Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 0d5efeeace | |||
| 955b93969c |
151 changed files with 3040 additions and 12299 deletions
167
Cargo.lock
generated
167
Cargo.lock
generated
|
|
@ -37,6 +37,17 @@ dependencies = [
|
|||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.7.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9"
|
||||
dependencies = [
|
||||
"getrandom 0.2.17",
|
||||
"once_cell",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.8.12"
|
||||
|
|
@ -184,6 +195,12 @@ version = "0.8.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "1.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.13.2"
|
||||
|
|
@ -352,21 +369,18 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "cultcache-rs"
|
||||
version = "0.3.1"
|
||||
source = "git+https://github.com/GameCult/CultLib.git?rev=75188fdefa0b4baf131dbfd94e77d06a965f0cc6#75188fdefa0b4baf131dbfd94e77d06a965f0cc6"
|
||||
version = "0.2.0"
|
||||
source = "git+https://github.com/GameCult/CultLib.git?rev=8fc74c709444b884a2bd339e0c306bf9484d99d2#8fc74c709444b884a2bd339e0c306bf9484d99d2"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"chrono",
|
||||
"cultcache-rs-derive 0.2.0 (git+https://github.com/GameCult/CultLib.git?rev=75188fdefa0b4baf131dbfd94e77d06a965f0cc6)",
|
||||
"cultcache-rs-derive 0.1.0",
|
||||
"fs2",
|
||||
"redb",
|
||||
"rmp-serde",
|
||||
"serde",
|
||||
"serde_bytes",
|
||||
"serde_json",
|
||||
"serde_stacker",
|
||||
"sha2",
|
||||
"stacker",
|
||||
"uuid",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
|
@ -378,7 +392,7 @@ source = "git+https://github.com/GameCult/CultLib.git?rev=a4eae9056152335662f111
|
|||
dependencies = [
|
||||
"anyhow",
|
||||
"chrono",
|
||||
"cultcache-rs-derive 0.2.0 (git+https://github.com/GameCult/CultLib.git?rev=a4eae9056152335662f1115f97bf35fb00ab6d23)",
|
||||
"cultcache-rs-derive 0.2.0",
|
||||
"fs2",
|
||||
"redb",
|
||||
"rmp-serde",
|
||||
|
|
@ -393,8 +407,8 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "cultcache-rs-derive"
|
||||
version = "0.2.0"
|
||||
source = "git+https://github.com/GameCult/CultLib.git?rev=75188fdefa0b4baf131dbfd94e77d06a965f0cc6#75188fdefa0b4baf131dbfd94e77d06a965f0cc6"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/GameCult/CultLib.git?rev=8fc74c709444b884a2bd339e0c306bf9484d99d2#8fc74c709444b884a2bd339e0c306bf9484d99d2"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -413,17 +427,18 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "cultnet-rs"
|
||||
version = "0.2.0"
|
||||
source = "git+https://github.com/GameCult/CultLib.git?rev=a4eae9056152335662f1115f97bf35fb00ab6d23#a4eae9056152335662f1115f97bf35fb00ab6d23"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/GameCult/CultLib.git?rev=8fc74c709444b884a2bd339e0c306bf9484d99d2#8fc74c709444b884a2bd339e0c306bf9484d99d2"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"anyhow",
|
||||
"base64",
|
||||
"chrono",
|
||||
"cultcache-rs 0.3.1 (git+https://github.com/GameCult/CultLib.git?rev=a4eae9056152335662f1115f97bf35fb00ab6d23)",
|
||||
"cultcache-rs 0.2.0",
|
||||
"ed25519-dalek",
|
||||
"hmac",
|
||||
"rand",
|
||||
"reed-solomon-erasure",
|
||||
"rmp-serde",
|
||||
"rmpv",
|
||||
"serde",
|
||||
|
|
@ -438,18 +453,16 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "cultnet-rs"
|
||||
version = "0.3.0"
|
||||
source = "git+https://github.com/GameCult/CultLib.git?rev=75188fdefa0b4baf131dbfd94e77d06a965f0cc6#75188fdefa0b4baf131dbfd94e77d06a965f0cc6"
|
||||
version = "0.2.0"
|
||||
source = "git+https://github.com/GameCult/CultLib.git?rev=a4eae9056152335662f1115f97bf35fb00ab6d23#a4eae9056152335662f1115f97bf35fb00ab6d23"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"anyhow",
|
||||
"base64",
|
||||
"chrono",
|
||||
"cultcache-rs 0.3.1 (git+https://github.com/GameCult/CultLib.git?rev=75188fdefa0b4baf131dbfd94e77d06a965f0cc6)",
|
||||
"cultcache-rs 0.3.1",
|
||||
"ed25519-dalek",
|
||||
"fs2",
|
||||
"hmac",
|
||||
"libc",
|
||||
"rand",
|
||||
"rmp-serde",
|
||||
"rmpv",
|
||||
|
|
@ -612,7 +625,7 @@ version = "0.1.0"
|
|||
dependencies = [
|
||||
"anyhow",
|
||||
"chrono",
|
||||
"cultcache-rs 0.3.1 (git+https://github.com/GameCult/CultLib.git?rev=75188fdefa0b4baf131dbfd94e77d06a965f0cc6)",
|
||||
"cultcache-rs 0.2.0",
|
||||
"rmp-serde",
|
||||
"schemars",
|
||||
"serde",
|
||||
|
|
@ -626,7 +639,7 @@ dependencies = [
|
|||
"anyhow",
|
||||
"chrono",
|
||||
"clap",
|
||||
"cultnet-rs 0.3.0",
|
||||
"cultnet-rs 0.1.0",
|
||||
"huginn-daemon",
|
||||
"huginn-mind",
|
||||
"rmcp",
|
||||
|
|
@ -861,6 +874,15 @@ dependencies = [
|
|||
"unicode-segmentation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.12.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
|
||||
dependencies = [
|
||||
"ahash 0.7.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.17.1"
|
||||
|
|
@ -910,7 +932,7 @@ dependencies = [
|
|||
"anyhow",
|
||||
"base64",
|
||||
"chrono",
|
||||
"cultnet-rs 0.3.0",
|
||||
"cultnet-rs 0.1.0",
|
||||
"huginn-mind",
|
||||
"rmp-serde",
|
||||
"rustix",
|
||||
|
|
@ -929,8 +951,8 @@ dependencies = [
|
|||
"anyhow",
|
||||
"base64",
|
||||
"chrono",
|
||||
"cultcache-rs 0.3.1 (git+https://github.com/GameCult/CultLib.git?rev=75188fdefa0b4baf131dbfd94e77d06a965f0cc6)",
|
||||
"cultnet-rs 0.3.0",
|
||||
"cultcache-rs 0.2.0",
|
||||
"cultnet-rs 0.1.0",
|
||||
"eureka-pipeline",
|
||||
"jsonschema",
|
||||
"rmp-serde",
|
||||
|
|
@ -982,6 +1004,15 @@ dependencies = [
|
|||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "instant"
|
||||
version = "0.1.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "is_terminal_polyfill"
|
||||
version = "1.70.2"
|
||||
|
|
@ -1011,7 +1042,7 @@ version = "0.58.6"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f622c574a29fb3034a0a90645afee194dfedc75fc124228801b6e77041075bd1"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"ahash 0.8.12",
|
||||
"bytecount",
|
||||
"data-encoding",
|
||||
"email_address",
|
||||
|
|
@ -1048,7 +1079,7 @@ version = "0.58.6"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39b7670345e5ce87371339d13705dbb9276b7e3e934bbe1efa2c6734a1f65a96"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"ahash 0.8.12",
|
||||
"bytecount",
|
||||
"fraction",
|
||||
"getrandom 0.3.4",
|
||||
|
|
@ -1064,6 +1095,12 @@ version = "0.2.189"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
|
||||
|
||||
[[package]]
|
||||
name = "libm"
|
||||
version = "0.2.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
version = "0.12.1"
|
||||
|
|
@ -1085,6 +1122,15 @@ version = "0.4.34"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
|
||||
|
||||
[[package]]
|
||||
name = "lru"
|
||||
version = "0.7.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a"
|
||||
dependencies = [
|
||||
"hashbrown 0.12.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.3"
|
||||
|
|
@ -1104,8 +1150,8 @@ dependencies = [
|
|||
"anyhow",
|
||||
"base64",
|
||||
"chrono",
|
||||
"cultcache-rs 0.3.1 (git+https://github.com/GameCult/CultLib.git?rev=75188fdefa0b4baf131dbfd94e77d06a965f0cc6)",
|
||||
"cultnet-rs 0.3.0",
|
||||
"cultcache-rs 0.2.0",
|
||||
"cultnet-rs 0.1.0",
|
||||
"eureka-pipeline",
|
||||
"eureka-state",
|
||||
"ghostlight-persona-projection",
|
||||
|
|
@ -1133,7 +1179,7 @@ source = "git+https://eureka.gamecult.org/forge/GameCult/ModelConnector.git?rev=
|
|||
dependencies = [
|
||||
"aes-gcm",
|
||||
"base64",
|
||||
"cultcache-rs 0.3.1 (git+https://github.com/GameCult/CultLib.git?rev=a4eae9056152335662f1115f97bf35fb00ab6d23)",
|
||||
"cultcache-rs 0.3.1",
|
||||
"cultnet-rs 0.2.0",
|
||||
"rand_core 0.6.4",
|
||||
"rmp-serde",
|
||||
|
|
@ -1258,6 +1304,17 @@ version = "0.5.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.11.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
|
||||
dependencies = [
|
||||
"instant",
|
||||
"lock_api",
|
||||
"parking_lot_core 0.8.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.12.5"
|
||||
|
|
@ -1265,7 +1322,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
|
||||
dependencies = [
|
||||
"lock_api",
|
||||
"parking_lot_core",
|
||||
"parking_lot_core 0.9.12",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot_core"
|
||||
version = "0.8.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"instant",
|
||||
"libc",
|
||||
"redox_syscall 0.2.16",
|
||||
"smallvec",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1276,7 +1347,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
|
|||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"redox_syscall",
|
||||
"redox_syscall 0.5.18",
|
||||
"smallvec",
|
||||
"windows-link",
|
||||
]
|
||||
|
|
@ -1417,13 +1488,35 @@ dependencies = [
|
|||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.2.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
|
||||
dependencies = [
|
||||
"bitflags 1.3.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.5.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"bitflags 2.13.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "reed-solomon-erasure"
|
||||
version = "6.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7263373d500d4d4f505d43a2a662d475a894aa94503a1ee28e9188b5f3960d4f"
|
||||
dependencies = [
|
||||
"libm",
|
||||
"lru",
|
||||
"parking_lot 0.11.2",
|
||||
"smallvec",
|
||||
"spin",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1452,13 +1545,13 @@ version = "0.58.6"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "558f3855529c0a7e78a48e409fa653456e55f1b167eab4109c16a255b7727e3f"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"ahash 0.8.12",
|
||||
"fluent-uri",
|
||||
"getrandom 0.3.4",
|
||||
"hashbrown",
|
||||
"hashbrown 0.17.1",
|
||||
"itoa",
|
||||
"micromap",
|
||||
"parking_lot",
|
||||
"parking_lot 0.12.5",
|
||||
"percent-encoding",
|
||||
"serde_json",
|
||||
]
|
||||
|
|
@ -1571,7 +1664,7 @@ version = "1.1.4"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"bitflags 2.13.2",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
|
|
@ -1782,6 +1875,12 @@ dependencies = [
|
|||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "spin"
|
||||
version = "0.9.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e"
|
||||
|
||||
[[package]]
|
||||
name = "spki"
|
||||
version = "0.7.3"
|
||||
|
|
|
|||
|
|
@ -16,5 +16,5 @@ publish = false
|
|||
# The one CultLib revision the workspace speaks. Every crate inherits these
|
||||
# entries; a bump is a change to this table and nowhere else.
|
||||
[workspace.dependencies]
|
||||
cultcache-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "75188fdefa0b4baf131dbfd94e77d06a965f0cc6", package = "cultcache-rs" }
|
||||
cultnet-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "75188fdefa0b4baf131dbfd94e77d06a965f0cc6", package = "cultnet-rs" }
|
||||
cultcache-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "8fc74c709444b884a2bd339e0c306bf9484d99d2", package = "cultcache-rs" }
|
||||
cultnet-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "8fc74c709444b884a2bd339e0c306bf9484d99d2", package = "cultnet-rs" }
|
||||
|
|
|
|||
|
|
@ -32,8 +32,7 @@ sample.research eureka-state:research:p1.scope epiphany.pipeline.research.v2 Epi
|
|||
sample.idea eureka-state:idea:p1.i1 epiphany.pipeline.idea.v2 EpiphanyPipelineIdeaDocument 9189a863616d706169676eac657572656b612d7374617465a470617373a27031a56c6162656ca26931a57469746c65b24f6e65206e6f74652070657220636c61696da469646561d9204b65657020612070617373206173206d616e7920736d616c6c206e6f7465732ea974726164656f666673b74d6f726520646f63756d656e747320746f206c6973742ea872657374735f6f6e9182a46b696e64a85265736561726368a26964be657572656b612d73746174653a72657365617263683a70312e73636f7065a6746f776172649182a46b696e64a85175657374696f6ea26964b8657572656b612d73746174653a7175657374696f6e3a5131ab70726f706f7365645f6f6eaa323032362d30392d3135
|
||||
sample.reply yggdrasil:reply:mind-2026-10-09-1-r1 epiphany.pipeline.reply.v2 EpiphanyPipelineReplyDocument 9185a8696e7374616e6365a979676764726173696ca56c6162656cb46d696e642d323032362d31302d30392d312d7231a372756e82a46b696e64a352756ea26964bf79676764726173696c3a72756e3a6d696e642d323032362d31302d30392d31a7616e73776572739182a46b696e64a8466f6c6c6f775570a26964d923657572656b612d73746174653a666f6c6c6f775f75703a61736b2d7468652d73656c66a474657874d93054686520736861646572207061737320697320646f6e653b2074686520736f756e647320617265204c6962627927732e
|
||||
sample.thought yggdrasil:thought:t1793000000 epiphany.pipeline.thought.v2 EpiphanyPipelineThoughtDocument 91de0014a8696e7374616e6365a979676764726173696ca56c6162656cab7431373933303030303030a474657874d93d54686520736f756e647320617265204c6962627927733b2074686520736861646572732077657265206e6576657220746865206861726420706172742ea572656164739182a46b696e64a64d656d6f7279a26964d92879676764726173696c3a6d656d6f72793a323032362d31302d3331743034313230302d74302d6d30af706572736f6e615f72656365697074b06d696e642d636f6d6d69742d30613162a86964656e74697479be457572656b612c207368652f6865722c20706c61696e2d73706f6b656e2ea7636f6e74657874d92056616c7565733a20636f686572656e6365206f7665722076656c6f636974792ea87374696d756c7573bd4c696262792061736b65642061626f75742074686520736f756e64732eaf70726f6a6563746f725f6d6f64656cb070726f6a6563746f723a6c6174657374aa70726f6a656374696f6ed92949206b656570207475726e696e67206f7665722077686f20686f6c64732074686520736f756e64732eaa626173655f6d6f64656cab626173653a6c6174657374b474656d70657261747572655f7065726d696c6c65cd03e8ae6d696e5f705f7065726d696c6c6532aa6d61785f746f6b656e73cca0aa63616e6469646174657392d93d54686520736f756e647320617265204c6962627927733b2074686520736861646572732077657265206e6576657220746865206861726420706172742eaf41207365636f6e642064726966742eb06e6561726573745f7065726d696c6c6592cd0136cd0366a663686f73656e00ac70726f6a6563746f725f6d73cd1068ac636f6e74696e7565725f6d73cd238cae70726f6d70745f76657273696f6ebd67686f73746c696768743d633433656235343b20776f7264733d313830
|
||||
sample.desk_answer eureka-state:desk_answer:Q1.n1 epiphany.pipeline.desk_answer.v2 EpiphanyPipelineDeskAnswerDocument 918ea863616d706169676eac657572656b612d7374617465a4666f726bb8657572656b612d73746174653a7175657374696f6e3a5131a873657175656e636501ac6f7074696f6e735f68617368d94063636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363a663686f696365a161a46e6f7465b853686172652069742c207769746820746865206d61726b2eab616e7377657265645f6174b4323032362d31312d31305431353a30303a30305aab72656365697665645f6174b4323032362d31312d31305431353a30303a30335aaf636c61696d5f696e5f667574757265c2aa726571756573745f6964d92436663163306135322d336237652d346431612d396335382d306532643761346239663133ab626f64795f736861323536d94064646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464a8616e737765726572d92435663063326231652d386134642d346333622d396537662d316132623363346435653666a9617574686f72697479a64d656d626572a76f7574636f6d6581a85265766973696f6e81a26f66be657572656b612d73746174653a6465736b5f616e737765723a51312e6e31
|
||||
sample.desk_listing eureka-state:desk_listing:Q1 epiphany.pipeline.desk_listing.v2 EpiphanyPipelineDeskListingDocument 9183a863616d706169676eac657572656b612d7374617465a66c6973746564b8657572656b612d73746174653a7175657374696f6e3a5131a6726561736f6ed9324d656d62657273206d617920726561642077687920746865204d696e64206c6973747320697473206f776e20666f726b732e
|
||||
sample.desk_answer eureka-state:desk_answer:Q1.n1 epiphany.pipeline.desk_answer.v2 EpiphanyPipelineDeskAnswerDocument 918ea863616d706169676eac657572656b612d7374617465a4666f726bb8657572656b612d73746174653a7175657374696f6e3a5131a873657175656e636501ac6f7074696f6e735f68617368d94063636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363a663686f696365a161a46e6f7465b853686172652069742c207769746820746865206d61726b2eab616e7377657265645f6174b4323032362d31312d31305431353a30303a30305aab72656365697665645f6174b4323032362d31312d31305431353a30303a30335aaf636c61696d5f696e5f667574757265c2aa726571756573745f6964d92436663163306135322d336237652d346431612d396335382d306532643761346239663133ab626f64795f736861323536d94064646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464a8616e737765726572a6757365722d37a9617574686f72697479a64d656d626572a76f7574636f6d6581a85265766973696f6e81a26f66be657572656b612d73746174653a6465736b5f616e737765723a51312e6e31
|
||||
resolution.superseded eureka-state:resolution:question.Q1.n1 epiphany.pipeline.resolution.v2 EpiphanyPipelineResolutionDocument 9185a77375626a65637482a46b696e64a85175657374696f6ea26964b8657572656b612d73746174653a7175657374696f6e3a5131a873657175656e636501a76f7574636f6d6581aa5375706572736564656481a262799282a46b696e64a652756c696e67a26964b6657572656b612d73746174653a72756c696e673a523882a46b696e64a85175657374696f6ea26964b8657572656b612d73746174653a7175657374696f6e3a5131a9726174696f6e616c65a852756c656420412eab7265736f6c7665645f6f6eaa323032362d30392d3135
|
||||
resolution.fixed-by-none eureka-state:resolution:question.Q1.n1 epiphany.pipeline.resolution.v2 EpiphanyPipelineResolutionDocument 9185a77375626a65637482a46b696e64a85175657374696f6ea26964b8657572656b612d73746174653a7175657374696f6e3a5131a873657175656e636501a76f7574636f6d6581a5466978656482a6636f6d6d6974a83566393832323864a26279c0a9726174696f6e616c65a852756c656420412eab7265736f6c7665645f6f6eaa323032362d30392d3135
|
||||
resolution.fixed-by-some eureka-state:resolution:question.Q1.n1 epiphany.pipeline.resolution.v2 EpiphanyPipelineResolutionDocument 9185a77375626a65637482a46b696e64a85175657374696f6ea26964b8657572656b612d73746174653a7175657374696f6e3a5131a873657175656e636501a76f7574636f6d6581a5466978656482a6636f6d6d6974a83566393832323864a2627982a46b696e64a94375745265706f7274a26964d921657572656b612d73746174653a6375745f7265706f72743a6375742d33612e6831a9726174696f6e616c65a852756c656420412eab7265736f6c7665645f6f6eaa323032362d30392d3135
|
||||
|
|
|
|||
|
|
@ -316,10 +316,6 @@ bounded_text! {
|
|||
Words = 4000, words_text;
|
||||
/// A client's retry token: a lowercase D-format UUID.
|
||||
RequestId = 36, uuid_text;
|
||||
/// A Bifrost account id: a lowercase D-format UUID (`UserAccount.Id`). A Matrix id, an email or a
|
||||
/// handle is never one.
|
||||
#[schemars(extend("pattern" = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"))]
|
||||
AccountId = 36, uuid_text;
|
||||
/// A key label: `[A-Za-z0-9_-]{1,64}`.
|
||||
Label = 64, label_text;
|
||||
/// A campaign slug or key segment: label parts joined by `.`.
|
||||
|
|
@ -889,13 +885,8 @@ value_types! {
|
|||
pub struct PipelineDeskAnswer {
|
||||
campaign: Slug, fork: Short, sequence: u32, options_hash: Sha256Hex, choice: Option<Short>, note: Option<Words>,
|
||||
answered_at: Short, received_at: Short, claim_in_future: bool, request_id: RequestId, body_sha256: Sha256Hex,
|
||||
answerer: AccountId, authority: DeskAuthority, outcome: DeskOutcome,
|
||||
answerer: Short, authority: DeskAuthority, outcome: DeskOutcome,
|
||||
}
|
||||
/// A fork the Mind lists: members who are not on a call's decider list may read the question
|
||||
/// while this document is in force (unlist = a `Withdrawn` resolution on it; relist = withdraw
|
||||
/// that). One per question, keyed by the question's label; the receipt says who listed it and
|
||||
/// when, `reason` why members may read it. Admitted on the Mind's own door only, never the desk's.
|
||||
pub struct PipelineDeskListing { campaign: Slug, listed: Short, reason: Line }
|
||||
pub struct PipelineVerdict { campaign: Slug, cut_report: Short, pass: u32, range: CommitRange, claims: Vec<VerdictClaim>[64] }
|
||||
pub struct PipelineFinding {
|
||||
campaign: Slug, verdict: Short, label: Label, range: CommitRange, confidence: FindingConfidence,
|
||||
|
|
@ -1434,16 +1425,6 @@ pipeline_kinds! {
|
|||
"epiphany.pipeline.thought.v2", "EpiphanyPipelineThoughtDocument";
|
||||
DeskAnswer(PipelineDeskAnswer) => EpiphanyPipelineDeskAnswerDocument, "desk_answer",
|
||||
"epiphany.pipeline.desk_answer.v2", "EpiphanyPipelineDeskAnswerDocument";
|
||||
DeskListing(PipelineDeskListing) => EpiphanyPipelineDeskListingDocument, "desk_listing",
|
||||
"epiphany.pipeline.desk_listing.v2", "EpiphanyPipelineDeskListingDocument";
|
||||
}
|
||||
|
||||
impl AccountId {
|
||||
/// Whether this is a lowercase D-format UUID. For the callers outside the crate, which have no way
|
||||
/// to reach `Bounded`; on the pattern of `Slug::validate_slug`.
|
||||
pub fn validate_account(&self) -> Result<(), PipelineRefusal> {
|
||||
self.validate("account")
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses a full document id: the grammar read backwards. A key and an id are
|
||||
|
|
@ -1535,7 +1516,7 @@ const SUBJECT_LOCAL_MAX: usize = 64;
|
|||
|
||||
/// The longest kind name a resolution can name as its subject (`stewardship`),
|
||||
/// checked against `PipelineKind::ALL` by a test.
|
||||
const LONGEST_KIND_NAME: usize = "desk_listing".len();
|
||||
const LONGEST_KIND_NAME: usize = "stewardship".len();
|
||||
|
||||
/// A `.n<sequence>` part at `u32::MAX`.
|
||||
const SEQUENCE_PART_MAX: usize = ".n".len() + "4294967295".len();
|
||||
|
|
@ -1670,10 +1651,6 @@ pub fn pipeline_key(document: &PipelineDocument) -> Result<String, PipelineRefus
|
|||
let sequence = format!("n{}", value.sequence);
|
||||
(campaign_field.as_str(), value.campaign.0.as_str(), local(&key_field, document_kind, &[label, &sequence])?)
|
||||
}
|
||||
D::DeskListing(value) => {
|
||||
let label = parent_local("desk_listing.listed", &value.listed.0, &value.campaign.0, PipelineKind::Question)?;
|
||||
(campaign_field.as_str(), value.campaign.0.as_str(), local(&key_field, document_kind, &[label])?)
|
||||
}
|
||||
D::Verdict(value) => {
|
||||
let report = parent_local("verdict.cut_report", &value.cut_report.0, &value.campaign.0, PipelineKind::CutReport)?;
|
||||
let cut = parent_cut("verdict.cut_report", report, 'h')?;
|
||||
|
|
@ -1950,12 +1927,9 @@ mod tests {
|
|||
campaign: slug(CAMPAIGN), fork: id("question", "Q1"), sequence: 1, options_hash: Sha256Hex("c".repeat(64)),
|
||||
choice: Some(s("a")), note: Some("Share it, with the mark.".into()), answered_at: s("2026-11-10T15:00:00Z"),
|
||||
received_at: s("2026-11-10T15:00:03Z"), claim_in_future: false, request_id: RequestId("6f1c0a52-3b7e-4d1a-9c58-0e2d7a4b9f13".into()), body_sha256: Sha256Hex("d".repeat(64)),
|
||||
answerer: AccountId("5f0c2b1e-8a4d-4c3b-9e7f-1a2b3c4d5e6f".into()), authority: DeskAuthority::Member,
|
||||
answerer: s("user-7"), authority: DeskAuthority::Member,
|
||||
outcome: DeskOutcome::Revision { of: id("desk_answer", "Q1.n1") },
|
||||
}), format!("{CAMPAIGN}:desk_answer:Q1.n1")),
|
||||
(D::DeskListing(PipelineDeskListing {
|
||||
campaign: slug(CAMPAIGN), listed: id("question", "Q1"), reason: "Members may read why the Mind lists its own forks.".into(),
|
||||
}), format!("{CAMPAIGN}:desk_listing:Q1")),
|
||||
]
|
||||
}
|
||||
|
||||
|
|
@ -3613,7 +3587,7 @@ mod tests {
|
|||
);
|
||||
}
|
||||
assert_eq!(widest, RESOLUTION_LOCAL_MAX, "the bound is exactly the widest depth-two chain");
|
||||
assert_eq!(RESOLUTION_LOCAL_MAX, 112);
|
||||
assert_eq!(RESOLUTION_LOCAL_MAX, 111);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -4135,7 +4109,7 @@ mod tests {
|
|||
};
|
||||
let document_schema = serde_json::to_value(schemars::schema_for!(PipelineDocument))?;
|
||||
let variants = document_schema["oneOf"].as_array().expect("the document schema is a oneOf");
|
||||
assert_eq!(PipelineKind::ALL.len(), 22, "twenty-two kinds");
|
||||
assert_eq!(PipelineKind::ALL.len(), 21, "twenty-one kinds");
|
||||
assert_eq!(
|
||||
variants.iter().map(tag).collect::<Vec<_>>(),
|
||||
PipelineKind::ALL.iter().map(|kind| kind.name().to_owned()).collect::<Vec<_>>(),
|
||||
|
|
@ -4665,7 +4639,7 @@ mod tests {
|
|||
}
|
||||
|
||||
fn desk_answer_sample() -> PipelineDeskAnswer {
|
||||
let PipelineDocument::DeskAnswer(answer) = samples().remove(20).0 else { unreachable!() };
|
||||
let PipelineDocument::DeskAnswer(answer) = samples().pop().unwrap().0 else { unreachable!() };
|
||||
answer
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ macro_rules! kinds {
|
|||
|
||||
kinds!(
|
||||
campaign, target, question, ruling, cut_spec, cut_report, verdict, finding, follow_up, resolution, instance, stewardship,
|
||||
hand_off, run, pulse, memory, research, idea, reply, thought, desk_answer, desk_listing,
|
||||
hand_off, run, pulse, memory, research, idea, reply, thought, desk_answer,
|
||||
);
|
||||
|
||||
/// The schema named, or the names when none is: the exit code is 2 for a name
|
||||
|
|
|
|||
|
|
@ -1145,12 +1145,12 @@ fn schema_prints_the_published_files() {
|
|||
let names = run(&[], &["schema"], None);
|
||||
assert_eq!(names.code, 0);
|
||||
let names: Vec<&str> = names.out.lines().collect();
|
||||
assert_eq!(names.len(), 23, "{names:?}");
|
||||
assert_eq!(names.len(), 22, "{names:?}");
|
||||
assert_eq!(names[0], "campaign");
|
||||
assert_eq!(names[14], "pulse");
|
||||
assert_eq!(names[15], "memory");
|
||||
assert_eq!((names[16], names[17]), ("research", "idea"));
|
||||
assert_eq!((names[18], names[19], names[20], names[21], names[22]), ("reply", "thought", "desk_answer", "desk_listing", "selection"));
|
||||
assert_eq!((names[18], names[19], names[20], names[21]), ("reply", "thought", "desk_answer", "selection"));
|
||||
for kind in PipelineKind::ALL {
|
||||
assert!(names.contains(&kind.name()), "{}", kind.name());
|
||||
}
|
||||
|
|
@ -1303,22 +1303,6 @@ impl huginn_daemon::sources::SourceIndex for Desk {
|
|||
anyhow::bail!("a daemon never writes the source index")
|
||||
}
|
||||
|
||||
fn chat_tail(&mut self, _room_id: &str) -> anyhow::Result<Option<huginn_daemon::sources::StoredChat>> {
|
||||
anyhow::bail!("a daemon never reads the ingest job's side of the source index")
|
||||
}
|
||||
|
||||
fn chat_chunks_holding(&mut self, _room_id: &str, _event_ids: &[String]) -> anyhow::Result<Vec<huginn_daemon::sources::StoredChat>> {
|
||||
anyhow::bail!("a daemon never reads the ingest job's side of the source index")
|
||||
}
|
||||
|
||||
fn chat_chunks_overlapping(&mut self, _room_id: &str, _from_ms: u64, _to_ms: u64) -> anyhow::Result<Vec<huginn_daemon::sources::StoredChat>> {
|
||||
anyhow::bail!("a daemon never reads the ingest job's side of the source index")
|
||||
}
|
||||
|
||||
fn delete_chat(&mut self, _room_id: &str, _ids: Option<&[String]>) -> anyhow::Result<()> {
|
||||
anyhow::bail!("a daemon never writes the source index")
|
||||
}
|
||||
|
||||
fn chat_nearest(
|
||||
&mut self,
|
||||
_vector: &[f32],
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ use chrono::{DateTime, Utc};
|
|||
use cultnet_rs::Selection;
|
||||
use crate::sources::forge::HttpForge;
|
||||
use huginn_mind::desk_wire::{DeskRequest, DeskResponse};
|
||||
use huginn_mind::eureka_pipeline::{AccountId, PipelineRef, Short, Slug};
|
||||
use huginn_mind::eureka_pipeline::{PipelineRef, Slug};
|
||||
use huginn_mind::wire::{ACTIVITY_REPOS_MAX, HuginnMindRequest, HuginnMindResponse, IndexStatus};
|
||||
use huginn_mind::{ChatHits, ChatQuery, ChatWindow, ChatWindowQuery, Members, Mind, MindRefusal, MindStore, PipelineAdmissionOutcome, SemanticQuery, SourceHits, SourceQuery};
|
||||
|
||||
|
|
@ -109,15 +109,6 @@ impl Handled {
|
|||
}
|
||||
}
|
||||
|
||||
/// The door checks an account id; the Mind's desk calls take it as the text it is.
|
||||
fn short_id(id: AccountId) -> Short {
|
||||
Short(id.0)
|
||||
}
|
||||
|
||||
fn short_ids(ids: &[AccountId]) -> Vec<Short> {
|
||||
ids.iter().map(|id| Short(id.0.clone())).collect()
|
||||
}
|
||||
|
||||
/// What the log says of the hits the mind held no document for: how many, and
|
||||
/// the first few by name (a restored mind can leave hundreds), or nothing when
|
||||
/// the mind held them all.
|
||||
|
|
@ -272,18 +263,18 @@ impl<S: MindStore, I: IndexSink<S>> Daemon<S, I> {
|
|||
/// One desk request, answered by the Mind method it names. The Mind is the instance's own
|
||||
/// (`mind` is checked against it); `deciders` is this call's list, handed to the Mind and
|
||||
/// dropped. A desk answer is not indexed (`index_text` gives it no text), so nothing here tells the index.
|
||||
/// The reads are handed `subject` and `deciders`: the Mind builds a decider's view or a
|
||||
/// non-decider's (ruling `desk-read-visibility-withheld-ids`).
|
||||
/// The reads answer with the whole record and nothing here gates them (ruling
|
||||
/// `desk-read-visibility-whole-record`): they are handed `deciders` and not `subject`.
|
||||
pub fn handle_desk(&mut self, request: DeskRequest, now: DateTime<Utc>) -> DeskResponse {
|
||||
if let Err(refusal) = self.mind.require_instance(request.mind()) {
|
||||
return DeskResponse::Refused(refusal);
|
||||
}
|
||||
match request {
|
||||
DeskRequest::Answer { subject, authority, deciders, request, .. } => {
|
||||
self.mind.desk_answer(&short_ids(&deciders), short_id(subject), authority, request, now)
|
||||
self.mind.desk_answer(&deciders, subject, authority, request, now)
|
||||
}
|
||||
DeskRequest::Forks { subject, deciders, status, cursor, limit, .. } => self.mind.desk_forks(&short_id(subject), &short_ids(&deciders), status, cursor, limit),
|
||||
DeskRequest::Fork { subject, deciders, fork_id, .. } => self.mind.desk_fork(&short_id(subject), &short_ids(&deciders), &fork_id),
|
||||
DeskRequest::Forks { deciders, status, cursor, limit, .. } => self.mind.desk_forks(&deciders, status, cursor, limit),
|
||||
DeskRequest::Fork { deciders, fork_id, .. } => self.mind.desk_fork(&deciders, &fork_id),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -10,22 +10,16 @@
|
|||
//! deadline like one that is silent), a reply past `DESK_MAX_RESPONSE_BYTES` refused by name (a read of one
|
||||
//! fork past it is that fork's `fork_too_large`).
|
||||
//!
|
||||
//! The trust boundary of the reads (ruling `desk-read-visibility-withheld-ids`). The peer-uid check
|
||||
//! The trust boundary of the reads (ruling `desk-read-visibility-whole-record`). The peer-uid check
|
||||
//! IS the whole authentication: Huginn must be reachable only by Bifrost's peer-checked socket, so
|
||||
//! the configured uid is Bifrost's and no other process's. `Forks` and `Fork` carry every decider's
|
||||
//! note to whoever passes that check, and Bifrost strips notes at its own door (a stripped read
|
||||
//! serves `note: null`, CultLib `fork-card.md`); a second reader of this socket is therefore a note
|
||||
//! leak, and a deployment that gives the socket to any uid but Bifrost's has no reader check left.
|
||||
//! Account ids are Huginn's to withhold: a reader whose `subject` is not in the call's `deciders`
|
||||
//! gets `deciders` empty, `answerer` nil and the counts instead. The reads carry no `authority` (a
|
||||
//! payload that names one is refused whole), and the door has no other way to ask who is reading.
|
||||
//!
|
||||
//! The listing gate: a reader who is not a decider sees a fork only while a `desk_listing` for it is
|
||||
//! in force, and a fork it may not see reads `fork_unknown`, the same bytes as a missing one, on
|
||||
//! every desk operation, a replay of a stored answer included. The door never writes a listing; it
|
||||
//! arrives by the RUDP Admit door only. Unlisting is a `Withdrawn` resolution on the listing, and
|
||||
//! admitting the listing again after that returns `AlreadyAdmitted` and the fork stays hidden:
|
||||
//! relisting is withdrawing the withdrawal, not a second admission.
|
||||
//! the configured uid is Bifrost's and no other process's. `Forks` and `Fork` return the WHOLE record
|
||||
//! (cards, outcomes, every decider's note) to whoever passes that check; nothing in Huginn strips a
|
||||
//! note or gates a read on `subject` or `deciders` (`deciders` only fills the card and the
|
||||
//! showability gate). Bifrost applies visibility at its own door (a stripped read serves `note: null`,
|
||||
//! CultLib `fork-card.md`). A second reader of this socket is therefore a note leak, and a deployment
|
||||
//! that gives the socket to any uid but Bifrost's has no reader check left. The reads carry no
|
||||
//! `authority` (a payload that names one is refused whole), and the door has no other way to ask who
|
||||
//! is reading.
|
||||
//!
|
||||
//! It owns no rule. Which Mind method answers is `Daemon::handle_desk`'s; when the door is
|
||||
//! served is `serve::run`'s, once per pass in the thread that owns the Mind. A peer that is
|
||||
|
|
|
|||
|
|
@ -85,26 +85,20 @@ fn wire(hash: &str, request_id: &str) -> ForkAnswerRequest {
|
|||
}
|
||||
}
|
||||
|
||||
/// A test's name for someone as the Bifrost account id the door requires: a deterministic lowercase D-format UUID.
|
||||
fn acct(name: &str) -> huginn_mind::eureka_pipeline::AccountId {
|
||||
let hash = name.bytes().fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| (hash ^ u64::from(byte)).wrapping_mul(0x0100_0000_01b3));
|
||||
huginn_mind::eureka_pipeline::AccountId(format!("{:08x}-{:04x}-4000-8000-{:012x}", hash >> 32, (hash >> 16) & 0xffff, hash & 0xffff_ffff_ffff))
|
||||
}
|
||||
|
||||
const R1: &str = "00000000-0000-4000-8000-000000000001";
|
||||
const R2: &str = "00000000-0000-4000-8000-000000000002";
|
||||
const R3: &str = "00000000-0000-4000-8000-000000000003";
|
||||
|
||||
fn answer(who: &str, authority: DeskAuthority, deciders: &[&str], request: ForkAnswerRequest) -> DeskRequest {
|
||||
DeskRequest::Answer { mind: slug(INSTANCE), subject: acct(who), authority, deciders: deciders.iter().map(|d| acct(d)).collect(), request }
|
||||
DeskRequest::Answer { mind: slug(INSTANCE), subject: short(who), authority, deciders: deciders.iter().map(|d| short(d)).collect(), request }
|
||||
}
|
||||
|
||||
/// A list read by `who` against `deciders`.
|
||||
fn forks(who: &str, deciders: &[&str], status: Option<ListStatus>, cursor: Option<ForkCursor>, limit: u8) -> DeskRequest {
|
||||
DeskRequest::Forks {
|
||||
mind: slug(INSTANCE),
|
||||
subject: acct(who),
|
||||
deciders: deciders.iter().map(|d| acct(d)).collect(),
|
||||
subject: short(who),
|
||||
deciders: deciders.iter().map(|d| short(d)).collect(),
|
||||
status,
|
||||
cursor,
|
||||
limit: ForkLimit::new(limit).unwrap(),
|
||||
|
|
@ -116,7 +110,7 @@ fn forks_all(deciders: &[&str]) -> DeskRequest {
|
|||
}
|
||||
|
||||
fn fork_read(who: &str, deciders: &[&str], fork_id: &str) -> DeskRequest {
|
||||
DeskRequest::Fork { mind: slug(INSTANCE), subject: acct(who), deciders: deciders.iter().map(|d| acct(d)).collect(), fork_id: fork_id.into() }
|
||||
DeskRequest::Fork { mind: slug(INSTANCE), subject: short(who), deciders: deciders.iter().map(|d| short(d)).collect(), fork_id: fork_id.into() }
|
||||
}
|
||||
|
||||
fn page_of(response: DeskResponse) -> (ForkList, u64) {
|
||||
|
|
@ -211,13 +205,6 @@ fn rig(peer_uid: u32) -> Rig {
|
|||
}
|
||||
|
||||
impl Rig {
|
||||
/// The Mind lists the seeded fork, on its own door, so a non-decider reaches it.
|
||||
fn list_the_fork(&mut self) {
|
||||
let listing = PipelineDocument::DeskListing(huginn_mind::eureka_pipeline::PipelineDeskListing { campaign: slug("eureka-state"), listed: short(FORK), reason: "Listed.".into() });
|
||||
let admitted = self.daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![listing])), now()).answered();
|
||||
assert!(matches!(admitted, HuginnMindResponse::Admit(huginn_mind::PipelineAdmissionOutcome::Committed { .. })), "{admitted:?}");
|
||||
}
|
||||
|
||||
/// Runs `client` on its own thread while this one serves the door, then drains what is still waiting.
|
||||
fn serving<T: Send>(&mut self, client: impl FnOnce(&Path) -> T + Send) -> T {
|
||||
let path = self.path.clone();
|
||||
|
|
@ -580,7 +567,7 @@ mod door_smoke {
|
|||
claim_in_future: false,
|
||||
request_id: RequestId(R1.into()),
|
||||
body_sha256: Sha256Hex("0".repeat(64)),
|
||||
answerer: acct("x"),
|
||||
answerer: short("x"),
|
||||
authority: DeskAuthority::Operator,
|
||||
outcome: DeskOutcome::Counted,
|
||||
});
|
||||
|
|
@ -614,7 +601,6 @@ mod door_smoke {
|
|||
#[test]
|
||||
fn deciders_ride_each_request() {
|
||||
let mut rig = rig(my_uid());
|
||||
rig.list_the_fork();
|
||||
let (before, hash) = (rig.state(), rig.hash.clone());
|
||||
let out = rig.serving(|path| {
|
||||
let ask = |id: &str, who: &str, deciders: &[&str], request_id: &str| {
|
||||
|
|
@ -790,7 +776,6 @@ mod door_smoke {
|
|||
#[test]
|
||||
fn a_mind_refusal_and_a_stored_refusal_are_typed_replies() {
|
||||
let mut rig = rig(my_uid());
|
||||
rig.list_the_fork();
|
||||
let (before, hash) = (rig.state(), rig.hash.clone());
|
||||
let mut foreign = answer("x", DeskAuthority::Member, &["x"], wire(&hash, R1));
|
||||
let DeskRequest::Answer { mind, .. } = &mut foreign else { unreachable!() };
|
||||
|
|
@ -816,7 +801,7 @@ mod door_smoke {
|
|||
assert_eq!((listed_at, read_at), (before.1 as u64, before.1 as u64), "both read at the head the mind held");
|
||||
assert_eq!(list.stamp, format!("s{listed_at}"));
|
||||
assert_eq!(list.forks[0], entry);
|
||||
assert_eq!((entry.card.fork_id.as_str(), entry.card.options_hash.as_str(), entry.card.deciders.as_slice()), (FORK, hash.as_str(), [acct("x").0, acct("y").0].as_slice()));
|
||||
assert_eq!((entry.card.fork_id.as_str(), entry.card.options_hash.as_str(), entry.card.deciders.as_slice()), (FORK, hash.as_str(), ["x".to_string(), "y".to_string()].as_slice()));
|
||||
assert_eq!(rig.state(), before, "a read stores nothing");
|
||||
assert!(rig.index.seen.lock().unwrap().is_empty());
|
||||
// An unknown fork is the answer path's refusal.
|
||||
|
|
@ -870,43 +855,29 @@ mod door_smoke {
|
|||
}
|
||||
}
|
||||
|
||||
/// Ruling `desk-read-visibility-withheld-ids`: a subject the call's deciders do not name gets counts
|
||||
/// and no account id, over the socket too; notes stay whole (Bifrost strips them). A decider gets the ids.
|
||||
/// Ruling `desk-read-visibility-whole-record`: Huginn returns the whole record, notes included, to
|
||||
/// whoever the socket carries. The subject is not in the deciders; the decider's note is there.
|
||||
#[test]
|
||||
fn a_subject_not_in_deciders_gets_counts_and_no_account_id() {
|
||||
fn a_subject_not_in_deciders_reads_the_whole_record_with_the_notes() {
|
||||
let mut rig = rig(my_uid());
|
||||
let hash = rig.hash.clone();
|
||||
let note = "Keep it local: the author's call.";
|
||||
rig.serving(|path| counted(reply_of(path, "m-1", &DeskRequest::Answer {
|
||||
mind: slug(INSTANCE),
|
||||
subject: acct("y"),
|
||||
subject: short("y"),
|
||||
authority: DeskAuthority::Member,
|
||||
deciders: vec![acct("y")],
|
||||
deciders: vec![short("y")],
|
||||
request: noted(&hash, R1, note),
|
||||
})));
|
||||
// Unlisted: the outsider is shown nothing at all, and a read is `fork_unknown`.
|
||||
let (hidden, unknown) = rig.serving(|path| (reply_of(path, "m-1", &forks("outsider", &["x"], None, None, 50)), reply_of(path, "m-1b", &fork_read("outsider", &["x"], FORK))));
|
||||
assert_eq!(page_of(hidden).0.forks.len(), 0);
|
||||
assert!(matches!(unknown, DeskResponse::NotStored(_)), "{unknown:?}");
|
||||
// The Mind lists it, on its own door.
|
||||
let listing = PipelineDocument::DeskListing(huginn_mind::eureka_pipeline::PipelineDeskListing { campaign: slug("eureka-state"), listed: short(FORK), reason: "Members may read why.".into() });
|
||||
let admitted = rig.daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![listing])), now()).answered();
|
||||
assert!(matches!(admitted, HuginnMindResponse::Admit(huginn_mind::PipelineAdmissionOutcome::Committed { .. })), "{admitted:?}");
|
||||
let (list, read) = rig.serving(|path| (reply_of(path, "m-2", &forks("outsider", &["x"], None, None, 50)), reply_of(path, "m-3", &fork_read("outsider", &["x"], FORK))));
|
||||
let (list, _) = page_of(list);
|
||||
let (read, _) = entry_of(read);
|
||||
for entry in [&list.forks[0], &read] {
|
||||
assert_eq!(entry.record.answers.len(), 1);
|
||||
assert_eq!(entry.record.answers[0].note.as_deref(), Some(note));
|
||||
assert_eq!(entry.record.answers[0].answerer, None);
|
||||
assert_eq!(entry.record.answerer_count, 1);
|
||||
assert!(entry.card.deciders.is_empty());
|
||||
assert_eq!(entry.card.decider_count, 1);
|
||||
assert_eq!(entry.record.answers[0].answerer, "y");
|
||||
assert_eq!(entry.card.deciders, ["x"], "the list is the call's, and only fills the card");
|
||||
}
|
||||
let seen = rig.serving(|path| reply_of(path, "m-4", &fork_read("x", &["x"], FORK)));
|
||||
let (seen, _) = entry_of(seen);
|
||||
assert_eq!(seen.card.deciders, [acct("x").0]);
|
||||
assert_eq!(seen.record.answers[0].answerer.as_deref(), Some(acct("y").0.as_str()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -925,7 +896,7 @@ mod door_smoke {
|
|||
let refused = rig.serving(|path| failure_of(path, &bytes));
|
||||
assert_eq!(refused.code, "payload-not-a-request", "{what}");
|
||||
assert!(refused.message.contains("deciders"), "{what}: {}", refused.message);
|
||||
assert!(!refused.message.contains(&acct("d64").0), "{what} echoes a decider: {}", refused.message);
|
||||
assert!(!refused.message.contains("d64"), "{what} echoes a decider: {}", refused.message);
|
||||
}
|
||||
assert_eq!(rig.state(), before, "nothing stored, head unchanged");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ use std::time::Duration;
|
|||
|
||||
use anyhow::{Result, bail, ensure};
|
||||
|
||||
use crate::sources::{ChatFilter, ChatOrder, ChatPayload, ChatPoint, ChatRow, DescribedSources, SourceIndex, SourcePoint, SourceRow, SourcesMeta, StoredChat};
|
||||
use crate::sources::{ChatFilter, ChatOrder, ChatPayload, ChatPoint, ChatRow, DescribedSources, SourceIndex, SourcePoint, SourceRow, SourcesMeta};
|
||||
|
||||
use super::{CollectionMeta, Described, Embedder, Hit, ModelIdentity, Point, VectorIndex, point_id};
|
||||
|
||||
|
|
@ -264,16 +264,6 @@ pub(crate) struct SourcesState {
|
|||
pub(crate) deletes: Vec<(String, Option<Vec<String>>)>,
|
||||
/// A delete naming this repo refuses.
|
||||
pub(crate) fail_deletes_of: Option<String>,
|
||||
/// Every cursor write refuses once this many have been written.
|
||||
pub(crate) fail_cursor_writes_after: Option<usize>,
|
||||
/// Every chat upsert refuses while this is set.
|
||||
pub(crate) fail_chat_upserts: bool,
|
||||
/// Every chat delete refuses while this is set.
|
||||
pub(crate) fail_chat_deletes: bool,
|
||||
/// A chat delete naming one of these rooms refuses.
|
||||
pub(crate) fail_chat_deletes_for: BTreeSet<String>,
|
||||
/// Every chat delete that went through: the room and the ids (`None` is all of the room).
|
||||
pub(crate) chat_deletes: Vec<(String, Option<Vec<String>>)>,
|
||||
/// The repo filter and limit of every search, in order.
|
||||
pub(crate) searches: Vec<(Option<String>, u32)>,
|
||||
}
|
||||
|
|
@ -336,9 +326,6 @@ impl SourceIndex for FakeSources {
|
|||
|
||||
fn set_cursors(&mut self, cursors: &BTreeMap<String, String>) -> Result<()> {
|
||||
let mut state = self.state.lock().unwrap();
|
||||
if state.fail_cursor_writes_after.is_some_and(|after| state.cursor_writes.len() >= after) {
|
||||
bail!("the vector store refused the cursor write");
|
||||
}
|
||||
state.cursor_writes.push(cursors.clone());
|
||||
let Some(DescribedSources::Labelled { cursors: held, .. }) = &mut state.described else { bail!("no source collection") };
|
||||
*held = cursors.clone();
|
||||
|
|
@ -388,7 +375,6 @@ impl SourceIndex for FakeSources {
|
|||
|
||||
fn upsert_chat(&mut self, points: &[ChatPoint]) -> Result<()> {
|
||||
let mut state = self.state.lock().unwrap();
|
||||
ensure!(!state.fail_chat_upserts, "the vector store refused the chat upsert");
|
||||
for point in points {
|
||||
ensure!(!point.payload.messages.is_empty(), "a chat chunk carries no messages");
|
||||
state.chat.insert(point.id.clone(), point.clone());
|
||||
|
|
@ -418,44 +404,6 @@ impl SourceIndex for FakeSources {
|
|||
}
|
||||
Ok(kept.into_iter().take(limit as usize).cloned().collect())
|
||||
}
|
||||
|
||||
fn chat_tail(&mut self, room_id: &str) -> Result<Option<StoredChat>> {
|
||||
let state = self.state.lock().unwrap();
|
||||
Ok(state.chat.iter().filter(|(_, point)| point.payload.room_id == room_id).max_by_key(|(id, point)| (span(&point.payload).1, (*id).clone())).map(stored))
|
||||
}
|
||||
|
||||
fn chat_chunks_holding(&mut self, room_id: &str, event_ids: &[String]) -> Result<Vec<StoredChat>> {
|
||||
let state = self.state.lock().unwrap();
|
||||
Ok(state
|
||||
.chat
|
||||
.iter()
|
||||
.filter(|(_, point)| point.payload.room_id == room_id && point.payload.messages.iter().any(|message| event_ids.contains(&message.event_id)))
|
||||
.map(stored)
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn chat_chunks_overlapping(&mut self, room_id: &str, from_ms: u64, to_ms: u64) -> Result<Vec<StoredChat>> {
|
||||
let state = self.state.lock().unwrap();
|
||||
Ok(state.chat.iter().filter(|(_, point)| point.payload.room_id == room_id && span(&point.payload).1 >= from_ms && span(&point.payload).0 < to_ms).map(stored).collect())
|
||||
}
|
||||
|
||||
fn delete_chat(&mut self, room_id: &str, ids: Option<&[String]>) -> Result<()> {
|
||||
let mut state = self.state.lock().unwrap();
|
||||
ensure!(!state.fail_chat_deletes && !state.fail_chat_deletes_for.contains(room_id), "the vector store refused the chat delete");
|
||||
state.chat_deletes.push((room_id.to_owned(), ids.map(<[String]>::to_vec)));
|
||||
state.chat.retain(|id, point| !(point.payload.room_id == room_id && ids.is_none_or(|ids| ids.contains(id))));
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn stored((id, point): (&String, &ChatPoint)) -> StoredChat {
|
||||
StoredChat { id: id.clone(), payload: point.payload.clone() }
|
||||
}
|
||||
|
||||
/// (start, end) of the times a chunk's messages make.
|
||||
fn span(payload: &ChatPayload) -> (u64, u64) {
|
||||
let times = || payload.messages.iter().map(|message| message.ts_ms);
|
||||
(times().min().unwrap_or(0), times().max().unwrap_or(0))
|
||||
}
|
||||
|
||||
/// A chat chunk under a filter, by the span its messages make.
|
||||
|
|
@ -495,16 +443,4 @@ impl SourceIndex for FakeIndex {
|
|||
fn chat_scroll(&mut self, filter: &ChatFilter, order: ChatOrder, limit: u32, within: Duration) -> Result<Vec<ChatPayload>> {
|
||||
self.sources.chat_scroll(filter, order, limit, within)
|
||||
}
|
||||
fn chat_tail(&mut self, room_id: &str) -> Result<Option<StoredChat>> {
|
||||
self.sources.chat_tail(room_id)
|
||||
}
|
||||
fn chat_chunks_holding(&mut self, room_id: &str, event_ids: &[String]) -> Result<Vec<StoredChat>> {
|
||||
self.sources.chat_chunks_holding(room_id, event_ids)
|
||||
}
|
||||
fn chat_chunks_overlapping(&mut self, room_id: &str, from_ms: u64, to_ms: u64) -> Result<Vec<StoredChat>> {
|
||||
self.sources.chat_chunks_overlapping(room_id, from_ms, to_ms)
|
||||
}
|
||||
fn delete_chat(&mut self, room_id: &str, ids: Option<&[String]>) -> Result<()> {
|
||||
self.sources.delete_chat(room_id, ids)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ use huginn_mind::ChatMessage;
|
|||
|
||||
use crate::sources::{
|
||||
COLLECTION, CORPUS_CHAT, CORPUS_FORGE, ChatFilter, ChatOrder, ChatPayload, ChatPoint, ChatRow, DescribedSources, FRONT_PAGE_FILES, SourceIndex, SourcePayload, SourcePoint, SourceRow,
|
||||
SourcesMeta, StoredChat,
|
||||
SourcesMeta,
|
||||
};
|
||||
|
||||
use super::{CollectionMeta, INDEX_CALL_TIMEOUT, Described, Hit, Http, Method, Point, VectorIndex};
|
||||
|
|
@ -464,74 +464,6 @@ impl SourceIndex for QdrantIndex {
|
|||
let found = answer["result"]["points"].as_array().context("Qdrant's scroll has no points")?;
|
||||
found.iter().map(|point| chat_payload_of(&point["payload"])).collect()
|
||||
}
|
||||
|
||||
fn chat_tail(&mut self, room_id: &str) -> Result<Option<StoredChat>> {
|
||||
let order = json!({ "key": "ts_end_ms", "direction": "desc" });
|
||||
let Some(newest) = self.scroll_chat(chat_room_filter(room_id, vec![]), Some((order, 1)))?.into_iter().next() else { return Ok(None) };
|
||||
// Chunks tied on the greatest end are all read and the greatest id wins, so the tail is the same chunk on every run.
|
||||
let end = newest.payload.messages.iter().map(|message| message.ts_ms).max().unwrap_or(0);
|
||||
let tied = self.scroll_chat(chat_room_filter(room_id, vec![json!({ "key": "ts_end_ms", "range": { "gte": end } })]), None)?;
|
||||
Ok(tied.into_iter().max_by(|a, b| a.id.cmp(&b.id)))
|
||||
}
|
||||
|
||||
fn chat_chunks_holding(&mut self, room_id: &str, event_ids: &[String]) -> Result<Vec<StoredChat>> {
|
||||
if event_ids.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
self.scroll_chat(chat_room_filter(room_id, vec![json!({ "key": "event_ids", "match": { "any": event_ids } })]), None)
|
||||
}
|
||||
|
||||
fn chat_chunks_overlapping(&mut self, room_id: &str, from_ms: u64, to_ms: u64) -> Result<Vec<StoredChat>> {
|
||||
let span = vec![json!({ "key": "ts_end_ms", "range": { "gte": from_ms } }), json!({ "key": "ts_start_ms", "range": { "lt": to_ms } })];
|
||||
self.scroll_chat(chat_room_filter(room_id, span), None)
|
||||
}
|
||||
|
||||
fn delete_chat(&mut self, room_id: &str, ids: Option<&[String]>) -> Result<()> {
|
||||
let collection = self.sources.clone();
|
||||
let body = json!({ "filter": chat_delete_filter(room_id, ids) });
|
||||
self.ask(Method::Post, &format!("/collections/{collection}/points/delete?wait=true"), Some(&body), &[200])?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl QdrantIndex {
|
||||
/// Every chat chunk `filter` keeps, with its point id, paged; or, given an
|
||||
/// order and a limit, the first of them in that order.
|
||||
fn scroll_chat(&mut self, filter: Value, ordered: Option<(Value, u64)>) -> Result<Vec<StoredChat>> {
|
||||
let collection = self.sources.clone();
|
||||
let mut out = Vec::new();
|
||||
let mut offset = Value::Null;
|
||||
loop {
|
||||
let mut body = json!({ "filter": filter, "limit": SCROLL_PAGE, "with_payload": true, "with_vector": false });
|
||||
if let Some((order, limit)) = &ordered {
|
||||
body["order_by"] = order.clone();
|
||||
body["limit"] = json!(limit);
|
||||
} else if !offset.is_null() {
|
||||
body["offset"] = offset;
|
||||
}
|
||||
let (_, answer) = self.ask(Method::Post, &format!("/collections/{collection}/points/scroll"), Some(&body), &[200])?;
|
||||
for point in answer["result"]["points"].as_array().context("Qdrant's scroll has no points")? {
|
||||
out.push(StoredChat { id: point["id"].as_str().context("a point id is not a string")?.to_owned(), payload: chat_payload_of(&point["payload"])? });
|
||||
}
|
||||
offset = answer["result"]["next_page_offset"].clone();
|
||||
if ordered.is_some() || offset.is_null() {
|
||||
return Ok(out);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// What a chat delete removes: the room's chat points, all of them or those with `ids`. The corpus
|
||||
/// and room conditions hold either way, so a forge point or another room's point cannot match.
|
||||
fn chat_delete_filter(room_id: &str, ids: Option<&[String]>) -> Value {
|
||||
chat_room_filter(room_id, ids.map(|ids| json!({ "has_id": ids })).into_iter().collect())
|
||||
}
|
||||
|
||||
/// The chat points of one room, narrowed by `more` conditions.
|
||||
fn chat_room_filter(room_id: &str, more: Vec<Value>) -> Value {
|
||||
let mut must = vec![json!({ "key": "corpus", "match": { "value": CORPUS_CHAT } }), json!({ "key": "room_id", "match": { "value": room_id } })];
|
||||
must.extend(more);
|
||||
json!({ "must": must })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
@ -594,29 +526,6 @@ mod source_filter_tests {
|
|||
assert!(chat_payload_of(&forge).is_err());
|
||||
}
|
||||
|
||||
/// The writer-side filters name the chat corpus and the room, so a forge point never matches.
|
||||
#[test]
|
||||
fn the_writer_filters_are_always_chat_corpus_and_the_room() {
|
||||
let filter = chat_room_filter("!r:x", vec![json!({ "key": "event_ids", "match": { "any": ["$a"] } })]);
|
||||
assert_eq!(
|
||||
filter,
|
||||
json!({ "must": [
|
||||
{ "key": "corpus", "match": { "value": "chat" } },
|
||||
{ "key": "room_id", "match": { "value": "!r:x" } },
|
||||
{ "key": "event_ids", "match": { "any": ["$a"] } },
|
||||
] })
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_chat_delete_names_the_corpus_and_room_with_or_without_ids() {
|
||||
let all = chat_delete_filter("!r:x", None);
|
||||
assert_eq!(all["must"].as_array().unwrap().len(), 2);
|
||||
let some = chat_delete_filter("!r:x", Some(&["p1".to_owned()]));
|
||||
assert_eq!(some["must"][2], json!({ "has_id": ["p1"] }));
|
||||
assert_eq!(some["must"][0], json!({ "key": "corpus", "match": { "value": "chat" } }));
|
||||
}
|
||||
|
||||
/// The chat filter is the chat corpus, the rooms as an any-of `should` split
|
||||
/// by id and alias, and the two ranges that keep an overlapping span.
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -1,638 +0,0 @@
|
|||
//! Chat ingest: joined-room history into the shared source collection, beside
|
||||
//! the forge's repos. `refresh_chat` is the only writer of `corpus=chat` points
|
||||
//! and of the cursors object's `chat:` keys; `chat_purge::request_purge` is the
|
||||
//! only writer of `chat-purge:` keys, and `refresh_chat` the only applier.
|
||||
//!
|
||||
//! The Matrix client that implements `RoomHistory` is the next cut's and holds no
|
||||
//! policy. What is in scope is `room_scope`'s; what may be embedded is
|
||||
//! `huginn_mind::redact::scrub`'s; this file folds events into chunks and writes
|
||||
//! them, write-then-delete and cursor-last, so any crash replays to the same state.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet, HashSet};
|
||||
use std::time::Instant;
|
||||
|
||||
use anyhow::{Context, Result, bail, ensure};
|
||||
use huginn_mind::ChatMessage;
|
||||
use huginn_mind::redact::{ScrubKind, scrub};
|
||||
|
||||
use super::chat_chunk::{chat_version, chunk_id, chunk_messages, chunk_payload, cut_body};
|
||||
use super::chat_purge::{PurgeWindow, covers, parse_windows, purge_key};
|
||||
use super::room_scope::{Estate, Refusal, RoomFacts, admit_room, refuse_event};
|
||||
use super::{CORPUS, ChatPoint, DescribedSources, SourceIndex, StoredChat};
|
||||
use crate::index::{BATCH, BULK_EMBED_TIMEOUT, Embedder, INDEX_CALL_TIMEOUT};
|
||||
|
||||
/// Events asked of the port per page.
|
||||
pub const CHAT_PAGE: u32 = 200;
|
||||
|
||||
/// A chat cursor is the key `chat:<room_id>`; a forge repo name cannot contain
|
||||
/// a colon. The only definition of the prefix.
|
||||
pub const CHAT_CURSOR_PREFIX: &str = "chat:";
|
||||
|
||||
/// A steward's purge tombstone is the key `chat-purge:<room_id>`. The only
|
||||
/// definition of the prefix.
|
||||
pub const CHAT_PURGE_PREFIX: &str = "chat-purge:";
|
||||
|
||||
/// Whether a cursors-object key is chat's (a room cursor or a purge tombstone)
|
||||
/// and not a forge repo name. The only test that tells the two apart.
|
||||
pub fn is_chat_key(key: &str) -> bool {
|
||||
key.starts_with(CHAT_CURSOR_PREFIX) || key.starts_with(CHAT_PURGE_PREFIX)
|
||||
}
|
||||
|
||||
/// What one room event is to the ingest. Only text, notice and emote messages
|
||||
/// become `Message`; everything else (encrypted, media, state, server-redacted)
|
||||
/// is `Other` and is never stored.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum EventKind {
|
||||
Message { body: String },
|
||||
/// A replacement of `target`'s body.
|
||||
Edit { target: String, body: String },
|
||||
Redaction { target: String },
|
||||
Other,
|
||||
}
|
||||
|
||||
/// One event of a room, as the port hands it over: the Matrix client maps its
|
||||
/// JSON to this and filters nothing. `content_keys` are the top-level keys of
|
||||
/// the event's content (the scope reads them).
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct RoomEvent {
|
||||
pub event_id: String,
|
||||
pub sender: String,
|
||||
pub ts_ms: u64,
|
||||
pub content_keys: Vec<String>,
|
||||
pub kind: EventKind,
|
||||
}
|
||||
|
||||
/// One page of a room's events after a cursor event, oldest first.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct EventPage {
|
||||
pub events: Vec<RoomEvent>,
|
||||
/// More events follow this page.
|
||||
pub more: bool,
|
||||
}
|
||||
|
||||
/// The reader user's view of Synapse as ingest uses it. Holds no policy: it maps
|
||||
/// the server's answers to these types and filters nothing.
|
||||
pub trait RoomHistory {
|
||||
/// The rooms the reader has joined. The only source of rooms; there is no allow list.
|
||||
fn joined_rooms(&mut self) -> Result<Vec<String>>;
|
||||
fn facts(&mut self, room_id: &str) -> Result<RoomFacts>;
|
||||
/// Up to `limit` events after the event `after` (from the room's start when `None`).
|
||||
/// `Ok(None)` means `after` is unknown to the server.
|
||||
fn events_after(&mut self, room_id: &str, after: Option<&str>, limit: u32) -> Result<Option<EventPage>>;
|
||||
}
|
||||
|
||||
/// The cursors-object key of a room's chat cursor.
|
||||
pub fn cursor_key(room_id: &str) -> String {
|
||||
format!("{CHAT_CURSOR_PREFIX}{room_id}")
|
||||
}
|
||||
|
||||
/// The room a chat cursor key belongs to; `None` for any other key (a purge tombstone included).
|
||||
pub fn cursor_room(key: &str) -> Option<&str> {
|
||||
key.strip_prefix(CHAT_CURSOR_PREFIX)
|
||||
}
|
||||
|
||||
/// A cursor value, `<version>:<last event id>`; the event is empty before a room's first upsert.
|
||||
pub fn parse_cursor(value: &str) -> Option<(&str, &str)> {
|
||||
value.split_once(':')
|
||||
}
|
||||
|
||||
/// What one admitted room's run did, for the journal. Never carries a body or a sender.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct ChatReport {
|
||||
pub room_id: String,
|
||||
pub alias: Option<String>,
|
||||
pub first: Option<String>,
|
||||
pub last: Option<String>,
|
||||
pub events: u32,
|
||||
pub chunks: u32,
|
||||
pub scrubbed: u32,
|
||||
pub purged: u32,
|
||||
pub seconds: f64,
|
||||
/// Why the room could not be read or written this run; what the index held for it stays as it was.
|
||||
pub failure: Option<String>,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ChatReport {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(
|
||||
formatter,
|
||||
"{} ({}) {}..{} events={} chunks={} scrubbed={} purged={} seconds={:.1}",
|
||||
self.alias.as_deref().unwrap_or("-"),
|
||||
self.room_id,
|
||||
self.first.as_deref().unwrap_or("-"),
|
||||
self.last.as_deref().unwrap_or("-"),
|
||||
self.events,
|
||||
self.chunks,
|
||||
self.scrubbed,
|
||||
self.purged,
|
||||
self.seconds
|
||||
)?;
|
||||
match &self.failure {
|
||||
Some(why) => write!(formatter, " failed: {why}"),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A joined room the scope refused, whole, and why; `event_id` when an event did it.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct RefusedRoom {
|
||||
pub room_id: String,
|
||||
pub reason: Refusal,
|
||||
pub event_id: Option<String>,
|
||||
}
|
||||
|
||||
/// A credential the scrub replaced: where, never what.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ScrubHit {
|
||||
pub room_id: String,
|
||||
pub event_id: String,
|
||||
pub kind: ScrubKind,
|
||||
}
|
||||
|
||||
/// What a chat refresh did.
|
||||
#[derive(Clone, Debug, Default, PartialEq)]
|
||||
pub struct ChatRefreshed {
|
||||
pub rooms: Vec<ChatReport>,
|
||||
/// Rooms (or steps) that failed; the others went on.
|
||||
pub failed: Vec<String>,
|
||||
pub refused: Vec<RefusedRoom>,
|
||||
pub scrubbed: Vec<ScrubHit>,
|
||||
}
|
||||
|
||||
impl ChatReport {
|
||||
fn empty(room_id: &str, alias: Option<String>) -> Self {
|
||||
Self { room_id: room_id.to_owned(), alias, first: None, last: None, events: 0, chunks: 0, scrubbed: 0, purged: 0, seconds: 0.0, failure: None }
|
||||
}
|
||||
|
||||
/// A room that could not be handled this run. Its points and cursor are untouched: an error is not a reclassification.
|
||||
fn failed(room_id: &str, alias: Option<String>, error: &anyhow::Error) -> Self {
|
||||
Self { failure: Some(format!("{error:#}")), ..Self::empty(room_id, alias) }
|
||||
}
|
||||
}
|
||||
|
||||
impl ChatRefreshed {
|
||||
/// Nothing failed and no joined room had to be refused: a joined estate, direct,
|
||||
/// encrypted or bridged room means the reader's roster is wrong. A scrub hit does not make a run unclean.
|
||||
pub fn is_clean(&self) -> bool {
|
||||
self.failed.is_empty() && self.refused.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
fn read_cursors<S: SourceIndex>(index: &mut S) -> Result<BTreeMap<String, String>> {
|
||||
match index.describe_sources(INDEX_CALL_TIMEOUT).context("describing the source collection")? {
|
||||
DescribedSources::Labelled { cursors, .. } => Ok(cursors),
|
||||
_ => bail!("the source collection is gone; the forge refresh creates it"),
|
||||
}
|
||||
}
|
||||
|
||||
fn put_cursor<S: SourceIndex>(index: &mut S, key: &str, value: &str) -> Result<()> {
|
||||
let mut cursors = read_cursors(index)?;
|
||||
if cursors.insert(key.to_owned(), value.to_owned()).as_deref() != Some(value) {
|
||||
index.set_cursors(&cursors).context("writing a chat cursor")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn drop_cursor<S: SourceIndex>(index: &mut S, key: &str) -> Result<()> {
|
||||
let mut cursors = read_cursors(index)?;
|
||||
if cursors.remove(key).is_some() {
|
||||
index.set_cursors(&cursors).context("dropping a chat cursor")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A room's points, then its `chat:` cursor. Its purge tombstone stays.
|
||||
fn purge_room<S: SourceIndex>(index: &mut S, room_id: &str) -> Result<()> {
|
||||
index.delete_chat(room_id, None).with_context(|| format!("deleting the chunks of {room_id}"))?;
|
||||
drop_cursor(index, &cursor_key(room_id))
|
||||
}
|
||||
|
||||
/// What `retire_chat` did: the rooms it removed, and those it could not (each with why), which it went on past.
|
||||
#[derive(Debug, Default, PartialEq, Eq)]
|
||||
pub struct Retired {
|
||||
pub retired: Vec<String>,
|
||||
pub failed: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
/// Deletes the points and `chat:` cursor of every room not in `keep` (the reader left it,
|
||||
/// or it is refused or excluded). A room whose delete fails is reported and the others go on. Purge
|
||||
/// tombstones and forge repos are never touched. With the empty set it removes the whole chat corpus.
|
||||
pub fn retire_chat<S: SourceIndex>(index: &mut S, keep: &BTreeSet<String>) -> Result<Retired> {
|
||||
let rooms: Vec<String> = read_cursors(index)?.keys().filter_map(|key| cursor_room(key)).filter(|room| !keep.contains(*room)).map(str::to_owned).collect();
|
||||
let mut done = Retired::default();
|
||||
for room in rooms {
|
||||
match purge_room(index, &room) {
|
||||
Ok(()) => done.retired.push(room),
|
||||
Err(error) => done.failed.push((room, format!("{error:#}"))),
|
||||
}
|
||||
}
|
||||
Ok(done)
|
||||
}
|
||||
|
||||
/// A change to chunks already stored, in event order.
|
||||
enum Op {
|
||||
/// An edit by `sender` of `target`'s body; applies only to the original's sender.
|
||||
Edit { id: String, target: String, sender: String, body: String },
|
||||
/// A held message's body is what the page says it is (a replay).
|
||||
Set { target: String, body: String },
|
||||
Redact { target: String },
|
||||
}
|
||||
|
||||
impl Op {
|
||||
fn target(&self) -> &str {
|
||||
match self {
|
||||
Self::Edit { target, .. } | Self::Set { target, .. } | Self::Redact { target } => target,
|
||||
}
|
||||
}
|
||||
|
||||
/// Applies to `messages`; whether they changed.
|
||||
fn apply(&self, messages: &mut Vec<ChatMessage>) -> bool {
|
||||
let Some(at) = messages.iter().position(|message| message.event_id == self.target()) else { return false };
|
||||
match self {
|
||||
Self::Edit { sender, body, .. } if messages[at].sender == *sender && messages[at].body != *body => messages[at].body.clone_from(body),
|
||||
Self::Set { body, .. } if messages[at].body != *body => messages[at].body.clone_from(body),
|
||||
Self::Redact { .. } => {
|
||||
messages.remove(at);
|
||||
}
|
||||
_ => return false,
|
||||
}
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
/// A message of the page with the edits of the page that apply to it: the body is the last edit's, and a
|
||||
/// redacted edit leaves the list, so the body falls back to the one before it.
|
||||
struct Pending {
|
||||
message: ChatMessage,
|
||||
original: String,
|
||||
edits: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
struct Folded {
|
||||
pending: Vec<ChatMessage>,
|
||||
ops: Vec<Op>,
|
||||
/// Every event id of the page: a redaction of one of them is known.
|
||||
seen: HashSet<String>,
|
||||
}
|
||||
|
||||
struct Room<'a> {
|
||||
facts: &'a RoomFacts,
|
||||
label: String,
|
||||
windows: Vec<PurgeWindow>,
|
||||
}
|
||||
|
||||
/// The page folded in order: the purge filter, then the credential scrub, then the
|
||||
/// event's effect. After this no raw body exists.
|
||||
fn fold(room: &Room, events: &[RoomEvent], report: &mut ChatReport, out: &mut ChatRefreshed) -> Folded {
|
||||
let mut pending: Vec<Pending> = Vec::new();
|
||||
let mut ops: Vec<Op> = Vec::new();
|
||||
let mut scrubbed = |event: &RoomEvent, body: &str| {
|
||||
let clean = scrub(body);
|
||||
for kind in clean.hits {
|
||||
report.scrubbed += 1;
|
||||
out.scrubbed.push(ScrubHit { room_id: room.facts.room_id.clone(), event_id: event.event_id.clone(), kind });
|
||||
}
|
||||
cut_body(&clean.text)
|
||||
};
|
||||
for event in events {
|
||||
match &event.kind {
|
||||
EventKind::Message { body } => {
|
||||
if covers(&room.windows, event.ts_ms) {
|
||||
report.purged += 1;
|
||||
} else {
|
||||
let body = scrubbed(event, body);
|
||||
if !body.trim().is_empty() {
|
||||
let message = ChatMessage { event_id: event.event_id.clone(), sender: event.sender.clone(), ts_ms: event.ts_ms, body: body.clone() };
|
||||
pending.push(Pending { message, original: body, edits: Vec::new() });
|
||||
}
|
||||
}
|
||||
}
|
||||
EventKind::Edit { target, body } => {
|
||||
if covers(&room.windows, event.ts_ms) {
|
||||
report.purged += 1;
|
||||
continue;
|
||||
}
|
||||
let body = scrubbed(event, body);
|
||||
match pending.iter_mut().find(|held| held.message.event_id == *target) {
|
||||
Some(held) if held.message.sender == event.sender && !body.trim().is_empty() => {
|
||||
held.edits.push((event.event_id.clone(), body.clone()));
|
||||
held.message.body = body;
|
||||
}
|
||||
Some(_) => {}
|
||||
None => ops.push(Op::Edit { id: event.event_id.clone(), target: target.clone(), sender: event.sender.clone(), body }),
|
||||
}
|
||||
}
|
||||
EventKind::Redaction { target } => {
|
||||
// The target may be a message, or an edit whose own text must stop applying.
|
||||
pending.retain(|held| held.message.event_id != *target);
|
||||
for held in &mut pending {
|
||||
held.edits.retain(|(id, _)| id != target);
|
||||
held.message.body = held.edits.last().map_or_else(|| held.original.clone(), |(_, body)| body.clone());
|
||||
}
|
||||
ops.retain(|op| !matches!(op, Op::Edit { id, .. } if id == target));
|
||||
ops.push(Op::Redact { target: target.clone() });
|
||||
}
|
||||
EventKind::Other => {}
|
||||
}
|
||||
}
|
||||
Folded { pending: pending.into_iter().map(|held| held.message).collect(), ops, seen: events.iter().map(|event| event.event_id.clone()).collect() }
|
||||
}
|
||||
|
||||
/// Chunks to rewrite: the stored chunk ids they replace and the messages that now make them.
|
||||
type Rewrite = (Vec<String>, Vec<ChatMessage>);
|
||||
|
||||
/// Chunks `rewrites`, embeds and writes the new chunks (acknowledged), and only then deletes the old
|
||||
/// ids that are not among the new. Returns how many chunks were written.
|
||||
///
|
||||
/// A chunk written by an earlier run whose delete failed can still sit beside the chunk that replaced
|
||||
/// it, and a rewrite of the stale one then names the same first event, hence the same id, as the
|
||||
/// newer chunk. The newer chunk (`standing`, the other chunks the page found) stays; the stale one is deleted.
|
||||
fn write_rewrites<E: Embedder, S: SourceIndex>(
|
||||
embedder: &mut E,
|
||||
index: &mut S,
|
||||
dimensions: u32,
|
||||
room: &Room,
|
||||
rewrites: Vec<Rewrite>,
|
||||
standing: &BTreeSet<String>,
|
||||
) -> Result<u32> {
|
||||
let mut old: BTreeSet<String> = BTreeSet::new();
|
||||
let mut points: Vec<(String, super::ChatPayload)> = Vec::new();
|
||||
for (ids, messages) in rewrites {
|
||||
for group in chunk_messages(&room.label, &messages) {
|
||||
let id = chunk_id(&room.facts.room_id, &group[0].event_id);
|
||||
if standing.contains(&id) && !ids.contains(&id) {
|
||||
continue;
|
||||
}
|
||||
points.push((id, chunk_payload(&room.facts.room_id, room.facts.alias.as_deref(), &room.label, group)));
|
||||
}
|
||||
old.extend(ids);
|
||||
}
|
||||
let new: BTreeSet<&String> = points.iter().map(|(id, _)| id).collect();
|
||||
let stale: Vec<String> = old.iter().filter(|id| !new.contains(id)).cloned().collect();
|
||||
let written = u32::try_from(points.len()).unwrap_or(u32::MAX);
|
||||
for slice in points.chunks(BATCH) {
|
||||
let texts: Vec<String> = slice.iter().map(|(_, payload)| payload.text.clone()).collect();
|
||||
let vectors = embedder.embed(&texts, BULK_EMBED_TIMEOUT).context("embedding chat chunks")?;
|
||||
ensure!(vectors.len() == texts.len(), "the embedder returned {} vectors for {} chunks", vectors.len(), texts.len());
|
||||
let mut batch = Vec::with_capacity(slice.len());
|
||||
for ((id, payload), vector) in slice.iter().zip(vectors) {
|
||||
ensure!(vector.len() == dimensions as usize, "the embedder returned a vector of {} where the collection holds {dimensions}", vector.len());
|
||||
batch.push(ChatPoint { id: id.clone(), vector, payload: payload.clone() });
|
||||
}
|
||||
index.upsert_chat(&batch).context("writing chat chunks")?;
|
||||
}
|
||||
if !stale.is_empty() {
|
||||
index.delete_chat(&room.facts.room_id, Some(&stale)).context("deleting superseded chat chunks")?;
|
||||
}
|
||||
Ok(written)
|
||||
}
|
||||
|
||||
/// Applies the room's purge windows to the chunks it already holds.
|
||||
fn apply_purge<E: Embedder, S: SourceIndex>(embedder: &mut E, index: &mut S, dimensions: u32, room: &Room, report: &mut ChatReport) -> Result<()> {
|
||||
let mut hit: BTreeMap<String, StoredChat> = BTreeMap::new();
|
||||
for window in &room.windows {
|
||||
for chunk in index.chat_chunks_overlapping(&room.facts.room_id, window.from_ms, window.to_ms)? {
|
||||
hit.insert(chunk.id.clone(), chunk);
|
||||
}
|
||||
}
|
||||
let mut rewrites = Vec::new();
|
||||
for chunk in hit.into_values() {
|
||||
let kept: Vec<ChatMessage> = chunk.payload.messages.iter().filter(|message| !covers(&room.windows, message.ts_ms)).cloned().collect();
|
||||
let removed = chunk.payload.messages.len() - kept.len();
|
||||
if removed > 0 {
|
||||
report.purged += u32::try_from(removed).unwrap_or(u32::MAX);
|
||||
rewrites.push((vec![chunk.id], kept));
|
||||
}
|
||||
}
|
||||
report.chunks += write_rewrites(embedder, index, dimensions, room, rewrites, &BTreeSet::new())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Folds one page into the stored chunks: edits and redactions rewrite the chunk that held
|
||||
/// the target alone, new messages extend the open tail.
|
||||
fn write_page<E: Embedder, S: SourceIndex>(
|
||||
embedder: &mut E,
|
||||
index: &mut S,
|
||||
dimensions: u32,
|
||||
room: &Room,
|
||||
mut folded: Folded,
|
||||
may_rebuild: bool,
|
||||
) -> Result<Option<u32>> {
|
||||
let room_id = &room.facts.room_id;
|
||||
let mut asked: Vec<String> = folded.pending.iter().map(|message| message.event_id.clone()).collect();
|
||||
asked.extend(folded.ops.iter().map(|op| op.target().to_owned()));
|
||||
if asked.is_empty() {
|
||||
return Ok(Some(0));
|
||||
}
|
||||
let holding = index.chat_chunks_holding(room_id, &asked)?;
|
||||
// A message the index already holds is a replay: its page body is asserted, never appended again.
|
||||
let held: HashSet<&str> = holding.iter().flat_map(|chunk| chunk.payload.messages.iter().map(|message| message.event_id.as_str())).collect();
|
||||
// A redaction of an event this room neither holds nor shows on the page may be of an edit whose own id is
|
||||
// never stored, so a body it changed cannot be taken back here: the room is rebuilt from the history, which
|
||||
// serves a redacted edit without its text.
|
||||
if may_rebuild && folded.ops.iter().any(|op| matches!(op, Op::Redact { target } if !folded.seen.contains(target) && !held.contains(target.as_str()))) {
|
||||
return Ok(None);
|
||||
}
|
||||
let (replayed, fresh): (Vec<ChatMessage>, Vec<ChatMessage>) = folded.pending.drain(..).partition(|message| held.contains(message.event_id.as_str()));
|
||||
let mut ops: Vec<Op> = replayed.into_iter().map(|message| Op::Set { target: message.event_id, body: message.body }).collect();
|
||||
ops.append(&mut folded.ops);
|
||||
|
||||
let tail = index.chat_tail(room_id)?;
|
||||
let tail_id = tail.as_ref().map(|tail| tail.id.clone());
|
||||
let standing: BTreeSet<String> = holding.iter().map(|chunk| chunk.id.clone()).chain(tail_id.clone()).collect();
|
||||
let mut tail_messages = tail.map(|tail| tail.payload.messages).unwrap_or_default();
|
||||
let mut tail_changed = false;
|
||||
let mut rewrites: Vec<Rewrite> = Vec::new();
|
||||
for chunk in holding {
|
||||
let mut messages = chunk.payload.messages.clone();
|
||||
let mut changed = false;
|
||||
for op in &ops {
|
||||
changed |= op.apply(&mut messages);
|
||||
}
|
||||
if Some(&chunk.id) == tail_id.as_ref() {
|
||||
(tail_messages, tail_changed) = (messages, changed);
|
||||
} else if changed {
|
||||
rewrites.push((vec![chunk.id], messages));
|
||||
}
|
||||
}
|
||||
if !fresh.is_empty() || tail_changed {
|
||||
tail_messages.extend(fresh);
|
||||
rewrites.push((tail_id.into_iter().collect(), tail_messages));
|
||||
}
|
||||
write_rewrites(embedder, index, dimensions, room, rewrites, &standing).map(Some)
|
||||
}
|
||||
|
||||
enum Outcome {
|
||||
Done(ChatReport),
|
||||
Refused(RefusedRoom),
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn ingest_room<E: Embedder, S: SourceIndex, H: RoomHistory>(
|
||||
embedder: &mut E,
|
||||
index: &mut S,
|
||||
history: &mut H,
|
||||
dimensions: u32,
|
||||
facts: &RoomFacts,
|
||||
out: &mut ChatRefreshed,
|
||||
) -> Result<Outcome> {
|
||||
let started = Instant::now();
|
||||
let room_id = facts.room_id.as_str();
|
||||
let (key, version) = (cursor_key(room_id), chat_version());
|
||||
let cursors = read_cursors(index)?;
|
||||
let mut may_rebuild = false;
|
||||
let mut after: Option<String> = match cursors.get(&key).map(|value| parse_cursor(value)) {
|
||||
None => None,
|
||||
Some(Some((held, event))) if held == version => {
|
||||
may_rebuild = true;
|
||||
Some(event.to_owned()).filter(|event| !event.is_empty())
|
||||
}
|
||||
// Another chunker or scrub version, or a value nobody here wrote: this room alone starts over.
|
||||
Some(_) => {
|
||||
index.delete_chat(room_id, None).with_context(|| format!("deleting the chunks of {room_id} for a re-ingest"))?;
|
||||
None
|
||||
}
|
||||
};
|
||||
if after.is_none() {
|
||||
put_cursor(index, &key, &format!("{version}:"))?;
|
||||
}
|
||||
let windows = cursors.get(&purge_key(room_id)).map(|value| parse_windows(value)).transpose()?.unwrap_or_default();
|
||||
let mut report = ChatReport::empty(room_id, facts.alias.clone());
|
||||
// The room's name is embedded and stored in every chunk, so it passes the scrub like a body.
|
||||
let label = match &facts.name {
|
||||
Some(name) => {
|
||||
let clean = scrub(name);
|
||||
for kind in clean.hits {
|
||||
report.scrubbed += 1;
|
||||
out.scrubbed.push(ScrubHit { room_id: room_id.to_owned(), event_id: "(name)".to_owned(), kind });
|
||||
}
|
||||
clean.text
|
||||
}
|
||||
None => facts.alias.clone().unwrap_or_else(|| room_id.to_owned()),
|
||||
};
|
||||
let room = Room { facts, label, windows };
|
||||
apply_purge(embedder, index, dimensions, &room, &mut report)?;
|
||||
let mut restarted = false;
|
||||
loop {
|
||||
let Some(page) = history.events_after(room_id, after.as_deref(), CHAT_PAGE)? else {
|
||||
ensure!(after.is_some() && !restarted, "the server does not know the start of {room_id}'s history");
|
||||
restarted = true;
|
||||
purge_room(index, room_id)?;
|
||||
put_cursor(index, &key, &format!("{version}:"))?;
|
||||
after = None;
|
||||
continue;
|
||||
};
|
||||
let Some(last) = page.events.last().map(|event| event.event_id.clone()) else { break };
|
||||
// A refusing event refuses the room before anything of the page is chunked or embedded.
|
||||
if let Some((event, reason)) = page.events.iter().find_map(|event| refuse_event(event).map(|reason| (event, reason))) {
|
||||
purge_room(index, room_id)?;
|
||||
return Ok(Outcome::Refused(RefusedRoom { room_id: room_id.to_owned(), reason, event_id: Some(event.event_id.clone()) }));
|
||||
}
|
||||
report.first.get_or_insert_with(|| page.events[0].event_id.clone());
|
||||
report.events += u32::try_from(page.events.len()).unwrap_or(u32::MAX);
|
||||
let folded = fold(&room, &page.events, &mut report, out);
|
||||
let Some(written) = write_page(embedder, index, dimensions, &room, folded, may_rebuild)? else {
|
||||
// Start this room over from the history; nothing it read is trusted to be complete.
|
||||
purge_room(index, room_id)?;
|
||||
put_cursor(index, &key, &format!("{version}:"))?;
|
||||
(after, may_rebuild) = (None, false);
|
||||
report = ChatReport { scrubbed: report.scrubbed, ..ChatReport::empty(room_id, facts.alias.clone()) };
|
||||
continue;
|
||||
};
|
||||
report.chunks += written;
|
||||
put_cursor(index, &key, &format!("{version}:{last}"))?;
|
||||
report.last = Some(last.clone());
|
||||
after = Some(last);
|
||||
if !page.more {
|
||||
break;
|
||||
}
|
||||
}
|
||||
report.seconds = started.elapsed().as_secs_f64();
|
||||
Ok(Outcome::Done(report))
|
||||
}
|
||||
|
||||
/// Brings the chat corpus to the reader's joined rooms. The collection must already exist and be
|
||||
/// Huginn's, built under this embedder's model: the forge refresh creates and rebuilds it, and this never does.
|
||||
/// A joined room the scope refuses is purged and named; a room the reader left is retired; the rest
|
||||
/// are ingested from their cursors. One room's failure is journaled and the others go on.
|
||||
/// `journal` receives every line the run prints: reports, scrub hits, refusals and failures.
|
||||
pub fn refresh_chat<E: Embedder, S: SourceIndex, H: RoomHistory>(
|
||||
embedder: &mut E,
|
||||
index: &mut S,
|
||||
history: &mut H,
|
||||
estate: &Estate,
|
||||
journal: &mut dyn FnMut(String),
|
||||
) -> Result<ChatRefreshed> {
|
||||
let DescribedSources::Labelled { meta, .. } = index.describe_sources(INDEX_CALL_TIMEOUT).context("describing the source collection")? else {
|
||||
bail!("the source collection is absent or not Huginn's; the forge refresh creates it and chat never does");
|
||||
};
|
||||
ensure!(meta.corpus == CORPUS, "the source collection was built for corpus {} and chat ingest writes {CORPUS}; the forge refresh rebuilds it", meta.corpus);
|
||||
embedder.built_under(Some(&meta.model_identity()));
|
||||
let identity = embedder.model_identity(BULK_EMBED_TIMEOUT).context("reading the embedding model's identity")?;
|
||||
ensure!(meta.model_identity() == identity, "the source collection was built under {} {} and chat embeds under {} {}", meta.model, meta.model_digest, identity.name, identity.digest);
|
||||
|
||||
let mut out = ChatRefreshed::default();
|
||||
let mut joined = history.joined_rooms().context("listing the reader's joined rooms")?;
|
||||
let mut seen = HashSet::new();
|
||||
joined.retain(|room| seen.insert(room.clone()));
|
||||
let mut keep: BTreeSet<String> = joined.iter().cloned().collect();
|
||||
let mut admitted: Vec<RoomFacts> = Vec::new();
|
||||
for room_id in &joined {
|
||||
let verdict = if estate.holds_id(room_id) {
|
||||
Ok(Err(Refusal::Estate))
|
||||
} else {
|
||||
history.facts(room_id).map(|facts| admit_room(&facts, estate).map(|()| facts))
|
||||
};
|
||||
match verdict {
|
||||
Ok(Ok(facts)) => admitted.push(facts),
|
||||
Ok(Err(reason)) => {
|
||||
// Not kept, so the retirement below purges whatever it held.
|
||||
keep.remove(room_id);
|
||||
journal(format!("refused: {room_id} {reason}"));
|
||||
out.refused.push(RefusedRoom { room_id: room_id.clone(), reason, event_id: None });
|
||||
}
|
||||
// An error is not a reclassification: the room keeps its points and cursor, and the report says why.
|
||||
Err(error) => {
|
||||
let report = ChatReport::failed(room_id, None, &error.context("reading the room's facts"));
|
||||
journal(report.to_string());
|
||||
out.failed.push(room_id.clone());
|
||||
out.rooms.push(report);
|
||||
}
|
||||
}
|
||||
}
|
||||
match retire_chat(index, &keep) {
|
||||
Ok(done) => {
|
||||
for (room, why) in done.failed {
|
||||
journal(format!("huginn-sources: retiring {room} failed, the others go on: {why}"));
|
||||
out.failed.push(room);
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
journal(format!("huginn-sources: retiring rooms failed: {error:#}"));
|
||||
out.failed.push("retire".to_owned());
|
||||
}
|
||||
}
|
||||
for facts in admitted {
|
||||
match ingest_room(embedder, index, history, identity.dimensions, &facts, &mut out) {
|
||||
Ok(Outcome::Done(report)) => {
|
||||
journal(report.to_string());
|
||||
out.rooms.push(report);
|
||||
}
|
||||
Ok(Outcome::Refused(refused)) => {
|
||||
journal(format!("refused: {} {} [{}]", refused.room_id, refused.reason, refused.event_id.as_deref().unwrap_or("")));
|
||||
out.refused.push(refused);
|
||||
}
|
||||
Err(error) => {
|
||||
let report = ChatReport::failed(&facts.room_id, facts.alias.clone(), &error);
|
||||
journal(report.to_string());
|
||||
out.failed.push(facts.room_id.clone());
|
||||
out.rooms.push(report);
|
||||
}
|
||||
}
|
||||
}
|
||||
for hit in &out.scrubbed {
|
||||
journal(format!("scrubbed: {} {} {}", hit.room_id, hit.event_id, hit.kind));
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
|
@ -1,166 +0,0 @@
|
|||
//! The chat chunker: greedy, deterministic and local. A chunk is consecutive
|
||||
//! messages of one room; it closes at a time gap, at a message count, or when
|
||||
//! its embedded text would pass a character bound. A message is never split.
|
||||
|
||||
use chrono::DateTime;
|
||||
use huginn_mind::redact::SCRUB_VERSION;
|
||||
use huginn_mind::ChatMessage;
|
||||
|
||||
use super::ChatPayload;
|
||||
use crate::index::point_id;
|
||||
|
||||
/// Bumped whenever chunk boundaries or the embedded text would change for the
|
||||
/// same messages. Part of `chat_version`.
|
||||
pub const CHAT_CHUNKER_VERSION: u32 = 1;
|
||||
|
||||
/// A chunk closes when the next message is more than this much later.
|
||||
pub const CHAT_GAP_MS: u64 = 30 * 60 * 1000;
|
||||
/// The most messages a chunk holds.
|
||||
pub const CHAT_MESSAGES_MAX: usize = 24;
|
||||
/// The most characters a chunk's embedded text holds.
|
||||
pub const CHAT_CHARS_MAX: usize = 1_600;
|
||||
/// The most characters a stored message body holds, marker included.
|
||||
pub const CHAT_BODY_CHARS_MAX: usize = 1_200;
|
||||
|
||||
/// Ends a body that was cut.
|
||||
const CUT_MARKER: &str = "…[cut]";
|
||||
|
||||
/// What the room's cursor and chunk ids are stamped with: the chunker and the
|
||||
/// scrub. Either changing re-ingests the room.
|
||||
pub fn chat_version() -> String {
|
||||
format!("{CHAT_CHUNKER_VERSION}.{SCRUB_VERSION}")
|
||||
}
|
||||
|
||||
/// A chunk's id: where it starts and under which version it was cut.
|
||||
pub fn chunk_id(room_id: &str, first_event_id: &str) -> String {
|
||||
point_id(&format!("{room_id}\0{first_event_id}\0{}", chat_version()))
|
||||
}
|
||||
|
||||
/// A scrubbed body cut to `CHAT_BODY_CHARS_MAX` characters on a character boundary, with a marker.
|
||||
pub fn cut_body(body: &str) -> String {
|
||||
if body.chars().count() <= CHAT_BODY_CHARS_MAX {
|
||||
return body.to_owned();
|
||||
}
|
||||
let keep = CHAT_BODY_CHARS_MAX - CUT_MARKER.chars().count();
|
||||
body.chars().take(keep).chain(CUT_MARKER.chars()).collect()
|
||||
}
|
||||
|
||||
fn line_of(message: &ChatMessage) -> String {
|
||||
let when = DateTime::from_timestamp_millis(i64::try_from(message.ts_ms).unwrap_or(i64::MAX)).map_or_else(String::new, |at| at.format("%Y-%m-%d %H:%M").to_string());
|
||||
format!("{when} {}: {}", message.sender, message.body)
|
||||
}
|
||||
|
||||
/// What is embedded for a chunk: the room's label (its name, else its alias,
|
||||
/// else its id), then one `YYYY-MM-DD HH:MM sender: body` line per message, UTC.
|
||||
pub fn embedded_text(label: &str, messages: &[ChatMessage]) -> String {
|
||||
let mut text = label.to_owned();
|
||||
for message in messages {
|
||||
text.push('\n');
|
||||
text.push_str(&line_of(message));
|
||||
}
|
||||
text
|
||||
}
|
||||
|
||||
/// `messages` (one room, in order) cut into chunks, each a run of consecutive messages.
|
||||
pub fn chunk_messages(label: &str, messages: &[ChatMessage]) -> Vec<Vec<ChatMessage>> {
|
||||
let mut chunks: Vec<Vec<ChatMessage>> = Vec::new();
|
||||
let mut chars = 0_usize;
|
||||
for message in messages {
|
||||
let line = line_of(message).chars().count() + 1;
|
||||
let closes = chunks.last().is_some_and(|open| {
|
||||
let last = open.last().map_or(0, |last| last.ts_ms);
|
||||
message.ts_ms.saturating_sub(last) > CHAT_GAP_MS || open.len() >= CHAT_MESSAGES_MAX || chars + line > CHAT_CHARS_MAX
|
||||
});
|
||||
if closes || chunks.is_empty() {
|
||||
chunks.push(Vec::new());
|
||||
chars = label.chars().count();
|
||||
}
|
||||
chunks.last_mut().expect("a chunk is open").push(message.clone());
|
||||
chars += line;
|
||||
}
|
||||
chunks
|
||||
}
|
||||
|
||||
/// The point payload of one chunk.
|
||||
pub fn chunk_payload(room_id: &str, alias: Option<&str>, label: &str, messages: Vec<ChatMessage>) -> ChatPayload {
|
||||
ChatPayload { room_id: room_id.to_owned(), room_alias: alias.map(str::to_owned), text: embedded_text(label, &messages), messages }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn message(n: u64, ts_ms: u64, body: &str) -> ChatMessage {
|
||||
ChatMessage { event_id: format!("$e{n}"), sender: "@a:s".into(), ts_ms, body: body.into() }
|
||||
}
|
||||
|
||||
fn ids(chunks: &[Vec<ChatMessage>]) -> Vec<Vec<&str>> {
|
||||
chunks.iter().map(|chunk| chunk.iter().map(|message| message.event_id.as_str()).collect()).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_gap_a_count_and_a_character_bound_each_close_a_chunk() {
|
||||
let gap = [message(1, 0, "a"), message(2, CHAT_GAP_MS, "b"), message(3, 2 * CHAT_GAP_MS + 1, "c")];
|
||||
assert_eq!(ids(&chunk_messages("r", &gap)), [["$e1", "$e2"].as_slice(), ["$e3"].as_slice()], "a gap of exactly the bound does not close");
|
||||
let many: Vec<ChatMessage> = (0..25).map(|n| message(n, n, "x")).collect();
|
||||
let chunks = chunk_messages("r", &many);
|
||||
assert_eq!((chunks[0].len(), chunks[1].len()), (CHAT_MESSAGES_MAX, 1));
|
||||
let long: Vec<ChatMessage> = (0..4).map(|n| message(n, n, &"w".repeat(500))).collect();
|
||||
let chunks = chunk_messages("r", &long);
|
||||
assert!(chunks.iter().all(|chunk| embedded_text("r", chunk).chars().count() <= CHAT_CHARS_MAX), "{:?}", ids(&chunks));
|
||||
assert!(chunks.len() > 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_embedded_text_is_the_label_then_utc_lines() {
|
||||
let text = embedded_text("Swarm", &[message(1, 1_700_000_000_000, "hello")]);
|
||||
assert_eq!(text, "Swarm\n2023-11-14 22:13 @a:s: hello");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_long_body_is_cut_at_the_bound_with_a_marker_on_a_char_boundary_and_a_short_one_is_whole() {
|
||||
let cut = cut_body(&"é".repeat(CHAT_BODY_CHARS_MAX + 50));
|
||||
assert_eq!(cut.chars().count(), CHAT_BODY_CHARS_MAX);
|
||||
assert!(cut.ends_with(CUT_MARKER));
|
||||
assert_eq!(cut_body(&"é".repeat(CHAT_BODY_CHARS_MAX)), "é".repeat(CHAT_BODY_CHARS_MAX));
|
||||
let one = chunk_messages("r", &[message(1, 0, &cut)]);
|
||||
assert_eq!(one.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_chunk_id_is_its_room_first_event_and_version() {
|
||||
assert_eq!(chunk_id("!r:s", "$a"), chunk_id("!r:s", "$a"));
|
||||
assert_ne!(chunk_id("!r:s", "$a"), chunk_id("!r:s", "$b"));
|
||||
assert_ne!(chunk_id("!r:s", "$a"), chunk_id("!q:s", "$a"));
|
||||
assert_eq!(chat_version(), format!("{CHAT_CHUNKER_VERSION}.{SCRUB_VERSION}"));
|
||||
}
|
||||
|
||||
/// Chunking the whole sequence equals re-chunking the open tail with each new batch appended.
|
||||
#[test]
|
||||
fn appending_to_the_open_tail_gives_the_boundaries_of_chunking_all_at_once() {
|
||||
let mut seed = 0x2545_f491_4f6c_dd1d_u64;
|
||||
let mut next = move |bound: u64| {
|
||||
seed ^= seed << 13;
|
||||
seed ^= seed >> 7;
|
||||
seed ^= seed << 17;
|
||||
seed % bound
|
||||
};
|
||||
let mut ts = 0;
|
||||
let all: Vec<ChatMessage> = (0..300)
|
||||
.map(|n| {
|
||||
ts += if next(7) == 0 { CHAT_GAP_MS + 1 } else { next(60_000) };
|
||||
message(n, ts, &"z".repeat(next(700) as usize + 1))
|
||||
})
|
||||
.collect();
|
||||
let whole = chunk_messages("r", &all);
|
||||
for step in [1_usize, 5, 17, 100] {
|
||||
let mut held: Vec<Vec<ChatMessage>> = Vec::new();
|
||||
for batch in all.chunks(step) {
|
||||
let mut tail: Vec<ChatMessage> = held.pop().unwrap_or_default();
|
||||
tail.extend_from_slice(batch);
|
||||
held.extend(chunk_messages("r", &tail));
|
||||
}
|
||||
assert_eq!(held, whole, "step {step}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,103 +0,0 @@
|
|||
//! A steward's purge of a room's time window. The tombstone is the one chat
|
||||
//! state that is not derived from Synapse: a durable request, kept in the
|
||||
//! collection's cursors object under `chat-purge:<room_id>`, that must outlive
|
||||
//! the points it removes, so a re-ingest never brings the window back.
|
||||
//!
|
||||
//! `request_purge` is the only writer; `refresh_chat` reads the tombstone and
|
||||
//! applies it at its next run. The code does not authenticate the steward: whoever
|
||||
//! can run the job as the unit's user (the host admin) may ask, and the request
|
||||
//! carries a ref in the job's journal for a trace. Tombstones live in the
|
||||
//! collection, so a full collection rebuild forgets them; re-issue them from
|
||||
//! their refs.
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
|
||||
use super::chat::CHAT_PURGE_PREFIX;
|
||||
use super::{DescribedSources, SourceIndex};
|
||||
use crate::index::INDEX_CALL_TIMEOUT;
|
||||
|
||||
/// The most windows a room's tombstone holds.
|
||||
pub const PURGE_WINDOWS_MAX: usize = 32;
|
||||
|
||||
/// Messages with `from_ms <= ts_ms < to_ms` are purged.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct PurgeWindow {
|
||||
pub from_ms: u64,
|
||||
pub to_ms: u64,
|
||||
}
|
||||
|
||||
impl PurgeWindow {
|
||||
pub fn covers(&self, ts_ms: u64) -> bool {
|
||||
self.from_ms <= ts_ms && ts_ms < self.to_ms
|
||||
}
|
||||
}
|
||||
|
||||
/// The tombstone key of a room.
|
||||
pub fn purge_key(room_id: &str) -> String {
|
||||
format!("{CHAT_PURGE_PREFIX}{room_id}")
|
||||
}
|
||||
|
||||
/// A tombstone value, `<from>..<to>[,<from>..<to>...]`.
|
||||
pub fn parse_windows(value: &str) -> Result<Vec<PurgeWindow>> {
|
||||
value
|
||||
.split(',')
|
||||
.map(|part| {
|
||||
let (from, to) = part.split_once("..").with_context(|| format!("purge window {part:?} is not <from>..<to>"))?;
|
||||
let window = PurgeWindow { from_ms: from.parse().context("a purge window's start is not milliseconds")?, to_ms: to.parse().context("a purge window's end is not milliseconds")? };
|
||||
if window.from_ms >= window.to_ms {
|
||||
bail!("purge window {part:?} is empty");
|
||||
}
|
||||
Ok(window)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Windows sorted, with overlapping and touching ones joined.
|
||||
pub fn merge(mut windows: Vec<PurgeWindow>) -> Vec<PurgeWindow> {
|
||||
windows.sort();
|
||||
let mut merged: Vec<PurgeWindow> = Vec::new();
|
||||
for window in windows {
|
||||
match merged.last_mut() {
|
||||
Some(last) if window.from_ms <= last.to_ms => last.to_ms = last.to_ms.max(window.to_ms),
|
||||
_ => merged.push(window),
|
||||
}
|
||||
}
|
||||
merged
|
||||
}
|
||||
|
||||
pub fn encode(windows: &[PurgeWindow]) -> String {
|
||||
windows.iter().map(|window| format!("{}..{}", window.from_ms, window.to_ms)).collect::<Vec<_>>().join(",")
|
||||
}
|
||||
|
||||
/// Whether any window covers `ts_ms`.
|
||||
pub fn covers(windows: &[PurgeWindow], ts_ms: u64) -> bool {
|
||||
windows.iter().any(|window| window.covers(ts_ms))
|
||||
}
|
||||
|
||||
/// Records that `window` of `room_id` must leave the index and never return. Writes the cursors
|
||||
/// object only if the tombstone changed, and refuses a room's 33rd window. Writes no point and
|
||||
/// embeds nothing; the next `refresh_chat` applies it. The job's `chat-purge` subcommand calls
|
||||
/// it under the job's lock, because `set_cursors` replaces the cursors object whole.
|
||||
/// Returns whether the tombstone changed.
|
||||
pub fn request_purge<S: SourceIndex>(index: &mut S, room_id: &str, window: PurgeWindow) -> Result<bool> {
|
||||
if window.from_ms >= window.to_ms {
|
||||
bail!("a purge window must start before it ends");
|
||||
}
|
||||
let DescribedSources::Labelled { mut cursors, .. } = index.describe_sources(INDEX_CALL_TIMEOUT).context("describing the source collection")? else {
|
||||
bail!("the source collection is absent or not Huginn's; the forge refresh creates it");
|
||||
};
|
||||
let key = purge_key(room_id);
|
||||
let mut windows = cursors.get(&key).map(|held| parse_windows(held)).transpose()?.unwrap_or_default();
|
||||
windows.push(window);
|
||||
let windows = merge(windows);
|
||||
if windows.len() > PURGE_WINDOWS_MAX {
|
||||
bail!("room {room_id} already holds {PURGE_WINDOWS_MAX} purge windows; widen one instead");
|
||||
}
|
||||
let encoded = encode(&windows);
|
||||
if cursors.get(&key) == Some(&encoded) {
|
||||
return Ok(false);
|
||||
}
|
||||
cursors.insert(key, encoded);
|
||||
index.set_cursors(&cursors).context("writing the purge tombstone")?;
|
||||
Ok(true)
|
||||
}
|
||||
|
|
@ -1,964 +0,0 @@
|
|||
//! Chat ingest against a scripted room history, a fake vector store and the fake embedder.
|
||||
//! The refusals come first: what must never be read, embedded or stored.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Result, bail};
|
||||
use huginn_mind::ChatMessage;
|
||||
use huginn_mind::redact::ScrubKind;
|
||||
|
||||
use super::chat::*;
|
||||
use super::chat_chunk::{CHAT_BODY_CHARS_MAX, chat_version, chunk_id};
|
||||
use super::chat_purge::{PURGE_WINDOWS_MAX, PurgeWindow, request_purge};
|
||||
use super::room_scope::{ESTATE_ALIASES, Estate, Refusal, RoomFacts};
|
||||
use super::*;
|
||||
use crate::index::MANAGED_BY;
|
||||
use crate::index::fakes::{FakeEmbedder, FakeSources};
|
||||
|
||||
// ---- the rig ----
|
||||
|
||||
fn plain(id: &str) -> RoomFacts {
|
||||
RoomFacts { room_id: id.into(), alias: None, alt_aliases: vec![], name: None, encrypted: false, direct: false, bridge: None }
|
||||
}
|
||||
|
||||
fn estate(ids: &[&str]) -> Estate {
|
||||
Estate::new(ids.iter().chain(["!pad1:s", "!pad2:s", "!pad3:s"].iter()).take(3).map(|id| (*id).to_owned()).collect()).unwrap()
|
||||
}
|
||||
|
||||
struct Scripted {
|
||||
facts: RoomFacts,
|
||||
events: Vec<RoomEvent>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct FakeHistory {
|
||||
joined: Vec<String>,
|
||||
rooms: BTreeMap<String, Scripted>,
|
||||
calls: Vec<String>,
|
||||
/// Rooms whose state cannot be read.
|
||||
facts_fail: BTreeSet<String>,
|
||||
/// Serve an event a redaction targets as the server does, without its content.
|
||||
prune: bool,
|
||||
}
|
||||
|
||||
impl FakeHistory {
|
||||
fn room(&mut self, id: &str, events: Vec<RoomEvent>) -> &mut Self {
|
||||
self.rooms.insert(id.into(), Scripted { facts: plain(id), events });
|
||||
self.joined.push(id.into());
|
||||
self
|
||||
}
|
||||
|
||||
fn facts(&mut self, id: &str, edit: impl FnOnce(&mut RoomFacts)) -> &mut Self {
|
||||
edit(&mut self.rooms.get_mut(id).unwrap().facts);
|
||||
self
|
||||
}
|
||||
|
||||
fn push(&mut self, id: &str, events: Vec<RoomEvent>) {
|
||||
self.rooms.get_mut(id).unwrap().events.extend(events);
|
||||
}
|
||||
|
||||
fn called(&self, prefix: &str, room: &str) -> bool {
|
||||
self.calls.iter().any(|call| call == &format!("{prefix}:{room}"))
|
||||
}
|
||||
}
|
||||
|
||||
impl RoomHistory for FakeHistory {
|
||||
fn joined_rooms(&mut self) -> Result<Vec<String>> {
|
||||
self.calls.push("joined".into());
|
||||
Ok(self.joined.clone())
|
||||
}
|
||||
|
||||
fn facts(&mut self, room_id: &str) -> Result<RoomFacts> {
|
||||
self.calls.push(format!("facts:{room_id}"));
|
||||
if self.facts_fail.contains(room_id) {
|
||||
bail!("the server did not answer for the state of {room_id}");
|
||||
}
|
||||
match self.rooms.get(room_id) {
|
||||
Some(room) => Ok(room.facts.clone()),
|
||||
None => bail!("no such room"),
|
||||
}
|
||||
}
|
||||
|
||||
fn events_after(&mut self, room_id: &str, after: Option<&str>, limit: u32) -> Result<Option<EventPage>> {
|
||||
self.calls.push(format!("events:{room_id}"));
|
||||
let Some(room) = self.rooms.get(room_id) else { bail!("no such room") };
|
||||
let from = match after {
|
||||
None => 0,
|
||||
Some(after) => match room.events.iter().position(|event| event.event_id == after) {
|
||||
Some(at) => at + 1,
|
||||
None => return Ok(None),
|
||||
},
|
||||
};
|
||||
let redacted: BTreeSet<&str> = room.events.iter().filter_map(|event| if let EventKind::Redaction { target } = &event.kind { Some(target.as_str()) } else { None }).collect();
|
||||
let served: Vec<RoomEvent> = room.events[from..]
|
||||
.iter()
|
||||
.map(|event| if self.prune && redacted.contains(event.event_id.as_str()) { RoomEvent { kind: EventKind::Other, content_keys: vec![], ..event.clone() } } else { event.clone() })
|
||||
.collect();
|
||||
let take = served.len().min(limit as usize);
|
||||
Ok(Some(EventPage { events: served[..take].to_vec(), more: take < served.len() }))
|
||||
}
|
||||
}
|
||||
|
||||
fn event(id: &str, sender: &str, ts_ms: u64, kind: EventKind) -> RoomEvent {
|
||||
RoomEvent { event_id: id.into(), sender: sender.into(), ts_ms, content_keys: vec!["body".into(), "msgtype".into()], kind }
|
||||
}
|
||||
|
||||
fn said(id: &str, ts_ms: u64, body: &str) -> RoomEvent {
|
||||
event(id, "@a:s", ts_ms, EventKind::Message { body: body.into() })
|
||||
}
|
||||
|
||||
fn edit(id: &str, sender: &str, ts_ms: u64, target: &str, body: &str) -> RoomEvent {
|
||||
event(id, sender, ts_ms, EventKind::Edit { target: target.into(), body: body.into() })
|
||||
}
|
||||
|
||||
fn redaction(id: &str, ts_ms: u64, target: &str) -> RoomEvent {
|
||||
event(id, "@a:s", ts_ms, EventKind::Redaction { target: target.into() })
|
||||
}
|
||||
|
||||
struct Rig {
|
||||
embedder: FakeEmbedder,
|
||||
index: FakeSources,
|
||||
estate: Estate,
|
||||
}
|
||||
|
||||
impl Rig {
|
||||
fn new() -> Self {
|
||||
let mut embedder = FakeEmbedder::new("d1");
|
||||
let index = FakeSources::default();
|
||||
let identity = crate::index::Embedder::model_identity(&mut embedder, Duration::from_secs(1)).unwrap();
|
||||
let meta = SourcesMeta {
|
||||
managed_by: MANAGED_BY.into(),
|
||||
corpus: CORPUS.into(),
|
||||
model: identity.name,
|
||||
model_digest: identity.digest,
|
||||
dimensions: identity.dimensions,
|
||||
chunker_version: chunk::CHUNKER_VERSION,
|
||||
};
|
||||
index.plant(DescribedSources::Labelled { meta, cursors: BTreeMap::new() });
|
||||
Self { embedder, index, estate: estate(&[]) }
|
||||
}
|
||||
|
||||
fn run(&mut self, history: &mut FakeHistory) -> (Result<ChatRefreshed>, Vec<String>) {
|
||||
let mut journal = Vec::new();
|
||||
let out = refresh_chat(&mut self.embedder, &mut self.index, history, &self.estate, &mut |line| journal.push(line));
|
||||
(out, journal)
|
||||
}
|
||||
|
||||
fn ok(&mut self, history: &mut FakeHistory) -> ChatRefreshed {
|
||||
self.run(history).0.unwrap()
|
||||
}
|
||||
|
||||
fn points(&self) -> Vec<ChatPoint> {
|
||||
self.index.state.lock().unwrap().chat.values().cloned().collect()
|
||||
}
|
||||
|
||||
fn ids(&self) -> BTreeSet<String> {
|
||||
self.index.state.lock().unwrap().chat.keys().cloned().collect()
|
||||
}
|
||||
|
||||
fn bodies(&self) -> Vec<String> {
|
||||
let mut bodies: Vec<String> = self.points().iter().flat_map(|point| point.payload.messages.iter().map(|message| message.body.clone())).collect();
|
||||
bodies.sort();
|
||||
bodies
|
||||
}
|
||||
|
||||
fn held_ids(&self, room: &str) -> Vec<String> {
|
||||
let mut ids: Vec<String> =
|
||||
self.points().iter().filter(|point| point.payload.room_id == room).flat_map(|point| point.payload.messages.iter().map(|message| message.event_id.clone())).collect();
|
||||
ids.sort();
|
||||
ids
|
||||
}
|
||||
|
||||
fn cursor(&self, room: &str) -> Option<String> {
|
||||
self.index.cursors().get(&cursor_key(room)).cloned()
|
||||
}
|
||||
|
||||
fn embedded(&self) -> Vec<String> {
|
||||
self.embedder.texts()
|
||||
}
|
||||
|
||||
fn seed(&mut self, room: &str, event_id: &str, ts: u64) {
|
||||
let mut seeded = vec![ChatPoint {
|
||||
id: chunk_id(room, event_id),
|
||||
vector: vec![1.0; 4],
|
||||
payload: chunk_payload_of(room, &[(event_id, ts)]),
|
||||
}];
|
||||
self.index.upsert_chat(&seeded.split_off(0)).unwrap();
|
||||
let mut cursors = self.index.cursors();
|
||||
cursors.insert(cursor_key(room), format!("{}:{event_id}", chat_version()));
|
||||
self.index.set_cursors(&cursors).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
fn chunk_payload_of(room: &str, messages: &[(&str, u64)]) -> ChatPayload {
|
||||
let messages: Vec<ChatMessage> = messages.iter().map(|(id, ts)| ChatMessage { event_id: (*id).into(), sender: "@a:s".into(), ts_ms: *ts, body: format!("seeded {id}") }).collect();
|
||||
ChatPayload { room_id: room.into(), room_alias: None, text: "seeded".into(), messages }
|
||||
}
|
||||
|
||||
fn tombstone(rig: &Rig, room: &str) -> Option<String> {
|
||||
rig.index.cursors().get(&format!("{CHAT_PURGE_PREFIX}{room}")).cloned()
|
||||
}
|
||||
|
||||
// ---- scope: refusals first ----
|
||||
|
||||
#[test]
|
||||
fn only_joined_rooms_are_looked_at_and_a_join_rule_is_not_the_test() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!a:s", vec![said("$a1", 1, "in a")]).room("!b:s", vec![said("$b1", 2, "in b")]);
|
||||
// A room the reader has not joined: scripted, but never listed.
|
||||
history.rooms.insert("!c:s".into(), Scripted { facts: plain("!c:s"), events: vec![said("$c1", 3, "in c")] });
|
||||
let mut rig = Rig::new();
|
||||
let done = rig.ok(&mut history);
|
||||
assert!(done.is_clean());
|
||||
assert!(!history.calls.iter().any(|call| call.contains("!c:s")), "{:?}", history.calls);
|
||||
assert_eq!(rig.held_ids("!a:s"), ["$a1"]);
|
||||
assert_eq!(rig.held_ids("!b:s"), ["$b1"]);
|
||||
assert!(rig.held_ids("!c:s").is_empty() && done.rooms.iter().all(|report| report.room_id != "!c:s"));
|
||||
let code = include_str!("room_scope.rs");
|
||||
assert!(!code[..code.find("#[cfg(test)]").unwrap()].contains(&["join", "_rule"].concat()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_dm_room_is_refused_unread_unindexed_and_purged() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!dm:s", vec![said("$d1", 1, "private")]).facts("!dm:s", |facts| facts.direct = true);
|
||||
let mut rig = Rig::new();
|
||||
rig.seed("!dm:s", "$old", 5);
|
||||
let (done, journal) = rig.run(&mut history);
|
||||
let done = done.unwrap();
|
||||
assert!(!history.called("events", "!dm:s"));
|
||||
assert!(rig.points().is_empty() && rig.cursor("!dm:s").is_none() && rig.embedded().is_empty());
|
||||
assert_eq!(done.refused, [RefusedRoom { room_id: "!dm:s".into(), reason: Refusal::Direct, event_id: None }]);
|
||||
assert!(!done.is_clean() && journal.iter().any(|line| line == "refused: !dm:s direct room"), "{journal:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn each_estate_room_is_refused_by_id_and_by_alias_and_an_excluded_room_in_the_index_is_purged() {
|
||||
let mut rig = Rig::new();
|
||||
rig.estate = estate(&["!byid:s"]);
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!byid:s", vec![said("$i1", 1, "x")]);
|
||||
for (n, alias) in ESTATE_ALIASES.iter().map(|alias| alias.to_string()).chain(["#USERS:GAMECULT.ORG".to_owned()]).enumerate() {
|
||||
history.room(&format!("!al{n}:s"), vec![said("$x", 1, "x")]).facts(&format!("!al{n}:s"), |facts| facts.alias = Some(alias));
|
||||
}
|
||||
history.room("!alt:s", vec![said("$x", 1, "x")]).facts("!alt:s", |facts| facts.alt_aliases = vec!["#other:s".into(), "#patrons:gamecult.org".into()]);
|
||||
history.room("!ok:s", vec![said("$ok1", 1, "fine")]);
|
||||
history.room("!guide:s", vec![said("$g1", 1, "fine")]).facts("!guide:s", |facts| facts.alias = Some("#users-guide:gamecult.org".into()));
|
||||
history.room("!club:s", vec![said("$c1", 1, "fine")]).facts("!club:s", |facts| facts.alias = Some("#patrons-club:gamecult.org".into()));
|
||||
for room in ["!byid:s", "!al0:s", "!al1:s", "!al2:s", "!al3:s", "!alt:s"] {
|
||||
rig.seed(room, "$old", 5);
|
||||
}
|
||||
let mut cursors = rig.index.cursors();
|
||||
cursors.insert(format!("{CHAT_PURGE_PREFIX}!al0:s"), "1..2".into());
|
||||
rig.index.set_cursors(&cursors).unwrap();
|
||||
let done = rig.ok(&mut history);
|
||||
assert!(!history.called("facts", "!byid:s") && !history.called("events", "!byid:s"));
|
||||
for room in ["!al0:s", "!al1:s", "!al2:s", "!al3:s", "!alt:s"] {
|
||||
assert!(history.called("facts", room) && !history.called("events", room), "{room}");
|
||||
}
|
||||
let refused: BTreeSet<&str> = done.refused.iter().map(|refused| refused.room_id.as_str()).collect();
|
||||
assert_eq!(refused, ["!al0:s", "!al1:s", "!al2:s", "!al3:s", "!alt:s", "!byid:s"].into());
|
||||
assert!(done.refused.iter().all(|refused| refused.reason == Refusal::Estate) && !done.is_clean());
|
||||
let held: BTreeSet<String> = rig.points().iter().map(|point| point.payload.room_id.clone()).collect();
|
||||
assert_eq!(held, ["!guide:s".to_owned(), "!club:s".to_owned(), "!ok:s".to_owned()].into(), "lookalikes and plain rooms are in; every estate room is out");
|
||||
assert!(rig.index.cursors().keys().all(|key| !key.starts_with("chat:!al") && !key.starts_with("chat:!byid") && !key.starts_with("chat:!alt")));
|
||||
assert_eq!(tombstone(&rig, "!al0:s").as_deref(), Some("1..2"), "a tombstone is never touched by a refusal");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_encrypted_room_is_refused_and_an_encrypted_event_in_a_plain_room_is_never_stored() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!enc:s", vec![said("$e1", 1, "x")]).facts("!enc:s", |facts| facts.encrypted = true);
|
||||
history.room("!plain:s", vec![said("$p1", 1, "visible"), event("$p2", "@a:s", 2, EventKind::Other), said("$p3", 3, "also visible")]);
|
||||
let mut rig = Rig::new();
|
||||
let done = rig.ok(&mut history);
|
||||
assert_eq!(done.refused[0].reason, Refusal::Encrypted);
|
||||
assert!(!history.called("events", "!enc:s"));
|
||||
assert_eq!(rig.held_ids("!plain:s"), ["$p1", "$p3"]);
|
||||
assert_eq!(rig.cursor("!plain:s").unwrap(), format!("{}:$p3", chat_version()), "the cursor passes an Other event");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_bridged_room_is_refused_whole_before_any_event_is_read_and_its_chunks_are_purged() {
|
||||
let mut history = FakeHistory::default();
|
||||
for (n, state_type) in ["m.bridge", "uk.half-shot.bridge"].into_iter().enumerate() {
|
||||
let room = format!("!br{n}:s");
|
||||
history.room(&room, vec![said("$x", 1, "x")]).facts(&room, |facts| facts.bridge = Some(state_type.into()));
|
||||
}
|
||||
let mut rig = Rig::new();
|
||||
rig.seed("!br0:s", "$old", 5);
|
||||
let done = rig.ok(&mut history);
|
||||
assert_eq!(done.refused.len(), 2);
|
||||
assert!(done.refused.iter().all(|refused| matches!(refused.reason, Refusal::Bridged(_))));
|
||||
assert!(!history.calls.iter().any(|call| call.starts_with("events:")));
|
||||
assert!(rig.points().is_empty() && rig.index.cursors().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_bridged_platforms_sender_or_content_refuses_the_room_before_anything_is_embedded_and_purges_what_it_held() {
|
||||
let cases = [
|
||||
event("$bad", "@discord_123:s", 2, EventKind::Message { body: "SECRET-BODY-TEXT".into() }),
|
||||
event("$bad", "@_discord_9:s", 2, EventKind::Message { body: "SECRET-BODY-TEXT".into() }),
|
||||
event("$bad", "@discordbot:s", 2, EventKind::Message { body: "SECRET-BODY-TEXT".into() }),
|
||||
event("$bad", "@mautrix_discord:s", 2, EventKind::Message { body: "SECRET-BODY-TEXT".into() }),
|
||||
event("$bad", "@_bridge_discordbot:s", 2, EventKind::Message { body: "SECRET-BODY-TEXT".into() }),
|
||||
RoomEvent { content_keys: vec!["body".into(), "fi.mau.discord".into()], ..said("$bad", 2, "SECRET-BODY-TEXT") },
|
||||
RoomEvent { content_keys: vec!["uk.half-shot.discord".into()], ..said("$bad", 2, "SECRET-BODY-TEXT") },
|
||||
];
|
||||
for bad in cases {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$ok", 1, "before"), bad]);
|
||||
history.room("!other:s", vec![said("$o1", 1, "other room")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.seed("!r:s", "$prev", 0);
|
||||
let (done, journal) = rig.run(&mut history);
|
||||
let done = done.unwrap();
|
||||
assert!(!rig.embedded().iter().any(|text| text.contains("SECRET-BODY-TEXT") || text.contains("before")), "nothing of the page is embedded");
|
||||
assert!(rig.held_ids("!r:s").is_empty() && rig.cursor("!r:s").is_none());
|
||||
assert_eq!(rig.held_ids("!other:s"), ["$o1"]);
|
||||
assert_eq!(done.refused.len(), 1);
|
||||
assert_eq!((done.refused[0].room_id.as_str(), done.refused[0].event_id.as_deref()), ("!r:s", Some("$bad")));
|
||||
assert!(!done.is_clean());
|
||||
assert!(journal.iter().all(|line| !line.contains("SECRET-BODY-TEXT")), "{journal:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_lookalike_of_the_bridged_platform_is_kept() {
|
||||
let mut history = FakeHistory::default();
|
||||
let kept = ["@discourse:s", "@disco:s", "@eureka:s"].map(|sender| event(&format!("${}", &sender[1..6]), sender, 1, EventKind::Message { body: "hi".into() }));
|
||||
let mut keyed = said("$k", 2, "hi");
|
||||
keyed.content_keys.push("discourse".into());
|
||||
let mut events = kept.to_vec();
|
||||
events.push(keyed);
|
||||
history.room("!r:s", events);
|
||||
let mut rig = Rig::new();
|
||||
assert!(rig.ok(&mut history).is_clean());
|
||||
assert_eq!(rig.held_ids("!r:s").len(), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_collection_is_never_created_by_chat() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 1, "x")]);
|
||||
for described in [None, Some(DescribedSources::Absent), Some(DescribedSources::Unlabelled)] {
|
||||
let mut rig = Rig::new();
|
||||
match described {
|
||||
None => rig.index.state.lock().unwrap().described = None,
|
||||
Some(described) => rig.index.plant(described),
|
||||
}
|
||||
let error = format!("{:#}", rig.run(&mut history).0.unwrap_err());
|
||||
assert!(error.contains("forge refresh creates it"), "{error}");
|
||||
assert_eq!(rig.index.state.lock().unwrap().recreated, 0);
|
||||
assert!(rig.points().is_empty() && rig.embedded().is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
// ---- redactions, edits, replays ----
|
||||
|
||||
#[test]
|
||||
fn a_redacted_event_is_absent_after_the_next_refresh_and_a_first_or_only_message_changes_or_drops_its_chunk() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 10, "alpha"), said("$b", 20, "beta"), said("$c", 30, "gamma")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
let first_id = chunk_id("!r:s", "$a");
|
||||
assert_eq!(rig.ids(), [first_id.clone()].into());
|
||||
history.push("!r:s", vec![redaction("$r1", 40, "$b")]);
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.held_ids("!r:s"), ["$a", "$c"]);
|
||||
assert!(!rig.points()[0].payload.text.contains("beta"));
|
||||
history.push("!r:s", vec![redaction("$r2", 50, "$a")]);
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.ids(), [chunk_id("!r:s", "$c")].into(), "the first message left, so the id changed and the old one is gone");
|
||||
history.push("!r:s", vec![redaction("$r3", 60, "$c")]);
|
||||
rig.ok(&mut history);
|
||||
assert!(rig.points().is_empty(), "a chunk the redaction empties is deleted");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_redaction_in_the_same_page_as_its_message_never_reaches_the_embedder() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 10, "keep me"), said("$gone", 20, "REDACTED-BODY"), redaction("$r", 30, "$gone")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
assert!(rig.embedded().iter().all(|text| !text.contains("REDACTED-BODY")));
|
||||
assert_eq!(rig.held_ids("!r:s"), ["$a"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_edit_replaces_the_body_in_place_and_a_stranger_an_unknown_target_or_an_edit_of_an_edit_change_nothing() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 10, "old words"), said("$b", 20, "other")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
let before = rig.points();
|
||||
history.push("!r:s", vec![edit("$e1", "@a:s", 30, "$a", "new words")]);
|
||||
rig.ok(&mut history);
|
||||
let after = rig.points();
|
||||
assert_eq!(after.len(), 1);
|
||||
let first = &after[0].payload.messages[0];
|
||||
assert_eq!((first.event_id.as_str(), first.sender.as_str(), first.ts_ms, first.body.as_str()), ("$a", "@a:s", 10, "new words"));
|
||||
assert_eq!(after[0].id, before[0].id);
|
||||
assert!(after[0].payload.text.contains("new words") && !after[0].payload.text.contains("old words"));
|
||||
assert!(rig.embedded().last().unwrap().contains("new words"));
|
||||
let calls = rig.embedder.state.lock().unwrap().embedded.len();
|
||||
history.push("!r:s", vec![edit("$e2", "@stranger:s", 40, "$a", "hijack"), edit("$e3", "@a:s", 41, "$nothing", "lost"), edit("$e4", "@a:s", 42, "$e1", "edit of an edit")]);
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.points(), after);
|
||||
assert_eq!(rig.embedder.state.lock().unwrap().embedded.len(), calls, "no embed call for edits that change nothing");
|
||||
assert_eq!(rig.cursor("!r:s").unwrap(), format!("{}:$e4", chat_version()), "the cursor is the last event of any kind");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_after_a_crash_between_the_upsert_and_the_cursor_neither_duplicates_nor_loses() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 10, "alpha"), said("$b", 20, "beta"), edit("$e", "@a:s", 25, "$a", "alpha two"), said("$c", 30, "gamma")]);
|
||||
let mut rig = Rig::new();
|
||||
// One cursor write (the empty start) succeeds, then the page's upsert is acknowledged and the next write fails.
|
||||
rig.index.state.lock().unwrap().fail_cursor_writes_after = Some(1);
|
||||
assert!(!rig.run(&mut history).0.unwrap().failed.is_empty());
|
||||
let (ids, held) = (rig.ids(), rig.held_ids("!r:s"));
|
||||
assert_eq!(held, ["$a", "$b", "$c"]);
|
||||
rig.index.state.lock().unwrap().fail_cursor_writes_after = None;
|
||||
let done = rig.ok(&mut history);
|
||||
assert!(done.is_clean());
|
||||
assert_eq!((rig.ids(), rig.held_ids("!r:s")), (ids, held));
|
||||
assert!(rig.bodies().contains(&"alpha two".to_owned()));
|
||||
assert_eq!(rig.cursor("!r:s").unwrap(), format!("{}:$c", chat_version()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failed_upsert_leaves_the_cursor_and_a_failed_embed_or_delete_deletes_nothing_the_index_held() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 10, "alpha"), said("$b", 20, "beta")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
let (cursor, points) = (rig.cursor("!r:s"), rig.points());
|
||||
history.push("!r:s", vec![said("$c", 30, "gamma")]);
|
||||
rig.index.state.lock().unwrap().fail_chat_upserts = true;
|
||||
assert_eq!(rig.run(&mut history).0.unwrap().failed, ["!r:s"]);
|
||||
assert_eq!((rig.cursor("!r:s"), rig.points()), (cursor.clone(), points.clone()));
|
||||
rig.index.state.lock().unwrap().fail_chat_upserts = false;
|
||||
rig.embedder.state.lock().unwrap().embed_fails = true;
|
||||
assert_eq!(rig.run(&mut history).0.unwrap().failed, ["!r:s"]);
|
||||
assert_eq!((rig.cursor("!r:s"), rig.points()), (cursor.clone(), points.clone()));
|
||||
rig.embedder.state.lock().unwrap().embed_fails = false;
|
||||
// A redaction of the first message writes a new chunk, then fails to delete the old one: the cursor stays.
|
||||
history.push("!r:s", vec![redaction("$r", 40, "$a")]);
|
||||
rig.index.state.lock().unwrap().fail_chat_deletes = true;
|
||||
assert_eq!(rig.run(&mut history).0.unwrap().failed, ["!r:s"]);
|
||||
assert_eq!(rig.cursor("!r:s"), cursor);
|
||||
rig.index.state.lock().unwrap().fail_chat_deletes = false;
|
||||
assert!(rig.ok(&mut history).is_clean());
|
||||
assert_eq!(rig.held_ids("!r:s"), ["$b", "$c"]);
|
||||
}
|
||||
|
||||
// ---- chunking through the job ----
|
||||
|
||||
#[test]
|
||||
fn appending_to_the_tail_over_many_runs_equals_one_run_over_all() {
|
||||
let all: Vec<RoomEvent> = (0..80).map(|n| said(&format!("$m{n:03}"), 1_000 * n + if n >= 40 { 5_000_000 } else { 0 }, &format!("message number {n} {}", "x".repeat((n as usize * 37) % 400)))).collect();
|
||||
let mut whole_history = FakeHistory::default();
|
||||
whole_history.room("!r:s", all.clone());
|
||||
let mut whole = Rig::new();
|
||||
whole.ok(&mut whole_history);
|
||||
let mut grown_history = FakeHistory::default();
|
||||
grown_history.room("!r:s", vec![]);
|
||||
let mut grown = Rig::new();
|
||||
for batch in all.chunks(7) {
|
||||
grown_history.push("!r:s", batch.to_vec());
|
||||
grown.ok(&mut grown_history);
|
||||
}
|
||||
assert!(whole.points().len() > 2);
|
||||
let shape = |rig: &Rig| rig.points().into_iter().map(|point| (point.id, point.payload)).collect::<Vec<_>>();
|
||||
assert_eq!(shape(&grown), shape(&whole));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_long_message_is_cut_with_a_marker_and_every_point_satisfies_the_payload_contract() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 10, &"é".repeat(CHAT_BODY_CHARS_MAX * 2)), said("$b", 20, "short"), said("$c", 99_000_000, "later")]);
|
||||
history.facts("!r:s", |facts| {
|
||||
facts.alias = Some("#swarm:gamecult.org".into());
|
||||
facts.name = Some("Swarm".into());
|
||||
});
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
let points = rig.points();
|
||||
assert_eq!(points.len(), 2);
|
||||
for point in &points {
|
||||
let payload = &point.payload;
|
||||
assert_eq!((payload.room_id.as_str(), payload.room_alias.as_deref()), ("!r:s", Some("#swarm:gamecult.org")));
|
||||
assert_eq!(point.id, chunk_id("!r:s", &payload.messages[0].event_id));
|
||||
assert_eq!(payload.text, super::chat_chunk::embedded_text("Swarm", &payload.messages));
|
||||
assert!(!payload.messages.is_empty());
|
||||
}
|
||||
let long = &points.iter().find(|point| point.payload.messages[0].event_id == "$a").unwrap().payload.messages[0];
|
||||
assert_eq!(long.body.chars().count(), CHAT_BODY_CHARS_MAX);
|
||||
assert!(long.body.ends_with("[cut]"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_lost_cursor_event_or_a_version_bump_re_ingests_that_room_alone() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 10, "alpha"), said("$b", 20, "beta")]).room("!q:s", vec![said("$q1", 10, "quiet")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
let q_ids: BTreeSet<String> = rig.points().iter().filter(|point| point.payload.room_id == "!q:s").map(|point| point.id.clone()).collect();
|
||||
let recreated = rig.index.state.lock().unwrap().recreated;
|
||||
for bump in [Some("0.0".to_owned()), None] {
|
||||
let mut cursors = rig.index.cursors();
|
||||
let value = match &bump {
|
||||
Some(old) => format!("{old}:$b"),
|
||||
None => format!("{}:$gone-from-the-server", chat_version()),
|
||||
};
|
||||
cursors.insert(cursor_key("!r:s"), value);
|
||||
rig.index.set_cursors(&cursors).unwrap();
|
||||
let before = rig.embedded().len();
|
||||
assert!(rig.ok(&mut history).is_clean());
|
||||
assert_eq!(rig.held_ids("!r:s"), ["$a", "$b"]);
|
||||
assert!(rig.embedded().len() > before, "the room was re-embedded");
|
||||
assert_eq!(rig.points().iter().filter(|point| point.payload.room_id == "!q:s").map(|point| point.id.clone()).collect::<BTreeSet<_>>(), q_ids);
|
||||
assert_eq!(rig.cursor("!r:s").unwrap(), format!("{}:$b", chat_version()));
|
||||
}
|
||||
assert_eq!(rig.index.state.lock().unwrap().recreated, recreated);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retire_chat_removes_rooms_no_longer_kept_and_given_none_all_chat_and_touches_nothing_else() {
|
||||
let mut rig = Rig::new();
|
||||
rig.index.upsert_sources(&[SourcePoint {
|
||||
id: "forge-1".into(),
|
||||
vector: vec![1.0; 4],
|
||||
payload: SourcePayload { repo: "Repo".into(), path: "a".into(), commit: "c".into(), blob: "b".into(), line_start: 1, line_end: 1, chunk_index: 0, ext: "rs".into(), text: "t".into() },
|
||||
}])
|
||||
.unwrap();
|
||||
for room in ["!a:s", "!b:s"] {
|
||||
rig.seed(room, "$e", 1);
|
||||
}
|
||||
let mut cursors = rig.index.cursors();
|
||||
cursors.insert("Repo".into(), "c".into());
|
||||
cursors.insert(format!("{CHAT_PURGE_PREFIX}!a:s"), "1..2".into());
|
||||
rig.index.set_cursors(&cursors).unwrap();
|
||||
let retired = retire_chat(&mut rig.index, &["!a:s".to_owned()].into()).unwrap();
|
||||
assert_eq!(retired.retired, ["!b:s"]);
|
||||
assert_eq!(rig.held_ids("!a:s"), ["$e"]);
|
||||
assert!(rig.held_ids("!b:s").is_empty() && rig.cursor("!b:s").is_none());
|
||||
assert_eq!(retire_chat(&mut rig.index, &BTreeSet::new()).unwrap().retired, ["!a:s"]);
|
||||
assert!(rig.points().is_empty());
|
||||
assert_eq!(rig.index.state.lock().unwrap().points.len(), 1, "a forge point stays");
|
||||
let kept: Vec<String> = rig.index.cursors().into_keys().collect();
|
||||
assert_eq!(kept, ["Repo".to_owned(), format!("{CHAT_PURGE_PREFIX}!a:s")]);
|
||||
// A room the reader left is retired by the refresh itself.
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 1, "x")]);
|
||||
rig.seed("!gone:s", "$e", 1);
|
||||
rig.ok(&mut history);
|
||||
assert!(rig.held_ids("!gone:s").is_empty());
|
||||
}
|
||||
|
||||
// ---- the credential scrub through the job ----
|
||||
|
||||
fn planted() -> Vec<(ScrubKind, String, String)> {
|
||||
let t = |kind, secret: String, needle: &str| (kind, secret, needle.to_owned());
|
||||
vec![
|
||||
t(ScrubKind::AuthorizationHeader, format!("Authorization: Bearer {}", "a1B2".repeat(8)), "a1B2a1B2a1B2"),
|
||||
t(ScrubKind::Bearer, format!("Bearer {}", "z9Y8".repeat(6)), "z9Y8z9Y8z9Y8"),
|
||||
t(ScrubKind::Bearer, format!("bearer:{}", "z9Y8".repeat(6)), "z9Y8z9Y8z9Y8"),
|
||||
t(ScrubKind::KeyShape, format!("sk-{}", "q7W3".repeat(8)), "q7W3q7W3q7W3"),
|
||||
t(ScrubKind::KeyShape, format!("sk_live_{}", "m4N5".repeat(6)), "m4N5m4N5m4N5"),
|
||||
t(ScrubKind::KeyShape, format!("ghp_{}", "k2L9".repeat(9)), "k2L9k2L9k2L9"),
|
||||
t(ScrubKind::KeyShape, format!("xxx-ghp_{}", "k2L9".repeat(9)), "k2L9k2L9k2L9"),
|
||||
t(ScrubKind::KeyShape, format!("github_pat_{}_{}", "A1b2C3".repeat(4) + "ab", "d4E5f6".repeat(9) + "abcde"), "A1b2C3A1b2C3"),
|
||||
t(ScrubKind::KeyShape, format!("glpat-{}", "r6T7".repeat(5)), "r6T7r6T7r6T7"),
|
||||
t(ScrubKind::KeyShape, "xoxb-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL"),
|
||||
t(ScrubKind::KeyShape, "xoxa-2-123456789012-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL"),
|
||||
t(ScrubKind::KeyShape, "xoxp-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL"),
|
||||
t(ScrubKind::KeyShape, "xoxr-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL"),
|
||||
t(ScrubKind::KeyShape, "xoxs-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL"),
|
||||
t(ScrubKind::KeyShape, "AKIA0123456789ABCDEF".into(), "0123456789ABCDEF"),
|
||||
t(ScrubKind::KeyShape, "xAKIA0123456789ABCDEF".into(), "0123456789ABCDEF"),
|
||||
t(ScrubKind::KeyShape, format!("AIza{}", "Sy1_".repeat(9)), "Sy1_Sy1_Sy1_"),
|
||||
t(ScrubKind::KeyShape, format!("syt_{}_{}_{}", "YWxpY2U", "aBcDeFgHiJkLmNoPqR", "0a1B2c"), "aBcDeFgHiJkLmNoPqR"),
|
||||
t(ScrubKind::KeyShape, format!("mct_{}", "w4X5".repeat(8)), "w4X5w4X5w4X5"),
|
||||
t(ScrubKind::AccessToken, "https://h/_matrix/client/v3/sync?access_token=abcDEF123456xyz&since=s1".into(), "abcDEF123456xyz"),
|
||||
t(ScrubKind::AccessToken, "{\"access_token\": \"tokenValue987654\", \"user\": 1}".into(), "tokenValue987654"),
|
||||
t(ScrubKind::Jwt, "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJmYWtlIn0.c2lnbmF0dXJlZmFrZQ".into(), "c2lnbmF0dXJlZmFrZQ"),
|
||||
t(ScrubKind::PemBlock, "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAfake\n-----END RSA PRIVATE KEY-----".into(), "MIIEowIBAAKCAQEAfake"),
|
||||
t(ScrubKind::PemBlock, "-----BEGIN PRIVATE KEY-----\nMIIBVQIBADANfakeNOEND".into(), "MIIBVQIBADANfake"),
|
||||
t(ScrubKind::PemBlock, "-----BEGIN PGP PRIVATE KEY BLOCK-----\nlQOYBFakeFakeFake\n-----END PGP PRIVATE KEY BLOCK-----".into(), "lQOYBFakeFake"),
|
||||
t(ScrubKind::UrlUserinfo, "https://user:hunter2hunter2@host.example/r.git".into(), "hunter2hunter2"),
|
||||
t(ScrubKind::UrlUserinfo, "https://tokenvalue12345@host.example/".into(), "tokenvalue12345"),
|
||||
]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn planted_credentials_never_reach_the_chunk_the_embedder_or_the_journal_in_a_message_or_an_edit() {
|
||||
for (kind, secret, needle) in planted() {
|
||||
for shape in ["see {} now", "see {}", "{}\nnext line"] {
|
||||
let (dirty, clean) = (shape.replace("{}", &secret), "nothing here".to_owned());
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$m", 10, &dirty), said("$e", 20, &clean), edit("$ed", "@a:s", 30, "$e", &dirty)]);
|
||||
let mut rig = Rig::new();
|
||||
let (done, journal) = rig.run(&mut history);
|
||||
let done = done.unwrap();
|
||||
assert!(done.is_clean(), "a scrub hit does not make a run unclean");
|
||||
let stored = format!("{:?}", rig.points().iter().map(|point| (&point.payload.messages, &point.payload.text)).collect::<Vec<_>>());
|
||||
for seen in [stored, rig.embedded().join("\n"), journal.join("\n")] {
|
||||
assert!(!seen.contains(&needle), "{kind}: {secret:?} leaked into {seen}");
|
||||
}
|
||||
assert!(rig.bodies().iter().filter(|body| body.contains("<redacted>")).count() >= 2, "{kind}: both bodies show the replacement");
|
||||
let hits: Vec<(&str, ScrubKind)> = done.scrubbed.iter().map(|hit| (hit.event_id.as_str(), hit.kind)).collect();
|
||||
assert!(hits.contains(&("$m", kind)) && hits.contains(&("$ed", kind)), "{kind}: {hits:?}");
|
||||
assert!(done.scrubbed.iter().all(|hit| hit.room_id == "!r:s"));
|
||||
assert!(journal.contains(&format!("scrubbed: !r:s $m {kind}")), "{journal:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_scrub_meets_edits_redactions_the_body_cut_replays_and_version_bumps() {
|
||||
let token = format!("ghp_{}", "k2L9".repeat(9));
|
||||
let mut history = FakeHistory::default();
|
||||
history.room(
|
||||
"!r:s",
|
||||
vec![
|
||||
said("$clean", 10, "all clean"),
|
||||
said("$dirty", 20, &format!("token {token} inside")),
|
||||
edit("$adds", "@a:s", 30, "$clean", &format!("now with {token}")),
|
||||
edit("$removes", "@a:s", 40, "$dirty", "token removed"),
|
||||
said("$gone", 50, &format!("secret {token}")),
|
||||
redaction("$r", 60, "$gone"),
|
||||
said("$straddle", 70, &format!("{} {token} {}", "x".repeat(CHAT_BODY_CHARS_MAX - 10), "y".repeat(100))),
|
||||
],
|
||||
);
|
||||
let mut rig = Rig::new();
|
||||
let done = rig.ok(&mut history);
|
||||
assert!(rig.bodies().iter().all(|body| !body.contains("k2L9k2L9")));
|
||||
assert!(rig.embedded().iter().all(|text| !text.contains("k2L9k2L9")));
|
||||
let by_id: BTreeMap<String, String> = rig.points().iter().flat_map(|point| point.payload.messages.iter().map(|message| (message.event_id.clone(), message.body.clone()))).collect();
|
||||
assert_eq!(by_id["$clean"], "now with <redacted>");
|
||||
assert_eq!(by_id["$dirty"], "token removed");
|
||||
assert!(!by_id.contains_key("$gone"));
|
||||
assert!(by_id["$straddle"].chars().count() <= CHAT_BODY_CHARS_MAX && !by_id["$straddle"].contains("ghp_"));
|
||||
assert!(done.scrubbed.iter().any(|hit| hit.event_id == "$adds") && done.scrubbed.iter().any(|hit| hit.event_id == "$gone"));
|
||||
let (ids, texts) = (rig.ids(), rig.points().into_iter().map(|point| point.payload.text).collect::<Vec<_>>());
|
||||
// Replay from the start, as after a lost cursor: identical ids and text.
|
||||
let mut cursors = rig.index.cursors();
|
||||
cursors.remove(&cursor_key("!r:s"));
|
||||
rig.index.set_cursors(&cursors).unwrap();
|
||||
rig.ok(&mut history);
|
||||
assert_eq!((rig.ids(), rig.points().into_iter().map(|point| point.payload.text).collect::<Vec<_>>()), (ids, texts));
|
||||
// An older scrub version re-ingests the room and scrubs it again from the history.
|
||||
let mut cursors = rig.index.cursors();
|
||||
cursors.insert(cursor_key("!r:s"), format!("{}.0:$straddle", huginn_mind::redact::SCRUB_VERSION - 1));
|
||||
rig.index.set_cursors(&cursors).unwrap();
|
||||
rig.ok(&mut history);
|
||||
assert!(rig.bodies().iter().all(|body| !body.contains("k2L9k2L9")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ordinary_text_is_stored_byte_for_byte() {
|
||||
let mut history = FakeHistory::default();
|
||||
let same = "risk-averse task-force sk- 3f2b8c1d9e7a4b5c6d0e1f2a3b4c5d6e7f8a9b0c !abc:s user@example.com ssh://git@host/r.git the bearer of bad news";
|
||||
history.room("!r:s", vec![said("$a", 1, same)]);
|
||||
let mut rig = Rig::new();
|
||||
let done = rig.ok(&mut history);
|
||||
assert_eq!(rig.bodies(), [same]);
|
||||
assert!(done.scrubbed.is_empty());
|
||||
}
|
||||
|
||||
// ---- steward purge ----
|
||||
|
||||
fn window(from_ms: u64, to_ms: u64) -> PurgeWindow {
|
||||
PurgeWindow { from_ms, to_ms }
|
||||
}
|
||||
|
||||
fn purge_history() -> FakeHistory {
|
||||
let mut history = FakeHistory::default();
|
||||
// t1 = 1000, t2 = 3000: $b at t1 and $c inside are purged, $d at t2 is kept.
|
||||
history.room("!r:s", vec![said("$a", 500, "before"), said("$b", 1000, "PURGED-B"), said("$c", 2000, "PURGED-C"), said("$d", 3000, "after"), said("$e", 4000, "later")]);
|
||||
history.room("!other:s", vec![said("$o1", 1000, "same time other room"), said("$o2", 2000, "same time")]);
|
||||
history
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_purge_window_removes_exactly_its_events_and_re_chunks_the_neighbours() {
|
||||
let mut history = purge_history();
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
let other_ids: BTreeSet<String> = rig.points().iter().filter(|point| point.payload.room_id == "!other:s").map(|point| point.id.clone()).collect();
|
||||
assert_eq!(rig.held_ids("!r:s"), ["$a", "$b", "$c", "$d", "$e"]);
|
||||
assert!(request_purge(&mut rig.index, "!r:s", window(1000, 3000)).unwrap());
|
||||
let embedded_before = rig.embedded().len();
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.held_ids("!r:s"), ["$a", "$d", "$e"], "t1 is inside, t2 is not");
|
||||
assert!(rig.embedded()[embedded_before..].iter().all(|text| !text.contains("PURGED")));
|
||||
assert!(rig.points().iter().all(|point| !point.payload.text.contains("PURGED")));
|
||||
assert_eq!(rig.points().iter().filter(|point| point.payload.room_id == "!other:s").map(|point| point.id.clone()).collect::<BTreeSet<_>>(), other_ids);
|
||||
assert_eq!(rig.held_ids("!other:s"), ["$o1", "$o2"]);
|
||||
assert_eq!(tombstone(&rig, "!r:s").as_deref(), Some("1000..3000"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_purged_first_message_changes_the_chunk_id_and_an_emptied_chunk_is_deleted() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 1000, "x"), said("$b", 2000, "y"), said("$c", 99_000_000, "far")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.ids().len(), 2);
|
||||
request_purge(&mut rig.index, "!r:s", window(900, 1500)).unwrap();
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.ids(), [chunk_id("!r:s", "$b"), chunk_id("!r:s", "$c")].into());
|
||||
request_purge(&mut rig.index, "!r:s", window(99_000_000, 99_000_001)).unwrap();
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.held_ids("!r:s"), ["$b"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_purged_window_does_not_come_back() {
|
||||
let mut history = purge_history();
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
request_purge(&mut rig.index, "!r:s", window(1000, 3000)).unwrap();
|
||||
rig.ok(&mut history);
|
||||
let want = ["$a", "$d", "$e"];
|
||||
// New events, an in-window edit of an outside message, a redaction.
|
||||
history.push("!r:s", vec![said("$f", 5000, "new"), edit("$ed", "@a:s", 2500, "$a", "EDIT-IN-WINDOW"), redaction("$rd", 6000, "$e")]);
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.held_ids("!r:s"), ["$a", "$d", "$f"]);
|
||||
assert!(rig.bodies().contains(&"before".to_owned()) && !rig.bodies().iter().any(|body| body.contains("EDIT-IN-WINDOW")), "the in-window edit is dropped and the original stays");
|
||||
// A lost cursor re-ingests from the start; a version bump re-ingests too.
|
||||
for cursor in [None, Some("0.0:$f".to_owned())] {
|
||||
let mut cursors = rig.index.cursors();
|
||||
match cursor {
|
||||
None => cursors.remove(&cursor_key("!r:s")),
|
||||
Some(old) => cursors.insert(cursor_key("!r:s"), old),
|
||||
};
|
||||
rig.index.set_cursors(&cursors).unwrap();
|
||||
rig.ok(&mut history);
|
||||
assert!(rig.bodies().iter().all(|body| !body.contains("PURGED")), "{:?}", rig.held_ids("!r:s"));
|
||||
assert_eq!(rig.held_ids("!r:s"), ["$a", "$d", "$f"], "$e stays redacted by the history; {want:?} before the new events");
|
||||
}
|
||||
// The reader leaves the room and joins again.
|
||||
let mut left = FakeHistory::default();
|
||||
left.room("!other:s", vec![said("$o1", 1000, "x")]);
|
||||
rig.ok(&mut left);
|
||||
assert!(rig.held_ids("!r:s").is_empty() && tombstone(&rig, "!r:s").is_some());
|
||||
rig.ok(&mut history);
|
||||
assert!(rig.bodies().iter().all(|body| !body.contains("PURGED")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_purge_of_an_unindexed_window_writes_only_the_tombstone_and_still_holds() {
|
||||
let mut rig = Rig::new();
|
||||
let writes = rig.index.state.lock().unwrap().cursor_writes.len();
|
||||
assert!(request_purge(&mut rig.index, "!never:s", window(10, 20)).unwrap());
|
||||
assert_eq!(rig.index.state.lock().unwrap().cursor_writes.len(), writes + 1);
|
||||
assert!(rig.points().is_empty() && rig.embedded().is_empty());
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!never:s", vec![said("$a", 5, "ok"), said("$b", 15, "PURGED"), said("$c", 25, "ok")]);
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.held_ids("!never:s"), ["$a", "$c"]);
|
||||
assert!(rig.embedded().iter().all(|text| !text.contains("PURGED")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_repeated_or_overlapping_request_changes_nothing_or_merges_and_the_33rd_window_is_refused() {
|
||||
let mut rig = Rig::new();
|
||||
assert!(request_purge(&mut rig.index, "!r:s", window(10, 20)).unwrap());
|
||||
let writes = rig.index.state.lock().unwrap().cursor_writes.len();
|
||||
assert!(!request_purge(&mut rig.index, "!r:s", window(10, 20)).unwrap());
|
||||
assert!(!request_purge(&mut rig.index, "!r:s", window(12, 18)).unwrap());
|
||||
assert_eq!(rig.index.state.lock().unwrap().cursor_writes.len(), writes, "no write for a request the tombstone already holds");
|
||||
assert!(request_purge(&mut rig.index, "!r:s", window(15, 30)).unwrap());
|
||||
assert_eq!(tombstone(&rig, "!r:s").as_deref(), Some("10..30"));
|
||||
for n in 0..(PURGE_WINDOWS_MAX as u64 - 1) {
|
||||
request_purge(&mut rig.index, "!r:s", window(100 + n * 10, 105 + n * 10)).unwrap();
|
||||
}
|
||||
let error = format!("{:#}", request_purge(&mut rig.index, "!r:s", window(10_000, 10_001)).unwrap_err());
|
||||
assert!(error.contains("32"), "{error}");
|
||||
assert!(request_purge(&mut Rig::new().index, "!r:s", window(5, 5)).is_err());
|
||||
}
|
||||
|
||||
// ---- the source stays in its lane ----
|
||||
|
||||
#[test]
|
||||
fn the_ingest_code_is_pure_over_its_ports_and_names_no_credential_shape_no_second_literal_and_no_join_rule() {
|
||||
let sources = [("chat.rs", include_str!("chat.rs")), ("chat_chunk.rs", include_str!("chat_chunk.rs")), ("chat_purge.rs", include_str!("chat_purge.rs")), ("room_scope.rs", include_str!("room_scope.rs"))];
|
||||
let process = ["Command", "::new"].concat();
|
||||
for (name, source) in sources {
|
||||
let code = source.split("#[cfg(test)]").next().unwrap();
|
||||
assert!(!code.contains(&process) && !code.contains("std::process"), "{name}");
|
||||
assert!(!code.to_ascii_lowercase().contains("join_rule"), "{name}");
|
||||
if name != "room_scope.rs" {
|
||||
assert!(!code.to_ascii_lowercase().contains("discord"), "{name}");
|
||||
}
|
||||
for shape in ["sk-", "sk_live_", "ghp_", "github_pat_", "glpat-", "AKIA", "xoxb-", "PRIVATE KEY", "Bearer"] {
|
||||
assert!(!code.contains(shape), "{name} names {shape}");
|
||||
}
|
||||
let defined = usize::from(name == "chat.rs");
|
||||
assert_eq!((code.matches("\"chat:").count(), code.matches("\"chat-purge:").count()), (defined, defined), "{name}: each prefix is defined once");
|
||||
}
|
||||
let chat = include_str!("chat.rs");
|
||||
assert_eq!(chat.matches("scrub(").count(), 2, "scrub is called for a body and for the room name, nowhere else");
|
||||
}
|
||||
|
||||
// ---- redacted edits, redacted originals, late edits ----
|
||||
|
||||
const EDIT_SECRET: &str = "EDIT-SECRET";
|
||||
|
||||
fn nothing_secret_stored(rig: &Rig) {
|
||||
let stored = format!("{:?}", rig.points().iter().map(|point| (&point.payload.messages, &point.payload.text)).collect::<Vec<_>>());
|
||||
assert!(!stored.contains(EDIT_SECRET), "{stored}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_redacted_edit_in_the_same_page_reverts_the_message_to_its_earlier_body() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$m", 10, "original words"), edit("$e1", "@a:s", 20, "$m", "first edit"), edit("$e2", "@a:s", 30, "$m", EDIT_SECRET), redaction("$r", 40, "$e2")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.bodies(), ["first edit"]);
|
||||
assert!(rig.embedded().iter().all(|text| !text.contains(EDIT_SECRET)));
|
||||
history.room("!q:s", vec![said("$m", 10, "orig"), edit("$e", "@a:s", 20, "$m", EDIT_SECRET), redaction("$r", 30, "$e")]);
|
||||
rig.ok(&mut history);
|
||||
assert!(rig.bodies().contains(&"orig".to_owned()));
|
||||
nothing_secret_stored(&rig);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_redaction_of_an_edit_a_later_run_stored_rebuilds_the_room_and_the_secret_does_not_return() {
|
||||
let mut history = FakeHistory { prune: true, ..FakeHistory::default() };
|
||||
history.room("!r:s", vec![said("$m", 10, "original"), edit("$e", "@a:s", 20, "$m", EDIT_SECRET)]);
|
||||
history.room("!q:s", vec![said("$q", 10, "quiet")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
assert!(rig.bodies().iter().any(|body| body == EDIT_SECRET), "the edit applied while it stood");
|
||||
let recreated = rig.index.state.lock().unwrap().recreated;
|
||||
history.push("!r:s", vec![redaction("$r", 30, "$e")]);
|
||||
let done = rig.ok(&mut history);
|
||||
assert!(done.is_clean());
|
||||
nothing_secret_stored(&rig);
|
||||
assert!(rig.bodies().contains(&"original".to_owned()));
|
||||
assert_eq!(rig.held_ids("!q:s"), ["$q"]);
|
||||
assert_eq!(rig.index.state.lock().unwrap().recreated, recreated);
|
||||
assert_eq!(rig.cursor("!r:s").unwrap(), format!("{}:$r", chat_version()));
|
||||
// The rebuild does not loop: the redaction it replays is not a trigger the second time, and nothing changes.
|
||||
let ids = rig.ids();
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.ids(), ids);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_redacted_original_takes_its_live_and_late_edits_with_it_in_a_page_across_runs_and_across_a_rebuild() {
|
||||
let script = |history: &mut FakeHistory| {
|
||||
history.room("!r:s", vec![said("$m", 10, "original"), edit("$e1", "@a:s", 20, "$m", "live edit"), redaction("$re", 25, "$e1"), redaction("$rm", 30, "$m"), edit("$late", "@a:s", 40, "$m", EDIT_SECRET)]);
|
||||
history.prune = true;
|
||||
};
|
||||
// One page.
|
||||
let (mut history, mut rig) = (FakeHistory::default(), Rig::new());
|
||||
script(&mut history);
|
||||
rig.ok(&mut history);
|
||||
nothing_secret_stored(&rig);
|
||||
assert!(rig.points().is_empty());
|
||||
// Across runs: the original and a live edit stand, then it is redacted, then a late edit arrives.
|
||||
let (mut history, mut rig) = (FakeHistory { prune: true, ..FakeHistory::default() }, Rig::new());
|
||||
history.room("!r:s", vec![said("$m", 10, "original"), edit("$e1", "@a:s", 20, "$m", EDIT_SECRET)]);
|
||||
rig.ok(&mut history);
|
||||
history.push("!r:s", vec![redaction("$rm", 30, "$m")]);
|
||||
rig.ok(&mut history);
|
||||
nothing_secret_stored(&rig);
|
||||
history.push("!r:s", vec![edit("$late", "@a:s", 40, "$m", EDIT_SECRET)]);
|
||||
rig.ok(&mut history);
|
||||
nothing_secret_stored(&rig);
|
||||
assert!(rig.points().is_empty());
|
||||
// Across a rebuild: a lost cursor re-reads the whole history.
|
||||
let mut cursors = rig.index.cursors();
|
||||
cursors.remove(&cursor_key("!r:s"));
|
||||
rig.index.set_cursors(&cursors).unwrap();
|
||||
rig.ok(&mut history);
|
||||
nothing_secret_stored(&rig);
|
||||
assert!(rig.points().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stranger_cannot_edit_a_message_of_the_same_page() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$m", 10, "mine"), edit("$e", "@stranger:s", 20, "$m", "hijacked")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.bodies(), ["mine"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_room_name_is_scrubbed_before_it_is_embedded_or_stored() {
|
||||
let mut history = FakeHistory::default();
|
||||
let token = format!("ghp_{}", "k2L9".repeat(9));
|
||||
history.room("!r:s", vec![said("$m", 10, "hello")]).facts("!r:s", |facts| facts.name = Some(format!("Swarm {token}")));
|
||||
let mut rig = Rig::new();
|
||||
let (done, journal) = rig.run(&mut history);
|
||||
let done = done.unwrap();
|
||||
assert!(rig.embedded().iter().all(|text| !text.contains("k2L9k2L9")));
|
||||
assert!(rig.points().iter().all(|point| !point.payload.text.contains("k2L9k2L9") && point.payload.text.starts_with("Swarm <redacted>")));
|
||||
assert!(done.scrubbed.iter().any(|hit| hit.event_id == "(name)") && journal.iter().all(|line| !line.contains("k2L9k2L9")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_token_split_by_zero_width_characters_leaves_no_tail_in_a_stored_body() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$m", 10, &format!("key sk-{}\u{200b}{} done", "q7W3".repeat(4), "q7W3".repeat(4)))]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.bodies(), ["key <redacted> done"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retire_goes_on_past_a_failed_delete_and_names_each_failure() {
|
||||
let mut rig = Rig::new();
|
||||
for room in ["!a:s", "!b:s", "!c:s"] {
|
||||
rig.seed(room, "$e", 1);
|
||||
}
|
||||
rig.index.state.lock().unwrap().fail_chat_deletes_for = ["!a:s".to_owned()].into();
|
||||
let done = retire_chat(&mut rig.index, &BTreeSet::new()).unwrap();
|
||||
assert_eq!(done.retired, ["!b:s", "!c:s"]);
|
||||
assert_eq!(done.failed.len(), 1);
|
||||
assert_eq!(done.failed[0].0, "!a:s");
|
||||
assert_eq!(rig.held_ids("!a:s"), ["$e"]);
|
||||
assert!(rig.cursor("!a:s").is_some(), "the cursor stays so the next run finds it again");
|
||||
// The refresh reports it too.
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!z:s", vec![said("$z", 1, "x")]);
|
||||
let done = rig.ok(&mut history);
|
||||
assert!(done.failed.contains(&"!a:s".to_owned()) && !done.is_clean());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_facts_error_on_a_joined_room_keeps_its_points_and_cursor_and_is_reported() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 1, "x")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
let (cursor, points) = (rig.cursor("!r:s"), rig.points());
|
||||
history.facts_fail.insert("!r:s".into());
|
||||
let (done, journal) = rig.run(&mut history);
|
||||
let done = done.unwrap();
|
||||
assert_eq!((rig.cursor("!r:s"), rig.points()), (cursor, points));
|
||||
assert_eq!(done.failed, ["!r:s"]);
|
||||
assert!(done.rooms.iter().any(|report| report.room_id == "!r:s" && report.failure.is_some()));
|
||||
assert!(journal.iter().any(|line| line.contains("!r:s") && line.contains("failed")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_tail_on_equal_ends_is_the_greatest_chunk_id() {
|
||||
let mut rig = Rig::new();
|
||||
let (a, b) = (chunk_id("!r:s", "$a"), chunk_id("!r:s", "$b"));
|
||||
rig.index.upsert_chat(&[
|
||||
ChatPoint { id: a.clone(), vector: vec![1.0; 4], payload: chunk_payload_of("!r:s", &[("$a", 5)]) },
|
||||
ChatPoint { id: b.clone(), vector: vec![1.0; 4], payload: chunk_payload_of("!r:s", &[("$b", 5)]) },
|
||||
])
|
||||
.unwrap();
|
||||
assert_eq!(rig.index.chat_tail("!r:s").unwrap().unwrap().id, a.max(b));
|
||||
}
|
||||
|
|
@ -3,37 +3,26 @@
|
|||
//! for semantic search. Each point's `corpus` payload (`forge` or `chat`) keeps
|
||||
//! the two apart; a forge search filters `forge` and a chat search `chat`.
|
||||
//!
|
||||
//! Two truths, one job. The forge is the truth of which repos exist and what
|
||||
//! they hold, Synapse of what the Minds' rooms said; the collection is derived
|
||||
//! from them and can be deleted whenever (but for a steward's purge tombstones, see
|
||||
//! `chat_purge`). `refresh` is the forge's writer and the only creator of the
|
||||
//! collection, run by the `huginn-sources` oneshot job: first build, incremental
|
||||
//! refresh, rebuild after a model or chunker change, and retirement are all the one
|
||||
//! routine (delete by filter, ingest at a commit, rewrite the cursors). `chat::refresh_chat`
|
||||
//! joins it in the same job and the same cursors object, as the only writer of chat points;
|
||||
//! it requires the collection to exist and never recreates it. The daemon only reads: `search`
|
||||
//! and `search_chat` are its whole part, and nothing in `serve` or the worker calls a write.
|
||||
//! Two truths, one writer. The forge is the truth of which repos exist and what
|
||||
//! they hold; the collection is derived from it and can be deleted whenever.
|
||||
//! `refresh` is the only writer of the collection, run by the `huginn-sources`
|
||||
//! oneshot job: first build, incremental refresh, rebuild after a model or
|
||||
//! chunker change, and retirement are all the one routine (delete by filter,
|
||||
//! ingest at a commit, rewrite the cursors). The daemon only reads: `search`
|
||||
//! is its whole part, and nothing in `serve` or the worker calls a write.
|
||||
//!
|
||||
//! A cursor is the commit (or, for a room, the event) whose chunks the collection
|
||||
//! holds. It lives in the collection's own metadata, so it is lost with the collection
|
||||
//! A cursor is the commit whose chunks the collection holds for a repo. It
|
||||
//! lives in the collection's own metadata, so it is lost with the collection
|
||||
//! and the two cannot disagree about what is held. It is advanced only after
|
||||
//! the source's deletes and upserts are acknowledged: it may lag and never leads.
|
||||
//! A repo's is dropped before its first delete, so a run that fails between
|
||||
//! the delete and the upserts leaves no cursor, and the repo is ingested whole; a room's
|
||||
//! chat chunks are written before anything they replace is deleted. The cursors
|
||||
//! object is written whole by this job alone. A key is a repo name, or a chat key
|
||||
//! (`chat::is_chat_key`), never both.
|
||||
//! the repo's deletes and upserts are acknowledged: it may lag and never leads.
|
||||
//! It is dropped before the repo's first delete, so a run that fails between
|
||||
//! the delete and the upserts leaves no cursor, and the repo is ingested whole.
|
||||
//! The cursors object is written whole by this job alone, once per repo.
|
||||
|
||||
pub mod chat;
|
||||
pub mod chat_chunk;
|
||||
pub mod chat_purge;
|
||||
pub mod chunk;
|
||||
pub mod forge;
|
||||
pub mod git;
|
||||
pub mod room_scope;
|
||||
|
||||
#[cfg(test)]
|
||||
mod chat_tests;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
|
|
@ -201,14 +190,6 @@ impl ChatFilter {
|
|||
}
|
||||
}
|
||||
|
||||
/// A chat chunk as the index holds it, with its point id: what the ingest job
|
||||
/// reads back to rewrite a chunk.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct StoredChat {
|
||||
pub id: String,
|
||||
pub payload: ChatPayload,
|
||||
}
|
||||
|
||||
/// One chunk a vector search returned and how near it scored.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct SourceRow {
|
||||
|
|
@ -246,16 +227,6 @@ pub trait SourceIndex {
|
|||
/// Up to `limit` chat chunks `filter` keeps, by span start in `order`.
|
||||
/// Reads only `corpus=chat` points; the daemon's.
|
||||
fn chat_scroll(&mut self, filter: &ChatFilter, order: ChatOrder, limit: u32, within: Duration) -> Result<Vec<ChatPayload>>;
|
||||
/// The room's chunk with the greatest `ts_end_ms`, if it has any. The ingest job's alone.
|
||||
fn chat_tail(&mut self, room_id: &str) -> Result<Option<StoredChat>>;
|
||||
/// The room's chunks holding any of `event_ids`. The ingest job's alone.
|
||||
fn chat_chunks_holding(&mut self, room_id: &str, event_ids: &[String]) -> Result<Vec<StoredChat>>;
|
||||
/// The room's chunks whose span reaches into [`from_ms`, `to_ms`): `ts_end_ms >= from_ms`
|
||||
/// and `ts_start_ms < to_ms`. The ingest job's alone.
|
||||
fn chat_chunks_overlapping(&mut self, room_id: &str, from_ms: u64, to_ms: u64) -> Result<Vec<StoredChat>>;
|
||||
/// Deletes a room's chat chunks: those with the point ids `ids`, or all of them when `None`.
|
||||
/// Acknowledged before it returns. Never touches a forge point. The ingest job's alone.
|
||||
fn delete_chat(&mut self, room_id: &str, ids: Option<&[String]>) -> Result<()>;
|
||||
}
|
||||
|
||||
/// A repo the forge lists as indexable.
|
||||
|
|
@ -424,7 +395,7 @@ fn retire<S: SourceIndex>(
|
|||
failed: &mut Vec<String>,
|
||||
) {
|
||||
let listed = |name: &str| catalog.iter().any(|repo| repo.name == name);
|
||||
let mut gone: BTreeSet<String> = cursors.keys().filter(|name| !chat::is_chat_key(name) && !listed(name)).cloned().collect();
|
||||
let mut gone: BTreeSet<String> = cursors.keys().filter(|name| !listed(name)).cloned().collect();
|
||||
if let Ok(entries) = std::fs::read_dir(mirrors) {
|
||||
for entry in entries.flatten() {
|
||||
if let Some(name) = entry.file_name().to_string_lossy().strip_suffix(".git")
|
||||
|
|
|
|||
|
|
@ -1,184 +0,0 @@
|
|||
//! Which joined rooms and which senders are in the chat corpus. The only code
|
||||
//! that decides it, and the only file outside tests that names the bridged
|
||||
//! platform whose content is refused.
|
||||
//!
|
||||
//! Scope (ruling `gamecult-chat:ruling:chat-index-minds-rooms`): the rooms the
|
||||
//! reader user has joined, minus this exclusion. A room's join rule is read by
|
||||
//! nobody (`RoomFacts` has no field for it): a joined invite-only room qualifies
|
||||
//! like any other. The reader roster is the first lock; this file is the second.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use super::chat::RoomEvent;
|
||||
|
||||
/// The estate rooms by canonical alias, ASCII case-insensitive. Code owns the
|
||||
/// aliases and the rule; the unit supplies the room ids (`Estate`), and either alone excludes.
|
||||
pub const ESTATE_ALIASES: [&str; 3] = ["#users:gamecult.org", "#members:gamecult.org", "#patrons:gamecult.org"];
|
||||
|
||||
/// What the server says about a room, as far as admission needs it. The Matrix
|
||||
/// client fills it from the room's state and decides nothing. It has no `Default`: a
|
||||
/// room whose state could not be read is a port error, never a plain room.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct RoomFacts {
|
||||
pub room_id: String,
|
||||
pub alias: Option<String>,
|
||||
pub alt_aliases: Vec<String>,
|
||||
pub name: Option<String>,
|
||||
/// An `m.room.encryption` state event exists.
|
||||
pub encrypted: bool,
|
||||
/// Any `m.room.member` in the state has `is_direct` true.
|
||||
pub direct: bool,
|
||||
/// The state type (`m.bridge` or `uk.half-shot.bridge`) of a bridge event, when the room has one.
|
||||
pub bridge: Option<String>,
|
||||
}
|
||||
|
||||
/// The estate room ids the caller knows (they exist only on the live server): exactly one per
|
||||
/// estate alias, so a job that was handed none, or too few, cannot be built.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Estate {
|
||||
ids: BTreeSet<String>,
|
||||
}
|
||||
|
||||
impl Estate {
|
||||
pub fn new(ids: BTreeSet<String>) -> anyhow::Result<Self> {
|
||||
anyhow::ensure!(
|
||||
ids.len() == ESTATE_ALIASES.len() && ids.iter().all(|id| id.starts_with('!')),
|
||||
"the estate is exactly {} room ids ({}), got {}",
|
||||
ESTATE_ALIASES.len(),
|
||||
ESTATE_ALIASES.join(", "),
|
||||
ids.len()
|
||||
);
|
||||
Ok(Self { ids })
|
||||
}
|
||||
|
||||
/// Whether `room_id` is a named estate room. Decided before the room's state is read.
|
||||
pub fn holds_id(&self, room_id: &str) -> bool {
|
||||
self.ids.contains(room_id)
|
||||
}
|
||||
}
|
||||
|
||||
/// Why a joined room, or an event in it, is refused.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum Refusal {
|
||||
Estate,
|
||||
Direct,
|
||||
Encrypted,
|
||||
Bridged(String),
|
||||
DiscordSender,
|
||||
DiscordContent,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for Refusal {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::Estate => formatter.write_str("estate room"),
|
||||
Self::Direct => formatter.write_str("direct room"),
|
||||
Self::Encrypted => formatter.write_str("encrypted room"),
|
||||
Self::Bridged(state_type) => write!(formatter, "bridged room ({state_type})"),
|
||||
Self::DiscordSender => formatter.write_str("discord sender"),
|
||||
Self::DiscordContent => formatter.write_str("discord content"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a joined room may be read at all.
|
||||
pub fn admit_room(facts: &RoomFacts, estate: &Estate) -> Result<(), Refusal> {
|
||||
let estate_alias = |alias: &str| ESTATE_ALIASES.iter().any(|estate| estate.eq_ignore_ascii_case(alias));
|
||||
if estate.holds_id(&facts.room_id) || facts.alias.iter().chain(&facts.alt_aliases).any(|alias| estate_alias(alias)) {
|
||||
return Err(Refusal::Estate);
|
||||
}
|
||||
if facts.direct {
|
||||
return Err(Refusal::Direct);
|
||||
}
|
||||
if facts.encrypted {
|
||||
return Err(Refusal::Encrypted);
|
||||
}
|
||||
if let Some(state_type) = &facts.bridge {
|
||||
return Err(Refusal::Bridged(state_type.clone()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Whether an event refuses its whole room: a sender whose localpart, lowercased, starts with the
|
||||
/// platform's name after any leading underscores, has it after an underscore (`mautrix_discord`,
|
||||
/// `_bridge_discordbot`), or has it followed by `bot` or an underscore; or content with a top-level key
|
||||
/// naming it. Deterministic identifier rules, not language reading; `discourse` is not it.
|
||||
pub fn refuse_event(event: &RoomEvent) -> Option<Refusal> {
|
||||
let localpart = event.sender.trim_start_matches('@').split(':').next().unwrap_or("").to_ascii_lowercase();
|
||||
let name = ["disc", "ord"].concat();
|
||||
if localpart.trim_start_matches('_').starts_with(&name) || localpart.contains(&format!("_{name}")) || localpart.contains(&format!("{name}bot")) || localpart.contains(&format!("{name}_")) {
|
||||
return Some(Refusal::DiscordSender);
|
||||
}
|
||||
event.content_keys.iter().any(|key| key.to_ascii_lowercase().contains("discord")).then_some(Refusal::DiscordContent)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::chat::EventKind;
|
||||
use super::*;
|
||||
|
||||
fn facts(id: &str) -> RoomFacts {
|
||||
RoomFacts { room_id: id.into(), alias: None, alt_aliases: vec![], name: None, encrypted: false, direct: false, bridge: None }
|
||||
}
|
||||
|
||||
fn estate(ids: &[&str]) -> Estate {
|
||||
Estate::new(ids.iter().chain(["!pad1:s", "!pad2:s", "!pad3:s"].iter()).take(3).map(|id| (*id).to_owned()).collect()).unwrap()
|
||||
}
|
||||
|
||||
fn event(sender: &str, keys: &[&str]) -> RoomEvent {
|
||||
RoomEvent { event_id: "$e".into(), sender: sender.into(), ts_ms: 1, content_keys: keys.iter().map(|key| (*key).into()).collect(), kind: EventKind::Other }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_direct_encrypted_or_bridged_room_is_refused_and_a_plain_room_is_admitted() {
|
||||
let estate = estate(&[]);
|
||||
assert_eq!(admit_room(&RoomFacts { direct: true, ..facts("!a:s") }, &estate), Err(Refusal::Direct));
|
||||
assert_eq!(admit_room(&RoomFacts { encrypted: true, ..facts("!a:s") }, &estate), Err(Refusal::Encrypted));
|
||||
for state_type in ["m.bridge", "uk.half-shot.bridge"] {
|
||||
assert_eq!(admit_room(&RoomFacts { bridge: Some(state_type.into()), ..facts("!a:s") }, &estate), Err(Refusal::Bridged(state_type.into())));
|
||||
}
|
||||
assert_eq!(admit_room(&RoomFacts { alias: Some("#swarm:gamecult.org".into()), name: Some("Swarm".into()), ..facts("!a:s") }, &estate), Ok(()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_estate_is_refused_by_id_and_by_alias_in_any_case_and_a_lookalike_is_not() {
|
||||
let named = estate(&["!ids:s"]);
|
||||
assert!(named.holds_id("!ids:s") && !named.holds_id("!other:s"));
|
||||
assert_eq!(admit_room(&facts("!ids:s"), &named), Err(Refusal::Estate));
|
||||
for alias in ESTATE_ALIASES.iter().map(|alias| alias.to_string()).chain(["#USERS:Gamecult.Org".to_owned()]) {
|
||||
assert_eq!(admit_room(&RoomFacts { alias: Some(alias.clone()), ..facts("!x:s") }, &estate(&[])), Err(Refusal::Estate), "{alias}");
|
||||
assert_eq!(admit_room(&RoomFacts { alt_aliases: vec!["#a:s".into(), alias.clone()], ..facts("!x:s") }, &estate(&[])), Err(Refusal::Estate), "alt {alias}");
|
||||
}
|
||||
for lookalike in ["#users-guide:gamecult.org", "#patrons-club:gamecult.org", "#users:gamecult.org.evil", "#users:other.org"] {
|
||||
assert_eq!(admit_room(&RoomFacts { alias: Some(lookalike.into()), ..facts("!x:s") }, &estate(&[])), Ok(()), "{lookalike}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_bridged_platform_sender_or_content_key_refuses_and_a_lookalike_does_not() {
|
||||
for sender in ["@discord_123:s", "@_discord_9:s", "@__DiscordBot:s", "@discordbot:s", "@mautrix_discord:s", "@_bridge_discordbot:s", "@a_discord:s"] {
|
||||
assert_eq!(refuse_event(&event(sender, &[])), Some(Refusal::DiscordSender), "{sender}");
|
||||
}
|
||||
for key in ["fi.mau.discord", "uk.half-shot.discord", "X.DISCORD.y"] {
|
||||
assert_eq!(refuse_event(&event("@a:s", &["body", key])), Some(Refusal::DiscordContent), "{key}");
|
||||
}
|
||||
for sender in ["@discourse:s", "@disco:s", "@eureka:s", "@x:discord.com"] {
|
||||
assert_eq!(refuse_event(&event(sender, &["discourse", "body"])), None, "{sender}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_estate_needs_exactly_its_room_ids() {
|
||||
for ids in [vec![], vec!["!a:s"], vec!["!a:s", "!b:s", "!c:s", "!d:s"], vec!["!a:s", "!b:s", "notaroom"]] {
|
||||
assert!(Estate::new(ids.iter().map(|id| (*id).to_owned()).collect()).is_err(), "{ids:?}");
|
||||
}
|
||||
assert!(Estate::new(["!a:s", "!b:s", "!c:s"].map(str::to_owned).into()).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_room_facts_carries_no_join_rule() {
|
||||
let source = include_str!("room_scope.rs");
|
||||
let code = &source[..source.find("#[cfg(test)]").unwrap()];
|
||||
assert!(!code.contains(&["join", "_rule"].concat()));
|
||||
}
|
||||
}
|
||||
|
|
@ -1155,24 +1155,3 @@ fn a_window_asks_the_index_only_and_needs_no_embedder() {
|
|||
let signature = &signature[..signature.find('{').unwrap()];
|
||||
assert!(!signature.contains("Embedder"), "{signature}");
|
||||
}
|
||||
|
||||
/// A chat cursor and a purge tombstone share the forge's cursors object; the
|
||||
/// forge retire never reads either as a repo name, and still retires a repo.
|
||||
#[test]
|
||||
fn chat_cursor_and_purge_tombstone_keys_survive_the_forge_retire() {
|
||||
let (keep, gone) = (Remote::new(), Remote::new());
|
||||
keep.commit(&[("a.rs", b"a\n")], &[]);
|
||||
gone.commit(&[("b.rs", b"b\n")], &[]);
|
||||
let mut rig = Rig::new();
|
||||
rig.converged(&mut FakeForge::of(&[("Keep", &keep), ("Gone", &gone)]));
|
||||
let mut cursors = rig.index.cursors();
|
||||
cursors.insert("chat:!r:s".into(), "1:$e".into());
|
||||
cursors.insert("chat-purge:!r:s".into(), "1..2".into());
|
||||
rig.index.set_cursors(&cursors).unwrap();
|
||||
rig.index.upsert_chat(&[chat_point("!r:s", None, 0, 4, &[("$e", 5)])]).unwrap();
|
||||
rig.converged(&mut FakeForge::of(&[("Keep", &keep)]));
|
||||
let after = rig.index.cursors();
|
||||
assert_eq!(after.keys().map(String::as_str).collect::<Vec<_>>(), ["Keep", "chat-purge:!r:s", "chat:!r:s"], "Gone retired; both chat keys kept");
|
||||
assert!(rig.index.state.lock().unwrap().deletes.iter().all(|(repo, _)| !repo.starts_with("chat")));
|
||||
assert_eq!(rig.index.state.lock().unwrap().chat.len(), 1);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -191,18 +191,12 @@ impl Drop for Daemon {
|
|||
}
|
||||
}
|
||||
|
||||
/// A test's name for someone as the Bifrost account id the door requires: a deterministic lowercase D-format UUID.
|
||||
fn acct(name: &str) -> huginn_mind::eureka_pipeline::AccountId {
|
||||
let hash = name.bytes().fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| (hash ^ u64::from(byte)).wrapping_mul(0x0100_0000_01b3));
|
||||
huginn_mind::eureka_pipeline::AccountId(format!("{:08x}-{:04x}-4000-8000-{:012x}", hash >> 32, (hash >> 16) & 0xffff, hash & 0xffff_ffff_ffff))
|
||||
}
|
||||
|
||||
fn answer(who: &str, hash: &str, request_id: &str) -> DeskRequest {
|
||||
DeskRequest::Answer {
|
||||
mind: INSTANCE.into(),
|
||||
subject: acct(who),
|
||||
subject: short(who),
|
||||
authority: DeskAuthority::Operator,
|
||||
deciders: vec![acct(who)],
|
||||
deciders: vec![short(who)],
|
||||
request: ForkAnswerRequest {
|
||||
schema_version: RequestSchema::V0,
|
||||
fork_id: FORK.into(),
|
||||
|
|
@ -219,8 +213,8 @@ fn answer(who: &str, hash: &str, request_id: &str) -> DeskRequest {
|
|||
fn forks_request(who: &str, deciders: &[&str]) -> DeskRequest {
|
||||
DeskRequest::Forks {
|
||||
mind: INSTANCE.into(),
|
||||
subject: acct(who),
|
||||
deciders: deciders.iter().map(|d| acct(d)).collect(),
|
||||
subject: short(who),
|
||||
deciders: deciders.iter().map(|d| short(d)).collect(),
|
||||
status: None,
|
||||
cursor: None,
|
||||
limit: huginn_mind::desk_wire::ForkLimit::new(50).unwrap(),
|
||||
|
|
@ -283,21 +277,21 @@ fn the_binary_serves_both_doors_and_survives_restart() {
|
|||
let catalog = format!("{:?}", exchange(&desk, &framed(&catalog_request())).expect("a catalog"));
|
||||
assert!(catalog.contains("huginn.desk_request.v1") && catalog.contains("huginn.desk_response.v1"), "{catalog}");
|
||||
assert!(!catalog.contains("huginn.mind_request.v1"), "{catalog}");
|
||||
// The reads answer and store nothing; a decider reading its own list is told the ids.
|
||||
// The reads answer, store nothing, and carry the whole record to a subject the deciders do not name.
|
||||
let (list, listed_at) = match ask(&desk, "m-r1", &forks_request("x", &["x"])) {
|
||||
DeskResponse::Forks { list, as_of } => (list, as_of),
|
||||
other => panic!("{other:?}"),
|
||||
};
|
||||
assert_eq!((list.open_count, list.recorded_count, list.forks.len(), list.stamp.clone()), (1, 0, 1, format!("s{listed_at}")));
|
||||
assert_eq!(list.forks[0].card.options_hash, hash);
|
||||
let DeskResponse::Fork { entry, as_of } = ask(&desk, "m-r2", &DeskRequest::Fork { mind: INSTANCE.into(), subject: acct("x"), deciders: vec![acct("x")], fork_id: FORK.into() })
|
||||
let DeskResponse::Fork { entry, as_of } = ask(&desk, "m-r2", &DeskRequest::Fork { mind: INSTANCE.into(), subject: short("x"), deciders: vec![short("x")], fork_id: FORK.into() })
|
||||
else {
|
||||
panic!("a read of the seeded fork")
|
||||
};
|
||||
assert_eq!((entry, as_of), (list.forks[0].clone(), listed_at));
|
||||
assert_eq!(daemon.documents(), seeded);
|
||||
// `authority` is no field of a read: the payload is refused whole and nothing is answered or stored.
|
||||
for (operation, request) in [("forks", forks_request("x", &["x"])), ("fork", DeskRequest::Fork { mind: INSTANCE.into(), subject: acct("x"), deciders: vec![acct("x")], fork_id: FORK.into() })] {
|
||||
for (operation, request) in [("forks", forks_request("x", &["x"])), ("fork", DeskRequest::Fork { mind: INSTANCE.into(), subject: short("x"), deciders: vec![short("x")], fork_id: FORK.into() })] {
|
||||
let mut value = serde_json::to_value(&request).unwrap();
|
||||
let variant = if operation == "forks" { "Forks" } else { "Fork" };
|
||||
value[variant]["authority"] = serde_json::json!("Operator");
|
||||
|
|
@ -325,20 +319,15 @@ fn the_binary_serves_both_doors_and_survives_restart() {
|
|||
assert_eq!((stale.code, counted_answer_id.as_deref()), (Some(ForkAnswerCode::StaleFork), Some("n1")));
|
||||
let stored = daemon.documents();
|
||||
assert_eq!(stored, seeded + 2);
|
||||
// A subject the call's deciders do not name is a non-decider: the fork is unlisted, so it reads as a missing one.
|
||||
let outsider = ask(&desk, "m-r3", &DeskRequest::Fork { mind: INSTANCE.into(), subject: acct("outsider"), deciders: vec![acct("y")], fork_id: FORK.into() });
|
||||
assert!(matches!(outsider, DeskResponse::NotStored(ref refusal) if refusal.code == huginn_mind::desk_wire::ForkRefusalCode::ForkUnknown), "{outsider:?}");
|
||||
// A decider reads both answers, the first with its choice and the stale one with its code, and is told who answered.
|
||||
// A subject the call's deciders do not name reads both answers, the first with its choice and the stale one with its code.
|
||||
let DeskResponse::Fork { entry, .. } =
|
||||
ask(&desk, "m-r4", &DeskRequest::Fork { mind: INSTANCE.into(), subject: acct("y"), deciders: vec![acct("y")], fork_id: FORK.into() })
|
||||
ask(&desk, "m-r3", &DeskRequest::Fork { mind: INSTANCE.into(), subject: short("outsider"), deciders: vec![short("y")], fork_id: FORK.into() })
|
||||
else {
|
||||
panic!("a read")
|
||||
};
|
||||
let seen: Vec<_> = entry.record.answers.iter().map(|answer| (answer.answer_id.as_str(), answer.answerer.as_deref(), answer.code)).collect();
|
||||
let x = acct("x").0;
|
||||
assert_eq!(seen, [("n1", Some(x.as_str()), None), ("n2", Some(x.as_str()), Some(ForkAnswerCode::StaleFork))]);
|
||||
assert_eq!((entry.record.counted_answer_id.as_deref(), entry.record.answerer_count), (Some("n1"), 1));
|
||||
assert_eq!((entry.card.deciders.as_slice(), entry.card.decider_count), (&[acct("y").0][..], 1));
|
||||
let seen: Vec<_> = entry.record.answers.iter().map(|answer| (answer.answer_id.as_str(), answer.answerer.as_str(), answer.code)).collect();
|
||||
assert_eq!(seen, [("n1", "x", None), ("n2", "x", Some(ForkAnswerCode::StaleFork))]);
|
||||
assert_eq!((entry.record.counted_answer_id.as_deref(), entry.card.deciders.as_slice()), (Some("n1"), ["y".to_string()].as_slice()));
|
||||
assert_eq!(daemon.documents(), stored);
|
||||
|
||||
// A stop takes the socket with it; the stored answers are the retry's memory.
|
||||
|
|
@ -590,7 +579,7 @@ fn forks_page(who: &str, cursor: Option<&str>) -> DeskRequest {
|
|||
}
|
||||
|
||||
fn fork_read(who: &str, fork_id: &str) -> DeskRequest {
|
||||
DeskRequest::Fork { mind: INSTANCE.into(), subject: acct(who), deciders: vec![acct(who)], fork_id: fork_id.into() }
|
||||
DeskRequest::Fork { mind: INSTANCE.into(), subject: short(who), deciders: vec![short(who)], fork_id: fork_id.into() }
|
||||
}
|
||||
|
||||
/// The wire spelling of a not-stored refusal's code, and its text.
|
||||
|
|
|
|||
|
|
@ -62,18 +62,12 @@ pub(crate) enum Door {
|
|||
Desk,
|
||||
}
|
||||
|
||||
/// The door split. Every staged `desk_answer`, whether the caller sent it or a derivation produced it,
|
||||
/// is refused unless it came by the desk door; and a staged `desk_listing` is refused if it came by the
|
||||
/// desk door: listing is the Mind's own act, on the Admit door, with a receipt saying who.
|
||||
fn refuse_documents_at_the_wrong_door(door: Door, docs: &Docs<'_>) -> Result<(), MindRefusal> {
|
||||
match docs.batch.iter().find(|staged| match staged.kind {
|
||||
PipelineKind::DeskAnswer => door != Door::Desk,
|
||||
PipelineKind::DeskListing => door == Door::Desk,
|
||||
_ => false,
|
||||
}) {
|
||||
Some(staged) if staged.kind == PipelineKind::DeskListing => Err(MindRefusal::DeskRequestInvalid { field: "desk_listing".into() }),
|
||||
Some(staged) => Err(MindRefusal::DeskAnswerOutsideDesk { id: staged.key.clone() }),
|
||||
None => Ok(()),
|
||||
/// Every staged `desk_answer`, whether the caller sent it or a derivation
|
||||
/// produced it, is refused unless it came by the desk door.
|
||||
fn refuse_desk_answers_outside_desk(door: Door, docs: &Docs<'_>) -> Result<(), MindRefusal> {
|
||||
match docs.batch.iter().find(|staged| staged.kind == PipelineKind::DeskAnswer) {
|
||||
Some(staged) if door != Door::Desk => Err(MindRefusal::DeskAnswerOutsideDesk { id: staged.key.clone() }),
|
||||
_ => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -166,7 +160,7 @@ impl<S: MindStore> Mind<S> {
|
|||
for envelope in envelopes {
|
||||
docs.push(stage(envelope, &mind)?)?;
|
||||
}
|
||||
refuse_documents_at_the_wrong_door(door, &docs)?;
|
||||
refuse_desk_answers_outside_desk(door, &docs)?;
|
||||
// A6
|
||||
let carries_instance = docs.batch.iter().filter(|staged| staged.kind == PipelineKind::Instance).count() == 1;
|
||||
if self.is_empty() && !carries_instance {
|
||||
|
|
@ -190,7 +184,7 @@ impl<S: MindStore> Mind<S> {
|
|||
docs.push(stage(envelope, &mind)?)?;
|
||||
}
|
||||
}
|
||||
refuse_documents_at_the_wrong_door(door, &docs)?;
|
||||
refuse_desk_answers_outside_desk(door, &docs)?;
|
||||
for staged in &docs.batch[originals..] {
|
||||
resolve(&docs, staged, &mind, external, &mut strong)?;
|
||||
}
|
||||
|
|
@ -633,9 +627,6 @@ fn check(docs: &Docs<'_>, staged: &Staged, mind: &Slug, import: Option<&ImportSo
|
|||
}
|
||||
// Decided in `answer_desk` before the document exists; no other door can stage one.
|
||||
D::DeskAnswer(_) => Ok(()),
|
||||
// The question is a citation (`listed`), so a missing one is refused with every other missing referent, and
|
||||
// the key derivation holds it to the listing's own campaign.
|
||||
D::DeskListing(_) => Ok(()),
|
||||
D::Ruling(ruling) => {
|
||||
if ruling.operator_quote.is_some() && ruling.authority != RulingAuthority::Operator {
|
||||
return Err(MindRefusal::QuoteWithoutOperator { ruling: key.into() });
|
||||
|
|
@ -914,7 +905,6 @@ fn matrix(subject: PipelineKind, outcome: &ResolutionOutcome) -> bool {
|
|||
(K::Target, O::Superseded { by }) => all(by, K::Target),
|
||||
(K::Question, O::Answered { by }) => by.kind == K::Ruling,
|
||||
(K::Question, O::Withdrawn { .. }) => true,
|
||||
(K::DeskListing, O::Withdrawn { .. }) => true,
|
||||
(K::Ruling, O::Superseded { by }) => all(by, K::Ruling),
|
||||
(K::CutSpec, O::Superseded { by }) => all(by, K::CutSpec),
|
||||
(K::CutSpec, O::Withdrawn { .. }) => true,
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@
|
|||
|
||||
use chrono::{DateTime, Duration, SecondsFormat, SubsecRound, Utc};
|
||||
use eureka_pipeline::{
|
||||
AccountId,
|
||||
DeskAuthority, DeskOutcome, DeskRefusal, PipelineDeskAnswer, PipelineDocument, PipelineKind, PipelineQuestion, PipelineRefusal,
|
||||
PipelineResolution, RequestId, ResolutionOutcome, Sha256Hex, Short, Slug, Words,
|
||||
};
|
||||
|
|
@ -146,7 +145,7 @@ fn counts_under(answer: &PipelineDeskAnswer, current: &Sha256Hex) -> bool {
|
|||
/// `answer_desk` runs before it reads the fork and the door runs before its showability gate.
|
||||
pub(crate) fn bound_answer<'a>(docs: &'a Docs<'_>, answerer: &Short, request_id: &RequestId) -> Option<(&'a str, &'a PipelineDeskAnswer)> {
|
||||
docs.of_kind(PipelineKind::DeskAnswer).find_map(|(key, document)| match document {
|
||||
PipelineDocument::DeskAnswer(held) if held.answerer.0 == answerer.0 && held.request_id == *request_id => Some((key, held)),
|
||||
PipelineDocument::DeskAnswer(held) if held.answerer == *answerer && held.request_id == *request_id => Some((key, held)),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
|
@ -315,7 +314,7 @@ impl<S: MindStore> Mind<S> {
|
|||
claim_in_future,
|
||||
request_id: request.request_id.clone(),
|
||||
body_sha256: body_sha256.clone(),
|
||||
answerer: AccountId(stamp.answerer.0.clone()),
|
||||
answerer: stamp.answerer.clone(),
|
||||
authority: stamp.authority,
|
||||
outcome,
|
||||
};
|
||||
|
|
@ -426,7 +425,7 @@ mod tests {
|
|||
const LIST: [&str; 6] = ["x", "y", "z", "a", "op", "dec"];
|
||||
|
||||
fn stamp_as(who: &str, authority: DeskAuthority, list: &[&str]) -> DeskStamp {
|
||||
DeskStamp { answerer: account(who), authority, deciders: list.iter().map(|id| account(id)).collect() }
|
||||
DeskStamp { answerer: s(who), authority, deciders: list.iter().map(|id| s(id)).collect() }
|
||||
}
|
||||
|
||||
fn decider(who: &str) -> DeskStamp {
|
||||
|
|
@ -532,7 +531,7 @@ mod tests {
|
|||
let mut mind = desk();
|
||||
stored(ask(&mut mind, &decider("z"), pick(FORK, "Q1", "A", "2026-11-10T12:00:00Z")));
|
||||
stored(ask(&mut mind, &decider("a"), pick(FORK, "Q1", "A", "2026-11-10T12:00:00Z")));
|
||||
assert_eq!(held(&mind, &counted(&mind, FORK).unwrap()).answerer, acct("z"));
|
||||
assert_eq!(held(&mind, &counted(&mind, FORK).unwrap()).answerer, s("z"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -781,7 +780,7 @@ mod tests {
|
|||
let record = held(&mind, &answer);
|
||||
assert_eq!(record.outcome, outcome);
|
||||
assert_eq!(outcome, DeskOutcome::Refused { code: DeskRefusal::NotADecider, retracts: None });
|
||||
assert_eq!((record.answerer, record.authority, record.answered_at), (acct("guest"), DeskAuthority::Steward, s(at)));
|
||||
assert_eq!((record.answerer, record.authority, record.answered_at), (s("guest"), DeskAuthority::Steward, s(at)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -791,7 +790,7 @@ mod tests {
|
|||
for (who, authority) in [("op", DeskAuthority::Operator), ("dec", DeskAuthority::Steward), ("x", DeskAuthority::Member)] {
|
||||
let (answer, _) = stored(ask(&mut mind, &stamp_as(who, authority, &LIST), pick(FORK, "Q1", "A", at)));
|
||||
let record = held(&mind, &answer);
|
||||
assert_eq!((record.answerer, record.authority), (acct(who), authority));
|
||||
assert_eq!((record.answerer, record.authority), (s(who), authority));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -937,7 +936,7 @@ mod tests {
|
|||
assert_eq!(counted(&mind, FORK), Some(answer));
|
||||
}
|
||||
|
||||
/// Libby's vectors (CultLib d3025d3c, `contracts/cultnet/governance/fork-card-vectors.json`, the `optionsHash` array):
|
||||
/// Libby's vectors (CultLib 6231ecad, `contracts/cultnet/governance/fork-card-vectors.json`, the `optionsHash` array):
|
||||
/// the seven of the binding file, A to G, and the eight she adds. Each is recomputed from its shown members.
|
||||
fn vectors() -> Vec<serde_json::Value> {
|
||||
let file: serde_json::Value = serde_json::from_str(include_str!("../vectors/fork-card-vectors.json")).unwrap();
|
||||
|
|
@ -1040,7 +1039,7 @@ mod tests {
|
|||
let DeskAnswerOutcome::Stored { receipt_id, .. } = outcome else { panic!("{outcome:?}") };
|
||||
let receipt = mind.receipts().unwrap().into_iter().find(|receipt| receipt.receipt_id == receipt_id).unwrap();
|
||||
assert_eq!(receipt.provenance.faculty, faculty, "{authority:?}");
|
||||
assert_eq!(receipt.provenance.agent, account("x"));
|
||||
assert_eq!(receipt.provenance.agent, s("x"));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1207,10 +1206,10 @@ mod tests {
|
|||
let image = first.image().unwrap();
|
||||
let found = Docs::at(image, image.head).find(K::DeskAnswer, &key("Q1", 1)).cloned();
|
||||
let Some(PipelineDocument::DeskAnswer(mut held)) = found else { panic!("stored") };
|
||||
held.answerer = eureka_pipeline::AccountId(long.clone());
|
||||
held.answerer = s(&long);
|
||||
store.plant(PipelineDocument::DeskAnswer(held).prepare(&cache).unwrap());
|
||||
let mut mind = opened(store, INSTANCE);
|
||||
let stamp = DeskStamp { answerer: s(&long), ..stamp_as("x", DeskAuthority::Member, &LIST) };
|
||||
let stamp = stamp_as(&long, DeskAuthority::Member, &LIST);
|
||||
let before = head(&mind);
|
||||
assert_eq!(replayed(ask(&mut mind, &stamp, request)), (key("Q1", 1), DeskOutcome::Counted));
|
||||
assert_eq!(head(&mind), before);
|
||||
|
|
@ -1263,8 +1262,8 @@ mod tests {
|
|||
assert_eq!(stored(ask(&mut mind, &decider("x"), sent.clone())), (key("Q1", 1), DeskOutcome::Counted));
|
||||
let (y_answer, y_outcome) = stored(ask(&mut mind, &decider("y"), sent.clone()));
|
||||
assert_eq!((y_answer.as_str(), &y_outcome), (key("Q1", 2).as_str(), &DeskOutcome::Revision { of: s(&key("Q1", 1)) }));
|
||||
assert_eq!(held(&mind, &y_answer).answerer, acct("y"));
|
||||
assert_eq!(held(&mind, &key("Q1", 1)).answerer, acct("x"));
|
||||
assert_eq!(held(&mind, &y_answer).answerer, s("y"));
|
||||
assert_eq!(held(&mind, &key("Q1", 1)).answerer, s("x"));
|
||||
// x's later retry still returns n1; y's returns n2.
|
||||
assert_eq!(replayed(ask(&mut mind, &decider("x"), sent.clone())).0, key("Q1", 1));
|
||||
assert_eq!(replayed(ask(&mut mind, &decider("y"), sent)).0, key("Q1", 2));
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
//! The fork desk's answer wire, Huginn's half: Libby's answer-side documents
|
||||
//! (CultLib `contracts/cultnet/governance/`, the contract file and its vectors at d3025d3c)
|
||||
//! (CultLib `contracts/cultnet/governance/`, the contract file and its vectors at e57fbaee)
|
||||
//! as Rust types, the one conversion between her `n<seq>` answer ids and the
|
||||
//! record keys, the projection of a stored answer, the refusal mapping and the
|
||||
//! showability predicate, and `Mind::desk_answer`, the one call that runs them in order.
|
||||
|
|
@ -20,7 +20,7 @@ pub use reads::{CardSchema, CardStatus, EntrySchema, ForkCard, ForkCursor, ForkE
|
|||
|
||||
use chrono::{DateTime, Utc};
|
||||
use eureka_pipeline::{
|
||||
AccountId, DeskAuthority, DeskOutcome, DeskRefusal, PipelineDeskAnswer, PipelineDocument, PipelineKind, PipelineQuestion, RequestId, Sha256Hex, Short, Slug, Words,
|
||||
DeskAuthority, DeskOutcome, DeskRefusal, PipelineDeskAnswer, PipelineDocument, PipelineKind, PipelineQuestion, RequestId, Sha256Hex, Short, Slug, Words,
|
||||
};
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
|
@ -147,10 +147,7 @@ pub struct ForkDeskAnswer {
|
|||
pub answered_at: String,
|
||||
pub received_at: String,
|
||||
pub claim_in_future: bool,
|
||||
/// The answering account, or nil in a non-decider's view (the key is kept). A stored answer always has one.
|
||||
#[serde(deserialize_with = "required")]
|
||||
#[schemars(required)]
|
||||
pub answerer: Option<String>,
|
||||
pub answerer: String,
|
||||
pub authority: DeskAuthority,
|
||||
pub request_id: String,
|
||||
}
|
||||
|
|
@ -183,12 +180,12 @@ pub struct ForkRefusal {
|
|||
/// `Refused` is any other Mind refusal; an `Unavailable` is Bifrost's to turn into its 503, and
|
||||
/// Huginn writes no document for it.
|
||||
///
|
||||
/// The trust boundary of the reads (ruling `desk-read-visibility-withheld-ids`): `Forks` and `Fork`
|
||||
/// carry every answer's note, to whatever the peer-checked socket carries; Bifrost strips notes
|
||||
/// at its door (a stripped read serves `note: null`, CultLib `fork-card.md`). Huginn itself
|
||||
/// withholds the account ids from a reader who is not a decider (`deciders` empty, `answerer` nil,
|
||||
/// counts in their place). Huginn must be reachable only by Bifrost's peer-checked socket: the
|
||||
/// peer-uid check IS the whole authentication, and a second reader of that socket is a note leak.
|
||||
/// The trust boundary of the reads (ruling `desk-read-visibility-whole-record`): `Forks` and `Fork`
|
||||
/// carry the WHOLE record, every answer's note included, to whatever the peer-checked socket
|
||||
/// carries. Huginn strips nothing and gates nothing; Bifrost applies visibility at its door (a
|
||||
/// stripped read serves `note: null`, CultLib `fork-card.md`). Huginn must therefore be reachable
|
||||
/// only by Bifrost's peer-checked socket: the peer-uid check IS the whole authentication, and a
|
||||
/// second reader of that socket is a note leak.
|
||||
// One value per call, built and encoded once; a box would only add a deref to every reader.
|
||||
#[allow(clippy::large_enum_variant)]
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
|
|
@ -232,33 +229,18 @@ pub const DESK_PAGE_BUDGET: usize = DESK_MAX_RESPONSE_BYTES - DESK_ENVELOPE_MARG
|
|||
/// Most deciders one call carries (CultLib `card.deciders` `maxItems`).
|
||||
pub const DESK_DECIDERS_MAX: usize = 64;
|
||||
|
||||
/// An account id read from the wire: a lowercase D-format UUID, or the decoder's refusal, which names the
|
||||
/// field and never quotes the value.
|
||||
fn account_id<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result<AccountId, D::Error> {
|
||||
let id = AccountId(String::deserialize(deserializer)?);
|
||||
id.validate_account().map_err(|_| serde::de::Error::custom("not an account id: a lowercase D-format UUID"))?;
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
/// A decider list, read whole or not at all: one to `DESK_DECIDERS_MAX` account ids, none twice. A
|
||||
/// bad list is the decoder's refusal, which names the field and never quotes an id.
|
||||
fn deciders<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result<Vec<AccountId>, D::Error> {
|
||||
/// A decider list, read whole or not at all: one to `DESK_DECIDERS_MAX` subjects, none twice. A
|
||||
/// bad list is the decoder's refusal, which names the field and never quotes a subject.
|
||||
fn deciders<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result<Vec<Short>, D::Error> {
|
||||
struct List;
|
||||
/// One list entry, through the same grammar as `subject`.
|
||||
struct Account(AccountId);
|
||||
impl<'de> Deserialize<'de> for Account {
|
||||
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
account_id(deserializer).map(Self)
|
||||
}
|
||||
}
|
||||
impl<'de> serde::de::Visitor<'de> for List {
|
||||
type Value = Vec<AccountId>;
|
||||
type Value = Vec<Short>;
|
||||
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
formatter.write_str("a list of distinct deciders")
|
||||
}
|
||||
fn visit_seq<A: serde::de::SeqAccess<'de>>(self, mut seq: A) -> Result<Vec<AccountId>, A::Error> {
|
||||
fn visit_seq<A: serde::de::SeqAccess<'de>>(self, mut seq: A) -> Result<Vec<Short>, A::Error> {
|
||||
let mut list = Vec::new();
|
||||
while let Some(Account(subject)) = seq.next_element::<Account>()? {
|
||||
while let Some(subject) = seq.next_element::<Short>()? {
|
||||
if list.len() == DESK_DECIDERS_MAX {
|
||||
return Err(serde::de::Error::custom("too many deciders"));
|
||||
}
|
||||
|
|
@ -279,7 +261,7 @@ fn deciders<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result<Vec<Acc
|
|||
fn deciders_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema {
|
||||
schemars::json_schema!({
|
||||
"type": "array",
|
||||
"items": generator.subschema_for::<AccountId>(),
|
||||
"items": generator.subschema_for::<Short>(),
|
||||
"minItems": 1,
|
||||
"maxItems": DESK_DECIDERS_MAX,
|
||||
"uniqueItems": true
|
||||
|
|
@ -293,33 +275,31 @@ fn deciders_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schem
|
|||
/// the set refuses the whole payload.
|
||||
///
|
||||
/// The reads, `Forks` and `Fork`, carry NO `authority` (CultLib `list_request` and
|
||||
/// `read_request` are closed and have none; one that names it is refused whole). Their `deciders`
|
||||
/// feed the showability gate, and `subject` decides the view: a subject in `deciders` is a decider,
|
||||
/// sees every showable fork and is told the ids; any other reads only the forks the Mind lists
|
||||
/// (a `desk_listing` in force) and is told counts in the ids' place. `Forks` takes the contract's `list_request` fields flat
|
||||
/// beside them (`status`, `cursor`, `limit`), each key written, `null` where absent. Bifrost
|
||||
/// strips notes; see `DeskResponse` for the trust boundary. These docs and the doc of `Mind::desk_forks`/`Mind::desk_fork` are the
|
||||
/// `read_request` are closed and have none; one that names it is refused whole). Their `subject`
|
||||
/// and `deciders` gate nothing: `deciders` only fills the card's `deciders` and the showability
|
||||
/// gate, `subject` is read by no read. `Forks` takes the contract's `list_request` fields flat
|
||||
/// beside them (`status`, `cursor`, `limit`), each key written, `null` where absent. Huginn
|
||||
/// answers every read with the whole record and Bifrost applies visibility; see `DeskResponse`
|
||||
/// for the trust boundary. These docs and the doc of `Mind::desk_forks`/`Mind::desk_fork` are the
|
||||
/// reads spec.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub enum DeskRequest {
|
||||
Answer {
|
||||
mind: Slug,
|
||||
#[serde(deserialize_with = "account_id")]
|
||||
subject: AccountId,
|
||||
subject: Short,
|
||||
authority: DeskAuthority,
|
||||
#[serde(deserialize_with = "deciders")]
|
||||
#[schemars(schema_with = "deciders_schema")]
|
||||
deciders: Vec<AccountId>,
|
||||
deciders: Vec<Short>,
|
||||
request: ForkAnswerRequest,
|
||||
},
|
||||
Forks {
|
||||
mind: Slug,
|
||||
#[serde(deserialize_with = "account_id")]
|
||||
subject: AccountId,
|
||||
subject: Short,
|
||||
#[serde(deserialize_with = "deciders")]
|
||||
#[schemars(schema_with = "deciders_schema")]
|
||||
deciders: Vec<AccountId>,
|
||||
deciders: Vec<Short>,
|
||||
#[serde(deserialize_with = "required")]
|
||||
#[schemars(required)]
|
||||
status: Option<ListStatus>,
|
||||
|
|
@ -330,11 +310,10 @@ pub enum DeskRequest {
|
|||
},
|
||||
Fork {
|
||||
mind: Slug,
|
||||
#[serde(deserialize_with = "account_id")]
|
||||
subject: AccountId,
|
||||
subject: Short,
|
||||
#[serde(deserialize_with = "deciders")]
|
||||
#[schemars(schema_with = "deciders_schema")]
|
||||
deciders: Vec<AccountId>,
|
||||
deciders: Vec<Short>,
|
||||
fork_id: String,
|
||||
},
|
||||
}
|
||||
|
|
@ -356,7 +335,7 @@ impl DeskRequest {
|
|||
}
|
||||
}
|
||||
|
||||
pub fn deciders(&self) -> &[AccountId] {
|
||||
pub fn deciders(&self) -> &[Short] {
|
||||
match self {
|
||||
Self::Answer { deciders, .. } | Self::Forks { deciders, .. } | Self::Fork { deciders, .. } => deciders,
|
||||
}
|
||||
|
|
@ -480,7 +459,7 @@ fn project(key: &str, stored: &PipelineDeskAnswer) -> Result<ForkDeskAnswer, Min
|
|||
answered_at: stored.answered_at.0.clone(),
|
||||
received_at: stored.received_at.0.clone(),
|
||||
claim_in_future: stored.claim_in_future,
|
||||
answerer: Some(stored.answerer.0.clone()),
|
||||
answerer: stored.answerer.0.clone(),
|
||||
authority: stored.authority,
|
||||
request_id: stored.request_id.0.clone(),
|
||||
})
|
||||
|
|
@ -549,13 +528,10 @@ impl<S: MindStore> Mind<S> {
|
|||
};
|
||||
let image = self.image()?;
|
||||
let docs = Docs::at(image, image.head);
|
||||
// A fork this reader may not see is unshowable here too, by the read side's own `visible`: the answer path
|
||||
// must not tell a hidden fork from a missing one, nor store a refusal for it. A request already stored replays
|
||||
// whatever the decider list is now (the lookup is `answer_desk`'s, asked here only so `showable` does not
|
||||
// shadow it), but never to a reader who can no longer see the fork: the counted answer id and `as_of` it
|
||||
// would carry are the fork's current state, so that replay is `fork_unknown` too (ruling clause 3).
|
||||
let replay = bound_answer(&docs, &subject, &sent.request_id).is_some();
|
||||
let unshowable = matches!(docs.find(PipelineKind::Question, &sent.fork.0), Some(PipelineDocument::Question(question)) if (!replay && !showable(question, deciders)) || !reads::visible(&docs, &sent.fork.0, deciders.contains(&subject)));
|
||||
// A request already stored replays whatever the list is now: the lookup is `answer_desk`'s,
|
||||
// asked here only so the gate does not shadow it.
|
||||
let unshowable = bound_answer(&docs, &subject, &sent.request_id).is_none()
|
||||
&& matches!(docs.find(PipelineKind::Question, &sent.fork.0), Some(PipelineDocument::Question(question)) if !showable(question, deciders));
|
||||
if unshowable {
|
||||
return Ok(DeskResponse::NotStored(fork_unknown()));
|
||||
}
|
||||
|
|
@ -590,7 +566,7 @@ mod tests {
|
|||
const LIST: [&str; 4] = ["x", "y", "z", "op"];
|
||||
const AT: &str = "2026-11-10T12:00:00Z";
|
||||
|
||||
// ---- Libby's vectors: CultLib d3025d3c, `contracts/cultnet/governance/fork-card-vectors.json` ----
|
||||
// ---- Libby's vectors: CultLib 6231ecad, `contracts/cultnet/governance/fork-card-vectors.json` ----
|
||||
|
||||
fn vectors() -> Value {
|
||||
serde_json::from_str(include_str!("../vectors/fork-card-vectors.json")).unwrap()
|
||||
|
|
@ -619,11 +595,6 @@ mod tests {
|
|||
fn desk() -> Mind<MemoryStore> {
|
||||
let mut mind = seeded();
|
||||
committed(admit(&mut mind, vec![question_n(1), question_n(2)]));
|
||||
// Listed, so a reader who is not a decider reaches them (an unlisted fork is the reads' own tests').
|
||||
for fork in [FORK, FORK2] {
|
||||
let listing = D::DeskListing(eureka_pipeline::PipelineDeskListing { campaign: slug("eureka-state"), listed: s(fork), reason: "Listed.".into() });
|
||||
committed(admit(&mut mind, vec![listing]));
|
||||
}
|
||||
mind
|
||||
}
|
||||
|
||||
|
|
@ -659,14 +630,14 @@ mod tests {
|
|||
}
|
||||
|
||||
fn deciders(list: &[&str]) -> Vec<Short> {
|
||||
list.iter().map(|who| account(who)).collect()
|
||||
list.iter().map(|who| s(who)).collect()
|
||||
}
|
||||
|
||||
/// The response and whether the call committed anything: a replay, a refused body and a gate
|
||||
/// refusal store nothing, so the head does not move.
|
||||
fn ask_as(mind: &mut Mind<MemoryStore>, who: &str, list: &[&str], request: ForkAnswerRequest) -> (DeskResponse, bool) {
|
||||
let before = head(mind);
|
||||
let response = mind.desk_answer(&deciders(list), account(who), DeskAuthority::Member, request, now());
|
||||
let response = mind.desk_answer(&deciders(list), s(who), DeskAuthority::Member, request, now());
|
||||
(response, head(mind) != before)
|
||||
}
|
||||
|
||||
|
|
@ -805,8 +776,7 @@ mod tests {
|
|||
claim_in_future: message["claim_in_future"].as_bool().unwrap(),
|
||||
request_id: RequestId(text("request_id").unwrap()),
|
||||
body_sha256: Sha256Hex("0".repeat(64)),
|
||||
// A withheld vector names no answerer: store a placeholder, the vector is the non-decider's view of it.
|
||||
answerer: AccountId(text("answerer").unwrap_or_else(|| "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f".into())),
|
||||
answerer: Short(text("answerer").unwrap()),
|
||||
authority,
|
||||
outcome,
|
||||
};
|
||||
|
|
@ -817,14 +787,10 @@ mod tests {
|
|||
fn stored_answers_project_to_the_desk_answer_vectors() {
|
||||
let file = vectors();
|
||||
let valid = rows(&file["deskAnswer"]["valid"]);
|
||||
assert_eq!(valid.len(), 8);
|
||||
assert_eq!(valid.len(), 5);
|
||||
for (name, message, bytes) in &valid {
|
||||
let (key, stored) = stored_from(message);
|
||||
let mut projected = project(&key, &stored).unwrap();
|
||||
// A null answerer is a non-decider's view of the stored answer.
|
||||
if message["answerer"].is_null() {
|
||||
projected.answerer = None;
|
||||
}
|
||||
let projected = project(&key, &stored).unwrap();
|
||||
assert_eq!(&encode(&projected), bytes, "{name}");
|
||||
}
|
||||
// Each answer inside her results is the projection of an equivalent record too.
|
||||
|
|
@ -1313,54 +1279,6 @@ mod tests {
|
|||
}
|
||||
}
|
||||
|
||||
/// Ruling clause 3: a stored request replays whatever the decider list is now, but never to a reader who can no
|
||||
/// longer see the fork. The replay would carry the fork's current counted answer id and head, so it is the same
|
||||
/// bytes a missing fork answers with; the stored word itself comes back once the fork is listed again.
|
||||
#[test]
|
||||
fn a_stored_answer_does_not_replay_to_a_reader_who_can_no_longer_see_the_fork() {
|
||||
let mut mind = desk();
|
||||
let hash = hash_of(&mind, FORK);
|
||||
// Dan is not a decider: his answer to the listed fork is a stored refusal and counts nothing.
|
||||
let dan_request = wire(FORK, &hash);
|
||||
let (first, wrote) = ask(&mut mind, "d", dan_request.clone());
|
||||
let (refused, counted, _) = answered(first);
|
||||
assert!((wrote, refused.outcome.as_str(), refused.code.is_some(), counted) == (true, "Refused", true, None), "{refused:?}");
|
||||
// A decider answers, so the fork now has a counted answer a replay could name.
|
||||
let x_request = wire(FORK, &hash);
|
||||
let (counted_by_x, _) = ask(&mut mind, "x", x_request.clone());
|
||||
assert!(answered(counted_by_x).1.is_some());
|
||||
// Still listed: the replay is the stored word, with the fork's current state.
|
||||
let before = head(&mind);
|
||||
let (listed_replay, wrote) = ask(&mut mind, "d", dan_request.clone());
|
||||
assert!(!wrote);
|
||||
let (_, counted, as_of) = answered(listed_replay);
|
||||
assert_eq!((counted.is_some(), as_of), (true, before));
|
||||
// Unlisted: the replay is fork_unknown, byte for byte a fork that does not exist, and writes nothing.
|
||||
let withdrawal = crate::fixtures::resolution_n(
|
||||
eureka_pipeline::PipelineRef { kind: PipelineKind::DeskListing, id: s("eureka-state:desk_listing:Q1") },
|
||||
1,
|
||||
eureka_pipeline::ResolutionOutcome::Withdrawn { reason: "unlisted".into() },
|
||||
);
|
||||
let withdrawal_key = eureka_pipeline::pipeline_key(&withdrawal).unwrap();
|
||||
committed(admit(&mut mind, vec![withdrawal]));
|
||||
let before = head(&mind);
|
||||
let (hidden, wrote) = ask(&mut mind, "d", dan_request.clone());
|
||||
assert!(!wrote);
|
||||
let (missing, _) = ask(&mut mind, "d", ForkAnswerRequest { fork_id: "eureka-state:question:Q99".into(), ..wire(FORK, &hash) });
|
||||
assert_eq!(not_stored(hidden.clone()).code, ForkRefusalCode::ForkUnknown);
|
||||
assert_eq!(encode(&hidden), encode(&missing));
|
||||
// A decider is told the fork whether or not it is listed.
|
||||
let (decider_replay, wrote) = ask(&mut mind, "x", x_request);
|
||||
assert!(!wrote && answered(decider_replay).1.is_some());
|
||||
assert_eq!(head(&mind), before);
|
||||
// Relisted: the stored word replays again.
|
||||
let subject = eureka_pipeline::PipelineRef { kind: PipelineKind::Resolution, id: s(&withdrawal_key) };
|
||||
committed(admit(&mut mind, vec![crate::fixtures::resolution(subject, eureka_pipeline::ResolutionOutcome::Withdrawn { reason: "listed again".into() })]));
|
||||
let (relisted, wrote) = ask(&mut mind, "d", dan_request);
|
||||
assert!(!wrote);
|
||||
assert_eq!(answered(relisted).0.outcome, "Refused");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn showable_needs_two_options_text_depends_and_a_decider() {
|
||||
let D::Question(plain) = question("Q9", &["A", "B"], "A") else { unreachable!() };
|
||||
|
|
@ -1430,9 +1348,9 @@ mod tests {
|
|||
let (other, committed) = ask(&mut mind, "y", request.clone());
|
||||
let (other, _, _) = answered(other);
|
||||
assert!(committed);
|
||||
assert_eq!((other.answer_id.as_str(), other.answerer.as_deref().unwrap(), other.outcome.as_str(), other.request_id.as_str()), ("n2", account("y").0.as_str(), "Revision", request.request_id.as_str()));
|
||||
assert_eq!((other.answer_id.as_str(), other.answerer.as_str(), other.outcome.as_str(), other.request_id.as_str()), ("n2", "y", "Revision", request.request_id.as_str()));
|
||||
let (original, _, _) = answered(ask(&mut mind, "x", request.clone()).0);
|
||||
assert_eq!((original.answer_id.as_str(), original.answerer.as_deref().unwrap()), ("n1", account("x").0.as_str()));
|
||||
assert_eq!((original.answer_id.as_str(), original.answerer.as_str()), ("n1", "x"));
|
||||
// V17 and V18: an id that is no lowercase D-format UUID is body_invalid naming request_id, and binds nothing.
|
||||
let before = head(&mind);
|
||||
for bad in ["retry-1", "0000000A-0000-4000-8000-00000000000B", "{00000000-0000-4000-8000-000000000001}", "00000000-0000-4000-8000-0000000000010", "", " 00000000-0000-4000-8000-000000000001", "00000000-0000-4000-8000-000000000001 ", "00000000-0000-4000-8000-000000000001\n", "00000000-0000-4000-8000-000000000001x"] {
|
||||
|
|
@ -1547,7 +1465,6 @@ mod tests {
|
|||
assert_eq!(shown_hash(&question).0, her_hash, "{name}: the card's hash");
|
||||
let mut mind = seeded();
|
||||
committed(admit(&mut mind, vec![D::Question(question)]));
|
||||
committed(admit(&mut mind, vec![D::DeskListing(eureka_pipeline::PipelineDeskListing { campaign: slug("eureka-state"), listed: s(FORK), reason: "Listed.".into() })]));
|
||||
let list: Vec<&str> = card["deciders"].as_array().unwrap().iter().map(|who| who.as_str().unwrap()).collect();
|
||||
if card["status"] == "closed" {
|
||||
close(&mut mind, FORK, "RQ1", "no");
|
||||
|
|
@ -1644,27 +1561,27 @@ mod tests {
|
|||
let mut mind = desk();
|
||||
let hash = hash_of(&mind, FORK);
|
||||
// The operator's own account, off the list, is no decider; a member on the list is.
|
||||
let off = mind.desk_answer(&deciders(&["x"]), account("op"), DeskAuthority::Operator, wire(FORK, &hash), now());
|
||||
let off = mind.desk_answer(&deciders(&["x"]), s("op"), DeskAuthority::Operator, wire(FORK, &hash), now());
|
||||
assert_eq!(answered(off).0.code, Some(ForkAnswerCode::NotADecider));
|
||||
let on = mind.desk_answer(&deciders(&["m"]), account("m"), DeskAuthority::Member, wire(FORK, &hash), now());
|
||||
let on = mind.desk_answer(&deciders(&["m"]), s("m"), DeskAuthority::Member, wire(FORK, &hash), now());
|
||||
let (answer, _, _) = answered(on);
|
||||
assert_eq!((answer.outcome.as_str(), answer.authority, answer.answerer.as_deref().unwrap()), ("Counted", DeskAuthority::Member, account("m").0.as_str()));
|
||||
assert_eq!((answer.outcome.as_str(), answer.authority, answer.answerer.as_str()), ("Counted", DeskAuthority::Member, "m"));
|
||||
}
|
||||
|
||||
// ---- the desk request on the wire: deciders ride each call, at most 64, never stored ----
|
||||
|
||||
use crate::envelope::{OperationFailure, decode_desk_request, encode_desk_request};
|
||||
|
||||
fn list_of(n: usize) -> Vec<AccountId> {
|
||||
(0..n).map(|at| acct(&format!("d{at}"))).collect()
|
||||
fn list_of(n: usize) -> Vec<Short> {
|
||||
(0..n).map(|at| s(&format!("d{at}"))).collect()
|
||||
}
|
||||
|
||||
/// Each request kind of the desk, carrying `list`.
|
||||
fn desk_requests(mind: &str, list: &[AccountId], hash: &str) -> [DeskRequest; 3] {
|
||||
fn desk_requests(mind: &str, list: &[Short], hash: &str) -> [DeskRequest; 3] {
|
||||
[
|
||||
DeskRequest::Answer { mind: slug(mind), subject: acct("d0"), authority: DeskAuthority::Member, deciders: list.to_vec(), request: wire(FORK, hash) },
|
||||
DeskRequest::Forks { mind: slug(mind), subject: acct("d0"), deciders: list.to_vec(), status: None, cursor: None, limit: ForkLimit::new(10).unwrap() },
|
||||
DeskRequest::Fork { mind: slug(mind), subject: acct("d0"), deciders: list.to_vec(), fork_id: FORK.into() },
|
||||
DeskRequest::Answer { mind: slug(mind), subject: s("d0"), authority: DeskAuthority::Member, deciders: list.to_vec(), request: wire(FORK, hash) },
|
||||
DeskRequest::Forks { mind: slug(mind), subject: s("d0"), deciders: list.to_vec(), status: None, cursor: None, limit: ForkLimit::new(10).unwrap() },
|
||||
DeskRequest::Fork { mind: slug(mind), subject: s("d0"), deciders: list.to_vec(), fork_id: FORK.into() },
|
||||
]
|
||||
}
|
||||
|
||||
|
|
@ -1687,7 +1604,7 @@ mod tests {
|
|||
// 64 deciders through to the Mind: the subject is listed and counts.
|
||||
let DeskRequest::Answer { subject, authority, deciders, request, .. } = through_the_door(&desk_requests(INSTANCE, &list_of(64), &hash)[0]).unwrap() else { panic!("not an answer") };
|
||||
let before = head(&mind);
|
||||
let response = mind.desk_answer(&deciders.iter().map(|id| s(&id.0)).collect::<Vec<_>>(), s(&subject.0), authority, request, now());
|
||||
let response = mind.desk_answer(&deciders, subject, authority, request, now());
|
||||
assert!(head(&mind) > before);
|
||||
assert_eq!(answered(response).0.outcome, "Counted");
|
||||
|
||||
|
|
@ -1707,7 +1624,7 @@ mod tests {
|
|||
let hash = hash_of(&mind, FORK);
|
||||
let before = (head(&mind), stored_count(&mind, FORK));
|
||||
let mut twice = list_of(3);
|
||||
twice.push(acct("d1"));
|
||||
twice.push(s("d1"));
|
||||
for (what, list) in [("empty", vec![]), ("duplicate", twice)] {
|
||||
for sent in desk_requests(INSTANCE, &list, &hash) {
|
||||
let failure = through_the_door(&sent).unwrap_err();
|
||||
|
|
@ -1743,6 +1660,6 @@ mod tests {
|
|||
// The same request to the mind it names is answered.
|
||||
let DeskRequest::Answer { mind: named, subject, authority, deciders, request } = through_the_door(&desk_requests(INSTANCE, &list_of(4), &hash)[0]).unwrap() else { panic!("not an answer") };
|
||||
mind.require_instance(&named).unwrap();
|
||||
assert_eq!(answered(mind.desk_answer(&deciders.iter().map(|id| s(&id.0)).collect::<Vec<_>>(), s(&subject.0), authority, request, now())).0.outcome, "Counted");
|
||||
assert_eq!(answered(mind.desk_answer(&deciders, subject, authority, request, now())).0.outcome, "Counted");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -74,19 +74,6 @@ impl<'i> Docs<'i> {
|
|||
self.image.id_of(kind, id).filter(|row| self.visible(*row)).map(|row| self.image.row(row).ordinal)
|
||||
}
|
||||
|
||||
/// When the document was first admitted, in UTC: an imported document's source receipt
|
||||
/// (`imported_from.committed_at`) wins over this mind's own `admitted_at`. `None` when the
|
||||
/// document is not held at this snapshot or the time does not parse as RFC3339 (an import's
|
||||
/// `committed_at` is only checked as non-empty text at admission). Also `None` when the UTC year is
|
||||
/// outside 0..=9999: CultLib's instant pattern has four year digits, and a local time near the edge
|
||||
/// can convert out of range.
|
||||
pub(crate) fn first_admitted_at(&self, kind: PipelineKind, id: &str) -> Option<chrono::DateTime<chrono::Utc>> {
|
||||
let row = self.image.id_of(kind, id).filter(|row| self.visible(*row))?;
|
||||
let admission = &self.image.row(row).view.admission;
|
||||
let text = admission.provenance.imported_from.as_ref().map_or(admission.admitted_at.as_str(), |source| source.committed_at.0.as_str());
|
||||
chrono::DateTime::parse_from_rfc3339(text).ok().map(|time| time.with_timezone(&chrono::Utc)).filter(|time| (0..=9999).contains(&chrono::Datelike::year(time)))
|
||||
}
|
||||
|
||||
pub(crate) fn find(&self, kind: PipelineKind, id: &str) -> Option<&PipelineDocument> {
|
||||
self.in_batch(kind, id).or_else(|| self.in_image(kind, id))
|
||||
}
|
||||
|
|
@ -401,8 +388,7 @@ pub(crate) fn line_of(document: &PipelineDocument) -> Option<Lineage<'_>> {
|
|||
| D::Pulse(_)
|
||||
| D::Reply(_)
|
||||
| D::Thought(_)
|
||||
| D::DeskAnswer(_)
|
||||
| D::DeskListing(_) => None,
|
||||
| D::DeskAnswer(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -456,11 +442,10 @@ pub enum CitationRole {
|
|||
DayThrough,
|
||||
Fork,
|
||||
Answer,
|
||||
Listed,
|
||||
}
|
||||
|
||||
impl CitationRole {
|
||||
pub const ALL: [CitationRole; 27] = [
|
||||
pub const ALL: [CitationRole; 26] = [
|
||||
Self::RaisedIn,
|
||||
Self::Answers,
|
||||
Self::Rulings,
|
||||
|
|
@ -487,7 +472,6 @@ impl CitationRole {
|
|||
Self::DayThrough,
|
||||
Self::Fork,
|
||||
Self::Answer,
|
||||
Self::Listed,
|
||||
];
|
||||
|
||||
/// The wire spelling, equal to the referring field's name.
|
||||
|
|
@ -519,7 +503,6 @@ impl CitationRole {
|
|||
Self::DayThrough => "day_through",
|
||||
Self::Fork => "fork",
|
||||
Self::Answer => "answer",
|
||||
Self::Listed => "listed",
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -547,7 +530,6 @@ impl CitationRole {
|
|||
Self::Run => K::Reply,
|
||||
Self::Reads => K::Thought,
|
||||
Self::Fork | Self::Answer => K::DeskAnswer,
|
||||
Self::Listed => K::DeskListing,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -557,7 +539,7 @@ impl CitationRole {
|
|||
pub fn target_kinds(self) -> Vec<PipelineKind> {
|
||||
use PipelineKind as K;
|
||||
match self {
|
||||
Self::Answers | Self::Questions | Self::Forks | Self::Fork | Self::Listed => vec![K::Question],
|
||||
Self::Answers | Self::Questions | Self::Forks | Self::Fork => vec![K::Question],
|
||||
Self::Rulings => vec![K::Ruling],
|
||||
Self::CutSpec => vec![K::CutSpec],
|
||||
Self::CutReport => vec![K::CutReport],
|
||||
|
|
@ -675,7 +657,6 @@ pub(crate) fn citations(document: &PipelineDocument) -> Vec<(CitationRole, Pipel
|
|||
DeskOutcome::Counted | DeskOutcome::Refused { .. } => {}
|
||||
}
|
||||
}
|
||||
D::DeskListing(listing) => edges.push((R::Listed, cited(K::Question, &listing.listed))),
|
||||
D::Campaign(_) | D::Target(_) | D::Instance(_) | D::Stewardship(_) | D::Pulse(_) => {}
|
||||
}
|
||||
edges
|
||||
|
|
|
|||
|
|
@ -408,9 +408,9 @@ mod tests {
|
|||
fn desk_answer() -> DeskRequest {
|
||||
DeskRequest::Answer {
|
||||
mind: slug(INSTANCE),
|
||||
subject: crate::fixtures::acct("user-7"),
|
||||
subject: crate::fixtures::s("user-7"),
|
||||
authority: DeskAuthority::Member,
|
||||
deciders: vec![crate::fixtures::acct("user-7")],
|
||||
deciders: vec![crate::fixtures::s("user-7")],
|
||||
request: ForkAnswerRequest {
|
||||
schema_version: crate::desk_wire::RequestSchema::V0,
|
||||
fork_id: "eureka-state:question:Q1".into(),
|
||||
|
|
@ -425,7 +425,7 @@ mod tests {
|
|||
}
|
||||
|
||||
fn desk_requests() -> [(DeskRequest, &'static str); 3] {
|
||||
let (mind, subject, deciders) = (slug(INSTANCE), crate::fixtures::acct("user-7"), vec![crate::fixtures::acct("user-7")]);
|
||||
let (mind, subject, deciders) = (slug(INSTANCE), crate::fixtures::s("user-7"), vec![crate::fixtures::s("user-7")]);
|
||||
[
|
||||
(desk_answer(), "Answer"),
|
||||
(DeskRequest::Forks { mind: mind.clone(), subject: subject.clone(), deciders: deciders.clone(), status: Some(crate::desk_wire::ListStatus::Open), cursor: crate::desk_wire::ForkCursor::new(7), limit: crate::desk_wire::ForkLimit::new(50).unwrap() }, "Forks"),
|
||||
|
|
@ -549,7 +549,7 @@ mod tests {
|
|||
answered_at: "2026-11-10T12:00:00Z".into(),
|
||||
received_at: "2026-11-10T12:00:01Z".into(),
|
||||
claim_in_future: false,
|
||||
answerer: Some("user-7".into()),
|
||||
answerer: "user-7".into(),
|
||||
authority: DeskAuthority::Member,
|
||||
request_id: "00000000-0000-4000-8000-000000000001".into(),
|
||||
},
|
||||
|
|
@ -579,13 +579,13 @@ mod tests {
|
|||
fn a_maximal_desk_message_fits_the_frame() {
|
||||
let long = |n: usize| "x".repeat(n);
|
||||
let wire = |message: &CultNetMessage| encode_cultnet_message_to_vec(message, CultNetWireContract::CultNetSchemaV0).unwrap().len();
|
||||
let deciders: Vec<_> = (0..64).map(|at| crate::fixtures::acct(&format!("decider {at}"))).collect();
|
||||
let deciders: Vec<_> = (0..64).map(|at| crate::fixtures::s(&format!("{at:0>200}"))).collect();
|
||||
let mut sized = vec![];
|
||||
for (request, _) in desk_requests() {
|
||||
let widest = match request {
|
||||
DeskRequest::Answer { mind, request, authority, .. } => DeskRequest::Answer {
|
||||
mind,
|
||||
subject: crate::fixtures::acct("the widest subject"),
|
||||
subject: crate::fixtures::s(&long(200)),
|
||||
authority,
|
||||
deciders: deciders.clone(),
|
||||
request: ForkAnswerRequest {
|
||||
|
|
@ -596,10 +596,10 @@ mod tests {
|
|||
},
|
||||
},
|
||||
DeskRequest::Forks { mind, status, cursor, limit, .. } => {
|
||||
DeskRequest::Forks { mind, subject: crate::fixtures::acct("the widest subject"), deciders: deciders.clone(), status, cursor, limit }
|
||||
DeskRequest::Forks { mind, subject: crate::fixtures::s(&long(200)), deciders: deciders.clone(), status, cursor, limit }
|
||||
}
|
||||
DeskRequest::Fork { mind, .. } => {
|
||||
DeskRequest::Fork { mind, subject: crate::fixtures::acct("the widest subject"), deciders: deciders.clone(), fork_id: long(200) }
|
||||
DeskRequest::Fork { mind, subject: crate::fixtures::s(&long(200)), deciders: deciders.clone(), fork_id: long(200) }
|
||||
}
|
||||
};
|
||||
let message = encode_desk_request("m-10", &widest, Some(long(200))).unwrap();
|
||||
|
|
@ -607,8 +607,7 @@ mod tests {
|
|||
sized.push(wire(&message));
|
||||
}
|
||||
assert!(sized.iter().all(|size| *size <= DESK_MAX_REQUEST_BYTES), "{sized:?} against {DESK_MAX_REQUEST_BYTES}");
|
||||
// 64 account ids of 36 bytes and a 4,000-byte note: the widest answer a request can carry.
|
||||
assert!(sized[0] > 4000 + 64 * 36, "the test builds a request that is actually large: {sized:?}");
|
||||
assert!(sized[0] > DESK_MAX_REQUEST_BYTES / 4, "the test builds a request that is actually large: {sized:?}");
|
||||
|
||||
let answer = ForkDeskAnswer {
|
||||
schema_version: crate::desk_wire::AnswerSchema::V0,
|
||||
|
|
@ -624,7 +623,7 @@ mod tests {
|
|||
answered_at: "2026-11-10T12:00:00.123456789+00:00".into(),
|
||||
received_at: "2026-11-10T12:00:00.123456789+00:00".into(),
|
||||
claim_in_future: true,
|
||||
answerer: Some(long(200)),
|
||||
answerer: long(200),
|
||||
authority: DeskAuthority::Operator,
|
||||
request_id: "00000000-0000-4000-8000-000000000001".into(),
|
||||
};
|
||||
|
|
|
|||
|
|
@ -56,21 +56,6 @@ pub(crate) fn s(value: &str) -> Short {
|
|||
value.into()
|
||||
}
|
||||
|
||||
/// A deterministic Bifrost account id for a test's name for someone: a lowercase D-format UUID.
|
||||
pub(crate) fn account(name: &str) -> Short {
|
||||
// An id already in account form stands for itself.
|
||||
if name.len() == 36 && name.bytes().all(|byte| matches!(byte, b'0'..=b'9' | b'a'..=b'f' | b'-')) {
|
||||
return Short(name.into());
|
||||
}
|
||||
let hash = name.bytes().fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| (hash ^ u64::from(byte)).wrapping_mul(0x0100_0000_01b3));
|
||||
Short(format!("{:08x}-{:04x}-4000-8000-{:012x}", hash >> 32, (hash >> 16) & 0xffff, hash & 0xffff_ffff_ffff))
|
||||
}
|
||||
|
||||
/// The same, as the stored type.
|
||||
pub(crate) fn acct(name: &str) -> eureka_pipeline::AccountId {
|
||||
eureka_pipeline::AccountId(account(name).0)
|
||||
}
|
||||
|
||||
pub(crate) fn l(value: &str) -> Label {
|
||||
value.into()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -105,8 +105,8 @@ pub fn index_text(document: &PipelineDocument) -> Option<String> {
|
|||
lines.push(&idea.idea.0);
|
||||
lines.push(&idea.tradeoffs.0);
|
||||
}
|
||||
// A desk answer is a person's words and a listing's reason is no retrieval text: neither is indexed.
|
||||
D::Instance(_) | D::Stewardship(_) | D::Run(_) | D::Pulse(_) | D::Reply(_) | D::DeskAnswer(_) | D::DeskListing(_) => {}
|
||||
// A desk answer is a person's words, not indexed.
|
||||
D::Instance(_) | D::Stewardship(_) | D::Run(_) | D::Pulse(_) | D::Reply(_) | D::DeskAnswer(_) => {}
|
||||
}
|
||||
lines.retain(|line| !line.is_empty());
|
||||
if lines.is_empty() {
|
||||
|
|
|
|||
|
|
@ -41,7 +41,6 @@ mod rows;
|
|||
pub mod oracle;
|
||||
pub mod query;
|
||||
pub mod receipt;
|
||||
pub mod redact;
|
||||
pub mod refusal;
|
||||
pub mod store;
|
||||
pub mod wire;
|
||||
|
|
|
|||
|
|
@ -166,8 +166,6 @@ pub enum PipelineFacts {
|
|||
/// the one that counts for its fork at the read's snapshot, derived by `counted_answer` and never
|
||||
/// stored: a retracted `Counted` answer reads `counts: false`, the promoted next answer `true`.
|
||||
DeskAnswer { fork: Short, sequence: u32, outcome: Short, counts: bool, authority: DeskAuthority },
|
||||
/// The question this listing lets members read.
|
||||
DeskListing { listed: Short },
|
||||
}
|
||||
|
||||
impl PipelineFacts {
|
||||
|
|
@ -308,7 +306,6 @@ impl PipelineFacts {
|
|||
counts: counted_of(&value.fork.0).as_deref() == Some(id.id.0.as_str()),
|
||||
authority: value.authority,
|
||||
},
|
||||
D::DeskListing(value) => Self::DeskListing { listed: value.listed.clone() },
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1852,7 +1849,7 @@ mod tests {
|
|||
|
||||
// Two desk answers on the open fork Q2: the second is a Revision of the first.
|
||||
let D::Question(shown) = question("Q2", &["A", "B"], "A") else { unreachable!() };
|
||||
let stamp = crate::desk::DeskStamp { answerer: crate::fixtures::account("x"), authority: eureka_pipeline::DeskAuthority::Member, deciders: vec![crate::fixtures::account("x")] };
|
||||
let stamp = crate::desk::DeskStamp { answerer: s("x"), authority: eureka_pipeline::DeskAuthority::Member, deciders: vec![s("x")] };
|
||||
let pick = |n: u32| crate::desk::DeskAnswerRequest {
|
||||
fork: s(&q2),
|
||||
options_hash: crate::desk::options_hash(&shown),
|
||||
|
|
@ -1865,10 +1862,6 @@ mod tests {
|
|||
assert!(matches!(mind.answer_desk(stamp.clone(), pick(n), now()), crate::desk::DeskAnswerOutcome::Stored { .. }));
|
||||
}
|
||||
|
||||
// The Mind lists Q2: one `listed` edge.
|
||||
committed(admit(&mut mind, vec![D::DeskListing(eureka_pipeline::PipelineDeskListing { campaign: slug(CAMPAIGN), listed: s(&q2), reason: "Members may read why.".into() })]));
|
||||
let listing = id("desk_listing", "Q2");
|
||||
|
||||
let (n1, f1_closed, f2_closed) = (
|
||||
id("resolution", "question.Q1.n1"),
|
||||
id("resolution", &format!("finding.{}.n1", f1.rsplit(':').next().unwrap())),
|
||||
|
|
@ -1917,9 +1910,8 @@ mod tests {
|
|||
(CitationRole::Fork, &answer_1, &q2),
|
||||
(CitationRole::Fork, &answer_2, &q2),
|
||||
(CitationRole::Answer, &answer_2, &answer_1),
|
||||
(CitationRole::Listed, &listing, &q2),
|
||||
];
|
||||
assert_eq!(CitationRole::ALL.len(), 27);
|
||||
assert_eq!(CitationRole::ALL.len(), 26);
|
||||
for role in CitationRole::ALL {
|
||||
let selection =
|
||||
Selection { cited: Some(Incoming { role: role.name().into(), exists: true }), ..Selection::default() };
|
||||
|
|
|
|||
|
|
@ -330,13 +330,13 @@ impl CacheBackingStore for LandsThenFails {
|
|||
fn pull_all(&self) -> Result<Vec<CultCacheEnvelope>> {
|
||||
self.inner.pull_all()
|
||||
}
|
||||
fn push(&mut self, entry: &CultCacheEnvelope) -> Result<String> {
|
||||
fn push(&mut self, entry: &CultCacheEnvelope) -> Result<()> {
|
||||
self.inner.push(entry)
|
||||
}
|
||||
fn delete(&mut self, entry: &CultCacheEnvelope) -> Result<()> {
|
||||
self.inner.delete(entry)
|
||||
}
|
||||
fn push_all(&mut self, entries: &[CultCacheEnvelope], options: PushAllOptions) -> Result<Vec<String>> {
|
||||
fn push_all(&mut self, entries: &[CultCacheEnvelope], options: PushAllOptions) -> Result<()> {
|
||||
self.inner.push_all(entries, options)
|
||||
}
|
||||
}
|
||||
|
|
@ -547,13 +547,13 @@ impl CacheBackingStore for FailsTheLook {
|
|||
}
|
||||
self.inner.pull_all()
|
||||
}
|
||||
fn push(&mut self, entry: &CultCacheEnvelope) -> Result<String> {
|
||||
fn push(&mut self, entry: &CultCacheEnvelope) -> Result<()> {
|
||||
self.inner.push(entry)
|
||||
}
|
||||
fn delete(&mut self, entry: &CultCacheEnvelope) -> Result<()> {
|
||||
self.inner.delete(entry)
|
||||
}
|
||||
fn push_all(&mut self, entries: &[CultCacheEnvelope], options: PushAllOptions) -> Result<Vec<String>> {
|
||||
fn push_all(&mut self, entries: &[CultCacheEnvelope], options: PushAllOptions) -> Result<()> {
|
||||
self.inner.push_all(entries, options)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -374,7 +374,7 @@ mod tests {
|
|||
self.inner.pull_all()
|
||||
}
|
||||
|
||||
fn push(&mut self, entry: &CultCacheEnvelope) -> anyhow::Result<String> {
|
||||
fn push(&mut self, entry: &CultCacheEnvelope) -> anyhow::Result<()> {
|
||||
self.inner.push(entry)
|
||||
}
|
||||
|
||||
|
|
@ -382,7 +382,7 @@ mod tests {
|
|||
self.inner.delete(entry)
|
||||
}
|
||||
|
||||
fn push_all(&mut self, entries: &[CultCacheEnvelope], options: PushAllOptions) -> anyhow::Result<Vec<String>> {
|
||||
fn push_all(&mut self, entries: &[CultCacheEnvelope], options: PushAllOptions) -> anyhow::Result<()> {
|
||||
self.inner.push_all(entries, options)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,514 +0,0 @@
|
|||
//! Credential redaction: the only code that knows the shape of a secret in text.
|
||||
//!
|
||||
//! `redact_url_userinfo` redacts URL credentials (the Self run's journal and reply,
|
||||
//! the Persona's structured output). `scrub` is chat ingest's door: every message body
|
||||
//! passes it before it is chunked, embedded or stored, and it also runs the URL rule.
|
||||
//! Shapes only; nothing here reads meaning, and a hit is a kind, never a value.
|
||||
|
||||
/// The one redaction of URL credentials. Call it at each place a string the CLI produced enters the journal, her mind or chat: today
|
||||
/// the Self result (`result_text`: journal line and Reply) and the structured output of a Persona or Sleep result
|
||||
/// (`persona::structured_output`, string by string). It does not cover claude's stderr (inherited) or the CLI's transcript files.
|
||||
///
|
||||
/// A forge token rides in a remote URL, so every `scheme://` and every protocol-relative `//` opens an authority, read as git, curl and
|
||||
/// RFC 3986 read it: it ends at the first `/`, `?`, `#`, backslash, quote, backtick, parenthesis, bracket, angle bracket or ASCII whitespace
|
||||
/// (`is_authority_end`). Its userinfo is everything before the authority's LAST `@` (a raw `@` in a password still redacts), and non-ASCII
|
||||
/// characters, NBSP and U+2028 included, stay inside it. The userinfo becomes `<redacted>`: `scheme://<redacted>@host`. With a colon it is
|
||||
/// always redacted; without one it is a bare token on http, https, ftp, ftps, ws, wss, any `+http`/`+https` scheme (`git+https`), protocol-relative `//` and an unparseable scheme, and a plain
|
||||
/// username on any other scheme (`ssh://git@host`), which stays. A `@` after the authority (`github.com/@user`, `pkg@1.2.3`) is path, not
|
||||
/// userinfo, and is left alone. Scheme-less credentials are covered only in the shape `word:secret@host.tld/` at the start of an
|
||||
/// ASCII-whitespace-delimited word. Not covered: a password holding `/ ? # ` a quote or ASCII whitespace (no parser reads it as userinfo),
|
||||
/// scp-style `user:secret@host:path`, tokens in query strings, claude's stderr, the kept transcripts. Syntax only; no reading of meaning.
|
||||
pub fn redact_url_userinfo(text: &str) -> String {
|
||||
let mut out = String::with_capacity(text.len());
|
||||
let mut rest = text;
|
||||
while let Some(at) = rest.find("//") {
|
||||
let (before, after) = rest.split_at(at);
|
||||
let body = &after[2..];
|
||||
out.push_str(before);
|
||||
out.push_str("//");
|
||||
// The scheme is the ASCII run before `:`. Without a colon, `//` opens an authority only at a word start (`a//b@c` is not one).
|
||||
let scheme = match before.strip_suffix(':') {
|
||||
// Only ASCII bytes match, so the byte after the last non-match is a character boundary.
|
||||
Some(head) => head[head.bytes().rposition(|c| !(c.is_ascii_alphanumeric() || matches!(c, b'+' | b'-' | b'.'))).map_or(0, |i| i + 1)..].to_ascii_lowercase(),
|
||||
None if before.chars().next_back().is_none_or(|c| is_break(c) || matches!(c, '"' | '\'' | '`' | '(' | '[' | '{' | '<' | '=' | ',')) => String::new(),
|
||||
None => {
|
||||
rest = body;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let end = body.find(is_authority_end).unwrap_or(body.len());
|
||||
let authority = &body[..end];
|
||||
match authority.rfind('@') {
|
||||
Some(i) if i > 0 && (authority[..i].contains(':') || carries_bare_token(&scheme)) => {
|
||||
out.push_str("<redacted>");
|
||||
out.push_str(&authority[i..]);
|
||||
}
|
||||
_ => out.push_str(authority),
|
||||
}
|
||||
rest = &body[end..];
|
||||
}
|
||||
out.push_str(rest);
|
||||
redact_schemeless(&out)
|
||||
}
|
||||
|
||||
/// Schemes whose lone userinfo is a token, not a username: web and ftp schemes, `ws`/`wss`, and `x+http`/`x+https` wrappers; `scheme` is lowercase.
|
||||
fn carries_bare_token(scheme: &str) -> bool {
|
||||
matches!(scheme, "" | "http" | "https" | "ftp" | "ftps" | "ws" | "wss") || scheme.ends_with("+http") || scheme.ends_with("+https")
|
||||
}
|
||||
|
||||
/// ASCII whitespace; NBSP, U+2028 and other Unicode whitespace are not, so a secret holding them is redacted whole.
|
||||
fn is_break(c: char) -> bool {
|
||||
matches!(c, ' ' | '\t' | '\n' | '\r' | '\x0b' | '\x0c')
|
||||
}
|
||||
|
||||
/// The end of a URL's authority.
|
||||
fn is_authority_end(c: char) -> bool {
|
||||
is_break(c) || matches!(c, '/' | '?' | '#' | '\\' | '"' | '\'' | '`' | '(' | ')' | '[' | ']' | '{' | '}' | '<' | '>')
|
||||
}
|
||||
|
||||
/// `word:secret@host.tld/` at the start of a whitespace-delimited word that holds no `//` becomes `<redacted>@host.tld/`.
|
||||
fn redact_schemeless(text: &str) -> String {
|
||||
let mut out = String::with_capacity(text.len());
|
||||
let mut word = String::new();
|
||||
let flush = |word: &mut String, out: &mut String| {
|
||||
out.push_str(&redact_word(word));
|
||||
word.clear();
|
||||
};
|
||||
for c in text.chars() {
|
||||
if is_break(c) {
|
||||
flush(&mut word, &mut out);
|
||||
out.push(c);
|
||||
} else {
|
||||
word.push(c);
|
||||
}
|
||||
}
|
||||
flush(&mut word, &mut out);
|
||||
out
|
||||
}
|
||||
|
||||
fn redact_word(word: &str) -> String {
|
||||
let shaped = (|| {
|
||||
if word.contains("//") {
|
||||
return None;
|
||||
}
|
||||
let at = word.rfind('@')?;
|
||||
let colon = word.find(':')?;
|
||||
let (user, host) = (&word[..colon], &word[at + 1..]);
|
||||
let host_name = &host[..host.find('/')?];
|
||||
let user_ok = !user.is_empty() && user.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-'));
|
||||
let host_ok = host_name.contains('.') && host_name.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-'));
|
||||
(colon + 1 < at && user_ok && host_ok).then_some(at)
|
||||
})();
|
||||
match shaped {
|
||||
Some(at) => format!("<redacted>{}", &word[at..]),
|
||||
None => word.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The version of the scrub's rules. Part of the chat version: a bump makes every
|
||||
/// room re-ingest from the room's history and scrub it again.
|
||||
pub const SCRUB_VERSION: u32 = 2;
|
||||
|
||||
/// What replaces a secret.
|
||||
pub const REDACTED: &str = "<redacted>";
|
||||
|
||||
/// The shape of secret a scrub found. Never its value.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub enum ScrubKind {
|
||||
UrlUserinfo,
|
||||
Bearer,
|
||||
AuthorizationHeader,
|
||||
KeyShape,
|
||||
Jwt,
|
||||
PemBlock,
|
||||
AccessToken,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ScrubKind {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter.write_str(match self {
|
||||
Self::UrlUserinfo => "url-userinfo",
|
||||
Self::Bearer => "bearer",
|
||||
Self::AuthorizationHeader => "authorization-header",
|
||||
Self::KeyShape => "key-shape",
|
||||
Self::Jwt => "jwt",
|
||||
Self::PemBlock => "pem-block",
|
||||
Self::AccessToken => "access-token",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// A body with its secrets replaced by `<redacted>`, and the kinds found, one per secret.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Scrubbed {
|
||||
pub text: String,
|
||||
pub hits: Vec<ScrubKind>,
|
||||
}
|
||||
|
||||
/// Zero-width characters a secret could be split with.
|
||||
fn zero_width(c: char) -> bool {
|
||||
matches!(c, '\u{200b}' | '\u{200c}' | '\u{200d}' | '\u{2060}' | '\u{feff}' | '\u{00ad}' | '\u{180e}')
|
||||
}
|
||||
|
||||
/// Replaces credential shapes in `text`: a PEM or PGP private key block, an `authorization:` header
|
||||
/// line, `Bearer`/`Basic`/`token` followed by a long token (blank or colon between), a vendor key
|
||||
/// shape (Matrix tokens included), an `access_token` query or JSON value, a JWT, and URL userinfo.
|
||||
/// Each hit becomes `<redacted>`. Token shapes only; ordinary prose, SHAs, UUIDs and ids pass byte for byte.
|
||||
/// Zero-width characters are removed before matching; when anything matched, the returned text is the
|
||||
/// stripped one, so a token split by them leaves no tail, and when nothing matched the text is returned as it came (emoji joiners stay).
|
||||
pub fn scrub(text: &str) -> Scrubbed {
|
||||
let mut hits = Vec::new();
|
||||
let mut stripped: String = text.chars().filter(|c| !zero_width(*c)).collect();
|
||||
let passes: [(ScrubKind, fn(&str, usize) -> Option<usize>); 6] = [
|
||||
(ScrubKind::PemBlock, pem_block),
|
||||
(ScrubKind::AuthorizationHeader, authorization_line),
|
||||
(ScrubKind::AccessToken, access_token),
|
||||
(ScrubKind::Bearer, bearer_token),
|
||||
(ScrubKind::KeyShape, key_shape),
|
||||
(ScrubKind::Jwt, jwt),
|
||||
];
|
||||
for (kind, find) in passes {
|
||||
stripped = replace_matches(&stripped, kind, find, &mut hits);
|
||||
}
|
||||
let urls = redact_url_userinfo(&stripped);
|
||||
let (before, after) = (stripped.matches(REDACTED).count(), urls.matches(REDACTED).count());
|
||||
hits.extend(std::iter::repeat_n(ScrubKind::UrlUserinfo, after.saturating_sub(before)));
|
||||
if hits.is_empty() { Scrubbed { text: text.to_owned(), hits } } else { Scrubbed { text: urls, hits } }
|
||||
}
|
||||
|
||||
fn replace_matches(text: &str, kind: ScrubKind, find: fn(&str, usize) -> Option<usize>, hits: &mut Vec<ScrubKind>) -> String {
|
||||
let mut out = String::with_capacity(text.len());
|
||||
let mut at = 0;
|
||||
while at < text.len() {
|
||||
match find(text, at) {
|
||||
Some(end) if end > at => {
|
||||
out.push_str(REDACTED);
|
||||
hits.push(kind);
|
||||
at = end;
|
||||
}
|
||||
_ => {
|
||||
let next = at + text[at..].chars().next().map_or(1, char::len_utf8);
|
||||
out.push_str(&text[at..next]);
|
||||
at = next;
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Whether `at` starts a word: the byte before it is not an ASCII letter or digit.
|
||||
fn word_start(text: &str, at: usize) -> bool {
|
||||
at == 0 || !text.as_bytes()[at - 1].is_ascii_alphanumeric()
|
||||
}
|
||||
|
||||
/// The end of the run of bytes at `from` that `member` accepts.
|
||||
fn run_end(text: &str, from: usize, member: impl Fn(u8) -> bool) -> usize {
|
||||
from + text.as_bytes()[from..].iter().take_while(|byte| member(**byte)).count()
|
||||
}
|
||||
|
||||
fn starts_with_ignore_case(text: &str, at: usize, word: &str) -> bool {
|
||||
text.as_bytes().get(at..at + word.len()).is_some_and(|held| held.eq_ignore_ascii_case(word.as_bytes()))
|
||||
}
|
||||
|
||||
fn token_char(byte: u8) -> bool {
|
||||
byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'~' | b'+' | b'/' | b'=' | b'-')
|
||||
}
|
||||
|
||||
fn key_char(byte: u8) -> bool {
|
||||
byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-')
|
||||
}
|
||||
|
||||
/// `-----BEGIN ... PRIVATE KEY-----` through its `-----END ... PRIVATE KEY-----`, or the text's end.
|
||||
fn pem_block(text: &str, at: usize) -> Option<usize> {
|
||||
let rest = &text[at..];
|
||||
let label = rest.strip_prefix("-----BEGIN ")?;
|
||||
let words = run_end(label, 0, |byte| byte.is_ascii_uppercase() || byte == b' ');
|
||||
(label[..words].ends_with("PRIVATE KEY") || label[..words].ends_with("PRIVATE KEY BLOCK")).then_some(())?;
|
||||
label[words..].starts_with("-----").then_some(())?;
|
||||
let body_from = at + "-----BEGIN ".len() + words + 5;
|
||||
let end = text[body_from..].find("-----END ").and_then(|end| {
|
||||
let name = body_from + end + "-----END ".len();
|
||||
let words = run_end(text, name, |byte| byte.is_ascii_uppercase() || byte == b' ');
|
||||
text[words..].starts_with("-----").then_some(words + 5)
|
||||
});
|
||||
Some(end.unwrap_or(text.len()))
|
||||
}
|
||||
|
||||
/// `authorization:` in any case at a word start, through the end of the line.
|
||||
fn authorization_line(text: &str, at: usize) -> Option<usize> {
|
||||
(word_start(text, at) && starts_with_ignore_case(text, at, "authorization:")).then_some(())?;
|
||||
let from = at + "authorization:".len();
|
||||
let end = run_end(text, from, |byte| byte != b'\n' && byte != b'\r');
|
||||
(!text[from..end].trim().is_empty()).then_some(end)
|
||||
}
|
||||
|
||||
/// `Bearer`, `Basic` or `token`, blanks, then 16 or more token characters.
|
||||
fn bearer_token(text: &str, at: usize) -> Option<usize> {
|
||||
word_start(text, at).then_some(())?;
|
||||
let word = ["bearer", "basic", "token"].into_iter().find(|word| starts_with_ignore_case(text, at, word))?;
|
||||
let blanks = at + word.len();
|
||||
let colon = usize::from(text.as_bytes().get(blanks) == Some(&b':'));
|
||||
let value = run_end(text, blanks + colon, |byte| byte == b' ' || byte == b'\t');
|
||||
(value > blanks).then_some(())?;
|
||||
let end = run_end(text, value, token_char);
|
||||
(end - value >= 16).then_some(end)
|
||||
}
|
||||
|
||||
/// A vendor key: the prefix, then a long enough run of the key alphabet.
|
||||
fn key_shape(text: &str, at: usize) -> Option<usize> {
|
||||
let rest = &text[at..];
|
||||
let by_prefix = [("sk-", 16), ("sk_live_", 16), ("pk_live_", 16), ("github_pat_", 16), ("glpat-", 16), ("syt_", 16), ("mct_", 16)];
|
||||
let tokens = ["ghp_", "gho_", "ghu_", "ghs_", "ghr_"].into_iter().map(|prefix| (prefix, 16));
|
||||
for (prefix, least) in by_prefix.into_iter().chain(tokens) {
|
||||
// Only the generic prefix needs a word start (`risk-averse`, `task-force`); the distinctive ones are found glued to anything.
|
||||
if rest.starts_with(prefix) && (prefix != "sk-" || word_start(text, at)) {
|
||||
let end = run_end(text, at + prefix.len(), key_char);
|
||||
return (end - at - prefix.len() >= least).then_some(end);
|
||||
}
|
||||
}
|
||||
for prefix in ["xoxa-", "xoxb-", "xoxp-", "xoxr-", "xoxs-"] {
|
||||
if rest.starts_with(prefix) {
|
||||
let end = run_end(text, at + prefix.len(), |byte| byte.is_ascii_alphanumeric() || byte == b'-');
|
||||
return (end - at - prefix.len() >= 16).then_some(end);
|
||||
}
|
||||
}
|
||||
for prefix in ["AKIA", "ASIA"] {
|
||||
if rest.starts_with(prefix) {
|
||||
let end = run_end(text, at + prefix.len(), |byte| byte.is_ascii_uppercase() || byte.is_ascii_digit());
|
||||
return (end - at - prefix.len() >= 16).then_some(end);
|
||||
}
|
||||
}
|
||||
if rest.starts_with("AIza") {
|
||||
let end = run_end(text, at + 4, key_char);
|
||||
return (end - at - 4 >= 35).then_some(end);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// `access_token` as a query parameter or JSON member, through its value (8 or more characters).
|
||||
fn access_token(text: &str, at: usize) -> Option<usize> {
|
||||
starts_with_ignore_case(text, at, "access_token").then_some(())?;
|
||||
let mut at = at + "access_token".len();
|
||||
let skip = |at: usize, set: &[u8]| run_end(text, at, |byte| set.contains(&byte));
|
||||
at = skip(at, b"\"'");
|
||||
at = skip(at, b" \t");
|
||||
matches!(text.as_bytes().get(at), Some(b'=' | b':')).then_some(())?;
|
||||
at = skip(at + 1, b" \t");
|
||||
at = skip(at, b"\"'");
|
||||
let end = run_end(text, at, |byte| !byte.is_ascii_whitespace() && !matches!(byte, b'&' | b'"' | b'\'' | b',' | b'}' | b';' | b'#'));
|
||||
(end - at >= 8).then_some(end)
|
||||
}
|
||||
|
||||
/// Three base64url segments, the first starting `eyJ`.
|
||||
fn jwt(text: &str, at: usize) -> Option<usize> {
|
||||
text[at..].starts_with("eyJ").then_some(())?;
|
||||
let segment = |from: usize| run_end(text, from, key_char);
|
||||
let first = segment(at);
|
||||
let mut end = first;
|
||||
for _ in 0..2 {
|
||||
(text.as_bytes().get(end) == Some(&b'.')).then_some(())?;
|
||||
let next = segment(end + 1);
|
||||
(next > end + 1).then_some(())?;
|
||||
end = next;
|
||||
}
|
||||
Some(end)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const CANARY: &str = "CANARY-NOT-A-REAL-TOKEN-0000";
|
||||
|
||||
#[test]
|
||||
fn url_userinfo_is_redacted_where_it_is_a_secret_and_kept_where_it_is_a_username() {
|
||||
let said = format!(
|
||||
"a https://user:{CANARY}@host.example/r.git b https://{CANARY}@host.example/r.git c ssh://u:{CANARY}@h/x d http://{CANARY}@h e HTTPS://x:{CANARY}@h \
|
||||
f https://u:p@ss{CANARY}@h/x g ssh://git@host.example/r.git h git://git@h/x i https://host.example/no-userinfo j (https://u:{CANARY}@h) k ://@x"
|
||||
);
|
||||
let want = "a https://<redacted>@host.example/r.git b https://<redacted>@host.example/r.git c ssh://<redacted>@h/x d http://<redacted>@h e HTTPS://<redacted>@h \
|
||||
f https://<redacted>@h/x g ssh://git@host.example/r.git h git://git@h/x i https://host.example/no-userinfo j (https://<redacted>@h) k ://@x";
|
||||
assert_eq!(redact_url_userinfo(&said), want);
|
||||
assert!(!redact_url_userinfo(&said).contains(CANARY));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_secret_is_redacted_whole_by_the_authority_rule_and_commonplace_urls_are_left_alone() {
|
||||
let r = redact_url_userinfo;
|
||||
// Raw @, percent-encoding and non-ASCII (NBSP, U+2028) stay inside userinfo.
|
||||
for secret in ["pa@ssT", "pa%40ssT", "pa\u{a0}ssT", "pa\u{2028}ssT", "pa:ssT", "\u{e9}\u{65e5}ssT"] {
|
||||
let out = r(&format!("x https://u:{secret}@h/x"));
|
||||
assert_eq!(out, "x https://<redacted>@h/x", "{secret:?}");
|
||||
}
|
||||
// The authority ends at / ? # \\ quotes, brackets and ASCII whitespace: no parser reads those as userinfo, so what follows is path.
|
||||
assert_eq!(r("https://u:pa/ssT@h/x"), "https://u:pa/ssT@h/x");
|
||||
assert_eq!(r("https://u:pa ssT@h/x"), "https://u:pa ssT@h/x");
|
||||
assert_eq!(r("https://u:pa?ssT@h"), "https://u:pa?ssT@h");
|
||||
// Ordinary URLs with @ in the path are byte-identical.
|
||||
for same in [
|
||||
"https://github.com/@user/repo",
|
||||
"https://medium.com/@name/post-1",
|
||||
"https://registry.npmjs.org/@scope/pkg",
|
||||
"https://h/pkg@1.2.3",
|
||||
"a//b@c",
|
||||
"see //cdn.h/@x/y and https://h/?mail=a@b.c#f@g",
|
||||
"https://h:8080/p git@github.com:org/r.git ssh://git@host/r.git ssh://git@h:22/r git://git@h/x",
|
||||
] {
|
||||
assert_eq!(r(same), same);
|
||||
}
|
||||
// Compact JSON: only the credentialed URL changes.
|
||||
let json = format!(r#"{{"a":"https://u:{CANARY}@h/x","b":"https://h2/@v","c":"https://h3/p@1"}}"#);
|
||||
assert_eq!(r(&json), r#"{"a":"https://<redacted>@h/x","b":"https://h2/@v","c":"https://h3/p@1"}"#);
|
||||
assert_eq!(r("a/https://u:T@h/x,https://v:T@h2"), "a/https://<redacted>@h/x,https://<redacted>@h2");
|
||||
assert_eq!(r("HTTPS://TOKEN@h"), "HTTPS://<redacted>@h");
|
||||
assert_eq!(r("[l](https://T@h/x) <https://T@h> `https://u:T@h`"), "[l](https://<redacted>@h/x) <https://<redacted>@h> `https://<redacted>@h`");
|
||||
assert_eq!(r("a //u:T@host/x b //T@host c \"//T@h\" (//T@h)"), "a //<redacted>@host/x b //<redacted>@host c \"//<redacted>@h\" (//<redacted>@h)");
|
||||
// Token-bearing schemes beyond http: +http/+https wrappers and ws/wss redact a bare token; git+ssh and ssh keep their username.
|
||||
assert_eq!(r("git+https://T@github.com/a/b GIT+HTTPS://T@h wss://T@h/ ws://T@h/"), "git+https://<redacted>@github.com/a/b GIT+HTTPS://<redacted>@h wss://<redacted>@h/ ws://<redacted>@h/");
|
||||
for same in ["git+ssh://git@host/x", "ssh://git@host/x"] {
|
||||
assert_eq!(r(same), same);
|
||||
}
|
||||
// Scheme-less: only word:secret@host.tld/ . Each guard of the shape has its own case.
|
||||
assert_eq!(r("user:T@host.tld/path"), "<redacted>@host.tld/path");
|
||||
for same in ["a:b@c", "a:b@host/x", "a:b@ho_st.tld/x", "u$er:T@host.tld/x", "a:@host.tld/x", "mail me@x.org/ and :T@host.tld/x"] {
|
||||
assert_eq!(r(same), same);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multibyte_text_never_panics_the_redaction() {
|
||||
let r = redact_url_userinfo;
|
||||
for case in ["see \u{201c}https://h/x", "\u{2192}https://h/x", "docs\u{ff1a}https://h/x", "\u{e9}://x", "\u{65e5}\u{672c}://u:T@h", "https://x/\u{e9}://", "\u{e9}//u:T@h", "\u{e9}:u:T@h.tld/"] {
|
||||
let _ = r(case);
|
||||
}
|
||||
assert_eq!(r("\u{65e5}\u{672c}://u:T@h"), "\u{65e5}\u{672c}://<redacted>@h");
|
||||
assert_eq!(r("\u{2192}https://u:T@h/x"), "\u{2192}https://<redacted>@h/x");
|
||||
let seed = "\u{201c}\u{2192}\u{ff1a}\u{e9}\u{65e5}\u{a0}https://u:T@h/x//y:z@a.b/ ://@ //";
|
||||
let chars: Vec<char> = seed.chars().collect();
|
||||
for start in 0..=chars.len() {
|
||||
for end in start..=chars.len() {
|
||||
let _ = r(&chars[start..end].iter().collect::<String>());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// Every shape is built here from fake parts; none is a real credential.
|
||||
fn secrets() -> Vec<(ScrubKind, String, String)> {
|
||||
let pem = "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAfakefakefake\n-----END RSA PRIVATE KEY-----".to_owned();
|
||||
vec![
|
||||
(ScrubKind::AuthorizationHeader, format!("Authorization: Bearer {}", "a1B2".repeat(8)), "a1B2a1B2".into()),
|
||||
(ScrubKind::Bearer, format!("Bearer {}", "z9Y8x7".repeat(4)), "z9Y8x7z9Y8".into()),
|
||||
(ScrubKind::Bearer, format!("Basic {}", "dXNlcjpwYXNz".repeat(2)), "dXNlcjpwYXNzdXNl".into()),
|
||||
(ScrubKind::KeyShape, format!("sk-{}", "q7W3".repeat(8)), "q7W3q7W3".into()),
|
||||
(ScrubKind::KeyShape, format!("sk_live_{}", "m4N5".repeat(6)), "m4N5m4N5".into()),
|
||||
(ScrubKind::KeyShape, format!("ghp_{}", "k2L9".repeat(9)), "k2L9k2L9".into()),
|
||||
(ScrubKind::KeyShape, format!("github_pat_{}_{}", "A1b2C3".repeat(4) + "ab", "d4E5f6".repeat(9) + "abcde"), "A1b2C3A1b2".into()),
|
||||
(ScrubKind::KeyShape, format!("glpat-{}", "r6T7".repeat(5)), "r6T7r6T7".into()),
|
||||
(ScrubKind::KeyShape, "xoxb-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL".into()),
|
||||
(ScrubKind::KeyShape, "AKIA0123456789ABCDEF".into(), "0123456789ABCDEF".into()),
|
||||
(ScrubKind::KeyShape, format!("AIza{}", "Sy1_".repeat(9)), "Sy1_Sy1_Sy1_".into()),
|
||||
(ScrubKind::Jwt, "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJmYWtlIn0.c2lnbmF0dXJlZmFrZQ".into(), "c2lnbmF0dXJlZmFrZQ".into()),
|
||||
(ScrubKind::PemBlock, pem, "MIIEowIBAAKCAQEAfake".into()),
|
||||
(ScrubKind::PemBlock, "-----BEGIN PRIVATE KEY-----\nMIIBVQIBADANfakeNOEND".into(), "MIIBVQIBADANfake".into()),
|
||||
(ScrubKind::KeyShape, format!("syt_{}_{}_{}", "YWxpY2U", "aBcDeFgHiJkLmNoPqR", "0a1B2c"), "aBcDeFgHiJkLmNoPqR".into()),
|
||||
(ScrubKind::KeyShape, format!("mct_{}", "w4X5".repeat(8)), "w4X5w4X5w4X5".into()),
|
||||
(ScrubKind::KeyShape, "xoxa-2-123456789012-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL".into()),
|
||||
(ScrubKind::KeyShape, "xoxp-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL".into()),
|
||||
(ScrubKind::KeyShape, "xoxr-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL".into()),
|
||||
(ScrubKind::KeyShape, "xoxs-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL".into()),
|
||||
(ScrubKind::KeyShape, "ASIA0123456789ABCDEF".into(), "0123456789ABCDEF".into()),
|
||||
(ScrubKind::KeyShape, "gho_".to_owned() + &"p1Q2".repeat(9), "p1Q2p1Q2p1Q2".into()),
|
||||
(ScrubKind::KeyShape, "pk_live_".to_owned() + &"e3R4".repeat(6), "e3R4e3R4e3R4".into()),
|
||||
(ScrubKind::AccessToken, "https://h/_matrix/client/v3/sync?access_token=abcDEF123456xyz&since=s1".into(), "abcDEF123456xyz".into()),
|
||||
(ScrubKind::AccessToken, "{\"access_token\": \"tokenValue987654\", \"user\": 1}".into(), "tokenValue987654".into()),
|
||||
(ScrubKind::Bearer, format!("bearer:{}", "z9Y8x7".repeat(4)), "z9Y8x7z9Y8".into()),
|
||||
(ScrubKind::Bearer, format!("token: {}", "m1N2o3".repeat(4)), "m1N2o3m1N2".into()),
|
||||
(ScrubKind::PemBlock, "-----BEGIN PGP PRIVATE KEY BLOCK-----\nlQOYBFakeFakeFake\n-----END PGP PRIVATE KEY BLOCK-----".into(), "lQOYBFakeFake".into()),
|
||||
(ScrubKind::UrlUserinfo, "https://user:hunter2hunter2@host.example/r.git".into(), "hunter2hunter2".into()),
|
||||
(ScrubKind::UrlUserinfo, "https://tokenvalue12345@host.example/".into(), "tokenvalue12345".into()),
|
||||
]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_planted_credential_is_replaced_mid_line_and_at_line_end_with_its_kind() {
|
||||
for (kind, secret, distinct) in secrets() {
|
||||
for body in [format!("before {secret} after"), format!("before {secret}"), format!("{secret}\nnext line")] {
|
||||
let out = scrub(&body);
|
||||
assert!(!out.text.contains(&distinct), "{kind}: {}", out.text);
|
||||
assert!(out.text.contains(REDACTED), "{kind}: {}", out.text);
|
||||
assert!(out.hits.contains(&kind), "{kind} not in {:?} for {body:?}", out.hits);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_pem_block_without_its_end_runs_to_the_end_of_the_body_and_one_with_it_stops_there() {
|
||||
let open = scrub("see -----BEGIN EC PRIVATE KEY-----\nAAAA\nBBBB").text;
|
||||
assert_eq!(open, "see <redacted>");
|
||||
let closed = scrub("-----BEGIN PRIVATE KEY-----\nAAAA\n-----END PRIVATE KEY----- and then prose").text;
|
||||
assert_eq!(closed, "<redacted> and then prose");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_authorization_header_goes_to_the_end_of_its_line_only() {
|
||||
assert_eq!(scrub("hi\nAUTHORIZATION: Token abc\nbye").text, "hi\n<redacted>\nbye");
|
||||
assert_eq!(scrub("Authorization: x").hits, [ScrubKind::AuthorizationHeader]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ordinary_text_passes_byte_for_byte_and_reports_no_hit() {
|
||||
for same in [
|
||||
"risk-averse", "task-force", "sk-", "sk-short", "3f2b8c1d9e7a4b5c6d0e1f2a3b4c5d6e7f8a9b0c",
|
||||
"123e4567-e89b-12d3-a456-426614174000", "!abc:s", "$Ab3dEf-9_zQ", "user@example.com", "ssh://git@host/r.git",
|
||||
"https://github.com/@user", "pkg@1.2.3", "the bearer of bad news", "Authorization is a header",
|
||||
"padding like dGVzdA== in prose", "```\nlet x = 1;\n```", "the BEGIN PRIVATE KEY header", "a token of thanks",
|
||||
"AKIAshort", "ghp_short", "eyJ is a prefix", "naïve café — 日本語 with Bearer in it",
|
||||
] {
|
||||
let out = scrub(same);
|
||||
assert_eq!(out.text, same);
|
||||
assert!(out.hits.is_empty(), "{same}: {:?}", out.hits);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multibyte_text_around_a_secret_never_panics_the_scrub() {
|
||||
let body = format!("日本 \u{a0}é Bearer {} → ünï", "A1b2".repeat(6));
|
||||
assert_eq!(scrub(&body).text, "日本 \u{a0}é <redacted> → ünï");
|
||||
let seed = "é日 sk-aaaaaaaaaaaaaaaaaaaa -----BEGIN PRIVATE KEY----- authorization: é eyJa.b.c https://u:p@h/x";
|
||||
let chars: Vec<char> = seed.chars().collect();
|
||||
for start in 0..chars.len() {
|
||||
for end in start..=chars.len() {
|
||||
let _ = scrub(&chars[start..end].iter().collect::<String>());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_distinctive_key_is_found_glued_to_other_text_but_the_generic_prefix_still_needs_a_word_start() {
|
||||
let ghp = format!("xxx{}", "ghp_".to_owned() + &"k2L9".repeat(9));
|
||||
assert_eq!(scrub(&ghp).text, "xxx<redacted>");
|
||||
assert_eq!(scrub("idAKIA0123456789ABCDEFend").hits, [ScrubKind::KeyShape]);
|
||||
assert_eq!(scrub(&format!("zzz{}", "xoxb-123456789012-aBcDeFgHiJkLmNoP")).hits, [ScrubKind::KeyShape]);
|
||||
assert_eq!(scrub(&format!("q{}", "AIza".to_owned() + &"Sy1_".repeat(9))).hits, [ScrubKind::KeyShape]);
|
||||
assert!(scrub("risk-averse-thinking-process task-force-alpha-bravo").hits.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_width_characters_inside_a_token_leave_no_tail_and_emoji_joiners_survive_when_nothing_matched() {
|
||||
let split = format!("sk-{}\u{200b}{}", "q7W3".repeat(4), "q7W3".repeat(4));
|
||||
let out = scrub(&format!("key {split} end"));
|
||||
assert_eq!(out.text, "key <redacted> end");
|
||||
let jwt = "eyJhbGciOiJIUzI1NiJ9.eyJz\u{feff}dWIiOiJmYWtlIn0.c2lnbmF0dXJl\u{200d}ZmFrZQ";
|
||||
assert_eq!(scrub(jwt).text, "<redacted>");
|
||||
let family = "family \u{1f468}\u{200d}\u{1f469}\u{200d}\u{1f467} here";
|
||||
assert_eq!(scrub(family).text, family);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matrix_tokens_and_query_values_are_found_and_prose_about_them_is_not() {
|
||||
for same in ["the access_token is a thing", "access_token: short", "syt_ is the prefix", "mct_ tokens", "bearer:short"] {
|
||||
assert_eq!(scrub(same).text, same, "{same}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -248,7 +248,7 @@ impl SelectionRow {
|
|||
D::Stewardship(stewardship) => vec![stewardship.repo.identity()],
|
||||
D::HandOff(hand_off) => vec![hand_off.repo.identity()],
|
||||
D::Ruling(PipelineRuling { authority: RulingAuthority::Steward { repo, .. }, .. }) => vec![repo.identity()],
|
||||
D::Target(_) | D::Question(_) | D::Ruling(_) | D::Verdict(_) | D::Finding(_) | D::Instance(_) | D::Resolution(_) | D::Run(_) | D::Pulse(_) | D::Reply(_) | D::Thought(_) | D::Memory(_) | D::Research(_) | D::Idea(_) | D::DeskAnswer(_) | D::DeskListing(_) => {
|
||||
D::Target(_) | D::Question(_) | D::Ruling(_) | D::Verdict(_) | D::Finding(_) | D::Instance(_) | D::Resolution(_) | D::Run(_) | D::Pulse(_) | D::Reply(_) | D::Thought(_) | D::Memory(_) | D::Research(_) | D::Idea(_) | D::DeskAnswer(_) => {
|
||||
Vec::new()
|
||||
}
|
||||
},
|
||||
|
|
@ -466,7 +466,7 @@ mod tests {
|
|||
/// refused at the substrate's door.
|
||||
#[test]
|
||||
fn each_kind_declares_exactly_its_aliases_and_none_is_numeric() {
|
||||
let table: [(K, &[&str]); 22] = [
|
||||
let table: [(K, &[&str]); 21] = [
|
||||
(K::Campaign, &["repo"]),
|
||||
(K::Target, &[]),
|
||||
(K::Question, &[]),
|
||||
|
|
@ -488,7 +488,6 @@ mod tests {
|
|||
(K::Reply, &[]),
|
||||
(K::Thought, &[]),
|
||||
(K::DeskAnswer, &[]),
|
||||
(K::DeskListing, &[]),
|
||||
];
|
||||
for (kind, extra) in table {
|
||||
let want = [&["root", "in_force", "faculty"][..], extra].concat();
|
||||
|
|
@ -504,7 +503,7 @@ mod tests {
|
|||
/// twenty-one.
|
||||
#[test]
|
||||
fn each_kind_declares_the_roles_it_carries_and_their_targets() {
|
||||
let table: [(K, &[&str]); 22] = [
|
||||
let table: [(K, &[&str]); 21] = [
|
||||
(K::Campaign, &[]),
|
||||
(K::Target, &[]),
|
||||
(K::Question, &["raised_in"]),
|
||||
|
|
@ -526,14 +525,13 @@ mod tests {
|
|||
(K::Reply, &["run"]),
|
||||
(K::Thought, &["reads"]),
|
||||
(K::DeskAnswer, &["fork", "answer"]),
|
||||
(K::DeskListing, &["listed"]),
|
||||
];
|
||||
for (kind, roles) in table {
|
||||
assert_eq!(sorted(Vocabulary.declared_roles(kind.type_id())), strings(roles), "{kind:?}");
|
||||
}
|
||||
assert_eq!(Vocabulary.target_leaves("answers"), vec![K::Question.type_id().to_string()]);
|
||||
assert_eq!(Vocabulary.target_leaves("rulings"), vec![K::Ruling.type_id().to_string()]);
|
||||
assert_eq!(Vocabulary.target_leaves("subject").len(), 22);
|
||||
assert_eq!(Vocabulary.target_leaves("subject").len(), 21);
|
||||
assert_eq!(
|
||||
sorted(Vocabulary.target_leaves("claims")),
|
||||
sorted([K::CutSpec, K::Finding, K::FollowUp].iter().map(|kind| kind.type_id().to_string()).collect())
|
||||
|
|
|
|||
|
|
@ -116,9 +116,9 @@ pub(crate) mod test_stores {
|
|||
Ok(self.rows())
|
||||
}
|
||||
|
||||
fn push(&mut self, entry: &CultCacheEnvelope) -> Result<String> {
|
||||
fn push(&mut self, entry: &CultCacheEnvelope) -> Result<()> {
|
||||
self.plant(entry.clone());
|
||||
Ok(entry.stored_at.clone())
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn delete(&mut self, entry: &CultCacheEnvelope) -> Result<()> {
|
||||
|
|
@ -126,9 +126,9 @@ pub(crate) mod test_stores {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn push_all(&mut self, entries: &[CultCacheEnvelope], _options: PushAllOptions) -> Result<Vec<String>> {
|
||||
fn push_all(&mut self, entries: &[CultCacheEnvelope], _options: PushAllOptions) -> Result<()> {
|
||||
*self.rows.lock().unwrap() = entries.iter().map(|entry| (identity(entry), entry.clone())).collect();
|
||||
Ok(entries.iter().map(|entry| entry.stored_at.clone()).collect())
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -199,7 +199,7 @@ pub(crate) mod test_stores {
|
|||
self.inner.pull_all()
|
||||
}
|
||||
|
||||
fn push(&mut self, entry: &CultCacheEnvelope) -> Result<String> {
|
||||
fn push(&mut self, entry: &CultCacheEnvelope) -> Result<()> {
|
||||
self.inner.push(entry)
|
||||
}
|
||||
|
||||
|
|
@ -207,7 +207,7 @@ pub(crate) mod test_stores {
|
|||
self.inner.delete(entry)
|
||||
}
|
||||
|
||||
fn push_all(&mut self, entries: &[CultCacheEnvelope], options: PushAllOptions) -> Result<Vec<String>> {
|
||||
fn push_all(&mut self, entries: &[CultCacheEnvelope], options: PushAllOptions) -> Result<()> {
|
||||
self.inner.push_all(entries, options)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,10 +3,9 @@
|
|||
//! are what the door speaks. `docs/desk-feed-door.md` is the contract they pin; the README in the
|
||||
//! fixtures directory lists them.
|
||||
//!
|
||||
//! The entry-bearing fixtures carry the reader-count shapes (`raised_at`, `decider_count`,
|
||||
//! `answerer_count`, a nullable `answerer`), a decider's view and a non-decider's. CultLib's own bytes
|
||||
//! for those shapes are vendored at one commit in `fixtures/desk-feed/cultlib-d3025d3c/`; Huginn's
|
||||
//! real types decode and re-encode every one of them byte for byte (checked below).
|
||||
//! No fixture written here carries a card: Huginn's types do not carry `raised_at`, `decider_count`,
|
||||
//! `answerer_count` or a nullable `answerer` yet. Those shapes are CultLib's own bytes, vendored at
|
||||
//! one commit in `fixtures/desk-feed/cultlib-3a51bbab/` and checked below.
|
||||
//!
|
||||
//! `cargo test -p huginn-mind --test desk_feed_fixtures` compares every file with what the
|
||||
//! encoders produce now. `HUGINN_WRITE_DESK_FIXTURES=1` rewrites them instead.
|
||||
|
|
@ -18,11 +17,11 @@ use sha2::{Digest, Sha256};
|
|||
use cultnet_rs::{CultNetMessage, CultNetWireContract, encode_cultnet_message_to_vec, encode_frame};
|
||||
use huginn_mind::MindRefusal;
|
||||
use huginn_mind::desk_wire::{
|
||||
fork_too_large_read, AnswerSchema, CardSchema, CardStatus, DeskRequest, DeskResponse, EntrySchema, ForkCard, ForkCursor, ForkDeskAnswer, ForkEntry, ForkLimit,
|
||||
ForkList, ForkOption, ForkRecord, ForkRefusal, ForkRefusalCode, ForkRef, ListSchema, ListStatus, RecordSchema,
|
||||
fork_too_large_read, AnswerSchema, DeskRequest, DeskResponse, ForkCursor, ForkDeskAnswer, ForkLimit, ForkList, ForkRecord, ForkRefusal, ForkRefusalCode,
|
||||
ListSchema, ListStatus, RecordSchema,
|
||||
};
|
||||
use huginn_mind::envelope::{OperationFailure, decode_desk_request, encode_desk_failure, encode_desk_request, encode_desk_response};
|
||||
use huginn_mind::eureka_pipeline::{AccountId, DeskAuthority, Short, Slug};
|
||||
use huginn_mind::envelope::{OperationFailure, encode_desk_failure, encode_desk_request, encode_desk_response};
|
||||
use huginn_mind::eureka_pipeline::{DeskAuthority, Short, Slug};
|
||||
|
||||
const RUNTIME: &str = "huginn-eureka";
|
||||
/// Account ids as Bifrost sends them: `UserAccount.Id` rendered lowercase D-format
|
||||
|
|
@ -32,8 +31,6 @@ const READER: &str = "5f0c2b1e-8a4d-4c3b-9e7f-1a2b3c4d5e6f";
|
|||
const OTHER_DECIDER: &str = "0b9d7e3a-6c21-4f58-a0d4-9e8f7a6b5c4d";
|
||||
/// A member who answered but is not a decider: their answer is stored `Refused(not_a_decider)`.
|
||||
const OUTSIDER: &str = "c3a1f2e4-7b6d-4e8f-9a0b-1c2d3e4f5a6b";
|
||||
/// A member the call's deciders do not name, who reads as a non-decider.
|
||||
const NON_DECIDER: &str = "9e8d7c6b-5a49-4382-b1a0-f9e8d7c6b5a4";
|
||||
/// A question Eureka's mind holds (asked 2026-11-11). The answers, notes and hash in `record()` are
|
||||
/// synthetic: the hash is CultLib's sample digest, not this question's.
|
||||
const FORK: &str = "eureka-substrate:question:desk-listing-authority";
|
||||
|
|
@ -42,15 +39,11 @@ fn s(text: &str) -> Short {
|
|||
Short(text.into())
|
||||
}
|
||||
|
||||
fn a(text: &str) -> AccountId {
|
||||
AccountId(text.into())
|
||||
}
|
||||
|
||||
fn forks(status: Option<ListStatus>, cursor: Option<u64>, limit: u8) -> DeskRequest {
|
||||
DeskRequest::Forks {
|
||||
mind: Slug("eureka".into()),
|
||||
subject: a(READER),
|
||||
deciders: vec![a(READER), a(OTHER_DECIDER)],
|
||||
subject: s(READER),
|
||||
deciders: vec![s(READER), s(OTHER_DECIDER)],
|
||||
status,
|
||||
cursor: cursor.and_then(ForkCursor::new),
|
||||
limit: ForkLimit::new(limit).unwrap(),
|
||||
|
|
@ -58,23 +51,7 @@ fn forks(status: Option<ListStatus>, cursor: Option<u64>, limit: u8) -> DeskRequ
|
|||
}
|
||||
|
||||
fn read() -> DeskRequest {
|
||||
DeskRequest::Fork { mind: Slug("eureka".into()), subject: a(READER), deciders: vec![a(READER), a(OTHER_DECIDER)], fork_id: FORK.into() }
|
||||
}
|
||||
|
||||
/// The same reads from a subject the call's deciders do not name: a non-decider.
|
||||
fn forks_as_non_decider() -> DeskRequest {
|
||||
DeskRequest::Forks {
|
||||
mind: Slug("eureka".into()),
|
||||
subject: a(NON_DECIDER),
|
||||
deciders: vec![a(READER), a(OTHER_DECIDER)],
|
||||
status: Some(ListStatus::Open),
|
||||
cursor: None,
|
||||
limit: ForkLimit::new(50).unwrap(),
|
||||
}
|
||||
}
|
||||
|
||||
fn read_as_non_decider() -> DeskRequest {
|
||||
DeskRequest::Fork { mind: Slug("eureka".into()), subject: a(NON_DECIDER), deciders: vec![a(READER), a(OTHER_DECIDER)], fork_id: FORK.into() }
|
||||
DeskRequest::Fork { mind: Slug("eureka".into()), subject: s(READER), deciders: vec![s(READER), s(OTHER_DECIDER)], fork_id: FORK.into() }
|
||||
}
|
||||
|
||||
fn list(open_count: u64, recorded_count: u64) -> DeskResponse {
|
||||
|
|
@ -96,7 +73,7 @@ fn refusal(code: ForkRefusalCode, error: &str) -> DeskResponse {
|
|||
DeskResponse::NotStored(ForkRefusal { schema_version: huginn_mind::desk_wire::RefusalSchema::V0, code, error: error.into() })
|
||||
}
|
||||
|
||||
fn answer(id: &str, outcome: &str, of: Option<&str>, answerer: Option<&str>, note: &str, received_at: &str) -> ForkDeskAnswer {
|
||||
fn answer(id: &str, outcome: &str, of: Option<&str>, answerer: &str, note: &str, received_at: &str) -> ForkDeskAnswer {
|
||||
ForkDeskAnswer {
|
||||
schema_version: AnswerSchema::V0,
|
||||
answer_id: id.into(),
|
||||
|
|
@ -111,76 +88,35 @@ fn answer(id: &str, outcome: &str, of: Option<&str>, answerer: Option<&str>, not
|
|||
answered_at: received_at.into(),
|
||||
received_at: received_at.into(),
|
||||
claim_in_future: false,
|
||||
answerer: answerer.map(str::to_string),
|
||||
answerer: answerer.into(),
|
||||
authority: DeskAuthority::Member,
|
||||
request_id: format!("00000000-0000-4000-8000-00000000000{}", &id[1..]),
|
||||
}
|
||||
}
|
||||
|
||||
/// A fork's record with a counted answer, a decider's revision, a refusal and a retraction: four
|
||||
/// answers by three accounts. A decider's view names every answerer; a non-decider's names none and
|
||||
/// keeps the count.
|
||||
fn record(decider: bool) -> ForkRecord {
|
||||
let who = |id: &'static str| decider.then_some(id);
|
||||
/// A fork's record with a counted answer and a decider's revision, answerer ids raw: the door sends
|
||||
/// them raw and Bifrost masks them for a reader who is not a decider.
|
||||
fn record() -> ForkRecord {
|
||||
ForkRecord {
|
||||
schema_version: RecordSchema::V0,
|
||||
fork_id: FORK.into(),
|
||||
options_hash: HASH.into(),
|
||||
options_hash: "7d95fc87d532420611fe415c1def27a3447617bb68235f46e3af130aea361c48".into(),
|
||||
counted_answer_id: Some("n1".into()),
|
||||
answerer_count: 3,
|
||||
answers: vec![
|
||||
answer("n1", "Counted", None, who(READER), "The asker lists her own forks.", "2026-11-11T17:00:00Z"),
|
||||
answer("n2", "Revision", Some("n1"), who(OTHER_DECIDER), "Agreed.", "2026-11-11T17:05:00Z"),
|
||||
ForkDeskAnswer { code: Some(huginn_mind::desk_wire::ForkAnswerCode::NotADecider), ..answer("n3", "Refused", None, who(OUTSIDER), "Me too.", "2026-11-11T17:06:00Z") },
|
||||
answer("n1", "Counted", None, READER, "The asker lists her own forks.", "2026-11-11T17:00:00Z"),
|
||||
answer("n2", "Revision", Some("n1"), OTHER_DECIDER, "Agreed.", "2026-11-11T17:05:00Z"),
|
||||
ForkDeskAnswer { code: Some(huginn_mind::desk_wire::ForkAnswerCode::NotADecider), ..answer("n3", "Refused", None, OUTSIDER, "Me too.", "2026-11-11T17:06:00Z") },
|
||||
// A second answer by an account already counted: four answers, three answerers.
|
||||
ForkDeskAnswer {
|
||||
retracts: Some("n2".into()),
|
||||
choice: None,
|
||||
note: None,
|
||||
..answer("n4", "Retraction", None, who(OTHER_DECIDER), "", "2026-11-11T17:07:00Z")
|
||||
..answer("n4", "Retraction", None, OTHER_DECIDER, "", "2026-11-11T17:07:00Z")
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
const HASH: &str = "7d95fc87d532420611fe415c1def27a3447617bb68235f46e3af130aea361c48";
|
||||
|
||||
/// The fork's card. A decider's view lists the call's ids; a non-decider's lists none and keeps the count.
|
||||
fn card(decider: bool, raised_at: Option<&str>) -> ForkCard {
|
||||
ForkCard {
|
||||
schema_version: CardSchema::V0,
|
||||
fork_id: FORK.into(),
|
||||
title: "Who may list a fork?".into(),
|
||||
context: "A fork is read by members only once it is listed.".into(),
|
||||
options: vec![ForkOption { id: "asking-mind".into(), text: "The asking Mind lists her own forks.".into() }, ForkOption { id: "operator".into(), text: "An operator lists them.".into() }],
|
||||
recommendation: "asking-mind".into(),
|
||||
depends: Vec::new(),
|
||||
leans: None,
|
||||
flag: None,
|
||||
raised_in: Some(ForkRef { kind: "Idea".into(), id: "eureka-substrate:idea:desk-door-map".into() }),
|
||||
raised_at: raised_at.map(str::to_string),
|
||||
deciders: if decider { vec![READER.into(), OTHER_DECIDER.into()] } else { Vec::new() },
|
||||
decider_count: 2,
|
||||
status: CardStatus::Open,
|
||||
ruling: None,
|
||||
ruled_from: Vec::new(),
|
||||
order: 41,
|
||||
options_hash: HASH.into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn entry(decider: bool, raised_at: Option<&str>) -> ForkEntry {
|
||||
ForkEntry { schema_version: EntrySchema::V0, card: card(decider, raised_at), record: record(decider) }
|
||||
}
|
||||
|
||||
/// One open fork on the page; the counts are the page's visible set.
|
||||
fn list_of(entry: ForkEntry) -> DeskResponse {
|
||||
DeskResponse::Forks {
|
||||
list: ForkList { schema_version: ListSchema::V0, mind: "eureka".into(), stamp: "s10190".into(), open_count: 1, recorded_count: 0, forks: vec![entry], next_cursor: None },
|
||||
as_of: 10190,
|
||||
}
|
||||
}
|
||||
|
||||
/// The named MessagePack payload a CultNet operation message carries (base64 in the envelope).
|
||||
fn payload(message: &CultNetMessage) -> Vec<u8> {
|
||||
use base64::Engine;
|
||||
|
|
@ -200,24 +136,6 @@ fn hex(bytes: &[u8]) -> String {
|
|||
text
|
||||
}
|
||||
|
||||
/// A request message whose payload is `request` with one field changed to something the typed request
|
||||
/// cannot hold: what a client that skips the account grammar would send.
|
||||
fn malformed(id: &str, request: &DeskRequest, edit: impl Fn(&mut serde_json::Value)) -> CultNetMessage {
|
||||
use base64::Engine;
|
||||
let mut value = serde_json::to_value(request).unwrap();
|
||||
edit(&mut value);
|
||||
CultNetMessage::OperationRequest {
|
||||
message_id: id.into(),
|
||||
service_id: huginn_mind::desk_wire::DESK_SERVICE_ID.into(),
|
||||
operation: "fork".into(),
|
||||
payload_schema: huginn_mind::desk_wire::DESK_REQUEST_SCHEMA.into(),
|
||||
payload_encoding: "messagepack-base64".into(),
|
||||
payload: base64::engine::general_purpose::STANDARD.encode(rmp_serde::to_vec_named(&value).unwrap()),
|
||||
source_runtime_id: None,
|
||||
target_runtime_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Every fixture: its file stem and its bytes. A message gives `.frame.bin`, `.frame.hex` and,
|
||||
/// for an operation message, `.payload.msgpack` and `.payload.hex`; a bare payload gives the last two.
|
||||
fn fixtures() -> Vec<(String, Vec<u8>)> {
|
||||
|
|
@ -236,22 +154,6 @@ fn fixtures() -> Vec<(String, Vec<u8>)> {
|
|||
message("list-open-cursor.request", encode_desk_request("m-2", &forks(Some(ListStatus::Open), Some(42), 50), None).unwrap());
|
||||
message("list-counts.request", encode_desk_request("m-3", &forks(Some(ListStatus::Open), None, 0), None).unwrap());
|
||||
message("read.request", encode_desk_request("m-4", &read(), None).unwrap());
|
||||
message("list-non-decider.request", encode_desk_request("m-5", &forks_as_non_decider(), None).unwrap());
|
||||
message("read-non-decider.request", encode_desk_request("m-6", &read_as_non_decider(), None).unwrap());
|
||||
message("list-open.response", encode_desk_response("m-1", "forks", &list_of(entry(true, Some("2026-11-11T16:40:12Z"))), RUNTIME).unwrap());
|
||||
message("list-open-non-decider.response", encode_desk_response("m-5", "forks", &list_of(entry(false, Some("2026-11-11T16:40:12Z"))), RUNTIME).unwrap());
|
||||
// raised_at nil: the question's first admission is unknown.
|
||||
message("read.response", encode_desk_response("m-4", "fork", &DeskResponse::Fork { entry: entry(true, None), as_of: 10190 }, RUNTIME).unwrap());
|
||||
message("read-non-decider.response", encode_desk_response("m-6", "fork", &DeskResponse::Fork { entry: entry(false, None), as_of: 10190 }, RUNTIME).unwrap());
|
||||
// A decider or subject that is no account id is refused at decode, naming the field.
|
||||
for (name, id, request) in [
|
||||
("read-mxid-decider", "m-7", malformed("m-7", &read(), |value| value["Fork"]["deciders"][0] = "@heimdall:gamecult.org".into())),
|
||||
("read-email-subject", "m-8", malformed("m-8", &read(), |value| value["Fork"]["subject"] = "meta@gamecult.org".into())),
|
||||
] {
|
||||
message(&format!("{name}.request"), request.clone());
|
||||
let failure = decode_desk_request(&request).expect_err("the door refuses it");
|
||||
message(&format!("{name}.failure.response"), encode_desk_failure(id, "fork", &failure, RUNTIME));
|
||||
}
|
||||
message("list-empty.response", encode_desk_response("m-1", "forks", &list(0, 0), RUNTIME).unwrap());
|
||||
message("list-counts.response", encode_desk_response("m-3", "forks", &list(3, 5), RUNTIME).unwrap());
|
||||
message(
|
||||
|
|
@ -283,11 +185,9 @@ fn fixtures() -> Vec<(String, Vec<u8>)> {
|
|||
encode_desk_failure("m-1", "forks", &OperationFailure { code: "payload-not-a-request".into(), message: "field Forks.limit: out-of-range".into() }, RUNTIME),
|
||||
);
|
||||
message("error.response", CultNetMessage::Error { error: "huginn.desk refuses a message_id over 256 bytes".into(), code: None, details: None });
|
||||
for (name, decider) in [("record-whole", true), ("record-withheld", false)] {
|
||||
let bytes = rmp_serde::to_vec_named(&record(decider)).unwrap();
|
||||
files.push((format!("{name}.payload.msgpack"), bytes.clone()));
|
||||
files.push((format!("{name}.payload.hex"), hex(&bytes).into_bytes()));
|
||||
}
|
||||
let whole = rmp_serde::to_vec_named(&record()).unwrap();
|
||||
files.push(("record-whole.payload.msgpack".into(), whole.clone()));
|
||||
files.push(("record-whole.payload.hex".into(), hex(&whole).into_bytes()));
|
||||
files
|
||||
}
|
||||
|
||||
|
|
@ -321,7 +221,8 @@ fn account_id(text: &str) -> bool {
|
|||
}
|
||||
|
||||
/// Every account id the fixtures carry is a Bifrost account id, never a Matrix id, email or
|
||||
/// handle, and the door refuses any other shape at decode, naming the field and never the value.
|
||||
/// handle. The door does not refuse other shapes yet (`Short`, any 200 bytes); that refusal is owed
|
||||
/// by `cut-desk-feed-listing.r6`, and this pins the grammar it must enforce.
|
||||
#[test]
|
||||
fn every_account_id_in_the_fixtures_is_a_lowercase_d_format_uuid() {
|
||||
for request in [forks(Some(ListStatus::Open), None, 50), read()] {
|
||||
|
|
@ -331,15 +232,7 @@ fn every_account_id_in_the_fixtures_is_a_lowercase_d_format_uuid() {
|
|||
assert!(account_id(&subject.0), "subject");
|
||||
assert!(request.deciders().iter().all(|decider| account_id(&decider.0)), "deciders");
|
||||
}
|
||||
assert!(record(true).answers.iter().all(|answer| account_id(answer.answerer.as_deref().unwrap())), "answerer");
|
||||
for (request, field, value) in [
|
||||
(malformed("m-7", &read(), |value| value["Fork"]["deciders"][0] = "@heimdall:gamecult.org".into()), "deciders", "@heimdall:gamecult.org"),
|
||||
(malformed("m-8", &read(), |value| value["Fork"]["subject"] = "meta@gamecult.org".into()), "subject", "meta@gamecult.org"),
|
||||
] {
|
||||
let failure = decode_desk_request(&request).expect_err("a request outside the grammar is refused");
|
||||
assert_eq!(failure.code, "payload-not-a-request");
|
||||
assert!(failure.message.contains(field) && !failure.message.contains(value), "{}", failure.message);
|
||||
}
|
||||
assert!(record().answers.iter().all(|answer| account_id(&answer.answerer)), "answerer");
|
||||
for shape in [
|
||||
"@heimdall:gamecult.org",
|
||||
"meta@gamecult.org",
|
||||
|
|
@ -362,8 +255,8 @@ fn every_account_id_in_the_fixtures_is_a_lowercase_d_format_uuid() {
|
|||
|
||||
// ---- CultLib's bytes: the reader-count shapes, pinned at one commit. ----
|
||||
|
||||
/// The CultLib commit the vendored vectors come from (#118 on #117 on #115). Re-pin: see the README beside them.
|
||||
const CULTLIB_SHA: &str = "d3025d3c19674803ad32eb2335461b8cbc22bd28";
|
||||
/// The CultLib commit the vendored vectors come from (#117 on #115). Re-pin: see the README beside them.
|
||||
const CULTLIB_SHA: &str = "3a51bbabad244a56ec4521080530d88744001ce4";
|
||||
|
||||
/// Each vendored vector and the SHA-256 of its bytes.
|
||||
const VENDORED: &[(&str, &str)] = &[
|
||||
|
|
@ -374,8 +267,6 @@ const VENDORED: &[(&str, &str)] = &[
|
|||
("answer-revision", "19d7289395ae68cbaf6de1c0a1ef6c843211b2caf9462e1d2268cd0883386315"),
|
||||
("answer-retraction", "89156f41ec3ea54f5d47bd3318913ff013785f4727916875be97785935cb7735"),
|
||||
("answer-withheld-answerer", "b66b4f921cdd374be04af3de34c59e254a5e542027ae8c57f196a3ea3406e427"),
|
||||
("answer-withheld-answerer-revision", "93f7c875f963a7cfeab2a5980b6d5ce81e6753fc9f3f8bf0d7353f3f388144b7"),
|
||||
("answer-withheld-answerer-retraction", "7d0cc555bf9b6f7c345b0da7c504bcd1ed7b46040ff51d1e57c84b2f56655213"),
|
||||
("answer-refused-stale", "3183bc9c483f4a625332cda11ef22a9c8560868a7728b77daa63a1960a3cd3d6"),
|
||||
("entry-open-counted", "b51f499e8dd0465cf5df9d68d8fccbe94dd1339fd3157969693dd0d4ecb2549b"),
|
||||
("entry-withheld-open-counted", "62ccad7a9f41262d6e35d0247315d0bb75d1313b3c9dabe8a99d1aa1b844607f"),
|
||||
|
|
@ -384,7 +275,7 @@ const VENDORED: &[(&str, &str)] = &[
|
|||
];
|
||||
|
||||
fn vendored(name: &str) -> Vec<u8> {
|
||||
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/desk-feed/cultlib-d3025d3c").join(format!("{name}.msgpack"));
|
||||
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/desk-feed/cultlib-3a51bbab").join(format!("{name}.msgpack"));
|
||||
std::fs::read(&path).unwrap_or_else(|error| panic!("{}: {error}", path.display()))
|
||||
}
|
||||
|
||||
|
|
@ -488,7 +379,7 @@ const RECORD_KEYS: [&str; 6] = ["schemaVersion", "fork_id", "options_hash", "cou
|
|||
#[test]
|
||||
fn the_vendored_cultlib_vectors_are_the_pinned_bytes() {
|
||||
assert_eq!(CULTLIB_SHA.len(), 40);
|
||||
let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/desk-feed/cultlib-d3025d3c");
|
||||
let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/desk-feed/cultlib-3a51bbab");
|
||||
let docs = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../docs/desk-feed-door.md");
|
||||
for path in [dir.join("README.md"), docs] {
|
||||
let text = std::fs::read_to_string(&path).unwrap_or_else(|error| panic!("{}: {error}", path.display()));
|
||||
|
|
@ -529,8 +420,8 @@ fn the_cards_are_eighteen_keys_with_raised_at_and_decider_count() {
|
|||
}
|
||||
|
||||
/// `answerer` is a string or nil on every answer; `Value::Nil` only when `named` is false. The bytes
|
||||
/// null it on all four variants (`Counted`, `Revision`, `Retraction`, `Refused`), by ruling
|
||||
/// `desk-reader-counts-shape` and now by vector.
|
||||
/// null it on `Counted` and `Refused` only; nullable on all four variants is the ruling
|
||||
/// `desk-reader-counts-shape`, and CultLib has no vector with a `Revision` or `Retraction` nulled.
|
||||
fn check_answer(answer: &Value, named: bool) {
|
||||
assert_eq!(answer.keys().len(), 16);
|
||||
assert_eq!(answer.keys()[13], "answerer");
|
||||
|
|
@ -543,12 +434,12 @@ fn check_answer(answer: &Value, named: bool) {
|
|||
|
||||
#[test]
|
||||
fn the_answers_keep_sixteen_keys_and_answerer_is_a_string_or_nil() {
|
||||
for (name, named) in [("answer-counted", true), ("answer-revision", true), ("answer-retraction", true), ("answer-refused-stale", true), ("answer-withheld-answerer", false), ("answer-withheld-answerer-revision", false), ("answer-withheld-answerer-retraction", false)] {
|
||||
for (name, named) in [("answer-counted", true), ("answer-revision", true), ("answer-retraction", true), ("answer-refused-stale", true), ("answer-withheld-answerer", false)] {
|
||||
let bytes = vendored(name);
|
||||
assert_eq!(bytes[..3], [0xde, 0x00, 0x10], "{name}: header");
|
||||
check_answer(&decode(&bytes), named);
|
||||
}
|
||||
for (name, outcome) in [("answer-counted", "Counted"), ("answer-revision", "Revision"), ("answer-retraction", "Retraction"), ("answer-refused-stale", "Refused"), ("answer-withheld-answerer-revision", "Revision"), ("answer-withheld-answerer-retraction", "Retraction")] {
|
||||
for (name, outcome) in [("answer-counted", "Counted"), ("answer-revision", "Revision"), ("answer-retraction", "Retraction"), ("answer-refused-stale", "Refused")] {
|
||||
assert_eq!(decode(&vendored(name)).get("outcome").str(), outcome);
|
||||
}
|
||||
}
|
||||
|
|
@ -610,60 +501,26 @@ impl Value {
|
|||
}
|
||||
}
|
||||
|
||||
/// Huginn's real types against CultLib's bytes: each of the 14 vendored vectors decodes into the type
|
||||
/// that names it and re-encodes to the same bytes.
|
||||
/// Huginn's real types against CultLib's bytes. A decider's answers already decode (the answer keeps
|
||||
/// its 16 keys). The card and the record do not: `ForkCard` has no `raised_at` or `decider_count`,
|
||||
/// `ForkRecord` no `answerer_count`, and `ForkDeskAnswer.answerer` is a non-null string, and each type
|
||||
/// refuses unknown keys. This pins the gap `cut-desk-feed-listing` closes; when it does, these
|
||||
/// refusals become successful decodes of the same bytes and this test changes with them.
|
||||
#[test]
|
||||
fn huginn_types_decode_and_re_encode_every_vendored_vector_byte_for_byte() {
|
||||
fn round_trip<T: serde::Serialize + serde::de::DeserializeOwned>(name: &str) {
|
||||
let bytes = vendored(name);
|
||||
let value = rmp_serde::from_slice::<T>(&bytes).unwrap_or_else(|error| panic!("{name}: {error}"));
|
||||
assert!(rmp_serde::to_vec_named(&value).unwrap() == bytes, "{name} does not re-encode byte for byte");
|
||||
fn huginn_types_do_not_yet_carry_the_pinned_shapes() {
|
||||
use huginn_mind::desk_wire::ForkCard;
|
||||
for name in ["answer-counted", "answer-refused-stale"] {
|
||||
rmp_serde::from_slice::<ForkDeskAnswer>(&vendored(name)).unwrap_or_else(|error| panic!("{name}: {error}"));
|
||||
}
|
||||
assert_eq!(VENDORED.len(), 14);
|
||||
for (name, _) in VENDORED {
|
||||
match name.split('-').next().unwrap() {
|
||||
"card" => round_trip::<ForkCard>(name),
|
||||
"answer" => round_trip::<ForkDeskAnswer>(name),
|
||||
"entry" => round_trip::<ForkEntry>(name),
|
||||
other => panic!("{name}: no type for {other}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative first: a non-decider's bytes hold no account id, in any fixture Huginn writes for that view.
|
||||
#[test]
|
||||
fn a_non_decider_fixture_holds_no_account_id() {
|
||||
let ids = [READER, OTHER_DECIDER, OUTSIDER, NON_DECIDER];
|
||||
let withheld = |bytes: &[u8]| ids.iter().filter(|id| holds(bytes, id)).count();
|
||||
let files = fixtures();
|
||||
let bytes_of = |name: &str| files.iter().find(|(file, _)| file == name).unwrap_or_else(|| panic!("no fixture {name}")).1.clone();
|
||||
for name in ["list-open-non-decider.response.payload.msgpack", "read-non-decider.response.payload.msgpack", "record-withheld.payload.msgpack"] {
|
||||
let bytes = bytes_of(name);
|
||||
assert_eq!(withheld(&bytes), 0, "{name} carries an account id");
|
||||
}
|
||||
// The decider's twins do name them, so the search above can fail.
|
||||
for name in ["list-open.response.payload.msgpack", "record-whole.payload.msgpack"] {
|
||||
assert!(withheld(&bytes_of(name)) >= 3, "{name}");
|
||||
}
|
||||
}
|
||||
|
||||
/// The record header is `86` with `answerer_count` fifth, the card `DE 00 12` with `raised_at` and
|
||||
/// `decider_count` where CultLib puts them, in Huginn's own bytes.
|
||||
#[test]
|
||||
fn huginns_own_card_and_record_bytes_have_the_pinned_headers_and_keys() {
|
||||
let card_bytes = rmp_serde::to_vec_named(&card(true, Some("2026-11-11T16:40:12Z"))).unwrap();
|
||||
assert_eq!(card_bytes[..3], [0xde, 0x00, 0x12]);
|
||||
assert_eq!(decode(&card_bytes).keys(), CARD_KEYS);
|
||||
check_card(&decode(&card_bytes), false);
|
||||
let withheld_bytes = rmp_serde::to_vec_named(&card(false, None)).unwrap();
|
||||
check_card(&decode(&withheld_bytes), true);
|
||||
assert_eq!(decode(&withheld_bytes).get("raised_at"), &Value::Nil);
|
||||
for decider in [true, false] {
|
||||
let record_bytes = rmp_serde::to_vec_named(&record(decider)).unwrap();
|
||||
assert_eq!(record_bytes[0], 0x86);
|
||||
let decoded = decode(&record_bytes);
|
||||
assert_eq!(decoded.keys(), RECORD_KEYS);
|
||||
assert_eq!(decoded.get("answerer_count").int(), 3);
|
||||
decoded.list_of_answers().iter().for_each(|answer| check_answer(answer, decider));
|
||||
fn refused<T: serde::de::DeserializeOwned>(bytes: &[u8]) -> String {
|
||||
rmp_serde::from_slice::<T>(bytes).err().expect("the real type refuses these bytes").to_string()
|
||||
}
|
||||
assert!(refused::<ForkCard>(&vendored("card-operator-fork")).contains("raised_at"));
|
||||
assert!(refused::<ForkCard>(&vendored("card-withheld-deciders")).contains("raised_at"));
|
||||
assert!(refused::<ForkDeskAnswer>(&vendored("answer-withheld-answerer")).contains("string"));
|
||||
// The record is the entry's last key: its bytes run from the `86` after the key to the end.
|
||||
let entry = vendored("entry-open-counted");
|
||||
let key = [&[0xa6u8][..], b"record", &[0x86]].concat();
|
||||
let at = entry.windows(key.len()).position(|window| window == key).expect("the record key") + key.len() - 1;
|
||||
assert!(refused::<ForkRecord>(&entry[at..]).contains("answerer_count"));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,9 +15,9 @@ Each case has up to four files:
|
|||
Hex is lowercase with one trailing newline. The fork id is a question Eureka's mind holds. The
|
||||
record's answers, notes and hash are synthetic.
|
||||
|
||||
**The listing gate is live in Huginn.** A reader who is not on the call's deciders sees only forks the
|
||||
Mind lists (`docs/desk-feed-door.md`); `list-empty.response` is what such a reader gets from a Mind with
|
||||
nothing listed.
|
||||
**The listing gate is not live.** On `main` every showable question goes to every reader. Do not
|
||||
put a Mind that holds private questions behind a live receiver before `cut-desk-feed-listing`
|
||||
lands (see `docs/desk-feed-door.md`).
|
||||
|
||||
| case | direction | content |
|
||||
|---|---|---|
|
||||
|
|
@ -25,16 +25,6 @@ nothing listed.
|
|||
| `list-open-cursor.request` | Bifrost to Huginn | `Forks`, status open, cursor `c42`, limit 50 |
|
||||
| `list-counts.request` | Bifrost to Huginn | `Forks`, status open, limit 0 (counts and stamp only) |
|
||||
| `read.request` | Bifrost to Huginn | `Fork` for one fork id |
|
||||
| `list-non-decider.request` | Bifrost to Huginn | `Forks` as above, from a subject who is not on the call's deciders |
|
||||
| `read-non-decider.request` | Bifrost to Huginn | `Fork`, from a subject who is not on the call's deciders |
|
||||
| `list-open.response` | Huginn to Bifrost | a decider's page of one open fork: card with `raised_at` a string, `decider_count` 2 and both ids, record with `answerer_count` 3 and every `answerer` named |
|
||||
| `list-open-non-decider.response` | Huginn to Bifrost | the same page for a non-decider: `deciders` empty, `decider_count` 2, every `answerer` nil, `answerer_count` 3; no account id anywhere in its bytes |
|
||||
| `read.response` | Huginn to Bifrost | a decider's read of one fork, `raised_at` nil |
|
||||
| `read-non-decider.response` | Huginn to Bifrost | the same read for a non-decider, `raised_at` nil, no account id |
|
||||
| `read-mxid-decider.request` | client to Huginn | a `Fork` whose first decider is a Matrix id, so outside the account grammar; refused at decode |
|
||||
| `read-mxid-decider.failure.response` | Huginn to Bifrost | the operation failure for it: `payload-not-a-request`, naming `deciders`, never quoting the id |
|
||||
| `read-email-subject.request` | client to Huginn | a `Fork` whose subject is an email address; refused at decode |
|
||||
| `read-email-subject.failure.response` | Huginn to Bifrost | the operation failure for it, naming `subject` |
|
||||
| `list-empty.response` | Huginn to Bifrost | an empty Mind, or nothing listed to this reader: counts 0, `forks: []` |
|
||||
| `list-counts.response` | Huginn to Bifrost | the limit-0 answer: open 3, recorded 5, `forks: []` |
|
||||
| `read-unknown.response` | Huginn to Bifrost | `NotStored` `fork_unknown` (Bifrost: 404) |
|
||||
|
|
@ -45,10 +35,10 @@ nothing listed.
|
|||
| `unavailable.response` | Huginn to Bifrost | `Refused` `Unavailable` (Bifrost: 503 for this Mind) |
|
||||
| `failure.response` | Huginn to Bifrost | operation failure, `payloadSchema` `gamecult.cultnet.operation_failure.v1` (Bifrost: 503) |
|
||||
| `error.response` | Huginn to Bifrost | CultNet `Error` message, no payload (Bifrost: 503) |
|
||||
| `record-whole` | fragment | `gamecult.fork.record.v0` in a decider's view: all four answer variants (Counted, Revision, Refused, Retraction) by three accounts, answerer ids and notes whole, `answerer_count` 3 (6 keys) |
|
||||
| `record-withheld` | fragment | the same record in a non-decider's view: every `answerer` nil, notes whole, `answerer_count` 3 |
|
||||
| `record-whole` | fragment | `gamecult.fork.record.v0` with all four answer variants (Counted, Revision, Refused, Retraction) by three accounts, answerer ids and notes whole: what the door sends today (5 keys, no `answerer_count`) |
|
||||
|
||||
The reader-count shapes (`raised_at`, `decider_count`, `answerer_count`, nullable `answerer`) are
|
||||
also CultLib's own bytes, vendored unedited in `cultlib-d3025d3c/` (see its README for the pinned
|
||||
commit and each vector). Huginn's types decode every one of them and re-encode it byte for byte;
|
||||
the fixtures above are what Huginn's encoders write for the same shapes.
|
||||
CultLib's own bytes, vendored unedited in `cultlib-3a51bbab/` (see its README for the pinned commit and
|
||||
each vector). They are not Huginn's output: Huginn's types do not carry those shapes yet.
|
||||
`cut-desk-feed-listing` and `cut-desk-feed-raised-at` replace them with entry-bearing fixtures the
|
||||
encoders write, after decoding these bytes into the real types.
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
# CultLib desk-feed vectors, vendored
|
||||
|
||||
Source: `GameCult/CultLib` at commit `d3025d3c19674803ad32eb2335461b8cbc22bd28` (#118, on
|
||||
`3a51bbab` #117, on `af914253` #115), file `contracts/cultnet/governance/fork-card-vectors.json`. Each `.msgpack` file is
|
||||
Source: `GameCult/CultLib` at commit `3a51bbabad244a56ec4521080530d88744001ce4` (#117, on
|
||||
`af914253` #115), file `contracts/cultnet/governance/fork-card-vectors.json`. Each `.msgpack` file is
|
||||
the vector's `messagePackHex` decoded to bytes, nothing else changed. Do not edit them or rewrite them
|
||||
from Huginn's encoders: they are the reference Huginn's types are checked against (see
|
||||
from Huginn's encoders: Huginn's own types do not carry these shapes yet (see
|
||||
`docs/desk-feed-door.md`). To re-pin, take the new commit's vectors and update the sha here, in
|
||||
`docs/desk-feed-door.md` and in `CULTLIB_SHA` in `desk_feed_fixtures.rs`.
|
||||
|
||||
|
|
@ -13,11 +13,9 @@ from Huginn's encoders: they are the reference Huginn's types are checked agains
|
|||
| `card-raised-at-nil` | `card.valid` `raised_at_nil` | `raised_at` nil, key kept |
|
||||
| `card-withheld-deciders` | `card.valid` `withheld_deciders` | empty `deciders`, `decider_count` 2 |
|
||||
| `answer-counted` | `deskAnswer.valid` `counted` | 16 keys (`DE 00 10`), `answerer` a string |
|
||||
| `answer-revision` | `deskAnswer.valid` `revision` | a `Revision`, `answerer` a string |
|
||||
| `answer-retraction` | `deskAnswer.valid` `retraction` | a `Retraction`, `answerer` a string |
|
||||
| `answer-revision` | `deskAnswer.valid` `revision` | a `Revision`, `answerer` a string (no CultLib vector nulls a Revision) |
|
||||
| `answer-retraction` | `deskAnswer.valid` `retraction` | a `Retraction`, `answerer` a string (no CultLib vector nulls a Retraction) |
|
||||
| `answer-withheld-answerer` | `deskAnswer.valid` `withheld_answerer` | `answerer` nil |
|
||||
| `answer-withheld-answerer-revision` | `deskAnswer.valid` `withheld_answerer_revision` | a `Revision`, `answerer` nil |
|
||||
| `answer-withheld-answerer-retraction` | `deskAnswer.valid` `withheld_answerer_retraction` | a `Retraction`, `answerer` nil |
|
||||
| `answer-refused-stale` | `deskAnswer.valid` `refused_stale` | a `Refused` answer, `answerer` a string |
|
||||
| `entry-open-counted` | `transport.entry.valid` `open_counted` | decider view: entry `83`, record `86`, `answerer_count` 1 |
|
||||
| `entry-withheld-open-counted` | `transport.entry.valid` `withheld_open_counted` | non-decider twin: `deciders` empty, `answerer` nil, count 1 |
|
||||
|
|
@ -25,4 +23,4 @@ from Huginn's encoders: they are the reference Huginn's types are checked agains
|
|||
| `entry-withheld-closed-with-ruling-and-a-refusal` | `transport.entry.valid` `withheld_closed_with_ruling_and_a_refusal` | its non-decider twin, count 2 |
|
||||
|
||||
The test `desk_feed_fixtures.rs` checks each file's SHA-256, decodes it with a decoder of its own
|
||||
(not `rmp_serde`), then decodes it into Huginn's real type and re-encodes it, byte for byte.
|
||||
(not `rmp_serde`), and then tries Huginn's real types.
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -1 +0,0 @@
|
|||
000001a287a96d6573736167654964a36d2d35a96f7065726174696f6ea5666f726b73a77061796c6f6164d9f06761564762334a726334616b62576c755a4b5a6c64584a6c6132476e63335669616d566a644e6b6b4f5755345a44646a4e6d49744e5745304f5330304d7a67794c574978595441745a6a6c6c4f475133597a5a694e5745307147526c59326c6b5a584a7a6b746b6b4e575977597a4a694d5755744f4745305a433030597a4e694c546c6c4e3259744d574579596a4e6a4e4751315a545a6d32535177596a6c6b4e32557a59533032597a49784c54526d4e5467745954426b4e4330355a54686d4e324532596a566a4e47536d633352686448567a704739775a57366d5933567963323979774b5673615731706444493daf7061796c6f6164456e636f64696e67b26d6573736167657061636b2d626173653634ad7061796c6f6164536368656d61b6687567696e6e2e6465736b5f726571756573742e7631ad736368656d6156657273696f6ebc63756c746e65742e6f7065726174696f6e5f726571756573742e7630a9736572766963654964ab687567696e6e2e6465736b
|
||||
|
|
@ -1 +0,0 @@
|
|||
81a5466f726b7386a46d696e64a6657572656b61a77375626a656374d92439653864376336622d356134392d343338322d623161302d663965386437633662356134a8646563696465727392d92435663063326231652d386134642d346333622d396537662d316132623363346435653666d92430623964376533612d366332312d346635382d613064342d396538663761366235633464a6737461747573a46f70656ea6637572736f72c0a56c696d697432
|
||||
|
|
@ -1 +0,0 @@
|
|||
<EFBFBD>¥Forks†¤mind¦eureka§subjectÙ$9e8d7c6b-5a49-4382-b1a0-f9e8d7c6b5a4¨deciders’Ù$5f0c2b1e-8a4d-4c3b-9e7f-1a2b3c4d5e6fÙ$0b9d7e3a-6c21-4f58-a0d4-9e8f7a6b5c4d¦status¤open¦cursorÀ¥limit2
|
||||
Binary file not shown.
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Binary file not shown.
Binary file not shown.
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Binary file not shown.
Binary file not shown.
|
|
@ -1 +0,0 @@
|
|||
000001738aab646961676e6f737469637391b57061796c6f61642d6e6f742d612d72657175657374a96d6573736167654964a36d2d38a96f7065726174696f6ea4666f726ba77061796c6f6164d9606771526a6232526c745842686557787659575174626d39304c574574636d56786457567a644b64745a584e7a5957646c3253466d615756735a43424762334a724c6e4e31596d706c5933513649476c75646d467361575174646d46736457553daf7061796c6f6164456e636f64696e67b26d6573736167657061636b2d626173653634ad7061796c6f6164536368656d61d92567616d6563756c742e63756c746e65742e6f7065726174696f6e5f6661696c7572652e7631ad736368656d6156657273696f6ebd63756c746e65742e6f7065726174696f6e5f726573706f6e73652e7630a9736572766963654964ab687567696e6e2e6465736baf736f7572636552756e74696d654964ad687567696e6e2d657572656b61a6737461747573a872656a6563746564
|
||||
|
|
@ -1 +0,0 @@
|
|||
82a4636f6465b57061796c6f61642d6e6f742d612d72657175657374a76d657373616765d9216669656c6420466f726b2e7375626a6563743a20696e76616c69642d76616c7565
|
||||
|
|
@ -1 +0,0 @@
|
|||
うcodeオpayload-not-a-requestァmessageル!field Fork.subject: invalid-value
|
||||
Binary file not shown.
|
|
@ -1 +0,0 @@
|
|||
000001ad87a96d6573736167654964a36d2d38a96f7065726174696f6ea4666f726ba77061796c6f6164d9fc6761524762334a72684b686b5a574e705a47567963354c5a4a44566d4d474d79596a466c4c5468684e4751744e474d7a596930355a54646d4c5446684d6d497a597a526b4e5755325a746b6b4d4749355a44646c4d3245744e6d4d794d5330305a6a55344c5745775a4451744f5755345a6a64684e6d4931597a526b70325a76636d74666157545a4d475631636d56725953317a64574a7a64484a68644755366358566c63335270623234365a47567a6179317361584e306157356e4c57463164476876636d6c30656152746157356b706d5631636d56725961647a64574a715a574e307357316c644746415a3246745a574e316248517562334a6eaf7061796c6f6164456e636f64696e67b26d6573736167657061636b2d626173653634ad7061796c6f6164536368656d61b6687567696e6e2e6465736b5f726571756573742e7631ad736368656d6156657273696f6ebc63756c746e65742e6f7065726174696f6e5f726571756573742e7630a9736572766963654964ab687567696e6e2e6465736b
|
||||
|
|
@ -1 +0,0 @@
|
|||
81a4466f726b84a8646563696465727392d92435663063326231652d386134642d346333622d396537662d316132623363346435653666d92430623964376533612d366332312d346635382d613064342d396538663761366235633464a7666f726b5f6964d930657572656b612d7375627374726174653a7175657374696f6e3a6465736b2d6c697374696e672d617574686f72697479a46d696e64a6657572656b61a77375626a656374b16d6574614067616d6563756c742e6f7267
|
||||
|
|
@ -1 +0,0 @@
|
|||
<EFBFBD>¤Fork„¨deciders’Ù$5f0c2b1e-8a4d-4c3b-9e7f-1a2b3c4d5e6fÙ$0b9d7e3a-6c21-4f58-a0d4-9e8f7a6b5c4d§fork_idÙ0eureka-substrate:question:desk-listing-authority¤mind¦eureka§subject±meta@gamecult.org
|
||||
Binary file not shown.
|
|
@ -1 +0,0 @@
|
|||
000001778aab646961676e6f737469637391b57061796c6f61642d6e6f742d612d72657175657374a96d6573736167654964a36d2d37a96f7065726174696f6ea4666f726ba77061796c6f6164d9646771526a6232526c745842686557787659575174626d39304c574574636d56786457567a644b64745a584e7a5957646c3253566d615756735a43424762334a724c6d526c59326c6b5a584a7a577a42644f694270626e5a6862476c6b4c585a686248566caf7061796c6f6164456e636f64696e67b26d6573736167657061636b2d626173653634ad7061796c6f6164536368656d61d92567616d6563756c742e63756c746e65742e6f7065726174696f6e5f6661696c7572652e7631ad736368656d6156657273696f6ebd63756c746e65742e6f7065726174696f6e5f726573706f6e73652e7630a9736572766963654964ab687567696e6e2e6465736baf736f7572636552756e74696d654964ad687567696e6e2d657572656b61a6737461747573a872656a6563746564
|
||||
|
|
@ -1 +0,0 @@
|
|||
82a4636f6465b57061796c6f61642d6e6f742d612d72657175657374a76d657373616765d9256669656c6420466f726b2e64656369646572735b305d3a20696e76616c69642d76616c7565
|
||||
|
|
@ -1 +0,0 @@
|
|||
うcodeオpayload-not-a-requestァmessageル%field Fork.deciders[0]: invalid-value
|
||||
Binary file not shown.
|
|
@ -1 +0,0 @@
|
|||
000001b687a96d6573736167654964a36d2d37a96f7065726174696f6ea4666f726ba77061796c6f6164da01046761524762334a72684b686b5a574e705a47567963354b325147686c6157316b595778734f6d64686257566a645778304c6d39795a396b6b4d4749355a44646c4d3245744e6d4d794d5330305a6a55344c5745775a4451744f5755345a6a64684e6d4931597a526b70325a76636d74666157545a4d475631636d56725953317a64574a7a64484a68644755366358566c63335270623234365a47567a6179317361584e306157356e4c57463164476876636d6c30656152746157356b706d5631636d56725961647a64574a715a574e30325351315a6a426a4d6d49785a5330345954526b4c54526a4d3249744f5755335a69307859544a694d324d305a44566c4e6d593daf7061796c6f6164456e636f64696e67b26d6573736167657061636b2d626173653634ad7061796c6f6164536368656d61b6687567696e6e2e6465736b5f726571756573742e7631ad736368656d6156657273696f6ebc63756c746e65742e6f7065726174696f6e5f726571756573742e7630a9736572766963654964ab687567696e6e2e6465736b
|
||||
|
|
@ -1 +0,0 @@
|
|||
81a4466f726b84a8646563696465727392b6406865696d64616c6c3a67616d6563756c742e6f7267d92430623964376533612d366332312d346635382d613064342d396538663761366235633464a7666f726b5f6964d930657572656b612d7375627374726174653a7175657374696f6e3a6465736b2d6c697374696e672d617574686f72697479a46d696e64a6657572656b61a77375626a656374d92435663063326231652d386134642d346333622d396537662d316132623363346435653666
|
||||
|
|
@ -1 +0,0 @@
|
|||
<EFBFBD>¤Fork„¨deciders’¶@heimdall:gamecult.orgÙ$0b9d7e3a-6c21-4f58-a0d4-9e8f7a6b5c4d§fork_idÙ0eureka-substrate:question:desk-listing-authority¤mind¦eureka§subjectÙ$5f0c2b1e-8a4d-4c3b-9e7f-1a2b3c4d5e6f
|
||||
Binary file not shown.
|
|
@ -1 +0,0 @@
|
|||
000001ca87a96d6573736167654964a36d2d36a96f7065726174696f6ea4666f726ba77061796c6f6164da01186761524762334a72684b52746157356b706d5631636d56725961647a64574a715a574e30325351355a54686b4e324d3259693031595451354c54517a4f444974596a46684d43316d4f5755345a44646a4e6d49315954536f5a47566a6157526c636e4f53325351315a6a426a4d6d49785a5330345954526b4c54526a4d3249744f5755335a69307859544a694d324d305a44566c4e6d625a4a4442694f5751335a544e684c545a6a4d6a45744e4759314f4331684d4751304c546c6c4f47593359545a694e574d305a4b646d62334a7258326c6b3254426c64584a6c6132457463335669633352795958526c4f6e46315a584e30615739754f6d526c6332737462476c7a64476c755a7931686458526f62334a7064486b3daf7061796c6f6164456e636f64696e67b26d6573736167657061636b2d626173653634ad7061796c6f6164536368656d61b6687567696e6e2e6465736b5f726571756573742e7631ad736368656d6156657273696f6ebc63756c746e65742e6f7065726174696f6e5f726571756573742e7630a9736572766963654964ab687567696e6e2e6465736b
|
||||
|
|
@ -1 +0,0 @@
|
|||
81a4466f726b84a46d696e64a6657572656b61a77375626a656374d92439653864376336622d356134392d343338322d623161302d663965386437633662356134a8646563696465727392d92435663063326231652d386134642d346333622d396537662d316132623363346435653666d92430623964376533612d366332312d346635382d613064342d396538663761366235633464a7666f726b5f6964d930657572656b612d7375627374726174653a7175657374696f6e3a6465736b2d6c697374696e672d617574686f72697479
|
||||
|
|
@ -1 +0,0 @@
|
|||
<EFBFBD>¤Fork„¤mind¦eureka§subjectÙ$9e8d7c6b-5a49-4382-b1a0-f9e8d7c6b5a4¨deciders’Ù$5f0c2b1e-8a4d-4c3b-9e7f-1a2b3c4d5e6fÙ$0b9d7e3a-6c21-4f58-a0d4-9e8f7a6b5c4d§fork_idÙ0eureka-substrate:question:desk-listing-authority
|
||||
Binary file not shown.
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Binary file not shown.
Binary file not shown.
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Binary file not shown.
|
|
@ -1 +1 @@
|
|||
86ad736368656d6156657273696f6eb767616d6563756c742e666f726b2e7265636f72642e7630a7666f726b5f6964d930657572656b612d7375627374726174653a7175657374696f6e3a6465736b2d6c697374696e672d617574686f72697479ac6f7074696f6e735f68617368d94037643935666338376435333234323036313166653431356331646566323761333434373631376262363832333566343665336166313330616561333631633438b1636f756e7465645f616e737765725f6964a26e31ae616e7377657265725f636f756e7403a7616e737765727394de0010ad736368656d6156657273696f6eb767616d6563756c742e6465736b5f616e737765722e7630a9616e737765725f6964a26e31a7666f726b5f6964d930657572656b612d7375627374726174653a7175657374696f6e3a6465736b2d6c697374696e672d617574686f72697479a76f7574636f6d65a7436f756e746564a26f66c0a87265747261637473c0a4636f6465c0ac6f7074696f6e735f68617368d94037643935666338376435333234323036313166653431356331646566323761333434373631376262363832333566343665336166313330616561333631633438a663686f696365ab61736b696e672d6d696e64a46e6f7465be5468652061736b6572206c6973747320686572206f776e20666f726b732eab616e7377657265645f6174b4323032362d31312d31315431373a30303a30305aab72656365697665645f6174b4323032362d31312d31315431373a30303a30305aaf636c61696d5f696e5f667574757265c2a8616e737765726572d92435663063326231652d386134642d346333622d396537662d316132623363346435653666a9617574686f72697479a64d656d626572aa726571756573745f6964d92430303030303030302d303030302d343030302d383030302d303030303030303030303031de0010ad736368656d6156657273696f6eb767616d6563756c742e6465736b5f616e737765722e7630a9616e737765725f6964a26e32a7666f726b5f6964d930657572656b612d7375627374726174653a7175657374696f6e3a6465736b2d6c697374696e672d617574686f72697479a76f7574636f6d65a85265766973696f6ea26f66a26e31a87265747261637473c0a4636f6465c0ac6f7074696f6e735f68617368d94037643935666338376435333234323036313166653431356331646566323761333434373631376262363832333566343665336166313330616561333631633438a663686f696365ab61736b696e672d6d696e64a46e6f7465a74167726565642eab616e7377657265645f6174b4323032362d31312d31315431373a30353a30305aab72656365697665645f6174b4323032362d31312d31315431373a30353a30305aaf636c61696d5f696e5f667574757265c2a8616e737765726572d92430623964376533612d366332312d346635382d613064342d396538663761366235633464a9617574686f72697479a64d656d626572aa726571756573745f6964d92430303030303030302d303030302d343030302d383030302d303030303030303030303032de0010ad736368656d6156657273696f6eb767616d6563756c742e6465736b5f616e737765722e7630a9616e737765725f6964a26e33a7666f726b5f6964d930657572656b612d7375627374726174653a7175657374696f6e3a6465736b2d6c697374696e672d617574686f72697479a76f7574636f6d65a752656675736564a26f66c0a87265747261637473c0a4636f6465ad6e6f745f615f64656369646572ac6f7074696f6e735f68617368d94037643935666338376435333234323036313166653431356331646566323761333434373631376262363832333566343665336166313330616561333631633438a663686f696365ab61736b696e672d6d696e64a46e6f7465a74d6520746f6f2eab616e7377657265645f6174b4323032362d31312d31315431373a30363a30305aab72656365697665645f6174b4323032362d31312d31315431373a30363a30305aaf636c61696d5f696e5f667574757265c2a8616e737765726572d92463336131663265342d376236642d346538662d396130622d316332643365346635613662a9617574686f72697479a64d656d626572aa726571756573745f6964d92430303030303030302d303030302d343030302d383030302d303030303030303030303033de0010ad736368656d6156657273696f6eb767616d6563756c742e6465736b5f616e737765722e7630a9616e737765725f6964a26e34a7666f726b5f6964d930657572656b612d7375627374726174653a7175657374696f6e3a6465736b2d6c697374696e672d617574686f72697479a76f7574636f6d65aa52657472616374696f6ea26f66c0a87265747261637473a26e32a4636f6465c0ac6f7074696f6e735f68617368d94037643935666338376435333234323036313166653431356331646566323761333434373631376262363832333566343665336166313330616561333631633438a663686f696365c0a46e6f7465c0ab616e7377657265645f6174b4323032362d31312d31315431373a30373a30305aab72656365697665645f6174b4323032362d31312d31315431373a30373a30305aaf636c61696d5f696e5f667574757265c2a8616e737765726572d92430623964376533612d366332312d346635382d613064342d396538663761366235633464a9617574686f72697479a64d656d626572aa726571756573745f6964d92430303030303030302d303030302d343030302d383030302d303030303030303030303034
|
||||
85ad736368656d6156657273696f6eb767616d6563756c742e666f726b2e7265636f72642e7630a7666f726b5f6964d930657572656b612d7375627374726174653a7175657374696f6e3a6465736b2d6c697374696e672d617574686f72697479ac6f7074696f6e735f68617368d94037643935666338376435333234323036313166653431356331646566323761333434373631376262363832333566343665336166313330616561333631633438b1636f756e7465645f616e737765725f6964a26e31a7616e737765727394de0010ad736368656d6156657273696f6eb767616d6563756c742e6465736b5f616e737765722e7630a9616e737765725f6964a26e31a7666f726b5f6964d930657572656b612d7375627374726174653a7175657374696f6e3a6465736b2d6c697374696e672d617574686f72697479a76f7574636f6d65a7436f756e746564a26f66c0a87265747261637473c0a4636f6465c0ac6f7074696f6e735f68617368d94037643935666338376435333234323036313166653431356331646566323761333434373631376262363832333566343665336166313330616561333631633438a663686f696365ab61736b696e672d6d696e64a46e6f7465be5468652061736b6572206c6973747320686572206f776e20666f726b732eab616e7377657265645f6174b4323032362d31312d31315431373a30303a30305aab72656365697665645f6174b4323032362d31312d31315431373a30303a30305aaf636c61696d5f696e5f667574757265c2a8616e737765726572d92435663063326231652d386134642d346333622d396537662d316132623363346435653666a9617574686f72697479a64d656d626572aa726571756573745f6964d92430303030303030302d303030302d343030302d383030302d303030303030303030303031de0010ad736368656d6156657273696f6eb767616d6563756c742e6465736b5f616e737765722e7630a9616e737765725f6964a26e32a7666f726b5f6964d930657572656b612d7375627374726174653a7175657374696f6e3a6465736b2d6c697374696e672d617574686f72697479a76f7574636f6d65a85265766973696f6ea26f66a26e31a87265747261637473c0a4636f6465c0ac6f7074696f6e735f68617368d94037643935666338376435333234323036313166653431356331646566323761333434373631376262363832333566343665336166313330616561333631633438a663686f696365ab61736b696e672d6d696e64a46e6f7465a74167726565642eab616e7377657265645f6174b4323032362d31312d31315431373a30353a30305aab72656365697665645f6174b4323032362d31312d31315431373a30353a30305aaf636c61696d5f696e5f667574757265c2a8616e737765726572d92430623964376533612d366332312d346635382d613064342d396538663761366235633464a9617574686f72697479a64d656d626572aa726571756573745f6964d92430303030303030302d303030302d343030302d383030302d303030303030303030303032de0010ad736368656d6156657273696f6eb767616d6563756c742e6465736b5f616e737765722e7630a9616e737765725f6964a26e33a7666f726b5f6964d930657572656b612d7375627374726174653a7175657374696f6e3a6465736b2d6c697374696e672d617574686f72697479a76f7574636f6d65a752656675736564a26f66c0a87265747261637473c0a4636f6465ad6e6f745f615f64656369646572ac6f7074696f6e735f68617368d94037643935666338376435333234323036313166653431356331646566323761333434373631376262363832333566343665336166313330616561333631633438a663686f696365ab61736b696e672d6d696e64a46e6f7465a74d6520746f6f2eab616e7377657265645f6174b4323032362d31312d31315431373a30363a30305aab72656365697665645f6174b4323032362d31312d31315431373a30363a30305aaf636c61696d5f696e5f667574757265c2a8616e737765726572d92463336131663265342d376236642d346538662d396130622d316332643365346635613662a9617574686f72697479a64d656d626572aa726571756573745f6964d92430303030303030302d303030302d343030302d383030302d303030303030303030303033de0010ad736368656d6156657273696f6eb767616d6563756c742e6465736b5f616e737765722e7630a9616e737765725f6964a26e34a7666f726b5f6964d930657572656b612d7375627374726174653a7175657374696f6e3a6465736b2d6c697374696e672d617574686f72697479a76f7574636f6d65aa52657472616374696f6ea26f66c0a87265747261637473a26e32a4636f6465c0ac6f7074696f6e735f68617368d94037643935666338376435333234323036313166653431356331646566323761333434373631376262363832333566343665336166313330616561333631633438a663686f696365c0a46e6f7465c0ab616e7377657265645f6174b4323032362d31312d31315431373a30373a30305aab72656365697665645f6174b4323032362d31312d31315431373a30373a30305aaf636c61696d5f696e5f667574757265c2a8616e737765726572d92430623964376533612d366332312d346635382d613064342d396538663761366235633464a9617574686f72697479a64d656d626572aa726571756573745f6964d92430303030303030302d303030302d343030302d383030302d303030303030303030303034
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -1 +0,0 @@
|
|||
86ad736368656d6156657273696f6eb767616d6563756c742e666f726b2e7265636f72642e7630a7666f726b5f6964d930657572656b612d7375627374726174653a7175657374696f6e3a6465736b2d6c697374696e672d617574686f72697479ac6f7074696f6e735f68617368d94037643935666338376435333234323036313166653431356331646566323761333434373631376262363832333566343665336166313330616561333631633438b1636f756e7465645f616e737765725f6964a26e31ae616e7377657265725f636f756e7403a7616e737765727394de0010ad736368656d6156657273696f6eb767616d6563756c742e6465736b5f616e737765722e7630a9616e737765725f6964a26e31a7666f726b5f6964d930657572656b612d7375627374726174653a7175657374696f6e3a6465736b2d6c697374696e672d617574686f72697479a76f7574636f6d65a7436f756e746564a26f66c0a87265747261637473c0a4636f6465c0ac6f7074696f6e735f68617368d94037643935666338376435333234323036313166653431356331646566323761333434373631376262363832333566343665336166313330616561333631633438a663686f696365ab61736b696e672d6d696e64a46e6f7465be5468652061736b6572206c6973747320686572206f776e20666f726b732eab616e7377657265645f6174b4323032362d31312d31315431373a30303a30305aab72656365697665645f6174b4323032362d31312d31315431373a30303a30305aaf636c61696d5f696e5f667574757265c2a8616e737765726572c0a9617574686f72697479a64d656d626572aa726571756573745f6964d92430303030303030302d303030302d343030302d383030302d303030303030303030303031de0010ad736368656d6156657273696f6eb767616d6563756c742e6465736b5f616e737765722e7630a9616e737765725f6964a26e32a7666f726b5f6964d930657572656b612d7375627374726174653a7175657374696f6e3a6465736b2d6c697374696e672d617574686f72697479a76f7574636f6d65a85265766973696f6ea26f66a26e31a87265747261637473c0a4636f6465c0ac6f7074696f6e735f68617368d94037643935666338376435333234323036313166653431356331646566323761333434373631376262363832333566343665336166313330616561333631633438a663686f696365ab61736b696e672d6d696e64a46e6f7465a74167726565642eab616e7377657265645f6174b4323032362d31312d31315431373a30353a30305aab72656365697665645f6174b4323032362d31312d31315431373a30353a30305aaf636c61696d5f696e5f667574757265c2a8616e737765726572c0a9617574686f72697479a64d656d626572aa726571756573745f6964d92430303030303030302d303030302d343030302d383030302d303030303030303030303032de0010ad736368656d6156657273696f6eb767616d6563756c742e6465736b5f616e737765722e7630a9616e737765725f6964a26e33a7666f726b5f6964d930657572656b612d7375627374726174653a7175657374696f6e3a6465736b2d6c697374696e672d617574686f72697479a76f7574636f6d65a752656675736564a26f66c0a87265747261637473c0a4636f6465ad6e6f745f615f64656369646572ac6f7074696f6e735f68617368d94037643935666338376435333234323036313166653431356331646566323761333434373631376262363832333566343665336166313330616561333631633438a663686f696365ab61736b696e672d6d696e64a46e6f7465a74d6520746f6f2eab616e7377657265645f6174b4323032362d31312d31315431373a30363a30305aab72656365697665645f6174b4323032362d31312d31315431373a30363a30305aaf636c61696d5f696e5f667574757265c2a8616e737765726572c0a9617574686f72697479a64d656d626572aa726571756573745f6964d92430303030303030302d303030302d343030302d383030302d303030303030303030303033de0010ad736368656d6156657273696f6eb767616d6563756c742e6465736b5f616e737765722e7630a9616e737765725f6964a26e34a7666f726b5f6964d930657572656b612d7375627374726174653a7175657374696f6e3a6465736b2d6c697374696e672d617574686f72697479a76f7574636f6d65aa52657472616374696f6ea26f66c0a87265747261637473a26e32a4636f6465c0ac6f7074696f6e735f68617368d94037643935666338376435333234323036313166653431356331646566323761333434373631376262363832333566343665336166313330616561333631633438a663686f696365c0a46e6f7465c0ab616e7377657265645f6174b4323032362d31312d31315431373a30373a30305aab72656365697665645f6174b4323032362d31312d31315431373a30373a30305aaf636c61696d5f696e5f667574757265c2a8616e737765726572c0a9617574686f72697479a64d656d626572aa726571756573745f6964d92430303030303030302d303030302d343030302d383030302d303030303030303030303034
|
||||
Binary file not shown.
File diff suppressed because one or more lines are too long
|
|
@ -54,9 +54,8 @@ uuid = { version = "1", default-features = false, features = ["v5"] }
|
|||
libc = "0.2"
|
||||
# The lease socket: SCM_RIGHTS receive with MSG_CMSG_CLOEXEC and the descriptor
|
||||
# duplicates, as safe calls (the library forbids unsafe code). Already in the
|
||||
# graph through tempfile; this adds its `net` feature, and `fs` for the attachment
|
||||
# saver's descriptor-relative writes (openat, linkat, unlinkat).
|
||||
rustix = { version = "1", default-features = false, features = ["std", "net", "fs"] }
|
||||
# graph through tempfile; this adds its `net` feature only.
|
||||
rustix = { version = "1", default-features = false, features = ["std", "net"] }
|
||||
|
||||
# `syn`, `quote` and `proc-macro2` read the library's own source for
|
||||
# tests/public_surface.rs, which pins its public surface. All three are already
|
||||
|
|
|
|||
|
|
@ -65,7 +65,7 @@ pub(crate) fn addresses(who: &Addressee, delivery: &ChatDelivery) -> bool {
|
|||
/// Whether a delivery the mouth gave `own` is for her: her DM (`is_direct`, which every mouth sends, so an old mouth's DM is hers too),
|
||||
/// Matrix's word that it is a tag of hers or a reply to her, or what it says (`addresses`). The basis `Room` (`@room`) is for no one, and no basis is unproven, not "not for her": the names still decide.
|
||||
pub(crate) fn is_for(own: &Addressee, delivery: &ChatDelivery) -> bool {
|
||||
(delivery.form == Form::Message && (delivery.is_direct || matches!(delivery.basis, Some(Basis::Direct | Basis::Tag | Basis::Reply | Basis::FollowUp)))) || addresses(own, delivery)
|
||||
(delivery.form == Form::Message && (delivery.is_direct || matches!(delivery.basis, Some(Basis::Direct | Basis::Tag | Basis::Reply)))) || addresses(own, delivery)
|
||||
}
|
||||
|
||||
/// The chat names of one stewarded repo. Listed names are exactly those, each matched exactly as the steward wrote it (capitalization
|
||||
|
|
|
|||
|
|
@ -117,7 +117,7 @@ fn the_localpart_is_the_name_between_the_at_and_the_colon() {
|
|||
fn only_a_message_can_address_and_the_mouths_basis_stands_for_the_polling_mind_alone() {
|
||||
let nibu = steward("GameCult/Thing", &[]);
|
||||
let with = |basis, form, text: &str, mentions: &[&str]| ChatDelivery { basis, form, ..said(text, mentions) };
|
||||
for basis in [Basis::Direct, Basis::Tag, Basis::Reply, Basis::FollowUp] {
|
||||
for basis in [Basis::Direct, Basis::Tag, Basis::Reply] {
|
||||
assert!(is_for(&nibu, &with(Some(basis), Form::Message, "nothing named", &[])), "{basis:?}");
|
||||
assert!(!addresses(&nibu, &with(Some(basis), Form::Message, "nothing named", &[])), "a basis is a fact about the Persona that polled, not about every Mind: {basis:?}");
|
||||
assert!(!is_for(&nibu, &with(Some(basis), Form::Notice, "nothing named", &[])), "a notice is for no one: {basis:?}");
|
||||
|
|
|
|||
|
|
@ -1,154 +0,0 @@
|
|||
//! The attachment saver: a verified chat image becomes one file in the Persona's sandbox.
|
||||
//!
|
||||
//! `save` re-verifies what the mouth stored through `chat_image::load_bytes` (the one verifier; nothing here sniffs, hashes or
|
||||
//! sizes a file itself) and writes `attachments/<sha256>.<ext>` below the sandbox root. The name derives from the verified hash
|
||||
//! and one of four extensions; no component comes from the event, the reply, the mouth or the stored path.
|
||||
//!
|
||||
//! Containment is by directory descriptor, not by a path check. The sandbox root is opened once by its configured path (the
|
||||
//! operator's, not hers); every open below it is relative to a held descriptor and refuses a symlink, so a shell that swaps
|
||||
//! `attachments/` for a link between two calls finds the next call refusing it. The bytes go to a temporary name in that directory
|
||||
//! and are linked to the final name, which fails where a rename would replace her file; a crash leaves only the temporary.
|
||||
//!
|
||||
//! The file_received receipt means only that the mouth fetched and stored the file for her. It does not prove the file reached her sandbox. The receipt stays inside the mouth's fetch, so a failed save (Exists, WriteFailed, RepoTree, a Load skip) leaves the receipt standing; she is told the save failed in her own turn.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::chat_client::ChatImage;
|
||||
use crate::chat_image::{self, Skip};
|
||||
|
||||
/// Why a verified image was not saved. `Load` carries the verifier's own refusal, told by its own fixed reason.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum SaveFault {
|
||||
/// A git work tree at the sandbox root or in `attachments/`: a Matrix file does not go into one.
|
||||
RepoTree,
|
||||
/// The sandbox root or `attachments/` is a link or not a directory.
|
||||
OutsideSandbox,
|
||||
/// A different file already holds the name.
|
||||
Exists,
|
||||
WriteFailed,
|
||||
Load(Skip),
|
||||
}
|
||||
|
||||
/// The extension of a verified media type: one of four, never from outside.
|
||||
fn extension(media_type: &str) -> Option<&'static str> {
|
||||
match media_type {
|
||||
"image/png" => Some("png"),
|
||||
"image/jpeg" => Some("jpg"),
|
||||
"image/gif" => Some("gif"),
|
||||
"image/webp" => Some("webp"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Saves `image`, which the mouth stored for `room_id` under `store`, into `sandbox`. The path is relative to the sandbox:
|
||||
/// `attachments/<sha256>.<ext>`. An identical file already there counts as saved.
|
||||
pub(crate) fn save(store: &Path, sandbox: &Path, room_id: &str, image: &ChatImage) -> Result<PathBuf, SaveFault> {
|
||||
let (bytes, media_type) = chat_image::load_bytes(store, room_id, image).map_err(SaveFault::Load)?;
|
||||
let extension = extension(media_type).ok_or(SaveFault::Load(Skip::NotAnImage))?;
|
||||
let name = format!("{}.{extension}", image.sha256);
|
||||
write(sandbox, &name, &bytes)?;
|
||||
Ok(Path::new("attachments").join(name))
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn write(_sandbox: &Path, _name: &str, _bytes: &[u8]) -> Result<(), SaveFault> {
|
||||
// The organ runs under bubblewrap on Linux; there is no descriptor-relative write here.
|
||||
Err(SaveFault::WriteFailed)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn write(sandbox: &Path, name: &str, bytes: &[u8]) -> Result<(), SaveFault> {
|
||||
use rustix::fs::{self, AtFlags, Mode, OFlags};
|
||||
use rustix::io::Errno;
|
||||
|
||||
let dir_flags = OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC;
|
||||
let refused = |errno: Errno| if matches!(errno, Errno::LOOP | Errno::NOTDIR) { SaveFault::OutsideSandbox } else { SaveFault::WriteFailed };
|
||||
let work_tree = |dir: &rustix::fd::OwnedFd| match fs::statat(dir, ".git", AtFlags::SYMLINK_NOFOLLOW) {
|
||||
Ok(_) => Err(SaveFault::RepoTree),
|
||||
Err(Errno::NOENT) => Ok(()),
|
||||
Err(_) => Err(SaveFault::WriteFailed),
|
||||
};
|
||||
|
||||
let root = fs::open(sandbox, dir_flags, Mode::empty()).map_err(refused)?;
|
||||
work_tree(&root)?;
|
||||
match fs::mkdirat(&root, "attachments", Mode::RWXU) {
|
||||
Ok(()) | Err(Errno::EXIST) => {}
|
||||
Err(_) => return Err(SaveFault::WriteFailed),
|
||||
}
|
||||
let dir = fs::openat(&root, "attachments", dir_flags, Mode::empty()).map_err(refused)?;
|
||||
work_tree(&dir)?;
|
||||
|
||||
let temp = format!(".{name}.part");
|
||||
// A crashed save's leavings, even a FIFO or a link, are removed without being followed.
|
||||
match fs::unlinkat(&dir, temp.as_str(), AtFlags::empty()) {
|
||||
Ok(()) | Err(Errno::NOENT) => {}
|
||||
Err(_) => return Err(SaveFault::WriteFailed),
|
||||
}
|
||||
#[cfg(test)]
|
||||
tests::between_unlink_and_create();
|
||||
let file = fs::openat(&dir, temp.as_str(), OFlags::WRONLY | OFlags::CREATE | OFlags::EXCL | OFlags::NOFOLLOW | OFlags::CLOEXEC, Mode::RUSR | Mode::WUSR)
|
||||
.map_err(|_| SaveFault::WriteFailed)?;
|
||||
let landed = (|| {
|
||||
let mut rest = bytes;
|
||||
while !rest.is_empty() {
|
||||
#[cfg(test)]
|
||||
if tests::crash_after(bytes.len() - rest.len()) {
|
||||
return Err(Errno::IO);
|
||||
}
|
||||
let chunk = &rest[..rest.len().min(8192)];
|
||||
match rustix::io::write(&file, chunk) {
|
||||
Ok(0) => return Err(Errno::IO),
|
||||
Ok(n) => rest = &rest[n..],
|
||||
Err(Errno::INTR) => {}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
fs::fsync(&file)?;
|
||||
fs::linkat(&dir, temp.as_str(), &dir, name, AtFlags::empty())
|
||||
})();
|
||||
match landed {
|
||||
Ok(()) => {
|
||||
let _ = fs::unlinkat(&dir, temp.as_str(), AtFlags::empty());
|
||||
Ok(())
|
||||
}
|
||||
Err(Errno::EXIST) => {
|
||||
let _ = fs::unlinkat(&dir, temp.as_str(), AtFlags::empty());
|
||||
same_file(&dir, name, bytes)
|
||||
}
|
||||
// A simulated crash leaves the temporary as a real one would.
|
||||
#[cfg(test)]
|
||||
Err(Errno::IO) if tests::crashed() => Err(SaveFault::WriteFailed),
|
||||
Err(_) => {
|
||||
let _ = fs::unlinkat(&dir, temp.as_str(), AtFlags::empty());
|
||||
Err(SaveFault::WriteFailed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the file at `name` holds exactly `bytes`. It is a name her shell controls, so it is opened without following a link
|
||||
/// and without waiting (a FIFO opens), must be a regular file of exactly the image's length, and is read for at most that length
|
||||
/// plus one byte. Anything else is `Exists` with nothing read.
|
||||
#[cfg(unix)]
|
||||
fn same_file(dir: &rustix::fd::OwnedFd, name: &str, bytes: &[u8]) -> Result<(), SaveFault> {
|
||||
use rustix::fs::{self, FileType, Mode, OFlags};
|
||||
|
||||
let file = fs::openat(dir, name, OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::NONBLOCK | OFlags::CLOEXEC, Mode::empty()).map_err(|_| SaveFault::Exists)?;
|
||||
let stat = fs::fstat(&file).map_err(|_| SaveFault::Exists)?;
|
||||
if FileType::from_raw_mode(stat.st_mode) != FileType::RegularFile || u64::try_from(stat.st_size) != Ok(bytes.len() as u64) {
|
||||
return Err(SaveFault::Exists);
|
||||
}
|
||||
let mut held = vec![0u8; bytes.len() + 1];
|
||||
let mut filled = 0;
|
||||
while filled < held.len() {
|
||||
match rustix::io::read(&file, &mut held[filled..]) {
|
||||
Ok(0) => break,
|
||||
Ok(n) => filled += n,
|
||||
Err(rustix::io::Errno::INTR) => {}
|
||||
Err(_) => return Err(SaveFault::Exists),
|
||||
}
|
||||
}
|
||||
if held[..filled] == *bytes { Ok(()) } else { Err(SaveFault::Exists) }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue