Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ecf147dbd7 | ||
|
|
b721c3eac7 | ||
|
|
51a13b7e82 | ||
|
|
8b18434282 | ||
|
|
43c3b1c047 | ||
|
|
65642dcf3c | ||
| dcae9c5c2c | |||
|
|
b920f3f81f | ||
|
|
fbedfdce2f | ||
|
|
e719aa3766 | ||
| b4f3a48a8f | |||
| f1d5c4039b | |||
| 36dd8ce3ca | |||
| 421ace5432 | |||
| b299d6f732 | |||
| 79d954cad1 | |||
| 776367ef10 | |||
| 0115915b37 | |||
| ea4193e0e4 | |||
| d08b8880ff | |||
| da90d7e74e | |||
| bf9a8a7f13 | |||
| 9dde93e5f5 | |||
| 86372ae57b | |||
| 85b92f7f96 | |||
| ec8ba4b84c | |||
| 650e5e262e | |||
| 8a801d09dc | |||
| 66e6fe8d7d | |||
| 51835636b8 | |||
| 5f7314c668 | |||
| e988ecd521 | |||
| 5bd58b6e3f | |||
| 6ff4c50b29 | |||
| 0ff04c396e | |||
| 0e96552a3b | |||
| eea4d7edd5 | |||
| 94eb9401a5 | |||
| 2fb649e6ff | |||
| ec2d11efe8 | |||
| cd3066dde9 | |||
| aad5e204a2 | |||
| b373c2a5b1 | |||
| 5e8573235b | |||
| 6f66e048a5 | |||
| fece9a7737 | |||
| 99bc6c6c76 | |||
| c8b66eda85 | |||
| e939d246b8 | |||
| 1c412ebb8d | |||
| c812a18b18 | |||
| 2a099d82ca | |||
| 6d6aea7059 | |||
| 4e4f3a7f3b | |||
| 08afef830f | |||
| f4951db649 | |||
| b19e57a6d3 | |||
| 2d9e959c56 | |||
| 9bd6936ea6 | |||
| 480f94875b | |||
| ceb1dbdb86 | |||
| f8558a12a3 | |||
| 692586d607 | |||
| bbb05b51a4 | |||
| 0f2652d7fa | |||
| a26bce249d | |||
| 96a88d3cb6 | |||
| 3389ead5a2 | |||
| c243d731eb | |||
| aa15dfb36e | |||
| bf3f610b66 | |||
| 416fd7f719 | |||
| c2f251cb26 | |||
| 4e9d1dbd23 | |||
| c9a8e73dbc | |||
| 8a324186cf | |||
| ee72561716 | |||
| 8018322b2a | |||
| 69f9003b9b | |||
| 995a5b88f9 | |||
| 37be2cd305 | |||
| 4674d5f283 | |||
| cc0709039f | |||
| 0ee9c0228e | |||
| 7eb0440e00 | |||
| 9c1a5dba07 | |||
| 95713735aa | |||
| fa90daf8e6 | |||
| 2f04e13ea5 | |||
| 50d8e60102 | |||
| 470c188d4c | |||
| aae8fe2571 | |||
| d6068e7c04 | |||
| 598f1db7f9 | |||
| 797255eefe | |||
| 711514602b | |||
| 9d0a30947a | |||
| 4c1f247898 | |||
| 5af0cbea65 | |||
| d91985b521 |
188 changed files with 18363 additions and 2713 deletions
2
Cargo.lock
generated
2
Cargo.lock
generated
|
|
@ -1167,6 +1167,7 @@ dependencies = [
|
|||
"sha2",
|
||||
"syn 2.0.119",
|
||||
"tempfile",
|
||||
"unicode-general-category",
|
||||
"ureq",
|
||||
"uuid",
|
||||
]
|
||||
|
|
@ -1178,6 +1179,7 @@ source = "git+https://eureka.gamecult.org/forge/GameCult/ModelConnector.git?rev=
|
|||
dependencies = [
|
||||
"aes-gcm",
|
||||
"base64",
|
||||
"cultcache-rs 0.3.1",
|
||||
"cultnet-rs 0.2.0",
|
||||
"rand_core 0.6.4",
|
||||
"rmp-serde",
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ finding.plausible-blocker-preexisting-kinds eureka-state:finding:cut-3a.s2.F4 ep
|
|||
finding.severity-medium eureka-state:finding:cut-3a.s2.F4 epiphany.pipeline.finding.v2 EpiphanyPipelineFindingDocument 918da863616d706169676eac657572656b612d7374617465a776657264696374be657572656b612d73746174653a766572646963743a6375742d33612e7332a56c6162656ca24634a572616e676582a462617365a83566393832323864a468656164a83566393832323864aa636f6e666964656e6365a9436f6e6669726d6564a87365766572697479a64d656469756da5636c61696dd9214120646f74746564206c6162656c20636f6d706f7365732074776f206b6579732eaa696e76617269616e747391ab6d696e642d61646d697473a96c6f636174696f6e739183a470617468d9216372617465732f657572656b612d706970656c696e652f7372632f6c69622e7273a46c696e6501a8656e645f6c696e6509b06661696c7572655f7363656e6172696fbc54776f20646f63756d656e747320636c61696d206f6e65206b65792ea865766964656e63659183a46b696e64a454657374a76c6f6361746f72aa636172676f2074657374a6726573756c74a26f6baa707265636564656e747390a66f726967696eaa496e74726f6475636564
|
||||
finding.severity-low eureka-state:finding:cut-3a.s2.F4 epiphany.pipeline.finding.v2 EpiphanyPipelineFindingDocument 918da863616d706169676eac657572656b612d7374617465a776657264696374be657572656b612d73746174653a766572646963743a6375742d33612e7332a56c6162656ca24634a572616e676582a462617365a83566393832323864a468656164a83566393832323864aa636f6e666964656e6365a9436f6e6669726d6564a87365766572697479a34c6f77a5636c61696dd9214120646f74746564206c6162656c20636f6d706f7365732074776f206b6579732eaa696e76617269616e747391ab6d696e642d61646d697473a96c6f636174696f6e739183a470617468d9216372617465732f657572656b612d706970656c696e652f7372632f6c69622e7273a46c696e6501a8656e645f6c696e6509b06661696c7572655f7363656e6172696fbc54776f20646f63756d656e747320636c61696d206f6e65206b65792ea865766964656e63659183a46b696e64a454657374a76c6f6361746f72aa636172676f2074657374a6726573756c74a26f6baa707265636564656e747390a66f726967696eaa496e74726f6475636564
|
||||
ruling.authority-mind eureka-state:ruling:R8 epiphany.pipeline.ruling.v2 EpiphanyPipelineRulingDocument 918aa863616d706169676eac657572656b612d7374617465a56c6162656ca25238a57469746c65b9416e20696e7374616e6365206f776e7320697473206d696e64a7616e7377657273c0a663686f696365c0a672756c696e67ba416e20696e7374616e6365206f776e7320697473206d696e642eae6f70657261746f725f71756f7465c0a872756c65645f6f6eaa323032362d30392d3135aa707265636564656e74739185a47265706fb147616d6543756c742f4165746865726961a6636f6d6d6974d92861616161616161616161616161616161616161616161616161616161616161616161616161616161a46b696e64a652756c696e67a26964b363756c7463616368653a72756c696e673a5231ae7061796c6f61645f736861323536d94062626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262a9617574686f72697479a44d696e64
|
||||
ruling.authority-steward eureka-state:ruling:R8 epiphany.pipeline.ruling.v2 EpiphanyPipelineRulingDocument 918aa863616d706169676eac657572656b612d7374617465a56c6162656ca25238a57469746c65b9416e20696e7374616e6365206f776e7320697473206d696e64a7616e7377657273c0a663686f696365c0a672756c696e67ba416e20696e7374616e6365206f776e7320697473206d696e642eae6f70657261746f725f71756f7465c0a872756c65645f6f6eaa323032362d30392d3135aa707265636564656e74739185a47265706fb147616d6543756c742f4165746865726961a6636f6d6d6974d92861616161616161616161616161616161616161616161616161616161616161616161616161616161a46b696e64a652756c696e67a26964b363756c7463616368653a72756c696e673a5231ae7061796c6f61645f736861323536d94062626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262a9617574686f7269747981a75374657761726484a773746577617264ae406865696d64616c6c3a63686174a47265706fb047616d6543756c742f426966726f7374a5776f726473b4726566757365206c656164696e67207a65726f73a56576656e74a424657631
|
||||
cut_spec.reads-and-symbol eureka-state:cut_spec:cut-3a.r1 epiphany.pipeline.cut_spec.v2 EpiphanyPipelineCutSpecDocument 91de0013a863616d706169676eac657572656b612d7374617465a3637574a23361a87265766973696f6e01a57469746c65b2506970656c696e6520646f63756d656e7473a47265706fb147616d6543756c742f4570697068616e79a66272616e6368bb636f6465782f657572656b612d706970656c696e652d7374617465a462617365a83566393832323864aa646570656e64735f6f6e91a132a5666972737491ae526561642074686520737065632ea764656c657465739183a470617468a66f6c642e7273a56c696e657303a46e6f7465a464656164ab6b656570735f6d6f76657391ac636f6d6d6974206f776e6572a46164647391b5706970656c696e655f646f63756d656e74732e7273ac66696c655f6368616e6765739182a86c6f636174696f6e84a470617468d9216372617465732f657572656b612d706970656c696e652f7372632f6c69622e7273a46c696e6501a8656e645f6c696e6509a673796d626f6cab76616c75655f7479706573a66368616e6765a3616464a572656164739282a86c6f636174696f6e84a470617468d9216372617465732f657572656b612d706970656c696e652f7372632f6c69622e7273a46c696e650aa8656e645f6c696e6514a673796d626f6caf506970656c696e6543757453706563a3776879b274686520646f63756d656e7420736861706582a86c6f636174696f6e83a470617468d9216372617465732f657572656b612d706970656c696e652f7372632f6c69622e7273a46c696e651ea8656e645f6c696e65c0a3776879a86f6e65206c696e65ad617574686f726974795f6d617087a56f776e6572a4636f7265a6696e7075747391af747970656420646f63756d656e7473a76f75747075747391a9656e76656c6f706573ad646572697665645f737461746591ac64657269766564206b657973b1666f7262696464656e5f7772697465727391a34d4350ac7368617265645f706174687391a961646d697373696f6ead64656c6574696f6e5f6c696e65a36e2f61ac766572696669636174696f6e84a66275696c647391ab636172676f20636865636ba574657374739182a46e616d65a46b657973a470696e73af6f6e652064657269766564206b6579a86e656761746976659182a77061747465726ea75665633c75383ea573636f7065a9646f63756d656e7473a86f70657261746f7291a46e6f6e65a8657374696d61746588ab6c696e65735f6164646564cd0384ad6c696e65735f72656d6f76656400b2646570656e64656e636965735f616464656491a8736368656d617273b4646570656e64656e636965735f72656d6f76656490ad666f726d6174735f616464656491b66570697068616e792e706970656c696e652e2a2e7631af666f726d6174735f72656d6f76656490ad746172676574735f616464656490af746172676574735f72656d6f76656490a772756c696e677391b6657572656b612d73746174653a72756c696e673a5238a97175657374696f6e7391b8657572656b612d73746174653a7175657374696f6e3a5131
|
||||
finding.location-symbol eureka-state:finding:cut-3a.s2.F4 epiphany.pipeline.finding.v2 EpiphanyPipelineFindingDocument 918da863616d706169676eac657572656b612d7374617465a776657264696374be657572656b612d73746174653a766572646963743a6375742d33612e7332a56c6162656ca24634a572616e676582a462617365a83566393832323864a468656164a83566393832323864aa636f6e666964656e6365a9436f6e6669726d6564a87365766572697479a448696768a5636c61696dd9214120646f74746564206c6162656c20636f6d706f7365732074776f206b6579732eaa696e76617269616e747391ab6d696e642d61646d697473a96c6f636174696f6e739184a470617468d9216372617465732f657572656b612d706970656c696e652f7372632f6c69622e7273a46c696e6501a8656e645f6c696e6509a673796d626f6cab2631313533303930343938b06661696c7572655f7363656e6172696fbc54776f20646f63756d656e747320636c61696d206f6e65206b65792ea865766964656e63659183a46b696e64a454657374a76c6f6361746f72aa636172676f2074657374a6726573756c74a26f6baa707265636564656e747390a66f726967696eaa496e74726f6475636564
|
||||
target.repos eureka-state:target:r2 epiphany.pipeline.target.v2 EpiphanyPipelineTargetDocument 9187a863616d706169676eac657572656b612d7374617465a87265766973696f6e02aa696e76617269616e74739182a56c6162656cab6d696e642d61646d697473a973746174656d656e74b64f6e6c792061646d697373696f6e207772697465732eac6e6f745f696e5f73636f706591ac4576652062726f7773696e67b963616e6f6e6963616c5f696d706c656d656e746174696f6e7391a743756c744c6962a57265706f7392b147616d6543756c742f4570697068616e79af47616d6543756c742f487567696e6ea3646f6384a470617468d9256e6f7465732f657572656b612d706970656c696e652d73746174652d7461726765742e6d64aa73746172745f6c696e6501a8656e645f6c696e6509a6636f6d6d6974a83566393832323864
|
||||
|
|
|
|||
|
|
@ -759,7 +759,6 @@ unit_enums! {
|
|||
ClaimOutcome { Holds, Falsified, Unproven }
|
||||
FindingConfidence { Confirmed, Plausible }
|
||||
FindingSeverity { Blocker, High, Medium, Low }
|
||||
RulingAuthority { Operator, Standing, Defaulted, Mind }
|
||||
FindingOrigin { Introduced, PreExisting }
|
||||
RunOperator { Operator, Mind }
|
||||
RunTurn { PersonaTurn, SelfRun, Sleep }
|
||||
|
|
@ -1127,6 +1126,55 @@ impl Bounded for ResearchSource {
|
|||
}
|
||||
}
|
||||
|
||||
/// Who decided a ruling, and under what. `Mind` is this mind's own decision, admitted by her Self. `Steward` is
|
||||
/// a named peer's decision inside the peer's stewardship, as this mind heard it: `steward` is the deciding Mind's
|
||||
/// Matrix id, `repo` the stewardship she decided under, `words` her own words copied from the line this mind heard,
|
||||
/// `event` that line's Matrix event id when it was a delivery. The four unit variants stay bare strings on the wire.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, JsonSchema)]
|
||||
pub enum RulingAuthority {
|
||||
Operator,
|
||||
Standing,
|
||||
Defaulted,
|
||||
Mind,
|
||||
Steward { steward: Short, repo: OrgRepo, words: Para, event: Option<Short> },
|
||||
}
|
||||
|
||||
impl RulingAuthority {
|
||||
/// The variants' spellings, in declaration order: the names the wire and listings use.
|
||||
pub const NAMES: &'static [&'static str] = &["Operator", "Standing", "Defaulted", "Mind", "Steward"];
|
||||
|
||||
/// The variant's spelling, for rows, headers and listings.
|
||||
pub fn name(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Operator => "Operator",
|
||||
Self::Standing => "Standing",
|
||||
Self::Defaulted => "Defaulted",
|
||||
Self::Mind => "Mind",
|
||||
Self::Steward { .. } => "Steward",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Bounded for RulingAuthority {
|
||||
fn validate(&self, field: &str) -> Result<(), PipelineRefusal> {
|
||||
match self {
|
||||
Self::Steward { steward, repo, words, event } => {
|
||||
steward.validate(&format!("{field}.steward"))?;
|
||||
if steward.0.trim().is_empty() {
|
||||
return Err(format_error(&format!("{field}.steward"), "a steward naming no Mind"));
|
||||
}
|
||||
repo.validate(&format!("{field}.repo"))?;
|
||||
words.validate(&format!("{field}.words"))?;
|
||||
if words.0.trim().is_empty() {
|
||||
return Err(format_error(&format!("{field}.words"), "a steward ruling with no words"));
|
||||
}
|
||||
event.validate(&format!("{field}.event"))
|
||||
}
|
||||
_ => Ok(()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// How a subject was resolved. Every referent is a parsed `PipelineRef`, so a
|
||||
/// resolution names its records by ids of the kinds they declare, validated
|
||||
/// where every other referent is; a `Fixed` commit is a `Sha` whose referent
|
||||
|
|
@ -1923,6 +1971,50 @@ mod tests {
|
|||
ruling
|
||||
}
|
||||
|
||||
fn steward_ruling() -> PipelineRuling {
|
||||
let mut ruling = mind_ruling();
|
||||
ruling.authority = RulingAuthority::Steward {
|
||||
steward: "@heimdall:chat".into(),
|
||||
repo: OrgRepo("GameCult/Bifrost".into()),
|
||||
words: "refuse leading zeros".into(),
|
||||
event: Some("$ev1".into()),
|
||||
};
|
||||
ruling
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_steward_ruling_round_trips_and_leaves_the_unit_authorities_bare_strings() -> Result<()> {
|
||||
let cache = schema_cache()?;
|
||||
let document = PipelineDocument::Ruling(steward_ruling());
|
||||
document.validate()?;
|
||||
assert_eq!(PipelineDocument::decode(&document.prepare(&cache)?)?, document);
|
||||
let json = serde_json::to_value(&document)?;
|
||||
assert_eq!(json["value"]["authority"]["Steward"]["steward"], "@heimdall:chat");
|
||||
let mut unit = ruling_sample();
|
||||
unit.authority = RulingAuthority::Defaulted;
|
||||
assert_eq!(serde_json::to_value(PipelineDocument::Ruling(unit))?["value"]["authority"], "Defaulted");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_steward_ruling_refuses_blank_words_blank_steward_and_a_bad_repo() {
|
||||
let refused = |edit: fn(&mut RulingAuthority)| {
|
||||
let mut ruling = steward_ruling();
|
||||
edit(&mut ruling.authority);
|
||||
PipelineDocument::Ruling(ruling).validate().expect_err("refused").to_string()
|
||||
};
|
||||
let fields = [
|
||||
refused(|a| if let RulingAuthority::Steward { words, .. } = a { words.0 = " ".into() }),
|
||||
refused(|a| if let RulingAuthority::Steward { steward, .. } = a { steward.0 = String::new() }),
|
||||
refused(|a| if let RulingAuthority::Steward { repo, .. } = a { repo.0 = "nope".into() }),
|
||||
refused(|a| if let RulingAuthority::Steward { words, .. } = a { words.0 = "w".repeat(4001) }),
|
||||
refused(|a| if let RulingAuthority::Steward { event, .. } = a { *event = Some("e".repeat(201).as_str().into()) }),
|
||||
];
|
||||
for (fault, field) in fields.iter().zip(["authority.words", "authority.steward", "authority.repo", "authority.words", "authority.event"]) {
|
||||
assert!(fault.contains(field), "{fault}");
|
||||
}
|
||||
}
|
||||
|
||||
/// A withdrawal is a pulse that says only `tempo_vote_withdrawn`: it must survive
|
||||
/// encoding to an envelope, JSON and named MessagePack as a withdrawal, and an
|
||||
/// unwithdrawn pulse must not write the key.
|
||||
|
|
@ -1967,7 +2059,7 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn unit_enum_names_are_the_variants_in_declaration_order() {
|
||||
assert_eq!(RulingAuthority::NAMES, ["Operator", "Standing", "Defaulted", "Mind"]);
|
||||
assert_eq!(RulingAuthority::NAMES, ["Operator", "Standing", "Defaulted", "Mind", "Steward"]);
|
||||
assert_eq!(FindingOrigin::NAMES, ["Introduced", "PreExisting"]);
|
||||
}
|
||||
|
||||
|
|
@ -2044,6 +2136,7 @@ mod tests {
|
|||
out.push((format!("finding.severity-{name}"), D::Finding(f)));
|
||||
}
|
||||
out.push(("ruling.authority-mind".into(), D::Ruling(mind_ruling())));
|
||||
out.push(("ruling.authority-steward".into(), D::Ruling(steward_ruling())));
|
||||
let PipelineDocument::CutSpec(mut spec) = samples().remove(4).0 else { unreachable!() };
|
||||
spec.file_changes[0].location.symbol = Some("value_types".into());
|
||||
spec.reads = vec![
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ required-features = ["cli"]
|
|||
|
||||
[dependencies]
|
||||
anyhow = "1"
|
||||
chrono = "0.4.44"
|
||||
clap = { version = "4", features = ["derive"], optional = true }
|
||||
cultnet-rs.workspace = true
|
||||
huginn-mind = { path = "../huginn-mind" }
|
||||
|
|
@ -48,6 +49,5 @@ unicode-general-category = "1"
|
|||
# Tests run a real daemon in-process against a temporary mind, and the real
|
||||
# binary against it.
|
||||
[dev-dependencies]
|
||||
chrono = "0.4.44"
|
||||
huginn-daemon = { path = "../huginn-daemon" }
|
||||
tempfile = "3"
|
||||
|
|
|
|||
57
crates/eureka-state/src/bin/huginn/chat.rs
Normal file
57
crates/eureka-state/src/bin/huginn/chat.rs
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
//! The `huginn chat` verbs' arguments: one instant parser for `--after` and
|
||||
//! `--before`. Display and the request are in `main` and `render`.
|
||||
|
||||
use chrono::{DateTime, SecondsFormat, Utc};
|
||||
|
||||
use crate::trouble::Trouble;
|
||||
|
||||
/// An instant as Unix milliseconds: an RFC 3339 time (`2026-11-11T10:00:00Z`)
|
||||
/// or an age before `now` (`90s`, `15m`, `6h`, `2d`, `1w`). The one parse of
|
||||
/// both flags.
|
||||
pub fn instant_ms(flag: &str, text: &str, now: DateTime<Utc>) -> Result<u64, Trouble> {
|
||||
let wrong = || Trouble::invalid(format!("{flag} is an RFC 3339 time or an age such as 6h or 2d, not `{text}`"));
|
||||
let moment = if let Ok(time) = DateTime::parse_from_rfc3339(text) {
|
||||
time.with_timezone(&Utc)
|
||||
} else {
|
||||
let (digits, unit) = text.split_at(text.char_indices().next_back().map_or(0, |(at, _)| at));
|
||||
if digits.is_empty() || !digits.bytes().all(|byte| byte.is_ascii_digit()) {
|
||||
return Err(wrong());
|
||||
}
|
||||
let count: u32 = digits.parse().map_err(|_| wrong())?;
|
||||
let seconds = match unit {
|
||||
"s" => 1,
|
||||
"m" => 60,
|
||||
"h" => 3600,
|
||||
"d" => 86_400,
|
||||
"w" => 604_800,
|
||||
_ => return Err(wrong()),
|
||||
};
|
||||
now.checked_sub_signed(chrono::Duration::try_seconds(i64::from(count) * seconds).ok_or_else(wrong)?).ok_or_else(wrong)?
|
||||
};
|
||||
u64::try_from(moment.timestamp_millis()).map_err(|_| wrong())
|
||||
}
|
||||
|
||||
/// An instant as the CLI prints it.
|
||||
pub fn shown(ms: u64) -> String {
|
||||
DateTime::from_timestamp_millis(i64::try_from(ms).unwrap_or(i64::MAX))
|
||||
.map_or_else(|| ms.to_string(), |time| time.to_rfc3339_opts(SecondsFormat::Millis, true))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn an_instant_is_rfc3339_or_an_age_before_now() {
|
||||
let now = DateTime::parse_from_rfc3339("2026-11-11T12:00:00Z").unwrap().with_timezone(&Utc);
|
||||
let at = |text: &str| instant_ms("--after", text, now).map(shown);
|
||||
assert_eq!(at("2026-11-11T10:00:00Z").unwrap(), "2026-11-11T10:00:00.000Z");
|
||||
assert_eq!(at("2026-11-11T12:00:00+02:00").unwrap(), "2026-11-11T10:00:00.000Z");
|
||||
assert_eq!(at("6h").unwrap(), "2026-11-11T06:00:00.000Z");
|
||||
assert_eq!(at("2d").unwrap(), "2026-11-09T12:00:00.000Z");
|
||||
assert_eq!(at("90s").unwrap(), "2026-11-11T11:58:30.000Z");
|
||||
for bad in ["", "h", "6", "6x", "-2d", "yesterday", "1e3h", "+2d", "6日", "日", "4000000000w", "99999999999999999999d"] {
|
||||
assert!(matches!(at(bad), Err(Trouble::InvalidInput(ref detail)) if detail.contains("--after")), "{bad}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -15,6 +15,10 @@ use crate::trouble::Trouble;
|
|||
pub const TOOL: &str = "huginn";
|
||||
/// How long one call may take, connection to last byte.
|
||||
const TIMEOUT: Duration = Duration::from_secs(15);
|
||||
/// The environment variable a Self run sets to its run label: the session an admitting verb attributes to when `--session` is not given.
|
||||
pub const SESSION_VARIABLE: &str = "HUGINN_SESSION";
|
||||
/// `Short`'s bound in bytes (the leaf keeps it private); the daemon enforces it again.
|
||||
const SESSION_MAX: usize = 200;
|
||||
/// What `ask_until` leaves of its deadline for the caller to print and exit.
|
||||
const RESERVE: Duration = Duration::from_millis(100);
|
||||
|
||||
|
|
@ -39,6 +43,22 @@ impl Settings {
|
|||
self.client.as_ref().map_err(|detail| Trouble::Misconfigured(detail.clone()))
|
||||
}
|
||||
|
||||
/// The session an admission is attributed to: the flag as given; else `HUGINN_SESSION` when it
|
||||
/// is set and not empty; else a refusal. Never guessed and never any other variable.
|
||||
pub fn session(&self, flag: Option<String>) -> Result<String, Trouble> {
|
||||
if let Some(flag) = flag {
|
||||
return Ok(flag);
|
||||
}
|
||||
match variable(SESSION_VARIABLE) {
|
||||
Err(detail) => Err(Trouble::invalid(detail)),
|
||||
Ok(Some(value)) if value.len() > SESSION_MAX => {
|
||||
Err(Trouble::invalid(format!("{SESSION_VARIABLE} is longer than {SESSION_MAX} bytes")))
|
||||
}
|
||||
Ok(Some(value)) if !value.is_empty() => Ok(value),
|
||||
Ok(_) => Err(Trouble::invalid(format!("--session is required (or {SESSION_VARIABLE} set to the session's label)"))),
|
||||
}
|
||||
}
|
||||
|
||||
/// A client for another mind, read from: `instance` at `endpoint` (`rudp://<ip>:<port>`), both
|
||||
/// given as arguments. It refuses this process's own instance before any call, so the one
|
||||
/// mind a verb writes is never also its source. Every request through it declares `instance`,
|
||||
|
|
|
|||
|
|
@ -4,9 +4,9 @@
|
|||
//! strict decode of agent-written JSON into the leaf's types (`door`), the
|
||||
//! recipe selections (`recipes`), the text rendering (`render`), the exit code
|
||||
//! and the read-back after an admission. It owns no state, writes no file and
|
||||
//! reads no configuration but two environment variables: `EUREKA_INSTANCE`
|
||||
//! reads no configuration but these environment variables: `EUREKA_INSTANCE`
|
||||
//! names the mind and `HUGINN_ENDPOINT` (`rudp://host:port`) the daemon that
|
||||
//! holds it. Each call is a new process; there is no retry inside it, the
|
||||
//! holds it. The admitting verbs take `--session`, else `HUGINN_SESSION`. Each call is a new process; there is no retry inside it, the
|
||||
//! caller's rule retries `Unavailable` once.
|
||||
//!
|
||||
//! Answers go to stdout and the exit code says what the answer was: 0 answered,
|
||||
|
|
@ -17,6 +17,7 @@
|
|||
//! exits 1 when the campaign is not moved to this mind or this mind holds records of its own under it, and 4 when the history cannot be replayed whole.
|
||||
|
||||
mod admit_hold;
|
||||
mod chat;
|
||||
mod config;
|
||||
mod door;
|
||||
mod import;
|
||||
|
|
@ -40,7 +41,7 @@ use huginn_mind::eureka_pipeline::{
|
|||
Line, PipelineDocument, PipelineRef, Short, Slug, pipeline_key,
|
||||
};
|
||||
use huginn_mind::{
|
||||
BATCH_MAX, Faculty, HuginnMindRequest, HuginnMindResponse, MindRefusal, PipelineAdmissionBatch, PipelineAdmissionOutcome,
|
||||
BATCH_MAX, ChatQuery, ChatWindowQuery, Faculty, HuginnMindRequest, HuginnMindResponse, MindRefusal, PipelineAdmissionBatch, PipelineAdmissionOutcome,
|
||||
PersonaPutOutcome, PipelinePageItems, PipelineProvenance, PipelineSelectionPage, SemanticQuery, SourceQuery,
|
||||
};
|
||||
use serde_json::Value;
|
||||
|
|
@ -55,6 +56,7 @@ const DEFAULT_LIMIT: u32 = 50;
|
|||
const HOLD_CHECK: Duration = Duration::from_secs(10);
|
||||
/// The neighbours `--near` asks for when `--top-k` is not given.
|
||||
const DEFAULT_TOP_K: u32 = 10;
|
||||
const DEFAULT_WINDOW: u32 = 10;
|
||||
|
||||
const ADMIT_HELP: &str = "\
|
||||
The file is a JSON array of 1 to 64 documents, each {\"kind\": <kind>, \"value\": {...}}, or - for stdin; one document object alone is a batch of one. \
|
||||
|
|
@ -71,6 +73,50 @@ struct Cli {
|
|||
command: Command,
|
||||
}
|
||||
|
||||
#[derive(Subcommand, Debug)]
|
||||
enum ChatCommand {
|
||||
/// Chunks of the indexed rooms, each citing its room,
|
||||
/// event ids and times. With texts, ranked by meaning; with none, a listing
|
||||
/// oldest first, at most --top-k, ending in the `--after` to continue from.
|
||||
Search {
|
||||
/// Up to eight topics to find; the ranked lists of several are fused.
|
||||
#[arg(num_args = 0..=8)]
|
||||
texts: Vec<String>,
|
||||
/// Only this room: an id (`!abc:gamecult.org`) or a canonical alias
|
||||
/// (`#swarm:gamecult.org`). Repeat for several.
|
||||
#[arg(long = "room")]
|
||||
rooms: Vec<String>,
|
||||
/// From this instant: an RFC 3339 time, or an age such as 6h or 2d.
|
||||
#[arg(long)]
|
||||
after: Option<String>,
|
||||
/// Until this instant, as --after.
|
||||
#[arg(long)]
|
||||
before: Option<String>,
|
||||
/// How many chunks (default 10, at most 50).
|
||||
#[arg(long)]
|
||||
top_k: Option<u32>,
|
||||
/// Print the answer as JSON.
|
||||
#[arg(long)]
|
||||
json: bool,
|
||||
},
|
||||
/// One indexed message and the messages around it in the same room, by the
|
||||
/// event id a chat hit cites. Read from the index, so a removed message is
|
||||
/// refused, not shown.
|
||||
Window {
|
||||
/// The event id, beginning `$`.
|
||||
event_id: String,
|
||||
/// Messages before it (default 10, at most 50).
|
||||
#[arg(long)]
|
||||
before: Option<u32>,
|
||||
/// Messages after it (default 10, at most 50).
|
||||
#[arg(long)]
|
||||
after: Option<u32>,
|
||||
/// Print the answer as JSON.
|
||||
#[arg(long)]
|
||||
json: bool,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Subcommand, Debug)]
|
||||
enum Command {
|
||||
/// Who this process acts for and whether the daemon answers.
|
||||
|
|
@ -99,8 +145,11 @@ enum Command {
|
|||
},
|
||||
/// Search the public forge repos' code and docs by meaning.
|
||||
Sources {
|
||||
/// What to find, in words.
|
||||
text: String,
|
||||
/// What to find, in words: one to eight texts. Each is a topic; the
|
||||
/// ranked lists of several are fused, so a brief with many topics
|
||||
/// retrieves by every one.
|
||||
#[arg(required = true, num_args = 1..=8)]
|
||||
texts: Vec<String>,
|
||||
/// Look in this repo alone (its forge name).
|
||||
#[arg(long)]
|
||||
repo: Option<String>,
|
||||
|
|
@ -115,6 +164,11 @@ enum Command {
|
|||
#[arg(long)]
|
||||
json: bool,
|
||||
},
|
||||
/// Search the Minds' chat rooms.
|
||||
Chat {
|
||||
#[command(subcommand)]
|
||||
command: ChatCommand,
|
||||
},
|
||||
#[command(flatten)]
|
||||
Recipe(Recipe),
|
||||
/// Admit a batch of documents, whole or not at all, and read it back.
|
||||
|
|
@ -125,9 +179,9 @@ enum Command {
|
|||
/// The admitting agent's name.
|
||||
#[arg(long)]
|
||||
agent: String,
|
||||
/// The admitting session.
|
||||
/// The admitting session; else the environment's HUGINN_SESSION, which a Self run sets to its run.
|
||||
#[arg(long)]
|
||||
session: String,
|
||||
session: Option<String>,
|
||||
/// The token of the hold this exact batch got: admit it.
|
||||
#[arg(long)]
|
||||
seen: Option<String>,
|
||||
|
|
@ -152,9 +206,9 @@ enum Command {
|
|||
/// The importing agent's name.
|
||||
#[arg(long)]
|
||||
agent: String,
|
||||
/// The importing session.
|
||||
/// The importing session; else the environment's HUGINN_SESSION.
|
||||
#[arg(long)]
|
||||
session: String,
|
||||
session: Option<String>,
|
||||
},
|
||||
/// Recall into a dispatched brief.
|
||||
Recall {
|
||||
|
|
@ -198,9 +252,9 @@ enum PersonaCommand {
|
|||
/// The writing agent's name.
|
||||
#[arg(long)]
|
||||
agent: String,
|
||||
/// The writing session.
|
||||
/// The writing session; else the environment's HUGINN_SESSION.
|
||||
#[arg(long)]
|
||||
session: String,
|
||||
session: Option<String>,
|
||||
/// The stored updatedAt this put replaces; omit for the first put.
|
||||
#[arg(long)]
|
||||
expected_updated_at: Option<String>,
|
||||
|
|
@ -275,12 +329,16 @@ fn run(settings: &Settings, command: Command) -> Result<u8, Trouble> {
|
|||
query(settings, selection, semantic, None, documents)
|
||||
}
|
||||
Command::Sources {
|
||||
text,
|
||||
texts,
|
||||
repo,
|
||||
front,
|
||||
top_k,
|
||||
json,
|
||||
} => sources(settings, text, repo, front, top_k, json),
|
||||
} => sources(settings, texts, repo, front, top_k, json),
|
||||
Command::Chat { command: ChatCommand::Search { texts, rooms, after, before, top_k, json } } => {
|
||||
chat_search(settings, texts, rooms, after.as_deref(), before.as_deref(), top_k, json)
|
||||
}
|
||||
Command::Chat { command: ChatCommand::Window { event_id, before, after, json } } => chat_window(settings, event_id, before, after, json),
|
||||
Command::Recipe(Recipe::Blocked { spec_id, page }) => {
|
||||
blocked(settings, &spec_id, page.cursor, page.documents)
|
||||
}
|
||||
|
|
@ -290,6 +348,7 @@ fn run(settings: &Settings, command: Command) -> Result<u8, Trouble> {
|
|||
}
|
||||
Command::Admit { faculty, agent, session, seen, file } => admit(settings, faculty, agent, session, seen, &file),
|
||||
Command::Import { campaign, from, from_endpoint, faculty, agent, session } => {
|
||||
let session = settings.session(session)?;
|
||||
import::run(settings, import::Flags { campaign, from, from_endpoint, faculty, agent, session })
|
||||
}
|
||||
Command::Persona { command: PersonaCommand::Get } => persona_get(settings),
|
||||
|
|
@ -437,14 +496,15 @@ fn blocked(
|
|||
|
||||
fn sources(
|
||||
settings: &Settings,
|
||||
text: String,
|
||||
texts: Vec<String>,
|
||||
repo: Option<String>,
|
||||
front: bool,
|
||||
top_k: Option<u32>,
|
||||
json: bool,
|
||||
) -> Result<u8, Trouble> {
|
||||
let query = SourceQuery {
|
||||
text: Line(text),
|
||||
text: Line::default(),
|
||||
texts: texts.into_iter().map(Line).collect(),
|
||||
top_k: top_k.unwrap_or(DEFAULT_TOP_K),
|
||||
repo: repo.map(Short),
|
||||
front,
|
||||
|
|
@ -466,6 +526,53 @@ fn sources(
|
|||
}
|
||||
}
|
||||
|
||||
fn chat_search(
|
||||
settings: &Settings,
|
||||
texts: Vec<String>,
|
||||
rooms: Vec<String>,
|
||||
after: Option<&str>,
|
||||
before: Option<&str>,
|
||||
top_k: Option<u32>,
|
||||
json: bool,
|
||||
) -> Result<u8, Trouble> {
|
||||
let now = chrono::Utc::now();
|
||||
let query = ChatQuery {
|
||||
texts: texts.into_iter().map(Line).collect(),
|
||||
rooms: rooms.into_iter().map(Short).collect(),
|
||||
after_ms: after.map(|text| chat::instant_ms("--after", text, now)).transpose()?,
|
||||
before_ms: before.map(|text| chat::instant_ms("--before", text, now)).transpose()?,
|
||||
top_k: top_k.unwrap_or(DEFAULT_TOP_K),
|
||||
};
|
||||
match settings.ask(|instance| HuginnMindRequest::SearchChat { instance: instance.clone(), query })? {
|
||||
HuginnMindResponse::Chat(found) => {
|
||||
if json {
|
||||
say(serde_json::to_string_pretty(&found).map_err(|error| Trouble::Internal(error.to_string()))?);
|
||||
} else {
|
||||
say(render::chat_hits(&found));
|
||||
}
|
||||
Ok(0)
|
||||
}
|
||||
HuginnMindResponse::Refused(refusal) => Ok(refused(&refusal)),
|
||||
other => Err(unexpected(&other)),
|
||||
}
|
||||
}
|
||||
|
||||
fn chat_window(settings: &Settings, event_id: String, before: Option<u32>, after: Option<u32>, json: bool) -> Result<u8, Trouble> {
|
||||
let query = ChatWindowQuery { event_id: Short(event_id), before: before.unwrap_or(DEFAULT_WINDOW), after: after.unwrap_or(DEFAULT_WINDOW) };
|
||||
match settings.ask(|instance| HuginnMindRequest::ChatWindow { instance: instance.clone(), query })? {
|
||||
HuginnMindResponse::ChatWindow(found) => {
|
||||
if json {
|
||||
say(serde_json::to_string_pretty(&found).map_err(|error| Trouble::Internal(error.to_string()))?);
|
||||
} else {
|
||||
say(render::chat_window(&found));
|
||||
}
|
||||
Ok(0)
|
||||
}
|
||||
HuginnMindResponse::Refused(refusal) => Ok(refused(&refusal)),
|
||||
other => Err(unexpected(&other)),
|
||||
}
|
||||
}
|
||||
|
||||
fn show(page: &PipelineSelectionPage) {
|
||||
match &page.items {
|
||||
PipelinePageItems::Headers(headers) => {
|
||||
|
|
@ -489,10 +596,11 @@ fn admit(
|
|||
settings: &Settings,
|
||||
faculty: Faculty,
|
||||
agent: String,
|
||||
session: String,
|
||||
session: Option<String>,
|
||||
seen: Option<String>,
|
||||
file: &str,
|
||||
) -> Result<u8, Trouble> {
|
||||
let session = settings.session(session)?;
|
||||
let documents = door::documents(&door::file_or_stdin(file)?)?;
|
||||
if documents.is_empty() || documents.len() > BATCH_MAX {
|
||||
return Err(Trouble::invalid(format!("a batch is 1 to {BATCH_MAX} documents, this one has {}", documents.len())));
|
||||
|
|
@ -649,10 +757,11 @@ fn persona_put(
|
|||
settings: &Settings,
|
||||
faculty: Faculty,
|
||||
agent: String,
|
||||
session: String,
|
||||
session: Option<String>,
|
||||
expected_updated_at: Option<String>,
|
||||
file: &str,
|
||||
) -> Result<u8, Trouble> {
|
||||
let session = settings.session(session)?;
|
||||
let text = door::file_or_stdin(file)?;
|
||||
let state: Value = serde_json::from_str(&text)
|
||||
.map_err(|error| Trouble::invalid(format!("the persona document is not JSON: {error}")))?;
|
||||
|
|
@ -681,6 +790,8 @@ fn unexpected(answer: &HuginnMindResponse) -> Trouble {
|
|||
HuginnMindResponse::Persona(_) => "a persona",
|
||||
HuginnMindResponse::PersonaPut(_) => "a persona put",
|
||||
HuginnMindResponse::Sources(_) => "source hits",
|
||||
HuginnMindResponse::Chat(_) => "chat hits",
|
||||
HuginnMindResponse::ChatWindow(_) => "a chat window",
|
||||
HuginnMindResponse::Activity(_) => "repo activity",
|
||||
HuginnMindResponse::RootsTouched(_) => "roots touched",
|
||||
HuginnMindResponse::Refused(_) => "a refusal",
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
use std::fmt::Write;
|
||||
|
||||
use huginn_mind::eureka_pipeline::{PipelineResolution, ResolutionOutcome};
|
||||
use huginn_mind::{SourceHits, IndexStatus, PersonaStatus, PipelineDocumentSummary, PipelineDocumentView, PipelineStatus, PipelineStatusSummary};
|
||||
use huginn_mind::{ChatHits, ChatWindow, SourceHits, IndexStatus, PersonaStatus, PipelineDocumentSummary, PipelineDocumentView, PipelineStatus, PipelineStatusSummary};
|
||||
use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
|
||||
|
|
@ -100,7 +100,17 @@ pub fn header(item: &PipelineDocumentSummary) -> String {
|
|||
let _ = write!(line, " {name}");
|
||||
}
|
||||
for (key, value) in facts {
|
||||
if key != "title"
|
||||
if key == "authority"
|
||||
&& let Some(Value::Object(steward)) = value.get("Steward")
|
||||
{
|
||||
// Her words stay out of the header: it names who decided and under what.
|
||||
let _ = write!(line, " authority=Steward");
|
||||
for field in ["steward", "repo"] {
|
||||
if let Some(shown) = steward.get(field).and_then(scalar) {
|
||||
let _ = write!(line, " {field}={shown}");
|
||||
}
|
||||
}
|
||||
} else if key != "title"
|
||||
&& key != "label"
|
||||
&& let Some(shown) = scalar(value)
|
||||
{
|
||||
|
|
@ -137,6 +147,68 @@ pub fn source_hits(found: &SourceHits) -> String {
|
|||
")
|
||||
}
|
||||
|
||||
/// Chat hits as the CLI prints them. Per hit the organ's own header line,
|
||||
/// `#alias (!room) first..last time..time score` (the alias and the score only
|
||||
/// when there are some), then one `time sender event_id body` line per message,
|
||||
/// the body as the room held it with line breaks written `\n`: quoted data
|
||||
/// under our header, never a message framed as if sent to the reader. A blank line between hits; a
|
||||
/// truncated listing ends with the `--after` that continues it.
|
||||
pub fn chat_hits(found: &ChatHits) -> String {
|
||||
let mut out: Vec<String> = found
|
||||
.hits
|
||||
.iter()
|
||||
.map(|hit| {
|
||||
let room = hit.room_alias.as_ref().map_or_else(|| hit.room_id.clone(), |alias| format!("{alias} ({})", hit.room_id));
|
||||
let score = hit.score.map_or_else(String::new, |score| format!(" {score:.3}"));
|
||||
let mut text = format!(
|
||||
"{room} {}..{} {}..{}{score}\n",
|
||||
hit.first_event_id,
|
||||
hit.last_event_id,
|
||||
crate::chat::shown(hit.ts_start_ms),
|
||||
crate::chat::shown(hit.ts_end_ms)
|
||||
);
|
||||
for message in &hit.messages {
|
||||
let _ = writeln!(text, "{} {} {} {}", crate::chat::shown(message.ts_ms), message.sender, message.event_id, one_line(&message.body));
|
||||
}
|
||||
text
|
||||
})
|
||||
.collect();
|
||||
if found.truncated
|
||||
&& let Some(last) = found.hits.last()
|
||||
{
|
||||
out.push(format!("truncated: more follow, continue with --after {}\n", crate::chat::shown(last.ts_end_ms + 1)));
|
||||
}
|
||||
out.join("\n")
|
||||
}
|
||||
|
||||
/// A message body on one line: a line break is written `\n` (and a carriage
|
||||
/// return `\r`), so a body cannot start a line of its own. `--json` has the
|
||||
/// body exactly.
|
||||
fn one_line(body: &str) -> String {
|
||||
body.replace('\r', "\\r").replace('\n', "\\n")
|
||||
}
|
||||
|
||||
/// A chat window as the CLI prints it: `#alias (!room)` (the id alone with no
|
||||
/// alias), then one `time sender event_id body` line per message, the anchor
|
||||
/// marked `>` and the rest indented to match; a line says when messages lie
|
||||
/// beyond either end.
|
||||
pub fn chat_window(found: &ChatWindow) -> String {
|
||||
let room = found.room_alias.as_ref().map_or_else(|| found.room_id.clone(), |alias| format!("{alias} ({})", found.room_id));
|
||||
let mut out = format!("{room}\n");
|
||||
if found.more_before {
|
||||
out.push_str(" (earlier messages exist)\n");
|
||||
}
|
||||
for message in &found.messages {
|
||||
let mark = if message.event_id == found.anchor_event_id { '>' } else { ' ' };
|
||||
let _ = writeln!(out, "{mark} {} {} {} {}", crate::chat::shown(message.ts_ms), message.sender, message.event_id, one_line(&message.body));
|
||||
}
|
||||
if found.more_after {
|
||||
out.push_str(" (later messages exist)\n");
|
||||
}
|
||||
out.pop();
|
||||
out
|
||||
}
|
||||
|
||||
/// A fact as one token: a string, a number, a flag, a reference's id, or a
|
||||
/// list of those joined by commas. Anything deeper is not a header fact.
|
||||
fn scalar(value: &Value) -> Option<String> {
|
||||
|
|
@ -256,6 +328,32 @@ two
|
|||
assert_eq!(serde_json::from_str::<SourceHits>(&json).unwrap(), found, "--json round-trips");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_steward_ruling_header_names_the_decider_and_her_stewardship_but_not_her_words() {
|
||||
use huginn_mind::eureka_pipeline::{OrgRepo, PipelineRuling, RulingAuthority};
|
||||
let mut view = viewed(None, PipelineStatus::InForce);
|
||||
view.document = PipelineDocument::Ruling(PipelineRuling {
|
||||
campaign: Slug("c".into()),
|
||||
label: "R1".into(),
|
||||
title: "Zeros".into(),
|
||||
answers: None,
|
||||
choice: None,
|
||||
ruling: "Refuse them.".into(),
|
||||
operator_quote: None,
|
||||
ruled_on: Date("2026-11-11".into()),
|
||||
precedents: vec![],
|
||||
authority: RulingAuthority::Steward {
|
||||
steward: "@heimdall:chat".into(),
|
||||
repo: OrgRepo("GameCult/Bifrost".into()),
|
||||
words: "refuse leading zeros".into(),
|
||||
event: None,
|
||||
},
|
||||
});
|
||||
let line = header(&PipelineDocumentSummary::of(&view, &|_| None));
|
||||
assert!(line.contains("authority=Steward steward=@heimdall:chat repo=GameCult/Bifrost"), "{line}");
|
||||
assert!(!line.contains("zeros"), "{line}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_fact_is_one_token_or_nothing() {
|
||||
assert_eq!(scalar(&json!("")), None);
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ fn run(env: &[(&str, &str)], args: &[&str], stdin: Option<&str>) -> Run {
|
|||
command
|
||||
.env_remove("EUREKA_INSTANCE")
|
||||
.env_remove("HUGINN_ENDPOINT")
|
||||
.env_remove("HUGINN_SESSION")
|
||||
.envs(env.iter().copied())
|
||||
.args(args)
|
||||
.stdin(if stdin.is_some() { Stdio::piped() } else { Stdio::null() })
|
||||
|
|
@ -1232,7 +1233,8 @@ struct Desk {
|
|||
embeds_under: &'static str,
|
||||
held: Vec<huginn_daemon::sources::SourcePayload>,
|
||||
tickets: u64,
|
||||
done: Vec<(SearchTicket, Result<huginn_mind::SourceHits, String>)>,
|
||||
chat: Vec<huginn_daemon::sources::ChatPayload>,
|
||||
done: Vec<(SearchTicket, huginn_daemon::daemon::Found)>,
|
||||
}
|
||||
|
||||
/// An embedder that sends every text to one vector, under a digest.
|
||||
|
|
@ -1296,6 +1298,55 @@ impl huginn_daemon::sources::SourceIndex for Desk {
|
|||
.map(|payload| huginn_daemon::sources::SourceRow { score: 0.9, payload: payload.clone() })
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn upsert_chat(&mut self, _points: &[huginn_daemon::sources::ChatPoint]) -> anyhow::Result<()> {
|
||||
anyhow::bail!("a daemon never writes the source index")
|
||||
}
|
||||
|
||||
fn chat_tail(&mut self, _room_id: &str) -> anyhow::Result<Option<huginn_daemon::sources::StoredChat>> {
|
||||
anyhow::bail!("a daemon never reads the ingest job's side of the source index")
|
||||
}
|
||||
|
||||
fn chat_chunks_holding(&mut self, _room_id: &str, _event_ids: &[String]) -> anyhow::Result<Vec<huginn_daemon::sources::StoredChat>> {
|
||||
anyhow::bail!("a daemon never reads the ingest job's side of the source index")
|
||||
}
|
||||
|
||||
fn chat_chunks_overlapping(&mut self, _room_id: &str, _from_ms: u64, _to_ms: u64) -> anyhow::Result<Vec<huginn_daemon::sources::StoredChat>> {
|
||||
anyhow::bail!("a daemon never reads the ingest job's side of the source index")
|
||||
}
|
||||
|
||||
fn delete_chat(&mut self, _room_id: &str, _ids: Option<&[String]>) -> anyhow::Result<()> {
|
||||
anyhow::bail!("a daemon never writes the source index")
|
||||
}
|
||||
|
||||
fn chat_nearest(
|
||||
&mut self,
|
||||
_vector: &[f32],
|
||||
filter: &huginn_daemon::sources::ChatFilter,
|
||||
limit: u32,
|
||||
_within: Duration,
|
||||
) -> anyhow::Result<Vec<huginn_daemon::sources::ChatRow>> {
|
||||
Ok(self.chat_scroll(filter, huginn_daemon::sources::ChatOrder::OldestFirst, limit, _within)?.into_iter().map(|payload| huginn_daemon::sources::ChatRow { score: 0.9, payload }).collect())
|
||||
}
|
||||
|
||||
fn chat_scroll(&mut self, filter: &huginn_daemon::sources::ChatFilter, order: huginn_daemon::sources::ChatOrder, limit: u32, _within: Duration) -> anyhow::Result<Vec<huginn_daemon::sources::ChatPayload>> {
|
||||
let mut kept: Vec<_> = self
|
||||
.chat
|
||||
.iter()
|
||||
.filter(|payload| {
|
||||
let times = payload.messages.iter().map(|message| message.ts_ms);
|
||||
filter.event_id.as_ref().is_none_or(|id| payload.messages.iter().any(|message| &message.event_id == id))
|
||||
&& filter.keeps(&payload.room_id, payload.room_alias.as_deref(), times.clone().min().unwrap(), times.max().unwrap())
|
||||
})
|
||||
.cloned()
|
||||
.collect();
|
||||
kept.sort_by_key(|payload| payload.messages[0].ts_ms);
|
||||
if order == huginn_daemon::sources::ChatOrder::NewestFirst {
|
||||
kept.reverse();
|
||||
}
|
||||
kept.truncate(limit as usize);
|
||||
Ok(kept)
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: MindStore> IndexSink<S> for Desk {
|
||||
|
|
@ -1312,11 +1363,32 @@ impl<S: MindStore> IndexSink<S> for Desk {
|
|||
self.tickets += 1;
|
||||
let ticket = SearchTicket(self.tickets);
|
||||
let mut embedder = Flat(self.embeds_under);
|
||||
let found = huginn_daemon::sources::search(&mut embedder, self, query, Duration::from_secs(5)).map_err(|error| format!("{error:#}"));
|
||||
let found = huginn_daemon::sources::search(&mut embedder, self, query, Duration::from_secs(5))
|
||||
.map_or_else(|error| huginn_daemon::daemon::Found::Failed(format!("{error:#}")), huginn_daemon::daemon::Found::Sources);
|
||||
self.done.push((ticket, found));
|
||||
Ok(ticket)
|
||||
}
|
||||
fn sources_searched(&mut self) -> Vec<(SearchTicket, Result<huginn_mind::SourceHits, String>)> {
|
||||
fn search_chat(&mut self, query: &huginn_mind::ChatQuery) -> Result<SearchTicket, String> {
|
||||
self.tickets += 1;
|
||||
let ticket = SearchTicket(self.tickets);
|
||||
let mut embedder = Flat(self.embeds_under);
|
||||
let found = huginn_daemon::sources::search_chat(&mut embedder, self, query, Duration::from_secs(5))
|
||||
.map_or_else(|error| huginn_daemon::daemon::Found::Failed(format!("{error:#}")), huginn_daemon::daemon::Found::Chat);
|
||||
self.done.push((ticket, found));
|
||||
Ok(ticket)
|
||||
}
|
||||
fn chat_window(&mut self, query: &huginn_mind::ChatWindowQuery) -> Result<SearchTicket, String> {
|
||||
self.tickets += 1;
|
||||
let ticket = SearchTicket(self.tickets);
|
||||
let found = match huginn_daemon::sources::window(self, query, Duration::from_secs(5)) {
|
||||
Ok(Ok(window)) => huginn_daemon::daemon::Found::Window(window),
|
||||
Ok(Err(refusal)) => huginn_daemon::daemon::Found::Refused(refusal),
|
||||
Err(error) => huginn_daemon::daemon::Found::Failed(format!("{error:#}")),
|
||||
};
|
||||
self.done.push((ticket, found));
|
||||
Ok(ticket)
|
||||
}
|
||||
fn sources_searched(&mut self) -> Vec<(SearchTicket, huginn_daemon::daemon::Found)> {
|
||||
std::mem::take(&mut self.done)
|
||||
}
|
||||
fn search_deadline(&self) -> Duration {
|
||||
|
|
@ -1350,6 +1422,15 @@ fn source_daemon(built_under: &'static str, embeds_under: &'static str) -> Serve
|
|||
chunk("Demo", "README.md", 1, 2, "# Demo"),
|
||||
chunk("Other", "x.rs", 1, 1, "fn other() {}"),
|
||||
],
|
||||
chat: vec![huginn_daemon::sources::ChatPayload {
|
||||
room_id: "!r:gamecult.org".into(),
|
||||
room_alias: Some("#swarm:gamecult.org".into()),
|
||||
messages: vec![
|
||||
huginn_mind::ChatMessage { event_id: "$e1".into(), sender: "@nibu:gamecult.org".into(), ts_ms: 1_700_000_000_000, body: "ship the cut\nthen rest".into() },
|
||||
huginn_mind::ChatMessage { event_id: "$e2".into(), sender: "@eureka:gamecult.org".into(), ts_ms: 1_700_000_060_000, body: "done".into() },
|
||||
],
|
||||
text: "swarm".into(),
|
||||
}],
|
||||
tickets: 0,
|
||||
done: Vec::new(),
|
||||
};
|
||||
|
|
@ -1382,6 +1463,31 @@ fn sources_front_prints_only_the_front_page_with_its_lines() {
|
|||
assert_eq!(found.out.trim_end(), "Demo:README.md:1-2 @abcdef1 0.900\n# Demo", "{}", found.out);
|
||||
}
|
||||
|
||||
/// `huginn sources <TEXT>...` sends every text in `texts` and none in `text`, the
|
||||
/// daemon fuses them into one answer, and a ninth or no text never leaves the CLI.
|
||||
#[test]
|
||||
fn sources_sends_each_text_as_a_topic_and_prints_the_fused_answer() {
|
||||
let (server, seen) = replying(
|
||||
"search_sources",
|
||||
HuginnMindResponse::Sources(huginn_mind::SourceHits { indexed_under: huginn_mind::SourceModel { name: "m".into(), digest: "d".into() }, hits: vec![] }),
|
||||
);
|
||||
let ran = at(server.addr, &["sources", "qdrant filter", "reciprocal rank", "--top-k", "5"], None);
|
||||
assert_eq!(ran.code, 0, "{}", ran.err);
|
||||
let HuginnMindRequest::SearchSources { query, .. } = seen.lock().unwrap().remove(0) else { panic!("a source search") };
|
||||
let texts: Vec<&str> = query.texts.iter().map(|text| text.0.as_str()).collect();
|
||||
assert_eq!((texts, query.text.0.as_str(), query.top_k), (vec!["qdrant filter", "reciprocal rank"], "", 5));
|
||||
|
||||
let daemon = source_daemon("d1", "d1");
|
||||
let found = at(daemon.addr, &["sources", "cursor", "readme", "--repo", "Demo"], None);
|
||||
assert_eq!(found.code, 0, "{}", found.err);
|
||||
assert!(found.out.contains("Demo:src/lib.rs:3-9") && found.out.contains("Demo:README.md:1-2"), "{}", found.out);
|
||||
|
||||
let nine = ["sources", "a", "b", "c", "d", "e", "f", "g", "h", "i"];
|
||||
assert_ne!(at(server.addr, &nine, None).code, 0);
|
||||
assert_ne!(at(server.addr, &["sources"], None).code, 0);
|
||||
assert!(seen.lock().unwrap().is_empty(), "a refused command asked nothing");
|
||||
}
|
||||
|
||||
/// A daemon whose embedder is not the model the index was built under refuses
|
||||
/// the search naming both, and the CLI exits 1 with that word.
|
||||
#[test]
|
||||
|
|
@ -2782,3 +2888,152 @@ fn import_wrong_endpoint_is_foreign_instance() {
|
|||
assert_eq!(own.code, 2, "{}{}", own.out, own.err);
|
||||
assert!(own.err.starts_with("InvalidInput: --from heimdall is this process's own instance"), "{}", own.err);
|
||||
}
|
||||
|
||||
/// Configured for the test instance against a daemon at `addr`, with `HUGINN_SESSION` as given.
|
||||
fn with_session(addr: SocketAddr, session: &str, args: &[&str], stdin: Option<&str>) -> Run {
|
||||
let endpoint = format!("rudp://{addr}");
|
||||
run(&[("EUREKA_INSTANCE", INSTANCE), ("HUGINN_ENDPOINT", &endpoint), ("HUGINN_SESSION", session)], args, stdin)
|
||||
}
|
||||
|
||||
fn admit_without_session(file: &str) -> Vec<&str> {
|
||||
vec!["admit", "--faculty", "Hands", "--agent", "claude", file]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_session_is_the_flag_else_huginn_session_else_a_refusal() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let file = write(&dir, "campaign.json", &json!([campaign_json()]));
|
||||
let server = live();
|
||||
|
||||
// Neither: refused before any send, nothing admitted.
|
||||
let neither = at(server.addr, &admit_without_session(&file), None);
|
||||
assert_eq!(neither.code, 2, "{}", neither.err);
|
||||
assert!(neither.err.starts_with("InvalidInput: --session is required"), "{}", neither.err);
|
||||
assert_eq!(neither.out, "");
|
||||
let count = at(server.addr, &["whoami"], None);
|
||||
assert!(count.out.contains("receipts 2 "), "{}", count.out);
|
||||
|
||||
// An empty variable is absent.
|
||||
let empty = with_session(server.addr, "", &admit_without_session(&file), None);
|
||||
assert_eq!(empty.code, 2, "{}", empty.err);
|
||||
assert!(empty.err.starts_with("InvalidInput: --session is required"), "{}", empty.err);
|
||||
assert!(at(server.addr, &["whoami"], None).out.contains("receipts 2 "));
|
||||
|
||||
// The variable alone: the receipt carries it.
|
||||
let env_only = with_session(server.addr, "mind-20261111T072944197Z", &admit_without_session(&file), None);
|
||||
assert_eq!(env_only.code, 0, "{}", env_only.err);
|
||||
let viewed = at(server.addr, &["view", &format!("{CAMPAIGN}:campaign:self")], None);
|
||||
assert!(viewed.out.contains(" Hands/claude session mind-20261111T072944197Z tool huginn "), "{}", viewed.out);
|
||||
assert!(at(server.addr, &["whoami"], None).out.contains("receipts 3 "));
|
||||
|
||||
// The persona door takes it the same way.
|
||||
let persona_file = write(&dir, "persona.json", &persona(INSTANCE, "2026-10-10T10:00:00Z"));
|
||||
let put = with_session(server.addr, "mind-run-p", &["persona", "put", "--faculty", "SelfFaculty", "--agent", "claude", &persona_file], None);
|
||||
assert_eq!(put.code, 0, "{}", put.err);
|
||||
let receipt = put.out.strip_prefix("Committed ").and_then(|rest| rest.strip_suffix('\n')).expect(&put.out);
|
||||
let got = at(server.addr, &["persona", "get"], None);
|
||||
assert!(got.out.contains(&format!("receipt {receipt}")), "{}", got.out);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_flag_wins_over_huginn_session() {
|
||||
let (server, seen) = replying("admit", HuginnMindResponse::Admit(PipelineAdmissionOutcome::Committed { receipt_id: "mind-commit-x".into(), committed_at: "2026-11-11T00:00:00Z".into(), writes: vec![] }));
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let file = write(&dir, "campaign.json", &json!([campaign_json()]));
|
||||
let _ = with_session(server.addr, "from-env", &admit_args(&file), None);
|
||||
let seen = seen.lock().unwrap();
|
||||
let sessions: Vec<&str> = seen
|
||||
.iter()
|
||||
.filter_map(|request| if let HuginnMindRequest::Admit(batch) = request { Some(batch.provenance.session.0.as_str()) } else { None })
|
||||
.collect();
|
||||
assert_eq!(sessions, ["session-7"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_malformed_huginn_session_is_refused_naming_the_variable_without_echoing_it() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let file = write(&dir, "campaign.json", &json!([campaign_json()]));
|
||||
let (server, count) = counting(HuginnMindResponse::View(None), "view");
|
||||
let long = "L".repeat(201);
|
||||
let ran = with_session(server.addr, &long, &admit_without_session(&file), None);
|
||||
assert_eq!(ran.code, 2, "{}", ran.err);
|
||||
assert!(ran.err.starts_with("InvalidInput: HUGINN_SESSION is longer than 200 bytes"), "{}", ran.err);
|
||||
assert!(!ran.err.contains("LLLL"), "{}", ran.err);
|
||||
assert_eq!(*count.lock().unwrap(), 0, "nothing was sent");
|
||||
|
||||
// The bound itself is allowed.
|
||||
let live = live();
|
||||
let edge = with_session(live.addr, &"E".repeat(200), &admit_without_session(&file), None);
|
||||
assert_eq!(edge.code, 0, "{}", edge.err);
|
||||
}
|
||||
|
||||
/// `huginn chat search` end to end: a listing through the daemon's worker path
|
||||
/// prints the organ's header line and each message verbatim with its event id,
|
||||
/// a room that holds nothing prints nothing, and the flags reach the request as
|
||||
/// the instants and rooms they name.
|
||||
#[test]
|
||||
fn chat_search_prints_cited_messages_and_sends_its_selectors() {
|
||||
let server = source_daemon("d1", "d1");
|
||||
let found = at(server.addr, &["chat", "search", "--room", "#swarm:gamecult.org", "--after", "2023-11-14T22:00:00Z"], None);
|
||||
assert_eq!(found.code, 0, "{}", found.err);
|
||||
assert_eq!(
|
||||
found.out,
|
||||
"#swarm:gamecult.org (!r:gamecult.org) $e1..$e2 2023-11-14T22:13:20.000Z..2023-11-14T22:14:20.000Z\n\
|
||||
2023-11-14T22:13:20.000Z @nibu:gamecult.org $e1 ship the cut\\nthen rest\n\
|
||||
2023-11-14T22:14:20.000Z @eureka:gamecult.org $e2 done\n\n"
|
||||
);
|
||||
let other = at(server.addr, &["chat", "search", "--room", "!elsewhere:gamecult.org"], None);
|
||||
assert_eq!((other.code, other.out.trim()), (0, ""));
|
||||
let by_meaning = at(server.addr, &["chat", "search", "who shipped it", "--top-k", "1"], None);
|
||||
assert!(by_meaning.out.starts_with("#swarm:gamecult.org (!r:gamecult.org) $e1..$e2 ") && by_meaning.out.lines().next().unwrap().ends_with(" 0.900"), "{}", by_meaning.out);
|
||||
|
||||
let (spy, seen) = replying(
|
||||
"search_chat",
|
||||
HuginnMindResponse::Chat(huginn_mind::ChatHits { indexed_under: huginn_mind::SourceModel { name: "m".into(), digest: "d".into() }, truncated: false, hits: vec![] }),
|
||||
);
|
||||
let ran = at(spy.addr, &["chat", "search", "a", "b", "--room", "!x:y", "--room", "#z:y", "--after", "2023-11-14T22:13:20Z", "--before", "2023-11-14T22:14:20Z", "--top-k", "3"], None);
|
||||
assert_eq!(ran.code, 0, "{}", ran.err);
|
||||
let HuginnMindRequest::SearchChat { query, .. } = seen.lock().unwrap().remove(0) else { panic!("a chat search") };
|
||||
let texts: Vec<&str> = query.texts.iter().map(|text| text.0.as_str()).collect();
|
||||
let rooms: Vec<&str> = query.rooms.iter().map(|room| room.0.as_str()).collect();
|
||||
assert_eq!((texts, rooms, query.after_ms, query.before_ms, query.top_k), (vec!["a", "b"], vec!["!x:y", "#z:y"], Some(1_700_000_000_000), Some(1_700_000_060_000), 3));
|
||||
|
||||
assert_ne!(at(spy.addr, &["chat", "search", "--after", "yesterday"], None).code, 0);
|
||||
assert!(seen.lock().unwrap().is_empty(), "a bad instant never left the CLI");
|
||||
for bad in [&["chat", "search", "--room", "swarm"][..], &["chat", "search", "--top-k", "0"]] {
|
||||
let ran = at(server.addr, bad, None);
|
||||
assert_eq!(ran.code, 1, "{bad:?} is a refusal by the daemon's door: {}", ran.err);
|
||||
}
|
||||
}
|
||||
|
||||
/// `huginn chat window` end to end: an unknown or malformed event id is a
|
||||
/// refusal naming the field; a known one prints the room header and the
|
||||
/// messages with the anchor marked `>`; the flags reach the request.
|
||||
#[test]
|
||||
fn chat_window_prints_the_anchor_marked_and_refuses_what_the_index_lacks() {
|
||||
let server = source_daemon("d1", "d1");
|
||||
for (bad, field) in [("$unknown:gamecult.org", "event_id"), ("e1", "event_id")] {
|
||||
let ran = at(server.addr, &["chat", "window", bad], None);
|
||||
assert_eq!(ran.code, 1, "{}", ran.err);
|
||||
assert!(ran.out.starts_with("Refused SelectionInvalid") && ran.out.contains(field), "no window for an id the index lacks: {}", ran.out);
|
||||
}
|
||||
let found = at(server.addr, &["chat", "window", "$e2"], None);
|
||||
assert_eq!(found.code, 0, "{}", found.err);
|
||||
assert_eq!(
|
||||
found.out,
|
||||
"#swarm:gamecult.org (!r:gamecult.org)\n\
|
||||
\x20 2023-11-14T22:13:20.000Z @nibu:gamecult.org $e1 ship the cut\\nthen rest\n\
|
||||
> 2023-11-14T22:14:20.000Z @eureka:gamecult.org $e2 done\n"
|
||||
);
|
||||
let over = at(server.addr, &["chat", "window", "$e2", "--before", "51"], None);
|
||||
assert_eq!(over.code, 1, "{}", over.err);
|
||||
|
||||
let (spy, seen) = replying(
|
||||
"chat_window",
|
||||
HuginnMindResponse::ChatWindow(huginn_mind::ChatWindow { room_id: "!r:x".into(), room_alias: None, anchor_event_id: "$a".into(), messages: vec![], more_before: false, more_after: false }),
|
||||
);
|
||||
let ran = at(spy.addr, &["chat", "window", "$a", "--before", "3", "--after", "0"], None);
|
||||
assert_eq!(ran.code, 0, "{}", ran.err);
|
||||
let HuginnMindRequest::ChatWindow { query, .. } = seen.lock().unwrap().remove(0) else { panic!("a chat window") };
|
||||
assert_eq!((query.event_id.0.as_str(), query.before, query.after), ("$a", 3, 0));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -93,10 +93,18 @@ impl<S: MindStore> IndexSink<S> for Inert {
|
|||
fn search(&mut self, _text: &str, _top_k: u32, _among: Vec<String>) -> Result<SearchTicket, String> {
|
||||
Err("this index cannot search".into())
|
||||
}
|
||||
fn search_chat(&mut self, _query: &huginn_mind::ChatQuery) -> Result<SearchTicket, String> {
|
||||
Err("this index cannot search".into())
|
||||
}
|
||||
|
||||
fn chat_window(&mut self, _query: &huginn_mind::ChatWindowQuery) -> Result<SearchTicket, String> {
|
||||
Err("this index cannot search".into())
|
||||
}
|
||||
|
||||
fn search_sources(&mut self, _query: &huginn_mind::SourceQuery) -> Result<SearchTicket, String> {
|
||||
Err("this index cannot search".into())
|
||||
}
|
||||
fn sources_searched(&mut self) -> Vec<(SearchTicket, Result<huginn_mind::SourceHits, String>)> {
|
||||
fn sources_searched(&mut self) -> Vec<(SearchTicket, huginn_daemon::daemon::Found)> {
|
||||
Vec::new()
|
||||
}
|
||||
fn search_deadline(&self) -> Duration {
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ use crate::sources::forge::HttpForge;
|
|||
use huginn_mind::desk_wire::{DeskRequest, DeskResponse};
|
||||
use huginn_mind::eureka_pipeline::{PipelineRef, Slug};
|
||||
use huginn_mind::wire::{ACTIVITY_REPOS_MAX, HuginnMindRequest, HuginnMindResponse, IndexStatus};
|
||||
use huginn_mind::{Members, Mind, MindRefusal, MindStore, PipelineAdmissionOutcome, SemanticQuery, SourceHits, SourceQuery};
|
||||
use huginn_mind::{ChatHits, ChatQuery, ChatWindow, ChatWindowQuery, Members, Mind, MindRefusal, MindStore, PipelineAdmissionOutcome, SemanticQuery, SourceHits, SourceQuery};
|
||||
|
||||
/// Where the index attaches. Called after a batch commits, with the refs the
|
||||
/// batch landed, derived writes included; the sink may read each through
|
||||
|
|
@ -34,8 +34,14 @@ pub trait IndexSink<S: MindStore> {
|
|||
/// Starts a source search. The source index is not the mind's: it needs no
|
||||
/// member ids, and nothing about the mind's own index state refuses it.
|
||||
fn search_sources(&mut self, query: &SourceQuery) -> Result<SearchTicket, String>;
|
||||
/// Every source search that finished since the last call, by ticket.
|
||||
fn sources_searched(&mut self) -> Vec<(SearchTicket, Result<SourceHits, String>)>;
|
||||
/// Starts a chat search: the same queue and worker as a source search,
|
||||
/// over the same collection.
|
||||
fn search_chat(&mut self, query: &ChatQuery) -> Result<SearchTicket, String>;
|
||||
/// Starts a chat window: the same queue and worker, no embedding.
|
||||
fn chat_window(&mut self, query: &ChatWindowQuery) -> Result<SearchTicket, String>;
|
||||
/// Every source or chat search that finished since the last call, by
|
||||
/// ticket: `Found::Sources`, `Found::Chat` or `Found::Failed`.
|
||||
fn sources_searched(&mut self) -> Vec<(SearchTicket, Found)>;
|
||||
fn search_deadline(&self) -> std::time::Duration;
|
||||
/// Every search that finished since the last call, by ticket: its
|
||||
/// candidates and scores, or why it failed.
|
||||
|
|
@ -78,6 +84,10 @@ struct MindSearch {
|
|||
pub enum Found {
|
||||
Mind(Hits),
|
||||
Sources(SourceHits),
|
||||
Chat(ChatHits),
|
||||
Window(ChatWindow),
|
||||
/// The index answered with a typed refusal rather than failing.
|
||||
Refused(MindRefusal),
|
||||
Failed(String),
|
||||
}
|
||||
|
||||
|
|
@ -205,6 +215,24 @@ impl<S: MindStore, I: IndexSink<S>> Daemon<S, I> {
|
|||
Ok(roots) => HuginnMindResponse::RootsTouched(roots),
|
||||
Err(refusal) => HuginnMindResponse::Refused(refusal),
|
||||
},
|
||||
HuginnMindRequest::SearchChat { query, .. } => {
|
||||
if let Err(refusal) = query.check() {
|
||||
return Handled::Answered(HuginnMindResponse::Refused(refusal));
|
||||
}
|
||||
return match self.index.search_chat(&query) {
|
||||
Ok(ticket) => Handled::Searching(Search { ticket, mind: None }),
|
||||
Err(detail) => Handled::Answered(HuginnMindResponse::Refused(MindRefusal::Unavailable { detail })),
|
||||
};
|
||||
}
|
||||
HuginnMindRequest::ChatWindow { query, .. } => {
|
||||
if let Err(refusal) = query.check() {
|
||||
return Handled::Answered(HuginnMindResponse::Refused(refusal));
|
||||
}
|
||||
return match self.index.chat_window(&query) {
|
||||
Ok(ticket) => Handled::Searching(Search { ticket, mind: None }),
|
||||
Err(detail) => Handled::Answered(HuginnMindResponse::Refused(MindRefusal::Unavailable { detail })),
|
||||
};
|
||||
}
|
||||
HuginnMindRequest::StewardedActivity { instance, repos, after } => {
|
||||
let refused = |message: &str| MindRefusal::SelectionInvalid { field: "repos".into(), value: None, message: message.into() };
|
||||
if repos.len() > ACTIVITY_REPOS_MAX {
|
||||
|
|
@ -235,8 +263,8 @@ impl<S: MindStore, I: IndexSink<S>> Daemon<S, I> {
|
|||
/// One desk request, answered by the Mind method it names. The Mind is the instance's own
|
||||
/// (`mind` is checked against it); `deciders` is this call's list, handed to the Mind and
|
||||
/// dropped. A desk answer is not indexed (`index_text` gives it no text), so nothing here tells the index.
|
||||
/// The reads answer with the whole record and nothing here gates them (ruling
|
||||
/// `desk-read-visibility-whole-record`): they are handed `deciders` and not `subject`.
|
||||
/// The reads are handed `subject` and `deciders`: the Mind builds a decider's view or a
|
||||
/// non-decider's (ruling `desk-read-visibility-withheld-ids`).
|
||||
pub fn handle_desk(&mut self, request: DeskRequest, now: DateTime<Utc>) -> DeskResponse {
|
||||
if let Err(refusal) = self.mind.require_instance(request.mind()) {
|
||||
return DeskResponse::Refused(refusal);
|
||||
|
|
@ -245,8 +273,8 @@ impl<S: MindStore, I: IndexSink<S>> Daemon<S, I> {
|
|||
DeskRequest::Answer { subject, authority, deciders, request, .. } => {
|
||||
self.mind.desk_answer(&deciders, subject, authority, request, now)
|
||||
}
|
||||
DeskRequest::Forks { deciders, status, cursor, limit, .. } => self.mind.desk_forks(&deciders, status, cursor, limit),
|
||||
DeskRequest::Fork { deciders, fork_id, .. } => self.mind.desk_fork(&deciders, &fork_id),
|
||||
DeskRequest::Forks { subject, deciders, status, cursor, limit, .. } => self.mind.desk_forks(&subject, &deciders, status, cursor, limit),
|
||||
DeskRequest::Fork { subject, deciders, fork_id, .. } => self.mind.desk_fork(&subject, &deciders, &fork_id),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -282,8 +310,7 @@ impl<S: MindStore, I: IndexSink<S>> Daemon<S, I> {
|
|||
/// Searches the index has finished since this was last called.
|
||||
pub fn searched(&mut self) -> Vec<(SearchTicket, Found)> {
|
||||
let mind = self.index.searched().into_iter().map(|(ticket, found)| (ticket, found.map_or_else(Found::Failed, Found::Mind)));
|
||||
let sources = self.index.sources_searched().into_iter().map(|(ticket, found)| (ticket, found.map_or_else(Found::Failed, Found::Sources)));
|
||||
mind.chain(sources).collect()
|
||||
mind.chain(self.index.sources_searched()).collect()
|
||||
}
|
||||
|
||||
/// The client has stopped waiting for `ticket`.
|
||||
|
|
@ -300,6 +327,9 @@ impl<S: MindStore, I: IndexSink<S>> Daemon<S, I> {
|
|||
let (selection, semantic, members, hits) = match (search.mind, found) {
|
||||
(_, Found::Failed(detail)) => return unavailable(detail),
|
||||
(None, Found::Sources(hits)) => return HuginnMindResponse::Sources(hits),
|
||||
(None, Found::Chat(hits)) => return HuginnMindResponse::Chat(hits),
|
||||
(None, Found::Window(window)) => return HuginnMindResponse::ChatWindow(window),
|
||||
(None, Found::Refused(refusal)) => return HuginnMindResponse::Refused(refusal),
|
||||
(Some(asked), Found::Mind(hits)) => (asked.selection, asked.semantic, asked.members, hits),
|
||||
_ => return unavailable("the index answered another kind of search than the one asked".into()),
|
||||
};
|
||||
|
|
@ -326,6 +356,7 @@ pub(crate) mod tests {
|
|||
StructuralDelta, Title, VerificationTest,
|
||||
};
|
||||
use huginn_mind::wire::MindStatus;
|
||||
use huginn_mind::ChatQuery;
|
||||
use huginn_mind::{MindRefusal, OwnedRedbMessagePackBackingStore};
|
||||
use std::path::Path;
|
||||
use cultnet_rs::Selection;
|
||||
|
|
@ -580,11 +611,19 @@ pub(crate) mod tests {
|
|||
Err("this index cannot search".into())
|
||||
}
|
||||
|
||||
fn search_chat(&mut self, _query: &ChatQuery) -> Result<SearchTicket, String> {
|
||||
Err("this index cannot search".into())
|
||||
}
|
||||
|
||||
fn chat_window(&mut self, _query: &ChatWindowQuery) -> Result<SearchTicket, String> {
|
||||
Err("this index cannot search".into())
|
||||
}
|
||||
|
||||
fn search_sources(&mut self, _query: &SourceQuery) -> Result<SearchTicket, String> {
|
||||
Err("this index cannot search".into())
|
||||
}
|
||||
|
||||
fn sources_searched(&mut self) -> Vec<(SearchTicket, Result<SourceHits, String>)> {
|
||||
fn sources_searched(&mut self) -> Vec<(SearchTicket, Found)> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
|
|
@ -619,13 +658,21 @@ pub(crate) mod tests {
|
|||
Err("this index cannot search the mind".into())
|
||||
}
|
||||
|
||||
fn search_chat(&mut self, _query: &ChatQuery) -> Result<SearchTicket, String> {
|
||||
Err("this index cannot search".into())
|
||||
}
|
||||
|
||||
fn chat_window(&mut self, _query: &ChatWindowQuery) -> Result<SearchTicket, String> {
|
||||
Err("this index cannot search".into())
|
||||
}
|
||||
|
||||
fn search_sources(&mut self, _query: &SourceQuery) -> Result<SearchTicket, String> {
|
||||
*self.asked.lock().unwrap() += 1;
|
||||
Ok(SearchTicket(7))
|
||||
}
|
||||
|
||||
fn sources_searched(&mut self) -> Vec<(SearchTicket, Result<SourceHits, String>)> {
|
||||
vec![(SearchTicket(7), Ok(self.hits.clone()))]
|
||||
fn sources_searched(&mut self) -> Vec<(SearchTicket, Found)> {
|
||||
vec![(SearchTicket(7), Found::Sources(self.hits.clone()))]
|
||||
}
|
||||
|
||||
fn search_deadline(&self) -> std::time::Duration {
|
||||
|
|
@ -655,7 +702,7 @@ pub(crate) mod tests {
|
|||
let mut daemon = Daemon::new(mind, SourcesIndex { asked: Arc::clone(&asked), hits: hits.clone() });
|
||||
let request = |instance: &str, top_k: u32| HuginnMindRequest::SearchSources {
|
||||
instance: slug(instance),
|
||||
query: SourceQuery { text: huginn_mind::eureka_pipeline::Line("find".into()), top_k, repo: None, front: false },
|
||||
query: SourceQuery { text: huginn_mind::eureka_pipeline::Line("find".into()), texts: vec![], top_k, repo: None, front: false },
|
||||
};
|
||||
|
||||
let Handled::Searching(search) = daemon.handle(request(INSTANCE, 5), now()) else { panic!("a well-formed source search is a search") };
|
||||
|
|
@ -673,7 +720,41 @@ pub(crate) mod tests {
|
|||
let answer = daemon.handle(refused, now()).answered();
|
||||
assert!(matches!(answer, HuginnMindResponse::Refused(_)), "{answer:?}");
|
||||
}
|
||||
assert_eq!(*asked.lock().unwrap(), 2, "a refused query never reached the index");
|
||||
let texts = |texts: &[&str]| HuginnMindRequest::SearchSources {
|
||||
instance: slug(INSTANCE),
|
||||
query: SourceQuery { text: huginn_mind::eureka_pipeline::Line::default(), texts: texts.iter().map(|text| huginn_mind::eureka_pipeline::Line((*text).into())).collect(), top_k: 5, repo: None, front: false },
|
||||
};
|
||||
for refused in [texts(&[]), texts(&["a"; 9]), texts(&["a", " "])] {
|
||||
let answer = daemon.handle(refused, now()).answered();
|
||||
assert!(matches!(answer, HuginnMindResponse::Refused(MindRefusal::SelectionInvalid { ref field, .. }) if field == "texts"), "{answer:?}");
|
||||
}
|
||||
assert!(matches!(daemon.handle(texts(&["a", "b"]), now()), Handled::Searching(_)), "two texts are one queued search");
|
||||
assert_eq!(*asked.lock().unwrap(), 3, "a refused query never reached the index");
|
||||
}
|
||||
|
||||
/// A chat search is checked at the door before the index is asked, is
|
||||
/// refused for another mind, turns an index that cannot be asked into
|
||||
/// `Unavailable`, and its finished hits are the answer as they are.
|
||||
#[test]
|
||||
fn a_chat_search_is_checked_at_the_door_and_answers_its_hits() {
|
||||
use huginn_mind::eureka_pipeline::{Line, Short};
|
||||
use huginn_mind::{ChatHits, SourceModel};
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let mut daemon = Daemon::new(Mind::open(root.path(), &slug(INSTANCE)).unwrap(), NoIndex);
|
||||
let request = |instance: &str, rooms: &[&str], top_k: u32| HuginnMindRequest::SearchChat {
|
||||
instance: slug(instance),
|
||||
query: ChatQuery { texts: vec![Line("find".into())], rooms: rooms.iter().map(|room| Short((*room).into())).collect(), after_ms: None, before_ms: None, top_k },
|
||||
};
|
||||
for (refused, field) in [(request(INSTANCE, &[], 0), "top_k"), (request(INSTANCE, &["swarm"], 5), "rooms")] {
|
||||
let answer = daemon.handle(refused, now()).answered();
|
||||
assert!(matches!(answer, HuginnMindResponse::Refused(MindRefusal::SelectionInvalid { field: ref named, .. }) if named == field), "{answer:?}");
|
||||
}
|
||||
assert!(matches!(daemon.handle(request(OTHER, &[], 5), now()).answered(), HuginnMindResponse::Refused(_)));
|
||||
let answer = daemon.handle(request(INSTANCE, &["#swarm:x"], 5), now()).answered();
|
||||
assert!(matches!(answer, HuginnMindResponse::Refused(MindRefusal::Unavailable { .. })), "{answer:?}");
|
||||
let hits = ChatHits { indexed_under: SourceModel { name: "m".into(), digest: "d".into() }, truncated: false, hits: vec![] };
|
||||
let search = Search { ticket: SearchTicket(1), mind: None };
|
||||
assert_eq!(daemon.finish(search, Found::Chat(hits.clone())), HuginnMindResponse::Chat(hits));
|
||||
}
|
||||
|
||||
/// A daemon over the instance's mind under `state_root`, with no index.
|
||||
|
|
@ -707,11 +788,19 @@ pub(crate) mod tests {
|
|||
Err("this index cannot search".into())
|
||||
}
|
||||
|
||||
fn search_chat(&mut self, _query: &ChatQuery) -> Result<SearchTicket, String> {
|
||||
Err("this index cannot search".into())
|
||||
}
|
||||
|
||||
fn chat_window(&mut self, _query: &ChatWindowQuery) -> Result<SearchTicket, String> {
|
||||
Err("this index cannot search".into())
|
||||
}
|
||||
|
||||
fn search_sources(&mut self, _query: &SourceQuery) -> Result<SearchTicket, String> {
|
||||
Err("this index cannot search".into())
|
||||
}
|
||||
|
||||
fn sources_searched(&mut self) -> Vec<(SearchTicket, Result<SourceHits, String>)> {
|
||||
fn sources_searched(&mut self) -> Vec<(SearchTicket, Found)> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
|
|
@ -741,11 +830,19 @@ pub(crate) mod tests {
|
|||
Err("this index cannot search".into())
|
||||
}
|
||||
|
||||
fn search_chat(&mut self, _query: &ChatQuery) -> Result<SearchTicket, String> {
|
||||
Err("this index cannot search".into())
|
||||
}
|
||||
|
||||
fn chat_window(&mut self, _query: &ChatWindowQuery) -> Result<SearchTicket, String> {
|
||||
Err("this index cannot search".into())
|
||||
}
|
||||
|
||||
fn search_sources(&mut self, _query: &SourceQuery) -> Result<SearchTicket, String> {
|
||||
Err("this index cannot search".into())
|
||||
}
|
||||
|
||||
fn sources_searched(&mut self) -> Vec<(SearchTicket, Result<SourceHits, String>)> {
|
||||
fn sources_searched(&mut self) -> Vec<(SearchTicket, Found)> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,18 +7,18 @@
|
|||
//! bounds: one request per connection, one reply, a header over `DESK_MAX_REQUEST_BYTES` refused
|
||||
//! before anything is allocated, one `DESK_CONNECTION_DEADLINE` for the whole connection (every
|
||||
//! read and write spends what is left of it, so a peer that trickles bytes is cut off at the
|
||||
//! deadline like one that is silent), a reply past `DESK_MAX_RESPONSE_BYTES` refused by name.
|
||||
//! deadline like one that is silent), a reply past `DESK_MAX_RESPONSE_BYTES` refused by name (a read of one
|
||||
//! fork past it is that fork's `fork_too_large`).
|
||||
//!
|
||||
//! The trust boundary of the reads (ruling `desk-read-visibility-whole-record`). The peer-uid check
|
||||
//! The trust boundary of the reads (ruling `desk-read-visibility-withheld-ids`). The peer-uid check
|
||||
//! IS the whole authentication: Huginn must be reachable only by Bifrost's peer-checked socket, so
|
||||
//! the configured uid is Bifrost's and no other process's. `Forks` and `Fork` return the WHOLE record
|
||||
//! (cards, outcomes, every decider's note) to whoever passes that check; nothing in Huginn strips a
|
||||
//! note or gates a read on `subject` or `deciders` (`deciders` only fills the card and the
|
||||
//! showability gate). Bifrost applies visibility at its own door (a stripped read serves `note: null`,
|
||||
//! CultLib `fork-card.md`). A second reader of this socket is therefore a note leak, and a deployment
|
||||
//! that gives the socket to any uid but Bifrost's has no reader check left. The reads carry no
|
||||
//! `authority` (a payload that names one is refused whole), and the door has no other way to ask who
|
||||
//! is reading.
|
||||
//! the configured uid is Bifrost's and no other process's. `Forks` and `Fork` carry every decider's
|
||||
//! note to whoever passes that check, and Bifrost strips notes at its own door (a stripped read
|
||||
//! serves `note: null`, CultLib `fork-card.md`); a second reader of this socket is therefore a note
|
||||
//! leak, and a deployment that gives the socket to any uid but Bifrost's has no reader check left.
|
||||
//! Account ids are Huginn's to withhold: a reader whose `subject` is not in the call's `deciders`
|
||||
//! gets `deciders` empty, `answerer` nil and the counts instead. The reads carry no `authority` (a
|
||||
//! payload that names one is refused whole), and the door has no other way to ask who is reading.
|
||||
//!
|
||||
//! It owns no rule. Which Mind method answers is `Daemon::handle_desk`'s; when the door is
|
||||
//! served is `serve::run`'s, once per pass in the thread that owns the Mind. A peer that is
|
||||
|
|
@ -46,7 +46,7 @@ use cultnet_rs::{
|
|||
};
|
||||
use huginn_mind::desk_wire::{
|
||||
DESK_MAX_REQUEST_BYTES, DESK_MAX_RESPONSE_BYTES, DESK_REQUEST_SCHEMA, DESK_REQUEST_SCHEMA_JSON, DESK_RESPONSE_SCHEMA,
|
||||
DESK_RESPONSE_SCHEMA_JSON, DESK_SERVICE_ID, DeskResponse,
|
||||
DESK_RESPONSE_SCHEMA_JSON, DESK_SERVICE_ID, DeskResponse, fork_too_large_read,
|
||||
};
|
||||
use huginn_mind::envelope::{OperationFailure, decode_desk_request, encode_desk_failure, encode_desk_response};
|
||||
use huginn_mind::{MindRefusal, MindStore};
|
||||
|
|
@ -321,7 +321,11 @@ pub(crate) fn respond(response: &DeskResponse, message_id: &str, operation: &str
|
|||
};
|
||||
bounded(reply, limit, |size| match size {
|
||||
Ok(bytes) => {
|
||||
let refusal = DeskResponse::Refused(MindRefusal::ResponseTooLarge { bytes: bytes as u64, limit: limit as u64 });
|
||||
// A read of one fork past the cap is that fork's refusal; every other reply is the generic one.
|
||||
let refusal = match response {
|
||||
DeskResponse::Fork { entry, .. } => DeskResponse::NotStored(fork_too_large_read(entry.card.fork_id.as_str())),
|
||||
_ => DeskResponse::Refused(MindRefusal::ResponseTooLarge { bytes: bytes as u64, limit: limit as u64 }),
|
||||
};
|
||||
encode_desk_response(message_id, operation, &refusal, runtime_id).unwrap_or_else(|error| not_encodable(format!("{error:#}")))
|
||||
}
|
||||
Err(detail) => not_encodable(detail),
|
||||
|
|
|
|||
|
|
@ -855,10 +855,10 @@ mod door_smoke {
|
|||
}
|
||||
}
|
||||
|
||||
/// Ruling `desk-read-visibility-whole-record`: Huginn returns the whole record, notes included, to
|
||||
/// whoever the socket carries. The subject is not in the deciders; the decider's note is there.
|
||||
/// Ruling `desk-read-visibility-withheld-ids`: a subject the call's deciders do not name gets counts
|
||||
/// and no account id, over the socket too; notes stay whole (Bifrost strips them). A decider gets the ids.
|
||||
#[test]
|
||||
fn a_subject_not_in_deciders_reads_the_whole_record_with_the_notes() {
|
||||
fn a_subject_not_in_deciders_gets_counts_and_no_account_id() {
|
||||
let mut rig = rig(my_uid());
|
||||
let hash = rig.hash.clone();
|
||||
let note = "Keep it local: the author's call.";
|
||||
|
|
@ -875,9 +875,15 @@ mod door_smoke {
|
|||
for entry in [&list.forks[0], &read] {
|
||||
assert_eq!(entry.record.answers.len(), 1);
|
||||
assert_eq!(entry.record.answers[0].note.as_deref(), Some(note));
|
||||
assert_eq!(entry.record.answers[0].answerer, "y");
|
||||
assert_eq!(entry.card.deciders, ["x"], "the list is the call's, and only fills the card");
|
||||
assert_eq!(entry.record.answers[0].answerer, None);
|
||||
assert_eq!(entry.record.answerer_count, 1);
|
||||
assert!(entry.card.deciders.is_empty());
|
||||
assert_eq!(entry.card.decider_count, 1);
|
||||
}
|
||||
let seen = rig.serving(|path| reply_of(path, "m-4", &fork_read("x", &["x"], FORK)));
|
||||
let (seen, _) = entry_of(seen);
|
||||
assert_eq!(seen.card.deciders, ["x"]);
|
||||
assert_eq!(seen.record.answers[0].answerer.as_deref(), Some("y"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ use std::time::Duration;
|
|||
|
||||
use anyhow::{Result, bail, ensure};
|
||||
|
||||
use crate::sources::{DescribedSources, SourceIndex, SourcePoint, SourceRow, SourcesMeta};
|
||||
use crate::sources::{ChatFilter, ChatOrder, ChatPayload, ChatPoint, ChatRow, DescribedSources, SourceIndex, SourcePoint, SourceRow, SourcesMeta, StoredChat};
|
||||
|
||||
use super::{CollectionMeta, Described, Embedder, Hit, ModelIdentity, Point, VectorIndex, point_id};
|
||||
|
||||
|
|
@ -253,6 +253,8 @@ pub(crate) fn pair() -> (FakeEmbedder, FakeIndex) {
|
|||
pub(crate) struct SourcesState {
|
||||
pub(crate) described: Option<DescribedSources>,
|
||||
pub(crate) points: BTreeMap<String, SourcePoint>,
|
||||
/// The chat points, by id: the same collection, the other corpus.
|
||||
pub(crate) chat: BTreeMap<String, ChatPoint>,
|
||||
/// How many times the collection was recreated.
|
||||
pub(crate) recreated: u32,
|
||||
pub(crate) upsert_calls: u32,
|
||||
|
|
@ -262,6 +264,16 @@ pub(crate) struct SourcesState {
|
|||
pub(crate) deletes: Vec<(String, Option<Vec<String>>)>,
|
||||
/// A delete naming this repo refuses.
|
||||
pub(crate) fail_deletes_of: Option<String>,
|
||||
/// Every cursor write refuses once this many have been written.
|
||||
pub(crate) fail_cursor_writes_after: Option<usize>,
|
||||
/// Every chat upsert refuses while this is set.
|
||||
pub(crate) fail_chat_upserts: bool,
|
||||
/// Every chat delete refuses while this is set.
|
||||
pub(crate) fail_chat_deletes: bool,
|
||||
/// A chat delete naming one of these rooms refuses.
|
||||
pub(crate) fail_chat_deletes_for: BTreeSet<String>,
|
||||
/// Every chat delete that went through: the room and the ids (`None` is all of the room).
|
||||
pub(crate) chat_deletes: Vec<(String, Option<Vec<String>>)>,
|
||||
/// The repo filter and limit of every search, in order.
|
||||
pub(crate) searches: Vec<(Option<String>, u32)>,
|
||||
}
|
||||
|
|
@ -317,12 +329,16 @@ impl SourceIndex for FakeSources {
|
|||
let mut state = self.state.lock().unwrap();
|
||||
state.recreated += 1;
|
||||
state.points.clear();
|
||||
state.chat.clear();
|
||||
state.described = Some(DescribedSources::Labelled { meta: meta.clone(), cursors: BTreeMap::new() });
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn set_cursors(&mut self, cursors: &BTreeMap<String, String>) -> Result<()> {
|
||||
let mut state = self.state.lock().unwrap();
|
||||
if state.fail_cursor_writes_after.is_some_and(|after| state.cursor_writes.len() >= after) {
|
||||
bail!("the vector store refused the cursor write");
|
||||
}
|
||||
state.cursor_writes.push(cursors.clone());
|
||||
let Some(DescribedSources::Labelled { cursors: held, .. }) = &mut state.described else { bail!("no source collection") };
|
||||
*held = cursors.clone();
|
||||
|
|
@ -369,6 +385,86 @@ impl SourceIndex for FakeSources {
|
|||
rows.truncate(limit as usize);
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
fn upsert_chat(&mut self, points: &[ChatPoint]) -> Result<()> {
|
||||
let mut state = self.state.lock().unwrap();
|
||||
ensure!(!state.fail_chat_upserts, "the vector store refused the chat upsert");
|
||||
for point in points {
|
||||
ensure!(!point.payload.messages.is_empty(), "a chat chunk carries no messages");
|
||||
state.chat.insert(point.id.clone(), point.clone());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn chat_nearest(&mut self, vector: &[f32], filter: &ChatFilter, limit: u32, _within: Duration) -> Result<Vec<ChatRow>> {
|
||||
let state = self.state.lock().unwrap();
|
||||
let mut rows: Vec<ChatRow> = state
|
||||
.chat
|
||||
.values()
|
||||
.filter(|point| chat_kept(filter, &point.payload))
|
||||
.map(|point| ChatRow { score: cosine(vector, &point.vector), payload: point.payload.clone() })
|
||||
.collect();
|
||||
rows.sort_by(|a, b| b.score.total_cmp(&a.score));
|
||||
rows.truncate(limit as usize);
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
fn chat_scroll(&mut self, filter: &ChatFilter, order: ChatOrder, limit: u32, _within: Duration) -> Result<Vec<ChatPayload>> {
|
||||
let state = self.state.lock().unwrap();
|
||||
let mut kept: Vec<&ChatPayload> = state.chat.values().map(|point| &point.payload).filter(|payload| chat_kept(filter, payload)).collect();
|
||||
kept.sort_by_key(|payload| payload.messages.iter().map(|message| message.ts_ms).min());
|
||||
if order == ChatOrder::NewestFirst {
|
||||
kept.reverse();
|
||||
}
|
||||
Ok(kept.into_iter().take(limit as usize).cloned().collect())
|
||||
}
|
||||
|
||||
fn chat_tail(&mut self, room_id: &str) -> Result<Option<StoredChat>> {
|
||||
let state = self.state.lock().unwrap();
|
||||
Ok(state.chat.iter().filter(|(_, point)| point.payload.room_id == room_id).max_by_key(|(id, point)| (span(&point.payload).1, (*id).clone())).map(stored))
|
||||
}
|
||||
|
||||
fn chat_chunks_holding(&mut self, room_id: &str, event_ids: &[String]) -> Result<Vec<StoredChat>> {
|
||||
let state = self.state.lock().unwrap();
|
||||
Ok(state
|
||||
.chat
|
||||
.iter()
|
||||
.filter(|(_, point)| point.payload.room_id == room_id && point.payload.messages.iter().any(|message| event_ids.contains(&message.event_id)))
|
||||
.map(stored)
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn chat_chunks_overlapping(&mut self, room_id: &str, from_ms: u64, to_ms: u64) -> Result<Vec<StoredChat>> {
|
||||
let state = self.state.lock().unwrap();
|
||||
Ok(state.chat.iter().filter(|(_, point)| point.payload.room_id == room_id && span(&point.payload).1 >= from_ms && span(&point.payload).0 < to_ms).map(stored).collect())
|
||||
}
|
||||
|
||||
fn delete_chat(&mut self, room_id: &str, ids: Option<&[String]>) -> Result<()> {
|
||||
let mut state = self.state.lock().unwrap();
|
||||
ensure!(!state.fail_chat_deletes && !state.fail_chat_deletes_for.contains(room_id), "the vector store refused the chat delete");
|
||||
state.chat_deletes.push((room_id.to_owned(), ids.map(<[String]>::to_vec)));
|
||||
state.chat.retain(|id, point| !(point.payload.room_id == room_id && ids.is_none_or(|ids| ids.contains(id))));
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn stored((id, point): (&String, &ChatPoint)) -> StoredChat {
|
||||
StoredChat { id: id.clone(), payload: point.payload.clone() }
|
||||
}
|
||||
|
||||
/// (start, end) of the times a chunk's messages make.
|
||||
fn span(payload: &ChatPayload) -> (u64, u64) {
|
||||
let times = || payload.messages.iter().map(|message| message.ts_ms);
|
||||
(times().min().unwrap_or(0), times().max().unwrap_or(0))
|
||||
}
|
||||
|
||||
/// A chat chunk under a filter, by the span its messages make.
|
||||
fn chat_kept(filter: &ChatFilter, payload: &ChatPayload) -> bool {
|
||||
let times = || payload.messages.iter().map(|message| message.ts_ms);
|
||||
match (times().min(), times().max()) {
|
||||
(Some(start), Some(end)) => filter.event_id.as_ref().is_none_or(|id| payload.messages.iter().any(|message| &message.event_id == id)) && filter.keeps(&payload.room_id, payload.room_alias.as_deref(), start, end),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
impl SourceIndex for FakeIndex {
|
||||
|
|
@ -390,4 +486,25 @@ impl SourceIndex for FakeIndex {
|
|||
fn search_sources(&mut self, vector: &[f32], repo: Option<&str>, front: bool, limit: u32, within: Duration) -> Result<Vec<SourceRow>> {
|
||||
self.sources.search_sources(vector, repo, front, limit, within)
|
||||
}
|
||||
fn upsert_chat(&mut self, points: &[ChatPoint]) -> Result<()> {
|
||||
self.sources.upsert_chat(points)
|
||||
}
|
||||
fn chat_nearest(&mut self, vector: &[f32], filter: &ChatFilter, limit: u32, within: Duration) -> Result<Vec<ChatRow>> {
|
||||
self.sources.chat_nearest(vector, filter, limit, within)
|
||||
}
|
||||
fn chat_scroll(&mut self, filter: &ChatFilter, order: ChatOrder, limit: u32, within: Duration) -> Result<Vec<ChatPayload>> {
|
||||
self.sources.chat_scroll(filter, order, limit, within)
|
||||
}
|
||||
fn chat_tail(&mut self, room_id: &str) -> Result<Option<StoredChat>> {
|
||||
self.sources.chat_tail(room_id)
|
||||
}
|
||||
fn chat_chunks_holding(&mut self, room_id: &str, event_ids: &[String]) -> Result<Vec<StoredChat>> {
|
||||
self.sources.chat_chunks_holding(room_id, event_ids)
|
||||
}
|
||||
fn chat_chunks_overlapping(&mut self, room_id: &str, from_ms: u64, to_ms: u64) -> Result<Vec<StoredChat>> {
|
||||
self.sources.chat_chunks_overlapping(room_id, from_ms, to_ms)
|
||||
}
|
||||
fn delete_chat(&mut self, room_id: &str, ids: Option<&[String]>) -> Result<()> {
|
||||
self.sources.delete_chat(room_id, ids)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -105,9 +105,9 @@ use huginn_mind::wire::IndexStatus;
|
|||
use huginn_mind::{INDEX_TEXT_VERSION, IndexEntry, Mind, MindStore};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::daemon::{Hits, IndexSink, SearchTicket};
|
||||
use crate::daemon::{Found, Hits, IndexSink, SearchTicket};
|
||||
use crate::sources::{self, SourceIndex};
|
||||
use huginn_mind::{SourceHits, SourceQuery};
|
||||
use huginn_mind::{ChatQuery, ChatWindowQuery, SourceQuery};
|
||||
use ollama::QUERY_INSTRUCTION;
|
||||
|
||||
/// The `managed_by` a collection carries when this organ made it.
|
||||
|
|
@ -685,11 +685,18 @@ struct Queued {
|
|||
due: Instant,
|
||||
}
|
||||
|
||||
/// A source search waiting for the worker. It is not the mind's: nothing about
|
||||
/// the mind's reconciliation gates it.
|
||||
/// What a source-collection search asks: forge chunks or chat chunks.
|
||||
enum SourceAsk {
|
||||
Sources(SourceQuery),
|
||||
Chat(ChatQuery),
|
||||
Window(ChatWindowQuery),
|
||||
}
|
||||
|
||||
/// A source or chat search waiting for the worker. It is not the mind's:
|
||||
/// nothing about the mind's reconciliation gates it.
|
||||
struct QueuedSource {
|
||||
ticket: SearchTicket,
|
||||
query: SourceQuery,
|
||||
ask: SourceAsk,
|
||||
due: Instant,
|
||||
}
|
||||
|
||||
|
|
@ -706,7 +713,7 @@ struct Shared {
|
|||
/// Searches the worker finished and the serve loop has not collected.
|
||||
found: Vec<(SearchTicket, Result<Hits, String>)>,
|
||||
source_searches: VecDeque<QueuedSource>,
|
||||
sources_found: Vec<(SearchTicket, Result<SourceHits, String>)>,
|
||||
sources_found: Vec<(SearchTicket, Found)>,
|
||||
next_ticket: u64,
|
||||
}
|
||||
|
||||
|
|
@ -821,10 +828,15 @@ fn serve_source_search<E: Embedder, V: VectorIndex + SourceIndex>(projector: &mu
|
|||
if guard.closed {
|
||||
return;
|
||||
}
|
||||
let Some(QueuedSource { ticket, query, due }) = guard.source_searches.pop_front() else { return };
|
||||
let Some(QueuedSource { ticket, ask, due }) = guard.source_searches.pop_front() else { return };
|
||||
drop(guard);
|
||||
let found = sources::search(&mut projector.embedder, &mut projector.index, &query, due.saturating_duration_since(Instant::now()))
|
||||
.map_err(|error| format!("{error:#}"));
|
||||
let within = due.saturating_duration_since(Instant::now());
|
||||
let found = match ask {
|
||||
SourceAsk::Sources(query) => sources::search(&mut projector.embedder, &mut projector.index, &query, within).map(Found::Sources),
|
||||
SourceAsk::Chat(query) => sources::search_chat(&mut projector.embedder, &mut projector.index, &query, within).map(Found::Chat),
|
||||
SourceAsk::Window(query) => sources::window(&mut projector.index, &query, within).map(|window| window.map_or_else(Found::Refused, Found::Window)),
|
||||
}
|
||||
.unwrap_or_else(|error| Found::Failed(format!("{error:#}")));
|
||||
lock(shared).sources_found.push((ticket, found));
|
||||
}
|
||||
|
||||
|
|
@ -966,6 +978,24 @@ pub struct WorkerSink {
|
|||
}
|
||||
|
||||
impl WorkerSink {
|
||||
/// Queues a source or chat search behind the same bound as every other
|
||||
/// search; the mind's index state does not refuse it.
|
||||
fn queue_source_ask(&mut self, ask: SourceAsk) -> Result<SearchTicket, String> {
|
||||
let mut guard = lock(&self.shared);
|
||||
if guard.searches.len() + guard.source_searches.len() >= SEARCHES_QUEUED_MAX {
|
||||
return Err(format!(
|
||||
"the index has {} searches waiting and takes no more until it has answered some",
|
||||
guard.searches.len() + guard.source_searches.len()
|
||||
));
|
||||
}
|
||||
let ticket = SearchTicket(guard.next_ticket);
|
||||
guard.next_ticket += 1;
|
||||
guard.source_searches.push_back(QueuedSource { ticket, ask, due: Instant::now() + self.deadline });
|
||||
drop(guard);
|
||||
self.shared.1.notify_all();
|
||||
Ok(ticket)
|
||||
}
|
||||
|
||||
/// Starts the worker. `mind` is the mind's genesis receipt id, `None` for
|
||||
/// a mind not yet written. `startup` is every indexable document the mind
|
||||
/// holds now; the worker's first act is to reconcile the collection
|
||||
|
|
@ -1086,22 +1116,18 @@ impl<S: MindStore> IndexSink<S> for WorkerSink {
|
|||
}
|
||||
|
||||
fn search_sources(&mut self, query: &SourceQuery) -> Result<SearchTicket, String> {
|
||||
let mut guard = lock(&self.shared);
|
||||
if guard.searches.len() + guard.source_searches.len() >= SEARCHES_QUEUED_MAX {
|
||||
return Err(format!(
|
||||
"the index has {} searches waiting and takes no more until it has answered some",
|
||||
guard.searches.len() + guard.source_searches.len()
|
||||
));
|
||||
}
|
||||
let ticket = SearchTicket(guard.next_ticket);
|
||||
guard.next_ticket += 1;
|
||||
guard.source_searches.push_back(QueuedSource { ticket, query: query.clone(), due: Instant::now() + self.deadline });
|
||||
drop(guard);
|
||||
self.shared.1.notify_all();
|
||||
Ok(ticket)
|
||||
self.queue_source_ask(SourceAsk::Sources(query.clone()))
|
||||
}
|
||||
|
||||
fn sources_searched(&mut self) -> Vec<(SearchTicket, Result<SourceHits, String>)> {
|
||||
fn search_chat(&mut self, query: &ChatQuery) -> Result<SearchTicket, String> {
|
||||
self.queue_source_ask(SourceAsk::Chat(query.clone()))
|
||||
}
|
||||
|
||||
fn chat_window(&mut self, query: &ChatWindowQuery) -> Result<SearchTicket, String> {
|
||||
self.queue_source_ask(SourceAsk::Window(query.clone()))
|
||||
}
|
||||
|
||||
fn sources_searched(&mut self) -> Vec<(SearchTicket, Found)> {
|
||||
std::mem::take(&mut lock(&self.shared).sources_found)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,12 @@ use std::time::Duration;
|
|||
use anyhow::{Context, Result, ensure};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::sources::{COLLECTION, FRONT_PAGE_FILES, DescribedSources, SourceIndex, SourcePayload, SourcePoint, SourceRow, SourcesMeta};
|
||||
use huginn_mind::ChatMessage;
|
||||
|
||||
use crate::sources::{
|
||||
COLLECTION, CORPUS_CHAT, CORPUS_FORGE, ChatFilter, ChatOrder, ChatPayload, ChatPoint, ChatRow, DescribedSources, FRONT_PAGE_FILES, SourceIndex, SourcePayload, SourcePoint, SourceRow,
|
||||
SourcesMeta, StoredChat,
|
||||
};
|
||||
|
||||
use super::{CollectionMeta, INDEX_CALL_TIMEOUT, Described, Hit, Http, Method, Point, VectorIndex};
|
||||
|
||||
|
|
@ -175,21 +180,103 @@ impl VectorIndex for QdrantIndex {
|
|||
}
|
||||
}
|
||||
|
||||
/// The payload filter of a source search: the repo, and the front-page paths
|
||||
/// (exact matches on the keyword-indexed `path`), both when both are asked.
|
||||
fn source_filter(repo: Option<&str>, front: bool) -> Option<Value> {
|
||||
let mut must = Vec::new();
|
||||
/// The payload filter of a source search: forge points only, then the repo and
|
||||
/// the front-page paths (exact matches on the keyword-indexed `path`) when
|
||||
/// asked. A chat point carries no repo or path and can never pass it.
|
||||
fn source_filter(repo: Option<&str>, front: bool) -> Value {
|
||||
let mut must = vec![json!({ "key": "corpus", "match": { "value": CORPUS_FORGE } })];
|
||||
if let Some(repo) = repo {
|
||||
must.push(json!({ "key": "repo", "match": { "value": repo } }));
|
||||
}
|
||||
if front {
|
||||
must.push(json!({ "key": "path", "match": { "any": FRONT_PAGE_FILES } }));
|
||||
}
|
||||
(!must.is_empty()).then(|| json!({ "must": must }))
|
||||
json!({ "must": must })
|
||||
}
|
||||
|
||||
/// The payload filter of a chat read: chat points only; the rooms any-of by id
|
||||
/// (`!`) or alias (`#`) as a `should`; the span as ranges that keep a chunk
|
||||
/// overlapping [after, before].
|
||||
fn chat_filter(filter: &ChatFilter) -> Value {
|
||||
let mut must = vec![json!({ "key": "corpus", "match": { "value": CORPUS_CHAT } })];
|
||||
if let Some(after) = filter.after_ms {
|
||||
must.push(json!({ "key": "ts_end_ms", "range": { "gte": after } }));
|
||||
}
|
||||
if let Some(before) = filter.before_ms {
|
||||
must.push(json!({ "key": "ts_start_ms", "range": { "lte": before } }));
|
||||
}
|
||||
if let Some(from) = filter.starts_from_ms {
|
||||
must.push(json!({ "key": "ts_start_ms", "range": { "gte": from } }));
|
||||
}
|
||||
if let Some(to) = filter.starts_to_ms {
|
||||
must.push(json!({ "key": "ts_start_ms", "range": { "lte": to } }));
|
||||
}
|
||||
if let Some(event_id) = &filter.event_id {
|
||||
must.push(json!({ "key": "event_ids", "match": { "value": event_id } }));
|
||||
}
|
||||
let mut out = json!({ "must": must });
|
||||
let (ids, aliases): (Vec<&String>, Vec<&String>) = filter.rooms.iter().partition(|room| room.starts_with('!'));
|
||||
let mut should = Vec::new();
|
||||
if !ids.is_empty() {
|
||||
should.push(json!({ "key": "room_id", "match": { "any": ids } }));
|
||||
}
|
||||
if !aliases.is_empty() {
|
||||
should.push(json!({ "key": "room_alias", "match": { "any": aliases } }));
|
||||
}
|
||||
if !should.is_empty() {
|
||||
out["should"] = Value::Array(should);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// A chat chunk's payload. The span, first and last ids and the id list are
|
||||
/// derived from `messages`, so they cannot disagree with them.
|
||||
fn chat_payload_json(payload: &ChatPayload) -> Result<Value> {
|
||||
let (Some(first), Some(last)) = (payload.messages.first(), payload.messages.last()) else {
|
||||
anyhow::bail!("a chat chunk of {} carries no messages", payload.room_id);
|
||||
};
|
||||
let mut out = json!({
|
||||
"corpus": CORPUS_CHAT,
|
||||
"room_id": payload.room_id,
|
||||
"ts_start_ms": payload.messages.iter().map(|message| message.ts_ms).min(),
|
||||
"ts_end_ms": payload.messages.iter().map(|message| message.ts_ms).max(),
|
||||
"first_event_id": first.event_id,
|
||||
"last_event_id": last.event_id,
|
||||
"event_ids": payload.messages.iter().map(|message| &message.event_id).collect::<Vec<_>>(),
|
||||
"messages": payload.messages.iter().map(|message| json!({
|
||||
"event_id": message.event_id, "sender": message.sender, "ts_ms": message.ts_ms, "body": message.body,
|
||||
})).collect::<Vec<_>>(),
|
||||
"text": payload.text,
|
||||
});
|
||||
if let Some(alias) = &payload.room_alias {
|
||||
out["room_alias"] = json!(alias);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn chat_payload_of(payload: &Value) -> Result<ChatPayload> {
|
||||
ensure!(payload["corpus"] == CORPUS_CHAT, "a chat read was handed a point of another corpus");
|
||||
let text = |value: &Value, key: &str| value[key].as_str().map(str::to_owned).with_context(|| format!("a chat chunk carries no {key}"));
|
||||
let messages = payload["messages"]
|
||||
.as_array()
|
||||
.context("a chat chunk carries no messages")?
|
||||
.iter()
|
||||
.map(|message| {
|
||||
Ok(ChatMessage {
|
||||
event_id: text(message, "event_id")?,
|
||||
sender: text(message, "sender")?,
|
||||
ts_ms: message["ts_ms"].as_u64().context("a chat message carries no ts_ms")?,
|
||||
body: text(message, "body")?,
|
||||
})
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
ensure!(!messages.is_empty(), "a chat chunk carries no messages");
|
||||
Ok(ChatPayload { room_id: text(payload, "room_id")?, room_alias: payload["room_alias"].as_str().map(str::to_owned), messages, text: text(payload, "text")? })
|
||||
}
|
||||
|
||||
fn payload_json(payload: &SourcePayload) -> Value {
|
||||
json!({
|
||||
"corpus": CORPUS_FORGE,
|
||||
"repo": payload.repo,
|
||||
"path": payload.path,
|
||||
"commit": payload.commit,
|
||||
|
|
@ -270,8 +357,17 @@ impl SourceIndex for QdrantIndex {
|
|||
},
|
||||
});
|
||||
self.ask(Method::Put, &format!("{path}?wait=true"), Some(&body), &[200])?;
|
||||
for field in ["repo", "path"] {
|
||||
let index = json!({ "field_name": field, "field_schema": "keyword" });
|
||||
for (field, schema) in [
|
||||
("repo", "keyword"),
|
||||
("path", "keyword"),
|
||||
("corpus", "keyword"),
|
||||
("room_id", "keyword"),
|
||||
("room_alias", "keyword"),
|
||||
("event_ids", "keyword"),
|
||||
("ts_start_ms", "integer"),
|
||||
("ts_end_ms", "integer"),
|
||||
] {
|
||||
let index = json!({ "field_name": field, "field_schema": schema });
|
||||
self.ask(Method::Put, &format!("{path}/index?wait=true"), Some(&index), &[200])?;
|
||||
}
|
||||
Ok(())
|
||||
|
|
@ -315,9 +411,7 @@ impl SourceIndex for QdrantIndex {
|
|||
let collection = self.sources.clone();
|
||||
let http = self.http.bounded(within);
|
||||
let mut body = json!({ "vector": vector, "limit": limit, "with_payload": true, "with_vector": false });
|
||||
if let Some(filter) = source_filter(repo, front) {
|
||||
body["filter"] = filter;
|
||||
}
|
||||
body["filter"] = source_filter(repo, front);
|
||||
let (_, answer) =
|
||||
self.ask_with(&http, Method::Post, &format!("/collections/{collection}/points/search"), Some(&body), &[200])?;
|
||||
let found = answer["result"].as_array().context("Qdrant's search has no result")?;
|
||||
|
|
@ -331,6 +425,113 @@ impl SourceIndex for QdrantIndex {
|
|||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn upsert_chat(&mut self, points: &[ChatPoint]) -> Result<()> {
|
||||
let collection = self.sources.clone();
|
||||
let points: Vec<Value> = points
|
||||
.iter()
|
||||
.map(|point| Ok(json!({ "id": point.id, "vector": point.vector, "payload": chat_payload_json(&point.payload)? })))
|
||||
.collect::<Result<_>>()?;
|
||||
self.ask(Method::Put, &format!("/collections/{collection}/points?wait=true"), Some(&json!({ "points": points })), &[200])?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn chat_nearest(&mut self, vector: &[f32], filter: &ChatFilter, limit: u32, within: Duration) -> Result<Vec<ChatRow>> {
|
||||
let collection = self.sources.clone();
|
||||
let http = self.http.bounded(within);
|
||||
let body = json!({ "vector": vector, "limit": limit, "filter": chat_filter(filter), "with_payload": true, "with_vector": false });
|
||||
let (_, answer) =
|
||||
self.ask_with(&http, Method::Post, &format!("/collections/{collection}/points/search"), Some(&body), &[200])?;
|
||||
let found = answer["result"].as_array().context("Qdrant's search has no result")?;
|
||||
found
|
||||
.iter()
|
||||
.map(|point| Ok(ChatRow { score: point["score"].as_f64().context("a hit carries no score")? as f32, payload: chat_payload_of(&point["payload"])? }))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn chat_scroll(&mut self, filter: &ChatFilter, order: ChatOrder, limit: u32, within: Duration) -> Result<Vec<ChatPayload>> {
|
||||
let collection = self.sources.clone();
|
||||
let http = self.http.bounded(within);
|
||||
let body = json!({
|
||||
"filter": chat_filter(filter),
|
||||
"limit": limit,
|
||||
"order_by": { "key": "ts_start_ms", "direction": match order { ChatOrder::OldestFirst => "asc", ChatOrder::NewestFirst => "desc" } },
|
||||
"with_payload": true,
|
||||
"with_vector": false,
|
||||
});
|
||||
let (_, answer) =
|
||||
self.ask_with(&http, Method::Post, &format!("/collections/{collection}/points/scroll"), Some(&body), &[200])?;
|
||||
let found = answer["result"]["points"].as_array().context("Qdrant's scroll has no points")?;
|
||||
found.iter().map(|point| chat_payload_of(&point["payload"])).collect()
|
||||
}
|
||||
|
||||
fn chat_tail(&mut self, room_id: &str) -> Result<Option<StoredChat>> {
|
||||
let order = json!({ "key": "ts_end_ms", "direction": "desc" });
|
||||
let Some(newest) = self.scroll_chat(chat_room_filter(room_id, vec![]), Some((order, 1)))?.into_iter().next() else { return Ok(None) };
|
||||
// Chunks tied on the greatest end are all read and the greatest id wins, so the tail is the same chunk on every run.
|
||||
let end = newest.payload.messages.iter().map(|message| message.ts_ms).max().unwrap_or(0);
|
||||
let tied = self.scroll_chat(chat_room_filter(room_id, vec![json!({ "key": "ts_end_ms", "range": { "gte": end } })]), None)?;
|
||||
Ok(tied.into_iter().max_by(|a, b| a.id.cmp(&b.id)))
|
||||
}
|
||||
|
||||
fn chat_chunks_holding(&mut self, room_id: &str, event_ids: &[String]) -> Result<Vec<StoredChat>> {
|
||||
if event_ids.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
self.scroll_chat(chat_room_filter(room_id, vec![json!({ "key": "event_ids", "match": { "any": event_ids } })]), None)
|
||||
}
|
||||
|
||||
fn chat_chunks_overlapping(&mut self, room_id: &str, from_ms: u64, to_ms: u64) -> Result<Vec<StoredChat>> {
|
||||
let span = vec![json!({ "key": "ts_end_ms", "range": { "gte": from_ms } }), json!({ "key": "ts_start_ms", "range": { "lt": to_ms } })];
|
||||
self.scroll_chat(chat_room_filter(room_id, span), None)
|
||||
}
|
||||
|
||||
fn delete_chat(&mut self, room_id: &str, ids: Option<&[String]>) -> Result<()> {
|
||||
let collection = self.sources.clone();
|
||||
let body = json!({ "filter": chat_delete_filter(room_id, ids) });
|
||||
self.ask(Method::Post, &format!("/collections/{collection}/points/delete?wait=true"), Some(&body), &[200])?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl QdrantIndex {
|
||||
/// Every chat chunk `filter` keeps, with its point id, paged; or, given an
|
||||
/// order and a limit, the first of them in that order.
|
||||
fn scroll_chat(&mut self, filter: Value, ordered: Option<(Value, u64)>) -> Result<Vec<StoredChat>> {
|
||||
let collection = self.sources.clone();
|
||||
let mut out = Vec::new();
|
||||
let mut offset = Value::Null;
|
||||
loop {
|
||||
let mut body = json!({ "filter": filter, "limit": SCROLL_PAGE, "with_payload": true, "with_vector": false });
|
||||
if let Some((order, limit)) = &ordered {
|
||||
body["order_by"] = order.clone();
|
||||
body["limit"] = json!(limit);
|
||||
} else if !offset.is_null() {
|
||||
body["offset"] = offset;
|
||||
}
|
||||
let (_, answer) = self.ask(Method::Post, &format!("/collections/{collection}/points/scroll"), Some(&body), &[200])?;
|
||||
for point in answer["result"]["points"].as_array().context("Qdrant's scroll has no points")? {
|
||||
out.push(StoredChat { id: point["id"].as_str().context("a point id is not a string")?.to_owned(), payload: chat_payload_of(&point["payload"])? });
|
||||
}
|
||||
offset = answer["result"]["next_page_offset"].clone();
|
||||
if ordered.is_some() || offset.is_null() {
|
||||
return Ok(out);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// What a chat delete removes: the room's chat points, all of them or those with `ids`. The corpus
|
||||
/// and room conditions hold either way, so a forge point or another room's point cannot match.
|
||||
fn chat_delete_filter(room_id: &str, ids: Option<&[String]>) -> Value {
|
||||
chat_room_filter(room_id, ids.map(|ids| json!({ "has_id": ids })).into_iter().collect())
|
||||
}
|
||||
|
||||
/// The chat points of one room, narrowed by `more` conditions.
|
||||
fn chat_room_filter(room_id: &str, more: Vec<Value>) -> Value {
|
||||
let mut must = vec![json!({ "key": "corpus", "match": { "value": CORPUS_CHAT } }), json!({ "key": "room_id", "match": { "value": room_id } })];
|
||||
must.extend(more);
|
||||
json!({ "must": must })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
@ -339,9 +540,10 @@ mod source_filter_tests {
|
|||
|
||||
/// No ask, no filter; a repo alone is the repo's match.
|
||||
#[test]
|
||||
fn no_ask_is_no_filter_and_a_repo_is_its_own_match() {
|
||||
assert_eq!(source_filter(None, false), None);
|
||||
assert_eq!(source_filter(Some("Huginn"), false), Some(json!({ "must": [{ "key": "repo", "match": { "value": "Huginn" } }] })));
|
||||
fn no_ask_is_the_forge_corpus_and_a_repo_is_its_own_match() {
|
||||
let forge = json!({ "key": "corpus", "match": { "value": "forge" } });
|
||||
assert_eq!(source_filter(None, false), json!({ "must": [forge] }));
|
||||
assert_eq!(source_filter(Some("Huginn"), false), json!({ "must": [forge, { "key": "repo", "match": { "value": "Huginn" } }] }));
|
||||
}
|
||||
|
||||
/// The front filter matches the path against exactly the front-page files
|
||||
|
|
@ -354,15 +556,104 @@ mod source_filter_tests {
|
|||
assert_eq!(path.len(), 1);
|
||||
path[0]["match"]["any"].as_array().unwrap().iter().map(|p| p.as_str().unwrap().to_owned()).collect()
|
||||
};
|
||||
let front = source_filter(None, true).unwrap();
|
||||
let front = source_filter(None, true);
|
||||
assert_eq!(any(&front), ["README.md", "README", "readme.md", "README.markdown", "CLAUDE.md", "AGENTS.md"]);
|
||||
assert_eq!(front["must"].as_array().unwrap().len(), 1, "no repo condition when none was asked");
|
||||
assert_eq!(front["must"].as_array().unwrap().len(), 2, "the corpus and the path; no repo condition when none was asked");
|
||||
|
||||
let both = source_filter(Some("Zyphos"), true).unwrap();
|
||||
let both = source_filter(Some("Zyphos"), true);
|
||||
assert_eq!(any(&both), any(&front));
|
||||
let repos: Vec<&Value> = both["must"].as_array().unwrap().iter().filter(|c| c["key"] == "repo").collect();
|
||||
assert_eq!(repos.len(), 1);
|
||||
assert_eq!(repos[0]["match"]["value"], "Zyphos");
|
||||
assert_eq!(both["must"].as_array().unwrap().len(), 2);
|
||||
assert_eq!(both["must"].as_array().unwrap().len(), 3);
|
||||
}
|
||||
|
||||
fn payload() -> ChatPayload {
|
||||
let message = |id: &str, ts: u64| ChatMessage { event_id: id.into(), sender: "@a:gamecult.org".into(), ts_ms: ts, body: format!("body {id}") };
|
||||
ChatPayload { room_id: "!r:gamecult.org".into(), room_alias: Some("#swarm:gamecult.org".into()), messages: vec![message("$a", 10), message("$b", 20)], text: "swarm\n10 @a: body".into() }
|
||||
}
|
||||
|
||||
/// The chat codec is its own inverse, writes the contract's fields derived
|
||||
/// from the messages, and refuses a chunk with none and a point of another corpus.
|
||||
#[test]
|
||||
fn the_chat_payload_round_trips_the_contract() {
|
||||
let written = chat_payload_json(&payload()).unwrap();
|
||||
assert_eq!(written["corpus"], "chat");
|
||||
assert_eq!((written["ts_start_ms"].as_u64(), written["ts_end_ms"].as_u64()), (Some(10), Some(20)));
|
||||
assert_eq!((written["first_event_id"].as_str(), written["last_event_id"].as_str()), (Some("$a"), Some("$b")));
|
||||
assert_eq!(written["event_ids"], json!(["$a", "$b"]));
|
||||
assert_eq!(written["room_alias"], "#swarm:gamecult.org");
|
||||
assert_eq!(chat_payload_of(&written).unwrap(), payload());
|
||||
let mut no_alias = payload();
|
||||
no_alias.room_alias = None;
|
||||
assert!(chat_payload_json(&no_alias).unwrap().get("room_alias").is_none());
|
||||
assert_eq!(chat_payload_of(&chat_payload_json(&no_alias).unwrap()).unwrap(), no_alias);
|
||||
assert!(chat_payload_json(&ChatPayload { messages: vec![], ..payload() }).is_err());
|
||||
let forge = payload_json(&SourcePayload { repo: "r".into(), path: "p".into(), commit: "c".into(), blob: "b".into(), line_start: 1, line_end: 2, chunk_index: 0, ext: "rs".into(), text: "t".into() });
|
||||
assert_eq!(forge["corpus"], "forge");
|
||||
assert!(chat_payload_of(&forge).is_err());
|
||||
}
|
||||
|
||||
/// The writer-side filters name the chat corpus and the room, so a forge point never matches.
|
||||
#[test]
|
||||
fn the_writer_filters_are_always_chat_corpus_and_the_room() {
|
||||
let filter = chat_room_filter("!r:x", vec![json!({ "key": "event_ids", "match": { "any": ["$a"] } })]);
|
||||
assert_eq!(
|
||||
filter,
|
||||
json!({ "must": [
|
||||
{ "key": "corpus", "match": { "value": "chat" } },
|
||||
{ "key": "room_id", "match": { "value": "!r:x" } },
|
||||
{ "key": "event_ids", "match": { "any": ["$a"] } },
|
||||
] })
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_chat_delete_names_the_corpus_and_room_with_or_without_ids() {
|
||||
let all = chat_delete_filter("!r:x", None);
|
||||
assert_eq!(all["must"].as_array().unwrap().len(), 2);
|
||||
let some = chat_delete_filter("!r:x", Some(&["p1".to_owned()]));
|
||||
assert_eq!(some["must"][2], json!({ "has_id": ["p1"] }));
|
||||
assert_eq!(some["must"][0], json!({ "key": "corpus", "match": { "value": "chat" } }));
|
||||
}
|
||||
|
||||
/// The chat filter is the chat corpus, the rooms as an any-of `should` split
|
||||
/// by id and alias, and the two ranges that keep an overlapping span.
|
||||
#[test]
|
||||
fn the_chat_filter_is_corpus_rooms_and_overlap() {
|
||||
let filter = chat_filter(&ChatFilter { rooms: vec!["!r:x".into(), "#a:x".into(), "#b:x".into()], after_ms: Some(5), before_ms: Some(9), ..ChatFilter::default() });
|
||||
assert_eq!(
|
||||
filter,
|
||||
json!({
|
||||
"must": [
|
||||
{ "key": "corpus", "match": { "value": "chat" } },
|
||||
{ "key": "ts_end_ms", "range": { "gte": 5 } },
|
||||
{ "key": "ts_start_ms", "range": { "lte": 9 } },
|
||||
],
|
||||
"should": [
|
||||
{ "key": "room_id", "match": { "any": ["!r:x"] } },
|
||||
{ "key": "room_alias", "match": { "any": ["#a:x", "#b:x"] } },
|
||||
],
|
||||
})
|
||||
);
|
||||
assert_eq!(chat_filter(&ChatFilter::default()), json!({ "must": [{ "key": "corpus", "match": { "value": "chat" } }] }));
|
||||
}
|
||||
|
||||
/// An event id is a match on the keyword array, and the start bounds are
|
||||
/// ranges on the chunk's own start.
|
||||
#[test]
|
||||
fn the_chat_filter_matches_an_event_and_bounds_chunk_starts() {
|
||||
let filter = chat_filter(&ChatFilter { event_id: Some("$e".into()), starts_from_ms: Some(3), starts_to_ms: Some(8), ..ChatFilter::default() });
|
||||
assert_eq!(
|
||||
filter["must"],
|
||||
json!([
|
||||
{ "key": "corpus", "match": { "value": "chat" } },
|
||||
{ "key": "ts_start_ms", "range": { "gte": 3 } },
|
||||
{ "key": "ts_start_ms", "range": { "lte": 8 } },
|
||||
{ "key": "event_ids", "match": { "value": "$e" } },
|
||||
])
|
||||
);
|
||||
let corpus = json!({ "key": "corpus", "match": { "value": "chat" } });
|
||||
assert_eq!(filter["must"][0], corpus, "a point with no corpus, or another, does not match: the chat corpus clause is always first");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1995,14 +1995,23 @@ fn the_sink_owns_the_searchs_deadline_and_tells_the_loop() {
|
|||
}
|
||||
|
||||
fn ask_sources(sink: &mut WorkerSink, text: &str) -> Result<SearchTicket, String> {
|
||||
let query = SourceQuery { text: huginn_mind::eureka_pipeline::Line(text.into()), top_k: 5, repo: None, front: false };
|
||||
let query = SourceQuery { text: huginn_mind::eureka_pipeline::Line(text.into()), texts: vec![], top_k: 5, repo: None, front: false };
|
||||
<WorkerSink as IndexSink<OwnedRedbMessagePackBackingStore>>::search_sources(sink, &query)
|
||||
}
|
||||
|
||||
fn sources_answers(sink: &mut WorkerSink, count: usize) -> Vec<(SearchTicket, Result<SourceHits, String>)> {
|
||||
fn sources_answers(sink: &mut WorkerSink, count: usize) -> Vec<(SearchTicket, Result<huginn_mind::SourceHits, String>)> {
|
||||
let mut all = Vec::new();
|
||||
eventually("the source searches being answered", || {
|
||||
all.extend(<WorkerSink as IndexSink<OwnedRedbMessagePackBackingStore>>::sources_searched(sink));
|
||||
all.extend(<WorkerSink as IndexSink<OwnedRedbMessagePackBackingStore>>::sources_searched(sink).into_iter().map(|(ticket, found)| {
|
||||
(
|
||||
ticket,
|
||||
match found {
|
||||
crate::daemon::Found::Sources(hits) => Ok(hits),
|
||||
crate::daemon::Found::Failed(why) => Err(why),
|
||||
other => panic!("a source search was answered with another kind: {}", matches!(other, crate::daemon::Found::Chat(_))),
|
||||
},
|
||||
)
|
||||
}));
|
||||
all.len() >= count
|
||||
});
|
||||
all
|
||||
|
|
@ -2046,7 +2055,7 @@ fn a_worker_stuck_on_the_minds_collection_still_answers_a_source_search() {
|
|||
index.plant(&collection_name(&slug(INSTANCE)), mind_label);
|
||||
let meta = SourcesMeta {
|
||||
managed_by: MANAGED_BY.into(),
|
||||
corpus: "forge:GameCult".into(),
|
||||
corpus: crate::sources::CORPUS.into(),
|
||||
model: "model".into(),
|
||||
model_digest: "d1".into(),
|
||||
dimensions: 4,
|
||||
|
|
@ -2081,6 +2090,60 @@ fn a_worker_stuck_on_the_minds_collection_still_answers_a_source_search() {
|
|||
assert_eq!(answered.indexed_under.digest, "d1");
|
||||
}
|
||||
|
||||
/// A chat search rides the same queue and worker as a source search, answers
|
||||
/// from the same collection while the mind's own collection is stuck, and comes
|
||||
/// back as chat hits.
|
||||
#[test]
|
||||
fn a_worker_answers_a_chat_search_from_the_shared_collection() {
|
||||
use crate::sources::{ChatPayload, ChatPoint, DescribedSources, SourcesMeta};
|
||||
use huginn_mind::ChatMessage;
|
||||
let (embedder, index) = pair();
|
||||
index.plant(
|
||||
&collection_name(&slug(INSTANCE)),
|
||||
Described::Labelled(CollectionMeta {
|
||||
managed_by: MANAGED_BY.into(),
|
||||
instance: INSTANCE.into(),
|
||||
mind: "another-mind".into(),
|
||||
model: "model".into(),
|
||||
model_digest: "d1".into(),
|
||||
dimensions: 4,
|
||||
index_text_version: INDEX_TEXT_VERSION,
|
||||
}),
|
||||
);
|
||||
let meta = SourcesMeta { managed_by: MANAGED_BY.into(), corpus: crate::sources::CORPUS.into(), model: "model".into(), model_digest: "d1".into(), dimensions: 4, chunker_version: 1 };
|
||||
index.sources.plant(DescribedSources::Labelled { meta, cursors: Default::default() });
|
||||
index
|
||||
.sources
|
||||
.state
|
||||
.lock()
|
||||
.unwrap()
|
||||
.chat
|
||||
.insert(
|
||||
"c1".into(),
|
||||
ChatPoint {
|
||||
id: "c1".into(),
|
||||
vector: vec![1.0; 4],
|
||||
payload: ChatPayload {
|
||||
room_id: "!r:x".into(),
|
||||
room_alias: None,
|
||||
messages: vec![ChatMessage { event_id: "$e".into(), sender: "@a:x".into(), ts_ms: 5, body: "hi".into() }],
|
||||
text: "t".into(),
|
||||
},
|
||||
},
|
||||
);
|
||||
let mut sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries_of(1), quick());
|
||||
sink.wait_status(|status| matches!(status, IndexStatus::Refused { .. }));
|
||||
let query = huginn_mind::ChatQuery { texts: vec![], rooms: vec![], after_ms: None, before_ms: None, top_k: 5 };
|
||||
let ticket = <WorkerSink as IndexSink<OwnedRedbMessagePackBackingStore>>::search_chat(&mut sink, &query).unwrap();
|
||||
let mut found = Vec::new();
|
||||
eventually("the chat search being answered", || {
|
||||
found.extend(<WorkerSink as IndexSink<OwnedRedbMessagePackBackingStore>>::sources_searched(&mut sink));
|
||||
!found.is_empty()
|
||||
});
|
||||
let (answered, Found::Chat(hits)) = found.remove(0) else { panic!("a chat ask is answered with chat hits") };
|
||||
assert_eq!((answered, hits.hits.len(), hits.hits[0].first_event_id.as_str(), hits.truncated), (ticket, 1, "$e", false));
|
||||
}
|
||||
|
||||
/// Source searches take their own tickets, share the queue's bound with the
|
||||
/// mind's searches, and an abandon removes the one ticket named from either.
|
||||
#[test]
|
||||
|
|
@ -2107,3 +2170,47 @@ fn source_searches_have_distinct_tickets_share_the_queue_bound_and_abandon_by_ti
|
|||
abandon(&mut sink, second);
|
||||
assert!(ask_sink(&mut sink, "now", 3).is_ok(), "a freed slot is a slot");
|
||||
}
|
||||
|
||||
/// The chat reads state their request: a scroll is the filter, oldest chunk
|
||||
/// start first; a nearest search is the filter too.
|
||||
#[test]
|
||||
fn qdrant_chat_scroll_and_nearest_send_the_filter_and_the_order() {
|
||||
use crate::sources::{ChatFilter, ChatOrder, SourceIndex};
|
||||
let filter = ChatFilter { rooms: vec!["!r:x".into()], after_ms: Some(5), before_ms: None, ..ChatFilter::default() };
|
||||
let scrolling = stub(&[("POST /collections/", 200, r#"{"result":{"points":[]}}"#)]);
|
||||
QdrantIndex::new(&scrolling.url).chat_scroll(&filter, ChatOrder::OldestFirst, 7, TIME).unwrap();
|
||||
let scroll: serde_json::Value = serde_json::from_str(&scrolling.seen.lock().unwrap()[0].2).unwrap();
|
||||
assert_eq!(scroll["order_by"], serde_json::json!({ "key": "ts_start_ms", "direction": "asc" }));
|
||||
assert_eq!(scroll["filter"]["must"][0], serde_json::json!({ "key": "corpus", "match": { "value": "chat" } }));
|
||||
assert_eq!(scroll["filter"]["must"][1]["range"], serde_json::json!({ "gte": 5 }));
|
||||
assert_eq!(scroll["limit"], 7);
|
||||
|
||||
let searching = stub(&[("POST /collections/", 200, r#"{"result":[]}"#)]);
|
||||
QdrantIndex::new(&searching.url).chat_nearest(&[1.0, 0.0], &filter, 3, TIME).unwrap();
|
||||
let near: serde_json::Value = serde_json::from_str(&searching.seen.lock().unwrap()[0].2).unwrap();
|
||||
assert_eq!(near["filter"]["must"][0], serde_json::json!({ "key": "corpus", "match": { "value": "chat" } }));
|
||||
assert_eq!(near["filter"]["must"][1]["range"], serde_json::json!({ "gte": 5 }));
|
||||
assert_eq!(near["limit"], 3);
|
||||
}
|
||||
|
||||
/// A new source collection indexes `corpus` as a keyword and both times as
|
||||
/// integers, which the range filters and ordered scroll need.
|
||||
#[test]
|
||||
fn qdrant_recreate_sources_creates_the_corpus_and_time_indexes() {
|
||||
use crate::sources::{CORPUS, SourceIndex, SourcesMeta};
|
||||
let building = stub(&[("DELETE /collections/", 200, "{}"), ("PUT /collections/", 200, r#"{"result":true}"#)]);
|
||||
let meta = SourcesMeta { managed_by: MANAGED_BY.into(), corpus: CORPUS.into(), model: "m".into(), model_digest: "d".into(), dimensions: 4, chunker_version: 1 };
|
||||
QdrantIndex::new(&building.url).recreate_sources(&meta).unwrap();
|
||||
let seen = building.seen.lock().unwrap();
|
||||
let indexed: Vec<(String, String)> = seen
|
||||
.iter()
|
||||
.filter(|(_, path, _)| path.contains("/index"))
|
||||
.map(|(_, _, body)| {
|
||||
let body: serde_json::Value = serde_json::from_str(body).unwrap();
|
||||
(body["field_name"].as_str().unwrap().to_owned(), body["field_schema"].as_str().unwrap().to_owned())
|
||||
})
|
||||
.collect();
|
||||
let has = |field: &str, schema: &str| indexed.iter().any(|(f, s)| f == field && s == schema);
|
||||
assert!(has("corpus", "keyword"), "{indexed:?}");
|
||||
assert!(has("ts_start_ms", "integer") && has("ts_end_ms", "integer"), "{indexed:?}");
|
||||
}
|
||||
|
|
|
|||
638
crates/huginn-daemon/src/sources/chat.rs
Normal file
638
crates/huginn-daemon/src/sources/chat.rs
Normal file
|
|
@ -0,0 +1,638 @@
|
|||
//! Chat ingest: joined-room history into the shared source collection, beside
|
||||
//! the forge's repos. `refresh_chat` is the only writer of `corpus=chat` points
|
||||
//! and of the cursors object's `chat:` keys; `chat_purge::request_purge` is the
|
||||
//! only writer of `chat-purge:` keys, and `refresh_chat` the only applier.
|
||||
//!
|
||||
//! The Matrix client that implements `RoomHistory` is the next cut's and holds no
|
||||
//! policy. What is in scope is `room_scope`'s; what may be embedded is
|
||||
//! `huginn_mind::redact::scrub`'s; this file folds events into chunks and writes
|
||||
//! them, write-then-delete and cursor-last, so any crash replays to the same state.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet, HashSet};
|
||||
use std::time::Instant;
|
||||
|
||||
use anyhow::{Context, Result, bail, ensure};
|
||||
use huginn_mind::ChatMessage;
|
||||
use huginn_mind::redact::{ScrubKind, scrub};
|
||||
|
||||
use super::chat_chunk::{chat_version, chunk_id, chunk_messages, chunk_payload, cut_body};
|
||||
use super::chat_purge::{PurgeWindow, covers, parse_windows, purge_key};
|
||||
use super::room_scope::{Estate, Refusal, RoomFacts, admit_room, refuse_event};
|
||||
use super::{CORPUS, ChatPoint, DescribedSources, SourceIndex, StoredChat};
|
||||
use crate::index::{BATCH, BULK_EMBED_TIMEOUT, Embedder, INDEX_CALL_TIMEOUT};
|
||||
|
||||
/// Events asked of the port per page.
|
||||
pub const CHAT_PAGE: u32 = 200;
|
||||
|
||||
/// A chat cursor is the key `chat:<room_id>`; a forge repo name cannot contain
|
||||
/// a colon. The only definition of the prefix.
|
||||
pub const CHAT_CURSOR_PREFIX: &str = "chat:";
|
||||
|
||||
/// A steward's purge tombstone is the key `chat-purge:<room_id>`. The only
|
||||
/// definition of the prefix.
|
||||
pub const CHAT_PURGE_PREFIX: &str = "chat-purge:";
|
||||
|
||||
/// Whether a cursors-object key is chat's (a room cursor or a purge tombstone)
|
||||
/// and not a forge repo name. The only test that tells the two apart.
|
||||
pub fn is_chat_key(key: &str) -> bool {
|
||||
key.starts_with(CHAT_CURSOR_PREFIX) || key.starts_with(CHAT_PURGE_PREFIX)
|
||||
}
|
||||
|
||||
/// What one room event is to the ingest. Only text, notice and emote messages
|
||||
/// become `Message`; everything else (encrypted, media, state, server-redacted)
|
||||
/// is `Other` and is never stored.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum EventKind {
|
||||
Message { body: String },
|
||||
/// A replacement of `target`'s body.
|
||||
Edit { target: String, body: String },
|
||||
Redaction { target: String },
|
||||
Other,
|
||||
}
|
||||
|
||||
/// One event of a room, as the port hands it over: the Matrix client maps its
|
||||
/// JSON to this and filters nothing. `content_keys` are the top-level keys of
|
||||
/// the event's content (the scope reads them).
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct RoomEvent {
|
||||
pub event_id: String,
|
||||
pub sender: String,
|
||||
pub ts_ms: u64,
|
||||
pub content_keys: Vec<String>,
|
||||
pub kind: EventKind,
|
||||
}
|
||||
|
||||
/// One page of a room's events after a cursor event, oldest first.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct EventPage {
|
||||
pub events: Vec<RoomEvent>,
|
||||
/// More events follow this page.
|
||||
pub more: bool,
|
||||
}
|
||||
|
||||
/// The reader user's view of Synapse as ingest uses it. Holds no policy: it maps
|
||||
/// the server's answers to these types and filters nothing.
|
||||
pub trait RoomHistory {
|
||||
/// The rooms the reader has joined. The only source of rooms; there is no allow list.
|
||||
fn joined_rooms(&mut self) -> Result<Vec<String>>;
|
||||
fn facts(&mut self, room_id: &str) -> Result<RoomFacts>;
|
||||
/// Up to `limit` events after the event `after` (from the room's start when `None`).
|
||||
/// `Ok(None)` means `after` is unknown to the server.
|
||||
fn events_after(&mut self, room_id: &str, after: Option<&str>, limit: u32) -> Result<Option<EventPage>>;
|
||||
}
|
||||
|
||||
/// The cursors-object key of a room's chat cursor.
|
||||
pub fn cursor_key(room_id: &str) -> String {
|
||||
format!("{CHAT_CURSOR_PREFIX}{room_id}")
|
||||
}
|
||||
|
||||
/// The room a chat cursor key belongs to; `None` for any other key (a purge tombstone included).
|
||||
pub fn cursor_room(key: &str) -> Option<&str> {
|
||||
key.strip_prefix(CHAT_CURSOR_PREFIX)
|
||||
}
|
||||
|
||||
/// A cursor value, `<version>:<last event id>`; the event is empty before a room's first upsert.
|
||||
pub fn parse_cursor(value: &str) -> Option<(&str, &str)> {
|
||||
value.split_once(':')
|
||||
}
|
||||
|
||||
/// What one admitted room's run did, for the journal. Never carries a body or a sender.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct ChatReport {
|
||||
pub room_id: String,
|
||||
pub alias: Option<String>,
|
||||
pub first: Option<String>,
|
||||
pub last: Option<String>,
|
||||
pub events: u32,
|
||||
pub chunks: u32,
|
||||
pub scrubbed: u32,
|
||||
pub purged: u32,
|
||||
pub seconds: f64,
|
||||
/// Why the room could not be read or written this run; what the index held for it stays as it was.
|
||||
pub failure: Option<String>,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ChatReport {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(
|
||||
formatter,
|
||||
"{} ({}) {}..{} events={} chunks={} scrubbed={} purged={} seconds={:.1}",
|
||||
self.alias.as_deref().unwrap_or("-"),
|
||||
self.room_id,
|
||||
self.first.as_deref().unwrap_or("-"),
|
||||
self.last.as_deref().unwrap_or("-"),
|
||||
self.events,
|
||||
self.chunks,
|
||||
self.scrubbed,
|
||||
self.purged,
|
||||
self.seconds
|
||||
)?;
|
||||
match &self.failure {
|
||||
Some(why) => write!(formatter, " failed: {why}"),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A joined room the scope refused, whole, and why; `event_id` when an event did it.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct RefusedRoom {
|
||||
pub room_id: String,
|
||||
pub reason: Refusal,
|
||||
pub event_id: Option<String>,
|
||||
}
|
||||
|
||||
/// A credential the scrub replaced: where, never what.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ScrubHit {
|
||||
pub room_id: String,
|
||||
pub event_id: String,
|
||||
pub kind: ScrubKind,
|
||||
}
|
||||
|
||||
/// What a chat refresh did.
|
||||
#[derive(Clone, Debug, Default, PartialEq)]
|
||||
pub struct ChatRefreshed {
|
||||
pub rooms: Vec<ChatReport>,
|
||||
/// Rooms (or steps) that failed; the others went on.
|
||||
pub failed: Vec<String>,
|
||||
pub refused: Vec<RefusedRoom>,
|
||||
pub scrubbed: Vec<ScrubHit>,
|
||||
}
|
||||
|
||||
impl ChatReport {
|
||||
fn empty(room_id: &str, alias: Option<String>) -> Self {
|
||||
Self { room_id: room_id.to_owned(), alias, first: None, last: None, events: 0, chunks: 0, scrubbed: 0, purged: 0, seconds: 0.0, failure: None }
|
||||
}
|
||||
|
||||
/// A room that could not be handled this run. Its points and cursor are untouched: an error is not a reclassification.
|
||||
fn failed(room_id: &str, alias: Option<String>, error: &anyhow::Error) -> Self {
|
||||
Self { failure: Some(format!("{error:#}")), ..Self::empty(room_id, alias) }
|
||||
}
|
||||
}
|
||||
|
||||
impl ChatRefreshed {
|
||||
/// Nothing failed and no joined room had to be refused: a joined estate, direct,
|
||||
/// encrypted or bridged room means the reader's roster is wrong. A scrub hit does not make a run unclean.
|
||||
pub fn is_clean(&self) -> bool {
|
||||
self.failed.is_empty() && self.refused.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
fn read_cursors<S: SourceIndex>(index: &mut S) -> Result<BTreeMap<String, String>> {
|
||||
match index.describe_sources(INDEX_CALL_TIMEOUT).context("describing the source collection")? {
|
||||
DescribedSources::Labelled { cursors, .. } => Ok(cursors),
|
||||
_ => bail!("the source collection is gone; the forge refresh creates it"),
|
||||
}
|
||||
}
|
||||
|
||||
fn put_cursor<S: SourceIndex>(index: &mut S, key: &str, value: &str) -> Result<()> {
|
||||
let mut cursors = read_cursors(index)?;
|
||||
if cursors.insert(key.to_owned(), value.to_owned()).as_deref() != Some(value) {
|
||||
index.set_cursors(&cursors).context("writing a chat cursor")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn drop_cursor<S: SourceIndex>(index: &mut S, key: &str) -> Result<()> {
|
||||
let mut cursors = read_cursors(index)?;
|
||||
if cursors.remove(key).is_some() {
|
||||
index.set_cursors(&cursors).context("dropping a chat cursor")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A room's points, then its `chat:` cursor. Its purge tombstone stays.
|
||||
fn purge_room<S: SourceIndex>(index: &mut S, room_id: &str) -> Result<()> {
|
||||
index.delete_chat(room_id, None).with_context(|| format!("deleting the chunks of {room_id}"))?;
|
||||
drop_cursor(index, &cursor_key(room_id))
|
||||
}
|
||||
|
||||
/// What `retire_chat` did: the rooms it removed, and those it could not (each with why), which it went on past.
|
||||
#[derive(Debug, Default, PartialEq, Eq)]
|
||||
pub struct Retired {
|
||||
pub retired: Vec<String>,
|
||||
pub failed: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
/// Deletes the points and `chat:` cursor of every room not in `keep` (the reader left it,
|
||||
/// or it is refused or excluded). A room whose delete fails is reported and the others go on. Purge
|
||||
/// tombstones and forge repos are never touched. With the empty set it removes the whole chat corpus.
|
||||
pub fn retire_chat<S: SourceIndex>(index: &mut S, keep: &BTreeSet<String>) -> Result<Retired> {
|
||||
let rooms: Vec<String> = read_cursors(index)?.keys().filter_map(|key| cursor_room(key)).filter(|room| !keep.contains(*room)).map(str::to_owned).collect();
|
||||
let mut done = Retired::default();
|
||||
for room in rooms {
|
||||
match purge_room(index, &room) {
|
||||
Ok(()) => done.retired.push(room),
|
||||
Err(error) => done.failed.push((room, format!("{error:#}"))),
|
||||
}
|
||||
}
|
||||
Ok(done)
|
||||
}
|
||||
|
||||
/// A change to chunks already stored, in event order.
|
||||
enum Op {
|
||||
/// An edit by `sender` of `target`'s body; applies only to the original's sender.
|
||||
Edit { id: String, target: String, sender: String, body: String },
|
||||
/// A held message's body is what the page says it is (a replay).
|
||||
Set { target: String, body: String },
|
||||
Redact { target: String },
|
||||
}
|
||||
|
||||
impl Op {
|
||||
fn target(&self) -> &str {
|
||||
match self {
|
||||
Self::Edit { target, .. } | Self::Set { target, .. } | Self::Redact { target } => target,
|
||||
}
|
||||
}
|
||||
|
||||
/// Applies to `messages`; whether they changed.
|
||||
fn apply(&self, messages: &mut Vec<ChatMessage>) -> bool {
|
||||
let Some(at) = messages.iter().position(|message| message.event_id == self.target()) else { return false };
|
||||
match self {
|
||||
Self::Edit { sender, body, .. } if messages[at].sender == *sender && messages[at].body != *body => messages[at].body.clone_from(body),
|
||||
Self::Set { body, .. } if messages[at].body != *body => messages[at].body.clone_from(body),
|
||||
Self::Redact { .. } => {
|
||||
messages.remove(at);
|
||||
}
|
||||
_ => return false,
|
||||
}
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
/// A message of the page with the edits of the page that apply to it: the body is the last edit's, and a
|
||||
/// redacted edit leaves the list, so the body falls back to the one before it.
|
||||
struct Pending {
|
||||
message: ChatMessage,
|
||||
original: String,
|
||||
edits: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
struct Folded {
|
||||
pending: Vec<ChatMessage>,
|
||||
ops: Vec<Op>,
|
||||
/// Every event id of the page: a redaction of one of them is known.
|
||||
seen: HashSet<String>,
|
||||
}
|
||||
|
||||
struct Room<'a> {
|
||||
facts: &'a RoomFacts,
|
||||
label: String,
|
||||
windows: Vec<PurgeWindow>,
|
||||
}
|
||||
|
||||
/// The page folded in order: the purge filter, then the credential scrub, then the
|
||||
/// event's effect. After this no raw body exists.
|
||||
fn fold(room: &Room, events: &[RoomEvent], report: &mut ChatReport, out: &mut ChatRefreshed) -> Folded {
|
||||
let mut pending: Vec<Pending> = Vec::new();
|
||||
let mut ops: Vec<Op> = Vec::new();
|
||||
let mut scrubbed = |event: &RoomEvent, body: &str| {
|
||||
let clean = scrub(body);
|
||||
for kind in clean.hits {
|
||||
report.scrubbed += 1;
|
||||
out.scrubbed.push(ScrubHit { room_id: room.facts.room_id.clone(), event_id: event.event_id.clone(), kind });
|
||||
}
|
||||
cut_body(&clean.text)
|
||||
};
|
||||
for event in events {
|
||||
match &event.kind {
|
||||
EventKind::Message { body } => {
|
||||
if covers(&room.windows, event.ts_ms) {
|
||||
report.purged += 1;
|
||||
} else {
|
||||
let body = scrubbed(event, body);
|
||||
if !body.trim().is_empty() {
|
||||
let message = ChatMessage { event_id: event.event_id.clone(), sender: event.sender.clone(), ts_ms: event.ts_ms, body: body.clone() };
|
||||
pending.push(Pending { message, original: body, edits: Vec::new() });
|
||||
}
|
||||
}
|
||||
}
|
||||
EventKind::Edit { target, body } => {
|
||||
if covers(&room.windows, event.ts_ms) {
|
||||
report.purged += 1;
|
||||
continue;
|
||||
}
|
||||
let body = scrubbed(event, body);
|
||||
match pending.iter_mut().find(|held| held.message.event_id == *target) {
|
||||
Some(held) if held.message.sender == event.sender && !body.trim().is_empty() => {
|
||||
held.edits.push((event.event_id.clone(), body.clone()));
|
||||
held.message.body = body;
|
||||
}
|
||||
Some(_) => {}
|
||||
None => ops.push(Op::Edit { id: event.event_id.clone(), target: target.clone(), sender: event.sender.clone(), body }),
|
||||
}
|
||||
}
|
||||
EventKind::Redaction { target } => {
|
||||
// The target may be a message, or an edit whose own text must stop applying.
|
||||
pending.retain(|held| held.message.event_id != *target);
|
||||
for held in &mut pending {
|
||||
held.edits.retain(|(id, _)| id != target);
|
||||
held.message.body = held.edits.last().map_or_else(|| held.original.clone(), |(_, body)| body.clone());
|
||||
}
|
||||
ops.retain(|op| !matches!(op, Op::Edit { id, .. } if id == target));
|
||||
ops.push(Op::Redact { target: target.clone() });
|
||||
}
|
||||
EventKind::Other => {}
|
||||
}
|
||||
}
|
||||
Folded { pending: pending.into_iter().map(|held| held.message).collect(), ops, seen: events.iter().map(|event| event.event_id.clone()).collect() }
|
||||
}
|
||||
|
||||
/// Chunks to rewrite: the stored chunk ids they replace and the messages that now make them.
|
||||
type Rewrite = (Vec<String>, Vec<ChatMessage>);
|
||||
|
||||
/// Chunks `rewrites`, embeds and writes the new chunks (acknowledged), and only then deletes the old
|
||||
/// ids that are not among the new. Returns how many chunks were written.
|
||||
///
|
||||
/// A chunk written by an earlier run whose delete failed can still sit beside the chunk that replaced
|
||||
/// it, and a rewrite of the stale one then names the same first event, hence the same id, as the
|
||||
/// newer chunk. The newer chunk (`standing`, the other chunks the page found) stays; the stale one is deleted.
|
||||
fn write_rewrites<E: Embedder, S: SourceIndex>(
|
||||
embedder: &mut E,
|
||||
index: &mut S,
|
||||
dimensions: u32,
|
||||
room: &Room,
|
||||
rewrites: Vec<Rewrite>,
|
||||
standing: &BTreeSet<String>,
|
||||
) -> Result<u32> {
|
||||
let mut old: BTreeSet<String> = BTreeSet::new();
|
||||
let mut points: Vec<(String, super::ChatPayload)> = Vec::new();
|
||||
for (ids, messages) in rewrites {
|
||||
for group in chunk_messages(&room.label, &messages) {
|
||||
let id = chunk_id(&room.facts.room_id, &group[0].event_id);
|
||||
if standing.contains(&id) && !ids.contains(&id) {
|
||||
continue;
|
||||
}
|
||||
points.push((id, chunk_payload(&room.facts.room_id, room.facts.alias.as_deref(), &room.label, group)));
|
||||
}
|
||||
old.extend(ids);
|
||||
}
|
||||
let new: BTreeSet<&String> = points.iter().map(|(id, _)| id).collect();
|
||||
let stale: Vec<String> = old.iter().filter(|id| !new.contains(id)).cloned().collect();
|
||||
let written = u32::try_from(points.len()).unwrap_or(u32::MAX);
|
||||
for slice in points.chunks(BATCH) {
|
||||
let texts: Vec<String> = slice.iter().map(|(_, payload)| payload.text.clone()).collect();
|
||||
let vectors = embedder.embed(&texts, BULK_EMBED_TIMEOUT).context("embedding chat chunks")?;
|
||||
ensure!(vectors.len() == texts.len(), "the embedder returned {} vectors for {} chunks", vectors.len(), texts.len());
|
||||
let mut batch = Vec::with_capacity(slice.len());
|
||||
for ((id, payload), vector) in slice.iter().zip(vectors) {
|
||||
ensure!(vector.len() == dimensions as usize, "the embedder returned a vector of {} where the collection holds {dimensions}", vector.len());
|
||||
batch.push(ChatPoint { id: id.clone(), vector, payload: payload.clone() });
|
||||
}
|
||||
index.upsert_chat(&batch).context("writing chat chunks")?;
|
||||
}
|
||||
if !stale.is_empty() {
|
||||
index.delete_chat(&room.facts.room_id, Some(&stale)).context("deleting superseded chat chunks")?;
|
||||
}
|
||||
Ok(written)
|
||||
}
|
||||
|
||||
/// Applies the room's purge windows to the chunks it already holds.
|
||||
fn apply_purge<E: Embedder, S: SourceIndex>(embedder: &mut E, index: &mut S, dimensions: u32, room: &Room, report: &mut ChatReport) -> Result<()> {
|
||||
let mut hit: BTreeMap<String, StoredChat> = BTreeMap::new();
|
||||
for window in &room.windows {
|
||||
for chunk in index.chat_chunks_overlapping(&room.facts.room_id, window.from_ms, window.to_ms)? {
|
||||
hit.insert(chunk.id.clone(), chunk);
|
||||
}
|
||||
}
|
||||
let mut rewrites = Vec::new();
|
||||
for chunk in hit.into_values() {
|
||||
let kept: Vec<ChatMessage> = chunk.payload.messages.iter().filter(|message| !covers(&room.windows, message.ts_ms)).cloned().collect();
|
||||
let removed = chunk.payload.messages.len() - kept.len();
|
||||
if removed > 0 {
|
||||
report.purged += u32::try_from(removed).unwrap_or(u32::MAX);
|
||||
rewrites.push((vec![chunk.id], kept));
|
||||
}
|
||||
}
|
||||
report.chunks += write_rewrites(embedder, index, dimensions, room, rewrites, &BTreeSet::new())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Folds one page into the stored chunks: edits and redactions rewrite the chunk that held
|
||||
/// the target alone, new messages extend the open tail.
|
||||
fn write_page<E: Embedder, S: SourceIndex>(
|
||||
embedder: &mut E,
|
||||
index: &mut S,
|
||||
dimensions: u32,
|
||||
room: &Room,
|
||||
mut folded: Folded,
|
||||
may_rebuild: bool,
|
||||
) -> Result<Option<u32>> {
|
||||
let room_id = &room.facts.room_id;
|
||||
let mut asked: Vec<String> = folded.pending.iter().map(|message| message.event_id.clone()).collect();
|
||||
asked.extend(folded.ops.iter().map(|op| op.target().to_owned()));
|
||||
if asked.is_empty() {
|
||||
return Ok(Some(0));
|
||||
}
|
||||
let holding = index.chat_chunks_holding(room_id, &asked)?;
|
||||
// A message the index already holds is a replay: its page body is asserted, never appended again.
|
||||
let held: HashSet<&str> = holding.iter().flat_map(|chunk| chunk.payload.messages.iter().map(|message| message.event_id.as_str())).collect();
|
||||
// A redaction of an event this room neither holds nor shows on the page may be of an edit whose own id is
|
||||
// never stored, so a body it changed cannot be taken back here: the room is rebuilt from the history, which
|
||||
// serves a redacted edit without its text.
|
||||
if may_rebuild && folded.ops.iter().any(|op| matches!(op, Op::Redact { target } if !folded.seen.contains(target) && !held.contains(target.as_str()))) {
|
||||
return Ok(None);
|
||||
}
|
||||
let (replayed, fresh): (Vec<ChatMessage>, Vec<ChatMessage>) = folded.pending.drain(..).partition(|message| held.contains(message.event_id.as_str()));
|
||||
let mut ops: Vec<Op> = replayed.into_iter().map(|message| Op::Set { target: message.event_id, body: message.body }).collect();
|
||||
ops.append(&mut folded.ops);
|
||||
|
||||
let tail = index.chat_tail(room_id)?;
|
||||
let tail_id = tail.as_ref().map(|tail| tail.id.clone());
|
||||
let standing: BTreeSet<String> = holding.iter().map(|chunk| chunk.id.clone()).chain(tail_id.clone()).collect();
|
||||
let mut tail_messages = tail.map(|tail| tail.payload.messages).unwrap_or_default();
|
||||
let mut tail_changed = false;
|
||||
let mut rewrites: Vec<Rewrite> = Vec::new();
|
||||
for chunk in holding {
|
||||
let mut messages = chunk.payload.messages.clone();
|
||||
let mut changed = false;
|
||||
for op in &ops {
|
||||
changed |= op.apply(&mut messages);
|
||||
}
|
||||
if Some(&chunk.id) == tail_id.as_ref() {
|
||||
(tail_messages, tail_changed) = (messages, changed);
|
||||
} else if changed {
|
||||
rewrites.push((vec![chunk.id], messages));
|
||||
}
|
||||
}
|
||||
if !fresh.is_empty() || tail_changed {
|
||||
tail_messages.extend(fresh);
|
||||
rewrites.push((tail_id.into_iter().collect(), tail_messages));
|
||||
}
|
||||
write_rewrites(embedder, index, dimensions, room, rewrites, &standing).map(Some)
|
||||
}
|
||||
|
||||
enum Outcome {
|
||||
Done(ChatReport),
|
||||
Refused(RefusedRoom),
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn ingest_room<E: Embedder, S: SourceIndex, H: RoomHistory>(
|
||||
embedder: &mut E,
|
||||
index: &mut S,
|
||||
history: &mut H,
|
||||
dimensions: u32,
|
||||
facts: &RoomFacts,
|
||||
out: &mut ChatRefreshed,
|
||||
) -> Result<Outcome> {
|
||||
let started = Instant::now();
|
||||
let room_id = facts.room_id.as_str();
|
||||
let (key, version) = (cursor_key(room_id), chat_version());
|
||||
let cursors = read_cursors(index)?;
|
||||
let mut may_rebuild = false;
|
||||
let mut after: Option<String> = match cursors.get(&key).map(|value| parse_cursor(value)) {
|
||||
None => None,
|
||||
Some(Some((held, event))) if held == version => {
|
||||
may_rebuild = true;
|
||||
Some(event.to_owned()).filter(|event| !event.is_empty())
|
||||
}
|
||||
// Another chunker or scrub version, or a value nobody here wrote: this room alone starts over.
|
||||
Some(_) => {
|
||||
index.delete_chat(room_id, None).with_context(|| format!("deleting the chunks of {room_id} for a re-ingest"))?;
|
||||
None
|
||||
}
|
||||
};
|
||||
if after.is_none() {
|
||||
put_cursor(index, &key, &format!("{version}:"))?;
|
||||
}
|
||||
let windows = cursors.get(&purge_key(room_id)).map(|value| parse_windows(value)).transpose()?.unwrap_or_default();
|
||||
let mut report = ChatReport::empty(room_id, facts.alias.clone());
|
||||
// The room's name is embedded and stored in every chunk, so it passes the scrub like a body.
|
||||
let label = match &facts.name {
|
||||
Some(name) => {
|
||||
let clean = scrub(name);
|
||||
for kind in clean.hits {
|
||||
report.scrubbed += 1;
|
||||
out.scrubbed.push(ScrubHit { room_id: room_id.to_owned(), event_id: "(name)".to_owned(), kind });
|
||||
}
|
||||
clean.text
|
||||
}
|
||||
None => facts.alias.clone().unwrap_or_else(|| room_id.to_owned()),
|
||||
};
|
||||
let room = Room { facts, label, windows };
|
||||
apply_purge(embedder, index, dimensions, &room, &mut report)?;
|
||||
let mut restarted = false;
|
||||
loop {
|
||||
let Some(page) = history.events_after(room_id, after.as_deref(), CHAT_PAGE)? else {
|
||||
ensure!(after.is_some() && !restarted, "the server does not know the start of {room_id}'s history");
|
||||
restarted = true;
|
||||
purge_room(index, room_id)?;
|
||||
put_cursor(index, &key, &format!("{version}:"))?;
|
||||
after = None;
|
||||
continue;
|
||||
};
|
||||
let Some(last) = page.events.last().map(|event| event.event_id.clone()) else { break };
|
||||
// A refusing event refuses the room before anything of the page is chunked or embedded.
|
||||
if let Some((event, reason)) = page.events.iter().find_map(|event| refuse_event(event).map(|reason| (event, reason))) {
|
||||
purge_room(index, room_id)?;
|
||||
return Ok(Outcome::Refused(RefusedRoom { room_id: room_id.to_owned(), reason, event_id: Some(event.event_id.clone()) }));
|
||||
}
|
||||
report.first.get_or_insert_with(|| page.events[0].event_id.clone());
|
||||
report.events += u32::try_from(page.events.len()).unwrap_or(u32::MAX);
|
||||
let folded = fold(&room, &page.events, &mut report, out);
|
||||
let Some(written) = write_page(embedder, index, dimensions, &room, folded, may_rebuild)? else {
|
||||
// Start this room over from the history; nothing it read is trusted to be complete.
|
||||
purge_room(index, room_id)?;
|
||||
put_cursor(index, &key, &format!("{version}:"))?;
|
||||
(after, may_rebuild) = (None, false);
|
||||
report = ChatReport { scrubbed: report.scrubbed, ..ChatReport::empty(room_id, facts.alias.clone()) };
|
||||
continue;
|
||||
};
|
||||
report.chunks += written;
|
||||
put_cursor(index, &key, &format!("{version}:{last}"))?;
|
||||
report.last = Some(last.clone());
|
||||
after = Some(last);
|
||||
if !page.more {
|
||||
break;
|
||||
}
|
||||
}
|
||||
report.seconds = started.elapsed().as_secs_f64();
|
||||
Ok(Outcome::Done(report))
|
||||
}
|
||||
|
||||
/// Brings the chat corpus to the reader's joined rooms. The collection must already exist and be
|
||||
/// Huginn's, built under this embedder's model: the forge refresh creates and rebuilds it, and this never does.
|
||||
/// A joined room the scope refuses is purged and named; a room the reader left is retired; the rest
|
||||
/// are ingested from their cursors. One room's failure is journaled and the others go on.
|
||||
/// `journal` receives every line the run prints: reports, scrub hits, refusals and failures.
|
||||
pub fn refresh_chat<E: Embedder, S: SourceIndex, H: RoomHistory>(
|
||||
embedder: &mut E,
|
||||
index: &mut S,
|
||||
history: &mut H,
|
||||
estate: &Estate,
|
||||
journal: &mut dyn FnMut(String),
|
||||
) -> Result<ChatRefreshed> {
|
||||
let DescribedSources::Labelled { meta, .. } = index.describe_sources(INDEX_CALL_TIMEOUT).context("describing the source collection")? else {
|
||||
bail!("the source collection is absent or not Huginn's; the forge refresh creates it and chat never does");
|
||||
};
|
||||
ensure!(meta.corpus == CORPUS, "the source collection was built for corpus {} and chat ingest writes {CORPUS}; the forge refresh rebuilds it", meta.corpus);
|
||||
embedder.built_under(Some(&meta.model_identity()));
|
||||
let identity = embedder.model_identity(BULK_EMBED_TIMEOUT).context("reading the embedding model's identity")?;
|
||||
ensure!(meta.model_identity() == identity, "the source collection was built under {} {} and chat embeds under {} {}", meta.model, meta.model_digest, identity.name, identity.digest);
|
||||
|
||||
let mut out = ChatRefreshed::default();
|
||||
let mut joined = history.joined_rooms().context("listing the reader's joined rooms")?;
|
||||
let mut seen = HashSet::new();
|
||||
joined.retain(|room| seen.insert(room.clone()));
|
||||
let mut keep: BTreeSet<String> = joined.iter().cloned().collect();
|
||||
let mut admitted: Vec<RoomFacts> = Vec::new();
|
||||
for room_id in &joined {
|
||||
let verdict = if estate.holds_id(room_id) {
|
||||
Ok(Err(Refusal::Estate))
|
||||
} else {
|
||||
history.facts(room_id).map(|facts| admit_room(&facts, estate).map(|()| facts))
|
||||
};
|
||||
match verdict {
|
||||
Ok(Ok(facts)) => admitted.push(facts),
|
||||
Ok(Err(reason)) => {
|
||||
// Not kept, so the retirement below purges whatever it held.
|
||||
keep.remove(room_id);
|
||||
journal(format!("refused: {room_id} {reason}"));
|
||||
out.refused.push(RefusedRoom { room_id: room_id.clone(), reason, event_id: None });
|
||||
}
|
||||
// An error is not a reclassification: the room keeps its points and cursor, and the report says why.
|
||||
Err(error) => {
|
||||
let report = ChatReport::failed(room_id, None, &error.context("reading the room's facts"));
|
||||
journal(report.to_string());
|
||||
out.failed.push(room_id.clone());
|
||||
out.rooms.push(report);
|
||||
}
|
||||
}
|
||||
}
|
||||
match retire_chat(index, &keep) {
|
||||
Ok(done) => {
|
||||
for (room, why) in done.failed {
|
||||
journal(format!("huginn-sources: retiring {room} failed, the others go on: {why}"));
|
||||
out.failed.push(room);
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
journal(format!("huginn-sources: retiring rooms failed: {error:#}"));
|
||||
out.failed.push("retire".to_owned());
|
||||
}
|
||||
}
|
||||
for facts in admitted {
|
||||
match ingest_room(embedder, index, history, identity.dimensions, &facts, &mut out) {
|
||||
Ok(Outcome::Done(report)) => {
|
||||
journal(report.to_string());
|
||||
out.rooms.push(report);
|
||||
}
|
||||
Ok(Outcome::Refused(refused)) => {
|
||||
journal(format!("refused: {} {} [{}]", refused.room_id, refused.reason, refused.event_id.as_deref().unwrap_or("")));
|
||||
out.refused.push(refused);
|
||||
}
|
||||
Err(error) => {
|
||||
let report = ChatReport::failed(&facts.room_id, facts.alias.clone(), &error);
|
||||
journal(report.to_string());
|
||||
out.failed.push(facts.room_id.clone());
|
||||
out.rooms.push(report);
|
||||
}
|
||||
}
|
||||
}
|
||||
for hit in &out.scrubbed {
|
||||
journal(format!("scrubbed: {} {} {}", hit.room_id, hit.event_id, hit.kind));
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
166
crates/huginn-daemon/src/sources/chat_chunk.rs
Normal file
166
crates/huginn-daemon/src/sources/chat_chunk.rs
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
//! The chat chunker: greedy, deterministic and local. A chunk is consecutive
|
||||
//! messages of one room; it closes at a time gap, at a message count, or when
|
||||
//! its embedded text would pass a character bound. A message is never split.
|
||||
|
||||
use chrono::DateTime;
|
||||
use huginn_mind::redact::SCRUB_VERSION;
|
||||
use huginn_mind::ChatMessage;
|
||||
|
||||
use super::ChatPayload;
|
||||
use crate::index::point_id;
|
||||
|
||||
/// Bumped whenever chunk boundaries or the embedded text would change for the
|
||||
/// same messages. Part of `chat_version`.
|
||||
pub const CHAT_CHUNKER_VERSION: u32 = 1;
|
||||
|
||||
/// A chunk closes when the next message is more than this much later.
|
||||
pub const CHAT_GAP_MS: u64 = 30 * 60 * 1000;
|
||||
/// The most messages a chunk holds.
|
||||
pub const CHAT_MESSAGES_MAX: usize = 24;
|
||||
/// The most characters a chunk's embedded text holds.
|
||||
pub const CHAT_CHARS_MAX: usize = 1_600;
|
||||
/// The most characters a stored message body holds, marker included.
|
||||
pub const CHAT_BODY_CHARS_MAX: usize = 1_200;
|
||||
|
||||
/// Ends a body that was cut.
|
||||
const CUT_MARKER: &str = "…[cut]";
|
||||
|
||||
/// What the room's cursor and chunk ids are stamped with: the chunker and the
|
||||
/// scrub. Either changing re-ingests the room.
|
||||
pub fn chat_version() -> String {
|
||||
format!("{CHAT_CHUNKER_VERSION}.{SCRUB_VERSION}")
|
||||
}
|
||||
|
||||
/// A chunk's id: where it starts and under which version it was cut.
|
||||
pub fn chunk_id(room_id: &str, first_event_id: &str) -> String {
|
||||
point_id(&format!("{room_id}\0{first_event_id}\0{}", chat_version()))
|
||||
}
|
||||
|
||||
/// A scrubbed body cut to `CHAT_BODY_CHARS_MAX` characters on a character boundary, with a marker.
|
||||
pub fn cut_body(body: &str) -> String {
|
||||
if body.chars().count() <= CHAT_BODY_CHARS_MAX {
|
||||
return body.to_owned();
|
||||
}
|
||||
let keep = CHAT_BODY_CHARS_MAX - CUT_MARKER.chars().count();
|
||||
body.chars().take(keep).chain(CUT_MARKER.chars()).collect()
|
||||
}
|
||||
|
||||
fn line_of(message: &ChatMessage) -> String {
|
||||
let when = DateTime::from_timestamp_millis(i64::try_from(message.ts_ms).unwrap_or(i64::MAX)).map_or_else(String::new, |at| at.format("%Y-%m-%d %H:%M").to_string());
|
||||
format!("{when} {}: {}", message.sender, message.body)
|
||||
}
|
||||
|
||||
/// What is embedded for a chunk: the room's label (its name, else its alias,
|
||||
/// else its id), then one `YYYY-MM-DD HH:MM sender: body` line per message, UTC.
|
||||
pub fn embedded_text(label: &str, messages: &[ChatMessage]) -> String {
|
||||
let mut text = label.to_owned();
|
||||
for message in messages {
|
||||
text.push('\n');
|
||||
text.push_str(&line_of(message));
|
||||
}
|
||||
text
|
||||
}
|
||||
|
||||
/// `messages` (one room, in order) cut into chunks, each a run of consecutive messages.
|
||||
pub fn chunk_messages(label: &str, messages: &[ChatMessage]) -> Vec<Vec<ChatMessage>> {
|
||||
let mut chunks: Vec<Vec<ChatMessage>> = Vec::new();
|
||||
let mut chars = 0_usize;
|
||||
for message in messages {
|
||||
let line = line_of(message).chars().count() + 1;
|
||||
let closes = chunks.last().is_some_and(|open| {
|
||||
let last = open.last().map_or(0, |last| last.ts_ms);
|
||||
message.ts_ms.saturating_sub(last) > CHAT_GAP_MS || open.len() >= CHAT_MESSAGES_MAX || chars + line > CHAT_CHARS_MAX
|
||||
});
|
||||
if closes || chunks.is_empty() {
|
||||
chunks.push(Vec::new());
|
||||
chars = label.chars().count();
|
||||
}
|
||||
chunks.last_mut().expect("a chunk is open").push(message.clone());
|
||||
chars += line;
|
||||
}
|
||||
chunks
|
||||
}
|
||||
|
||||
/// The point payload of one chunk.
|
||||
pub fn chunk_payload(room_id: &str, alias: Option<&str>, label: &str, messages: Vec<ChatMessage>) -> ChatPayload {
|
||||
ChatPayload { room_id: room_id.to_owned(), room_alias: alias.map(str::to_owned), text: embedded_text(label, &messages), messages }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn message(n: u64, ts_ms: u64, body: &str) -> ChatMessage {
|
||||
ChatMessage { event_id: format!("$e{n}"), sender: "@a:s".into(), ts_ms, body: body.into() }
|
||||
}
|
||||
|
||||
fn ids(chunks: &[Vec<ChatMessage>]) -> Vec<Vec<&str>> {
|
||||
chunks.iter().map(|chunk| chunk.iter().map(|message| message.event_id.as_str()).collect()).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_gap_a_count_and_a_character_bound_each_close_a_chunk() {
|
||||
let gap = [message(1, 0, "a"), message(2, CHAT_GAP_MS, "b"), message(3, 2 * CHAT_GAP_MS + 1, "c")];
|
||||
assert_eq!(ids(&chunk_messages("r", &gap)), [["$e1", "$e2"].as_slice(), ["$e3"].as_slice()], "a gap of exactly the bound does not close");
|
||||
let many: Vec<ChatMessage> = (0..25).map(|n| message(n, n, "x")).collect();
|
||||
let chunks = chunk_messages("r", &many);
|
||||
assert_eq!((chunks[0].len(), chunks[1].len()), (CHAT_MESSAGES_MAX, 1));
|
||||
let long: Vec<ChatMessage> = (0..4).map(|n| message(n, n, &"w".repeat(500))).collect();
|
||||
let chunks = chunk_messages("r", &long);
|
||||
assert!(chunks.iter().all(|chunk| embedded_text("r", chunk).chars().count() <= CHAT_CHARS_MAX), "{:?}", ids(&chunks));
|
||||
assert!(chunks.len() > 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_embedded_text_is_the_label_then_utc_lines() {
|
||||
let text = embedded_text("Swarm", &[message(1, 1_700_000_000_000, "hello")]);
|
||||
assert_eq!(text, "Swarm\n2023-11-14 22:13 @a:s: hello");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_long_body_is_cut_at_the_bound_with_a_marker_on_a_char_boundary_and_a_short_one_is_whole() {
|
||||
let cut = cut_body(&"é".repeat(CHAT_BODY_CHARS_MAX + 50));
|
||||
assert_eq!(cut.chars().count(), CHAT_BODY_CHARS_MAX);
|
||||
assert!(cut.ends_with(CUT_MARKER));
|
||||
assert_eq!(cut_body(&"é".repeat(CHAT_BODY_CHARS_MAX)), "é".repeat(CHAT_BODY_CHARS_MAX));
|
||||
let one = chunk_messages("r", &[message(1, 0, &cut)]);
|
||||
assert_eq!(one.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_chunk_id_is_its_room_first_event_and_version() {
|
||||
assert_eq!(chunk_id("!r:s", "$a"), chunk_id("!r:s", "$a"));
|
||||
assert_ne!(chunk_id("!r:s", "$a"), chunk_id("!r:s", "$b"));
|
||||
assert_ne!(chunk_id("!r:s", "$a"), chunk_id("!q:s", "$a"));
|
||||
assert_eq!(chat_version(), format!("{CHAT_CHUNKER_VERSION}.{SCRUB_VERSION}"));
|
||||
}
|
||||
|
||||
/// Chunking the whole sequence equals re-chunking the open tail with each new batch appended.
|
||||
#[test]
|
||||
fn appending_to_the_open_tail_gives_the_boundaries_of_chunking_all_at_once() {
|
||||
let mut seed = 0x2545_f491_4f6c_dd1d_u64;
|
||||
let mut next = move |bound: u64| {
|
||||
seed ^= seed << 13;
|
||||
seed ^= seed >> 7;
|
||||
seed ^= seed << 17;
|
||||
seed % bound
|
||||
};
|
||||
let mut ts = 0;
|
||||
let all: Vec<ChatMessage> = (0..300)
|
||||
.map(|n| {
|
||||
ts += if next(7) == 0 { CHAT_GAP_MS + 1 } else { next(60_000) };
|
||||
message(n, ts, &"z".repeat(next(700) as usize + 1))
|
||||
})
|
||||
.collect();
|
||||
let whole = chunk_messages("r", &all);
|
||||
for step in [1_usize, 5, 17, 100] {
|
||||
let mut held: Vec<Vec<ChatMessage>> = Vec::new();
|
||||
for batch in all.chunks(step) {
|
||||
let mut tail: Vec<ChatMessage> = held.pop().unwrap_or_default();
|
||||
tail.extend_from_slice(batch);
|
||||
held.extend(chunk_messages("r", &tail));
|
||||
}
|
||||
assert_eq!(held, whole, "step {step}");
|
||||
}
|
||||
}
|
||||
}
|
||||
103
crates/huginn-daemon/src/sources/chat_purge.rs
Normal file
103
crates/huginn-daemon/src/sources/chat_purge.rs
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
//! A steward's purge of a room's time window. The tombstone is the one chat
|
||||
//! state that is not derived from Synapse: a durable request, kept in the
|
||||
//! collection's cursors object under `chat-purge:<room_id>`, that must outlive
|
||||
//! the points it removes, so a re-ingest never brings the window back.
|
||||
//!
|
||||
//! `request_purge` is the only writer; `refresh_chat` reads the tombstone and
|
||||
//! applies it at its next run. The code does not authenticate the steward: whoever
|
||||
//! can run the job as the unit's user (the host admin) may ask, and the request
|
||||
//! carries a ref in the job's journal for a trace. Tombstones live in the
|
||||
//! collection, so a full collection rebuild forgets them; re-issue them from
|
||||
//! their refs.
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
|
||||
use super::chat::CHAT_PURGE_PREFIX;
|
||||
use super::{DescribedSources, SourceIndex};
|
||||
use crate::index::INDEX_CALL_TIMEOUT;
|
||||
|
||||
/// The most windows a room's tombstone holds.
|
||||
pub const PURGE_WINDOWS_MAX: usize = 32;
|
||||
|
||||
/// Messages with `from_ms <= ts_ms < to_ms` are purged.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct PurgeWindow {
|
||||
pub from_ms: u64,
|
||||
pub to_ms: u64,
|
||||
}
|
||||
|
||||
impl PurgeWindow {
|
||||
pub fn covers(&self, ts_ms: u64) -> bool {
|
||||
self.from_ms <= ts_ms && ts_ms < self.to_ms
|
||||
}
|
||||
}
|
||||
|
||||
/// The tombstone key of a room.
|
||||
pub fn purge_key(room_id: &str) -> String {
|
||||
format!("{CHAT_PURGE_PREFIX}{room_id}")
|
||||
}
|
||||
|
||||
/// A tombstone value, `<from>..<to>[,<from>..<to>...]`.
|
||||
pub fn parse_windows(value: &str) -> Result<Vec<PurgeWindow>> {
|
||||
value
|
||||
.split(',')
|
||||
.map(|part| {
|
||||
let (from, to) = part.split_once("..").with_context(|| format!("purge window {part:?} is not <from>..<to>"))?;
|
||||
let window = PurgeWindow { from_ms: from.parse().context("a purge window's start is not milliseconds")?, to_ms: to.parse().context("a purge window's end is not milliseconds")? };
|
||||
if window.from_ms >= window.to_ms {
|
||||
bail!("purge window {part:?} is empty");
|
||||
}
|
||||
Ok(window)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Windows sorted, with overlapping and touching ones joined.
|
||||
pub fn merge(mut windows: Vec<PurgeWindow>) -> Vec<PurgeWindow> {
|
||||
windows.sort();
|
||||
let mut merged: Vec<PurgeWindow> = Vec::new();
|
||||
for window in windows {
|
||||
match merged.last_mut() {
|
||||
Some(last) if window.from_ms <= last.to_ms => last.to_ms = last.to_ms.max(window.to_ms),
|
||||
_ => merged.push(window),
|
||||
}
|
||||
}
|
||||
merged
|
||||
}
|
||||
|
||||
pub fn encode(windows: &[PurgeWindow]) -> String {
|
||||
windows.iter().map(|window| format!("{}..{}", window.from_ms, window.to_ms)).collect::<Vec<_>>().join(",")
|
||||
}
|
||||
|
||||
/// Whether any window covers `ts_ms`.
|
||||
pub fn covers(windows: &[PurgeWindow], ts_ms: u64) -> bool {
|
||||
windows.iter().any(|window| window.covers(ts_ms))
|
||||
}
|
||||
|
||||
/// Records that `window` of `room_id` must leave the index and never return. Writes the cursors
|
||||
/// object only if the tombstone changed, and refuses a room's 33rd window. Writes no point and
|
||||
/// embeds nothing; the next `refresh_chat` applies it. The job's `chat-purge` subcommand calls
|
||||
/// it under the job's lock, because `set_cursors` replaces the cursors object whole.
|
||||
/// Returns whether the tombstone changed.
|
||||
pub fn request_purge<S: SourceIndex>(index: &mut S, room_id: &str, window: PurgeWindow) -> Result<bool> {
|
||||
if window.from_ms >= window.to_ms {
|
||||
bail!("a purge window must start before it ends");
|
||||
}
|
||||
let DescribedSources::Labelled { mut cursors, .. } = index.describe_sources(INDEX_CALL_TIMEOUT).context("describing the source collection")? else {
|
||||
bail!("the source collection is absent or not Huginn's; the forge refresh creates it");
|
||||
};
|
||||
let key = purge_key(room_id);
|
||||
let mut windows = cursors.get(&key).map(|held| parse_windows(held)).transpose()?.unwrap_or_default();
|
||||
windows.push(window);
|
||||
let windows = merge(windows);
|
||||
if windows.len() > PURGE_WINDOWS_MAX {
|
||||
bail!("room {room_id} already holds {PURGE_WINDOWS_MAX} purge windows; widen one instead");
|
||||
}
|
||||
let encoded = encode(&windows);
|
||||
if cursors.get(&key) == Some(&encoded) {
|
||||
return Ok(false);
|
||||
}
|
||||
cursors.insert(key, encoded);
|
||||
index.set_cursors(&cursors).context("writing the purge tombstone")?;
|
||||
Ok(true)
|
||||
}
|
||||
964
crates/huginn-daemon/src/sources/chat_tests.rs
Normal file
964
crates/huginn-daemon/src/sources/chat_tests.rs
Normal file
|
|
@ -0,0 +1,964 @@
|
|||
//! Chat ingest against a scripted room history, a fake vector store and the fake embedder.
|
||||
//! The refusals come first: what must never be read, embedded or stored.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Result, bail};
|
||||
use huginn_mind::ChatMessage;
|
||||
use huginn_mind::redact::ScrubKind;
|
||||
|
||||
use super::chat::*;
|
||||
use super::chat_chunk::{CHAT_BODY_CHARS_MAX, chat_version, chunk_id};
|
||||
use super::chat_purge::{PURGE_WINDOWS_MAX, PurgeWindow, request_purge};
|
||||
use super::room_scope::{ESTATE_ALIASES, Estate, Refusal, RoomFacts};
|
||||
use super::*;
|
||||
use crate::index::MANAGED_BY;
|
||||
use crate::index::fakes::{FakeEmbedder, FakeSources};
|
||||
|
||||
// ---- the rig ----
|
||||
|
||||
fn plain(id: &str) -> RoomFacts {
|
||||
RoomFacts { room_id: id.into(), alias: None, alt_aliases: vec![], name: None, encrypted: false, direct: false, bridge: None }
|
||||
}
|
||||
|
||||
fn estate(ids: &[&str]) -> Estate {
|
||||
Estate::new(ids.iter().chain(["!pad1:s", "!pad2:s", "!pad3:s"].iter()).take(3).map(|id| (*id).to_owned()).collect()).unwrap()
|
||||
}
|
||||
|
||||
struct Scripted {
|
||||
facts: RoomFacts,
|
||||
events: Vec<RoomEvent>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct FakeHistory {
|
||||
joined: Vec<String>,
|
||||
rooms: BTreeMap<String, Scripted>,
|
||||
calls: Vec<String>,
|
||||
/// Rooms whose state cannot be read.
|
||||
facts_fail: BTreeSet<String>,
|
||||
/// Serve an event a redaction targets as the server does, without its content.
|
||||
prune: bool,
|
||||
}
|
||||
|
||||
impl FakeHistory {
|
||||
fn room(&mut self, id: &str, events: Vec<RoomEvent>) -> &mut Self {
|
||||
self.rooms.insert(id.into(), Scripted { facts: plain(id), events });
|
||||
self.joined.push(id.into());
|
||||
self
|
||||
}
|
||||
|
||||
fn facts(&mut self, id: &str, edit: impl FnOnce(&mut RoomFacts)) -> &mut Self {
|
||||
edit(&mut self.rooms.get_mut(id).unwrap().facts);
|
||||
self
|
||||
}
|
||||
|
||||
fn push(&mut self, id: &str, events: Vec<RoomEvent>) {
|
||||
self.rooms.get_mut(id).unwrap().events.extend(events);
|
||||
}
|
||||
|
||||
fn called(&self, prefix: &str, room: &str) -> bool {
|
||||
self.calls.iter().any(|call| call == &format!("{prefix}:{room}"))
|
||||
}
|
||||
}
|
||||
|
||||
impl RoomHistory for FakeHistory {
|
||||
fn joined_rooms(&mut self) -> Result<Vec<String>> {
|
||||
self.calls.push("joined".into());
|
||||
Ok(self.joined.clone())
|
||||
}
|
||||
|
||||
fn facts(&mut self, room_id: &str) -> Result<RoomFacts> {
|
||||
self.calls.push(format!("facts:{room_id}"));
|
||||
if self.facts_fail.contains(room_id) {
|
||||
bail!("the server did not answer for the state of {room_id}");
|
||||
}
|
||||
match self.rooms.get(room_id) {
|
||||
Some(room) => Ok(room.facts.clone()),
|
||||
None => bail!("no such room"),
|
||||
}
|
||||
}
|
||||
|
||||
fn events_after(&mut self, room_id: &str, after: Option<&str>, limit: u32) -> Result<Option<EventPage>> {
|
||||
self.calls.push(format!("events:{room_id}"));
|
||||
let Some(room) = self.rooms.get(room_id) else { bail!("no such room") };
|
||||
let from = match after {
|
||||
None => 0,
|
||||
Some(after) => match room.events.iter().position(|event| event.event_id == after) {
|
||||
Some(at) => at + 1,
|
||||
None => return Ok(None),
|
||||
},
|
||||
};
|
||||
let redacted: BTreeSet<&str> = room.events.iter().filter_map(|event| if let EventKind::Redaction { target } = &event.kind { Some(target.as_str()) } else { None }).collect();
|
||||
let served: Vec<RoomEvent> = room.events[from..]
|
||||
.iter()
|
||||
.map(|event| if self.prune && redacted.contains(event.event_id.as_str()) { RoomEvent { kind: EventKind::Other, content_keys: vec![], ..event.clone() } } else { event.clone() })
|
||||
.collect();
|
||||
let take = served.len().min(limit as usize);
|
||||
Ok(Some(EventPage { events: served[..take].to_vec(), more: take < served.len() }))
|
||||
}
|
||||
}
|
||||
|
||||
fn event(id: &str, sender: &str, ts_ms: u64, kind: EventKind) -> RoomEvent {
|
||||
RoomEvent { event_id: id.into(), sender: sender.into(), ts_ms, content_keys: vec!["body".into(), "msgtype".into()], kind }
|
||||
}
|
||||
|
||||
fn said(id: &str, ts_ms: u64, body: &str) -> RoomEvent {
|
||||
event(id, "@a:s", ts_ms, EventKind::Message { body: body.into() })
|
||||
}
|
||||
|
||||
fn edit(id: &str, sender: &str, ts_ms: u64, target: &str, body: &str) -> RoomEvent {
|
||||
event(id, sender, ts_ms, EventKind::Edit { target: target.into(), body: body.into() })
|
||||
}
|
||||
|
||||
fn redaction(id: &str, ts_ms: u64, target: &str) -> RoomEvent {
|
||||
event(id, "@a:s", ts_ms, EventKind::Redaction { target: target.into() })
|
||||
}
|
||||
|
||||
struct Rig {
|
||||
embedder: FakeEmbedder,
|
||||
index: FakeSources,
|
||||
estate: Estate,
|
||||
}
|
||||
|
||||
impl Rig {
|
||||
fn new() -> Self {
|
||||
let mut embedder = FakeEmbedder::new("d1");
|
||||
let index = FakeSources::default();
|
||||
let identity = crate::index::Embedder::model_identity(&mut embedder, Duration::from_secs(1)).unwrap();
|
||||
let meta = SourcesMeta {
|
||||
managed_by: MANAGED_BY.into(),
|
||||
corpus: CORPUS.into(),
|
||||
model: identity.name,
|
||||
model_digest: identity.digest,
|
||||
dimensions: identity.dimensions,
|
||||
chunker_version: chunk::CHUNKER_VERSION,
|
||||
};
|
||||
index.plant(DescribedSources::Labelled { meta, cursors: BTreeMap::new() });
|
||||
Self { embedder, index, estate: estate(&[]) }
|
||||
}
|
||||
|
||||
fn run(&mut self, history: &mut FakeHistory) -> (Result<ChatRefreshed>, Vec<String>) {
|
||||
let mut journal = Vec::new();
|
||||
let out = refresh_chat(&mut self.embedder, &mut self.index, history, &self.estate, &mut |line| journal.push(line));
|
||||
(out, journal)
|
||||
}
|
||||
|
||||
fn ok(&mut self, history: &mut FakeHistory) -> ChatRefreshed {
|
||||
self.run(history).0.unwrap()
|
||||
}
|
||||
|
||||
fn points(&self) -> Vec<ChatPoint> {
|
||||
self.index.state.lock().unwrap().chat.values().cloned().collect()
|
||||
}
|
||||
|
||||
fn ids(&self) -> BTreeSet<String> {
|
||||
self.index.state.lock().unwrap().chat.keys().cloned().collect()
|
||||
}
|
||||
|
||||
fn bodies(&self) -> Vec<String> {
|
||||
let mut bodies: Vec<String> = self.points().iter().flat_map(|point| point.payload.messages.iter().map(|message| message.body.clone())).collect();
|
||||
bodies.sort();
|
||||
bodies
|
||||
}
|
||||
|
||||
fn held_ids(&self, room: &str) -> Vec<String> {
|
||||
let mut ids: Vec<String> =
|
||||
self.points().iter().filter(|point| point.payload.room_id == room).flat_map(|point| point.payload.messages.iter().map(|message| message.event_id.clone())).collect();
|
||||
ids.sort();
|
||||
ids
|
||||
}
|
||||
|
||||
fn cursor(&self, room: &str) -> Option<String> {
|
||||
self.index.cursors().get(&cursor_key(room)).cloned()
|
||||
}
|
||||
|
||||
fn embedded(&self) -> Vec<String> {
|
||||
self.embedder.texts()
|
||||
}
|
||||
|
||||
fn seed(&mut self, room: &str, event_id: &str, ts: u64) {
|
||||
let mut seeded = vec![ChatPoint {
|
||||
id: chunk_id(room, event_id),
|
||||
vector: vec![1.0; 4],
|
||||
payload: chunk_payload_of(room, &[(event_id, ts)]),
|
||||
}];
|
||||
self.index.upsert_chat(&seeded.split_off(0)).unwrap();
|
||||
let mut cursors = self.index.cursors();
|
||||
cursors.insert(cursor_key(room), format!("{}:{event_id}", chat_version()));
|
||||
self.index.set_cursors(&cursors).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
fn chunk_payload_of(room: &str, messages: &[(&str, u64)]) -> ChatPayload {
|
||||
let messages: Vec<ChatMessage> = messages.iter().map(|(id, ts)| ChatMessage { event_id: (*id).into(), sender: "@a:s".into(), ts_ms: *ts, body: format!("seeded {id}") }).collect();
|
||||
ChatPayload { room_id: room.into(), room_alias: None, text: "seeded".into(), messages }
|
||||
}
|
||||
|
||||
fn tombstone(rig: &Rig, room: &str) -> Option<String> {
|
||||
rig.index.cursors().get(&format!("{CHAT_PURGE_PREFIX}{room}")).cloned()
|
||||
}
|
||||
|
||||
// ---- scope: refusals first ----
|
||||
|
||||
#[test]
|
||||
fn only_joined_rooms_are_looked_at_and_a_join_rule_is_not_the_test() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!a:s", vec![said("$a1", 1, "in a")]).room("!b:s", vec![said("$b1", 2, "in b")]);
|
||||
// A room the reader has not joined: scripted, but never listed.
|
||||
history.rooms.insert("!c:s".into(), Scripted { facts: plain("!c:s"), events: vec![said("$c1", 3, "in c")] });
|
||||
let mut rig = Rig::new();
|
||||
let done = rig.ok(&mut history);
|
||||
assert!(done.is_clean());
|
||||
assert!(!history.calls.iter().any(|call| call.contains("!c:s")), "{:?}", history.calls);
|
||||
assert_eq!(rig.held_ids("!a:s"), ["$a1"]);
|
||||
assert_eq!(rig.held_ids("!b:s"), ["$b1"]);
|
||||
assert!(rig.held_ids("!c:s").is_empty() && done.rooms.iter().all(|report| report.room_id != "!c:s"));
|
||||
let code = include_str!("room_scope.rs");
|
||||
assert!(!code[..code.find("#[cfg(test)]").unwrap()].contains(&["join", "_rule"].concat()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_dm_room_is_refused_unread_unindexed_and_purged() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!dm:s", vec![said("$d1", 1, "private")]).facts("!dm:s", |facts| facts.direct = true);
|
||||
let mut rig = Rig::new();
|
||||
rig.seed("!dm:s", "$old", 5);
|
||||
let (done, journal) = rig.run(&mut history);
|
||||
let done = done.unwrap();
|
||||
assert!(!history.called("events", "!dm:s"));
|
||||
assert!(rig.points().is_empty() && rig.cursor("!dm:s").is_none() && rig.embedded().is_empty());
|
||||
assert_eq!(done.refused, [RefusedRoom { room_id: "!dm:s".into(), reason: Refusal::Direct, event_id: None }]);
|
||||
assert!(!done.is_clean() && journal.iter().any(|line| line == "refused: !dm:s direct room"), "{journal:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn each_estate_room_is_refused_by_id_and_by_alias_and_an_excluded_room_in_the_index_is_purged() {
|
||||
let mut rig = Rig::new();
|
||||
rig.estate = estate(&["!byid:s"]);
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!byid:s", vec![said("$i1", 1, "x")]);
|
||||
for (n, alias) in ESTATE_ALIASES.iter().map(|alias| alias.to_string()).chain(["#USERS:GAMECULT.ORG".to_owned()]).enumerate() {
|
||||
history.room(&format!("!al{n}:s"), vec![said("$x", 1, "x")]).facts(&format!("!al{n}:s"), |facts| facts.alias = Some(alias));
|
||||
}
|
||||
history.room("!alt:s", vec![said("$x", 1, "x")]).facts("!alt:s", |facts| facts.alt_aliases = vec!["#other:s".into(), "#patrons:gamecult.org".into()]);
|
||||
history.room("!ok:s", vec![said("$ok1", 1, "fine")]);
|
||||
history.room("!guide:s", vec![said("$g1", 1, "fine")]).facts("!guide:s", |facts| facts.alias = Some("#users-guide:gamecult.org".into()));
|
||||
history.room("!club:s", vec![said("$c1", 1, "fine")]).facts("!club:s", |facts| facts.alias = Some("#patrons-club:gamecult.org".into()));
|
||||
for room in ["!byid:s", "!al0:s", "!al1:s", "!al2:s", "!al3:s", "!alt:s"] {
|
||||
rig.seed(room, "$old", 5);
|
||||
}
|
||||
let mut cursors = rig.index.cursors();
|
||||
cursors.insert(format!("{CHAT_PURGE_PREFIX}!al0:s"), "1..2".into());
|
||||
rig.index.set_cursors(&cursors).unwrap();
|
||||
let done = rig.ok(&mut history);
|
||||
assert!(!history.called("facts", "!byid:s") && !history.called("events", "!byid:s"));
|
||||
for room in ["!al0:s", "!al1:s", "!al2:s", "!al3:s", "!alt:s"] {
|
||||
assert!(history.called("facts", room) && !history.called("events", room), "{room}");
|
||||
}
|
||||
let refused: BTreeSet<&str> = done.refused.iter().map(|refused| refused.room_id.as_str()).collect();
|
||||
assert_eq!(refused, ["!al0:s", "!al1:s", "!al2:s", "!al3:s", "!alt:s", "!byid:s"].into());
|
||||
assert!(done.refused.iter().all(|refused| refused.reason == Refusal::Estate) && !done.is_clean());
|
||||
let held: BTreeSet<String> = rig.points().iter().map(|point| point.payload.room_id.clone()).collect();
|
||||
assert_eq!(held, ["!guide:s".to_owned(), "!club:s".to_owned(), "!ok:s".to_owned()].into(), "lookalikes and plain rooms are in; every estate room is out");
|
||||
assert!(rig.index.cursors().keys().all(|key| !key.starts_with("chat:!al") && !key.starts_with("chat:!byid") && !key.starts_with("chat:!alt")));
|
||||
assert_eq!(tombstone(&rig, "!al0:s").as_deref(), Some("1..2"), "a tombstone is never touched by a refusal");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_encrypted_room_is_refused_and_an_encrypted_event_in_a_plain_room_is_never_stored() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!enc:s", vec![said("$e1", 1, "x")]).facts("!enc:s", |facts| facts.encrypted = true);
|
||||
history.room("!plain:s", vec![said("$p1", 1, "visible"), event("$p2", "@a:s", 2, EventKind::Other), said("$p3", 3, "also visible")]);
|
||||
let mut rig = Rig::new();
|
||||
let done = rig.ok(&mut history);
|
||||
assert_eq!(done.refused[0].reason, Refusal::Encrypted);
|
||||
assert!(!history.called("events", "!enc:s"));
|
||||
assert_eq!(rig.held_ids("!plain:s"), ["$p1", "$p3"]);
|
||||
assert_eq!(rig.cursor("!plain:s").unwrap(), format!("{}:$p3", chat_version()), "the cursor passes an Other event");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_bridged_room_is_refused_whole_before_any_event_is_read_and_its_chunks_are_purged() {
|
||||
let mut history = FakeHistory::default();
|
||||
for (n, state_type) in ["m.bridge", "uk.half-shot.bridge"].into_iter().enumerate() {
|
||||
let room = format!("!br{n}:s");
|
||||
history.room(&room, vec![said("$x", 1, "x")]).facts(&room, |facts| facts.bridge = Some(state_type.into()));
|
||||
}
|
||||
let mut rig = Rig::new();
|
||||
rig.seed("!br0:s", "$old", 5);
|
||||
let done = rig.ok(&mut history);
|
||||
assert_eq!(done.refused.len(), 2);
|
||||
assert!(done.refused.iter().all(|refused| matches!(refused.reason, Refusal::Bridged(_))));
|
||||
assert!(!history.calls.iter().any(|call| call.starts_with("events:")));
|
||||
assert!(rig.points().is_empty() && rig.index.cursors().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_bridged_platforms_sender_or_content_refuses_the_room_before_anything_is_embedded_and_purges_what_it_held() {
|
||||
let cases = [
|
||||
event("$bad", "@discord_123:s", 2, EventKind::Message { body: "SECRET-BODY-TEXT".into() }),
|
||||
event("$bad", "@_discord_9:s", 2, EventKind::Message { body: "SECRET-BODY-TEXT".into() }),
|
||||
event("$bad", "@discordbot:s", 2, EventKind::Message { body: "SECRET-BODY-TEXT".into() }),
|
||||
event("$bad", "@mautrix_discord:s", 2, EventKind::Message { body: "SECRET-BODY-TEXT".into() }),
|
||||
event("$bad", "@_bridge_discordbot:s", 2, EventKind::Message { body: "SECRET-BODY-TEXT".into() }),
|
||||
RoomEvent { content_keys: vec!["body".into(), "fi.mau.discord".into()], ..said("$bad", 2, "SECRET-BODY-TEXT") },
|
||||
RoomEvent { content_keys: vec!["uk.half-shot.discord".into()], ..said("$bad", 2, "SECRET-BODY-TEXT") },
|
||||
];
|
||||
for bad in cases {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$ok", 1, "before"), bad]);
|
||||
history.room("!other:s", vec![said("$o1", 1, "other room")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.seed("!r:s", "$prev", 0);
|
||||
let (done, journal) = rig.run(&mut history);
|
||||
let done = done.unwrap();
|
||||
assert!(!rig.embedded().iter().any(|text| text.contains("SECRET-BODY-TEXT") || text.contains("before")), "nothing of the page is embedded");
|
||||
assert!(rig.held_ids("!r:s").is_empty() && rig.cursor("!r:s").is_none());
|
||||
assert_eq!(rig.held_ids("!other:s"), ["$o1"]);
|
||||
assert_eq!(done.refused.len(), 1);
|
||||
assert_eq!((done.refused[0].room_id.as_str(), done.refused[0].event_id.as_deref()), ("!r:s", Some("$bad")));
|
||||
assert!(!done.is_clean());
|
||||
assert!(journal.iter().all(|line| !line.contains("SECRET-BODY-TEXT")), "{journal:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_lookalike_of_the_bridged_platform_is_kept() {
|
||||
let mut history = FakeHistory::default();
|
||||
let kept = ["@discourse:s", "@disco:s", "@eureka:s"].map(|sender| event(&format!("${}", &sender[1..6]), sender, 1, EventKind::Message { body: "hi".into() }));
|
||||
let mut keyed = said("$k", 2, "hi");
|
||||
keyed.content_keys.push("discourse".into());
|
||||
let mut events = kept.to_vec();
|
||||
events.push(keyed);
|
||||
history.room("!r:s", events);
|
||||
let mut rig = Rig::new();
|
||||
assert!(rig.ok(&mut history).is_clean());
|
||||
assert_eq!(rig.held_ids("!r:s").len(), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_collection_is_never_created_by_chat() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 1, "x")]);
|
||||
for described in [None, Some(DescribedSources::Absent), Some(DescribedSources::Unlabelled)] {
|
||||
let mut rig = Rig::new();
|
||||
match described {
|
||||
None => rig.index.state.lock().unwrap().described = None,
|
||||
Some(described) => rig.index.plant(described),
|
||||
}
|
||||
let error = format!("{:#}", rig.run(&mut history).0.unwrap_err());
|
||||
assert!(error.contains("forge refresh creates it"), "{error}");
|
||||
assert_eq!(rig.index.state.lock().unwrap().recreated, 0);
|
||||
assert!(rig.points().is_empty() && rig.embedded().is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
// ---- redactions, edits, replays ----
|
||||
|
||||
#[test]
|
||||
fn a_redacted_event_is_absent_after_the_next_refresh_and_a_first_or_only_message_changes_or_drops_its_chunk() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 10, "alpha"), said("$b", 20, "beta"), said("$c", 30, "gamma")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
let first_id = chunk_id("!r:s", "$a");
|
||||
assert_eq!(rig.ids(), [first_id.clone()].into());
|
||||
history.push("!r:s", vec![redaction("$r1", 40, "$b")]);
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.held_ids("!r:s"), ["$a", "$c"]);
|
||||
assert!(!rig.points()[0].payload.text.contains("beta"));
|
||||
history.push("!r:s", vec![redaction("$r2", 50, "$a")]);
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.ids(), [chunk_id("!r:s", "$c")].into(), "the first message left, so the id changed and the old one is gone");
|
||||
history.push("!r:s", vec![redaction("$r3", 60, "$c")]);
|
||||
rig.ok(&mut history);
|
||||
assert!(rig.points().is_empty(), "a chunk the redaction empties is deleted");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_redaction_in_the_same_page_as_its_message_never_reaches_the_embedder() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 10, "keep me"), said("$gone", 20, "REDACTED-BODY"), redaction("$r", 30, "$gone")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
assert!(rig.embedded().iter().all(|text| !text.contains("REDACTED-BODY")));
|
||||
assert_eq!(rig.held_ids("!r:s"), ["$a"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_edit_replaces_the_body_in_place_and_a_stranger_an_unknown_target_or_an_edit_of_an_edit_change_nothing() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 10, "old words"), said("$b", 20, "other")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
let before = rig.points();
|
||||
history.push("!r:s", vec![edit("$e1", "@a:s", 30, "$a", "new words")]);
|
||||
rig.ok(&mut history);
|
||||
let after = rig.points();
|
||||
assert_eq!(after.len(), 1);
|
||||
let first = &after[0].payload.messages[0];
|
||||
assert_eq!((first.event_id.as_str(), first.sender.as_str(), first.ts_ms, first.body.as_str()), ("$a", "@a:s", 10, "new words"));
|
||||
assert_eq!(after[0].id, before[0].id);
|
||||
assert!(after[0].payload.text.contains("new words") && !after[0].payload.text.contains("old words"));
|
||||
assert!(rig.embedded().last().unwrap().contains("new words"));
|
||||
let calls = rig.embedder.state.lock().unwrap().embedded.len();
|
||||
history.push("!r:s", vec![edit("$e2", "@stranger:s", 40, "$a", "hijack"), edit("$e3", "@a:s", 41, "$nothing", "lost"), edit("$e4", "@a:s", 42, "$e1", "edit of an edit")]);
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.points(), after);
|
||||
assert_eq!(rig.embedder.state.lock().unwrap().embedded.len(), calls, "no embed call for edits that change nothing");
|
||||
assert_eq!(rig.cursor("!r:s").unwrap(), format!("{}:$e4", chat_version()), "the cursor is the last event of any kind");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_after_a_crash_between_the_upsert_and_the_cursor_neither_duplicates_nor_loses() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 10, "alpha"), said("$b", 20, "beta"), edit("$e", "@a:s", 25, "$a", "alpha two"), said("$c", 30, "gamma")]);
|
||||
let mut rig = Rig::new();
|
||||
// One cursor write (the empty start) succeeds, then the page's upsert is acknowledged and the next write fails.
|
||||
rig.index.state.lock().unwrap().fail_cursor_writes_after = Some(1);
|
||||
assert!(!rig.run(&mut history).0.unwrap().failed.is_empty());
|
||||
let (ids, held) = (rig.ids(), rig.held_ids("!r:s"));
|
||||
assert_eq!(held, ["$a", "$b", "$c"]);
|
||||
rig.index.state.lock().unwrap().fail_cursor_writes_after = None;
|
||||
let done = rig.ok(&mut history);
|
||||
assert!(done.is_clean());
|
||||
assert_eq!((rig.ids(), rig.held_ids("!r:s")), (ids, held));
|
||||
assert!(rig.bodies().contains(&"alpha two".to_owned()));
|
||||
assert_eq!(rig.cursor("!r:s").unwrap(), format!("{}:$c", chat_version()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failed_upsert_leaves_the_cursor_and_a_failed_embed_or_delete_deletes_nothing_the_index_held() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 10, "alpha"), said("$b", 20, "beta")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
let (cursor, points) = (rig.cursor("!r:s"), rig.points());
|
||||
history.push("!r:s", vec![said("$c", 30, "gamma")]);
|
||||
rig.index.state.lock().unwrap().fail_chat_upserts = true;
|
||||
assert_eq!(rig.run(&mut history).0.unwrap().failed, ["!r:s"]);
|
||||
assert_eq!((rig.cursor("!r:s"), rig.points()), (cursor.clone(), points.clone()));
|
||||
rig.index.state.lock().unwrap().fail_chat_upserts = false;
|
||||
rig.embedder.state.lock().unwrap().embed_fails = true;
|
||||
assert_eq!(rig.run(&mut history).0.unwrap().failed, ["!r:s"]);
|
||||
assert_eq!((rig.cursor("!r:s"), rig.points()), (cursor.clone(), points.clone()));
|
||||
rig.embedder.state.lock().unwrap().embed_fails = false;
|
||||
// A redaction of the first message writes a new chunk, then fails to delete the old one: the cursor stays.
|
||||
history.push("!r:s", vec![redaction("$r", 40, "$a")]);
|
||||
rig.index.state.lock().unwrap().fail_chat_deletes = true;
|
||||
assert_eq!(rig.run(&mut history).0.unwrap().failed, ["!r:s"]);
|
||||
assert_eq!(rig.cursor("!r:s"), cursor);
|
||||
rig.index.state.lock().unwrap().fail_chat_deletes = false;
|
||||
assert!(rig.ok(&mut history).is_clean());
|
||||
assert_eq!(rig.held_ids("!r:s"), ["$b", "$c"]);
|
||||
}
|
||||
|
||||
// ---- chunking through the job ----
|
||||
|
||||
#[test]
|
||||
fn appending_to_the_tail_over_many_runs_equals_one_run_over_all() {
|
||||
let all: Vec<RoomEvent> = (0..80).map(|n| said(&format!("$m{n:03}"), 1_000 * n + if n >= 40 { 5_000_000 } else { 0 }, &format!("message number {n} {}", "x".repeat((n as usize * 37) % 400)))).collect();
|
||||
let mut whole_history = FakeHistory::default();
|
||||
whole_history.room("!r:s", all.clone());
|
||||
let mut whole = Rig::new();
|
||||
whole.ok(&mut whole_history);
|
||||
let mut grown_history = FakeHistory::default();
|
||||
grown_history.room("!r:s", vec![]);
|
||||
let mut grown = Rig::new();
|
||||
for batch in all.chunks(7) {
|
||||
grown_history.push("!r:s", batch.to_vec());
|
||||
grown.ok(&mut grown_history);
|
||||
}
|
||||
assert!(whole.points().len() > 2);
|
||||
let shape = |rig: &Rig| rig.points().into_iter().map(|point| (point.id, point.payload)).collect::<Vec<_>>();
|
||||
assert_eq!(shape(&grown), shape(&whole));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_long_message_is_cut_with_a_marker_and_every_point_satisfies_the_payload_contract() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 10, &"é".repeat(CHAT_BODY_CHARS_MAX * 2)), said("$b", 20, "short"), said("$c", 99_000_000, "later")]);
|
||||
history.facts("!r:s", |facts| {
|
||||
facts.alias = Some("#swarm:gamecult.org".into());
|
||||
facts.name = Some("Swarm".into());
|
||||
});
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
let points = rig.points();
|
||||
assert_eq!(points.len(), 2);
|
||||
for point in &points {
|
||||
let payload = &point.payload;
|
||||
assert_eq!((payload.room_id.as_str(), payload.room_alias.as_deref()), ("!r:s", Some("#swarm:gamecult.org")));
|
||||
assert_eq!(point.id, chunk_id("!r:s", &payload.messages[0].event_id));
|
||||
assert_eq!(payload.text, super::chat_chunk::embedded_text("Swarm", &payload.messages));
|
||||
assert!(!payload.messages.is_empty());
|
||||
}
|
||||
let long = &points.iter().find(|point| point.payload.messages[0].event_id == "$a").unwrap().payload.messages[0];
|
||||
assert_eq!(long.body.chars().count(), CHAT_BODY_CHARS_MAX);
|
||||
assert!(long.body.ends_with("[cut]"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_lost_cursor_event_or_a_version_bump_re_ingests_that_room_alone() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 10, "alpha"), said("$b", 20, "beta")]).room("!q:s", vec![said("$q1", 10, "quiet")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
let q_ids: BTreeSet<String> = rig.points().iter().filter(|point| point.payload.room_id == "!q:s").map(|point| point.id.clone()).collect();
|
||||
let recreated = rig.index.state.lock().unwrap().recreated;
|
||||
for bump in [Some("0.0".to_owned()), None] {
|
||||
let mut cursors = rig.index.cursors();
|
||||
let value = match &bump {
|
||||
Some(old) => format!("{old}:$b"),
|
||||
None => format!("{}:$gone-from-the-server", chat_version()),
|
||||
};
|
||||
cursors.insert(cursor_key("!r:s"), value);
|
||||
rig.index.set_cursors(&cursors).unwrap();
|
||||
let before = rig.embedded().len();
|
||||
assert!(rig.ok(&mut history).is_clean());
|
||||
assert_eq!(rig.held_ids("!r:s"), ["$a", "$b"]);
|
||||
assert!(rig.embedded().len() > before, "the room was re-embedded");
|
||||
assert_eq!(rig.points().iter().filter(|point| point.payload.room_id == "!q:s").map(|point| point.id.clone()).collect::<BTreeSet<_>>(), q_ids);
|
||||
assert_eq!(rig.cursor("!r:s").unwrap(), format!("{}:$b", chat_version()));
|
||||
}
|
||||
assert_eq!(rig.index.state.lock().unwrap().recreated, recreated);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retire_chat_removes_rooms_no_longer_kept_and_given_none_all_chat_and_touches_nothing_else() {
|
||||
let mut rig = Rig::new();
|
||||
rig.index.upsert_sources(&[SourcePoint {
|
||||
id: "forge-1".into(),
|
||||
vector: vec![1.0; 4],
|
||||
payload: SourcePayload { repo: "Repo".into(), path: "a".into(), commit: "c".into(), blob: "b".into(), line_start: 1, line_end: 1, chunk_index: 0, ext: "rs".into(), text: "t".into() },
|
||||
}])
|
||||
.unwrap();
|
||||
for room in ["!a:s", "!b:s"] {
|
||||
rig.seed(room, "$e", 1);
|
||||
}
|
||||
let mut cursors = rig.index.cursors();
|
||||
cursors.insert("Repo".into(), "c".into());
|
||||
cursors.insert(format!("{CHAT_PURGE_PREFIX}!a:s"), "1..2".into());
|
||||
rig.index.set_cursors(&cursors).unwrap();
|
||||
let retired = retire_chat(&mut rig.index, &["!a:s".to_owned()].into()).unwrap();
|
||||
assert_eq!(retired.retired, ["!b:s"]);
|
||||
assert_eq!(rig.held_ids("!a:s"), ["$e"]);
|
||||
assert!(rig.held_ids("!b:s").is_empty() && rig.cursor("!b:s").is_none());
|
||||
assert_eq!(retire_chat(&mut rig.index, &BTreeSet::new()).unwrap().retired, ["!a:s"]);
|
||||
assert!(rig.points().is_empty());
|
||||
assert_eq!(rig.index.state.lock().unwrap().points.len(), 1, "a forge point stays");
|
||||
let kept: Vec<String> = rig.index.cursors().into_keys().collect();
|
||||
assert_eq!(kept, ["Repo".to_owned(), format!("{CHAT_PURGE_PREFIX}!a:s")]);
|
||||
// A room the reader left is retired by the refresh itself.
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 1, "x")]);
|
||||
rig.seed("!gone:s", "$e", 1);
|
||||
rig.ok(&mut history);
|
||||
assert!(rig.held_ids("!gone:s").is_empty());
|
||||
}
|
||||
|
||||
// ---- the credential scrub through the job ----
|
||||
|
||||
fn planted() -> Vec<(ScrubKind, String, String)> {
|
||||
let t = |kind, secret: String, needle: &str| (kind, secret, needle.to_owned());
|
||||
vec![
|
||||
t(ScrubKind::AuthorizationHeader, format!("Authorization: Bearer {}", "a1B2".repeat(8)), "a1B2a1B2a1B2"),
|
||||
t(ScrubKind::Bearer, format!("Bearer {}", "z9Y8".repeat(6)), "z9Y8z9Y8z9Y8"),
|
||||
t(ScrubKind::Bearer, format!("bearer:{}", "z9Y8".repeat(6)), "z9Y8z9Y8z9Y8"),
|
||||
t(ScrubKind::KeyShape, format!("sk-{}", "q7W3".repeat(8)), "q7W3q7W3q7W3"),
|
||||
t(ScrubKind::KeyShape, format!("sk_live_{}", "m4N5".repeat(6)), "m4N5m4N5m4N5"),
|
||||
t(ScrubKind::KeyShape, format!("ghp_{}", "k2L9".repeat(9)), "k2L9k2L9k2L9"),
|
||||
t(ScrubKind::KeyShape, format!("xxx-ghp_{}", "k2L9".repeat(9)), "k2L9k2L9k2L9"),
|
||||
t(ScrubKind::KeyShape, format!("github_pat_{}_{}", "A1b2C3".repeat(4) + "ab", "d4E5f6".repeat(9) + "abcde"), "A1b2C3A1b2C3"),
|
||||
t(ScrubKind::KeyShape, format!("glpat-{}", "r6T7".repeat(5)), "r6T7r6T7r6T7"),
|
||||
t(ScrubKind::KeyShape, "xoxb-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL"),
|
||||
t(ScrubKind::KeyShape, "xoxa-2-123456789012-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL"),
|
||||
t(ScrubKind::KeyShape, "xoxp-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL"),
|
||||
t(ScrubKind::KeyShape, "xoxr-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL"),
|
||||
t(ScrubKind::KeyShape, "xoxs-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL"),
|
||||
t(ScrubKind::KeyShape, "AKIA0123456789ABCDEF".into(), "0123456789ABCDEF"),
|
||||
t(ScrubKind::KeyShape, "xAKIA0123456789ABCDEF".into(), "0123456789ABCDEF"),
|
||||
t(ScrubKind::KeyShape, format!("AIza{}", "Sy1_".repeat(9)), "Sy1_Sy1_Sy1_"),
|
||||
t(ScrubKind::KeyShape, format!("syt_{}_{}_{}", "YWxpY2U", "aBcDeFgHiJkLmNoPqR", "0a1B2c"), "aBcDeFgHiJkLmNoPqR"),
|
||||
t(ScrubKind::KeyShape, format!("mct_{}", "w4X5".repeat(8)), "w4X5w4X5w4X5"),
|
||||
t(ScrubKind::AccessToken, "https://h/_matrix/client/v3/sync?access_token=abcDEF123456xyz&since=s1".into(), "abcDEF123456xyz"),
|
||||
t(ScrubKind::AccessToken, "{\"access_token\": \"tokenValue987654\", \"user\": 1}".into(), "tokenValue987654"),
|
||||
t(ScrubKind::Jwt, "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJmYWtlIn0.c2lnbmF0dXJlZmFrZQ".into(), "c2lnbmF0dXJlZmFrZQ"),
|
||||
t(ScrubKind::PemBlock, "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAfake\n-----END RSA PRIVATE KEY-----".into(), "MIIEowIBAAKCAQEAfake"),
|
||||
t(ScrubKind::PemBlock, "-----BEGIN PRIVATE KEY-----\nMIIBVQIBADANfakeNOEND".into(), "MIIBVQIBADANfake"),
|
||||
t(ScrubKind::PemBlock, "-----BEGIN PGP PRIVATE KEY BLOCK-----\nlQOYBFakeFakeFake\n-----END PGP PRIVATE KEY BLOCK-----".into(), "lQOYBFakeFake"),
|
||||
t(ScrubKind::UrlUserinfo, "https://user:hunter2hunter2@host.example/r.git".into(), "hunter2hunter2"),
|
||||
t(ScrubKind::UrlUserinfo, "https://tokenvalue12345@host.example/".into(), "tokenvalue12345"),
|
||||
]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn planted_credentials_never_reach_the_chunk_the_embedder_or_the_journal_in_a_message_or_an_edit() {
|
||||
for (kind, secret, needle) in planted() {
|
||||
for shape in ["see {} now", "see {}", "{}\nnext line"] {
|
||||
let (dirty, clean) = (shape.replace("{}", &secret), "nothing here".to_owned());
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$m", 10, &dirty), said("$e", 20, &clean), edit("$ed", "@a:s", 30, "$e", &dirty)]);
|
||||
let mut rig = Rig::new();
|
||||
let (done, journal) = rig.run(&mut history);
|
||||
let done = done.unwrap();
|
||||
assert!(done.is_clean(), "a scrub hit does not make a run unclean");
|
||||
let stored = format!("{:?}", rig.points().iter().map(|point| (&point.payload.messages, &point.payload.text)).collect::<Vec<_>>());
|
||||
for seen in [stored, rig.embedded().join("\n"), journal.join("\n")] {
|
||||
assert!(!seen.contains(&needle), "{kind}: {secret:?} leaked into {seen}");
|
||||
}
|
||||
assert!(rig.bodies().iter().filter(|body| body.contains("<redacted>")).count() >= 2, "{kind}: both bodies show the replacement");
|
||||
let hits: Vec<(&str, ScrubKind)> = done.scrubbed.iter().map(|hit| (hit.event_id.as_str(), hit.kind)).collect();
|
||||
assert!(hits.contains(&("$m", kind)) && hits.contains(&("$ed", kind)), "{kind}: {hits:?}");
|
||||
assert!(done.scrubbed.iter().all(|hit| hit.room_id == "!r:s"));
|
||||
assert!(journal.contains(&format!("scrubbed: !r:s $m {kind}")), "{journal:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_scrub_meets_edits_redactions_the_body_cut_replays_and_version_bumps() {
|
||||
let token = format!("ghp_{}", "k2L9".repeat(9));
|
||||
let mut history = FakeHistory::default();
|
||||
history.room(
|
||||
"!r:s",
|
||||
vec![
|
||||
said("$clean", 10, "all clean"),
|
||||
said("$dirty", 20, &format!("token {token} inside")),
|
||||
edit("$adds", "@a:s", 30, "$clean", &format!("now with {token}")),
|
||||
edit("$removes", "@a:s", 40, "$dirty", "token removed"),
|
||||
said("$gone", 50, &format!("secret {token}")),
|
||||
redaction("$r", 60, "$gone"),
|
||||
said("$straddle", 70, &format!("{} {token} {}", "x".repeat(CHAT_BODY_CHARS_MAX - 10), "y".repeat(100))),
|
||||
],
|
||||
);
|
||||
let mut rig = Rig::new();
|
||||
let done = rig.ok(&mut history);
|
||||
assert!(rig.bodies().iter().all(|body| !body.contains("k2L9k2L9")));
|
||||
assert!(rig.embedded().iter().all(|text| !text.contains("k2L9k2L9")));
|
||||
let by_id: BTreeMap<String, String> = rig.points().iter().flat_map(|point| point.payload.messages.iter().map(|message| (message.event_id.clone(), message.body.clone()))).collect();
|
||||
assert_eq!(by_id["$clean"], "now with <redacted>");
|
||||
assert_eq!(by_id["$dirty"], "token removed");
|
||||
assert!(!by_id.contains_key("$gone"));
|
||||
assert!(by_id["$straddle"].chars().count() <= CHAT_BODY_CHARS_MAX && !by_id["$straddle"].contains("ghp_"));
|
||||
assert!(done.scrubbed.iter().any(|hit| hit.event_id == "$adds") && done.scrubbed.iter().any(|hit| hit.event_id == "$gone"));
|
||||
let (ids, texts) = (rig.ids(), rig.points().into_iter().map(|point| point.payload.text).collect::<Vec<_>>());
|
||||
// Replay from the start, as after a lost cursor: identical ids and text.
|
||||
let mut cursors = rig.index.cursors();
|
||||
cursors.remove(&cursor_key("!r:s"));
|
||||
rig.index.set_cursors(&cursors).unwrap();
|
||||
rig.ok(&mut history);
|
||||
assert_eq!((rig.ids(), rig.points().into_iter().map(|point| point.payload.text).collect::<Vec<_>>()), (ids, texts));
|
||||
// An older scrub version re-ingests the room and scrubs it again from the history.
|
||||
let mut cursors = rig.index.cursors();
|
||||
cursors.insert(cursor_key("!r:s"), format!("{}.0:$straddle", huginn_mind::redact::SCRUB_VERSION - 1));
|
||||
rig.index.set_cursors(&cursors).unwrap();
|
||||
rig.ok(&mut history);
|
||||
assert!(rig.bodies().iter().all(|body| !body.contains("k2L9k2L9")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ordinary_text_is_stored_byte_for_byte() {
|
||||
let mut history = FakeHistory::default();
|
||||
let same = "risk-averse task-force sk- 3f2b8c1d9e7a4b5c6d0e1f2a3b4c5d6e7f8a9b0c !abc:s user@example.com ssh://git@host/r.git the bearer of bad news";
|
||||
history.room("!r:s", vec![said("$a", 1, same)]);
|
||||
let mut rig = Rig::new();
|
||||
let done = rig.ok(&mut history);
|
||||
assert_eq!(rig.bodies(), [same]);
|
||||
assert!(done.scrubbed.is_empty());
|
||||
}
|
||||
|
||||
// ---- steward purge ----
|
||||
|
||||
fn window(from_ms: u64, to_ms: u64) -> PurgeWindow {
|
||||
PurgeWindow { from_ms, to_ms }
|
||||
}
|
||||
|
||||
fn purge_history() -> FakeHistory {
|
||||
let mut history = FakeHistory::default();
|
||||
// t1 = 1000, t2 = 3000: $b at t1 and $c inside are purged, $d at t2 is kept.
|
||||
history.room("!r:s", vec![said("$a", 500, "before"), said("$b", 1000, "PURGED-B"), said("$c", 2000, "PURGED-C"), said("$d", 3000, "after"), said("$e", 4000, "later")]);
|
||||
history.room("!other:s", vec![said("$o1", 1000, "same time other room"), said("$o2", 2000, "same time")]);
|
||||
history
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_purge_window_removes_exactly_its_events_and_re_chunks_the_neighbours() {
|
||||
let mut history = purge_history();
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
let other_ids: BTreeSet<String> = rig.points().iter().filter(|point| point.payload.room_id == "!other:s").map(|point| point.id.clone()).collect();
|
||||
assert_eq!(rig.held_ids("!r:s"), ["$a", "$b", "$c", "$d", "$e"]);
|
||||
assert!(request_purge(&mut rig.index, "!r:s", window(1000, 3000)).unwrap());
|
||||
let embedded_before = rig.embedded().len();
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.held_ids("!r:s"), ["$a", "$d", "$e"], "t1 is inside, t2 is not");
|
||||
assert!(rig.embedded()[embedded_before..].iter().all(|text| !text.contains("PURGED")));
|
||||
assert!(rig.points().iter().all(|point| !point.payload.text.contains("PURGED")));
|
||||
assert_eq!(rig.points().iter().filter(|point| point.payload.room_id == "!other:s").map(|point| point.id.clone()).collect::<BTreeSet<_>>(), other_ids);
|
||||
assert_eq!(rig.held_ids("!other:s"), ["$o1", "$o2"]);
|
||||
assert_eq!(tombstone(&rig, "!r:s").as_deref(), Some("1000..3000"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_purged_first_message_changes_the_chunk_id_and_an_emptied_chunk_is_deleted() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 1000, "x"), said("$b", 2000, "y"), said("$c", 99_000_000, "far")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.ids().len(), 2);
|
||||
request_purge(&mut rig.index, "!r:s", window(900, 1500)).unwrap();
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.ids(), [chunk_id("!r:s", "$b"), chunk_id("!r:s", "$c")].into());
|
||||
request_purge(&mut rig.index, "!r:s", window(99_000_000, 99_000_001)).unwrap();
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.held_ids("!r:s"), ["$b"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_purged_window_does_not_come_back() {
|
||||
let mut history = purge_history();
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
request_purge(&mut rig.index, "!r:s", window(1000, 3000)).unwrap();
|
||||
rig.ok(&mut history);
|
||||
let want = ["$a", "$d", "$e"];
|
||||
// New events, an in-window edit of an outside message, a redaction.
|
||||
history.push("!r:s", vec![said("$f", 5000, "new"), edit("$ed", "@a:s", 2500, "$a", "EDIT-IN-WINDOW"), redaction("$rd", 6000, "$e")]);
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.held_ids("!r:s"), ["$a", "$d", "$f"]);
|
||||
assert!(rig.bodies().contains(&"before".to_owned()) && !rig.bodies().iter().any(|body| body.contains("EDIT-IN-WINDOW")), "the in-window edit is dropped and the original stays");
|
||||
// A lost cursor re-ingests from the start; a version bump re-ingests too.
|
||||
for cursor in [None, Some("0.0:$f".to_owned())] {
|
||||
let mut cursors = rig.index.cursors();
|
||||
match cursor {
|
||||
None => cursors.remove(&cursor_key("!r:s")),
|
||||
Some(old) => cursors.insert(cursor_key("!r:s"), old),
|
||||
};
|
||||
rig.index.set_cursors(&cursors).unwrap();
|
||||
rig.ok(&mut history);
|
||||
assert!(rig.bodies().iter().all(|body| !body.contains("PURGED")), "{:?}", rig.held_ids("!r:s"));
|
||||
assert_eq!(rig.held_ids("!r:s"), ["$a", "$d", "$f"], "$e stays redacted by the history; {want:?} before the new events");
|
||||
}
|
||||
// The reader leaves the room and joins again.
|
||||
let mut left = FakeHistory::default();
|
||||
left.room("!other:s", vec![said("$o1", 1000, "x")]);
|
||||
rig.ok(&mut left);
|
||||
assert!(rig.held_ids("!r:s").is_empty() && tombstone(&rig, "!r:s").is_some());
|
||||
rig.ok(&mut history);
|
||||
assert!(rig.bodies().iter().all(|body| !body.contains("PURGED")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_purge_of_an_unindexed_window_writes_only_the_tombstone_and_still_holds() {
|
||||
let mut rig = Rig::new();
|
||||
let writes = rig.index.state.lock().unwrap().cursor_writes.len();
|
||||
assert!(request_purge(&mut rig.index, "!never:s", window(10, 20)).unwrap());
|
||||
assert_eq!(rig.index.state.lock().unwrap().cursor_writes.len(), writes + 1);
|
||||
assert!(rig.points().is_empty() && rig.embedded().is_empty());
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!never:s", vec![said("$a", 5, "ok"), said("$b", 15, "PURGED"), said("$c", 25, "ok")]);
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.held_ids("!never:s"), ["$a", "$c"]);
|
||||
assert!(rig.embedded().iter().all(|text| !text.contains("PURGED")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_repeated_or_overlapping_request_changes_nothing_or_merges_and_the_33rd_window_is_refused() {
|
||||
let mut rig = Rig::new();
|
||||
assert!(request_purge(&mut rig.index, "!r:s", window(10, 20)).unwrap());
|
||||
let writes = rig.index.state.lock().unwrap().cursor_writes.len();
|
||||
assert!(!request_purge(&mut rig.index, "!r:s", window(10, 20)).unwrap());
|
||||
assert!(!request_purge(&mut rig.index, "!r:s", window(12, 18)).unwrap());
|
||||
assert_eq!(rig.index.state.lock().unwrap().cursor_writes.len(), writes, "no write for a request the tombstone already holds");
|
||||
assert!(request_purge(&mut rig.index, "!r:s", window(15, 30)).unwrap());
|
||||
assert_eq!(tombstone(&rig, "!r:s").as_deref(), Some("10..30"));
|
||||
for n in 0..(PURGE_WINDOWS_MAX as u64 - 1) {
|
||||
request_purge(&mut rig.index, "!r:s", window(100 + n * 10, 105 + n * 10)).unwrap();
|
||||
}
|
||||
let error = format!("{:#}", request_purge(&mut rig.index, "!r:s", window(10_000, 10_001)).unwrap_err());
|
||||
assert!(error.contains("32"), "{error}");
|
||||
assert!(request_purge(&mut Rig::new().index, "!r:s", window(5, 5)).is_err());
|
||||
}
|
||||
|
||||
// ---- the source stays in its lane ----
|
||||
|
||||
#[test]
|
||||
fn the_ingest_code_is_pure_over_its_ports_and_names_no_credential_shape_no_second_literal_and_no_join_rule() {
|
||||
let sources = [("chat.rs", include_str!("chat.rs")), ("chat_chunk.rs", include_str!("chat_chunk.rs")), ("chat_purge.rs", include_str!("chat_purge.rs")), ("room_scope.rs", include_str!("room_scope.rs"))];
|
||||
let process = ["Command", "::new"].concat();
|
||||
for (name, source) in sources {
|
||||
let code = source.split("#[cfg(test)]").next().unwrap();
|
||||
assert!(!code.contains(&process) && !code.contains("std::process"), "{name}");
|
||||
assert!(!code.to_ascii_lowercase().contains("join_rule"), "{name}");
|
||||
if name != "room_scope.rs" {
|
||||
assert!(!code.to_ascii_lowercase().contains("discord"), "{name}");
|
||||
}
|
||||
for shape in ["sk-", "sk_live_", "ghp_", "github_pat_", "glpat-", "AKIA", "xoxb-", "PRIVATE KEY", "Bearer"] {
|
||||
assert!(!code.contains(shape), "{name} names {shape}");
|
||||
}
|
||||
let defined = usize::from(name == "chat.rs");
|
||||
assert_eq!((code.matches("\"chat:").count(), code.matches("\"chat-purge:").count()), (defined, defined), "{name}: each prefix is defined once");
|
||||
}
|
||||
let chat = include_str!("chat.rs");
|
||||
assert_eq!(chat.matches("scrub(").count(), 2, "scrub is called for a body and for the room name, nowhere else");
|
||||
}
|
||||
|
||||
// ---- redacted edits, redacted originals, late edits ----
|
||||
|
||||
const EDIT_SECRET: &str = "EDIT-SECRET";
|
||||
|
||||
fn nothing_secret_stored(rig: &Rig) {
|
||||
let stored = format!("{:?}", rig.points().iter().map(|point| (&point.payload.messages, &point.payload.text)).collect::<Vec<_>>());
|
||||
assert!(!stored.contains(EDIT_SECRET), "{stored}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_redacted_edit_in_the_same_page_reverts_the_message_to_its_earlier_body() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$m", 10, "original words"), edit("$e1", "@a:s", 20, "$m", "first edit"), edit("$e2", "@a:s", 30, "$m", EDIT_SECRET), redaction("$r", 40, "$e2")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.bodies(), ["first edit"]);
|
||||
assert!(rig.embedded().iter().all(|text| !text.contains(EDIT_SECRET)));
|
||||
history.room("!q:s", vec![said("$m", 10, "orig"), edit("$e", "@a:s", 20, "$m", EDIT_SECRET), redaction("$r", 30, "$e")]);
|
||||
rig.ok(&mut history);
|
||||
assert!(rig.bodies().contains(&"orig".to_owned()));
|
||||
nothing_secret_stored(&rig);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_redaction_of_an_edit_a_later_run_stored_rebuilds_the_room_and_the_secret_does_not_return() {
|
||||
let mut history = FakeHistory { prune: true, ..FakeHistory::default() };
|
||||
history.room("!r:s", vec![said("$m", 10, "original"), edit("$e", "@a:s", 20, "$m", EDIT_SECRET)]);
|
||||
history.room("!q:s", vec![said("$q", 10, "quiet")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
assert!(rig.bodies().iter().any(|body| body == EDIT_SECRET), "the edit applied while it stood");
|
||||
let recreated = rig.index.state.lock().unwrap().recreated;
|
||||
history.push("!r:s", vec![redaction("$r", 30, "$e")]);
|
||||
let done = rig.ok(&mut history);
|
||||
assert!(done.is_clean());
|
||||
nothing_secret_stored(&rig);
|
||||
assert!(rig.bodies().contains(&"original".to_owned()));
|
||||
assert_eq!(rig.held_ids("!q:s"), ["$q"]);
|
||||
assert_eq!(rig.index.state.lock().unwrap().recreated, recreated);
|
||||
assert_eq!(rig.cursor("!r:s").unwrap(), format!("{}:$r", chat_version()));
|
||||
// The rebuild does not loop: the redaction it replays is not a trigger the second time, and nothing changes.
|
||||
let ids = rig.ids();
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.ids(), ids);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_redacted_original_takes_its_live_and_late_edits_with_it_in_a_page_across_runs_and_across_a_rebuild() {
|
||||
let script = |history: &mut FakeHistory| {
|
||||
history.room("!r:s", vec![said("$m", 10, "original"), edit("$e1", "@a:s", 20, "$m", "live edit"), redaction("$re", 25, "$e1"), redaction("$rm", 30, "$m"), edit("$late", "@a:s", 40, "$m", EDIT_SECRET)]);
|
||||
history.prune = true;
|
||||
};
|
||||
// One page.
|
||||
let (mut history, mut rig) = (FakeHistory::default(), Rig::new());
|
||||
script(&mut history);
|
||||
rig.ok(&mut history);
|
||||
nothing_secret_stored(&rig);
|
||||
assert!(rig.points().is_empty());
|
||||
// Across runs: the original and a live edit stand, then it is redacted, then a late edit arrives.
|
||||
let (mut history, mut rig) = (FakeHistory { prune: true, ..FakeHistory::default() }, Rig::new());
|
||||
history.room("!r:s", vec![said("$m", 10, "original"), edit("$e1", "@a:s", 20, "$m", EDIT_SECRET)]);
|
||||
rig.ok(&mut history);
|
||||
history.push("!r:s", vec![redaction("$rm", 30, "$m")]);
|
||||
rig.ok(&mut history);
|
||||
nothing_secret_stored(&rig);
|
||||
history.push("!r:s", vec![edit("$late", "@a:s", 40, "$m", EDIT_SECRET)]);
|
||||
rig.ok(&mut history);
|
||||
nothing_secret_stored(&rig);
|
||||
assert!(rig.points().is_empty());
|
||||
// Across a rebuild: a lost cursor re-reads the whole history.
|
||||
let mut cursors = rig.index.cursors();
|
||||
cursors.remove(&cursor_key("!r:s"));
|
||||
rig.index.set_cursors(&cursors).unwrap();
|
||||
rig.ok(&mut history);
|
||||
nothing_secret_stored(&rig);
|
||||
assert!(rig.points().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stranger_cannot_edit_a_message_of_the_same_page() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$m", 10, "mine"), edit("$e", "@stranger:s", 20, "$m", "hijacked")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.bodies(), ["mine"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_room_name_is_scrubbed_before_it_is_embedded_or_stored() {
|
||||
let mut history = FakeHistory::default();
|
||||
let token = format!("ghp_{}", "k2L9".repeat(9));
|
||||
history.room("!r:s", vec![said("$m", 10, "hello")]).facts("!r:s", |facts| facts.name = Some(format!("Swarm {token}")));
|
||||
let mut rig = Rig::new();
|
||||
let (done, journal) = rig.run(&mut history);
|
||||
let done = done.unwrap();
|
||||
assert!(rig.embedded().iter().all(|text| !text.contains("k2L9k2L9")));
|
||||
assert!(rig.points().iter().all(|point| !point.payload.text.contains("k2L9k2L9") && point.payload.text.starts_with("Swarm <redacted>")));
|
||||
assert!(done.scrubbed.iter().any(|hit| hit.event_id == "(name)") && journal.iter().all(|line| !line.contains("k2L9k2L9")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_token_split_by_zero_width_characters_leaves_no_tail_in_a_stored_body() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$m", 10, &format!("key sk-{}\u{200b}{} done", "q7W3".repeat(4), "q7W3".repeat(4)))]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.bodies(), ["key <redacted> done"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retire_goes_on_past_a_failed_delete_and_names_each_failure() {
|
||||
let mut rig = Rig::new();
|
||||
for room in ["!a:s", "!b:s", "!c:s"] {
|
||||
rig.seed(room, "$e", 1);
|
||||
}
|
||||
rig.index.state.lock().unwrap().fail_chat_deletes_for = ["!a:s".to_owned()].into();
|
||||
let done = retire_chat(&mut rig.index, &BTreeSet::new()).unwrap();
|
||||
assert_eq!(done.retired, ["!b:s", "!c:s"]);
|
||||
assert_eq!(done.failed.len(), 1);
|
||||
assert_eq!(done.failed[0].0, "!a:s");
|
||||
assert_eq!(rig.held_ids("!a:s"), ["$e"]);
|
||||
assert!(rig.cursor("!a:s").is_some(), "the cursor stays so the next run finds it again");
|
||||
// The refresh reports it too.
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!z:s", vec![said("$z", 1, "x")]);
|
||||
let done = rig.ok(&mut history);
|
||||
assert!(done.failed.contains(&"!a:s".to_owned()) && !done.is_clean());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_facts_error_on_a_joined_room_keeps_its_points_and_cursor_and_is_reported() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 1, "x")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
let (cursor, points) = (rig.cursor("!r:s"), rig.points());
|
||||
history.facts_fail.insert("!r:s".into());
|
||||
let (done, journal) = rig.run(&mut history);
|
||||
let done = done.unwrap();
|
||||
assert_eq!((rig.cursor("!r:s"), rig.points()), (cursor, points));
|
||||
assert_eq!(done.failed, ["!r:s"]);
|
||||
assert!(done.rooms.iter().any(|report| report.room_id == "!r:s" && report.failure.is_some()));
|
||||
assert!(journal.iter().any(|line| line.contains("!r:s") && line.contains("failed")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_tail_on_equal_ends_is_the_greatest_chunk_id() {
|
||||
let mut rig = Rig::new();
|
||||
let (a, b) = (chunk_id("!r:s", "$a"), chunk_id("!r:s", "$b"));
|
||||
rig.index.upsert_chat(&[
|
||||
ChatPoint { id: a.clone(), vector: vec![1.0; 4], payload: chunk_payload_of("!r:s", &[("$a", 5)]) },
|
||||
ChatPoint { id: b.clone(), vector: vec![1.0; 4], payload: chunk_payload_of("!r:s", &[("$b", 5)]) },
|
||||
])
|
||||
.unwrap();
|
||||
assert_eq!(rig.index.chat_tail("!r:s").unwrap().unwrap().id, a.max(b));
|
||||
}
|
||||
|
|
@ -1,35 +1,49 @@
|
|||
//! The source index: the public GameCult forge repos' code and docs as chunks
|
||||
//! in one Qdrant collection, `huginn_sources`, for semantic search.
|
||||
//! The source index: the public GameCult forge repos' code and docs, and the
|
||||
//! Minds' chat rooms, as chunks in one Qdrant collection, `huginn_sources`,
|
||||
//! for semantic search. Each point's `corpus` payload (`forge` or `chat`) keeps
|
||||
//! the two apart; a forge search filters `forge` and a chat search `chat`.
|
||||
//!
|
||||
//! Two truths, one writer. The forge is the truth of which repos exist and what
|
||||
//! they hold; the collection is derived from it and can be deleted whenever.
|
||||
//! `refresh` is the only writer of the collection, run by the `huginn-sources`
|
||||
//! oneshot job: first build, incremental refresh, rebuild after a model or
|
||||
//! chunker change, and retirement are all the one routine (delete by filter,
|
||||
//! ingest at a commit, rewrite the cursors). The daemon only reads: `search`
|
||||
//! is its whole part, and nothing in `serve` or the worker calls a write.
|
||||
//! Two truths, one job. The forge is the truth of which repos exist and what
|
||||
//! they hold, Synapse of what the Minds' rooms said; the collection is derived
|
||||
//! from them and can be deleted whenever (but for a steward's purge tombstones, see
|
||||
//! `chat_purge`). `refresh` is the forge's writer and the only creator of the
|
||||
//! collection, run by the `huginn-sources` oneshot job: first build, incremental
|
||||
//! refresh, rebuild after a model or chunker change, and retirement are all the one
|
||||
//! routine (delete by filter, ingest at a commit, rewrite the cursors). `chat::refresh_chat`
|
||||
//! joins it in the same job and the same cursors object, as the only writer of chat points;
|
||||
//! it requires the collection to exist and never recreates it. The daemon only reads: `search`
|
||||
//! and `search_chat` are its whole part, and nothing in `serve` or the worker calls a write.
|
||||
//!
|
||||
//! A cursor is the commit whose chunks the collection holds for a repo. It
|
||||
//! lives in the collection's own metadata, so it is lost with the collection
|
||||
//! A cursor is the commit (or, for a room, the event) whose chunks the collection
|
||||
//! holds. It lives in the collection's own metadata, so it is lost with the collection
|
||||
//! and the two cannot disagree about what is held. It is advanced only after
|
||||
//! the repo's deletes and upserts are acknowledged: it may lag and never leads.
|
||||
//! It is dropped before the repo's first delete, so a run that fails between
|
||||
//! the delete and the upserts leaves no cursor, and the repo is ingested whole.
|
||||
//! The cursors object is written whole by this job alone, once per repo.
|
||||
//! the source's deletes and upserts are acknowledged: it may lag and never leads.
|
||||
//! A repo's is dropped before its first delete, so a run that fails between
|
||||
//! the delete and the upserts leaves no cursor, and the repo is ingested whole; a room's
|
||||
//! chat chunks are written before anything they replace is deleted. The cursors
|
||||
//! object is written whole by this job alone. A key is a repo name, or a chat key
|
||||
//! (`chat::is_chat_key`), never both.
|
||||
|
||||
pub mod chat;
|
||||
pub mod chat_chunk;
|
||||
pub mod chat_purge;
|
||||
pub mod chunk;
|
||||
pub mod forge;
|
||||
pub mod git;
|
||||
pub mod room_scope;
|
||||
|
||||
#[cfg(test)]
|
||||
mod chat_tests;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::collections::hash_map::Entry;
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||
use std::path::Path;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use anyhow::{Context, Result, bail, ensure};
|
||||
use huginn_mind::{SourceHit, SourceHits, SourceModel, SourceQuery};
|
||||
use huginn_mind::{ChatHit, ChatHits, ChatMessage, ChatQuery, ChatWindow, ChatWindowQuery, MindRefusal, SOURCE_TOP_K_MAX, SourceHit, SourceHits, SourceModel, SourceQuery};
|
||||
|
||||
use crate::index::{BATCH, BULK_EMBED_TIMEOUT, Budget, Embedder, INDEX_CALL_TIMEOUT, MANAGED_BY, ModelIdentity, point_id};
|
||||
use chunk::{CHUNKER_VERSION, embedded_text};
|
||||
|
|
@ -42,13 +56,32 @@ pub const COLLECTION: &str = "huginn_sources";
|
|||
/// agent reads first to learn what the repo is. The only owner of the list.
|
||||
pub const FRONT_PAGE_FILES: [&str; 6] = ["README.md", "README", "readme.md", "README.markdown", "CLAUDE.md", "AGENTS.md"];
|
||||
|
||||
/// What the collection says its corpus is.
|
||||
pub const CORPUS: &str = "forge:GameCult";
|
||||
/// What the collection says its corpus is: the set of corpora it holds, each
|
||||
/// point's `corpus` payload naming which (`forge` or `chat`). A collection
|
||||
/// labelled with any other set is rebuilt.
|
||||
pub const CORPUS: &str = "forge:GameCult+chat:rooms";
|
||||
|
||||
/// The `corpus` payload of a forge point, and of a chat point.
|
||||
pub const CORPUS_FORGE: &str = "forge";
|
||||
pub const CORPUS_CHAT: &str = "chat";
|
||||
|
||||
/// The instruction Qwen3 embedding models take in front of a question about
|
||||
/// past conversation.
|
||||
pub const CHAT_QUERY_INSTRUCTION: &str =
|
||||
"Given a question about past conversation in chat rooms, retrieve the chat passages that answer it";
|
||||
|
||||
/// The instruction Qwen3 embedding models take in front of a query about code.
|
||||
pub const SOURCE_QUERY_INSTRUCTION: &str =
|
||||
"Given a question about a codebase, retrieve the source code and documentation passages that answer it";
|
||||
|
||||
/// The constant of reciprocal rank fusion, score = sum over lists of
|
||||
/// 1 / (FUSE_K + rank), rank 1-based: Cormack, Clarke and Buttcher, SIGIR 2009,
|
||||
/// whose value this is. The one calibration constant of a multi-text search.
|
||||
pub const FUSE_K: u32 = 60;
|
||||
|
||||
/// How many chunks each text's list holds before the lists are fused.
|
||||
pub const FUSE_DEPTH: u32 = SOURCE_TOP_K_MAX;
|
||||
|
||||
/// Paths named in one delete call.
|
||||
const DELETE_PATHS_PER_CALL: usize = 256;
|
||||
|
||||
|
|
@ -102,6 +135,80 @@ pub struct SourcePoint {
|
|||
pub payload: SourcePayload,
|
||||
}
|
||||
|
||||
/// What travels beside a chat chunk's vector: consecutive messages of one
|
||||
/// room, never spanning rooms. The single owner of the chat point's shape; the
|
||||
/// ingest job is its only writer (through `upsert_chat`) and nothing else of the
|
||||
/// payload is read. `messages` are the display truth and hold, in order, at
|
||||
/// least one message. The chunk's span, its first and last event ids and its
|
||||
/// `event_ids` are derived from them by the codec, so they cannot disagree.
|
||||
/// `text` is what was embedded: the room's name, then `time sender: body` per
|
||||
/// message.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct ChatPayload {
|
||||
pub room_id: String,
|
||||
/// Absent when the room has no canonical alias.
|
||||
pub room_alias: Option<String>,
|
||||
pub messages: Vec<ChatMessage>,
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct ChatPoint {
|
||||
pub id: String,
|
||||
pub vector: Vec<f32>,
|
||||
pub payload: ChatPayload,
|
||||
}
|
||||
|
||||
/// One chat chunk a vector search returned and how near it scored.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct ChatRow {
|
||||
pub score: f32,
|
||||
pub payload: ChatPayload,
|
||||
}
|
||||
|
||||
/// Which chat chunks a read keeps; the selectors compose as AND. `rooms` are
|
||||
/// any-of room ids (`!`) or aliases (`#`); the span keeps chunks whose own
|
||||
/// span overlaps [`after_ms`, `before_ms`]. `event_id` keeps the chunk holding
|
||||
/// that message. `starts_from_ms` and `starts_to_ms` bound a chunk's own start
|
||||
/// (inclusive), which is what a neighbour read walks by.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct ChatFilter {
|
||||
pub rooms: Vec<String>,
|
||||
pub after_ms: Option<u64>,
|
||||
pub before_ms: Option<u64>,
|
||||
pub event_id: Option<String>,
|
||||
pub starts_from_ms: Option<u64>,
|
||||
pub starts_to_ms: Option<u64>,
|
||||
}
|
||||
|
||||
/// Which end of the room's history a chat scroll begins at.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum ChatOrder {
|
||||
OldestFirst,
|
||||
NewestFirst,
|
||||
}
|
||||
|
||||
impl ChatFilter {
|
||||
/// Whether a chunk of `room_id`/`room_alias` spanning [`start`, `end`]
|
||||
/// passes. The fake index and the tests read the filter through this; the
|
||||
/// Qdrant index states the same thing as conditions.
|
||||
pub fn keeps(&self, room_id: &str, room_alias: Option<&str>, start: u64, end: u64) -> bool {
|
||||
(self.rooms.is_empty() || self.rooms.iter().any(|room| room == room_id || Some(room.as_str()) == room_alias))
|
||||
&& self.after_ms.is_none_or(|after| end >= after)
|
||||
&& self.before_ms.is_none_or(|before| start <= before)
|
||||
&& self.starts_from_ms.is_none_or(|from| start >= from)
|
||||
&& self.starts_to_ms.is_none_or(|to| start <= to)
|
||||
}
|
||||
}
|
||||
|
||||
/// A chat chunk as the index holds it, with its point id: what the ingest job
|
||||
/// reads back to rewrite a chunk.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct StoredChat {
|
||||
pub id: String,
|
||||
pub payload: ChatPayload,
|
||||
}
|
||||
|
||||
/// One chunk a vector search returned and how near it scored.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct SourceRow {
|
||||
|
|
@ -115,7 +222,9 @@ pub struct SourceRow {
|
|||
pub trait SourceIndex {
|
||||
fn describe_sources(&mut self, within: Duration) -> Result<DescribedSources>;
|
||||
/// Drops the collection if it exists and creates it empty with this
|
||||
/// metadata, no cursors, and keyword indexes on `repo` and `path`.
|
||||
/// metadata, no cursors, keyword indexes on `repo`, `path`, `corpus`,
|
||||
/// `room_id`, `room_alias` and `event_ids`, and integer indexes on
|
||||
/// `ts_start_ms` and `ts_end_ms`.
|
||||
fn recreate_sources(&mut self, meta: &SourcesMeta) -> Result<()>;
|
||||
/// Replaces the cursors object whole.
|
||||
fn set_cursors(&mut self, cursors: &BTreeMap<String, String>) -> Result<()>;
|
||||
|
|
@ -128,6 +237,25 @@ pub trait SourceIndex {
|
|||
/// The `limit` nearest chunks, best first, in `repo` alone when given, and
|
||||
/// among the `FRONT_PAGE_FILES` alone when `front`.
|
||||
fn search_sources(&mut self, vector: &[f32], repo: Option<&str>, front: bool, limit: u32, within: Duration) -> Result<Vec<SourceRow>>;
|
||||
/// Writes chat chunks, acknowledged before it returns. The ingest job's
|
||||
/// alone: the daemon never calls it. A chunk with no messages is refused.
|
||||
fn upsert_chat(&mut self, points: &[ChatPoint]) -> Result<()>;
|
||||
/// The `limit` chat chunks nearest `vector`, best first, among those
|
||||
/// `filter` keeps. Reads only `corpus=chat` points; the daemon's.
|
||||
fn chat_nearest(&mut self, vector: &[f32], filter: &ChatFilter, limit: u32, within: Duration) -> Result<Vec<ChatRow>>;
|
||||
/// Up to `limit` chat chunks `filter` keeps, by span start in `order`.
|
||||
/// Reads only `corpus=chat` points; the daemon's.
|
||||
fn chat_scroll(&mut self, filter: &ChatFilter, order: ChatOrder, limit: u32, within: Duration) -> Result<Vec<ChatPayload>>;
|
||||
/// The room's chunk with the greatest `ts_end_ms`, if it has any. The ingest job's alone.
|
||||
fn chat_tail(&mut self, room_id: &str) -> Result<Option<StoredChat>>;
|
||||
/// The room's chunks holding any of `event_ids`. The ingest job's alone.
|
||||
fn chat_chunks_holding(&mut self, room_id: &str, event_ids: &[String]) -> Result<Vec<StoredChat>>;
|
||||
/// The room's chunks whose span reaches into [`from_ms`, `to_ms`): `ts_end_ms >= from_ms`
|
||||
/// and `ts_start_ms < to_ms`. The ingest job's alone.
|
||||
fn chat_chunks_overlapping(&mut self, room_id: &str, from_ms: u64, to_ms: u64) -> Result<Vec<StoredChat>>;
|
||||
/// Deletes a room's chat chunks: those with the point ids `ids`, or all of them when `None`.
|
||||
/// Acknowledged before it returns. Never touches a forge point. The ingest job's alone.
|
||||
fn delete_chat(&mut self, room_id: &str, ids: Option<&[String]>) -> Result<()>;
|
||||
}
|
||||
|
||||
/// A repo the forge lists as indexable.
|
||||
|
|
@ -296,7 +424,7 @@ fn retire<S: SourceIndex>(
|
|||
failed: &mut Vec<String>,
|
||||
) {
|
||||
let listed = |name: &str| catalog.iter().any(|repo| repo.name == name);
|
||||
let mut gone: BTreeSet<String> = cursors.keys().filter(|name| !listed(name)).cloned().collect();
|
||||
let mut gone: BTreeSet<String> = cursors.keys().filter(|name| !chat::is_chat_key(name) && !listed(name)).cloned().collect();
|
||||
if let Ok(entries) = std::fs::read_dir(mirrors) {
|
||||
for entry in entries.flatten() {
|
||||
if let Some(name) = entry.file_name().to_string_lossy().strip_suffix(".git")
|
||||
|
|
@ -438,16 +566,56 @@ impl Batch {
|
|||
}
|
||||
}
|
||||
|
||||
/// A source search, which a daemon answers from the collection as it finds it:
|
||||
/// it embeds the query with its own embedder, and refuses if the collection
|
||||
/// was built under another model, because vectors of two models do not compare.
|
||||
pub fn search<E: Embedder, S: SourceIndex>(embedder: &mut E, index: &mut S, query: &SourceQuery, within: Duration) -> Result<SourceHits> {
|
||||
let budget = Budget::new(within);
|
||||
let built = match index.describe_sources(budget.left()?).context("describing the source collection")? {
|
||||
DescribedSources::Absent => bail!("source index not built"),
|
||||
DescribedSources::Unlabelled => bail!("source index not built: {COLLECTION} carries no Huginn metadata"),
|
||||
DescribedSources::Labelled { meta, .. } => meta,
|
||||
};
|
||||
/// Reciprocal rank fusion of ranked lists (best first): an item's score is the
|
||||
/// sum over the lists that hold it of `1 / (k + rank)`, rank 1-based, and items
|
||||
/// with the same `key` are one item (the first list's copy is kept). Best first,
|
||||
/// the first-seen item ahead on a tie, cut to `top_k`. Pure: the one fuse, which
|
||||
/// every search that ranks across lists calls and none copies.
|
||||
pub fn fuse<T>(lists: Vec<Vec<T>>, key: impl Fn(&T) -> String, k: u32, top_k: usize) -> Vec<(f32, T)> {
|
||||
let mut seen: HashMap<String, usize> = HashMap::new();
|
||||
let mut fused: Vec<(f32, T)> = Vec::new();
|
||||
for list in lists {
|
||||
for (rank, item) in list.into_iter().enumerate() {
|
||||
let share = 1.0 / (k as f32 + rank as f32 + 1.0);
|
||||
match seen.entry(key(&item)) {
|
||||
Entry::Occupied(at) => fused[*at.get()].0 += share,
|
||||
Entry::Vacant(at) => {
|
||||
at.insert(fused.len());
|
||||
fused.push((share, item));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fused.sort_by(|a, b| b.0.total_cmp(&a.0));
|
||||
fused.truncate(top_k);
|
||||
fused
|
||||
}
|
||||
|
||||
/// The texts a query searches by: `texts`, or an old client's `text` as a list
|
||||
/// of one, trimmed, and each distinct text once, in the order asked.
|
||||
fn query_texts(query: &SourceQuery) -> Vec<&str> {
|
||||
let asked: Vec<&str> = if query.texts.is_empty() { vec![query.text.0.as_str()] } else { query.texts.iter().map(|text| text.0.as_str()).collect() };
|
||||
distinct_texts(asked)
|
||||
}
|
||||
|
||||
fn distinct_texts(asked: Vec<&str>) -> Vec<&str> {
|
||||
let mut distinct: Vec<&str> = Vec::with_capacity(asked.len());
|
||||
for text in asked {
|
||||
if !distinct.iter().any(|seen| seen.trim() == text.trim()) {
|
||||
distinct.push(text);
|
||||
}
|
||||
}
|
||||
distinct
|
||||
}
|
||||
|
||||
/// The label of the collection a daemon is about to read, if it was built
|
||||
/// under the model this daemon embeds with. Absent, unlabelled and other-model
|
||||
/// collections are loud refusals: vectors of two models do not compare, and an
|
||||
/// empty page is not an answer. A collection built for another corpus set (an
|
||||
/// old forge-only one) is refused the same way until the rebuild lands, since
|
||||
/// it would answer a chat search with an empty page.
|
||||
fn readable_under_this_model<E: Embedder, S: SourceIndex>(embedder: &mut E, index: &mut S, budget: &Budget) -> Result<(SourcesMeta, ModelIdentity)> {
|
||||
let built = built_label(index, budget)?;
|
||||
let identity = embedder.model_identity(budget.left()?).context("reading the embedding model's identity")?;
|
||||
ensure!(
|
||||
built.model == identity.name && built.model_digest == identity.digest && built.dimensions == identity.dimensions,
|
||||
|
|
@ -459,12 +627,47 @@ pub fn search<E: Embedder, S: SourceIndex>(embedder: &mut E, index: &mut S, quer
|
|||
identity.digest,
|
||||
identity.dimensions
|
||||
);
|
||||
let text = format!("Instruct: {SOURCE_QUERY_INSTRUCTION}\nQuery: {}", query.text.0);
|
||||
let mut vectors = embedder.embed(&[text], budget.left()?).context("embedding the query")?;
|
||||
ensure!(vectors.len() == 1, "the embedder returned {} vectors for one query", vectors.len());
|
||||
let vector = vectors.remove(0);
|
||||
ensure!(vector.len() == identity.dimensions as usize, "the embedder returned a query vector of {} where the collection holds {}", vector.len(), identity.dimensions);
|
||||
let rows = index.search_sources(&vector, query.repo.as_ref().map(|repo| repo.0.as_str()), query.front, query.top_k, budget.left()?)?;
|
||||
Ok((built, identity))
|
||||
}
|
||||
|
||||
fn built_label<S: SourceIndex>(index: &mut S, budget: &Budget) -> Result<SourcesMeta> {
|
||||
match index.describe_sources(budget.left()?).context("describing the source collection")? {
|
||||
DescribedSources::Absent => bail!("source index not built"),
|
||||
DescribedSources::Unlabelled => bail!("source index not built: {COLLECTION} carries no Huginn metadata"),
|
||||
DescribedSources::Labelled { meta, .. } => {
|
||||
ensure!(
|
||||
meta.corpus == CORPUS,
|
||||
"the source index is stale: it was built for corpus {} and this daemon reads {CORPUS}; the rebuild is pending",
|
||||
meta.corpus
|
||||
);
|
||||
Ok(meta)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A source search, which a daemon answers from the collection as it finds it:
|
||||
/// it embeds the query with its own embedder, and refuses if the collection
|
||||
/// was built under another model, because vectors of two models do not compare.
|
||||
pub fn search<E: Embedder, S: SourceIndex>(embedder: &mut E, index: &mut S, query: &SourceQuery, within: Duration) -> Result<SourceHits> {
|
||||
let budget = Budget::new(within);
|
||||
let (built, identity) = readable_under_this_model(embedder, index, &budget)?;
|
||||
let topics = query_texts(query);
|
||||
let prompts: Vec<String> = topics.iter().map(|topic| format!("Instruct: {SOURCE_QUERY_INSTRUCTION}\nQuery: {topic}")).collect();
|
||||
let vectors = embedder.embed(&prompts, budget.left()?).context("embedding the query")?;
|
||||
ensure!(vectors.len() == prompts.len(), "the embedder returned {} vectors for {} query texts", vectors.len(), prompts.len());
|
||||
let repo = query.repo.as_ref().map(|repo| repo.0.as_str());
|
||||
let depth = if vectors.len() == 1 { query.top_k } else { FUSE_DEPTH };
|
||||
let mut lists = Vec::with_capacity(vectors.len());
|
||||
for vector in &vectors {
|
||||
ensure!(vector.len() == identity.dimensions as usize, "the embedder returned a query vector of {} where the collection holds {}", vector.len(), identity.dimensions);
|
||||
lists.push(index.search_sources(vector, repo, query.front, depth, budget.left()?)?);
|
||||
}
|
||||
let rows = if lists.len() == 1 {
|
||||
lists.remove(0)
|
||||
} else {
|
||||
let key = |row: &SourceRow| chunk_id(&row.payload.repo, &row.payload.path, &row.payload.blob, row.payload.chunk_index);
|
||||
fuse(lists, key, FUSE_K, query.top_k as usize).into_iter().map(|(score, row)| SourceRow { score, ..row }).collect()
|
||||
};
|
||||
Ok(SourceHits {
|
||||
indexed_under: SourceModel { name: built.model, digest: built.model_digest },
|
||||
hits: rows
|
||||
|
|
@ -481,3 +684,161 @@ pub fn search<E: Embedder, S: SourceIndex>(embedder: &mut E, index: &mut S, quer
|
|||
.collect(),
|
||||
})
|
||||
}
|
||||
|
||||
/// A chat search. With texts, the chunks nearest them among those the room and
|
||||
/// span selectors keep, several texts fused by `fuse`; with none, a listing of
|
||||
/// what the selectors keep, oldest first, `top_k` of it and `truncated` saying
|
||||
/// more exist (no embedding, so the model's identity is not consulted, though
|
||||
/// an absent collection is still refused). A hit outside the span is not
|
||||
/// returned and a boundary hit loses its messages outside [after, before], its
|
||||
/// span and first and last ids recomputed from what is left. Never writes.
|
||||
pub fn search_chat<E: Embedder, S: SourceIndex>(embedder: &mut E, index: &mut S, query: &ChatQuery, within: Duration) -> Result<ChatHits> {
|
||||
let budget = Budget::new(within);
|
||||
let filter = ChatFilter { rooms: query.rooms.iter().map(|room| room.0.clone()).collect(), after_ms: query.after_ms, before_ms: query.before_ms, ..ChatFilter::default() };
|
||||
let top_k = query.top_k as usize;
|
||||
let topics = distinct_texts(query.texts.iter().map(|text| text.0.as_str()).collect());
|
||||
let (built, rows, truncated) = if topics.is_empty() {
|
||||
let built = built_label(index, &budget)?;
|
||||
let mut held = index.chat_scroll(&filter, ChatOrder::OldestFirst, query.top_k + 1, budget.left()?)?;
|
||||
let truncated = held.len() > top_k;
|
||||
held.truncate(top_k);
|
||||
(built, held.into_iter().map(|payload| (None, payload)).collect::<Vec<_>>(), truncated)
|
||||
} else {
|
||||
let (built, identity) = readable_under_this_model(embedder, index, &budget)?;
|
||||
let prompts: Vec<String> = topics.iter().map(|topic| format!("Instruct: {CHAT_QUERY_INSTRUCTION}\nQuery: {topic}")).collect();
|
||||
let vectors = embedder.embed(&prompts, budget.left()?).context("embedding the query")?;
|
||||
ensure!(vectors.len() == prompts.len(), "the embedder returned {} vectors for {} query texts", vectors.len(), prompts.len());
|
||||
let depth = if vectors.len() == 1 { query.top_k } else { FUSE_DEPTH };
|
||||
let mut lists = Vec::with_capacity(vectors.len());
|
||||
for vector in &vectors {
|
||||
ensure!(vector.len() == identity.dimensions as usize, "the embedder returned a query vector of {} where the collection holds {}", vector.len(), identity.dimensions);
|
||||
lists.push(index.chat_nearest(vector, &filter, depth, budget.left()?)?);
|
||||
}
|
||||
let rows: Vec<ChatRow> = if lists.len() == 1 {
|
||||
lists.remove(0)
|
||||
} else {
|
||||
let key = |row: &ChatRow| format!("{}\0{}", row.payload.room_id, row.payload.messages.first().map_or("", |message| message.event_id.as_str()));
|
||||
fuse(lists, key, FUSE_K, top_k).into_iter().map(|(score, row)| ChatRow { score, ..row }).collect()
|
||||
};
|
||||
(built, rows.into_iter().map(|row| (Some(row.score), row.payload)).collect(), false)
|
||||
};
|
||||
Ok(ChatHits {
|
||||
indexed_under: SourceModel { name: built.model, digest: built.model_digest },
|
||||
truncated,
|
||||
hits: rows.into_iter().filter_map(|(score, payload)| chat_hit(score, payload, &filter)).collect(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Chunks a neighbour read asks for at a time, and the most such reads each way.
|
||||
const WINDOW_CHUNKS_PER_SCROLL: u32 = 5;
|
||||
const WINDOW_SCROLLS_MAX: usize = 6;
|
||||
|
||||
/// The message a chat window is about and its neighbours, read from the chat
|
||||
/// index alone: the chunk holding `event_id`, then chunks of that room before
|
||||
/// and after it until the asked count of messages is held, the room ends or
|
||||
/// `WINDOW_SCROLLS_MAX` reads are spent. An id the index does not hold is a
|
||||
/// typed refusal naming `event_id`, never an empty window; an id in two chunks
|
||||
/// is an index fault. Needs no embedder. Never writes.
|
||||
pub fn window<S: SourceIndex>(index: &mut S, query: &ChatWindowQuery, within: Duration) -> Result<Result<ChatWindow, MindRefusal>> {
|
||||
let budget = Budget::new(within);
|
||||
built_label(index, &budget)?;
|
||||
let id = query.event_id.0.as_str();
|
||||
let anchor_filter = ChatFilter { event_id: Some(id.to_owned()), ..ChatFilter::default() };
|
||||
let mut found = index.chat_scroll(&anchor_filter, ChatOrder::OldestFirst, 2, budget.left()?)?;
|
||||
ensure!(found.len() < 2, "the chat index holds event {id} in two chunks: an ingest fault, not a window");
|
||||
let Some(anchor) = found.pop() else {
|
||||
return Ok(Err(MindRefusal::SelectionInvalid {
|
||||
field: "event_id".into(),
|
||||
value: Some(id.to_owned()),
|
||||
message: "the chat index does not hold this event: it may not be ingested yet, it may have been removed (redacted, or purged by a steward request), or it may be in a room the index does not read (a DM, #users, #members, #patrons, any room the reader has not joined)".into(),
|
||||
}));
|
||||
};
|
||||
let start_of = |chunk: &ChatPayload| chunk.messages.iter().map(|message| message.ts_ms).min().unwrap_or(0);
|
||||
let first_id = |chunk: &ChatPayload| chunk.messages.first().map(|message| message.event_id.clone()).unwrap_or_default();
|
||||
let room = ChatFilter { rooms: vec![anchor.room_id.clone()], ..ChatFilter::default() };
|
||||
let at = anchor.messages.iter().position(|message| message.event_id == id).context("the chat index returned a chunk that does not hold the event it was asked for")?;
|
||||
// Chunks held so far as (first event id, start): the anchor's, then each walk's.
|
||||
let mut held = vec![(first_id(&anchor), start_of(&anchor))];
|
||||
let mut walk = |forward: bool, want: usize, mine: usize| -> Result<(Vec<ChatPayload>, bool)> {
|
||||
let (mut chunks, mut count, mut edge, mut exhausted) = (Vec::new(), mine, start_of(&anchor), false);
|
||||
for _ in 0..WINDOW_SCROLLS_MAX {
|
||||
if count > want {
|
||||
break;
|
||||
}
|
||||
let (filter, order) = if forward {
|
||||
(ChatFilter { starts_from_ms: Some(edge), ..room.clone() }, ChatOrder::OldestFirst)
|
||||
} else {
|
||||
(ChatFilter { starts_to_ms: Some(edge), ..room.clone() }, ChatOrder::NewestFirst)
|
||||
};
|
||||
// Held chunks starting at the edge come back first with any others
|
||||
// that start there; asking for that many extra means a run of
|
||||
// chunks at one millisecond cannot fill a read with chunks held.
|
||||
let limit = WINDOW_CHUNKS_PER_SCROLL + held.iter().filter(|(_, start)| *start == edge).count() as u32;
|
||||
let read = index.chat_scroll(&filter, order, limit, budget.left()?)?;
|
||||
exhausted = (read.len() as u32) < limit;
|
||||
for chunk in read {
|
||||
let key = (first_id(&chunk), start_of(&chunk));
|
||||
if held.contains(&key) {
|
||||
continue;
|
||||
}
|
||||
edge = if forward { key.1.max(edge) } else { key.1.min(edge) };
|
||||
held.push(key);
|
||||
count += chunk.messages.len();
|
||||
chunks.push(chunk);
|
||||
}
|
||||
if exhausted {
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Stopping short of the asked count without reaching the room's end
|
||||
// means messages remain that this read did not reach.
|
||||
Ok((chunks, count <= want && !exhausted))
|
||||
};
|
||||
let (earlier, unreached_before) = walk(false, query.before as usize, at)?;
|
||||
let (later, unreached_after) = walk(true, query.after as usize, anchor.messages.len() - at - 1)?;
|
||||
// Chronological by (ts, chunk start, chunk first event id, position), so
|
||||
// the order does not depend on which message anchored the read.
|
||||
let mut keyed: Vec<(u64, u64, String, usize, ChatMessage)> = Vec::new();
|
||||
for chunk in earlier.iter().chain(std::iter::once(&anchor)).chain(later.iter()) {
|
||||
let (start, first) = (start_of(chunk), first_id(chunk));
|
||||
keyed.extend(chunk.messages.iter().enumerate().map(|(position, message)| (message.ts_ms, start, first.clone(), position, message.clone())));
|
||||
}
|
||||
keyed.sort_by(|a, b| (a.0, a.1, &a.2, a.3).cmp(&(b.0, b.1, &b.2, b.3)));
|
||||
let (mut seen, mut messages) = (std::collections::BTreeSet::new(), Vec::new());
|
||||
for (.., message) in keyed {
|
||||
if seen.insert(message.event_id.clone()) {
|
||||
messages.push(message);
|
||||
}
|
||||
}
|
||||
let at = messages.iter().position(|message| message.event_id == id).context("the anchor fell out of its own window")?;
|
||||
let (lo, hi) = (at.saturating_sub(query.before as usize), (at + 1 + query.after as usize).min(messages.len()));
|
||||
Ok(Ok(ChatWindow {
|
||||
room_id: anchor.room_id,
|
||||
room_alias: anchor.room_alias,
|
||||
anchor_event_id: id.to_owned(),
|
||||
more_before: unreached_before || lo > 0,
|
||||
more_after: unreached_after || hi < messages.len(),
|
||||
messages: messages[lo..hi].to_vec(),
|
||||
}))
|
||||
}
|
||||
|
||||
/// A held chunk as a hit: only the messages inside the asked span, the span and
|
||||
/// ids those messages make, and nothing at all when none is inside.
|
||||
fn chat_hit(score: Option<f32>, payload: ChatPayload, filter: &ChatFilter) -> Option<ChatHit> {
|
||||
let messages: Vec<ChatMessage> = payload
|
||||
.messages
|
||||
.into_iter()
|
||||
.filter(|message| filter.after_ms.is_none_or(|after| message.ts_ms >= after) && filter.before_ms.is_none_or(|before| message.ts_ms <= before))
|
||||
.collect();
|
||||
let (first, last) = (messages.first()?, messages.last()?);
|
||||
Some(ChatHit {
|
||||
room_id: payload.room_id,
|
||||
room_alias: payload.room_alias,
|
||||
first_event_id: first.event_id.clone(),
|
||||
last_event_id: last.event_id.clone(),
|
||||
ts_start_ms: messages.iter().map(|message| message.ts_ms).min()?,
|
||||
ts_end_ms: messages.iter().map(|message| message.ts_ms).max()?,
|
||||
score,
|
||||
messages,
|
||||
})
|
||||
}
|
||||
|
|
|
|||
184
crates/huginn-daemon/src/sources/room_scope.rs
Normal file
184
crates/huginn-daemon/src/sources/room_scope.rs
Normal file
|
|
@ -0,0 +1,184 @@
|
|||
//! Which joined rooms and which senders are in the chat corpus. The only code
|
||||
//! that decides it, and the only file outside tests that names the bridged
|
||||
//! platform whose content is refused.
|
||||
//!
|
||||
//! Scope (ruling `gamecult-chat:ruling:chat-index-minds-rooms`): the rooms the
|
||||
//! reader user has joined, minus this exclusion. A room's join rule is read by
|
||||
//! nobody (`RoomFacts` has no field for it): a joined invite-only room qualifies
|
||||
//! like any other. The reader roster is the first lock; this file is the second.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use super::chat::RoomEvent;
|
||||
|
||||
/// The estate rooms by canonical alias, ASCII case-insensitive. Code owns the
|
||||
/// aliases and the rule; the unit supplies the room ids (`Estate`), and either alone excludes.
|
||||
pub const ESTATE_ALIASES: [&str; 3] = ["#users:gamecult.org", "#members:gamecult.org", "#patrons:gamecult.org"];
|
||||
|
||||
/// What the server says about a room, as far as admission needs it. The Matrix
|
||||
/// client fills it from the room's state and decides nothing. It has no `Default`: a
|
||||
/// room whose state could not be read is a port error, never a plain room.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct RoomFacts {
|
||||
pub room_id: String,
|
||||
pub alias: Option<String>,
|
||||
pub alt_aliases: Vec<String>,
|
||||
pub name: Option<String>,
|
||||
/// An `m.room.encryption` state event exists.
|
||||
pub encrypted: bool,
|
||||
/// Any `m.room.member` in the state has `is_direct` true.
|
||||
pub direct: bool,
|
||||
/// The state type (`m.bridge` or `uk.half-shot.bridge`) of a bridge event, when the room has one.
|
||||
pub bridge: Option<String>,
|
||||
}
|
||||
|
||||
/// The estate room ids the caller knows (they exist only on the live server): exactly one per
|
||||
/// estate alias, so a job that was handed none, or too few, cannot be built.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Estate {
|
||||
ids: BTreeSet<String>,
|
||||
}
|
||||
|
||||
impl Estate {
|
||||
pub fn new(ids: BTreeSet<String>) -> anyhow::Result<Self> {
|
||||
anyhow::ensure!(
|
||||
ids.len() == ESTATE_ALIASES.len() && ids.iter().all(|id| id.starts_with('!')),
|
||||
"the estate is exactly {} room ids ({}), got {}",
|
||||
ESTATE_ALIASES.len(),
|
||||
ESTATE_ALIASES.join(", "),
|
||||
ids.len()
|
||||
);
|
||||
Ok(Self { ids })
|
||||
}
|
||||
|
||||
/// Whether `room_id` is a named estate room. Decided before the room's state is read.
|
||||
pub fn holds_id(&self, room_id: &str) -> bool {
|
||||
self.ids.contains(room_id)
|
||||
}
|
||||
}
|
||||
|
||||
/// Why a joined room, or an event in it, is refused.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum Refusal {
|
||||
Estate,
|
||||
Direct,
|
||||
Encrypted,
|
||||
Bridged(String),
|
||||
DiscordSender,
|
||||
DiscordContent,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for Refusal {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::Estate => formatter.write_str("estate room"),
|
||||
Self::Direct => formatter.write_str("direct room"),
|
||||
Self::Encrypted => formatter.write_str("encrypted room"),
|
||||
Self::Bridged(state_type) => write!(formatter, "bridged room ({state_type})"),
|
||||
Self::DiscordSender => formatter.write_str("discord sender"),
|
||||
Self::DiscordContent => formatter.write_str("discord content"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a joined room may be read at all.
|
||||
pub fn admit_room(facts: &RoomFacts, estate: &Estate) -> Result<(), Refusal> {
|
||||
let estate_alias = |alias: &str| ESTATE_ALIASES.iter().any(|estate| estate.eq_ignore_ascii_case(alias));
|
||||
if estate.holds_id(&facts.room_id) || facts.alias.iter().chain(&facts.alt_aliases).any(|alias| estate_alias(alias)) {
|
||||
return Err(Refusal::Estate);
|
||||
}
|
||||
if facts.direct {
|
||||
return Err(Refusal::Direct);
|
||||
}
|
||||
if facts.encrypted {
|
||||
return Err(Refusal::Encrypted);
|
||||
}
|
||||
if let Some(state_type) = &facts.bridge {
|
||||
return Err(Refusal::Bridged(state_type.clone()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Whether an event refuses its whole room: a sender whose localpart, lowercased, starts with the
|
||||
/// platform's name after any leading underscores, has it after an underscore (`mautrix_discord`,
|
||||
/// `_bridge_discordbot`), or has it followed by `bot` or an underscore; or content with a top-level key
|
||||
/// naming it. Deterministic identifier rules, not language reading; `discourse` is not it.
|
||||
pub fn refuse_event(event: &RoomEvent) -> Option<Refusal> {
|
||||
let localpart = event.sender.trim_start_matches('@').split(':').next().unwrap_or("").to_ascii_lowercase();
|
||||
let name = ["disc", "ord"].concat();
|
||||
if localpart.trim_start_matches('_').starts_with(&name) || localpart.contains(&format!("_{name}")) || localpart.contains(&format!("{name}bot")) || localpart.contains(&format!("{name}_")) {
|
||||
return Some(Refusal::DiscordSender);
|
||||
}
|
||||
event.content_keys.iter().any(|key| key.to_ascii_lowercase().contains("discord")).then_some(Refusal::DiscordContent)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::chat::EventKind;
|
||||
use super::*;
|
||||
|
||||
fn facts(id: &str) -> RoomFacts {
|
||||
RoomFacts { room_id: id.into(), alias: None, alt_aliases: vec![], name: None, encrypted: false, direct: false, bridge: None }
|
||||
}
|
||||
|
||||
fn estate(ids: &[&str]) -> Estate {
|
||||
Estate::new(ids.iter().chain(["!pad1:s", "!pad2:s", "!pad3:s"].iter()).take(3).map(|id| (*id).to_owned()).collect()).unwrap()
|
||||
}
|
||||
|
||||
fn event(sender: &str, keys: &[&str]) -> RoomEvent {
|
||||
RoomEvent { event_id: "$e".into(), sender: sender.into(), ts_ms: 1, content_keys: keys.iter().map(|key| (*key).into()).collect(), kind: EventKind::Other }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_direct_encrypted_or_bridged_room_is_refused_and_a_plain_room_is_admitted() {
|
||||
let estate = estate(&[]);
|
||||
assert_eq!(admit_room(&RoomFacts { direct: true, ..facts("!a:s") }, &estate), Err(Refusal::Direct));
|
||||
assert_eq!(admit_room(&RoomFacts { encrypted: true, ..facts("!a:s") }, &estate), Err(Refusal::Encrypted));
|
||||
for state_type in ["m.bridge", "uk.half-shot.bridge"] {
|
||||
assert_eq!(admit_room(&RoomFacts { bridge: Some(state_type.into()), ..facts("!a:s") }, &estate), Err(Refusal::Bridged(state_type.into())));
|
||||
}
|
||||
assert_eq!(admit_room(&RoomFacts { alias: Some("#swarm:gamecult.org".into()), name: Some("Swarm".into()), ..facts("!a:s") }, &estate), Ok(()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_estate_is_refused_by_id_and_by_alias_in_any_case_and_a_lookalike_is_not() {
|
||||
let named = estate(&["!ids:s"]);
|
||||
assert!(named.holds_id("!ids:s") && !named.holds_id("!other:s"));
|
||||
assert_eq!(admit_room(&facts("!ids:s"), &named), Err(Refusal::Estate));
|
||||
for alias in ESTATE_ALIASES.iter().map(|alias| alias.to_string()).chain(["#USERS:Gamecult.Org".to_owned()]) {
|
||||
assert_eq!(admit_room(&RoomFacts { alias: Some(alias.clone()), ..facts("!x:s") }, &estate(&[])), Err(Refusal::Estate), "{alias}");
|
||||
assert_eq!(admit_room(&RoomFacts { alt_aliases: vec!["#a:s".into(), alias.clone()], ..facts("!x:s") }, &estate(&[])), Err(Refusal::Estate), "alt {alias}");
|
||||
}
|
||||
for lookalike in ["#users-guide:gamecult.org", "#patrons-club:gamecult.org", "#users:gamecult.org.evil", "#users:other.org"] {
|
||||
assert_eq!(admit_room(&RoomFacts { alias: Some(lookalike.into()), ..facts("!x:s") }, &estate(&[])), Ok(()), "{lookalike}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_bridged_platform_sender_or_content_key_refuses_and_a_lookalike_does_not() {
|
||||
for sender in ["@discord_123:s", "@_discord_9:s", "@__DiscordBot:s", "@discordbot:s", "@mautrix_discord:s", "@_bridge_discordbot:s", "@a_discord:s"] {
|
||||
assert_eq!(refuse_event(&event(sender, &[])), Some(Refusal::DiscordSender), "{sender}");
|
||||
}
|
||||
for key in ["fi.mau.discord", "uk.half-shot.discord", "X.DISCORD.y"] {
|
||||
assert_eq!(refuse_event(&event("@a:s", &["body", key])), Some(Refusal::DiscordContent), "{key}");
|
||||
}
|
||||
for sender in ["@discourse:s", "@disco:s", "@eureka:s", "@x:discord.com"] {
|
||||
assert_eq!(refuse_event(&event(sender, &["discourse", "body"])), None, "{sender}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_estate_needs_exactly_its_room_ids() {
|
||||
for ids in [vec![], vec!["!a:s"], vec!["!a:s", "!b:s", "!c:s", "!d:s"], vec!["!a:s", "!b:s", "notaroom"]] {
|
||||
assert!(Estate::new(ids.iter().map(|id| (*id).to_owned()).collect()).is_err(), "{ids:?}");
|
||||
}
|
||||
assert!(Estate::new(["!a:s", "!b:s", "!c:s"].map(str::to_owned).into()).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_room_facts_carries_no_join_rule() {
|
||||
let source = include_str!("room_scope.rs");
|
||||
let code = &source[..source.find("#[cfg(test)]").unwrap()];
|
||||
assert!(!code.contains(&["join", "_rule"].concat()));
|
||||
}
|
||||
}
|
||||
|
|
@ -7,7 +7,7 @@ use std::process::Command;
|
|||
|
||||
use anyhow::Result;
|
||||
use huginn_mind::eureka_pipeline::{Line, Short};
|
||||
use huginn_mind::SourceQuery;
|
||||
use huginn_mind::{ChatHits, ChatQuery, ChatWindow, ChatWindowQuery, MindRefusal, SourceQuery};
|
||||
|
||||
use super::*;
|
||||
use crate::index::fakes::{FakeEmbedder, FakeSources};
|
||||
|
|
@ -505,7 +505,7 @@ fn front_query(text: &str, top_k: u32, repo: Option<&str>) -> SourceQuery {
|
|||
}
|
||||
|
||||
fn query(text: &str, top_k: u32, repo: Option<&str>) -> SourceQuery {
|
||||
SourceQuery { text: Line(text.into()), top_k, repo: repo.map(|repo| Short(repo.into())), front: false }
|
||||
SourceQuery { text: Line(text.into()), texts: vec![], top_k, repo: repo.map(|repo| Short(repo.into())), front: false }
|
||||
}
|
||||
|
||||
/// A daemon whose embedder is not the one the collection was built under
|
||||
|
|
@ -606,3 +606,573 @@ fn chunks_are_embedded_in_bounded_batches() {
|
|||
assert_eq!(sizes, [BATCH, BATCH, 6]);
|
||||
assert_eq!(rig.index.held().len(), 70);
|
||||
}
|
||||
|
||||
fn fused(lists: &[&[&str]], top_k: usize) -> Vec<(f32, String)> {
|
||||
fuse(lists.iter().map(|list| list.iter().map(|item| (*item).to_owned()).collect()).collect(), |item| item.clone(), FUSE_K, top_k)
|
||||
}
|
||||
|
||||
/// A chunk that two lists put first outranks one that a single list puts first:
|
||||
/// 1/61 + 1/61 against 1/61, with k = 60 and ranks counted from 1.
|
||||
#[test]
|
||||
fn a_chunk_first_in_two_lists_outranks_a_chunk_first_in_one() {
|
||||
let found = fused(&[&["both", "x"], &["both", "y"]], 10);
|
||||
let names: Vec<&str> = found.iter().map(|(_, item)| item.as_str()).collect();
|
||||
assert_eq!(names[0], "both");
|
||||
assert!((found[0].0 - 2.0 / 61.0).abs() < 1e-6, "{}", found[0].0);
|
||||
assert!((found[1].0 - 1.0 / 62.0).abs() < 1e-6, "second place in one list is 1/(60+2)");
|
||||
assert_eq!(names, ["both", "x", "y"], "a tie keeps the earlier list's item ahead");
|
||||
let deeper = fused(&[&["a", "b", "c"], &["c"]], 10);
|
||||
assert_eq!(deeper[0].1, "c", "1/63 + 1/61 beats 1/61");
|
||||
}
|
||||
|
||||
/// The fused list is cut to `top_k`, and an empty input fuses to nothing.
|
||||
#[test]
|
||||
fn a_fused_list_is_cut_to_top_k() {
|
||||
assert_eq!(fused(&[&["a", "b", "c"]], 2).len(), 2);
|
||||
assert!(fused(&[], 5).is_empty());
|
||||
}
|
||||
|
||||
/// An embedder whose vector is the axis of the topic a text names (`topic-N`
|
||||
/// anywhere in it), so a test says which chunk each text is near.
|
||||
struct TopicEmbedder(FakeEmbedder);
|
||||
|
||||
impl Embedder for TopicEmbedder {
|
||||
fn model_identity(&mut self, within: Duration) -> Result<crate::index::ModelIdentity> {
|
||||
self.0.model_identity(within)
|
||||
}
|
||||
|
||||
fn embed(&mut self, texts: &[String], within: Duration) -> Result<Vec<Vec<f32>>> {
|
||||
self.0.embed(texts, within)?;
|
||||
let dimensions = self.0.state.lock().unwrap().identity.dimensions as usize;
|
||||
Ok(texts
|
||||
.iter()
|
||||
.map(|text| {
|
||||
let axis: usize = text.split("topic-").nth(1).and_then(|rest| rest.get(..1)).and_then(|digit| digit.parse().ok()).expect("a text names topic-N");
|
||||
(0..dimensions).map(|at| if at == axis { 1.0 } else { 0.0 }).collect()
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
}
|
||||
|
||||
/// A rig whose collection holds one chunk per axis, `chunk-N` near `topic-N`,
|
||||
/// built under an embedder of `dimensions`.
|
||||
fn topic_rig(dimensions: u32) -> (Rig, TopicEmbedder) {
|
||||
let remote = Remote::new();
|
||||
remote.commit(&[("seed.rs", b"seed\n")], &[]);
|
||||
let mut rig = Rig::new();
|
||||
rig.embedder.state.lock().unwrap().identity.dimensions = dimensions;
|
||||
rig.converged(&mut FakeForge::of(&[("Demo", &remote)]));
|
||||
{
|
||||
let mut state = rig.index.state.lock().unwrap();
|
||||
state.points.clear();
|
||||
for axis in 0..dimensions as usize {
|
||||
let payload = SourcePayload {
|
||||
repo: "Demo".into(),
|
||||
path: format!("chunk-{axis}.rs"),
|
||||
commit: "c".into(),
|
||||
blob: format!("b{axis}"),
|
||||
line_start: 1,
|
||||
line_end: 1,
|
||||
chunk_index: 0,
|
||||
ext: "rs".into(),
|
||||
text: format!("chunk {axis}"),
|
||||
};
|
||||
let id = chunk_id(&payload.repo, &payload.path, &payload.blob, 0);
|
||||
state.points.insert(id.clone(), SourcePoint { id, vector: (0..dimensions as usize).map(|at| if at == axis { 1.0 } else { 0.0 }).collect(), payload });
|
||||
}
|
||||
state.searches.clear();
|
||||
}
|
||||
rig.embedder.state.lock().unwrap().embedded.clear();
|
||||
let embedder = TopicEmbedder(rig.embedder.clone());
|
||||
(rig, embedder)
|
||||
}
|
||||
|
||||
fn queries(texts: &[&str], top_k: u32) -> SourceQuery {
|
||||
SourceQuery { text: Line::default(), texts: texts.iter().map(|text| Line((*text).into())).collect(), top_k, repo: None, front: false }
|
||||
}
|
||||
|
||||
fn paths(hits: &SourceHits) -> Vec<&str> {
|
||||
hits.hits.iter().map(|hit| hit.path.as_str()).collect()
|
||||
}
|
||||
|
||||
/// One text answers exactly as it did before texts existed, whether it came in
|
||||
/// `text` or in `texts`: the index's own cosine scores, unfused, top_k deep.
|
||||
#[test]
|
||||
fn one_text_answers_exactly_as_before() {
|
||||
let (mut rig, mut embedder) = topic_rig(4);
|
||||
let within = Duration::from_secs(5);
|
||||
let old = search(&mut embedder, &mut rig.index, &query("topic-2", 3, None), within).unwrap();
|
||||
let new = search(&mut embedder, &mut rig.index, &queries(&["topic-2"], 3), within).unwrap();
|
||||
assert_eq!(old, new);
|
||||
assert_eq!(old.hits.len(), 3);
|
||||
assert_eq!((old.hits[0].path.as_str(), old.hits[0].score), ("chunk-2.rs", 1.0), "the cosine, not a reciprocal-rank sum");
|
||||
assert!(old.hits[1..].iter().all(|hit| hit.score == 0.0));
|
||||
let searches = rig.index.state.lock().unwrap().searches.clone();
|
||||
assert_eq!(searches, [(None, 3), (None, 3)], "unfused, so asked for top_k and no more");
|
||||
}
|
||||
|
||||
/// Two texts that are equal once trimmed are one text: one search, and the
|
||||
/// answer of the single text.
|
||||
#[test]
|
||||
fn a_duplicated_text_does_not_double_count() {
|
||||
let (mut rig, mut embedder) = topic_rig(4);
|
||||
let within = Duration::from_secs(5);
|
||||
let single = search(&mut embedder, &mut rig.index, &queries(&["topic-1"], 4), within).unwrap();
|
||||
rig.index.state.lock().unwrap().searches.clear();
|
||||
let twice = search(&mut embedder, &mut rig.index, &queries(&["topic-1", " topic-1\n"], 4), within).unwrap();
|
||||
assert_eq!(twice, single);
|
||||
assert_eq!(rig.index.state.lock().unwrap().searches.len(), 1);
|
||||
}
|
||||
|
||||
/// n distinct texts are one embed call of n prompts and n searches, each
|
||||
/// `FUSE_DEPTH` deep; the answer is fused and cut to top_k.
|
||||
#[test]
|
||||
fn n_texts_make_n_searches_and_one_embed_call() {
|
||||
let (mut rig, mut embedder) = topic_rig(4);
|
||||
let within = Duration::from_secs(5);
|
||||
let found = search(&mut embedder, &mut rig.index, &queries(&["topic-0", "topic-1", "topic-3"], 2), within).unwrap();
|
||||
let embedded = rig.embedder.state.lock().unwrap().embedded.clone();
|
||||
assert_eq!(embedded.len(), 1, "one embed call");
|
||||
assert_eq!(embedded[0].len(), 3);
|
||||
assert!(embedded[0].iter().all(|prompt| prompt.starts_with("Instruct: ")));
|
||||
let searches = rig.index.state.lock().unwrap().searches.clone();
|
||||
assert_eq!(searches, vec![(None, FUSE_DEPTH); 3]);
|
||||
assert_eq!(found.hits.len(), 2);
|
||||
assert!(found.hits.iter().all(|hit| hit.score >= 1.0 / 61.0), "a reciprocal-rank sum, at least the 1/(60+1) of a first place");
|
||||
assert!(found.hits[0].score >= found.hits[1].score);
|
||||
}
|
||||
|
||||
/// The reason the cut exists: a brief of eight 1,000-byte topics reaches the
|
||||
/// chunk that only its last topic is near.
|
||||
#[test]
|
||||
fn a_long_brief_of_eight_topics_reaches_the_chunk_only_the_last_is_near() {
|
||||
let (mut rig, mut embedder) = topic_rig(8);
|
||||
let within = Duration::from_secs(5);
|
||||
let topics: Vec<String> = (0..8).map(|n| format!("topic-{n} {}", "x".repeat(1000 - 8))).collect();
|
||||
assert!(topics.iter().all(|topic| topic.len() == 1000));
|
||||
let refs: Vec<&str> = topics.iter().map(String::as_str).collect();
|
||||
let found = search(&mut embedder, &mut rig.index, &queries(&refs, 8), within).unwrap();
|
||||
assert!(paths(&found).contains(&"chunk-7.rs"), "{:?}", paths(&found));
|
||||
assert_eq!(paths(&found).len(), 8);
|
||||
}
|
||||
|
||||
// ---- chat search ----
|
||||
|
||||
fn msg(event_id: &str, ts_ms: u64) -> ChatMessage {
|
||||
ChatMessage { event_id: event_id.into(), sender: "@a:gamecult.org".into(), ts_ms, body: format!("body of {event_id}") }
|
||||
}
|
||||
|
||||
/// A chunk in `room` (with `alias`) of messages `(id, ts)`, near `axis`.
|
||||
fn chat_point(room: &str, alias: Option<&str>, axis: usize, dimensions: u32, messages: &[(&str, u64)]) -> ChatPoint {
|
||||
let messages: Vec<ChatMessage> = messages.iter().map(|(id, ts)| msg(id, *ts)).collect();
|
||||
ChatPoint {
|
||||
id: point_id(&format!("{room}{}", messages[0].event_id)),
|
||||
vector: (0..dimensions as usize).map(|at| if at == axis { 1.0 } else { 0.0 }).collect(),
|
||||
payload: ChatPayload { room_id: room.into(), room_alias: alias.map(str::to_owned), messages, text: format!("chunk near topic-{axis}") },
|
||||
}
|
||||
}
|
||||
|
||||
fn chat_query(texts: &[&str], rooms: &[&str], after: Option<u64>, before: Option<u64>, top_k: u32) -> ChatQuery {
|
||||
ChatQuery {
|
||||
texts: texts.iter().map(|text| Line((*text).into())).collect(),
|
||||
rooms: rooms.iter().map(|room| Short((*room).into())).collect(),
|
||||
after_ms: after,
|
||||
before_ms: before,
|
||||
top_k,
|
||||
}
|
||||
}
|
||||
|
||||
/// A labelled collection that also holds the chat chunks given. Two rooms:
|
||||
/// `!a` (alias `#swarm`), `!b` (no alias).
|
||||
fn chat_rig(points: Vec<ChatPoint>) -> (Rig, TopicEmbedder) {
|
||||
let (mut rig, embedder) = topic_rig(4);
|
||||
rig.index.state.lock().unwrap().points.clear();
|
||||
rig.index.upsert_chat(&points).unwrap();
|
||||
(rig, embedder)
|
||||
}
|
||||
|
||||
fn ids(hits: &ChatHits) -> Vec<&str> {
|
||||
hits.hits.iter().map(|hit| hit.first_event_id.as_str()).collect()
|
||||
}
|
||||
|
||||
const WITHIN: Duration = Duration::from_secs(5);
|
||||
|
||||
fn three_rooms() -> Vec<ChatPoint> {
|
||||
vec![
|
||||
chat_point("!a:x", Some("#swarm:x"), 0, 4, &[("$a1", 100), ("$a2", 200)]),
|
||||
chat_point("!a:x", Some("#swarm:x"), 1, 4, &[("$a3", 300), ("$a4", 400)]),
|
||||
chat_point("!b:x", None, 0, 4, &[("$b1", 150)]),
|
||||
]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_room_filter_by_id_and_by_alias_returns_only_that_room() {
|
||||
let (mut rig, mut embedder) = chat_rig(three_rooms());
|
||||
let by_alias = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &["#swarm:x"], None, None, 10), WITHIN).unwrap();
|
||||
assert_eq!(ids(&by_alias), ["$a1", "$a3"]);
|
||||
let by_id = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &["!b:x"], None, None, 10), WITHIN).unwrap();
|
||||
assert_eq!(ids(&by_id), ["$b1"], "a room with no alias is reachable by id");
|
||||
assert_eq!(by_id.hits[0].room_alias, None);
|
||||
let both = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &["#swarm:x", "!b:x"], None, None, 10), WITHIN).unwrap();
|
||||
assert_eq!(ids(&both), ["$a1", "$b1", "$a3"], "any-of, oldest first");
|
||||
let nobody = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &["#nobody:x"], None, None, 10), WITHIN).unwrap();
|
||||
assert!(nobody.hits.is_empty() && !nobody.truncated, "an unknown alias is zero hits, not an error");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_time_span_returns_the_overlapping_chunks_and_trims_boundary_messages() {
|
||||
let (mut rig, mut embedder) = chat_rig(three_rooms());
|
||||
let hits = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &["#swarm:x"], Some(150), Some(350), 10), WITHIN).unwrap();
|
||||
assert_eq!(ids(&hits), ["$a2", "$a3"], "the boundary chunks keep only in-span messages and first ids are recomputed");
|
||||
let (first, second) = (&hits.hits[0], &hits.hits[1]);
|
||||
assert_eq!((first.ts_start_ms, first.ts_end_ms, first.last_event_id.as_str(), first.messages.len()), (200, 200, "$a2", 1));
|
||||
assert_eq!((second.ts_start_ms, second.ts_end_ms, second.last_event_id.as_str(), second.messages.len()), (300, 300, "$a3", 1));
|
||||
let none = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &["#swarm:x"], Some(1000), None, 10), WITHIN).unwrap();
|
||||
assert!(none.hits.is_empty(), "a chunk wholly outside is absent");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_mode_is_oldest_first_truncated_when_more_exist_and_needs_no_embed() {
|
||||
let (mut rig, mut embedder) = chat_rig(three_rooms());
|
||||
let page = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &[], None, None, 2), WITHIN).unwrap();
|
||||
assert_eq!(ids(&page), ["$a1", "$b1"]);
|
||||
assert!(page.truncated);
|
||||
assert!(embedder.0.state.lock().unwrap().embedded.is_empty(), "no texts, no embed call");
|
||||
// Within one room chunks never overlap, so the last hit's end + 1 continues without a repeat.
|
||||
let first = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &["!a:x"], None, None, 1), WITHIN).unwrap();
|
||||
assert_eq!((ids(&first), first.truncated), (vec!["$a1"], true));
|
||||
let rest = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &["!a:x"], Some(first.hits[0].ts_end_ms + 1), None, 1), WITHIN).unwrap();
|
||||
assert_eq!((ids(&rest), rest.truncated), (vec!["$a3"], false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn texts_rank_by_meaning_and_compose_with_room_and_span() {
|
||||
let (mut rig, mut embedder) = chat_rig(three_rooms());
|
||||
let near = search_chat(&mut embedder, &mut rig.index, &chat_query(&["topic-0"], &[], None, None, 5), WITHIN).unwrap();
|
||||
assert_eq!(near.hits.len(), 3);
|
||||
assert!(near.hits[0].score.unwrap() > near.hits[2].score.unwrap());
|
||||
assert_eq!(near.hits[2].first_event_id, "$a3", "the far chunk is last");
|
||||
let narrowed = search_chat(&mut embedder, &mut rig.index, &chat_query(&["topic-0"], &["!a:x"], Some(250), None, 5), WITHIN).unwrap();
|
||||
assert_eq!(ids(&narrowed), ["$a3"], "a near chunk in another room or outside the span never appears");
|
||||
let fused = search_chat(&mut embedder, &mut rig.index, &chat_query(&["topic-0", "topic-1"], &["!a:x"], None, None, 5), WITHIN).unwrap();
|
||||
assert_eq!(fused.hits.len(), 2);
|
||||
assert!(fused.hits[0].score.unwrap() > 0.0 && fused.hits[0].score != Some(1.0), "two texts carry the fused reciprocal-rank score");
|
||||
assert_eq!(embedder.0.state.lock().unwrap().embedded.last().unwrap().len(), 2, "one embed call for two texts");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_hit_cites_room_event_ids_and_timestamps() {
|
||||
let (mut rig, mut embedder) = chat_rig(three_rooms());
|
||||
for texts in [&[][..], &["topic-0"][..]] {
|
||||
let hits = search_chat(&mut embedder, &mut rig.index, &chat_query(texts, &[], None, None, 10), WITHIN).unwrap();
|
||||
assert!(!hits.hits.is_empty());
|
||||
for hit in &hits.hits {
|
||||
assert!(hit.room_id.starts_with('!') && !hit.first_event_id.is_empty() && !hit.last_event_id.is_empty());
|
||||
assert!(hit.ts_start_ms <= hit.ts_end_ms && !hit.messages.is_empty());
|
||||
assert_eq!((hit.messages[0].event_id.as_str(), hit.messages.last().unwrap().event_id.as_str()), (hit.first_event_id.as_str(), hit.last_event_id.as_str()));
|
||||
for message in &hit.messages {
|
||||
assert!(message.event_id.starts_with('$') && message.sender.starts_with('@') && message.ts_ms > 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_forge_search_never_returns_a_chat_point_and_a_chat_search_never_a_forge_point() {
|
||||
let (mut rig, mut embedder) = topic_rig(4);
|
||||
rig.index.upsert_chat(&three_rooms()).unwrap();
|
||||
let forge = search(&mut embedder, &mut rig.index, &queries(&["topic-0"], 10), WITHIN).unwrap();
|
||||
assert_eq!(forge.hits.len(), 4, "only the four forge chunks");
|
||||
let chat = search_chat(&mut embedder, &mut rig.index, &chat_query(&["topic-0"], &[], None, None, 10), WITHIN).unwrap();
|
||||
assert_eq!(chat.hits.len(), 3, "only the three chat chunks");
|
||||
let listed = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &[], None, None, 10), WITHIN).unwrap();
|
||||
assert_eq!(listed.hits.len(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_absent_unlabelled_or_other_model_collection_is_a_loud_refusal_for_chat() {
|
||||
let mut rig = Rig::new();
|
||||
let mut embedder = TopicEmbedder(FakeEmbedder::new("d1"));
|
||||
for texts in [&[][..], &["topic-0"][..]] {
|
||||
let error = search_chat(&mut embedder, &mut rig.index, &chat_query(texts, &[], None, None, 3), WITHIN).unwrap_err();
|
||||
assert_eq!(format!("{error:#}"), "source index not built");
|
||||
}
|
||||
rig.index.plant(DescribedSources::Unlabelled);
|
||||
assert!(search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &[], None, None, 3), WITHIN).is_err());
|
||||
let (mut rig, _) = chat_rig(three_rooms());
|
||||
let mut foreign = TopicEmbedder(FakeEmbedder::new("d2"));
|
||||
let error = format!("{:#}", search_chat(&mut foreign, &mut rig.index, &chat_query(&["topic-0"], &[], None, None, 3), WITHIN).unwrap_err());
|
||||
assert!(error.contains("built under model d1") && error.contains("embeds under model d2"), "{error}");
|
||||
}
|
||||
|
||||
/// An old forge-only collection (labelled with the old corpus) is rebuilt once
|
||||
/// under the new corpus, and a second refresh leaves it.
|
||||
#[test]
|
||||
fn an_old_forge_only_collection_rebuilds_once() {
|
||||
let remote = Remote::new();
|
||||
remote.commit(&[("a.rs", b"one\n")], &[]);
|
||||
let mut rig = Rig::new();
|
||||
let mut forge = FakeForge::of(&[("Demo", &remote)]);
|
||||
rig.converged(&mut forge);
|
||||
let mut old = match rig.index.state.lock().unwrap().described.clone() {
|
||||
Some(DescribedSources::Labelled { meta, cursors }) => (meta, cursors),
|
||||
other => panic!("{other:?}"),
|
||||
};
|
||||
assert_eq!(old.0.corpus, "forge:GameCult+chat:rooms");
|
||||
old.0.corpus = "forge:GameCult".into();
|
||||
rig.index.plant(DescribedSources::Labelled { meta: old.0, cursors: old.1 });
|
||||
let before = rig.index.state.lock().unwrap().recreated;
|
||||
rig.converged(&mut forge);
|
||||
rig.converged(&mut forge);
|
||||
assert_eq!(rig.index.state.lock().unwrap().recreated, before + 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_chat_read_path_holds_no_discord_matrix_or_second_fuse() {
|
||||
let source = include_str!("mod.rs");
|
||||
let read = &source[source.find("pub fn search_chat").unwrap()..];
|
||||
for banned in ["discord", "puppet", "bridge", "matrix", "synapse", "access_token"] {
|
||||
assert!(!read.to_ascii_lowercase().contains(banned), "{banned} in the chat read path");
|
||||
}
|
||||
assert_eq!(source.matches("pub fn fuse").count(), 1);
|
||||
}
|
||||
|
||||
/// An old forge-only collection answers both searches with the stale-index
|
||||
/// error until the rebuild lands, not with an empty page.
|
||||
#[test]
|
||||
fn an_old_corpus_collection_is_refused_until_it_is_rebuilt() {
|
||||
let (mut rig, mut embedder) = chat_rig(three_rooms());
|
||||
let Some(DescribedSources::Labelled { mut meta, cursors }) = rig.index.state.lock().unwrap().described.clone() else { panic!() };
|
||||
meta.corpus = "forge:GameCult".into();
|
||||
rig.index.plant(DescribedSources::Labelled { meta, cursors });
|
||||
let error = format!("{:#}", search(&mut embedder, &mut rig.index, &query("one", 3, None), WITHIN).unwrap_err());
|
||||
assert!(error.contains("stale") && error.contains("forge:GameCult") && error.contains("rebuild is pending"), "{error}");
|
||||
for texts in [&[][..], &["topic-0"][..]] {
|
||||
let error = format!("{:#}", search_chat(&mut embedder, &mut rig.index, &chat_query(texts, &[], None, None, 3), WITHIN).unwrap_err());
|
||||
assert!(error.contains("stale") && error.contains("rebuild is pending"), "{error}");
|
||||
}
|
||||
}
|
||||
|
||||
/// A message exactly on `after` or `before` is in the span, and so is a chunk
|
||||
/// whose end is exactly `after` or whose start is exactly `before`.
|
||||
#[test]
|
||||
fn the_span_bounds_are_inclusive_for_chunks_and_for_messages() {
|
||||
let (mut rig, mut embedder) = chat_rig(three_rooms());
|
||||
let mut hits = |after, before| {
|
||||
let found = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &["#swarm:x"], after, before, 10), WITHIN).unwrap();
|
||||
found.hits.iter().map(|hit| (hit.first_event_id.clone(), hit.messages.len())).collect::<Vec<_>>()
|
||||
};
|
||||
let owned = |pairs: &[(&str, usize)]| pairs.iter().map(|(id, n)| ((*id).to_owned(), *n)).collect::<Vec<_>>();
|
||||
// Chunk $a1..$a2 ends at 200: after=200 keeps it, trimmed to the message on the bound.
|
||||
assert_eq!(hits(Some(200), None), owned(&[("$a2", 1), ("$a3", 2)]));
|
||||
// Chunk $a3..$a4 starts at 300: before=300 keeps it, trimmed to the message on the bound.
|
||||
assert_eq!(hits(None, Some(300)), owned(&[("$a1", 2), ("$a3", 1)]));
|
||||
// Both bounds on one message.
|
||||
assert_eq!(hits(Some(200), Some(200)), owned(&[("$a2", 1)]));
|
||||
|
||||
let filter = ChatFilter { rooms: vec![], after_ms: Some(200), before_ms: Some(300), ..ChatFilter::default() };
|
||||
assert!(filter.keeps("!a:x", None, 100, 200) && filter.keeps("!a:x", None, 300, 400));
|
||||
assert!(!filter.keeps("!a:x", None, 100, 199) && !filter.keeps("!a:x", None, 301, 400));
|
||||
}
|
||||
|
||||
// ---- chat window ----
|
||||
|
||||
fn wq(event_id: &str, before: u32, after: u32) -> ChatWindowQuery {
|
||||
ChatWindowQuery { event_id: Short(event_id.into()), before, after }
|
||||
}
|
||||
|
||||
/// Room `!a`: five two-message chunks, $a0..$a9 at 100..1000. Room `!b`:
|
||||
/// chunks whose times fall between them ($b0 at 450, $b1 at 550).
|
||||
fn window_rig() -> Rig {
|
||||
let mut points: Vec<ChatPoint> = (0..5)
|
||||
.map(|chunk| {
|
||||
let (first, second) = (format!("$a{}", chunk * 2), format!("$a{}", chunk * 2 + 1));
|
||||
let at = chunk as u64 * 200;
|
||||
chat_point("!a:x", Some("#swarm:x"), 0, 4, &[(&first, at + 100), (&second, at + 200)])
|
||||
})
|
||||
.collect();
|
||||
points.push(chat_point("!b:x", None, 0, 4, &[("$b0", 450), ("$b1", 550)]));
|
||||
chat_rig(points).0
|
||||
}
|
||||
|
||||
fn window_ids(window: &ChatWindow) -> Vec<&str> {
|
||||
window.messages.iter().map(|message| message.event_id.as_str()).collect()
|
||||
}
|
||||
|
||||
fn refusal_field(answer: Result<Result<ChatWindow, MindRefusal>>) -> (String, String) {
|
||||
match answer.unwrap() {
|
||||
Err(MindRefusal::SelectionInvalid { field, message, .. }) => (field, message),
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unknown_event_id_is_refused_naming_the_field_and_the_three_reasons() {
|
||||
let mut rig = window_rig();
|
||||
let (field, message) = refusal_field(window(&mut rig.index, &wq("$nope", 3, 3), WITHIN));
|
||||
assert_eq!(field, "event_id");
|
||||
for reason in ["not be ingested yet", "removed (redacted, or purged", "a room the index does not read"] {
|
||||
assert!(message.contains(reason), "{message}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_redacted_event_is_refused_and_absent_from_its_neighbours_window() {
|
||||
// The refresh re-chunked the window without $a5: the index holds only $a4.
|
||||
let mut rig = chat_rig(vec![chat_point("!a:x", None, 0, 4, &[("$a3", 300), ("$a4", 400)]), chat_point("!a:x", None, 0, 4, &[("$a6", 600)])]).0;
|
||||
assert_eq!(refusal_field(window(&mut rig.index, &wq("$a5", 3, 3), WITHIN)).0, "event_id");
|
||||
let around = window(&mut rig.index, &wq("$a4", 3, 3), WITHIN).unwrap().unwrap();
|
||||
assert_eq!(window_ids(&around), ["$a3", "$a4", "$a6"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_event_in_two_chunks_is_an_index_fault_not_a_window() {
|
||||
let mut rig = chat_rig(vec![chat_point("!a:x", None, 0, 4, &[("$a1", 100), ("$a2", 200)]), chat_point("!a:x", None, 0, 4, &[("$a2", 200), ("$a3", 300)])]).0;
|
||||
let error = window(&mut rig.index, &wq("$a2", 1, 1), WITHIN).unwrap_err().to_string();
|
||||
assert!(error.contains("$a2") && error.contains("two chunks"), "{error}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_window_stays_in_the_anchors_room_though_another_room_is_time_adjacent() {
|
||||
let mut rig = window_rig();
|
||||
let around = window(&mut rig.index, &wq("$a4", 4, 4), WITHIN).unwrap().unwrap();
|
||||
assert_eq!(window_ids(&around), ["$a0", "$a1", "$a2", "$a3", "$a4", "$a5", "$a6", "$a7", "$a8"], "no $b between them");
|
||||
assert_eq!((around.room_id.as_str(), around.room_alias.as_deref()), ("!a:x", Some("#swarm:x")));
|
||||
let other = window(&mut rig.index, &wq("$b1", 5, 5), WITHIN).unwrap().unwrap();
|
||||
assert_eq!(window_ids(&other), ["$b0", "$b1"], "the other room's window holds none of !a");
|
||||
assert!(!other.more_before && !other.more_after && other.room_alias.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_window_across_chunks_has_exactly_before_and_after_in_order_with_the_anchor_once() {
|
||||
let mut rig = window_rig();
|
||||
let around = window(&mut rig.index, &wq("$a4", 3, 3), WITHIN).unwrap().unwrap();
|
||||
assert_eq!(window_ids(&around), ["$a1", "$a2", "$a3", "$a4", "$a5", "$a6", "$a7"]);
|
||||
assert_eq!(around.anchor_event_id, "$a4");
|
||||
assert!(around.more_before && around.more_after);
|
||||
assert!(around.messages.iter().all(|message| !message.event_id.is_empty() && !message.sender.is_empty() && message.ts_ms > 0));
|
||||
let exact = window(&mut rig.index, &wq("$a4", 4, 5), WITHIN).unwrap().unwrap();
|
||||
assert_eq!(window_ids(&exact).len(), 10);
|
||||
assert!(!exact.more_before && !exact.more_after, "nothing remains beyond exactly the room's ends");
|
||||
let alone = window(&mut rig.index, &wq("$a4", 0, 0), WITHIN).unwrap().unwrap();
|
||||
assert_eq!(window_ids(&alone), ["$a4"]);
|
||||
assert!(alone.more_before && alone.more_after);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_window_at_a_rooms_edge_returns_fewer_and_says_nothing_more_lies_beyond() {
|
||||
let mut rig = window_rig();
|
||||
let start = window(&mut rig.index, &wq("$a0", 5, 1), WITHIN).unwrap().unwrap();
|
||||
assert_eq!(window_ids(&start), ["$a0", "$a1"]);
|
||||
assert!(!start.more_before && start.more_after);
|
||||
let end = window(&mut rig.index, &wq("$a9", 1, 5), WITHIN).unwrap().unwrap();
|
||||
assert_eq!(window_ids(&end), ["$a8", "$a9"]);
|
||||
assert!(end.more_before && !end.more_after);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_window_stops_after_six_reads_of_five_chunks_and_says_more_remain() {
|
||||
let points: Vec<ChatPoint> = (0..80).map(|n| chat_point("!a:x", None, 0, 4, &[(&format!("$m{n:02}"), 1000 + n as u64)])).collect();
|
||||
let mut rig = chat_rig(points).0;
|
||||
let around = window(&mut rig.index, &wq("$m70", 50, 0), WITHIN).unwrap().unwrap();
|
||||
assert_eq!(around.messages.len(), 31, "six reads of five new chunks: thirty earlier messages and the anchor");
|
||||
assert_eq!(around.messages[0].event_id, "$m40");
|
||||
assert_eq!(around.messages.last().unwrap().event_id, "$m70");
|
||||
assert!(around.more_before && around.more_after, "the cap is not the room's start, and $m71 follows the anchor");
|
||||
// Asking for exactly what the six reads reach leaves the same honest flag.
|
||||
let exact = window(&mut rig.index, &wq("$m70", 30, 0), WITHIN).unwrap().unwrap();
|
||||
assert_eq!((exact.messages.len(), exact.more_before), (31, true), "thirty held is not yet 'more than asked': the sixth read ended the walk, not the room");
|
||||
let fewer = window(&mut rig.index, &wq("$m70", 29, 0), WITHIN).unwrap().unwrap();
|
||||
assert_eq!((fewer.messages.len(), fewer.more_before), (30, true), "the cut leaves $m40 beyond");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_window_does_not_change_with_the_message_that_anchors_it_when_chunks_share_a_start() {
|
||||
let chunks = [("$t1", "$t2", 150), ("$u1", "$u2", 160), ("$v1", "$v2", 170)];
|
||||
let points: Vec<ChatPoint> = chunks.iter().map(|(first, second, at)| chat_point("!a:x", None, 0, 4, &[(first, 100), (second, *at)])).collect();
|
||||
let mut rig = chat_rig(points).0;
|
||||
let want = ["$t1", "$u1", "$v1", "$t2", "$u2", "$v2"];
|
||||
for anchor in want {
|
||||
let around = window(&mut rig.index, &wq(anchor, 10, 10), WITHIN).unwrap().unwrap();
|
||||
assert_eq!(window_ids(&around), want, "anchored at {anchor}");
|
||||
assert!(!around.more_before && !around.more_after);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_run_of_chunks_at_one_millisecond_is_all_reached() {
|
||||
let points: Vec<ChatPoint> = (0..8).map(|n| chat_point("!a:x", None, 0, 4, &[(&format!("$c{n}"), 100)])).collect();
|
||||
let mut rig = chat_rig(points).0;
|
||||
for anchor in ["$c0", "$c3", "$c7"] {
|
||||
let around = window(&mut rig.index, &wq(anchor, 20, 20), WITHIN).unwrap().unwrap();
|
||||
assert_eq!(window_ids(&around), ["$c0", "$c1", "$c2", "$c3", "$c4", "$c5", "$c6", "$c7"], "anchored at {anchor}");
|
||||
assert!(!around.more_before && !around.more_after, "the room ends in both directions");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_chunk_starting_with_the_anchors_chunk_is_reached_going_forward_alone() {
|
||||
// The anchor's own chunk already holds a message before it, so no earlier read happens.
|
||||
let mut rig = chat_rig(vec![chat_point("!a:x", None, 0, 4, &[("$x0", 100), ("$x1", 120)]), chat_point("!a:x", None, 0, 4, &[("$y0", 100), ("$y1", 130)])]).0;
|
||||
let around = window(&mut rig.index, &wq("$x1", 0, 5), WITHIN).unwrap().unwrap();
|
||||
assert_eq!(window_ids(&around), ["$x1", "$y1"]);
|
||||
assert!(around.more_before && !around.more_after, "$x0 and $y0 precede; nothing follows $y1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_walk_that_holds_exactly_the_asked_count_still_reads_on_to_learn_the_rooms_end() {
|
||||
let points: Vec<ChatPoint> = (0..10).map(|n| chat_point("!a:x", None, 0, 4, &[(&format!("$m{n}"), 1000 + n as u64)])).collect();
|
||||
let mut rig = chat_rig(points).0;
|
||||
let around = window(&mut rig.index, &wq("$m5", 5, 0), WITHIN).unwrap().unwrap();
|
||||
assert_eq!(window_ids(&around), ["$m0", "$m1", "$m2", "$m3", "$m4", "$m5"]);
|
||||
assert!(!around.more_before, "the room starts at $m0");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_event_repeated_in_a_neighbour_chunk_appears_once() {
|
||||
let mut rig = chat_rig(vec![chat_point("!a:x", None, 0, 4, &[("$a1", 100), ("$a2", 200)]), chat_point("!a:x", None, 0, 4, &[("$a2", 200), ("$a3", 300)])]).0;
|
||||
let around = window(&mut rig.index, &wq("$a1", 0, 5), WITHIN).unwrap().unwrap();
|
||||
assert_eq!(window_ids(&around), ["$a1", "$a2", "$a3"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_forge_point_is_not_an_event_the_window_can_anchor_on() {
|
||||
let (mut rig, _) = topic_rig(4);
|
||||
let mut state = rig.index.state.lock().unwrap();
|
||||
let mut point = state.points.values().next().unwrap().clone();
|
||||
point.id = "forge-with-an-event-id".into();
|
||||
point.payload.text = "$f1 looks like an event id".into();
|
||||
state.points.insert(point.id.clone(), point);
|
||||
drop(state);
|
||||
let (field, _) = refusal_field(window(&mut rig.index, &wq("$f1", 1, 1), WITHIN));
|
||||
assert_eq!(field, "event_id", "only corpus=chat points are read, as for an unknown id");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_window_asks_the_index_only_and_needs_no_embedder() {
|
||||
let source = include_str!("mod.rs");
|
||||
let signature = &source[source.find("pub fn window").unwrap()..];
|
||||
let signature = &signature[..signature.find('{').unwrap()];
|
||||
assert!(!signature.contains("Embedder"), "{signature}");
|
||||
}
|
||||
|
||||
/// A chat cursor and a purge tombstone share the forge's cursors object; the
|
||||
/// forge retire never reads either as a repo name, and still retires a repo.
|
||||
#[test]
|
||||
fn chat_cursor_and_purge_tombstone_keys_survive_the_forge_retire() {
|
||||
let (keep, gone) = (Remote::new(), Remote::new());
|
||||
keep.commit(&[("a.rs", b"a\n")], &[]);
|
||||
gone.commit(&[("b.rs", b"b\n")], &[]);
|
||||
let mut rig = Rig::new();
|
||||
rig.converged(&mut FakeForge::of(&[("Keep", &keep), ("Gone", &gone)]));
|
||||
let mut cursors = rig.index.cursors();
|
||||
cursors.insert("chat:!r:s".into(), "1:$e".into());
|
||||
cursors.insert("chat-purge:!r:s".into(), "1..2".into());
|
||||
rig.index.set_cursors(&cursors).unwrap();
|
||||
rig.index.upsert_chat(&[chat_point("!r:s", None, 0, 4, &[("$e", 5)])]).unwrap();
|
||||
rig.converged(&mut FakeForge::of(&[("Keep", &keep)]));
|
||||
let after = rig.index.cursors();
|
||||
assert_eq!(after.keys().map(String::as_str).collect::<Vec<_>>(), ["Keep", "chat-purge:!r:s", "chat:!r:s"], "Gone retired; both chat keys kept");
|
||||
assert!(rig.index.state.lock().unwrap().deletes.iter().all(|(repo, _)| !repo.starts_with("chat")));
|
||||
assert_eq!(rig.index.state.lock().unwrap().chat.len(), 1);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -277,7 +277,7 @@ fn the_binary_serves_both_doors_and_survives_restart() {
|
|||
let catalog = format!("{:?}", exchange(&desk, &framed(&catalog_request())).expect("a catalog"));
|
||||
assert!(catalog.contains("huginn.desk_request.v1") && catalog.contains("huginn.desk_response.v1"), "{catalog}");
|
||||
assert!(!catalog.contains("huginn.mind_request.v1"), "{catalog}");
|
||||
// The reads answer, store nothing, and carry the whole record to a subject the deciders do not name.
|
||||
// The reads answer and store nothing; a decider reading its own list is told the ids.
|
||||
let (list, listed_at) = match ask(&desk, "m-r1", &forks_request("x", &["x"])) {
|
||||
DeskResponse::Forks { list, as_of } => (list, as_of),
|
||||
other => panic!("{other:?}"),
|
||||
|
|
@ -319,15 +319,16 @@ fn the_binary_serves_both_doors_and_survives_restart() {
|
|||
assert_eq!((stale.code, counted_answer_id.as_deref()), (Some(ForkAnswerCode::StaleFork), Some("n1")));
|
||||
let stored = daemon.documents();
|
||||
assert_eq!(stored, seeded + 2);
|
||||
// A subject the call's deciders do not name reads both answers, the first with its choice and the stale one with its code.
|
||||
// A subject the call's deciders do not name reads both answers, the first with its choice and the stale one with its code, and is told no account id.
|
||||
let DeskResponse::Fork { entry, .. } =
|
||||
ask(&desk, "m-r3", &DeskRequest::Fork { mind: INSTANCE.into(), subject: short("outsider"), deciders: vec![short("y")], fork_id: FORK.into() })
|
||||
else {
|
||||
panic!("a read")
|
||||
};
|
||||
let seen: Vec<_> = entry.record.answers.iter().map(|answer| (answer.answer_id.as_str(), answer.answerer.as_str(), answer.code)).collect();
|
||||
assert_eq!(seen, [("n1", "x", None), ("n2", "x", Some(ForkAnswerCode::StaleFork))]);
|
||||
assert_eq!((entry.record.counted_answer_id.as_deref(), entry.card.deciders.as_slice()), (Some("n1"), ["y".to_string()].as_slice()));
|
||||
let seen: Vec<_> = entry.record.answers.iter().map(|answer| (answer.answer_id.as_str(), answer.answerer.as_deref(), answer.code)).collect();
|
||||
assert_eq!(seen, [("n1", None, None), ("n2", None, Some(ForkAnswerCode::StaleFork))]);
|
||||
assert_eq!((entry.record.counted_answer_id.as_deref(), entry.record.answerer_count), (Some("n1"), 1));
|
||||
assert_eq!((entry.card.deciders.as_slice(), entry.card.decider_count), (&[][..], 1));
|
||||
assert_eq!(daemon.documents(), stored);
|
||||
|
||||
// A stop takes the socket with it; the stored answers are the retry's memory.
|
||||
|
|
@ -590,8 +591,8 @@ fn refusal_of(response: &DeskResponse) -> Option<(String, String)> {
|
|||
|
||||
/// Three forks, the middle one's record grown over the socket until its card alone is past the page
|
||||
/// budget. The list names it and its order (`fork_too_large`), the cursor `c<order>` walks on to the
|
||||
/// older fork and ends at order 0, a READ of it never returns that code (an entry while it fits the
|
||||
/// response bound, the door's named response-too-large past it), and `authority` on a read is still
|
||||
/// older fork and ends at order 0, a READ of it never carries an order or cursor (an entry while it fits the
|
||||
/// response bound, the fork-only `fork_too_large` past it), and `authority` on a read is still
|
||||
/// refused.
|
||||
#[test]
|
||||
fn a_fork_too_large_to_page_is_named_by_the_list_and_never_by_the_read() {
|
||||
|
|
@ -664,11 +665,11 @@ fn a_fork_too_large_to_page_is_named_by_the_list_and_never_by_the_read() {
|
|||
let read = ask(&desk, "m-r", &fork_read("x", big));
|
||||
let DeskResponse::Fork { entry, .. } = &read else { panic!("a read of the oversize fork is its entry, not a refusal: {read:?}") };
|
||||
assert_eq!((entry.card.fork_id.as_str(), entry.card.order), (big, 1));
|
||||
// Past DESK_MAX_RESPONSE_BYTES the read is the door's named response-too-large refusal, still not fork_too_large.
|
||||
// Past DESK_MAX_RESPONSE_BYTES the read is the fork-only fork_too_large (no order, no cursor).
|
||||
let mut more = 0_u32;
|
||||
let too_big = loop {
|
||||
let read = ask(&desk, "m-r", &fork_read("x", big));
|
||||
if matches!(read, DeskResponse::Refused(_)) {
|
||||
if matches!(read, DeskResponse::NotStored(_)) {
|
||||
break read;
|
||||
}
|
||||
assert!(matches!(read, DeskResponse::Fork { .. }), "{read:?}");
|
||||
|
|
@ -681,7 +682,8 @@ fn a_fork_too_large_to_page_is_named_by_the_list_and_never_by_the_read() {
|
|||
}
|
||||
assert!(matches!(ask(&desk, "m-w", &grow), DeskResponse::Answered { .. }));
|
||||
};
|
||||
assert!(matches!(too_big, DeskResponse::Refused(huginn_mind::MindRefusal::ResponseTooLarge { .. })), "{too_big:?}");
|
||||
let (code, text) = refusal_of(&too_big).unwrap();
|
||||
assert!(code == "fork_too_large" && text.contains(big) && !text.contains("order"), "{text}");
|
||||
assert!(refusal_of(&ask(&desk, "m-g", &forks_page("x", Some("c2")))).is_some_and(|(code, _)| code == "fork_too_large"));
|
||||
|
||||
// (4) `authority` on a read is still refused whole (the contract's request_invalid is the door's payload-not-a-request).
|
||||
|
|
@ -783,3 +785,130 @@ fn the_newest_fork_too_large_is_named_and_its_cursor_resumes_to_the_older_forks(
|
|||
let rest = page_ids(ask(&desk, "m-p", &forks_page("x", Some("c2"))));
|
||||
assert_eq!(rest, (vec!["eureka-state:question:Q2".to_string(), FORK.to_string()], None));
|
||||
}
|
||||
|
||||
/// Grows `fork` over the socket until a READ of it no longer fits the reply cap; returns the raw reply.
|
||||
fn grown_until_read_overflows(desk: &Path, hash: &str, fork: &str) -> CultNetMessage {
|
||||
let note = "a long reason, ".repeat(60);
|
||||
for sent in 1..400_u32 {
|
||||
let reply = exchange(desk, &desk_bytes("m-read", &fork_read("x", fork))).expect("the door replied");
|
||||
if !matches!(decode_desk_response(&reply).unwrap().1, Ok(DeskResponse::Fork { .. })) {
|
||||
return reply;
|
||||
}
|
||||
let mut grow = answer("x", hash, &format!("00000000-0000-4000-8000-{sent:012}"));
|
||||
if let DeskRequest::Answer { request, .. } = &mut grow {
|
||||
request.fork_id = fork.into();
|
||||
request.answer = Some(ForkAnswer { choice: None, note: Some(note.clone()) });
|
||||
}
|
||||
assert!(matches!(ask(desk, "m-w", &grow), DeskResponse::Answered { .. }));
|
||||
}
|
||||
panic!("the record does not grow")
|
||||
}
|
||||
|
||||
/// A READ of a fork whose entry is past the reply cap is `fork_too_large` naming the fork, with no
|
||||
/// order and no cursor; a fitting fork still reads; the list's refusal is unchanged; the generic
|
||||
/// response-too-large stays for every other reply.
|
||||
#[test]
|
||||
fn a_read_of_an_oversize_fork_is_fork_too_large_naming_only_the_fork() {
|
||||
let _serial = serial();
|
||||
let (_root, _run, hash, desk, _daemon) = three_forks();
|
||||
let big = "eureka-state:question:Q2";
|
||||
let reply = grown_until_read_overflows(&desk, &hash, big);
|
||||
let (id, response) = decode_desk_response(&reply).unwrap();
|
||||
assert_eq!(id, "m-read");
|
||||
let (code, text) = refusal_of(&response.unwrap()).expect("a not-stored refusal");
|
||||
assert_eq!(code, "fork_too_large");
|
||||
assert!(text.contains(big) && !text.contains("order") && !text.contains("cursor") && !text.contains(" c1"), "{text}");
|
||||
// Fitting forks read, the others untouched.
|
||||
for fork in [FORK, "eureka-state:question:Q3"] {
|
||||
assert!(matches!(ask(&desk, "m-ok", &fork_read("x", fork)), DeskResponse::Fork { .. }));
|
||||
}
|
||||
// The list still names the fork and its order, and the cursor goes on.
|
||||
let listed = ask(&desk, "m-l", &forks_page("x", Some("c2")));
|
||||
let (code, text) = refusal_of(&listed).unwrap();
|
||||
assert_eq!(code, "fork_too_large");
|
||||
assert!(text.contains(big) && text.contains("order 1") && text.contains("c1"), "{text}");
|
||||
}
|
||||
|
||||
/// The encoded size of the reply to a read of `fork` sent under `id`, and the reply. The door echoes the id,
|
||||
/// so its length moves the reply one byte at a time where a record's growth moves it in base64 steps of four.
|
||||
fn read_size(desk: &Path, fork: &str, id: &str) -> (usize, CultNetMessage) {
|
||||
let mut stream = UnixStream::connect(desk).unwrap();
|
||||
stream.set_read_timeout(Some(Duration::from_secs(10))).unwrap();
|
||||
stream.write_all(&desk_bytes(id, &fork_read("x", fork))).unwrap();
|
||||
let mut header = [0_u8; 4];
|
||||
stream.read_exact(&mut header).unwrap();
|
||||
let mut payload = vec![0_u8; u32::from_be_bytes(header) as usize];
|
||||
stream.read_exact(&mut payload).unwrap();
|
||||
(payload.len(), decode_cultnet_message_from_slice(&payload, CultNetWireContract::CultNetSchemaV0).unwrap())
|
||||
}
|
||||
|
||||
/// The reply cap is exact on a READ: a reply of exactly DESK_MAX_RESPONSE_BYTES is the entry, and the same
|
||||
/// read one byte longer is `fork_too_large` naming the fork. The record is grown to within 250 bytes of the
|
||||
/// cap, then the echoed message id is padded byte by byte to land on it.
|
||||
#[test]
|
||||
fn a_read_reply_of_exactly_the_cap_is_the_entry_and_one_byte_more_is_fork_too_large() {
|
||||
let _serial = serial();
|
||||
let cap = huginn_mind::desk_wire::DESK_MAX_RESPONSE_BYTES;
|
||||
let (_root, _run, hash, desk, _daemon) = three_forks();
|
||||
let big = "eureka-state:question:Q2";
|
||||
let mut sent = 0_u32;
|
||||
loop {
|
||||
let (size, reply) = read_size(&desk, big, "m-size");
|
||||
assert!(matches!(decode_desk_response(&reply).unwrap().1, Ok(DeskResponse::Fork { .. })), "grew past the cap at {size}");
|
||||
let remaining = cap - size;
|
||||
if remaining <= 250 {
|
||||
break;
|
||||
}
|
||||
// An answer costs about 513 plus 4/3 of its note (base64); aim the last one at 170 bytes short of the cap.
|
||||
let note = if remaining > 5000 { 2000 } else { (remaining - 170 - 513) * 3 / 4 };
|
||||
sent += 1;
|
||||
let mut request = answer("x", &hash, &format!("00000000-0000-4000-8000-{sent:012}"));
|
||||
if let DeskRequest::Answer { request, .. } = &mut request {
|
||||
request.fork_id = big.into();
|
||||
request.answer = Some(ForkAnswer { choice: None, note: Some("a".repeat(note)) });
|
||||
}
|
||||
assert!(matches!(ask(&desk, "m-w", &request), DeskResponse::Answered { .. }));
|
||||
}
|
||||
let sizes: Vec<(usize, String)> = (0..250).map(|pad| format!("m-size{}", "x".repeat(pad))).map(|id| (read_size(&desk, big, &id).0, id)).collect();
|
||||
let (at_cap, id) = sizes.iter().find(|(size, _)| *size == cap).expect("a padded id lands exactly on the cap").clone();
|
||||
let (_, reply) = read_size(&desk, big, &id);
|
||||
let response = decode_desk_response(&reply).unwrap().1;
|
||||
assert!(matches!(response, Ok(DeskResponse::Fork { .. })), "{at_cap} bytes: {response:?}");
|
||||
let (over, reply) = read_size(&desk, big, &format!("{id}x"));
|
||||
let (_, response) = decode_desk_response(&reply).unwrap();
|
||||
let (code, text) = refusal_of(&response.unwrap()).expect("a not-stored refusal");
|
||||
assert!(code == "fork_too_large" && text.contains(big) && !text.contains("cursor") && !text.contains("order"), "{text}");
|
||||
assert!(over < cap, "the refusal itself is small: {over}");
|
||||
}
|
||||
|
||||
/// A stale answer is STORED as a refusal (`stale_fork`), never counted, and its replay returns the same
|
||||
/// stored refusal from the door even once the fork is too large to read: nothing new is stored, and the
|
||||
/// reply is neither `fork_too_large` nor `ResponseTooLarge` (a stored record carries no card, so it is small).
|
||||
#[test]
|
||||
fn a_stale_answer_is_stored_refused_and_its_replay_survives_the_fork_growing_too_large() {
|
||||
let _serial = serial();
|
||||
let (_root, _run, hash, desk, daemon) = three_forks();
|
||||
let fork = "eureka-state:question:Q2";
|
||||
let mut stale = answer("x", &"0".repeat(64), "00000000-0000-4000-8000-0000ffff0001");
|
||||
if let DeskRequest::Answer { request, .. } = &mut stale {
|
||||
request.fork_id = fork.into();
|
||||
}
|
||||
let before = daemon.documents();
|
||||
let DeskResponse::Answered { answer: first, counted_answer_id, .. } = ask(&desk, "m-s1", &stale) else { panic!("a stale answer is stored") };
|
||||
assert_eq!((first.code, first.outcome.as_str(), counted_answer_id), (Some(ForkAnswerCode::StaleFork), "Refused", None));
|
||||
let stored = daemon.documents();
|
||||
assert_eq!(stored, before + 1, "the refusal is a stored record");
|
||||
let DeskResponse::Fork { entry, .. } = ask(&desk, "m-s2", &fork_read("x", fork)) else { panic!("a read") };
|
||||
assert!(entry.record.answers.iter().any(|a| a.answer_id == first.answer_id && a.code == Some(ForkAnswerCode::StaleFork)));
|
||||
assert_eq!(entry.record.counted_answer_id, None);
|
||||
// The fork grows until it cannot be read.
|
||||
let reply = grown_until_read_overflows(&desk, &hash, fork);
|
||||
let (_, response) = decode_desk_response(&reply).unwrap();
|
||||
assert_eq!(refusal_of(&response.unwrap()).unwrap().0, "fork_too_large");
|
||||
let grown = daemon.documents();
|
||||
// The identical request replays the stored refusal: same answer id, nothing stored.
|
||||
let DeskResponse::Answered { answer: replay, counted_answer_id, .. } = ask(&desk, "m-s1", &stale) else { panic!("the replay is the stored answer") };
|
||||
assert_eq!((replay.answer_id.clone(), replay.code, replay.outcome.as_str()), (first.answer_id.clone(), Some(ForkAnswerCode::StaleFork), "Refused"));
|
||||
assert_ne!(counted_answer_id.as_deref(), Some(replay.answer_id.as_str()), "the refusal is never the counted answer");
|
||||
assert_eq!(daemon.documents(), grown, "the replay stored nothing");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -179,6 +179,7 @@ fn a_small_repo_is_refreshed_into_a_scratch_source_collection_and_found() {
|
|||
.and_then(|_| {
|
||||
let query = huginn_mind::SourceQuery {
|
||||
text: huginn_mind::eureka_pipeline::Line("what does this repository do".into()),
|
||||
texts: vec![],
|
||||
top_k: 3,
|
||||
repo: None,
|
||||
front: false,
|
||||
|
|
|
|||
|
|
@ -2345,6 +2345,22 @@ mod tests {
|
|||
committed(admit(&mut mind, vec![D::Ruling(quoted)]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_steward_ruling_takes_no_operator_quote() {
|
||||
let mut mind = seeded();
|
||||
let mut peer = ruling("R1");
|
||||
peer.authority = RulingAuthority::Steward {
|
||||
steward: "@heimdall:chat".into(),
|
||||
repo: eureka_pipeline::OrgRepo("GameCult/Bifrost".into()),
|
||||
words: "refuse leading zeros".into(),
|
||||
event: None,
|
||||
};
|
||||
peer.operator_quote = Some("all recommendations, go ahead".into());
|
||||
assert_eq!(refusal(admit(&mut mind, vec![D::Ruling(peer.clone())])), MindRefusal::QuoteWithoutOperator { ruling: id("ruling", "R1") });
|
||||
peer.operator_quote = None;
|
||||
committed(admit(&mut mind, vec![D::Ruling(peer)]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_mind_ruling_without_a_quote_is_admitted() {
|
||||
let mut mind = seeded();
|
||||
|
|
|
|||
|
|
@ -936,7 +936,7 @@ mod tests {
|
|||
assert_eq!(counted(&mind, FORK), Some(answer));
|
||||
}
|
||||
|
||||
/// Libby's vectors (CultLib 6231ecad, `contracts/cultnet/governance/fork-card-vectors.json`, the `optionsHash` array):
|
||||
/// Libby's vectors (CultLib d3025d3c, `contracts/cultnet/governance/fork-card-vectors.json`, the `optionsHash` array):
|
||||
/// the seven of the binding file, A to G, and the eight she adds. Each is recomputed from its shown members.
|
||||
fn vectors() -> Vec<serde_json::Value> {
|
||||
let file: serde_json::Value = serde_json::from_str(include_str!("../vectors/fork-card-vectors.json")).unwrap();
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
//! The fork desk's answer wire, Huginn's half: Libby's answer-side documents
|
||||
//! (CultLib `contracts/cultnet/governance/`, the contract file and its vectors at e57fbaee)
|
||||
//! (CultLib `contracts/cultnet/governance/`, the contract file and its vectors at d3025d3c)
|
||||
//! as Rust types, the one conversion between her `n<seq>` answer ids and the
|
||||
//! record keys, the projection of a stored answer, the refusal mapping and the
|
||||
//! showability predicate, and `Mind::desk_answer`, the one call that runs them in order.
|
||||
|
|
@ -147,7 +147,10 @@ pub struct ForkDeskAnswer {
|
|||
pub answered_at: String,
|
||||
pub received_at: String,
|
||||
pub claim_in_future: bool,
|
||||
pub answerer: String,
|
||||
/// The answering account, or nil in a non-decider's view (the key is kept). A stored answer always has one.
|
||||
#[serde(deserialize_with = "required")]
|
||||
#[schemars(required)]
|
||||
pub answerer: Option<String>,
|
||||
pub authority: DeskAuthority,
|
||||
pub request_id: String,
|
||||
}
|
||||
|
|
@ -180,12 +183,12 @@ pub struct ForkRefusal {
|
|||
/// `Refused` is any other Mind refusal; an `Unavailable` is Bifrost's to turn into its 503, and
|
||||
/// Huginn writes no document for it.
|
||||
///
|
||||
/// The trust boundary of the reads (ruling `desk-read-visibility-whole-record`): `Forks` and `Fork`
|
||||
/// carry the WHOLE record, every answer's note included, to whatever the peer-checked socket
|
||||
/// carries. Huginn strips nothing and gates nothing; Bifrost applies visibility at its door (a
|
||||
/// stripped read serves `note: null`, CultLib `fork-card.md`). Huginn must therefore be reachable
|
||||
/// only by Bifrost's peer-checked socket: the peer-uid check IS the whole authentication, and a
|
||||
/// second reader of that socket is a note leak.
|
||||
/// The trust boundary of the reads (ruling `desk-read-visibility-withheld-ids`): `Forks` and `Fork`
|
||||
/// carry every answer's note, to whatever the peer-checked socket carries; Bifrost strips notes
|
||||
/// at its door (a stripped read serves `note: null`, CultLib `fork-card.md`). Huginn itself
|
||||
/// withholds the account ids from a reader who is not a decider (`deciders` empty, `answerer` nil,
|
||||
/// counts in their place). Huginn must be reachable only by Bifrost's peer-checked socket: the
|
||||
/// peer-uid check IS the whole authentication, and a second reader of that socket is a note leak.
|
||||
// One value per call, built and encoded once; a box would only add a deref to every reader.
|
||||
#[allow(clippy::large_enum_variant)]
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
|
|
@ -275,12 +278,11 @@ fn deciders_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schem
|
|||
/// the set refuses the whole payload.
|
||||
///
|
||||
/// The reads, `Forks` and `Fork`, carry NO `authority` (CultLib `list_request` and
|
||||
/// `read_request` are closed and have none; one that names it is refused whole). Their `subject`
|
||||
/// and `deciders` gate nothing: `deciders` only fills the card's `deciders` and the showability
|
||||
/// gate, `subject` is read by no read. `Forks` takes the contract's `list_request` fields flat
|
||||
/// beside them (`status`, `cursor`, `limit`), each key written, `null` where absent. Huginn
|
||||
/// answers every read with the whole record and Bifrost applies visibility; see `DeskResponse`
|
||||
/// for the trust boundary. These docs and the doc of `Mind::desk_forks`/`Mind::desk_fork` are the
|
||||
/// `read_request` are closed and have none; one that names it is refused whole). Their `deciders`
|
||||
/// feed the showability gate, and `subject` decides the view: a subject in `deciders` is a decider
|
||||
/// and is told the ids; any other is told counts in their place. `Forks` takes the contract's `list_request` fields flat
|
||||
/// beside them (`status`, `cursor`, `limit`), each key written, `null` where absent. Bifrost
|
||||
/// strips notes; see `DeskResponse` for the trust boundary. These docs and the doc of `Mind::desk_forks`/`Mind::desk_fork` are the
|
||||
/// reads spec.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
|
|
@ -390,6 +392,12 @@ fn fork_too_large(fork: &str, order: u64) -> ForkRefusal {
|
|||
fork_refusal(ForkRefusalCode::ForkTooLarge, format!("fork_too_large: fork {fork} at order {order} is past the page budget by itself; {onward}"))
|
||||
}
|
||||
|
||||
/// A READ of one fork whose entry is past the reply cap: names the fork, nothing else (a read has no
|
||||
/// order to continue from and no cursor). The door raises it where it measures the reply.
|
||||
pub fn fork_too_large_read(fork: &str) -> ForkRefusal {
|
||||
fork_refusal(ForkRefusalCode::ForkTooLarge, format!("fork_too_large: fork {fork} is past the response bound by itself"))
|
||||
}
|
||||
|
||||
fn body_invalid(field: &str) -> ForkRefusal {
|
||||
fork_refusal(ForkRefusalCode::BodyInvalid, format!("body_invalid: the body does not satisfy gamecult.fork.answer_request.v0 ({field})"))
|
||||
}
|
||||
|
|
@ -453,7 +461,7 @@ fn project(key: &str, stored: &PipelineDeskAnswer) -> Result<ForkDeskAnswer, Min
|
|||
answered_at: stored.answered_at.0.clone(),
|
||||
received_at: stored.received_at.0.clone(),
|
||||
claim_in_future: stored.claim_in_future,
|
||||
answerer: stored.answerer.0.clone(),
|
||||
answerer: Some(stored.answerer.0.clone()),
|
||||
authority: stored.authority,
|
||||
request_id: stored.request_id.0.clone(),
|
||||
})
|
||||
|
|
@ -560,7 +568,7 @@ mod tests {
|
|||
const LIST: [&str; 4] = ["x", "y", "z", "op"];
|
||||
const AT: &str = "2026-11-10T12:00:00Z";
|
||||
|
||||
// ---- Libby's vectors: CultLib 6231ecad, `contracts/cultnet/governance/fork-card-vectors.json` ----
|
||||
// ---- Libby's vectors: CultLib d3025d3c, `contracts/cultnet/governance/fork-card-vectors.json` ----
|
||||
|
||||
fn vectors() -> Value {
|
||||
serde_json::from_str(include_str!("../vectors/fork-card-vectors.json")).unwrap()
|
||||
|
|
@ -770,7 +778,8 @@ mod tests {
|
|||
claim_in_future: message["claim_in_future"].as_bool().unwrap(),
|
||||
request_id: RequestId(text("request_id").unwrap()),
|
||||
body_sha256: Sha256Hex("0".repeat(64)),
|
||||
answerer: Short(text("answerer").unwrap()),
|
||||
// A withheld vector names no answerer: store a placeholder, the vector is the non-decider's view of it.
|
||||
answerer: Short(text("answerer").unwrap_or_else(|| "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f".into())),
|
||||
authority,
|
||||
outcome,
|
||||
};
|
||||
|
|
@ -781,10 +790,14 @@ mod tests {
|
|||
fn stored_answers_project_to_the_desk_answer_vectors() {
|
||||
let file = vectors();
|
||||
let valid = rows(&file["deskAnswer"]["valid"]);
|
||||
assert_eq!(valid.len(), 5);
|
||||
assert_eq!(valid.len(), 8);
|
||||
for (name, message, bytes) in &valid {
|
||||
let (key, stored) = stored_from(message);
|
||||
let projected = project(&key, &stored).unwrap();
|
||||
let mut projected = project(&key, &stored).unwrap();
|
||||
// A null answerer is a non-decider's view of the stored answer.
|
||||
if message["answerer"].is_null() {
|
||||
projected.answerer = None;
|
||||
}
|
||||
assert_eq!(&encode(&projected), bytes, "{name}");
|
||||
}
|
||||
// Each answer inside her results is the projection of an equivalent record too.
|
||||
|
|
@ -1342,9 +1355,9 @@ mod tests {
|
|||
let (other, committed) = ask(&mut mind, "y", request.clone());
|
||||
let (other, _, _) = answered(other);
|
||||
assert!(committed);
|
||||
assert_eq!((other.answer_id.as_str(), other.answerer.as_str(), other.outcome.as_str(), other.request_id.as_str()), ("n2", "y", "Revision", request.request_id.as_str()));
|
||||
assert_eq!((other.answer_id.as_str(), other.answerer.as_deref().unwrap(), other.outcome.as_str(), other.request_id.as_str()), ("n2", "y", "Revision", request.request_id.as_str()));
|
||||
let (original, _, _) = answered(ask(&mut mind, "x", request.clone()).0);
|
||||
assert_eq!((original.answer_id.as_str(), original.answerer.as_str()), ("n1", "x"));
|
||||
assert_eq!((original.answer_id.as_str(), original.answerer.as_deref().unwrap()), ("n1", "x"));
|
||||
// V17 and V18: an id that is no lowercase D-format UUID is body_invalid naming request_id, and binds nothing.
|
||||
let before = head(&mind);
|
||||
for bad in ["retry-1", "0000000A-0000-4000-8000-00000000000B", "{00000000-0000-4000-8000-000000000001}", "00000000-0000-4000-8000-0000000000010", "", " 00000000-0000-4000-8000-000000000001", "00000000-0000-4000-8000-000000000001 ", "00000000-0000-4000-8000-000000000001\n", "00000000-0000-4000-8000-000000000001x"] {
|
||||
|
|
@ -1559,7 +1572,7 @@ mod tests {
|
|||
assert_eq!(answered(off).0.code, Some(ForkAnswerCode::NotADecider));
|
||||
let on = mind.desk_answer(&deciders(&["m"]), s("m"), DeskAuthority::Member, wire(FORK, &hash), now());
|
||||
let (answer, _, _) = answered(on);
|
||||
assert_eq!((answer.outcome.as_str(), answer.authority, answer.answerer.as_str()), ("Counted", DeskAuthority::Member, "m"));
|
||||
assert_eq!((answer.outcome.as_str(), answer.authority, answer.answerer.as_deref().unwrap()), ("Counted", DeskAuthority::Member, "m"));
|
||||
}
|
||||
|
||||
// ---- the desk request on the wire: deciders ride each call, at most 64, never stored ----
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -74,6 +74,19 @@ impl<'i> Docs<'i> {
|
|||
self.image.id_of(kind, id).filter(|row| self.visible(*row)).map(|row| self.image.row(row).ordinal)
|
||||
}
|
||||
|
||||
/// When the document was first admitted, in UTC: an imported document's source receipt
|
||||
/// (`imported_from.committed_at`) wins over this mind's own `admitted_at`. `None` when the
|
||||
/// document is not held at this snapshot or the time does not parse as RFC3339 (an import's
|
||||
/// `committed_at` is only checked as non-empty text at admission). Also `None` when the UTC year is
|
||||
/// outside 0..=9999: CultLib's instant pattern has four year digits, and a local time near the edge
|
||||
/// can convert out of range.
|
||||
pub(crate) fn first_admitted_at(&self, kind: PipelineKind, id: &str) -> Option<chrono::DateTime<chrono::Utc>> {
|
||||
let row = self.image.id_of(kind, id).filter(|row| self.visible(*row))?;
|
||||
let admission = &self.image.row(row).view.admission;
|
||||
let text = admission.provenance.imported_from.as_ref().map_or(admission.admitted_at.as_str(), |source| source.committed_at.0.as_str());
|
||||
chrono::DateTime::parse_from_rfc3339(text).ok().map(|time| time.with_timezone(&chrono::Utc)).filter(|time| (0..=9999).contains(&chrono::Datelike::year(time)))
|
||||
}
|
||||
|
||||
pub(crate) fn find(&self, kind: PipelineKind, id: &str) -> Option<&PipelineDocument> {
|
||||
self.in_batch(kind, id).or_else(|| self.in_image(kind, id))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -549,7 +549,7 @@ mod tests {
|
|||
answered_at: "2026-11-10T12:00:00Z".into(),
|
||||
received_at: "2026-11-10T12:00:01Z".into(),
|
||||
claim_in_future: false,
|
||||
answerer: "user-7".into(),
|
||||
answerer: Some("user-7".into()),
|
||||
authority: DeskAuthority::Member,
|
||||
request_id: "00000000-0000-4000-8000-000000000001".into(),
|
||||
},
|
||||
|
|
@ -623,7 +623,7 @@ mod tests {
|
|||
answered_at: "2026-11-10T12:00:00.123456789+00:00".into(),
|
||||
received_at: "2026-11-10T12:00:00.123456789+00:00".into(),
|
||||
claim_in_future: true,
|
||||
answerer: long(200),
|
||||
answerer: Some(long(200)),
|
||||
authority: DeskAuthority::Operator,
|
||||
request_id: "00000000-0000-4000-8000-000000000001".into(),
|
||||
};
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
//! how a change to it makes every projection rebuild. No network, no JSON, no
|
||||
//! store handle: the daemon owns the vector store and the embedder.
|
||||
|
||||
use eureka_pipeline::{PipelineDocument, PipelineKind, PipelineRef};
|
||||
use eureka_pipeline::{PipelineDocument, PipelineKind, PipelineRef, RulingAuthority};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::mind::Mind;
|
||||
|
|
@ -56,6 +56,9 @@ pub fn index_text(document: &PipelineDocument) -> Option<String> {
|
|||
lines.push(&ruling.title.0);
|
||||
lines.push(&ruling.ruling.0);
|
||||
lines.extend(ruling.operator_quote.iter().map(|quote| quote.0.as_str()));
|
||||
if let RulingAuthority::Steward { words, .. } = &ruling.authority {
|
||||
lines.push(&words.0);
|
||||
}
|
||||
}
|
||||
D::CutSpec(spec) => {
|
||||
lines.push(&spec.title.0);
|
||||
|
|
@ -247,6 +250,15 @@ More documents.");
|
|||
assert_eq!(INDEX_TEXT_VERSION, 1);
|
||||
}
|
||||
|
||||
/// A Steward ruling is found by the words the peer said as well as by its title and ruling.
|
||||
#[test]
|
||||
fn a_steward_rulings_words_are_indexed() {
|
||||
let mut peer = ruling("R1");
|
||||
peer.operator_quote = None;
|
||||
peer.authority = RulingAuthority::Steward { steward: s("@heimdall:chat"), repo: eureka_pipeline::OrgRepo(REPO.into()), words: "refuse leading zeros".into(), event: None };
|
||||
assert!(index_text(&D::Ruling(peer)).unwrap().lines().any(|line| line == "refuse leading zeros"));
|
||||
}
|
||||
|
||||
fn texts(mind: &Mind<MemoryStore>) -> BTreeMap<String, String> {
|
||||
mind.index_entries(None).unwrap().into_iter().map(|entry| (entry.id.id.0, entry.text)).collect()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ mod rows;
|
|||
pub mod oracle;
|
||||
pub mod query;
|
||||
pub mod receipt;
|
||||
pub mod redact;
|
||||
pub mod refusal;
|
||||
pub mod store;
|
||||
pub mod wire;
|
||||
|
|
@ -72,7 +73,7 @@ pub use store::{MindStore, OwnedRedbMessagePackBackingStore};
|
|||
pub use wire::{
|
||||
DeferredAnswer, HuginnMindRequest, HuginnMindResponse, IndexStatus, MAX_DEFERRED_BODY_BYTES, MIND_REQUEST_SCHEMA, MIND_REQUEST_SCHEMA_JSON,
|
||||
MIND_RESPONSE_SCHEMA, MIND_RESPONSE_SCHEMA_JSON, MIND_SERVICE_ID, MindStatus, PersonaStatus,
|
||||
ACTIVITY_REPOS_MAX, OpenPr, PrAction, RepoActivity, RepoChange, RepoEvent, RepoMoved, SOURCE_TOP_K_MAX, SourceHit, SourceHits, SourceModel, SourceQuery,
|
||||
ACTIVITY_REPOS_MAX, CHAT_ROOMS_MAX, CHAT_WINDOW_MAX, ChatHit, ChatHits, ChatMessage, ChatQuery, ChatWindow, ChatWindowQuery, EVENT_ID_MAX, OpenPr, PrAction, RepoActivity, RepoChange, RepoEvent, RepoMoved, SOURCE_TEXTS_MAX, SOURCE_TOP_K_MAX, SourceHit, SourceHits, SourceModel, SourceQuery,
|
||||
};
|
||||
|
||||
/// The leaf, whole, through the one crate that pins its rev: the daemon and
|
||||
|
|
|
|||
|
|
@ -191,7 +191,7 @@ impl PipelineFacts {
|
|||
title: value.title.clone(),
|
||||
answers: value.answers.clone(),
|
||||
choice: value.choice.clone(),
|
||||
authority: value.authority,
|
||||
authority: value.authority.clone(),
|
||||
ruled_on: value.ruled_on.clone(),
|
||||
},
|
||||
D::CutSpec(value) => Self::CutSpec {
|
||||
|
|
@ -1504,6 +1504,25 @@ mod tests {
|
|||
assert!(matches!(refused(&mind, &misspelt), MindRefusal::SelectionInvalid { value: Some(v), .. } if v == "Minds"));
|
||||
}
|
||||
|
||||
/// A peer's ruling is found by the Mind who decided, by the stewardship she decided under, and by authority.
|
||||
#[test]
|
||||
fn a_peers_rulings_are_selected_by_mind_and_by_repo() {
|
||||
let mut mind = seeded();
|
||||
let mut peer = ruling("R1");
|
||||
peer.authority = RulingAuthority::Steward {
|
||||
steward: "@heimdall:chat".into(),
|
||||
repo: eureka_pipeline::OrgRepo("GameCult/Bifrost".into()),
|
||||
words: "refuse leading zeros".into(),
|
||||
event: None,
|
||||
};
|
||||
committed(admit(&mut mind, vec![D::Ruling(peer), D::Ruling(ruling("R2"))]));
|
||||
let found = |alias: &str, word: &str| mind.query(&with(of_kinds(&[K::Ruling]), any_of(alias, &[word]))).unwrap().matched;
|
||||
assert_eq!(found("authority", "Steward"), 1);
|
||||
assert_eq!(found("about", "@heimdall:chat"), 1);
|
||||
assert_eq!(found("repo", "GameCult/Bifrost"), 1);
|
||||
assert_eq!(found("repo", "GameCult/CultLib"), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unknown_schema_or_malformed_key_is_refused() {
|
||||
let mind = worked();
|
||||
|
|
|
|||
514
crates/huginn-mind/src/redact.rs
Normal file
514
crates/huginn-mind/src/redact.rs
Normal file
|
|
@ -0,0 +1,514 @@
|
|||
//! Credential redaction: the only code that knows the shape of a secret in text.
|
||||
//!
|
||||
//! `redact_url_userinfo` redacts URL credentials (the Self run's journal and reply,
|
||||
//! the Persona's structured output). `scrub` is chat ingest's door: every message body
|
||||
//! passes it before it is chunked, embedded or stored, and it also runs the URL rule.
|
||||
//! Shapes only; nothing here reads meaning, and a hit is a kind, never a value.
|
||||
|
||||
/// The one redaction of URL credentials. Call it at each place a string the CLI produced enters the journal, her mind or chat: today
|
||||
/// the Self result (`result_text`: journal line and Reply) and the structured output of a Persona or Sleep result
|
||||
/// (`persona::structured_output`, string by string). It does not cover claude's stderr (inherited) or the CLI's transcript files.
|
||||
///
|
||||
/// A forge token rides in a remote URL, so every `scheme://` and every protocol-relative `//` opens an authority, read as git, curl and
|
||||
/// RFC 3986 read it: it ends at the first `/`, `?`, `#`, backslash, quote, backtick, parenthesis, bracket, angle bracket or ASCII whitespace
|
||||
/// (`is_authority_end`). Its userinfo is everything before the authority's LAST `@` (a raw `@` in a password still redacts), and non-ASCII
|
||||
/// characters, NBSP and U+2028 included, stay inside it. The userinfo becomes `<redacted>`: `scheme://<redacted>@host`. With a colon it is
|
||||
/// always redacted; without one it is a bare token on http, https, ftp, ftps, ws, wss, any `+http`/`+https` scheme (`git+https`), protocol-relative `//` and an unparseable scheme, and a plain
|
||||
/// username on any other scheme (`ssh://git@host`), which stays. A `@` after the authority (`github.com/@user`, `pkg@1.2.3`) is path, not
|
||||
/// userinfo, and is left alone. Scheme-less credentials are covered only in the shape `word:secret@host.tld/` at the start of an
|
||||
/// ASCII-whitespace-delimited word. Not covered: a password holding `/ ? # ` a quote or ASCII whitespace (no parser reads it as userinfo),
|
||||
/// scp-style `user:secret@host:path`, tokens in query strings, claude's stderr, the kept transcripts. Syntax only; no reading of meaning.
|
||||
pub fn redact_url_userinfo(text: &str) -> String {
|
||||
let mut out = String::with_capacity(text.len());
|
||||
let mut rest = text;
|
||||
while let Some(at) = rest.find("//") {
|
||||
let (before, after) = rest.split_at(at);
|
||||
let body = &after[2..];
|
||||
out.push_str(before);
|
||||
out.push_str("//");
|
||||
// The scheme is the ASCII run before `:`. Without a colon, `//` opens an authority only at a word start (`a//b@c` is not one).
|
||||
let scheme = match before.strip_suffix(':') {
|
||||
// Only ASCII bytes match, so the byte after the last non-match is a character boundary.
|
||||
Some(head) => head[head.bytes().rposition(|c| !(c.is_ascii_alphanumeric() || matches!(c, b'+' | b'-' | b'.'))).map_or(0, |i| i + 1)..].to_ascii_lowercase(),
|
||||
None if before.chars().next_back().is_none_or(|c| is_break(c) || matches!(c, '"' | '\'' | '`' | '(' | '[' | '{' | '<' | '=' | ',')) => String::new(),
|
||||
None => {
|
||||
rest = body;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let end = body.find(is_authority_end).unwrap_or(body.len());
|
||||
let authority = &body[..end];
|
||||
match authority.rfind('@') {
|
||||
Some(i) if i > 0 && (authority[..i].contains(':') || carries_bare_token(&scheme)) => {
|
||||
out.push_str("<redacted>");
|
||||
out.push_str(&authority[i..]);
|
||||
}
|
||||
_ => out.push_str(authority),
|
||||
}
|
||||
rest = &body[end..];
|
||||
}
|
||||
out.push_str(rest);
|
||||
redact_schemeless(&out)
|
||||
}
|
||||
|
||||
/// Schemes whose lone userinfo is a token, not a username: web and ftp schemes, `ws`/`wss`, and `x+http`/`x+https` wrappers; `scheme` is lowercase.
|
||||
fn carries_bare_token(scheme: &str) -> bool {
|
||||
matches!(scheme, "" | "http" | "https" | "ftp" | "ftps" | "ws" | "wss") || scheme.ends_with("+http") || scheme.ends_with("+https")
|
||||
}
|
||||
|
||||
/// ASCII whitespace; NBSP, U+2028 and other Unicode whitespace are not, so a secret holding them is redacted whole.
|
||||
fn is_break(c: char) -> bool {
|
||||
matches!(c, ' ' | '\t' | '\n' | '\r' | '\x0b' | '\x0c')
|
||||
}
|
||||
|
||||
/// The end of a URL's authority.
|
||||
fn is_authority_end(c: char) -> bool {
|
||||
is_break(c) || matches!(c, '/' | '?' | '#' | '\\' | '"' | '\'' | '`' | '(' | ')' | '[' | ']' | '{' | '}' | '<' | '>')
|
||||
}
|
||||
|
||||
/// `word:secret@host.tld/` at the start of a whitespace-delimited word that holds no `//` becomes `<redacted>@host.tld/`.
|
||||
fn redact_schemeless(text: &str) -> String {
|
||||
let mut out = String::with_capacity(text.len());
|
||||
let mut word = String::new();
|
||||
let flush = |word: &mut String, out: &mut String| {
|
||||
out.push_str(&redact_word(word));
|
||||
word.clear();
|
||||
};
|
||||
for c in text.chars() {
|
||||
if is_break(c) {
|
||||
flush(&mut word, &mut out);
|
||||
out.push(c);
|
||||
} else {
|
||||
word.push(c);
|
||||
}
|
||||
}
|
||||
flush(&mut word, &mut out);
|
||||
out
|
||||
}
|
||||
|
||||
fn redact_word(word: &str) -> String {
|
||||
let shaped = (|| {
|
||||
if word.contains("//") {
|
||||
return None;
|
||||
}
|
||||
let at = word.rfind('@')?;
|
||||
let colon = word.find(':')?;
|
||||
let (user, host) = (&word[..colon], &word[at + 1..]);
|
||||
let host_name = &host[..host.find('/')?];
|
||||
let user_ok = !user.is_empty() && user.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-'));
|
||||
let host_ok = host_name.contains('.') && host_name.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-'));
|
||||
(colon + 1 < at && user_ok && host_ok).then_some(at)
|
||||
})();
|
||||
match shaped {
|
||||
Some(at) => format!("<redacted>{}", &word[at..]),
|
||||
None => word.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The version of the scrub's rules. Part of the chat version: a bump makes every
|
||||
/// room re-ingest from the room's history and scrub it again.
|
||||
pub const SCRUB_VERSION: u32 = 2;
|
||||
|
||||
/// What replaces a secret.
|
||||
pub const REDACTED: &str = "<redacted>";
|
||||
|
||||
/// The shape of secret a scrub found. Never its value.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub enum ScrubKind {
|
||||
UrlUserinfo,
|
||||
Bearer,
|
||||
AuthorizationHeader,
|
||||
KeyShape,
|
||||
Jwt,
|
||||
PemBlock,
|
||||
AccessToken,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ScrubKind {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter.write_str(match self {
|
||||
Self::UrlUserinfo => "url-userinfo",
|
||||
Self::Bearer => "bearer",
|
||||
Self::AuthorizationHeader => "authorization-header",
|
||||
Self::KeyShape => "key-shape",
|
||||
Self::Jwt => "jwt",
|
||||
Self::PemBlock => "pem-block",
|
||||
Self::AccessToken => "access-token",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// A body with its secrets replaced by `<redacted>`, and the kinds found, one per secret.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Scrubbed {
|
||||
pub text: String,
|
||||
pub hits: Vec<ScrubKind>,
|
||||
}
|
||||
|
||||
/// Zero-width characters a secret could be split with.
|
||||
fn zero_width(c: char) -> bool {
|
||||
matches!(c, '\u{200b}' | '\u{200c}' | '\u{200d}' | '\u{2060}' | '\u{feff}' | '\u{00ad}' | '\u{180e}')
|
||||
}
|
||||
|
||||
/// Replaces credential shapes in `text`: a PEM or PGP private key block, an `authorization:` header
|
||||
/// line, `Bearer`/`Basic`/`token` followed by a long token (blank or colon between), a vendor key
|
||||
/// shape (Matrix tokens included), an `access_token` query or JSON value, a JWT, and URL userinfo.
|
||||
/// Each hit becomes `<redacted>`. Token shapes only; ordinary prose, SHAs, UUIDs and ids pass byte for byte.
|
||||
/// Zero-width characters are removed before matching; when anything matched, the returned text is the
|
||||
/// stripped one, so a token split by them leaves no tail, and when nothing matched the text is returned as it came (emoji joiners stay).
|
||||
pub fn scrub(text: &str) -> Scrubbed {
|
||||
let mut hits = Vec::new();
|
||||
let mut stripped: String = text.chars().filter(|c| !zero_width(*c)).collect();
|
||||
let passes: [(ScrubKind, fn(&str, usize) -> Option<usize>); 6] = [
|
||||
(ScrubKind::PemBlock, pem_block),
|
||||
(ScrubKind::AuthorizationHeader, authorization_line),
|
||||
(ScrubKind::AccessToken, access_token),
|
||||
(ScrubKind::Bearer, bearer_token),
|
||||
(ScrubKind::KeyShape, key_shape),
|
||||
(ScrubKind::Jwt, jwt),
|
||||
];
|
||||
for (kind, find) in passes {
|
||||
stripped = replace_matches(&stripped, kind, find, &mut hits);
|
||||
}
|
||||
let urls = redact_url_userinfo(&stripped);
|
||||
let (before, after) = (stripped.matches(REDACTED).count(), urls.matches(REDACTED).count());
|
||||
hits.extend(std::iter::repeat_n(ScrubKind::UrlUserinfo, after.saturating_sub(before)));
|
||||
if hits.is_empty() { Scrubbed { text: text.to_owned(), hits } } else { Scrubbed { text: urls, hits } }
|
||||
}
|
||||
|
||||
fn replace_matches(text: &str, kind: ScrubKind, find: fn(&str, usize) -> Option<usize>, hits: &mut Vec<ScrubKind>) -> String {
|
||||
let mut out = String::with_capacity(text.len());
|
||||
let mut at = 0;
|
||||
while at < text.len() {
|
||||
match find(text, at) {
|
||||
Some(end) if end > at => {
|
||||
out.push_str(REDACTED);
|
||||
hits.push(kind);
|
||||
at = end;
|
||||
}
|
||||
_ => {
|
||||
let next = at + text[at..].chars().next().map_or(1, char::len_utf8);
|
||||
out.push_str(&text[at..next]);
|
||||
at = next;
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Whether `at` starts a word: the byte before it is not an ASCII letter or digit.
|
||||
fn word_start(text: &str, at: usize) -> bool {
|
||||
at == 0 || !text.as_bytes()[at - 1].is_ascii_alphanumeric()
|
||||
}
|
||||
|
||||
/// The end of the run of bytes at `from` that `member` accepts.
|
||||
fn run_end(text: &str, from: usize, member: impl Fn(u8) -> bool) -> usize {
|
||||
from + text.as_bytes()[from..].iter().take_while(|byte| member(**byte)).count()
|
||||
}
|
||||
|
||||
fn starts_with_ignore_case(text: &str, at: usize, word: &str) -> bool {
|
||||
text.as_bytes().get(at..at + word.len()).is_some_and(|held| held.eq_ignore_ascii_case(word.as_bytes()))
|
||||
}
|
||||
|
||||
fn token_char(byte: u8) -> bool {
|
||||
byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'~' | b'+' | b'/' | b'=' | b'-')
|
||||
}
|
||||
|
||||
fn key_char(byte: u8) -> bool {
|
||||
byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-')
|
||||
}
|
||||
|
||||
/// `-----BEGIN ... PRIVATE KEY-----` through its `-----END ... PRIVATE KEY-----`, or the text's end.
|
||||
fn pem_block(text: &str, at: usize) -> Option<usize> {
|
||||
let rest = &text[at..];
|
||||
let label = rest.strip_prefix("-----BEGIN ")?;
|
||||
let words = run_end(label, 0, |byte| byte.is_ascii_uppercase() || byte == b' ');
|
||||
(label[..words].ends_with("PRIVATE KEY") || label[..words].ends_with("PRIVATE KEY BLOCK")).then_some(())?;
|
||||
label[words..].starts_with("-----").then_some(())?;
|
||||
let body_from = at + "-----BEGIN ".len() + words + 5;
|
||||
let end = text[body_from..].find("-----END ").and_then(|end| {
|
||||
let name = body_from + end + "-----END ".len();
|
||||
let words = run_end(text, name, |byte| byte.is_ascii_uppercase() || byte == b' ');
|
||||
text[words..].starts_with("-----").then_some(words + 5)
|
||||
});
|
||||
Some(end.unwrap_or(text.len()))
|
||||
}
|
||||
|
||||
/// `authorization:` in any case at a word start, through the end of the line.
|
||||
fn authorization_line(text: &str, at: usize) -> Option<usize> {
|
||||
(word_start(text, at) && starts_with_ignore_case(text, at, "authorization:")).then_some(())?;
|
||||
let from = at + "authorization:".len();
|
||||
let end = run_end(text, from, |byte| byte != b'\n' && byte != b'\r');
|
||||
(!text[from..end].trim().is_empty()).then_some(end)
|
||||
}
|
||||
|
||||
/// `Bearer`, `Basic` or `token`, blanks, then 16 or more token characters.
|
||||
fn bearer_token(text: &str, at: usize) -> Option<usize> {
|
||||
word_start(text, at).then_some(())?;
|
||||
let word = ["bearer", "basic", "token"].into_iter().find(|word| starts_with_ignore_case(text, at, word))?;
|
||||
let blanks = at + word.len();
|
||||
let colon = usize::from(text.as_bytes().get(blanks) == Some(&b':'));
|
||||
let value = run_end(text, blanks + colon, |byte| byte == b' ' || byte == b'\t');
|
||||
(value > blanks).then_some(())?;
|
||||
let end = run_end(text, value, token_char);
|
||||
(end - value >= 16).then_some(end)
|
||||
}
|
||||
|
||||
/// A vendor key: the prefix, then a long enough run of the key alphabet.
|
||||
fn key_shape(text: &str, at: usize) -> Option<usize> {
|
||||
let rest = &text[at..];
|
||||
let by_prefix = [("sk-", 16), ("sk_live_", 16), ("pk_live_", 16), ("github_pat_", 16), ("glpat-", 16), ("syt_", 16), ("mct_", 16)];
|
||||
let tokens = ["ghp_", "gho_", "ghu_", "ghs_", "ghr_"].into_iter().map(|prefix| (prefix, 16));
|
||||
for (prefix, least) in by_prefix.into_iter().chain(tokens) {
|
||||
// Only the generic prefix needs a word start (`risk-averse`, `task-force`); the distinctive ones are found glued to anything.
|
||||
if rest.starts_with(prefix) && (prefix != "sk-" || word_start(text, at)) {
|
||||
let end = run_end(text, at + prefix.len(), key_char);
|
||||
return (end - at - prefix.len() >= least).then_some(end);
|
||||
}
|
||||
}
|
||||
for prefix in ["xoxa-", "xoxb-", "xoxp-", "xoxr-", "xoxs-"] {
|
||||
if rest.starts_with(prefix) {
|
||||
let end = run_end(text, at + prefix.len(), |byte| byte.is_ascii_alphanumeric() || byte == b'-');
|
||||
return (end - at - prefix.len() >= 16).then_some(end);
|
||||
}
|
||||
}
|
||||
for prefix in ["AKIA", "ASIA"] {
|
||||
if rest.starts_with(prefix) {
|
||||
let end = run_end(text, at + prefix.len(), |byte| byte.is_ascii_uppercase() || byte.is_ascii_digit());
|
||||
return (end - at - prefix.len() >= 16).then_some(end);
|
||||
}
|
||||
}
|
||||
if rest.starts_with("AIza") {
|
||||
let end = run_end(text, at + 4, key_char);
|
||||
return (end - at - 4 >= 35).then_some(end);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// `access_token` as a query parameter or JSON member, through its value (8 or more characters).
|
||||
fn access_token(text: &str, at: usize) -> Option<usize> {
|
||||
starts_with_ignore_case(text, at, "access_token").then_some(())?;
|
||||
let mut at = at + "access_token".len();
|
||||
let skip = |at: usize, set: &[u8]| run_end(text, at, |byte| set.contains(&byte));
|
||||
at = skip(at, b"\"'");
|
||||
at = skip(at, b" \t");
|
||||
matches!(text.as_bytes().get(at), Some(b'=' | b':')).then_some(())?;
|
||||
at = skip(at + 1, b" \t");
|
||||
at = skip(at, b"\"'");
|
||||
let end = run_end(text, at, |byte| !byte.is_ascii_whitespace() && !matches!(byte, b'&' | b'"' | b'\'' | b',' | b'}' | b';' | b'#'));
|
||||
(end - at >= 8).then_some(end)
|
||||
}
|
||||
|
||||
/// Three base64url segments, the first starting `eyJ`.
|
||||
fn jwt(text: &str, at: usize) -> Option<usize> {
|
||||
text[at..].starts_with("eyJ").then_some(())?;
|
||||
let segment = |from: usize| run_end(text, from, key_char);
|
||||
let first = segment(at);
|
||||
let mut end = first;
|
||||
for _ in 0..2 {
|
||||
(text.as_bytes().get(end) == Some(&b'.')).then_some(())?;
|
||||
let next = segment(end + 1);
|
||||
(next > end + 1).then_some(())?;
|
||||
end = next;
|
||||
}
|
||||
Some(end)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const CANARY: &str = "CANARY-NOT-A-REAL-TOKEN-0000";
|
||||
|
||||
#[test]
|
||||
fn url_userinfo_is_redacted_where_it_is_a_secret_and_kept_where_it_is_a_username() {
|
||||
let said = format!(
|
||||
"a https://user:{CANARY}@host.example/r.git b https://{CANARY}@host.example/r.git c ssh://u:{CANARY}@h/x d http://{CANARY}@h e HTTPS://x:{CANARY}@h \
|
||||
f https://u:p@ss{CANARY}@h/x g ssh://git@host.example/r.git h git://git@h/x i https://host.example/no-userinfo j (https://u:{CANARY}@h) k ://@x"
|
||||
);
|
||||
let want = "a https://<redacted>@host.example/r.git b https://<redacted>@host.example/r.git c ssh://<redacted>@h/x d http://<redacted>@h e HTTPS://<redacted>@h \
|
||||
f https://<redacted>@h/x g ssh://git@host.example/r.git h git://git@h/x i https://host.example/no-userinfo j (https://<redacted>@h) k ://@x";
|
||||
assert_eq!(redact_url_userinfo(&said), want);
|
||||
assert!(!redact_url_userinfo(&said).contains(CANARY));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_secret_is_redacted_whole_by_the_authority_rule_and_commonplace_urls_are_left_alone() {
|
||||
let r = redact_url_userinfo;
|
||||
// Raw @, percent-encoding and non-ASCII (NBSP, U+2028) stay inside userinfo.
|
||||
for secret in ["pa@ssT", "pa%40ssT", "pa\u{a0}ssT", "pa\u{2028}ssT", "pa:ssT", "\u{e9}\u{65e5}ssT"] {
|
||||
let out = r(&format!("x https://u:{secret}@h/x"));
|
||||
assert_eq!(out, "x https://<redacted>@h/x", "{secret:?}");
|
||||
}
|
||||
// The authority ends at / ? # \\ quotes, brackets and ASCII whitespace: no parser reads those as userinfo, so what follows is path.
|
||||
assert_eq!(r("https://u:pa/ssT@h/x"), "https://u:pa/ssT@h/x");
|
||||
assert_eq!(r("https://u:pa ssT@h/x"), "https://u:pa ssT@h/x");
|
||||
assert_eq!(r("https://u:pa?ssT@h"), "https://u:pa?ssT@h");
|
||||
// Ordinary URLs with @ in the path are byte-identical.
|
||||
for same in [
|
||||
"https://github.com/@user/repo",
|
||||
"https://medium.com/@name/post-1",
|
||||
"https://registry.npmjs.org/@scope/pkg",
|
||||
"https://h/pkg@1.2.3",
|
||||
"a//b@c",
|
||||
"see //cdn.h/@x/y and https://h/?mail=a@b.c#f@g",
|
||||
"https://h:8080/p git@github.com:org/r.git ssh://git@host/r.git ssh://git@h:22/r git://git@h/x",
|
||||
] {
|
||||
assert_eq!(r(same), same);
|
||||
}
|
||||
// Compact JSON: only the credentialed URL changes.
|
||||
let json = format!(r#"{{"a":"https://u:{CANARY}@h/x","b":"https://h2/@v","c":"https://h3/p@1"}}"#);
|
||||
assert_eq!(r(&json), r#"{"a":"https://<redacted>@h/x","b":"https://h2/@v","c":"https://h3/p@1"}"#);
|
||||
assert_eq!(r("a/https://u:T@h/x,https://v:T@h2"), "a/https://<redacted>@h/x,https://<redacted>@h2");
|
||||
assert_eq!(r("HTTPS://TOKEN@h"), "HTTPS://<redacted>@h");
|
||||
assert_eq!(r("[l](https://T@h/x) <https://T@h> `https://u:T@h`"), "[l](https://<redacted>@h/x) <https://<redacted>@h> `https://<redacted>@h`");
|
||||
assert_eq!(r("a //u:T@host/x b //T@host c \"//T@h\" (//T@h)"), "a //<redacted>@host/x b //<redacted>@host c \"//<redacted>@h\" (//<redacted>@h)");
|
||||
// Token-bearing schemes beyond http: +http/+https wrappers and ws/wss redact a bare token; git+ssh and ssh keep their username.
|
||||
assert_eq!(r("git+https://T@github.com/a/b GIT+HTTPS://T@h wss://T@h/ ws://T@h/"), "git+https://<redacted>@github.com/a/b GIT+HTTPS://<redacted>@h wss://<redacted>@h/ ws://<redacted>@h/");
|
||||
for same in ["git+ssh://git@host/x", "ssh://git@host/x"] {
|
||||
assert_eq!(r(same), same);
|
||||
}
|
||||
// Scheme-less: only word:secret@host.tld/ . Each guard of the shape has its own case.
|
||||
assert_eq!(r("user:T@host.tld/path"), "<redacted>@host.tld/path");
|
||||
for same in ["a:b@c", "a:b@host/x", "a:b@ho_st.tld/x", "u$er:T@host.tld/x", "a:@host.tld/x", "mail me@x.org/ and :T@host.tld/x"] {
|
||||
assert_eq!(r(same), same);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multibyte_text_never_panics_the_redaction() {
|
||||
let r = redact_url_userinfo;
|
||||
for case in ["see \u{201c}https://h/x", "\u{2192}https://h/x", "docs\u{ff1a}https://h/x", "\u{e9}://x", "\u{65e5}\u{672c}://u:T@h", "https://x/\u{e9}://", "\u{e9}//u:T@h", "\u{e9}:u:T@h.tld/"] {
|
||||
let _ = r(case);
|
||||
}
|
||||
assert_eq!(r("\u{65e5}\u{672c}://u:T@h"), "\u{65e5}\u{672c}://<redacted>@h");
|
||||
assert_eq!(r("\u{2192}https://u:T@h/x"), "\u{2192}https://<redacted>@h/x");
|
||||
let seed = "\u{201c}\u{2192}\u{ff1a}\u{e9}\u{65e5}\u{a0}https://u:T@h/x//y:z@a.b/ ://@ //";
|
||||
let chars: Vec<char> = seed.chars().collect();
|
||||
for start in 0..=chars.len() {
|
||||
for end in start..=chars.len() {
|
||||
let _ = r(&chars[start..end].iter().collect::<String>());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// Every shape is built here from fake parts; none is a real credential.
|
||||
fn secrets() -> Vec<(ScrubKind, String, String)> {
|
||||
let pem = "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAfakefakefake\n-----END RSA PRIVATE KEY-----".to_owned();
|
||||
vec![
|
||||
(ScrubKind::AuthorizationHeader, format!("Authorization: Bearer {}", "a1B2".repeat(8)), "a1B2a1B2".into()),
|
||||
(ScrubKind::Bearer, format!("Bearer {}", "z9Y8x7".repeat(4)), "z9Y8x7z9Y8".into()),
|
||||
(ScrubKind::Bearer, format!("Basic {}", "dXNlcjpwYXNz".repeat(2)), "dXNlcjpwYXNzdXNl".into()),
|
||||
(ScrubKind::KeyShape, format!("sk-{}", "q7W3".repeat(8)), "q7W3q7W3".into()),
|
||||
(ScrubKind::KeyShape, format!("sk_live_{}", "m4N5".repeat(6)), "m4N5m4N5".into()),
|
||||
(ScrubKind::KeyShape, format!("ghp_{}", "k2L9".repeat(9)), "k2L9k2L9".into()),
|
||||
(ScrubKind::KeyShape, format!("github_pat_{}_{}", "A1b2C3".repeat(4) + "ab", "d4E5f6".repeat(9) + "abcde"), "A1b2C3A1b2".into()),
|
||||
(ScrubKind::KeyShape, format!("glpat-{}", "r6T7".repeat(5)), "r6T7r6T7".into()),
|
||||
(ScrubKind::KeyShape, "xoxb-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL".into()),
|
||||
(ScrubKind::KeyShape, "AKIA0123456789ABCDEF".into(), "0123456789ABCDEF".into()),
|
||||
(ScrubKind::KeyShape, format!("AIza{}", "Sy1_".repeat(9)), "Sy1_Sy1_Sy1_".into()),
|
||||
(ScrubKind::Jwt, "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJmYWtlIn0.c2lnbmF0dXJlZmFrZQ".into(), "c2lnbmF0dXJlZmFrZQ".into()),
|
||||
(ScrubKind::PemBlock, pem, "MIIEowIBAAKCAQEAfake".into()),
|
||||
(ScrubKind::PemBlock, "-----BEGIN PRIVATE KEY-----\nMIIBVQIBADANfakeNOEND".into(), "MIIBVQIBADANfake".into()),
|
||||
(ScrubKind::KeyShape, format!("syt_{}_{}_{}", "YWxpY2U", "aBcDeFgHiJkLmNoPqR", "0a1B2c"), "aBcDeFgHiJkLmNoPqR".into()),
|
||||
(ScrubKind::KeyShape, format!("mct_{}", "w4X5".repeat(8)), "w4X5w4X5w4X5".into()),
|
||||
(ScrubKind::KeyShape, "xoxa-2-123456789012-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL".into()),
|
||||
(ScrubKind::KeyShape, "xoxp-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL".into()),
|
||||
(ScrubKind::KeyShape, "xoxr-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL".into()),
|
||||
(ScrubKind::KeyShape, "xoxs-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL".into()),
|
||||
(ScrubKind::KeyShape, "ASIA0123456789ABCDEF".into(), "0123456789ABCDEF".into()),
|
||||
(ScrubKind::KeyShape, "gho_".to_owned() + &"p1Q2".repeat(9), "p1Q2p1Q2p1Q2".into()),
|
||||
(ScrubKind::KeyShape, "pk_live_".to_owned() + &"e3R4".repeat(6), "e3R4e3R4e3R4".into()),
|
||||
(ScrubKind::AccessToken, "https://h/_matrix/client/v3/sync?access_token=abcDEF123456xyz&since=s1".into(), "abcDEF123456xyz".into()),
|
||||
(ScrubKind::AccessToken, "{\"access_token\": \"tokenValue987654\", \"user\": 1}".into(), "tokenValue987654".into()),
|
||||
(ScrubKind::Bearer, format!("bearer:{}", "z9Y8x7".repeat(4)), "z9Y8x7z9Y8".into()),
|
||||
(ScrubKind::Bearer, format!("token: {}", "m1N2o3".repeat(4)), "m1N2o3m1N2".into()),
|
||||
(ScrubKind::PemBlock, "-----BEGIN PGP PRIVATE KEY BLOCK-----\nlQOYBFakeFakeFake\n-----END PGP PRIVATE KEY BLOCK-----".into(), "lQOYBFakeFake".into()),
|
||||
(ScrubKind::UrlUserinfo, "https://user:hunter2hunter2@host.example/r.git".into(), "hunter2hunter2".into()),
|
||||
(ScrubKind::UrlUserinfo, "https://tokenvalue12345@host.example/".into(), "tokenvalue12345".into()),
|
||||
]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_planted_credential_is_replaced_mid_line_and_at_line_end_with_its_kind() {
|
||||
for (kind, secret, distinct) in secrets() {
|
||||
for body in [format!("before {secret} after"), format!("before {secret}"), format!("{secret}\nnext line")] {
|
||||
let out = scrub(&body);
|
||||
assert!(!out.text.contains(&distinct), "{kind}: {}", out.text);
|
||||
assert!(out.text.contains(REDACTED), "{kind}: {}", out.text);
|
||||
assert!(out.hits.contains(&kind), "{kind} not in {:?} for {body:?}", out.hits);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_pem_block_without_its_end_runs_to_the_end_of_the_body_and_one_with_it_stops_there() {
|
||||
let open = scrub("see -----BEGIN EC PRIVATE KEY-----\nAAAA\nBBBB").text;
|
||||
assert_eq!(open, "see <redacted>");
|
||||
let closed = scrub("-----BEGIN PRIVATE KEY-----\nAAAA\n-----END PRIVATE KEY----- and then prose").text;
|
||||
assert_eq!(closed, "<redacted> and then prose");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_authorization_header_goes_to_the_end_of_its_line_only() {
|
||||
assert_eq!(scrub("hi\nAUTHORIZATION: Token abc\nbye").text, "hi\n<redacted>\nbye");
|
||||
assert_eq!(scrub("Authorization: x").hits, [ScrubKind::AuthorizationHeader]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ordinary_text_passes_byte_for_byte_and_reports_no_hit() {
|
||||
for same in [
|
||||
"risk-averse", "task-force", "sk-", "sk-short", "3f2b8c1d9e7a4b5c6d0e1f2a3b4c5d6e7f8a9b0c",
|
||||
"123e4567-e89b-12d3-a456-426614174000", "!abc:s", "$Ab3dEf-9_zQ", "user@example.com", "ssh://git@host/r.git",
|
||||
"https://github.com/@user", "pkg@1.2.3", "the bearer of bad news", "Authorization is a header",
|
||||
"padding like dGVzdA== in prose", "```\nlet x = 1;\n```", "the BEGIN PRIVATE KEY header", "a token of thanks",
|
||||
"AKIAshort", "ghp_short", "eyJ is a prefix", "naïve café — 日本語 with Bearer in it",
|
||||
] {
|
||||
let out = scrub(same);
|
||||
assert_eq!(out.text, same);
|
||||
assert!(out.hits.is_empty(), "{same}: {:?}", out.hits);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multibyte_text_around_a_secret_never_panics_the_scrub() {
|
||||
let body = format!("日本 \u{a0}é Bearer {} → ünï", "A1b2".repeat(6));
|
||||
assert_eq!(scrub(&body).text, "日本 \u{a0}é <redacted> → ünï");
|
||||
let seed = "é日 sk-aaaaaaaaaaaaaaaaaaaa -----BEGIN PRIVATE KEY----- authorization: é eyJa.b.c https://u:p@h/x";
|
||||
let chars: Vec<char> = seed.chars().collect();
|
||||
for start in 0..chars.len() {
|
||||
for end in start..=chars.len() {
|
||||
let _ = scrub(&chars[start..end].iter().collect::<String>());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_distinctive_key_is_found_glued_to_other_text_but_the_generic_prefix_still_needs_a_word_start() {
|
||||
let ghp = format!("xxx{}", "ghp_".to_owned() + &"k2L9".repeat(9));
|
||||
assert_eq!(scrub(&ghp).text, "xxx<redacted>");
|
||||
assert_eq!(scrub("idAKIA0123456789ABCDEFend").hits, [ScrubKind::KeyShape]);
|
||||
assert_eq!(scrub(&format!("zzz{}", "xoxb-123456789012-aBcDeFgHiJkLmNoP")).hits, [ScrubKind::KeyShape]);
|
||||
assert_eq!(scrub(&format!("q{}", "AIza".to_owned() + &"Sy1_".repeat(9))).hits, [ScrubKind::KeyShape]);
|
||||
assert!(scrub("risk-averse-thinking-process task-force-alpha-bravo").hits.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_width_characters_inside_a_token_leave_no_tail_and_emoji_joiners_survive_when_nothing_matched() {
|
||||
let split = format!("sk-{}\u{200b}{}", "q7W3".repeat(4), "q7W3".repeat(4));
|
||||
let out = scrub(&format!("key {split} end"));
|
||||
assert_eq!(out.text, "key <redacted> end");
|
||||
let jwt = "eyJhbGciOiJIUzI1NiJ9.eyJz\u{feff}dWIiOiJmYWtlIn0.c2lnbmF0dXJl\u{200d}ZmFrZQ";
|
||||
assert_eq!(scrub(jwt).text, "<redacted>");
|
||||
let family = "family \u{1f468}\u{200d}\u{1f469}\u{200d}\u{1f467} here";
|
||||
assert_eq!(scrub(family).text, family);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matrix_tokens_and_query_values_are_found_and_prose_about_them_is_not() {
|
||||
for same in ["the access_token is a thing", "access_token: short", "syt_ is the prefix", "mct_ tokens", "bearer:short"] {
|
||||
assert_eq!(scrub(same).text, same, "{same}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -13,7 +13,7 @@ use std::collections::BTreeMap;
|
|||
|
||||
use cultnet_rs::{Row, RowSet, Selection};
|
||||
use eureka_pipeline::{
|
||||
FindingConfidence, FindingOrigin, FindingSeverity, Label, MemoryOrigin, MemorySort, OrgRepo, PipelineDocument, PipelineKind, PipelineRef, ResolutionOutcome,
|
||||
FindingConfidence, FindingOrigin, FindingSeverity, Label, MemoryOrigin, MemorySort, OrgRepo, PipelineDocument, PipelineKind, PipelineRef, PipelineRuling, ResolutionOutcome,
|
||||
RulingAuthority, RunOperator, RunTurn, Slug,
|
||||
};
|
||||
|
||||
|
|
@ -134,7 +134,7 @@ impl Alias {
|
|||
match self {
|
||||
Self::Root | Self::InForce | Self::Faculty => true,
|
||||
Self::Repo => {
|
||||
matches!(kind, K::Campaign | K::CutSpec | K::CutReport | K::FollowUp | K::Stewardship | K::HandOff | K::Resolution)
|
||||
matches!(kind, K::Campaign | K::CutSpec | K::CutReport | K::FollowUp | K::Stewardship | K::HandOff | K::Resolution | K::Ruling)
|
||||
}
|
||||
Self::Cut => matches!(kind, K::CutSpec | K::CutReport | K::Verdict | K::Finding | K::Resolution),
|
||||
Self::Severity | Self::Confidence | Self::Origin => kind == K::Finding,
|
||||
|
|
@ -143,7 +143,8 @@ impl Alias {
|
|||
Self::Outcome => kind == K::Resolution,
|
||||
Self::Turn | Self::OperatedBy | Self::Ran => kind == K::Run,
|
||||
Self::Voted | Self::Urged => kind == K::Pulse,
|
||||
Self::Sort | Self::About | Self::MemoryOrigin => kind == K::Memory,
|
||||
Self::Sort | Self::MemoryOrigin => kind == K::Memory,
|
||||
Self::About => matches!(kind, K::Memory | K::Ruling),
|
||||
Self::Pass => matches!(kind, K::Research | K::Idea),
|
||||
}
|
||||
}
|
||||
|
|
@ -246,6 +247,7 @@ impl SelectionRow {
|
|||
D::FollowUp(follow_up) => vec![follow_up.repo.identity()],
|
||||
D::Stewardship(stewardship) => vec![stewardship.repo.identity()],
|
||||
D::HandOff(hand_off) => vec![hand_off.repo.identity()],
|
||||
D::Ruling(PipelineRuling { authority: RulingAuthority::Steward { repo, .. }, .. }) => vec![repo.identity()],
|
||||
D::Target(_) | D::Question(_) | D::Ruling(_) | D::Verdict(_) | D::Finding(_) | D::Instance(_) | D::Resolution(_) | D::Run(_) | D::Pulse(_) | D::Reply(_) | D::Thought(_) | D::Memory(_) | D::Research(_) | D::Idea(_) | D::DeskAnswer(_) => {
|
||||
Vec::new()
|
||||
}
|
||||
|
|
@ -267,7 +269,12 @@ impl SelectionRow {
|
|||
put(Alias::Confidence, vec![format!("{:?}", finding.confidence)]);
|
||||
put(Alias::Origin, vec![format!("{:?}", finding.origin)]);
|
||||
}
|
||||
D::Ruling(ruling) => put(Alias::Authority, vec![format!("{:?}", ruling.authority)]),
|
||||
D::Ruling(ruling) => {
|
||||
put(Alias::Authority, vec![ruling.authority.name().to_string()]);
|
||||
if let RulingAuthority::Steward { steward, .. } = &ruling.authority {
|
||||
put(Alias::About, vec![steward.0.clone()]);
|
||||
}
|
||||
}
|
||||
D::Verdict(verdict) => {
|
||||
put(Alias::ClaimOutcome, verdict.claims.iter().map(|claim| format!("{:?}", claim.outcome)).collect())
|
||||
}
|
||||
|
|
@ -463,7 +470,7 @@ mod tests {
|
|||
(K::Campaign, &["repo"]),
|
||||
(K::Target, &[]),
|
||||
(K::Question, &[]),
|
||||
(K::Ruling, &["authority"]),
|
||||
(K::Ruling, &["authority", "repo", "about"]),
|
||||
(K::CutSpec, &["repo", "cut"]),
|
||||
(K::CutReport, &["repo", "cut"]),
|
||||
(K::Verdict, &["cut", "claim_outcome"]),
|
||||
|
|
|
|||
|
|
@ -92,6 +92,13 @@ pub enum HuginnMindRequest {
|
|||
/// docs, which no mind holds. The daemon embeds the text and asks the
|
||||
/// index it does not write.
|
||||
SearchSources { instance: Slug, query: SourceQuery },
|
||||
/// Search of the chat index: chunks of the rooms the Minds are meant to
|
||||
/// have, by room, by time span and by meaning. Read-only, like
|
||||
/// `SearchSources`, and queued with it.
|
||||
SearchChat { instance: Slug, query: ChatQuery },
|
||||
/// A chat message and its neighbours in the same room, read from the chat
|
||||
/// index (never from Synapse), by the event id a chat hit cites.
|
||||
ChatWindow { instance: Slug, query: ChatWindowQuery },
|
||||
/// What others did in `repos` on the forge since the `after` cursor: the
|
||||
/// daemon reads the forge and no document of the mind, and answers
|
||||
/// `Refused(Unavailable)` when it has no forge or the forge does not answer
|
||||
|
|
@ -181,13 +188,23 @@ pub struct RepoMoved {
|
|||
/// The most hits a source search answers.
|
||||
pub const SOURCE_TOP_K_MAX: u32 = 50;
|
||||
|
||||
/// A source search: the text to find chunks near, how many, optionally the
|
||||
/// one forge repo (by name) to look in, and whether to look only in repos'
|
||||
/// front-page files (`FRONT_PAGE_FILES` at the repo root). An old client's
|
||||
/// request has no `front` and means false.
|
||||
/// The most texts one source search takes.
|
||||
pub const SOURCE_TEXTS_MAX: usize = 8;
|
||||
|
||||
/// A source search: the texts to find chunks near (one to `SOURCE_TEXTS_MAX`,
|
||||
/// each a topic; several answer with their ranked lists fused), how many,
|
||||
/// optionally the one forge repo (by name) to look in, and whether to look
|
||||
/// only in repos' front-page files (`FRONT_PAGE_FILES` at the repo root). An
|
||||
/// old client's request has no `front` and means false.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct SourceQuery {
|
||||
/// Transitional: an old client's single text, read as `texts` of one. Exactly
|
||||
/// one of `text` and `texts` is given. To be deleted once no client sends it.
|
||||
#[serde(default)]
|
||||
pub text: Line,
|
||||
/// The topics to search by, one to `SOURCE_TEXTS_MAX`.
|
||||
#[serde(default)]
|
||||
pub texts: Vec<Line>,
|
||||
pub top_k: u32,
|
||||
pub repo: Option<Short>,
|
||||
#[serde(default)]
|
||||
|
|
@ -195,7 +212,8 @@ pub struct SourceQuery {
|
|||
}
|
||||
|
||||
impl SourceQuery {
|
||||
/// The door: text that says something, and between one and
|
||||
/// The door: exactly one of `text` and `texts`, each text saying something,
|
||||
/// at most `SOURCE_TEXTS_MAX` texts, and between one and
|
||||
/// `SOURCE_TOP_K_MAX` hits. A refusal names the field.
|
||||
pub fn check(&self) -> Result<(), MindRefusal> {
|
||||
let refused = |field: &str, value: Option<String>, message: &str| MindRefusal::SelectionInvalid {
|
||||
|
|
@ -203,9 +221,21 @@ impl SourceQuery {
|
|||
value,
|
||||
message: message.into(),
|
||||
};
|
||||
if self.text.0.trim().is_empty() {
|
||||
if !self.text.0.is_empty() && !self.texts.is_empty() {
|
||||
return Err(refused("texts", Some(self.texts.len().to_string()), "a source search takes `text` or `texts`, not both"));
|
||||
}
|
||||
if self.texts.is_empty() && self.text.0.is_empty() {
|
||||
return Err(refused("texts", None, "a source search needs texts to find"));
|
||||
}
|
||||
if self.texts.is_empty() && self.text.0.trim().is_empty() {
|
||||
return Err(refused("text", None, "a source search needs text to find"));
|
||||
}
|
||||
if self.texts.len() > SOURCE_TEXTS_MAX {
|
||||
return Err(refused("texts", Some(self.texts.len().to_string()), "a source search takes 1 to 8 texts"));
|
||||
}
|
||||
if self.texts.iter().any(|text| text.0.trim().is_empty()) {
|
||||
return Err(refused("texts", None, "every text of a source search says something"));
|
||||
}
|
||||
if !(1..=SOURCE_TOP_K_MAX).contains(&self.top_k) {
|
||||
return Err(refused("top_k", Some(self.top_k.to_string()), "a source search asks for 1 to 50 hits"));
|
||||
}
|
||||
|
|
@ -234,6 +264,8 @@ pub struct SourceHit {
|
|||
pub commit: String,
|
||||
pub line_start: u32,
|
||||
pub line_end: u32,
|
||||
/// Cosine similarity for a search of one text; the reciprocal-rank sum
|
||||
/// (k = 60) over the texts' lists for a search of several.
|
||||
pub score: f32,
|
||||
pub text: String,
|
||||
}
|
||||
|
|
@ -245,6 +277,140 @@ pub struct SourceHits {
|
|||
pub hits: Vec<SourceHit>,
|
||||
}
|
||||
|
||||
/// The most rooms one chat search names.
|
||||
pub const CHAT_ROOMS_MAX: usize = 16;
|
||||
|
||||
/// A chat search. The selectors compose as AND. `rooms` are any-of: a value
|
||||
/// beginning `!` is a room id, `#` a canonical alias. `after_ms` and
|
||||
/// `before_ms` (Unix milliseconds) keep the chunks whose span overlaps
|
||||
/// [after, before]. `texts` (none to `SOURCE_TEXTS_MAX`) is the meaning match,
|
||||
/// several fused as in a source search; with none the search is a listing:
|
||||
/// chunks oldest first, at most `top_k`, `ChatHits.truncated` saying more
|
||||
/// exist, and the caller narrows with `after_ms` = the last hit's
|
||||
/// `ts_end_ms + 1`.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct ChatQuery {
|
||||
pub texts: Vec<Line>,
|
||||
pub rooms: Vec<Short>,
|
||||
pub after_ms: Option<u64>,
|
||||
pub before_ms: Option<u64>,
|
||||
pub top_k: u32,
|
||||
}
|
||||
|
||||
impl ChatQuery {
|
||||
/// The door. A refusal names the field.
|
||||
pub fn check(&self) -> Result<(), MindRefusal> {
|
||||
let refused = |field: &str, value: Option<String>, message: &str| MindRefusal::SelectionInvalid {
|
||||
field: field.into(),
|
||||
value,
|
||||
message: message.into(),
|
||||
};
|
||||
if self.texts.len() > SOURCE_TEXTS_MAX {
|
||||
return Err(refused("texts", Some(self.texts.len().to_string()), "a chat search takes 0 to 8 texts"));
|
||||
}
|
||||
if self.texts.iter().any(|text| text.0.trim().is_empty()) {
|
||||
return Err(refused("texts", None, "every text of a chat search says something"));
|
||||
}
|
||||
if self.rooms.len() > CHAT_ROOMS_MAX {
|
||||
return Err(refused("rooms", Some(self.rooms.len().to_string()), "a chat search names at most 16 rooms"));
|
||||
}
|
||||
if let Some(room) = self.rooms.iter().find(|room| !(room.0.starts_with('!') || room.0.starts_with('#')) || room.0.len() < 2) {
|
||||
return Err(refused("rooms", Some(room.0.clone()), "a room is a room id beginning `!` or a canonical alias beginning `#`"));
|
||||
}
|
||||
if let (Some(after), Some(before)) = (self.after_ms, self.before_ms)
|
||||
&& before < after
|
||||
{
|
||||
return Err(refused("before_ms", Some(before.to_string()), "before_ms is not earlier than after_ms"));
|
||||
}
|
||||
if !(1..=SOURCE_TOP_K_MAX).contains(&self.top_k) {
|
||||
return Err(refused("top_k", Some(self.top_k.to_string()), "a chat search asks for 1 to 50 hits"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// One message of a chunk, as the room held it.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct ChatMessage {
|
||||
pub event_id: String,
|
||||
/// The sender's Matrix id.
|
||||
pub sender: String,
|
||||
pub ts_ms: u64,
|
||||
pub body: String,
|
||||
}
|
||||
|
||||
/// One chunk of consecutive messages in one room. Every citation is
|
||||
/// required: only the room's alias may be absent, because a room may have
|
||||
/// none. The span and the first and last ids are those of `messages` as
|
||||
/// returned, after any trimming to the asked span.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct ChatHit {
|
||||
pub room_id: String,
|
||||
pub room_alias: Option<String>,
|
||||
pub first_event_id: String,
|
||||
pub last_event_id: String,
|
||||
pub ts_start_ms: u64,
|
||||
pub ts_end_ms: u64,
|
||||
/// Cosine similarity for one text, the reciprocal-rank sum for several,
|
||||
/// none in a listing.
|
||||
pub score: Option<f32>,
|
||||
pub messages: Vec<ChatMessage>,
|
||||
}
|
||||
|
||||
/// The answer to a chat search: oldest first in a listing, nearest first with
|
||||
/// texts.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct ChatHits {
|
||||
pub indexed_under: SourceModel,
|
||||
/// A listing had more chunks than `top_k`.
|
||||
pub truncated: bool,
|
||||
pub hits: Vec<ChatHit>,
|
||||
}
|
||||
|
||||
/// The most messages a chat window reaches to either side of its anchor.
|
||||
pub const CHAT_WINDOW_MAX: u32 = 50;
|
||||
|
||||
/// The longest event id a chat window takes.
|
||||
pub const EVENT_ID_MAX: usize = 200;
|
||||
|
||||
/// A chat window: the message `event_id` names and up to `before` messages
|
||||
/// ahead of it and `after` behind it, in its room.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct ChatWindowQuery {
|
||||
pub event_id: Short,
|
||||
pub before: u32,
|
||||
pub after: u32,
|
||||
}
|
||||
|
||||
impl ChatWindowQuery {
|
||||
/// The door. A refusal names the field.
|
||||
pub fn check(&self) -> Result<(), MindRefusal> {
|
||||
let refused = |field: &str, value: String, message: &str| MindRefusal::SelectionInvalid { field: field.into(), value: Some(value), message: message.into() };
|
||||
if !self.event_id.0.starts_with('$') || self.event_id.0.len() > EVENT_ID_MAX {
|
||||
return Err(refused("event_id", self.event_id.0.clone(), "an event id begins `$` and is at most 200 bytes"));
|
||||
}
|
||||
for (field, count) in [("before", self.before), ("after", self.after)] {
|
||||
if count > CHAT_WINDOW_MAX {
|
||||
return Err(refused(field, count.to_string(), "a chat window reaches 0 to 50 messages to each side"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// The answer to a chat window: `messages` chronological with the anchor in
|
||||
/// them once, all of `room_id`, each carrying the citations a hit's do.
|
||||
/// `more_before` and `more_after` say messages exist beyond the cut.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct ChatWindow {
|
||||
pub room_id: String,
|
||||
pub room_alias: Option<String>,
|
||||
pub anchor_event_id: String,
|
||||
pub messages: Vec<ChatMessage>,
|
||||
pub more_before: bool,
|
||||
pub more_after: bool,
|
||||
}
|
||||
|
||||
impl HuginnMindRequest {
|
||||
/// The envelope's `operation`.
|
||||
pub fn operation(&self) -> &'static str {
|
||||
|
|
@ -256,6 +422,8 @@ impl HuginnMindRequest {
|
|||
Self::PersonaGet { .. } => "persona_get",
|
||||
Self::PersonaPut { .. } => "persona_put",
|
||||
Self::SearchSources { .. } => "search_sources",
|
||||
Self::SearchChat { .. } => "search_chat",
|
||||
Self::ChatWindow { .. } => "chat_window",
|
||||
Self::StewardedActivity { .. } => "stewarded_activity",
|
||||
Self::RootsTouched { .. } => "roots_touched",
|
||||
}
|
||||
|
|
@ -271,6 +439,8 @@ impl HuginnMindRequest {
|
|||
| Self::PersonaGet { instance }
|
||||
| Self::PersonaPut { instance, .. }
|
||||
| Self::SearchSources { instance, .. }
|
||||
| Self::SearchChat { instance, .. }
|
||||
| Self::ChatWindow { instance, .. }
|
||||
| Self::StewardedActivity { instance, .. }
|
||||
| Self::RootsTouched { instance } => Some(instance),
|
||||
}
|
||||
|
|
@ -290,6 +460,8 @@ pub enum HuginnMindResponse {
|
|||
Persona(Option<PersonaStateView>),
|
||||
PersonaPut(PersonaPutOutcome),
|
||||
Sources(SourceHits),
|
||||
Chat(ChatHits),
|
||||
ChatWindow(ChatWindow),
|
||||
Activity(RepoActivity),
|
||||
RootsTouched(Vec<RootTouched>),
|
||||
/// The answer did not fit one send and is being held for the client to
|
||||
|
|
@ -419,8 +591,17 @@ mod tests {
|
|||
},
|
||||
HuginnMindRequest::SearchSources {
|
||||
instance: slug(INSTANCE),
|
||||
query: SourceQuery { text: Line("reconcile".into()), top_k: 5, repo: Some(Short("Huginn".into())), front: true },
|
||||
query: SourceQuery { text: Line("reconcile".into()), texts: vec![], top_k: 5, repo: Some(Short("Huginn".into())), front: true },
|
||||
},
|
||||
HuginnMindRequest::SearchSources {
|
||||
instance: slug(INSTANCE),
|
||||
query: SourceQuery { text: Line::default(), texts: vec![Line("reconcile".into()), Line("fuse".into())], top_k: 5, repo: None, front: false },
|
||||
},
|
||||
HuginnMindRequest::SearchChat {
|
||||
instance: slug(INSTANCE),
|
||||
query: ChatQuery { texts: vec![Line("release".into())], rooms: vec![Short("#swarm:gamecult.org".into())], after_ms: Some(1), before_ms: None, top_k: 5 },
|
||||
},
|
||||
HuginnMindRequest::ChatWindow { instance: slug(INSTANCE), query: ChatWindowQuery { event_id: Short("$a".into()), before: 5, after: 0 } },
|
||||
HuginnMindRequest::RootsTouched { instance: slug(INSTANCE) },
|
||||
]
|
||||
}
|
||||
|
|
@ -471,6 +652,28 @@ mod tests {
|
|||
text: "x".into(),
|
||||
}],
|
||||
}),
|
||||
HuginnMindResponse::Chat(ChatHits {
|
||||
indexed_under: SourceModel { name: "m".into(), digest: "d".into() },
|
||||
truncated: false,
|
||||
hits: vec![ChatHit {
|
||||
room_id: "!r:gamecult.org".into(),
|
||||
room_alias: None,
|
||||
first_event_id: "$a".into(),
|
||||
last_event_id: "$a".into(),
|
||||
ts_start_ms: 1,
|
||||
ts_end_ms: 1,
|
||||
score: None,
|
||||
messages: vec![ChatMessage { event_id: "$a".into(), sender: "@x:gamecult.org".into(), ts_ms: 1, body: "hi".into() }],
|
||||
}],
|
||||
}),
|
||||
HuginnMindResponse::ChatWindow(ChatWindow {
|
||||
room_id: "!r:gamecult.org".into(),
|
||||
room_alias: Some("#swarm:gamecult.org".into()),
|
||||
anchor_event_id: "$a".into(),
|
||||
messages: vec![ChatMessage { event_id: "$a".into(), sender: "@x:gamecult.org".into(), ts_ms: 1, body: "hi".into() }],
|
||||
more_before: false,
|
||||
more_after: true,
|
||||
}),
|
||||
HuginnMindResponse::RootsTouched(seeded().roots_touched().unwrap()),
|
||||
HuginnMindResponse::Deferred(DeferredAnswer {
|
||||
manifest: cultnet_rs::pack_content("huginn.mind_response", "package", "", "", "", b"body", 4)
|
||||
|
|
@ -486,7 +689,7 @@ mod tests {
|
|||
#[test]
|
||||
fn the_wire_vocabulary_is_the_minds_methods_and_status_is_derived_from_the_response() {
|
||||
let names: Vec<&str> = requests().iter().map(|request| request.operation()).collect();
|
||||
assert_eq!(names, ["whoami", "admit", "view", "query", "persona_get", "persona_put", "search_sources", "roots_touched"]);
|
||||
assert_eq!(names, ["whoami", "admit", "view", "query", "persona_get", "persona_put", "search_sources", "search_sources", "search_chat", "chat_window", "roots_touched"]);
|
||||
for request in requests() {
|
||||
let declared = request.instance().is_none();
|
||||
assert_eq!(declared, matches!(request, HuginnMindRequest::Whoami), "{request:?}");
|
||||
|
|
@ -518,6 +721,7 @@ mod tests {
|
|||
fn a_source_query_without_text_or_within_one_to_fifty_hits_is_refused_by_field() {
|
||||
let query = |text: &str, top_k: u32, repo: Option<&str>| SourceQuery {
|
||||
text: Line(text.into()),
|
||||
texts: vec![],
|
||||
top_k,
|
||||
repo: repo.map(|repo| Short(repo.into())),
|
||||
front: false,
|
||||
|
|
@ -529,6 +733,17 @@ mod tests {
|
|||
assert_eq!(query("x", 1, None).check(), Ok(()));
|
||||
assert_eq!(query("x", SOURCE_TOP_K_MAX, Some("Huginn")).check(), Ok(()));
|
||||
assert_eq!(field(query(" ", 5, None)), "text");
|
||||
let many = |texts: &[&str]| SourceQuery { text: Line::default(), texts: texts.iter().map(|text| Line((*text).into())).collect(), ..query("", 5, None) };
|
||||
assert_eq!(many(&["a", "b"]).check(), Ok(()));
|
||||
assert_eq!(many(&["a"; SOURCE_TEXTS_MAX]).check(), Ok(()));
|
||||
assert_eq!(field(query("", 5, None)), "texts", "neither text nor texts");
|
||||
assert_eq!(field(many(&[])), "texts");
|
||||
assert_eq!(field(SourceQuery { text: Line("x".into()), ..many(&["a"]) }), "texts", "both");
|
||||
assert_eq!(field(many(&["a", " "])), "texts", "a blank entry");
|
||||
match many(&["a"; SOURCE_TEXTS_MAX + 1]).check() {
|
||||
Err(MindRefusal::SelectionInvalid { field, value, .. }) => assert_eq!((field.as_str(), value.as_deref()), ("texts", Some("9"))),
|
||||
other => panic!("expected a refusal, got {other:?}"),
|
||||
}
|
||||
assert_eq!(field(query("x", 0, None)), "top_k");
|
||||
assert_eq!(field(query("x", SOURCE_TOP_K_MAX + 1, None)), "top_k");
|
||||
assert_eq!(field(query("x", 5, Some(" "))), "repo");
|
||||
|
|
@ -549,6 +764,23 @@ mod tests {
|
|||
assert!(!rmp_serde::from_slice::<SourceQuery>(&bytes).unwrap().front);
|
||||
}
|
||||
|
||||
/// An old client's search has `text` and no `texts`: it decodes as the one
|
||||
/// text it sent, and passes the door.
|
||||
#[test]
|
||||
fn a_source_query_encoded_without_texts_decodes_as_its_one_text() {
|
||||
#[derive(Serialize)]
|
||||
struct OldQuery {
|
||||
text: Line,
|
||||
top_k: u32,
|
||||
repo: Option<Short>,
|
||||
front: bool,
|
||||
}
|
||||
let old = OldQuery { text: Line("x".into()), top_k: 3, repo: None, front: false };
|
||||
let bytes = rmp_serde::to_vec_named(&old).unwrap();
|
||||
let query = rmp_serde::from_slice::<SourceQuery>(&bytes).unwrap();
|
||||
assert_eq!((query.text.0.as_str(), query.texts.len(), query.check()), ("x", 0, Ok(())));
|
||||
}
|
||||
|
||||
/// A new client reads an old daemon's page: the five fields it had, no
|
||||
/// `scores`. This is what keeps rollback by flip safe.
|
||||
#[test]
|
||||
|
|
@ -567,6 +799,54 @@ mod tests {
|
|||
assert_eq!((page.matched, page.as_of, page.scores), (0, 7, None));
|
||||
}
|
||||
|
||||
/// A chat search is refused by field at the door, and the published
|
||||
/// response marks every citation required.
|
||||
#[test]
|
||||
fn a_chat_query_is_refused_by_field_and_a_hit_cites_room_ids_and_times() {
|
||||
let ok = ChatQuery { texts: vec![], rooms: vec![Short("!a:b".into()), Short("#c:b".into())], after_ms: Some(1), before_ms: Some(1), top_k: 50 };
|
||||
assert!(ok.check().is_ok());
|
||||
let field = |query: ChatQuery| match query.check() {
|
||||
Err(MindRefusal::SelectionInvalid { field, .. }) => field,
|
||||
other => panic!("{other:?}"),
|
||||
};
|
||||
assert_eq!(field(ChatQuery { rooms: vec![Short("swarm".into())], ..ok.clone() }), "rooms");
|
||||
assert_eq!(field(ChatQuery { rooms: vec![Short("#".into())], ..ok.clone() }), "rooms");
|
||||
assert_eq!(field(ChatQuery { before_ms: Some(0), ..ok.clone() }), "before_ms");
|
||||
assert_eq!(field(ChatQuery { top_k: 0, ..ok.clone() }), "top_k");
|
||||
assert_eq!(field(ChatQuery { top_k: 51, ..ok.clone() }), "top_k");
|
||||
assert_eq!(field(ChatQuery { texts: vec![Line(" ".into())], ..ok.clone() }), "texts");
|
||||
assert_eq!(field(ChatQuery { texts: vec![Line("x".into()); 9], ..ok.clone() }), "texts");
|
||||
assert_eq!(field(ChatQuery { rooms: vec![Short("!a:b".into()); 17], ..ok }), "rooms");
|
||||
let response = serde_json::to_value(schemars::schema_for!(HuginnMindResponse)).unwrap();
|
||||
let required: Vec<&str> = response["$defs"]["ChatHit"]["required"].as_array().unwrap().iter().filter_map(|v| v.as_str()).collect();
|
||||
for field in ["room_id", "first_event_id", "last_event_id", "ts_start_ms", "ts_end_ms", "messages"] {
|
||||
assert!(required.contains(&field), "{field} must be required");
|
||||
}
|
||||
let message: Vec<&str> = response["$defs"]["ChatMessage"]["required"].as_array().unwrap().iter().filter_map(|v| v.as_str()).collect();
|
||||
assert_eq!(message, ["event_id", "sender", "ts_ms", "body"]);
|
||||
}
|
||||
|
||||
/// A chat window is refused by field at the door.
|
||||
#[test]
|
||||
fn a_chat_window_query_is_refused_by_field() {
|
||||
let ok = ChatWindowQuery { event_id: Short("$a".into()), before: 50, after: 0 };
|
||||
assert!(ok.check().is_ok());
|
||||
let field = |query: ChatWindowQuery| match query.check() {
|
||||
Err(MindRefusal::SelectionInvalid { field, .. }) => field,
|
||||
other => panic!("{other:?}"),
|
||||
};
|
||||
assert_eq!(field(ChatWindowQuery { before: 51, ..ok.clone() }), "before");
|
||||
assert_eq!(field(ChatWindowQuery { after: 51, ..ok.clone() }), "after");
|
||||
assert_eq!(field(ChatWindowQuery { event_id: Short("a".into()), ..ok.clone() }), "event_id");
|
||||
assert_eq!(field(ChatWindowQuery { event_id: Short(format!("${}", "a".repeat(200))), ..ok.clone() }), "event_id");
|
||||
assert!(ChatWindowQuery { event_id: Short(format!("${}", "a".repeat(199))), ..ok }.check().is_ok());
|
||||
let response = serde_json::to_value(schemars::schema_for!(HuginnMindResponse)).unwrap();
|
||||
let required: Vec<&str> = response["$defs"]["ChatWindow"]["required"].as_array().unwrap().iter().filter_map(|v| v.as_str()).collect();
|
||||
for field in ["room_id", "anchor_event_id", "messages", "more_before", "more_after"] {
|
||||
assert!(required.contains(&field), "{field} must be required");
|
||||
}
|
||||
}
|
||||
|
||||
/// The two files the catalog publishes are the derivation of the two
|
||||
/// types, byte for byte.
|
||||
#[test]
|
||||
|
|
|
|||
631
crates/huginn-mind/tests/desk_feed_fixtures.rs
Normal file
631
crates/huginn-mind/tests/desk_feed_fixtures.rs
Normal file
|
|
@ -0,0 +1,631 @@
|
|||
//! The desk feed's golden fixtures: the bytes Bifrost writes to a Mind's desk socket for a list or
|
||||
//! a read, and the bytes Huginn writes back, built here with Huginn's own encoders so the files
|
||||
//! are what the door speaks. `docs/desk-feed-door.md` is the contract they pin; the README in the
|
||||
//! fixtures directory lists them.
|
||||
//!
|
||||
//! The entry-bearing fixtures carry the reader-count shapes (`raised_at`, `decider_count`,
|
||||
//! `answerer_count`, a nullable `answerer`), a decider's view and a non-decider's. CultLib's own bytes
|
||||
//! for those shapes are vendored at one commit in `fixtures/desk-feed/cultlib-d3025d3c/`; Huginn's
|
||||
//! real types decode and re-encode every one of them byte for byte (checked below).
|
||||
//!
|
||||
//! `cargo test -p huginn-mind --test desk_feed_fixtures` compares every file with what the
|
||||
//! encoders produce now. `HUGINN_WRITE_DESK_FIXTURES=1` rewrites them instead.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use cultnet_rs::{CultNetMessage, CultNetWireContract, encode_cultnet_message_to_vec, encode_frame};
|
||||
use huginn_mind::MindRefusal;
|
||||
use huginn_mind::desk_wire::{
|
||||
fork_too_large_read, AnswerSchema, CardSchema, CardStatus, DeskRequest, DeskResponse, EntrySchema, ForkCard, ForkCursor, ForkDeskAnswer, ForkEntry, ForkLimit,
|
||||
ForkList, ForkOption, ForkRecord, ForkRefusal, ForkRefusalCode, ForkRef, ListSchema, ListStatus, RecordSchema,
|
||||
};
|
||||
use huginn_mind::envelope::{OperationFailure, encode_desk_failure, encode_desk_request, encode_desk_response};
|
||||
use huginn_mind::eureka_pipeline::{DeskAuthority, Short, Slug};
|
||||
|
||||
const RUNTIME: &str = "huginn-eureka";
|
||||
/// Account ids as Bifrost sends them: `UserAccount.Id` rendered lowercase D-format
|
||||
/// (`DeskEndpoints.cs` `accountId.Value.ToString("D")`; `StartupConfigurationValidator` refuses any
|
||||
/// other decider). Synthetic values.
|
||||
const READER: &str = "5f0c2b1e-8a4d-4c3b-9e7f-1a2b3c4d5e6f";
|
||||
const OTHER_DECIDER: &str = "0b9d7e3a-6c21-4f58-a0d4-9e8f7a6b5c4d";
|
||||
/// A member who answered but is not a decider: their answer is stored `Refused(not_a_decider)`.
|
||||
const OUTSIDER: &str = "c3a1f2e4-7b6d-4e8f-9a0b-1c2d3e4f5a6b";
|
||||
/// A member the call's deciders do not name, who reads as a non-decider.
|
||||
const NON_DECIDER: &str = "9e8d7c6b-5a49-4382-b1a0-f9e8d7c6b5a4";
|
||||
/// A question Eureka's mind holds (asked 2026-11-11). The answers, notes and hash in `record()` are
|
||||
/// synthetic: the hash is CultLib's sample digest, not this question's.
|
||||
const FORK: &str = "eureka-substrate:question:desk-listing-authority";
|
||||
|
||||
fn s(text: &str) -> Short {
|
||||
Short(text.into())
|
||||
}
|
||||
|
||||
fn forks(status: Option<ListStatus>, cursor: Option<u64>, limit: u8) -> DeskRequest {
|
||||
DeskRequest::Forks {
|
||||
mind: Slug("eureka".into()),
|
||||
subject: s(READER),
|
||||
deciders: vec![s(READER), s(OTHER_DECIDER)],
|
||||
status,
|
||||
cursor: cursor.and_then(ForkCursor::new),
|
||||
limit: ForkLimit::new(limit).unwrap(),
|
||||
}
|
||||
}
|
||||
|
||||
fn read() -> DeskRequest {
|
||||
DeskRequest::Fork { mind: Slug("eureka".into()), subject: s(READER), deciders: vec![s(READER), s(OTHER_DECIDER)], fork_id: FORK.into() }
|
||||
}
|
||||
|
||||
/// The same reads from a subject the call's deciders do not name: a non-decider.
|
||||
fn forks_as_non_decider() -> DeskRequest {
|
||||
DeskRequest::Forks {
|
||||
mind: Slug("eureka".into()),
|
||||
subject: s(NON_DECIDER),
|
||||
deciders: vec![s(READER), s(OTHER_DECIDER)],
|
||||
status: Some(ListStatus::Open),
|
||||
cursor: None,
|
||||
limit: ForkLimit::new(50).unwrap(),
|
||||
}
|
||||
}
|
||||
|
||||
fn read_as_non_decider() -> DeskRequest {
|
||||
DeskRequest::Fork { mind: Slug("eureka".into()), subject: s(NON_DECIDER), deciders: vec![s(READER), s(OTHER_DECIDER)], fork_id: FORK.into() }
|
||||
}
|
||||
|
||||
fn list(open_count: u64, recorded_count: u64) -> DeskResponse {
|
||||
DeskResponse::Forks {
|
||||
list: ForkList {
|
||||
schema_version: ListSchema::V0,
|
||||
mind: "eureka".into(),
|
||||
stamp: "s10190".into(),
|
||||
open_count,
|
||||
recorded_count,
|
||||
forks: Vec::new(),
|
||||
next_cursor: None,
|
||||
},
|
||||
as_of: 10190,
|
||||
}
|
||||
}
|
||||
|
||||
fn refusal(code: ForkRefusalCode, error: &str) -> DeskResponse {
|
||||
DeskResponse::NotStored(ForkRefusal { schema_version: huginn_mind::desk_wire::RefusalSchema::V0, code, error: error.into() })
|
||||
}
|
||||
|
||||
fn answer(id: &str, outcome: &str, of: Option<&str>, answerer: Option<&str>, note: &str, received_at: &str) -> ForkDeskAnswer {
|
||||
ForkDeskAnswer {
|
||||
schema_version: AnswerSchema::V0,
|
||||
answer_id: id.into(),
|
||||
fork_id: FORK.into(),
|
||||
outcome: outcome.into(),
|
||||
of: of.map(str::to_string),
|
||||
retracts: None,
|
||||
code: None,
|
||||
options_hash: "7d95fc87d532420611fe415c1def27a3447617bb68235f46e3af130aea361c48".into(),
|
||||
choice: Some("asking-mind".into()),
|
||||
note: Some(note.into()),
|
||||
answered_at: received_at.into(),
|
||||
received_at: received_at.into(),
|
||||
claim_in_future: false,
|
||||
answerer: answerer.map(str::to_string),
|
||||
authority: DeskAuthority::Member,
|
||||
request_id: format!("00000000-0000-4000-8000-00000000000{}", &id[1..]),
|
||||
}
|
||||
}
|
||||
|
||||
/// A fork's record with a counted answer, a decider's revision, a refusal and a retraction: four
|
||||
/// answers by three accounts. A decider's view names every answerer; a non-decider's names none and
|
||||
/// keeps the count.
|
||||
fn record(decider: bool) -> ForkRecord {
|
||||
let who = |id: &'static str| decider.then_some(id);
|
||||
ForkRecord {
|
||||
schema_version: RecordSchema::V0,
|
||||
fork_id: FORK.into(),
|
||||
options_hash: HASH.into(),
|
||||
counted_answer_id: Some("n1".into()),
|
||||
answerer_count: 3,
|
||||
answers: vec![
|
||||
answer("n1", "Counted", None, who(READER), "The asker lists her own forks.", "2026-11-11T17:00:00Z"),
|
||||
answer("n2", "Revision", Some("n1"), who(OTHER_DECIDER), "Agreed.", "2026-11-11T17:05:00Z"),
|
||||
ForkDeskAnswer { code: Some(huginn_mind::desk_wire::ForkAnswerCode::NotADecider), ..answer("n3", "Refused", None, who(OUTSIDER), "Me too.", "2026-11-11T17:06:00Z") },
|
||||
// A second answer by an account already counted: four answers, three answerers.
|
||||
ForkDeskAnswer {
|
||||
retracts: Some("n2".into()),
|
||||
choice: None,
|
||||
note: None,
|
||||
..answer("n4", "Retraction", None, who(OTHER_DECIDER), "", "2026-11-11T17:07:00Z")
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
const HASH: &str = "7d95fc87d532420611fe415c1def27a3447617bb68235f46e3af130aea361c48";
|
||||
|
||||
/// The fork's card. A decider's view lists the call's ids; a non-decider's lists none and keeps the count.
|
||||
fn card(decider: bool, raised_at: Option<&str>) -> ForkCard {
|
||||
ForkCard {
|
||||
schema_version: CardSchema::V0,
|
||||
fork_id: FORK.into(),
|
||||
title: "Who may list a fork?".into(),
|
||||
context: "A fork is read by members only once it is listed.".into(),
|
||||
options: vec![ForkOption { id: "asking-mind".into(), text: "The asking Mind lists her own forks.".into() }, ForkOption { id: "operator".into(), text: "An operator lists them.".into() }],
|
||||
recommendation: "asking-mind".into(),
|
||||
depends: Vec::new(),
|
||||
leans: None,
|
||||
flag: None,
|
||||
raised_in: Some(ForkRef { kind: "Idea".into(), id: "eureka-substrate:idea:desk-door-map".into() }),
|
||||
raised_at: raised_at.map(str::to_string),
|
||||
deciders: if decider { vec![READER.into(), OTHER_DECIDER.into()] } else { Vec::new() },
|
||||
decider_count: 2,
|
||||
status: CardStatus::Open,
|
||||
ruling: None,
|
||||
ruled_from: Vec::new(),
|
||||
order: 41,
|
||||
options_hash: HASH.into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn entry(decider: bool, raised_at: Option<&str>) -> ForkEntry {
|
||||
ForkEntry { schema_version: EntrySchema::V0, card: card(decider, raised_at), record: record(decider) }
|
||||
}
|
||||
|
||||
/// One open fork on the page; the counts are the page's visible set.
|
||||
fn list_of(entry: ForkEntry) -> DeskResponse {
|
||||
DeskResponse::Forks {
|
||||
list: ForkList { schema_version: ListSchema::V0, mind: "eureka".into(), stamp: "s10190".into(), open_count: 1, recorded_count: 0, forks: vec![entry], next_cursor: None },
|
||||
as_of: 10190,
|
||||
}
|
||||
}
|
||||
|
||||
/// The named MessagePack payload a CultNet operation message carries (base64 in the envelope).
|
||||
fn payload(message: &CultNetMessage) -> Vec<u8> {
|
||||
use base64::Engine;
|
||||
let (CultNetMessage::OperationRequest { payload, .. } | CultNetMessage::OperationResponse { payload, .. }) = message else {
|
||||
panic!("an operation message")
|
||||
};
|
||||
base64::engine::general_purpose::STANDARD.decode(payload).unwrap()
|
||||
}
|
||||
|
||||
fn frame(message: &CultNetMessage) -> Vec<u8> {
|
||||
encode_frame(&encode_cultnet_message_to_vec(message, CultNetWireContract::CultNetSchemaV0).unwrap()).unwrap()
|
||||
}
|
||||
|
||||
fn hex(bytes: &[u8]) -> String {
|
||||
let mut text: String = bytes.iter().map(|byte| format!("{byte:02x}")).collect();
|
||||
text.push('\n');
|
||||
text
|
||||
}
|
||||
|
||||
/// Every fixture: its file stem and its bytes. A message gives `.frame.bin`, `.frame.hex` and,
|
||||
/// for an operation message, `.payload.msgpack` and `.payload.hex`; a bare payload gives the last two.
|
||||
fn fixtures() -> Vec<(String, Vec<u8>)> {
|
||||
let mut files = Vec::new();
|
||||
let mut message = |name: &str, message: CultNetMessage| {
|
||||
let framed = frame(&message);
|
||||
files.push((format!("{name}.frame.bin"), framed.clone()));
|
||||
files.push((format!("{name}.frame.hex"), hex(&framed).into_bytes()));
|
||||
if matches!(message, CultNetMessage::OperationRequest { .. } | CultNetMessage::OperationResponse { .. }) {
|
||||
let inner = payload(&message);
|
||||
files.push((format!("{name}.payload.msgpack"), inner.clone()));
|
||||
files.push((format!("{name}.payload.hex"), hex(&inner).into_bytes()));
|
||||
}
|
||||
};
|
||||
message("list-open.request", encode_desk_request("m-1", &forks(Some(ListStatus::Open), None, 50), None).unwrap());
|
||||
message("list-open-cursor.request", encode_desk_request("m-2", &forks(Some(ListStatus::Open), Some(42), 50), None).unwrap());
|
||||
message("list-counts.request", encode_desk_request("m-3", &forks(Some(ListStatus::Open), None, 0), None).unwrap());
|
||||
message("read.request", encode_desk_request("m-4", &read(), None).unwrap());
|
||||
message("list-non-decider.request", encode_desk_request("m-5", &forks_as_non_decider(), None).unwrap());
|
||||
message("read-non-decider.request", encode_desk_request("m-6", &read_as_non_decider(), None).unwrap());
|
||||
message("list-open.response", encode_desk_response("m-1", "forks", &list_of(entry(true, Some("2026-11-11T16:40:12Z"))), RUNTIME).unwrap());
|
||||
message("list-open-non-decider.response", encode_desk_response("m-5", "forks", &list_of(entry(false, Some("2026-11-11T16:40:12Z"))), RUNTIME).unwrap());
|
||||
// raised_at nil: the question's first admission is unknown.
|
||||
message("read.response", encode_desk_response("m-4", "fork", &DeskResponse::Fork { entry: entry(true, None), as_of: 10190 }, RUNTIME).unwrap());
|
||||
message("read-non-decider.response", encode_desk_response("m-6", "fork", &DeskResponse::Fork { entry: entry(false, None), as_of: 10190 }, RUNTIME).unwrap());
|
||||
message("list-empty.response", encode_desk_response("m-1", "forks", &list(0, 0), RUNTIME).unwrap());
|
||||
message("list-counts.response", encode_desk_response("m-3", "forks", &list(3, 5), RUNTIME).unwrap());
|
||||
message(
|
||||
"read-unknown.response",
|
||||
encode_desk_response("m-4", "fork", &refusal(ForkRefusalCode::ForkUnknown, "fork_unknown: no such fork"), RUNTIME).unwrap(),
|
||||
);
|
||||
message(
|
||||
"list-too-large.response",
|
||||
encode_desk_response(
|
||||
"m-1",
|
||||
"forks",
|
||||
&refusal(ForkRefusalCode::ForkTooLarge, &format!("fork_too_large: fork {FORK} at order 41 is past the page budget by itself; continue with cursor c41")),
|
||||
RUNTIME,
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
message("read-too-large.response", encode_desk_response("m-4", "fork", &DeskResponse::NotStored(fork_too_large_read(FORK)), RUNTIME).unwrap());
|
||||
message(
|
||||
"list-response-too-large.response",
|
||||
encode_desk_response("m-1", "forks", &DeskResponse::Refused(MindRefusal::ResponseTooLarge { bytes: 70000, limit: 65536 }), RUNTIME).unwrap(),
|
||||
);
|
||||
message("refused-unit.response", encode_desk_response("m-1", "forks", &DeskResponse::Refused(MindRefusal::MissingIdentity), RUNTIME).unwrap());
|
||||
message(
|
||||
"unavailable.response",
|
||||
encode_desk_response("m-1", "forks", &DeskResponse::Refused(MindRefusal::Unavailable { detail: "the mind store is not readable".into() }), RUNTIME).unwrap(),
|
||||
);
|
||||
message(
|
||||
"failure.response",
|
||||
encode_desk_failure("m-1", "forks", &OperationFailure { code: "payload-not-a-request".into(), message: "field Forks.limit: out-of-range".into() }, RUNTIME),
|
||||
);
|
||||
message("error.response", CultNetMessage::Error { error: "huginn.desk refuses a message_id over 256 bytes".into(), code: None, details: None });
|
||||
for (name, decider) in [("record-whole", true), ("record-withheld", false)] {
|
||||
let bytes = rmp_serde::to_vec_named(&record(decider)).unwrap();
|
||||
files.push((format!("{name}.payload.msgpack"), bytes.clone()));
|
||||
files.push((format!("{name}.payload.hex"), hex(&bytes).into_bytes()));
|
||||
}
|
||||
files
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn desk_feed_fixtures_are_what_the_encoders_write() {
|
||||
let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/desk-feed");
|
||||
let write = std::env::var_os("HUGINN_WRITE_DESK_FIXTURES").is_some();
|
||||
if write {
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
}
|
||||
for (name, bytes) in fixtures() {
|
||||
let path = dir.join(&name);
|
||||
if write {
|
||||
std::fs::write(&path, &bytes).unwrap();
|
||||
} else {
|
||||
let held = std::fs::read(&path).unwrap_or_else(|error| panic!("{}: {error}", path.display()));
|
||||
assert!(held == bytes, "{name} differs from what the encoders write now");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The one grammar for `subject`, every `deciders` entry and every `answerer`: a lowercase
|
||||
/// D-format UUID (8-4-4-4-12 lowercase hex), the grammar Huginn already names for `RequestId`
|
||||
/// (`eureka-pipeline` `uuid_text`). A Matrix id, an email or a handle never fits it.
|
||||
fn account_id(text: &str) -> bool {
|
||||
text.len() == 36
|
||||
&& text.bytes().enumerate().all(|(at, byte)| match at {
|
||||
8 | 13 | 18 | 23 => byte == b'-',
|
||||
_ => matches!(byte, b'0'..=b'9' | b'a'..=b'f'),
|
||||
})
|
||||
}
|
||||
|
||||
/// Every account id the fixtures carry is a Bifrost account id, never a Matrix id, email or
|
||||
/// handle. The door does not refuse other shapes yet (`Short`, any 200 bytes); that refusal is owed
|
||||
/// by `cut-desk-feed-listing.r6`, and this pins the grammar it must enforce.
|
||||
#[test]
|
||||
fn every_account_id_in_the_fixtures_is_a_lowercase_d_format_uuid() {
|
||||
for request in [forks(Some(ListStatus::Open), None, 50), read()] {
|
||||
let subject = match &request {
|
||||
DeskRequest::Forks { subject, .. } | DeskRequest::Fork { subject, .. } | DeskRequest::Answer { subject, .. } => subject,
|
||||
};
|
||||
assert!(account_id(&subject.0), "subject");
|
||||
assert!(request.deciders().iter().all(|decider| account_id(&decider.0)), "deciders");
|
||||
}
|
||||
assert!(record(true).answers.iter().all(|answer| account_id(answer.answerer.as_deref().unwrap())), "answerer");
|
||||
for shape in [
|
||||
"@heimdall:gamecult.org",
|
||||
"meta@gamecult.org",
|
||||
"heimdall",
|
||||
"7",
|
||||
"5F0C2B1E-8A4D-4C3B-9E7F-1A2B3C4D5E6F",
|
||||
"{5f0c2b1e-8a4d-4c3b-9e7f-1a2b3c4d5e6f}",
|
||||
"",
|
||||
// 36 hex digits and no dash: the length alone does not make an id.
|
||||
"5f0c2b1e08a4d04c3b09e7f01a2b3c4d5e6f",
|
||||
// 37 characters: a valid id with one more digit.
|
||||
"5f0c2b1e-8a4d-4c3b-9e7f-1a2b3c4d5e6f0",
|
||||
// A dash moved off its position.
|
||||
"5f0c2b1e8-a4d-4c3b-9e7f-1a2b3c4d5e6f",
|
||||
] {
|
||||
assert!(!account_id(shape), "{shape} is not an account id");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// ---- CultLib's bytes: the reader-count shapes, pinned at one commit. ----
|
||||
|
||||
/// The CultLib commit the vendored vectors come from (#118 on #117 on #115). Re-pin: see the README beside them.
|
||||
const CULTLIB_SHA: &str = "d3025d3c19674803ad32eb2335461b8cbc22bd28";
|
||||
|
||||
/// Each vendored vector and the SHA-256 of its bytes.
|
||||
const VENDORED: &[(&str, &str)] = &[
|
||||
("card-operator-fork", "56bfdb65ded60b562001d09d0b8da6e092900f404bf5f68f7084dff29bcc34f5"),
|
||||
("card-raised-at-nil", "b81bb825790ffe03fabe63c3f8a61c36bd50ca268aa6dff67b5bcfad3957e58c"),
|
||||
("card-withheld-deciders", "d13b9b77461cfdf78f943ef56c6aeab852ab6b6f0b355e26c5b361f99c79c7f7"),
|
||||
("answer-counted", "55d69ae3cc7ebbeeae7d501e8641640e115c3fb5aac399ea760588bca343b9e6"),
|
||||
("answer-revision", "19d7289395ae68cbaf6de1c0a1ef6c843211b2caf9462e1d2268cd0883386315"),
|
||||
("answer-retraction", "89156f41ec3ea54f5d47bd3318913ff013785f4727916875be97785935cb7735"),
|
||||
("answer-withheld-answerer", "b66b4f921cdd374be04af3de34c59e254a5e542027ae8c57f196a3ea3406e427"),
|
||||
("answer-withheld-answerer-revision", "93f7c875f963a7cfeab2a5980b6d5ce81e6753fc9f3f8bf0d7353f3f388144b7"),
|
||||
("answer-withheld-answerer-retraction", "7d0cc555bf9b6f7c345b0da7c504bcd1ed7b46040ff51d1e57c84b2f56655213"),
|
||||
("answer-refused-stale", "3183bc9c483f4a625332cda11ef22a9c8560868a7728b77daa63a1960a3cd3d6"),
|
||||
("entry-open-counted", "b51f499e8dd0465cf5df9d68d8fccbe94dd1339fd3157969693dd0d4ecb2549b"),
|
||||
("entry-withheld-open-counted", "62ccad7a9f41262d6e35d0247315d0bb75d1313b3c9dabe8a99d1aa1b844607f"),
|
||||
("entry-closed-with-ruling-and-a-refusal", "f7859c7d04d338d6b1c634308b08ae58ae77b0ff459767859a5e40f0af2f16e5"),
|
||||
("entry-withheld-closed-with-ruling-and-a-refusal", "daaa819bfa2612616826ef3217dfcf8d8d8cb71d86547d7106426c779134f827"),
|
||||
];
|
||||
|
||||
fn vendored(name: &str) -> Vec<u8> {
|
||||
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/desk-feed/cultlib-d3025d3c").join(format!("{name}.msgpack"));
|
||||
std::fs::read(&path).unwrap_or_else(|error| panic!("{}: {error}", path.display()))
|
||||
}
|
||||
|
||||
/// A decoded MessagePack value that keeps map order. A small decoder of its own, so the key counts,
|
||||
/// headers and positions below are read off the bytes and not off `rmp_serde`'s idea of the types.
|
||||
#[derive(Debug, PartialEq)]
|
||||
enum Value {
|
||||
Nil,
|
||||
Bool(bool),
|
||||
Int(i64),
|
||||
Str(String),
|
||||
List(Vec<Value>),
|
||||
Map(Vec<(String, Value)>),
|
||||
}
|
||||
|
||||
impl Value {
|
||||
fn get(&self, key: &str) -> &Value {
|
||||
let Value::Map(pairs) = self else { panic!("not a map") };
|
||||
&pairs.iter().find(|(name, _)| name == key).unwrap_or_else(|| panic!("no key {key}")).1
|
||||
}
|
||||
fn keys(&self) -> Vec<&str> {
|
||||
let Value::Map(pairs) = self else { panic!("not a map") };
|
||||
pairs.iter().map(|(name, _)| name.as_str()).collect()
|
||||
}
|
||||
fn list(&self) -> &[Value] {
|
||||
let Value::List(items) = self else { panic!("not a list") };
|
||||
items
|
||||
}
|
||||
fn str(&self) -> &str {
|
||||
let Value::Str(text) = self else { panic!("not a string: {self:?}") };
|
||||
text
|
||||
}
|
||||
fn int(&self) -> i64 {
|
||||
let Value::Int(number) = self else { panic!("not an int: {self:?}") };
|
||||
*number
|
||||
}
|
||||
}
|
||||
|
||||
fn decode(bytes: &[u8]) -> Value {
|
||||
fn take<'a>(bytes: &'a [u8], at: &mut usize, n: usize) -> &'a [u8] {
|
||||
let slice = &bytes[*at..*at + n];
|
||||
*at += n;
|
||||
slice
|
||||
}
|
||||
fn be(slice: &[u8]) -> u64 {
|
||||
slice.iter().fold(0, |acc, byte| (acc << 8) | u64::from(*byte))
|
||||
}
|
||||
fn text(bytes: &[u8], at: &mut usize, n: usize) -> Value {
|
||||
Value::Str(String::from_utf8(take(bytes, at, n).to_vec()).unwrap())
|
||||
}
|
||||
fn items(bytes: &[u8], at: &mut usize, n: usize) -> Vec<Value> {
|
||||
(0..n).map(|_| value(bytes, at)).collect()
|
||||
}
|
||||
fn pairs(bytes: &[u8], at: &mut usize, n: usize) -> Value {
|
||||
Value::Map((0..n).map(|_| (value(bytes, at).str().to_string(), value(bytes, at))).collect())
|
||||
}
|
||||
fn value(bytes: &[u8], at: &mut usize) -> Value {
|
||||
let tag = take(bytes, at, 1)[0];
|
||||
match tag {
|
||||
0x00..=0x7f => Value::Int(i64::from(tag)),
|
||||
0x80..=0x8f => pairs(bytes, at, usize::from(tag & 0x0f)),
|
||||
0x90..=0x9f => Value::List(items(bytes, at, usize::from(tag & 0x0f))),
|
||||
0xa0..=0xbf => text(bytes, at, usize::from(tag & 0x1f)),
|
||||
0xc0 => Value::Nil,
|
||||
0xc2 => Value::Bool(false),
|
||||
0xc3 => Value::Bool(true),
|
||||
0xcc => Value::Int(be(take(bytes, at, 1)) as i64),
|
||||
0xcd => Value::Int(be(take(bytes, at, 2)) as i64),
|
||||
0xce => Value::Int(be(take(bytes, at, 4)) as i64),
|
||||
0xd9 => {
|
||||
let n = be(take(bytes, at, 1)) as usize;
|
||||
text(bytes, at, n)
|
||||
}
|
||||
0xda => {
|
||||
let n = be(take(bytes, at, 2)) as usize;
|
||||
text(bytes, at, n)
|
||||
}
|
||||
0xdc => {
|
||||
let n = be(take(bytes, at, 2)) as usize;
|
||||
Value::List(items(bytes, at, n))
|
||||
}
|
||||
0xde => {
|
||||
let n = be(take(bytes, at, 2)) as usize;
|
||||
pairs(bytes, at, n)
|
||||
}
|
||||
other => panic!("MessagePack tag {other:#04x} at {} is not in this decoder", *at - 1),
|
||||
}
|
||||
}
|
||||
let mut at = 0;
|
||||
let decoded = value(bytes, &mut at);
|
||||
assert_eq!(at, bytes.len(), "trailing bytes");
|
||||
decoded
|
||||
}
|
||||
|
||||
const CARD_KEYS: [&str; 18] = [
|
||||
"schemaVersion", "fork_id", "title", "context", "options", "recommendation", "depends", "leans", "flag", "raised_in", "raised_at", "deciders",
|
||||
"decider_count", "status", "ruling", "ruled_from", "order", "options_hash",
|
||||
];
|
||||
const RECORD_KEYS: [&str; 6] = ["schemaVersion", "fork_id", "options_hash", "counted_answer_id", "answerer_count", "answers"];
|
||||
|
||||
#[test]
|
||||
fn the_vendored_cultlib_vectors_are_the_pinned_bytes() {
|
||||
assert_eq!(CULTLIB_SHA.len(), 40);
|
||||
let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/desk-feed/cultlib-d3025d3c");
|
||||
let docs = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../docs/desk-feed-door.md");
|
||||
for path in [dir.join("README.md"), docs] {
|
||||
let text = std::fs::read_to_string(&path).unwrap_or_else(|error| panic!("{}: {error}", path.display()));
|
||||
assert!(text.contains(CULTLIB_SHA), "{} does not name the pinned sha {CULTLIB_SHA}", path.display());
|
||||
}
|
||||
assert!(dir.to_str().unwrap().ends_with(&format!("cultlib-{}", &CULTLIB_SHA[..8])), "the directory is named for the pin");
|
||||
for (name, sha) in VENDORED {
|
||||
let digest: String = Sha256::digest(vendored(name)).iter().map(|byte| format!("{byte:02x}")).collect();
|
||||
assert_eq!(&digest, sha, "{name} is not CultLib {CULTLIB_SHA}'s bytes");
|
||||
}
|
||||
}
|
||||
|
||||
/// The card: header `DE 00 12` (map16, 18 entries: `schemaVersion` and 17 fields), the keys in this
|
||||
/// order, `raised_at` a string or nil, `decider_count` 1..64 right after `deciders`, and `deciders` empty
|
||||
/// (withheld) or exactly `decider_count` account ids.
|
||||
fn check_card(card: &Value, withheld: bool) {
|
||||
assert_eq!(card.keys(), CARD_KEYS);
|
||||
assert!(matches!(card.get("raised_at"), Value::Nil | Value::Str(_)));
|
||||
let count = card.get("decider_count").int();
|
||||
assert!((1..=64).contains(&count));
|
||||
let ids = card.get("deciders").list();
|
||||
assert_eq!(ids.is_empty(), withheld);
|
||||
assert!(ids.is_empty() || ids.len() as i64 == count);
|
||||
assert!(ids.iter().all(|id| account_id(id.str())), "deciders are lowercase D-format UUIDs");
|
||||
assert_eq!(card.get("options_hash").str().len(), 64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_cards_are_eighteen_keys_with_raised_at_and_decider_count() {
|
||||
for name in ["card-operator-fork", "card-raised-at-nil", "card-withheld-deciders"] {
|
||||
let bytes = vendored(name);
|
||||
assert_eq!(bytes[..3], [0xde, 0x00, 0x12], "{name}: header");
|
||||
check_card(&decode(&bytes), name == "card-withheld-deciders");
|
||||
}
|
||||
assert!(matches!(decode(&vendored("card-operator-fork")).get("raised_at"), Value::Str(_)));
|
||||
assert_eq!(decode(&vendored("card-raised-at-nil")).get("raised_at"), &Value::Nil);
|
||||
assert_eq!(decode(&vendored("card-withheld-deciders")).get("decider_count").int(), 2);
|
||||
}
|
||||
|
||||
/// `answerer` is a string or nil on every answer; `Value::Nil` only when `named` is false. The bytes
|
||||
/// null it on all four variants (`Counted`, `Revision`, `Retraction`, `Refused`), by ruling
|
||||
/// `desk-reader-counts-shape` and now by vector.
|
||||
fn check_answer(answer: &Value, named: bool) {
|
||||
assert_eq!(answer.keys().len(), 16);
|
||||
assert_eq!(answer.keys()[13], "answerer");
|
||||
match answer.get("answerer") {
|
||||
Value::Str(id) => assert!(named && account_id(id), "a named answerer is a lowercase D-format UUID"),
|
||||
Value::Nil => assert!(!named),
|
||||
other => panic!("answerer is {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_answers_keep_sixteen_keys_and_answerer_is_a_string_or_nil() {
|
||||
for (name, named) in [("answer-counted", true), ("answer-revision", true), ("answer-retraction", true), ("answer-refused-stale", true), ("answer-withheld-answerer", false), ("answer-withheld-answerer-revision", false), ("answer-withheld-answerer-retraction", false)] {
|
||||
let bytes = vendored(name);
|
||||
assert_eq!(bytes[..3], [0xde, 0x00, 0x10], "{name}: header");
|
||||
check_answer(&decode(&bytes), named);
|
||||
}
|
||||
for (name, outcome) in [("answer-counted", "Counted"), ("answer-revision", "Revision"), ("answer-retraction", "Retraction"), ("answer-refused-stale", "Refused"), ("answer-withheld-answerer-revision", "Revision"), ("answer-withheld-answerer-retraction", "Retraction")] {
|
||||
assert_eq!(decode(&vendored(name)).get("outcome").str(), outcome);
|
||||
}
|
||||
}
|
||||
|
||||
/// The entry is `83` and its record `86`, `answerer_count` the fifth key. In a decider's view the count
|
||||
/// is the number of distinct answerer ids over every stored answer, refusals included; in a
|
||||
/// non-decider's view no answerer is named and the count is between 1 and the number of answers (0 for
|
||||
/// none). A non-decider entry holds none of its twin's account ids anywhere in its bytes.
|
||||
#[test]
|
||||
fn the_entries_are_83_over_a_record_of_86_with_answerer_count_before_answers() {
|
||||
for (decider, non_decider) in [("entry-open-counted", "entry-withheld-open-counted"), ("entry-closed-with-ruling-and-a-refusal", "entry-withheld-closed-with-ruling-and-a-refusal")] {
|
||||
let (decider_bytes, non_decider_bytes) = (vendored(decider), vendored(non_decider));
|
||||
let mut ids = Vec::new();
|
||||
for (bytes, named) in [(&decider_bytes, true), (&non_decider_bytes, false)] {
|
||||
assert_eq!(bytes[0], 0x83, "entry header");
|
||||
let entry = decode(bytes);
|
||||
assert_eq!(entry.keys(), ["schemaVersion", "card", "record"]);
|
||||
check_card(entry.get("card"), !named);
|
||||
let record = entry.get("record");
|
||||
assert_eq!(record.keys(), RECORD_KEYS);
|
||||
let answers = record.list_of_answers();
|
||||
let count = record.get("answerer_count").int();
|
||||
answers.iter().for_each(|answer| check_answer(answer, named));
|
||||
if named {
|
||||
let mut distinct: Vec<&str> = answers.iter().map(|answer| answer.get("answerer").str()).collect();
|
||||
distinct.sort_unstable();
|
||||
distinct.dedup();
|
||||
assert_eq!(count, distinct.len() as i64, "{decider}: distinct answerers");
|
||||
ids.extend(distinct.iter().map(|id| id.to_string()));
|
||||
ids.extend(entry.get("card").get("deciders").list().iter().map(|id| id.str().to_string()));
|
||||
} else if answers.is_empty() {
|
||||
assert_eq!(count, 0);
|
||||
} else {
|
||||
assert!((1..=answers.len() as i64).contains(&count));
|
||||
}
|
||||
}
|
||||
assert!(!ids.is_empty());
|
||||
assert!(ids.iter().all(|id| !holds(&non_decider_bytes, id)), "{non_decider} carries an account id");
|
||||
assert!(ids.iter().all(|id| holds(&decider_bytes, id)), "{decider} names every id");
|
||||
// The twins agree on everything the view does not move.
|
||||
let (a, b) = (decode(&decider_bytes), decode(&non_decider_bytes));
|
||||
assert_eq!(a.get("card").get("options_hash"), b.get("card").get("options_hash"));
|
||||
assert_eq!(a.get("card").get("decider_count"), b.get("card").get("decider_count"));
|
||||
assert_eq!(a.get("record").get("answerer_count"), b.get("record").get("answerer_count"));
|
||||
}
|
||||
// A refusal is an answer and its answerer counts: the closed fork's two answers are two accounts.
|
||||
let closed = decode(&vendored("entry-closed-with-ruling-and-a-refusal"));
|
||||
assert!(closed.get("record").list_of_answers().iter().any(|answer| answer.get("outcome").str() == "Refused"));
|
||||
assert_eq!(closed.get("record").get("answerer_count").int(), 2);
|
||||
}
|
||||
|
||||
fn holds(bytes: &[u8], text: &str) -> bool {
|
||||
bytes.windows(text.len()).any(|window| window == text.as_bytes())
|
||||
}
|
||||
|
||||
impl Value {
|
||||
fn list_of_answers(&self) -> &[Value] {
|
||||
self.get("answers").list()
|
||||
}
|
||||
}
|
||||
|
||||
/// Huginn's real types against CultLib's bytes: each of the 14 vendored vectors decodes into the type
|
||||
/// that names it and re-encodes to the same bytes.
|
||||
#[test]
|
||||
fn huginn_types_decode_and_re_encode_every_vendored_vector_byte_for_byte() {
|
||||
fn round_trip<T: serde::Serialize + serde::de::DeserializeOwned>(name: &str) {
|
||||
let bytes = vendored(name);
|
||||
let value = rmp_serde::from_slice::<T>(&bytes).unwrap_or_else(|error| panic!("{name}: {error}"));
|
||||
assert!(rmp_serde::to_vec_named(&value).unwrap() == bytes, "{name} does not re-encode byte for byte");
|
||||
}
|
||||
assert_eq!(VENDORED.len(), 14);
|
||||
for (name, _) in VENDORED {
|
||||
match name.split('-').next().unwrap() {
|
||||
"card" => round_trip::<ForkCard>(name),
|
||||
"answer" => round_trip::<ForkDeskAnswer>(name),
|
||||
"entry" => round_trip::<ForkEntry>(name),
|
||||
other => panic!("{name}: no type for {other}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative first: a non-decider's bytes hold no account id, in any fixture Huginn writes for that view.
|
||||
#[test]
|
||||
fn a_non_decider_fixture_holds_no_account_id() {
|
||||
let ids = [READER, OTHER_DECIDER, OUTSIDER, NON_DECIDER];
|
||||
let withheld = |bytes: &[u8]| ids.iter().filter(|id| holds(bytes, id)).count();
|
||||
let files = fixtures();
|
||||
let bytes_of = |name: &str| files.iter().find(|(file, _)| file == name).unwrap_or_else(|| panic!("no fixture {name}")).1.clone();
|
||||
for name in ["list-open-non-decider.response.payload.msgpack", "read-non-decider.response.payload.msgpack", "record-withheld.payload.msgpack"] {
|
||||
let bytes = bytes_of(name);
|
||||
assert_eq!(withheld(&bytes), 0, "{name} carries an account id");
|
||||
}
|
||||
// The decider's twins do name them, so the search above can fail.
|
||||
for name in ["list-open.response.payload.msgpack", "record-whole.payload.msgpack"] {
|
||||
assert!(withheld(&bytes_of(name)) >= 3, "{name}");
|
||||
}
|
||||
}
|
||||
|
||||
/// The record header is `86` with `answerer_count` fifth, the card `DE 00 12` with `raised_at` and
|
||||
/// `decider_count` where CultLib puts them, in Huginn's own bytes.
|
||||
#[test]
|
||||
fn huginns_own_card_and_record_bytes_have_the_pinned_headers_and_keys() {
|
||||
let card_bytes = rmp_serde::to_vec_named(&card(true, Some("2026-11-11T16:40:12Z"))).unwrap();
|
||||
assert_eq!(card_bytes[..3], [0xde, 0x00, 0x12]);
|
||||
assert_eq!(decode(&card_bytes).keys(), CARD_KEYS);
|
||||
check_card(&decode(&card_bytes), false);
|
||||
let withheld_bytes = rmp_serde::to_vec_named(&card(false, None)).unwrap();
|
||||
check_card(&decode(&withheld_bytes), true);
|
||||
assert_eq!(decode(&withheld_bytes).get("raised_at"), &Value::Nil);
|
||||
for decider in [true, false] {
|
||||
let record_bytes = rmp_serde::to_vec_named(&record(decider)).unwrap();
|
||||
assert_eq!(record_bytes[0], 0x86);
|
||||
let decoded = decode(&record_bytes);
|
||||
assert_eq!(decoded.keys(), RECORD_KEYS);
|
||||
assert_eq!(decoded.get("answerer_count").int(), 3);
|
||||
decoded.list_of_answers().iter().for_each(|answer| check_answer(answer, decider));
|
||||
}
|
||||
}
|
||||
3
crates/huginn-mind/tests/fixtures/desk-feed/.gitattributes
vendored
Normal file
3
crates/huginn-mind/tests/fixtures/desk-feed/.gitattributes
vendored
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
*.bin binary
|
||||
*.msgpack binary
|
||||
*.hex text eol=lf
|
||||
50
crates/huginn-mind/tests/fixtures/desk-feed/README.md
vendored
Normal file
50
crates/huginn-mind/tests/fixtures/desk-feed/README.md
vendored
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
# Desk feed golden fixtures
|
||||
|
||||
The bytes on a Mind's desk socket for a list or a read. The contract is `docs/desk-feed-door.md`.
|
||||
`crates/huginn-mind/tests/desk_feed_fixtures.rs` writes these files with Huginn's own encoders and
|
||||
checks them on every `cargo test -p huginn-mind --test desk_feed_fixtures`. Set
|
||||
`HUGINN_WRITE_DESK_FIXTURES=1` to rewrite them. Do not edit them by hand.
|
||||
|
||||
Each case has up to four files:
|
||||
|
||||
- `<case>.frame.bin` and `<case>.frame.hex`: the whole frame on the socket (u32 big-endian length,
|
||||
then the CultNet message).
|
||||
- `<case>.payload.msgpack` and `<case>.payload.hex`: the base64-decoded `payload` of an operation
|
||||
message (named MessagePack). The `record-*` cases are bare payload fragments with no frame.
|
||||
|
||||
Hex is lowercase with one trailing newline. The fork id is a question Eureka's mind holds. The
|
||||
record's answers, notes and hash are synthetic.
|
||||
|
||||
**The listing gate is not live.** On `main` every showable question goes to every reader. Do not
|
||||
put a Mind that holds private questions behind a live receiver before `cut-desk-feed-listing`
|
||||
lands (see `docs/desk-feed-door.md`).
|
||||
|
||||
| case | direction | content |
|
||||
|---|---|---|
|
||||
| `list-open.request` | Bifrost to Huginn | `Forks`, status open, no cursor, limit 50 |
|
||||
| `list-open-cursor.request` | Bifrost to Huginn | `Forks`, status open, cursor `c42`, limit 50 |
|
||||
| `list-counts.request` | Bifrost to Huginn | `Forks`, status open, limit 0 (counts and stamp only) |
|
||||
| `read.request` | Bifrost to Huginn | `Fork` for one fork id |
|
||||
| `list-non-decider.request` | Bifrost to Huginn | `Forks` as above, from a subject who is not on the call's deciders |
|
||||
| `read-non-decider.request` | Bifrost to Huginn | `Fork`, from a subject who is not on the call's deciders |
|
||||
| `list-open.response` | Huginn to Bifrost | a decider's page of one open fork: card with `raised_at` a string, `decider_count` 2 and both ids, record with `answerer_count` 3 and every `answerer` named |
|
||||
| `list-open-non-decider.response` | Huginn to Bifrost | the same page for a non-decider: `deciders` empty, `decider_count` 2, every `answerer` nil, `answerer_count` 3; no account id anywhere in its bytes |
|
||||
| `read.response` | Huginn to Bifrost | a decider's read of one fork, `raised_at` nil |
|
||||
| `read-non-decider.response` | Huginn to Bifrost | the same read for a non-decider, `raised_at` nil, no account id |
|
||||
| `list-empty.response` | Huginn to Bifrost | an empty Mind, or nothing listed to this reader: counts 0, `forks: []` |
|
||||
| `list-counts.response` | Huginn to Bifrost | the limit-0 answer: open 3, recorded 5, `forks: []` |
|
||||
| `read-unknown.response` | Huginn to Bifrost | `NotStored` `fork_unknown` (Bifrost: 404) |
|
||||
| `list-too-large.response` | Huginn to Bifrost | `NotStored` `fork_too_large` naming the cursor to continue (Bifrost: 422) |
|
||||
| `read-too-large.response` | Huginn to Bifrost | `NotStored` `fork_too_large`, read form: names the fork, no order or cursor (Bifrost: 422) |
|
||||
| `list-response-too-large.response` | Huginn to Bifrost | `Refused` `ResponseTooLarge` (Bifrost: 503) |
|
||||
| `refused-unit.response` | Huginn to Bifrost | `Refused` with a unit variant: a bare string `"MissingIdentity"` (shape only; Bifrost: 503) |
|
||||
| `unavailable.response` | Huginn to Bifrost | `Refused` `Unavailable` (Bifrost: 503 for this Mind) |
|
||||
| `failure.response` | Huginn to Bifrost | operation failure, `payloadSchema` `gamecult.cultnet.operation_failure.v1` (Bifrost: 503) |
|
||||
| `error.response` | Huginn to Bifrost | CultNet `Error` message, no payload (Bifrost: 503) |
|
||||
| `record-whole` | fragment | `gamecult.fork.record.v0` in a decider's view: all four answer variants (Counted, Revision, Refused, Retraction) by three accounts, answerer ids and notes whole, `answerer_count` 3 (6 keys) |
|
||||
| `record-withheld` | fragment | the same record in a non-decider's view: every `answerer` nil, notes whole, `answerer_count` 3 |
|
||||
|
||||
The reader-count shapes (`raised_at`, `decider_count`, `answerer_count`, nullable `answerer`) are
|
||||
also CultLib's own bytes, vendored unedited in `cultlib-d3025d3c/` (see its README for the pinned
|
||||
commit and each vector). Huginn's types decode every one of them and re-encode it byte for byte;
|
||||
the fixtures above are what Huginn's encoders write for the same shapes.
|
||||
28
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/README.md
vendored
Normal file
28
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/README.md
vendored
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
# CultLib desk-feed vectors, vendored
|
||||
|
||||
Source: `GameCult/CultLib` at commit `d3025d3c19674803ad32eb2335461b8cbc22bd28` (#118, on
|
||||
`3a51bbab` #117, on `af914253` #115), file `contracts/cultnet/governance/fork-card-vectors.json`. Each `.msgpack` file is
|
||||
the vector's `messagePackHex` decoded to bytes, nothing else changed. Do not edit them or rewrite them
|
||||
from Huginn's encoders: they are the reference Huginn's types are checked against (see
|
||||
`docs/desk-feed-door.md`). To re-pin, take the new commit's vectors and update the sha here, in
|
||||
`docs/desk-feed-door.md` and in `CULTLIB_SHA` in `desk_feed_fixtures.rs`.
|
||||
|
||||
| file | upstream vector | what it pins |
|
||||
|---|---|---|
|
||||
| `card-operator-fork` | `card.valid` `operator_fork` | 18 keys (`DE 00 12`), `raised_at` a string, a decider's `deciders` and `decider_count` 1 |
|
||||
| `card-raised-at-nil` | `card.valid` `raised_at_nil` | `raised_at` nil, key kept |
|
||||
| `card-withheld-deciders` | `card.valid` `withheld_deciders` | empty `deciders`, `decider_count` 2 |
|
||||
| `answer-counted` | `deskAnswer.valid` `counted` | 16 keys (`DE 00 10`), `answerer` a string |
|
||||
| `answer-revision` | `deskAnswer.valid` `revision` | a `Revision`, `answerer` a string |
|
||||
| `answer-retraction` | `deskAnswer.valid` `retraction` | a `Retraction`, `answerer` a string |
|
||||
| `answer-withheld-answerer` | `deskAnswer.valid` `withheld_answerer` | `answerer` nil |
|
||||
| `answer-withheld-answerer-revision` | `deskAnswer.valid` `withheld_answerer_revision` | a `Revision`, `answerer` nil |
|
||||
| `answer-withheld-answerer-retraction` | `deskAnswer.valid` `withheld_answerer_retraction` | a `Retraction`, `answerer` nil |
|
||||
| `answer-refused-stale` | `deskAnswer.valid` `refused_stale` | a `Refused` answer, `answerer` a string |
|
||||
| `entry-open-counted` | `transport.entry.valid` `open_counted` | decider view: entry `83`, record `86`, `answerer_count` 1 |
|
||||
| `entry-withheld-open-counted` | `transport.entry.valid` `withheld_open_counted` | non-decider twin: `deciders` empty, `answerer` nil, count 1 |
|
||||
| `entry-closed-with-ruling-and-a-refusal` | `transport.entry.valid` `closed_with_ruling_and_a_refusal` | decider view, a refusal among the answers, `answerer_count` 2 |
|
||||
| `entry-withheld-closed-with-ruling-and-a-refusal` | `transport.entry.valid` `withheld_closed_with_ruling_and_a_refusal` | its non-decider twin, count 2 |
|
||||
|
||||
The test `desk_feed_fixtures.rs` checks each file's SHA-256, decodes it with a decoder of its own
|
||||
(not `rmp_serde`), then decodes it into Huginn's real type and re-encodes it, byte for byte.
|
||||
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/answer-counted.msgpack
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/answer-counted.msgpack
vendored
Normal file
Binary file not shown.
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/answer-refused-stale.msgpack
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/answer-refused-stale.msgpack
vendored
Normal file
Binary file not shown.
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/answer-retraction.msgpack
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/answer-retraction.msgpack
vendored
Normal file
Binary file not shown.
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/answer-revision.msgpack
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/answer-revision.msgpack
vendored
Normal file
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/answer-withheld-answerer.msgpack
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/answer-withheld-answerer.msgpack
vendored
Normal file
Binary file not shown.
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/card-operator-fork.msgpack
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/card-operator-fork.msgpack
vendored
Normal file
Binary file not shown.
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/card-raised-at-nil.msgpack
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/card-raised-at-nil.msgpack
vendored
Normal file
Binary file not shown.
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/card-withheld-deciders.msgpack
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/card-withheld-deciders.msgpack
vendored
Normal file
Binary file not shown.
Binary file not shown.
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/entry-open-counted.msgpack
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/entry-open-counted.msgpack
vendored
Normal file
Binary file not shown.
Binary file not shown.
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/entry-withheld-open-counted.msgpack
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/entry-withheld-open-counted.msgpack
vendored
Normal file
Binary file not shown.
BIN
crates/huginn-mind/tests/fixtures/desk-feed/error.response.frame.bin
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/error.response.frame.bin
vendored
Normal file
Binary file not shown.
1
crates/huginn-mind/tests/fixtures/desk-feed/error.response.frame.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/error.response.frame.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
0000007385ad736368656d6156657273696f6eb063756c746e65742e6572726f722e7630a56572726f72d92f687567696e6e2e6465736b20726566757365732061206d6573736167655f6964206f76657220323536206279746573ab726f7574696e6748696e74c0a4636f6465c0a764657461696c73c0
|
||||
BIN
crates/huginn-mind/tests/fixtures/desk-feed/failure.response.frame.bin
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/failure.response.frame.bin
vendored
Normal file
Binary file not shown.
1
crates/huginn-mind/tests/fixtures/desk-feed/failure.response.frame.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/failure.response.frame.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
000001708aab646961676e6f737469637391b57061796c6f61642d6e6f742d612d72657175657374a96d6573736167654964a36d2d31a96f7065726174696f6ea5666f726b73a77061796c6f6164d95c6771526a6232526c745842686557787659575174626d39304c574574636d56786457567a644b64745a584e7a5957646c76325a705a57786b49455a76636d747a4c6d787062576c304f6942766458517462325974636d46755a32553daf7061796c6f6164456e636f64696e67b26d6573736167657061636b2d626173653634ad7061796c6f6164536368656d61d92567616d6563756c742e63756c746e65742e6f7065726174696f6e5f6661696c7572652e7631ad736368656d6156657273696f6ebd63756c746e65742e6f7065726174696f6e5f726573706f6e73652e7630a9736572766963654964ab687567696e6e2e6465736baf736f7572636552756e74696d654964ad687567696e6e2d657572656b61a6737461747573a872656a6563746564
|
||||
1
crates/huginn-mind/tests/fixtures/desk-feed/failure.response.payload.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/failure.response.payload.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
82a4636f6465b57061796c6f61642d6e6f742d612d72657175657374a76d657373616765bf6669656c6420466f726b732e6c696d69743a206f75742d6f662d72616e6765
|
||||
1
crates/huginn-mind/tests/fixtures/desk-feed/failure.response.payload.msgpack
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/failure.response.payload.msgpack
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
うcodeオpayload-not-a-requestァmessageソfield Forks.limit: out-of-range
|
||||
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.request.frame.bin
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.request.frame.bin
vendored
Normal file
Binary file not shown.
1
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.request.frame.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.request.frame.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
000001a287a96d6573736167654964a36d2d33a96f7065726174696f6ea5666f726b73a77061796c6f6164d9f06761564762334a726334616b62576c755a4b5a6c64584a6c6132476e63335669616d566a644e6b6b4e575977597a4a694d5755744f4745305a433030597a4e694c546c6c4e3259744d574579596a4e6a4e4751315a545a6d7147526c59326c6b5a584a7a6b746b6b4e575977597a4a694d5755744f4745305a433030597a4e694c546c6c4e3259744d574579596a4e6a4e4751315a545a6d32535177596a6c6b4e32557a59533032597a49784c54526d4e5467745954426b4e4330355a54686d4e324532596a566a4e47536d633352686448567a704739775a57366d5933567963323979774b5673615731706441413daf7061796c6f6164456e636f64696e67b26d6573736167657061636b2d626173653634ad7061796c6f6164536368656d61b6687567696e6e2e6465736b5f726571756573742e7631ad736368656d6156657273696f6ebc63756c746e65742e6f7065726174696f6e5f726571756573742e7630a9736572766963654964ab687567696e6e2e6465736b
|
||||
1
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.request.payload.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.request.payload.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
81a5466f726b7386a46d696e64a6657572656b61a77375626a656374d92435663063326231652d386134642d346333622d396537662d316132623363346435653666a8646563696465727392d92435663063326231652d386134642d346333622d396537662d316132623363346435653666d92430623964376533612d366332312d346635382d613064342d396538663761366235633464a6737461747573a46f70656ea6637572736f72c0a56c696d697400
|
||||
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.request.payload.msgpack
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.request.payload.msgpack
vendored
Normal file
Binary file not shown.
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.response.frame.bin
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.response.frame.bin
vendored
Normal file
Binary file not shown.
1
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.response.frame.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.response.frame.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
0000019289a96d6573736167654964a36d2d33a96f7065726174696f6ea5666f726b73a77061796c6f6164d9b06761564762334a7263344b6b62476c7a6449657463324e6f5a573168566d567963326c766272566e5957316c593356736443356d62334a724c6d787063335175646a436b62576c755a4b5a6c64584a6c6132476c633352686258436d637a45774d546b77716d39775a57356659323931626e5144726e4a6c593239795a47566b58324e76645735304261566d62334a7263354372626d56346446396a64584a7a62334c417057467a5832396d7a53664faf7061796c6f6164456e636f64696e67b26d6573736167657061636b2d626173653634ad7061796c6f6164536368656d61b7687567696e6e2e6465736b5f726573706f6e73652e7631ad736368656d6156657273696f6ebd63756c746e65742e6f7065726174696f6e5f726573706f6e73652e7630a9736572766963654964ab687567696e6e2e6465736baf736f7572636552756e74696d654964ad687567696e6e2d657572656b61a6737461747573a86163636570746564
|
||||
1
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.response.payload.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.response.payload.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
81a5466f726b7382a46c69737487ad736368656d6156657273696f6eb567616d6563756c742e666f726b2e6c6973742e7630a46d696e64a6657572656b61a57374616d70a6733130313930aa6f70656e5f636f756e7403ae7265636f726465645f636f756e7405a5666f726b7390ab6e6578745f637572736f72c0a561735f6f66cd27ce
|
||||
1
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.response.payload.msgpack
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.response.payload.msgpack
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
▼Forksうlist<EFBFBD>schemaVersionオgamecult.fork.list.v0、mindヲeureka・stampヲs10190ェopen_countョrecorded_count・forks性next_cursorタ・as_ofヘ'ホ
|
||||
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-empty.response.frame.bin
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-empty.response.frame.bin
vendored
Normal file
Binary file not shown.
1
crates/huginn-mind/tests/fixtures/desk-feed/list-empty.response.frame.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-empty.response.frame.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
0000019289a96d6573736167654964a36d2d31a96f7065726174696f6ea5666f726b73a77061796c6f6164d9b06761564762334a7263344b6b62476c7a6449657463324e6f5a573168566d567963326c766272566e5957316c593356736443356d62334a724c6d787063335175646a436b62576c755a4b5a6c64584a6c6132476c633352686258436d637a45774d546b77716d39775a57356659323931626e5141726e4a6c593239795a47566b58324e7664573530414b566d62334a7263354372626d56346446396a64584a7a62334c417057467a5832396d7a53664faf7061796c6f6164456e636f64696e67b26d6573736167657061636b2d626173653634ad7061796c6f6164536368656d61b7687567696e6e2e6465736b5f726573706f6e73652e7631ad736368656d6156657273696f6ebd63756c746e65742e6f7065726174696f6e5f726573706f6e73652e7630a9736572766963654964ab687567696e6e2e6465736baf736f7572636552756e74696d654964ad687567696e6e2d657572656b61a6737461747573a86163636570746564
|
||||
1
crates/huginn-mind/tests/fixtures/desk-feed/list-empty.response.payload.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-empty.response.payload.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
81a5466f726b7382a46c69737487ad736368656d6156657273696f6eb567616d6563756c742e666f726b2e6c6973742e7630a46d696e64a6657572656b61a57374616d70a6733130313930aa6f70656e5f636f756e7400ae7265636f726465645f636f756e7400a5666f726b7390ab6e6578745f637572736f72c0a561735f6f66cd27ce
|
||||
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-empty.response.payload.msgpack
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-empty.response.payload.msgpack
vendored
Normal file
Binary file not shown.
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-non-decider.request.frame.bin
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-non-decider.request.frame.bin
vendored
Normal file
Binary file not shown.
1
crates/huginn-mind/tests/fixtures/desk-feed/list-non-decider.request.frame.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-non-decider.request.frame.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
000001a287a96d6573736167654964a36d2d35a96f7065726174696f6ea5666f726b73a77061796c6f6164d9f06761564762334a726334616b62576c755a4b5a6c64584a6c6132476e63335669616d566a644e6b6b4f5755345a44646a4e6d49744e5745304f5330304d7a67794c574978595441745a6a6c6c4f475133597a5a694e5745307147526c59326c6b5a584a7a6b746b6b4e575977597a4a694d5755744f4745305a433030597a4e694c546c6c4e3259744d574579596a4e6a4e4751315a545a6d32535177596a6c6b4e32557a59533032597a49784c54526d4e5467745954426b4e4330355a54686d4e324532596a566a4e47536d633352686448567a704739775a57366d5933567963323979774b5673615731706444493daf7061796c6f6164456e636f64696e67b26d6573736167657061636b2d626173653634ad7061796c6f6164536368656d61b6687567696e6e2e6465736b5f726571756573742e7631ad736368656d6156657273696f6ebc63756c746e65742e6f7065726174696f6e5f726571756573742e7630a9736572766963654964ab687567696e6e2e6465736b
|
||||
1
crates/huginn-mind/tests/fixtures/desk-feed/list-non-decider.request.payload.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-non-decider.request.payload.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
81a5466f726b7386a46d696e64a6657572656b61a77375626a656374d92439653864376336622d356134392d343338322d623161302d663965386437633662356134a8646563696465727392d92435663063326231652d386134642d346333622d396537662d316132623363346435653666d92430623964376533612d366332312d346635382d613064342d396538663761366235633464a6737461747573a46f70656ea6637572736f72c0a56c696d697432
|
||||
1
crates/huginn-mind/tests/fixtures/desk-feed/list-non-decider.request.payload.msgpack
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-non-decider.request.payload.msgpack
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
<EFBFBD>¥Forks†¤mind¦eureka§subjectÙ$9e8d7c6b-5a49-4382-b1a0-f9e8d7c6b5a4¨deciders’Ù$5f0c2b1e-8a4d-4c3b-9e7f-1a2b3c4d5e6fÙ$0b9d7e3a-6c21-4f58-a0d4-9e8f7a6b5c4d¦status¤open¦cursorÀ¥limit2
|
||||
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-open-cursor.request.frame.bin
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-open-cursor.request.frame.bin
vendored
Normal file
Binary file not shown.
1
crates/huginn-mind/tests/fixtures/desk-feed/list-open-cursor.request.frame.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-open-cursor.request.frame.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
000001a687a96d6573736167654964a36d2d32a96f7065726174696f6ea5666f726b73a77061796c6f6164d9f46761564762334a726334616b62576c755a4b5a6c64584a6c6132476e63335669616d566a644e6b6b4e575977597a4a694d5755744f4745305a433030597a4e694c546c6c4e3259744d574579596a4e6a4e4751315a545a6d7147526c59326c6b5a584a7a6b746b6b4e575977597a4a694d5755744f4745305a433030597a4e694c546c6c4e3259744d574579596a4e6a4e4751315a545a6d32535177596a6c6b4e32557a59533032597a49784c54526d4e5467745954426b4e4330355a54686d4e324532596a566a4e47536d633352686448567a704739775a57366d59335679633239796f324d304d715673615731706444493daf7061796c6f6164456e636f64696e67b26d6573736167657061636b2d626173653634ad7061796c6f6164536368656d61b6687567696e6e2e6465736b5f726571756573742e7631ad736368656d6156657273696f6ebc63756c746e65742e6f7065726174696f6e5f726571756573742e7630a9736572766963654964ab687567696e6e2e6465736b
|
||||
1
crates/huginn-mind/tests/fixtures/desk-feed/list-open-cursor.request.payload.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-open-cursor.request.payload.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
81a5466f726b7386a46d696e64a6657572656b61a77375626a656374d92435663063326231652d386134642d346333622d396537662d316132623363346435653666a8646563696465727392d92435663063326231652d386134642d346333622d396537662d316132623363346435653666d92430623964376533612d366332312d346635382d613064342d396538663761366235633464a6737461747573a46f70656ea6637572736f72a3633432a56c696d697432
|
||||
1
crates/huginn-mind/tests/fixtures/desk-feed/list-open-cursor.request.payload.msgpack
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-open-cursor.request.payload.msgpack
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
<EFBFBD>¥Forks†¤mind¦eureka§subjectÙ$5f0c2b1e-8a4d-4c3b-9e7f-1a2b3c4d5e6f¨deciders’Ù$5f0c2b1e-8a4d-4c3b-9e7f-1a2b3c4d5e6fÙ$0b9d7e3a-6c21-4f58-a0d4-9e8f7a6b5c4d¦status¤open¦cursor£c42¥limit2
|
||||
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-open-non-decider.response.frame.bin
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-open-non-decider.response.frame.bin
vendored
Normal file
Binary file not shown.
1
crates/huginn-mind/tests/fixtures/desk-feed/list-open-non-decider.response.frame.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-open-non-decider.response.frame.hex
vendored
Normal file
File diff suppressed because one or more lines are too long
1
crates/huginn-mind/tests/fixtures/desk-feed/list-open-non-decider.response.payload.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-open-non-decider.response.payload.hex
vendored
Normal file
File diff suppressed because one or more lines are too long
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-open-non-decider.response.payload.msgpack
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-open-non-decider.response.payload.msgpack
vendored
Normal file
Binary file not shown.
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-open.request.frame.bin
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-open.request.frame.bin
vendored
Normal file
Binary file not shown.
1
crates/huginn-mind/tests/fixtures/desk-feed/list-open.request.frame.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-open.request.frame.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
000001a287a96d6573736167654964a36d2d31a96f7065726174696f6ea5666f726b73a77061796c6f6164d9f06761564762334a726334616b62576c755a4b5a6c64584a6c6132476e63335669616d566a644e6b6b4e575977597a4a694d5755744f4745305a433030597a4e694c546c6c4e3259744d574579596a4e6a4e4751315a545a6d7147526c59326c6b5a584a7a6b746b6b4e575977597a4a694d5755744f4745305a433030597a4e694c546c6c4e3259744d574579596a4e6a4e4751315a545a6d32535177596a6c6b4e32557a59533032597a49784c54526d4e5467745954426b4e4330355a54686d4e324532596a566a4e47536d633352686448567a704739775a57366d5933567963323979774b5673615731706444493daf7061796c6f6164456e636f64696e67b26d6573736167657061636b2d626173653634ad7061796c6f6164536368656d61b6687567696e6e2e6465736b5f726571756573742e7631ad736368656d6156657273696f6ebc63756c746e65742e6f7065726174696f6e5f726571756573742e7630a9736572766963654964ab687567696e6e2e6465736b
|
||||
1
crates/huginn-mind/tests/fixtures/desk-feed/list-open.request.payload.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-open.request.payload.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
81a5466f726b7386a46d696e64a6657572656b61a77375626a656374d92435663063326231652d386134642d346333622d396537662d316132623363346435653666a8646563696465727392d92435663063326231652d386134642d346333622d396537662d316132623363346435653666d92430623964376533612d366332312d346635382d613064342d396538663761366235633464a6737461747573a46f70656ea6637572736f72c0a56c696d697432
|
||||
1
crates/huginn-mind/tests/fixtures/desk-feed/list-open.request.payload.msgpack
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-open.request.payload.msgpack
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
<EFBFBD>¥Forks†¤mind¦eureka§subjectÙ$5f0c2b1e-8a4d-4c3b-9e7f-1a2b3c4d5e6f¨deciders’Ù$5f0c2b1e-8a4d-4c3b-9e7f-1a2b3c4d5e6fÙ$0b9d7e3a-6c21-4f58-a0d4-9e8f7a6b5c4d¦status¤open¦cursorÀ¥limit2
|
||||
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-open.response.frame.bin
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-open.response.frame.bin
vendored
Normal file
Binary file not shown.
1
crates/huginn-mind/tests/fixtures/desk-feed/list-open.response.frame.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-open.response.frame.hex
vendored
Normal file
File diff suppressed because one or more lines are too long
1
crates/huginn-mind/tests/fixtures/desk-feed/list-open.response.payload.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-open.response.payload.hex
vendored
Normal file
File diff suppressed because one or more lines are too long
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-open.response.payload.msgpack
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-open.response.payload.msgpack
vendored
Normal file
Binary file not shown.
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-response-too-large.response.frame.bin
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-response-too-large.response.frame.bin
vendored
Normal file
Binary file not shown.
1
crates/huginn-mind/tests/fixtures/desk-feed/list-response-too-large.response.frame.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-response-too-large.response.frame.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
0000012689a96d6573736167654964a36d2d31a96f7065726174696f6ea5666f726b73a77061796c6f6164d944676164535a575a316332566b676242535a584e776232357a5a56527662307868636d646c677156696558526c63383441415246777057787062576c307a6741424141413daf7061796c6f6164456e636f64696e67b26d6573736167657061636b2d626173653634ad7061796c6f6164536368656d61b7687567696e6e2e6465736b5f726573706f6e73652e7631ad736368656d6156657273696f6ebd63756c746e65742e6f7065726174696f6e5f726573706f6e73652e7630a9736572766963654964ab687567696e6e2e6465736baf736f7572636552756e74696d654964ad687567696e6e2d657572656b61a6737461747573a872656a6563746564
|
||||
1
crates/huginn-mind/tests/fixtures/desk-feed/list-response-too-large.response.payload.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-response-too-large.response.payload.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
81a75265667573656481b0526573706f6e7365546f6f4c6172676582a56279746573ce00011170a56c696d6974ce00010000
|
||||
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-response-too-large.response.payload.msgpack
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-response-too-large.response.payload.msgpack
vendored
Normal file
Binary file not shown.
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-too-large.response.frame.bin
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-too-large.response.frame.bin
vendored
Normal file
Binary file not shown.
1
crates/huginn-mind/tests/fixtures/desk-feed/list-too-large.response.frame.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-too-large.response.frame.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
0000020b89a96d6573736167654964a36d2d31a96f7065726174696f6ea5666f726b73a77061796c6f6164da012867616c4f62335254644739795a57534472584e6a6147567459565a6c636e4e70623236345a3246745a574e31624851755a6d3979617935795a575a31633246734c6e597770474e765a4757755a6d39796131393062323966624746795a32576c5a584a7962334c5a6a575a76636d74666447397658327868636d646c4f69426d62334a7249475631636d56725953317a64574a7a64484a68644755366358566c63335270623234365a47567a6179317361584e306157356e4c57463164476876636d6c306553426864434276636d526c636941304d5342706379427759584e304948526f5a5342775957646c49474a315a47646c644342696553427064484e6c6247593749474e76626e5270626e566c4948647064476767593356796332397949474d304d513d3daf7061796c6f6164456e636f64696e67b26d6573736167657061636b2d626173653634ad7061796c6f6164536368656d61b7687567696e6e2e6465736b5f726573706f6e73652e7631ad736368656d6156657273696f6ebd63756c746e65742e6f7065726174696f6e5f726573706f6e73652e7630a9736572766963654964ab687567696e6e2e6465736baf736f7572636552756e74696d654964ad687567696e6e2d657572656b61a6737461747573a872656a6563746564
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue