From f146be2e653968e587c16fd4ca41f5f05de352d1 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 20 Nov 2026 00:28:55 +0100 Subject: [PATCH] Projector after Soul's third pass: a glued-name boundary rule; one selection for both renderings of the messages to her; JSON escapes for bodies and display names; failures keep the report and their usage; a 60% floor and CJK words in the cap; addressing age; bounds under the connector frame MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit T1: address.rs names_whole_word: a name is glued only by a letter or digit, or a single hyphen with one beyond it; every other character is a boundary, so _Eureka_, Eureka/Libby, ok...Eureka? and Huginn.toml name her again and mind-eureka does not. Soul's 72 forms pinned. T2: said_to_her decides once; stimulus and heard both carry the count. T4: Model::ask fails with Failed { reason, usage }; a failed render or judge call is recorded with its usage and the report goes on. T5: the sentence cut keeps at least 60% of the budget; 。!? end sentences; CJK characters are words. T6/T7: spoken is a JSON string; a display name is quoted, capped at 80 bytes and followed by its Matrix id. T8: the addressing report gives the table's age and why it was not used. T9: own beat lines excluded (tested); words, stimulus, item and the other parts bounded. Co-Authored-By: Claude Opus 5.5 --- crates/mind-body/src/address.rs | 32 ++--- crates/mind-body/src/address/tests.rs | 37 ++++- crates/mind-body/src/bin/mind-persona.rs | 15 +- crates/mind-body/src/heartbeat.rs | 8 ++ crates/mind-body/src/projection.rs | 67 ++++++--- crates/mind-body/src/projection/narrate.rs | 130 +++++++++++------ crates/mind-body/src/projection/select.rs | 89 +++++++----- crates/mind-body/src/projection/tests.rs | 154 ++++++++++++++++++++- crates/mind-body/src/projection/tuning.rs | 23 +++ 9 files changed, 421 insertions(+), 134 deletions(-) diff --git a/crates/mind-body/src/address.rs b/crates/mind-body/src/address.rs index 91bebeb..396375b 100644 --- a/crates/mind-body/src/address.rs +++ b/crates/mind-body/src/address.rs @@ -93,33 +93,27 @@ pub(crate) fn project_names(repo: &OrgRepo, listed: &[Short]) -> Vec { names } -/// Whether `text` holds `name` as a whole word, in any case or exactly as written: the characters on either side are not letters or -/// digits, and not the joints of a path or an identifier (`-`, `_`, `/`, `\`, and a `.` with more of the token after it), so -/// `/var/lib/mind-eureka/` or `eureka.cc` never names her while "Eureka." ends a sentence that does. An identity match on a name, -/// not a judgement of meaning. +/// Whether `text` holds `name` as a whole word, in any case or exactly as written. A name is glued, and so not her name, +/// when a letter or digit touches it, or a single hyphen joins it to one (`mind-eureka`, `Eureka-chan`); everything else +/// is a boundary on either side: whitespace, punctuation, `/`, `_`, `.`, `\`, and a double hyphen or a dash. So +/// `/var/lib/mind-eureka/` never names her, and `_Eureka_`, `Eureka/Libby`, `ok...Eureka?` and `Huginn.toml` do. An +/// identity match on a name, not a judgement of meaning. pub(crate) fn names_whole_word(text: &str, name: &str, exact: bool) -> bool { let (text, name) = if exact { (text.to_string(), name.to_string()) } else { (text.to_lowercase(), name.to_lowercase()) }; if name.is_empty() { return false; } // Every character offset, not every non-overlapping match: a name may overlap itself ("ab ab" in "xab ab ab"). - text.char_indices().any(|(start, _)| { - text[start..].starts_with(&name) && !text[..start].chars().next_back().is_some_and(|before| joins(before) || before == '.') && !continues(&text[start + name.len()..]) - }) + text.char_indices().any(|(start, _)| text[start..].starts_with(&name) && !glued(text[..start].chars().rev()) && !glued(text[start + name.len()..].chars())) } -/// A character that joins a word to more of the same token: a letter, a digit, or a path or identifier joint. -fn joins(c: char) -> bool { - c.is_alphanumeric() || matches!(c, '-' | '_' | '/' | '\\') -} - -/// Whether the text after a name carries the token on: a joining character, or a `.` with a letter or digit after it. -fn continues(after: &str) -> bool { - let mut chars = after.chars(); - match chars.next() { - Some('.') => chars.next().is_some_and(char::is_alphanumeric), - Some(c) => joins(c), - None => false, +/// Whether the characters leading away from a name, nearest first, glue it to more of a word: a letter or digit, or a +/// single hyphen with a letter or digit beyond it. +fn glued(mut away: impl Iterator) -> bool { + match away.next() { + Some(c) if c.is_alphanumeric() => true, + Some('-') => away.next().is_some_and(char::is_alphanumeric), + _ => false, } } diff --git a/crates/mind-body/src/address/tests.rs b/crates/mind-body/src/address/tests.rs index fc10941..8ac8ccb 100644 --- a/crates/mind-body/src/address/tests.rs +++ b/crates/mind-body/src/address/tests.rs @@ -140,12 +140,37 @@ fn a_direct_message_is_for_her_whatever_the_basis() { } #[test] -fn a_name_inside_a_path_or_an_identifier_is_not_her_name() { - let eureka = Addressee::new("@eureka:chat", ["Eureka"]); - for path in ["the trace at /var/lib/mind-eureka/persona-trace.cc grew", "see eureka.cc", "eureka_state is a crate", "C:\\eureka\\x", "mind-eureka@ is a unit"] { - assert!(!addresses(&eureka, &said(path, &[])), "{path:?} does not name her"); +fn a_name_glued_to_a_word_is_not_her_name_and_every_other_boundary_is() { + // Soul's 72 forms (third pass on C3a): every one names her except the six glued to a word. + let eureka = Addressee::new("@eureka:chat", ["eureka", "Eureka"]); + let glued = ["Eurekas idea", "Eurekaさん", "ÉEureka", "Eureka-chan", "/var/lib/mind-eureka/x", "mind-eureka"]; + let forms = [ + "Eureka, are you there", "Eureka: look", "eureka?", "Eureka!", "Eureka.", "Eureka's turn", "Eureka’s turn", "Eurekas idea", "the Eureka's", "ask eureka about it", + "thanks eureka", "eureka", "Eureka\n", "`eureka`", "**Eureka**", "*Eureka*", "_Eureka_", "__Eureka__", "~~Eureka~~", "[Eureka](https://matrix.to/#/@eureka:gamecult.org)", + "https://matrix.to/#/@eureka:gamecult.org hi", "@eureka:gamecult.org hi", "@eureka hi", "<@eureka>", "(Eureka)", "\"Eureka\"", "«Eureka»", "—Eureka—", "Eureka—can you", + "Eureka--can you", "Eureka - can you", "Eureka/Libby, can you two", "Libby/Eureka can you", "Eureka/Nibu", "ok...Eureka?", "ok… Eureka?", "Hmm.Eureka?", "Eureka.Can you", + "Eureka...", "Eureka…", "#eureka", "Eurekaさん", "エウレカ Eureka", "Eureka、", "Eureka。", "ÉEureka", "Eureka-chan", "Eureka_bot", "Eureka\\", "\\Eureka", + "Eureka:\n> quoted", "> Eureka hi", "Eureka;", "Eureka'", "'Eureka'", "Eureka(?)", "Eureka+Libby", "Eureka&Libby", "Eureka|Libby", "Eureka=", "Eureka. ", + "/var/lib/mind-eureka/x", "eureka.cc", "eureka_state", "mind-eureka", "eureka.gamecult.org", "https://eureka.gamecult.org/forge", "C:\\eureka\\x", "Eureka.rs", + ]; + assert_eq!(forms.len(), 72); + for form in forms { + assert_eq!(addresses(&eureka, &said(form, &[])), !glued.contains(&form), "{form:?}"); } - for spoken in ["Eureka.", "Eureka, are you there?", "ask eureka.", "(Eureka)", "@Eureka hi", "\"Eureka\" said it", "Eureka's turn"] { - assert!(addresses(&eureka, &said(spoken, &[])), "{spoken:?} names her"); + // Each side of the rule on its own: a hyphen joins only with a letter or digit beyond it, before and after. + for (text, names) in [("x-eureka", false), ("eureka-x", false), ("--eureka", true), ("eureka--", true), ("x--eureka", true), ("eureka--x", true), ("-eureka", true), ("eureka-", true), (".eureka", true), ("eureka.x", true), ("x.eureka", true)] { + assert_eq!(addresses(&eureka, &said(text, &[])), names, "{text:?}"); + } +} + +#[test] +fn a_project_name_holds_at_every_boundary_but_a_glued_one() { + for repo in ["GameCult/Huginn", "GameCult/CultLib", "GameCult/gamecult-ops", "GameCult/cultcache-rs"] { + let steward = steward(repo, &[]); + let short = repo.split_once('/').unwrap().1; + for text in [format!("{repo} PR #76"), format!("{short}/main is red"), format!("_{short}_ is down"), format!("see {short}--now"), format!("{short}.toml"), format!("{short}'s build")] { + assert!(addresses(&steward, &said(&text, &[])), "{text:?}"); + } + assert!(!addresses(&steward, &said(&format!("mind-{short}"), &[])), "{short} glued by a hyphen"); } } diff --git a/crates/mind-body/src/bin/mind-persona.rs b/crates/mind-body/src/bin/mind-persona.rs index 6a9cebe..92ab948 100644 --- a/crates/mind-body/src/bin/mind-persona.rs +++ b/crates/mind-body/src/bin/mind-persona.rs @@ -19,7 +19,9 @@ //! `--state`), and with `--renders` the Projector's narrations of K successive turns; `--judge` renders a second state //! too and asks the model which narration came from which. It writes nothing and reads no brake. It reads //! `MIND_PERSONA_OPERATOR`, `MIND_PERSONA_MINDS` and `MIND_PERSONA_SELF_SENDER` (required), and the heartbeat's addressing -//! from `MIND_HEARTBEAT_STATE` when set; the room comes from the mouth when `BIFROST_CHAT_ENDPOINT` +//! from `MIND_HEARTBEAT_STATE`, which a run by hand must set as the unit does (`/var/lib/mind-heartbeat/heartbeat.cc`): +//! without it "said to her" knows only her own names, never her projects', and stderr says so before the report; the +//! room comes from the mouth when `BIFROST_CHAT_ENDPOINT` //! and `BIFROST_CHAT_KEY_FILE` are set; the tuning is `MIND_PERSONA_TUNING`, else //! `/etc/gamecult/minds//persona-tuning.toml`; renders need `MIND_PERSONA_CONNECTOR_ENDPOINT` //! (`:`) and the `connector-key` credential in `CREDENTIALS_DIRECTORY`. Without `--select-only` or @@ -150,7 +152,16 @@ fn run_project(args: &[String]) -> Result { own_sender: required("MIND_PERSONA_SELF_SENDER")?, tuning, }; - Ok(serde_json::to_string_pretty(&project(&organs, &request, now)?)?) + let report = project(&organs, &request, now)?; + let addressing = &report["addressing"]; + // The addressing source is what decides "said to her": said first, where a person running this by hand sees it. + eprintln!( + "mind-persona project: addressing from {} ({}); table written {}", + addressing["source"].as_str().unwrap_or("?"), + addressing["why"].as_str().unwrap_or("?"), + addressing["age_minutes"].as_i64().map_or_else(|| "never".to_owned(), |minutes| format!("{minutes} min ago")), + ); + Ok(serde_json::to_string_pretty(&report)?) } #[allow(clippy::disallowed_methods, reason = "the binary reads its own arguments")] diff --git a/crates/mind-body/src/heartbeat.rs b/crates/mind-body/src/heartbeat.rs index 8b146fb..063d216 100644 --- a/crates/mind-body/src/heartbeat.rs +++ b/crates/mind-body/src/heartbeat.rs @@ -220,6 +220,10 @@ pub trait Store { /// What a Persona unit may read of `heartbeat.cc`: the swarm's addressing. Read-only. pub trait HeartbeatView { fn addressing(&self) -> Result>; + /// When the store was last written, so a reader can tell a stale table; none when there is no store at all. + fn written_at(&self) -> Option> { + None + } } /// The store as a CultCache single-file store under the unit's state directory. @@ -315,6 +319,10 @@ impl HeartbeatView for FileStore { fn addressing(&self) -> Result> { self.load_addressing() } + + fn written_at(&self) -> Option> { + std::fs::metadata(&self.path).and_then(|meta| meta.modified()).ok().map(DateTime::::from) + } } /// What the operator stores allow: the swarm brake, and each instance's own diff --git a/crates/mind-body/src/projection.rs b/crates/mind-body/src/projection.rs index 1e5ccd3..28f2ce9 100644 --- a/crates/mind-body/src/projection.rs +++ b/crates/mind-body/src/projection.rs @@ -30,7 +30,7 @@ use crate::address::{Addressee, is_for, localpart}; use crate::chat_client::{Chat, ChatPending, Form}; use crate::heartbeat::HeartbeatView; -pub use narrate::{Answer, KitModel, Model, Spent, connector_key}; +pub use narrate::{Answer, Failed, KitModel, Model, Spent, connector_key}; pub use select::TurnKind; pub use tuning::{TUNING_FILE, TUNING_ROOT}; @@ -129,19 +129,28 @@ pub struct Request { } /// Her addressee: the heartbeat's row for her when its table holds one, else her own names (handle and public name). -fn own_addressee(addressing: Option<&dyn HeartbeatView>, own_sender: &str, public_name: &str) -> (Addressee, Value) { +/// Her addressee and where it came from: `{source, why, written_at, age_minutes}`. The heartbeat's row when its table +/// holds one; else her own names (handle and public name), with why: no store configured, no heartbeat file, a file +/// that could not be read, or a table without her row. The table's age is the store's last write: under a held swarm +/// brake the heartbeat does not poll, so an old table is a stale one. +fn own_addressee(addressing: Option<&dyn HeartbeatView>, own_sender: &str, public_name: &str, now: DateTime) -> (Addressee, Value) { + let written_at = addressing.and_then(HeartbeatView::written_at); + let age = written_at.map(|at| (now - at).num_minutes()); + let report = |source: &str, why: &str| json!({ "source": source, "why": why, "written_at": written_at.map(|at| at.to_rfc3339()), "age_minutes": age }); let table = addressing.map(|view| view.addressing()); if let Some(Ok(Some(table))) = &table && let Some(row) = table.minds.iter().find(|row| row.matrix_id() == own_sender) { - return (row.clone(), json!("heartbeat")); + return (row.clone(), report("heartbeat", "her row in the heartbeat's addressing table")); } - let why = match table { - None => "own names only: no heartbeat addressing is configured".to_owned(), - Some(Err(fault)) => format!("own names only: the heartbeat addressing could not be read: {fault:#}"), - Some(Ok(_)) => "own names only: the heartbeat addressing has no row for her".to_owned(), + let why = match (&table, written_at) { + (None, _) => "no heartbeat addressing is configured (MIND_HEARTBEAT_STATE is not set)".to_owned(), + (Some(Err(fault)), _) => format!("the heartbeat addressing could not be read: {fault:#}"), + (Some(Ok(_)), None) => "there is no heartbeat file".to_owned(), + (Some(Ok(None)), Some(_)) => "the heartbeat file holds no addressing table".to_owned(), + (Some(Ok(Some(_))), Some(_)) => "the heartbeat's addressing table has no row for her".to_owned(), }; - (Addressee::new(own_sender, [localpart(own_sender), public_name]), json!(why)) + (Addressee::new(own_sender, [localpart(own_sender), public_name]), report("own names only", &why)) } /// The room through the mouth's pending read, which acks nothing: new messages and the tail, own lines kept for the @@ -195,15 +204,19 @@ fn packet_json(packet: &Packet) -> Value { /// One narration of one packet: the request id names the instance, the seed and the moment, because the kit's replay /// store answers a repeated id with its first answer. -fn narrate_one(model: &dyn Model, instance: &str, packet: &Packet, tuning: &tuning::Tuning, tag: &str, now: DateTime) -> Result { +/// A failed call is a record with its reason and what it cost, never the end of the report. +fn narrate_one(model: &dyn Model, instance: &str, packet: &Packet, tuning: &tuning::Tuning, tag: &str, now: DateTime) -> Value { let request_id = format!("{AGENT}:{instance}:project:{tag}:{}:{}", packet.seed, now.timestamp_millis()); - let answer = model.ask(&request_id, &tuning.projector.model, &projector_prompt(packet, tuning))?; + let ids: Vec = packet.selected.iter().map(|pick| pick.id.clone()).collect(); + let answer = match model.ask(&request_id, &tuning.projector.model, &projector_prompt(packet, tuning)) { + Ok(answer) => answer, + Err(failed) => return json!({ "seed": packet.seed, "lens": packet.lens, "selected": ids, "failed": failed.reason, "usage": failed.usage }), + }; let raw = answer.text; let budget = word_budget(packet.kind, tuning); let narration = cap_words(&raw, budget); - let ids: Vec = packet.selected.iter().map(|pick| pick.id.clone()).collect(); let named: Vec = packet.ids.clone(); - Ok(json!({ + json!({ "seed": packet.seed, "lens": packet.lens, "selected": ids, @@ -212,7 +225,7 @@ fn narrate_one(model: &dyn Model, instance: &str, packet: &Packet, tuning: &tuni "field_names": field_names(&narration, &named), "usage": answer.usage, "narration": narration, - })) + }) } /// K successive turns over one state: each packet under the freshness the earlier ones left. @@ -242,7 +255,7 @@ pub fn project(organs: &Organs, request: &Request, now: DateTime) -> Result } }; let public_name = state.get("publicName").and_then(Value::as_str).unwrap_or_default().to_owned(); - let (own, addressing) = own_addressee(organs.addressing, &request.own_sender, &public_name); + let (own, addressing) = own_addressee(organs.addressing, &request.own_sender, &public_name, now); let (room, room_report) = match organs.chat { Some(chat) => read_room(chat, request, &own), None => (Room { here: request.room.clone(), ..Room::default() }, json!({ "read": false, "why": "no mouth is configured" })), @@ -269,8 +282,10 @@ pub fn project(organs: &Organs, request: &Request, now: DateTime) -> Result report["model"] = json!(tuning.projector.model); let packets = successive(&inputs, tuning, renders.max(1)); report["packet"] = packet_json(&packets[0]); - let narrations: Vec = packets.iter().map(|packet| narrate_one(model, &request.instance, packet, tuning, "a", now)).collect::>()?; - let texts = |narrations: &[Value]| narrations.iter().map(|one| one["narration"].as_str().unwrap_or_default().to_owned()).collect::>(); + let narrations: Vec = packets.iter().map(|packet| narrate_one(model, &request.instance, packet, tuning, "a", now)).collect(); + let texts = |narrations: &[Value]| narrations.iter().filter_map(|one| one["narration"].as_str().map(str::to_owned)).collect::>(); + let failed = |narrations: &[Value]| narrations.iter().filter(|one| one.get("failed").is_some()).count(); + report["failed_renders"] = json!(failed(&narrations)); report["distinct_2"] = json!(distinct_2(&texts(&narrations))); report["embedding_distance"] = json!("not measured: the daemon offers no embedding verb"); let leaked = narrations.iter().any(|one| one["field_names"].as_array().is_some_and(|names| !names.is_empty())); @@ -279,7 +294,7 @@ pub fn project(organs: &Organs, request: &Request, now: DateTime) -> Result if let Some(other) = &request.judge_against { let other_inputs = Inputs { state: other, ..inputs }; let others = successive(&other_inputs, tuning, renders.max(1)); - let other_narrations: Vec = others.iter().map(|packet| narrate_one(model, &request.instance, packet, tuning, "b", now)).collect::>()?; + let other_narrations: Vec = others.iter().map(|packet| narrate_one(model, &request.instance, packet, tuning, "b", now)).collect(); let mut verdicts = Vec::new(); let mut right = 0usize; for (at, (narration, truth)) in narrations.iter().map(|one| (one, 'A')).chain(other_narrations.iter().map(|one| (one, 'B'))).enumerate() { @@ -287,16 +302,26 @@ pub fn project(organs: &Organs, request: &Request, now: DateTime) -> Result // The two descriptions swap places on odd turns, so a judge that always answers one letter scores half. let swapped = at % 2 == 1; let (first, second) = if swapped { (b, a) } else { (a, b) }; - let text = narration["narration"].as_str().unwrap_or_default(); - let answer = model.ask(&format!("{AGENT}:{}:judge:{at}:{}", request.instance, now.timestamp_millis()), &tuning.projector.model, &judge_prompt(first, second, text))?; + // A narration that failed has nothing to judge; it is counted under the renders' failures. + let Some(text) = narration["narration"].as_str() else { continue }; + let answer = match model.ask(&format!("{AGENT}:{}:judge:{at}:{}", request.instance, now.timestamp_millis()), &tuning.projector.model, &judge_prompt(first, second, text)) { + Ok(answer) => answer, + Err(failed) => { + verdicts.push(json!({ "from": truth.to_string(), "failed": failed.reason, "usage": failed.usage })); + continue; + } + }; let letter = judged(&answer.text).map(|letter| if swapped { if letter == 'A' { 'B' } else { 'A' } } else { letter }); let correct = letter == Some(truth); right += usize::from(correct); verdicts.push(json!({ "from": truth.to_string(), "judged": letter.map(String::from), "correct": correct, "usage": answer.usage })); } - let accuracy = right as f64 / verdicts.len().max(1) as f64; + // Accuracy over the verdicts the judge gave; a failed call is no verdict, and any failure fails the eval. + let judged_count = verdicts.iter().filter(|verdict| verdict.get("failed").is_none()).count(); + let failures = verdicts.len() - judged_count + failed(&narrations) + failed(&other_narrations); + let accuracy = right as f64 / judged_count.max(1) as f64; report["against"] = json!({ "packet": packet_json(&others[0]), "renders": other_narrations, "distinct_2": distinct_2(&texts(&other_narrations)) }); - report["judge"] = json!({ "accuracy": accuracy, "pass_at": JUDGE_PASS, "passed": accuracy >= JUDGE_PASS && !leaked, "verdicts": verdicts }); + report["judge"] = json!({ "accuracy": accuracy, "pass_at": JUDGE_PASS, "failures": failures, "passed": accuracy >= JUDGE_PASS && !leaked && failures == 0, "verdicts": verdicts }); } report["renders"] = Value::Array(narrations); Ok(report) diff --git a/crates/mind-body/src/projection/narrate.rs b/crates/mind-body/src/projection/narrate.rs index 717cb16..53e76e5 100644 --- a/crates/mind-body/src/projection/narrate.rs +++ b/crates/mind-body/src/projection/narrate.rs @@ -20,8 +20,34 @@ use super::tuning::Tuning; /// One tool-less model turn. `KitModel` is the one live owner; a test answers with recorded text. pub trait Model { - /// One turn of `model`: `prompt` is its one user text and `request_id` its identity on the connector. - fn ask(&self, request_id: &str, model: &str, prompt: &str) -> Result; + /// One turn of `model`: `prompt` is its one user text and `request_id` its identity on the connector. A failed turn + /// keeps whatever usage the connector reported for it. + fn ask(&self, request_id: &str, model: &str, prompt: &str) -> std::result::Result; +} + +/// A turn that produced no answer: why, and what it cost when the connector's receipt says (a refusal, a fault before +/// the provider, or a failure with no usage block costs none). A receipt whose usage block is missing or malformed does +/// not decode at all: that is model-connector's error, reported as the reason. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Failed { + pub reason: String, + pub usage: Option, +} + +impl std::fmt::Display for Failed { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.reason) + } +} + +impl Failed { + fn without_usage(reason: impl Into) -> Self { + Self { reason: reason.into(), usage: None } + } +} + +fn spent(model: &str, usage: &model_connector::Usage) -> Spent { + Spent { model: model.to_owned(), input_tokens: usage.input_tokens, output_tokens: usage.output_tokens, cache_read_tokens: usage.cache_read_tokens, cache_write_tokens: usage.cache_write_tokens, reasoning_tokens: usage.reasoning_tokens } } /// A model's answer, and what the turn cost as the connector counted it: none when the answerer counts nothing. @@ -75,15 +101,15 @@ impl KitModel { } impl Model for KitModel { - fn ask(&self, request_id: &str, model: &str, prompt: &str) -> Result { + fn ask(&self, request_id: &str, model: &str, prompt: &str) -> std::result::Result { let mut request = ProviderRequest::new(request_id, request_id, model, INSTRUCTIONS, MAX_OUTPUT_TOKENS); request.input = vec![InputItem::UserText { text: prompt.to_owned() }]; request.reasoning_effort = Some(ReasoningEffort::Low); let expires = u64::try_from(Utc::now().timestamp_millis()).unwrap_or_default() + INVOCATION_LIFE_MS; - let invocation = TransportInvocation::new(&self.caller, expires, Sha256::digest(prompt.as_bytes()).into(), request).map_err(|_| anyhow!("the request is not a valid connector request"))?; - let result = self.client.execute(&invocation).map_err(|fault| anyhow!("the connector did not answer: {fault}"))?; + let invocation = TransportInvocation::new(&self.caller, expires, Sha256::digest(prompt.as_bytes()).into(), request).map_err(|_| Failed::without_usage("the request is not a valid connector request"))?; + let result = self.client.execute(&invocation).map_err(|fault| Failed::without_usage(format!("the connector did not answer: {fault}")))?; match result.disposition { - TransportDisposition::Refused(reason) => bail!("the connector refused the call: {reason:?}"), + TransportDisposition::Refused(reason) => Err(Failed::without_usage(format!("the connector refused the call: {reason:?}"))), TransportDisposition::Transported { events, receipt } => match receipt.outcome { TransportOutcome::Completed { stop_reason: StopReason::EndTurn, usage, .. } => Ok(Answer { text: events @@ -93,17 +119,10 @@ impl Model for KitModel { _ => None, }) .collect(), - usage: Some(Spent { - model: model.to_owned(), - input_tokens: usage.input_tokens, - output_tokens: usage.output_tokens, - cache_read_tokens: usage.cache_read_tokens, - cache_write_tokens: usage.cache_write_tokens, - reasoning_tokens: usage.reasoning_tokens, - }), + usage: Some(spent(model, &usage)), }), - TransportOutcome::Completed { stop_reason, .. } => bail!("the model's turn ended early: {stop_reason:?}"), - TransportOutcome::Failed { kind, .. } => bail!("the provider failed the call: {kind:?}"), + TransportOutcome::Completed { stop_reason, usage, .. } => Err(Failed { reason: format!("the model's turn ended early: {stop_reason:?}"), usage: Some(spent(model, &usage)) }), + TransportOutcome::Failed { kind, usage, .. } => Err(Failed { reason: format!("the provider failed the call: {kind:?}"), usage: usage.as_ref().map(|usage| spent(model, usage)) }), }, } } @@ -122,52 +141,79 @@ pub(crate) fn projector_prompt(packet: &Packet, tuning: &Tuning) -> String { build_projector_prompt(&ProjectorPrompt { identity: &packet.identity, typed_context: &packet.context, visible_stimulus: &packet.stimulus, domain_guidance: &guidance, word_budget: word_budget(packet.kind, tuning) }) } -/// `text` held to `max` whitespace-separated words; the model's count is not trusted. An overrun is cut at the last -/// sentence end inside the budget, so the stream ends on a whole sentence; only when the budget holds no sentence end -/// is it cut after the `max`-th word. -pub(crate) fn cap_words(text: &str, max: usize) -> String { - let text = text.trim(); - let mut words = 0; - let mut in_word = false; - for (at, c) in text.char_indices() { - if c.is_whitespace() { - in_word = false; - } else if !in_word { - in_word = true; - words += 1; - if words > max { - let within = text[..at].trim_end(); - return last_sentence_end(within).map_or_else(|| within.to_owned(), |end| within[..end].to_owned()); - } - } - } - text.to_owned() +/// A character that is a word on its own: Han, kana and Hangul are written without spaces, so each counts as one. +fn cjk(c: char) -> bool { + matches!(u32::from(c), 0x3040..=0x30FF | 0x3400..=0x4DBF | 0x4E00..=0x9FFF | 0xAC00..=0xD7AF | 0xF900..=0xFAFF | 0x20000..=0x2FFFF) } -/// The byte just past the last sentence end in `text`: a `.`, `!`, `?` or `…`, with any closing quotes or brackets after -/// it, followed by whitespace or the end of the text. +/// CJK and full-width punctuation (`。`, `,`, `!`, `?`, `「」`): it belongs to the word before it and starts none. +fn cjk_punctuation(c: char) -> bool { + matches!(u32::from(c), 0x3000..=0x303F | 0xFF01..=0xFF0F | 0xFF1A..=0xFF20 | 0xFF3B..=0xFF40 | 0xFF5B..=0xFF65) +} + +/// The byte offset where each word of `text` begins: a run of non-space characters is one word, and a CJK character is +/// a word of its own. +fn word_starts(text: &str) -> Vec { + let mut starts = Vec::new(); + let mut in_word = false; + for (at, c) in text.char_indices() { + if c.is_whitespace() || cjk_punctuation(c) { + in_word = false; + } else if cjk(c) { + starts.push(at); + in_word = false; + } else if !in_word { + starts.push(at); + in_word = true; + } + } + starts +} + +/// The share of the budget a sentence cut must keep; below it the word cut keeps more of the stream. +const SENTENCE_CUT_FLOOR: f64 = 0.6; + +/// `text` held to `max` words (whitespace-separated, each CJK character one); the model's count is not trusted. An +/// overrun is cut at the last sentence end inside the budget, so the stream ends on a whole sentence, when that keeps +/// at least 60% of the budget; otherwise, or with no sentence end inside it, it is cut after the `max`-th word. No +/// abbreviation list: a cut after "e.g." is accepted. +pub(crate) fn cap_words(text: &str, max: usize) -> String { + let text = text.trim(); + let starts = word_starts(text); + let Some(&over) = starts.get(max) else { return text.to_owned() }; + let within = text[..over].trim_end(); + match last_sentence_end(within) { + Some(end) if words(&within[..end]) as f64 >= SENTENCE_CUT_FLOOR * max as f64 => within[..end].to_owned(), + _ => within.to_owned(), + } +} + +/// The byte just past the last sentence end in `text`: a `.`, `!`, `?` or `…` with any closing quotes or brackets after +/// it, followed by whitespace or the end of the text; or a `。`, `!` or `?`, which ends a sentence where it stands. fn last_sentence_end(text: &str) -> Option { let chars: Vec<(usize, char)> = text.char_indices().collect(); let mut found = None; for (index, (at, c)) in chars.iter().enumerate() { - if !matches!(c, '.' | '!' | '?' | '…') { + let full_width = matches!(c, '。' | '!' | '?'); + if !full_width && !matches!(c, '.' | '!' | '?' | '…') { continue; } let mut end = at + c.len_utf8(); let mut next = index + 1; - while let Some((close_at, close)) = chars.get(next).filter(|(_, close)| matches!(close, '"' | '\'' | '”' | '’' | ')' | ']')) { + while let Some((close_at, close)) = chars.get(next).filter(|(_, close)| matches!(close, '"' | '\'' | '”' | '’' | ')' | ']' | '」' | '』')) { end = close_at + close.len_utf8(); next += 1; } - if chars.get(next).is_none_or(|(_, after)| after.is_whitespace()) { + if full_width || chars.get(next).is_none_or(|(_, after)| after.is_whitespace()) { found = Some(end); } } found } +/// Words as the cap counts them. pub(crate) fn words(text: &str) -> usize { - text.split_whitespace().count() + word_starts(text).len() } /// The one kebab form that is ordinary English: "short-term" folds to the schema key `shortTerm`, and prose must not fail diff --git a/crates/mind-body/src/projection/select.rs b/crates/mind-body/src/projection/select.rs index c43e597..3436b7d 100644 --- a/crates/mind-body/src/projection/select.rs +++ b/crates/mind-body/src/projection/select.rs @@ -14,9 +14,9 @@ //! //! Who a message is to is `address::is_for`'s, the rule the heartbeat woke her by, and nothing else here. People are //! keyed by their full Matrix id; only the operator and the Minds of the swarm are keyed by the names their bonds use, -//! and an unbonded speaker is shown with the id, never under a bonded person's name. Every message body is rendered by -//! one escape (`spoken`): quotes escaped, a line break shown as ` ⏎ `, so no text can forge a line; speaker and room come -//! from the mouth's metadata alone. +//! and an unbonded speaker is shown by the display name the mouth reports, quoted and escaped, with the Matrix id after +//! it, never under a bonded person's name. Every message body and display name is rendered by one escape (`spoken`, a +//! JSON string), so no text can forge a line; speaker and room come from the mouth's metadata alone. //! //! Never read: `agencyPressure`, `candidateActions`, `privateNotes` and `provenance`. A memory tagged with a direct //! message room (`room:!…`) is selectable only in that room. @@ -262,22 +262,11 @@ fn relevance_log(item_text: &str, touching: &BTreeSet) -> f64 { (1.0 + (shared as f64 / 3.0).min(1.0)).ln() } -/// A message body as every line renders it: in quotation marks, its own quotes and backslashes escaped, a line break -/// shown as ` ⏎ ` and other control characters dropped, so a body can never end its line or start another. +/// A message body or a display name as every line renders it: a JSON string, quotes, backslashes and every control +/// character escaped (`\n`, `\t`, `\r`, `\u…`, and the Unicode line separators too), so no text can end its line or +/// start another, and the escape is reversible: nothing is dropped. pub(crate) fn spoken(text: &str) -> String { - let mut out = String::with_capacity(text.len() + 2); - out.push('"'); - for c in text.chars() { - match c { - '\\' => out.push_str("\\\\"), - '"' => out.push_str("\\\""), - '\n' | '\u{2028}' | '\u{2029}' | '\u{0085}' => out.push_str(" ⏎ "), - c if c.is_control() => {} - c => out.push(c), - } - } - out.push('"'); - out + crate::persona::quoted(text) } // --------------------------------------------------------------------------------------------- @@ -520,11 +509,12 @@ fn cap_of(pool: Pool, tuning: &Tuning) -> usize { fn name_of(line: &RoomLine, inputs: &Inputs) -> String { let person = inputs.people.person_of(&line.sender); let bonded = items(inputs.state, &["affect", "socialBonds"]).iter().find(|bond| live(bond) && text(bond, &["object", "id"]).to_lowercase() == person).map(|bond| text(bond, &["object", "label"]).to_owned()).filter(|label| !label.is_empty()); - let name = bonded.unwrap_or_else(|| { - let display = one_line(&line.display); - if display.is_empty() { line.sender.clone() } else { format!("{display} ({})", line.sender) } - }); - clip(&one_line(&name), NAME_BYTES) + match bonded { + Some(label) => clip(&one_line(&label), NAME_BYTES), + // A display name is the speaker's own text: quoted and escaped, at most `NAME_BYTES`, the real id always after it. + None if !line.display.trim().is_empty() => format!("{} ({})", spoken(&clip(line.display.trim(), NAME_BYTES)), line.sender), + None => line.sender.clone(), + } } /// Lines appended under a byte cap: a line that would pass it is refused whole. @@ -588,34 +578,55 @@ fn newest_section(heading: &str, lines: &[String], cap: usize) -> String { if shown { out.out } else { String::new() } } -/// The messages to her as lines under the `addressed_bytes` ceiling: whole messages, newest first, until the ceiling; -/// a note counting the older ones left out; and a single message larger than the ceiling clipped on a grapheme boundary -/// with how many bytes more it held, never paraphrased. Chronological order, with the counts. -fn said_to_her(to_her: &[&RoomLine], render: impl Fn(&RoomLine, &str) -> String, ceiling: usize) -> (Vec, usize, usize) { - let mut kept: Vec = Vec::new(); +/// Which messages to her a turn shows: whole messages, newest first, until the `addressed_bytes` ceiling, and a single +/// message larger than the ceiling clipped on a grapheme boundary (`cut`, the bytes of its text kept). Decided once, on +/// the stimulus's lines, and rendered for the stimulus and the heard lines alike. +struct Addressed<'a> { + /// Chronological: each message shown, and where its text is cut if it is. + shown: Vec<(&'a RoomLine, Option)>, + left_out: usize, + clipped: usize, +} + +impl Addressed<'_> { + fn lines(&self, render: impl Fn(&RoomLine, &str) -> String) -> Vec { + let mut lines: Vec = Vec::new(); + if self.left_out > 0 { + lines.push(format!("({} more said to her, not shown)", self.left_out)); + } + lines.extend(self.shown.iter().map(|(line, cut)| match cut { + None => render(line, &spoken(&line.text)), + Some(cut) => render(line, &format!("{} [… {} bytes more]", spoken(&line.text[..*cut]), line.text.len() - cut)), + })); + lines + } +} + +fn said_to_her<'a>(to_her: &[&'a RoomLine], render: impl Fn(&RoomLine, &str) -> String, ceiling: usize) -> Addressed<'a> { + let mut shown: Vec<(&RoomLine, Option)> = Vec::new(); let mut used = 0; let mut clipped = 0; for line in to_her.iter().rev() { let whole = render(line, &spoken(&line.text)); if used + whole.len() < ceiling { used += whole.len() + 1; - kept.push(whole); + shown.push((line, None)); continue; } - if kept.is_empty() { + if shown.is_empty() { // One message larger than the ceiling: as much of it as fits, cut between graphemes. let ends: Vec = line.text.grapheme_indices(true).map(|(at, grapheme)| at + grapheme.len()).collect(); let fitting = |end: usize| render(line, &format!("{} [… {} bytes more]", spoken(&line.text[..end]), line.text.len() - end)); let fit = ends.partition_point(|end| fitting(*end).len() < ceiling); let cut = if fit == 0 { 0 } else { ends[fit - 1] }; clipped = line.text.len() - cut; - kept.push(fitting(cut)); + shown.push((line, Some(cut))); } break; } - let left_out = to_her.len() - kept.len(); - kept.reverse(); - (kept, left_out, clipped) + let left_out = to_her.len() - shown.len(); + shown.reverse(); + Addressed { shown, left_out, clipped } } fn chat_stimulus(inputs: &Inputs, tuning: &Tuning) -> (String, String, Overflow) { @@ -628,14 +639,16 @@ fn chat_stimulus(inputs: &Inputs, tuning: &Tuning) -> (String, String, Overflow) // Heard lines are prose for the Persona: the speaker and the room from the mouth, then the escaped words. let heard = |line: &RoomLine, body: &str| format!("{}, in {}: {body}", name_of(line, inputs), line.room); let ceiling = tuning.select.addressed_bytes; - let (typed_to_her, left_out, clipped) = said_to_her(&to_her, typed, ceiling); - let (heard_to_her, _, _) = said_to_her(&to_her, heard, ceiling); - let note = (left_out > 0).then(|| format!("({left_out} more said to her, not shown)")); + // One selection, rendered twice: the stimulus and the heard lines show the same messages to her and the same count. + let addressed = said_to_her(&to_her, typed, ceiling); + let (left_out, clipped) = (addressed.left_out, addressed.clipped); + let typed_to_her = addressed.lines(typed); + let heard_to_her = addressed.lines(heard); let mut stimulus = String::new(); if !typed_to_her.is_empty() { stimulus.push_str("Said to her:"); - for line in note.iter().chain(&typed_to_her) { + for line in &typed_to_her { stimulus.push('\n'); stimulus.push_str(line); } diff --git a/crates/mind-body/src/projection/tests.rs b/crates/mind-body/src/projection/tests.rs index 82ff845..ad35aa1 100644 --- a/crates/mind-body/src/projection/tests.rs +++ b/crates/mind-body/src/projection/tests.rs @@ -283,7 +283,7 @@ fn messages_addressed_to_her_appear_verbatim() { let said = "Eureka — \"is the door\" fixed?\nTell me straight."; let room = Room { messages: vec![message("#swarm:gamecult.org", "@libby:gamecult.org", "Libby", said, 2, true)], tail: vec![], here: None }; let packet = packet_of(TurnKind::Chat, &state(), &room, &History::default(), 1); - let escaped = "\"Eureka — \\\"is the door\\\" fixed? ⏎ Tell me straight.\""; + let escaped = "\"Eureka — \\\"is the door\\\" fixed?\\nTell me straight.\""; assert_eq!(packet.heard, format!("Libby, in #swarm:gamecult.org: {escaped}"), "the Persona hears the exact words, escaped on one line"); assert!(packet.stimulus.starts_with("Said to her:\n") && packet.stimulus.contains(escaped), "{}", packet.stimulus); let own = Room { messages: vec![message("#swarm:gamecult.org", OWN, "Eureka", "my own line", 1, false)], tail: vec![], here: None }; @@ -585,7 +585,7 @@ impl Recorded { } impl Model for Recorded { - fn ask(&self, request_id: &str, model: &str, prompt: &str) -> Result { + fn ask(&self, request_id: &str, model: &str, prompt: &str) -> std::result::Result { self.calls.borrow_mut().push((request_id.to_owned(), model.to_owned(), prompt.to_owned())); let text = if prompt.starts_with("