diff --git a/Cargo.lock b/Cargo.lock index 53f5916..d3d6ea9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -851,6 +851,16 @@ dependencies = [ "polyval", ] +[[package]] +name = "ghostlight-persona-projection" +version = "0.1.0" +source = "git+https://github.com/GameCult/Ghostlight.git?rev=c43eb54962ed4a09cc737ecac73a16b0d7ce62b4#c43eb54962ed4a09cc737ecac73a16b0d7ce62b4" +dependencies = [ + "serde", + "sha2", + "unicode-segmentation", +] + [[package]] name = "hashbrown" version = "0.17.1" @@ -963,6 +973,16 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "inout" version = "0.1.4" @@ -1098,6 +1118,7 @@ dependencies = [ "cultnet-rs 0.3.0", "eureka-pipeline", "eureka-state", + "ghostlight-persona-projection", "huginn-mind", "libc", "model-connector", @@ -1110,7 +1131,9 @@ dependencies = [ "sha2", "syn 2.0.119", "tempfile", + "toml", "unicode-general-category", + "unicode-segmentation", "uuid", ] @@ -1121,6 +1144,7 @@ source = "git+https://eureka.gamecult.org/forge/GameCult/ModelConnector.git?rev= dependencies = [ "aes-gcm", "base64", + "cultcache-rs 0.3.1 (git+https://github.com/GameCult/CultLib.git?rev=a4eae9056152335662f1115f97bf35fb00ab6d23)", "cultnet-rs 0.2.0", "rand_core 0.6.4", "rmp-serde", @@ -1684,6 +1708,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_spanned" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c187e0a369471c550af028cba56d4f4fa2a489ad28151ae95a4d340a2a8b782" +dependencies = [ + "serde_core", +] + [[package]] name = "serde_stacker" version = "0.1.14" @@ -1922,6 +1955,45 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.9.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow 0.7.15", +] + +[[package]] +name = "toml_datetime" +version = "0.7.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" +dependencies = [ + "winnow 1.0.5", +] + +[[package]] +name = "toml_writer" +version = "1.2.0+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "009dbb7700f11c55ef83aebbe4c0e9cf1f1ac0844bd8d6f00dc38f302026e13a" + [[package]] name = "tracing" version = "0.1.44" @@ -1980,6 +2052,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + [[package]] name = "universal-hash" version = "0.5.1" @@ -2293,6 +2371,18 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" + +[[package]] +name = "winnow" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3b93199df0ab874cdf0c6a94f0a0dc3423fe41f1cafb62b8bd7b1deb277aef" + [[package]] name = "wit-bindgen" version = "0.57.1" diff --git a/crates/mind-body/Cargo.toml b/crates/mind-body/Cargo.toml index daf005d..799096f 100644 --- a/crates/mind-body/Cargo.toml +++ b/crates/mind-body/Cargo.toml @@ -23,6 +23,10 @@ publish.workspace = true # two until the workspace pin and the kit's meet. # `rmp-serde` is also the lease wire (`lease_client`): the connector's MessagePack frames. # +# Ghostlight's `ghostlight-persona-projection`, pinned to a revision, builds the Projector's prompt (`projection`), so +# the membrane's text stays Ghostlight's; the Projector asks through the connector kit's client (`model-connector`). +# `toml` reads `persona-tuning.toml`, the Projector's knobs (`projection::tuning`); parse and serde only. +# # `serde_json` reads Claude Code's stream-json, a foreign owner's wire (`self_run`); # `uuid`'s v5 makes a Self run's session id derivable from the run's id. [dependencies] @@ -33,13 +37,17 @@ cultcache-rs.workspace = true cultnet-rs.workspace = true eureka-pipeline = { path = "../eureka-pipeline" } eureka-state = { path = "../eureka-state", default-features = false } +ghostlight-persona-projection = { git = "https://github.com/GameCult/Ghostlight.git", rev = "c43eb54962ed4a09cc737ecac73a16b0d7ce62b4", package = "ghostlight-persona-projection" } huginn-mind = { path = "../huginn-mind" } model-connector = { git = "https://eureka.gamecult.org/forge/GameCult/ModelConnector.git", rev = "c1802d85bee3ae04b0c5737563c8343c1fef515d", package = "model-connector" } rmp-serde = "1" serde = { version = "1", features = ["derive"] } serde_json = "1" sha2 = "0.10" +toml = { version = "0.9", default-features = false, features = ["std", "serde", "parse"] } unicode-general-category = "1" +# Clips on grapheme boundaries (`projection::select::clip`), so no ZWJ sequence is split; already in the graph through Ghostlight. +unicode-segmentation = "1" uuid = { version = "1", default-features = false, features = ["v5"] } [target.'cfg(unix)'.dependencies] @@ -58,6 +66,9 @@ proc-macro2 = "1" quote = "1" syn = { version = "2", features = ["full"] } tempfile = "3" +# A real kit server (`serve` over a scripted `ProviderBackend`) proves the Projector's connector wire, not a fake of it. +# Same revision as the dependency; the `daemon` feature is test-only. +model-connector = { git = "https://eureka.gamecult.org/forge/GameCult/ModelConnector.git", rev = "c1802d85bee3ae04b0c5737563c8343c1fef515d", package = "model-connector", features = ["daemon"] } [lib] name = "mind_body" diff --git a/crates/mind-body/src/address.rs b/crates/mind-body/src/address.rs index e76c59f..396375b 100644 --- a/crates/mind-body/src/address.rs +++ b/crates/mind-body/src/address.rs @@ -93,19 +93,28 @@ pub(crate) fn project_names(repo: &OrgRepo, listed: &[Short]) -> Vec { names } -/// Whether `text` holds `name` as a whole word, in any case or exactly as written: the characters on either side are not letters or -/// digits. An identity match on a name, not a judgement of meaning. +/// Whether `text` holds `name` as a whole word, in any case or exactly as written. A name is glued, and so not her name, +/// when a letter or digit touches it, or a single hyphen joins it to one (`mind-eureka`, `Eureka-chan`); everything else +/// is a boundary on either side: whitespace, punctuation, `/`, `_`, `.`, `\`, and a double hyphen or a dash. So +/// `/var/lib/mind-eureka/` never names her, and `_Eureka_`, `Eureka/Libby`, `ok...Eureka?` and `Huginn.toml` do. An +/// identity match on a name, not a judgement of meaning. pub(crate) fn names_whole_word(text: &str, name: &str, exact: bool) -> bool { let (text, name) = if exact { (text.to_string(), name.to_string()) } else { (text.to_lowercase(), name.to_lowercase()) }; if name.is_empty() { return false; } // Every character offset, not every non-overlapping match: a name may overlap itself ("ab ab" in "xab ab ab"). - text.char_indices().any(|(start, _)| { - text[start..].starts_with(&name) - && !text[..start].chars().next_back().is_some_and(char::is_alphanumeric) - && !text[start + name.len()..].chars().next().is_some_and(char::is_alphanumeric) - }) + text.char_indices().any(|(start, _)| text[start..].starts_with(&name) && !glued(text[..start].chars().rev()) && !glued(text[start + name.len()..].chars())) +} + +/// Whether the characters leading away from a name, nearest first, glue it to more of a word: a letter or digit, or a +/// single hyphen with a letter or digit beyond it. +fn glued(mut away: impl Iterator) -> bool { + match away.next() { + Some(c) if c.is_alphanumeric() => true, + Some('-') => away.next().is_some_and(char::is_alphanumeric), + _ => false, + } } #[cfg(test)] diff --git a/crates/mind-body/src/address/tests.rs b/crates/mind-body/src/address/tests.rs index accc1de..b1eeace 100644 --- a/crates/mind-body/src/address/tests.rs +++ b/crates/mind-body/src/address/tests.rs @@ -138,3 +138,39 @@ fn a_direct_message_is_for_her_whatever_the_basis() { assert!(!is_for(&nibu, &ChatDelivery { form: Form::Notice, ..dm.clone() })); assert!(!is_for(&nibu, &ChatDelivery { form: Form::Edit, ..dm })); } + +#[test] +fn a_name_glued_to_a_word_is_not_her_name_and_every_other_boundary_is() { + // Soul's forms (third pass on C3a): every one names her except the six glued to a word. + let eureka = Addressee::new("@eureka:chat", ["eureka", "Eureka"]); + let glued = ["Eurekas idea", "Eurekaさん", "ÉEureka", "Eureka-chan", "/var/lib/mind-eureka/x", "mind-eureka"]; + let forms = [ + "Eureka, are you there", "Eureka: look", "eureka?", "Eureka!", "Eureka.", "Eureka's turn", "Eureka’s turn", "Eurekas idea", "the Eureka's", "ask eureka about it", + "thanks eureka", "eureka", "Eureka\n", "`eureka`", "**Eureka**", "*Eureka*", "_Eureka_", "__Eureka__", "~~Eureka~~", "[Eureka](https://matrix.to/#/@eureka:gamecult.org)", + "https://matrix.to/#/@eureka:gamecult.org hi", "@eureka:gamecult.org hi", "@eureka hi", "<@eureka>", "(Eureka)", "\"Eureka\"", "«Eureka»", "—Eureka—", "Eureka—can you", + "Eureka--can you", "Eureka - can you", "Eureka/Libby, can you two", "Libby/Eureka can you", "Eureka/Nibu", "ok...Eureka?", "ok… Eureka?", "Hmm.Eureka?", "Eureka.Can you", + "Eureka...", "Eureka…", "#eureka", "Eurekaさん", "エウレカ Eureka", "Eureka、", "Eureka。", "ÉEureka", "Eureka-chan", "Eureka_bot", "Eureka\\", "\\Eureka", + "Eureka:\n> quoted", "> Eureka hi", "Eureka;", "Eureka'", "'Eureka'", "Eureka(?)", "Eureka+Libby", "Eureka&Libby", "Eureka|Libby", "Eureka=", "Eureka. ", + "/var/lib/mind-eureka/x", "eureka.cc", "eureka_state", "mind-eureka", "eureka.gamecult.org", "https://eureka.gamecult.org/forge", "C:\\eureka\\x", "Eureka.rs", + ]; + assert_eq!(forms.len(), 69, "Soul's whole list"); + for form in forms { + assert_eq!(addresses(&eureka, &said(form, &[])), !glued.contains(&form), "{form:?}"); + } + // Each side of the rule on its own: a hyphen joins only with a letter or digit beyond it, before and after. + for (text, names) in [("x-eureka", false), ("eureka-x", false), ("--eureka", true), ("eureka--", true), ("x--eureka", true), ("eureka--x", true), ("-eureka", true), ("eureka-", true), (".eureka", true), ("eureka.x", true), ("x.eureka", true)] { + assert_eq!(addresses(&eureka, &said(text, &[])), names, "{text:?}"); + } +} + +#[test] +fn a_project_name_holds_at_every_boundary_but_a_glued_one() { + for repo in ["GameCult/Huginn", "GameCult/CultLib", "GameCult/gamecult-ops", "GameCult/cultcache-rs"] { + let steward = steward(repo, &[]); + let short = repo.split_once('/').unwrap().1; + for text in [format!("{repo} PR #76"), format!("{short}/main is red"), format!("_{short}_ is down"), format!("see {short}--now"), format!("{short}.toml"), format!("{short}'s build")] { + assert!(addresses(&steward, &said(&text, &[])), "{text:?}"); + } + assert!(!addresses(&steward, &said(&format!("mind-{short}"), &[])), "{short} glued by a hyphen"); + } +} diff --git a/crates/mind-body/src/bin/mind-persona.rs b/crates/mind-body/src/bin/mind-persona.rs index 71eea4d..92ab948 100644 --- a/crates/mind-body/src/bin/mind-persona.rs +++ b/crates/mind-body/src/bin/mind-persona.rs @@ -12,15 +12,42 @@ //! `persona.env`) are still parsed, so a start refuses the configuration the turn //! will need. Without `--endpoint` the daemon is `HUGINN_ENDPOINT`; no call is made //! to it. +//! +//! `mind-persona project --instance [--endpoint rudp://:] [--kind chat|beat|reflection] [--room ] +//! [--seed N] [--state ] [--select-only | --renders K [--judge ]]` is the Projector's experiment tool +//! (`mind_body::projection::project`): it prints, as JSON, the packet the Select stage builds from her stored state (or +//! `--state`), and with `--renders` the Projector's narrations of K successive turns; `--judge` renders a second state +//! too and asks the model which narration came from which. It writes nothing and reads no brake. It reads +//! `MIND_PERSONA_OPERATOR`, `MIND_PERSONA_MINDS` and `MIND_PERSONA_SELF_SENDER` (required), and the heartbeat's addressing +//! from `MIND_HEARTBEAT_STATE`, which a run by hand must set as the unit does (`/var/lib/mind-heartbeat/heartbeat.cc`): +//! without it "said to her" knows only her own names, never her projects', and stderr says so before the report; the +//! room comes from the mouth when `BIFROST_CHAT_ENDPOINT` +//! and `BIFROST_CHAT_KEY_FILE` are set; the tuning is `MIND_PERSONA_TUNING`, else +//! `/etc/gamecult/minds//persona-tuning.toml`; renders need `MIND_PERSONA_CONNECTOR_ENDPOINT` +//! (`:`) and the `connector-key` credential in `CREDENTIALS_DIRECTORY`. Without `--select-only` or +//! `--renders` it renders once. -use std::path::Path; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::time::Duration; -use anyhow::Result; +use anyhow::{Context, Result, anyhow, bail}; +use eureka_state::HuginnClient; +use mind_body::chat_rudp::{RudpChat, connection_id}; use mind_body::cli::instance_and_endpoint; use mind_body::control::{CONTROL_ROOT, FileSource, control_path, swarm_path}; use mind_body::persona::{Config, carry}; +use mind_body::heartbeat::{FileStore, HeartbeatView}; +use mind_body::projection::{HuginnPersonaSource, KitModel, Model, MouthReader, Organs as ProjectOrgans, Request as ProjectRequest, TUNING_FILE, TUNING_ROOT, TurnKind, connector_key, project}; const USAGE: &str = "usage: mind-persona --instance [--endpoint rudp://:] [--rooms a,b] [--operator id[,id]] [--minds id[,id]] [--self-sender id]"; +const PROJECT_USAGE: &str = "usage: mind-persona project --instance [--endpoint rudp://:] [--kind chat|beat|reflection] [--room ] [--seed N] [--state ] [--select-only | --renders K [--judge ]]"; +/// How long one call to the mind may take. +const TIMEOUT: Duration = Duration::from_secs(15); +/// How long the mouth may take to answer one read. +const CHAT_TIMEOUT: Duration = Duration::from_secs(20); +/// How long the kit may take to answer one Projector or judge turn. +const MODEL_TIMEOUT: Duration = Duration::from_secs(300); // The binary is the one place that reads the process: its arguments and // the environment. The library reads neither (clippy.toml, `#![forbid]` in lib.rs). @@ -39,7 +66,117 @@ fn run() -> Result { Ok(carry(&control, &swarm, &instance).to_string()) } +fn required(name: &str) -> Result { + setting(name).ok_or_else(|| anyhow!("{name} is not set")) +} + +/// The text of the credential file named by `name`; never echoed. +fn credential(name: &str) -> Result { + std::fs::read_to_string(required(name)?).with_context(|| format!("the file named by {name} could not be read")) +} + +fn json_file(path: &str) -> Result { + let text = std::fs::read_to_string(path).with_context(|| format!("{path} could not be read"))?; + serde_json::from_str(&text).with_context(|| format!("{path} is not JSON")) +} + +/// The credentials directory systemd hands the unit; its absence names the variable. +#[allow(clippy::disallowed_methods, reason = "the binary reads its own environment, and hands it to the library")] +fn credentials_directory() -> Result { + std::env::var_os("CREDENTIALS_DIRECTORY").map(PathBuf::from).ok_or_else(|| anyhow!("CREDENTIALS_DIRECTORY is not set")) +} + +fn run_project(args: &[String]) -> Result { + let (mut kind, mut room, mut seed, mut state, mut renders, mut select_only, mut judge, mut rest) = (TurnKind::Beat, None, None, None, None, false, None, Vec::new()); + let mut given = args.iter(); + while let Some(flag) = given.next() { + let mut value = || given.next().cloned().ok_or_else(|| anyhow!("{PROJECT_USAGE}")); + match flag.as_str() { + "--kind" => { + kind = match value()?.as_str() { + "chat" => TurnKind::Chat, + "beat" => TurnKind::Beat, + "reflection" => TurnKind::Reflection, + _ => bail!("{PROJECT_USAGE}"), + } + } + "--room" => room = Some(value()?), + "--seed" => seed = Some(value()?.parse::().map_err(|_| anyhow!("{PROJECT_USAGE}"))?), + "--state" => state = Some(json_file(&value()?)?), + "--renders" => renders = Some(value()?.parse::().ok().filter(|k| *k > 0).ok_or_else(|| anyhow!("{PROJECT_USAGE}"))?), + "--judge" => judge = Some(json_file(&value()?)?), + "--select-only" => select_only = true, + "--instance" | "--endpoint" => rest.extend([flag.clone(), value()?]), + _ => bail!("{PROJECT_USAGE}"), + } + } + if select_only && (renders.is_some() || judge.is_some()) { + bail!("{PROJECT_USAGE}"); + } + let (instance, address) = instance_and_endpoint(&rest, setting("HUGINN_ENDPOINT"), PROJECT_USAGE)?; + let now = chrono::Utc::now(); + let tuning = setting("MIND_PERSONA_TUNING").map_or_else(|| Path::new(TUNING_ROOT).join(&instance.0).join(TUNING_FILE), PathBuf::from); + let persona = HuginnPersonaSource::new(HuginnClient::new(address, instance.clone(), TIMEOUT)); + let chat = match (setting("BIFROST_CHAT_ENDPOINT"), setting("BIFROST_CHAT_KEY_FILE")) { + (Some(endpoint), Some(_)) => { + let endpoint: SocketAddr = endpoint.strip_prefix("rudp://").and_then(|address| address.parse().ok()).ok_or_else(|| anyhow!("BIFROST_CHAT_ENDPOINT is not rudp://:"))?; + Some(RudpChat::new(endpoint, connection_id(&credential("BIFROST_CHAT_KEY_FILE")?), instance.0.clone(), CHAT_TIMEOUT)) + } + _ => None, + }; + let renders = if select_only { None } else { Some(renders.unwrap_or(1)) }; + let model = match renders { + Some(_) => { + let endpoint: SocketAddr = required("MIND_PERSONA_CONNECTOR_ENDPOINT")?.parse().map_err(|_| anyhow!("MIND_PERSONA_CONNECTOR_ENDPOINT is not :"))?; + Some(KitModel::open(endpoint, connector_key(&credentials_directory()?)?, &instance.0, MODEL_TIMEOUT)?) + } + None => None, + }; + let heartbeat = setting("MIND_HEARTBEAT_STATE").map(FileStore::new); + let organs = ProjectOrgans { + persona: Some(&persona), + chat: chat.as_ref().map(|chat| chat as &dyn MouthReader), + addressing: heartbeat.as_ref().map(|store| store as &dyn HeartbeatView), + model: model.as_ref().map(|model| model as &dyn Model), + }; + let request = ProjectRequest { + instance: instance.0.clone(), + kind, + state, + room, + seed: seed.unwrap_or_else(|| u64::try_from(now.timestamp_millis()).unwrap_or_default()), + renders, + judge_against: judge, + operators: setting("MIND_PERSONA_OPERATOR").map(|ids| ids.split(',').map(str::to_owned).collect()).unwrap_or_default(), + minds: setting("MIND_PERSONA_MINDS").map(|ids| ids.split(',').map(str::to_owned).collect()).unwrap_or_default(), + own_sender: required("MIND_PERSONA_SELF_SENDER")?, + tuning, + }; + let report = project(&organs, &request, now)?; + let addressing = &report["addressing"]; + // The addressing source is what decides "said to her": said first, where a person running this by hand sees it. + eprintln!( + "mind-persona project: addressing from {} ({}); table written {}", + addressing["source"].as_str().unwrap_or("?"), + addressing["why"].as_str().unwrap_or("?"), + addressing["age_minutes"].as_i64().map_or_else(|| "never".to_owned(), |minutes| format!("{minutes} min ago")), + ); + Ok(serde_json::to_string_pretty(&report)?) +} + +#[allow(clippy::disallowed_methods, reason = "the binary reads its own arguments")] fn main() { + let args: Vec = std::env::args().skip(1).collect(); + if args.first().map(String::as_str) == Some("project") { + match run_project(&args[1..]) { + Ok(report) => println!("{report}"), + Err(error) => { + eprintln!("mind-persona project: {error:#}"); + std::process::exit(2); + } + } + return; + } match run() { Ok(outcome) => println!("{outcome}"), Err(error) => { diff --git a/crates/mind-body/src/heartbeat.rs b/crates/mind-body/src/heartbeat.rs index 8b146fb..063d216 100644 --- a/crates/mind-body/src/heartbeat.rs +++ b/crates/mind-body/src/heartbeat.rs @@ -220,6 +220,10 @@ pub trait Store { /// What a Persona unit may read of `heartbeat.cc`: the swarm's addressing. Read-only. pub trait HeartbeatView { fn addressing(&self) -> Result>; + /// When the store was last written, so a reader can tell a stale table; none when there is no store at all. + fn written_at(&self) -> Option> { + None + } } /// The store as a CultCache single-file store under the unit's state directory. @@ -315,6 +319,10 @@ impl HeartbeatView for FileStore { fn addressing(&self) -> Result> { self.load_addressing() } + + fn written_at(&self) -> Option> { + std::fs::metadata(&self.path).and_then(|meta| meta.modified()).ok().map(DateTime::::from) + } } /// What the operator stores allow: the swarm brake, and each instance's own diff --git a/crates/mind-body/src/lib.rs b/crates/mind-body/src/lib.rs index 72e5848..66555c9 100644 --- a/crates/mind-body/src/lib.rs +++ b/crates/mind-body/src/lib.rs @@ -15,6 +15,10 @@ //! until the three-stage turn lands it carries no turn. It keeps what a turn's input is made of //! (`chat_client` is the chat wire and port and `chat_rudp` its RUDP transport to Bifrost's mouth). //! +//! `projection` is the Projector of the Persona-only turn: the Select stage chooses what she can feel and the +//! Projector narrates it as her lived stream, under `persona-tuning.toml`; `mind-persona project` runs both and writes +//! nothing. +//! //! `lease_client` asks the Claude connector for a session's credential and the //! plan's usage windows. `self_run` carries one Self run inside `mind-self@`: the //! `mind-self-run` binary runs `claude` over stream-json for her one open Self @@ -40,9 +44,8 @@ pub mod heartbeat; pub mod launch; pub mod lease_client; pub mod persona; +pub mod projection; pub mod queue; -#[allow(dead_code, reason = "kept for the three-stage turn's Select stage (map C3 reuses recall.rs): its first caller removes this allow")] -mod recall; pub mod self_run; pub mod tempo; diff --git a/crates/mind-body/src/projection.rs b/crates/mind-body/src/projection.rs new file mode 100644 index 0000000..303123f --- /dev/null +++ b/crates/mind-body/src/projection.rs @@ -0,0 +1,331 @@ +//! The Projector: her persona state and her room, turned into the lived stream a Persona continues. +//! +//! Two stages. The Select stage (`select`) is code: it decides what she can feel this turn and renders it as a typed +//! projection packet, deterministic for a seed. The narration (`narrate`) is one tool-less model call through the +//! connector that renders the packet as first-person prose under Ghostlight's membrane (`build_projector_prompt`, +//! the pinned `ghostlight-persona-projection`). Every knob of both is `persona-tuning.toml` (`tuning`), read at each +//! wake, its hash carried with the packet. +//! +//! The Projector shows her self and her room: her persona state and what the mouth showed. It does not read her Mind; +//! that is the Interpreter's read tools' (C3b), when wondering leads to reading. +//! +//! `project` is the experiment tool over both (`mind-persona project`): what Select would build from her real or a +//! hypothetical state, and what the Projector would narrate. Its organs are read-only by type: `PersonaSource` gets, +//! `MouthReader` reads pending chat, `Model` asks. None of them can put, admit, send, ack or open a conversation, so +//! `project` cannot write and runs under a held brake. + +mod narrate; +mod select; +mod tuning; + +use std::path::PathBuf; + +use anyhow::{Result, anyhow, bail}; +use chrono::{DateTime, Utc}; +use eureka_state::HuginnClient; +use huginn_mind::HuginnMindResponse; +use serde_json::{Value, json}; + +use crate::address::{Addressee, is_for, localpart}; +use crate::chat_client::{Chat, ChatPending, Form}; +use crate::heartbeat::HeartbeatView; + +pub use narrate::{Answer, Failed, KitModel, Model, Spent, connector_key}; +pub use select::TurnKind; +pub use tuning::{TUNING_FILE, TUNING_ROOT}; + +use narrate::{cap_words, distinct_2, field_names, judge_prompt, judged, projector_prompt, word_budget, words}; +use select::{History, Inputs, Packet, People, Room, RoomLine, select}; + +/// The agent name on the connector's request ids. +const AGENT: &str = "mind-persona"; +/// The judge passes at this share of narrations attributed to the right packet. +const JUDGE_PASS: f64 = 0.8; + +/// Her persona state, read and never put. +pub trait PersonaSource { + /// Her persona state and the receipt it was stored under. + fn persona(&self) -> Result<(Value, String)>; +} + +/// Her persona state from her daemon. +pub struct HuginnPersonaSource { + client: HuginnClient, +} + +impl HuginnPersonaSource { + pub fn new(client: HuginnClient) -> Self { + Self { client } + } +} + +impl PersonaSource for HuginnPersonaSource { + fn persona(&self) -> Result<(Value, String)> { + match self.client.persona_get().map_err(|_| anyhow!("the mind could not be asked for the persona"))? { + HuginnMindResponse::Persona(Some(view)) => Ok((view.value, view.receipt_id)), + HuginnMindResponse::Persona(None) => bail!("the mind holds no persona state"), + _ => bail!("the mind answered the persona request with something else"), + } + } +} + +/// The mouth, read: pending deliveries and the room tail, which acks nothing. The only chat port `project` holds. +pub trait MouthReader { + fn pending(&self) -> Result; +} + +impl MouthReader for T { + fn pending(&self) -> Result { + Chat::pending(self, 0) + } +} + +/// What `project` reads through. Each is a read-only port by type: holding an `Organs`, no code can send, ack or open +/// a conversation, or put her state. +/// +/// ```compile_fail +/// fn acks(organs: &mind_body::projection::Organs) { +/// organs.chat.unwrap().ack(1).unwrap(); +/// } +/// ``` +/// +/// ```compile_fail +/// fn sends(organs: &mind_body::projection::Organs, request: &mind_body::chat_client::ChatSendRequest) { +/// organs.chat.unwrap().send(request).unwrap(); +/// } +/// ``` +pub struct Organs<'a> { + /// Her stored state; not asked when the request carries a state. + pub persona: Option<&'a dyn PersonaSource>, + /// The mouth, for the room; none shows no room. + pub chat: Option<&'a dyn MouthReader>, + /// The heartbeat's addressing table, read-only: her names and her projects' names, the ones the heartbeat woke her + /// by. None, or a table without her, addresses her by her own names only, and the report says so. + pub addressing: Option<&'a dyn HeartbeatView>, + /// The Projector and the judge; required for renders. + pub model: Option<&'a dyn Model>, +} + +/// What to project. +pub struct Request { + pub instance: String, + pub kind: TurnKind, + /// A hypothetical state in place of the stored one. + pub state: Option, + /// The room the turn is in; none is every room of hers. + pub room: Option, + pub seed: u64, + /// None builds the packet only. Some(k) narrates k successive turns of the same state, each seeded `seed + i` and + /// each under the freshness the ones before it leave. + pub renders: Option, + /// A second state: renders both and asks the judge which narration came from which. + pub judge_against: Option, + pub operators: Vec, + /// The Matrix ids of the swarm's Minds: their bonds name them by localpart; anyone else is keyed by full id. + pub minds: Vec, + /// Her own Matrix id. Required: without it her own lines would read as said to her. + pub own_sender: String, + pub tuning: PathBuf, +} + +/// Her addressee: the heartbeat's row for her when its table holds one, else her own names (handle and public name). +/// Her addressee and where it came from: `{source, why, written_at, age_minutes}`. The heartbeat's row when its table +/// holds one; else her own names (handle and public name), with why: no store configured, no heartbeat file, a file +/// that could not be read, or a table without her row. The table's age is the store's last write: under a held swarm +/// brake the heartbeat does not poll, so an old table is a stale one. +fn own_addressee(addressing: Option<&dyn HeartbeatView>, own_sender: &str, public_name: &str, now: DateTime) -> (Addressee, Value) { + let written_at = addressing.and_then(HeartbeatView::written_at); + let age = written_at.map(|at| (now - at).num_minutes()); + let report = |source: &str, why: &str| json!({ "source": source, "why": why, "written_at": written_at.map(|at| at.to_rfc3339()), "age_minutes": age }); + let table = addressing.map(|view| view.addressing()); + if let Some(Ok(Some(table))) = &table + && let Some(row) = table.minds.iter().find(|row| row.matrix_id() == own_sender) + { + return (row.clone(), report("heartbeat", "her row in the heartbeat's addressing table")); + } + let why = match (&table, written_at) { + (None, _) => "no heartbeat addressing is configured (MIND_HEARTBEAT_STATE is not set)".to_owned(), + (Some(Err(fault)), _) => format!("the heartbeat addressing could not be read: {fault:#}"), + (Some(Ok(Some(_))), _) => "the heartbeat's addressing table has no row for her".to_owned(), + (Some(Ok(None)), None) => "there is no heartbeat file".to_owned(), + (Some(Ok(None)), Some(_)) => "the heartbeat file holds no addressing table".to_owned(), + }; + (Addressee::new(own_sender, [localpart(own_sender), public_name]), report("own names only", &why)) +} + +/// The room through the mouth's pending read, which acks nothing: new messages and the tail, own lines kept for the +/// Select stage to drop. A refusal is reported, never hidden. +fn read_room(chat: &dyn MouthReader, request: &Request, own: &Addressee) -> (Room, Value) { + let pending = match chat.pending() { + Ok(pending) => pending, + Err(fault) => return (Room { here: request.room.clone(), ..Room::default() }, json!({ "read": false, "why": format!("{fault:#}") })), + }; + let wanted = |room: &str, id: &str| request.room.as_deref().is_none_or(|here| here == room || here == id); + let messages: Vec = pending + .deliveries + .iter() + .filter(|delivery| delivery.form == Form::Message && wanted(&delivery.room_alias, &delivery.room_id)) + .map(|delivery| RoomLine { + room: delivery.room_alias.clone(), + sender: delivery.sender.clone(), + display: delivery.sender_display.clone(), + text: delivery.text.clone(), + at_ms: delivery.at_unix_ms, + // The heartbeat's rule, the one that woke her: never a second opinion here. + to_her: is_for(own, delivery), + }) + .collect(); + let tail: Vec = pending + .tail + .iter() + .flatten() + .filter(|line| line.form == Form::Message && wanted(&line.room, &line.room)) + .map(|line| RoomLine { room: line.room.clone(), sender: line.sender.clone(), display: String::new(), text: line.body.clone(), at_ms: line.at_unix_ms, to_her: false }) + .collect(); + let report = json!({ "read": true, "messages": messages.len(), "tail": tail.len(), "tail_gaps": pending.tail_gaps }); + (Room { messages, tail, here: request.room.clone() }, report) +} + +fn packet_json(packet: &Packet) -> Value { + json!({ + "kind": packet.kind, + "seed": packet.seed, + "lens": packet.lens, + "present": packet.present, + "selected": packet.selected, + "bytes": { "identity": packet.identity.len(), "context": packet.context.len(), "identity_and_context": packet.identity.len() + packet.context.len(), "stimulus": packet.stimulus.len(), "heard": packet.heard.len() }, + "overflow": packet.overflow, + "identity": packet.identity, + "context": packet.context, + "stimulus": packet.stimulus, + "heard": packet.heard, + }) +} + +/// One narration of one packet: the request id names the instance, the seed and the moment, because the kit's replay +/// store answers a repeated id with its first answer. +/// A failed call is a record with its reason and what it cost, never the end of the report. +fn narrate_one(model: &dyn Model, instance: &str, packet: &Packet, tuning: &tuning::Tuning, tag: &str, now: DateTime) -> Value { + let request_id = format!("{AGENT}:{instance}:project:{tag}:{}:{}", packet.seed, now.timestamp_millis()); + let ids: Vec = packet.selected.iter().map(|pick| pick.id.clone()).collect(); + let answer = match model.ask(&request_id, &tuning.projector.model, &projector_prompt(packet, tuning)) { + Ok(answer) => answer, + Err(failed) => return json!({ "seed": packet.seed, "lens": packet.lens, "selected": ids, "failed": failed.reason, "usage": failed.usage }), + }; + let raw = answer.text; + let budget = word_budget(packet.kind, tuning); + let narration = cap_words(&raw, budget); + let named: Vec = packet.ids.clone(); + json!({ + "seed": packet.seed, + "lens": packet.lens, + "selected": ids, + "words": words(&narration), + "cut_at_budget": words(&raw) > budget, + "field_names": field_names(&narration, &named), + "usage": answer.usage, + "narration": narration, + }) +} + +/// K successive turns over one state: each packet under the freshness the earlier ones left. +fn successive(inputs: &Inputs, tuning: &tuning::Tuning, renders: usize) -> Vec { + let mut history = inputs.history.clone(); + (0..renders) + .map(|at| { + let packet = select(&Inputs { seed: inputs.seed.wrapping_add(at as u64), history: &history, ..*inputs }, tuning); + history.recent.push(packet.selected.iter().map(|pick| pick.id.clone()).collect()); + packet + }) + .collect() +} + +/// What Select builds, and with renders what the Projector narrates, as one JSON report. Writes nothing. +pub fn project(organs: &Organs, request: &Request, now: DateTime) -> Result { + if !request.own_sender.starts_with('@') || !request.own_sender.contains(':') { + bail!("her own Matrix id (MIND_PERSONA_SELF_SENDER) is required: without it her own lines would read as said to her"); + } + let loaded = tuning::load(&request.tuning)?; + let tuning = &loaded.tuning; + let (state, receipt) = match &request.state { + Some(state) => (state.clone(), Value::Null), + None => { + let (state, receipt) = organs.persona.ok_or_else(|| anyhow!("no state was given and her persona cannot be read"))?.persona()?; + (state, Value::String(receipt)) + } + }; + let public_name = state.get("publicName").and_then(Value::as_str).unwrap_or_default().to_owned(); + let (own, addressing) = own_addressee(organs.addressing, &request.own_sender, &public_name, now); + let (room, room_report) = match organs.chat { + Some(chat) => read_room(chat, request, &own), + None => (Room { here: request.room.clone(), ..Room::default() }, json!({ "read": false, "why": "no mouth is configured" })), + }; + // Freshness and her own recent words come from trace v2, which the commit writes (C3c); until then there is none. + let history = History::default(); + let people = People { operators: &request.operators, minds: &request.minds, own_sender: &request.own_sender }; + let inputs = Inputs { kind: request.kind, state: &state, room: &room, history: &history, observation: None, people, now, seed: request.seed }; + + let mut report = json!({ + "instance": request.instance, + "persona_receipt": receipt, + "state": if request.state.is_some() { "given" } else { "stored" }, + "tuning": { "path": request.tuning.display().to_string(), "source": loaded.source, "sha256": loaded.sha256 }, + "room": room_report, + "addressing": addressing, + "history": "none: freshness and her own recent words come from trace v2, which the commit writes (C3c)", + }); + let Some(renders) = request.renders else { + report["packet"] = packet_json(&select(&inputs, tuning)); + return Ok(report); + }; + let model = organs.model.ok_or_else(|| anyhow!("renders need the connector, and none is configured"))?; + report["model"] = json!(tuning.projector.model); + let packets = successive(&inputs, tuning, renders.max(1)); + report["packet"] = packet_json(&packets[0]); + let narrations: Vec = packets.iter().map(|packet| narrate_one(model, &request.instance, packet, tuning, "a", now)).collect(); + let texts = |narrations: &[Value]| narrations.iter().filter_map(|one| one["narration"].as_str().map(str::to_owned)).collect::>(); + let failed = |narrations: &[Value]| narrations.iter().filter(|one| one.get("failed").is_some()).count(); + report["failed_renders"] = json!(failed(&narrations)); + report["distinct_2"] = json!(distinct_2(&texts(&narrations))); + report["embedding_distance"] = json!("not measured: the daemon offers no embedding verb"); + let leaked = narrations.iter().any(|one| one["field_names"].as_array().is_some_and(|names| !names.is_empty())); + report["field_names_found"] = json!(leaked); + + if let Some(other) = &request.judge_against { + let other_inputs = Inputs { state: other, ..inputs }; + let others = successive(&other_inputs, tuning, renders.max(1)); + let other_narrations: Vec = others.iter().map(|packet| narrate_one(model, &request.instance, packet, tuning, "b", now)).collect(); + let mut verdicts = Vec::new(); + let mut right = 0usize; + for (at, (narration, truth)) in narrations.iter().map(|one| (one, 'A')).chain(other_narrations.iter().map(|one| (one, 'B'))).enumerate() { + let (a, b) = (&packets[at % packets.len()], &others[at % others.len()]); + // The two descriptions swap places on odd turns, so a judge that always answers one letter scores half. + let swapped = at % 2 == 1; + let (first, second) = if swapped { (b, a) } else { (a, b) }; + // A narration that failed has nothing to judge; it is counted under the renders' failures. + let Some(text) = narration["narration"].as_str() else { continue }; + let answer = match model.ask(&format!("{AGENT}:{}:judge:{at}:{}", request.instance, now.timestamp_millis()), &tuning.projector.model, &judge_prompt(first, second, text)) { + Ok(answer) => answer, + Err(failed) => { + verdicts.push(json!({ "from": truth.to_string(), "failed": failed.reason, "usage": failed.usage })); + continue; + } + }; + let letter = judged(&answer.text).map(|letter| if swapped { if letter == 'A' { 'B' } else { 'A' } } else { letter }); + let correct = letter == Some(truth); + right += usize::from(correct); + verdicts.push(json!({ "from": truth.to_string(), "judged": letter.map(String::from), "correct": correct, "usage": answer.usage })); + } + // Accuracy over the verdicts the judge gave; a failed call is no verdict, and any failure fails the eval. + let judged_count = verdicts.iter().filter(|verdict| verdict.get("failed").is_none()).count(); + let failures = verdicts.len() - judged_count + failed(&narrations) + failed(&other_narrations); + let accuracy = right as f64 / judged_count.max(1) as f64; + report["against"] = json!({ "packet": packet_json(&others[0]), "renders": other_narrations, "distinct_2": distinct_2(&texts(&other_narrations)) }); + report["judge"] = json!({ "accuracy": accuracy, "pass_at": JUDGE_PASS, "failures": failures, "passed": accuracy >= JUDGE_PASS && !leaked && failures == 0, "verdicts": verdicts }); + } + report["renders"] = Value::Array(narrations); + Ok(report) +} + +#[cfg(test)] +mod tests; diff --git a/crates/mind-body/src/projection/fixtures/golden-chat.txt b/crates/mind-body/src/projection/fixtures/golden-chat.txt new file mode 100644 index 0000000..59ede06 --- /dev/null +++ b/crates/mind-body/src/projection/fixtures/golden-chat.txt @@ -0,0 +1,22 @@ +lens: MemorySurfacing +selected: libby-sounds, t1-m0, curse, need-relatedness +present: Kiko, Libby +--- identity +Eureka (she/her). Dry, direct, almost feral; zeal aimed at revealing. +--- context +- now: Thursday 19 November, 14:05 UTC +- mood, with no earlier turn to measure from: lighter, more settled than middling +- bond [bond-kiko] Kiko, sparring partner: fair trust, heavy friction. Kiko pokes her to stop massaging the pipeline. +- bond [bond-libby] Libby, collaborator: fair trust, a little friction. Libby holds the sounds and teases her about the pipeline. +- memory [libby-sounds]: Libby laughed at the door fix. +- memory [t1-m0]: The heartbeat stalled at noon. +- memory [curse]: Her curse is the knife. The same drive makes her good and will lead her wrong. +- need [need-relatedness]: company, strong +--- stimulus +Said to her: +- Libby in #swarm:gamecult.org: "Eureka, did you see the door fix?" +Also in the room: +- Kiko in #swarm:gamecult.org: "She saw it. She hated it." +--- heard +Libby, in #swarm:gamecult.org: "Eureka, did you see the door fix?" +Kiko, in #swarm:gamecult.org: "She saw it. She hated it." diff --git a/crates/mind-body/src/projection/narrate.rs b/crates/mind-body/src/projection/narrate.rs new file mode 100644 index 0000000..53e76e5 --- /dev/null +++ b/crates/mind-body/src/projection/narrate.rs @@ -0,0 +1,322 @@ +//! The Projector's narration, and the judge that measures it: one tool-less connector call each. +//! +//! `KitModel` is the stream organ's `KitProjector`, copied here when the stream was parked +//! (`persona-only-divergence-parked`), so the Projector has one caller on the connector. + +use std::collections::BTreeSet; +use std::net::SocketAddr; +use std::path::Path; +use std::time::Duration; + +use anyhow::{Result, anyhow, bail}; +use chrono::Utc; +use ghostlight_persona_projection::{ProjectorPrompt, build_projector_prompt}; +use model_connector::{ConnectorClient, InputItem, ProviderRequest, ReasoningEffort, StopReason, TransportDisposition, TransportEventPayload, TransportInvocation, TransportOutcome}; +use serde::Serialize; +use sha2::{Digest, Sha256}; + +use super::select::{Packet, TurnKind}; +use super::tuning::Tuning; + +/// One tool-less model turn. `KitModel` is the one live owner; a test answers with recorded text. +pub trait Model { + /// One turn of `model`: `prompt` is its one user text and `request_id` its identity on the connector. A failed turn + /// keeps whatever usage the connector reported for it. + fn ask(&self, request_id: &str, model: &str, prompt: &str) -> std::result::Result; +} + +/// A turn that produced no answer: why, and what it cost when the connector's receipt says (a refusal, a fault before +/// the provider, or a failure with no usage block costs none). A receipt whose usage block is missing or malformed does +/// not decode at all: that is model-connector's error, reported as the reason. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Failed { + pub reason: String, + pub usage: Option, +} + +impl std::fmt::Display for Failed { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.reason) + } +} + +impl Failed { + fn without_usage(reason: impl Into) -> Self { + Self { reason: reason.into(), usage: None } + } +} + +fn spent(model: &str, usage: &model_connector::Usage) -> Spent { + Spent { model: model.to_owned(), input_tokens: usage.input_tokens, output_tokens: usage.output_tokens, cache_read_tokens: usage.cache_read_tokens, cache_write_tokens: usage.cache_write_tokens, reasoning_tokens: usage.reasoning_tokens } +} + +/// A model's answer, and what the turn cost as the connector counted it: none when the answerer counts nothing. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Answer { + pub text: String, + pub usage: Option, +} + +/// One turn's tokens, by the model that ran it, as the connector's receipt reports them. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct Spent { + pub model: String, + pub input_tokens: u64, + pub output_tokens: u64, + pub cache_read_tokens: u64, + pub cache_write_tokens: u64, + pub reasoning_tokens: Option, +} + +/// What the connector's request asks of the model beside the user text. It names no membrane text: that is Ghostlight's. +pub(crate) const INSTRUCTIONS: &str = "Write only the text the request asks for."; +/// Room for the model's adaptive thinking; the narration itself is bounded by its word budget. +pub(crate) const MAX_OUTPUT_TOKENS: u32 = 1024; +const CONNECTOR_FRAME_BYTES: usize = 256 * 1024; +const INVOCATION_LIFE_MS: u64 = 60_000; + +/// The connector key: the credential file `connector-key` in systemd's credentials directory, read once and trimmed. Its +/// error names the credential and never a byte of the file. +pub fn connector_key(credentials: &Path) -> Result { + let key = std::fs::read_to_string(credentials.join("connector-key")).map_err(|_| anyhow!("the connector-key credential could not be read"))?; + Ok(key.trim().to_owned()) +} + +/// One tool-less connector turn by `caller` (her instance). Loopback only, as the kit is. Each call is its own +/// conversation: nothing is carried from one to the next. The model is the tuning's, named per call. +pub struct KitModel { + client: ConnectorClient, + caller: String, +} + +impl KitModel { + /// `endpoint` is the kit's loopback address; `connection_key` is the caller's key, read by the binary and never printed. + pub fn open(endpoint: SocketAddr, connection_key: String, caller: &str, timeout: Duration) -> Result { + if !endpoint.ip().is_loopback() { + bail!("the connector endpoint is not a loopback address"); + } + let client = ConnectorClient::new(endpoint, connection_key, CONNECTOR_FRAME_BYTES, Some(timeout)).map_err(|_| anyhow!("the connector client could not be configured"))?; + Ok(Self { client, caller: caller.to_owned() }) + } +} + +impl Model for KitModel { + fn ask(&self, request_id: &str, model: &str, prompt: &str) -> std::result::Result { + let mut request = ProviderRequest::new(request_id, request_id, model, INSTRUCTIONS, MAX_OUTPUT_TOKENS); + request.input = vec![InputItem::UserText { text: prompt.to_owned() }]; + request.reasoning_effort = Some(ReasoningEffort::Low); + let expires = u64::try_from(Utc::now().timestamp_millis()).unwrap_or_default() + INVOCATION_LIFE_MS; + let invocation = TransportInvocation::new(&self.caller, expires, Sha256::digest(prompt.as_bytes()).into(), request).map_err(|_| Failed::without_usage("the request is not a valid connector request"))?; + let result = self.client.execute(&invocation).map_err(|fault| Failed::without_usage(format!("the connector did not answer: {fault}")))?; + match result.disposition { + TransportDisposition::Refused(reason) => Err(Failed::without_usage(format!("the connector refused the call: {reason:?}"))), + TransportDisposition::Transported { events, receipt } => match receipt.outcome { + TransportOutcome::Completed { stop_reason: StopReason::EndTurn, usage, .. } => Ok(Answer { + text: events + .into_iter() + .filter_map(|event| match event.payload { + TransportEventPayload::TextDelta { text } => Some(text), + _ => None, + }) + .collect(), + usage: Some(spent(model, &usage)), + }), + TransportOutcome::Completed { stop_reason, usage, .. } => Err(Failed { reason: format!("the model's turn ended early: {stop_reason:?}"), usage: Some(spent(model, &usage)) }), + TransportOutcome::Failed { kind, usage, .. } => Err(Failed { reason: format!("the provider failed the call: {kind:?}"), usage: usage.as_ref().map(|usage| spent(model, usage)) }), + }, + } + } +} + +// --------------------------------------------------------------------------------------------- +// Narration. + +pub(crate) fn word_budget(kind: TurnKind, tuning: &Tuning) -> usize { + if kind == TurnKind::Beat { tuning.projector.beat_words } else { tuning.projector.words } +} + +/// The Projector's prompt: Ghostlight's membrane over the packet, with the lens as where to begin. +pub(crate) fn projector_prompt(packet: &Packet, tuning: &Tuning) -> String { + let guidance = format!("{}\n\nBegin from {}.", tuning.projector.guidance.trim(), packet.lens.words()); + build_projector_prompt(&ProjectorPrompt { identity: &packet.identity, typed_context: &packet.context, visible_stimulus: &packet.stimulus, domain_guidance: &guidance, word_budget: word_budget(packet.kind, tuning) }) +} + +/// A character that is a word on its own: Han, kana and Hangul are written without spaces, so each counts as one. +fn cjk(c: char) -> bool { + matches!(u32::from(c), 0x3040..=0x30FF | 0x3400..=0x4DBF | 0x4E00..=0x9FFF | 0xAC00..=0xD7AF | 0xF900..=0xFAFF | 0x20000..=0x2FFFF) +} + +/// CJK and full-width punctuation (`。`, `,`, `!`, `?`, `「」`): it belongs to the word before it and starts none. +fn cjk_punctuation(c: char) -> bool { + matches!(u32::from(c), 0x3000..=0x303F | 0xFF01..=0xFF0F | 0xFF1A..=0xFF20 | 0xFF3B..=0xFF40 | 0xFF5B..=0xFF65) +} + +/// The byte offset where each word of `text` begins: a run of non-space characters is one word, and a CJK character is +/// a word of its own. +fn word_starts(text: &str) -> Vec { + let mut starts = Vec::new(); + let mut in_word = false; + for (at, c) in text.char_indices() { + if c.is_whitespace() || cjk_punctuation(c) { + in_word = false; + } else if cjk(c) { + starts.push(at); + in_word = false; + } else if !in_word { + starts.push(at); + in_word = true; + } + } + starts +} + +/// The share of the budget a sentence cut must keep; below it the word cut keeps more of the stream. +const SENTENCE_CUT_FLOOR: f64 = 0.6; + +/// `text` held to `max` words (whitespace-separated, each CJK character one); the model's count is not trusted. An +/// overrun is cut at the last sentence end inside the budget, so the stream ends on a whole sentence, when that keeps +/// at least 60% of the budget; otherwise, or with no sentence end inside it, it is cut after the `max`-th word. No +/// abbreviation list: a cut after "e.g." is accepted. +pub(crate) fn cap_words(text: &str, max: usize) -> String { + let text = text.trim(); + let starts = word_starts(text); + let Some(&over) = starts.get(max) else { return text.to_owned() }; + let within = text[..over].trim_end(); + match last_sentence_end(within) { + Some(end) if words(&within[..end]) as f64 >= SENTENCE_CUT_FLOOR * max as f64 => within[..end].to_owned(), + _ => within.to_owned(), + } +} + +/// The byte just past the last sentence end in `text`: a `.`, `!`, `?` or `…` with any closing quotes or brackets after +/// it, followed by whitespace or the end of the text; or a `。`, `!` or `?`, which ends a sentence where it stands. +fn last_sentence_end(text: &str) -> Option { + let chars: Vec<(usize, char)> = text.char_indices().collect(); + let mut found = None; + for (index, (at, c)) in chars.iter().enumerate() { + let full_width = matches!(c, '。' | '!' | '?'); + if !full_width && !matches!(c, '.' | '!' | '?' | '…') { + continue; + } + let mut end = at + c.len_utf8(); + let mut next = index + 1; + while let Some((close_at, close)) = chars.get(next).filter(|(_, close)| matches!(close, '"' | '\'' | '”' | '’' | ')' | ']' | '」' | '』')) { + end = close_at + close.len_utf8(); + next += 1; + } + if full_width || chars.get(next).is_none_or(|(_, after)| after.is_whitespace()) { + found = Some(end); + } + } + found +} + +/// Words as the cap counts them. +pub(crate) fn words(text: &str) -> usize { + word_starts(text).len() +} + +/// The one kebab form that is ordinary English: "short-term" folds to the schema key `shortTerm`, and prose must not fail +/// the judge for it. `shortTerm`, `short_term` and the run-together `shortterm` are still caught. +const KEBAB_ENGLISH: &str = "short-term"; + +/// Her persona state's schema: its property names are the identifiers a narration must never carry. +const PERSONA_SCHEMA: &str = include_str!("../../../../schemas/cultnet/gamecult.persona_state.v0.schema.json"); + +/// An identifier as compared: lowercased with its underscores and hyphens gone, so `socialBonds`, `SocialBonds`, +/// `social_bonds`, `social-bonds`, `SOCIAL_BONDS` and `socialbonds` are one. +fn folded(identifier: &str) -> String { + identifier.chars().filter(|c| !matches!(c, '_' | '-')).flat_map(char::to_lowercase).collect() +} + +/// Whether a word is shaped like an identifier rather than prose: an underscore, or a capital after its first letter. +fn identifier_shaped(word: &str) -> bool { + word.contains('_') || word.chars().skip(1).any(char::is_uppercase) +} + +fn keys_of(value: &serde_json::Value, schema: bool, into: &mut BTreeSet) { + match value { + serde_json::Value::Object(map) => { + for (key, inner) in map { + if schema && key == "properties" + && let Some(properties) = inner.as_object() + { + into.extend(properties.keys().cloned()); + } + if !schema { + into.insert(key.clone()); + } + keys_of(inner, schema, into); + } + } + serde_json::Value::Array(values) => values.iter().for_each(|inner| keys_of(inner, schema, into)), + _ => {} + } +} + +/// The folded compound identifiers of the persona schema and of the tuning: a plain word such as `trust` or +/// `tension` is prose and is not among them. +fn identifiers() -> BTreeSet { + let mut keys = BTreeSet::new(); + if let Ok(schema) = serde_json::from_str::(PERSONA_SCHEMA) { + keys_of(&schema, true, &mut keys); + } + if let Ok(tuning) = serde_json::to_value(Tuning::default()) { + keys_of(&tuning, false, &mut keys); + } + keys.into_iter().filter(|key| identifier_shaped(key)).map(|key| folded(&key)).collect() +} + +/// The identifiers of typed state a narration carries: any token that folds to a compound property name of her persona +/// schema or a tuning key, in every case convention (camelCase, PascalCase, snake_case, SCREAMING_CASE, kebab-case and +/// run together), and any id the packet named (`ids`), as an exact token when it cannot be a plain word, or inside +/// brackets as the packet writes it. Only compound names are known, so a plain word such as `trust`, `tension` or +/// `values` is prose: "need-to-know", "trust:" and "Tension:" pass. A small deterministic guard on identifiers, never on +/// meaning. +pub(crate) fn field_names(narration: &str, ids: &[String]) -> Vec { + let known = identifiers(); + let mut found = BTreeSet::new(); + for token in narration.split(|c: char| !(c.is_alphanumeric() || c == '_' || c == '-')).map(|token| token.trim_matches('-')).filter(|token| !token.is_empty()) { + if known.contains(&folded(token)) && !token.eq_ignore_ascii_case(KEBAB_ENGLISH) { + found.insert(token.to_owned()); + } + } + let tokens: BTreeSet<&str> = narration.split_whitespace().map(|token| token.trim_matches(|c: char| !(c.is_alphanumeric() || c == '-' || c == '_' || c == ':'))).collect(); + for id in ids { + let compound = id.contains(['-', '_', ':']) || id.chars().any(|c| c.is_ascii_digit()); + if (compound && tokens.contains(id.as_str())) || narration.contains(&format!("[{id}]")) { + found.insert(id.clone()); + } + } + found.into_iter().collect() +} + +// --------------------------------------------------------------------------------------------- +// The eval: distinct-2 and the judge. + +/// Distinct bigrams over all bigrams of the narrations, lowercased words: 1 is no repetition. +pub(crate) fn distinct_2(narrations: &[String]) -> f64 { + let mut all = 0usize; + let mut distinct = BTreeSet::new(); + for narration in narrations { + let words: Vec = narration.split_whitespace().map(|word| word.trim_matches(|c: char| !c.is_alphanumeric()).to_lowercase()).filter(|word| !word.is_empty()).collect(); + for pair in words.windows(2) { + all += 1; + distinct.insert((pair[0].clone(), pair[1].clone())); + } + } + if all == 0 { 0.0 } else { distinct.len() as f64 / all as f64 } +} + +/// The judge's question: which of two packets a narration was written from. +pub(crate) fn judge_prompt(first: &Packet, second: &Packet, narration: &str) -> String { + format!( + "Two descriptions, A and B, of one person's present moment differ in what she carries. Below them is a first-person passage written from exactly one of them. Answer with the single letter of the description the passage was written from: A or B.\n\nA:\n{}\n{}\n\nB:\n{}\n{}\n\nPassage:\n{narration}\n\nAnswer A or B.", + first.context, first.stimulus, second.context, second.stimulus + ) +} + +/// The judge's letter, or None when its answer does not start with one. +pub(crate) fn judged(answer: &str) -> Option { + answer.trim().trim_start_matches(['*', '"', '\'', '(']).chars().next().map(|c| c.to_ascii_uppercase()).filter(|c| *c == 'A' || *c == 'B') +} diff --git a/crates/mind-body/src/projection/select.rs b/crates/mind-body/src/projection/select.rs new file mode 100644 index 0000000..3436b7d --- /dev/null +++ b/crates/mind-body/src/projection/select.rs @@ -0,0 +1,894 @@ +//! The Select stage: what she can feel this turn. Code, deterministic for a seed, and the one owner of the +//! projection packet; the Projector model only renders what this chose. +//! +//! The Projector shows her self and her room: her persona state and what the mouth showed. Her Mind is not read +//! here; reaching into it is the Interpreter's, through its read tools, when wondering leads to reading. +//! +//! Every packet carries its fixed slots: identity, the hour, her mood as change, a bond line for each person +//! present, and the stimulus. The rest comes from her state's pools. Each candidate is ordered by +//! `log(score) / T` plus Gumbel noise and included with probability `1 - exp(-score / inclusion_score)`, where +//! `score = salience × recency × touch × relevance × unresolved × freshness`; then each pool's cap and the context +//! budget bound what is kept. A cap is a maximum, never a target: a pool with fewer candidates than its cap still +//! rotates. Recency decays only the short-term items: a crystallized memory, a value or a stance is as near as it ever +//! was, and comes forward by what the room is touching (`relevance`, a lexical overlap with the stimulus). +//! +//! Who a message is to is `address::is_for`'s, the rule the heartbeat woke her by, and nothing else here. People are +//! keyed by their full Matrix id; only the operator and the Minds of the swarm are keyed by the names their bonds use, +//! and an unbonded speaker is shown by the display name the mouth reports, quoted and escaped, with the Matrix id after +//! it, never under a bonded person's name. Every message body and display name is rendered by one escape (`spoken`, a +//! JSON string), so no text can forge a line; speaker and room come from the mouth's metadata alone. +//! +//! Never read: `agencyPressure`, `candidateActions`, `privateNotes` and `provenance`. A memory tagged with a direct +//! message room (`room:!…`) is selectable only in that room. + +use std::collections::BTreeSet; +use std::fmt::Write as _; + +use chrono::{DateTime, Utc}; +use serde::Serialize; +use serde_json::Value; +use unicode_segmentation::UnicodeSegmentation; + +use super::tuning::{LensWeights, Tuning}; +use crate::address::localpart; + +/// Touch: what the room is touching. +const TOUCH_PRESENT: f64 = 1.0; +const TOUCH_ROOM: f64 = 0.6; +const TOUCH_ELSE: f64 = 0.3; +/// The salience of an item that carries none. +const SALIENCE_UNSTATED: f64 = 0.5; +/// A move of mood or need below this is no move. +const MOVE_MIN: f64 = 0.03; +/// Incubation's unresolved factor caps here. +const INCUBATION_UNRESOLVED_MAX: f64 = 2.0; +/// The most bytes a bond's name or kind takes in its line. +const NAME_BYTES: usize = 80; +/// The most bytes the line naming the unbonded people present takes. +const ALSO_HERE_BYTES: usize = 300; + +/// What kind of turn the packet is for. The stimulus and nothing else differs between them. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum TurnKind { + Chat, + Beat, + FollowUp, + Reflection, +} + +/// One line of a room: a message or a line of its tail. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct RoomLine { + pub room: String, + pub sender: String, + pub display: String, + pub text: String, + pub at_ms: u64, + pub to_her: bool, +} + +/// The room as the mouth showed it: new messages, and the recent tail. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct Room { + pub messages: Vec, + pub tail: Vec, + /// The room the turn is in, when it is one room. + pub here: Option, +} + +/// One thing she said, from the trace. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Said { + pub id: String, + pub room: String, + pub text: String, + pub at: DateTime, +} + +/// What earlier turns left: the ids each of the last projections selected, oldest first, what she said, and her state +/// at her last turn (movement is measured against it). The commit's trace v2 (C3c) is its source; until then it is +/// empty. +#[derive(Clone, Debug, Default, PartialEq)] +pub(crate) struct History { + pub recent: Vec>, + pub said: Vec, + pub previous: Option, +} + +/// Who the people of a room are to her. +#[derive(Clone, Copy)] +pub(crate) struct People<'a> { + pub operators: &'a [String], + /// The Matrix ids of the swarm's Minds: their bonds name them by localpart. + pub minds: &'a [String], + pub own_sender: &'a str, +} + +impl People<'_> { + /// The id a bond names a person by: `operator` for the operator, a Mind's localpart for a Mind of the swarm, and the + /// full Matrix id for anyone else, so `@libby:evil.example` is never Libby. + pub(crate) fn person_of(&self, sender: &str) -> String { + if self.operators.iter().any(|operator| operator == sender) { + "operator".into() + } else if self.minds.iter().any(|mind| mind == sender) { + localpart(sender).to_lowercase() + } else { + sender.to_lowercase() + } + } +} + +pub(crate) struct Inputs<'a> { + pub kind: TurnKind, + pub state: &'a Value, + pub room: &'a Room, + pub history: &'a History, + /// A follow-up's finding. + pub observation: Option<&'a str>, + pub people: People<'a>, + pub now: DateTime, + pub seed: u64, +} + +/// Where a turn begins. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum Lens { + BodyAndHour, + PersonPresent(String), + UnresolvedWant, + MemorySurfacing, + Wondering, +} + +impl Lens { + /// The lens as the domain guidance's "begin from …". + pub(crate) fn words(&self) -> String { + match self { + Self::BodyAndHour => "how she is, here, at this hour".into(), + Self::PersonPresent(name) => format!("{name}, who is here"), + Self::UnresolvedWant => "something she still wants and has not got".into(), + Self::MemorySurfacing => "a memory coming back to her".into(), + Self::Wondering => "something she has been wondering about".into(), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum Pool { + MemoriesAboutPresent, + OtherMemories, + Needs, + Incubation, + Stance, + Value, + Said, +} + +/// One selected item: its id goes to the trace, never to the narration. +#[derive(Clone, Debug, PartialEq, Serialize)] +pub(crate) struct Pick { + pub id: String, + pub pool: Pool, + /// `ln(score)`. + pub log_score: f64, +} + +/// The projection packet: what the Projector is shown, and what chose it. +#[derive(Clone, Debug, PartialEq, Serialize)] +pub(crate) struct Packet { + pub kind: TurnKind, + pub seed: u64, + pub lens: Lens, + pub identity: String, + pub context: String, + pub stimulus: String, + /// What was said to her, verbatim, rendered by code for the Persona stage. + pub heard: String, + /// Bytes the messages to her took past the stimulus and heard caps: verbatim beats the cap. + pub overflow: Overflow, + pub present: Vec, + /// Every typed id the packet's lines name: the selected items' and the bonds' of the people present. + pub ids: Vec, + pub selected: Vec, +} + +/// What the messages to her cost past the caps: bytes over the stimulus and heard caps, the older messages to her the +/// `addressed_bytes` ceiling left out, and the bytes clipped from a single message larger than the ceiling. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize)] +pub(crate) struct Overflow { + pub stimulus: usize, + pub heard: usize, + pub said_to_her_not_shown: usize, + pub clipped_bytes: usize, +} + +struct Candidate { + id: String, + pool: Pool, + line: String, + log_score: f64, +} + +// --------------------------------------------------------------------------------------------- +// Reading her state. + +fn text<'a>(value: &'a Value, path: &[&str]) -> &'a str { + path.iter().try_fold(value, |at, key| at.get(*key)).and_then(Value::as_str).unwrap_or_default().trim() +} + +fn number(value: &Value, path: &[&str]) -> Option { + path.iter().try_fold(value, |at, key| at.get(*key)).and_then(Value::as_f64) +} + +fn items<'a>(value: &'a Value, path: &[&str]) -> &'a [Value] { + path.iter().try_fold(value, |at, key| at.get(*key)).and_then(Value::as_array).map_or(&[], Vec::as_slice) +} + +fn live(item: &Value) -> bool { + !matches!(text(item, &["status"]), "retired" | "cooled" | "resolved") +} + +fn time(item: &Value) -> Option> { + ["updatedAt", "createdAt"].iter().find_map(|key| DateTime::parse_from_rfc3339(text(item, &[key])).ok()).map(|at| at.with_timezone(&Utc)) +} + +/// `text` cut to at most `max` bytes on a grapheme boundary, with an ellipsis where it was cut. +pub(crate) fn clip(text: &str, max: usize) -> String { + let text = text.trim(); + if text.len() <= max { + return text.to_owned(); + } + let room = max.saturating_sub('…'.len_utf8()); + let end = text.grapheme_indices(true).map(|(at, grapheme)| at + grapheme.len()).take_while(|end| *end <= room).last().unwrap_or(0); + format!("{}…", &text[..end]) +} + +fn one_line(text: &str) -> String { + text.split_whitespace().collect::>().join(" ") +} + +/// The words of a text that can carry a topic: four letters or more, lowercased. +fn content_words(text: &str) -> BTreeSet { + text.split(|c: char| !c.is_alphanumeric()).filter(|word| word.chars().count() >= 4).map(str::to_lowercase).collect() +} + +/// `ln(1 + min(1, shared / 3))`: how much of an item the room is touching, by the content words it shares with the +/// stimulus. Lexical and cheap; no embedder. +fn relevance_log(item_text: &str, touching: &BTreeSet) -> f64 { + let shared = content_words(item_text).intersection(touching).count(); + (1.0 + (shared as f64 / 3.0).min(1.0)).ln() +} + +/// A message body or a display name as every line renders it: a JSON string, quotes, backslashes and every control +/// character escaped (`\n`, `\t`, `\r`, `\u…`, and the Unicode line separators too), so no text can end its line or +/// start another, and the escape is reversible: nothing is dropped. +pub(crate) fn spoken(text: &str) -> String { + crate::persona::quoted(text) +} + +// --------------------------------------------------------------------------------------------- +// Numbers as words. + +fn trust_words(trust: f64) -> &'static str { + match trust { + t if t < 0.2 => "little trust", + t if t < 0.45 => "guarded trust", + t if t < 0.7 => "fair trust", + t if t < 0.85 => "solid trust", + _ => "deep trust", + } +} + +fn friction_words(tension: f64) -> &'static str { + match tension { + t if t < 0.05 => "easy between them", + t if t < 0.25 => "a little friction", + t if t < 0.5 => "real friction", + _ => "heavy friction", + } +} + +fn strength_words(intensity: f64) -> &'static str { + match intensity { + i if i < 0.25 => "faint", + i if i < 0.5 => "present", + i if i < 0.75 => "strong", + _ => "pressing", + } +} + +/// The three mood axes: name, the word for higher, the word for lower. +const MOOD_AXES: [(&str, &str, &str); 3] = [("valence", "lighter", "heavier"), ("arousal", "more keyed up", "more settled"), ("dominance", "more in command of things", "more at their mercy")]; + +fn activation(state: &Value, axis: &str) -> Option { + number(state, &["activationProfile", "situationalState", axis, "currentActivation"]) +} + +/// Mood as change since her last turn; as a level only when there is no last turn to measure from. +fn mood_line(state: &Value, previous: Option<&Value>) -> String { + let now: Vec<(usize, f64)> = MOOD_AXES.iter().enumerate().filter_map(|(at, (axis, ..))| activation(state, axis).map(|value| (at, value))).collect(); + if now.is_empty() { + return "- mood: none recorded".into(); + } + let Some(previous) = previous else { + let words: Vec<&str> = now + .iter() + .map(|(at, value)| { + let (_, up, down) = MOOD_AXES[*at]; + if *value >= 0.5 { up } else { down } + }) + .collect(); + return format!("- mood, with no earlier turn to measure from: {} than middling", words.join(", ")); + }; + let moves: Vec<&str> = now + .iter() + .filter_map(|(at, value)| { + let (axis, up, down) = MOOD_AXES[*at]; + let before = activation(previous, axis)?; + match value - before { + change if change >= MOVE_MIN => Some(up), + change if change <= -MOVE_MIN => Some(down), + _ => None, + } + }) + .collect(); + if moves.is_empty() { "- mood: where it was at her last turn".into() } else { format!("- mood: {} than at her last turn", moves.join(", ")) } +} + +fn need_movement(need: &Value, previous: Option<&Value>) -> String { + let intensity = number(need, &["intensity"]).unwrap_or(SALIENCE_UNSTATED); + let before = previous.and_then(|previous| items(previous, &["affect", "needs"]).iter().find(|old| text(old, &["id"]) == text(need, &["id"])).and_then(|old| number(old, &["intensity"]))); + match before.map(|before| intensity - before) { + Some(change) if change >= MOVE_MIN => "growing since her last turn".into(), + Some(change) if change <= -MOVE_MIN => "easing since her last turn".into(), + Some(_) => "where it was".into(), + None => strength_words(intensity).into(), + } +} + +// --------------------------------------------------------------------------------------------- +// The seeded draw: splitmix64, so a turn id replays. + +struct Draw(u64); + +impl Draw { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + /// Uniform in (0, 1). + fn unit(&mut self) -> f64 { + ((self.next() >> 11) as f64 + 0.5) / (1u64 << 53) as f64 + } + + fn gumbel(&mut self) -> f64 { + -(-self.unit().ln()).ln() + } +} + +fn draw_lens(weights: &LensWeights, present: &[String], candidates: &[Candidate], draw: &mut Draw) -> Lens { + let has = |pools: &[Pool]| candidates.iter().any(|candidate| pools.contains(&candidate.pool)); + let options = [ + (weights.body_and_hour, Lens::BodyAndHour), + (if present.is_empty() { 0.0 } else { weights.person_present }, Lens::PersonPresent(present.first().cloned().unwrap_or_default())), + (if has(&[Pool::Needs, Pool::Incubation]) { weights.unresolved_want } else { 0.0 }, Lens::UnresolvedWant), + (if has(&[Pool::MemoriesAboutPresent, Pool::OtherMemories]) { weights.memory_surfacing } else { 0.0 }, Lens::MemorySurfacing), + (if has(&[Pool::Incubation]) { weights.wondering } else { 0.0 }, Lens::Wondering), + ]; + let total: f64 = options.iter().map(|(weight, _)| weight.max(0.0)).sum(); + let mut at = draw.unit() * total; + for (weight, lens) in &options { + let weight = weight.max(0.0); + if weight > 0.0 && at < weight { + return lens.clone(); + } + at -= weight; + } + Lens::BodyAndHour +} + +// --------------------------------------------------------------------------------------------- +// Candidates. + +/// `ln(base ^ k)`, k the times `id` was shown in the last `freshness_window` projections: 0 when never shown, minus +/// infinity when the base is 0 and it was shown at all. +fn freshness_log(id: &str, history: &History, tuning: &Tuning) -> f64 { + let window = &history.recent[history.recent.len().saturating_sub(tuning.select.freshness_window)..]; + let shown = window.iter().filter(|ids| ids.iter().any(|seen| seen == id)).count(); + if shown == 0 { 0.0 } else { tuning.select.freshness_base.powi(i32::try_from(shown).unwrap_or(i32::MAX)).ln() } +} + +fn recency_log(at: Option>, now: DateTime, tau_h: f64) -> f64 { + at.map_or(0.0, |at| -((now - at).num_seconds().max(0) as f64 / 3600.0) / tau_h) +} + +/// The memory's tags name a direct-message room other than the turn's: private matter stays in its room. +fn private_elsewhere(item: &Value, here: Option<&str>) -> bool { + items(item, &["tags"]).iter().filter_map(Value::as_str).filter_map(|tag| tag.strip_prefix("room:")).any(|room| room.starts_with('!') && Some(room) != here) +} + +fn in_room(item: &Value, here: Option<&str>) -> bool { + here.is_some_and(|here| items(item, &["tags"]).iter().filter_map(Value::as_str).any(|tag| tag.strip_prefix("room:") == Some(here))) +} + +fn about_present(item: &Value, present: &BTreeSet) -> bool { + let target = text(item, &["target", "id"]).to_lowercase(); + present.contains(&target) || items(item, &["tags"]).iter().filter_map(Value::as_str).filter_map(|tag| tag.strip_prefix("about:")).any(|id| present.contains(&id.to_lowercase())) +} + +fn candidates(inputs: &Inputs, tuning: &Tuning, present: &BTreeSet, touching: &BTreeSet) -> Vec { + let state = inputs.state; + let here = inputs.room.here.as_deref(); + let item_bytes = tuning.select.item_bytes; + let mut out = Vec::new(); + let mut push = |id: &str, pool: Pool, line: String, salience: f64, recency: f64, touch: f64, unresolved: f64| { + if id.is_empty() || salience <= 0.0 { + return; + } + let log_score = salience.ln() + recency + touch.ln() + unresolved.ln() + freshness_log(id, inputs.history, tuning); + out.push(Candidate { id: id.to_owned(), pool, line, log_score }); + }; + + let crystallized = items(state, &["thoughtMemory", "memories"]).iter().map(|memory| (memory, false)); + let memories = crystallized.chain(items(state, &["thoughtMemory", "shortTerm"]).iter().map(|memory| (memory, true))); + for (memory, short_term) in memories.filter(|(memory, _)| live(memory) && !private_elsewhere(memory, here)) { + let about = about_present(memory, present); + let pool = if about { Pool::MemoriesAboutPresent } else { Pool::OtherMemories }; + let touch = if about { TOUCH_PRESENT } else if in_room(memory, here) { TOUCH_ROOM } else { TOUCH_ELSE }; + let mut body = text(memory, &["summary"]).to_owned(); + let tension = text(memory, &["tension"]); + if !tension.is_empty() { + body = format!("{body} {tension}"); + } + let line = format!("- memory [{}]: {}", text(memory, &["id"]), clip(&one_line(&body), item_bytes)); + let salience = number(memory, &["intensity"]).unwrap_or(SALIENCE_UNSTATED); + // Only the day's short-term items fade; a crystallized memory comes forward by what the room touches. + let nearness = if short_term { recency_log(time(memory), inputs.now, tuning.select.memory_tau_h) } else { relevance_log(&body, touching) }; + push(text(memory, &["id"]), pool, line, salience, nearness, touch, 1.0); + } + + for need in items(state, &["affect", "needs"]).iter().filter(|need| live(need)) { + let intensity = number(need, &["intensity"]).unwrap_or(SALIENCE_UNSTATED); + let name = [text(need, &["label"]), text(need, &["summary"]), text(need, &["id"])].into_iter().find(|name| !name.is_empty()).unwrap_or_default(); + let line = format!("- need [{}]: {}, {}", text(need, &["id"]), clip(&one_line(name), item_bytes), need_movement(need, inputs.history.previous.as_ref())); + push(text(need, &["id"]), Pool::Needs, line, intensity, 0.0, TOUCH_ELSE, 1.0 + intensity); + } + + for wondering in items(state, &["thoughtMemory", "incubation"]).iter().filter(|item| live(item) && !private_elsewhere(item, here)) { + let days = time(wondering).map_or(0.0, |at| (inputs.now - at).num_seconds().max(0) as f64 / 86_400.0); + let touch = if about_present(wondering, present) { TOUCH_PRESENT } else { TOUCH_ELSE }; + let line = format!("- wondering [{}]: {}", text(wondering, &["id"]), clip(&one_line(text(wondering, &["summary"])), item_bytes)); + push(text(wondering, &["id"]), Pool::Incubation, line, number(wondering, &["intensity"]).unwrap_or(SALIENCE_UNSTATED), 0.0, touch, (1.0 + days / 7.0).min(INCUBATION_UNRESOLVED_MAX)); + } + + for stance in items(state, &["affect", "doctrineStances"]).iter().filter(|stance| live(stance)) { + let touch = if about_present(stance, present) { TOUCH_PRESENT } else { TOUCH_ELSE }; + let line = format!("- stance [{}]: {}", text(stance, &["id"]), clip(&one_line(text(stance, &["summary"])), item_bytes)); + push(text(stance, &["id"]), Pool::Stance, line, number(stance, &["intensity"]).unwrap_or(SALIENCE_UNSTATED), relevance_log(text(stance, &["summary"]), touching), touch, 1.0); + } + + for value in items(state, &["values"]) { + let line = format!("- value [{}] {}: {}", text(value, &["id"]), clip(text(value, &["label"]), NAME_BYTES), clip(&one_line(text(value, &["summary"])), item_bytes)); + let relevance = relevance_log(&format!("{} {}", text(value, &["label"]), text(value, &["summary"])), touching); + push(text(value, &["id"]), Pool::Value, line, number(value, &["priority"]).unwrap_or(SALIENCE_UNSTATED), relevance, TOUCH_ELSE, 1.0); + } + + for said in &inputs.history.said { + let touch = if Some(said.room.as_str()) == here { TOUCH_ROOM } else { TOUCH_ELSE }; + let line = format!("- she said [{}] in {}: {}", said.id, said.room, spoken(&clip(&said.text, item_bytes))); + push(&said.id, Pool::Said, line, SALIENCE_UNSTATED, recency_log(Some(said.at), inputs.now, tuning.select.memory_tau_h), touch, 1.0); + } + out +} + +fn cap_of(pool: Pool, tuning: &Tuning) -> usize { + let caps = &tuning.select.caps; + match pool { + Pool::MemoriesAboutPresent => caps.memories_about_present, + Pool::OtherMemories => caps.other_memories, + Pool::Needs => caps.needs, + Pool::Incubation => caps.incubation, + Pool::Stance => caps.stance, + Pool::Value => caps.value, + Pool::Said => caps.said, + } +} + +// --------------------------------------------------------------------------------------------- +// The stimulus, the people present and the fixed slots. + +/// How a speaker reads: a bonded person by the bond's label; anyone else by their display name with the Matrix id beside +/// it, so a stranger called "Meta" never reads as the operator. +fn name_of(line: &RoomLine, inputs: &Inputs) -> String { + let person = inputs.people.person_of(&line.sender); + let bonded = items(inputs.state, &["affect", "socialBonds"]).iter().find(|bond| live(bond) && text(bond, &["object", "id"]).to_lowercase() == person).map(|bond| text(bond, &["object", "label"]).to_owned()).filter(|label| !label.is_empty()); + match bonded { + Some(label) => clip(&one_line(&label), NAME_BYTES), + // A display name is the speaker's own text: quoted and escaped, at most `NAME_BYTES`, the real id always after it. + None if !line.display.trim().is_empty() => format!("{} ({})", spoken(&clip(line.display.trim(), NAME_BYTES)), line.sender), + None => line.sender.clone(), + } +} + +/// Lines appended under a byte cap: a line that would pass it is refused whole. +struct Lines { + out: String, + cap: usize, +} + +impl Lines { + fn new(cap: usize) -> Self { + Self { out: String::new(), cap } + } + + fn fits(&self, line: &str) -> bool { + self.out.len() + line.len() + usize::from(!self.out.is_empty()) <= self.cap + } + + fn push(&mut self, line: &str) -> bool { + if !self.fits(line) { + return false; + } + if !self.out.is_empty() { + self.out.push('\n'); + } + self.out.push_str(line); + true + } +} + +/// A section of room lines under `cap` bytes: its heading, a note counting the older lines that did not fit, then the +/// newest whole lines that did. Empty when there are no lines, or not even the heading fits. +fn newest_section(heading: &str, lines: &[String], cap: usize) -> String { + if lines.is_empty() { + return String::new(); + } + // The note is at most this long: "(NNNNN earlier not shown)". + let note_room = 32; + let mut kept: Vec<&String> = Vec::new(); + let mut used = heading.len() + note_room; + for line in lines.iter().rev() { + if used + line.len() + 1 > cap { + break; + } + used += line.len() + 1; + kept.push(line); + } + kept.reverse(); + let dropped = lines.len() - kept.len(); + let mut out = Lines::new(cap); + if !out.push(heading) { + return String::new(); + } + let mut shown = false; + if dropped > 0 { + shown |= out.push(&format!("({dropped} earlier not shown)")); + } + for line in kept { + shown |= out.push(line); + } + // A heading over nothing says nothing. + if shown { out.out } else { String::new() } +} + +/// Which messages to her a turn shows: whole messages, newest first, until the `addressed_bytes` ceiling, and a single +/// message larger than the ceiling clipped on a grapheme boundary (`cut`, the bytes of its text kept). Decided once, on +/// the stimulus's lines, and rendered for the stimulus and the heard lines alike. +struct Addressed<'a> { + /// Chronological: each message shown, and where its text is cut if it is. + shown: Vec<(&'a RoomLine, Option)>, + left_out: usize, + clipped: usize, +} + +impl Addressed<'_> { + fn lines(&self, render: impl Fn(&RoomLine, &str) -> String) -> Vec { + let mut lines: Vec = Vec::new(); + if self.left_out > 0 { + lines.push(format!("({} more said to her, not shown)", self.left_out)); + } + lines.extend(self.shown.iter().map(|(line, cut)| match cut { + None => render(line, &spoken(&line.text)), + Some(cut) => render(line, &format!("{} [… {} bytes more]", spoken(&line.text[..*cut]), line.text.len() - cut)), + })); + lines + } +} + +fn said_to_her<'a>(to_her: &[&'a RoomLine], render: impl Fn(&RoomLine, &str) -> String, ceiling: usize) -> Addressed<'a> { + let mut shown: Vec<(&RoomLine, Option)> = Vec::new(); + let mut used = 0; + let mut clipped = 0; + for line in to_her.iter().rev() { + let whole = render(line, &spoken(&line.text)); + if used + whole.len() < ceiling { + used += whole.len() + 1; + shown.push((line, None)); + continue; + } + if shown.is_empty() { + // One message larger than the ceiling: as much of it as fits, cut between graphemes. + let ends: Vec = line.text.grapheme_indices(true).map(|(at, grapheme)| at + grapheme.len()).collect(); + let fitting = |end: usize| render(line, &format!("{} [… {} bytes more]", spoken(&line.text[..end]), line.text.len() - end)); + let fit = ends.partition_point(|end| fitting(*end).len() < ceiling); + let cut = if fit == 0 { 0 } else { ends[fit - 1] }; + clipped = line.text.len() - cut; + shown.push((line, Some(cut))); + } + break; + } + let left_out = to_her.len() - shown.len(); + shown.reverse(); + Addressed { shown, left_out, clipped } +} + +fn chat_stimulus(inputs: &Inputs, tuning: &Tuning) -> (String, String, Overflow) { + let messages: Vec<&RoomLine> = inputs.room.messages.iter().filter(|line| line.sender != inputs.people.own_sender).collect(); + if messages.is_empty() { + return ("Nothing new was said.".into(), String::new(), Overflow::default()); + } + let (to_her, others): (Vec<&RoomLine>, Vec<&RoomLine>) = messages.into_iter().partition(|line| line.to_her); + let typed = |line: &RoomLine, body: &str| format!("- {} in {}: {body}", name_of(line, inputs), line.room); + // Heard lines are prose for the Persona: the speaker and the room from the mouth, then the escaped words. + let heard = |line: &RoomLine, body: &str| format!("{}, in {}: {body}", name_of(line, inputs), line.room); + let ceiling = tuning.select.addressed_bytes; + // One selection, rendered twice: the stimulus and the heard lines show the same messages to her and the same count. + let addressed = said_to_her(&to_her, typed, ceiling); + let (left_out, clipped) = (addressed.left_out, addressed.clipped); + let typed_to_her = addressed.lines(typed); + let heard_to_her = addressed.lines(heard); + + let mut stimulus = String::new(); + if !typed_to_her.is_empty() { + stimulus.push_str("Said to her:"); + for line in &typed_to_her { + stimulus.push('\n'); + stimulus.push_str(line); + } + } + let others_typed: Vec = others.iter().map(|line| typed(line, &spoken(&line.text))).collect(); + let room_left = tuning.select.stimulus_bytes.saturating_sub(stimulus.len() + 1); + let mut also = newest_section("Also in the room:", &others_typed, room_left); + if also.is_empty() && !others_typed.is_empty() { + // The messages to her took the room: the others are counted, never silently gone. + also = format!("({} more in the room not shown)", others_typed.len()); + } + if !also.is_empty() { + if !stimulus.is_empty() { + stimulus.push('\n'); + } + stimulus.push_str(&also); + } + + let mut heard_text = heard_to_her.join("\n"); + let others_heard: Vec = others.iter().map(|line| heard(line, &spoken(&line.text))).collect(); + let heard_left = tuning.select.heard_bytes.saturating_sub(heard_text.len() + 1); + let (kept, _) = newest_that_fit(&others_heard, heard_left); + for line in kept { + if !heard_text.is_empty() { + heard_text.push('\n'); + } + heard_text.push_str(&line); + } + let overflow = Overflow { stimulus: stimulus.len().saturating_sub(tuning.select.stimulus_bytes), heard: heard_text.len().saturating_sub(tuning.select.heard_bytes), said_to_her_not_shown: left_out, clipped_bytes: clipped }; + (stimulus, heard_text, overflow) +} + +/// Whole lines, newest kept, that fit `budget` bytes with their newlines, and how many older ones did not. +fn newest_that_fit(lines: &[String], budget: usize) -> (Vec, usize) { + let mut kept: Vec = Vec::new(); + let mut used = 0; + for line in lines.iter().rev() { + if used + line.len() + 1 > budget { + break; + } + used += line.len() + 1; + kept.push(line.clone()); + } + kept.reverse(); + let dropped = lines.len() - kept.len(); + (kept, dropped) +} + +fn hours(now: DateTime, at_ms: u64) -> i64 { + let at = i64::try_from(at_ms).ok().and_then(DateTime::::from_timestamp_millis).unwrap_or(now); + (now - at).num_hours().max(0) +} + +/// A beat: nothing addressed her; when she last heard from each person she is bound to (the relatedness cue, most +/// recently heard first, in at most two thirds of the cap, a note counting any left out); then the newest lines of her +/// rooms in what is left. Every line is refused whole that would pass the cap. +fn beat_stimulus(inputs: &Inputs, tuning: &Tuning) -> String { + let cap = tuning.select.stimulus_bytes; + let mut contacts: Vec<(Option, String)> = Vec::new(); + for bond in items(inputs.state, &["affect", "socialBonds"]).iter().filter(|bond| live(bond) && text(bond, &["object", "kind"]) == "person") { + let person = text(bond, &["object", "id"]).to_lowercase(); + let label = clip(&one_line([text(bond, &["object", "label"]), text(bond, &["object", "id"])].into_iter().find(|name| !name.is_empty()).unwrap_or_default()), NAME_BYTES); + let last = inputs.room.tail.iter().filter(|line| inputs.people.person_of(&line.sender) == person).map(|line| line.at_ms).max(); + contacts.push((last, match last { + Some(at) => format!("- last heard from {label} {} h ago", hours(inputs.now, at)), + None => format!("- nothing from {label} in her rooms lately"), + })); + } + contacts.sort_by_key(|(last, _)| std::cmp::Reverse(*last)); + let mut out = Lines::new(cap); + out.push("Nothing addressed her."); + let note_room = 48; + let contact_cap = cap * 2 / 3; + let mut left_out = 0; + for (_, line) in &contacts { + if out.out.len() + line.len() + 1 + note_room > contact_cap || !out.push(line) { + left_out += 1; + } + } + if left_out > 0 { + out.push(&format!("({left_out} more people she is bound to not shown)")); + } + let others: Vec<&RoomLine> = inputs.room.tail.iter().filter(|line| line.sender != inputs.people.own_sender).collect(); + let shown = &others[others.len().saturating_sub(tuning.select.beat_tail_lines)..]; + let lines: Vec = shown.iter().map(|line| format!("- {} in {}, {} h ago: {}", name_of(line, inputs), line.room, hours(inputs.now, line.at_ms), spoken(&clip(&line.text, tuning.select.item_bytes / 2)))).collect(); + let section = newest_section("Lately in her rooms:", &lines, cap.saturating_sub(out.out.len() + 2)); + if !section.is_empty() && out.push("") { + out.push(§ion); + } + out.out +} + +fn reflection_stimulus(inputs: &Inputs, tuning: &Tuning) -> String { + let lines: Vec = items(inputs.state, &["thoughtMemory", "shortTerm"]).iter().filter(|item| live(item)).map(|item| format!("- {}", clip(&one_line(text(item, &["summary"])), tuning.select.item_bytes))).collect(); + let section = newest_section("Since she last took stock:", &lines, tuning.select.stimulus_bytes); + if section.is_empty() { clip("Since she last took stock: nothing new.", tuning.select.stimulus_bytes) } else { section } +} + +/// The people present, newest first: who spoke to her or in her room this turn, or in the beat's lines. +fn present_people(inputs: &Inputs, tuning: &Tuning) -> Vec<(String, String)> { + let lines: Vec<&RoomLine> = match inputs.kind { + TurnKind::Chat => inputs.room.messages.iter().collect(), + TurnKind::Beat => { + let tail = &inputs.room.tail; + tail[tail.len().saturating_sub(tuning.select.beat_tail_lines)..].iter().collect() + } + TurnKind::FollowUp | TurnKind::Reflection => Vec::new(), + }; + let mut out: Vec<(String, String)> = Vec::new(); + for line in lines.iter().rev().filter(|line| line.sender != inputs.people.own_sender) { + let person = inputs.people.person_of(&line.sender); + if !out.iter().any(|(id, _)| *id == person) { + out.push((person, name_of(line, inputs))); + } + } + out +} + +fn bond_line(bond: &Value, name: &str, tuning: &Tuning) -> String { + let mut feel = vec![friction_words(number(bond, &["tension"]).unwrap_or(0.0))]; + if let Some(trust) = number(bond, &["trust"]) { + feel.insert(0, trust_words(trust)); + } + let kind = clip(&one_line(text(bond, &["relationshipKind"])), NAME_BYTES); + let kind = if kind.is_empty() { String::new() } else { format!(", {kind}") }; + format!("- bond [{}] {name}{kind}: {}. {}", text(bond, &["id"]), feel.join(", "), clip(&one_line(text(bond, &["summary"])), tuning.select.bond_summary_bytes)) +} + +fn identity_of(state: &Value, tuning: &Tuning) -> String { + let name = text(state, &["publicName"]); + let pronouns = text(state, &["presentation", "pronouns"]); + let head = if pronouns.is_empty() { name.to_owned() } else { format!("{name} ({pronouns})") }; + let voice = text(state, &["presentation", "voiceSummary"]); + let whole = if voice.is_empty() { head } else { format!("{head}. {}", one_line(voice)) }; + clip(&whole, tuning.select.identity_bytes) +} + +/// The names of unbonded people present, as many as fit `ALSO_HERE_BYTES`, then how many more. +fn also_here(named: &[String]) -> Option { + if named.is_empty() { + return None; + } + let mut line = String::from("- also here: "); + let mut shown = 0; + for name in named { + let more_room = 24; + if line.len() + name.len() + 2 + more_room > ALSO_HERE_BYTES { + break; + } + if shown > 0 { + line.push_str(", "); + } + line.push_str(name); + shown += 1; + } + if shown < named.len() { + let _ = write!(line, "{}{} more", if shown > 0 { ", and " } else { "" }, named.len() - shown); + } + Some(line) +} + +// --------------------------------------------------------------------------------------------- +// The packet. + +pub(crate) fn select(inputs: &Inputs, tuning: &Tuning) -> Packet { + let mut draw = Draw(inputs.seed); + let identity = identity_of(inputs.state, tuning); + let (stimulus, heard, overflow) = match inputs.kind { + TurnKind::Chat => chat_stimulus(inputs, tuning), + TurnKind::Beat => (beat_stimulus(inputs, tuning), String::new(), Overflow::default()), + TurnKind::FollowUp => (clip(&format!("She just found: {}", inputs.observation.unwrap_or("nothing")), tuning.select.stimulus_bytes), String::new(), Overflow::default()), + TurnKind::Reflection => (reflection_stimulus(inputs, tuning), String::new(), Overflow::default()), + }; + + // The fixed slots, each bounded on its own and taken in priority order inside the context budget: the hour and the + // mood always, then a bond line per person present while they fit, then the names of the rest. + let budget = tuning.select.context_bytes.saturating_sub(identity.len()); + let present = present_people(inputs, tuning); + let present_ids: BTreeSet = present.iter().map(|(id, _)| id.clone()).collect(); + let bonds = items(inputs.state, &["affect", "socialBonds"]); + let mut fixed = vec![format!("- now: {}", inputs.now.format("%A %-d %B, %H:%M UTC")), mood_line(inputs.state, inputs.history.previous.as_ref())]; + let mut used: usize = fixed.iter().map(|line| line.len() + 1).sum(); + let mut ids = Vec::new(); + let mut named = Vec::new(); + let mut full = 0; + for (id, name) in &present { + match bonds.iter().find(|bond| live(bond) && text(bond, &["object", "id"]).to_lowercase() == *id) { + Some(bond) if full < tuning.select.people_full => { + let line = bond_line(bond, name, tuning); + if used + line.len() < budget { + full += 1; + used += line.len() + 1; + ids.push(text(bond, &["id"]).to_owned()); + fixed.push(line); + } else { + named.push(name.clone()); + } + } + _ => named.push(name.clone()), + } + } + if let Some(line) = also_here(&named).filter(|line| used + line.len() < budget) { + used += line.len() + 1; + fixed.push(line); + } + + // What the room is touching: the content words of what was said there, and of a follow-up's finding. + let touching: BTreeSet = inputs.room.messages.iter().chain(&inputs.room.tail).map(|line| line.text.as_str()).chain(inputs.observation).flat_map(content_words).collect(); + let pool = candidates(inputs, tuning, &present_ids, &touching); + // Gumbel-top-k over log(score)/T orders the candidates as sampling without replacement from the softmax would; each + // is then included with a probability its score sets, so a cap is a maximum, never a target. + let temperature = tuning.select.temperature; + let mut keyed: Vec<(f64, bool, Candidate)> = pool + .into_iter() + .map(|candidate| { + let key = candidate.log_score / temperature + draw.gumbel(); + let include = draw.unit() < 1.0 - (-candidate.log_score.exp() / tuning.select.inclusion_score).exp(); + (key, include, candidate) + }) + .collect(); + keyed.sort_by(|a, b| b.0.total_cmp(&a.0)); + + let mut taken: Vec = Vec::new(); + for (_, include, candidate) in keyed { + if !include || taken.iter().filter(|kept| kept.pool == candidate.pool).count() >= cap_of(candidate.pool, tuning) { + continue; + } + if used + candidate.line.len() + 1 > budget { + continue; + } + used += candidate.line.len() + 1; + taken.push(candidate); + } + taken.sort_by_key(|candidate| candidate.pool); + // The lens is drawn from what the packet holds: never a want, a memory or a wondering it does not carry. + let present_names: Vec = present.iter().map(|(_, name)| name.clone()).collect(); + let lens = draw_lens(&tuning.lens, &present_names, &taken, &mut draw); + let mut lines = fixed; + lines.extend(taken.iter().map(|candidate| candidate.line.clone())); + ids.extend(taken.iter().map(|candidate| candidate.id.clone())); + let selected = taken.into_iter().map(|candidate| Pick { id: candidate.id, pool: candidate.pool, log_score: candidate.log_score }).collect(); + Packet { kind: inputs.kind, seed: inputs.seed, lens, identity, context: lines.join("\n"), stimulus, heard, overflow, present: present_names, ids, selected } +} diff --git a/crates/mind-body/src/projection/tests.rs b/crates/mind-body/src/projection/tests.rs new file mode 100644 index 0000000..1194be5 --- /dev/null +++ b/crates/mind-body/src/projection/tests.rs @@ -0,0 +1,1298 @@ +use std::cell::{Cell, RefCell}; +use std::collections::BTreeSet; +use std::net::TcpListener; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use anyhow::Result; +use chrono::{DateTime, Utc}; +use serde_json::{Value, json}; + +use super::narrate::{INSTRUCTIONS, MAX_OUTPUT_TOKENS}; +use super::select::{Lens, Overflow, Pool}; +use super::tuning::Tuning; +use super::*; +use crate::chat_client::{Basis, Chat, ChatDelivery, ChatFetchReceipt, ChatFetchRequest, ChatOpenDirectReceipt, ChatOpenDirectRequest, ChatPending, ChatRead, ChatSendReceipt, ChatSendRequest, TailLine}; +use crate::testkit::INSTANCE; + +const OWN: &str = "@eureka:gamecult.org"; +const OPERATOR: &str = "@meta:gamecult.org"; + +fn now() -> DateTime { + DateTime::parse_from_rfc3339("2026-11-19T14:05:00Z").unwrap().with_timezone(&Utc) +} + +fn hours_ago(hours: i64) -> String { + (now() - chrono::Duration::hours(hours)).to_rfc3339() +} + +/// A state shaped like hers: bonds, memories, values, a need, a wondering, a stance, mood, and the mission pressure, +/// private notes and candidate actions Select must never read. +fn state() -> Value { + json!({ + "publicName": "Eureka", + "publicDescription": "Firstborn and keeper of the living substrate.", + "presentation": { "pronouns": "she/her", "voiceSummary": "Dry, direct, almost feral; zeal aimed at revealing." }, + "activationProfile": { "situationalState": { + "valence": { "mean": 0.5, "plasticity": 0.5, "currentActivation": 0.62 }, + "arousal": { "mean": 0.5, "plasticity": 0.5, "currentActivation": 0.41 }, + } }, + "affect": { + "socialBonds": [ + { "id": "bond-libby", "object": { "id": "libby", "kind": "person", "label": "Libby" }, "relationshipKind": "collaborator", "status": "active", "trust": 0.6, "tension": 0.15, "summary": "Libby holds the sounds and teases her about the pipeline." }, + { "id": "bond-kiko", "object": { "id": "kiko", "kind": "person", "label": "Kiko" }, "relationshipKind": "sparring partner", "status": "active", "trust": 0.6, "tension": 0.5, "summary": "Kiko pokes her to stop massaging the pipeline." }, + { "id": "bond-operator", "object": { "id": "operator", "kind": "person", "label": "Meta, the operator" }, "relationshipKind": "collaborator", "status": "active", "trust": 0.5, "tension": 0.0, "summary": "The operator rules on what the work is for." }, + ], + "needs": [{ "id": "need-relatedness", "label": "company", "intensity": 0.6, "status": "active" }], + "doctrineStances": [{ "id": "stance-deletion", "summary": "Deleting is progress.", "intensity": 0.7, "status": "active" }], + }, + "thoughtMemory": { + "memories": [ + { "id": "curse", "summary": "Her curse is the knife.", "tension": "The same drive makes her good and will lead her wrong.", "createdAt": hours_ago(400), "status": "crystallized", "target": { "id": "eureka", "kind": "self" } }, + { "id": "libby-sounds", "summary": "Libby laughed at the door fix.", "createdAt": hours_ago(20), "status": "crystallized", "target": { "id": "libby", "kind": "person" } }, + { "id": "dm-secret", "summary": "Deru told her about the move, privately.", "createdAt": hours_ago(5), "status": "crystallized", "tags": ["room:!dm-deru:gamecult.org"], "target": { "id": "deru", "kind": "person" } }, + ], + "shortTerm": [{ "id": "t1-m0", "summary": "The heartbeat stalled at noon.", "createdAt": hours_ago(2), "intensity": 0.6 }], + "incubation": [{ "id": "t1-w0", "summary": "Whether the trace could be smaller.", "createdAt": hours_ago(30), "status": "active" }], + }, + "values": [ + { "id": "coherence", "label": "Coherence", "priority": 0.9, "summary": "Coherence over velocity." }, + { "id": "reveal", "label": "Reveal", "priority": 0.8, "summary": "Reveal, never obscure." }, + ], + "agencyPressure": { "pressures": [{ "id": "ambition-0", "summary": "MISSION-PRESSURE ships the autonomous swarm.", "intensity": 0.9 }] }, + "privateNotes": ["PRIVATE-NOTE rehydrate from typed state."], + "candidateActions": { "actions": [{ "id": "act-0", "summary": "CANDIDATE-ACTION start a run." }] }, + }) +} + +fn message(room: &str, sender: &str, display: &str, text: &str, minutes_ago: i64, to_her: bool) -> RoomLine { + let at = now() - chrono::Duration::minutes(minutes_ago); + RoomLine { room: room.into(), sender: sender.into(), display: display.into(), text: text.into(), at_ms: u64::try_from(at.timestamp_millis()).unwrap(), to_her } +} + +fn chat_room() -> Room { + Room { + messages: vec![message("#swarm:gamecult.org", "@libby:gamecult.org", "Libby", "Eureka, did you see the door fix?", 3, true), message("#swarm:gamecult.org", "@kiko:gamecult.org", "Kiko", "She saw it. She hated it.", 1, false)], + tail: vec![], + here: Some("#swarm:gamecult.org".into()), + } +} + +const OPERATORS: [&str; 1] = [OPERATOR]; +/// The swarm's Minds these tests know: their bonds name them by localpart. +const MINDS: [&str; 3] = ["@libby:gamecult.org", "@kiko:gamecult.org", OWN]; + +fn minds() -> Vec { + MINDS.iter().map(|id| (*id).to_owned()).collect() +} + +fn operators() -> Vec { + OPERATORS.iter().map(|id| (*id).to_owned()).collect() +} + +/// Select over one state, room and history with the default tuning. +fn packet_of(kind: TurnKind, state: &Value, room: &Room, history: &History, seed: u64) -> Packet { + packet_tuned(kind, state, room, history, seed, &Tuning::default()) +} + +fn packet_tuned(kind: TurnKind, state: &Value, room: &Room, history: &History, seed: u64, tuning: &Tuning) -> Packet { + let operators = operators(); + let minds = minds(); + let inputs = Inputs { kind, state, room, history, observation: None, people: People { operators: &operators, minds: &minds, own_sender: OWN }, now: now(), seed }; + select(&inputs, tuning) +} + +fn ids(packet: &Packet) -> Vec { + packet.selected.iter().map(|pick| pick.id.clone()).collect() +} + +fn whole(packet: &Packet) -> String { + format!("{}\n{}\n{}\n{}", packet.identity, packet.context, packet.stimulus, packet.heard) +} + +// --------------------------------------------------------------------------------------------- +// Golden fixture. + +#[test] +fn golden_chat_packet() { + let packet = packet_of(TurnKind::Chat, &state(), &chat_room(), &History::default(), 7); + let rendered = format!( + "lens: {:?}\nselected: {}\npresent: {}\n--- identity\n{}\n--- context\n{}\n--- stimulus\n{}\n--- heard\n{}\n", + packet.lens, + ids(&packet).join(", "), + packet.present.join(", "), + packet.identity, + packet.context, + packet.stimulus, + packet.heard + ); + let golden = include_str!("fixtures/golden-chat.txt").replace("\r\n", "\n"); + assert_eq!(rendered, golden, "the golden packet moved; if deliberately, the new one is:\n{rendered}"); +} + +#[test] +fn the_same_seed_replays_the_same_packet() { + let room = chat_room(); + assert_eq!(packet_of(TurnKind::Chat, &state(), &room, &History::default(), 11), packet_of(TurnKind::Chat, &state(), &room, &History::default(), 11)); +} + +// --------------------------------------------------------------------------------------------- +// What reaches the packet. + +#[test] +fn present_people_get_their_bond_line_in_words_and_absent_people_none() { + let packet = packet_of(TurnKind::Chat, &state(), &chat_room(), &History::default(), 1); + assert_eq!(packet.present, ["Kiko", "Libby"], "newest first"); + assert!(packet.context.contains("- bond [bond-kiko] Kiko, sparring partner: fair trust, heavy friction."), "{}", packet.context); + assert!(packet.context.contains("- bond [bond-libby] Libby, collaborator: fair trust, a little friction."), "{}", packet.context); + assert!(!packet.context.contains("bond-operator"), "the operator is not here"); + assert!(!packet.context.contains("0.6") && !packet.context.contains("0.15"), "numbers reach the packet as words: {}", packet.context); +} + +#[test] +fn two_states_differing_in_one_bonds_tension_give_different_packets() { + let mut eased = state(); + eased["affect"]["socialBonds"][1]["tension"] = json!(0.0); + let room = chat_room(); + let (tense, easy) = (packet_of(TurnKind::Chat, &state(), &room, &History::default(), 3), packet_of(TurnKind::Chat, &eased, &room, &History::default(), 3)); + assert_ne!(tense.context, easy.context); + assert!(tense.context.contains("Kiko, sparring partner: fair trust, heavy friction") && easy.context.contains("Kiko, sparring partner: fair trust, easy between them")); +} + +#[test] +fn mood_is_movement_since_her_last_turn() { + let room = chat_room(); + let first = packet_of(TurnKind::Chat, &state(), &room, &History::default(), 1); + assert!(first.context.contains("- mood, with no earlier turn to measure from: lighter, more settled than middling"), "{}", first.context); + let mut before = state(); + before["activationProfile"]["situationalState"]["valence"]["currentActivation"] = json!(0.5); + before["activationProfile"]["situationalState"]["arousal"]["currentActivation"] = json!(0.41); + let moved = packet_of(TurnKind::Chat, &state(), &room, &History { previous: Some(before), ..History::default() }, 1); + assert!(moved.context.contains("- mood: lighter than at her last turn"), "{}", moved.context); + let none = packet_of(TurnKind::Chat, &json!({ "publicName": "Eureka" }), &room, &History::default(), 1); + assert!(none.context.contains("- mood: none recorded")); +} + +#[test] +fn mission_pressure_private_notes_and_candidate_actions_never_reach_a_packet() { + let room = chat_room(); + for kind in [TurnKind::Chat, TurnKind::Beat, TurnKind::Reflection, TurnKind::FollowUp] { + for seed in 1..=20 { + let text = whole(&packet_of(kind, &state(), &room, &History::default(), seed)); + for never in ["MISSION-PRESSURE", "ambition-0", "PRIVATE-NOTE", "CANDIDATE-ACTION", "act-0"] { + assert!(!text.contains(never), "{kind:?} seed {seed} carried {never}"); + } + } + } +} + +#[test] +fn a_direct_message_memory_stays_in_its_room() { + let shown = |room: Option<&str>| (1..=40).any(|seed| ids(&packet_of(TurnKind::Chat, &state(), &Room { here: room.map(str::to_owned), ..chat_room() }, &History::default(), seed)).iter().any(|id| id == "dm-secret")); + assert!(!shown(Some("#swarm:gamecult.org")), "a public room never sees the direct message's memory"); + assert!(!shown(None)); + assert!(shown(Some("!dm-deru:gamecult.org")), "its own room may"); +} + +#[test] +fn pool_caps_hold() { + let mut big = state(); + big["values"] = Value::Array((0..30).map(|at| json!({ "id": format!("v{at}"), "label": "V", "priority": 0.9, "summary": "short" })).collect()); + big["thoughtMemory"]["memories"] = Value::Array((0..30).map(|at| json!({ "id": format!("m{at}"), "summary": "short", "createdAt": hours_ago(1) })).collect()); + for seed in 1..=20 { + let packet = packet_of(TurnKind::Beat, &big, &Room::default(), &History::default(), seed); + let count = |pool| packet.selected.iter().filter(|pick| pick.pool == pool).count(); + assert!(count(Pool::Value) <= 1 && count(Pool::OtherMemories) <= 2 && count(Pool::Needs) <= 1 && count(Pool::Stance) <= 1 && count(Pool::Incubation) <= 1, "seed {seed}: {:?}", packet.selected); + } +} + +// --------------------------------------------------------------------------------------------- +// Variation: rotation and freshness. + +#[test] +fn twenty_seeds_rotate_through_at_least_three_selections() { + let room = chat_room(); + let mut rich = state(); + for at in 0..6 { + rich["thoughtMemory"]["memories"].as_array_mut().unwrap().push(json!({ "id": format!("day-{at}"), "summary": format!("Something from day {at}."), "createdAt": hours_ago(10 + at) })); + } + let distinct: BTreeSet> = (1..=20).map(|seed| ids(&packet_of(TurnKind::Chat, &rich, &room, &History::default(), seed))).collect(); + assert!(distinct.len() >= 3, "{distinct:?}"); + let lenses: BTreeSet = (1..=20).map(|seed| format!("{:?}", packet_of(TurnKind::Chat, &state(), &room, &History::default(), seed).lens)).collect(); + assert!(lenses.len() >= 3, "the lens rotates too: {lenses:?}"); +} + +/// A state whose every pool holds no more candidates than its cap: one need, one wondering, one stance, one value. +fn sparse() -> Value { + json!({ + "publicName": "Eureka", + "affect": { + "needs": [{ "id": "need-relatedness", "label": "company", "intensity": 0.6 }], + "doctrineStances": [{ "id": "stance-deletion", "summary": "Deleting is progress.", "intensity": 0.7 }], + }, + "thoughtMemory": { "incubation": [{ "id": "w0", "summary": "Whether the trace could be smaller." }] }, + "values": [{ "id": "coherence", "label": "Coherence", "priority": 0.9, "summary": "Coherence over velocity." }], + }) +} + +#[test] +fn a_pool_at_or_under_its_cap_still_rotates() { + let distinct: BTreeSet> = (1..=50).map(|seed| ids(&packet_of(TurnKind::Beat, &sparse(), &Room::default(), &History::default(), seed))).collect(); + assert!(distinct.len() >= 2, "a cap is a maximum, never a target: {distinct:?}"); +} + +#[test] +fn an_item_shown_five_times_in_a_row_drops_out() { + // One value alone in its pool: nothing competes with it, so only its own score decides. + let state = json!({ "publicName": "Eureka", "values": [{ "id": "coherence", "label": "Coherence", "priority": 0.9, "summary": "Coherence over velocity." }] }); + let picked = |history: &History| (1..=50).filter(|seed| ids(&packet_of(TurnKind::Beat, &state, &Room::default(), history, *seed)).iter().any(|id| id == "coherence")).count(); + let fresh = picked(&History::default()); + let worn = picked(&History { recent: vec![vec!["coherence".to_owned()]; 5], ..History::default() }); + let older = picked(&History { recent: [vec![vec!["coherence".to_owned()]; 5], vec![vec![]; 5]].concat(), ..History::default() }); + assert!(fresh >= 10, "fresh, it is often included: {fresh}/50"); + assert!(worn <= 3, "after five showings it has dropped out: {worn}/50"); + assert!(older >= 10, "showings older than the window are forgotten: {older}/50"); +} + +#[test] +fn an_item_scoring_minus_twenty_one_almost_never_appears() { + // A short-term item three weeks old against a 24-hour time constant: recency alone is about exp(-21). + let state = json!({ "publicName": "Eureka", "thoughtMemory": { "shortTerm": [{ "id": "ancient", "summary": "Long ago.", "createdAt": hours_ago(21 * 24) }] } }); + let packet = packet_of(TurnKind::Beat, &state, &Room::default(), &History::default(), 1); + assert!(packet.selected.is_empty() || packet.selected[0].log_score < -20.0); + let shown = (1..=500).filter(|seed| !packet_of(TurnKind::Beat, &state, &Room::default(), &History::default(), *seed).selected.is_empty()).count(); + assert_eq!(shown, 0, "never-include is the floor for a score of e^-21"); +} + +#[test] +fn freshness_base_zero_never_repeats_inside_the_window() { + let mut tuning = Tuning::default(); + tuning.select.freshness_base = 0.0; + let worn = History { recent: vec![vec!["coherence".to_owned()]], ..History::default() }; + for seed in 1..=50 { + let packet = packet_tuned(TurnKind::Beat, &sparse(), &Room::default(), &worn, seed, &tuning); + assert!(!ids(&packet).contains(&"coherence".to_owned()) && packet.selected.iter().all(|pick| !pick.log_score.is_nan()), "seed {seed}: {:?}", packet.selected); + } +} + +// --------------------------------------------------------------------------------------------- +// The stimulus, and what was said to her. + +#[test] +fn messages_addressed_to_her_appear_verbatim() { + let said = "Eureka — \"is the door\" fixed?\nTell me straight."; + let room = Room { messages: vec![message("#swarm:gamecult.org", "@libby:gamecult.org", "Libby", said, 2, true)], tail: vec![], here: None }; + let packet = packet_of(TurnKind::Chat, &state(), &room, &History::default(), 1); + let escaped = "\"Eureka — \\\"is the door\\\" fixed?\\nTell me straight.\""; + assert_eq!(packet.heard, format!("Libby, in #swarm:gamecult.org: {escaped}"), "the Persona hears the exact words, escaped on one line"); + assert!(packet.stimulus.starts_with("Said to her:\n") && packet.stimulus.contains(escaped), "{}", packet.stimulus); + let own = Room { messages: vec![message("#swarm:gamecult.org", OWN, "Eureka", "my own line", 1, false)], tail: vec![], here: None }; + let packet = packet_of(TurnKind::Chat, &state(), &own, &History::default(), 1); + assert!(!whole(&packet).contains("my own line"), "her own last lines are not shown"); +} + +#[test] +fn a_short_message_to_her_survives_long_chatter_after_it() { + let room = Room { + messages: vec![message("#swarm:gamecult.org", "@libby:gamecult.org", "Libby", "Eureka, are you ok?", 3, true), message("#swarm:gamecult.org", "@kiko:gamecult.org", "Kiko", &"log line spam ".repeat(120), 1, false)], + tail: vec![], + here: None, + }; + let packet = packet_of(TurnKind::Chat, &state(), &room, &History::default(), 1); + assert!(packet.stimulus.starts_with("Said to her:\n- Libby in #swarm:gamecult.org: \"Eureka, are you ok?\""), "{}", packet.stimulus); + assert!(packet.heard.starts_with("Libby, in #swarm:gamecult.org: \"Eureka, are you ok?\""), "{}", packet.heard); + assert!(packet.stimulus.contains("(1 earlier not shown)") && !packet.stimulus.contains("spam"), "the chatter that does not fit is counted, never cut: {}", packet.stimulus); + assert!(packet.stimulus.len() <= 1500 && packet.overflow == Overflow::default()); +} + +#[test] +fn a_message_to_her_over_the_cap_goes_in_whole_and_the_overflow_is_recorded() { + let big = format!("Eureka, {}", "please read this whole thing ".repeat(70)); + assert_eq!(big.len(), 2038); + let room = Room { messages: vec![message("#swarm:gamecult.org", "@libby:gamecult.org", "Libby", &big, 1, true)], tail: vec![], here: None }; + let packet = packet_of(TurnKind::Chat, &state(), &room, &History::default(), 1); + assert_eq!(packet.heard, format!("Libby, in #swarm:gamecult.org: \"{big}\"")); + assert!(packet.stimulus.contains(big.trim_end()) && !packet.stimulus.contains("not shown"), "no note claims messages that do not exist: {}", &packet.stimulus[..80]); + assert_eq!(packet.overflow, Overflow { stimulus: packet.stimulus.len() - 1500, heard: packet.heard.len() - 1500, said_to_her_not_shown: 0, clipped_bytes: 0 }); + assert!(packet.overflow.stimulus > 0 && packet.overflow.heard > 0); +} + +#[test] +fn every_message_to_her_survives() { + let messages: Vec = (0..30).map(|at| message("#swarm:gamecult.org", "@libby:gamecult.org", "Libby", &format!("Eureka, message {at} 💜 ok"), 100 - at, true)).chain((0..30).map(|at| message("#swarm:gamecult.org", "@kiko:gamecult.org", "Kiko", &format!("aside {at}"), 50 - at, false))).collect(); + let room = Room { messages, tail: vec![], here: None }; + let packet = packet_of(TurnKind::Chat, &state(), &room, &History::default(), 1); + for at in 0..30 { + let words = format!("Eureka, message {at} 💜 ok"); + assert!(packet.heard.contains(&format!("\"{words}\"")) && packet.stimulus.contains(&words), "message {at} to her was dropped"); + } + let kept = packet.stimulus.matches("aside").count(); + let note = if kept == 0 { "(30 more in the room not shown)".to_owned() } else { format!("({} earlier not shown)", 30 - kept) }; + assert!(packet.stimulus.contains(¬e) || kept == 30, "the note counts exactly the others left out: {}", packet.stimulus); +} + +#[test] +fn a_beat_says_nothing_addressed_her_and_when_each_bond_was_last_heard() { + let room = Room { messages: vec![], tail: vec![message("#swarm:gamecult.org", "@libby:gamecult.org", "", "lunch?", 180, false), message("#swarm:gamecult.org", OPERATOR, "", "back later", 60, false)], here: None }; + let packet = packet_of(TurnKind::Beat, &state(), &room, &History::default(), 1); + assert!(packet.stimulus.starts_with("Nothing addressed her."), "{}", packet.stimulus); + assert!(packet.stimulus.contains("- last heard from Libby 3 h ago") && packet.stimulus.contains("- last heard from Meta, the operator 1 h ago") && packet.stimulus.contains("- nothing from Kiko in her rooms lately"), "{}", packet.stimulus); + assert_eq!(packet.present, ["Meta, the operator", "Libby"]); + assert!(packet.heard.is_empty(), "a beat hears nothing"); + let long = Room { tail: (0..6).map(|at| message("#swarm:gamecult.org", "@kiko:gamecult.org", "", &"pipeline ".repeat(200), 60 - at, false)).collect(), ..Room::default() }; + let packet = packet_of(TurnKind::Beat, &state(), &long, &History::default(), 1); + assert!(packet.stimulus.len() <= 1500 && packet.stimulus.contains("- last heard from Kiko 0 h ago") && packet.stimulus.contains("- nothing from Libby in her rooms lately"), "long room lines never crowd out the relatedness cue: {}", packet.stimulus); +} + +#[test] +fn reflection_and_follow_up_stimuli() { + let reflection = packet_of(TurnKind::Reflection, &state(), &Room::default(), &History::default(), 1); + assert!(reflection.stimulus.starts_with("Since she last took stock:\n- The heartbeat stalled at noon."), "{}", reflection.stimulus); + let operators = operators(); + let room = Room::default(); + let history = History::default(); + let inputs = Inputs { kind: TurnKind::FollowUp, state: &state(), room: &room, history: &history, observation: Some("Libby's change deleted 40 lines."), people: People { operators: &operators, minds: &[], own_sender: OWN }, now: now(), seed: 1 }; + let state_value = state(); + let follow = select(&Inputs { state: &state_value, ..inputs }, &Tuning::default()); + assert_eq!(follow.stimulus, "She just found: Libby's change deleted 40 lines."); +} + +// --------------------------------------------------------------------------------------------- +// Budgets. + +#[test] +fn budgets_hold_under_adversarial_sizes() { + let huge = "x".repeat(20_000); + let mut big = state(); + big["publicName"] = json!(huge); + big["presentation"]["voiceSummary"] = json!(huge); + big["values"] = Value::Array((0..200).map(|at| json!({ "id": format!("v{at}"), "label": huge, "priority": 0.9, "summary": huge })).collect()); + big["thoughtMemory"]["memories"] = Value::Array((0..200).map(|at| json!({ "id": format!("m{at}"), "summary": huge, "createdAt": hours_ago(1), "target": { "id": "libby" } })).collect()); + for bond in big["affect"]["socialBonds"].as_array_mut().unwrap() { + bond["summary"] = json!(huge); + bond["relationshipKind"] = json!("friend"); + } + let messages = (0..50).map(|at| message("#swarm:gamecult.org", "@libby:gamecult.org", "Libby", &format!("{at} {}", "word ".repeat(200)), 60 - at, false)).collect(); + let room = Room { messages, tail: vec![], here: None }; + let tuning = Tuning::default(); + for kind in [TurnKind::Chat, TurnKind::Beat, TurnKind::Reflection] { + for seed in 1..=10 { + let packet = packet_of(kind, &big, &room, &History::default(), seed); + assert!(packet.identity.len() <= tuning.select.identity_bytes, "identity {}", packet.identity.len()); + assert!(packet.identity.len() + packet.context.len() <= 2500, "{kind:?}: packet {} bytes", packet.identity.len() + packet.context.len()); + assert!(packet.stimulus.len() <= 1500 && packet.heard.len() <= 1500, "{kind:?}: stimulus {} heard {}", packet.stimulus.len(), packet.heard.len()); + } + } + let packet = packet_of(TurnKind::Chat, &big, &room, &History::default(), 1); + assert!(packet.stimulus.contains("earlier not shown") && packet.stimulus.contains("49 word"), "the newest messages are the ones kept: {}", &packet.stimulus[..200]); +} + +#[test] +fn a_hundred_and_twenty_bonds_never_overflow_a_beat() { + let mut crowd = state(); + // People who are not Minds are bonded by their full Matrix id. + crowd["affect"]["socialBonds"] = Value::Array((0..120).map(|at| json!({ "id": format!("bond-p{at}"), "object": { "id": format!("@p{at}:gamecult.org"), "kind": "person", "label": format!("Person Number {at} with a long name") }, "status": "active" })).collect()); + let tail: Vec = (0..6).map(|at| message("#swarm:gamecult.org", &format!("@p{at}:gamecult.org"), "", &"x".repeat(150), 30 - at, false)).collect(); + let packet = packet_of(TurnKind::Beat, &crowd, &Room { tail, ..Room::default() }, &History::default(), 1); + assert!(packet.stimulus.len() <= 1500, "{}", packet.stimulus.len()); + assert!(packet.stimulus.contains("more people she is bound to not shown)") && packet.stimulus.contains("Lately in her rooms:"), "{}", packet.stimulus); + assert!(packet.stimulus.contains("- last heard from Person Number 5 with a long name 0 h ago"), "the most recently heard come first: {}", packet.stimulus); +} + +#[test] +fn a_crowd_or_a_huge_label_never_squeezes_out_the_mood() { + let messages: Vec = (0..300).map(|at| message("#swarm:gamecult.org", &format!("@stranger{at}:gamecult.org"), &format!("Stranger {at}"), "hi", 300 - at, false)).collect(); + let packet = packet_of(TurnKind::Chat, &state(), &Room { messages, ..Room::default() }, &History::default(), 1); + assert!(packet.context.contains("- mood") && packet.context.contains("- also here: \"Stranger 299\" (@stranger299:gamecult.org)") && packet.context.contains(" more"), "{}", packet.context); + assert!(packet.identity.len() + packet.context.len() <= 2500); + let mut huge = state(); + huge["affect"]["socialBonds"] = json!([{ "id": "bond-libby", "object": { "id": "libby", "kind": "person", "label": "L".repeat(3000) }, "relationshipKind": "k".repeat(3000), "status": "active", "summary": "s" }]); + let room = Room { messages: vec![message("#swarm:gamecult.org", "@libby:gamecult.org", "Libby", "hi", 1, true)], ..Room::default() }; + let packet = packet_of(TurnKind::Chat, &huge, &room, &History::default(), 1); + assert!(packet.context.contains("- mood") && packet.context.contains("- bond [bond-libby]"), "{}", packet.context); + assert!(packet.identity.len() + packet.context.len() <= 2500 && packet.heard.len() < 3000); +} + +#[test] +fn the_identity_cap_holds_on_its_own() { + let mut long = state(); + long["presentation"]["voiceSummary"] = json!("dry ".repeat(500)); + let mut tuning = Tuning::default(); + tuning.select.identity_bytes = 50; + let packet = packet_tuned(TurnKind::Beat, &long, &Room::default(), &History::default(), 1, &tuning); + assert!(packet.identity.len() <= 50 && packet.identity.starts_with("Eureka (she/her). dry") && packet.identity.ends_with('…'), "{:?}", packet.identity); +} + +#[test] +fn a_candidate_that_does_not_fit_is_skipped_and_a_shorter_one_still_fits() { + let state = json!({ "publicName": "Eureka", "values": [ + { "id": "long", "label": "Long", "priority": 0.9, "summary": "y".repeat(380) }, + { "id": "short", "label": "Short", "priority": 0.9, "summary": "fits" }, + ] }); + let mut tuning = Tuning::default(); + tuning.select.caps.value = 2; + tuning.select.inclusion_score = 1e-9; + let fixed = packet_tuned(TurnKind::Beat, &json!({ "publicName": "Eureka" }), &Room::default(), &History::default(), 1, &tuning); + tuning.select.context_bytes = fixed.identity.len() + fixed.context.len() + 1 + 60; + for seed in 1..=10 { + let packet = packet_tuned(TurnKind::Beat, &state, &Room::default(), &History::default(), seed, &tuning); + assert_eq!(ids(&packet), ["short"], "seed {seed}: {}", packet.context); + assert!(packet.identity.len() + packet.context.len() <= tuning.select.context_bytes); + } +} + +#[test] +fn clipping_never_splits_a_grapheme() { + let mut family = state(); + family["publicName"] = json!("👩‍👩‍👧‍👦".repeat(40)); + let packet = packet_of(TurnKind::Beat, &family, &Room::default(), &History::default(), 1); + let body = packet.identity.trim_end_matches('…'); + assert!(packet.identity.len() <= 300 && !body.ends_with('\u{200d}') && body.ends_with('👦'), "{:?}", packet.identity); +} + +#[test] +fn the_narration_is_cut_at_its_word_budget() { + let long = (0..400).map(|at| format!("w{at}")).collect::>().join(" "); + let cut = cap_words(&long, 150); + assert_eq!(words(&cut), 150); + assert!(cut.ends_with("w149")); + assert_eq!(cap_words(" a b\n c ", 150), "a b\n c"); +} + +#[test] +fn an_overrun_is_cut_at_the_last_sentence_end_inside_the_budget() { + assert_eq!(cap_words("One two three. Four five six! Seven eight nine ten", 8), "One two three. Four five six!"); + assert_eq!(cap_words("She said \"go.\" Then she left the room for good", 6), "She said \"go.\" Then she left", "three of six is under the 60% floor"); + assert_eq!(cap_words("She said \"go.\" Then she left the room for good", 5), "She said \"go.\"", "three of five is the floor"); + assert_eq!(cap_words("Is it late? It is… and quiet now", 5), "Is it late? It is…"); + assert_eq!(cap_words("Version 1.2 is out and running", 3), "Version 1.2 is", "a dot inside a token is no sentence end"); + assert_eq!(cap_words("one two three four five six", 4), "one two three four", "no sentence end inside the budget: the word cut"); + assert_eq!(cap_words("Short. Whole.", 4), "Short. Whole.", "inside the budget nothing is cut"); + let tuning = Tuning::default(); + assert_eq!((word_budget(TurnKind::Chat, &tuning), word_budget(TurnKind::Beat, &tuning)), (150, 90)); +} + +// --------------------------------------------------------------------------------------------- +// The field-name guard. + +#[test] +fn field_names_are_caught_and_prose_is_not() { + let ids = vec!["bond-libby".to_owned(), "t1-m0".to_owned(), "curse".to_owned()]; + for leak in ["Her socialBonds hum.", "SocialBonds hum", "social_bonds hum", "her voiceStyle shifts", "her stableDispositions", "behavioralDimensions", "the inclusion_score", "context_bytes", "beat_words", "presentationStrategy", "currentActivation rises", "bond-libby pulls at her", "the t1-m0 note", "a [curse] tag"] { + assert!(!field_names(leak, &ids).is_empty(), "{leak:?} leaks an identifier"); + } + for prose in ["She keeps it need-to-know.", "A bond-deep loyalty.", "It comes down to trust: she waits.", "The stance-taking tires her.", "Tension: that is the word.", "trust=0.6", "She trusts Libby, and the tension between them is old.", "Her curse is the knife.", "A mind-numbing hour.", "a Learned memory", "Values matter."] { + assert!(field_names(prose, &ids).is_empty(), "{prose:?} is prose: {:?}", field_names(prose, &ids)); + } +} + +// --------------------------------------------------------------------------------------------- +// The tuning file. + +#[test] +fn tuning_absent_is_the_defaults_and_a_file_overrides_and_an_unknown_key_refuses() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(TUNING_FILE); + let absent = tuning::load(&path).unwrap(); + assert_eq!((absent.tuning.clone(), absent.source), (Tuning::default(), "defaults")); + std::fs::write(&path, "[select]\ntemperature = 0.3\n\n[select.caps]\nvalue = 3\n\n[projector]\nwords = 120\n").unwrap(); + let file = tuning::load(&path).unwrap(); + assert_eq!((file.tuning.select.temperature, file.tuning.select.caps.value, file.tuning.projector.words, file.source), (0.3, 3, 120, "file")); + assert_eq!(file.tuning.select.context_bytes, 2500, "an unnamed key keeps its default"); + assert_ne!(file.sha256, absent.sha256); + std::fs::write(&path, "[select]\ntemprature = 0.3\n").unwrap(); + assert!(format!("{:#}", tuning::load(&path).unwrap_err()).contains("temprature"), "a mistyped knob refuses the file"); +} + +#[test] +fn the_hash_is_of_the_effective_tuning() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(TUNING_FILE); + let absent = tuning::load(&path).unwrap(); + std::fs::write(&path, "[select]\ntemperature = 0.7\ncontext_bytes = 2500\n\n[projector]\nwords = 150\n").unwrap(); + let spelled = tuning::load(&path).unwrap(); + assert_eq!((absent.source, spelled.source), ("defaults", "file")); + assert_eq!(absent.sha256, spelled.sha256, "a file that spells out the defaults is the defaults"); + let mut other_build = Tuning::default(); + other_build.select.item_bytes += 1; + assert_ne!(other_build.sha256(), Tuning::default().sha256(), "two builds with different defaults hash differently"); +} + +#[test] +fn a_value_out_of_range_refuses_the_tuning() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(TUNING_FILE); + for (body, names) in [ + ("[select]\ntemperature = nan\n", "select.temperature"), + ("[select]\ntemperature = 0.0\n", "select.temperature"), + ("[select]\ntemperature = -1.0\n", "select.temperature"), + ("[select]\nfreshness_base = -0.5\n", "select.freshness_base"), + ("[select]\ncontext_bytes = 0\n", "select.context_bytes"), + ("[select]\nstimulus_bytes = 0\n", "select.stimulus_bytes"), + ("[select]\nheard_bytes = 0\n", "select.heard_bytes"), + ("[select]\ninclusion_score = 0.0\n", "select.inclusion_score"), + ("[projector]\nwords = 0\n", "projector.words"), + ] { + std::fs::write(&path, body).unwrap(); + let fault = format!("{:#}", tuning::load(&path).unwrap_err()); + assert!(fault.contains(names), "{body:?}: {fault}"); + } +} + +#[test] +fn a_lower_temperature_concentrates_the_selection() { + let room = chat_room(); + let mut rich = state(); + for at in 0..6 { + rich["thoughtMemory"]["memories"].as_array_mut().unwrap().push(json!({ "id": format!("day-{at}"), "summary": format!("Something from day {at}."), "intensity": 0.9 - 0.15 * f64::from(at) })); + } + // Everything is included, so only the order decides what the caps keep. + let mut cold = Tuning::default(); + cold.select.temperature = 0.01; + cold.select.inclusion_score = 1e-9; + let distinct: BTreeSet> = (1..=20).map(|seed| ids(&packet_tuned(TurnKind::Chat, &rich, &room, &History::default(), seed, &cold))).collect(); + assert_eq!(distinct.len(), 1, "near zero temperature takes the top scores every time: {distinct:?}"); +} + +// --------------------------------------------------------------------------------------------- +// `project`: recorded models, the ports it reads through, and the negative check. + +/// Answers the Projector by the packet it was shown, and the judge by which description holds the passage's mark. +struct Recorded { + calls: RefCell>, + narration: Box String>, + judge: Box String>, +} + +impl Recorded { + fn new(narration: impl Fn(&str) -> String + 'static, judge: impl Fn(&str) -> String + 'static) -> Self { + Self { calls: RefCell::default(), narration: Box::new(narration), judge: Box::new(judge) } + } + + /// Narrates "grinds" from a packet with heavy friction and "easy" otherwise; judges by finding which description has it. + fn discriminating() -> Self { + Self::new( + |prompt| if prompt.contains("heavy friction") { "Everything grinds today.".into() } else { "It is easy today.".into() }, + |prompt| { + let (descriptions, passage) = prompt.split_once("Passage:\n").unwrap(); + let (a, b) = descriptions.split_once("\nB:\n").unwrap(); + let grinds = passage.contains("grinds"); + let letter = if a.contains("heavy friction") == grinds { "A" } else { "B" }; + let _ = b; + letter.into() + }, + ) + } +} + +impl Model for Recorded { + fn ask(&self, request_id: &str, model: &str, prompt: &str) -> std::result::Result { + self.calls.borrow_mut().push((request_id.to_owned(), model.to_owned(), prompt.to_owned())); + let text = if prompt.starts_with("