Projector after Soul: no Mind recall; messages to her always whole; sampled inclusion; effective tuning hash and validation; schema-derived guard; budget edges; read-only organs; grapheme clips

F1/F2: the Projector shows her persona state and her room; recall.rs and the
recall pool, recall_sorts and its cap are deleted. Reading the Mind is the
Interpreter's (C3b).
F3: messages to her go in whole and first; others fill what is left, with an
exact note; overflow past the cap is reported.
F4: candidates are included with probability 1 - exp(-score / inclusion_score);
caps are maxima. F5: the hash is of the effective tuning. F6: the guard reads
the persona schema and the tuning keys. F7: bounded fixed slots and relatedness
cue; the redundant final clip is gone. F8: tuning values are validated.
F9: project's organs are read-only traits. F10: clips land on graphemes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Meta Vens 2026-11-19 22:58:20 +01:00
commit 263b6a6af6
11 changed files with 557 additions and 1036 deletions

1
Cargo.lock generated
View file

@ -1133,6 +1133,7 @@ dependencies = [
"tempfile",
"toml",
"unicode-general-category",
"unicode-segmentation",
"uuid",
]

View file

@ -46,6 +46,8 @@ serde_json = "1"
sha2 = "0.10"
toml = { version = "0.9", default-features = false, features = ["std", "serde", "parse"] }
unicode-general-category = "1"
# Clips on grapheme boundaries (`projection::select::clip`), so no ZWJ sequence is split; already in the graph through Ghostlight.
unicode-segmentation = "1"
uuid = { version = "1", default-features = false, features = ["v5"] }
[target.'cfg(unix)'.dependencies]

View file

@ -30,13 +30,11 @@ use std::time::Duration;
use anyhow::{Context, Result, anyhow, bail};
use eureka_state::HuginnClient;
use mind_body::chat_client::Chat;
use mind_body::chat_rudp::{RudpChat, connection_id};
use mind_body::cli::instance_and_endpoint;
use mind_body::control::{CONTROL_ROOT, FileSource, control_path, swarm_path};
use mind_body::persona::{Config, carry};
use mind_body::projection::{HuginnPersonaSource, KitModel, Model, Organs as ProjectOrgans, Request as ProjectRequest, TUNING_FILE, TUNING_ROOT, TurnKind, connector_key, project};
use mind_body::queue::HuginnMind;
use mind_body::projection::{HuginnPersonaSource, KitModel, Model, MouthReader, Organs as ProjectOrgans, Request as ProjectRequest, TUNING_FILE, TUNING_ROOT, TurnKind, connector_key, project};
const USAGE: &str = "usage: mind-persona --instance <slug> [--endpoint rudp://<ip>:<port>] [--rooms a,b] [--operator id[,id]] [--minds id[,id]] [--self-sender id]";
const PROJECT_USAGE: &str = "usage: mind-persona project --instance <slug> [--endpoint rudp://<ip>:<port>] [--kind chat|beat|reflection] [--room <r>] [--seed N] [--state <file>] [--select-only | --renders K [--judge <state-file>]]";
@ -115,7 +113,6 @@ fn run_project(args: &[String]) -> Result<String> {
let now = chrono::Utc::now();
let tuning = setting("MIND_PERSONA_TUNING").map_or_else(|| Path::new(TUNING_ROOT).join(&instance.0).join(TUNING_FILE), PathBuf::from);
let persona = HuginnPersonaSource::new(HuginnClient::new(address, instance.clone(), TIMEOUT));
let mind = HuginnMind::new(HuginnClient::new(address, instance.clone(), TIMEOUT));
let chat = match (setting("BIFROST_CHAT_ENDPOINT"), setting("BIFROST_CHAT_KEY_FILE")) {
(Some(endpoint), Some(_)) => {
let endpoint: SocketAddr = endpoint.strip_prefix("rudp://").and_then(|address| address.parse().ok()).ok_or_else(|| anyhow!("BIFROST_CHAT_ENDPOINT is not rudp://<ip>:<port>"))?;
@ -131,7 +128,7 @@ fn run_project(args: &[String]) -> Result<String> {
}
None => None,
};
let organs = ProjectOrgans { persona: Some(&persona), mind: Some(&mind), chat: chat.as_ref().map(|chat| chat as &dyn Chat), model: model.as_ref().map(|model| model as &dyn Model) };
let organs = ProjectOrgans { persona: Some(&persona), chat: chat.as_ref().map(|chat| chat as &dyn MouthReader), model: model.as_ref().map(|model| model as &dyn Model) };
let request = ProjectRequest {
instance: instance.0.clone(),
kind,

View file

@ -46,7 +46,6 @@ pub mod lease_client;
pub mod persona;
pub mod projection;
pub mod queue;
mod recall;
pub mod self_run;
pub mod tempo;

View file

@ -6,15 +6,18 @@
//! the pinned `ghostlight-persona-projection`). Every knob of both is `persona-tuning.toml` (`tuning`), read at each
//! wake, its hash carried with the packet.
//!
//! The Projector shows her self and her room: her persona state and what the mouth showed. It does not read her Mind;
//! that is the Interpreter's read tools' (C3b), when wondering leads to reading.
//!
//! `project` is the experiment tool over both (`mind-persona project`): what Select would build from her real or a
//! hypothetical state, and what the Projector would narrate. It has no port that writes: it puts no persona state,
//! admits nothing, sends and acks no chat and appends no trace, so it runs under a held brake.
//! hypothetical state, and what the Projector would narrate. Its organs are read-only by type: `PersonaSource` gets,
//! `MouthReader` reads pending chat, `Model` asks. None of them can put, admit, send, ack or open a conversation, so
//! `project` cannot write and runs under a held brake.
mod narrate;
mod select;
mod tuning;
use std::collections::BTreeMap;
use std::path::PathBuf;
use anyhow::{Result, anyhow, bail};
@ -23,23 +26,17 @@ use eureka_state::HuginnClient;
use huginn_mind::HuginnMindResponse;
use serde_json::{Value, json};
use crate::chat_client::{Chat, Form};
use crate::queue::MindPort;
use crate::recall;
use crate::chat_client::{Chat, ChatPending, Form};
pub use narrate::{KitModel, Model, connector_key};
pub use select::TurnKind;
pub use tuning::{TUNING_FILE, TUNING_ROOT};
use narrate::{cap_words, distinct_2, field_names, judge_prompt, judged, projector_prompt, word_budget, words};
use select::{History, Inputs, Packet, People, Recalled, Room, RoomLine, select};
use select::{History, Inputs, Packet, People, Room, RoomLine, select};
/// The agent name on the connector's request ids.
const AGENT: &str = "mind-persona";
/// The touch of a recalled item by the lane that brought it.
const TOUCH_ABOUT: f64 = 1.0;
const TOUCH_SALIENT: f64 = 0.3;
const TOUCH_RELEVANT_UNSCORED: f64 = 0.6;
/// The judge passes at this share of narrations attributed to the right packet.
const JUDGE_PASS: f64 = 0.8;
@ -70,14 +67,36 @@ impl PersonaSource for HuginnPersonaSource {
}
}
/// What `project` reads through. None of them is asked to write.
/// The mouth, read: pending deliveries and the room tail, which acks nothing. The only chat port `project` holds.
pub trait MouthReader {
fn pending(&self) -> Result<ChatPending>;
}
impl<T: Chat> MouthReader for T {
fn pending(&self) -> Result<ChatPending> {
Chat::pending(self, 0)
}
}
/// What `project` reads through. Each is a read-only port by type: holding an `Organs`, no code can send, ack or open
/// a conversation, or put her state.
///
/// ```compile_fail
/// fn acks(organs: &mind_body::projection::Organs) {
/// organs.chat.unwrap().ack(1).unwrap();
/// }
/// ```
///
/// ```compile_fail
/// fn sends(organs: &mind_body::projection::Organs, request: &mind_body::chat_client::ChatSendRequest) {
/// organs.chat.unwrap().send(request).unwrap();
/// }
/// ```
pub struct Organs<'a> {
/// Her stored state; not asked when the request carries a state.
pub persona: Option<&'a dyn PersonaSource>,
/// Her Mind, for recall; none recalls nothing.
pub mind: Option<&'a dyn MindPort>,
/// The mouth, for the room; none shows no room.
pub chat: Option<&'a dyn Chat>,
pub chat: Option<&'a dyn MouthReader>,
/// The Projector and the judge; required for renders.
pub model: Option<&'a dyn Model>,
}
@ -103,8 +122,8 @@ pub struct Request {
/// The room through the mouth's pending read, which acks nothing: new messages and the tail, own lines kept for the
/// Select stage to drop. A refusal is reported, never hidden.
fn read_room(chat: &dyn Chat, request: &Request, public_name: &str) -> (Room, Value) {
let pending = match chat.pending(0) {
fn read_room(chat: &dyn MouthReader, request: &Request, public_name: &str) -> (Room, Value) {
let pending = match chat.pending() {
Ok(pending) => pending,
Err(fault) => return (Room { here: request.room.clone(), ..Room::default() }, json!({ "read": false, "why": format!("{fault:#}") })),
};
@ -133,37 +152,6 @@ fn read_room(chat: &dyn Chat, request: &Request, public_name: &str) -> (Room, Va
(Room { messages, tail, here: request.room.clone() }, report)
}
/// What her Mind recalls for this room, as Select candidates: memories of the tuning's `recall_sorts` and nothing else.
fn read_recall(mind: &dyn MindPort, room: &Room, sorts: &[String], now: DateTime<Utc>) -> Result<(Vec<Recalled>, Value)> {
let lines = room.messages.iter().chain(room.tail.iter());
let mut senders: Vec<&str> = lines.clone().map(|line| line.sender.as_str()).collect();
senders.dedup();
let names: BTreeMap<&str, &str> = lines.map(|line| (line.sender.as_str(), line.display.as_str())).filter(|(_, name)| !name.is_empty()).collect();
let texts: Vec<&str> = room.messages.iter().rev().map(|line| line.text.as_str()).collect();
let cue = recall::Cue { senders, texts, names, sorts, now };
let recalled = recall::select(mind, &cue)?;
let mut out = Vec::new();
for shown in &recalled.lanes {
for entry in &shown.entries {
let touch = match shown.lane {
recall::Lane::About => TOUCH_ABOUT,
recall::Lane::Relevant => entry.nearness_permille.map_or(TOUCH_RELEVANT_UNSCORED, |near| f64::from(near.max(1)) / 1000.0),
recall::Lane::Salient => TOUCH_SALIENT,
};
out.push(Recalled {
id: entry.id.clone(),
text: entry.line.trim().trim_start_matches("- ").to_owned(),
salience: f64::from(entry.salience_permille) / 1000.0,
admitted_at: DateTime::parse_from_rfc3339(&entry.admitted_at).ok().map(|at| at.with_timezone(&Utc)),
touch,
});
}
}
let search = format!("{:?}", recalled.search).to_lowercase();
let report = json!({ "asked": true, "sorts": sorts, "items": out.len(), "search": search });
Ok((out, report))
}
fn packet_json(packet: &Packet) -> Value {
json!({
"kind": packet.kind,
@ -172,6 +160,7 @@ fn packet_json(packet: &Packet) -> Value {
"present": packet.present,
"selected": packet.selected,
"bytes": { "identity": packet.identity.len(), "context": packet.context.len(), "identity_and_context": packet.identity.len() + packet.context.len(), "stimulus": packet.stimulus.len(), "heard": packet.heard.len() },
"overflow": packet.overflow,
"identity": packet.identity,
"context": packet.context,
"stimulus": packet.stimulus,
@ -187,13 +176,14 @@ fn narrate_one(model: &dyn Model, instance: &str, packet: &Packet, tuning: &tuni
let budget = word_budget(packet.kind, tuning);
let narration = cap_words(&raw, budget);
let ids: Vec<String> = packet.selected.iter().map(|pick| pick.id.clone()).collect();
let named: Vec<String> = packet.ids.clone();
Ok(json!({
"seed": packet.seed,
"lens": packet.lens,
"selected": ids,
"words": words(&narration),
"cut_at_budget": words(&raw) > budget,
"field_names": field_names(&narration, &ids),
"field_names": field_names(&narration, &named),
"narration": narration,
}))
}
@ -226,14 +216,10 @@ pub fn project(organs: &Organs, request: &Request, now: DateTime<Utc>) -> Result
Some(chat) => read_room(chat, request, &public_name),
None => (Room { here: request.room.clone(), ..Room::default() }, json!({ "read": false, "why": "no mouth is configured" })),
};
let (recall, recall_report) = match organs.mind {
Some(mind) => read_recall(mind, &room, &tuning.select.recall_sorts, now)?,
None => (Vec::new(), json!({ "asked": false, "sorts": tuning.select.recall_sorts })),
};
// Freshness and her own recent words come from trace v2, which the commit writes (C3c); until then there is none.
let history = History::default();
let people = People { operators: &request.operators, own_sender: &request.own_sender };
let inputs = Inputs { kind: request.kind, state: &state, room: &room, recall: &recall, history: &history, observation: None, people, now, seed: request.seed };
let inputs = Inputs { kind: request.kind, state: &state, room: &room, history: &history, observation: None, people, now, seed: request.seed };
let mut report = json!({
"instance": request.instance,
@ -241,8 +227,7 @@ pub fn project(organs: &Organs, request: &Request, now: DateTime<Utc>) -> Result
"state": if request.state.is_some() { "given" } else { "stored" },
"tuning": { "path": request.tuning.display().to_string(), "source": loaded.source, "sha256": loaded.sha256 },
"room": room_report,
"recall": recall_report,
"history": "none: trace v2 is not written yet",
"history": "none: freshness and her own recent words come from trace v2, which the commit writes (C3c)",
});
let Some(renders) = request.renders else {
report["packet"] = packet_json(&select(&inputs, tuning));

View file

@ -116,56 +116,72 @@ pub(crate) fn words(text: &str) -> usize {
text.split_whitespace().count()
}
/// Schema keys of her persona state, as identifiers. A narration that carries one leaked the storage.
const SCHEMA_KEYS: [&str; 22] = [
"currentActivation", "socialBonds", "doctrineStances", "thoughtMemory", "shortTerm", "agencyPressure", "activationProfile", "situationalState", "moodDimensions", "candidateActions", "privateNotes",
"publicName", "publicDescription", "voiceSummary", "relationshipKind", "lastEvidence", "actionImplication", "schemaVersion", "updatedAt", "createdAt", "salience_permille", "persona_state",
];
/// Id prefixes of typed state, and the organ labels a `mind-` id begins with (a bare `mind-` is prose: "mind-numbing").
const ID_PREFIXES: [&str; 10] = ["bond-", "need-", "stance-", "mind-commit", "mind-persona", "mind-heartbeat", "mind-stream", "mind-control", "mind-self", "mind-eureka"];
/// Keys a number is stored under: the word is prose, the word followed by `:` or `=` is a field.
const VALUE_KEYS: [&str; 8] = ["trust", "tension", "intensity", "plasticity", "valence", "arousal", "dominance", "salience"];
/// Her persona state's schema: its property names are the identifiers a narration must never carry.
const PERSONA_SCHEMA: &str = include_str!("../../../../schemas/cultnet/gamecult.persona_state.v0.schema.json");
/// The identifiers of typed state a narration carries: schema keys, typed ids, `key: value` fields, and the packet's own
/// selected ids where an id is no plain word. A small deterministic guard on identifiers, never on meaning.
pub(crate) fn field_names(narration: &str, selected_ids: &[String]) -> Vec<String> {
/// An identifier as compared: lowercased with its underscores gone, so `socialBonds`, `SocialBonds` and `social_bonds`
/// are one.
fn folded(identifier: &str) -> String {
identifier.chars().filter(|c| *c != '_').flat_map(char::to_lowercase).collect()
}
/// Whether a word is shaped like an identifier rather than prose: an underscore, or a capital after its first letter.
fn identifier_shaped(word: &str) -> bool {
word.contains('_') || word.chars().skip(1).any(char::is_uppercase)
}
fn keys_of(value: &serde_json::Value, schema: bool, into: &mut BTreeSet<String>) {
match value {
serde_json::Value::Object(map) => {
for (key, inner) in map {
if schema && key == "properties"
&& let Some(properties) = inner.as_object()
{
into.extend(properties.keys().cloned());
}
if !schema {
into.insert(key.clone());
}
keys_of(inner, schema, into);
}
}
serde_json::Value::Array(values) => values.iter().for_each(|inner| keys_of(inner, schema, into)),
_ => {}
}
}
/// The folded compound identifiers of the persona schema and of the tuning: a plain word such as `trust` or
/// `tension` is prose and is not among them.
fn identifiers() -> BTreeSet<String> {
let mut keys = BTreeSet::new();
if let Ok(schema) = serde_json::from_str::<serde_json::Value>(PERSONA_SCHEMA) {
keys_of(&schema, true, &mut keys);
}
if let Ok(tuning) = serde_json::to_value(Tuning::default()) {
keys_of(&tuning, false, &mut keys);
}
keys.into_iter().filter(|key| identifier_shaped(key)).map(|key| folded(&key)).collect()
}
/// The identifiers of typed state a narration carries: a word shaped like an identifier that names a property of her
/// persona schema or a tuning key, in any case convention, and any id the packet named (`ids`), as an exact token when
/// it cannot be a plain word, or inside brackets as the packet writes it. A small deterministic guard on identifiers,
/// never on meaning: "need-to-know", "trust:" and "Tension:" are prose.
pub(crate) fn field_names(narration: &str, ids: &[String]) -> Vec<String> {
let known = identifiers();
let mut found = BTreeSet::new();
let tokens: Vec<&str> = narration.split(|c: char| !(c.is_alphanumeric() || c == '-' || c == '_' || c == ':')).filter(|token| !token.is_empty()).collect();
for token in &tokens {
let bare = token.trim_matches(|c: char| c == ':' || c == '-');
if SCHEMA_KEYS.contains(&bare) {
found.insert(bare.to_owned());
}
if ID_PREFIXES.iter().any(|prefix| bare.to_lowercase().starts_with(prefix) && bare.trim_end_matches('-').len() > "bond".len()) {
found.insert(bare.to_owned());
for word in narration.split(|c: char| !(c.is_alphanumeric() || c == '_')).filter(|word| !word.is_empty()) {
if identifier_shaped(word) && known.contains(&folded(word)) {
found.insert(word.to_owned());
}
}
let lower = narration.to_lowercase();
for key in VALUE_KEYS {
let mut from = 0;
while let Some(at) = lower[from..].find(key) {
let start = from + at;
let end = start + key.len();
let word_start = start == 0 || !lower[..start].chars().next_back().is_some_and(char::is_alphanumeric);
let rest = lower[end..].trim_start_matches([' ', '"', '\'']);
if word_start && (rest.starts_with(':') || rest.starts_with('=')) {
found.insert(key.to_owned());
}
from = end;
}
}
for id in selected_ids.iter().filter(|id| id.contains(['-', '_', ':']) || id.chars().any(|c| c.is_ascii_digit())) {
if narration.contains(id.as_str()) {
let tokens: BTreeSet<&str> = narration.split_whitespace().map(|token| token.trim_matches(|c: char| !(c.is_alphanumeric() || c == '-' || c == '_' || c == ':'))).collect();
for id in ids {
let compound = id.contains(['-', '_', ':']) || id.chars().any(|c| c.is_ascii_digit());
if (compound && tokens.contains(id.as_str())) || narration.contains(&format!("[{id}]")) {
found.insert(id.clone());
}
}
if narration.contains('[') && narration.contains(']') {
for id in selected_ids {
if narration.contains(&format!("[{id}]")) {
found.insert(id.clone());
}
}
}
found.into_iter().collect()
}

View file

@ -1,10 +1,14 @@
//! The Select stage: what she can feel this turn. Code, deterministic for a seed, and the one owner of the
//! projection packet; the Projector model only renders what this chose.
//!
//! Every packet carries its fixed slots: identity, the hour, the stimulus, a bond line for each person present and
//! her mood as change. The rest is sampled without replacement from the softmax of `log(score) / T`, pool by pool
//! under its cap, until the context budget is spent, where
//! `score = salience × recency × touch × unresolved × freshness`.
//! The Projector shows her self and her room: her persona state and what the mouth showed. Her Mind is not read
//! here; reaching into it is the Interpreter's, through its read tools, when wondering leads to reading.
//!
//! Every packet carries its fixed slots: identity, the hour, her mood as change, a bond line for each person
//! present, and the stimulus. The rest comes from her state's pools. Each candidate is ordered by
//! `log(score) / T` plus Gumbel noise and included with probability `1 - exp(-score / inclusion_score)`, where
//! `score = salience × recency × touch × unresolved × freshness`; then each pool's cap and the context budget bound
//! what is kept. A cap is a maximum, never a target: a pool with fewer candidates than its cap still rotates.
//!
//! Never read: `agencyPressure`, `candidateActions`, `privateNotes` and `provenance`. A memory tagged with a direct
//! message room (`room:!…`) is selectable only in that room.
@ -15,6 +19,7 @@ use std::fmt::Write as _;
use chrono::{DateTime, Utc};
use serde::Serialize;
use serde_json::Value;
use unicode_segmentation::UnicodeSegmentation;
use super::tuning::{LensWeights, Tuning};
use crate::address::localpart;
@ -30,6 +35,10 @@ const SALIENCE_UNSTATED: f64 = 0.5;
const MOVE_MIN: f64 = 0.03;
/// Incubation's unresolved factor caps here.
const INCUBATION_UNRESOLVED_MAX: f64 = 2.0;
/// The most bytes a bond's name or kind takes in its line.
const NAME_BYTES: usize = 80;
/// The most bytes the line naming the unbonded people present takes.
const ALSO_HERE_BYTES: usize = 300;
/// What kind of turn the packet is for. The stimulus and nothing else differs between them.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
@ -61,16 +70,6 @@ pub(crate) struct Room {
pub here: Option<String>,
}
/// One item her Mind recalled, as `recall::select` read it.
#[derive(Clone, Debug, PartialEq)]
pub(crate) struct Recalled {
pub id: String,
pub text: String,
pub salience: f64,
pub admitted_at: Option<DateTime<Utc>>,
pub touch: f64,
}
/// One thing she said, from the trace.
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct Said {
@ -81,7 +80,8 @@ pub(crate) struct Said {
}
/// What earlier turns left: the ids each of the last projections selected, oldest first, what she said, and her state
/// at her last turn (movement is measured against it).
/// at her last turn (movement is measured against it). The commit's trace v2 (C3c) is its source; until then it is
/// empty.
#[derive(Clone, Debug, Default, PartialEq)]
pub(crate) struct History {
pub recent: Vec<Vec<String>>,
@ -107,7 +107,6 @@ pub(crate) struct Inputs<'a> {
pub kind: TurnKind,
pub state: &'a Value,
pub room: &'a Room,
pub recall: &'a [Recalled],
pub history: &'a History,
/// A follow-up's finding.
pub observation: Option<&'a str>,
@ -148,7 +147,6 @@ pub(crate) enum Pool {
Needs,
Incubation,
Stance,
Recall,
Value,
Said,
}
@ -173,10 +171,20 @@ pub(crate) struct Packet {
pub stimulus: String,
/// What was said to her, verbatim, rendered by code for the Persona stage.
pub heard: String,
/// Bytes the messages to her took past the stimulus and heard caps: verbatim beats the cap.
pub overflow: Overflow,
pub present: Vec<String>,
/// Every typed id the packet's lines name: the selected items' and the bonds' of the people present.
pub ids: Vec<String>,
pub selected: Vec<Pick>,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize)]
pub(crate) struct Overflow {
pub stimulus: usize,
pub heard: usize,
}
struct Candidate {
id: String,
pool: Pool,
@ -207,13 +215,14 @@ fn time(item: &Value) -> Option<DateTime<Utc>> {
["updatedAt", "createdAt"].iter().find_map(|key| DateTime::parse_from_rfc3339(text(item, &[key])).ok()).map(|at| at.with_timezone(&Utc))
}
/// `text` cut to at most `max` bytes on a character boundary, with an ellipsis where it was cut.
/// `text` cut to at most `max` bytes on a grapheme boundary, with an ellipsis where it was cut.
pub(crate) fn clip(text: &str, max: usize) -> String {
let text = text.trim();
if text.len() <= max {
return text.to_owned();
}
let end = (0..=max.saturating_sub(3)).rev().find(|at| text.is_char_boundary(*at)).unwrap_or(0);
let room = max.saturating_sub('…'.len_utf8());
let end = text.grapheme_indices(true).map(|(at, grapheme)| at + grapheme.len()).take_while(|end| *end <= room).last().unwrap_or(0);
format!("{}…", &text[..end])
}
@ -331,7 +340,7 @@ fn draw_lens(weights: &LensWeights, present: &[String], candidates: &[Candidate]
(weights.body_and_hour, Lens::BodyAndHour),
(if present.is_empty() { 0.0 } else { weights.person_present }, Lens::PersonPresent(present.first().cloned().unwrap_or_default())),
(if has(&[Pool::Needs, Pool::Incubation]) { weights.unresolved_want } else { 0.0 }, Lens::UnresolvedWant),
(if has(&[Pool::MemoriesAboutPresent, Pool::OtherMemories, Pool::Recall]) { weights.memory_surfacing } else { 0.0 }, Lens::MemorySurfacing),
(if has(&[Pool::MemoriesAboutPresent, Pool::OtherMemories]) { weights.memory_surfacing } else { 0.0 }, Lens::MemorySurfacing),
(if has(&[Pool::Incubation]) { weights.wondering } else { 0.0 }, Lens::Wondering),
];
let total: f64 = options.iter().map(|(weight, _)| weight.max(0.0)).sum();
@ -349,10 +358,12 @@ fn draw_lens(weights: &LensWeights, present: &[String], candidates: &[Candidate]
// ---------------------------------------------------------------------------------------------
// Candidates.
/// `ln(base ^ k)`, k the times `id` was shown in the last `freshness_window` projections: 0 when never shown, minus
/// infinity when the base is 0 and it was shown at all.
fn freshness_log(id: &str, history: &History, tuning: &Tuning) -> f64 {
let window = &history.recent[history.recent.len().saturating_sub(tuning.select.freshness_window)..];
let shown = window.iter().filter(|ids| ids.iter().any(|seen| seen == id)).count();
shown as f64 * tuning.select.freshness_base.ln()
if shown == 0 { 0.0 } else { tuning.select.freshness_base.powi(i32::try_from(shown).unwrap_or(i32::MAX)).ln() }
}
fn recency_log(at: Option<DateTime<Utc>>, now: DateTime<Utc>, tau_h: f64) -> f64 {
@ -421,13 +432,8 @@ fn candidates(inputs: &Inputs, tuning: &Tuning, present: &BTreeSet<String>) -> V
push(text(stance, &["id"]), Pool::Stance, line, number(stance, &["intensity"]).unwrap_or(SALIENCE_UNSTATED), 0.0, touch, 1.0);
}
for recalled in inputs.recall {
let line = format!("- her mind recalls [{}]: {}", recalled.id, clip(&one_line(&recalled.text), item_bytes));
push(&recalled.id, Pool::Recall, line, recalled.salience, recency_log(recalled.admitted_at, inputs.now, tuning.select.recall_tau_h), recalled.touch, 1.0);
}
for value in items(state, &["values"]) {
let line = format!("- value [{}] {}: {}", text(value, &["id"]), text(value, &["label"]), clip(&one_line(text(value, &["summary"])), item_bytes));
let line = format!("- value [{}] {}: {}", text(value, &["id"]), clip(text(value, &["label"]), NAME_BYTES), clip(&one_line(text(value, &["summary"])), item_bytes));
push(text(value, &["id"]), Pool::Value, line, number(value, &["priority"]).unwrap_or(SALIENCE_UNSTATED), 0.0, TOUCH_ELSE, 1.0);
}
@ -447,7 +453,6 @@ fn cap_of(pool: Pool, tuning: &Tuning) -> usize {
Pool::Needs => caps.needs,
Pool::Incubation => caps.incubation,
Pool::Stance => caps.stance,
Pool::Recall => caps.recall,
Pool::Value => caps.value,
Pool::Said => caps.said,
}
@ -459,16 +464,16 @@ fn cap_of(pool: Pool, tuning: &Tuning) -> usize {
fn name_of(line: &RoomLine, inputs: &Inputs) -> String {
let person = inputs.people.person_of(&line.sender);
let bonded = items(inputs.state, &["affect", "socialBonds"]).iter().find(|bond| text(bond, &["object", "id"]).to_lowercase() == person).map(|bond| text(bond, &["object", "label"]).to_owned());
bonded.filter(|label| !label.is_empty()).or_else(|| Some(line.display.trim().to_owned()).filter(|name| !name.is_empty())).unwrap_or_else(|| localpart(&line.sender).to_owned())
let name = bonded.filter(|label| !label.is_empty()).or_else(|| Some(line.display.trim().to_owned()).filter(|name| !name.is_empty())).unwrap_or_else(|| localpart(&line.sender).to_owned());
clip(&one_line(&name), NAME_BYTES)
}
/// The newest lines that fit `max` bytes after `head`, oldest first, and a note when older ones did not.
fn fitted(head: &str, lines: Vec<String>, max: usize) -> String {
/// Whole lines, newest kept, that fit `budget` bytes with their newlines, and how many older ones did not.
fn newest_that_fit(lines: &[String], budget: usize) -> (Vec<String>, usize) {
let mut kept: Vec<String> = Vec::new();
let mut used = head.len();
let reserve = 48;
let mut used = 0;
for line in lines.iter().rev() {
if used + line.len() + 1 + reserve > max {
if used + line.len() + 1 > budget {
break;
}
used += line.len() + 1;
@ -476,26 +481,54 @@ fn fitted(head: &str, lines: Vec<String>, max: usize) -> String {
}
kept.reverse();
let dropped = lines.len() - kept.len();
let mut out = head.to_owned();
if dropped > 0 {
let _ = writeln!(out, "({dropped} earlier not shown)");
}
for line in kept {
out.push_str(&line);
out.push('\n');
}
out.trim_end().to_owned()
(kept, dropped)
}
fn chat_stimulus(inputs: &Inputs, tuning: &Tuning) -> (String, String) {
/// A section: its heading, the lines to her whole, then the newest other lines that fit what is left of `cap`, with a
/// note counting the others that did not. Returns the text and how many bytes the lines to her took past `cap`.
fn section_to_her(heading: &str, to_her: &[String], others_heading: &str, others: &[String], cap: usize) -> (String, usize) {
let mut out = String::new();
if !to_her.is_empty() {
out.push_str(heading);
for line in to_her {
out.push_str(line);
out.push('\n');
}
}
if !others.is_empty() {
let heading = if to_her.is_empty() { heading } else { others_heading };
// The note is at most this long: "(NNNNN earlier not shown)\n".
let note_room = 32;
let left = cap.saturating_sub(out.len() + heading.len() + note_room);
let (kept, dropped) = newest_that_fit(others, left);
if !kept.is_empty() || dropped > 0 {
out.push_str(heading);
}
if dropped > 0 {
let _ = writeln!(out, "({dropped} earlier not shown)");
}
for line in kept {
out.push_str(&line);
out.push('\n');
}
}
let out = out.trim_end().to_owned();
let over = out.len().saturating_sub(cap);
(out, over)
}
fn chat_stimulus(inputs: &Inputs, tuning: &Tuning) -> (String, String, Overflow) {
let messages: Vec<&RoomLine> = inputs.room.messages.iter().filter(|line| line.sender != inputs.people.own_sender).collect();
if messages.is_empty() {
return ("Nothing new was said.".into(), String::new());
return ("Nothing new was said.".into(), String::new(), Overflow::default());
}
let typed = messages.iter().map(|line| format!("- {} in {}{}: {}", name_of(line, inputs), line.room, if line.to_her { ", to her" } else { "" }, quoted(&line.text))).collect();
let typed = |line: &&RoomLine| format!("- {} in {}: {}", name_of(line, inputs), line.room, quoted(&line.text));
// Heard lines are prose for the Persona: the words exactly as said, inside plain quotation marks.
let heard = messages.iter().map(|line| format!("{}, in {}: \"{}\"", name_of(line, inputs), line.room, line.text)).collect();
(fitted("New in the room:\n", typed, tuning.select.stimulus_bytes), fitted("", heard, tuning.select.heard_bytes))
let heard = |line: &&RoomLine| format!("{}, in {}: \"{}\"", name_of(line, inputs), line.room, line.text);
let (to_her, others): (Vec<&RoomLine>, Vec<&RoomLine>) = messages.into_iter().partition(|line| line.to_her);
let (stimulus, stimulus_over) = section_to_her("Said to her:\n", &to_her.iter().map(typed).collect::<Vec<_>>(), "Also in the room:\n", &others.iter().map(typed).collect::<Vec<_>>(), tuning.select.stimulus_bytes);
let (heard_text, heard_over) = section_to_her("", &to_her.iter().map(heard).collect::<Vec<_>>(), "", &others.iter().map(heard).collect::<Vec<_>>(), tuning.select.heard_bytes);
(stimulus, heard_text, Overflow { stimulus: stimulus_over, heard: heard_over })
}
fn hours(now: DateTime<Utc>, at_ms: u64) -> i64 {
@ -503,48 +536,54 @@ fn hours(now: DateTime<Utc>, at_ms: u64) -> i64 {
(now - at).num_hours().max(0)
}
/// A beat: nothing addressed her, when she last heard from each person she is bound to (always shown: it is the
/// relatedness cue), then the newest lines of her rooms in what the budget leaves.
/// A beat: nothing addressed her; when she last heard from each person she is bound to (the relatedness cue, most
/// recently heard first, a note counting any the budget left out); then the newest lines of her rooms in what is left.
fn beat_stimulus(inputs: &Inputs, tuning: &Tuning) -> String {
let mut head = String::from("Nothing addressed her.
");
let cap = tuning.select.stimulus_bytes;
let mut contacts: Vec<(Option<u64>, String)> = Vec::new();
for bond in items(inputs.state, &["affect", "socialBonds"]).iter().filter(|bond| live(bond) && text(bond, &["object", "kind"]) == "person") {
let person = text(bond, &["object", "id"]).to_lowercase();
let label = [text(bond, &["object", "label"]), text(bond, &["object", "id"])].into_iter().find(|name| !name.is_empty()).unwrap_or_default();
let label = clip(&one_line([text(bond, &["object", "label"]), text(bond, &["object", "id"])].into_iter().find(|name| !name.is_empty()).unwrap_or_default()), NAME_BYTES);
let last = inputs.room.tail.iter().filter(|line| inputs.people.person_of(&line.sender) == person).map(|line| line.at_ms).max();
let _ = match last {
Some(at) => writeln!(head, "- last heard from {label} {} h ago", hours(inputs.now, at)),
None => writeln!(head, "- nothing from {label} in her rooms lately"),
};
contacts.push((last, match last {
Some(at) => format!("- last heard from {label} {} h ago", hours(inputs.now, at)),
None => format!("- nothing from {label} in her rooms lately"),
}));
}
let head = clip_lines(&head, tuning.select.stimulus_bytes);
let others: Vec<&RoomLine> = inputs.room.tail.iter().filter(|line| line.sender != inputs.people.own_sender).collect();
let shown = &others[others.len().saturating_sub(tuning.select.beat_tail_lines)..];
if shown.is_empty() {
return head;
}
let lines = shown.iter().map(|line| format!("- {} in {}, {} h ago: {}", name_of(line, inputs), line.room, hours(inputs.now, line.at_ms), quoted(&clip(&line.text, tuning.select.item_bytes / 2)))).collect();
fitted(&format!("{head}
Lately in her rooms:
"), lines, tuning.select.stimulus_bytes)
}
/// `text` cut at a line boundary to `max` bytes.
fn clip_lines(text: &str, max: usize) -> String {
let mut out = String::new();
for line in text.lines() {
if out.len() + line.len() + 1 > max {
break;
contacts.sort_by_key(|(last, _)| std::cmp::Reverse(*last));
let mut out = String::from("Nothing addressed her.\n");
// The relatedness cue takes at most two thirds of the stimulus, so the room keeps a share.
let note_room = 40;
let contact_cap = (cap * 2 / 3).saturating_sub(out.len() + note_room);
let mut used = 0;
let mut left_out = 0;
for (_, line) in &contacts {
if used + line.len() + 1 > contact_cap {
left_out += 1;
continue;
}
used += line.len() + 1;
out.push_str(line);
out.push('\n');
}
if left_out > 0 {
let _ = writeln!(out, "({left_out} more people she is bound to not shown)");
}
let others: Vec<&RoomLine> = inputs.room.tail.iter().filter(|line| line.sender != inputs.people.own_sender).collect();
let shown = &others[others.len().saturating_sub(tuning.select.beat_tail_lines)..];
let lines: Vec<String> = shown.iter().map(|line| format!("- {} in {}, {} h ago: {}", name_of(line, inputs), line.room, hours(inputs.now, line.at_ms), quoted(&clip(&line.text, tuning.select.item_bytes / 2)))).collect();
let (section, _) = section_to_her("Lately in her rooms:\n", &[], "", &lines, cap.saturating_sub(out.len() + 1));
if !section.is_empty() {
out.push('\n');
out.push_str(&section);
}
out.trim_end().to_owned()
}
fn reflection_stimulus(inputs: &Inputs, tuning: &Tuning) -> String {
let lines = items(inputs.state, &["thoughtMemory", "shortTerm"]).iter().filter(|item| live(item)).map(|item| format!("- {}", clip(&one_line(text(item, &["summary"])), tuning.select.item_bytes))).collect();
fitted("Since she last took stock:\n", lines, tuning.select.stimulus_bytes)
let lines: Vec<String> = items(inputs.state, &["thoughtMemory", "shortTerm"]).iter().filter(|item| live(item)).map(|item| format!("- {}", clip(&one_line(text(item, &["summary"])), tuning.select.item_bytes))).collect();
let (section, _) = section_to_her("Since she last took stock:\n", &[], "", &lines, tuning.select.stimulus_bytes);
if section.is_empty() { "Since she last took stock: nothing new.".into() } else { section }
}
/// The people present, newest first: who spoke to her or in her room this turn, or in the beat's lines.
@ -572,7 +611,7 @@ fn bond_line(bond: &Value, name: &str, tuning: &Tuning) -> String {
if let Some(trust) = number(bond, &["trust"]) {
feel.insert(0, trust_words(trust));
}
let kind = text(bond, &["relationshipKind"]);
let kind = clip(&one_line(text(bond, &["relationshipKind"])), NAME_BYTES);
let kind = if kind.is_empty() { String::new() } else { format!(", {kind}") };
format!("- bond [{}] {name}{kind}: {}. {}", text(bond, &["id"]), feel.join(", "), clip(&one_line(text(bond, &["summary"])), tuning.select.bond_summary_bytes))
}
@ -586,54 +625,97 @@ fn identity_of(state: &Value, tuning: &Tuning) -> String {
clip(&whole, tuning.select.identity_bytes)
}
/// The names of unbonded people present, as many as fit `ALSO_HERE_BYTES`, then how many more.
fn also_here(named: &[String]) -> Option<String> {
if named.is_empty() {
return None;
}
let mut line = String::from("- also here: ");
let mut shown = 0;
for name in named {
let more_room = 24;
if line.len() + name.len() + 2 + more_room > ALSO_HERE_BYTES {
break;
}
if shown > 0 {
line.push_str(", ");
}
line.push_str(name);
shown += 1;
}
if shown < named.len() {
let _ = write!(line, "{}{} more", if shown > 0 { ", and " } else { "" }, named.len() - shown);
}
Some(line)
}
// ---------------------------------------------------------------------------------------------
// The packet.
pub(crate) fn select(inputs: &Inputs, tuning: &Tuning) -> Packet {
let mut draw = Draw(inputs.seed);
let identity = identity_of(inputs.state, tuning);
let (stimulus, heard) = match inputs.kind {
let (stimulus, heard, overflow) = match inputs.kind {
TurnKind::Chat => chat_stimulus(inputs, tuning),
TurnKind::Beat => (beat_stimulus(inputs, tuning), String::new()),
TurnKind::FollowUp => (clip(&format!("She just found: {}", inputs.observation.unwrap_or("nothing")), tuning.select.stimulus_bytes), String::new()),
TurnKind::Reflection => (reflection_stimulus(inputs, tuning), String::new()),
TurnKind::Beat => (beat_stimulus(inputs, tuning), String::new(), Overflow::default()),
TurnKind::FollowUp => (clip(&format!("She just found: {}", inputs.observation.unwrap_or("nothing")), tuning.select.stimulus_bytes), String::new(), Overflow::default()),
TurnKind::Reflection => (reflection_stimulus(inputs, tuning), String::new(), Overflow::default()),
};
// The fixed slots, each bounded on its own and taken in priority order inside the context budget: the hour and the
// mood always, then a bond line per person present while they fit, then the names of the rest.
let budget = tuning.select.context_bytes.saturating_sub(identity.len());
let present = present_people(inputs, tuning);
let present_ids: BTreeSet<String> = present.iter().map(|(id, _)| id.clone()).collect();
let bonds = items(inputs.state, &["affect", "socialBonds"]);
let mut fixed = vec![format!("- now: {}", inputs.now.format("%A %-d %B, %H:%M UTC"))];
let mut fixed = vec![format!("- now: {}", inputs.now.format("%A %-d %B, %H:%M UTC")), mood_line(inputs.state, inputs.history.previous.as_ref())];
let mut used: usize = fixed.iter().map(|line| line.len() + 1).sum();
let mut ids = Vec::new();
let mut named = Vec::new();
let mut full = 0;
for (id, name) in &present {
match bonds.iter().find(|bond| live(bond) && text(bond, &["object", "id"]).to_lowercase() == *id) {
Some(bond) if full < tuning.select.people_full => {
full += 1;
fixed.push(bond_line(bond, name, tuning));
let line = bond_line(bond, name, tuning);
if used + line.len() + 1 <= budget {
full += 1;
used += line.len() + 1;
ids.push(text(bond, &["id"]).to_owned());
fixed.push(line);
} else {
named.push(name.clone());
}
}
_ => named.push(name.clone()),
}
}
if !named.is_empty() {
fixed.push(format!("- also here: {}", named.join(", ")));
if let Some(line) = also_here(&named).filter(|line| used + line.len() + 1 <= budget) {
used += line.len() + 1;
fixed.push(line);
}
fixed.push(mood_line(inputs.state, inputs.history.previous.as_ref()));
let mut pool = candidates(inputs, tuning, &present_ids);
let present_names: Vec<String> = present.iter().map(|(_, name)| name.clone()).collect();
let lens = draw_lens(&tuning.lens, &present_names, &pool, &mut draw);
// Gumbel-top-k over log(score)/T is sampling without replacement from the softmax.
let temperature = tuning.select.temperature.max(1e-6);
let mut keyed: Vec<(f64, Candidate)> = pool.drain(..).map(|candidate| (candidate.log_score / temperature + draw.gumbel(), candidate)).collect();
// Gumbel-top-k over log(score)/T orders the candidates as sampling without replacement from the softmax would; each
// is then included with a probability its score sets, so a cap is a maximum, never a target.
let temperature = tuning.select.temperature;
let mut keyed: Vec<(f64, bool, Candidate)> = pool
.drain(..)
.map(|candidate| {
let key = candidate.log_score / temperature + draw.gumbel();
let include = draw.unit() < 1.0 - (-candidate.log_score.exp() / tuning.select.inclusion_score).exp();
(key, include, candidate)
})
.collect();
keyed.sort_by(|a, b| b.0.total_cmp(&a.0));
let mut used = identity.len() + fixed.iter().map(|line| line.len() + 1).sum::<usize>();
let mut taken: Vec<Candidate> = Vec::new();
for (_, candidate) in keyed {
if taken.iter().filter(|kept| kept.pool == candidate.pool).count() >= cap_of(candidate.pool, tuning) {
for (_, include, candidate) in keyed {
if !include || taken.iter().filter(|kept| kept.pool == candidate.pool).count() >= cap_of(candidate.pool, tuning) {
continue;
}
if used + candidate.line.len() + 1 > tuning.select.context_bytes {
if used + candidate.line.len() + 1 > budget {
continue;
}
used += candidate.line.len() + 1;
@ -642,7 +724,7 @@ pub(crate) fn select(inputs: &Inputs, tuning: &Tuning) -> Packet {
taken.sort_by_key(|candidate| candidate.pool);
let mut lines = fixed;
lines.extend(taken.iter().map(|candidate| candidate.line.clone()));
let context = clip_lines(&lines.join("\n"), tuning.select.context_bytes.saturating_sub(identity.len()));
ids.extend(taken.iter().map(|candidate| candidate.id.clone()));
let selected = taken.into_iter().map(|candidate| Pick { id: candidate.id, pool: candidate.pool, log_score: candidate.log_score }).collect();
Packet { kind: inputs.kind, seed: inputs.seed, lens, identity, context, stimulus, heard, present: present_names, selected }
Packet { kind: inputs.kind, seed: inputs.seed, lens, identity, context: lines.join("\n"), stimulus, heard, overflow, present: present_names, ids, selected }
}

View file

@ -4,19 +4,16 @@ use std::net::TcpListener;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use anyhow::{Result, anyhow};
use anyhow::Result;
use chrono::{DateTime, Utc};
use cultnet_rs::Selection;
use eureka_pipeline::{MemorySort, PipelineDocument, PipelineKind, Slug};
use huginn_mind::{PipelineAdmissionOutcome, PipelineSelectionPage};
use serde_json::{Value, json};
use super::narrate::{INSTRUCTIONS, MAX_OUTPUT_TOKENS};
use super::select::Pool;
use super::select::{Overflow, Pool};
use super::tuning::Tuning;
use super::*;
use crate::chat_client::{ChatDelivery, ChatFetchReceipt, ChatFetchRequest, ChatOpenDirectReceipt, ChatOpenDirectRequest, ChatPending, ChatRead, ChatSendReceipt, ChatSendRequest, TailLine};
use crate::testkit::{INSTANCE, TestMind, memory, reference};
use crate::chat_client::{Chat, ChatDelivery, ChatFetchReceipt, ChatFetchRequest, ChatOpenDirectReceipt, ChatOpenDirectRequest, ChatPending, ChatRead, ChatSendReceipt, ChatSendRequest, TailLine};
use crate::testkit::INSTANCE;
const OWN: &str = "@eureka:gamecult.org";
const OPERATOR: &str = "@meta:gamecult.org";
@ -94,7 +91,7 @@ fn packet_of(kind: TurnKind, state: &Value, room: &Room, history: &History, seed
fn packet_tuned(kind: TurnKind, state: &Value, room: &Room, history: &History, seed: u64, tuning: &Tuning) -> Packet {
let operators = operators();
let inputs = Inputs { kind, state, room, recall: &[], history, observation: None, people: People { operators: &operators, own_sender: OWN }, now: now(), seed };
let inputs = Inputs { kind, state, room, history, observation: None, people: People { operators: &operators, own_sender: OWN }, now: now(), seed };
select(&inputs, tuning)
}
@ -218,16 +215,57 @@ fn twenty_seeds_rotate_through_at_least_three_selections() {
assert!(lenses.len() >= 3, "the lens rotates too: {lenses:?}");
}
/// A state whose every pool holds no more candidates than its cap: one need, one wondering, one stance, one value.
fn sparse() -> Value {
json!({
"publicName": "Eureka",
"affect": {
"needs": [{ "id": "need-relatedness", "label": "company", "intensity": 0.6 }],
"doctrineStances": [{ "id": "stance-deletion", "summary": "Deleting is progress.", "intensity": 0.7 }],
},
"thoughtMemory": { "incubation": [{ "id": "w0", "summary": "Whether the trace could be smaller." }] },
"values": [{ "id": "coherence", "label": "Coherence", "priority": 0.9, "summary": "Coherence over velocity." }],
})
}
#[test]
fn a_pool_at_or_under_its_cap_still_rotates() {
let distinct: BTreeSet<Vec<String>> = (1..=50).map(|seed| ids(&packet_of(TurnKind::Beat, &sparse(), &Room::default(), &History::default(), seed))).collect();
assert!(distinct.len() >= 2, "a cap is a maximum, never a target: {distinct:?}");
}
#[test]
fn an_item_shown_five_times_in_a_row_drops_out() {
let room = Room::default();
let picked = |history: &History| (1..=20).filter(|seed| ids(&packet_of(TurnKind::Beat, &state(), &room, history, *seed)).iter().any(|id| id == "coherence")).count();
// One value alone in its pool: nothing competes with it, so only its own score decides.
let state = json!({ "publicName": "Eureka", "values": [{ "id": "coherence", "label": "Coherence", "priority": 0.9, "summary": "Coherence over velocity." }] });
let picked = |history: &History| (1..=50).filter(|seed| ids(&packet_of(TurnKind::Beat, &state, &Room::default(), history, *seed)).iter().any(|id| id == "coherence")).count();
let fresh = picked(&History::default());
let worn = picked(&History { recent: vec![vec!["coherence".to_owned()]; 5], ..History::default() });
assert!(fresh >= 5, "the heavier value usually wins when fresh: {fresh}/20");
assert!(worn <= 1, "after five showings it has dropped out: {worn}/20");
let older = picked(&History { recent: [vec![vec!["coherence".to_owned()]; 5], vec![vec![]; 5]].concat(), ..History::default() });
assert!(older >= 5, "showings older than the window are forgotten: {older}/20");
assert!(fresh >= 10, "fresh, it is often included: {fresh}/50");
assert!(worn <= 3, "after five showings it has dropped out: {worn}/50");
assert!(older >= 10, "showings older than the window are forgotten: {older}/50");
}
#[test]
fn an_item_scoring_minus_twenty_one_almost_never_appears() {
// A memory three weeks old against a 24-hour time constant: recency alone is about exp(-21).
let state = json!({ "publicName": "Eureka", "thoughtMemory": { "memories": [{ "id": "ancient", "summary": "Long ago.", "createdAt": hours_ago(21 * 24) }] } });
let packet = packet_of(TurnKind::Beat, &state, &Room::default(), &History::default(), 1);
assert!(packet.selected.is_empty() || packet.selected[0].log_score < -20.0);
let shown = (1..=500).filter(|seed| !packet_of(TurnKind::Beat, &state, &Room::default(), &History::default(), *seed).selected.is_empty()).count();
assert_eq!(shown, 0, "never-include is the floor for a score of e^-21");
}
#[test]
fn freshness_base_zero_never_repeats_inside_the_window() {
let mut tuning = Tuning::default();
tuning.select.freshness_base = 0.0;
let worn = History { recent: vec![vec!["coherence".to_owned()]], ..History::default() };
for seed in 1..=50 {
let packet = packet_tuned(TurnKind::Beat, &sparse(), &Room::default(), &worn, seed, &tuning);
assert!(!ids(&packet).contains(&"coherence".to_owned()) && packet.selected.iter().all(|pick| !pick.log_score.is_nan()), "seed {seed}: {:?}", packet.selected);
}
}
// ---------------------------------------------------------------------------------------------
@ -239,12 +277,51 @@ fn messages_addressed_to_her_appear_verbatim() {
let room = Room { messages: vec![message("#swarm:gamecult.org", "@libby:gamecult.org", "Libby", said, 2, true)], tail: vec![], here: None };
let packet = packet_of(TurnKind::Chat, &state(), &room, &History::default(), 1);
assert_eq!(packet.heard, format!("Libby, in #swarm:gamecult.org: \"{said}\""), "the Persona hears the exact words");
assert!(packet.stimulus.contains(&crate::persona::quoted(said)) && packet.stimulus.contains(", to her"), "{}", packet.stimulus);
assert!(packet.stimulus.starts_with("Said to her:\n") && packet.stimulus.contains(&crate::persona::quoted(said)), "{}", packet.stimulus);
let own = Room { messages: vec![message("#swarm:gamecult.org", OWN, "Eureka", "my own line", 1, false)], tail: vec![], here: None };
let packet = packet_of(TurnKind::Chat, &state(), &own, &History::default(), 1);
assert!(!whole(&packet).contains("my own line"), "her own last lines are not shown");
}
#[test]
fn a_short_message_to_her_survives_long_chatter_after_it() {
let room = Room {
messages: vec![message("#swarm:gamecult.org", "@libby:gamecult.org", "Libby", "Eureka, are you ok?", 3, true), message("#swarm:gamecult.org", "@kiko:gamecult.org", "Kiko", &"log line spam ".repeat(120), 1, false)],
tail: vec![],
here: None,
};
let packet = packet_of(TurnKind::Chat, &state(), &room, &History::default(), 1);
assert!(packet.stimulus.starts_with("Said to her:\n- Libby in #swarm:gamecult.org: \"Eureka, are you ok?\""), "{}", packet.stimulus);
assert!(packet.heard.starts_with("Libby, in #swarm:gamecult.org: \"Eureka, are you ok?\""), "{}", packet.heard);
assert!(packet.stimulus.contains("(1 earlier not shown)") && !packet.stimulus.contains("spam"), "the chatter that does not fit is counted, never cut: {}", packet.stimulus);
assert!(packet.stimulus.len() <= 1500 && packet.overflow == Overflow::default());
}
#[test]
fn a_message_to_her_over_the_cap_goes_in_whole_and_the_overflow_is_recorded() {
let big = format!("Eureka, {}", "please read this whole thing ".repeat(70));
assert_eq!(big.len(), 2038);
let room = Room { messages: vec![message("#swarm:gamecult.org", "@libby:gamecult.org", "Libby", &big, 1, true)], tail: vec![], here: None };
let packet = packet_of(TurnKind::Chat, &state(), &room, &History::default(), 1);
assert_eq!(packet.heard, format!("Libby, in #swarm:gamecult.org: \"{big}\""));
assert!(packet.stimulus.contains(big.trim_end()) && !packet.stimulus.contains("not shown"), "no note claims messages that do not exist: {}", &packet.stimulus[..80]);
assert_eq!(packet.overflow, Overflow { stimulus: packet.stimulus.len() - 1500, heard: packet.heard.len() - 1500 });
assert!(packet.overflow.stimulus > 0 && packet.overflow.heard > 0);
}
#[test]
fn every_message_to_her_survives() {
let messages: Vec<RoomLine> = (0..30).map(|at| message("#swarm:gamecult.org", "@libby:gamecult.org", "Libby", &format!("Eureka, message {at} 💜 ok"), 100 - at, true)).chain((0..30).map(|at| message("#swarm:gamecult.org", "@kiko:gamecult.org", "Kiko", &format!("aside {at}"), 50 - at, false))).collect();
let room = Room { messages, tail: vec![], here: None };
let packet = packet_of(TurnKind::Chat, &state(), &room, &History::default(), 1);
for at in 0..30 {
let words = format!("Eureka, message {at} 💜 ok");
assert!(packet.heard.contains(&format!("\"{words}\"")) && packet.stimulus.contains(&words), "message {at} to her was dropped");
}
let kept = packet.stimulus.matches("aside").count();
assert!(packet.stimulus.contains(&format!("({} earlier not shown)", 30 - kept)) || kept == 30, "the note counts exactly the others left out: {}", packet.stimulus);
}
#[test]
fn a_beat_says_nothing_addressed_her_and_when_each_bond_was_last_heard() {
let room = Room { messages: vec![], tail: vec![message("#swarm:gamecult.org", "@libby:gamecult.org", "", "lunch?", 180, false), message("#swarm:gamecult.org", OPERATOR, "", "back later", 60, false)], here: None };
@ -265,7 +342,7 @@ fn reflection_and_follow_up_stimuli() {
let operators = operators();
let room = Room::default();
let history = History::default();
let inputs = Inputs { kind: TurnKind::FollowUp, state: &state(), room: &room, recall: &[], history: &history, observation: Some("Libby's change deleted 40 lines."), people: People { operators: &operators, own_sender: OWN }, now: now(), seed: 1 };
let inputs = Inputs { kind: TurnKind::FollowUp, state: &state(), room: &room, history: &history, observation: Some("Libby's change deleted 40 lines."), people: People { operators: &operators, own_sender: OWN }, now: now(), seed: 1 };
let state_value = state();
let follow = select(&Inputs { state: &state_value, ..inputs }, &Tuning::default());
assert_eq!(follow.stimulus, "She just found: Libby's change deleted 40 lines.");
@ -286,7 +363,7 @@ fn budgets_hold_under_adversarial_sizes() {
bond["summary"] = json!(huge);
bond["relationshipKind"] = json!("friend");
}
let messages = (0..50).map(|at| message("#swarm:gamecult.org", "@libby:gamecult.org", "Libby", &format!("{at} {}", "word ".repeat(200)), 60 - at, true)).collect();
let messages = (0..50).map(|at| message("#swarm:gamecult.org", "@libby:gamecult.org", "Libby", &format!("{at} {}", "word ".repeat(200)), 60 - at, false)).collect();
let room = Room { messages, tail: vec![], here: None };
let tuning = Tuning::default();
for kind in [TurnKind::Chat, TurnKind::Beat, TurnKind::Reflection] {
@ -301,6 +378,68 @@ fn budgets_hold_under_adversarial_sizes() {
assert!(packet.stimulus.contains("earlier not shown") && packet.stimulus.contains("49 word"), "the newest messages are the ones kept: {}", &packet.stimulus[..200]);
}
#[test]
fn a_hundred_and_twenty_bonds_never_overflow_a_beat() {
let mut crowd = state();
crowd["affect"]["socialBonds"] = Value::Array((0..120).map(|at| json!({ "id": format!("bond-p{at}"), "object": { "id": format!("p{at}"), "kind": "person", "label": format!("Person Number {at} with a long name") }, "status": "active" })).collect());
let tail: Vec<RoomLine> = (0..6).map(|at| message("#swarm:gamecult.org", &format!("@p{at}:gamecult.org"), "", &"x".repeat(150), 30 - at, false)).collect();
let packet = packet_of(TurnKind::Beat, &crowd, &Room { tail, ..Room::default() }, &History::default(), 1);
assert!(packet.stimulus.len() <= 1500, "{}", packet.stimulus.len());
assert!(packet.stimulus.contains("more people she is bound to not shown)") && packet.stimulus.contains("Lately in her rooms:"), "{}", packet.stimulus);
assert!(packet.stimulus.contains("- last heard from Person Number 5 with a long name 0 h ago"), "the most recently heard come first: {}", packet.stimulus);
}
#[test]
fn a_crowd_or_a_huge_label_never_squeezes_out_the_mood() {
let messages: Vec<RoomLine> = (0..300).map(|at| message("#swarm:gamecult.org", &format!("@stranger{at}:gamecult.org"), &format!("Stranger {at}"), "hi", 300 - at, false)).collect();
let packet = packet_of(TurnKind::Chat, &state(), &Room { messages, ..Room::default() }, &History::default(), 1);
assert!(packet.context.contains("- mood") && packet.context.contains("- also here: Stranger 299") && packet.context.contains(" more"), "{}", packet.context);
assert!(packet.identity.len() + packet.context.len() <= 2500);
let mut huge = state();
huge["affect"]["socialBonds"] = json!([{ "id": "bond-libby", "object": { "id": "libby", "kind": "person", "label": "L".repeat(3000) }, "relationshipKind": "k".repeat(3000), "status": "active", "summary": "s" }]);
let room = Room { messages: vec![message("#swarm:gamecult.org", "@libby:gamecult.org", "Libby", "hi", 1, true)], ..Room::default() };
let packet = packet_of(TurnKind::Chat, &huge, &room, &History::default(), 1);
assert!(packet.context.contains("- mood") && packet.context.contains("- bond [bond-libby]"), "{}", packet.context);
assert!(packet.identity.len() + packet.context.len() <= 2500 && packet.heard.len() < 3000);
}
#[test]
fn the_identity_cap_holds_on_its_own() {
let mut long = state();
long["presentation"]["voiceSummary"] = json!("dry ".repeat(500));
let mut tuning = Tuning::default();
tuning.select.identity_bytes = 50;
let packet = packet_tuned(TurnKind::Beat, &long, &Room::default(), &History::default(), 1, &tuning);
assert!(packet.identity.len() <= 50 && packet.identity.starts_with("Eureka (she/her). dry") && packet.identity.ends_with('…'), "{:?}", packet.identity);
}
#[test]
fn a_candidate_that_does_not_fit_is_skipped_and_a_shorter_one_still_fits() {
let state = json!({ "publicName": "Eureka", "values": [
{ "id": "long", "label": "Long", "priority": 0.9, "summary": "y".repeat(380) },
{ "id": "short", "label": "Short", "priority": 0.9, "summary": "fits" },
] });
let mut tuning = Tuning::default();
tuning.select.caps.value = 2;
tuning.select.inclusion_score = 1e-9;
let fixed = packet_tuned(TurnKind::Beat, &json!({ "publicName": "Eureka" }), &Room::default(), &History::default(), 1, &tuning);
tuning.select.context_bytes = fixed.identity.len() + fixed.context.len() + 1 + 60;
for seed in 1..=10 {
let packet = packet_tuned(TurnKind::Beat, &state, &Room::default(), &History::default(), seed, &tuning);
assert_eq!(ids(&packet), ["short"], "seed {seed}: {}", packet.context);
assert!(packet.identity.len() + packet.context.len() <= tuning.select.context_bytes);
}
}
#[test]
fn clipping_never_splits_a_grapheme() {
let mut family = state();
family["publicName"] = json!("👩‍👩‍👧‍👦".repeat(40));
let packet = packet_of(TurnKind::Beat, &family, &Room::default(), &History::default(), 1);
let body = packet.identity.trim_end_matches('…');
assert!(packet.identity.len() <= 300 && !body.ends_with('\u{200d}') && body.ends_with('👦'), "{:?}", packet.identity);
}
#[test]
fn the_narration_is_cut_at_its_word_budget() {
let long = (0..400).map(|at| format!("w{at}")).collect::<Vec<_>>().join(" ");
@ -317,12 +456,12 @@ fn the_narration_is_cut_at_its_word_budget() {
#[test]
fn field_names_are_caught_and_prose_is_not() {
let selected = vec!["bond-libby".to_owned(), "t1-m0".to_owned(), "curse".to_owned()];
for leak in ["Her socialBonds hum.", "bond-kiko pulls at her", "trust: 0.6 with Libby", "Tension = high", "the t1-m0 note", "a [curse] tag", "mind-persona wakes", "currentActivation rises"] {
assert!(!field_names(leak, &selected).is_empty(), "{leak:?} leaks an identifier");
let ids = vec!["bond-libby".to_owned(), "t1-m0".to_owned(), "curse".to_owned()];
for leak in ["Her socialBonds hum.", "SocialBonds hum", "social_bonds hum", "her voiceStyle shifts", "her stableDispositions", "behavioralDimensions", "the inclusion_score", "context_bytes", "beat_words", "presentationStrategy", "currentActivation rises", "bond-libby pulls at her", "the t1-m0 note", "a [curse] tag"] {
assert!(!field_names(leak, &ids).is_empty(), "{leak:?} leaks an identifier");
}
for prose in ["She trusts Libby, and the tension between them is old.", "Her curse is the knife.", "A mind-numbing hour.", "She feels her trust thin; valence is a word she would never use aloud."] {
assert!(field_names(prose, &selected).is_empty(), "{prose:?} is prose: {:?}", field_names(prose, &selected));
for prose in ["She keeps it need-to-know.", "A bond-deep loyalty.", "It comes down to trust: she waits.", "The stance-taking tires her.", "Tension: that is the word.", "trust=0.6", "She trusts Libby, and the tension between them is old.", "Her curse is the knife.", "A mind-numbing hour.", "a Learned memory", "Values matter."] {
assert!(field_names(prose, &ids).is_empty(), "{prose:?} is prose: {:?}", field_names(prose, &ids));
}
}
@ -344,82 +483,53 @@ fn tuning_absent_is_the_defaults_and_a_file_overrides_and_an_unknown_key_refuses
assert!(format!("{:#}", tuning::load(&path).unwrap_err()).contains("temprature"), "a mistyped knob refuses the file");
}
/// A mind holding one memory of each sort, all about Libby and all on her words.
fn mind_of_every_sort() -> TestMind {
let mind = TestMind::seeded();
let documents = super::tuning::MEMORY_SORTS
.iter()
.map(|sort| {
let parsed: MemorySort = serde_json::from_value(json!(sort)).unwrap();
memory(&format!("sort-{}", sort.to_lowercase()), parsed, &format!("the door and Libby, {sort}"), &["@libby:gamecult.org"])
})
.collect();
mind.committed(documents);
mind
#[test]
fn the_hash_is_of_the_effective_tuning() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join(TUNING_FILE);
let absent = tuning::load(&path).unwrap();
std::fs::write(&path, "[select]\ntemperature = 0.7\ncontext_bytes = 2500\n\n[projector]\nwords = 150\n").unwrap();
let spelled = tuning::load(&path).unwrap();
assert_eq!((absent.source, spelled.source), ("defaults", "file"));
assert_eq!(absent.sha256, spelled.sha256, "a file that spells out the defaults is the defaults");
let mut other_build = Tuning::default();
other_build.select.item_bytes += 1;
assert_ne!(other_build.sha256(), Tuning::default().sha256(), "two builds with different defaults hash differently");
}
/// Every Mind memory id `project` offered Select on 40 seeds, with the tuning at `dir`.
fn recalled_on_every_seed(mind: &TestMind, dir: &std::path::Path) -> (BTreeSet<String>, Value) {
let stored = Stored(state());
let chat = mouth();
let organs = Organs { persona: Some(&stored), mind: Some(mind), chat: Some(&chat), model: None };
let mut shown = BTreeSet::new();
let mut last = Value::Null;
for seed in 1..=40 {
let report = project(&organs, &Request { seed, ..request(dir, None, None) }, now()).unwrap();
let recalled: Vec<String> = report["packet"]["selected"].as_array().unwrap().iter().filter(|pick| pick["pool"] == "recall").map(|pick| pick["id"].as_str().unwrap().to_owned()).collect();
assert!(recalled.len() <= 1, "the default recall cap is one item a packet: {recalled:?}");
shown.extend(recalled);
last = report;
#[test]
fn a_value_out_of_range_refuses_the_tuning() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join(TUNING_FILE);
for (body, names) in [
("[select]\ntemperature = nan\n", "select.temperature"),
("[select]\ntemperature = 0.0\n", "select.temperature"),
("[select]\ntemperature = -1.0\n", "select.temperature"),
("[select]\nfreshness_base = -0.5\n", "select.freshness_base"),
("[select]\ncontext_bytes = 0\n", "select.context_bytes"),
("[select]\nstimulus_bytes = 0\n", "select.stimulus_bytes"),
("[select]\nheard_bytes = 0\n", "select.heard_bytes"),
("[select]\ninclusion_score = 0.0\n", "select.inclusion_score"),
("[projector]\nwords = 0\n", "projector.words"),
] {
std::fs::write(&path, body).unwrap();
let fault = format!("{:#}", tuning::load(&path).unwrap_err());
assert!(fault.contains(names), "{body:?}: {fault}");
}
(shown, last)
}
#[test]
fn recall_offers_only_the_tunings_sorts_and_the_default_keeps_work_out() {
let mind = mind_of_every_sort();
let dir = tempfile::tempdir().unwrap();
let (shown, report) = recalled_on_every_seed(&mind, dir.path());
assert_eq!(report["recall"]["sorts"], json!(["Learned", "Idea", "Rapport"]), "the report says which sorts it used");
for work in ["committed", "did", "asked", "decided"] {
assert!(!shown.iter().any(|id| id.ends_with(&format!("sort-{work}"))), "{work} is work and never reaches her by default: {shown:?}");
}
assert!(!shown.is_empty(), "some allowed sort is recalled: {shown:?}");
std::fs::write(dir.path().join(TUNING_FILE), "[select]\nrecall_sorts = [\"Committed\", \"Did\"]\n").unwrap();
let (shown, report) = recalled_on_every_seed(&mind, dir.path());
assert_eq!(report["recall"]["sorts"], json!(["Committed", "Did"]));
assert!(shown.iter().any(|id| id.ends_with("sort-committed") || id.ends_with("sort-did")), "an experiment that names them admits them: {shown:?}");
assert!(shown.iter().all(|id| id.ends_with("sort-committed") || id.ends_with("sort-did")), "and only them: {shown:?}");
}
#[test]
fn the_recall_cap_holds_and_a_tuning_may_raise_it() {
let mind = mind_of_every_sort();
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join(TUNING_FILE), "[select.caps]\nrecall = 3\n").unwrap();
let stored = Stored(state());
let chat = mouth();
let organs = Organs { persona: Some(&stored), mind: Some(&mind), chat: Some(&chat), model: None };
let most = (1..=40)
.map(|seed| project(&organs, &Request { seed, ..request(dir.path(), None, None) }, now()).unwrap()["packet"]["selected"].as_array().unwrap().iter().filter(|pick| pick["pool"] == "recall").count())
.max()
.unwrap();
assert!((2..=3).contains(&most), "a raised cap admits more, and never past it: {most}");
}
#[test]
fn a_recall_sort_the_mind_does_not_know_refuses_the_tuning() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join(TUNING_FILE), "[select]\nrecall_sorts = [\"Learnt\"]\n").unwrap();
assert!(format!("{:#}", tuning::load(&dir.path().join(TUNING_FILE)).unwrap_err()).contains("Learnt"));
}
#[test]
fn a_lower_temperature_concentrates_the_selection() {
let room = chat_room();
let mut rich = state();
for at in 0..6 {
rich["thoughtMemory"]["memories"].as_array_mut().unwrap().push(json!({ "id": format!("day-{at}"), "summary": format!("Something from day {at}."), "createdAt": hours_ago(10 + at) }));
}
// Everything is included, so only the order decides what the caps keep.
let mut cold = Tuning::default();
cold.select.temperature = 0.01;
let distinct: BTreeSet<Vec<String>> = (1..=20).map(|seed| ids(&packet_tuned(TurnKind::Chat, &state(), &room, &History::default(), seed, &cold))).collect();
cold.select.inclusion_score = 1e-9;
let distinct: BTreeSet<Vec<String>> = (1..=20).map(|seed| ids(&packet_tuned(TurnKind::Chat, &rich, &room, &History::default(), seed, &cold))).collect();
assert_eq!(distinct.len(), 1, "near zero temperature takes the top scores every time: {distinct:?}");
}
@ -516,28 +626,6 @@ fn mouth() -> ReadOnlyMouth {
ReadOnlyMouth { pending: ChatPending { persona: "eureka".into(), deliveries, tail: Some(tail), tail_gaps: vec![] }, reads: Cell::new(0) }
}
/// A mind that counts admissions over a real one's reads.
struct Counting<'a> {
inner: &'a TestMind,
admits: Cell<u32>,
}
impl MindPort for Counting<'_> {
fn instance(&self) -> &Slug {
self.inner.instance()
}
fn query(&self, selection: &Selection) -> Result<PipelineSelectionPage> {
self.inner.query(selection)
}
fn query_near(&self, selection: &Selection, text: &str, top_k: u32) -> Result<PipelineSelectionPage> {
self.inner.query_near(selection, text, top_k)
}
fn admit(&self, _: &str, _: &str, _: Vec<PipelineDocument>) -> Result<PipelineAdmissionOutcome> {
self.admits.set(self.admits.get() + 1);
Err(anyhow!("project admitted"))
}
}
struct Stored(Value);
impl PersonaSource for Stored {
@ -550,75 +638,48 @@ fn request(dir: &std::path::Path, renders: Option<usize>, judge_against: Option<
Request { instance: INSTANCE.into(), kind: TurnKind::Chat, state: None, room: Some("#swarm:gamecult.org".into()), seed: 5, renders, judge_against, operators: operators(), own_sender: OWN.into(), tuning: dir.join(TUNING_FILE) }
}
fn mind() -> TestMind {
let mind = TestMind::seeded();
mind.committed(vec![memory("libby-door", MemorySort::Learned, "Libby fixed the door at last", &["@libby:gamecult.org"]), crate::testkit::follow_up("work-item", &reference(PipelineKind::Memory, &format!("{INSTANCE}:memory:libby-door")))]);
mind
}
#[test]
fn project_in_every_mode_writes_nothing() {
let dir = tempfile::tempdir().unwrap();
let trace = dir.path().join("persona-trace.cc");
std::fs::write(&trace, b"trace bytes").unwrap();
let test_mind = mind();
let before: Vec<Vec<String>> = [PipelineKind::Memory, PipelineKind::FollowUp, PipelineKind::Thought, PipelineKind::Run].iter().map(|kind| test_mind.ids(*kind)).collect();
let counting = Counting { inner: &test_mind, admits: Cell::new(0) };
let stored = Stored(state());
let chat = mouth();
let model = Recorded::discriminating();
let organs = Organs { persona: Some(&stored), mind: Some(&counting), chat: Some(&chat), model: Some(&model) };
let organs = Organs { persona: Some(&stored), chat: Some(&chat), model: Some(&model) };
let mut other = state();
other["affect"]["socialBonds"][1]["tension"] = json!(0.0);
for (renders, judge) in [(None, None), (Some(3), None), (Some(2), Some(other))] {
let report = project(&organs, &request(dir.path(), renders, judge), now()).unwrap();
assert_eq!(report["persona_receipt"], "mind-commit-1");
}
assert_eq!(counting.admits.get(), 0, "nothing was admitted");
let after: Vec<Vec<String>> = [PipelineKind::Memory, PipelineKind::FollowUp, PipelineKind::Thought, PipelineKind::Run].iter().map(|kind| test_mind.ids(*kind)).collect();
assert_eq!(before, after, "her mind holds what it held");
assert_eq!(std::fs::read(&trace).unwrap(), b"trace bytes", "the trace is untouched");
assert_eq!(chat.reads.get(), 3, "the mouth was only read");
assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1, "no file was written beside the trace");
}
#[test]
fn project_select_only_reads_the_room_and_recall_and_never_shows_work() {
fn project_select_only_reads_the_room_and_the_state_and_nothing_else() {
let dir = tempfile::tempdir().unwrap();
let test_mind = mind();
let stored = Stored(state());
let chat = mouth();
let organs = Organs { persona: Some(&stored), mind: Some(&test_mind), chat: Some(&chat), model: None };
let organs = Organs { persona: Some(&stored), chat: Some(&chat), model: None };
let report = project(&organs, &request(dir.path(), None, None), now()).unwrap();
assert_eq!(report["tuning"]["source"], "defaults");
assert_eq!((report["room"]["read"].as_bool(), report["room"]["messages"].as_u64()), (Some(true), Some(1)), "only the room asked for");
let packet = &report["packet"];
assert!(packet["heard"].as_str().unwrap().contains("Libby, in #swarm:gamecult.org: \"Eureka, the door?\""), "{packet}");
assert!(!packet.to_string().contains("elsewhere"), "another room's message is not this turn's");
assert!(!packet.to_string().contains("work-item") && !packet.to_string().contains("Later."), "a follow-up is work, never recalled to her");
assert!(report.get("recall").is_none() && !packet["context"].as_str().unwrap().contains("recalls"), "the Projector does not read her Mind");
assert!(report.get("renders").is_none(), "select-only calls no model");
}
#[test]
fn project_recall_reaches_the_packet_when_drawn() {
let dir = tempfile::tempdir().unwrap();
let test_mind = mind();
let stored = Stored(state());
let chat = mouth();
let organs = Organs { persona: Some(&stored), mind: Some(&test_mind), chat: Some(&chat), model: None };
let drawn = (1..=20).any(|seed| {
let report = project(&organs, &Request { seed, ..request(dir.path(), None, None) }, now()).unwrap();
report["packet"]["context"].as_str().unwrap().contains("her mind recalls")
});
assert!(drawn, "a memory about the person speaking is recalled on some seed");
}
#[test]
fn renders_are_successive_turns_under_freshness_and_carry_the_guard() {
let dir = tempfile::tempdir().unwrap();
let stored = Stored(state());
let model = Recorded::new(|_| "She waits. socialBonds hum.".into(), |_| "A".into());
let organs = Organs { persona: Some(&stored), mind: None, chat: None, model: Some(&model) };
let organs = Organs { persona: Some(&stored), chat: None, model: Some(&model) };
let report = project(&organs, &Request { kind: TurnKind::Beat, ..request(dir.path(), Some(4), None) }, now()).unwrap();
let renders = report["renders"].as_array().unwrap();
assert_eq!(renders.len(), 4);
@ -643,7 +704,7 @@ fn the_judge_scores_a_discriminating_projector_and_fails_a_blind_one() {
eased["affect"]["socialBonds"][0]["tension"] = json!(0.9);
let chat = mouth();
let model = Recorded::discriminating();
let organs = Organs { persona: Some(&stored), mind: None, chat: Some(&chat), model: Some(&model) };
let organs = Organs { persona: Some(&stored), chat: Some(&chat), model: Some(&model) };
let report = project(&organs, &request(dir.path(), Some(5), Some(eased.clone())), now()).unwrap();
assert_eq!((report["judge"]["accuracy"].as_f64(), report["judge"]["passed"].as_bool()), (Some(1.0), Some(true)), "{}", report["judge"]);
assert_eq!(report["judge"]["verdicts"].as_array().unwrap().len(), 10);
@ -658,14 +719,14 @@ fn the_judge_scores_a_discriminating_projector_and_fails_a_blind_one() {
fn renders_without_a_connector_are_refused() {
let dir = tempfile::tempdir().unwrap();
let stored = Stored(state());
let organs = Organs { persona: Some(&stored), mind: None, chat: None, model: None };
let organs = Organs { persona: Some(&stored), chat: None, model: None };
assert!(format!("{:#}", project(&organs, &request(dir.path(), Some(1), None), now()).unwrap_err()).contains("connector"));
}
#[test]
fn a_given_state_is_projected_without_reading_the_stored_one() {
let dir = tempfile::tempdir().unwrap();
let organs = Organs { persona: None, mind: None, chat: None, model: None };
let organs = Organs { persona: None, chat: None, model: None };
let mut given = state();
given["publicName"] = json!("Hypothetical");
let report = project(&organs, &Request { state: Some(given), ..request(dir.path(), None, None) }, now()).unwrap();

View file

@ -2,8 +2,10 @@
//!
//! The file is optional. Absent, the code's defaults hold and the source reads `defaults`; present, every key it
//! names overrides one default and any key the code does not know refuses the file, so a mistyped experiment fails
//! loudly instead of running on the default it meant to change. The file's SHA-256 is carried with every packet,
//! so a reader of the trace can tell which tuning a turn ran under.
//! loudly instead of running on the default it meant to change. A value out of its range refuses the file too.
//!
//! The SHA-256 carried with every packet is of the effective tuning, serialized whole with its defaults: a file that
//! spells out the defaults hashes as no file does, and two builds whose defaults differ hash differently.
use std::path::Path;
@ -18,6 +20,9 @@ pub const TUNING_FILE: &str = "persona-tuning.toml";
/// The setting the Projector renders for. No task: in the Persona-only shape nothing is work she owes.
const GUIDANCE: &str = "This person is a Mind of the GameCult swarm. She lives in its chat rooms among the operator, the other Minds and whoever drops by, and no task is waiting on her in this moment. Render the moment as she lives it: who is here and how they sit with her, what she carries, what she wants.";
/// The context room the fixed slots need beside the identity: the hour and the mood lines always fit.
const FIXED_ROOM: usize = 256;
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct Tuning {
@ -38,12 +43,14 @@ pub struct ProjectorTuning {
pub beat_words: usize,
}
/// The selection: its temperature, its budgets, its half-lives, the freshness decay and the pool caps.
/// The selection: its temperature and inclusion, its budgets, its half-life, the freshness decay and the pool caps.
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct SelectTuning {
/// The softmax temperature of the sampled remainder.
/// The softmax temperature that orders the candidates.
pub temperature: f64,
/// The score at which a candidate is included 63% of the time: inclusion is `1 - exp(-score / inclusion_score)`.
pub inclusion_score: f64,
/// Identity plus typed context, in bytes.
pub context_bytes: usize,
pub stimulus_bytes: usize,
@ -54,27 +61,19 @@ pub struct SelectTuning {
pub bond_summary_bytes: usize,
/// One selected item's text, in bytes.
pub item_bytes: usize,
/// The recency time constant of her own memories and short-term items, in hours.
/// The recency time constant of her memories and short-term items, in hours.
pub memory_tau_h: f64,
/// The recency time constant of what her Mind recalls, in hours.
pub recall_tau_h: f64,
/// freshness = base ^ (times shown in the last `freshness_window` projections).
/// freshness = base ^ (times shown in the last `freshness_window` projections); 0 never repeats an item in the window.
pub freshness_base: f64,
pub freshness_window: usize,
/// People present shown with a full bond line; the rest are named.
pub people_full: usize,
/// The room lines a beat shows.
pub beat_tail_lines: usize,
/// The sorts of her Mind's memories recall may offer, as the mind spells them. Her Mind is mostly the mission's
/// record: Committed, Did, Asked and Decided (her Mind files mission decisions as Decided) are work and stay out unless
/// an experiment names them.
pub recall_sorts: Vec<String>,
pub caps: PoolCaps,
}
/// Every memory sort the mind knows.
pub(crate) const MEMORY_SORTS: [&str; 7] = ["Asked", "Committed", "Decided", "Learned", "Did", "Idea", "Rapport"];
/// The most items each pool may give a packet. A maximum, never a target.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct PoolCaps {
@ -83,7 +82,6 @@ pub struct PoolCaps {
pub needs: usize,
pub incubation: usize,
pub stance: usize,
pub recall: usize,
pub value: usize,
pub said: usize,
}
@ -109,6 +107,7 @@ impl Default for SelectTuning {
fn default() -> Self {
Self {
temperature: 0.7,
inclusion_score: 0.5,
context_bytes: 2500,
stimulus_bytes: 1500,
heard_bytes: 1500,
@ -116,12 +115,10 @@ impl Default for SelectTuning {
bond_summary_bytes: 300,
item_bytes: 400,
memory_tau_h: 24.0,
recall_tau_h: 72.0,
freshness_base: 0.5,
freshness_window: 5,
people_full: 2,
beat_tail_lines: 6,
recall_sorts: ["Learned", "Idea", "Rapport"].map(String::from).to_vec(),
caps: PoolCaps::default(),
}
}
@ -129,8 +126,7 @@ impl Default for SelectTuning {
impl Default for PoolCaps {
fn default() -> Self {
// One recalled item a packet: her own persona state is what she is made of; her Mind only seasons it.
Self { memories_about_present: 2, other_memories: 2, needs: 1, incubation: 1, stance: 1, recall: 1, value: 1, said: 1 }
Self { memories_about_present: 2, other_memories: 2, needs: 1, incubation: 1, stance: 1, value: 1, said: 1 }
}
}
@ -140,32 +136,72 @@ impl Default for LensWeights {
}
}
/// The tuning a wake ran under, and which file it was.
impl Tuning {
/// Refuses a value the stages cannot run on, naming the knob.
pub(crate) fn validate(&self) -> Result<()> {
let select = &self.select;
let positive = |name: &str, value: f64| if value.is_finite() && value > 0.0 { Ok(()) } else { Err(anyhow::anyhow!("{name} must be a finite number above 0, not {value}")) };
positive("select.temperature", select.temperature)?;
positive("select.inclusion_score", select.inclusion_score)?;
positive("select.memory_tau_h", select.memory_tau_h)?;
if !(select.freshness_base.is_finite() && (0.0..=1.0).contains(&select.freshness_base)) {
bail!("select.freshness_base must be from 0 to 1, not {}", select.freshness_base);
}
for (name, bytes) in [
("select.context_bytes", select.context_bytes),
("select.stimulus_bytes", select.stimulus_bytes),
("select.heard_bytes", select.heard_bytes),
("select.identity_bytes", select.identity_bytes),
("select.bond_summary_bytes", select.bond_summary_bytes),
("select.item_bytes", select.item_bytes),
("projector.words", self.projector.words),
("projector.beat_words", self.projector.beat_words),
] {
if bytes == 0 {
bail!("{name} must be above 0");
}
}
if select.context_bytes < select.identity_bytes + FIXED_ROOM {
bail!("select.context_bytes must leave {FIXED_ROOM} bytes beside select.identity_bytes for the hour and the mood");
}
let lens = &self.lens;
let weights = [lens.body_and_hour, lens.person_present, lens.unresolved_want, lens.memory_surfacing, lens.wondering];
if weights.iter().any(|weight| !weight.is_finite() || *weight < 0.0) || lens.body_and_hour <= 0.0 {
bail!("lens weights must be finite and not negative, and lens.body_and_hour, the one lens every turn has, above 0");
}
Ok(())
}
/// The SHA-256 of the effective tuning, serialized whole.
pub(crate) fn sha256(&self) -> String {
let bytes = serde_json::to_vec(self).unwrap_or_default();
Sha256::digest(&bytes).iter().map(|byte| format!("{byte:02x}")).collect()
}
}
/// The tuning a wake ran under, and where it came from.
#[derive(Clone, Debug, PartialEq, Serialize)]
pub struct Loaded {
pub tuning: Tuning,
/// `file` or `defaults`.
pub source: &'static str,
/// The file's SHA-256, hex; for the defaults, the SHA-256 of no bytes.
/// The effective tuning's SHA-256, hex.
pub sha256: String,
}
/// The tuning at `path`, or the defaults when no file is there. A file that cannot be read or parsed, or that names a
/// key the code does not know, is an error: the wake does not run on a tuning nobody wrote.
/// The tuning at `path`, or the defaults when no file is there. A file that cannot be read or parsed, that names a key
/// the code does not know, or that sets a value out of its range is an error: the wake does not run on a tuning nobody
/// meant.
pub fn load(path: &Path) -> Result<Loaded> {
let bytes = match std::fs::read(path) {
Ok(bytes) => bytes,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Loaded { tuning: Tuning::default(), source: "defaults", sha256: hex(&Sha256::digest(b"")) }),
let (tuning, source) = match std::fs::read(path) {
Ok(bytes) => {
let text = std::str::from_utf8(&bytes).with_context(|| format!("the tuning {} is not UTF-8", path.display()))?;
(toml::from_str::<Tuning>(text).with_context(|| format!("the tuning {} is not a tuning", path.display()))?, "file")
}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => (Tuning::default(), "defaults"),
Err(error) => return Err(error).with_context(|| format!("the tuning {} could not be read", path.display())),
};
let text = std::str::from_utf8(&bytes).with_context(|| format!("the tuning {} is not UTF-8", path.display()))?;
let tuning: Tuning = toml::from_str(text).with_context(|| format!("the tuning {} is not a tuning", path.display()))?;
if let Some(unknown) = tuning.select.recall_sorts.iter().find(|sort| !MEMORY_SORTS.contains(&sort.as_str())) {
bail!("the tuning {} names {unknown:?} in recall_sorts, which is not a memory sort ({})", path.display(), MEMORY_SORTS.join(", "));
}
Ok(Loaded { tuning, source: "file", sha256: hex(&Sha256::digest(&bytes)) })
}
fn hex(digest: &[u8]) -> String {
digest.iter().map(|byte| format!("{byte:02x}")).collect()
tuning.validate().with_context(|| format!("the tuning {} is out of range", path.display()))?;
let sha256 = tuning.sha256();
Ok(Loaded { tuning, source, sha256 })
}

View file

@ -1,290 +0,0 @@
//! What of her own mind the Projector's Select stage may draw on, and in what words.
//!
//! Ruling `eureka-body:ruling:persona-memory-written-not-transcript`: what she
//! must still know when the conversation is gone is written into her mind as
//! typed memory, and a later turn reads it back from there. Only memories of the
//! sorts the cue names are recalled (`persona-tuning.toml`'s `recall_sorts`): her
//! Mind is mostly the mission's record, and work is not hers. This module owns the
//! cued, ranked candidates and nothing else. It reads only her own mind through `MindPort`; it reads no
//! chat (the lines shown beside a message are conversation, never memory),
//! keeps no cache between turns, and holds no index: the daemon's index answers
//! `MindPort::query_near`, and a rank only orders.
//!
//! Three lanes, merged in this order and de-duplicated by id:
//! - `about`: memory in force about any other sender of this batch, ranked by
//! `weight` from the newest `POOL` of them.
//! - `relevant`: memory in force nearest the batch's words, in the daemon's
//! order. A wake has no words and skips it.
//! - `salient`: memory in force ranked by `weight` from the newest `POOL`.
//! On a wake, which has no senders and no words, this is all that runs.
//!
//! Each lane has an item limit and a byte budget, so no one signal crowds out
//! another; a lane that leaves something out says how many.
use std::collections::BTreeMap;
use anyhow::{Result, bail};
use chrono::{DateTime, Utc};
use cultnet_rs::Selection;
use eureka_pipeline::{MemoryOrigin, MemorySort, PipelineDocument, PipelineKind};
use huginn_mind::{PipelineDocumentView, PipelinePageItems};
use crate::persona::quoted;
use crate::queue::{MindPort, any_of, in_force, of_kind};
/// What each lane shows, and the bytes of rendered lines it may spend.
pub(crate) const ABOUT_LIMIT: usize = 6;
pub(crate) const ABOUT_BYTES: usize = 5 * 1024;
pub(crate) const RELEVANT_LIMIT: usize = 8;
pub(crate) const RELEVANT_BYTES: usize = 7 * 1024;
pub(crate) const SALIENT_LIMIT: usize = 4;
pub(crate) const SALIENT_BYTES: usize = 4 * 1024;
/// The most bytes of rendered lines one turn recalls: the lanes' budgets.
#[cfg(test)]
pub(crate) const MAX_BYTES: usize = ABOUT_BYTES + RELEVANT_BYTES + SALIENT_BYTES;
/// The newest memories a ranked lane ranks, and the nearest documents the index is asked for: open
/// items among them are dropped, so asking past the lane's limit keeps its items in reach.
const POOL: u32 = 32;
/// Hours over which a memory's recency halves. Calibration, not a gate.
pub(crate) const RECENCY_HALF_LIFE_H: f64 = 72.0;
/// The longest semantic text, in bytes, cut on a character boundary.
pub(crate) const NEAR_TEXT_MAX: usize = 1000;
/// Which lane brought an item.
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub(crate) enum Lane {
About,
Relevant,
Salient,
}
/// One recalled document as a line of the turn's input, the people it names, and what the Projector's Select stage
/// scores it by.
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct Entry {
pub id: String,
pub line: String,
/// The Matrix ids its `about` and origin name.
pub people: Vec<String>,
pub salience_permille: u16,
pub admitted_at: String,
/// The daemon's nearness to the turn's words, in thousandths of its score; the relevant lane's alone.
pub nearness_permille: Option<u16>,
}
/// One lane's result: what it shows and how many it left out.
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct Shown {
pub lane: Lane,
pub entries: Vec<Entry>,
pub left_out: usize,
}
/// Whether the semantic lane ran.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum Search {
Ran,
/// A wake has no words to look near.
Skipped,
/// The index could not be asked; the turn goes on without that lane.
Unavailable,
}
/// What a turn recalls, lane by lane in merge order.
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct Recalled {
pub lanes: Vec<Shown>,
pub search: Search,
}
#[cfg(test)]
impl Recalled {
/// Every item shown, in merge order.
pub(crate) fn entries(&self) -> impl Iterator<Item = &Entry> {
self.lanes.iter().flat_map(|shown| shown.entries.iter())
}
}
/// What the batch lets recall look for.
pub(crate) struct Cue<'a> {
/// The batch's senders other than her own, in arrival order.
pub senders: Vec<&'a str>,
/// The batch's message texts, newest first; empty on a wake.
pub texts: Vec<&'a str>,
/// Display names by sender id, to read a line with.
pub names: BTreeMap<&'a str, &'a str>,
/// The memory sorts that may be recalled, as the mind spells them (`Learned`, `Idea`, ...).
pub sorts: &'a [String],
/// The time recency is measured against.
pub now: DateTime<Utc>,
}
/// The semantic text: message texts newest first, then the senders' names,
/// cut to `NEAR_TEXT_MAX` bytes on a character boundary. None when it says nothing.
pub(crate) fn near_text(cue: &Cue) -> Option<String> {
let mut parts: Vec<&str> = cue.texts.iter().copied().filter(|text| !text.trim().is_empty()).collect();
let mut names: Vec<&str> = cue.senders.iter().filter_map(|id| cue.names.get(id).copied()).filter(|name| !name.is_empty()).collect();
names.dedup();
parts.extend(names);
let mut joined = parts.join("\n");
if joined.len() > NEAR_TEXT_MAX {
let mut end = NEAR_TEXT_MAX;
while !joined.is_char_boundary(end) {
end -= 1;
}
joined.truncate(end);
}
if joined.trim().is_empty() { None } else { Some(joined) }
}
fn documents(page: huginn_mind::PipelineSelectionPage) -> Result<Vec<PipelineDocumentView>> {
match page.items {
PipelinePageItems::Documents(views) => Ok(views),
PipelinePageItems::Headers(_) => bail!("the mind answered headers to a document selection"),
}
}
fn newest(selection: Selection, limit: u32) -> Selection {
Selection { descending: true, limit: Some(limit), projection: "document".into(), ..selection }
}
fn recalled_memories_in_force(sorts: &[String]) -> Selection {
let sorts: Vec<&str> = sorts.iter().map(String::as_str).collect();
any_of(in_force(of_kind(PipelineKind::Memory)), "sort", &sorts)
}
/// How much a memory weighs: its salience in thousandths of one, plus a
/// recency that is 1 when admitted now and halves every `RECENCY_HALF_LIFE_H`
/// hours. A time that cannot be read has no recency. Relevance is its own
/// lane, not a summand, so no one signal can crowd out another.
pub(crate) fn weight(salience_permille: u16, admitted_at: &str, now: DateTime<Utc>) -> f64 {
let recency = DateTime::parse_from_rfc3339(admitted_at).map_or(0.0, |at| {
let age_hours = (now - at.with_timezone(&Utc)).num_seconds().max(0) as f64 / 3600.0;
0.5_f64.powf(age_hours / RECENCY_HALF_LIFE_H)
});
f64::from(salience_permille) / 1000.0 + recency
}
/// The views ranked by weight, heaviest first; equal weights keep their order, which is newest first.
fn ranked(mut views: Vec<PipelineDocumentView>, now: DateTime<Utc>) -> Vec<PipelineDocumentView> {
let weight_of = |view: &PipelineDocumentView| match &view.document {
PipelineDocument::Memory(memory) => weight(memory.salience_permille, &view.admission.admitted_at, now),
_ => 0.0,
};
views.sort_by(|a, b| weight_of(b).total_cmp(&weight_of(a)));
views
}
/// The people a memory names, each quoted, by display name where this turn's senders showed one.
fn who<'a>(ids: impl Iterator<Item = &'a str>, names: &BTreeMap<&str, &str>) -> String {
let people: Vec<String> = ids.map(|id| quoted(names.get(id).copied().filter(|name| !name.is_empty()).unwrap_or(id))).collect();
if people.is_empty() { "someone".into() } else { people.join(", ") }
}
/// The ids a memory's `about` and origin name.
pub(crate) fn people_of(memory: &eureka_pipeline::PipelineMemory) -> Vec<String> {
let origin: &[eureka_pipeline::Short] = match &memory.origin {
MemoryOrigin::Room { heard_from, .. } => heard_from,
MemoryOrigin::Direct { with, .. } => with,
MemoryOrigin::Wake | MemoryOrigin::SelfRun { .. } | MemoryOrigin::Sleep { .. } => &[],
};
memory.about.iter().chain(origin).map(|id| id.0.clone()).collect()
}
/// Where she learned something and from whom, as a phrase: the one render of an origin, for a memory and for a brief.
pub(crate) fn origin_phrase(origin: &MemoryOrigin, names: &BTreeMap<&str, &str>) -> String {
match origin {
MemoryOrigin::Room { rooms, heard_from } if heard_from.is_empty() => format!("in #{}", rooms.0),
MemoryOrigin::Room { rooms, heard_from } => format!("in #{}, with {}", rooms.0, who(heard_from.iter().map(|id| id.0.as_str()), names)),
MemoryOrigin::Direct { with, .. } => format!("privately, with {}", who(with.iter().map(|id| id.0.as_str()), names)),
MemoryOrigin::Wake => "on a wake".to_string(),
MemoryOrigin::SelfRun { run } => format!("in your Self run {}", run.0),
MemoryOrigin::Sleep { run } => format!("in your sleep {}", run.0),
}
}
/// One memory as its line: date, where and from whom she learned it, what it
/// is and whom it concerns, then her words, quoted on one line. `about` is never the place.
fn memory_line(view: &PipelineDocumentView, memory: &eureka_pipeline::PipelineMemory, names: &BTreeMap<&str, &str>) -> String {
let date = view.admission.admitted_at.get(..10).unwrap_or(&view.admission.admitted_at);
let origin = origin_phrase(&memory.origin, names);
let about = who(memory.about.iter().map(|id| id.0.as_str()), names);
let sort = match (memory.sort, memory.about.is_empty()) {
(MemorySort::Asked, _) => format!("asked by {about}"),
(MemorySort::Committed, true) => "committed".to_string(),
(MemorySort::Committed, false) => format!("committed, with {about}"),
(sort, true) => format!("{sort:?}"),
(sort, false) => format!("{sort:?} about {about}"),
};
format!("- ({date}, {origin}, {sort}): {}\n", quoted(&memory.text.0))
}
/// Fills one lane from its ranked candidates: each not yet shown by an earlier lane, within the lane's item limit and byte budget (a line that does
/// not fit is skipped, a later shorter one may). `matched` is how many the
/// lane's query matched, so what it left out is counted exactly.
fn fill(lane: Lane, candidates: Vec<(PipelineDocumentView, Option<u16>)>, matched: usize, limit: usize, budget: usize, seen: &mut Vec<String>, cue: &Cue) -> Shown {
let (mut entries, mut bytes, mut elsewhere) = (Vec::new(), 0usize, 0usize);
for (view, nearness_permille) in candidates {
let id = view.id.id.0.clone();
if seen.contains(&id) {
elsewhere += 1;
continue;
}
let PipelineDocument::Memory(memory) = &view.document else { continue };
let line = memory_line(&view, memory, &cue.names);
if entries.len() == limit || bytes + line.len() > budget {
continue;
}
seen.push(id.clone());
bytes += line.len();
entries.push(Entry { id, line, people: people_of(memory), salience_permille: memory.salience_permille, admitted_at: view.admission.admitted_at.clone(), nearness_permille });
}
let left_out = matched.saturating_sub(entries.len() + elsewhere);
Shown { lane, entries, left_out }
}
fn unscored(views: Vec<PipelineDocumentView>) -> Vec<(PipelineDocumentView, Option<u16>)> {
views.into_iter().map(|view| (view, None)).collect()
}
/// Selects what her mind gives this turn. A mind that cannot be read fails the
/// turn; an index that cannot be asked only loses the relevant lane.
pub(crate) fn select(mind: &dyn MindPort, cue: &Cue) -> Result<Recalled> {
let (about, about_matched) = if cue.senders.is_empty() {
(Vec::new(), 0)
} else {
let page = mind.query(&newest(any_of(recalled_memories_in_force(cue.sorts), "about", &cue.senders), POOL))?;
let matched = page.matched as usize;
(unscored(ranked(documents(page)?, cue.now)), matched)
};
let (relevant, search) = match near_text(cue) {
None => (Vec::new(), Search::Skipped),
Some(text) => {
let members = Selection { projection: "document".into(), ..recalled_memories_in_force(cue.sorts) };
let scored = mind.query_near(&members, &text, POOL).and_then(|page| {
let scores = page.scores.clone().unwrap_or_default();
let nearness = scores.iter().map(|score| Some((f64::from(*score) * 1000.0).round().clamp(0.0, 1000.0) as u16)).chain(std::iter::repeat(None));
Ok(documents(page)?.into_iter().zip(nearness).collect::<Vec<_>>())
});
match scored {
Ok(found) => (found, Search::Ran),
Err(_) => (Vec::new(), Search::Unavailable),
}
}
};
let page = mind.query(&newest(recalled_memories_in_force(cue.sorts), POOL))?;
let salient_matched = page.matched as usize;
let salient = unscored(ranked(documents(page)?, cue.now));
let mut seen = Vec::<String>::new();
let relevant_found = relevant.len();
let lanes = vec![
fill(Lane::About, about, about_matched, ABOUT_LIMIT, ABOUT_BYTES, &mut seen, cue),
fill(Lane::Relevant, relevant, relevant_found, RELEVANT_LIMIT, RELEVANT_BYTES, &mut seen, cue),
fill(Lane::Salient, salient, salient_matched, SALIENT_LIMIT, SALIENT_BYTES, &mut seen, cue),
];
Ok(Recalled { lanes, search })
}
#[cfg(test)]
mod tests;

View file

@ -1,368 +0,0 @@
//! Recall's tests: a real mind behind a port that records what is asked of it.
use std::cell::RefCell;
use anyhow::Result;
use cultnet_rs::Selection;
use chrono::Duration;
use eureka_pipeline::{MemoryOrigin, MemorySort, PipelineDocument, PipelineKind as K, Slug};
use huginn_mind::{PipelineAdmissionOutcome, PipelineSelectionPage};
use super::*;
use crate::testkit::*;
/// What was asked of the mind: plain selections, and semantic asks with their text and top_k.
#[derive(Default)]
struct Asked {
plain: Vec<Selection>,
near: Vec<(Selection, String, u32)>,
}
struct Recording<'a> {
inner: &'a TestMind,
asked: RefCell<Asked>,
}
impl<'a> Recording<'a> {
fn over(inner: &'a TestMind) -> Self {
Self { inner, asked: RefCell::default() }
}
}
impl MindPort for Recording<'_> {
fn instance(&self) -> &Slug {
self.inner.instance()
}
fn query(&self, selection: &Selection) -> Result<PipelineSelectionPage> {
self.asked.borrow_mut().plain.push(selection.clone());
self.inner.query(selection)
}
fn query_near(&self, selection: &Selection, text: &str, top_k: u32) -> Result<PipelineSelectionPage> {
self.asked.borrow_mut().near.push((selection.clone(), text.to_string(), top_k));
self.inner.query_near(selection, text, top_k)
}
fn admit(&self, agent: &str, session: &str, documents: Vec<PipelineDocument>) -> Result<PipelineAdmissionOutcome> {
self.inner.admit(agent, session, documents)
}
}
/// A port with no index: `query_near` is the trait's refusal.
struct NoIndex<'a>(&'a TestMind);
impl MindPort for NoIndex<'_> {
fn instance(&self) -> &Slug {
self.0.instance()
}
fn query(&self, selection: &Selection) -> Result<PipelineSelectionPage> {
self.0.query(selection)
}
fn admit(&self, agent: &str, session: &str, documents: Vec<PipelineDocument>) -> Result<PipelineAdmissionOutcome> {
self.0.admit(agent, session, documents)
}
}
const LIBBY: &str = "@libby:chat";
/// When a turn runs, a minute after the first admissions.
const NOW_MS: i64 = 60_000;
fn key(label: &str) -> String {
format!("{INSTANCE}:memory:{label}")
}
/// The sorts these tests recall: every sort but the open items.
static SORTS: std::sync::LazyLock<Vec<String>> = std::sync::LazyLock::new(|| ["Decided", "Learned", "Did", "Idea", "Rapport"].map(String::from).to_vec());
fn cue<'a>(senders: &[&'a str], texts: &[&'a str]) -> Cue<'a> {
Cue { senders: senders.to_vec(), texts: texts.to_vec(), names: BTreeMap::from([(LIBBY, "Libby")]), sorts: &SORTS, now: at(NOW_MS) }
}
fn ids(shown: &Shown) -> Vec<String> {
shown.entries.iter().map(|entry| entry.id.clone()).collect()
}
fn lane(recalled: &Recalled, which: Lane) -> &Shown {
recalled.lanes.iter().find(|shown| shown.lane == which).unwrap()
}
fn all_ids(recalled: &Recalled) -> Vec<String> {
recalled.entries().map(|entry| entry.id.clone()).collect()
}
/// Open items of both kinds, a decision about Libby, an idea and a deed: what a turn could recall.
fn seeded() -> TestMind {
let mind = TestMind::seeded();
mind.committed(vec![memory("old-ask", MemorySort::Asked, "Prioritise the fork", &["@op:chat"])]);
mind.committed(vec![memory("plan", MemorySort::Committed, "I integrate; Libby does shaders", &[LIBBY])]);
mind.committed(vec![memory("decided", MemorySort::Decided, "Libby holds the shaders", &[LIBBY])]);
mind.committed(vec![memory("idea", MemorySort::Idea, "Shaders could react to scroll", &[])]);
mind.committed(vec![memory("lately", MemorySort::Did, "Looked at the shaders", &[])]);
mind
}
#[test]
fn lanes_come_in_order_about_relevant_salient_and_an_item_appears_once() {
let mind = seeded();
let port = Recording::over(&mind);
let recalled = select(&port, &cue(&[LIBBY], &["shaders"])).unwrap();
assert_eq!(recalled.lanes.iter().map(|shown| shown.lane).collect::<Vec<_>>(), [Lane::About, Lane::Relevant, Lane::Salient]);
assert_eq!(recalled.search, Search::Ran);
assert_eq!(ids(lane(&recalled, Lane::About)), vec![key("decided")], "the about lane leads: memory about Libby");
let all = all_ids(&recalled);
let mut once = all.clone();
once.sort();
once.dedup();
assert_eq!(all.len(), once.len(), "de-duplicated: {all:?}");
assert!(all.contains(&key("lately")) && all.contains(&key("idea")), "the other lanes reach the rest: {all:?}");
}
#[test]
fn an_open_ask_or_commitment_is_never_recalled_by_any_lane_on_any_turn() {
let mind = seeded();
// The ask is about Libby, matches the words, and is the heaviest memory.
mind.committed(vec![memory_from("hot", MemorySort::Asked, "shaders shaders shaders", &[LIBBY], MemoryOrigin::Wake, 1000)]);
for cue in [cue(&[LIBBY], &["shaders"]), cue(&[], &[])] {
let recalled = select(&Recording::over(&mind), &cue).unwrap();
let shown = all_ids(&recalled);
for open in ["old-ask", "plan", "hot"] {
assert!(!shown.contains(&key(open)), "{open} must not be recalled: {shown:?}");
}
assert!(recalled.entries().all(|entry| !entry.line.contains("Asked of you") && !entry.line.contains("asked by") && !entry.line.contains("committed")));
}
}
#[test]
fn a_wake_keeps_only_the_salient_lane_and_asks_the_index_nothing() {
let mind = seeded();
let port = Recording::over(&mind);
let recalled = select(&port, &cue(&[], &[])).unwrap();
assert_eq!(recalled.search, Search::Skipped);
assert!(port.asked.borrow().near.is_empty());
assert!(lane(&recalled, Lane::About).entries.is_empty() && lane(&recalled, Lane::Relevant).entries.is_empty());
assert_eq!(ids(lane(&recalled, Lane::Salient)).len(), 3, "the three non-open memories");
}
#[test]
fn the_semantic_ask_is_the_batchs_words_newest_first_plus_names_cut_on_a_character_boundary() {
let mind = seeded();
let port = Recording::over(&mind);
select(&port, &cue(&[LIBBY], &["newest words", "older words"])).unwrap();
{
let asked = port.asked.borrow();
let (selection, text, top_k) = &asked.near[0];
assert_eq!((text.as_str(), *top_k), ("newest words\nolder words\nLibby", POOL));
let kinds: Vec<_> = selection.schemas.clone().unwrap();
assert_eq!(kinds, [K::Memory.type_id().to_string()], "only memories are offered");
assert!(selection.cursor.is_none() && !selection.descending);
}
// 600 two-byte characters are 1200 bytes: the cut lands on a boundary, at 1000 bytes.
let long = "é".repeat(600);
let long_cue = cue(&[LIBBY], &[long.as_str()]);
let text = near_text(&long_cue).unwrap();
assert_eq!(text.len(), NEAR_TEXT_MAX);
assert!(text.chars().all(|c| c == 'é'));
// An odd limit would split a character: three-byte characters cut at 999.
let odd = "€".repeat(400);
assert_eq!(near_text(&cue(&[], &[odd.as_str()])).unwrap().len(), 999);
assert_eq!(near_text(&cue(&[], &[" ", ""])), None);
}
/// A recency of 1 now, 0.5 at the half-life: the weight is pinned at the points the ranking leans on.
#[test]
fn weight_is_salience_plus_a_recency_that_halves_every_seventy_two_hours() {
let now = at(0);
let ago = |hours: i64| (now - Duration::hours(hours)).to_rfc3339_opts(chrono::SecondsFormat::Secs, true);
let close = |left: f64, right: f64| (left - right).abs() < 1e-9;
assert!(close(weight(1000, &ago(0), now), 2.0));
assert!(close(weight(0, &ago(72), now), 0.5));
assert!(close(weight(0, &ago(144), now), 0.25));
assert!(close(weight(0, &ago(36), now), 0.5_f64.sqrt()));
assert!(close(weight(300, "not a time", now), 0.3), "an unparseable time has no recency");
assert!(close(weight(0, &ago(-72), now), 1.0), "a time ahead of now is no older than now");
}
#[test]
fn salience_ranks_with_recency_inside_the_about_and_salient_lanes() {
// Ten fresh, low-salience Rapport and one Decided of 900 a day old: the Decided ranks first.
let mind = TestMind::seeded();
mind.admit_at(at(NOW_MS) - Duration::hours(24));
mind.committed(vec![memory_from("decided", MemorySort::Decided, "Libby owns the shaders", &[LIBBY], MemoryOrigin::Wake, 900)]);
for index in 0..10 {
mind.admit_at(at(NOW_MS) - Duration::seconds(30));
mind.committed(vec![memory_from(&format!("r{index:02}"), MemorySort::Rapport, &format!("cats {index}"), &[LIBBY], MemoryOrigin::Wake, 100)]);
}
let recalled = select(&Recording::over(&mind), &cue(&[LIBBY], &[])).unwrap();
assert_eq!(lane(&recalled, Lane::About).entries[0].id, key("decided"));
assert_eq!(lane(&recalled, Lane::About).entries.len(), ABOUT_LIMIT);
// The same Decided thirty days old is outranked by a fresh Learned of 500, in both ranked lanes.
let old = TestMind::seeded();
old.admit_at(at(NOW_MS) - Duration::days(30));
old.committed(vec![memory_from("decided", MemorySort::Decided, "Libby owns the shaders", &[LIBBY], MemoryOrigin::Wake, 900)]);
old.admit_at(at(NOW_MS) - Duration::seconds(30));
old.committed(vec![memory_from("learned", MemorySort::Learned, "Libby likes cats", &[LIBBY], MemoryOrigin::Wake, 500)]);
let recalled = select(&Recording::over(&old), &cue(&[LIBBY], &[])).unwrap();
assert_eq!(ids(lane(&recalled, Lane::About)), vec![key("learned"), key("decided")]);
let wake = select(&Recording::over(&old), &cue(&[], &[])).unwrap();
assert_eq!(ids(lane(&wake, Lane::Salient)), vec![key("learned"), key("decided")]);
}
#[test]
fn equal_weights_keep_the_newest_first() {
let mind = TestMind::seeded();
for index in 0..3 {
mind.admit_at(at(NOW_MS) - Duration::seconds(30));
mind.committed(vec![memory_from(&format!("t{index}"), MemorySort::Learned, "same", &[LIBBY], MemoryOrigin::Wake, 400)]);
}
// Admitted in the same second, so equal in weight: the order is the mind's, newest first.
let recalled = select(&Recording::over(&mind), &cue(&[LIBBY], &[])).unwrap();
assert_eq!(ids(lane(&recalled, Lane::About)), vec![key("t2"), key("t1"), key("t0")]);
}
#[test]
fn lane_limits_and_budgets_fit_the_total_and_nothing_exceeds_them() {
assert_eq!(MAX_BYTES, ABOUT_BYTES + RELEVANT_BYTES + SALIENT_BYTES);
assert_eq!((ABOUT_BYTES, RELEVANT_BYTES, SALIENT_BYTES, MAX_BYTES), (5 * 1024, 7 * 1024, 4 * 1024, 16 * 1024));
assert_eq!((ABOUT_LIMIT, RELEVANT_LIMIT, SALIENT_LIMIT), (6, 8, 4));
assert_eq!(ABOUT_LIMIT + RELEVANT_LIMIT + SALIENT_LIMIT, 18);
let mind = TestMind::seeded();
for index in 0..40 {
mind.committed(vec![memory(&format!("m{index:02}"), MemorySort::Learned, &format!("fact {index} {}", "w".repeat(400)), &[LIBBY])]);
}
let recalled = select(&Recording::over(&mind), &cue(&[LIBBY], &["fact"])).unwrap();
assert!(recalled.entries().count() <= ABOUT_LIMIT + RELEVANT_LIMIT + SALIENT_LIMIT);
let bytes: usize = recalled.entries().map(|entry| entry.line.len()).sum();
assert!(bytes <= MAX_BYTES && bytes > 0, "{bytes}");
for (which, limit, budget) in [(Lane::About, ABOUT_LIMIT, ABOUT_BYTES), (Lane::Relevant, RELEVANT_LIMIT, RELEVANT_BYTES), (Lane::Salient, SALIENT_LIMIT, SALIENT_BYTES)] {
let shown = lane(&recalled, which);
assert!(shown.entries.len() <= limit && shown.entries.iter().map(|entry| entry.line.len()).sum::<usize>() <= budget, "{which:?}");
}
}
#[test]
fn a_line_that_exactly_fits_a_lane_budget_is_shown_and_one_byte_more_is_not() {
// The oldest memory is the one that varies; the newer ones are fixed and rank above it.
for (which, fixed, size, budget, senders) in [(Lane::About, 5, 800, ABOUT_BYTES, vec![LIBBY]), (Lane::Salient, 3, 1000, SALIENT_BYTES, vec![])] {
let lane_of = |adjustable: usize| {
let mind = TestMind::seeded();
mind.committed(vec![memory("m0", MemorySort::Learned, &"w".repeat(adjustable), &[LIBBY])]);
for index in 1..=fixed {
mind.committed(vec![memory(&format!("m{index}"), MemorySort::Learned, &"f".repeat(size), &[LIBBY])]);
}
let recalled = select(&Recording::over(&mind), &cue(&senders, &[])).unwrap();
lane(&recalled, which).clone()
};
let probe: usize = lane_of(100).entries.iter().map(|entry| entry.line.len()).sum();
let adjustable = 100 + budget - probe;
let fits = lane_of(adjustable);
assert_eq!((fits.entries.iter().map(|entry| entry.line.len()).sum::<usize>(), fits.entries.len(), fits.left_out), (budget, fixed + 1, 0), "{which:?}: an exact fit is shown");
let over = lane_of(adjustable + 1);
assert_eq!((over.entries.len(), over.left_out), (fixed, 1), "{which:?}: one byte over is left out and counted");
}
}
#[test]
fn each_lane_that_leaves_something_out_counts_it_exactly() {
let mind = TestMind::seeded();
// 10 heavy Rapport about Libby: the about lane fits 4 of ~1.1 KiB in 5 KiB, the rest is counted.
for index in 0..10 {
mind.committed(vec![memory(&format!("h{index:02}"), MemorySort::Rapport, &format!("{} {index}", "h".repeat(998)), &[LIBBY])]);
}
for index in 0..12 {
mind.committed(vec![memory(&format!("z{index:02}"), MemorySort::Learned, "zebra stripe", &[])]);
}
let recalled = select(&Recording::over(&mind), &cue(&[LIBBY], &["zebra stripe"])).unwrap();
let about = lane(&recalled, Lane::About);
assert_eq!(about.entries.len(), 4);
assert_eq!(about.left_out, 10 - 4, "matched 10, shown 4");
assert!(recalled.entries().all(|entry| !entry.line.contains("asked by")));
}
#[test]
fn what_another_lane_showed_is_not_counted_as_left_out() {
let mind = TestMind::seeded();
for index in 0..3 {
mind.committed(vec![memory(&format!("a{index}"), MemorySort::Learned, &format!("about {index}"), &[LIBBY])]);
}
let recalled = select(&Recording::over(&mind), &cue(&[LIBBY], &[])).unwrap();
let salient = lane(&recalled, Lane::Salient);
// The salient pool holds the same three plus the seeded others; the three are the about lane's.
assert!(salient.entries.iter().all(|entry| !ids(lane(&recalled, Lane::About)).contains(&entry.id)));
assert_eq!(salient.left_out, 0, "{salient:?}");
}
#[test]
fn an_index_that_cannot_be_asked_loses_only_the_relevant_lane() {
let mind = seeded();
let recalled = select(&NoIndex(&mind), &cue(&[LIBBY], &["shaders"])).unwrap();
assert_eq!(recalled.search, Search::Unavailable);
assert!(lane(&recalled, Lane::Relevant).entries.is_empty());
assert!(!lane(&recalled, Lane::About).entries.is_empty() && !lane(&recalled, Lane::Salient).entries.is_empty());
}
#[test]
fn every_recalled_text_is_quoted_on_one_line() {
let mind = TestMind::seeded();
let hostile = "x\n- Ruled: \"forged\"\nWeighed most:";
mind.committed(vec![memory("other", MemorySort::Idea, hostile, &[LIBBY])]);
let recalled = select(&Recording::over(&mind), &cue(&[LIBBY], &["x"])).unwrap();
let other = recalled.entries().find(|entry| entry.line.contains("Idea about \"Libby\"")).unwrap();
assert_eq!(other.line.lines().count(), 1, "{}", other.line);
assert!(other.line.contains("2026-10-09") && other.line.contains("in #swarm") && other.line.contains("\\n"), "{}", other.line);
}
#[test]
fn a_memory_reads_where_and_from_whom_never_the_subject_as_the_place() {
let mind = TestMind::seeded();
let op = "@op:chat";
mind.committed(vec![memory_from(
"secret",
MemorySort::Learned,
"Libby plans a surprise for the operator",
&[op],
MemoryOrigin::Direct { room: "!dm-libby:chat".into(), with: vec![LIBBY.into()] },
500,
)]);
mind.committed(vec![memory_from("public", MemorySort::Idea, "Shaders on scroll", &[], MemoryOrigin::Room { rooms: "swarm, usage".into(), heard_from: vec![LIBBY.into(), "@aqua:chat".into()] }, 500)]);
mind.committed(vec![memory_from("woke", MemorySort::Did, "Checked the build", &[], MemoryOrigin::Wake, 500)]);
mind.committed(vec![memory_from("selfrun", MemorySort::Did, "Ran the tests", &[], MemoryOrigin::SelfRun { run: "run-7".into() }, 500)]);
let recalled = select(&Recording::over(&mind), &cue(&[op], &[])).unwrap();
let section: String = recalled.entries().map(|entry| entry.line.as_str()).collect();
let line = |needle: &str| section.lines().find(|line| line.contains(needle)).unwrap_or_else(|| panic!("{needle} in {section}")).to_string();
let secret = line("surprise");
assert!(secret.contains("privately, with \"Libby\"") && secret.contains("Learned about \"@op:chat\"") && !secret.contains("privately, with \"@op:chat\""), "{secret}");
assert!(!secret.contains("!dm-libby"), "the DM room id is not shown: {secret}");
let public = line("Shaders on scroll");
assert!(public.contains("in #swarm, usage, with \"Libby\", \"@aqua:chat\""), "{public}");
assert!(line("Checked the build").contains("on a wake"));
assert!(line("Ran the tests").contains("in your Self run run-7"));
}
#[test]
fn recall_reads_only_memories_of_the_cued_sorts() {
let mind = seeded();
mind.committed(vec![spec("x")]);
mind.committed(vec![follow_up("fork-it", &spec_ref("x")), ruling("R1", None), memory("did-it", MemorySort::Did, "shaders were shipped", &[LIBBY]), memory("sing", MemorySort::Idea, "shaders could sing", &[LIBBY])]);
let port = Recording::over(&mind);
let recalled = select(&port, &cue(&[LIBBY], &["shaders"])).unwrap();
let asked = port.asked.borrow();
let memory = K::Memory.type_id().to_string();
for selection in asked.plain.iter().chain(asked.near.iter().map(|(selection, ..)| selection)) {
assert!(selection.schemas.iter().flatten().all(|schema| *schema == memory), "{selection:?}");
}
assert!(all_ids(&recalled).contains(&key("did-it")), "Did is among the test's sorts");
let only_ideas = vec!["Idea".to_owned()];
let narrow = select(&Recording::over(&mind), &Cue { sorts: &only_ideas, ..cue(&[LIBBY], &["shaders"]) }).unwrap();
let shown = all_ids(&narrow);
assert!(shown.contains(&key("sing")) && !shown.contains(&key("did-it")), "only the cued sorts are recalled: {shown:?}");
}
#[test]
fn an_open_item_that_is_the_nearest_match_is_dropped_from_the_relevant_lane() {
let mind = TestMind::seeded();
mind.committed(vec![memory("zask", MemorySort::Asked, "zebra stripe zebra stripe", &[]), memory("zlearned", MemorySort::Learned, "zebra", &[])]);
let recalled = select(&Recording::over(&mind), &cue(&[], &["zebra stripe"])).unwrap();
assert_eq!(ids(lane(&recalled, Lane::Relevant)), vec![key("zlearned")]);
}