diff --git a/crates/mind-body/src/bin/mind-stream.rs b/crates/mind-body/src/bin/mind-stream.rs index 407c41a..ee16e5a 100644 --- a/crates/mind-body/src/bin/mind-stream.rs +++ b/crates/mind-body/src/bin/mind-stream.rs @@ -1,109 +1,109 @@ -//! `mind-stream --instance [--once] [--dry-run]`: her stream of thought. -//! -//! Without `--once` it runs forever: one `mind_body::stream::tick` per `STREAM_POLL_S`, or per -//! `STREAM_MIN_GAP_S` after a blank or a fault. `--dry-run` runs one tick that admits nothing and -//! prints the thought it would admit as JSON. A model or mind fault ends the tick, never the -//! process: it is one line on stderr. This reads the process's arguments and environment, which -//! the library never does. -//! -//! Environment: `HUGINN_ENDPOINT` (`rudp://:`, her own daemon), `EUREKA_INSTANCE` -//! (when `--instance` is not given), `MIND_STREAM_ENDPOINT` (`http://:`, the -//! model host), `MIND_STREAM_PROJECTOR_MODEL` and `MIND_STREAM_BASE_MODEL`. - -use std::net::SocketAddr; -use std::process::ExitCode; -use std::time::Duration; - -use anyhow::{Result, anyhow, bail}; -use chrono::Utc; -use eureka_pipeline::Slug; -use eureka_state::HuginnClient; -use mind_body::control::read_effective; -use mind_body::queue::HuginnMind; -use mind_body::stream::{HuginnPersonaSource, Mode, OllamaStream, Organs, Outcome, STREAM_MIN_GAP_S, STREAM_POLL_S, tick}; - -const TIMEOUT: Duration = Duration::from_secs(30); - -#[allow(clippy::disallowed_methods, reason = "the binary reads its own environment, and hands it to the library")] -fn setting(name: &str) -> Option { - std::env::var(name).ok().filter(|value| !value.is_empty()) -} - -fn required(name: &str) -> Result { - setting(name).ok_or_else(|| anyhow!("{name} is not set")) -} - -struct Args { - instance: Option, - once: bool, - dry_run: bool, -} - -fn arguments() -> Result { - let mut args = Args { instance: None, once: false, dry_run: false }; - let mut given = std::env::args().skip(1); - while let Some(argument) = given.next() { - match argument.as_str() { - "--instance" => args.instance = Some(given.next().ok_or_else(|| anyhow!("--instance needs a slug"))?), - "--once" => args.once = true, - "--dry-run" => args.dry_run = true, - other => bail!("unknown argument {other}; usage: mind-stream --instance [--once] [--dry-run]"), - } - } - Ok(args) -} - -fn run() -> Result<()> { - let args = arguments()?; - let instance = Slug(args.instance.map_or_else(|| required("EUREKA_INSTANCE"), Ok)?); - let address: SocketAddr = required("HUGINN_ENDPOINT")? - .strip_prefix("rudp://") - .and_then(|address| address.parse().ok()) - .ok_or_else(|| anyhow!("HUGINN_ENDPOINT is not rudp://:"))?; - let projector_model = required("MIND_STREAM_PROJECTOR_MODEL")?; - let base_model = required("MIND_STREAM_BASE_MODEL")?; - let model = OllamaStream::open(&required("MIND_STREAM_ENDPOINT")?, &projector_model, &base_model)?; - let mind = HuginnMind::new(HuginnClient::new(address, instance.clone(), TIMEOUT)); - let persona = HuginnPersonaSource::new(HuginnClient::new(address, instance.clone(), TIMEOUT)); - let organs = Organs { mind: &mind, persona: &persona, model: &model, projector_model: &projector_model, base_model: &base_model }; - let mode = if args.dry_run { Mode::DryRun } else { Mode::Live }; - let once = args.once || args.dry_run; - loop { - let mut faulted = false; - let waits = match tick(&organs, &|| read_effective(&instance), mode, Utc::now()) { - Ok(Outcome::Thought { thought, admitted }) => { - if admitted { - println!("admitted {}:thought:{}", instance.0, thought.label.0); - } else { - println!("{}", serde_json::to_string_pretty(&*thought)?); - } - false - } - Ok(Outcome::Blank) => true, - Ok(Outcome::Held(reason)) => { - eprintln!("mind-stream: held ({reason:?}); nothing ticked"); - false - } - Ok(Outcome::NotDue) => false, - Err(fault) => { - eprintln!("mind-stream: {}", format!("{fault:#}").replace('\n', " ")); - faulted = true; - true - } - }; - if once { - return if faulted { Err(anyhow!("the tick faulted")) } else { Ok(()) }; - } - std::thread::sleep(Duration::from_secs(if waits { STREAM_MIN_GAP_S as u64 } else { STREAM_POLL_S as u64 })); - } -} - -fn main() -> ExitCode { - match run() { - Ok(()) => ExitCode::SUCCESS, - Err(fault) => { - eprintln!("mind-stream: {}", format!("{fault:#}").replace('\n', " ")); - ExitCode::FAILURE - } - } -} +//! `mind-stream --instance [--once] [--dry-run]`: her stream of thought. +//! +//! Without `--once` it runs forever: one `mind_body::stream::tick` per `STREAM_POLL_S`, or per +//! `STREAM_MIN_GAP_S` after a blank or a fault. `--dry-run` runs one tick that admits nothing and +//! prints the thought it would admit as JSON. A model or mind fault ends the tick, never the +//! process: it is one line on stderr. This reads the process's arguments and environment, which +//! the library never does. +//! +//! Environment: `HUGINN_ENDPOINT` (`rudp://:`, her own daemon), `EUREKA_INSTANCE` +//! (when `--instance` is not given), `MIND_STREAM_ENDPOINT` (`http://:`, the +//! model host), `MIND_STREAM_PROJECTOR_MODEL` and `MIND_STREAM_BASE_MODEL`. + +use std::net::SocketAddr; +use std::process::ExitCode; +use std::time::Duration; + +use anyhow::{Result, anyhow, bail}; +use chrono::Utc; +use eureka_pipeline::Slug; +use eureka_state::HuginnClient; +use mind_body::control::read_effective; +use mind_body::queue::HuginnMind; +use mind_body::stream::{HuginnPersonaSource, Mode, OllamaStream, Organs, Outcome, STREAM_MIN_GAP_S, STREAM_POLL_S, tick}; + +const TIMEOUT: Duration = Duration::from_secs(30); + +#[allow(clippy::disallowed_methods, reason = "the binary reads its own environment, and hands it to the library")] +fn setting(name: &str) -> Option { + std::env::var(name).ok().filter(|value| !value.is_empty()) +} + +fn required(name: &str) -> Result { + setting(name).ok_or_else(|| anyhow!("{name} is not set")) +} + +struct Args { + instance: Option, + once: bool, + dry_run: bool, +} + +fn arguments() -> Result { + let mut args = Args { instance: None, once: false, dry_run: false }; + let mut given = std::env::args().skip(1); + while let Some(argument) = given.next() { + match argument.as_str() { + "--instance" => args.instance = Some(given.next().ok_or_else(|| anyhow!("--instance needs a slug"))?), + "--once" => args.once = true, + "--dry-run" => args.dry_run = true, + other => bail!("unknown argument {other}; usage: mind-stream --instance [--once] [--dry-run]"), + } + } + Ok(args) +} + +fn run() -> Result<()> { + let args = arguments()?; + let instance = Slug(args.instance.map_or_else(|| required("EUREKA_INSTANCE"), Ok)?); + let address: SocketAddr = required("HUGINN_ENDPOINT")? + .strip_prefix("rudp://") + .and_then(|address| address.parse().ok()) + .ok_or_else(|| anyhow!("HUGINN_ENDPOINT is not rudp://:"))?; + let projector_model = required("MIND_STREAM_PROJECTOR_MODEL")?; + let base_model = required("MIND_STREAM_BASE_MODEL")?; + let model = OllamaStream::open(&required("MIND_STREAM_ENDPOINT")?, &projector_model, &base_model)?; + let mind = HuginnMind::new(HuginnClient::new(address, instance.clone(), TIMEOUT)); + let persona = HuginnPersonaSource::new(HuginnClient::new(address, instance.clone(), TIMEOUT)); + let organs = Organs { mind: &mind, persona: &persona, model: &model, projector_model: &projector_model, base_model: &base_model }; + let mode = if args.dry_run { Mode::DryRun } else { Mode::Live }; + let once = args.once || args.dry_run; + loop { + let mut faulted = false; + let waits = match tick(&organs, &|| read_effective(&instance), mode, Utc::now()) { + Ok(Outcome::Thought { thought, admitted }) => { + if admitted { + println!("admitted {}:thought:{}", instance.0, thought.label.0); + } else { + println!("{}", serde_json::to_string_pretty(&*thought)?); + } + false + } + Ok(Outcome::Blank) => true, + Ok(Outcome::Held(reason)) => { + eprintln!("mind-stream: held ({reason:?}); nothing ticked"); + false + } + Ok(Outcome::NotDue) => false, + Err(fault) => { + eprintln!("mind-stream: {}", format!("{fault:#}").replace('\n', " ")); + faulted = true; + true + } + }; + if once { + return if faulted { Err(anyhow!("the tick faulted")) } else { Ok(()) }; + } + std::thread::sleep(Duration::from_secs(if waits { STREAM_MIN_GAP_S as u64 } else { STREAM_POLL_S as u64 })); + } +} + +fn main() -> ExitCode { + match run() { + Ok(()) => ExitCode::SUCCESS, + Err(fault) => { + eprintln!("mind-stream: {}", format!("{fault:#}").replace('\n', " ")); + ExitCode::FAILURE + } + } +} diff --git a/crates/mind-body/src/stream.rs b/crates/mind-body/src/stream.rs index 696a633..ddef05f 100644 --- a/crates/mind-body/src/stream.rs +++ b/crates/mind-body/src/stream.rs @@ -1,642 +1,644 @@ -//! The stream organ: her stream of thought, offline. Once per tick it projects her own -//! state through Ghostlight's Projector, lets a base model continue the projection a few ways, -//! keeps the continuation least like anything she has thought before, and admits it as one -//! `thought` with its whole trace. `mind-stream@` (agent `mind-stream`) alone decides -//! whether a thought exists and what it says. -//! -//! Grounding: no selection the seed makes names kind `thought`. A thought's words reach a later -//! seed only through something she wrote (a memory, a reply, a run). Cadence is derived from her -//! newest thought's admission time and nothing else is kept: no state file, no embedder (the -//! daemon scores nearness), no second prompt builder. -//! -//! The stream never admits anything but thoughts, never puts her persona state, never sends chat -//! and never starts a run. Delivery to a Persona is another organ's. - -use std::collections::BTreeMap; -use std::net::IpAddr; -use std::time::Duration; - -use anyhow::{Context, Result, anyhow, bail}; -use chrono::{DateTime, Duration as Span, Utc}; -use eureka_pipeline::{ - Label, Line, Para, PipelineDocument, PipelineKind, PipelineRef, PipelineThought, ResolutionOutcome, Short, Slug, THOUGHT_CANDIDATES_MAX, - THOUGHT_NEAREST_NONE_PERMILLE, -}; -use eureka_state::HuginnClient; -use ghostlight_persona_projection::{ProjectorPrompt, build_projector_prompt}; -use huginn_mind::{HuginnMindResponse, PipelineAdmissionOutcome, PipelineDocumentView, PipelinePageItems}; -use serde_json::{Value, json}; - -use crate::control::Effective; -use crate::persona::{Responsibilities, outcome_words, quoted, recent_runs, render_responsibilities, repo_notice_line}; -use crate::queue::{MindPort, any_of, headers, in_force, of_kind}; -use crate::recall::{self, Cue}; - -pub const AGENT: &str = "mind-stream"; - -/// No tick sooner than this after her newest thought, whatever has happened. -pub const STREAM_MIN_GAP_S: i64 = 900; -/// How often the stream process polls: the unit in which a poll "crosses" a `STREAM_MIN_GAP_S` mark. -pub const STREAM_POLL_S: i64 = 60; -/// A tick is due after this long even when nothing has happened. -pub const STREAM_IDLE_GAP_S: i64 = 3600; -/// The most bytes of context and of stimulus the Projector is shown. -const SECTION_MAX_BYTES: usize = 3800; -/// An in-force memory this old may come unbidden when nothing is new. -const UNBIDDEN_AFTER_DAYS: i64 = 7; - -const PROJECTOR_WORDS: usize = 180; -const CONTINUATIONS: usize = THOUGHT_CANDIDATES_MAX; -const TEMPERATURE_PERMILLE: u16 = 1000; -const MIN_P_PERMILLE: u16 = 50; -const MAX_TOKENS: u16 = 160; -const NUM_THREAD: u32 = 4; - -/// The Ghostlight revision `build_projector_prompt` comes from; a test holds it to the manifest's pin. -const GHOSTLIGHT_REV: &str = "c43eb54962ed4a09cc737ecac73a16b0d7ce62b4"; -/// How the continuer frame writes the date. -const FRAME_DATE_FORMAT: &str = "%-d %B %Y"; - -/// The setting the Projector renders for: where she is between conversations. -const STREAM_GUIDANCE: &str = "This person is a Mind of the GameCult swarm in the quiet between conversations. What she holds is her repos, her fellow Minds and the people she works with. Render what is on her mind in this quiet."; - -/// What each part of the context may take. Their sum and the separators stay under -/// `SECTION_MAX_BYTES`, so no section can crowd another out. -const BUDGETS: [(&str, usize); 7] = [ - ("values", 350), - ("pressures", 450), - ("bonds", 500), - ("runs", 250), - ("responsibilities", 800), - ("recall", 800), - ("replies", 600), -]; - -const REPLIES_SHOWN: u32 = 3; -const RECENT_READ: u32 = 30; -const READS_MAX: usize = 64; -const LINE_MAX_BYTES: usize = 1000; -const PARA_MAX_BYTES: usize = 4000; - -// --------------------------------------------------------------------------------------------- -// The model port. - -/// How a continuation is sampled. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct Sampling { - pub temperature_permille: u16, - pub min_p_permille: u16, - pub max_tokens: u16, -} - -impl Sampling { - const STREAM: Self = Self { temperature_permille: TEMPERATURE_PERMILLE, min_p_permille: MIN_P_PERMILLE, max_tokens: MAX_TOKENS }; -} - -/// The two models of a tick. Faults are errors: the tick ends, the process does not. -pub trait StreamModel { - /// The Projector: one chat prompt in, one projection out. - fn project(&self, prompt: &str) -> Result; - /// The continuer: `n` raw continuations of `prompt`, no chat template. - fn continue_raw(&self, prompt: &str, sampling: &Sampling, n: usize) -> Result>; -} - -/// Ollama on this host. Loopback only, as Ghostlight's local binding is. -pub struct OllamaStream { - base: String, - projector_model: String, - base_model: String, - agent: ureq::Agent, -} - -impl OllamaStream { - /// `endpoint` is `http://:`; any other address is refused here. - pub fn open(endpoint: &str, projector_model: &str, base_model: &str) -> Result { - // Parsed as the HTTP client parses it, so the host checked is the host connected to. - let uri: ureq::http::Uri = endpoint.parse().map_err(|_| anyhow!("the stream endpoint is not a URL"))?; - let authority = uri.authority().filter(|_| uri.scheme_str() == Some("http")).ok_or_else(|| anyhow!("the stream endpoint must be http://:"))?; - if authority.as_str().contains('@') { - bail!("the stream endpoint carries userinfo"); - } - if uri.path() != "/" || uri.query().is_some() { - bail!("the stream endpoint is an origin, not a path"); - } - let host = authority.host().trim_start_matches('[').trim_end_matches(']'); - let loopback = host.eq_ignore_ascii_case("localhost") || host.parse::().is_ok_and(|ip| ip.is_loopback()); - if !loopback { - bail!("the stream endpoint is not a loopback address"); - } - let agent: ureq::Agent = ureq::Agent::config_builder() - .timeout_global(Some(Duration::from_secs(900))) - .timeout_connect(Some(Duration::from_secs(3))) - .http_status_as_error(false) - // The host checked above is the host connected to: no proxy from the environment, no redirect to elsewhere. - .proxy(None) - .max_redirects(0) - .build() - .into(); - Ok(Self { base: format!("http://{authority}"), projector_model: projector_model.to_owned(), base_model: base_model.to_owned(), agent }) - } - - fn post(&self, path: &str, body: &Value) -> Result { - let url = format!("{}{path}", self.base); - let mut response = self.agent.post(&url).header("content-type", "application/json").send(body.to_string()).with_context(|| format!("no answer from {url}"))?; - let status = response.status().as_u16(); - let text = response.body_mut().read_to_string().with_context(|| format!("reading the answer from {url}"))?; - if status != 200 { - bail!("the model host answered {status} to {path}: {}", text.chars().take(300).collect::()); - } - serde_json::from_str(&text).with_context(|| format!("the answer to {path} is not JSON")) - } -} - -impl StreamModel for OllamaStream { - fn project(&self, prompt: &str) -> Result { - let answer = self.post("/api/chat", &json!({ "model": self.projector_model, "stream": false, "messages": [{ "role": "user", "content": prompt }] }))?; - answer["message"]["content"].as_str().map(str::to_owned).ok_or_else(|| anyhow!("the Projector's answer has no message content")) - } - - fn continue_raw(&self, prompt: &str, sampling: &Sampling, n: usize) -> Result> { - let body = json!({ - "model": self.base_model, - "prompt": prompt, - "raw": true, - "stream": false, - "options": { - "temperature": f64::from(sampling.temperature_permille) / 1000.0, - "min_p": f64::from(sampling.min_p_permille) / 1000.0, - "num_predict": sampling.max_tokens, - "stop": ["\n\n"], - "num_thread": NUM_THREAD, - }, - }); - (0..n) - .map(|_| self.post("/api/generate", &body)?["response"].as_str().map(str::to_owned).ok_or_else(|| anyhow!("a continuation's answer has no response"))) - .collect() - } -} - -// --------------------------------------------------------------------------------------------- -// Her persona state, read and never put. - -pub struct PersonaRead { - pub state: Value, - pub receipt_id: String, -} - -pub trait PersonaSource { - fn persona(&self) -> Result; -} - -/// Her persona state from the daemon, as the Persona organ reads it. -pub struct HuginnPersonaSource { - client: HuginnClient, -} - -impl HuginnPersonaSource { - pub fn new(client: HuginnClient) -> Self { - Self { client } - } -} - -impl PersonaSource for HuginnPersonaSource { - fn persona(&self) -> Result { - match self.client.persona_get().map_err(|_| anyhow!("the mind could not be asked for the persona"))? { - HuginnMindResponse::Persona(Some(view)) => Ok(PersonaRead { state: view.value, receipt_id: view.receipt_id }), - HuginnMindResponse::Persona(None) => bail!("the mind holds no persona state"), - _ => bail!("the mind answered the persona request with something else"), - } - } -} - -// --------------------------------------------------------------------------------------------- -// The seed. - -/// What the Projector is shown, and the documents it came from. -#[derive(Clone, Debug, PartialEq, Eq)] -struct Seed { - pub identity: String, - pub context: String, - pub stimulus: String, - /// Whether anything new was admitted since her newest thought (the unbidden memory and the - /// nothing-new line are not news). - pub news: bool, - pub reads: Vec, -} - -impl Seed { - fn projector_prompt(&self) -> String { - build_projector_prompt(&ProjectorPrompt { - identity: &self.identity, - typed_context: &self.context, - visible_stimulus: &self.stimulus, - domain_guidance: STREAM_GUIDANCE, - word_budget: PROJECTOR_WORDS, - }) - } -} - -fn cap(text: &str, max: usize) -> String { - if text.len() <= max { - return text.to_owned(); - } - let mut end = max; - while !text.is_char_boundary(end) { - end -= 1; - } - let cut = &text[..end]; - match cut.rfind('\n') { - Some(at) if at > 0 => cut[..at].to_owned(), - _ => cut.to_owned(), - } -} - -fn text_of<'a>(value: &'a Value, path: &[&str]) -> &'a str { - path.iter().try_fold(value, |at, key| at.get(*key)).and_then(Value::as_str).unwrap_or_default().trim() -} - -fn items<'a>(value: &'a Value, path: &[&str]) -> &'a [Value] { - path.iter().try_fold(value, |at, key| at.get(*key)).and_then(Value::as_array).map_or(&[], Vec::as_slice) -} - -fn identity_of(state: &Value) -> String { - let mut parts = vec![text_of(state, &["publicName"]).to_owned()]; - for text in [text_of(state, &["publicDescription"]), text_of(state, &["presentation", "pronouns"]), text_of(state, &["presentation", "voiceSummary"])] { - if !text.is_empty() { - parts.push(text.to_owned()); - } - } - cap(&parts.join(". "), LINE_MAX_BYTES) -} - -fn values_section(state: &Value) -> String { - let mut values: Vec<&Value> = items(state, &["values"]).iter().collect(); - values.sort_by(|a, b| b["priority"].as_f64().unwrap_or(0.0).total_cmp(&a["priority"].as_f64().unwrap_or(0.0))); - let lines: Vec = values.iter().map(|value| format!("- {}: {}", text_of(value, &["label"]), text_of(value, &["summary"]))).collect(); - section("What she values:", lines) -} - -fn pressures_section(state: &Value) -> String { - let lines: Vec = items(state, &["agencyPressure", "pressures"]).iter().map(|pressure| format!("- {}", text_of(pressure, &["summary"]))).collect(); - section("What presses on her:", lines) -} - -fn bond_line(bond: &Value) -> String { - let who = [text_of(bond, &["object", "label"]), text_of(bond, &["object", "id"])].into_iter().find(|name| !name.is_empty()).unwrap_or("someone"); - format!("- {who} ({}): {}", text_of(bond, &["relationshipKind"]), text_of(bond, &["summary"])) -} - -fn bonds_section(state: &Value) -> String { - section("Who she is bound to:", items(state, &["affect", "socialBonds"]).iter().map(bond_line).collect()) -} - -fn section(heading: &str, lines: Vec) -> String { - if lines.is_empty() { String::new() } else { format!("{heading}\n{}", lines.join("\n")) } -} - -/// The people her bonds name, by the ids her memories say they are `about`. -fn bonded_people(state: &Value) -> Vec { - items(state, &["affect", "socialBonds"]) - .iter() - .filter(|bond| text_of(bond, &["object", "kind"]) == "person") - .map(|bond| text_of(bond, &["object", "id"]).to_owned()) - .filter(|id| !id.is_empty()) - .collect() -} - -fn documents(mind: &dyn MindPort, mut selection: cultnet_rs::Selection, limit: u32) -> Result> { - selection.projection = "document".into(); - selection.descending = true; - selection.limit = Some(limit); - match mind.query(&selection)?.items { - PipelinePageItems::Documents(views) => Ok(views), - PipelinePageItems::Headers(_) => bail!("the mind answered headers to a document selection"), - } -} - -fn admitted_at(view: &PipelineDocumentView) -> Option> { - DateTime::parse_from_rfc3339(&view.admission.admitted_at).ok().map(|at| at.with_timezone(&Utc)) -} - -fn mine(kind: PipelineKind, instance: &Slug) -> cultnet_rs::Selection { - any_of(of_kind(kind), "root", &[&instance.0]) -} - -/// Her Self's newest replies, whether or not her Persona has heard them. -fn replies(mind: &dyn MindPort, instance: &Slug) -> Result> { - let mut found: Vec<_> = documents(mind, mine(PipelineKind::Reply, instance), REPLIES_SHOWN)? - .into_iter() - .filter_map(|view| match view.document { - PipelineDocument::Reply(reply) => Some((view.admission.ordinal, view.id, reply.text.0)), - _ => None, - }) - .collect(); - found.sort_by_key(|(ordinal, ..)| std::cmp::Reverse(*ordinal)); - Ok(found.into_iter().take(REPLIES_SHOWN as usize).map(|(_, id, text)| (id, text)).collect()) -} - -/// What her mind admitted since `since`, newest first: one line and one citation each. -fn mind_news(mind: &dyn MindPort, instance: &Slug, since: Option>) -> Result)>> { - let after = |view: &PipelineDocumentView| since.is_none_or(|since| admitted_at(view).is_some_and(|at| at > since)); - let mut found: Vec<(u64, String, PipelineRef)> = Vec::new(); - for view in documents(mind, in_force(mine(PipelineKind::Memory, instance)), RECENT_READ)?.into_iter().filter(&after) { - if let PipelineDocument::Memory(memory) = &view.document { - found.push((view.admission.ordinal, format!("- You noted ({:?}): {}", memory.sort, quoted(&memory.text.0)), view.id.clone())); - } - } - for view in documents(mind, mine(PipelineKind::Reply, instance), RECENT_READ)?.into_iter().filter(&after) { - if let PipelineDocument::Reply(reply) = &view.document { - found.push((view.admission.ordinal, format!("- Your Self said: {}", quoted(&reply.text.0)), view.id.clone())); - } - } - for view in documents(mind, mine(PipelineKind::Resolution, instance), RECENT_READ)?.into_iter().filter(&after) { - if let PipelineDocument::Resolution(resolution) = &view.document - && resolution.subject.kind == PipelineKind::Run - { - let outcome: &ResolutionOutcome = &resolution.outcome; - found.push((view.admission.ordinal, format!("- Run {} closed: {}", resolution.subject.id.0, outcome_words(outcome)), view.id.clone())); - } - } - found.sort_by_key(|(ordinal, ..)| std::cmp::Reverse(*ordinal)); - Ok(found.into_iter().map(|(_, line, id)| (line, Some(id))).collect()) -} - -/// Her stewarded repos' activity by others is the forge's word, not a document; a daemon with no forge adds nothing. -/// Only what happened after `since` is news, and the notice is stamped with when it happened. -fn repo_news(mind: &dyn MindPort, since: Option>, responsibilities: &Responsibilities, now: DateTime) -> Option<(String, Option)> { - if responsibilities.repos.is_empty() { - return None; - } - let mut activity = mind.stewarded_activity(&responsibilities.repos, None).ok()?; - if let Some(since) = since { - let since_ms = u64::try_from(since.timestamp_millis()).unwrap_or(0); - // A capped repo's unread entries are older than the oldest it showed. Once the read shows an event of the repo at or before - // `since` they are older than `since` too; until then the cap is disclosed, whatever other repos showed. - let decided: Vec = activity.events.iter().filter(|event| event.at <= since_ms).map(|event| event.repo.identity()).collect(); - activity.capped.retain(|repo| !decided.contains(&repo.identity())); - activity.events.retain(|event| event.at > since_ms); - } - let happened = activity.events.iter().map(|event| event.at).max().unwrap_or_else(|| u64::try_from(now.timestamp_millis()).unwrap_or(0)); - repo_notice_line(&activity, happened).map(|line| (format!("- {line}"), None)) -} - -/// One in-force memory of hers older than `UNBIDDEN_AFTER_DAYS`, picked uniformly by `pick`. -fn unbidden(mind: &dyn MindPort, instance: &Slug, now: DateTime, pick: u64) -> Result> { - let mut old: Vec<_> = headers(mind, &in_force(mine(PipelineKind::Memory, instance)))? - .into_iter() - .filter(|header| DateTime::parse_from_rfc3339(&header.admission.admitted_at).is_ok_and(|at| now - at.with_timezone(&Utc) > Span::days(UNBIDDEN_AFTER_DAYS))) - .map(|header| (header.admission.ordinal, header.id)) - .collect(); - if old.is_empty() { - return Ok(None); - } - old.sort_by_key(|(ordinal, _)| *ordinal); - let (_, id) = old.swap_remove(usize::try_from(pick % old.len() as u64).unwrap_or(0)); - let one = cultnet_rs::Selection { keys: Some(vec![id.id.0.clone()]), projection: "document".into(), ..of_kind(PipelineKind::Memory) }; - let PipelinePageItems::Documents(views) = mind.query(&one)?.items else { bail!("the mind answered headers to a document selection") }; - Ok(views.into_iter().find_map(|view| match view.document { - PipelineDocument::Memory(memory) => Some((format!("Out of nowhere you recall: {}", quoted(&memory.text.0)), id.clone())), - _ => None, - })) -} - -/// The seed, rebuilt from her mind and nothing else. `since` is her newest thought's admission and `fresh` what `news` found. -fn build_seed(mind: &dyn MindPort, persona: &PersonaRead, since: Option>, now: DateTime, responsibilities: &Responsibilities, fresh: Vec<(String, Option)>) -> Result { - let instance = mind.instance().clone(); - let state = &persona.state; - let mut reads: Vec = Vec::new(); - let mut cite = |id: PipelineRef| { - if !reads.contains(&id) { - reads.push(id); - } - }; - - let runs = recent_runs(mind, &instance)?; - let people = bonded_people(state); - let cue = Cue { senders: people.iter().map(String::as_str).collect(), texts: Vec::new(), names: BTreeMap::new(), listed: &responsibilities.listed, now }; - let recalled = recall::select(mind, &cue)?; - let heard = replies(mind, &instance)?; - let heard_section = section("What your Self said lately:", heard.iter().map(|(_, text)| format!("- {}", quoted(text))).collect()); - - let sections = [ - values_section(state), - pressures_section(state), - bonds_section(state), - section("Your latest runs:", runs.iter().map(|run| format!("- {run}")).collect()), - responsibilities.text.clone(), - recall::render(&recalled), - heard_section, - ]; - let kept: Vec = sections.iter().zip(BUDGETS).map(|(text, (_, budget))| cap(text, budget)).collect(); - // The reads are what the Projector was shown after the bounds: the recall lines and replies the capped sections kept. - // Each shown line is one citation: a memory reading like another takes its own line, not a share of the first's. - let mut shown: Vec<&str> = kept[5].lines().collect(); - for entry in recalled.entries() { - if let Some(at) = shown.iter().position(|line| *line == entry.line.trim_end()) { - shown.remove(at); - cite(PipelineRef { kind: PipelineKind::Memory, id: Short(entry.id.clone()) }); - } - } - let replies_shown = kept[6].lines().count().saturating_sub(1); - heard.iter().take(replies_shown).for_each(|(id, _)| cite(id.clone())); - let context = kept.iter().filter(|text| !text.trim().is_empty()).cloned().collect::>().join("\n"); - - let has_news = !fresh.is_empty(); - let stimulus = if has_news { - let body: Vec<&str> = fresh.iter().map(|(line, _)| line.as_str()).collect(); - let kept = cap(&format!("Since your last thought:\n{}", body.join("\n")), SECTION_MAX_BYTES); - // Each line is one line (`quoted` escapes newlines), so the lines the bound kept are what the thought read. - let shown = kept.lines().count().saturating_sub(1); - fresh.into_iter().take(shown).filter_map(|(_, id)| id).for_each(&mut cite); - kept - } else if let Some((line, id)) = unbidden(mind, &instance, now, u64::try_from(now.timestamp()).unwrap_or(0))? { - cite(id); - line - } else { - let at = since.map_or_else(|| "you began".to_owned(), |at| at.format("%-d %B %H:%M UTC").to_string()); - format!("Nothing has happened since {at}.") - }; - reads.truncate(READS_MAX); - Ok(Seed { identity: identity_of(state), context: cap(&context, SECTION_MAX_BYTES), stimulus: cap(&stimulus, SECTION_MAX_BYTES), news: has_news, reads }) -} - -// --------------------------------------------------------------------------------------------- -// Continuation, selection and the tick. - -/// The frame the base model continues: her notebook heading, the date, the projection verbatim -/// and an open paragraph. It instructs nothing. -fn continuer_frame(name: &str, date: &str, projection: &str) -> String { - format!("{name}'s notebook\n\n{date}\n\n{projection}\n\n") -} - -/// The fixed parts of both prompts, recorded in every thought: the frame is the function's own output over placeholders. -fn prompt_version() -> String { - format!( - "ghostlight={GHOSTLIGHT_REV}; words={PROJECTOR_WORDS}; guidance={STREAM_GUIDANCE:?}; frame={:?}", - continuer_frame("", &format!(""), "") - ) -} - -/// Each candidate's nearness to her earlier thoughts, in thousandths of the daemon's top score; -/// 1000 where she has none. -fn nearness(mind: &dyn MindPort, candidates: &[String]) -> Result> { - let members = in_force(mine(PipelineKind::Thought, mind.instance())); - let any = mind.query(&cultnet_rs::Selection { limit: Some(1), ..members.clone() })?; - if any.matched == 0 { - return Ok(vec![THOUGHT_NEAREST_NONE_PERMILLE; candidates.len()]); - } - candidates - .iter() - .map(|text| { - let page = mind.query_near(&members, text, 1)?; - let top = page.scores.as_ref().and_then(|scores| scores.first().copied()).ok_or_else(|| anyhow!("the index scored no earlier thought"))?; - Ok((f64::from(top) * 1000.0).round().clamp(0.0, 1000.0) as u16) - }) - .collect() -} - -/// The least near candidate, the first on a tie. -fn least_near(nearest: &[u16]) -> usize { - nearest.iter().enumerate().min_by_key(|(at, near)| (**near, *at)).map_or(0, |(at, _)| at) -} - -/// How a tick ended. -#[derive(Clone, Debug, PartialEq)] -pub enum Outcome { - /// A brake is held: nothing was read, asked or admitted. - Held(crate::control::HeldReason), - /// Not due by the cadence. - NotDue, - /// Every continuation was blank; nothing was admitted. The loop waits `STREAM_MIN_GAP_S`. - Blank, - /// The thought; admitted unless the tick was a dry run. - Thought { thought: Box, admitted: bool }, -} - -/// Whether a tick is a real one or the operator's look at what one would say. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Mode { - Live, - /// Both models run and nothing is admitted; the cadence does not hold it back. - DryRun, -} - -pub struct Organs<'a> { - pub mind: &'a dyn MindPort, - pub persona: &'a dyn PersonaSource, - pub model: &'a dyn StreamModel, - pub projector_model: &'a str, - pub base_model: &'a str, -} - -/// Whether a tick is due: `STREAM_MIN_GAP_S` has passed and there is news, or `STREAM_IDLE_GAP_S` has. -fn due(since: Option>, now: DateTime, news: bool) -> bool { - match since { - None => true, - Some(at) => { - let gap = (now - at).num_seconds(); - gap >= STREAM_IDLE_GAP_S || (gap >= STREAM_MIN_GAP_S && news) - } - } -} - -/// When her newest thought was ticked: its label's instant, never later than its admission. The models take minutes between the -/// two, and what the mind or the forge admitted meanwhile is news to the next thought, which the admission time would hide. -fn newest_thought(mind: &dyn MindPort) -> Result>> { - let newest = cultnet_rs::Selection { descending: true, limit: Some(1), ..mine(PipelineKind::Thought, mind.instance()) }; - let PipelinePageItems::Headers(found) = mind.query(&newest)?.items else { bail!("the mind answered documents to a header selection") }; - Ok(found - .iter() - .filter_map(|header| { - let admitted = DateTime::parse_from_rfc3339(&header.admission.admitted_at).ok()?.with_timezone(&Utc); - let label = header.id.id.0.rsplit(':').next().and_then(|label| label.strip_prefix('t')).and_then(|label| label.parse::().ok()).and_then(|label| DateTime::from_timestamp(label, 0)); - Some(label.map_or(admitted, |at| at.min(admitted))) - }) - .max()) -} - -fn bounded(text: &str, max: usize) -> String { - cap(text.trim(), max) -} - -/// One tick. A fault is an `Err`; the caller prints it and the process goes on. -pub fn tick(organs: &Organs, brake: &dyn Fn() -> Effective, mode: Mode, now: DateTime) -> Result { - if let Effective::Held { reason } = brake() { - return Ok(Outcome::Held(reason)); - } - let mind = organs.mind; - let since = newest_thought(mind)?; - if mode == Mode::Live && since.is_some_and(|at| (now - at).num_seconds() < STREAM_MIN_GAP_S) { - return Ok(Outcome::NotDue); - } - // Due-ness is decided on the cheap reads; the seed (recall, runs, replies) is built only for a tick that may run. - let responsibilities = render_responsibilities(mind, mind.instance(), &BTreeMap::new())?; - let mut fresh = mind_news(mind, mind.instance(), since)?; - // The forge is read when a tick will run anyway, and on a quiet stretch only by the poll that crosses each minimum-gap mark. - let quiet = mode == Mode::Live && !due(since, now, !fresh.is_empty()); - if quiet && since.is_some_and(|at| (now - at).num_seconds() % STREAM_MIN_GAP_S >= STREAM_POLL_S) { - return Ok(Outcome::NotDue); - } - fresh.extend(repo_news(mind, since, &responsibilities, now)); - if quiet && fresh.is_empty() { - return Ok(Outcome::NotDue); - } - let persona = organs.persona.persona()?; - let seed = build_seed(mind, &persona, since, now, &responsibilities, fresh)?; - - let began = std::time::Instant::now(); - let projection = bounded(&organs.model.project(&seed.projector_prompt())?, PARA_MAX_BYTES); - let projector_ms = u32::try_from(began.elapsed().as_millis()).unwrap_or(u32::MAX); - if projection.is_empty() { - return Ok(Outcome::Blank); - } - let frame = continuer_frame(text_of(&persona.state, &["publicName"]), &now.format(FRAME_DATE_FORMAT).to_string(), &projection); - let began = std::time::Instant::now(); - let candidates: Vec = organs - .model - .continue_raw(&frame, &Sampling::STREAM, CONTINUATIONS)? - .iter() - .map(|text| bounded(text, LINE_MAX_BYTES)) - .filter(|text| !text.is_empty()) - .take(THOUGHT_CANDIDATES_MAX) - .collect(); - let continuer_ms = u32::try_from(began.elapsed().as_millis()).unwrap_or(u32::MAX); - if candidates.is_empty() { - return Ok(Outcome::Blank); - } - let nearest_permille = nearness(mind, &candidates)?; - let chosen = least_near(&nearest_permille); - let thought = PipelineThought { - instance: mind.instance().clone(), - label: Label::from(format!("t{}", now.timestamp()).as_str()), - text: Para::from(candidates[chosen].as_str()), - reads: seed.reads.clone(), - persona_receipt: Short(persona.receipt_id.clone()), - identity: Line::from(seed.identity.as_str()), - context: Para::from(seed.context.as_str()), - stimulus: Para::from(seed.stimulus.as_str()), - projector_model: Short(organs.projector_model.to_owned()), - projection: Para::from(projection.as_str()), - base_model: Short(organs.base_model.to_owned()), - temperature_permille: Sampling::STREAM.temperature_permille, - min_p_permille: Sampling::STREAM.min_p_permille, - max_tokens: Sampling::STREAM.max_tokens, - candidates: candidates.iter().map(|text| Line::from(text.as_str())).collect(), - nearest_permille, - chosen: u8::try_from(chosen).unwrap_or(0), - projector_ms, - continuer_ms, - prompt_version: Line::from(prompt_version().as_str()), - }; - if mode == Mode::DryRun { - return Ok(Outcome::Thought { thought: Box::new(thought), admitted: false }); - } - let label = thought.label.0.clone(); - match mind.admit(AGENT, &format!("stream-{label}"), vec![PipelineDocument::Thought(thought.clone())])? { - PipelineAdmissionOutcome::Committed { .. } => Ok(Outcome::Thought { thought: Box::new(thought), admitted: true }), - other => Err(anyhow!("the mind did not admit the thought: {other:?}")), - } -} - -#[cfg(test)] -mod tests; +//! The stream organ: her stream of thought, offline. Once per tick it projects her own +//! state through Ghostlight's Projector, lets a base model continue the projection a few ways, +//! keeps the continuation least like anything she has thought before, and admits it as one +//! `thought` with its whole trace. `mind-stream@` (agent `mind-stream`) alone decides +//! whether a thought exists and what it says. +//! +//! Grounding: no selection the seed makes names kind `thought`. A thought's words reach a later +//! seed only through something she wrote (a memory, a reply, a run). Cadence is derived from her +//! newest thought's admission time and nothing else is kept: no state file, no embedder (the +//! daemon scores nearness), no second prompt builder. +//! +//! The stream never admits anything but thoughts, never puts her persona state, never sends chat +//! and never starts a run. Delivery to a Persona is another organ's. + +use std::collections::BTreeMap; +use std::net::IpAddr; +use std::time::Duration; + +use anyhow::{Context, Result, anyhow, bail}; +use chrono::{DateTime, Duration as Span, Utc}; +use eureka_pipeline::{ + Label, Line, Para, PipelineDocument, PipelineKind, PipelineRef, PipelineThought, ResolutionOutcome, Short, Slug, THOUGHT_CANDIDATES_MAX, + THOUGHT_NEAREST_NONE_PERMILLE, +}; +use eureka_state::HuginnClient; +use ghostlight_persona_projection::{ProjectorPrompt, build_projector_prompt}; +use huginn_mind::{HuginnMindResponse, PipelineAdmissionOutcome, PipelineDocumentView, PipelinePageItems}; +use serde_json::{Value, json}; + +use crate::control::Effective; +use crate::persona::{Responsibilities, outcome_words, quoted, recent_runs, render_responsibilities, repo_notice_line}; +use crate::queue::{MindPort, any_of, headers, in_force, of_kind}; +use crate::recall::{self, Cue}; + +pub const AGENT: &str = "mind-stream"; + +/// No tick sooner than this after her newest thought, whatever has happened. The gap counts from the thought's label +/// (the tick time it was made for), not from its admission: a model run of M seconds leaves `STREAM_MIN_GAP_S - M` +/// seconds of quiet after the thought is admitted. +pub const STREAM_MIN_GAP_S: i64 = 900; +/// How often the stream process polls: the unit in which a poll "crosses" a `STREAM_MIN_GAP_S` mark. +pub const STREAM_POLL_S: i64 = 60; +/// A tick is due after this long even when nothing has happened. +pub const STREAM_IDLE_GAP_S: i64 = 3600; +/// The most bytes of context and of stimulus the Projector is shown. +const SECTION_MAX_BYTES: usize = 3800; +/// An in-force memory this old may come unbidden when nothing is new. +const UNBIDDEN_AFTER_DAYS: i64 = 7; + +const PROJECTOR_WORDS: usize = 180; +const CONTINUATIONS: usize = THOUGHT_CANDIDATES_MAX; +const TEMPERATURE_PERMILLE: u16 = 1000; +const MIN_P_PERMILLE: u16 = 50; +const MAX_TOKENS: u16 = 160; +const NUM_THREAD: u32 = 4; + +/// The Ghostlight revision `build_projector_prompt` comes from; a test holds it to the manifest's pin. +const GHOSTLIGHT_REV: &str = "c43eb54962ed4a09cc737ecac73a16b0d7ce62b4"; +/// How the continuer frame writes the date. +const FRAME_DATE_FORMAT: &str = "%-d %B %Y"; + +/// The setting the Projector renders for: where she is between conversations. +const STREAM_GUIDANCE: &str = "This person is a Mind of the GameCult swarm in the quiet between conversations. What she holds is her repos, her fellow Minds and the people she works with. Render what is on her mind in this quiet."; + +/// What each part of the context may take. Their sum and the separators stay under +/// `SECTION_MAX_BYTES`, so no section can crowd another out. +const BUDGETS: [(&str, usize); 7] = [ + ("values", 350), + ("pressures", 450), + ("bonds", 500), + ("runs", 250), + ("responsibilities", 800), + ("recall", 800), + ("replies", 600), +]; + +const REPLIES_SHOWN: u32 = 3; +const RECENT_READ: u32 = 30; +const READS_MAX: usize = 64; +const LINE_MAX_BYTES: usize = 1000; +const PARA_MAX_BYTES: usize = 4000; + +// --------------------------------------------------------------------------------------------- +// The model port. + +/// How a continuation is sampled. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Sampling { + pub temperature_permille: u16, + pub min_p_permille: u16, + pub max_tokens: u16, +} + +impl Sampling { + const STREAM: Self = Self { temperature_permille: TEMPERATURE_PERMILLE, min_p_permille: MIN_P_PERMILLE, max_tokens: MAX_TOKENS }; +} + +/// The two models of a tick. Faults are errors: the tick ends, the process does not. +pub trait StreamModel { + /// The Projector: one chat prompt in, one projection out. + fn project(&self, prompt: &str) -> Result; + /// The continuer: `n` raw continuations of `prompt`, no chat template. + fn continue_raw(&self, prompt: &str, sampling: &Sampling, n: usize) -> Result>; +} + +/// Ollama on this host. Loopback only, as Ghostlight's local binding is. +pub struct OllamaStream { + base: String, + projector_model: String, + base_model: String, + agent: ureq::Agent, +} + +impl OllamaStream { + /// `endpoint` is `http://:`; any other address is refused here. + pub fn open(endpoint: &str, projector_model: &str, base_model: &str) -> Result { + // Parsed as the HTTP client parses it, so the host checked is the host connected to. + let uri: ureq::http::Uri = endpoint.parse().map_err(|_| anyhow!("the stream endpoint is not a URL"))?; + let authority = uri.authority().filter(|_| uri.scheme_str() == Some("http")).ok_or_else(|| anyhow!("the stream endpoint must be http://:"))?; + if authority.as_str().contains('@') { + bail!("the stream endpoint carries userinfo"); + } + if uri.path() != "/" || uri.query().is_some() { + bail!("the stream endpoint is an origin, not a path"); + } + let host = authority.host().trim_start_matches('[').trim_end_matches(']'); + let loopback = host.eq_ignore_ascii_case("localhost") || host.parse::().is_ok_and(|ip| ip.is_loopback()); + if !loopback { + bail!("the stream endpoint is not a loopback address"); + } + let agent: ureq::Agent = ureq::Agent::config_builder() + .timeout_global(Some(Duration::from_secs(900))) + .timeout_connect(Some(Duration::from_secs(3))) + .http_status_as_error(false) + // The host checked above is the host connected to: no proxy from the environment, no redirect to elsewhere. + .proxy(None) + .max_redirects(0) + .build() + .into(); + Ok(Self { base: format!("http://{authority}"), projector_model: projector_model.to_owned(), base_model: base_model.to_owned(), agent }) + } + + fn post(&self, path: &str, body: &Value) -> Result { + let url = format!("{}{path}", self.base); + let mut response = self.agent.post(&url).header("content-type", "application/json").send(body.to_string()).with_context(|| format!("no answer from {url}"))?; + let status = response.status().as_u16(); + let text = response.body_mut().read_to_string().with_context(|| format!("reading the answer from {url}"))?; + if status != 200 { + bail!("the model host answered {status} to {path}: {}", text.chars().take(300).collect::()); + } + serde_json::from_str(&text).with_context(|| format!("the answer to {path} is not JSON")) + } +} + +impl StreamModel for OllamaStream { + fn project(&self, prompt: &str) -> Result { + let answer = self.post("/api/chat", &json!({ "model": self.projector_model, "stream": false, "messages": [{ "role": "user", "content": prompt }] }))?; + answer["message"]["content"].as_str().map(str::to_owned).ok_or_else(|| anyhow!("the Projector's answer has no message content")) + } + + fn continue_raw(&self, prompt: &str, sampling: &Sampling, n: usize) -> Result> { + let body = json!({ + "model": self.base_model, + "prompt": prompt, + "raw": true, + "stream": false, + "options": { + "temperature": f64::from(sampling.temperature_permille) / 1000.0, + "min_p": f64::from(sampling.min_p_permille) / 1000.0, + "num_predict": sampling.max_tokens, + "stop": ["\n\n"], + "num_thread": NUM_THREAD, + }, + }); + (0..n) + .map(|_| self.post("/api/generate", &body)?["response"].as_str().map(str::to_owned).ok_or_else(|| anyhow!("a continuation's answer has no response"))) + .collect() + } +} + +// --------------------------------------------------------------------------------------------- +// Her persona state, read and never put. + +pub struct PersonaRead { + pub state: Value, + pub receipt_id: String, +} + +pub trait PersonaSource { + fn persona(&self) -> Result; +} + +/// Her persona state from the daemon, as the Persona organ reads it. +pub struct HuginnPersonaSource { + client: HuginnClient, +} + +impl HuginnPersonaSource { + pub fn new(client: HuginnClient) -> Self { + Self { client } + } +} + +impl PersonaSource for HuginnPersonaSource { + fn persona(&self) -> Result { + match self.client.persona_get().map_err(|_| anyhow!("the mind could not be asked for the persona"))? { + HuginnMindResponse::Persona(Some(view)) => Ok(PersonaRead { state: view.value, receipt_id: view.receipt_id }), + HuginnMindResponse::Persona(None) => bail!("the mind holds no persona state"), + _ => bail!("the mind answered the persona request with something else"), + } + } +} + +// --------------------------------------------------------------------------------------------- +// The seed. + +/// What the Projector is shown, and the documents it came from. +#[derive(Clone, Debug, PartialEq, Eq)] +struct Seed { + pub identity: String, + pub context: String, + pub stimulus: String, + /// Whether anything new was admitted since her newest thought (the unbidden memory and the + /// nothing-new line are not news). + pub news: bool, + pub reads: Vec, +} + +impl Seed { + fn projector_prompt(&self) -> String { + build_projector_prompt(&ProjectorPrompt { + identity: &self.identity, + typed_context: &self.context, + visible_stimulus: &self.stimulus, + domain_guidance: STREAM_GUIDANCE, + word_budget: PROJECTOR_WORDS, + }) + } +} + +fn cap(text: &str, max: usize) -> String { + if text.len() <= max { + return text.to_owned(); + } + let mut end = max; + while !text.is_char_boundary(end) { + end -= 1; + } + let cut = &text[..end]; + match cut.rfind('\n') { + Some(at) if at > 0 => cut[..at].to_owned(), + _ => cut.to_owned(), + } +} + +fn text_of<'a>(value: &'a Value, path: &[&str]) -> &'a str { + path.iter().try_fold(value, |at, key| at.get(*key)).and_then(Value::as_str).unwrap_or_default().trim() +} + +fn items<'a>(value: &'a Value, path: &[&str]) -> &'a [Value] { + path.iter().try_fold(value, |at, key| at.get(*key)).and_then(Value::as_array).map_or(&[], Vec::as_slice) +} + +fn identity_of(state: &Value) -> String { + let mut parts = vec![text_of(state, &["publicName"]).to_owned()]; + for text in [text_of(state, &["publicDescription"]), text_of(state, &["presentation", "pronouns"]), text_of(state, &["presentation", "voiceSummary"])] { + if !text.is_empty() { + parts.push(text.to_owned()); + } + } + cap(&parts.join(". "), LINE_MAX_BYTES) +} + +fn values_section(state: &Value) -> String { + let mut values: Vec<&Value> = items(state, &["values"]).iter().collect(); + values.sort_by(|a, b| b["priority"].as_f64().unwrap_or(0.0).total_cmp(&a["priority"].as_f64().unwrap_or(0.0))); + let lines: Vec = values.iter().map(|value| format!("- {}: {}", text_of(value, &["label"]), text_of(value, &["summary"]))).collect(); + section("What she values:", lines) +} + +fn pressures_section(state: &Value) -> String { + let lines: Vec = items(state, &["agencyPressure", "pressures"]).iter().map(|pressure| format!("- {}", text_of(pressure, &["summary"]))).collect(); + section("What presses on her:", lines) +} + +fn bond_line(bond: &Value) -> String { + let who = [text_of(bond, &["object", "label"]), text_of(bond, &["object", "id"])].into_iter().find(|name| !name.is_empty()).unwrap_or("someone"); + format!("- {who} ({}): {}", text_of(bond, &["relationshipKind"]), text_of(bond, &["summary"])) +} + +fn bonds_section(state: &Value) -> String { + section("Who she is bound to:", items(state, &["affect", "socialBonds"]).iter().map(bond_line).collect()) +} + +fn section(heading: &str, lines: Vec) -> String { + if lines.is_empty() { String::new() } else { format!("{heading}\n{}", lines.join("\n")) } +} + +/// The people her bonds name, by the ids her memories say they are `about`. +fn bonded_people(state: &Value) -> Vec { + items(state, &["affect", "socialBonds"]) + .iter() + .filter(|bond| text_of(bond, &["object", "kind"]) == "person") + .map(|bond| text_of(bond, &["object", "id"]).to_owned()) + .filter(|id| !id.is_empty()) + .collect() +} + +fn documents(mind: &dyn MindPort, mut selection: cultnet_rs::Selection, limit: u32) -> Result> { + selection.projection = "document".into(); + selection.descending = true; + selection.limit = Some(limit); + match mind.query(&selection)?.items { + PipelinePageItems::Documents(views) => Ok(views), + PipelinePageItems::Headers(_) => bail!("the mind answered headers to a document selection"), + } +} + +fn admitted_at(view: &PipelineDocumentView) -> Option> { + DateTime::parse_from_rfc3339(&view.admission.admitted_at).ok().map(|at| at.with_timezone(&Utc)) +} + +fn mine(kind: PipelineKind, instance: &Slug) -> cultnet_rs::Selection { + any_of(of_kind(kind), "root", &[&instance.0]) +} + +/// Her Self's newest replies, whether or not her Persona has heard them. +fn replies(mind: &dyn MindPort, instance: &Slug) -> Result> { + let mut found: Vec<_> = documents(mind, mine(PipelineKind::Reply, instance), REPLIES_SHOWN)? + .into_iter() + .filter_map(|view| match view.document { + PipelineDocument::Reply(reply) => Some((view.admission.ordinal, view.id, reply.text.0)), + _ => None, + }) + .collect(); + found.sort_by_key(|(ordinal, ..)| std::cmp::Reverse(*ordinal)); + Ok(found.into_iter().take(REPLIES_SHOWN as usize).map(|(_, id, text)| (id, text)).collect()) +} + +/// What her mind admitted since `since`, newest first: one line and one citation each. +fn mind_news(mind: &dyn MindPort, instance: &Slug, since: Option>) -> Result)>> { + let after = |view: &PipelineDocumentView| since.is_none_or(|since| admitted_at(view).is_some_and(|at| at > since)); + let mut found: Vec<(u64, String, PipelineRef)> = Vec::new(); + for view in documents(mind, in_force(mine(PipelineKind::Memory, instance)), RECENT_READ)?.into_iter().filter(&after) { + if let PipelineDocument::Memory(memory) = &view.document { + found.push((view.admission.ordinal, format!("- You noted ({:?}): {}", memory.sort, quoted(&memory.text.0)), view.id.clone())); + } + } + for view in documents(mind, mine(PipelineKind::Reply, instance), RECENT_READ)?.into_iter().filter(&after) { + if let PipelineDocument::Reply(reply) = &view.document { + found.push((view.admission.ordinal, format!("- Your Self said: {}", quoted(&reply.text.0)), view.id.clone())); + } + } + for view in documents(mind, mine(PipelineKind::Resolution, instance), RECENT_READ)?.into_iter().filter(&after) { + if let PipelineDocument::Resolution(resolution) = &view.document + && resolution.subject.kind == PipelineKind::Run + { + let outcome: &ResolutionOutcome = &resolution.outcome; + found.push((view.admission.ordinal, format!("- Run {} closed: {}", resolution.subject.id.0, outcome_words(outcome)), view.id.clone())); + } + } + found.sort_by_key(|(ordinal, ..)| std::cmp::Reverse(*ordinal)); + Ok(found.into_iter().map(|(_, line, id)| (line, Some(id))).collect()) +} + +/// Her stewarded repos' activity by others is the forge's word, not a document; a daemon with no forge adds nothing. +/// Only what happened after `since` is news, and the notice is stamped with when it happened. +fn repo_news(mind: &dyn MindPort, since: Option>, responsibilities: &Responsibilities, now: DateTime) -> Option<(String, Option)> { + if responsibilities.repos.is_empty() { + return None; + } + let mut activity = mind.stewarded_activity(&responsibilities.repos, None).ok()?; + if let Some(since) = since { + let since_ms = u64::try_from(since.timestamp_millis()).unwrap_or(0); + // A capped repo's unread entries are older than the oldest it showed. Once the read shows an event of the repo at or before + // `since` they are older than `since` too; until then the cap is disclosed, whatever other repos showed. + let decided: Vec = activity.events.iter().filter(|event| event.at <= since_ms).map(|event| event.repo.identity()).collect(); + activity.capped.retain(|repo| !decided.contains(&repo.identity())); + activity.events.retain(|event| event.at > since_ms); + } + let happened = activity.events.iter().map(|event| event.at).max().unwrap_or_else(|| u64::try_from(now.timestamp_millis()).unwrap_or(0)); + repo_notice_line(&activity, happened).map(|line| (format!("- {line}"), None)) +} + +/// One in-force memory of hers older than `UNBIDDEN_AFTER_DAYS`, picked uniformly by `pick`. +fn unbidden(mind: &dyn MindPort, instance: &Slug, now: DateTime, pick: u64) -> Result> { + let mut old: Vec<_> = headers(mind, &in_force(mine(PipelineKind::Memory, instance)))? + .into_iter() + .filter(|header| DateTime::parse_from_rfc3339(&header.admission.admitted_at).is_ok_and(|at| now - at.with_timezone(&Utc) > Span::days(UNBIDDEN_AFTER_DAYS))) + .map(|header| (header.admission.ordinal, header.id)) + .collect(); + if old.is_empty() { + return Ok(None); + } + old.sort_by_key(|(ordinal, _)| *ordinal); + let (_, id) = old.swap_remove(usize::try_from(pick % old.len() as u64).unwrap_or(0)); + let one = cultnet_rs::Selection { keys: Some(vec![id.id.0.clone()]), projection: "document".into(), ..of_kind(PipelineKind::Memory) }; + let PipelinePageItems::Documents(views) = mind.query(&one)?.items else { bail!("the mind answered headers to a document selection") }; + Ok(views.into_iter().find_map(|view| match view.document { + PipelineDocument::Memory(memory) => Some((format!("Out of nowhere you recall: {}", quoted(&memory.text.0)), id.clone())), + _ => None, + })) +} + +/// The seed, rebuilt from her mind and nothing else. `since` is her newest thought's label tick time (never later than its admission) and `fresh` what `news` found. +fn build_seed(mind: &dyn MindPort, persona: &PersonaRead, since: Option>, now: DateTime, responsibilities: &Responsibilities, fresh: Vec<(String, Option)>) -> Result { + let instance = mind.instance().clone(); + let state = &persona.state; + let mut reads: Vec = Vec::new(); + let mut cite = |id: PipelineRef| { + if !reads.contains(&id) { + reads.push(id); + } + }; + + let runs = recent_runs(mind, &instance)?; + let people = bonded_people(state); + let cue = Cue { senders: people.iter().map(String::as_str).collect(), texts: Vec::new(), names: BTreeMap::new(), listed: &responsibilities.listed, now }; + let recalled = recall::select(mind, &cue)?; + let heard = replies(mind, &instance)?; + let heard_section = section("What your Self said lately:", heard.iter().map(|(_, text)| format!("- {}", quoted(text))).collect()); + + let sections = [ + values_section(state), + pressures_section(state), + bonds_section(state), + section("Your latest runs:", runs.iter().map(|run| format!("- {run}")).collect()), + responsibilities.text.clone(), + recall::render(&recalled), + heard_section, + ]; + let kept: Vec = sections.iter().zip(BUDGETS).map(|(text, (_, budget))| cap(text, budget)).collect(); + // The reads are what the Projector was shown after the bounds: the recall lines and replies the capped sections kept. + // Each shown line is one citation: a memory reading like another takes its own line, not a share of the first's. + let mut shown: Vec<&str> = kept[5].lines().collect(); + for entry in recalled.entries() { + if let Some(at) = shown.iter().position(|line| *line == entry.line.trim_end()) { + shown.remove(at); + cite(PipelineRef { kind: PipelineKind::Memory, id: Short(entry.id.clone()) }); + } + } + let replies_shown = kept[6].lines().count().saturating_sub(1); + heard.iter().take(replies_shown).for_each(|(id, _)| cite(id.clone())); + let context = kept.iter().filter(|text| !text.trim().is_empty()).cloned().collect::>().join("\n"); + + let has_news = !fresh.is_empty(); + let stimulus = if has_news { + let body: Vec<&str> = fresh.iter().map(|(line, _)| line.as_str()).collect(); + let kept = cap(&format!("Since your last thought:\n{}", body.join("\n")), SECTION_MAX_BYTES); + // Each line is one line (`quoted` escapes newlines), so the lines the bound kept are what the thought read. + let shown = kept.lines().count().saturating_sub(1); + fresh.into_iter().take(shown).filter_map(|(_, id)| id).for_each(&mut cite); + kept + } else if let Some((line, id)) = unbidden(mind, &instance, now, u64::try_from(now.timestamp()).unwrap_or(0))? { + cite(id); + line + } else { + let at = since.map_or_else(|| "you began".to_owned(), |at| at.format("%-d %B %H:%M UTC").to_string()); + format!("Nothing has happened since {at}.") + }; + reads.truncate(READS_MAX); + Ok(Seed { identity: identity_of(state), context: cap(&context, SECTION_MAX_BYTES), stimulus: cap(&stimulus, SECTION_MAX_BYTES), news: has_news, reads }) +} + +// --------------------------------------------------------------------------------------------- +// Continuation, selection and the tick. + +/// The frame the base model continues: her notebook heading, the date, the projection verbatim +/// and an open paragraph. It instructs nothing. +fn continuer_frame(name: &str, date: &str, projection: &str) -> String { + format!("{name}'s notebook\n\n{date}\n\n{projection}\n\n") +} + +/// The fixed parts of both prompts, recorded in every thought: the frame is the function's own output over placeholders. +fn prompt_version() -> String { + format!( + "ghostlight={GHOSTLIGHT_REV}; words={PROJECTOR_WORDS}; guidance={STREAM_GUIDANCE:?}; frame={:?}", + continuer_frame("", &format!(""), "") + ) +} + +/// Each candidate's nearness to her earlier thoughts, in thousandths of the daemon's top score; +/// 1000 where she has none. +fn nearness(mind: &dyn MindPort, candidates: &[String]) -> Result> { + let members = in_force(mine(PipelineKind::Thought, mind.instance())); + let any = mind.query(&cultnet_rs::Selection { limit: Some(1), ..members.clone() })?; + if any.matched == 0 { + return Ok(vec![THOUGHT_NEAREST_NONE_PERMILLE; candidates.len()]); + } + candidates + .iter() + .map(|text| { + let page = mind.query_near(&members, text, 1)?; + let top = page.scores.as_ref().and_then(|scores| scores.first().copied()).ok_or_else(|| anyhow!("the index scored no earlier thought"))?; + Ok((f64::from(top) * 1000.0).round().clamp(0.0, 1000.0) as u16) + }) + .collect() +} + +/// The least near candidate, the first on a tie. +fn least_near(nearest: &[u16]) -> usize { + nearest.iter().enumerate().min_by_key(|(at, near)| (**near, *at)).map_or(0, |(at, _)| at) +} + +/// How a tick ended. +#[derive(Clone, Debug, PartialEq)] +pub enum Outcome { + /// A brake is held: nothing was read, asked or admitted. + Held(crate::control::HeldReason), + /// Not due by the cadence. + NotDue, + /// Every continuation was blank; nothing was admitted. The loop waits `STREAM_MIN_GAP_S`. + Blank, + /// The thought; admitted unless the tick was a dry run. + Thought { thought: Box, admitted: bool }, +} + +/// Whether a tick is a real one or the operator's look at what one would say. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Mode { + Live, + /// Both models run and nothing is admitted; the cadence does not hold it back. + DryRun, +} + +pub struct Organs<'a> { + pub mind: &'a dyn MindPort, + pub persona: &'a dyn PersonaSource, + pub model: &'a dyn StreamModel, + pub projector_model: &'a str, + pub base_model: &'a str, +} + +/// Whether a tick is due: `STREAM_MIN_GAP_S` has passed and there is news, or `STREAM_IDLE_GAP_S` has. +fn due(since: Option>, now: DateTime, news: bool) -> bool { + match since { + None => true, + Some(at) => { + let gap = (now - at).num_seconds(); + gap >= STREAM_IDLE_GAP_S || (gap >= STREAM_MIN_GAP_S && news) + } + } +} + +/// When her newest thought was ticked: its label's instant, never later than its admission. The models take minutes between the +/// two, and what the mind or the forge admitted meanwhile is news to the next thought, which the admission time would hide. +fn newest_thought(mind: &dyn MindPort) -> Result>> { + let newest = cultnet_rs::Selection { descending: true, limit: Some(1), ..mine(PipelineKind::Thought, mind.instance()) }; + let PipelinePageItems::Headers(found) = mind.query(&newest)?.items else { bail!("the mind answered documents to a header selection") }; + Ok(found + .iter() + .filter_map(|header| { + let admitted = DateTime::parse_from_rfc3339(&header.admission.admitted_at).ok()?.with_timezone(&Utc); + let label = header.id.id.0.rsplit(':').next().and_then(|label| label.strip_prefix('t')).and_then(|label| label.parse::().ok()).and_then(|label| DateTime::from_timestamp(label, 0)); + Some(label.map_or(admitted, |at| at.min(admitted))) + }) + .max()) +} + +fn bounded(text: &str, max: usize) -> String { + cap(text.trim(), max) +} + +/// One tick. A fault is an `Err`; the caller prints it and the process goes on. +pub fn tick(organs: &Organs, brake: &dyn Fn() -> Effective, mode: Mode, now: DateTime) -> Result { + if let Effective::Held { reason } = brake() { + return Ok(Outcome::Held(reason)); + } + let mind = organs.mind; + let since = newest_thought(mind)?; + if mode == Mode::Live && since.is_some_and(|at| (now - at).num_seconds() < STREAM_MIN_GAP_S) { + return Ok(Outcome::NotDue); + } + // Due-ness is decided on the cheap reads; the seed (recall, runs, replies) is built only for a tick that may run. + let responsibilities = render_responsibilities(mind, mind.instance(), &BTreeMap::new())?; + let mut fresh = mind_news(mind, mind.instance(), since)?; + // The forge is read when a tick will run anyway, and on a quiet stretch only by the poll that crosses each minimum-gap mark. + let quiet = mode == Mode::Live && !due(since, now, !fresh.is_empty()); + if quiet && since.is_some_and(|at| (now - at).num_seconds() % STREAM_MIN_GAP_S >= STREAM_POLL_S) { + return Ok(Outcome::NotDue); + } + fresh.extend(repo_news(mind, since, &responsibilities, now)); + if quiet && fresh.is_empty() { + return Ok(Outcome::NotDue); + } + let persona = organs.persona.persona()?; + let seed = build_seed(mind, &persona, since, now, &responsibilities, fresh)?; + + let began = std::time::Instant::now(); + let projection = bounded(&organs.model.project(&seed.projector_prompt())?, PARA_MAX_BYTES); + let projector_ms = u32::try_from(began.elapsed().as_millis()).unwrap_or(u32::MAX); + if projection.is_empty() { + return Ok(Outcome::Blank); + } + let frame = continuer_frame(text_of(&persona.state, &["publicName"]), &now.format(FRAME_DATE_FORMAT).to_string(), &projection); + let began = std::time::Instant::now(); + let candidates: Vec = organs + .model + .continue_raw(&frame, &Sampling::STREAM, CONTINUATIONS)? + .iter() + .map(|text| bounded(text, LINE_MAX_BYTES)) + .filter(|text| !text.is_empty()) + .take(THOUGHT_CANDIDATES_MAX) + .collect(); + let continuer_ms = u32::try_from(began.elapsed().as_millis()).unwrap_or(u32::MAX); + if candidates.is_empty() { + return Ok(Outcome::Blank); + } + let nearest_permille = nearness(mind, &candidates)?; + let chosen = least_near(&nearest_permille); + let thought = PipelineThought { + instance: mind.instance().clone(), + label: Label::from(format!("t{}", now.timestamp()).as_str()), + text: Para::from(candidates[chosen].as_str()), + reads: seed.reads.clone(), + persona_receipt: Short(persona.receipt_id.clone()), + identity: Line::from(seed.identity.as_str()), + context: Para::from(seed.context.as_str()), + stimulus: Para::from(seed.stimulus.as_str()), + projector_model: Short(organs.projector_model.to_owned()), + projection: Para::from(projection.as_str()), + base_model: Short(organs.base_model.to_owned()), + temperature_permille: Sampling::STREAM.temperature_permille, + min_p_permille: Sampling::STREAM.min_p_permille, + max_tokens: Sampling::STREAM.max_tokens, + candidates: candidates.iter().map(|text| Line::from(text.as_str())).collect(), + nearest_permille, + chosen: u8::try_from(chosen).unwrap_or(0), + projector_ms, + continuer_ms, + prompt_version: Line::from(prompt_version().as_str()), + }; + if mode == Mode::DryRun { + return Ok(Outcome::Thought { thought: Box::new(thought), admitted: false }); + } + let label = thought.label.0.clone(); + match mind.admit(AGENT, &format!("stream-{label}"), vec![PipelineDocument::Thought(thought.clone())])? { + PipelineAdmissionOutcome::Committed { .. } => Ok(Outcome::Thought { thought: Box::new(thought), admitted: true }), + other => Err(anyhow!("the mind did not admit the thought: {other:?}")), + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/mind-body/src/stream/tests.rs b/crates/mind-body/src/stream/tests.rs index ddc716f..c0f131a 100644 --- a/crates/mind-body/src/stream/tests.rs +++ b/crates/mind-body/src/stream/tests.rs @@ -1,863 +1,863 @@ -use std::cell::RefCell; -use std::collections::VecDeque; -use std::io::{Read, Write}; -use std::net::TcpListener; -use std::sync::{Arc, Mutex}; - -use anyhow::{Result, anyhow}; -use chrono::{DateTime, Utc}; -use cultnet_rs::Selection; -use eureka_pipeline::{MemorySort, PipelineDocument, PipelineKind, PipelineReply, PipelineThought, RunOperator, RunTurn, Slug}; -use huginn_mind::{PipelineAdmissionOutcome, PipelineSelectionPage, RepoActivity, RepoChange, RepoEvent}; -use serde_json::{Value, json}; - -use super::*; -use crate::control::{Grant, HeldReason}; -use crate::testkit::{INSTANCE, TestMind, at, memory, reference, run, run_ref}; - -fn t(secs: i64) -> DateTime { - at(secs * 1000) -} - -/// The label of a thought ticked at test-clock `secs`: the Unix seconds of that instant. -fn ticked(secs: i64) -> String { - format!("t{}", t(secs).timestamp()) -} - -fn released() -> Effective { - Effective::Released(Grant::for_test(Slug(INSTANCE.into()))) -} - -fn state() -> Value { - json!({ - "publicName": "Yggdrasil", - "publicDescription": "A Mind of the swarm.", - "presentation": { "pronouns": "she/her", "voiceSummary": "Plain." }, - "values": [{ "id": "v1", "label": "Coherence", "priority": 0.9, "summary": "Coherence over velocity." }], - "agencyPressure": { "pressures": [{ "summary": "The shaders are late." }] }, - "affect": { "socialBonds": [{ - "object": { "kind": "person", "id": "@libby:chat", "label": "Libby" }, - "relationshipKind": "collaborator", "summary": "Libby holds the sounds." - }] }, - }) -} - -struct Persona(PersonaRead); - -impl PersonaSource for Persona { - fn persona(&self) -> Result { - Ok(PersonaRead { state: self.0.state.clone(), receipt_id: self.0.receipt_id.clone() }) - } -} - -fn persona() -> Persona { - Persona(PersonaRead { state: state(), receipt_id: "mind-commit-1".into() }) -} - -struct Fake { - projection: String, - candidates: Vec, - fault: bool, - projected: RefCell>, - continued: RefCell>, -} - -impl Fake { - fn saying(projection: &str, candidates: &[&str]) -> Self { - Self { projection: projection.into(), candidates: candidates.iter().map(|text| (*text).into()).collect(), fault: false, projected: RefCell::default(), continued: RefCell::default() } - } - - fn calls(&self) -> usize { - self.projected.borrow().len() + self.continued.borrow().len() - } -} - -impl StreamModel for Fake { - fn project(&self, prompt: &str) -> Result { - self.projected.borrow_mut().push(prompt.into()); - if self.fault { Err(anyhow!("the model host is down")) } else { Ok(self.projection.clone()) } - } - - fn continue_raw(&self, prompt: &str, sampling: &Sampling, n: usize) -> Result> { - self.continued.borrow_mut().push((prompt.into(), *sampling, n)); - Ok(self.candidates.iter().take(n).cloned().collect()) - } -} - -/// A real mind that records what is asked and admitted, and scores semantic asks as told. -struct Wrap<'a> { - inner: &'a TestMind, - scores: RefCell>, - selections: RefCell>, - admitted: RefCell>>, - forge: Option, - forge_reads: std::cell::Cell, -} - -impl<'a> Wrap<'a> { - fn over(inner: &'a TestMind, scores: &[f32]) -> Self { - Self { inner, scores: RefCell::new(scores.iter().copied().collect()), selections: RefCell::default(), admitted: RefCell::default(), forge: None, forge_reads: std::cell::Cell::new(0) } - } - - /// The same mind with a forge that answers `events`, whatever the asker's cursor. - fn with_forge(mut self, events: Vec) -> Self { - self.forge = Some(RepoActivity { head: 1, events, capped: vec![], open_prs: vec![], unseen: vec![], moved: vec![] }); - self - } -} - -impl MindPort for Wrap<'_> { - fn instance(&self) -> &Slug { - self.inner.instance() - } - - fn query(&self, selection: &Selection) -> Result { - self.selections.borrow_mut().push(selection.clone()); - self.inner.query(selection) - } - - fn query_near(&self, selection: &Selection, _text: &str, _top_k: u32) -> Result { - self.selections.borrow_mut().push(selection.clone()); - let mut page = self.inner.query(selection)?; - page.scores = Some(vec![self.scores.borrow_mut().pop_front().ok_or_else(|| anyhow!("no score was scripted"))?]); - Ok(page) - } - - fn roots_touched(&self) -> Result> { - self.inner.roots_touched() - } - - fn stewarded_activity(&self, repos: &[huginn_mind::eureka_pipeline::OrgRepo], after: Option) -> Result { - self.forge_reads.set(self.forge_reads.get() + 1); - match &self.forge { - Some(activity) => Ok(activity.clone()), - None => self.inner.stewarded_activity(repos, after), - } - } - - fn admit(&self, agent: &str, session: &str, documents: Vec) -> Result { - self.admitted.borrow_mut().push(documents.clone()); - self.inner.admit(agent, session, documents) - } -} - -/// The seed as the tick builds it: the cheap reads first, then the rest. -fn seed_of(mind: &dyn MindPort, read: &PersonaRead, since: Option>, now: DateTime) -> Result { - let responsibilities = render_responsibilities(mind, mind.instance(), &BTreeMap::new())?; - let mut fresh = mind_news(mind, mind.instance(), since)?; - fresh.extend(repo_news(mind, since, &responsibilities, now)); - build_seed(mind, read, since, now, &responsibilities, fresh) -} - -/// A push by someone else to her repo at `secs`. -fn push(id: u64, secs: i64) -> RepoEvent { - RepoEvent { id, at: u64::try_from(t(secs).timestamp_millis()).unwrap(), repo: "GameCult/Huginn".into(), actor: "libby".into(), change: RepoChange::Pushed { commits: 1 } } -} - -/// Counts how often her persona is read, which only a tick that may run does. -struct CountedPersona(Persona, std::cell::Cell); - -impl PersonaSource for CountedPersona { - fn persona(&self) -> Result { - self.1.set(self.1.get() + 1); - self.0.persona() - } -} - -fn thought_doc(label: &str, text: &str) -> PipelineDocument { - PipelineDocument::Thought(PipelineThought { - instance: Slug(INSTANCE.into()), - label: label.into(), - text: text.into(), - reads: vec![], - persona_receipt: "mind-commit-0".into(), - identity: "I".into(), - context: "".into(), - stimulus: "".into(), - projector_model: "projector".into(), - projection: "A projection.".into(), - base_model: "base".into(), - temperature_permille: 1000, - min_p_permille: 50, - max_tokens: 160, - candidates: vec![text.into()], - nearest_permille: vec![1000], - chosen: 0, - projector_ms: 0, - continuer_ms: 0, - prompt_version: "test".into(), - }) -} - -/// A mind with an old memory about Libby and one thought at `thought_at`. -fn mind_with_thought(thought_at: i64) -> TestMind { - let mind = TestMind::seeded(); - mind.admit_at(t(10)); - mind.committed(vec![memory("m1", MemorySort::Learned, "Libby wants the sounds before the shaders", &["@libby:chat"])]); - mind.admit_at(t(thought_at)); - mind.committed(vec![thought_doc(&ticked(thought_at), "An earlier drift")]); - mind -} - -fn organs<'a>(mind: &'a dyn MindPort, persona: &'a Persona, model: &'a Fake) -> Organs<'a> { - Organs { mind, persona, model, projector_model: "projector:test", base_model: "base:test" } -} - -fn thoughts(mind: &TestMind) -> Vec { - mind.ids(PipelineKind::Thought) -} - -#[test] -fn budgets_leave_room_for_every_section() { - let sum: usize = BUDGETS.iter().map(|(_, budget)| budget).sum(); - assert!(sum + BUDGETS.len() <= SECTION_MAX_BYTES, "{sum}"); -} - -#[test] -fn seed_reads_no_thought() { - let without = TestMind::seeded(); - without.committed(vec![memory("m1", MemorySort::Learned, "Libby wants the sounds first", &["@libby:chat"])]); - let with = TestMind::seeded(); - with.committed(vec![memory("m1", MemorySort::Learned, "Libby wants the sounds first", &["@libby:chat"])]); - with.committed(vec![thought_doc("t1", "A drift that must stay out of the seed")]); - let (plain, thoughtful) = (Wrap::over(&without, &[]), Wrap::over(&with, &[])); - let read = PersonaRead { state: state(), receipt_id: "r".into() }; - let one = seed_of(&plain, &read, None, t(100_000)).unwrap(); - let two = seed_of(&thoughtful, &read, None, t(100_000)).unwrap(); - assert_eq!(one.projector_prompt(), two.projector_prompt(), "a mind holding thoughts yields the prompt of the same mind without them"); - assert_eq!(one.reads, two.reads); - assert!(!two.projector_prompt().contains("must stay out")); - let thought_schema = PipelineKind::Thought.type_id(); - for selection in thoughtful.selections.borrow().iter() { - assert!(!selection.schemas.iter().flatten().any(|schema| schema == thought_schema), "a seed selection named thoughts: {selection:?}"); - } -} - -#[test] -fn seed_carries_relationships() { - let mind = TestMind::seeded(); - mind.committed(vec![ - memory("m1", MemorySort::Learned, "Libby wants the sounds before the shaders", &["@libby:chat"]), - memory("m2", MemorySort::Rapport, "Noor distrusts the new scheduler", &["@noor:chat"]), - run("w1", RunTurn::SelfRun, RunOperator::Mind, &[]), - ]); - mind.committed(vec![PipelineDocument::Reply(PipelineReply { - instance: Slug(INSTANCE.into()), - label: "w1-r1".into(), - run: run_ref("w1"), - answers: vec![], - text: "The shader pass is done; the sounds are Libby's.".into(), - })]); - let seed = seed_of(&Wrap::over(&mind, &[]), &PersonaRead { state: state(), receipt_id: "r".into() }, None, t(100_000)).unwrap(); - for wanted in ["Libby holds the sounds", "collaborator", "Libby wants the sounds before the shaders", "Noor distrusts the new scheduler", "The shader pass is done", "Coherence over velocity", "The shaders are late"] { - assert!(seed.context.contains(wanted), "{wanted:?} missing from {}", seed.context); - } - assert!(seed.identity.contains("Yggdrasil") && seed.identity.contains("she/her") && seed.identity.contains("Plain.")); - assert!(seed.reads.iter().any(|read| read.kind == PipelineKind::Reply)); - assert!(seed.reads.iter().any(|read| read.kind == PipelineKind::Memory)); -} - -#[test] -fn seed_sections_keep_budgets() { - let huge = "x".repeat(3000); - let mut crowded = state(); - crowded["values"] = json!((0..40).map(|at| json!({ "id": format!("v{at}"), "label": format!("Value {at}"), "priority": 0.5, "summary": huge })).collect::>()); - crowded["agencyPressure"]["pressures"] = json!((0..40).map(|_| json!({ "summary": huge })).collect::>()); - let mind = TestMind::seeded(); - mind.committed(vec![memory("m1", MemorySort::Learned, "Libby wants the sounds first", &["@libby:chat"])]); - let seed = seed_of(&Wrap::over(&mind, &[]), &PersonaRead { state: crowded, receipt_id: "r".into() }, None, t(100_000)).unwrap(); - assert!(seed.context.len() <= SECTION_MAX_BYTES, "{}", seed.context.len()); - assert!(seed.context.contains("Libby holds the sounds"), "oversized values and pressures crowded out the bonds"); - assert!(seed.context.contains("Libby wants the sounds first"), "oversized values and pressures crowded out the recall"); - let values = seed.context.split("What presses on her:").next().unwrap(); - assert!(values.len() <= BUDGETS[0].1 + 1, "values took {}", values.len()); -} - -#[test] -fn stimulus_is_what_is_new() { - let mind = TestMind::seeded(); - mind.admit_at(t(100)); - mind.committed(vec![memory("m1", MemorySort::Learned, "Before the thought, the kettle", &[])]); - mind.admit_at(t(200)); - mind.committed(vec![thought_doc(&ticked(200), "A drift")]); - mind.admit_at(t(300)); - mind.committed(vec![memory("m2", MemorySort::Decided, "After the thought, the window", &[]), run("w9", RunTurn::SelfRun, RunOperator::Mind, &[])]); - mind.admit_at(t(350)); - mind.committed(vec![crate::testkit::close(run_ref("w9"), crate::testkit::recorded())]); - let wrap = Wrap::over(&mind, &[]); - let read = PersonaRead { state: state(), receipt_id: "r".into() }; - let seed = seed_of(&wrap, &read, Some(t(200)), t(400)).unwrap(); - assert!(seed.news); - assert!(seed.stimulus.contains("After the thought, the window"), "{}", seed.stimulus); - assert!(seed.stimulus.contains("w9 closed"), "a run closure is news: {}", seed.stimulus); - assert!(!seed.stimulus.contains("Before the thought, the kettle"), "{}", seed.stimulus); - assert!(seed.stimulus.find("closed").unwrap() < seed.stimulus.find("After the thought").unwrap(), "newest first"); - assert!(seed.reads.contains(&reference(PipelineKind::Memory, &format!("{INSTANCE}:memory:m2")))); -} - -#[test] -fn stimulus_without_news_is_an_unbidden_memory_chosen_by_the_tick_label_else_the_nothing_line() { - let mind = TestMind::seeded(); - mind.admit_at(t(100)); - mind.committed(vec![memory("m1", MemorySort::Learned, "The first old thing", &[]), memory("m2", MemorySort::Learned, "The second old thing", &[])]); - let long_after = 100 * 86_400; - mind.admit_at(t(long_after - 100)); - mind.committed(vec![memory("m3", MemorySort::Learned, "A recent thing", &[])]); - let wrap = Wrap::over(&mind, &[]); - let read = PersonaRead { state: state(), receipt_id: "r".into() }; - let picks: Vec = (0..4) - .map(|offset| { - let seed = seed_of(&wrap, &read, Some(t(long_after - 10)), t(long_after + offset)).unwrap(); - assert!(!seed.news); - seed.stimulus - }) - .collect(); - assert!(picks.iter().all(|stimulus| stimulus.starts_with("Out of nowhere you recall")), "{picks:?}"); - assert!(picks.iter().all(|stimulus| !stimulus.contains("recent thing")), "a memory under 7 days old is not unbidden"); - assert!(picks.iter().any(|stimulus| stimulus.contains("first old")) && picks.iter().any(|stimulus| stimulus.contains("second old")), "the label seeds a uniform pick: {picks:?}"); - assert_eq!(picks[0], picks[2], "the same label seed picks the same memory"); - - let bare = TestMind::seeded(); - let seed = seed_of(&Wrap::over(&bare, &[]), &read, Some(t(1000)), t(5000)).unwrap(); - assert!(!seed.news); - assert!(seed.stimulus.starts_with("Nothing has happened since"), "{}", seed.stimulus); -} - -#[test] -fn cadence_from_newest_thought() { - let mind = mind_with_thought(1000); - let model = Fake::saying("I turn it over.", &["A new drift"]); - let persona = persona(); - let wrap = Wrap::over(&mind, &[0.1]); - let organs = organs(&wrap, &persona, &model); - let tick_at = |secs: i64| tick(&organs, &|| released(), Mode::Live, t(secs)).unwrap(); - assert_eq!(tick_at(1000 + STREAM_MIN_GAP_S - 1), Outcome::NotDue, "no tick before the minimum gap"); - assert_eq!(tick_at(1000 + STREAM_MIN_GAP_S), Outcome::NotDue, "past the minimum gap but nothing is new"); - assert_eq!(model.calls(), 0); - assert_eq!(tick_at(1000 + STREAM_IDLE_GAP_S - 1), Outcome::NotDue); - assert_eq!(thoughts(&mind).len(), 1); - // Something new, past the minimum gap but short of the idle gap. - mind.admit_at(t(1000 + STREAM_MIN_GAP_S)); - mind.committed(vec![memory("m9", MemorySort::Decided, "Something happened", &[])]); - let Outcome::Thought { admitted: true, .. } = tick_at(1000 + STREAM_MIN_GAP_S + 1) else { panic!("news past the minimum gap makes a thought") }; - assert_eq!(thoughts(&mind).len(), 2); -} - -#[test] -fn the_idle_gap_alone_makes_a_tick_due() { - let mind = mind_with_thought(1000); - let model = Fake::saying("I turn it over.", &["A new drift"]); - let persona = persona(); - let wrap = Wrap::over(&mind, &[0.1]); - let Outcome::Thought { admitted: true, thought } = tick(&organs(&wrap, &persona, &model), &|| released(), Mode::Live, t(1000 + STREAM_IDLE_GAP_S)).unwrap() else { panic!("idle gap") }; - assert_eq!(thought.label.0, format!("t{}", t(1000 + STREAM_IDLE_GAP_S).timestamp())); - assert!(due(None, t(0), false) && !due(Some(t(0)), t(STREAM_MIN_GAP_S), false) && due(Some(t(0)), t(STREAM_MIN_GAP_S), true)); -} - -#[test] -fn held_brake_ticks_nothing() { - let mind = mind_with_thought(1000); - let model = Fake::saying("I turn it over.", &["A new drift"]); - let persona = persona(); - let wrap = Wrap::over(&mind, &[0.1]); - for reason in [HeldReason::BrakeHeld, HeldReason::BrakeAbsent, HeldReason::Undecodable] { - let held = tick(&organs(&wrap, &persona, &model), &move || Effective::Held { reason }, Mode::Live, t(100_000)).unwrap(); - assert_eq!(held, Outcome::Held(reason)); - let dry = tick(&organs(&wrap, &persona, &model), &move || Effective::Held { reason }, Mode::DryRun, t(100_000)).unwrap(); - assert_eq!(dry, Outcome::Held(reason), "a dry run is held too"); - } - assert_eq!(model.calls(), 0, "no model call"); - assert!(wrap.admitted.borrow().is_empty() && wrap.selections.borrow().is_empty(), "no admission, and the mind was not read"); -} - -#[test] -fn least_near_candidate_chosen() { - let mind = mind_with_thought(1000); - let model = Fake::saying("I turn it over.", &["one", "two", "three", "four"]); - let persona = persona(); - let wrap = Wrap::over(&mind, &[0.9, 0.2, 0.2, 0.5]); - let Outcome::Thought { thought, admitted: true } = tick(&organs(&wrap, &persona, &model), &|| released(), Mode::Live, t(1000 + STREAM_IDLE_GAP_S)).unwrap() else { panic!("a thought") }; - assert_eq!(thought.nearest_permille, vec![900, 200, 200, 500]); - assert_eq!(thought.chosen, 1, "the least near, the first on a tie"); - assert_eq!(thought.text.0, "two"); - assert_eq!(thought.candidates.iter().map(|line| line.0.as_str()).collect::>(), ["one", "two", "three", "four"], "every candidate is kept in the trace"); - assert_eq!(least_near(&[500, 100, 100]), 1); - assert_eq!(least_near(&[]), 0); -} - -#[test] -fn no_earlier_thought_is_nearness_1000() { - let mind = TestMind::seeded(); - let model = Fake::saying("I turn it over.", &["one", "two"]); - let persona = persona(); - let wrap = Wrap::over(&mind, &[]); - let Outcome::Thought { thought, .. } = tick(&organs(&wrap, &persona, &model), &|| released(), Mode::Live, t(5000)).unwrap() else { panic!("a thought") }; - assert_eq!(thought.nearest_permille, vec![1000, 1000]); - assert_eq!(thought.chosen, 0); - assert_eq!((thought.projector_model.0.as_str(), thought.base_model.0.as_str(), thought.persona_receipt.0.as_str()), ("projector:test", "base:test", "mind-commit-1")); - assert_eq!((thought.temperature_permille, thought.min_p_permille, thought.max_tokens), (1000, 50, 160)); -} - -#[test] -fn frame_holds_projection_verbatim() { - assert_eq!(continuer_frame("Yggdrasil", "9 October 2026", "I turn it over."), "Yggdrasil's notebook\n\n9 October 2026\n\nI turn it over.\n\n"); - let mind = mind_with_thought(1000); - let model = Fake::saying("I keep turning over the sounds.", &["a", "b", "c", "d", "e"]); - let persona = persona(); - let wrap = Wrap::over(&mind, &[0.3, 0.3, 0.3, 0.3]); - tick(&organs(&wrap, &persona, &model), &|| released(), Mode::Live, t(1000 + STREAM_IDLE_GAP_S)).unwrap(); - let continued = model.continued.borrow(); - let (prompt, sampling, n) = &continued[0]; - assert_eq!(prompt, &continuer_frame("Yggdrasil", &t(1000 + STREAM_IDLE_GAP_S).format(FRAME_DATE_FORMAT).to_string(), "I keep turning over the sounds.")); - assert_eq!((*n, *sampling), (4, Sampling::STREAM)); - for foreign in ["Libby", "Coherence", "Plain.", "she/her", "private Projector"] { - assert!(!prompt.contains(foreign), "{foreign} reached the continuer"); - } - let projected = model.projected.borrow(); - assert!(projected[0].contains("private Projector") && projected[0].contains(STREAM_GUIDANCE), "the Projector prompt is Ghostlight's builder over her seed"); - assert!(projected[0].contains("Libby holds the sounds")); -} - -#[test] -fn blank_and_faulty_ticks_admit_nothing() { - let mind = TestMind::seeded(); - let persona = persona(); - let blank = Fake::saying("A projection.", &["", " \n", "\t"]); - let wrap = Wrap::over(&mind, &[]); - assert_eq!(tick(&organs(&wrap, &persona, &blank), &|| released(), Mode::Live, t(5000)).unwrap(), Outcome::Blank); - let silent_projector = Fake::saying(" ", &["x"]); - assert_eq!(tick(&organs(&wrap, &persona, &silent_projector), &|| released(), Mode::Live, t(5000)).unwrap(), Outcome::Blank); - let mut down = Fake::saying("x", &["x"]); - down.fault = true; - assert!(tick(&organs(&wrap, &persona, &down), &|| released(), Mode::Live, t(5000)).is_err()); - assert!(wrap.admitted.borrow().is_empty() && thoughts(&mind).is_empty()); -} - -#[test] -fn dry_run_admits_nothing() { - let mind = mind_with_thought(1000); - let model = Fake::saying("I turn it over.", &["A new drift", "Another"]); - let persona = persona(); - let wrap = Wrap::over(&mind, &[0.4, 0.6]); - let Outcome::Thought { thought, admitted } = tick(&organs(&wrap, &persona, &model), &|| released(), Mode::DryRun, t(1001)).unwrap() else { panic!("a dry run prints a thought") }; - assert!(!admitted); - assert_eq!(thought.text.0, "A new drift"); - assert!(wrap.admitted.borrow().is_empty()); - assert_eq!(thoughts(&mind).len(), 1); - assert!(serde_json::to_string(&*thought).unwrap().contains("\"candidates\"")); -} - -#[test] -fn stream_admits_only_thoughts() { - let mind = TestMind::seeded(); - mind.committed(vec![memory("m1", MemorySort::Learned, "Libby wants the sounds first", &["@libby:chat"])]); - let model = Fake::saying("I turn it over.", &["A drift", "Another"]); - let persona = persona(); - let wrap = Wrap::over(&mind, &[]); - let Outcome::Thought { thought, admitted: true } = tick(&organs(&wrap, &persona, &model), &|| released(), Mode::Live, t(100_000)).unwrap() else { panic!("a thought") }; - let batches = wrap.admitted.borrow(); - assert_eq!(batches.len(), 1); - assert!(matches!(batches[0].as_slice(), [PipelineDocument::Thought(_)]), "exactly one thought and nothing else"); - let stored = mind.view(PipelineKind::Thought, &format!("{INSTANCE}:thought:t{}", t(100_000).timestamp())).expect("admitted under its instance-rooted key"); - assert_eq!(stored.document, PipelineDocument::Thought(*thought.clone())); - assert!(thought.reads.iter().any(|read| read.kind == PipelineKind::Memory), "the thought cites what the seed read"); - assert_eq!(mind.ids(PipelineKind::Memory).len(), 1, "no memory was written"); -} - -// --------------------------------------------------------------------------------------------- -// The model port against a fake HTTP server. - -type Seen = Arc>>; - -fn serve(answers: Vec) -> (String, Seen) { - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let endpoint = format!("http://{}", listener.local_addr().unwrap()); - let seen: Seen = Arc::default(); - let record = Arc::clone(&seen); - std::thread::spawn(move || { - for answer in answers { - let (mut socket, _) = listener.accept().unwrap(); - let mut raw = Vec::new(); - let mut buffer = [0u8; 4096]; - let (head, length) = loop { - let read = socket.read(&mut buffer).unwrap(); - raw.extend_from_slice(&buffer[..read]); - if let Some(end) = raw.windows(4).position(|window| window == b"\r\n\r\n") { - let head = String::from_utf8_lossy(&raw[..end]).to_string(); - let length = head.lines().find_map(|line| line.to_ascii_lowercase().strip_prefix("content-length:").map(|value| value.trim().parse::().unwrap())).unwrap_or(0); - if raw.len() >= end + 4 + length { - break (head, (end + 4, length)); - } - } - assert!(read > 0, "the client closed early"); - }; - let body: Value = serde_json::from_slice(&raw[length.0..length.0 + length.1]).unwrap(); - let path = head.lines().next().unwrap().split_whitespace().nth(1).unwrap().to_owned(); - record.lock().unwrap().push((path, body)); - write!(socket, "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{answer}", answer.len()).unwrap(); - } - }); - (endpoint, seen) -} - -#[test] -fn raw_generate_is_raw() { - let (endpoint, seen) = serve(vec![json!({ "response": "one" }).to_string(), json!({ "response": "two" }).to_string()]); - let model = OllamaStream::open(&endpoint, "projector:x", "base:y").unwrap(); - let out = model.continue_raw("The frame\n\n", &Sampling::STREAM, 2).unwrap(); - assert_eq!(out, ["one", "two"]); - let seen = seen.lock().unwrap(); - assert_eq!(seen.len(), 2, "one call per continuation"); - for (path, body) in seen.iter() { - assert_eq!(path, "/api/generate"); - assert_eq!(body["raw"], json!(true)); - assert_eq!(body["stream"], json!(false)); - assert_eq!(body["model"], json!("base:y")); - assert_eq!(body["prompt"], json!("The frame\n\n")); - assert_eq!(body["options"], json!({ "temperature": 1.0, "min_p": 0.05, "num_predict": 160, "stop": ["\n\n"], "num_thread": 4 })); - } -} - -#[test] -fn the_projector_is_asked_in_chat() { - let (endpoint, seen) = serve(vec![json!({ "message": { "role": "assistant", "content": "I hold it." } }).to_string()]); - let model = OllamaStream::open(&endpoint, "projector:x", "base:y").unwrap(); - assert_eq!(model.project("Be the Projector.").unwrap(), "I hold it."); - let seen = seen.lock().unwrap(); - assert_eq!(seen[0].0, "/api/chat"); - assert_eq!(seen[0].1, json!({ "model": "projector:x", "stream": false, "messages": [{ "role": "user", "content": "Be the Projector." }] })); -} - -#[test] -fn a_model_host_that_answers_badly_is_a_fault() { - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let endpoint = format!("http://{}", listener.local_addr().unwrap()); - std::thread::spawn(move || { - let (mut socket, _) = listener.accept().unwrap(); - let mut buffer = [0u8; 4096]; - let _ = socket.read(&mut buffer); - let _ = socket.write_all(b"HTTP/1.1 500 Internal Server Error\r\ncontent-length: 4\r\nconnection: close\r\n\r\nboom"); - }); - let model = OllamaStream::open(&endpoint, "p", "b").unwrap(); - let fault = model.project("x").unwrap_err().to_string(); - assert!(fault.contains("500"), "{fault}"); -} - -#[test] -fn non_loopback_endpoint_refused() { - for refused in [ - "http://localhost:11434@example.com", - "http://localhost@example.com", - "http://127.0.0.1:11434@example.com:80", - "http://user:secret@127.0.0.1:11434", - "http://127.0.0.1:11434@localhost", - "http://[::1]:11434@example.com", - "http://localhost:11434\\@example.com", - "http://example.com#@127.0.0.1", - "http://example.com?@localhost", - "http://127.0.0.1:11434?x=1", - "http://localhost.example.com:11434", - "http://", - "", - ] { - assert!(OllamaStream::open(refused, "p", "b").is_err(), "{refused} must be refused"); - } - for refused in ["http://10.77.0.1:11434", "http://example.com:11434", "http://0.0.0.0:11434", "https://127.0.0.1:11434", "127.0.0.1:11434", "http://127.0.0.1:11434/api", "http://[2001:db8::1]:11434"] { - assert!(OllamaStream::open(refused, "p", "b").is_err(), "{refused} must be refused"); - } - for opened in ["http://127.0.0.1:11434", "http://localhost:11434", "http://[::1]:11434", "http://127.0.0.2:1/", "http://LOCALHOST:11434/"] { - assert!(OllamaStream::open(opened, "p", "b").is_ok(), "{opened} is loopback"); - } -} - -#[test] -fn cap_cuts_at_a_line_else_at_a_character() { - assert_eq!(cap("short", 10), "short"); - assert_eq!(cap("ab\ncd", 4), "ab", "whole lines only"); - assert_eq!(cap("abcdef", 4), "abcd", "no newline to cut at"); - assert_eq!(cap("\nabc", 3), "\nab", "a newline at the very start is no cut point"); - assert_eq!(cap("aé\nb", 2), "a", "never inside a character"); - assert_eq!(cap("ééé", 5), "éé"); -} - -#[test] -fn bonds_name_the_label_else_the_id_else_someone() { - let line = |bond: Value| bond_line(&bond); - assert_eq!(line(json!({ "object": { "label": "Libby", "id": "@libby:chat" }, "relationshipKind": "friend", "summary": "s" })), "- Libby (friend): s"); - assert_eq!(line(json!({ "object": { "label": "", "id": "@libby:chat" }, "relationshipKind": "friend", "summary": "s" })), "- @libby:chat (friend): s"); - assert_eq!(line(json!({ "object": {}, "relationshipKind": "friend", "summary": "s" })), "- someone (friend): s"); -} - -#[test] -fn only_person_bonds_with_ids_are_people_she_asks_her_memories_about() { - let state = json!({ "affect": { "socialBonds": [ - { "object": { "kind": "person", "id": "@libby:chat" } }, - { "object": { "kind": "repo", "id": "GameCult/Huginn" } }, - { "object": { "kind": "person", "id": "" } }, - { "object": { "kind": "person", "id": "@noor:chat" } }, - ] } }); - assert_eq!(bonded_people(&state), ["@libby:chat", "@noor:chat"]); - assert!(bonded_people(&json!({})).is_empty()); -} - -#[test] -fn news_is_strictly_after_her_newest_thought() { - let mind = TestMind::seeded(); - mind.admit_at(t(500)); - mind.committed(vec![memory("m1", MemorySort::Learned, "Admitted at the very instant of the thought", &[])]); - let read = PersonaRead { state: state(), receipt_id: "r".into() }; - let seed = seed_of(&Wrap::over(&mind, &[]), &read, Some(t(500)), t(600)).unwrap(); - assert!(!seed.news, "{}", seed.stimulus); - let seed = seed_of(&Wrap::over(&mind, &[]), &read, Some(t(499)), t(600)).unwrap(); - assert!(seed.news); -} - -#[test] -fn an_unbidden_memory_is_older_than_seven_days_not_exactly_seven() { - let mind = TestMind::seeded(); - mind.admit_at(t(1000)); - mind.committed(vec![memory("m1", MemorySort::Learned, "The week-old thing", &[])]); - let read = PersonaRead { state: state(), receipt_id: "r".into() }; - let week = 7 * 86_400; - let exactly = seed_of(&Wrap::over(&mind, &[]), &read, Some(t(1000)), t(1000 + week)).unwrap(); - assert!(exactly.stimulus.starts_with("Nothing has happened"), "{}", exactly.stimulus); - let over = seed_of(&Wrap::over(&mind, &[]), &read, Some(t(1000)), t(1001 + week)).unwrap(); - assert!(over.stimulus.contains("The week-old thing"), "{}", over.stimulus); -} - -#[test] -fn a_flood_of_news_fills_the_stimulus_with_whole_lines_newest_first() { - let mind = TestMind::seeded(); - let flood: Vec = (0..60).map(|at| memory(&format!("f{at:02}"), MemorySort::Learned, &format!("{at:02} {}", "w".repeat(150)), &[])).collect(); - mind.admit_at(t(100)); - mind.committed(flood); - let read = PersonaRead { state: state(), receipt_id: "r".into() }; - let seed = seed_of(&Wrap::over(&mind, &[]), &read, Some(t(50)), t(200)).unwrap(); - let lines: Vec<&str> = seed.stimulus.lines().skip(1).collect(); - assert!(seed.stimulus.len() <= SECTION_MAX_BYTES && seed.stimulus.len() > SECTION_MAX_BYTES - 250, "{}", seed.stimulus.len()); - assert!(lines.iter().all(|line| line.starts_with("- You noted") && line.ends_with('"')), "a line was cut: {lines:?}"); - assert!(lines.len() > 10 && lines.len() < 30, "{}", lines.len()); - let cited = seed.reads.iter().filter(|read| read.kind == PipelineKind::Memory).count(); - assert!(cited >= lines.len() && cited <= lines.len() + 12, "the thought cites what its stimulus showed: {cited} reads for {} lines", lines.len()); - let newest = lines[0]; - assert!(newest.contains("\"29 ") || newest.contains("\"59 "), "newest first: {newest}"); -} - -#[test] -fn the_newest_thought_sets_the_cadence_not_the_oldest() { - let mind = TestMind::seeded(); - mind.admit_at(t(100)); - mind.committed(vec![thought_doc(&ticked(100), "An old drift")]); - mind.admit_at(t(10_000)); - mind.committed(vec![thought_doc(&ticked(10_000), "A recent drift")]); - let model = Fake::saying("I turn it over.", &["A new drift"]); - let persona = persona(); - let wrap = Wrap::over(&mind, &[0.1]); - let tick_at = |secs: i64| tick(&organs(&wrap, &persona, &model), &|| released(), Mode::Live, t(secs)).unwrap(); - assert_eq!(tick_at(10_000 + STREAM_MIN_GAP_S - 1), Outcome::NotDue, "a recent thought holds the stream back whatever the oldest says"); - assert_eq!(model.calls(), 0); -} - -#[test] -fn news_at_exactly_the_minimum_gap_is_due() { - let mind = mind_with_thought(1000); - mind.admit_at(t(1500)); - mind.committed(vec![memory("m9", MemorySort::Decided, "Something happened", &[])]); - let model = Fake::saying("I turn it over.", &["A new drift"]); - let persona = persona(); - let wrap = Wrap::over(&mind, &[0.1]); - let Outcome::Thought { admitted: true, .. } = tick(&organs(&wrap, &persona, &model), &|| released(), Mode::Live, t(1000 + STREAM_MIN_GAP_S)).unwrap() else { panic!("due at the minimum gap") }; -} - -#[test] -fn repo_activity_is_news_only_after_her_newest_thought_and_is_stamped_when_it_happened() { - let mind = mind_with_thought(1000); - let read = PersonaRead { state: state(), receipt_id: "r".into() }; - let old = Wrap::over(&mind, &[]).with_forge(vec![push(1, 500)]); - let seed = seed_of(&old, &read, Some(t(1000)), t(1901)).unwrap(); - assert!(!seed.news && !seed.stimulus.contains("Changes in your repos"), "an event older than her newest thought is not news: {}", seed.stimulus); - let edge = Wrap::over(&mind, &[]).with_forge(vec![push(1, 1000)]); - assert!(!seed_of(&edge, &read, Some(t(1000)), t(1901)).unwrap().news, "an event at her newest thought's time is not after it"); - let mixed = Wrap::over(&mind, &[]).with_forge(vec![push(2, 1500), push(1, 500)]); - let seed = seed_of(&mixed, &read, Some(t(1000)), t(1901)).unwrap(); - assert!(seed.news && seed.stimulus.contains("Changes in your repos: GameCult/Huginn: 1 commit pushed by libby"), "{}", seed.stimulus); - let stamp = t(1500).to_rfc3339_opts(chrono::SecondsFormat::Secs, true); - assert!(seed.stimulus.contains(&format!("at={stamp} ")), "the notice carries the event's time, not the tick's: {}", seed.stimulus); - let first = Wrap::over(&mind, &[]).with_forge(vec![push(1, 500)]); - assert!(seed_of(&first, &read, None, t(1901)).unwrap().news, "with no thought yet, what the forge holds is news"); -} - -#[test] -fn a_capped_repo_is_news_only_beside_an_event_after_her_newest_thought() { - let mind = mind_with_thought(1000); - let read = PersonaRead { state: state(), receipt_id: "r".into() }; - let mut quiet = Wrap::over(&mind, &[]).with_forge(vec![push(1, 500)]); - quiet.forge.as_mut().unwrap().capped.push("GameCult/Huginn".into()); - assert!(!seed_of(&quiet, &read, Some(t(1000)), t(1901)).unwrap().news); - let mut busy = Wrap::over(&mind, &[]).with_forge(vec![push(2, 1500)]); - busy.forge.as_mut().unwrap().capped.push("GameCult/Huginn".into()); - let seed = seed_of(&busy, &read, Some(t(1000)), t(1901)).unwrap(); - assert!(seed.news && seed.stimulus.to_lowercase().contains("page limit"), "{}", seed.stimulus); -} - -#[test] -fn old_repo_activity_does_not_make_a_tick_due() { - let mind = mind_with_thought(1000); - let model = Fake::saying("I turn it over.", &["A new drift"]); - let persona = CountedPersona(persona(), std::cell::Cell::new(0)); - let wrap = Wrap::over(&mind, &[0.1]).with_forge(vec![push(1, 500)]); - let organs = Organs { mind: &wrap, persona: &persona, model: &model, projector_model: "projector:test", base_model: "base:test" }; - assert_eq!(tick(&organs, &|| released(), Mode::Live, t(1901)).unwrap(), Outcome::NotDue); - assert_eq!((model.calls(), persona.1.get(), thoughts(&mind).len()), (0, 0, 1), "no model call, no seed, no thought"); -} - -#[test] -fn new_repo_activity_makes_a_tick_due_and_the_hourly_memory_comes_for_an_active_steward() { - let week = 8 * 86_400; - let mind = TestMind::seeded(); - mind.admit_at(t(100)); - mind.committed(vec![memory("m1", MemorySort::Learned, "An old thing she holds", &[])]); - mind.admit_at(t(week)); - mind.committed(vec![thought_doc(&ticked(week), "An earlier drift")]); - let model = Fake::saying("I turn it over.", &["A new drift", "Another"]); - let persona = persona(); - let fresh = Wrap::over(&mind, &[0.1, 0.2]).with_forge(vec![push(2, week + 1000)]); - let Outcome::Thought { thought, admitted: true } = tick(&organs(&fresh, &persona, &model), &|| released(), Mode::Live, t(week + 2 * STREAM_MIN_GAP_S)).unwrap() else { panic!("new activity at the next gap mark") }; - assert!(thought.stimulus.0.contains("Changes in your repos"), "{}", thought.stimulus.0); - - let model = Fake::saying("I turn it over.", &["Yet another"]); - let quiet = Wrap::over(&mind, &[0.3]).with_forge(vec![push(1, week - 5000)]); - let at = week + 1000 + STREAM_MIN_GAP_S + STREAM_IDLE_GAP_S + 10; - let Outcome::Thought { thought, admitted: true } = tick(&organs(&quiet, &persona, &model), &|| released(), Mode::Live, t(at)).unwrap() else { panic!("the idle gap") }; - assert!(thought.stimulus.0.starts_with("Out of nowhere you recall"), "old activity is not news, so the hourly memory comes: {}", thought.stimulus.0); -} - -#[test] -fn reads_cite_only_what_the_context_showed() { - let mind = TestMind::seeded(); - let memories: Vec = (0..12).map(|at| memory(&format!("m{at:02}"), MemorySort::Learned, &format!("zz{at:02} {}", "x".repeat(290)), &["@libby:chat"])).collect(); - mind.admit_at(t(100)); - mind.committed(memories); - mind.admit_at(t(200)); - mind.committed(vec![run("w1", RunTurn::SelfRun, RunOperator::Mind, &[])]); - mind.committed( - (1..=3) - .map(|at| PipelineDocument::Reply(PipelineReply { instance: Slug(INSTANCE.into()), label: format!("w1-r{at}").as_str().into(), run: run_ref("w1"), answers: vec![], text: format!("rp{at} {}", "y".repeat(400)).as_str().into() })) - .collect(), - ); - let read = PersonaRead { state: state(), receipt_id: "r".into() }; - let seed = seed_of(&Wrap::over(&mind, &[]), &read, Some(t(300)), t(400)).unwrap(); - assert!(!seed.news); - let cited: Vec = seed.reads.iter().filter(|read| read.kind == PipelineKind::Memory).map(|read| read.id.0.rsplit(':').next().unwrap().to_owned()).collect(); - assert!(!cited.is_empty() && cited.len() < 12, "{cited:?}"); - for label in &cited { - assert!(seed.context.contains(&format!("zz{}", &label[1..])), "{label} is cited but not shown"); - } - let shown = (0..12).filter(|at| seed.context.contains(&format!("zz{at:02}"))).count(); - assert_eq!(shown, cited.len(), "every memory the context shows is cited"); - let replies_cited = seed.reads.iter().filter(|read| read.kind == PipelineKind::Reply).count(); - let replies_shown = (1..=3).filter(|at| seed.context.contains(&format!("rp{at} "))).count(); - assert!(replies_shown >= 1 && replies_shown < 3, "the replies section is bounded: {replies_shown}"); - assert_eq!(replies_cited, replies_shown, "replies cited are the replies shown"); -} - -#[test] -fn every_thought_records_the_fixed_parts_of_its_prompts() { - let mind = mind_with_thought(1000); - let model = Fake::saying("I turn it over.", &["A new drift"]); - let persona = persona(); - let wrap = Wrap::over(&mind, &[0.1]); - let Outcome::Thought { thought, .. } = tick(&organs(&wrap, &persona, &model), &|| released(), Mode::Live, t(1000 + STREAM_IDLE_GAP_S)).unwrap() else { panic!("idle gap") }; - let version = &thought.prompt_version.0; - assert_eq!(version, &prompt_version()); - let guidance = format!("{STREAM_GUIDANCE:?}"); - for part in [guidance.as_str(), GHOSTLIGHT_REV, "words=180", "'s notebook", "", ""] { - assert!(version.contains(part), "{part} missing from {version}"); - } - assert!(version.len() <= LINE_MAX_BYTES, "{}", version.len()); - assert!(include_str!("../../Cargo.toml").contains(GHOSTLIGHT_REV), "the recorded revision is the manifest's pin"); - let rebuilt = build_projector_prompt(&ProjectorPrompt { identity: &thought.identity.0, typed_context: &thought.context.0, visible_stimulus: &thought.stimulus.0, domain_guidance: STREAM_GUIDANCE, word_budget: PROJECTOR_WORDS }); - assert_eq!(model.projected.borrow()[0], rebuilt, "the trace rebuilds the Projector's exact prompt"); -} - - -/// A thought ticked at 900 s and admitted at 1000 s: what landed at 950 s while the models ran is news to the next one. -#[test] -fn what_lands_while_she_thinks_is_news_to_the_next_thought() { - let mind = TestMind::seeded(); - mind.admit_at(t(950)); - mind.committed(vec![memory("m9", MemorySort::Decided, "Landed while the Projector ran", &[])]); - mind.admit_at(t(1000)); - mind.committed(vec![thought_doc(&ticked(900), "A drift that took a hundred seconds")]); - let model = Fake::saying("I turn it over.", &["A new drift"]); - let persona = persona(); - let wrap = Wrap::over(&mind, &[0.1]).with_forge(vec![push(2, 1000), push(1, 950), push(0, 899)]); - let Outcome::Thought { thought, admitted: true } = tick(&organs(&wrap, &persona, &model), &|| released(), Mode::Live, t(1901)).unwrap() else { panic!("news past the gap") }; - assert!(thought.stimulus.0.contains("Landed while the Projector ran"), "{}", thought.stimulus.0); - assert!(thought.stimulus.0.contains("2 commits pushed by libby"), "the pushes at 950 s and 1000 s, not the one at 899 s: {}", thought.stimulus.0); -} - -#[test] -fn a_capped_repo_is_judged_by_its_own_events_and_disclosed_when_they_do_not_decide_it() { - let mind = mind_with_thought(1000); - let read = PersonaRead { state: state(), receipt_id: "r".into() }; - let cultlib = |id, secs| RepoEvent { repo: "GameCult/CultLib".into(), ..push(id, secs) }; - let mut two = Wrap::over(&mind, &[]).with_forge(vec![push(2, 1500), cultlib(1, 500)]); - two.forge.as_mut().unwrap().capped = vec!["GameCult/Huginn".into(), "GameCult/CultLib".into()]; - let seed = seed_of(&two, &read, Some(t(1000)), t(1901)).unwrap(); - assert!(seed.stimulus.to_lowercase().contains("page limit") && seed.stimulus.contains("GameCult/Huginn"), "the repo no event decides stays disclosed: {}", seed.stimulus); - assert!(!seed.stimulus.contains("GameCult/CultLib"), "an event before her thought decides CultLib's cap: {}", seed.stimulus); - let mut flooded = Wrap::over(&mind, &[]).with_forge(vec![]); - flooded.forge.as_mut().unwrap().capped = vec!["GameCult/Huginn".into()]; - let seed = seed_of(&flooded, &read, Some(t(1000)), t(1901)).unwrap(); - assert!(seed.news && seed.stimulus.to_lowercase().contains("page limit"), "a cap with no event at all is disclosed: {}", seed.stimulus); -} - -#[test] -fn a_quiet_hour_reads_the_forge_at_each_gap_mark_only() { - let mind = mind_with_thought(1000); - let model = Fake::saying("I turn it over.", &["A new drift"]); - let persona = persona(); - let wrap = Wrap::over(&mind, &[0.1]).with_forge(vec![push(1, 500)]); - let organs = organs(&wrap, &persona, &model); - let mut secs = 1000 + STREAM_POLL_S; - while secs < 1000 + STREAM_IDLE_GAP_S { - assert_eq!(tick(&organs, &|| released(), Mode::Live, t(secs)).unwrap(), Outcome::NotDue, "at {secs}"); - secs += STREAM_POLL_S; - } - assert_eq!(wrap.forge_reads.get(), 3, "one read for each mark crossed at 900, 1800 and 2700 s"); - assert_eq!(model.calls(), 0); -} - -#[test] -fn identical_memory_lines_are_each_cited_once() { - let mind = TestMind::seeded(); - mind.admit_at(t(100)); - mind.committed((0..12).map(|at| memory(&format!("m{at:02}"), MemorySort::Learned, &format!("the same thing {}", "x".repeat(290)), &["@libby:chat"])).collect()); - let read = PersonaRead { state: state(), receipt_id: "r".into() }; - let seed = seed_of(&Wrap::over(&mind, &[]), &read, Some(t(300)), t(400)).unwrap(); - let shown = seed.context.lines().filter(|line| line.contains("the same thing")).count(); - let cited = seed.reads.iter().filter(|read| read.kind == PipelineKind::Memory).count(); - assert!(shown >= 1 && cited == shown, "{shown} lines shown, {cited} cited"); -} - -/// The environment's proxy settings are read when an agent is built, so the check runs in a child process that has them. -#[test] -fn the_stream_agent_ignores_the_proxy_environment() { - if std::env::var("STREAM_PROXY_CHILD").is_ok() { - let plain: ureq::Agent = ureq::Agent::config_builder().build().into(); - assert!(plain.config().proxy().is_some(), "the child's environment names a proxy"); - let stream = OllamaStream::open("http://127.0.0.1:9", "p", "b").unwrap(); - assert!(stream.agent.config().proxy().is_none(), "the stream's agent took the environment's proxy"); - assert_eq!(stream.agent.config().max_redirects(), 0); - return; - } - let child = std::process::Command::new(std::env::current_exe().unwrap()) - .args(["--exact", "stream::tests::the_stream_agent_ignores_the_proxy_environment", "--nocapture"]) - .env("STREAM_PROXY_CHILD", "1") - .env("ALL_PROXY", "http://192.0.2.1:3128") - .env("HTTP_PROXY", "http://192.0.2.1:3128") - .output() - .unwrap(); - assert!(child.status.success(), "{}{}", String::from_utf8_lossy(&child.stdout), String::from_utf8_lossy(&child.stderr)); -} +use std::cell::RefCell; +use std::collections::VecDeque; +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::sync::{Arc, Mutex}; + +use anyhow::{Result, anyhow}; +use chrono::{DateTime, Utc}; +use cultnet_rs::Selection; +use eureka_pipeline::{MemorySort, PipelineDocument, PipelineKind, PipelineReply, PipelineThought, RunOperator, RunTurn, Slug}; +use huginn_mind::{PipelineAdmissionOutcome, PipelineSelectionPage, RepoActivity, RepoChange, RepoEvent}; +use serde_json::{Value, json}; + +use super::*; +use crate::control::{Grant, HeldReason}; +use crate::testkit::{INSTANCE, TestMind, at, memory, reference, run, run_ref}; + +fn t(secs: i64) -> DateTime { + at(secs * 1000) +} + +/// The label of a thought ticked at test-clock `secs`: the Unix seconds of that instant. +fn ticked(secs: i64) -> String { + format!("t{}", t(secs).timestamp()) +} + +fn released() -> Effective { + Effective::Released(Grant::for_test(Slug(INSTANCE.into()))) +} + +fn state() -> Value { + json!({ + "publicName": "Yggdrasil", + "publicDescription": "A Mind of the swarm.", + "presentation": { "pronouns": "she/her", "voiceSummary": "Plain." }, + "values": [{ "id": "v1", "label": "Coherence", "priority": 0.9, "summary": "Coherence over velocity." }], + "agencyPressure": { "pressures": [{ "summary": "The shaders are late." }] }, + "affect": { "socialBonds": [{ + "object": { "kind": "person", "id": "@libby:chat", "label": "Libby" }, + "relationshipKind": "collaborator", "summary": "Libby holds the sounds." + }] }, + }) +} + +struct Persona(PersonaRead); + +impl PersonaSource for Persona { + fn persona(&self) -> Result { + Ok(PersonaRead { state: self.0.state.clone(), receipt_id: self.0.receipt_id.clone() }) + } +} + +fn persona() -> Persona { + Persona(PersonaRead { state: state(), receipt_id: "mind-commit-1".into() }) +} + +struct Fake { + projection: String, + candidates: Vec, + fault: bool, + projected: RefCell>, + continued: RefCell>, +} + +impl Fake { + fn saying(projection: &str, candidates: &[&str]) -> Self { + Self { projection: projection.into(), candidates: candidates.iter().map(|text| (*text).into()).collect(), fault: false, projected: RefCell::default(), continued: RefCell::default() } + } + + fn calls(&self) -> usize { + self.projected.borrow().len() + self.continued.borrow().len() + } +} + +impl StreamModel for Fake { + fn project(&self, prompt: &str) -> Result { + self.projected.borrow_mut().push(prompt.into()); + if self.fault { Err(anyhow!("the model host is down")) } else { Ok(self.projection.clone()) } + } + + fn continue_raw(&self, prompt: &str, sampling: &Sampling, n: usize) -> Result> { + self.continued.borrow_mut().push((prompt.into(), *sampling, n)); + Ok(self.candidates.iter().take(n).cloned().collect()) + } +} + +/// A real mind that records what is asked and admitted, and scores semantic asks as told. +struct Wrap<'a> { + inner: &'a TestMind, + scores: RefCell>, + selections: RefCell>, + admitted: RefCell>>, + forge: Option, + forge_reads: std::cell::Cell, +} + +impl<'a> Wrap<'a> { + fn over(inner: &'a TestMind, scores: &[f32]) -> Self { + Self { inner, scores: RefCell::new(scores.iter().copied().collect()), selections: RefCell::default(), admitted: RefCell::default(), forge: None, forge_reads: std::cell::Cell::new(0) } + } + + /// The same mind with a forge that answers `events`, whatever the asker's cursor. + fn with_forge(mut self, events: Vec) -> Self { + self.forge = Some(RepoActivity { head: 1, events, capped: vec![], open_prs: vec![], unseen: vec![], moved: vec![] }); + self + } +} + +impl MindPort for Wrap<'_> { + fn instance(&self) -> &Slug { + self.inner.instance() + } + + fn query(&self, selection: &Selection) -> Result { + self.selections.borrow_mut().push(selection.clone()); + self.inner.query(selection) + } + + fn query_near(&self, selection: &Selection, _text: &str, _top_k: u32) -> Result { + self.selections.borrow_mut().push(selection.clone()); + let mut page = self.inner.query(selection)?; + page.scores = Some(vec![self.scores.borrow_mut().pop_front().ok_or_else(|| anyhow!("no score was scripted"))?]); + Ok(page) + } + + fn roots_touched(&self) -> Result> { + self.inner.roots_touched() + } + + fn stewarded_activity(&self, repos: &[huginn_mind::eureka_pipeline::OrgRepo], after: Option) -> Result { + self.forge_reads.set(self.forge_reads.get() + 1); + match &self.forge { + Some(activity) => Ok(activity.clone()), + None => self.inner.stewarded_activity(repos, after), + } + } + + fn admit(&self, agent: &str, session: &str, documents: Vec) -> Result { + self.admitted.borrow_mut().push(documents.clone()); + self.inner.admit(agent, session, documents) + } +} + +/// The seed as the tick builds it: the cheap reads first, then the rest. +fn seed_of(mind: &dyn MindPort, read: &PersonaRead, since: Option>, now: DateTime) -> Result { + let responsibilities = render_responsibilities(mind, mind.instance(), &BTreeMap::new())?; + let mut fresh = mind_news(mind, mind.instance(), since)?; + fresh.extend(repo_news(mind, since, &responsibilities, now)); + build_seed(mind, read, since, now, &responsibilities, fresh) +} + +/// A push by someone else to her repo at `secs`. +fn push(id: u64, secs: i64) -> RepoEvent { + RepoEvent { id, at: u64::try_from(t(secs).timestamp_millis()).unwrap(), repo: "GameCult/Huginn".into(), actor: "libby".into(), change: RepoChange::Pushed { commits: 1 } } +} + +/// Counts how often her persona is read, which only a tick that may run does. +struct CountedPersona(Persona, std::cell::Cell); + +impl PersonaSource for CountedPersona { + fn persona(&self) -> Result { + self.1.set(self.1.get() + 1); + self.0.persona() + } +} + +fn thought_doc(label: &str, text: &str) -> PipelineDocument { + PipelineDocument::Thought(PipelineThought { + instance: Slug(INSTANCE.into()), + label: label.into(), + text: text.into(), + reads: vec![], + persona_receipt: "mind-commit-0".into(), + identity: "I".into(), + context: "".into(), + stimulus: "".into(), + projector_model: "projector".into(), + projection: "A projection.".into(), + base_model: "base".into(), + temperature_permille: 1000, + min_p_permille: 50, + max_tokens: 160, + candidates: vec![text.into()], + nearest_permille: vec![1000], + chosen: 0, + projector_ms: 0, + continuer_ms: 0, + prompt_version: "test".into(), + }) +} + +/// A mind with an old memory about Libby and one thought at `thought_at`. +fn mind_with_thought(thought_at: i64) -> TestMind { + let mind = TestMind::seeded(); + mind.admit_at(t(10)); + mind.committed(vec![memory("m1", MemorySort::Learned, "Libby wants the sounds before the shaders", &["@libby:chat"])]); + mind.admit_at(t(thought_at)); + mind.committed(vec![thought_doc(&ticked(thought_at), "An earlier drift")]); + mind +} + +fn organs<'a>(mind: &'a dyn MindPort, persona: &'a Persona, model: &'a Fake) -> Organs<'a> { + Organs { mind, persona, model, projector_model: "projector:test", base_model: "base:test" } +} + +fn thoughts(mind: &TestMind) -> Vec { + mind.ids(PipelineKind::Thought) +} + +#[test] +fn budgets_leave_room_for_every_section() { + let sum: usize = BUDGETS.iter().map(|(_, budget)| budget).sum(); + assert!(sum + BUDGETS.len() <= SECTION_MAX_BYTES, "{sum}"); +} + +#[test] +fn seed_reads_no_thought() { + let without = TestMind::seeded(); + without.committed(vec![memory("m1", MemorySort::Learned, "Libby wants the sounds first", &["@libby:chat"])]); + let with = TestMind::seeded(); + with.committed(vec![memory("m1", MemorySort::Learned, "Libby wants the sounds first", &["@libby:chat"])]); + with.committed(vec![thought_doc("t1", "A drift that must stay out of the seed")]); + let (plain, thoughtful) = (Wrap::over(&without, &[]), Wrap::over(&with, &[])); + let read = PersonaRead { state: state(), receipt_id: "r".into() }; + let one = seed_of(&plain, &read, None, t(100_000)).unwrap(); + let two = seed_of(&thoughtful, &read, None, t(100_000)).unwrap(); + assert_eq!(one.projector_prompt(), two.projector_prompt(), "a mind holding thoughts yields the prompt of the same mind without them"); + assert_eq!(one.reads, two.reads); + assert!(!two.projector_prompt().contains("must stay out")); + let thought_schema = PipelineKind::Thought.type_id(); + for selection in thoughtful.selections.borrow().iter() { + assert!(!selection.schemas.iter().flatten().any(|schema| schema == thought_schema), "a seed selection named thoughts: {selection:?}"); + } +} + +#[test] +fn seed_carries_relationships() { + let mind = TestMind::seeded(); + mind.committed(vec![ + memory("m1", MemorySort::Learned, "Libby wants the sounds before the shaders", &["@libby:chat"]), + memory("m2", MemorySort::Rapport, "Noor distrusts the new scheduler", &["@noor:chat"]), + run("w1", RunTurn::SelfRun, RunOperator::Mind, &[]), + ]); + mind.committed(vec![PipelineDocument::Reply(PipelineReply { + instance: Slug(INSTANCE.into()), + label: "w1-r1".into(), + run: run_ref("w1"), + answers: vec![], + text: "The shader pass is done; the sounds are Libby's.".into(), + })]); + let seed = seed_of(&Wrap::over(&mind, &[]), &PersonaRead { state: state(), receipt_id: "r".into() }, None, t(100_000)).unwrap(); + for wanted in ["Libby holds the sounds", "collaborator", "Libby wants the sounds before the shaders", "Noor distrusts the new scheduler", "The shader pass is done", "Coherence over velocity", "The shaders are late"] { + assert!(seed.context.contains(wanted), "{wanted:?} missing from {}", seed.context); + } + assert!(seed.identity.contains("Yggdrasil") && seed.identity.contains("she/her") && seed.identity.contains("Plain.")); + assert!(seed.reads.iter().any(|read| read.kind == PipelineKind::Reply)); + assert!(seed.reads.iter().any(|read| read.kind == PipelineKind::Memory)); +} + +#[test] +fn seed_sections_keep_budgets() { + let huge = "x".repeat(3000); + let mut crowded = state(); + crowded["values"] = json!((0..40).map(|at| json!({ "id": format!("v{at}"), "label": format!("Value {at}"), "priority": 0.5, "summary": huge })).collect::>()); + crowded["agencyPressure"]["pressures"] = json!((0..40).map(|_| json!({ "summary": huge })).collect::>()); + let mind = TestMind::seeded(); + mind.committed(vec![memory("m1", MemorySort::Learned, "Libby wants the sounds first", &["@libby:chat"])]); + let seed = seed_of(&Wrap::over(&mind, &[]), &PersonaRead { state: crowded, receipt_id: "r".into() }, None, t(100_000)).unwrap(); + assert!(seed.context.len() <= SECTION_MAX_BYTES, "{}", seed.context.len()); + assert!(seed.context.contains("Libby holds the sounds"), "oversized values and pressures crowded out the bonds"); + assert!(seed.context.contains("Libby wants the sounds first"), "oversized values and pressures crowded out the recall"); + let values = seed.context.split("What presses on her:").next().unwrap(); + assert!(values.len() <= BUDGETS[0].1 + 1, "values took {}", values.len()); +} + +#[test] +fn stimulus_is_what_is_new() { + let mind = TestMind::seeded(); + mind.admit_at(t(100)); + mind.committed(vec![memory("m1", MemorySort::Learned, "Before the thought, the kettle", &[])]); + mind.admit_at(t(200)); + mind.committed(vec![thought_doc(&ticked(200), "A drift")]); + mind.admit_at(t(300)); + mind.committed(vec![memory("m2", MemorySort::Decided, "After the thought, the window", &[]), run("w9", RunTurn::SelfRun, RunOperator::Mind, &[])]); + mind.admit_at(t(350)); + mind.committed(vec![crate::testkit::close(run_ref("w9"), crate::testkit::recorded())]); + let wrap = Wrap::over(&mind, &[]); + let read = PersonaRead { state: state(), receipt_id: "r".into() }; + let seed = seed_of(&wrap, &read, Some(t(200)), t(400)).unwrap(); + assert!(seed.news); + assert!(seed.stimulus.contains("After the thought, the window"), "{}", seed.stimulus); + assert!(seed.stimulus.contains("w9 closed"), "a run closure is news: {}", seed.stimulus); + assert!(!seed.stimulus.contains("Before the thought, the kettle"), "{}", seed.stimulus); + assert!(seed.stimulus.find("closed").unwrap() < seed.stimulus.find("After the thought").unwrap(), "newest first"); + assert!(seed.reads.contains(&reference(PipelineKind::Memory, &format!("{INSTANCE}:memory:m2")))); +} + +#[test] +fn stimulus_without_news_is_an_unbidden_memory_chosen_by_the_tick_label_else_the_nothing_line() { + let mind = TestMind::seeded(); + mind.admit_at(t(100)); + mind.committed(vec![memory("m1", MemorySort::Learned, "The first old thing", &[]), memory("m2", MemorySort::Learned, "The second old thing", &[])]); + let long_after = 100 * 86_400; + mind.admit_at(t(long_after - 100)); + mind.committed(vec![memory("m3", MemorySort::Learned, "A recent thing", &[])]); + let wrap = Wrap::over(&mind, &[]); + let read = PersonaRead { state: state(), receipt_id: "r".into() }; + let picks: Vec = (0..4) + .map(|offset| { + let seed = seed_of(&wrap, &read, Some(t(long_after - 10)), t(long_after + offset)).unwrap(); + assert!(!seed.news); + seed.stimulus + }) + .collect(); + assert!(picks.iter().all(|stimulus| stimulus.starts_with("Out of nowhere you recall")), "{picks:?}"); + assert!(picks.iter().all(|stimulus| !stimulus.contains("recent thing")), "a memory under 7 days old is not unbidden"); + assert!(picks.iter().any(|stimulus| stimulus.contains("first old")) && picks.iter().any(|stimulus| stimulus.contains("second old")), "the label seeds a uniform pick: {picks:?}"); + assert_eq!(picks[0], picks[2], "the same label seed picks the same memory"); + + let bare = TestMind::seeded(); + let seed = seed_of(&Wrap::over(&bare, &[]), &read, Some(t(1000)), t(5000)).unwrap(); + assert!(!seed.news); + assert!(seed.stimulus.starts_with("Nothing has happened since"), "{}", seed.stimulus); +} + +#[test] +fn cadence_from_newest_thought() { + let mind = mind_with_thought(1000); + let model = Fake::saying("I turn it over.", &["A new drift"]); + let persona = persona(); + let wrap = Wrap::over(&mind, &[0.1]); + let organs = organs(&wrap, &persona, &model); + let tick_at = |secs: i64| tick(&organs, &|| released(), Mode::Live, t(secs)).unwrap(); + assert_eq!(tick_at(1000 + STREAM_MIN_GAP_S - 1), Outcome::NotDue, "no tick before the minimum gap"); + assert_eq!(tick_at(1000 + STREAM_MIN_GAP_S), Outcome::NotDue, "past the minimum gap but nothing is new"); + assert_eq!(model.calls(), 0); + assert_eq!(tick_at(1000 + STREAM_IDLE_GAP_S - 1), Outcome::NotDue); + assert_eq!(thoughts(&mind).len(), 1); + // Something new, past the minimum gap but short of the idle gap. + mind.admit_at(t(1000 + STREAM_MIN_GAP_S)); + mind.committed(vec![memory("m9", MemorySort::Decided, "Something happened", &[])]); + let Outcome::Thought { admitted: true, .. } = tick_at(1000 + STREAM_MIN_GAP_S + 1) else { panic!("news past the minimum gap makes a thought") }; + assert_eq!(thoughts(&mind).len(), 2); +} + +#[test] +fn the_idle_gap_alone_makes_a_tick_due() { + let mind = mind_with_thought(1000); + let model = Fake::saying("I turn it over.", &["A new drift"]); + let persona = persona(); + let wrap = Wrap::over(&mind, &[0.1]); + let Outcome::Thought { admitted: true, thought } = tick(&organs(&wrap, &persona, &model), &|| released(), Mode::Live, t(1000 + STREAM_IDLE_GAP_S)).unwrap() else { panic!("idle gap") }; + assert_eq!(thought.label.0, format!("t{}", t(1000 + STREAM_IDLE_GAP_S).timestamp())); + assert!(due(None, t(0), false) && !due(Some(t(0)), t(STREAM_MIN_GAP_S), false) && due(Some(t(0)), t(STREAM_MIN_GAP_S), true)); +} + +#[test] +fn held_brake_ticks_nothing() { + let mind = mind_with_thought(1000); + let model = Fake::saying("I turn it over.", &["A new drift"]); + let persona = persona(); + let wrap = Wrap::over(&mind, &[0.1]); + for reason in [HeldReason::BrakeHeld, HeldReason::BrakeAbsent, HeldReason::Undecodable] { + let held = tick(&organs(&wrap, &persona, &model), &move || Effective::Held { reason }, Mode::Live, t(100_000)).unwrap(); + assert_eq!(held, Outcome::Held(reason)); + let dry = tick(&organs(&wrap, &persona, &model), &move || Effective::Held { reason }, Mode::DryRun, t(100_000)).unwrap(); + assert_eq!(dry, Outcome::Held(reason), "a dry run is held too"); + } + assert_eq!(model.calls(), 0, "no model call"); + assert!(wrap.admitted.borrow().is_empty() && wrap.selections.borrow().is_empty(), "no admission, and the mind was not read"); +} + +#[test] +fn least_near_candidate_chosen() { + let mind = mind_with_thought(1000); + let model = Fake::saying("I turn it over.", &["one", "two", "three", "four"]); + let persona = persona(); + let wrap = Wrap::over(&mind, &[0.9, 0.2, 0.2, 0.5]); + let Outcome::Thought { thought, admitted: true } = tick(&organs(&wrap, &persona, &model), &|| released(), Mode::Live, t(1000 + STREAM_IDLE_GAP_S)).unwrap() else { panic!("a thought") }; + assert_eq!(thought.nearest_permille, vec![900, 200, 200, 500]); + assert_eq!(thought.chosen, 1, "the least near, the first on a tie"); + assert_eq!(thought.text.0, "two"); + assert_eq!(thought.candidates.iter().map(|line| line.0.as_str()).collect::>(), ["one", "two", "three", "four"], "every candidate is kept in the trace"); + assert_eq!(least_near(&[500, 100, 100]), 1); + assert_eq!(least_near(&[]), 0); +} + +#[test] +fn no_earlier_thought_is_nearness_1000() { + let mind = TestMind::seeded(); + let model = Fake::saying("I turn it over.", &["one", "two"]); + let persona = persona(); + let wrap = Wrap::over(&mind, &[]); + let Outcome::Thought { thought, .. } = tick(&organs(&wrap, &persona, &model), &|| released(), Mode::Live, t(5000)).unwrap() else { panic!("a thought") }; + assert_eq!(thought.nearest_permille, vec![1000, 1000]); + assert_eq!(thought.chosen, 0); + assert_eq!((thought.projector_model.0.as_str(), thought.base_model.0.as_str(), thought.persona_receipt.0.as_str()), ("projector:test", "base:test", "mind-commit-1")); + assert_eq!((thought.temperature_permille, thought.min_p_permille, thought.max_tokens), (1000, 50, 160)); +} + +#[test] +fn frame_holds_projection_verbatim() { + assert_eq!(continuer_frame("Yggdrasil", "9 October 2026", "I turn it over."), "Yggdrasil's notebook\n\n9 October 2026\n\nI turn it over.\n\n"); + let mind = mind_with_thought(1000); + let model = Fake::saying("I keep turning over the sounds.", &["a", "b", "c", "d", "e"]); + let persona = persona(); + let wrap = Wrap::over(&mind, &[0.3, 0.3, 0.3, 0.3]); + tick(&organs(&wrap, &persona, &model), &|| released(), Mode::Live, t(1000 + STREAM_IDLE_GAP_S)).unwrap(); + let continued = model.continued.borrow(); + let (prompt, sampling, n) = &continued[0]; + assert_eq!(prompt, &continuer_frame("Yggdrasil", &t(1000 + STREAM_IDLE_GAP_S).format(FRAME_DATE_FORMAT).to_string(), "I keep turning over the sounds.")); + assert_eq!((*n, *sampling), (4, Sampling::STREAM)); + for foreign in ["Libby", "Coherence", "Plain.", "she/her", "private Projector"] { + assert!(!prompt.contains(foreign), "{foreign} reached the continuer"); + } + let projected = model.projected.borrow(); + assert!(projected[0].contains("private Projector") && projected[0].contains(STREAM_GUIDANCE), "the Projector prompt is Ghostlight's builder over her seed"); + assert!(projected[0].contains("Libby holds the sounds")); +} + +#[test] +fn blank_and_faulty_ticks_admit_nothing() { + let mind = TestMind::seeded(); + let persona = persona(); + let blank = Fake::saying("A projection.", &["", " \n", "\t"]); + let wrap = Wrap::over(&mind, &[]); + assert_eq!(tick(&organs(&wrap, &persona, &blank), &|| released(), Mode::Live, t(5000)).unwrap(), Outcome::Blank); + let silent_projector = Fake::saying(" ", &["x"]); + assert_eq!(tick(&organs(&wrap, &persona, &silent_projector), &|| released(), Mode::Live, t(5000)).unwrap(), Outcome::Blank); + let mut down = Fake::saying("x", &["x"]); + down.fault = true; + assert!(tick(&organs(&wrap, &persona, &down), &|| released(), Mode::Live, t(5000)).is_err()); + assert!(wrap.admitted.borrow().is_empty() && thoughts(&mind).is_empty()); +} + +#[test] +fn dry_run_admits_nothing() { + let mind = mind_with_thought(1000); + let model = Fake::saying("I turn it over.", &["A new drift", "Another"]); + let persona = persona(); + let wrap = Wrap::over(&mind, &[0.4, 0.6]); + let Outcome::Thought { thought, admitted } = tick(&organs(&wrap, &persona, &model), &|| released(), Mode::DryRun, t(1001)).unwrap() else { panic!("a dry run prints a thought") }; + assert!(!admitted); + assert_eq!(thought.text.0, "A new drift"); + assert!(wrap.admitted.borrow().is_empty()); + assert_eq!(thoughts(&mind).len(), 1); + assert!(serde_json::to_string(&*thought).unwrap().contains("\"candidates\"")); +} + +#[test] +fn stream_admits_only_thoughts() { + let mind = TestMind::seeded(); + mind.committed(vec![memory("m1", MemorySort::Learned, "Libby wants the sounds first", &["@libby:chat"])]); + let model = Fake::saying("I turn it over.", &["A drift", "Another"]); + let persona = persona(); + let wrap = Wrap::over(&mind, &[]); + let Outcome::Thought { thought, admitted: true } = tick(&organs(&wrap, &persona, &model), &|| released(), Mode::Live, t(100_000)).unwrap() else { panic!("a thought") }; + let batches = wrap.admitted.borrow(); + assert_eq!(batches.len(), 1); + assert!(matches!(batches[0].as_slice(), [PipelineDocument::Thought(_)]), "exactly one thought and nothing else"); + let stored = mind.view(PipelineKind::Thought, &format!("{INSTANCE}:thought:t{}", t(100_000).timestamp())).expect("admitted under its instance-rooted key"); + assert_eq!(stored.document, PipelineDocument::Thought(*thought.clone())); + assert!(thought.reads.iter().any(|read| read.kind == PipelineKind::Memory), "the thought cites what the seed read"); + assert_eq!(mind.ids(PipelineKind::Memory).len(), 1, "no memory was written"); +} + +// --------------------------------------------------------------------------------------------- +// The model port against a fake HTTP server. + +type Seen = Arc>>; + +fn serve(answers: Vec) -> (String, Seen) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let seen: Seen = Arc::default(); + let record = Arc::clone(&seen); + std::thread::spawn(move || { + for answer in answers { + let (mut socket, _) = listener.accept().unwrap(); + let mut raw = Vec::new(); + let mut buffer = [0u8; 4096]; + let (head, length) = loop { + let read = socket.read(&mut buffer).unwrap(); + raw.extend_from_slice(&buffer[..read]); + if let Some(end) = raw.windows(4).position(|window| window == b"\r\n\r\n") { + let head = String::from_utf8_lossy(&raw[..end]).to_string(); + let length = head.lines().find_map(|line| line.to_ascii_lowercase().strip_prefix("content-length:").map(|value| value.trim().parse::().unwrap())).unwrap_or(0); + if raw.len() >= end + 4 + length { + break (head, (end + 4, length)); + } + } + assert!(read > 0, "the client closed early"); + }; + let body: Value = serde_json::from_slice(&raw[length.0..length.0 + length.1]).unwrap(); + let path = head.lines().next().unwrap().split_whitespace().nth(1).unwrap().to_owned(); + record.lock().unwrap().push((path, body)); + write!(socket, "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{answer}", answer.len()).unwrap(); + } + }); + (endpoint, seen) +} + +#[test] +fn raw_generate_is_raw() { + let (endpoint, seen) = serve(vec![json!({ "response": "one" }).to_string(), json!({ "response": "two" }).to_string()]); + let model = OllamaStream::open(&endpoint, "projector:x", "base:y").unwrap(); + let out = model.continue_raw("The frame\n\n", &Sampling::STREAM, 2).unwrap(); + assert_eq!(out, ["one", "two"]); + let seen = seen.lock().unwrap(); + assert_eq!(seen.len(), 2, "one call per continuation"); + for (path, body) in seen.iter() { + assert_eq!(path, "/api/generate"); + assert_eq!(body["raw"], json!(true)); + assert_eq!(body["stream"], json!(false)); + assert_eq!(body["model"], json!("base:y")); + assert_eq!(body["prompt"], json!("The frame\n\n")); + assert_eq!(body["options"], json!({ "temperature": 1.0, "min_p": 0.05, "num_predict": 160, "stop": ["\n\n"], "num_thread": 4 })); + } +} + +#[test] +fn the_projector_is_asked_in_chat() { + let (endpoint, seen) = serve(vec![json!({ "message": { "role": "assistant", "content": "I hold it." } }).to_string()]); + let model = OllamaStream::open(&endpoint, "projector:x", "base:y").unwrap(); + assert_eq!(model.project("Be the Projector.").unwrap(), "I hold it."); + let seen = seen.lock().unwrap(); + assert_eq!(seen[0].0, "/api/chat"); + assert_eq!(seen[0].1, json!({ "model": "projector:x", "stream": false, "messages": [{ "role": "user", "content": "Be the Projector." }] })); +} + +#[test] +fn a_model_host_that_answers_badly_is_a_fault() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + std::thread::spawn(move || { + let (mut socket, _) = listener.accept().unwrap(); + let mut buffer = [0u8; 4096]; + let _ = socket.read(&mut buffer); + let _ = socket.write_all(b"HTTP/1.1 500 Internal Server Error\r\ncontent-length: 4\r\nconnection: close\r\n\r\nboom"); + }); + let model = OllamaStream::open(&endpoint, "p", "b").unwrap(); + let fault = model.project("x").unwrap_err().to_string(); + assert!(fault.contains("500"), "{fault}"); +} + +#[test] +fn non_loopback_endpoint_refused() { + for refused in [ + "http://localhost:11434@example.com", + "http://localhost@example.com", + "http://127.0.0.1:11434@example.com:80", + "http://user:secret@127.0.0.1:11434", + "http://127.0.0.1:11434@localhost", + "http://[::1]:11434@example.com", + "http://localhost:11434\\@example.com", + "http://example.com#@127.0.0.1", + "http://example.com?@localhost", + "http://127.0.0.1:11434?x=1", + "http://localhost.example.com:11434", + "http://", + "", + ] { + assert!(OllamaStream::open(refused, "p", "b").is_err(), "{refused} must be refused"); + } + for refused in ["http://10.77.0.1:11434", "http://example.com:11434", "http://0.0.0.0:11434", "https://127.0.0.1:11434", "127.0.0.1:11434", "http://127.0.0.1:11434/api", "http://[2001:db8::1]:11434"] { + assert!(OllamaStream::open(refused, "p", "b").is_err(), "{refused} must be refused"); + } + for opened in ["http://127.0.0.1:11434", "http://localhost:11434", "http://[::1]:11434", "http://127.0.0.2:1/", "http://LOCALHOST:11434/"] { + assert!(OllamaStream::open(opened, "p", "b").is_ok(), "{opened} is loopback"); + } +} + +#[test] +fn cap_cuts_at_a_line_else_at_a_character() { + assert_eq!(cap("short", 10), "short"); + assert_eq!(cap("ab\ncd", 4), "ab", "whole lines only"); + assert_eq!(cap("abcdef", 4), "abcd", "no newline to cut at"); + assert_eq!(cap("\nabc", 3), "\nab", "a newline at the very start is no cut point"); + assert_eq!(cap("aé\nb", 2), "a", "never inside a character"); + assert_eq!(cap("ééé", 5), "éé"); +} + +#[test] +fn bonds_name_the_label_else_the_id_else_someone() { + let line = |bond: Value| bond_line(&bond); + assert_eq!(line(json!({ "object": { "label": "Libby", "id": "@libby:chat" }, "relationshipKind": "friend", "summary": "s" })), "- Libby (friend): s"); + assert_eq!(line(json!({ "object": { "label": "", "id": "@libby:chat" }, "relationshipKind": "friend", "summary": "s" })), "- @libby:chat (friend): s"); + assert_eq!(line(json!({ "object": {}, "relationshipKind": "friend", "summary": "s" })), "- someone (friend): s"); +} + +#[test] +fn only_person_bonds_with_ids_are_people_she_asks_her_memories_about() { + let state = json!({ "affect": { "socialBonds": [ + { "object": { "kind": "person", "id": "@libby:chat" } }, + { "object": { "kind": "repo", "id": "GameCult/Huginn" } }, + { "object": { "kind": "person", "id": "" } }, + { "object": { "kind": "person", "id": "@noor:chat" } }, + ] } }); + assert_eq!(bonded_people(&state), ["@libby:chat", "@noor:chat"]); + assert!(bonded_people(&json!({})).is_empty()); +} + +#[test] +fn news_is_strictly_after_her_newest_thought() { + let mind = TestMind::seeded(); + mind.admit_at(t(500)); + mind.committed(vec![memory("m1", MemorySort::Learned, "Admitted at the very instant of the thought", &[])]); + let read = PersonaRead { state: state(), receipt_id: "r".into() }; + let seed = seed_of(&Wrap::over(&mind, &[]), &read, Some(t(500)), t(600)).unwrap(); + assert!(!seed.news, "{}", seed.stimulus); + let seed = seed_of(&Wrap::over(&mind, &[]), &read, Some(t(499)), t(600)).unwrap(); + assert!(seed.news); +} + +#[test] +fn an_unbidden_memory_is_older_than_seven_days_not_exactly_seven() { + let mind = TestMind::seeded(); + mind.admit_at(t(1000)); + mind.committed(vec![memory("m1", MemorySort::Learned, "The week-old thing", &[])]); + let read = PersonaRead { state: state(), receipt_id: "r".into() }; + let week = 7 * 86_400; + let exactly = seed_of(&Wrap::over(&mind, &[]), &read, Some(t(1000)), t(1000 + week)).unwrap(); + assert!(exactly.stimulus.starts_with("Nothing has happened"), "{}", exactly.stimulus); + let over = seed_of(&Wrap::over(&mind, &[]), &read, Some(t(1000)), t(1001 + week)).unwrap(); + assert!(over.stimulus.contains("The week-old thing"), "{}", over.stimulus); +} + +#[test] +fn a_flood_of_news_fills_the_stimulus_with_whole_lines_newest_first() { + let mind = TestMind::seeded(); + let flood: Vec = (0..60).map(|at| memory(&format!("f{at:02}"), MemorySort::Learned, &format!("{at:02} {}", "w".repeat(150)), &[])).collect(); + mind.admit_at(t(100)); + mind.committed(flood); + let read = PersonaRead { state: state(), receipt_id: "r".into() }; + let seed = seed_of(&Wrap::over(&mind, &[]), &read, Some(t(50)), t(200)).unwrap(); + let lines: Vec<&str> = seed.stimulus.lines().skip(1).collect(); + assert!(seed.stimulus.len() <= SECTION_MAX_BYTES && seed.stimulus.len() > SECTION_MAX_BYTES - 250, "{}", seed.stimulus.len()); + assert!(lines.iter().all(|line| line.starts_with("- You noted") && line.ends_with('"')), "a line was cut: {lines:?}"); + assert!(lines.len() > 10 && lines.len() < 30, "{}", lines.len()); + let cited = seed.reads.iter().filter(|read| read.kind == PipelineKind::Memory).count(); + assert!(cited >= lines.len() && cited <= lines.len() + 12, "the thought cites what its stimulus showed: {cited} reads for {} lines", lines.len()); + let newest = lines[0]; + assert!(newest.contains("\"29 ") || newest.contains("\"59 "), "newest first: {newest}"); +} + +#[test] +fn the_newest_thought_sets_the_cadence_not_the_oldest() { + let mind = TestMind::seeded(); + mind.admit_at(t(100)); + mind.committed(vec![thought_doc(&ticked(100), "An old drift")]); + mind.admit_at(t(10_000)); + mind.committed(vec![thought_doc(&ticked(10_000), "A recent drift")]); + let model = Fake::saying("I turn it over.", &["A new drift"]); + let persona = persona(); + let wrap = Wrap::over(&mind, &[0.1]); + let tick_at = |secs: i64| tick(&organs(&wrap, &persona, &model), &|| released(), Mode::Live, t(secs)).unwrap(); + assert_eq!(tick_at(10_000 + STREAM_MIN_GAP_S - 1), Outcome::NotDue, "a recent thought holds the stream back whatever the oldest says"); + assert_eq!(model.calls(), 0); +} + +#[test] +fn news_at_exactly_the_minimum_gap_is_due() { + let mind = mind_with_thought(1000); + mind.admit_at(t(1500)); + mind.committed(vec![memory("m9", MemorySort::Decided, "Something happened", &[])]); + let model = Fake::saying("I turn it over.", &["A new drift"]); + let persona = persona(); + let wrap = Wrap::over(&mind, &[0.1]); + let Outcome::Thought { admitted: true, .. } = tick(&organs(&wrap, &persona, &model), &|| released(), Mode::Live, t(1000 + STREAM_MIN_GAP_S)).unwrap() else { panic!("due at the minimum gap") }; +} + +#[test] +fn repo_activity_is_news_only_after_her_newest_thought_and_is_stamped_when_it_happened() { + let mind = mind_with_thought(1000); + let read = PersonaRead { state: state(), receipt_id: "r".into() }; + let old = Wrap::over(&mind, &[]).with_forge(vec![push(1, 500)]); + let seed = seed_of(&old, &read, Some(t(1000)), t(1901)).unwrap(); + assert!(!seed.news && !seed.stimulus.contains("Changes in your repos"), "an event older than her newest thought is not news: {}", seed.stimulus); + let edge = Wrap::over(&mind, &[]).with_forge(vec![push(1, 1000)]); + assert!(!seed_of(&edge, &read, Some(t(1000)), t(1901)).unwrap().news, "an event at her newest thought's time is not after it"); + let mixed = Wrap::over(&mind, &[]).with_forge(vec![push(2, 1500), push(1, 500)]); + let seed = seed_of(&mixed, &read, Some(t(1000)), t(1901)).unwrap(); + assert!(seed.news && seed.stimulus.contains("Changes in your repos: GameCult/Huginn: 1 commit pushed by libby"), "{}", seed.stimulus); + let stamp = t(1500).to_rfc3339_opts(chrono::SecondsFormat::Secs, true); + assert!(seed.stimulus.contains(&format!("at={stamp} ")), "the notice carries the event's time, not the tick's: {}", seed.stimulus); + let first = Wrap::over(&mind, &[]).with_forge(vec![push(1, 500)]); + assert!(seed_of(&first, &read, None, t(1901)).unwrap().news, "with no thought yet, what the forge holds is news"); +} + +#[test] +fn a_capped_repo_is_news_only_beside_a_forge_event_after_her_newest_thoughts_label() { + let mind = mind_with_thought(1000); + let read = PersonaRead { state: state(), receipt_id: "r".into() }; + let mut quiet = Wrap::over(&mind, &[]).with_forge(vec![push(1, 500)]); + quiet.forge.as_mut().unwrap().capped.push("GameCult/Huginn".into()); + assert!(!seed_of(&quiet, &read, Some(t(1000)), t(1901)).unwrap().news); + let mut busy = Wrap::over(&mind, &[]).with_forge(vec![push(2, 1500)]); + busy.forge.as_mut().unwrap().capped.push("GameCult/Huginn".into()); + let seed = seed_of(&busy, &read, Some(t(1000)), t(1901)).unwrap(); + assert!(seed.news && seed.stimulus.to_lowercase().contains("page limit"), "{}", seed.stimulus); +} + +#[test] +fn old_repo_activity_does_not_make_a_tick_due() { + let mind = mind_with_thought(1000); + let model = Fake::saying("I turn it over.", &["A new drift"]); + let persona = CountedPersona(persona(), std::cell::Cell::new(0)); + let wrap = Wrap::over(&mind, &[0.1]).with_forge(vec![push(1, 500)]); + let organs = Organs { mind: &wrap, persona: &persona, model: &model, projector_model: "projector:test", base_model: "base:test" }; + assert_eq!(tick(&organs, &|| released(), Mode::Live, t(1901)).unwrap(), Outcome::NotDue); + assert_eq!((model.calls(), persona.1.get(), thoughts(&mind).len()), (0, 0, 1), "no model call, no seed, no thought"); +} + +#[test] +fn new_repo_activity_makes_a_tick_due_and_the_hourly_memory_comes_for_an_active_steward() { + let week = 8 * 86_400; + let mind = TestMind::seeded(); + mind.admit_at(t(100)); + mind.committed(vec![memory("m1", MemorySort::Learned, "An old thing she holds", &[])]); + mind.admit_at(t(week)); + mind.committed(vec![thought_doc(&ticked(week), "An earlier drift")]); + let model = Fake::saying("I turn it over.", &["A new drift", "Another"]); + let persona = persona(); + let fresh = Wrap::over(&mind, &[0.1, 0.2]).with_forge(vec![push(2, week + 1000)]); + let Outcome::Thought { thought, admitted: true } = tick(&organs(&fresh, &persona, &model), &|| released(), Mode::Live, t(week + 2 * STREAM_MIN_GAP_S)).unwrap() else { panic!("new activity at the next gap mark") }; + assert!(thought.stimulus.0.contains("Changes in your repos"), "{}", thought.stimulus.0); + + let model = Fake::saying("I turn it over.", &["Yet another"]); + let quiet = Wrap::over(&mind, &[0.3]).with_forge(vec![push(1, week - 5000)]); + let at = week + 1000 + STREAM_MIN_GAP_S + STREAM_IDLE_GAP_S + 10; + let Outcome::Thought { thought, admitted: true } = tick(&organs(&quiet, &persona, &model), &|| released(), Mode::Live, t(at)).unwrap() else { panic!("the idle gap") }; + assert!(thought.stimulus.0.starts_with("Out of nowhere you recall"), "old activity is not news, so the hourly memory comes: {}", thought.stimulus.0); +} + +#[test] +fn reads_cite_only_what_the_context_showed() { + let mind = TestMind::seeded(); + let memories: Vec = (0..12).map(|at| memory(&format!("m{at:02}"), MemorySort::Learned, &format!("zz{at:02} {}", "x".repeat(290)), &["@libby:chat"])).collect(); + mind.admit_at(t(100)); + mind.committed(memories); + mind.admit_at(t(200)); + mind.committed(vec![run("w1", RunTurn::SelfRun, RunOperator::Mind, &[])]); + mind.committed( + (1..=3) + .map(|at| PipelineDocument::Reply(PipelineReply { instance: Slug(INSTANCE.into()), label: format!("w1-r{at}").as_str().into(), run: run_ref("w1"), answers: vec![], text: format!("rp{at} {}", "y".repeat(400)).as_str().into() })) + .collect(), + ); + let read = PersonaRead { state: state(), receipt_id: "r".into() }; + let seed = seed_of(&Wrap::over(&mind, &[]), &read, Some(t(300)), t(400)).unwrap(); + assert!(!seed.news); + let cited: Vec = seed.reads.iter().filter(|read| read.kind == PipelineKind::Memory).map(|read| read.id.0.rsplit(':').next().unwrap().to_owned()).collect(); + assert!(!cited.is_empty() && cited.len() < 12, "{cited:?}"); + for label in &cited { + assert!(seed.context.contains(&format!("zz{}", &label[1..])), "{label} is cited but not shown"); + } + let shown = (0..12).filter(|at| seed.context.contains(&format!("zz{at:02}"))).count(); + assert_eq!(shown, cited.len(), "every memory the context shows is cited"); + let replies_cited = seed.reads.iter().filter(|read| read.kind == PipelineKind::Reply).count(); + let replies_shown = (1..=3).filter(|at| seed.context.contains(&format!("rp{at} "))).count(); + assert!(replies_shown >= 1 && replies_shown < 3, "the replies section is bounded: {replies_shown}"); + assert_eq!(replies_cited, replies_shown, "replies cited are the replies shown"); +} + +#[test] +fn every_thought_records_the_fixed_parts_of_its_prompts() { + let mind = mind_with_thought(1000); + let model = Fake::saying("I turn it over.", &["A new drift"]); + let persona = persona(); + let wrap = Wrap::over(&mind, &[0.1]); + let Outcome::Thought { thought, .. } = tick(&organs(&wrap, &persona, &model), &|| released(), Mode::Live, t(1000 + STREAM_IDLE_GAP_S)).unwrap() else { panic!("idle gap") }; + let version = &thought.prompt_version.0; + assert_eq!(version, &prompt_version()); + let guidance = format!("{STREAM_GUIDANCE:?}"); + for part in [guidance.as_str(), GHOSTLIGHT_REV, "words=180", "'s notebook", "", ""] { + assert!(version.contains(part), "{part} missing from {version}"); + } + assert!(version.len() <= LINE_MAX_BYTES, "{}", version.len()); + assert!(include_str!("../../Cargo.toml").contains(GHOSTLIGHT_REV), "the recorded revision is the manifest's pin"); + let rebuilt = build_projector_prompt(&ProjectorPrompt { identity: &thought.identity.0, typed_context: &thought.context.0, visible_stimulus: &thought.stimulus.0, domain_guidance: STREAM_GUIDANCE, word_budget: PROJECTOR_WORDS }); + assert_eq!(model.projected.borrow()[0], rebuilt, "the trace rebuilds the Projector's exact prompt"); +} + + +/// A thought ticked at 900 s and admitted at 1000 s: what landed at 950 s while the models ran is news to the next one. +#[test] +fn what_lands_while_she_thinks_is_news_to_the_next_thought() { + let mind = TestMind::seeded(); + mind.admit_at(t(950)); + mind.committed(vec![memory("m9", MemorySort::Decided, "Landed while the Projector ran", &[])]); + mind.admit_at(t(1000)); + mind.committed(vec![thought_doc(&ticked(900), "A drift that took a hundred seconds")]); + let model = Fake::saying("I turn it over.", &["A new drift"]); + let persona = persona(); + let wrap = Wrap::over(&mind, &[0.1]).with_forge(vec![push(2, 1000), push(1, 950), push(0, 899)]); + let Outcome::Thought { thought, admitted: true } = tick(&organs(&wrap, &persona, &model), &|| released(), Mode::Live, t(1901)).unwrap() else { panic!("news past the gap") }; + assert!(thought.stimulus.0.contains("Landed while the Projector ran"), "{}", thought.stimulus.0); + assert!(thought.stimulus.0.contains("2 commits pushed by libby"), "the pushes at 950 s and 1000 s, not the one at 899 s: {}", thought.stimulus.0); +} + +#[test] +fn a_capped_repo_is_judged_by_its_own_events_and_disclosed_when_they_do_not_decide_it() { + let mind = mind_with_thought(1000); + let read = PersonaRead { state: state(), receipt_id: "r".into() }; + let cultlib = |id, secs| RepoEvent { repo: "GameCult/CultLib".into(), ..push(id, secs) }; + let mut two = Wrap::over(&mind, &[]).with_forge(vec![push(2, 1500), cultlib(1, 500)]); + two.forge.as_mut().unwrap().capped = vec!["GameCult/Huginn".into(), "GameCult/CultLib".into()]; + let seed = seed_of(&two, &read, Some(t(1000)), t(1901)).unwrap(); + assert!(seed.stimulus.to_lowercase().contains("page limit") && seed.stimulus.contains("GameCult/Huginn"), "the repo no event decides stays disclosed: {}", seed.stimulus); + assert!(!seed.stimulus.contains("GameCult/CultLib"), "an event before her thought decides CultLib's cap: {}", seed.stimulus); + let mut flooded = Wrap::over(&mind, &[]).with_forge(vec![]); + flooded.forge.as_mut().unwrap().capped = vec!["GameCult/Huginn".into()]; + let seed = seed_of(&flooded, &read, Some(t(1000)), t(1901)).unwrap(); + assert!(seed.news && seed.stimulus.to_lowercase().contains("page limit"), "a cap with no event at all is disclosed: {}", seed.stimulus); +} + +#[test] +fn a_quiet_hour_reads_the_forge_at_each_gap_mark_only() { + let mind = mind_with_thought(1000); + let model = Fake::saying("I turn it over.", &["A new drift"]); + let persona = persona(); + let wrap = Wrap::over(&mind, &[0.1]).with_forge(vec![push(1, 500)]); + let organs = organs(&wrap, &persona, &model); + let mut secs = 1000 + STREAM_POLL_S; + while secs < 1000 + STREAM_IDLE_GAP_S { + assert_eq!(tick(&organs, &|| released(), Mode::Live, t(secs)).unwrap(), Outcome::NotDue, "at {secs}"); + secs += STREAM_POLL_S; + } + assert_eq!(wrap.forge_reads.get(), 3, "one read for each mark crossed at 900, 1800 and 2700 s"); + assert_eq!(model.calls(), 0); +} + +#[test] +fn identical_memory_lines_are_each_cited_once() { + let mind = TestMind::seeded(); + mind.admit_at(t(100)); + mind.committed((0..12).map(|at| memory(&format!("m{at:02}"), MemorySort::Learned, &format!("the same thing {}", "x".repeat(290)), &["@libby:chat"])).collect()); + let read = PersonaRead { state: state(), receipt_id: "r".into() }; + let seed = seed_of(&Wrap::over(&mind, &[]), &read, Some(t(300)), t(400)).unwrap(); + let shown = seed.context.lines().filter(|line| line.contains("the same thing")).count(); + let cited = seed.reads.iter().filter(|read| read.kind == PipelineKind::Memory).count(); + assert!(shown >= 1 && cited == shown, "{shown} lines shown, {cited} cited"); +} + +/// The environment's proxy settings are read when an agent is built, so the check runs in a child process that has them. +#[test] +fn the_stream_agent_ignores_the_proxy_environment() { + if std::env::var("STREAM_PROXY_CHILD").is_ok() { + let plain: ureq::Agent = ureq::Agent::config_builder().build().into(); + assert!(plain.config().proxy().is_some(), "the child's environment names a proxy"); + let stream = OllamaStream::open("http://127.0.0.1:9", "p", "b").unwrap(); + assert!(stream.agent.config().proxy().is_none(), "the stream's agent took the environment's proxy"); + assert_eq!(stream.agent.config().max_redirects(), 0); + return; + } + let child = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "stream::tests::the_stream_agent_ignores_the_proxy_environment", "--nocapture"]) + .env("STREAM_PROXY_CHILD", "1") + .env("ALL_PROXY", "http://192.0.2.1:3128") + .env("HTTP_PROXY", "http://192.0.2.1:3128") + .output() + .unwrap(); + assert!(child.status.success(), "{}{}", String::from_utf8_lossy(&child.stdout), String::from_utf8_lossy(&child.stderr)); +}