diff --git a/Cargo.lock b/Cargo.lock index dfffd46..251c9fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -730,6 +730,18 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "ghostlight-verse" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "ghostlight", + "serde_json", + "tempfile", + "tokio", +] + [[package]] name = "hmac" version = "0.12.1" diff --git a/Cargo.toml b/Cargo.toml index 7bfeb7e..681ebc9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ "crates/ghostlight", "crates/ghostlight-dungeon", "crates/ghostlight-persona-projection", + "crates/ghostlight-verse", ] resolver = "2" diff --git a/crates/ghostlight-verse/Cargo.toml b/crates/ghostlight-verse/Cargo.toml new file mode 100644 index 0000000..89dd37f --- /dev/null +++ b/crates/ghostlight-verse/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "ghostlight-verse" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +ghostlight = { path = "../ghostlight" } +tokio.workspace = true + +[dev-dependencies] +serde_json.workspace = true +tempfile = "3" diff --git a/crates/ghostlight-verse/src/main.rs b/crates/ghostlight-verse/src/main.rs new file mode 100644 index 0000000..9c70d06 --- /dev/null +++ b/crates/ghostlight-verse/src/main.rs @@ -0,0 +1,346 @@ +//! The verse runner's operator command line. +//! +//! `genesis` creates the canonical Aetheria verse world in a state root, and +//! `run` advances it a given number of ticks under a token cap, printing per +//! tick what each faction did and what it cost. A run starts only when this +//! command is invoked, and ends when it has done what it was asked. +//! +//! The state root holds `world.cc`, `controller-work.cc`, `runs.cc` and +//! `calls.cc`. The provider key is read from a file at run time and appears +//! nowhere else. + +use anyhow::{Context, Result, anyhow, bail}; +use chrono::{Duration, Utc}; +use ghostlight::{ + CommandBody, CommandId, ControllerModels, ControllerOpenError, ControllerRunner, DEFAULT_HOSTED_MODEL_PREFIX, + DEFAULT_LOCAL_MODEL_PREFIX, CallRecordStore, CoverBudget, CreateVerseWorldIntent, + HostedBinding, Lens, LensWeights, LocalBinding, PrincipalCommandIntent, RecordingInferencePort, + RunStart, RunStatus, TickMinutes, VerifiedPrincipalEvidence, VerseRunConfig, VerseRunStore, + WorldMailbox, call_run_id, id_text, open_controller_work, open_inference, + read_aetheria_catalog, run_manual, +}; +use std::collections::BTreeMap; +use std::net::SocketAddr; +use std::ops::ControlFlow; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +/// What a manual run spends per tick on cognition cells. The operator's lever +/// on cost is the token cap; the partition always covers every faction. +const COVER: CoverBudget = CoverBudget { + cells: 8, + constituent_cap: 8, + urgency_slots: 2, +}; + +const RUNTIME_ID: &str = "ghostlight-verse"; + +#[tokio::main] +async fn main() -> ExitCode { + let arguments: Vec = std::env::args().skip(1).collect(); + match command(arguments).await { + Ok(code) => code, + Err(error) => { + eprintln!("ghostlight-verse: {error:#}"); + ExitCode::FAILURE + } + } +} + +async fn command(arguments: Vec) -> Result { + let Some((name, rest)) = arguments.split_first() else { + bail!("usage: ghostlight-verse genesis|run [flags]"); + }; + match name.as_str() { + "genesis" => genesis(Flags::parse(rest, &GENESIS_FLAGS)?).await, + "run" => run(Flags::parse(rest, &RUN_FLAGS)?).await, + _ => bail!("unknown command, expected genesis or run"), + } +} + +const GENESIS_FLAGS: [&str; 4] = ["catalog", "aetheria-commit", "state", "owner-hash"]; +const RUN_FLAGS: [&str; 7] = [ + "state", + "ticks", + "token-cap", + "tick-span-minutes", + "model", + "base-url", + "key-file", +]; +/// Flags a command may omit. +const OPTIONAL: [&str; 1] = ["key-file"]; + +/// `--name value` pairs, each named once and from the command's own list. +struct Flags(BTreeMap<&'static str, String>); + +impl Flags { + fn parse(arguments: &[String], allowed: &[&'static str]) -> Result { + let mut seen = BTreeMap::new(); + let mut rest = arguments.iter(); + while let Some(flag) = rest.next() { + let Some(name) = flag.strip_prefix("--") else { + bail!("expected a --flag, found a bare argument"); + }; + let Some(&known) = allowed.iter().find(|candidate| **candidate == name) else { + bail!("a flag this command does not take was given"); + }; + let Some(value) = rest.next() else { + bail!("--{known} needs a value"); + }; + if seen.insert(known, value.clone()).is_some() { + bail!("--{known} was given twice"); + } + } + for &required in allowed { + if !OPTIONAL.contains(&required) && !seen.contains_key(required) { + bail!("--{required} is required"); + } + } + Ok(Self(seen)) + } + + fn text(&self, name: &str) -> &str { + self.0.get(name).map_or("", String::as_str) + } + + fn path(&self, name: &str) -> PathBuf { + PathBuf::from(self.text(name)) + } + + fn number(&self, name: &str) -> Result { + match self.text(name).parse::() { + Ok(value) if value > T::default() => Ok(value), + _ => bail!("--{name} is not a positive integer"), + } + } +} + +async fn genesis(flags: Flags) -> Result { + let state = flags.path("state"); + std::fs::create_dir_all(&state).context("the state directory cannot be created")?; + let catalog = read_aetheria_catalog(flags.path("catalog")) + .context("the Aetheria catalog cannot be read")?; + let factions = catalog.factions.len(); + let principal = VerifiedPrincipalEvidence::new(flags.text("owner-hash"), Utc::now() + Duration::hours(1)); + + let (mailbox, task) = WorldMailbox::open(state.join("world.cc")) + .map_err(store_fault("the world file in --state cannot be opened"))?; + let receipt = mailbox + .create_verse( + CreateVerseWorldIntent { + id: CommandId::new(), + title: "The Aetheria Verse".into(), + brief: "The Aetheria verse, simulated from its catalog.".into(), + catalog, + aetheria_commit: flags.text("aetheria-commit").into(), + lens_weights: LensWeights::new(Lens::ALL.into_iter().map(|lens| (lens, 1)).collect()), + targets: BTreeMap::new(), + }, + &principal, + ) + .await + .context("the verse world cannot be created")?; + // A created world is a draft: the owner approves and activates it, so the + // factions hold opportunities. + for body in [CommandBody::ApproveDraft, CommandBody::ActivateWorld] { + let revision = mailbox.snapshot().await.context("the world cannot be read")?.revision; + mailbox + .submit_principal( + PrincipalCommandIntent { + id: CommandId::new(), + world_id: receipt.world_id, + expected_revision: revision, + body, + }, + &principal, + ) + .await + .context("the world cannot be activated")?; + } + drop(mailbox); + task.await.context("the world file did not close cleanly")?; + println!("world {}", id_text(receipt.world_id)); + println!("factions {factions}"); + println!("phase Active"); + Ok(ExitCode::SUCCESS) +} + +async fn run(flags: Flags) -> Result { + let state = flags.path("state"); + let ticks: u32 = flags.number("ticks")?; + let cap: u64 = flags.number("token-cap")?; + let span: u32 = flags.number("tick-span-minutes")?; + let tick_span = TickMinutes::new(span).context("--tick-span-minutes is outside the clock's span")?; + if !state.join("world.cc").is_file() { + bail!("the state directory holds no world; run genesis first"); + } + + let model = flags.text("model").to_owned(); + let models = ControllerModels { + projector: model.clone(), + persona: model.clone(), + interpreter: model.clone(), + operational_agent: model.clone(), + elaborator: model.clone(), + }; + let inner = open_lane(&flags, &model, &models)?; + + let (mailbox, task) = WorldMailbox::open(state.join("world.cc")) + .map_err(store_fault("the world file in --state cannot be opened"))?; + let world = mailbox.snapshot().await.context("the world cannot be read")?; + let calls = Arc::new(CallRecordStore::open(state.join("calls.cc")).map_err(store_fault("the call records in --state cannot be opened"))?); + let runs = VerseRunStore::open(state.join("runs.cc")).map_err(store_fault("the run records in --state cannot be opened"))?; + let work = open_controller_work(state.join("controller-work.cc")) + .map_err(store_fault("the controller work store in --state cannot be opened"))?; + let config = VerseRunConfig { + ticks, + tick_span, + cover: COVER, + }; + let claim = runs + .claim(&world, cap, &config) + .context("the run cannot be claimed")?; + let sequence = claim.sequence(); + // A start that fails after the claim ends the claimed run, so it binds + // nothing: the next claim takes the next number. + let started = RecordingInferencePort::open( + inner, + calls, + call_run_id(sequence), + cap, + &RunStart::of(&world), + ) + .context("the run's call records cannot be opened") + .and_then(|port| { + let port = Arc::new(port); + ControllerRunner::open(mailbox.clone(), port.clone(), work, models) + .map(|runner| (port, runner)) + .context("the controller organ cannot open") + }); + let (port, runner) = match started { + Ok(started) => started, + Err(error) => { + return Err(match runs.abandon(claim, "start_refused") { + Ok(()) => error, + Err(_) => error.context("the refused run's record could not be ended"), + }); + } + }; + + println!( + "run {sequence}: {ticks} ticks of {span} minutes, cap {cap} tokens, model {model}" + ); + let interrupted = Arc::new(AtomicBool::new(false)); + { + let interrupted = interrupted.clone(); + tokio::spawn(async move { + if tokio::signal::ctrl_c().await.is_ok() { + interrupted.store(true, Ordering::SeqCst); + } + }); + } + let outcome = run_manual( + &mailbox, + &runner, + &port, + &runs, + claim, + &config, + |report| { + print!("{report}"); + if interrupted.load(Ordering::SeqCst) { + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) + } + }, + ) + .await + .context("the run could not be recorded")?; + println!( + "status: {}; {} of {} ticks closed", + outcome.record.status, + outcome.record.ticks_closed(), + outcome.record.ticks_requested + ); + drop(runner); + drop(port); + drop(mailbox); + task.await.context("the world file did not close cleanly")?; + Ok(match outcome.record.status { + RunStatus::Failed { .. } => ExitCode::FAILURE, + _ => ExitCode::SUCCESS, + }) +} + +/// The inference lane the model's prefix names. A hosted model needs an https +/// origin and a key file; a local one reaches a loopback server and needs +/// neither, so a key file beside it is refused. +fn open_lane( + flags: &Flags, + model: &str, + models: &ControllerModels, +) -> Result> { + let (mut hosted, mut local) = (None, None); + if model.starts_with(DEFAULT_HOSTED_MODEL_PREFIX) { + let Some(key) = flags.0.get("key-file") else { + bail!("a hosted model needs --key-file"); + }; + hosted = Some(HostedBinding { + base_url: flags.text("base-url").to_owned(), + key_path: Path::new(key).to_owned(), + model_prefix: DEFAULT_HOSTED_MODEL_PREFIX.into(), + caller_runtime_id: RUNTIME_ID.into(), + }); + } else if model.starts_with(DEFAULT_LOCAL_MODEL_PREFIX) { + if flags.0.contains_key("key-file") { + bail!("a local model takes no --key-file"); + } + let endpoint: SocketAddr = flags + .text("base-url") + .strip_prefix("http://") + .and_then(|address| address.trim_end_matches('/').parse().ok()) + .context("--base-url for a local model is not http://
:")?; + local = Some(LocalBinding { + endpoint, + model_prefix: DEFAULT_LOCAL_MODEL_PREFIX.into(), + caller_runtime_id: RUNTIME_ID.into(), + }); + } else { + bail!("--model names no inference lane (expected a hosted/ or local/ prefix)"); + } + open_inference(None, None, local, hosted, &models.each()) + .map_err(|error| anyhow!(lane_fault(&error))) +} + +/// A store that cannot open says which one and that it is under `--state`, +/// never the path or the store's own text, which carries the path. +fn store_fault(message: &'static str) -> impl FnOnce(E) -> anyhow::Error { + move |_| anyhow!(message) +} + +/// What a lane that cannot open says: the flag or the kind of fault, never a +/// value the operator passed. +fn lane_fault(error: &ControllerOpenError) -> &'static str { + match error { + ControllerOpenError::InvalidModels => "--model is not an exact nonempty identifier", + ControllerOpenError::Connector(_) => "the connector lane cannot open", + ControllerOpenError::WorkStore(_) => "the controller work journal cannot open", + ControllerOpenError::UnroutableModel { .. } => "--model is claimed by no inference lane", + ControllerOpenError::SdkSidecarMissing { .. } => "the SDK sidecar entry is not a file", + ControllerOpenError::LocalEndpointNotLoopback { .. } => { + "--base-url for a local model must be a loopback address" + } + ControllerOpenError::HostedEndpointNotHttps => { + "--base-url for a hosted model must be an https URL" + } + ControllerOpenError::HostedKeyUnreadable => { + "--key-file is unreadable, empty, or not one run of visible ASCII" + } + ControllerOpenError::SharedModelPrefix { .. } => "two inference lanes claim one model prefix", + ControllerOpenError::EmptyModelPrefix { .. } => "an inference lane's model prefix is empty", + } +} diff --git a/crates/ghostlight-verse/tests/verse_binary.rs b/crates/ghostlight-verse/tests/verse_binary.rs new file mode 100644 index 0000000..28cf75d --- /dev/null +++ b/crates/ghostlight-verse/tests/verse_binary.rs @@ -0,0 +1,445 @@ +//! The `ghostlight-verse` command against a loopback scripted responder. No +//! real model is called and no key file is read: the responder answers the +//! OpenAI-compatible chat-completions request the local lane sends. + +use serde_json::{Value, json}; +use std::io::{Read, Write}; +use std::net::{SocketAddr, TcpListener, TcpStream}; +use std::path::Path; +use std::process::{Command, Output}; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +const BINARY: &str = env!("CARGO_BIN_EXE_ghostlight-verse"); +const CATALOG: &str = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../ghostlight/tests/fixtures/aetheria-verse-catalog.cc" +); +const COMMIT: &str = "92b82e9aa5bb422c568c4bfc85a8f641426a41fc"; +/// What the responder reports each call cost: 120 prompt and 30 completion. +const PER_CALL: u64 = 150; + +fn verse(arguments: &[&str]) -> Output { + Command::new(BINARY).args(arguments).output().unwrap() +} + +fn stdout(output: &Output) -> String { + String::from_utf8_lossy(&output.stdout).into_owned() +} + +fn stderr(output: &Output) -> String { + String::from_utf8_lossy(&output.stderr).into_owned() +} + +fn genesis(state: &Path) -> Output { + verse(&[ + "genesis", + "--catalog", + CATALOG, + "--aetheria-commit", + COMMIT, + "--state", + state.to_str().unwrap(), + "--owner-hash", + "verse-owner", + ]) +} + +/// Answers every chat-completions request with one tool call: the faction the +/// prompt names speaks its own line. Counts the requests it served. +struct Responder { + address: SocketAddr, + served: Arc, +} + +impl Responder { + fn start() -> Self { + Self::answering(200) + } + + /// A provider that refuses every call with this status. + fn refusing(status: u16) -> Self { + Self::answering(status) + } + + fn answering(status: u16) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let served = Arc::new(AtomicUsize::new(0)); + let counter = served.clone(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(stream) = stream else { return }; + let counter = counter.clone(); + std::thread::spawn(move || serve(stream, &counter, status)); + } + }); + Self { address, served } + } + + fn url(&self) -> String { + format!("http://{}", self.address) + } +} + +fn serve(mut stream: TcpStream, served: &AtomicUsize, status: u16) { + let mut raw = Vec::new(); + let mut chunk = [0u8; 4096]; + let (head_end, length) = loop { + let read = stream.read(&mut chunk).unwrap_or(0); + if read == 0 { + return; + } + raw.extend_from_slice(&chunk[..read]); + if let Some(end) = raw.windows(4).position(|window| window == b"\r\n\r\n") { + let head = String::from_utf8_lossy(&raw[..end]).to_ascii_lowercase(); + let length = head + .lines() + .find_map(|line| line.strip_prefix("content-length:")) + .and_then(|value| value.trim().parse::().ok()) + .unwrap_or(0); + break (end + 4, length); + } + }; + while raw.len() < head_end + length { + let read = stream.read(&mut chunk).unwrap_or(0); + if read == 0 { + return; + } + raw.extend_from_slice(&chunk[..read]); + } + let request: Value = serde_json::from_slice(&raw[head_end..head_end + length]).unwrap(); + let text = request.to_string(); + let faction = ["Adrasteia", "Brannoch"] + .into_iter() + .find(|name| text.contains(&format!("Decision owner:\\n{name}\\n"))) + .expect("a request names its faction"); + let sequence = served.fetch_add(1, Ordering::SeqCst); + let reply = json!({ + "id": format!("reply-{sequence}"), + "choices": [{ + "message": { + "role": "assistant", + "content": null, + "tool_calls": [{ + "id": format!("call-{sequence}"), + "type": "function", + "function": { + "name": "speak", + "arguments": json!({ "text": format!("{faction} holds the line.") }).to_string(), + }, + }], + }, + "finish_reason": "tool_calls", + }], + "usage": { "prompt_tokens": 120, "completion_tokens": 30 }, + }) + .to_string(); + let response = format!( + "HTTP/1.1 {status} Scripted\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{reply}", + reply.len() + ); + let _ = stream.write_all(response.as_bytes()); +} + +fn run(state: &Path, responder: &Responder, ticks: &str, cap: &str) -> Output { + verse(&[ + "run", + "--state", + state.to_str().unwrap(), + "--ticks", + ticks, + "--token-cap", + cap, + "--tick-span-minutes", + "60", + "--model", + "local/scripted", + "--base-url", + &responder.url(), + ]) +} + +#[test] +fn binary_run_prints_acts_and_spend() { + let state = tempfile::tempdir().unwrap(); + let made = genesis(state.path()); + assert!(made.status.success(), "{}", stderr(&made)); + let made = stdout(&made); + assert!(made.contains("factions 2\n") && made.contains("phase Active\n"), "{made}"); + assert!(state.path().join("world.cc").is_file()); + + let responder = Responder::start(); + let output = run(state.path(), &responder, "2", "1000000"); + assert!(output.status.success(), "{}", stderr(&output)); + let text = stdout(&output); + + // Two ticks, each with both factions named by verb, and the spend as it + // accrues against the cap. + assert!(text.starts_with("run 1: 2 ticks of 60 minutes, cap 1000000 tokens"), "{text}"); + for tick in 0..2u64 { + let spent = PER_CALL * 2 * (tick + 1); + assert!( + text.contains(&format!("tick {tick} closed: 2 cells, 2 acts, 0 refused\n")), + "{text}" + ); + assert!( + text.contains(&format!( + " tokens: {} this tick, {spent} spent, {} remaining\n", + PER_CALL * 2, + 1_000_000 - spent + )), + "{text}" + ); + } + for faction in ["Adrasteia", "Brannoch"] { + assert_eq!( + text.matches(&format!(" {faction} speak [")).count(), + 2, + "{text}" + ); + assert!(text.contains(&format!("says \"{faction} holds the line.\"")), "{text}"); + } + assert!(text.ends_with("status: Completed; 2 of 2 ticks closed\n"), "{text}"); + assert_eq!(responder.served.load(Ordering::SeqCst), 4); + + // The state root holds each store the run wrote. + for file in ["world.cc", "controller-work.cc", "runs.cc", "calls.cc"] { + assert!(state.path().join(file).is_file(), "{file} was not written"); + } + + // A second run is the next in the world and carries on from its clock. + let again = run(state.path(), &responder, "1", "1000000"); + assert!(again.status.success(), "{}", stderr(&again)); + let text = stdout(&again); + assert!(text.starts_with("run 2: 1 ticks of 60 minutes"), "{text}"); + assert!(text.contains("tick 2 closed: 2 cells, 2 acts, 0 refused\n"), "{text}"); +} + +#[test] +fn a_cap_the_first_call_could_pass_ends_the_run_with_nothing_spent() { + let state = tempfile::tempdir().unwrap(); + assert!(genesis(state.path()).status.success()); + let responder = Responder::start(); + let output = run(state.path(), &responder, "3", "1000"); + assert!(output.status.success(), "{}", stderr(&output)); + let text = stdout(&output); + assert!(text.contains("tick 0 open: "), "{text}"); + assert!(text.contains("tokens: 0 this tick, 0 spent, 1000 remaining"), "{text}"); + assert!(text.ends_with("status: BudgetExhausted; 0 of 3 ticks closed\n"), "{text}"); + assert_eq!(responder.served.load(Ordering::SeqCst), 0, "no call was made"); +} + +#[test] +fn a_run_that_fails_says_so_and_exits_nonzero() { + let state = tempfile::tempdir().unwrap(); + assert!(genesis(state.path()).status.success()); + let responder = Responder::refusing(500); + let output = run(state.path(), &responder, "2", "1000000"); + assert!(!output.status.success()); + let text = stdout(&output); + assert!(text.contains("tick 0 open: "), "{text}"); + assert!( + text.ends_with("status: Failed (inference_recovery_required); 0 of 2 ticks closed\n"), + "{text}" + ); + assert_eq!( + responder.served.load(Ordering::SeqCst), + 1, + "the run stopped at the first refusal" + ); +} + +#[test] +fn the_command_refuses_what_it_cannot_run_and_never_echoes_a_value() { + let state = tempfile::tempdir().unwrap(); + let responder = Responder::start(); + + // No world yet. + let output = run(state.path(), &responder, "1", "1000"); + assert!(!output.status.success()); + assert!(stderr(&output).contains("run genesis first"), "{}", stderr(&output)); + + assert!(genesis(state.path()).status.success()); + // A second genesis does not overwrite the first. + assert!(!genesis(state.path()).status.success()); + + let secret = "sk-not-a-real-key-0000"; + let key = state.path().join(secret); + let cases: [(Vec<&str>, &str); 9] = [ + (vec![], "usage"), + (vec!["fly"], "unknown command"), + ( + vec!["run", "--state", state.path().to_str().unwrap(), "--ticks", "0"], + "--token-cap is required", + ), + ( + vec![ + "run", "--state", state.path().to_str().unwrap(), "--ticks", "many", + "--token-cap", "1", "--tick-span-minutes", "60", "--model", "local/x", + "--base-url", "http://127.0.0.1:1", + ], + "--ticks is not a positive integer", + ), + ( + vec![ + "run", "--state", state.path().to_str().unwrap(), "--ticks", "0", + "--token-cap", "1", "--tick-span-minutes", "60", "--model", "local/x", + "--base-url", "http://127.0.0.1:1", + ], + "--ticks is not a positive integer", + ), + ( + vec![ + "run", "--state", state.path().to_str().unwrap(), "--ticks", "1", + "--token-cap", "0", "--tick-span-minutes", "60", "--model", "local/x", + "--base-url", "http://127.0.0.1:1", + ], + "--token-cap is not a positive integer", + ), + ( + vec![ + "run", "--state", state.path().to_str().unwrap(), "--ticks", "1", + "--token-cap", "1", "--tick-span-minutes", "60", "--model", "gpt-4", + "--base-url", "http://127.0.0.1:1", + ], + "--model names no inference lane", + ), + ( + // A hosted model needs its key file; a missing one is named by flag. + vec![ + "run", "--state", state.path().to_str().unwrap(), "--ticks", "1", + "--token-cap", "1", "--tick-span-minutes", "60", "--model", "hosted/x", + "--base-url", "https://api.example.test", "--key-file", key.to_str().unwrap(), + ], + "--key-file is unreadable", + ), + ( + vec![ + "run", "--state", state.path().to_str().unwrap(), "--ticks", "1", + "--token-cap", "1", "--tick-span-minutes", "60", "--model", "local/x", + "--base-url", "http://127.0.0.1:1", "--key-file", key.to_str().unwrap(), + ], + "a local model takes no --key-file", + ), + ]; + for (arguments, expected) in cases { + let output = verse(&arguments); + assert!(!output.status.success(), "{arguments:?} succeeded"); + let error = stderr(&output); + assert!(error.contains(expected), "{arguments:?}: {error}"); + assert!(!error.contains(secret), "{arguments:?} echoed a path: {error}"); + assert!(!error.contains("many"), "{arguments:?} echoed a value: {error}"); + } +} + +#[test] +fn a_non_loopback_local_endpoint_is_refused_without_echoing_it() { + let state = tempfile::tempdir().unwrap(); + assert!(genesis(state.path()).status.success()); + let output = verse(&[ + "run", + "--state", + state.path().to_str().unwrap(), + "--ticks", + "1", + "--token-cap", + "1000", + "--tick-span-minutes", + "60", + "--model", + "local/x", + "--base-url", + "http://10.77.0.99:4242", + ]); + assert!(!output.status.success()); + let error = stderr(&output); + assert!(error.contains("--base-url"), "{error}"); + assert!(!error.contains("10.77.0.99"), "the endpoint was echoed: {error}"); + // The lane failed before any run was claimed. + assert!(!stdout(&output).contains("run 1"), "{}", stdout(&output)); +} + +/// A start refused after its claim ends the claimed record Failed, so the run +/// binds nothing. Reached from the command by removing the run records while +/// the call ledger still holds run 1 under another cap. +#[test] +fn a_start_the_ledger_refuses_ends_its_claimed_record_failed() { + let parent = tempfile::tempdir().unwrap(); + let state = parent.path().join("state"); + std::fs::create_dir_all(&state).unwrap(); + assert!(genesis(&state).status.success()); + let responder = Responder::start(); + let first = run(&state, &responder, "1", "1000000"); + assert!(first.status.success(), "{}", stderr(&first)); + std::fs::remove_file(state.join("runs.cc")).unwrap(); + + let refused = run(&state, &responder, "1", "500000"); + assert!(!refused.status.success()); + + let world = { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let _entered = runtime.enter(); + let (mailbox, task) = ghostlight::WorldMailbox::open(state.join("world.cc")).unwrap(); + let world = runtime.block_on(mailbox.snapshot()).unwrap().world_id; + drop(mailbox); + runtime.block_on(task).unwrap(); + world + }; + let runs = ghostlight::VerseRunStore::open(state.join("runs.cc")) + .unwrap() + .runs(world) + .unwrap(); + assert_eq!(runs.len(), 1); + assert_eq!( + runs[0].status, + ghostlight::RunStatus::Failed { + reason: "start_refused".into() + } + ); +} + +/// A store that cannot open names the flag and the fault, never the path. +#[test] +fn a_store_that_cannot_open_never_echoes_the_state_path() { + let responder = Responder::start(); + for file in ["world.cc", "controller-work.cc"] { + for held in [false, true] { + let parent = tempfile::tempdir().unwrap(); + let state = parent.path().join("distinctivestatename"); + std::fs::create_dir_all(&state).unwrap(); + assert!(genesis(&state).status.success()); + let first = run(&state, &responder, "1", "1000000"); + assert!(first.status.success(), "{}", stderr(&first)); + + let target = state.join(file); + let _guard: Box = if !held { + std::fs::write(&target, b"not a database").unwrap(); + Box::new(()) + } else if file == "world.cc" { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let entered = runtime.enter(); + let opened = ghostlight::WorldMailbox::open(&target).unwrap(); + drop(entered); + Box::new((runtime, opened)) + } else { + Box::new(ghostlight::open_controller_work(&target).unwrap()) + }; + + let output = run(&state, &responder, "1", "1000000"); + assert!(!output.status.success(), "{file} held={held}"); + let error = stderr(&output); + assert!(error.contains("--state"), "{file} held={held}: {error}"); + assert!( + !error.contains("distinctivestatename"), + "{file} held={held} echoed the path: {error}" + ); + } + } +} diff --git a/crates/ghostlight/src/consumer.rs b/crates/ghostlight/src/consumer.rs index 2df43ad..e764dde 100644 --- a/crates/ghostlight/src/consumer.rs +++ b/crates/ghostlight/src/consumer.rs @@ -469,7 +469,7 @@ mod tests { AuthorityTargetRef, CallerId, CausalBoundary, CommandBody, CommandEnvelope, CommitmentKind, ComponentOp, CoverBudget, DecisionOpportunity, DecisionScope, Declaration, DraftHandle, EntityDeclaration, EntityId, EntityKind, EvidenceRef, FactDeclaration, FactStandingRef, - FictionalMinutes, JurisdictionKey, NewController, PatchGround, Quantity, Ref, + FictionalMinutes, JurisdictionKey, NewController, PatchGround, Quantity, Ref, TickIndex, RouteDeclaration, Statement, SubjectDeclaration, SubjectId, SubjectKind, SystemCapability, WorldEffect, WorldId, WorldKernel, WorldMailbox, WorldScaleIntentRef, agency_graph, commit_digest, derive_boundaries, derive_cover, derive_opportunities, @@ -736,8 +736,7 @@ mod tests { assert!(!graph.subjects.contains(&mirror.first)); let cover = derive_cover( snapshot.world_id, - snapshot.now, - 30, + TickIndex(0), &opportunities, &graph, CoverBudget { diff --git a/crates/ghostlight/src/controllers.rs b/crates/ghostlight/src/controllers.rs index 5d3e573..0f551c5 100644 --- a/crates/ghostlight/src/controllers.rs +++ b/crates/ghostlight/src/controllers.rs @@ -924,6 +924,11 @@ impl ControllerWork { } impl NarrativeCheckpoint { + /// The row holds its submission body: the next step is the world's answer. + fn awaits_world(&self) -> bool { + matches!(self, Self::ReadyToSubmit { .. } | Self::NoProposal { .. }) + } + fn command_id(&self) -> CommandId { match self { Self::Projector { command_id, .. } @@ -1100,6 +1105,11 @@ impl OperationalCheckpoint { } } + /// The row holds its submission body: the next step is the world's answer. + fn awaits_world(&self) -> bool { + matches!(self, Self::ReadyToSubmit { .. } | Self::NoProposal { .. }) + } + fn opportunity(&self) -> &DecisionOpportunity { match self { Self::AgentInFlight { opportunity, .. } @@ -1446,6 +1456,34 @@ fn record_persona_turn( )) } +/// A narrative turn the world overtook, for a driver test that has no +/// narrative subject to run one: the same value `interrupted` builds, bound to +/// the opportunity it is given. +#[cfg(test)] +pub(crate) fn overtaken_narrative_run(opportunity: &DecisionOpportunity) -> NarrativeRun { + let turn = record_persona_turn( + opportunity, + "projector".into(), + "persona".into(), + "Prose.".into(), + ) + .expect("a fixture opportunity encodes"); + NarrativeRun::Interrupted(NarrativeInterruption { + subject: opportunity.scope.subject_id, + bound_scope_digest: opportunity.scope_digest.as_str().to_owned(), + fresh_scope_digest: None, + gap: TranslationGapSummary { + kind: TranslationGapKind::Unresolved, + source: SourceRange { + start_byte: 0, + end_byte: 6, + }, + detail: OVERTAKEN_DETAIL.into(), + }, + turn, + }) +} + /// One opportunity's selection against one snapshot. /// /// The expected controller mode is read from the opportunity rather than @@ -1584,12 +1622,26 @@ fn turn_matches_opportunity(turn: &PersonaTurn, opportunity: &DecisionOpportunit && opportunity.scope_digest.as_str() == turn.binding().scope_digest } -/// Two opportunity values that bind the same proposal. The revision is a receipt -/// and an ordering value; the scope digest is the binding. -fn binds_same_scope(left: &DecisionOpportunity, right: &DecisionOpportunity) -> bool { - left.world_id == right.world_id - && left.scope == right.scope - && left.scope_digest == right.scope_digest +/// Two opportunity values that name the same decision: the same world and the +/// same decision scope. The scope digest is not part of the name. It says what +/// the scope looked like when a proposal was reasoned, so it gates work that is +/// still reasoning, never a row whose submission body the world answers by +/// command id. +fn same_decision(left: &DecisionOpportunity, right: &DecisionOpportunity) -> bool { + left.world_id == right.world_id && left.scope == right.scope +} + +/// Whether a persisted row may be resumed against the caller's opportunity. +/// A row that awaits the world carries its submission body, and the world +/// answers it by command id (its receipt, or a `ScopeChanged` refusal), so the +/// caller's digest cannot change what that command is. A row still reasoning +/// keeps the digest check, so no tokens are spent on a scope that moved. +fn row_answers_caller( + row: &DecisionOpportunity, + caller: &DecisionOpportunity, + awaits_world: bool, +) -> bool { + same_decision(row, caller) && (awaits_world || row.scope_digest == caller.scope_digest) } fn derive_narrative_capture( @@ -2843,6 +2895,35 @@ pub struct ConstituentSubmission { pub submission: SubmissionDisposition, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ConstituentRefusal { + pub subject: SubjectId, + pub reason: ConstituentRefusalReason, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ConstituentRefusalReason { + /// The subject's opportunity moved between the cover and the cell, so it was + /// not selected and nothing was submitted for it. + LeftBeforeSelection, + /// The kernel refused its proposal because the scope it was reasoned from + /// no longer held (a neighbour's act moved it). + ScopeChanged, + /// The kernel refused its command for any other reason. + WorldRefused, + /// The member's own tool call could not be decoded and it proposed nothing + /// else, so no turn was submitted for it: a decode failure is not a + /// decision to decline. + ProposalUndecodable, + /// The persisted row of a resumed cell does not hold this member, so the + /// cognition behind the row never saw it. Nothing is submitted for it. + RowDisagreesWithCell, + /// A tool call in the cell named no handle and this member finished + /// without a choice of its own, so whether the call was meant for it is + /// unknown. Declining it would pass the model's gap off as a decision. + CallUnattributed, +} + /// One coarse cell's outcome: one inference, N ordinary one-opportunity /// submissions, and whatever the decode could not attribute. #[derive(Debug)] @@ -2850,7 +2931,14 @@ pub struct GroupedRun { pub cell: CellId, pub resolution: Resolution, pub submissions: Vec, + /// Decode problems the grouped decode could not attribute to a constituent. pub needs: Vec, + /// The tool names of the calls that named no handle, already valid under + /// the provider contract; never their arguments. + pub unattributed: Vec, + /// Constituents of this cell that did not commit, one entry each, with the + /// reason. A constituent is in `submissions` or here, never lost. + pub refused: Vec, /// Set when the run stopped before every constituent was submitted. The /// constituents already in `submissions` are committed; the rest resume from /// the persisted row against the kernel's idempotency ledger. @@ -3036,7 +3124,7 @@ impl ControllerRunner { } match self.work.lookup(command_id).await? { ControllerWorkLookup::Confirmed(ControllerWork::Narrative(checkpoint)) => { - if !binds_same_scope(checkpoint.opportunity(), opportunity) { + if !row_answers_caller(checkpoint.opportunity(), opportunity, checkpoint.awaits_world()) { return Err(ControllerError::OpportunityMismatch); } return self @@ -3044,7 +3132,7 @@ impl ControllerRunner { .await; } ControllerWorkLookup::CustodyUncertain(ControllerWork::Narrative(checkpoint)) => { - if !binds_same_scope(checkpoint.opportunity(), opportunity) { + if !row_answers_caller(checkpoint.opportunity(), opportunity, checkpoint.awaits_world()) { return Err(ControllerError::OpportunityMismatch); } return Ok(narrative_pending( @@ -3116,7 +3204,7 @@ impl ControllerRunner { } match self.work.lookup(command_id).await? { ControllerWorkLookup::Confirmed(ControllerWork::Operational(checkpoint)) => { - if !binds_same_scope(checkpoint.opportunity(), opportunity) { + if !row_answers_caller(checkpoint.opportunity(), opportunity, checkpoint.awaits_world()) { return Err(ControllerError::OpportunityMismatch); } return self @@ -3124,7 +3212,7 @@ impl ControllerRunner { .await; } ControllerWorkLookup::CustodyUncertain(ControllerWork::Operational(checkpoint)) => { - if !binds_same_scope(checkpoint.opportunity(), opportunity) { + if !row_answers_caller(checkpoint.opportunity(), opportunity, checkpoint.awaits_world()) { return Err(ControllerError::OpportunityMismatch); } return Ok(operational_pending( @@ -3372,7 +3460,7 @@ impl ControllerRunner { if checkpoint.cell() != cell { return Err(ControllerError::OpportunityMismatch); } - return self.resume_persisted_group(command_id, checkpoint).await; + return self.resume_row(command_id, checkpoint, members).await; } ControllerWorkLookup::CustodyUncertain(ControllerWork::Grouped(checkpoint)) => { if checkpoint.cell() != cell { @@ -3411,7 +3499,7 @@ impl ControllerRunner { .map_err(ControllerError::Snapshot)?; let mut selected = Vec::new(); let mut constituents = Vec::new(); - let mut needs = Vec::new(); + let mut refused = Vec::new(); for member in members { match select_one(&snapshot, &member.opportunity) { Ok(decision) => { @@ -3432,8 +3520,9 @@ impl ControllerRunner { // cell was not active when the cell ran. It is dropped rather // than declined: declining a stale opportunity would be refused // anyway, and the drop is recorded where a reader can see it. - Err(error) => needs.push(ControllerNeed { - detail: format!("a constituent left the cell before selection: {error}"), + Err(_) => refused.push(ConstituentRefusal { + subject: member.subject, + reason: ConstituentRefusalReason::LeftBeforeSelection, }), } } @@ -3442,7 +3531,9 @@ impl ControllerRunner { cell, resolution: Resolution::Coarse { constituents: 0 }, submissions: Vec::new(), - needs, + needs: Vec::new(), + unattributed: Vec::new(), + refused, pending: None, }); } @@ -3495,12 +3586,43 @@ impl ControllerRunner { { grouped_pending(checkpoint, ControllerPendingReason::StoreReopenRequired) } else { - self.resume_persisted_group(command_id, checkpoint).await? + self.resume_persisted_group(command_id, checkpoint, members) + .await? }; // The constituents that left the cell are reported beside the run they // were dropped from, not written into the row: they produced no // cognition, so they are not cognition evidence. - run.needs.splice(0..0, needs); + run.refused.splice(0..0, refused); + Ok(run) + } + + /// A persisted row resumed for the cell the driver froze. The cell owns + /// the member set: a cell member the row does not hold is refused here, + /// once, with the reason that is true of it, and a row member the cell + /// lacks is not run. Nothing downstream repairs the lists. + async fn resume_row( + &self, + command_id: CommandId, + checkpoint: GroupedCheckpoint, + members: &[Constituent], + ) -> Result { + let lacked: Vec = members + .iter() + .filter(|member| { + !checkpoint + .constituents() + .iter() + .any(|held| held.subject == member.subject) + }) + .map(|member| ConstituentRefusal { + subject: member.subject, + reason: ConstituentRefusalReason::RowDisagreesWithCell, + }) + .collect(); + let mut run = self + .resume_persisted_group(command_id, checkpoint, members) + .await?; + run.refused.splice(0..0, lacked); Ok(run) } @@ -3508,16 +3630,17 @@ impl ControllerRunner { &self, command_id: CommandId, checkpoint: GroupedCheckpoint, + members: &[Constituent], ) -> Result { if checkpoint.command_id() != command_id { return Err(ControllerError::CommandMismatch); } match checkpoint { checkpoint @ GroupedCheckpoint::AgentInFlight { .. } => { - self.run_group_pending(checkpoint).await + self.run_group_pending(checkpoint, members).await } checkpoint @ GroupedCheckpoint::Submitting { .. } => { - self.submit_group(checkpoint).await + self.submit_group(checkpoint, members).await } } } @@ -3525,6 +3648,7 @@ impl ControllerRunner { async fn run_group_pending( &self, mut checkpoint: GroupedCheckpoint, + members: &[Constituent], ) -> Result { loop { let GroupedCheckpoint::AgentInFlight { @@ -3572,7 +3696,7 @@ impl ControllerRunner { ControllerPendingReason::StoreReopenRequired, )); } - return self.submit_group(next).await; + return self.submit_group(next, members).await; } Ok(GroupedLoopEvaluation::Continue { conversation }) => { let round = completed.len(); @@ -3623,6 +3747,7 @@ impl ControllerRunner { async fn submit_group( &self, checkpoint: GroupedCheckpoint, + members: &[Constituent], ) -> Result { let GroupedCheckpoint::Submitting { cell, @@ -3649,8 +3774,30 @@ impl ControllerRunner { let resolution = Resolution::Coarse { constituents: constituents.len(), }; - let mut needs = capture.needs; + let needs = capture.needs; + let unattributed = capture.unattributed; let mut submissions = Vec::new(); + let mut refused = Vec::new(); + // The world's attention order, read once: among exercises the subject the + // world would attend first lands first, so a member refused behind a + // neighbour's act (and so ahead of it next tick) is the one that lands + // next time. `order_opportunities` starves nobody. + let attention = match self.mailbox.snapshot().await { + Ok(snapshot) => snapshot.opportunities, + Err(MailboxError::Unavailable) => { + return Ok(grouped_pending( + checkpoint.clone(), + ControllerPendingReason::WorldUnavailable, + )); + } + Err(MailboxError::OutcomeUnknown { .. }) => { + return Ok(grouped_pending( + checkpoint.clone(), + ControllerPendingReason::WorldOutcomeUnknown, + )); + } + Err(error) => return Err(ControllerError::Snapshot(error)), + }; // Declines first. A decline changes no other subject's scope, so every // silent constituent's turn is consumed. An exercise can change what its // neighbours bound to, and co-located subjects are exactly @@ -3659,8 +3806,34 @@ impl ControllerRunner { // outcome; ordering the non-mutating submissions ahead of it loses no // turns and costs nothing. let mut ordered: Vec<(usize, &ConstituentWork)> = constituents.iter().enumerate().collect(); - ordered.sort_by_key(|(handle, _)| capture.proposals.contains_key(handle)); + ordered.sort_by_key(|(handle, constituent)| { + let exercises = capture.proposals.contains_key(handle); + let place = attention + .iter() + .position(|opportunity| opportunity.scope.subject_id == constituent.subject) + .unwrap_or(usize::MAX); + (exercises, if exercises { place } else { 0 }) + }); for (handle, constituent) in ordered { + // The cell being run owns who acts. A row member the cell lacks + // has left it (the driver accounts for it there) and is not run. + if !members.iter().any(|member| member.subject == constituent.subject) { + continue; + } + if capture.undecodable.contains(&handle) { + refused.push(ConstituentRefusal { + subject: constituent.subject, + reason: ConstituentRefusalReason::ProposalUndecodable, + }); + continue; + } + if capture.call_unattributed.contains(&handle) { + refused.push(ConstituentRefusal { + subject: constituent.subject, + reason: ConstituentRefusalReason::CallUnattributed, + }); + continue; + } let command = match capture.proposals.get(&handle) { Some(invocation) => ControllerWorldCommand::Exercise(invocation.clone()), None => ControllerWorldCommand::Decline, @@ -3681,14 +3854,20 @@ impl ControllerRunner { resolution, submissions, needs, + unattributed, + refused, pending: Some(reason), }); } // One constituent's refusal is that constituent's outcome. The // cell is not a transaction: every other handle's proposal was // bound to its own opportunity and is unaffected. - Err(ControllerError::World(error)) => needs.push(ControllerNeed { - detail: format!("constituent c{handle} was refused by the world: {error}"), + Err(ControllerError::World(error)) => refused.push(ConstituentRefusal { + subject: constituent.subject, + reason: match error { + KernelError::ScopeChanged { .. } => ConstituentRefusalReason::ScopeChanged, + _ => ConstituentRefusalReason::WorldRefused, + }, }), Err(error) => return Err(error), } @@ -3704,6 +3883,8 @@ impl ControllerRunner { resolution, submissions, needs, + unattributed, + refused, pending: None, }) } @@ -5764,8 +5945,17 @@ fn split_handle(name: &str, constituents: usize) -> Option<(usize, &str)> { /// The batched decode. Vector-valued by construction: the singleton evaluator /// keeps its scalar proposal and is not touched. struct GroupedCapture { - /// Handle to proposal. A handle absent here proposed nothing and declines. + /// Handle to proposal. A handle absent here proposed nothing and declines, + /// unless it is in `undecodable`. proposals: BTreeMap, + /// Handles whose own tool call failed to decode and which never finished + /// otherwise: nothing is submitted for them. + undecodable: BTreeSet, + /// Handles that finished with no choice while some call in the cell named + /// no handle: nothing says the call was not theirs, so they do not decline. + call_unattributed: BTreeSet, + /// The names of the calls that named no handle. + unattributed: Vec, needs: Vec, } @@ -5779,6 +5969,11 @@ enum GroupedLoopEvaluation { pub(super) struct GroupedFold { proposals: BTreeMap, terminal: BTreeSet, + /// Handles with a call that failed to decode, whether or not a later call + /// finished them. + decode_failed: BTreeSet, + /// The names of the calls that named no handle, capped. + unattributed: Vec, needs: Vec, } @@ -5787,6 +5982,8 @@ impl GroupedFold { Self { proposals: BTreeMap::new(), terminal: BTreeSet::new(), + decode_failed: BTreeSet::new(), + unattributed: Vec::new(), needs: Vec::new(), } } @@ -5805,6 +6002,7 @@ pub(super) fn grouped_tool_result( // A model that writes `c99__carry` has not proposed anything. It has // produced a gap. None => { + fold.unattributed.push(name.to_string()); fold.needs.push(tool_decode_need( name, arguments, @@ -5835,6 +6033,7 @@ pub(super) fn grouped_tool_result( "invocation captured".into() } Err(detail) => { + fold.decode_failed.insert(handle); fold.needs.push(tool_decode_need(name, arguments, &detail)); "arguments recorded as a need".into() } @@ -5849,6 +6048,7 @@ pub(super) fn grouped_tool_result( "need recorded".into() } Err(error) => { + fold.decode_failed.insert(handle); fold.needs .push(tool_decode_need(name, arguments, &error.to_string())); "arguments recorded as a need".into() @@ -5869,6 +6069,7 @@ pub(super) fn grouped_tool_result( "decision finished without a proposal".into() } Err(error) => { + fold.decode_failed.insert(handle); fold.needs .push(tool_decode_need(name, arguments, &error.to_string())); "arguments recorded as a need".into() @@ -5876,6 +6077,7 @@ pub(super) fn grouped_tool_result( } } _ => { + fold.decode_failed.insert(handle); fold.needs.push(tool_decode_need( name, arguments, @@ -5993,6 +6195,22 @@ fn evaluate_grouped_loop( return Ok(GroupedLoopEvaluation::Complete { capture: GroupedCapture { proposals: fold.proposals, + undecodable: fold + .decode_failed + .difference(&fold.terminal) + .copied() + .collect(), + call_unattributed: if fold.unattributed.is_empty() { + BTreeSet::new() + } else { + (0..constituents.len()) + .filter(|handle| { + !fold.terminal.contains(handle) + && !fold.decode_failed.contains(handle) + }) + .collect() + }, + unattributed: fold.unattributed, needs: fold.needs, }, }); @@ -6055,6 +6273,8 @@ fn grouped_pending(checkpoint: GroupedCheckpoint, reason: ControllerPendingReaso }, submissions: Vec::new(), needs: Vec::new(), + unattributed: Vec::new(), + refused: Vec::new(), pending: Some(reason), } } @@ -14699,8 +14919,7 @@ mod tests { let graph = mailbox.agency_graph().await.unwrap(); let cover = derive_cover( snapshot.world_id, - snapshot.now, - 60, + TickIndex(snapshot.now.0 / 60), &snapshot.opportunities, &graph, CoverBudget { @@ -14747,8 +14966,9 @@ mod tests { /// Verification 20. One inference proposes for `c0`, names a handle outside /// the cell, and says nothing for the rest. The valid call commits under its /// own opportunity; the out-of-cell name produces a gap and reaches no - /// mailbox; every silent handle declines, so "attended and stayed silent" is - /// distinguishable from "was never attended". + /// mailbox; every handle that finished with no choice is refused as + /// `CallUnattributed` and not declined, because the call that named no + /// handle may have been meant for it. #[tokio::test] async fn soul_a_batched_turn_attributes_by_tool_identity_and_refuses_an_outside_handle() { let (_directory, mailbox, task) = active_cell_mailbox(vec![ @@ -14781,7 +15001,8 @@ mod tests { }; assert!(run.pending.is_none()); assert_eq!(run.resolution, Resolution::Coarse { constituents: 3 }); - assert_eq!(run.submissions.len(), 3, "every constituent finished"); + assert_eq!(run.submissions.len(), 1, "only the attributed call committed"); + assert_eq!(run.unattributed, vec!["c7__speak".to_owned()]); let first = cell.members()[0].subject; assert!(matches!( run.submissions @@ -14790,16 +15011,17 @@ mod tests { .map(|entry| &entry.submission), Some(SubmissionDisposition::Completed(_)) )); - for entry in run - .submissions - .iter() - .filter(|entry| entry.subject != first) - { - assert!( - matches!(entry.submission, SubmissionDisposition::NoProposal(_)), - "a silent constituent did not decline" - ); - } + assert_eq!( + run.refused, + cell.members()[1..] + .iter() + .map(|member| ConstituentRefusal { + subject: member.subject, + reason: ConstituentRefusalReason::CallUnattributed, + }) + .collect::>(), + "a silent constituent was declined behind a call that named no handle" + ); assert!( run.needs .iter() @@ -14817,6 +15039,241 @@ mod tests { task.await.unwrap(); } + /// A resumed row is run for the cell the driver froze. The cell owns the + /// member set: a member the row lacks is refused once as + /// `RowDisagreesWithCell` wherever it stands in a three-member cell, and a + /// row member the cell lacks is not run. + #[tokio::test] + async fn a_resumed_row_is_run_for_the_cell_and_refuses_the_member_it_lacks() { + let (_directory, mailbox, task) = active_cell_mailbox(vec![ + NewController::OperationalAgent, + NewController::OperationalAgent, + NewController::OperationalAgent, + ]) + .await; + let Cell::Group { id, tick, members } = one_group(&mailbox).await else { + panic!("the cell is not coarse") + }; + let [m0, m1, m2] = [0, 1, 2].map(|index| members[index].clone()); + let group = |members: Vec| Cell::Group { id, tick, members }; + let (runner, _store) = grouped_runner( + &mailbox, + vec![output( + vec![ + speak_call("c0", "c0__speak", "Hold the bridge."), + InferenceEvent::ToolCall { + call_id: "c1".into(), + name: "c1__finish_without_proposal".into(), + arguments: "{}".into(), + }, + ], + "row", + )], + ); + let subjects = |run: &GroupedRun| -> Vec { + run.submissions.iter().map(|entry| entry.subject).collect() + }; + // The row is persisted for [m0, m2]. + let CellRun::Grouped(first) = runner + .run_cell(&group(vec![m0.clone(), m2.clone()])) + .await + .unwrap() + else { + panic!("a coarse cell did not run the grouped lane") + }; + assert_eq!(subjects(&first), vec![m0.subject, m2.subject]); + let committed = mailbox.operator_log().await.unwrap().len(); + + // The cell is [m0, m1, m2]: the member in the middle is the one refused. + let CellRun::Grouped(second) = runner + .run_cell(&group(vec![m0.clone(), m1.clone(), m2.clone()])) + .await + .unwrap() + else { + panic!("a resumed coarse cell did not run the grouped lane") + }; + assert_eq!( + second.refused, + vec![ConstituentRefusal { + subject: m1.subject, + reason: ConstituentRefusalReason::RowDisagreesWithCell, + }] + ); + assert_eq!(subjects(&second), vec![m0.subject, m2.subject]); + + // The cell is [m2, m1]: m1 is refused, m2 runs, and m0 (in the row, + // not in the cell) is neither run nor named. + let CellRun::Grouped(third) = runner + .run_cell(&group(vec![m2.clone(), m1.clone()])) + .await + .unwrap() + else { + panic!("a resumed coarse cell did not run the grouped lane") + }; + assert_eq!( + third.refused, + vec![ConstituentRefusal { + subject: m1.subject, + reason: ConstituentRefusalReason::RowDisagreesWithCell, + }] + ); + assert_eq!(subjects(&third), vec![m2.subject]); + assert_eq!(mailbox.operator_log().await.unwrap().len(), committed); + + drop(runner); + drop(mailbox); + task.await.unwrap(); + } + + /// Every site that marks a member's own call undecodable refuses that + /// member, for either handle; a well-formed `record_need` does not. + #[tokio::test] + async fn a_member_whose_own_call_cannot_be_decoded_is_refused_at_every_site() { + let bad_speak = || InferenceEvent::ToolCall { + call_id: "bad".into(), + name: "c0__speak".into(), + arguments: json!({ "text": 5 }).to_string(), + }; + let call = |name: &str, arguments: &str| InferenceEvent::ToolCall { + call_id: "bad".into(), + name: name.into(), + arguments: arguments.into(), + }; + let sites: Vec<(&str, InferenceEvent, bool)> = vec![ + ("catalog arguments", bad_speak(), true), + ( + "record_need arguments", + call("c0__record_need", "not json at all"), + true, + ), + ( + "finish_without_proposal arguments", + call("c0__finish_without_proposal", "not json at all"), + true, + ), + ("tool not offered", call("c0__teleport", "{}"), true), + ( + "a well-formed record_need", + call("c0__record_need", r#"{"detail":"no route"}"#), + false, + ), + ]; + for (site, event, refuses) in sites { + let (_directory, mailbox, task) = active_cell_mailbox(vec![ + NewController::OperationalAgent, + NewController::OperationalAgent, + ]) + .await; + let cell = one_group(&mailbox).await; + let (runner, _store) = grouped_runner( + &mailbox, + vec![ + output( + vec![event, speak_call("c1", "c1__speak", "Hold the bridge.")], + "site-one", + ), + output(vec![InferenceEvent::Text("Nothing further.".into())], "site-two"), + ], + ); + let CellRun::Grouped(run) = runner.run_cell(&cell).await.unwrap() else { + panic!("a coarse cell did not run the grouped lane") + }; + let (zero, one) = (cell.members()[0].subject, cell.members()[1].subject); + if refuses { + assert_eq!( + run.refused, + vec![ConstituentRefusal { + subject: zero, + reason: ConstituentRefusalReason::ProposalUndecodable, + }], + "{site}" + ); + assert_eq!(run.submissions.len(), 1, "{site}"); + assert_eq!(run.submissions[0].subject, one, "{site}"); + } else { + assert!(run.refused.is_empty(), "{site}: {:?}", run.refused); + assert_eq!(run.submissions.len(), 2, "{site}"); + assert!( + run.needs.iter().any(|need| need.detail.contains("c0: no route")), + "{site}: the need was not recorded" + ); + } + drop(runner); + drop(mailbox); + task.await.unwrap(); + } + } + + /// A call that names no handle is reported by tool name alone, and a + /// member that finished with no choice behind it is refused, not + /// declined; a member that decided to finish still declines. + #[tokio::test] + async fn a_call_naming_no_handle_is_reported_and_refuses_the_members_it_may_have_meant() { + for finishes in [false, true] { + let (_directory, mailbox, task) = active_cell_mailbox(vec![ + NewController::OperationalAgent, + NewController::OperationalAgent, + ]) + .await; + let cell = one_group(&mailbox).await; + let mut events = vec![ + speak_call("c0", "c0__speak", "Hold the bridge."), + InferenceEvent::ToolCall { + call_id: "stray".into(), + name: "c9__speak".into(), + arguments: json!({ "text": "SENTINEL-ARGUMENT" }).to_string(), + }, + ]; + if finishes { + events.push(InferenceEvent::ToolCall { + call_id: "c1".into(), + name: "c1__finish_without_proposal".into(), + arguments: "{}".into(), + }); + } + let (runner, _store) = grouped_runner( + &mailbox, + vec![ + output(events, "stray-one"), + output(vec![InferenceEvent::Text("Nothing further.".into())], "stray-two"), + ], + ); + let CellRun::Grouped(run) = runner.run_cell(&cell).await.unwrap() else { + panic!("a coarse cell did not run the grouped lane") + }; + assert_eq!(run.unattributed, vec!["c9__speak".to_owned()]); + assert!( + !format!("{run:?}").contains("SENTINEL-ARGUMENT"), + "the call's arguments were kept" + ); + if finishes { + assert!(run.refused.is_empty(), "{:?}", run.refused); + assert_eq!(run.submissions.len(), 2); + } else { + assert_eq!(run.submissions.len(), 1); + assert_eq!( + run.refused, + vec![ConstituentRefusal { + subject: cell.members()[1].subject, + reason: ConstituentRefusalReason::CallUnattributed, + }] + ); + } + drop(runner); + drop(mailbox); + task.await.unwrap(); + } + } + + /// The call's arguments are not kept at all: only the name is. + #[test] + fn the_name_of_a_call_naming_no_handle_is_kept_and_its_arguments_are_not() { + let mut fold = GroupedFold::new(); + grouped_tool_result(&[], &mut fold, "c9__speak", r#"{"text":"SENTINEL-ARGUMENT"}"#) + .unwrap(); + assert_eq!(fold.unattributed, vec!["c9__speak".to_string()]); + } + /// Verification 20b. No spelling of a handle outside the cell decodes, and /// the index-to-subject mapping is scheduler-owned rather than /// model-supplied: a shared tool with a `subject` argument would be the @@ -15160,13 +15617,8 @@ mod tests { run.submissions[0].submission, SubmissionDisposition::Completed(_) )); - assert!( - run.needs - .iter() - .any(|need| need.detail.contains("scope changed")), - "the refused act was not recorded: {:?}", - run.needs - ); + assert_eq!(run.refused.len(), 1, "the refused act was not recorded"); + assert_eq!(run.refused[0].reason, ConstituentRefusalReason::ScopeChanged); assert_eq!(mailbox.operator_log().await.unwrap().len(), 1); drop(runner); @@ -15334,12 +15786,9 @@ mod tests { panic!("a coarse cell did not run the grouped lane") }; assert_eq!(run.submissions.len(), 1); - assert!( - run.needs - .iter() - .any(|need| need.detail.contains("scope changed")), - "{:?}", - run.needs + assert_eq!( + run.refused.iter().map(|entry| (entry.subject, entry.reason)).collect::>(), + vec![(second, ConstituentRefusalReason::ScopeChanged)] ); let after = digest_of(&mailbox.snapshot().await.unwrap(), second) @@ -15476,7 +15925,8 @@ mod tests { panic!("a coarse cell did not run the grouped lane") }; assert!(run.pending.is_none()); - assert_eq!(run.submissions.len(), 3); + assert_eq!(run.submissions.len(), 1); + assert_eq!(run.refused.len(), 2); let live = mailbox.snapshot().await.unwrap(); assert_eq!(mailbox.operator_log().await.unwrap().len(), 1); @@ -19131,4 +19581,1141 @@ mod tests { drop(mailbox); task.await.unwrap(); } + + /// The lookup-first tests use two subjects in one room, so one subject's + /// act moves the other's scope digest as well as its own. + fn fresh_opportunity(snapshot: &WorldSnapshot, subject: SubjectId) -> DecisionOpportunity { + snapshot + .opportunities + .iter() + .find(|opportunity| opportunity.scope.subject_id == subject) + .expect("the subject still holds an opportunity") + .clone() + } + + fn operational_speaks(text: &str, receipt: &str) -> Result { + output( + vec![InferenceEvent::ToolCall { + call_id: "speak".into(), + name: INTERPRETER_SPEAK_TOOL.into(), + arguments: json!({ "text": text }).to_string(), + }], + receipt, + ) + } + + fn narrative_turn_outputs( + tag: &str, + speech: &str, + ) -> Vec> { + vec![ + output( + vec![InferenceEvent::Text(format!("{tag} watches the water."))], + &format!("{tag}-projector"), + ), + output( + vec![InferenceEvent::Text(format!("I say, \"{speech}\""))], + &format!("{tag}-persona"), + ), + output( + vec![ + InferenceEvent::ToolCall { + call_id: "call_speak".into(), + name: INTERPRETER_SPEAK_TOOL.into(), + arguments: json!({ "source_quote": speech }).to_string(), + }, + InferenceEvent::ToolCall { + call_id: "call_finish".into(), + name: FINISH_INTERPRETATION_TOOL.into(), + arguments: "{}".into(), + }, + ], + &format!("{tag}-interpreter"), + ), + ] + } + + #[tokio::test] + async fn a_committed_operational_row_is_answered_with_its_receipt_after_its_scope_moved() { + let (_directory, mailbox, task) = active_cell_mailbox(vec![ + NewController::OperationalAgent, + NewController::OperationalAgent, + ]) + .await; + // Exactly two provider answers: a third call would panic the port. + let (runner, _store) = grouped_runner( + &mailbox, + vec![ + operational_speaks("The hinge is flooding.", "first"), + operational_speaks("Then we close the span.", "second"), + ], + ); + let snapshot = mailbox.snapshot().await.unwrap(); + let first = snapshot.opportunities[0].clone(); + let second = snapshot.opportunities[1].clone(); + let first_command = CommandId::new(); + let OperationalRun::Completed(committed) = runner + .run_operational(first_command, &first) + .await + .unwrap() + else { + panic!("the first act remained pending") + }; + let SubmissionDisposition::Completed(SubmitReceipt::Applied(original)) = + committed.submission + else { + panic!("the first act did not commit") + }; + // Another subject speaks in the commons. + let after_first = mailbox.snapshot().await.unwrap(); + let second_now = fresh_opportunity(&after_first, second.scope.subject_id); + assert!(matches!( + runner + .run_operational(CommandId::new(), &second_now) + .await + .unwrap(), + OperationalRun::Completed(_) + )); + + let moved = mailbox.snapshot().await.unwrap(); + let fresh = fresh_opportunity(&moved, first.scope.subject_id); + assert_ne!(fresh.scope_digest, first.scope_digest, "the scope did not move"); + let OperationalRun::Completed(answered) = runner + .run_operational(first_command, &fresh) + .await + .unwrap() + else { + panic!("the committed row was not answered") + }; + assert!( + matches!( + &answered.submission, + SubmissionDisposition::PreviouslyConfirmed(receipt) if *receipt == original + ), + "{:?}", + answered.submission + ); + assert_eq!(mailbox.operator_log().await.unwrap().len(), 2); + + drop(runner); + drop(mailbox); + task.await.unwrap(); + } + + #[tokio::test] + async fn a_committed_narrative_row_is_answered_with_its_receipt_after_its_scope_moved() { + let (_directory, mailbox, task) = active_cell_mailbox(vec![ + NewController::NarrativePersona, + NewController::NarrativePersona, + ]) + .await; + let mut outputs = narrative_turn_outputs("Mara", "The rain has teeth tonight."); + outputs.extend(narrative_turn_outputs("Tobin", "Then we close the span.")); + let (runner, _store) = grouped_runner(&mailbox, outputs); + let snapshot = mailbox.snapshot().await.unwrap(); + let first = snapshot.opportunities[0].clone(); + let second = snapshot.opportunities[1].clone(); + let first_command = CommandId::new(); + let NarrativeRun::Completed(committed) = runner + .run_narrative(first_command, &first) + .await + .unwrap() + else { + panic!("the first turn remained pending") + }; + let SubmissionDisposition::Completed(SubmitReceipt::Applied(original)) = + committed.submission + else { + panic!("the first turn did not commit") + }; + let after_first = mailbox.snapshot().await.unwrap(); + let second_now = fresh_opportunity(&after_first, second.scope.subject_id); + assert!(matches!( + runner + .run_narrative(CommandId::new(), &second_now) + .await + .unwrap(), + NarrativeRun::Completed(_) + )); + + let moved = mailbox.snapshot().await.unwrap(); + let fresh = fresh_opportunity(&moved, first.scope.subject_id); + assert_ne!(fresh.scope_digest, first.scope_digest, "the scope did not move"); + let NarrativeRun::Completed(answered) = runner + .run_narrative(first_command, &fresh) + .await + .unwrap() + else { + panic!("the committed row was not answered") + }; + assert!( + matches!( + &answered.submission, + SubmissionDisposition::PreviouslyConfirmed(receipt) if *receipt == original + ), + "{:?}", + answered.submission + ); + assert_eq!(mailbox.operator_log().await.unwrap().len(), 2); + + drop(runner); + drop(mailbox); + task.await.unwrap(); + } + + #[tokio::test] + async fn a_row_in_flight_whose_scope_moved_is_refused_before_inference() { + let (_directory, mailbox, task) = active_cell_mailbox(vec![ + NewController::OperationalAgent, + NewController::OperationalAgent, + ]) + .await; + // The first call is pending, the second commits, and a third would + // panic the port: the refused rerun must not reach the provider. + let (runner, _store) = grouped_runner( + &mailbox, + vec![ + Err(InferenceFault::retryable("the provider is busy")), + operational_speaks("Then we close the span.", "second"), + ], + ); + let snapshot = mailbox.snapshot().await.unwrap(); + let first = snapshot.opportunities[0].clone(); + let second = snapshot.opportunities[1].clone(); + let first_command = CommandId::new(); + assert!(matches!( + runner + .run_operational(first_command, &first) + .await + .unwrap(), + OperationalRun::Pending(_) + )); + assert!(matches!( + runner + .run_operational(CommandId::new(), &second) + .await + .unwrap(), + OperationalRun::Completed(_) + )); + let moved = mailbox.snapshot().await.unwrap(); + let fresh = fresh_opportunity(&moved, first.scope.subject_id); + assert_ne!(fresh.scope_digest, first.scope_digest, "the scope did not move"); + let error = runner + .run_operational(first_command, &fresh) + .await + .unwrap_err(); + assert!(matches!(error, ControllerError::OpportunityMismatch)); + assert_eq!(mailbox.operator_log().await.unwrap().len(), 1); + + drop(runner); + drop(mailbox); + task.await.unwrap(); + } + + #[tokio::test] + async fn a_committed_row_is_never_read_for_another_decision() { + let (_directory, mailbox, task) = active_cell_mailbox(vec![ + NewController::OperationalAgent, + NewController::OperationalAgent, + ]) + .await; + let (runner, _store) = grouped_runner( + &mailbox, + vec![operational_speaks("The hinge is flooding.", "first")], + ); + let snapshot = mailbox.snapshot().await.unwrap(); + let first = snapshot.opportunities[0].clone(); + let other = snapshot.opportunities[1].clone(); + let command = CommandId::new(); + assert!(matches!( + runner.run_operational(command, &first).await.unwrap(), + OperationalRun::Completed(_) + )); + let error = runner.run_operational(command, &other).await.unwrap_err(); + assert!(matches!(error, ControllerError::OpportunityMismatch)); + let mut foreign = first.clone(); + foreign.world_id = WorldId::issue(); + let error = runner.run_operational(command, &foreign).await.unwrap_err(); + assert!(matches!(error, ControllerError::OpportunityMismatch)); + + drop(runner); + drop(mailbox); + task.await.unwrap(); + } + + #[tokio::test] + async fn a_grouped_member_the_world_refuses_is_named_in_refused() { + let (_directory, mailbox, task) = active_cell_mailbox(vec![ + NewController::OperationalAgent, + NewController::OperationalAgent, + ]) + .await; + let cell = one_group(&mailbox).await; + let Cell::Group { members, .. } = &cell else { + panic!("the cell is not coarse") + }; + let (runner, _store) = grouped_runner( + &mailbox, + vec![output( + vec![ + speak_call("c0", "c0__speak", "The hinge is flooding."), + speak_call("c1", "c1__speak", "Then we close the span."), + ], + "two-acts", + )], + ); + let CellRun::Grouped(run) = runner.run_cell(&cell).await.unwrap() else { + panic!("a coarse cell did not run the grouped lane") + }; + assert_eq!(run.submissions.len(), 1); + let committed = run.submissions[0].subject; + let refused_subject = members + .iter() + .map(|member| member.subject) + .find(|subject| *subject != committed) + .unwrap(); + assert_eq!( + run.refused, + vec![ConstituentRefusal { + subject: refused_subject, + reason: ConstituentRefusalReason::ScopeChanged, + }] + ); + assert!(run.needs.is_empty(), "{:?}", run.needs); + + drop(runner); + drop(mailbox); + task.await.unwrap(); + } + + #[tokio::test] + async fn a_grouped_member_that_left_the_cell_is_named_in_refused() { + let (_directory, mailbox, task) = active_cell_mailbox(vec![ + NewController::OperationalAgent, + NewController::OperationalAgent, + ]) + .await; + let cell = one_group(&mailbox).await; + let Cell::Group { id, tick, members } = cell else { + panic!("the cell is not coarse") + }; + // One member's bound opportunity is stale (its digest is not the + // world's); the other is bound to the world as it is. + let world = mailbox.snapshot().await.unwrap(); + let leaver = members[1].subject; + let mut stale = members[1].opportunity.clone(); + stale.scope_digest = members[0].opportunity.scope_digest.clone(); + let bound = vec![ + Constituent { + subject: members[0].subject, + opportunity: fresh_opportunity(&world, members[0].subject), + }, + Constituent { + subject: leaver, + opportunity: stale, + }, + ]; + let (runner, _store) = grouped_runner( + &mailbox, + vec![output( + vec![speak_call("c0", "c0__speak", "The hinge is flooding.")], + "lone-act", + )], + ); + let cell = Cell::Group { + id, + tick, + members: bound, + }; + let CellRun::Grouped(run) = runner.run_cell(&cell).await.unwrap() else { + panic!("a coarse cell did not run the grouped lane") + }; + assert_eq!( + run.refused, + vec![ConstituentRefusal { + subject: leaver, + reason: ConstituentRefusalReason::LeftBeforeSelection, + }] + ); + assert!(run.needs.is_empty(), "{:?}", run.needs); + + drop(runner); + drop(mailbox); + task.await.unwrap(); + } + + #[tokio::test] + async fn a_grouped_cell_submits_exercises_in_attention_order() { + let (_directory, mailbox, task) = active_cell_mailbox(vec![ + NewController::OperationalAgent, + NewController::OperationalAgent, + ]) + .await; + let both_act = |tag: &str| { + output( + vec![ + speak_call("c0", "c0__speak", "The hinge is flooding."), + speak_call("c1", "c1__speak", "Then we close the span."), + ], + tag, + ) + }; + let (runner, _store) = grouped_runner(&mailbox, vec![both_act("one"), both_act("two")]); + + let first_cell = one_group(&mailbox).await; + let Cell::Group { members, .. } = &first_cell else { + panic!("the cell is not coarse") + }; + let low = members[0].subject; + let CellRun::Grouped(first) = runner.run_cell(&first_cell).await.unwrap() else { + panic!("a coarse cell did not run the grouped lane") + }; + assert_eq!(first.submissions.len(), 1); + assert_eq!(first.refused.len(), 1); + let refused_first = first.refused[0].subject; + assert_eq!(first.submissions[0].subject, low); + assert_ne!(refused_first, low); + + // Time passes; the refused member has waited, so the world attends it first. + let owner = PrincipalId::new("owner"); + let before = mailbox.snapshot().await.unwrap(); + mailbox + .submit_fixture( + CommandEnvelope { + id: CommandId::new(), + world_id: before.world_id, + expected_revision: before.revision, + caller: CallerId::Principal(owner.clone()), + body: CommandBody::AdvanceTime { + minutes: crate::TickMinutes::new(60).unwrap(), + }, + }, + &AuthenticatedCaller::fixture(CallerId::Principal(owner)), + ) + .await + .unwrap(); + let second_cell = one_group(&mailbox).await; + let CellRun::Grouped(second) = runner.run_cell(&second_cell).await.unwrap() else { + panic!("a coarse cell did not run the grouped lane") + }; + assert_eq!(second.submissions.len(), 1); + assert_eq!( + second.submissions[0].subject, refused_first, + "the member refused last time did not land first this time" + ); + + drop(runner); + drop(mailbox); + task.await.unwrap(); + } + + /// The same rows as the store sees them while its custody is uncertain. + struct UncertainStore(Arc); + + #[async_trait] + impl ControllerWorkStore for UncertainStore { + async fn lookup( + &self, + command_id: CommandId, + ) -> Result { + Ok(match self.0.lookup(command_id).await? { + ControllerWorkLookup::Confirmed(work) => ControllerWorkLookup::CustodyUncertain(work), + other => other, + }) + } + + async fn persist( + &self, + work: &ControllerWork, + ) -> Result { + self.0.persist(work).await + } + + async fn custody_probe(&self) -> Result { + self.0.custody_probe().await + } + + async fn elaboration_in_flight( + &self, + ) -> Result, ControllerWorkStoreError> { + self.0.elaboration_in_flight().await + } + } + + fn uncertain_runner(mailbox: &WorldMailbox, store: &Arc) -> ControllerRunner { + let port = Arc::new(RecordingPort { + outputs: Mutex::new(Vec::new()), + persisted_before_interpreter: Arc::new(AtomicBool::new(true)), + }); + ControllerRunner::open(mailbox.clone(), port, Arc::new(UncertainStore(store.clone())), models()) + .expect("the fixture ports open") + } + + #[tokio::test] + async fn a_row_read_in_uncertain_custody_is_reported_pending_for_its_own_decision_only() { + let (_directory, mailbox, task) = active_cell_mailbox(vec![ + NewController::OperationalAgent, + NewController::OperationalAgent, + NewController::NarrativePersona, + NewController::NarrativePersona, + ]) + .await; + let snapshot = mailbox.snapshot().await.unwrap(); + let by_mode = |mode: ControllerMode| { + snapshot + .opportunities + .iter() + .filter(|opportunity| opportunity.controller_mode == mode) + .cloned() + .collect::>() + }; + let operational = by_mode(ControllerMode::OperationalAgent); + let narrative = by_mode(ControllerMode::NarrativePersona); + let mut outputs = vec![operational_speaks("The hinge is flooding.", "operational")]; + outputs.extend(narrative_turn_outputs("Mara", "The rain has teeth tonight.")); + outputs.push(operational_speaks("Then we close the span.", "other")); + let (runner, store) = grouped_runner(&mailbox, outputs); + let operational_command = CommandId::new(); + let narrative_command = CommandId::new(); + runner + .run_operational(operational_command, &operational[0]) + .await + .unwrap(); + // The operational act moved the narrative subject's scope: bind it as + // the world now holds it. + let narrative_bound = fresh_opportunity( + &mailbox.snapshot().await.unwrap(), + narrative[0].scope.subject_id, + ); + runner + .run_narrative(narrative_command, &narrative_bound) + .await + .unwrap(); + // Another subject speaks in the commons, so every scope moves. + let other_bound = fresh_opportunity( + &mailbox.snapshot().await.unwrap(), + operational[1].scope.subject_id, + ); + runner + .run_operational(CommandId::new(), &other_bound) + .await + .unwrap(); + + let uncertain = uncertain_runner(&mailbox, &store); + let moved = mailbox.snapshot().await.unwrap(); + let operational_now = fresh_opportunity(&moved, operational[0].scope.subject_id); + let narrative_now = fresh_opportunity(&moved, narrative[0].scope.subject_id); + assert_ne!(operational_now.scope_digest, operational[0].scope_digest); + assert_ne!(narrative_now.scope_digest, narrative_bound.scope_digest); + let OperationalRun::Pending(pending) = uncertain + .run_operational(operational_command, &operational_now) + .await + .unwrap() + else { + panic!("an operational row in uncertain custody was not reported pending") + }; + assert_eq!(pending.reason(), ControllerPendingReason::StoreReopenRequired); + let NarrativeRun::Pending(pending) = uncertain + .run_narrative(narrative_command, &narrative_now) + .await + .unwrap() + else { + panic!("a narrative row in uncertain custody was not reported pending") + }; + assert_eq!(pending.reason(), ControllerPendingReason::StoreReopenRequired); + + let other_operational = fresh_opportunity(&moved, operational[1].scope.subject_id); + let other_narrative = fresh_opportunity(&moved, narrative[1].scope.subject_id); + assert!(matches!( + uncertain + .run_operational(operational_command, &other_operational) + .await, + Err(ControllerError::OpportunityMismatch) + )); + assert!(matches!( + uncertain + .run_narrative(narrative_command, &other_narrative) + .await, + Err(ControllerError::OpportunityMismatch) + )); + + drop((runner, uncertain)); + drop(mailbox); + task.await.unwrap(); + } + + #[tokio::test] + async fn a_narrative_row_in_flight_whose_scope_moved_is_refused_before_inference() { + let (_directory, mailbox, task) = active_cell_mailbox(vec![ + NewController::NarrativePersona, + NewController::NarrativePersona, + ]) + .await; + // The first turn's projector call is pending; the second subject's turn + // commits; a further call would panic the port. + let mut outputs = vec![Err(InferenceFault::retryable("the provider is busy"))]; + outputs.extend(narrative_turn_outputs("Tobin", "Then we close the span.")); + let (runner, _store) = grouped_runner(&mailbox, outputs); + let snapshot = mailbox.snapshot().await.unwrap(); + let first = snapshot.opportunities[0].clone(); + let second = snapshot.opportunities[1].clone(); + let command = CommandId::new(); + assert!(matches!( + runner.run_narrative(command, &first).await.unwrap(), + NarrativeRun::Pending(_) + )); + assert!(matches!( + runner.run_narrative(CommandId::new(), &second).await.unwrap(), + NarrativeRun::Completed(_) + )); + let moved = mailbox.snapshot().await.unwrap(); + let fresh = fresh_opportunity(&moved, first.scope.subject_id); + assert_ne!(fresh.scope_digest, first.scope_digest, "the scope did not move"); + let error = runner.run_narrative(command, &fresh).await.unwrap_err(); + assert!(matches!(error, ControllerError::OpportunityMismatch)); + assert_eq!(mailbox.operator_log().await.unwrap().len(), 1); + + drop(runner); + drop(mailbox); + task.await.unwrap(); + } + + fn operational_declines(receipt: &str) -> Result { + output( + vec![ + InferenceEvent::ToolCall { + call_id: "need".into(), + name: RECORD_NEED_TOOL.into(), + arguments: json!({"detail":"No current intervention is warranted."}) + .to_string(), + }, + InferenceEvent::ToolCall { + call_id: "finish".into(), + name: FINISH_WITHOUT_PROPOSAL_TOOL.into(), + arguments: "{}".into(), + }, + ], + receipt, + ) + } + + fn narrative_declines(tag: &str) -> Vec> { + let source = format!("{tag} lets the question pass."); + vec![ + output( + vec![InferenceEvent::Text(format!("{tag} watches the water."))], + &format!("{tag}-projector"), + ), + output( + vec![InferenceEvent::Text(source.clone())], + &format!("{tag}-persona"), + ), + output( + vec![ + InferenceEvent::ToolCall { + call_id: "gap".into(), + name: INTERPRETER_RECORD_GAP_TOOL.into(), + arguments: json!({ + "kind":"unresolved", + "source_quote":source, + "detail":"The prose expresses no supported world action." + }) + .to_string(), + }, + InferenceEvent::ToolCall { + call_id: "finish".into(), + name: FINISH_INTERPRETATION_TOOL.into(), + arguments: "{}".into(), + }, + ], + &format!("{tag}-interpreter"), + ), + ] + } + + #[tokio::test] + async fn a_committed_operational_decline_is_answered_with_its_receipt_after_its_scope_moved() { + let (_directory, mailbox, task) = active_cell_mailbox(vec![ + NewController::OperationalAgent, + NewController::OperationalAgent, + ]) + .await; + // Exactly two provider answers: a third call would panic the port. + let (runner, _store) = grouped_runner( + &mailbox, + vec![ + operational_declines("first"), + operational_speaks("Then we close the span.", "second"), + ], + ); + let snapshot = mailbox.snapshot().await.unwrap(); + let first = snapshot.opportunities[0].clone(); + let second = snapshot.opportunities[1].clone(); + let first_command = CommandId::new(); + let OperationalRun::Completed(committed) = runner + .run_operational(first_command, &first) + .await + .unwrap() + else { + panic!("the first decline remained pending") + }; + let SubmissionDisposition::NoProposal(SubmitReceipt::Applied(original)) = + committed.submission + else { + panic!("the first turn did not commit a decline") + }; + let second_now = + fresh_opportunity(&mailbox.snapshot().await.unwrap(), second.scope.subject_id); + assert!(matches!( + runner + .run_operational(CommandId::new(), &second_now) + .await + .unwrap(), + OperationalRun::Completed(_) + )); + let fresh = fresh_opportunity(&mailbox.snapshot().await.unwrap(), first.scope.subject_id); + assert_ne!(fresh.scope_digest, first.scope_digest, "the scope did not move"); + let OperationalRun::Completed(answered) = runner + .run_operational(first_command, &fresh) + .await + .unwrap() + else { + panic!("the committed decline was not answered") + }; + assert!( + matches!( + &answered.submission, + SubmissionDisposition::NoProposal(SubmitReceipt::AlreadyApplied(receipt)) + if *receipt == original + ), + "{:?}", + answered.submission + ); + assert_eq!(mailbox.operator_log().await.unwrap().len(), 1); + + drop(runner); + drop(mailbox); + task.await.unwrap(); + } + + #[tokio::test] + async fn a_committed_narrative_decline_is_answered_with_its_receipt_after_its_scope_moved() { + let (_directory, mailbox, task) = active_cell_mailbox(vec![ + NewController::NarrativePersona, + NewController::NarrativePersona, + ]) + .await; + let mut outputs = narrative_declines("Mara"); + outputs.extend(narrative_turn_outputs("Tobin", "Then we close the span.")); + let (runner, _store) = grouped_runner(&mailbox, outputs); + let snapshot = mailbox.snapshot().await.unwrap(); + let first = snapshot.opportunities[0].clone(); + let second = snapshot.opportunities[1].clone(); + let first_command = CommandId::new(); + let NarrativeRun::Completed(committed) = runner + .run_narrative(first_command, &first) + .await + .unwrap() + else { + panic!("the first decline remained pending") + }; + let SubmissionDisposition::NoProposal(SubmitReceipt::Applied(original)) = + committed.submission + else { + panic!("the first turn did not commit a decline") + }; + let second_now = + fresh_opportunity(&mailbox.snapshot().await.unwrap(), second.scope.subject_id); + assert!(matches!( + runner + .run_narrative(CommandId::new(), &second_now) + .await + .unwrap(), + NarrativeRun::Completed(_) + )); + let fresh = fresh_opportunity(&mailbox.snapshot().await.unwrap(), first.scope.subject_id); + assert_ne!(fresh.scope_digest, first.scope_digest, "the scope did not move"); + let NarrativeRun::Completed(answered) = runner + .run_narrative(first_command, &fresh) + .await + .unwrap() + else { + panic!("the committed decline was not answered") + }; + assert!( + matches!( + &answered.submission, + SubmissionDisposition::NoProposal(SubmitReceipt::AlreadyApplied(receipt)) + if *receipt == original + ), + "{:?}", + answered.submission + ); + assert_eq!(mailbox.operator_log().await.unwrap().len(), 1); + + drop(runner); + drop(mailbox); + task.await.unwrap(); + } + + /// An in-flight operational row read in uncertain custody after its scope + /// moved is refused as a mismatch, not reported pending; read against its + /// own scope it is pending. + #[tokio::test] + async fn an_operational_row_in_flight_in_uncertain_custody_keeps_the_digest_gate() { + let (_directory, mailbox, task) = active_cell_mailbox(vec![ + NewController::OperationalAgent, + NewController::OperationalAgent, + ]) + .await; + let (runner, store) = grouped_runner( + &mailbox, + vec![ + Err(InferenceFault::retryable("the provider is busy")), + operational_speaks("Then we close the span.", "second"), + ], + ); + let snapshot = mailbox.snapshot().await.unwrap(); + let first = snapshot.opportunities[0].clone(); + let second = snapshot.opportunities[1].clone(); + let command = CommandId::new(); + assert!(matches!( + runner.run_operational(command, &first).await.unwrap(), + OperationalRun::Pending(_) + )); + assert!(matches!( + runner + .run_operational(CommandId::new(), &second) + .await + .unwrap(), + OperationalRun::Completed(_) + )); + let fresh = fresh_opportunity(&mailbox.snapshot().await.unwrap(), first.scope.subject_id); + assert_ne!(fresh.scope_digest, first.scope_digest, "the scope did not move"); + let uncertain = uncertain_runner(&mailbox, &store); + assert!(matches!( + uncertain.run_operational(command, &fresh).await, + Err(ControllerError::OpportunityMismatch) + )); + let OperationalRun::Pending(pending) = + uncertain.run_operational(command, &first).await.unwrap() + else { + panic!("a row bound to the caller's own scope was not reported pending") + }; + assert_eq!(pending.reason(), ControllerPendingReason::StoreReopenRequired); + + drop((runner, uncertain)); + drop(mailbox); + task.await.unwrap(); + } + + #[tokio::test] + async fn a_narrative_row_in_flight_in_uncertain_custody_keeps_the_digest_gate() { + let (_directory, mailbox, task) = active_cell_mailbox(vec![ + NewController::NarrativePersona, + NewController::NarrativePersona, + ]) + .await; + let mut outputs = vec![Err(InferenceFault::retryable("the provider is busy"))]; + outputs.extend(narrative_turn_outputs("Tobin", "Then we close the span.")); + let (runner, store) = grouped_runner(&mailbox, outputs); + let snapshot = mailbox.snapshot().await.unwrap(); + let first = snapshot.opportunities[0].clone(); + let second = snapshot.opportunities[1].clone(); + let command = CommandId::new(); + assert!(matches!( + runner.run_narrative(command, &first).await.unwrap(), + NarrativeRun::Pending(_) + )); + assert!(matches!( + runner + .run_narrative(CommandId::new(), &second) + .await + .unwrap(), + NarrativeRun::Completed(_) + )); + let fresh = fresh_opportunity(&mailbox.snapshot().await.unwrap(), first.scope.subject_id); + assert_ne!(fresh.scope_digest, first.scope_digest, "the scope did not move"); + let uncertain = uncertain_runner(&mailbox, &store); + assert!(matches!( + uncertain.run_narrative(command, &fresh).await, + Err(ControllerError::OpportunityMismatch) + )); + let NarrativeRun::Pending(pending) = + uncertain.run_narrative(command, &first).await.unwrap() + else { + panic!("a row bound to the caller's own scope was not reported pending") + }; + assert_eq!(pending.reason(), ControllerPendingReason::StoreReopenRequired); + + drop((runner, uncertain)); + drop(mailbox); + task.await.unwrap(); + } + + /// A provider port that severs the world's owner task once the answer is + /// in, so the next read of the world finds the mailbox unavailable. The + /// fixture's only fault injection: production code has no such seam. + struct SeversWorldAfterAnswer { + inner: RecordingPort, + world: tokio::task::AbortHandle, + } + + #[async_trait] + impl InferencePort for SeversWorldAfterAnswer { + fn prepare(&self, request: InferenceRequest) -> Result { + self.inner.prepare(request) + } + + async fn infer( + &self, + request: PreparedInference, + ) -> Result { + let answer = self.inner.infer(request).await; + self.world.abort(); + tokio::task::yield_now().await; + answer + } + } + + #[tokio::test] + async fn a_grouped_cell_whose_world_cannot_be_read_at_submission_is_pending_with_nothing_submitted() + { + let (_directory, mailbox, task) = active_cell_mailbox(vec![ + NewController::OperationalAgent, + NewController::OperationalAgent, + ]) + .await; + let both_act = output( + vec![ + speak_call("c0", "c0__speak", "The hinge is flooding."), + speak_call("c1", "c1__speak", "Then we close the span."), + ], + "one", + ); + let persisted = Arc::new(AtomicBool::new(true)); + let port = Arc::new(SeversWorldAfterAnswer { + inner: RecordingPort { + outputs: Mutex::new(vec![both_act]), + persisted_before_interpreter: persisted.clone(), + }, + world: task.abort_handle(), + }); + let store = Arc::new(RecordingWorkStore { + persisted, + work: Mutex::new(BTreeMap::new()), + }); + let runner = ControllerRunner::open(mailbox.clone(), port, store.clone(), models()) + .expect("the fixture ports open"); + let cell = one_group(&mailbox).await; + let CellRun::Grouped(run) = runner.run_cell(&cell).await.unwrap() else { + panic!("a coarse cell did not run the grouped lane") + }; + assert_eq!(run.pending, Some(ControllerPendingReason::WorldUnavailable)); + assert!(run.submissions.is_empty(), "{:?}", run.submissions); + assert!(run.refused.is_empty(), "{:?}", run.refused); + assert!( + !store.work.lock().unwrap().is_empty(), + "the cell's row was not kept for the resume" + ); + + drop(runner); + drop(mailbox); + assert!(task.await.unwrap_err().is_cancelled()); + } + + /// A store that answers the write of a row that awaits the world with + /// uncertain custody, so the run ends pending with the submission body + /// persisted but never sent. + struct NeverSentStore { + inner: Arc, + armed: AtomicBool, + } + + #[async_trait] + impl ControllerWorkStore for NeverSentStore { + async fn lookup( + &self, + command_id: CommandId, + ) -> Result { + self.inner.lookup(command_id).await + } + + async fn persist( + &self, + work: &ControllerWork, + ) -> Result { + let written = self.inner.persist(work).await?; + let awaiting = match work { + ControllerWork::Narrative(row) => row.awaits_world(), + ControllerWork::Operational(row) => row.awaits_world(), + _ => false, + }; + if awaiting && self.armed.load(Ordering::SeqCst) { + return Ok(ControllerWorkWrite::CustodyUncertain); + } + Ok(written) + } + + async fn custody_probe(&self) -> Result { + self.inner.custody_probe().await + } + + async fn elaboration_in_flight( + &self, + ) -> Result, ControllerWorkStoreError> { + self.inner.elaboration_in_flight().await + } + } + + fn runner_over( + mailbox: &WorldMailbox, + outputs: Vec>, + store: Arc, + ) -> (ControllerRunner, Arc) { + let port = Arc::new(RecordingPort { + outputs: Mutex::new(outputs), + persisted_before_interpreter: Arc::new(AtomicBool::new(true)), + }); + ( + ControllerRunner::open(mailbox.clone(), port.clone(), store, models()) + .expect("the fixture ports open"), + port, + ) + } + + fn recording_store() -> Arc { + Arc::new(RecordingWorkStore { + persisted: Arc::new(AtomicBool::new(true)), + work: Mutex::new(BTreeMap::new()), + }) + } + + /// An operational row that awaits the world but never reached it, then a + /// neighbour moves the scope, then the same command is rerun against the + /// scope as it now stands. The row is submitted and the world refuses its + /// stale body: no provider call, nothing new in the log. + async fn operational_row_never_sent_then_scope_moved( + first_output: Result, + ) { + let (_directory, mailbox, task) = active_cell_mailbox(vec![ + NewController::OperationalAgent, + NewController::OperationalAgent, + ]) + .await; + let inner = recording_store(); + let never_sent = Arc::new(NeverSentStore { + inner: inner.clone(), + armed: AtomicBool::new(true), + }); + let (a, _) = runner_over(&mailbox, vec![first_output], never_sent.clone()); + let snapshot = mailbox.snapshot().await.unwrap(); + let first = snapshot.opportunities[0].clone(); + let second = snapshot.opportunities[1].clone(); + let command = CommandId::new(); + assert!(matches!( + a.run_operational(command, &first).await, + Ok(OperationalRun::Pending(_)) + )); + never_sent.armed.store(false, Ordering::SeqCst); + let (b, _) = runner_over( + &mailbox, + vec![operational_speaks("Then we close the span.", "second")], + inner.clone(), + ); + assert!(matches!( + b.run_operational(CommandId::new(), &second).await.unwrap(), + OperationalRun::Completed(_) + )); + let log_before = mailbox.operator_log().await.unwrap().len(); + let fresh = fresh_opportunity(&mailbox.snapshot().await.unwrap(), first.scope.subject_id); + assert_ne!(fresh.scope_digest, first.scope_digest, "the scope did not move"); + let (c, port) = runner_over(&mailbox, Vec::new(), inner.clone()); + assert!(matches!( + c.run_operational(command, &fresh).await, + Err(ControllerError::World(KernelError::ScopeChanged { .. })) + )); + assert_eq!(mailbox.operator_log().await.unwrap().len(), log_before); + assert!(port.outputs.lock().unwrap().is_empty()); + + drop((a, b, c)); + drop(mailbox); + task.await.unwrap(); + } + + #[tokio::test] + async fn an_operational_row_ready_to_submit_but_never_sent_is_refused_by_the_world_after_its_scope_moved() + { + operational_row_never_sent_then_scope_moved(operational_speaks( + "The hinge is flooding.", + "first", + )) + .await; + } + + #[tokio::test] + async fn an_operational_decline_never_sent_is_refused_by_the_world_after_its_scope_moved() { + operational_row_never_sent_then_scope_moved(operational_declines("first")).await; + } + + /// The narrative lane answers the same refusal with its one bounded + /// re-lowering: one Interpreter call against the scope as it now stands, + /// and the re-lowered act commits. + #[tokio::test] + async fn a_narrative_row_ready_to_submit_but_never_sent_is_lowered_once_more_after_its_scope_moved() + { + let (_directory, mailbox, task) = active_cell_mailbox(vec![ + NewController::NarrativePersona, + NewController::NarrativePersona, + ]) + .await; + let inner = recording_store(); + let never_sent = Arc::new(NeverSentStore { + inner: inner.clone(), + armed: AtomicBool::new(true), + }); + let (a, _) = runner_over( + &mailbox, + narrative_turn_outputs("Mara", "The rain has teeth tonight."), + never_sent.clone(), + ); + let snapshot = mailbox.snapshot().await.unwrap(); + let first = snapshot.opportunities[0].clone(); + let second = snapshot.opportunities[1].clone(); + let command = CommandId::new(); + assert!(matches!( + a.run_narrative(command, &first).await, + Ok(NarrativeRun::Pending(_)) + )); + never_sent.armed.store(false, Ordering::SeqCst); + let (b, _) = runner_over( + &mailbox, + narrative_turn_outputs("Tobin", "Then we close the span."), + inner.clone(), + ); + assert!(matches!( + b.run_narrative(CommandId::new(), &second).await.unwrap(), + NarrativeRun::Completed(_) + )); + let log_before = mailbox.operator_log().await.unwrap().len(); + let fresh = fresh_opportunity(&mailbox.snapshot().await.unwrap(), first.scope.subject_id); + assert_ne!(fresh.scope_digest, first.scope_digest, "the scope did not move"); + // One answer is the bound; the second would be a second round. + let relowering = || narrative_turn_outputs("Mara", "The rain has teeth tonight.").remove(2); + let (c, port) = runner_over(&mailbox, vec![relowering(), relowering()], inner.clone()); + assert!(matches!( + c.run_narrative(command, &fresh).await, + Ok(NarrativeRun::Completed(_)) + )); + assert_eq!( + port.outputs.lock().unwrap().len(), + 1, + "the re-lowering did not spend exactly one provider call" + ); + assert_eq!(mailbox.operator_log().await.unwrap().len(), log_before + 1); + + drop((a, b, c)); + drop(mailbox); + task.await.unwrap(); + } } diff --git a/crates/ghostlight/src/cover.rs b/crates/ghostlight/src/cover.rs index 02c0adb..5d1ce42 100644 --- a/crates/ghostlight/src/cover.rs +++ b/crates/ghostlight/src/cover.rs @@ -11,7 +11,6 @@ //! The cover is a partition, never a filter: every active subject is in exactly //! one cell every tick. The budget decides at what resolution, not whether. -use super::clock::FictionalMinutes; use super::{ControllerMode, DecisionOpportunity, SubjectId, WorldId}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; @@ -29,25 +28,18 @@ pub(crate) const AGENCY_STAR_THRESHOLD: usize = 16; /// the name, not by a second namespace. const COVER_NAMESPACE: &str = "ghostlight.cover.v1"; -/// Monotone tick index. Derived from the world clock, never stored. It is -/// threaded into every derived command id, so a resumed tick over the same -/// snapshot re-derives the same ids and the kernel's idempotency ledger answers -/// with the original receipts instead of double-committing. +/// A tick's ordinal: the number of clock advances the world had committed when +/// the tick opened. The driver reads it from the world and freezes it in the +/// tick document; it is never derived from `now`, so it does not depend on the +/// span. It is threaded into every derived command id, so a resumed tick +/// re-submits the same ids and the kernel's idempotency ledger answers with the +/// original receipts instead of double-committing. #[derive( Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, Default, )] #[serde(transparent)] pub struct TickIndex(pub u64); -impl TickIndex { - /// Floor division by the span one `AdvanceTime` carries. The driver submits - /// the clock *after* the cells run, so every cell in one tick sees the same - /// `now` and therefore the same index. - pub(crate) fn of(now: FictionalMinutes, tick_minutes: u32) -> Self { - Self(now.0 / u64::from(tick_minutes.max(1))) - } -} - /// How a subject was represented this tick. Derived from the cell, carried /// nowhere in world state: representation is a compute budget, not world truth. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -254,7 +246,7 @@ impl AgencyGraph { } } -/// The partition. Pure over `(world, now, opportunities, graph, budget)`. +/// The partition. Pure over `(world, tick, opportunities, graph, budget)`. /// /// `opportunities` arrives in the kernel's one attention order (pressure, then /// attention debt, then id) and is never re-sorted and never filtered here: a @@ -262,13 +254,11 @@ impl AgencyGraph { /// is a fact about cognition rather than a scheduling preference. pub fn derive_cover( world: WorldId, - now: FictionalMinutes, - tick_minutes: u32, + tick: TickIndex, opportunities: &[DecisionOpportunity], graph: &AgencyGraph, budget: CoverBudget, ) -> Cover { - let tick = TickIndex::of(now, tick_minutes); let active: Vec<&DecisionOpportunity> = opportunities .iter() .filter(|opportunity| opportunity.controller_mode != ControllerMode::Human) @@ -601,8 +591,7 @@ mod tests { ) -> Cover { derive_cover( WorldId::nil_for_test(), - FictionalMinutes(tick * 60), - 60, + TickIndex(tick), opportunities, graph, budget, diff --git a/crates/ghostlight/src/journal.rs b/crates/ghostlight/src/journal.rs index 8163d3a..a2d36fb 100644 --- a/crates/ghostlight/src/journal.rs +++ b/crates/ghostlight/src/journal.rs @@ -181,6 +181,11 @@ impl WorldJournal { self.commits.get(&command_id) } + /// Every commit the journal holds, in no particular order. + pub(super) fn commits(&self) -> impl Iterator { + self.commits.values() + } + #[cfg(test)] pub(super) fn commit_count(&self) -> usize { self.commits.len() diff --git a/crates/ghostlight/src/lib.rs b/crates/ghostlight/src/lib.rs index f373b42..3f64965 100644 --- a/crates/ghostlight/src/lib.rs +++ b/crates/ghostlight/src/lib.rs @@ -85,6 +85,7 @@ pub use consumer::{ }; pub use controllers::{ CONTROLLER_WORK_SCHEMA, CellRun, ConnectorBinding, ControllerError, ControllerModels, + ControllerOpenError, ControllerNeed, ControllerPendingReason, ControllerRunner, ControllerWork, ControllerWorkCustody, ControllerWorkLookup, ControllerWorkStore, ControllerWorkStoreError, ControllerWorkWrite, @@ -145,7 +146,10 @@ pub use table::{ pub use vault::{VaultError, VaultEvidenceSource}; pub use verse::{ CALL_RECORD_SCHEMA, CallRecordError, CallRecordStore, CreateVerseWorldIntent, ProviderCallRecord, - RecordingInferencePort, ReplayInferencePort, RunField, RunId, RunStart, VerseGenesisError, + RUN_RECORD_SCHEMA, RecordedCoverBudget, RecordingInferencePort, Refusal, RefusalReason, RunClaim, + ReplayInferencePort, RunField, RunId, RunOutcome, RunStart, RunStatus, TickCheckpoint, + TickReport, UnattributedCall, VerseGenesisError, VerseRunConfig, VerseRunError, VerseRunRecord, VerseRunStore, + call_run_id, run_manual, }; use patch::{EdgeRecord, EntityRecord, LedgerDelta, ResolvedOp, ResolvedPatch}; @@ -1275,6 +1279,21 @@ pub struct OperatorEvent { pub speaker_label: String, pub speech: Option, pub display: Option, + /// The affordance kind the act exercised. + pub verb: String, + /// How the world's grammar renders that verb; none in a world with no + /// grammar. + pub render_path: Option, + /// What the act bound each of the verb's roles to, in the invocation's + /// order. + pub roles: Vec, +} + +/// One role an act bound, with the label of what it was bound to. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BoundRole { + pub role: String, + pub target: String, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -1565,7 +1584,43 @@ impl WorldKernel { /// no consumer may read in full would be a loaded gun with a comment on it. fn operator_log(&self) -> Result, KernelError> { self.journal.ensure_healthy()?; - operator_log(&self.state) + // An event records what the world did; the invocation, which names the + // bound roles, lives on the commit that produced it. + let invocations: BTreeMap = self + .journal + .commits() + .filter_map(|commit| match &commit.effect { + WorldEffect::DecisionExercised { + invocation, event, .. + } => Some((event.id, invocation)), + _ => None, + }) + .collect(); + operator_log(&self.state, &invocations) + } + + /// The tick driver's opportunity list, in the same order a snapshot + /// carries it. Separate from `snapshot` for the reason `agency_graph` is: + /// only the tick driver reads it. + fn opportunities(&self) -> Result, KernelError> { + self.journal.ensure_healthy()?; + Ok(order_opportunities( + &self.state, + derive_opportunities(&self.state)?, + )) + } + + /// How many clock advances the world has committed: the next tick's + /// ordinal, held by the world and read from its journal. Separate from + /// `snapshot` for the reason `opportunities` is: only the tick driver + /// reads it. + fn clock_advances(&self) -> Result { + self.journal.ensure_healthy()?; + Ok(self + .journal + .commits() + .filter(|commit| matches!(commit.effect, WorldEffect::TimeAdvanced { .. })) + .count() as u64) } /// The scheduler's adjacency projection. Separate from `snapshot` for the @@ -3740,7 +3795,10 @@ fn scope_components(state: &WorldState, subject_id: SubjectId) -> ScopeComponent /// The human operator's story feed, derived from the causal record. The events /// log is not a perception surface: `knowledge` is the sole answer to what a /// subject may act on and what its projection may contain. -fn operator_log(state: &WorldState) -> Result, KernelError> { +fn operator_log( + state: &WorldState, + invocations: &BTreeMap, +) -> Result, KernelError> { state .events .iter() @@ -3749,6 +3807,30 @@ fn operator_log(state: &WorldState) -> Result, KernelError> { .subjects .get(&event.scope.subject_id) .ok_or_else(|| KernelError::Invariant("event names no live subject".into()))?; + let invocation = invocations.get(&event.id).ok_or_else(|| { + KernelError::Invariant("event has no committed invocation".into()) + })?; + let entry = state + .affordance_catalog + .get(&event.affordance) + .ok_or_else(|| KernelError::Invariant("event names no catalog entry".into()))?; + let roles = invocation + .bindings + .iter() + .map(|binding| { + let label = match binding.target { + Target::Subject(id) => state.subjects.get(&id).map(|s| s.label.clone()), + Target::Entity(id) => state.entities.get(&id).map(|e| e.label.clone()), + Target::Edge(id) => state.edges.contains_key(&id).then(|| "edge".to_owned()), + }; + label + .map(|target| BoundRole { + role: binding.role.0.clone(), + target, + }) + .ok_or_else(|| KernelError::Invariant("event binds no live referent".into())) + }) + .collect::, _>>()?; Ok(OperatorEvent { revision: event.revision, speaker: event.scope.subject_id, @@ -3761,6 +3843,13 @@ fn operator_log(state: &WorldState) -> Result, KernelError> { .display .and_then(|fact| state.facts.get(&fact)) .map(|record| record.statement.clone()), + verb: entry.kind.0.clone(), + render_path: state + .grammar + .as_ref() + .and_then(|grammar| grammar.verb(&entry.kind.0)) + .map(|verb| verb.render_path), + roles, }) }) .collect() @@ -11533,7 +11622,7 @@ mod soul_knowledge_tests { // Nobody holds it. assert!(kernel.state.knowledge.is_empty()); // The operator reads it anyway, with the speaker resolved by label. - let log = operator_log(&kernel.state).unwrap(); + let log = kernel.operator_log().unwrap(); assert_eq!( log, vec![OperatorEvent { @@ -11542,6 +11631,9 @@ mod soul_knowledge_tests { speaker_label: "The Yard Bystander".into(), speech: Some(Statement::new(text).unwrap()), display: None, + verb: "speak".into(), + render_path: None, + roles: Vec::new(), }] ); // And no subject snapshot carries it. diff --git a/crates/ghostlight/src/mailbox.rs b/crates/ghostlight/src/mailbox.rs index 76f76da..c9b232b 100644 --- a/crates/ghostlight/src/mailbox.rs +++ b/crates/ghostlight/src/mailbox.rs @@ -82,6 +82,16 @@ enum Request { AgencyGraph { reply: oneshot::Sender>, }, + /// The tick driver's opportunity list. Separate from `Snapshot` so the + /// list a cover is derived from has one door, held by the driver. + Opportunities { + reply: oneshot::Sender, KernelError>>, + }, + /// The tick ordinal's one door: how many clock advances the world has + /// committed. Held by the tick driver, as `Opportunities` is. + ClockAdvances { + reply: oneshot::Sender>, + }, ControllerReceipt { command_id: CommandId, opportunity: super::DecisionOpportunity, @@ -486,6 +496,37 @@ impl WorldMailbox { .map_err(MailboxError::Kernel) } + /// The tick driver's opportunity list, in snapshot order. It is on + /// `WorldMailbox` and on no controller port, as `agency_graph` is. + pub async fn opportunities(&self) -> Result, MailboxError> { + let (reply, response) = oneshot::channel(); + self.sender + .send(Request::Opportunities { reply }) + .await + .map_err(|_| MailboxError::Unavailable)?; + + response + .await + .map_err(|_| MailboxError::Unavailable)? + .map_err(MailboxError::Kernel) + } + + /// The tick driver's ordinal: how many clock advances the world has + /// committed. It is on `WorldMailbox` and on no controller port, as + /// `opportunities` is. + pub async fn clock_advances(&self) -> Result { + let (reply, response) = oneshot::channel(); + self.sender + .send(Request::ClockAdvances { reply }) + .await + .map_err(|_| MailboxError::Unavailable)?; + + response + .await + .map_err(|_| MailboxError::Unavailable)? + .map_err(MailboxError::Kernel) + } + pub async fn operator_log(&self) -> Result, MailboxError> { let (reply, response) = oneshot::channel(); self.sender @@ -955,6 +996,32 @@ async fn run_owner(mut owned: OwnedWorld, mut receiver: mpsc::Receiver) return; } } + Request::Opportunities { reply } => { + let result = match &owned { + OwnedWorld::Empty(_) => Err(KernelError::WorldNotCreated), + OwnedWorld::Live(kernel) => kernel.opportunities(), + }; + let owner_must_stop = result + .as_ref() + .is_err_and(KernelError::requires_owner_restart); + let _ = reply.send(result); + if owner_must_stop { + return; + } + } + Request::ClockAdvances { reply } => { + let result = match &owned { + OwnedWorld::Empty(_) => Err(KernelError::WorldNotCreated), + OwnedWorld::Live(kernel) => kernel.clock_advances(), + }; + let owner_must_stop = result + .as_ref() + .is_err_and(KernelError::requires_owner_restart); + let _ = reply.send(result); + if owner_must_stop { + return; + } + } Request::ControllerReceipt { command_id, opportunity, diff --git a/crates/ghostlight/src/verse/calls.rs b/crates/ghostlight/src/verse/calls.rs index d86a92d..079fa44 100644 --- a/crates/ghostlight/src/verse/calls.rs +++ b/crates/ghostlight/src/verse/calls.rs @@ -256,7 +256,7 @@ enum DecodedRow { Ledger(RunLedger), } -fn canonical Deserialize<'de>>( +pub(super) fn canonical Deserialize<'de>>( row: &CultCacheEnvelope, ) -> Result { let value: T = rmp_serde::from_slice(&row.payload).map_err(|_| CallRecordError::Corrupt)?; @@ -585,6 +585,16 @@ impl RecordingInferencePort { }) } + /// The run this port records. + pub fn run_id(&self) -> &RunId { + &self.run + } + + /// The token cap the run was opened under. + pub fn cap(&self) -> u64 { + self.cap + } + /// Tokens reserved or charged to the run so far. pub fn spent(&self) -> u64 { self.store.spent(&self.run).unwrap_or(u64::MAX) diff --git a/crates/ghostlight/src/verse/mod.rs b/crates/ghostlight/src/verse/mod.rs index 2600127..abef1b3 100644 --- a/crates/ghostlight/src/verse/mod.rs +++ b/crates/ghostlight/src/verse/mod.rs @@ -1,13 +1,19 @@ -//! The Aetheria verse run's own organs. The verse binary and its run record -//! live above this module; here sit the parts of a run that belong to the -//! library: the record of every provider call and the spend it implies, and the -//! genesis of the canonical verse world. +//! The Aetheria verse run's own organs. The verse binary sits above this +//! module; here sit the parts of a run that belong to the library: the record +//! of every provider call and the spend it implies, the genesis of the +//! canonical verse world, and the manual run that advances it with its record. mod calls; mod genesis; +mod run; pub use genesis::{CreateVerseWorldIntent, VerseGenesisError}; pub use calls::{ CALL_RECORD_SCHEMA, CallRecordError, CallRecordStore, ProviderCallRecord, RecordingInferencePort, ReplayInferencePort, RunField, RunId, RunStart, }; +pub use run::{ + RUN_RECORD_SCHEMA, RecordedCoverBudget, Refusal, RefusalReason, RunClaim, RunOutcome, RunStatus, + TickCheckpoint, TickReport, UnattributedCall, VerseRunConfig, VerseRunError, VerseRunRecord, VerseRunStore, + call_run_id, run_manual, +}; diff --git a/crates/ghostlight/src/verse/run.rs b/crates/ghostlight/src/verse/run.rs new file mode 100644 index 0000000..6557b61 --- /dev/null +++ b/crates/ghostlight/src/verse/run.rs @@ -0,0 +1,1162 @@ +//! The manual verse run: the tick driver and its record. +//! +//! One invocation advances the world tick by tick. A tick reads the world, +//! opens (or finds open) its tick document, runs each frozen cell through the +//! controller organ, advances the clock once by the frozen span, and +//! checkpoints. The driver owns the order of those steps and nothing else: +//! cells decide what factions do, the recording port owns spend, the kernel +//! owns time. +//! +//! A tick is named by the world's clock-advance count, never by `now`. The +//! resume point is the open tick document (world, ordinal), written before the +//! tick's first cell and read by any later run: it froze the span, the cover +//! budget and the cells when the tick opened, so a run under another span or +//! budget finishes the open tick as it was opened and applies its own config +//! from the next tick it opens. A tick whose cells did not all finish never +//! advances the clock, so the same ordinal, cells and command ids come back, +//! the controller-work rows pick up where they stopped, and the kernel answers +//! every re-submission with the original receipt. A run record reports a run +//! and is never read to resume one. +//! +//! A run starts only when it is invoked: nothing here waits, schedules or +//! repeats. + +use super::calls::{RecordingInferencePort, canonical}; +use crate::controllers::{ + CellRun, ConstituentRefusalReason, ControllerError, ControllerPendingReason, ControllerRunner, InferenceFaultClass, + NarrativeRun, OperationalRun, SubmissionDisposition, +}; +use crate::cover::{ + Cell, CellId, Constituent, CoverBudget, CoverBudgetError, TickIndex, derive_cover, +}; +use crate::{ + AgencyGraph, CommandId, DecisionOpportunity, KernelError, MailboxError, OperatorEvent, + RenderPath, + SubjectId, SubmitReceipt, TickMinutes, WorldId, WorldMailbox, WorldSnapshot, +}; +use chrono::Utc; +use cultcache_rs::{CacheBackingStore, CultCacheEnvelope, OwnedRedbMessagePackBackingStore}; +use serde::{Deserialize, Serialize}; +use std::{collections::BTreeMap, ops::ControlFlow, path::Path, sync::Mutex}; +use thiserror::Error; + +/// The run record document's schema. +pub const RUN_RECORD_SCHEMA: &str = "ghostlight.verse_run.v1"; +const RUN_ROW: &str = "verse_run.v1"; +/// The tick document's schema. +pub const TICK_RECORD_SCHEMA: &str = "ghostlight.verse_tick.v1"; +const TICK_ROW: &str = "verse_tick.v1"; +const CLOCK_NAMESPACE: &str = "ghostlight.verse.clock"; + +/// Why a run could not be recorded or started. Every variant names a kind of +/// fault; none carries a value the run held. +#[derive(Debug, Error)] +pub enum VerseRunError { + #[error("the run record store could not be read or written")] + Store, + #[error("the run record store holds a row that is not a canonical run record")] + Corrupt, + #[error("a run record's sequence is not an existing run or the next in its world")] + OutOfSequence, + #[error("the recording port belongs to another run")] + PortNotThisRun, + #[error("the run's configuration is not the one its claim recorded")] + ConfigNotThisClaim, + #[error("the world is not at the revision its claim was made from")] + WorldMovedSinceClaim, + #[error("the cover budget is not valid: {0}")] + Budget(#[from] CoverBudgetError), + #[error(transparent)] + Mailbox(#[from] MailboxError), +} + +/// Where a run stands. `Running` is only ever the record of a run in flight, +/// or of one whose process died before it could say how it ended. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum RunStatus { + Running, + Completed, + /// A provider call could pass the token cap, so the run ended before the + /// open tick's clock advance. + BudgetExhausted, + /// The run ended without finishing its open tick. `reason` is a fixed code + /// naming the kind of fault, never a value. + Failed { + reason: String, + }, + /// The caller asked the run to end after a finished tick. + Stopped, +} + +/// Why a cell committed nothing and the run went on. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum RefusalReason { + /// A narrative turn the world moved under and could not lower again. + Interrupted, + /// The world refused the cell's submission. + WorldRejected, + /// The cell's subject no longer holds an opportunity. + OpportunityGone, + /// The cell's work was bound to a scope that has since moved. Work still + /// reasoning is refused before any inference. Work whose submission body + /// was persisted but never sent is submitted and refused by the world with + /// `ScopeChanged`. A cell whose command already committed does not end + /// this way: the controller answers it with its receipt whatever the scope + /// has done since. + ScopeMoved, + /// A grouped member's own tool call could not be decoded and it proposed + /// nothing else. No turn was submitted for it. + ProposalUndecodable, + /// A resumed cell's persisted row did not hold this member. No turn was + /// submitted for it. + RowDisagreesWithCell, + /// A tool call in the member's cell named no handle and the member + /// finished with no choice of its own, so no turn was submitted for it. + CallUnattributed, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Refusal { + /// The cell's position in the tick's cover. + pub cell: u32, + /// The frozen member that committed nothing. Every member of a closed + /// tick's cells is in `committed` or here, once. + pub subject: SubjectId, + pub reason: RefusalReason, +} + +/// A tool call in a grouped cell that named no handle in it. The name is the +/// call's own, capped; its arguments are never kept. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct UnattributedCall { + /// The cell's position in the tick's cover. + pub cell: u32, + pub tool: String, +} + +/// A tick as the run left it. A closed tick has all its cells run and the +/// clock advanced by the tick's frozen span. The tick a run ended inside is +/// recorded too, unclosed, so what its cells committed is not lost; a later +/// run finds that tick's document open and works in it. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TickCheckpoint { + /// The tick's ordinal, as its tick document names it. + pub tick: u64, + pub closed: bool, + pub cells: u32, + /// Every command the tick's cells hold a receipt for, in cell order. A + /// decline is a commit, so it is listed. + pub committed: Vec, + pub refusals: Vec, + /// The calls grouped cells received that named no handle, in cell order. + pub unattributed: Vec, + /// Tokens the recording port charged during the tick. + pub tokens: u64, +} + +/// The cover budget as the record carries it. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RecordedCoverBudget { + pub cells: u16, + pub constituent_cap: u16, + pub urgency_slots: u16, +} + +/// A cell as the tick document froze it: the partition's identity and its +/// members, decided once when the tick opened. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct FrozenCell { + pub id: CellId, + /// True for a `Cell::Group`, even one a refresh has left a single member. + pub group: bool, + pub members: Vec, +} + +impl FrozenCell { + fn of(cell: &Cell) -> Self { + let (id, group) = match cell { + Cell::Singleton { id, .. } => (*id, false), + Cell::Group { id, .. } => (*id, true), + }; + Self { + id, + group, + members: cell.members().iter().map(|member| member.subject).collect(), + } + } +} + +/// What closed a tick. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum TickClose { + /// The run with this sequence submitted the tick's clock advance. + Clock { run: u64 }, + /// A later run found the world's ordinal past the tick with the document + /// still open: its clock landed and the close was lost, or another clock + /// writer moved time. + World, +} + +/// What a tick decided when it opened. Document `ghostlight.verse_tick.v1`, +/// keyed (world, ordinal). Written once before the tick's first cell, closed +/// once, never reopened or deleted. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct VerseTick { + pub world_id: WorldId, + pub ordinal: u64, + pub opened_now: u64, + pub opened_revision: u64, + pub span_minutes: u32, + pub cover_budget: RecordedCoverBudget, + pub oversubscribed: bool, + pub cells: Vec, + pub opened_by_run: u64, + pub closed: Option, +} + +/// One invocation of the verse runner: what it was asked, what it did. +/// Document `ghostlight.verse_run.v1`, keyed (world, run sequence). +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct VerseRunRecord { + pub world_id: WorldId, + pub sequence: u64, + pub token_cap: u64, + pub ticks_requested: u32, + pub tick_span_minutes: u32, + pub cover_budget: RecordedCoverBudget, + /// The world's revision when the run began. + pub start_revision: u64, + pub checkpoints: Vec, + pub status: RunStatus, +} + +impl VerseRunRecord { + /// The ticks this run finished: every cell run and the clock advanced. + pub fn ticks_closed(&self) -> usize { + self.checkpoints.iter().filter(|c| c.closed).count() + } +} + +/// What a caller asks of one run. The token cap is not here: the recording +/// port owns it. +#[derive(Clone, Copy, Debug)] +pub struct VerseRunConfig { + pub ticks: u32, + pub tick_span: TickMinutes, + pub cover: CoverBudget, +} + +/// A tick as the operator reads it, including the one a run ended inside. +#[derive(Clone, Debug)] +pub struct TickReport { + pub checkpoint: TickCheckpoint, + /// Every act the tick committed, from the operator log. + pub acts: Vec, + /// The labels of the subjects the checkpoint's refusals name, from the + /// snapshot the tick opened on. + pub labels: BTreeMap, + pub spent: u64, + pub remaining: u64, +} + +impl std::fmt::Display for RunStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Running => f.write_str("Running"), + Self::Completed => f.write_str("Completed"), + Self::BudgetExhausted => f.write_str("BudgetExhausted"), + Self::Failed { reason } => write!(f, "Failed ({reason})"), + Self::Stopped => f.write_str("Stopped"), + } + } +} + +/// The tick as the operator reads it: a header, one line per committed act +/// (who, which verb, how it renders, what its roles are bound to, what was +/// said or shown), one per refusal, and the spend. +impl std::fmt::Display for TickReport { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let checkpoint = &self.checkpoint; + writeln!( + f, + "tick {} {}: {} cells, {} acts, {} refused", + checkpoint.tick, + if checkpoint.closed { "closed" } else { "open" }, + checkpoint.cells, + self.acts.len(), + checkpoint.refusals.len() + )?; + for act in &self.acts { + let path = match act.render_path { + Some(RenderPath::ShipAction) => "ship_action", + Some(RenderPath::Conversation) => "conversation", + None => "-", + }; + write!(f, " {} {} [{path}]", act.speaker_label, act.verb)?; + for role in &act.roles { + write!(f, " {}={}", role.role, role.target)?; + } + if let Some(speech) = &act.speech { + write!(f, " says {:?}", speech.as_str())?; + } + if let Some(display) = &act.display { + write!(f, " shows {:?}", display.as_str())?; + } + writeln!(f)?; + } + for refusal in &checkpoint.refusals { + let reason = match refusal.reason { + RefusalReason::Interrupted => "interrupted", + RefusalReason::WorldRejected => "world_rejected", + RefusalReason::OpportunityGone => "opportunity_gone", + RefusalReason::ScopeMoved => "scope_moved", + RefusalReason::ProposalUndecodable => "proposal_undecodable", + RefusalReason::RowDisagreesWithCell => "row_disagrees_with_cell", + RefusalReason::CallUnattributed => "call_unattributed", + }; + let label = self.labels.get(&refusal.subject).map_or("-", String::as_str); + writeln!(f, " refused: cell {} {label} {reason}", refusal.cell)?; + } + for call in &checkpoint.unattributed { + writeln!(f, " note: cell {} call names no handle: tool {:?}", call.cell, call.tool)?; + } + writeln!( + f, + " tokens: {} this tick, {} spent, {} remaining", + checkpoint.tokens, self.spent, self.remaining + ) + } +} + +#[derive(Clone, Debug)] +pub struct RunOutcome { + pub record: VerseRunRecord, +} + +fn row_key(world: WorldId, sequence: u64) -> String { + format!("{}/{sequence:020}", world.key()) +} + +fn tick_key(world: WorldId, ordinal: u64) -> String { + format!("{}/{ordinal:020}", world.key()) +} + +fn tick_row(tick: &VerseTick) -> Result { + Ok(CultCacheEnvelope { + key: tick_key(tick.world_id, tick.ordinal), + r#type: TICK_ROW.into(), + payload: rmp_serde::to_vec_named(tick).map_err(|_| VerseRunError::Store)?, + stored_at: Utc::now().to_rfc3339(), + schema_id: Some(TICK_RECORD_SCHEMA.into()), + }) +} + +fn run_row(record: &VerseRunRecord) -> Result { + Ok(CultCacheEnvelope { + key: row_key(record.world_id, record.sequence), + r#type: RUN_ROW.into(), + payload: rmp_serde::to_vec_named(record).map_err(|_| VerseRunError::Store)?, + stored_at: Utc::now().to_rfc3339(), + schema_id: Some(RUN_RECORD_SCHEMA.into()), + }) +} + +/// A run's number and cap, claimed from the run store before anything of the +/// run exists. The call ledger's run id derives from it ([`call_run_id`]), and +/// [`run_manual`] runs only the record it names. Not `Clone`: a claim runs once +/// or is abandoned. +#[derive(Debug)] +pub struct RunClaim { + world_id: WorldId, + sequence: u64, + token_cap: u64, +} + +impl RunClaim { + /// The run's number, the one [`call_run_id`] names. + pub fn sequence(&self) -> u64 { + self.sequence + } +} + +/// Every row of the file, split by type. +struct Rows { + runs: Vec, + ticks: Vec, +} + +/// Run records and tick documents in one CultCache file. A run is claimed +/// when it begins and saved again each time it moves, so a row is replaced by +/// its own later state. A tick is opened once and closed once, and only by the tick +/// driver: the writers are private to this module. +/// +/// ```compile_fail +/// fn wedge(store: &ghostlight::VerseRunStore, world: ghostlight::WorldId) { +/// store.close_ticks_before(world, 1).unwrap(); +/// } +/// ``` +pub struct VerseRunStore { + store: Mutex, +} + +impl VerseRunStore { + /// Opens the file, creating it empty when absent. Every row must be a + /// canonical run record of this schema under its own key. + pub fn open(path: impl AsRef) -> Result { + let store = OwnedRedbMessagePackBackingStore::new(path.as_ref()) + .map_err(|_| VerseRunError::Store)?; + store + .validate_path_identity() + .map_err(|_| VerseRunError::Store)?; + let opened = Self { + store: Mutex::new(store), + }; + opened.records()?; + Ok(opened) + } + + fn records(&self) -> Result { + let rows = self + .store + .lock() + .map_err(|_| VerseRunError::Store)? + .pull_all() + .map_err(|_| VerseRunError::Store)?; + let mut runs = Vec::new(); + let mut ticks = Vec::new(); + for row in &rows { + match (row.r#type.as_str(), row.schema_id.as_deref()) { + (RUN_ROW, Some(RUN_RECORD_SCHEMA)) => { + let record: VerseRunRecord = + canonical(row).map_err(|_| VerseRunError::Corrupt)?; + if row.key != row_key(record.world_id, record.sequence) + || record.sequence == 0 + { + return Err(VerseRunError::Corrupt); + } + runs.push(record); + } + (TICK_ROW, Some(TICK_RECORD_SCHEMA)) => { + let tick: VerseTick = canonical(row).map_err(|_| VerseRunError::Corrupt)?; + if row.key != tick_key(tick.world_id, tick.ordinal) { + return Err(VerseRunError::Corrupt); + } + ticks.push(tick); + } + _ => return Err(VerseRunError::Corrupt), + } + } + runs.sort_by_key(|record| (record.world_id.key(), record.sequence)); + ticks.sort_by_key(|tick| (tick.world_id.key(), tick.ordinal)); + Ok(Rows { runs, ticks }) + } + + /// The sequence the next claim of this world takes. Sequences count from + /// one. Private: a caller never chooses a run's number, `claim` does. + fn next_sequence(&self, world: WorldId) -> Result { + Ok(self + .records()? + .runs + .iter() + .filter(|record| record.world_id == world) + .map(|record| record.sequence) + .max() + .unwrap_or(0) + + 1) + } + + /// Every run of the world, in sequence. + pub fn runs(&self, world: WorldId) -> Result, VerseRunError> { + Ok(self + .records()? + .runs + .into_iter() + .filter(|record| record.world_id == world) + .collect()) + } + + /// The tick document of (world, ordinal), when one was opened. + pub fn tick(&self, world: WorldId, ordinal: u64) -> Result, VerseRunError> { + Ok(self + .records()? + .ticks + .into_iter() + .find(|tick| tick.world_id == world && tick.ordinal == ordinal)) + } + + /// Writes a tick document that does not exist yet, open. The store's own + /// insert-if-absent decides, so two opens of one ordinal cannot both win. + /// Private: the tick driver is the only writer of tick rows. + fn open_tick(&self, tick: &VerseTick) -> Result<(), VerseRunError> { + if tick.closed.is_some() { + return Err(VerseRunError::OutOfSequence); + } + let row = tick_row(tick)?; + let inserted = self + .store + .lock() + .map_err(|_| VerseRunError::Store)? + .insert_entry_if_absent(row) + .map_err(|_| VerseRunError::Store)?; + if inserted { + Ok(()) + } else { + Err(VerseRunError::OutOfSequence) + } + } + + /// Closes an open tick document. Private, like every tick writer. + fn close_tick( + &self, + world: WorldId, + ordinal: u64, + close: TickClose, + ) -> Result<(), VerseRunError> { + let Some(mut tick) = self.tick(world, ordinal)? else { + return Err(VerseRunError::OutOfSequence); + }; + if tick.closed.is_some() { + return Err(VerseRunError::OutOfSequence); + } + tick.closed = Some(close); + self.put_tick(&tick) + } + + /// Closes, as the world's, every open tick of the world below `ordinal`. + fn close_ticks_before(&self, world: WorldId, ordinal: u64) -> Result<(), VerseRunError> { + for mut tick in self.records()?.ticks { + if tick.world_id == world && tick.ordinal < ordinal && tick.closed.is_none() { + tick.closed = Some(TickClose::World); + self.put_tick(&tick)?; + } + } + Ok(()) + } + + fn put_tick(&self, tick: &VerseTick) -> Result<(), VerseRunError> { + let row = tick_row(tick)?; + self.store + .lock() + .map_err(|_| VerseRunError::Store)? + .push(&row) + .map_err(|_| VerseRunError::Store) + } + + /// Claims the next run of the snapshot's world: writes its `Running` + /// record, with the cap and config it was claimed under, in one insert that + /// cannot overwrite a record. The returned claim is the only way to run it; + /// a claim that never runs stays `Running` (a process that died) or is + /// ended by [`abandon`](Self::abandon), and its number is never issued + /// again. + pub fn claim( + &self, + snapshot: &WorldSnapshot, + token_cap: u64, + config: &VerseRunConfig, + ) -> Result { + let cover_budget = config.cover.validated()?; + let record = VerseRunRecord { + world_id: snapshot.world_id, + sequence: self.next_sequence(snapshot.world_id)?, + token_cap, + ticks_requested: config.ticks, + tick_span_minutes: config.tick_span.minutes(), + cover_budget: RecordedCoverBudget { + cells: cover_budget.cells, + constituent_cap: cover_budget.constituent_cap, + urgency_slots: cover_budget.urgency_slots, + }, + start_revision: snapshot.revision, + checkpoints: Vec::new(), + status: RunStatus::Running, + }; + self.insert_run(&record)?; + Ok(RunClaim { + world_id: record.world_id, + sequence: record.sequence, + token_cap, + }) + } + + /// Writes a run record that does not exist yet. The store's own + /// insert-if-absent decides, so two claims of one sequence cannot both win. + fn insert_run(&self, record: &VerseRunRecord) -> Result<(), VerseRunError> { + let inserted = self + .store + .lock() + .map_err(|_| VerseRunError::Store)? + .insert_entry_if_absent(run_row(record)?) + .map_err(|_| VerseRunError::Store)?; + if inserted { + Ok(()) + } else { + Err(VerseRunError::OutOfSequence) + } + } + + /// Ends a claim that never ran: its record becomes `Failed { reason }`. + pub fn abandon(&self, claim: RunClaim, reason: &'static str) -> Result<(), VerseRunError> { + let mut record = self.claimed(&claim)?; + record.status = failed(reason); + self.save(&record) + } + + /// The record a claim names. + fn claimed(&self, claim: &RunClaim) -> Result { + self.records()? + .runs + .into_iter() + .find(|record| record.world_id == claim.world_id && record.sequence == claim.sequence) + .ok_or(VerseRunError::OutOfSequence) + } + + /// Replaces a claimed record with its later state. A sequence with no + /// record is refused: only `claim` creates one. + fn save(&self, record: &VerseRunRecord) -> Result<(), VerseRunError> { + self.claimed(&RunClaim { + world_id: record.world_id, + sequence: record.sequence, + token_cap: record.token_cap, + })?; + self.store + .lock() + .map_err(|_| VerseRunError::Store)? + .push(&run_row(record)?) + .map_err(|_| VerseRunError::Store) + } +} + +/// How an error ends a cell. +enum CellEnd { + Budget, + Fatal(&'static str), + Refused(RefusalReason), +} + +fn classify(error: &ControllerError) -> CellEnd { + match error { + ControllerError::Inference { source, .. } + if source.class() == InferenceFaultClass::BudgetExhausted => + { + CellEnd::Budget + } + _ if error.requires_quarantine() => CellEnd::Fatal("controller_quarantined"), + ControllerError::World(KernelError::ScopeChanged { .. }) => { + CellEnd::Refused(RefusalReason::ScopeMoved) + } + ControllerError::World(_) => CellEnd::Refused(RefusalReason::WorldRejected), + ControllerError::NoOpportunity { .. } | ControllerError::AmbiguousOpportunity => { + CellEnd::Refused(RefusalReason::OpportunityGone) + } + ControllerError::OpportunityMismatch => CellEnd::Refused(RefusalReason::ScopeMoved), + _ => CellEnd::Fatal("controller_error"), + } +} + +fn pending_code(reason: ControllerPendingReason) -> &'static str { + match reason { + ControllerPendingReason::InferenceRetryable => "inference_retryable", + ControllerPendingReason::InferenceRecoveryRequired => "inference_recovery_required", + ControllerPendingReason::WorldUnavailable => "world_unavailable", + ControllerPendingReason::WorldOutcomeUnknown => "world_outcome_unknown", + ControllerPendingReason::StoreReopenRequired => "store_reopen_required", + } +} + +fn command_of(submission: &SubmissionDisposition) -> CommandId { + match submission { + SubmissionDisposition::NoProposal(receipt) | SubmissionDisposition::Completed(receipt) => { + match receipt { + SubmitReceipt::Applied(commit) | SubmitReceipt::AlreadyApplied(commit) => { + commit.command_id + } + } + } + SubmissionDisposition::PreviouslyConfirmed(commit) => commit.command_id, + } +} + +/// Folds one cell's outcome into the tick. Returns the status that ends the +/// run, when the cell did not finish. A pending cell's unreached members are +/// not refusals: the tick stays open and they resume. +fn absorb( + result: Result, + cell: &Cell, + position: u32, + committed: &mut Vec, + refusals: &mut Vec, + unattributed: &mut Vec, +) -> Option { + match result { + Ok(run) => { + absorb_run(run, cell, position, committed, refusals, unattributed).map(failed) + } + Err(error) => match classify(&error) { + CellEnd::Budget => Some(RunStatus::BudgetExhausted), + CellEnd::Fatal(code) => Some(failed(code)), + CellEnd::Refused(reason) => { + refusals.extend(refuse_all(cell, position, reason)); + None + } + }, + } +} + +/// A finished cell's commits and refusals; the code of the pending state when +/// it did not finish. +fn absorb_run( + run: CellRun, + cell: &Cell, + position: u32, + committed: &mut Vec, + refusals: &mut Vec, + unattributed: &mut Vec, +) -> Option<&'static str> { + match run { + CellRun::Narrative(NarrativeRun::Completed(decision)) => { + committed.push(command_of(decision.submission())); + None + } + CellRun::Narrative(NarrativeRun::Pending(pending)) => Some(pending_code(pending.reason())), + CellRun::Narrative(NarrativeRun::Interrupted(_)) => { + refusals.extend(refuse_all(cell, position, RefusalReason::Interrupted)); + None + } + CellRun::Operational(OperationalRun::Completed(decision)) => { + committed.push(command_of(decision.submission())); + None + } + CellRun::Operational(OperationalRun::Pending(pending)) => { + Some(pending_code(pending.reason())) + } + CellRun::Grouped(run) => { + // The controller owns who is in the cell and says, for each + // member, committed or refused. The driver copies. A finished + // cell whose lists miss a member, or name one it lacks, is a + // broken invariant and ends the run, not a relabel. + unattributed.extend(run.unattributed.iter().map(|tool| UnattributedCall { + cell: position, + tool: tool.clone(), + })); + for entry in &run.submissions { + committed.push(command_of(&entry.submission)); + } + for refused in &run.refused { + refusals.push(Refusal { + cell: position, + subject: refused.subject, + reason: match refused.reason { + ConstituentRefusalReason::LeftBeforeSelection => { + RefusalReason::OpportunityGone + } + ConstituentRefusalReason::ScopeChanged => RefusalReason::ScopeMoved, + ConstituentRefusalReason::WorldRefused => RefusalReason::WorldRejected, + ConstituentRefusalReason::ProposalUndecodable => { + RefusalReason::ProposalUndecodable + } + ConstituentRefusalReason::RowDisagreesWithCell => { + RefusalReason::RowDisagreesWithCell + } + ConstituentRefusalReason::CallUnattributed => { + RefusalReason::CallUnattributed + } + }, + }); + } + if run.pending.is_none() { + let listed: Vec = run + .submissions + .iter() + .map(|entry| entry.subject) + .chain(run.refused.iter().map(|refused| refused.subject)) + .collect(); + let members = cell.members(); + let exact = listed.len() == members.len() + && members.iter().all(|member| listed.contains(&member.subject)); + if !exact { + return Some("cell_membership"); + } + } + run.pending.map(pending_code) + } + } +} + +/// One refusal per member of the cell. +fn refuse_all(cell: &Cell, position: u32, reason: RefusalReason) -> impl Iterator { + cell.members().iter().map(move |member| Refusal { + cell: position, + subject: member.subject, + reason, + }) +} + +/// The frozen cell with each member bound to the opportunity the world holds +/// for that subject now, and the frozen members that hold none. The partition +/// is fixed when the tick opens; an opportunity is the world's current binding +/// for a subject, and an earlier cell's act can move it (a word spoken in a +/// shared room changes what its listeners know). A subject that no longer holds +/// an opportunity leaves the cell, and an empty cell is gone. +fn refreshed( + frozen: &FrozenCell, + tick: TickIndex, + fresh: &[DecisionOpportunity], +) -> (Option, Vec) { + let mut gone = Vec::new(); + let mut members: Vec = Vec::new(); + for subject in &frozen.members { + match fresh + .iter() + .find(|opportunity| opportunity.scope.subject_id == *subject) + { + Some(opportunity) => members.push(Constituent { + subject: *subject, + opportunity: opportunity.clone(), + }), + None => gone.push(*subject), + } + } + let cell = if frozen.group { + (!members.is_empty()).then(|| Cell::Group { + id: frozen.id, + tick, + members, + }) + } else { + members.into_iter().next().map(|member| Cell::Singleton { + id: frozen.id, + tick, + member, + }) + }; + (cell, gone) +} + +enum TickEnd { + /// Every cell finished and the clock advanced. + Closed, + /// The run ends inside this tick with this status. + Open(RunStatus), +} + +fn failed(reason: &str) -> RunStatus { + RunStatus::Failed { + reason: reason.to_owned(), + } +} + +/// Runs `config.ticks` ticks from the world's present clock under the cap the +/// recording port was opened with. `observe` sees each tick as it ends, +/// including the one a run ends inside, and may stop the run after a finished +/// tick by breaking. `claim` is the run's number from +/// [`VerseRunStore::claim`]; the port must have been opened for it +/// ([`call_run_id`]) under the cap the claim recorded, and `config` is the one +/// the claim was made with. A claim this refuses is abandoned, so a refused +/// start binds nothing. +pub async fn run_manual( + mailbox: &WorldMailbox, + runner: &ControllerRunner, + port: &RecordingInferencePort, + store: &VerseRunStore, + claim: RunClaim, + config: &VerseRunConfig, + mut observe: impl FnMut(&TickReport) -> ControlFlow<()>, +) -> Result { + let begun = match mailbox.snapshot().await { + Ok(begun) => begun, + Err(error) => { + store.abandon(claim, "world_unavailable")?; + return Err(error.into()); + } + }; + if port.run_id() != &call_run_id(claim.sequence) + || port.cap() != claim.token_cap + || begun.world_id != claim.world_id + { + store.abandon(claim, "port_not_this_run")?; + return Err(VerseRunError::PortNotThisRun); + } + let sequence = claim.sequence; + let mut record = store.claimed(&claim)?; + let cover_budget = match config.cover.validated() { + Ok(cover_budget) if record_matches(&record, config, cover_budget) => cover_budget, + _ => { + store.abandon(claim, "config_not_this_claim")?; + return Err(VerseRunError::ConfigNotThisClaim); + } + }; + // The record names the revision the run started from, so the run starts + // from it. + if record.start_revision != begun.revision { + store.abandon(claim, "world_moved_since_claim")?; + return Err(VerseRunError::WorldMovedSinceClaim); + } + + let mut spent_before = port.spent(); + for completed in 0..config.ticks { + let (end, report) = run_tick( + mailbox, + runner, + port, + store, + sequence, + config, + cover_budget, + spent_before, + || mailbox.operator_log(), + ) + .await; + spent_before = port.spent(); + match end { + TickEnd::Open(status) => { + record.status = status; + if let Some(report) = report { + record.checkpoints.push(report.checkpoint.clone()); + store.save(&record)?; + let _ = observe(&report); + } else { + store.save(&record)?; + } + return Ok(RunOutcome { record }); + } + TickEnd::Closed => { + let report = report.expect("a closed tick has its report"); + record.checkpoints.push(report.checkpoint.clone()); + store.save(&record)?; + if observe(&report).is_break() && completed + 1 < config.ticks { + record.status = RunStatus::Stopped; + store.save(&record)?; + return Ok(RunOutcome { record }); + } + } + } + } + record.status = RunStatus::Completed; + store.save(&record)?; + Ok(RunOutcome { record }) +} + +/// Whether the record a claim wrote was made under this config. +fn record_matches(record: &VerseRunRecord, config: &VerseRunConfig, cover: CoverBudget) -> bool { + record.ticks_requested == config.ticks + && record.tick_span_minutes == config.tick_span.minutes() + && record.cover_budget + == RecordedCoverBudget { + cells: cover.cells, + constituent_cap: cover.constituent_cap, + urgency_slots: cover.urgency_slots, + } +} + +/// The key a run's provider calls are recorded under: the run's sequence, so a +/// run record and its call records name each other. +pub fn call_run_id(sequence: u64) -> super::RunId { + super::RunId::new(&format!("run-{sequence}")) + .expect("a decimal sequence is a canonical run id") +} + +/// The id of a tick's clock advance: the world and the ordinal, nothing the +/// run holds. A retry in the same tick has the same body and is answered +/// `AlreadyApplied`. +pub(crate) fn clock_command_id(world: WorldId, ordinal: u64) -> CommandId { + CommandId::derived(CLOCK_NAMESPACE, &[&world.text(), &ordinal.to_string()]) +} + +/// One tick. The report is absent only when the world or the tick document +/// could not be read at all. +#[allow(clippy::too_many_arguments)] +async fn run_tick( + mailbox: &WorldMailbox, + runner: &ControllerRunner, + port: &RecordingInferencePort, + store: &VerseRunStore, + sequence: u64, + config: &VerseRunConfig, + cover_budget: CoverBudget, + spent_before: u64, + read_log: ReadLog, +) -> (TickEnd, Option) +where + ReadLog: FnOnce() -> Log, + Log: Future, MailboxError>>, +{ + let read = async { + let snapshot = mailbox.snapshot().await?; + let ordinal = mailbox.clock_advances().await?; + let opportunities = mailbox.opportunities().await?; + let graph = mailbox.agency_graph().await?; + Ok::<_, MailboxError>((snapshot, ordinal, opportunities, graph)) + } + .await; + let Ok((snapshot, ordinal, opportunities, graph)) = read else { + return (TickEnd::Open(failed("world_unavailable")), None); + }; + let world = snapshot.world_id; + let Ok(tick) = open_or_find( + store, + ordinal, + sequence, + config, + cover_budget, + &snapshot, + &opportunities, + &graph, + ) else { + return (TickEnd::Open(failed("run_store")), None); + }; + let index = TickIndex(ordinal); + let Some(span) = TickMinutes::new(tick.span_minutes) else { + return (TickEnd::Open(failed("run_store")), None); + }; + + let mut committed = Vec::new(); + let mut refusals = Vec::new(); + let mut unattributed = Vec::new(); + let mut open: Option = None; + for (position, frozen) in tick.cells.iter().enumerate() { + let position = position as u32; + let fresh = match mailbox.opportunities().await { + Ok(fresh) => fresh, + Err(_) => { + open = Some(failed("world_unavailable")); + break; + } + }; + let (cell, gone) = refreshed(frozen, index, &fresh); + refusals.extend(gone.iter().map(|&subject| Refusal { + cell: position, + subject, + reason: RefusalReason::OpportunityGone, + })); + let Some(cell) = cell else { + continue; + }; + let result = runner.run_cell(&cell).await; + if let Some(status) = absorb( + result, + &cell, + position, + &mut committed, + &mut refusals, + &mut unattributed, + ) { + open = Some(status); + break; + } + } + + // The act listing is the tick's report to the operator: a tick whose log + // cannot be read is not closed, and the run ends there. + let acts = match acts_since(read_log().await, snapshot.revision) { + Some(acts) => acts, + None => { + open = Some(failed("operator_log_unavailable")); + Vec::new() + } + }; + + if open.is_none() { + if mailbox + .submit_clock(clock_command_id(world, ordinal), span) + .await + .is_err() + { + open = Some(failed("clock_refused")); + } else if store + .close_tick(world, ordinal, TickClose::Clock { run: sequence }) + .is_err() + { + open = Some(failed("run_store")); + } + } + + let spent = port.spent(); + let checkpoint_closed = open.is_none(); + let labels = refusals + .iter() + .filter_map(|refusal| { + snapshot + .subjects + .iter() + .find(|subject| subject.id == refusal.subject) + .map(|subject| (refusal.subject, subject.label.clone())) + }) + .collect(); + let report = TickReport { + checkpoint: TickCheckpoint { + tick: ordinal, + closed: checkpoint_closed, + cells: tick.cells.len() as u32, + committed, + refusals, + unattributed, + tokens: spent.saturating_sub(spent_before), + }, + acts, + labels, + spent, + remaining: port.cap().saturating_sub(spent), + }; + (open.map_or(TickEnd::Closed, TickEnd::Open), Some(report)) +} + +/// The acts a tick committed: the operator log's events past the tick's +/// opening revision. A log that cannot be read is `None`, never an empty tick. +fn acts_since( + log: Result, MailboxError>, + since: u64, +) -> Option> { + log.ok().map(|log| { + log.into_iter() + .filter(|event| event.revision > since) + .collect() + }) +} + +/// The tick document for `ordinal`: the open one a run left, or a new one +/// frozen from a cover derived now. An open tick the world has moved past is +/// closed as the world's first. A document already closed at the world's +/// current ordinal is a store that disagrees with the world. +#[allow(clippy::too_many_arguments)] +fn open_or_find( + store: &VerseRunStore, + ordinal: u64, + sequence: u64, + config: &VerseRunConfig, + cover_budget: CoverBudget, + snapshot: &WorldSnapshot, + opportunities: &[DecisionOpportunity], + graph: &AgencyGraph, +) -> Result { + let world = snapshot.world_id; + store.close_ticks_before(world, ordinal)?; + if let Some(tick) = store.tick(world, ordinal)? { + return if tick.closed.is_none() { + Ok(tick) + } else { + Err(VerseRunError::OutOfSequence) + }; + } + let cover = derive_cover(world, TickIndex(ordinal), opportunities, graph, cover_budget); + let tick = VerseTick { + world_id: world, + ordinal, + opened_now: snapshot.now.0, + opened_revision: snapshot.revision, + span_minutes: config.tick_span.minutes(), + cover_budget: RecordedCoverBudget { + cells: cover_budget.cells, + constituent_cap: cover_budget.constituent_cap, + urgency_slots: cover_budget.urgency_slots, + }, + oversubscribed: cover.oversubscribed, + cells: cover.cells.iter().map(FrozenCell::of).collect(), + opened_by_run: sequence, + closed: None, + }; + store.open_tick(&tick)?; + Ok(tick) +} + +#[cfg(test)] +mod tests; diff --git a/crates/ghostlight/src/verse/run/tests.rs b/crates/ghostlight/src/verse/run/tests.rs new file mode 100644 index 0000000..fcac9a1 --- /dev/null +++ b/crates/ghostlight/src/verse/run/tests.rs @@ -0,0 +1,2758 @@ +use super::*; +use crate::aetheria::read_aetheria_catalog; +use crate::controllers::{ + ControllerModels, ControllerWorkStore, InferenceEvent, InferenceFault, InferenceOutput, + InferencePort, InferencePurpose, InferenceRequest, PreparedInference, TokenUsage, + open_controller_work, +}; +use crate::cover::derive_cover; +use crate::tests::stock_weights; +use crate::verse::{ + CallRecordStore, CreateVerseWorldIntent, RecordingInferencePort, ReplayInferencePort, + RunStart, +}; +use crate::{ + BoundRole, CommandBody, ControllerMode, FictionalMinutes, KernelError, + PrincipalCommandIntent, RenderPath, VerifiedPrincipalEvidence, id_text, +}; +use async_trait::async_trait; +use chrono::Duration; +use serde_json::json; +use std::collections::{BTreeMap, VecDeque}; +use std::path::Path; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use tokio::task::JoinHandle; + +const COMMIT: &str = "92b82e9aa5bb422c568c4bfc85a8f641426a41fc"; +const FIXTURE: &str = concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/aetheria-verse-catalog.cc" +); +const SPAN: u32 = 60; + +/// Creates the verse world at `dir/world.cc` from the fixture catalog and +/// activates it, then lets go of the file so a copy of it is a restored world. +async fn create_world(dir: &Path) { + let (mailbox, task) = WorldMailbox::open(dir.join("world.cc")).unwrap(); + let principal = + VerifiedPrincipalEvidence::fixture("verse-owner", Utc::now() + Duration::hours(1)); + let receipt = mailbox + .create_verse( + CreateVerseWorldIntent { + id: CommandId::new(), + title: "The Verse".into(), + brief: "Aetheria.".into(), + catalog: read_aetheria_catalog(FIXTURE).unwrap(), + aetheria_commit: COMMIT.into(), + lens_weights: stock_weights(), + targets: BTreeMap::new(), + }, + &principal, + ) + .await + .unwrap(); + for body in [CommandBody::ApproveDraft, CommandBody::ActivateWorld] { + let revision = mailbox.snapshot().await.unwrap().revision; + mailbox + .submit_principal( + PrincipalCommandIntent { + id: CommandId::new(), + world_id: receipt.world_id, + expected_revision: revision, + body, + }, + &principal, + ) + .await + .unwrap(); + } + drop(mailbox); + task.await.unwrap(); +} + +/// A scripted provider. `reply` sees the serialized request and answers with +/// the events a model would. It reports usage as a fixed share of the bound the +/// recording port holds the call to, so the charge is a function of the +/// request. +struct ScriptedPort { + reply: Box Vec + Send + Sync>, + faults: std::sync::Mutex>, + bounds: std::sync::Mutex>, + calls: AtomicUsize, +} + +impl ScriptedPort { + fn new(reply: impl Fn(&str) -> Vec + Send + Sync + 'static) -> Arc { + Arc::new(Self { + reply: Box::new(reply), + faults: std::sync::Mutex::new(VecDeque::new()), + bounds: std::sync::Mutex::new(Vec::new()), + calls: AtomicUsize::new(0), + }) + } + + /// Every faction speaks its own line. + fn speaking() -> Arc { + Self::new(|request| { + let name = who(request); + vec![speak_call(&format!("{name} holds the line."))] + }) + } + + fn fail_next(&self, fault: InferenceFault) { + self.faults.lock().unwrap().push_back(fault); + } + + fn calls(&self) -> usize { + self.calls.load(Ordering::SeqCst) + } + + fn bounds(&self) -> Vec { + self.bounds.lock().unwrap().clone() + } +} + +/// Which faction a request is for: the prompt names its decision owner. +fn who(request: &str) -> String { + for name in ["Adrasteia", "Brannoch"] { + if request.contains(&format!("Decision owner:\\n{name}\\n")) { + return name.to_owned(); + } + } + panic!("a request names no faction"); +} + +fn speak_call(text: &str) -> InferenceEvent { + InferenceEvent::ToolCall { + call_id: "act".into(), + name: "speak".into(), + arguments: json!({ "text": text }).to_string(), + } +} + +#[async_trait] +impl InferencePort for ScriptedPort { + fn prepare(&self, request: InferenceRequest) -> Result { + PreparedInference::prepare("verse-run-test", 4_102_444_800_000, request) + } + + async fn infer(&self, request: PreparedInference) -> Result { + let sequence = self.calls.fetch_add(1, Ordering::SeqCst); + let wire = &request.invocation.request; + let bound = serde_json::to_vec(wire).unwrap().len() as u64 + + u64::from(wire.max_output_tokens.unwrap()); + self.bounds.lock().unwrap().push(bound); + if let Some(fault) = self.faults.lock().unwrap().pop_front() { + return Err(fault); + } + let text = serde_json::to_string(wire).unwrap(); + Ok( + InferenceOutput::new((self.reply)(&text), format!("sha256:scripted-{sequence}")) + .with_usage(TokenUsage { + prompt: bound / 2, + completion: bound / 4, + cached_prompt: None, + }), + ) + } +} + +fn models() -> ControllerModels { + ControllerModels { + projector: "scripted".into(), + persona: "scripted".into(), + interpreter: "scripted".into(), + operational_agent: "scripted".into(), + elaborator: "scripted".into(), + } +} + +fn two_cells() -> CoverBudget { + CoverBudget { + cells: 2, + constituent_cap: 8, + urgency_slots: 0, + } +} + +fn config(ticks: u32) -> VerseRunConfig { + VerseRunConfig { + ticks, + tick_span: TickMinutes::new(SPAN).unwrap(), + cover: two_cells(), + } +} + +/// An opened verse state root: the world, its call records, its run records +/// and its controller work, as the binary holds them. +struct Harness { + mailbox: WorldMailbox, + _task: JoinHandle<()>, + calls: Arc, + runs: VerseRunStore, + work: Arc, +} + +impl Harness { + async fn open(dir: &Path) -> Self { + let (mailbox, task) = WorldMailbox::open(dir.join("world.cc")).unwrap(); + Self { + mailbox, + _task: task, + calls: Arc::new(CallRecordStore::open(dir.join("calls.cc")).unwrap()), + runs: VerseRunStore::open(dir.join("runs.cc")).unwrap(), + work: open_controller_work(dir.join("controller-work.cc")).unwrap(), + } + } + + async fn fresh() -> (tempfile::TempDir, Self) { + let dir = tempfile::tempdir().unwrap(); + create_world(dir.path()).await; + let harness = Self::open(dir.path()).await; + (dir, harness) + } + + /// One run over `inner`, claimed and recorded under the next sequence. + async fn run( + &self, + inner: Arc, + cap: u64, + ticks: u32, + observe: impl FnMut(&TickReport) -> ControlFlow<()>, + ) -> RunOutcome { + self.run_under(inner, cap, config(ticks), observe).await + } + + /// One run under the given config, claimed and recorded under the next + /// sequence. + async fn run_under( + &self, + inner: Arc, + cap: u64, + config: VerseRunConfig, + observe: impl FnMut(&TickReport) -> ControlFlow<()>, + ) -> RunOutcome { + let world = self.mailbox.snapshot().await.unwrap(); + let claim = self.runs.claim(&world, cap, &config).unwrap(); + let sequence = claim.sequence(); + let port = Arc::new( + RecordingInferencePort::open( + inner, + self.calls.clone(), + call_run_id(sequence), + cap, + &RunStart::of(&world), + ) + .unwrap(), + ); + let runner = + ControllerRunner::open(self.mailbox.clone(), port.clone(), self.work.clone(), models()) + .unwrap(); + run_manual( + &self.mailbox, + &runner, + &port, + &self.runs, + claim, + &config, + observe, + ) + .await + .unwrap() + } + + async fn run_quietly(&self, inner: Arc, cap: u64, ticks: u32) -> RunOutcome { + self.run(inner, cap, ticks, |_| ControlFlow::Continue(())) + .await + } + + async fn run_quietly_under( + &self, + inner: Arc, + config: VerseRunConfig, + ) -> RunOutcome { + self.run_under(inner, u64::MAX, config, |_| ControlFlow::Continue(())) + .await + } + + async fn now(&self) -> u64 { + self.mailbox.snapshot().await.unwrap().now.0 + } + + async fn acts(&self) -> usize { + self.mailbox.operator_log().await.unwrap().len() + } + + async fn subject(&self, label: &str) -> crate::SubjectId { + self.mailbox + .snapshot() + .await + .unwrap() + .subjects + .iter() + .find(|subject| subject.label == label) + .unwrap() + .id + } +} + +#[tokio::test] +async fn scripted_run_commits_and_checkpoints_every_tick() { + let (_dir, verse) = Harness::fresh().await; + let world = verse.mailbox.snapshot().await.unwrap(); + let port = ScriptedPort::speaking(); + let mut reports = Vec::new(); + let outcome = verse + .run(port.clone(), 10_000_000, 3, |report| { + reports.push(report.clone()); + ControlFlow::Continue(()) + }) + .await; + + // One checkpoint per tick, tick indexes 0 to 2, the clock moved three spans. + assert_eq!(outcome.record.status, RunStatus::Completed); + let ticks: Vec = outcome.record.checkpoints.iter().map(|c| c.tick).collect(); + assert_eq!(ticks, vec![0, 1, 2]); + assert_eq!(outcome.record.ticks_closed(), 3); + assert_eq!(verse.now().await, 3 * u64::from(SPAN)); + // Both factions acted every tick, the second on a scope the first moved. + assert_eq!(verse.acts().await, 6); + assert_eq!(port.calls(), 6); + + // Every committed command id is the one the tick document's frozen cells + // derive, in cell order, and the document is closed by this run's clock. + for checkpoint in &outcome.record.checkpoints { + let tick = verse + .runs + .tick(world.world_id, checkpoint.tick) + .unwrap() + .unwrap(); + assert_eq!(tick.closed, Some(TickClose::Clock { run: 1 })); + assert_eq!(tick.opened_by_run, 1); + assert_eq!(tick.span_minutes, SPAN); + let expected: Vec = tick + .cells + .iter() + .map(|cell| { + assert!(!cell.group, "two factions under two cells are singletons"); + CommandId::for_cell_constituent( + world.world_id, + cell.id, + cell.members[0], + TickIndex(checkpoint.tick), + ) + }) + .collect(); + assert_eq!(expected.len(), 2); + assert_eq!(checkpoint.committed, expected); + assert_eq!(checkpoint.cells, 2); + assert!(checkpoint.closed); + assert!(checkpoint.refusals.is_empty()); + } + + // The record in the store is the record the run returned, and it says what + // the run was asked. + assert_eq!( + verse.runs.runs(world.world_id).unwrap(), + vec![outcome.record.clone()] + ); + assert_eq!(outcome.record.world_id, world.world_id); + assert_eq!(outcome.record.sequence, 1); + assert_eq!(outcome.record.token_cap, 10_000_000); + assert_eq!(outcome.record.ticks_requested, 3); + assert_eq!(outcome.record.tick_span_minutes, SPAN); + assert_eq!(outcome.record.start_revision, world.revision); + assert_eq!( + outcome.record.cover_budget, + RecordedCoverBudget { + cells: 2, + constituent_cap: 8, + urgency_slots: 0 + } + ); + + // A tick's tokens are what the call store charged for its two calls. + let records = verse.calls.records(&call_run_id(1)).unwrap(); + assert_eq!(records.len(), 6); + for (tick, checkpoint) in outcome.record.checkpoints.iter().enumerate() { + let charged = records[2 * tick].charged() + records[2 * tick + 1].charged(); + assert!(charged > 0); + assert_eq!(checkpoint.tokens, charged); + } + let total: u64 = outcome.record.checkpoints.iter().map(|c| c.tokens).sum(); + assert_eq!(total, verse.calls.spent(&call_run_id(1)).unwrap()); + + // The operator sees each tick as it ends: its acts, and the spend so far. + assert_eq!(reports.len(), 3); + let mut spent = 0; + for (tick, report) in reports.iter().enumerate() { + spent += report.checkpoint.tokens; + assert!(report.checkpoint.closed); + assert_eq!(report.acts.len(), 2); + assert_eq!(report.spent, spent); + assert_eq!(report.remaining, 10_000_000 - spent); + assert_eq!(report.checkpoint, outcome.record.checkpoints[tick]); + for act in &report.acts { + assert_eq!(act.verb, "speak"); + assert_eq!( + act.speech.as_ref().unwrap().as_str(), + format!("{} holds the line.", act.speaker_label) + ); + } + } + // A tick reports only its own acts. + let revisions: Vec = reports + .iter() + .flat_map(|report| report.acts.iter().map(|act| act.revision)) + .collect(); + assert_eq!(revisions.len(), 6); + assert!(revisions.windows(2).all(|pair| pair[0] < pair[1])); +} + +#[tokio::test] +async fn budget_exhausted_stops_before_the_clock_and_resumes() { + // What the first call is held to, measured on a world of its own. + let (_probe_dir, probe) = Harness::fresh().await; + let measuring = ScriptedPort::speaking(); + probe.run_quietly(measuring.clone(), u64::MAX, 1).await; + let first_bound = measuring.bounds()[0]; + + let (_dir, verse) = Harness::fresh().await; + let world = verse.mailbox.snapshot().await.unwrap(); + // Room for the first call, and not for a second after the first's charge. + let first = ScriptedPort::speaking(); + let mut reports = Vec::new(); + let ended = verse + .run(first.clone(), first_bound + 64, 2, |report| { + reports.push(report.clone()); + ControlFlow::Continue(()) + }) + .await; + assert_eq!(ended.record.status, RunStatus::BudgetExhausted); + // The tick it ended inside is recorded, unclosed, with the act it committed. + let [open] = ended.record.checkpoints.as_slice() else { + panic!("the open tick is the run's one checkpoint") + }; + assert!(!open.closed); + assert_eq!(open.tick, 0); + assert_eq!(open.committed.len(), 1); + assert_eq!(ended.record.ticks_closed(), 0); + assert_eq!(verse.now().await, 0, "the open tick's clock did not advance"); + assert_eq!(first.calls(), 1, "the second call never reached the provider"); + assert_eq!(verse.acts().await, 1); + // The open tick is reported, unclosed, with what it committed. + assert_eq!(reports.len(), 1); + assert!(!reports[0].checkpoint.closed); + assert_eq!(reports[0].acts.len(), 1); + assert_eq!(&reports[0].checkpoint, open); + assert!(reports[0].remaining < first_bound); + assert_eq!( + verse.runs.runs(world.world_id).unwrap(), + vec![ended.record.clone()] + ); + + // A later run, under its own cap, finishes the tick and costs no call for + // the act already committed. + let second = ScriptedPort::speaking(); + let mut resumed_reports = Vec::new(); + let resumed = verse + .run(second.clone(), u64::MAX, 2, |report| { + resumed_reports.push(report.clone()); + ControlFlow::Continue(()) + }) + .await; + assert_eq!(resumed.record.status, RunStatus::Completed); + assert_eq!(resumed.record.sequence, 2); + // The resumed tick is the document run 1 opened, closed by run 2. + let opened = verse.runs.tick(world.world_id, 0).unwrap().unwrap(); + assert_eq!(opened.opened_by_run, 1); + assert_eq!(opened.closed, Some(TickClose::Clock { run: 2 })); + let ticks: Vec = resumed.record.checkpoints.iter().map(|c| c.tick).collect(); + assert_eq!(ticks, vec![0, 1], "the open tick is the first this run closes"); + assert_eq!(verse.now().await, 2 * u64::from(SPAN)); + assert_eq!(second.calls(), 3, "one call finished tick 0, two ran tick 1"); + assert_eq!(verse.acts().await, 4, "no act was committed twice"); + // The frozen cells run in their frozen order: the cell that already acted + // answers with its original receipt and costs no call, and the one the + // first run left in flight finishes. + let closing = &resumed.record.checkpoints[0]; + assert!(closing.closed); + assert_eq!(closing.cells, 2); + assert_eq!(closing.committed.len(), 2, "{closing:?}"); + assert_eq!(closing.committed[0], open.committed[0], "{closing:?}"); + assert_ne!(closing.committed[1], open.committed[0], "{closing:?}"); + assert!(closing.refusals.is_empty(), "{closing:?}"); + assert_eq!(resumed.record.ticks_closed(), 2); + // The tick reports the act this run committed and not the one it found. + assert_eq!(resumed_reports[0].acts.len(), 1); + assert!(resumed_reports[0].acts[0].revision > reports[0].acts[0].revision); + assert_ne!( + resumed_reports[0].acts[0].speaker_label, + reports[0].acts[0].speaker_label + ); + assert_eq!(resumed.record.checkpoints[1].committed.len(), 2); +} + +/// Both cells of a tick committed through another driver before any run opened +/// it, one after the other, so each one's act has since moved the other's +/// scope. The run that opens the tick reaches both committed rows, in whatever +/// order it froze, and answers each with its receipt: no inference, no refusal, +/// no second act. `reverse` flips which cell the other driver committed first, +/// so the answer is shown not to depend on the order the cells are reached in. +async fn resumed_after_both_cells_committed_elsewhere(reverse: bool) { + let (_dir, verse) = Harness::fresh().await; + let snapshot = verse.mailbox.snapshot().await.unwrap(); + let graph = verse.mailbox.agency_graph().await.unwrap(); + let cover = derive_cover( + snapshot.world_id, + TickIndex(0), + &snapshot.opportunities, + &graph, + CoverBudget { + cells: 2, + constituent_cap: 8, + urgency_slots: 0, + }, + ); + assert_eq!(cover.cells.len(), 2); + let elsewhere = ScriptedPort::speaking(); + let runner = ControllerRunner::open( + verse.mailbox.clone(), + elsewhere.clone(), + verse.work.clone(), + models(), + ) + .unwrap(); + let order: Vec<&Cell> = if reverse { + cover.cells.iter().rev().collect() + } else { + cover.cells.iter().collect() + }; + for cell in order { + let fresh = verse.mailbox.opportunities().await.unwrap(); + let bound = refreshed(&FrozenCell::of(cell), TickIndex(0), &fresh).0.unwrap(); + runner.run_cell(&bound).await.unwrap(); + } + assert_eq!(verse.acts().await, 2); + + let resuming = ScriptedPort::speaking(); + let ran = verse + .run_quietly_under(resuming.clone(), config_of(1, 60, 2)) + .await; + assert_eq!(ran.record.status, RunStatus::Completed); + let [closing] = ran.record.checkpoints.as_slice() else { + panic!("the run closed one tick") + }; + assert!(closing.closed); + assert!(closing.refusals.is_empty(), "{closing:?}"); + assert_eq!(closing.committed.len(), 2, "{closing:?}"); + assert_eq!(resuming.calls(), 0, "a committed cell was asked again"); + assert_eq!(verse.acts().await, 2, "no act was committed twice"); +} + +#[tokio::test] +async fn committed_cells_are_answered_in_the_order_the_tick_froze() { + resumed_after_both_cells_committed_elsewhere(false).await; +} + +#[tokio::test] +async fn committed_cells_are_answered_against_the_order_the_tick_froze() { + resumed_after_both_cells_committed_elsewhere(true).await; +} + +#[tokio::test] +async fn recorded_run_replays_to_the_same_journal() { + let live_dir = tempfile::tempdir().unwrap(); + create_world(live_dir.path()).await; + let replay_dir = tempfile::tempdir().unwrap(); + std::fs::copy( + live_dir.path().join("world.cc"), + replay_dir.path().join("world.cc"), + ) + .unwrap(); + let live = Harness::open(live_dir.path()).await; + let replayed = Harness::open(replay_dir.path()).await; + + let provider = ScriptedPort::speaking(); + let recorded = live.run_quietly(provider.clone(), u64::MAX, 3).await; + assert_eq!(recorded.record.status, RunStatus::Completed); + + // The same run again, from the recorded replies and no provider. + let start = RunStart::of(&replayed.mailbox.snapshot().await.unwrap()); + let replay = Arc::new( + ReplayInferencePort::new("verse-replay", &live.calls, &call_run_id(1), &start).unwrap(), + ); + let again = replayed.run_quietly(replay, u64::MAX, 3).await; + assert_eq!(again.record.status, RunStatus::Completed); + assert_eq!(provider.calls(), 6, "replay made no provider call"); + + assert_eq!(again.record.checkpoints, recorded.record.checkpoints); + let live_world = live.mailbox.snapshot().await.unwrap(); + let replayed_world = replayed.mailbox.snapshot().await.unwrap(); + assert_eq!(replayed_world.revision, live_world.revision); + assert_eq!(replayed_world.state_digest, live_world.state_digest); + assert_eq!( + replayed.mailbox.operator_log().await.unwrap(), + live.mailbox.operator_log().await.unwrap() + ); + // Both stores hold the same calls, charged the same. + assert_eq!( + replayed.calls.records(&call_run_id(1)).unwrap(), + live.calls.records(&call_run_id(1)).unwrap() + ); +} + +#[tokio::test] +async fn a_run_the_caller_stops_ends_after_a_finished_tick() { + let (_dir, verse) = Harness::fresh().await; + let stopped = verse + .run(ScriptedPort::speaking(), u64::MAX, 3, |_| { + ControlFlow::Break(()) + }) + .await; + assert_eq!(stopped.record.status, RunStatus::Stopped); + assert_eq!(stopped.record.checkpoints.len(), 1); + assert_eq!(verse.now().await, u64::from(SPAN)); + + // A stop asked on the last tick is not a stop: the run was done. + let done = verse + .run(ScriptedPort::speaking(), u64::MAX, 1, |_| { + ControlFlow::Break(()) + }) + .await; + assert_eq!(done.record.status, RunStatus::Completed); +} + +#[tokio::test] +async fn a_pending_cell_ends_the_run_without_the_clock_and_a_later_run_finishes_it() { + let (_dir, verse) = Harness::fresh().await; + let flaky = ScriptedPort::speaking(); + flaky.fail_next(InferenceFault::retryable("the provider is busy")); + let ended = verse.run_quietly(flaky.clone(), u64::MAX, 2).await; + assert_eq!( + ended.record.status, + RunStatus::Failed { + reason: "inference_retryable".into() + } + ); + assert_eq!(ended.record.ticks_closed(), 0); + let [open] = ended.record.checkpoints.as_slice() else { + panic!("the open tick is recorded") + }; + assert!(!open.closed && open.committed.is_empty()); + assert_eq!(verse.now().await, 0); + assert_eq!(verse.acts().await, 0); + + let finished = verse + .run_quietly(ScriptedPort::speaking(), u64::MAX, 1) + .await; + assert_eq!(finished.record.status, RunStatus::Completed); + assert_eq!(verse.now().await, u64::from(SPAN)); + assert_eq!(verse.acts().await, 2); +} + +#[tokio::test] +async fn a_provider_answer_against_the_tool_contract_ends_the_run_without_the_clock() { + let (_dir, verse) = Harness::fresh().await; + // The faction acts and then calls finish_without_proposal, which + // contradicts its own choice. + let port = ScriptedPort::new(|request| { + let name = who(request); + vec![ + speak_call(&format!("{name} speaks.")), + InferenceEvent::ToolCall { + call_id: "finish".into(), + name: "finish_without_proposal".into(), + arguments: "{}".into(), + }, + ] + }); + let outcome = verse.run_quietly(port, u64::MAX, 1).await; + assert_eq!( + outcome.record.status, + RunStatus::Failed { + reason: "inference_recovery_required".into() + } + ); + assert_eq!(outcome.record.ticks_closed(), 0); + assert_eq!(verse.now().await, 0); +} + +#[tokio::test] +async fn a_cell_the_world_refuses_is_a_refusal_and_the_tick_closes() { + let (_dir, verse) = Harness::fresh().await; + // Adrasteia attacks something that is not in the world; Brannoch speaks. + let nothing = id_text(CommandId::new()); + let port = ScriptedPort::new(move |request| { + let name = who(request); + if name == "Adrasteia" { + vec![InferenceEvent::ToolCall { + call_id: "act".into(), + name: "attack".into(), + arguments: json!({ "target": nothing, "text": "Adrasteia strikes." }).to_string(), + }] + } else { + vec![speak_call("Brannoch speaks.")] + } + }); + let outcome = verse.run_quietly(port, u64::MAX, 1).await; + assert_eq!(outcome.record.status, RunStatus::Completed); + let [checkpoint] = outcome.record.checkpoints.as_slice() else { + panic!("one tick, one checkpoint") + }; + assert_eq!(checkpoint.cells, 2); + assert_eq!(checkpoint.committed.len(), 1); + assert_eq!(checkpoint.refusals.len(), 1); + assert_eq!(checkpoint.refusals[0].reason, RefusalReason::WorldRejected); + assert!(checkpoint.refusals[0].cell < 2); + assert_eq!(checkpoint.refusals[0].subject, verse.subject("Adrasteia").await); + assert_eq!(verse.now().await, u64::from(SPAN)); + assert_eq!(verse.acts().await, 1); +} + +#[test] +fn the_operator_reads_refusals_and_how_a_run_ended() { + let [one, two, three, four, five, six] = [(); 6].map(|()| crate::SubjectId::issue()); + let refusal = |cell, subject, reason| Refusal { + cell, + subject, + reason, + }; + let report = TickReport { + checkpoint: TickCheckpoint { + tick: 4, + closed: false, + cells: 3, + committed: Vec::new(), + refusals: vec![ + refusal(1, one, RefusalReason::WorldRejected), + refusal(2, two, RefusalReason::OpportunityGone), + refusal(0, three, RefusalReason::Interrupted), + refusal(5, four, RefusalReason::ScopeMoved), + refusal(3, five, RefusalReason::RowDisagreesWithCell), + refusal(3, six, RefusalReason::CallUnattributed), + ], + unattributed: vec![UnattributedCall { + cell: 3, + tool: "c9__speak".into(), + }], + tokens: 7, + }, + acts: Vec::new(), + labels: BTreeMap::from([ + (one, "Ada".to_owned()), + (two, "Brann".to_owned()), + (three, "Cato".to_owned()), + (four, "Dura".to_owned()), + (five, "Eve".to_owned()), + (six, "Fay".to_owned()), + ]), + spent: 40, + remaining: 60, + }; + assert_eq!( + report.to_string(), + concat!( + "tick 4 open: 3 cells, 0 acts, 6 refused\n", + " refused: cell 1 Ada world_rejected\n", + " refused: cell 2 Brann opportunity_gone\n", + " refused: cell 0 Cato interrupted\n", + " refused: cell 5 Dura scope_moved\n", + " refused: cell 3 Eve row_disagrees_with_cell\n", + " refused: cell 3 Fay call_unattributed\n", + " note: cell 3 call names no handle: tool \"c9__speak\"\n", + " tokens: 7 this tick, 40 spent, 60 remaining\n", + ) + ); + let ended: Vec = [ + RunStatus::Running, + RunStatus::Completed, + RunStatus::BudgetExhausted, + RunStatus::Failed { + reason: "clock_refused".into(), + }, + RunStatus::Stopped, + ] + .iter() + .map(ToString::to_string) + .collect(); + assert_eq!( + ended, + ["Running", "Completed", "BudgetExhausted", "Failed (clock_refused)", "Stopped"] + ); +} + +#[tokio::test] +async fn an_act_names_its_verb_render_path_and_bound_roles() { + let (_dir, verse) = Harness::fresh().await; + let target = id_text(verse.subject("Brannoch").await); + let port = ScriptedPort::new(move |request| { + if who(request) == "Adrasteia" { + vec![InferenceEvent::ToolCall { + call_id: "act".into(), + name: "attack".into(), + arguments: json!({ "target": target, "text": "Adrasteia strikes." }).to_string(), + }] + } else { + vec![speak_call("Brannoch answers.")] + } + }); + let mut reports = Vec::new(); + let outcome = verse + .run(port, u64::MAX, 1, |report| { + reports.push(report.clone()); + ControlFlow::Continue(()) + }) + .await; + assert_eq!(outcome.record.status, RunStatus::Completed); + let catalog = read_aetheria_catalog(FIXTURE).unwrap(); + let acts = &reports[0].acts; + let attack = acts.iter().find(|act| act.verb == "attack").unwrap(); + assert_eq!(attack.speaker_label, "Adrasteia"); + assert_eq!( + attack.render_path, + Some(catalog.grammar.verb("attack").unwrap().render_path) + ); + assert_eq!( + attack.roles, + vec![BoundRole { + role: "target".into(), + target: "Brannoch".into() + }] + ); + + // And the operator reads them so. + let name = |verb: &str| match catalog.grammar.verb(verb).unwrap().render_path { + RenderPath::ShipAction => "ship_action", + RenderPath::Conversation => "conversation", + }; + let text = reports[0].to_string(); + let checkpoint = &reports[0].checkpoint; + assert!( + text.starts_with("tick 0 closed: 2 cells, 2 acts, 0 refused\n"), + "{text}" + ); + let attack_line = format!( + "\n Adrasteia attack [{}] target=Brannoch says \"Adrasteia strikes.\"\n", + name("attack") + ); + assert!(text.contains(&attack_line), "{text}"); + let speak_line = format!( + "\n Brannoch speak [{}] says \"Brannoch answers.\"\n", + name("speak") + ); + assert!(text.contains(&speak_line), "{text}"); + let tokens_line = format!( + " tokens: {} this tick, {} spent, {} remaining\n", + checkpoint.tokens, reports[0].spent, reports[0].remaining + ); + assert!(text.ends_with(&tokens_line), "{text}"); + let speak = acts.iter().find(|act| act.verb == "speak").unwrap(); + assert_eq!(speak.speaker_label, "Brannoch"); + assert_eq!( + speak.render_path, + Some(catalog.grammar.verb("speak").unwrap().render_path) + ); + assert!(speak.roles.is_empty()); +} + +#[tokio::test] +async fn the_opportunity_door_answers_what_a_snapshot_carries() { + let (_dir, verse) = Harness::fresh().await; + let snapshot = verse.mailbox.snapshot().await.unwrap(); + assert_eq!(snapshot.opportunities.len(), 2); + assert_eq!( + verse.mailbox.opportunities().await.unwrap(), + snapshot.opportunities + ); +} + +#[tokio::test] +async fn a_cell_is_run_on_what_the_world_holds_for_its_members_now() { + let (_dir, verse) = Harness::fresh().await; + let opportunities = verse.mailbox.opportunities().await.unwrap(); + let graph = verse.mailbox.agency_graph().await.unwrap(); + let world = verse.mailbox.snapshot().await.unwrap().world_id; + let singles = derive_cover( + world, + TickIndex(0), + &opportunities, + &graph, + two_cells(), + ); + let group = derive_cover( + world, + TickIndex(0), + &opportunities, + &graph, + CoverBudget { + cells: 1, + constituent_cap: 8, + urgency_slots: 0, + }, + ); + let [grouped @ Cell::Group { members, .. }] = group.cells.as_slice() else { + panic!("one cell for two subjects is a group") + }; + assert_eq!(members.len(), 2); + + // Changed opportunities replace the ones the cover held. + let mut moved = opportunities.clone(); + for opportunity in &mut moved { + opportunity.revision += 1; + } + for cell in &singles.cells { + let Cell::Singleton { member: held, .. } = cell else { + panic!("two subjects under two cells are singletons") + }; + let frozen = FrozenCell::of(cell); + let (Some(Cell::Singleton { member, .. }), gone) = refreshed(&frozen, TickIndex(0), &moved) else { + panic!("a subject that still has an opportunity keeps its cell") + }; + assert_ne!(member.opportunity, held.opportunity); + assert!(moved.contains(&member.opportunity)); + // The member is bound to its own subject's opportunity, not a neighbour's. + assert_eq!(member.subject, held.subject); + assert_eq!(member.opportunity.scope.subject_id, member.subject); + assert!(gone.is_empty()); + } + let frozen_group = FrozenCell::of(grouped); + let (Some(Cell::Group { members: now, .. }), gone) = + refreshed(&frozen_group, TickIndex(0), &moved) + else { + panic!("a group keeps its members") + }; + assert!(gone.is_empty()); + assert_eq!(now.len(), 2); + assert!(now.iter().all(|member| moved.contains(&member.opportunity))); + assert!( + now.iter() + .all(|member| member.opportunity.scope.subject_id == member.subject) + ); + + // A subject with no opportunity leaves its cell, and an empty cell is gone. + let one = &opportunities[..1]; + let (Some(Cell::Group { members: left, .. }), left_out) = + refreshed(&frozen_group, TickIndex(0), one) + else { + panic!("one member remains") + }; + assert_eq!(left.len(), 1); + assert_eq!(left[0].opportunity.scope, one[0].scope); + let gone = singles + .cells + .iter() + .filter(|cell| refreshed(&FrozenCell::of(cell), TickIndex(0), one).0.is_none()) + .count(); + assert_eq!(gone, 1); + let (nothing, all_gone) = refreshed(&frozen_group, TickIndex(0), &[]); + assert!(nothing.is_none()); + assert_eq!(all_gone, frozen_group.members); + assert_eq!(left_out, vec![opportunities[1].scope.subject_id]); +} + +#[test] +fn an_error_ends_a_cell_as_the_run_must_treat_it() { + let budget = ControllerError::Inference { + purpose: InferencePurpose::OperationalAgent, + source: InferenceFault::budget_exhausted(), + }; + assert!(matches!(classify(&budget), CellEnd::Budget)); + // Any other fault that stops the cognition organ stops the run. + let corrupted = ControllerError::Inference { + purpose: InferencePurpose::OperationalAgent, + source: InferenceFault::integrity_violation("the reply cannot be trusted"), + }; + assert!(matches!( + classify(&corrupted), + CellEnd::Fatal("controller_quarantined") + )); + assert!(matches!( + classify(&ControllerError::Serialization("x".into())), + CellEnd::Fatal("controller_quarantined") + )); + // What the world or the subject's standing refused stays with its cell. + assert!(matches!( + classify(&ControllerError::World(KernelError::AffordanceDenied)), + CellEnd::Refused(RefusalReason::WorldRejected) + )); + for gone in [ + ControllerError::NoOpportunity { + expected: ControllerMode::OperationalAgent, + }, + ControllerError::AmbiguousOpportunity, + ] { + assert!(matches!( + classify(&gone), + CellEnd::Refused(RefusalReason::OpportunityGone) + )); + } + assert!(matches!( + classify(&ControllerError::OpportunityMismatch), + CellEnd::Refused(RefusalReason::ScopeMoved) + )); + assert!(matches!( + classify(&ControllerError::CommandMismatch), + CellEnd::Fatal("controller_error") + )); +} + +/// One fact, one label: the world refusing a body whose scope moved is +/// `ScopeMoved`, the same label the controller's own refusal of in-flight work +/// carries, and not the catch-all `WorldRejected`. +#[tokio::test] +async fn a_world_refusal_for_a_moved_scope_is_scope_moved() { + let (_dir, verse) = Harness::fresh().await; + let opportunity = verse.mailbox.snapshot().await.unwrap().opportunities[0].clone(); + let moved = ControllerError::World(KernelError::ScopeChanged { + scope: opportunity.scope.clone(), + expected: opportunity.scope_digest.clone(), + actual: opportunity.scope_digest.clone(), + }); + assert!(matches!( + classify(&moved), + CellEnd::Refused(RefusalReason::ScopeMoved) + )); + assert!(matches!( + classify(&ControllerError::World(KernelError::AffordanceDenied)), + CellEnd::Refused(RefusalReason::WorldRejected) + )); +} + +/// A recording port over the harness's call store for `sequence` under `cap`, +/// and a runner on it. +fn port_and_runner( + verse: &Harness, + world: &WorldSnapshot, + sequence: u64, + cap: u64, +) -> (Arc, ControllerRunner) { + let port = Arc::new( + RecordingInferencePort::open( + ScriptedPort::speaking(), + verse.calls.clone(), + call_run_id(sequence), + cap, + &RunStart::of(world), + ) + .unwrap(), + ); + let runner = ControllerRunner::open( + verse.mailbox.clone(), + port.clone(), + verse.work.clone(), + models(), + ) + .unwrap(); + (port, runner) +} + +/// Runs `claim` on the given port and runner, which must refuse it. +async fn refused( + verse: &Harness, + port: &RecordingInferencePort, + runner: &ControllerRunner, + claim: RunClaim, + config: &VerseRunConfig, +) -> VerseRunError { + run_manual( + &verse.mailbox, + runner, + port, + &verse.runs, + claim, + config, + |_| ControlFlow::Continue(()), + ) + .await + .unwrap_err() +} + +#[tokio::test] +async fn a_run_started_on_the_wrong_port_world_or_config_is_refused_and_its_claim_ends_failed() { + let (_dir, verse) = Harness::fresh().await; + let world = verse.mailbox.snapshot().await.unwrap(); + let (_other_dir, elsewhere) = Harness::fresh().await; + let strange = elsewhere.mailbox.snapshot().await.unwrap(); + + // A cover budget with no cells cannot be claimed: no record exists. + let mut bad = config(1); + bad.cover.cells = 0; + assert!(matches!( + verse.runs.claim(&world, 1_000, &bad), + Err(VerseRunError::Budget(CoverBudgetError::NoCells)) + )); + assert!(verse.runs.runs(world.world_id).unwrap().is_empty()); + + let mut expected: Vec<(u64, RunStatus)> = Vec::new(); + + // A port opened for another run's number. + let claim = verse.runs.claim(&world, 1_000, &config(1)).unwrap(); + expected.push((claim.sequence(), failed("port_not_this_run"))); + let (port, runner) = port_and_runner(&verse, &world, 70, 1_000); + let error = refused(&verse, &port, &runner, claim, &config(1)).await; + assert!(matches!(error, VerseRunError::PortNotThisRun)); + + // A port under a cap other than the claim recorded, either side of it. + for port_cap in [999, 1_001] { + let claim = verse.runs.claim(&world, 1_000, &config(1)).unwrap(); + expected.push((claim.sequence(), failed("port_not_this_run"))); + let (port, runner) = port_and_runner(&verse, &world, claim.sequence(), port_cap); + let error = refused(&verse, &port, &runner, claim, &config(1)).await; + assert!(matches!(error, VerseRunError::PortNotThisRun), "cap {port_cap}"); + } + + // A config differing from the claim's in any one field. + let differing: Vec<(&str, VerseRunConfig)> = vec![ + ("ticks", config(2)), + ("tick span", VerseRunConfig { + tick_span: TickMinutes::new(SPAN + 1).unwrap(), + ..config(1) + }), + ("cells", config_of(1, SPAN, 3)), + ("constituent cap", VerseRunConfig { + cover: CoverBudget { constituent_cap: 9, ..two_cells() }, + ..config(1) + }), + ("urgency slots", VerseRunConfig { + cover: CoverBudget { urgency_slots: 1, ..two_cells() }, + ..config(1) + }), + ]; + for (field, other) in &differing { + let claim = verse.runs.claim(&world, 1_000, &config(1)).unwrap(); + expected.push((claim.sequence(), failed("config_not_this_claim"))); + let (port, runner) = port_and_runner(&verse, &world, claim.sequence(), 1_000); + let error = refused(&verse, &port, &runner, claim, other).await; + assert!(matches!(error, VerseRunError::ConfigNotThisClaim), "{field}"); + } + + // A claim made from a world revision ahead of the one the run starts from. + let mut ahead = world.clone(); + ahead.revision += 3; + let claim = verse.runs.claim(&ahead, 1_000, &config(1)).unwrap(); + expected.push((claim.sequence(), failed("world_moved_since_claim"))); + let (port, runner) = port_and_runner(&verse, &world, claim.sequence(), 1_000); + let error = refused(&verse, &port, &runner, claim, &config(1)).await; + assert!(matches!(error, VerseRunError::WorldMovedSinceClaim), "ahead"); + + // Another world's claim 1, on this world's mailbox and a port that fits it. + let claim = verse.runs.claim(&strange, 1_000, &config(1)).unwrap(); + let (port, runner) = port_and_runner(&verse, &world, claim.sequence(), 1_000); + let error = refused(&verse, &port, &runner, claim, &config(1)).await; + assert!(matches!(error, VerseRunError::PortNotThisRun)); + + let statuses = |world: &WorldSnapshot| -> Vec<(u64, RunStatus)> { + verse + .runs + .runs(world.world_id) + .unwrap() + .into_iter() + .map(|record| (record.sequence, record.status)) + .collect() + }; + assert_eq!(statuses(&world), expected); + assert_eq!(statuses(&strange), vec![(1, failed("port_not_this_run"))]); + assert!(verse.runs.tick(world.world_id, 0).unwrap().is_none()); + assert!(verse.runs.tick(strange.world_id, 0).unwrap().is_none()); + assert_eq!(verse.mailbox.operator_log().await.unwrap().len(), 0); +} + +/// The record names the revision the run started from: a claim made from a +/// world the run no longer starts from is refused and ends Failed. +#[tokio::test] +async fn a_claim_from_a_world_that_has_since_moved_is_refused() { + let (_dir, verse) = Harness::fresh().await; + let stale = verse.mailbox.snapshot().await.unwrap(); + verse + .run_quietly(ScriptedPort::speaking(), 1_000_000, 1) + .await; + let moved = verse.mailbox.snapshot().await.unwrap(); + assert!(moved.revision > stale.revision); + + let claim = verse.runs.claim(&stale, 1_000, &config(1)).unwrap(); + let sequence = claim.sequence(); + let (port, runner) = port_and_runner(&verse, &moved, sequence, 1_000); + let error = refused(&verse, &port, &runner, claim, &config(1)).await; + assert!(matches!(error, VerseRunError::WorldMovedSinceClaim)); + let runs = verse.runs.runs(moved.world_id).unwrap(); + assert_eq!(runs[1].sequence, sequence); + assert_eq!(runs[1].status, failed("world_moved_since_claim")); + assert!(runs[1].checkpoints.is_empty()); + + // A claim from the world as it stands is run, and records that revision. + let outcome = verse + .run_quietly(ScriptedPort::speaking(), 1_000_000, 1) + .await; + assert_eq!(outcome.record.start_revision, moved.revision); +} + +fn record(world: WorldId, sequence: u64) -> VerseRunRecord { + VerseRunRecord { + world_id: world, + sequence, + token_cap: 10, + ticks_requested: 1, + tick_span_minutes: SPAN, + cover_budget: RecordedCoverBudget { + cells: 2, + constituent_cap: 8, + urgency_slots: 0, + }, + start_revision: 0, + checkpoints: Vec::new(), + status: RunStatus::Running, + } +} + +#[tokio::test] +async fn the_run_store_claims_runs_per_world_and_replaces_a_claimed_run_with_its_later_state() { + let (_one_dir, one) = Harness::fresh().await; + let (_other_dir, other) = Harness::fresh().await; + let one = one.mailbox.snapshot().await.unwrap(); + let other = other.mailbox.snapshot().await.unwrap(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("runs.cc"); + let store = VerseRunStore::open(&path).unwrap(); + + // Claims number runs from one, per world, and write the Running record. + let first = store.claim(&one, 10, &config(3)).unwrap(); + let second = store.claim(&one, 20, &config(1)).unwrap(); + let elsewhere = store.claim(&other, 10, &config(1)).unwrap(); + assert_eq!( + (first.sequence(), second.sequence(), elsewhere.sequence()), + (1, 2, 1) + ); + let claimed = store.runs(one.world_id).unwrap(); + assert_eq!(claimed.len(), 2); + assert_eq!(claimed[0].token_cap, 10); + assert_eq!(claimed[0].ticks_requested, 3); + assert_eq!(claimed[1].token_cap, 20); + assert_eq!(claimed[0].start_revision, one.revision); + assert_eq!(claimed[0].status, RunStatus::Running); + + // A record is only ever a claimed one: a sequence with no record is refused. + assert!(matches!( + store.save(&record(one.world_id, 3)), + Err(VerseRunError::OutOfSequence) + )); + assert!(matches!( + store.save(&record(one.world_id, 0)), + Err(VerseRunError::OutOfSequence) + )); + assert_eq!(store.runs(one.world_id).unwrap().len(), 2); + + // Saving a claimed run again replaces it, and the file reopens to the + // same rows. + let mut finished = claimed[0].clone(); + finished.status = RunStatus::Completed; + store.save(&finished).unwrap(); + drop(store); + let store = VerseRunStore::open(&path).unwrap(); + assert_eq!( + store.runs(one.world_id).unwrap(), + vec![finished, claimed[1].clone()] + ); + assert_eq!(store.runs(other.world_id).unwrap().len(), 1); +} + +#[tokio::test] +async fn a_claim_is_never_reissued_even_when_its_run_never_started() { + let (_dir, verse) = Harness::fresh().await; + let world = verse.mailbox.snapshot().await.unwrap(); + + // Run 1 is claimed, then nothing happens: the process died. + let lost = verse.runs.claim(&world, 1_000_000_000, &config(1)).unwrap(); + assert_eq!(lost.sequence(), 1); + let before = verse.runs.runs(world.world_id).unwrap(); + drop(lost); + + // The next run is run 2 and leaves run 1's record exactly as it was. + let outcome = verse + .run_quietly(ScriptedPort::speaking(), 1_000_000_000, 1) + .await; + assert_eq!(outcome.record.sequence, 2); + assert_eq!(outcome.record.status, RunStatus::Completed); + let after = verse.runs.runs(world.world_id).unwrap(); + assert_eq!(after.len(), 2); + assert_eq!(after[0], before[0], "run 1's record was overwritten"); + assert_eq!(after[0].status, RunStatus::Running); + assert_eq!(after[1], outcome.record); +} + +/// The store's insert-if-absent, not a read before it, refuses a second record +/// of one sequence: concurrent claims on one handle read the same next number. +#[tokio::test] +async fn two_claims_of_one_sequence_cannot_both_succeed() { + let (_dir, verse) = Harness::fresh().await; + let world = verse.mailbox.snapshot().await.unwrap(); + let claim = verse.runs.claim(&world, 10, &config(1)).unwrap(); + let held = verse.runs.runs(world.world_id).unwrap(); + + let mut colliding = record(world.world_id, claim.sequence()); + colliding.token_cap = 99; + assert!(matches!( + verse.runs.insert_run(&colliding), + Err(VerseRunError::OutOfSequence) + )); + assert_eq!(verse.runs.runs(world.world_id).unwrap(), held); +} + +#[tokio::test] +async fn a_start_refused_after_the_claim_leaves_the_next_run_free_to_choose_its_cap() { + let (_dir, verse) = Harness::fresh().await; + let world = verse.mailbox.snapshot().await.unwrap(); + + // Run 1 is claimed and its call ledger opens under a cap, then the start + // is refused before the run begins. + let claim = verse.runs.claim(&world, 1_000, &config(1)).unwrap(); + let ledger = RecordingInferencePort::open( + ScriptedPort::speaking(), + verse.calls.clone(), + call_run_id(claim.sequence()), + 1_000, + &RunStart::of(&world), + ) + .unwrap(); + drop(ledger); + verse.runs.abandon(claim, "start_refused").unwrap(); + + // The operator tries again under another cap: it is a new run, and runs. + let outcome = verse + .run_quietly(ScriptedPort::speaking(), 50_000, 1) + .await; + assert_eq!(outcome.record.sequence, 2); + assert_eq!(outcome.record.token_cap, 50_000); + assert_eq!(outcome.record.status, RunStatus::Completed); + let runs = verse.runs.runs(world.world_id).unwrap(); + assert_eq!(runs[0].status, failed("start_refused")); + assert_eq!(runs[0].token_cap, 1_000); +} + +#[tokio::test] +async fn an_unreadable_operator_log_is_no_tick_report() { + let (_dir, verse) = Harness::fresh().await; + verse.run_quietly(ScriptedPort::speaking(), u64::MAX, 2).await; + let log = verse.mailbox.operator_log().await.unwrap(); + assert!(log.len() >= 4, "two ticks of two cells acted"); + + // A log that cannot be read is no tick report, whatever the revision. + assert!(acts_since(Err(MailboxError::Unavailable), 0).is_none()); + assert!(acts_since(Err(MailboxError::Unavailable), u64::MAX).is_none()); + + // A log that can is cut at the tick's opening revision. + assert_eq!(acts_since(Ok(log.clone()), 0), Some(log.clone())); + let middle = log[1].revision; + let later: Vec<_> = log.iter().filter(|e| e.revision > middle).cloned().collect(); + assert!(!later.is_empty() && later.len() < log.len()); + assert_eq!(acts_since(Ok(log.clone()), middle), Some(later)); + assert_eq!( + acts_since(Ok(log), u64::MAX), + Some(Vec::new()), + "an empty tick is a readable one" + ); +} + +/// A tick whose operator log cannot be read is not closed: the cells' work is +/// checkpointed, the clock does not advance, and the run ends Failed. +#[tokio::test] +async fn an_unreadable_operator_log_ends_the_run_with_no_clock_advance() { + let (_dir, verse) = Harness::fresh().await; + let world = verse.mailbox.snapshot().await.unwrap(); + let (port, runner) = port_and_runner(&verse, &world, 1, u64::MAX); + let config = config(1); + let (end, report) = run_tick( + &verse.mailbox, + &runner, + &port, + &verse.runs, + 1, + &config, + config.cover.validated().unwrap(), + 0, + || async { Err(MailboxError::Unavailable) }, + ) + .await; + + assert!( + matches!(&end, TickEnd::Open(status) if *status == failed("operator_log_unavailable")) + ); + let report = report.expect("the tick's checkpoint is kept"); + assert!(!report.checkpoint.closed); + assert!(!report.checkpoint.committed.is_empty(), "the cells ran"); + assert!(report.acts.is_empty()); + assert_eq!(verse.mailbox.clock_advances().await.unwrap(), 0); + assert_eq!(verse.now().await, world.now.0); + let tick = verse.runs.tick(world.world_id, 0).unwrap().unwrap(); + assert!(tick.closed.is_none()); +} + +#[test] +fn the_run_store_refuses_rows_that_are_not_canonical_run_records() { + let dir = tempfile::tempdir().unwrap(); + let world = WorldId::nil_for_test(); + let good = record(world, 1); + let envelope = |key: String, schema: &str, payload: Vec| CultCacheEnvelope { + key, + r#type: RUN_ROW.into(), + payload, + stored_at: Utc::now().to_rfc3339(), + schema_id: Some(schema.into()), + }; + let named = rmp_serde::to_vec_named(&good).unwrap(); + let cases = [ + // Another schema version is not migrated. + envelope(row_key(world, 1), "ghostlight.verse_run.v2", named.clone()), + // A row under a key that is not its own. + envelope(row_key(world, 9), RUN_RECORD_SCHEMA, named.clone()), + // A payload that is not the record. + envelope(row_key(world, 1), RUN_RECORD_SCHEMA, vec![0xc0]), + // The record in a non-canonical encoding. + envelope( + row_key(world, 1), + RUN_RECORD_SCHEMA, + rmp_serde::to_vec(&good).unwrap(), + ), + // A sequence of zero. + envelope( + row_key(world, 0), + RUN_RECORD_SCHEMA, + rmp_serde::to_vec_named(&record(world, 0)).unwrap(), + ), + ]; + for (index, row) in cases.into_iter().enumerate() { + let file = dir.path().join(format!("case-{index}.cc")); + { + let mut raw = OwnedRedbMessagePackBackingStore::new(&file).unwrap(); + raw.push(&row).unwrap(); + } + assert!( + matches!(VerseRunStore::open(&file), Err(VerseRunError::Corrupt)), + "case {index} was accepted" + ); + } +} + +/// A port that answers like `ScriptedPort::speaking` and fails the `at`th +/// provider call it sees, retryably. +struct FaultAt { + inner: Arc, + at: usize, + seen: AtomicUsize, +} + +#[async_trait] +impl InferencePort for FaultAt { + fn prepare(&self, request: InferenceRequest) -> Result { + self.inner.prepare(request) + } + + async fn infer(&self, request: PreparedInference) -> Result { + if self.seen.fetch_add(1, Ordering::SeqCst) == self.at { + return Err(InferenceFault::retryable("the provider is busy")); + } + self.inner.infer(request).await + } +} + +fn fault_at(at: usize) -> Arc { + Arc::new(FaultAt { + inner: ScriptedPort::speaking(), + at, + seen: AtomicUsize::new(0), + }) +} + +fn config_of(ticks: u32, span: u32, cells: u16) -> VerseRunConfig { + VerseRunConfig { + ticks, + tick_span: TickMinutes::new(span).unwrap(), + cover: CoverBudget { + cells, + constituent_cap: 8, + urgency_slots: 0, + }, + } +} + +/// Run 1 closes tick 0 (two cells, 60 minutes). Run 2 commits one faction's act +/// in tick 1 and ends pending. Run 3 resumes under `resume`. Asserts that each +/// faction acted exactly once between the end of run 1 and the end of run 3. +async fn resumed_after_a_pending_cell( + resume: VerseRunConfig, +) -> (tempfile::TempDir, Harness, RunOutcome) { + let (dir, verse) = Harness::fresh().await; + let first = verse + .run_quietly_under(ScriptedPort::speaking(), config_of(1, 60, 2)) + .await; + assert_eq!(first.record.status, RunStatus::Completed); + let mark = verse.mailbox.snapshot().await.unwrap().revision; + + let ended = verse + .run_quietly_under(fault_at(1), config_of(1, 60, 2)) + .await; + assert_eq!( + ended.record.status, + RunStatus::Failed { + reason: "inference_retryable".into() + } + ); + let opened: Vec<_> = verse + .mailbox + .operator_log() + .await + .unwrap() + .into_iter() + .filter(|event| event.revision > mark) + .map(|event| event.speaker) + .collect(); + assert_eq!(opened.len(), 1, "one faction acted in the open tick"); + + let resumed = verse + .run_quietly_under(ScriptedPort::speaking(), resume) + .await; + assert_eq!(resumed.record.status, RunStatus::Completed); + let since: Vec<_> = verse + .mailbox + .operator_log() + .await + .unwrap() + .into_iter() + .filter(|event| event.revision > mark) + .map(|event| event.speaker) + .collect(); + assert_eq!(since.len(), 2, "each faction acted once in the open tick"); + assert_eq!( + since.iter().filter(|speaker| **speaker == opened[0]).count(), + 1, + "the faction that acted before the pause did not act again" + ); + (dir, verse, resumed) +} + +#[tokio::test] +async fn a_resume_under_another_span_finishes_the_open_tick_under_its_own() { + let (_dir, verse, resumed) = resumed_after_a_pending_cell(config_of(1, 30, 2)).await; + // Tick 1's clock advance is the frozen 60 minutes, not the resuming 30. + assert_eq!(verse.now().await, 120); + let world = verse.mailbox.snapshot().await.unwrap().world_id; + let tick = verse.runs.tick(world, 1).unwrap().unwrap(); + assert_eq!(tick.span_minutes, 60); + assert_eq!(tick.opened_by_run, 2); + assert_eq!(tick.closed, Some(TickClose::Clock { run: 3 })); + let [closing] = resumed.record.checkpoints.as_slice() else { + panic!("the resuming run closed one tick") + }; + assert_eq!(closing.tick, 1); + assert!(closing.closed); +} + +#[tokio::test] +async fn a_resume_under_another_cover_budget_keeps_the_frozen_cells() { + // One cell would be a group of both factions; the open tick keeps its two + // singletons. + let (_dir, verse, resumed) = resumed_after_a_pending_cell(config_of(1, 60, 1)).await; + let world = verse.mailbox.snapshot().await.unwrap().world_id; + let tick = verse.runs.tick(world, 1).unwrap().unwrap(); + assert_eq!(tick.cover_budget.cells, 2); + assert_eq!(tick.cells.len(), 2); + assert!(tick.cells.iter().all(|cell| !cell.group)); + assert_eq!(resumed.record.checkpoints[0].cells, 2); + assert_eq!(verse.now().await, 120); +} + +#[tokio::test] +async fn a_new_span_opens_a_new_tick_and_never_reuses_a_closed_one() { + let (_dir, verse) = Harness::fresh().await; + let first = verse + .run_quietly_under(ScriptedPort::speaking(), config_of(2, 60, 2)) + .await; + assert_eq!(first.record.status, RunStatus::Completed); + assert_eq!(verse.now().await, 120); + let acts = verse.acts().await; + + let port = ScriptedPort::speaking(); + let second = verse + .run_quietly_under(port.clone(), config_of(2, 120, 2)) + .await; + assert_eq!(second.record.status, RunStatus::Completed); + let ordinals: Vec = second.record.checkpoints.iter().map(|c| c.tick).collect(); + assert_eq!(ordinals, vec![2, 3]); + assert_eq!(verse.now().await, 120 + 240); + assert_eq!(verse.acts().await, acts + 4); + assert_eq!(port.calls(), 4); + let before: Vec = first + .record + .checkpoints + .iter() + .flat_map(|c| c.committed.clone()) + .collect(); + for checkpoint in &second.record.checkpoints { + assert!(checkpoint.committed.iter().all(|id| !before.contains(id))); + } +} + +#[tokio::test] +async fn a_closed_ticks_clock_is_answered_from_the_kernels_ledger() { + let (_dir, verse) = Harness::fresh().await; + verse.run_quietly(ScriptedPort::speaking(), u64::MAX, 2).await; + let world = verse.mailbox.snapshot().await.unwrap().world_id; + assert_eq!(verse.now().await, 120); + for ordinal in 0..2 { + let receipt = verse + .mailbox + .submit_clock( + clock_command_id(world, ordinal), + TickMinutes::new(SPAN).unwrap(), + ) + .await + .unwrap(); + assert!( + matches!(receipt, SubmitReceipt::AlreadyApplied(_)), + "the clock of tick {ordinal} was not the id the run submitted" + ); + } + assert_eq!(verse.now().await, 120); +} + +#[tokio::test] +async fn an_open_tick_the_world_moved_past_is_closed_by_the_world() { + let (_probe_dir, probe) = Harness::fresh().await; + let measuring = ScriptedPort::speaking(); + probe.run_quietly(measuring.clone(), u64::MAX, 1).await; + let first_bound = measuring.bounds()[0]; + + let (_dir, verse) = Harness::fresh().await; + let world = verse.mailbox.snapshot().await.unwrap().world_id; + let ended = verse + .run_under( + ScriptedPort::speaking(), + first_bound + 64, + config(1), + |_| ControlFlow::Continue(()), + ) + .await; + assert_eq!(ended.record.status, RunStatus::BudgetExhausted); + assert_eq!(verse.acts().await, 1); + // Another writer moves time without closing the tick. + verse + .mailbox + .submit_clock(CommandId::new(), TickMinutes::new(SPAN).unwrap()) + .await + .unwrap(); + assert_eq!(verse.now().await, 60); + + let resumed = verse + .run_quietly(ScriptedPort::speaking(), u64::MAX, 1) + .await; + assert_eq!(resumed.record.status, RunStatus::Completed); + let abandoned = verse.runs.tick(world, 0).unwrap().unwrap(); + assert_eq!(abandoned.closed, Some(TickClose::World)); + let next = verse.runs.tick(world, 1).unwrap().unwrap(); + assert_eq!(next.opened_by_run, 2); + assert_eq!(next.closed, Some(TickClose::Clock { run: 2 })); + let [checkpoint] = resumed.record.checkpoints.as_slice() else { + panic!("one tick, one checkpoint") + }; + assert_eq!(checkpoint.tick, 1); + assert_eq!(checkpoint.committed.len(), 2, "a fresh cover ran both factions"); + assert_eq!(verse.acts().await, 3); + assert_eq!(verse.now().await, 120); + // No faction has two acts in one ordinal: ordinal 1 opened after run 1's + // act, so its acts are the ones past the tick's opening revision. + let log = verse.mailbox.operator_log().await.unwrap(); + for speaker in log.iter().map(|event| event.speaker) { + let in_first = log + .iter() + .filter(|event| event.speaker == speaker && event.revision <= next.opened_revision) + .count(); + let in_second = log + .iter() + .filter(|event| event.speaker == speaker && event.revision > next.opened_revision) + .count(); + assert!(in_first <= 1 && in_second <= 1); + } +} + +fn tick_doc(world: WorldId, ordinal: u64) -> VerseTick { + VerseTick { + world_id: world, + ordinal, + opened_now: 0, + opened_revision: 3, + span_minutes: SPAN, + cover_budget: RecordedCoverBudget { + cells: 2, + constituent_cap: 8, + urgency_slots: 0, + }, + oversubscribed: false, + cells: Vec::new(), + opened_by_run: 1, + closed: None, + } +} + +#[test] +fn the_tick_store_opens_once_closes_once_and_refuses_non_canonical_rows() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("runs.cc"); + let store = VerseRunStore::open(&path).unwrap(); + let world = WorldId::nil_for_test(); + let other = WorldId::issue(); + + store.open_tick(&tick_doc(world, 0)).unwrap(); + assert!(matches!( + store.open_tick(&tick_doc(world, 0)), + Err(VerseRunError::OutOfSequence) + )); + // A tick opened already closed is not an open. + let mut born_closed = tick_doc(world, 5); + born_closed.closed = Some(TickClose::World); + assert!(matches!( + store.open_tick(&born_closed), + Err(VerseRunError::OutOfSequence) + )); + assert!(matches!( + store.close_tick(world, 9, TickClose::World), + Err(VerseRunError::OutOfSequence) + )); + store + .close_tick(world, 0, TickClose::Clock { run: 4 }) + .unwrap(); + assert!(matches!( + store.close_tick(world, 0, TickClose::World), + Err(VerseRunError::OutOfSequence) + )); + + // The world closes the open ticks below an ordinal and no others. + for ordinal in 1..4 { + store.open_tick(&tick_doc(world, ordinal)).unwrap(); + } + store.open_tick(&tick_doc(other, 1)).unwrap(); + store.close_ticks_before(world, 3).unwrap(); + let closed_by = |store: &VerseRunStore, world, ordinal| { + store.tick(world, ordinal).unwrap().unwrap().closed + }; + assert_eq!(closed_by(&store, world, 0), Some(TickClose::Clock { run: 4 })); + assert_eq!(closed_by(&store, world, 1), Some(TickClose::World)); + assert_eq!(closed_by(&store, world, 2), Some(TickClose::World)); + assert_eq!(closed_by(&store, world, 3), None); + assert_eq!(closed_by(&store, other, 1), None); + assert!(store.tick(world, 7).unwrap().is_none()); + + // Tick documents and run records share the file and reopen intact. + store + .store + .lock() + .unwrap() + .push(&run_row(&record(world, 1)).unwrap()) + .unwrap(); + drop(store); + let store = VerseRunStore::open(&path).unwrap(); + assert_eq!(store.runs(world).unwrap(), vec![record(world, 1)]); + assert_eq!(closed_by(&store, world, 2), Some(TickClose::World)); + + let good = tick_doc(world, 0); + let envelope = |key: String, row_type: &str, schema: &str, payload: Vec| CultCacheEnvelope { + key, + r#type: row_type.into(), + payload, + stored_at: Utc::now().to_rfc3339(), + schema_id: Some(schema.into()), + }; + let named = rmp_serde::to_vec_named(&good).unwrap(); + let cases = [ + envelope(tick_key(world, 9), TICK_ROW, TICK_RECORD_SCHEMA, named.clone()), + envelope(tick_key(world, 0), TICK_ROW, "ghostlight.verse_tick.v2", named.clone()), + envelope(tick_key(world, 0), TICK_ROW, TICK_RECORD_SCHEMA, vec![0xc0]), + envelope( + tick_key(world, 0), + TICK_ROW, + TICK_RECORD_SCHEMA, + rmp_serde::to_vec(&good).unwrap(), + ), + envelope(tick_key(world, 0), "something_else.v1", TICK_RECORD_SCHEMA, named), + ]; + for (index, row) in cases.into_iter().enumerate() { + let file = dir.path().join(format!("tick-case-{index}.cc")); + { + let mut raw = OwnedRedbMessagePackBackingStore::new(&file).unwrap(); + raw.push(&row).unwrap(); + } + assert!( + matches!(VerseRunStore::open(&file), Err(VerseRunError::Corrupt)), + "tick case {index} was accepted" + ); + } +} + +/// A restored world keeps its run store: the closed tick document at the +/// world's own ordinal is refused before any provider call or act. +#[tokio::test] +async fn a_closed_tick_at_the_worlds_current_ordinal_is_refused_before_any_cell() { + let live = tempfile::tempdir().unwrap(); + create_world(live.path()).await; + let backup = live.path().join("world.backup"); + std::fs::copy(live.path().join("world.cc"), &backup).unwrap(); + { + let verse = Harness::open(live.path()).await; + let ran = verse + .run_quietly_under(ScriptedPort::speaking(), config_of(2, 60, 2)) + .await; + assert_eq!(ran.record.status, RunStatus::Completed); + let Harness { + mailbox, + _task, + calls, + runs, + work, + } = verse; + drop((calls, runs, work, mailbox)); + _task.await.unwrap(); + } + let restored = tempfile::tempdir().unwrap(); + std::fs::copy(&backup, restored.path().join("world.cc")).unwrap(); + std::fs::copy(live.path().join("runs.cc"), restored.path().join("runs.cc")).unwrap(); + let verse = Harness::open(restored.path()).await; + let world = verse.mailbox.snapshot().await.unwrap().world_id; + assert_eq!(verse.mailbox.clock_advances().await.unwrap(), 0); + assert!(verse.runs.tick(world, 0).unwrap().unwrap().closed.is_some()); + let acts = verse.acts().await; + + let port = ScriptedPort::speaking(); + let refused = verse.run_quietly_under(port.clone(), config_of(1, 60, 2)).await; + assert_eq!( + refused.record.status, + RunStatus::Failed { + reason: "run_store".into() + } + ); + assert_eq!(port.calls(), 0, "no provider call before the refusal"); + assert_eq!(verse.acts().await, acts); + assert_eq!(verse.now().await, 0); +} + +/// A port that moves the world's clock through another writer on its first +/// call, then answers like the wrapped port. +struct ClockMovedMid { + inner: Arc, + mailbox: WorldMailbox, + seen: AtomicUsize, + minutes: u32, +} + +#[async_trait] +impl InferencePort for ClockMovedMid { + fn prepare(&self, request: InferenceRequest) -> Result { + self.inner.prepare(request) + } + + async fn infer(&self, request: PreparedInference) -> Result { + if self.seen.fetch_add(1, Ordering::SeqCst) == 0 { + self.mailbox + .submit_clock(CommandId::new(), TickMinutes::new(self.minutes).unwrap()) + .await + .unwrap(); + } + self.inner.infer(request).await + } +} + +/// The ordinal counts every clock advance the world committed, whatever its +/// length: a 45-minute advance by another writer inside an open tick moves +/// it, so the next tick opens at ordinal 2 and ordinal 1 never has a document. +#[tokio::test] +async fn every_clock_advance_moves_the_ordinal_whatever_its_length() { + let (_dir, verse) = Harness::fresh().await; + let world = verse.mailbox.snapshot().await.unwrap().world_id; + let port = Arc::new(ClockMovedMid { + inner: ScriptedPort::speaking(), + mailbox: verse.mailbox.clone(), + seen: AtomicUsize::new(0), + minutes: 45, + }); + let first = verse.run_quietly_under(port, config_of(1, 60, 2)).await; + assert_eq!(first.record.status, RunStatus::Completed); + assert_eq!(verse.mailbox.clock_advances().await.unwrap(), 2); + assert_eq!( + verse.runs.tick(world, 0).unwrap().unwrap().closed, + Some(TickClose::Clock { run: 1 }) + ); + + let second = verse + .run_quietly_under(ScriptedPort::speaking(), config_of(1, 60, 2)) + .await; + assert_eq!(second.record.status, RunStatus::Completed); + assert_eq!(second.record.checkpoints[0].tick, 2); + assert!(verse.runs.tick(world, 1).unwrap().is_none()); + assert_eq!(verse.now().await, 45 + 60 + 60); +} + +/// Two opens of one ordinal cannot both succeed, even raced. +#[test] +fn two_racing_opens_of_one_tick_cannot_both_succeed() { + let dir = tempfile::tempdir().unwrap(); + let store = VerseRunStore::open(dir.path().join("runs.cc")).unwrap(); + let world = WorldId::nil_for_test(); + for ordinal in 0..100u64 { + let barrier = std::sync::Barrier::new(2); + let opened = std::thread::scope(|scope| { + let handles: Vec<_> = (1..=2u64) + .map(|run| { + let (store, barrier) = (&store, &barrier); + scope.spawn(move || { + let mut doc = tick_doc(world, ordinal); + doc.opened_by_run = run; + barrier.wait(); + store.open_tick(&doc).is_ok() + }) + }) + .collect(); + handles + .into_iter() + .map(|handle| handle.join().unwrap()) + .filter(|ok| *ok) + .count() + }); + assert_eq!(opened, 1, "ordinal {ordinal} was opened {opened} times"); + } +} + + +/// A reply for the one-cell cover: every constituent the request offers +/// speaks, so the grouped cell gets its one inference answered for both. +fn grouped_reply(request: &str) -> Vec { + let handles: Vec = (0..8) + .filter(|i| request.contains(&format!("c{i}__speak"))) + .collect(); + assert!(!handles.is_empty(), "a one-cell cover offers grouped handles"); + handles + .into_iter() + .map(|i| InferenceEvent::ToolCall { + call_id: format!("act{i}"), + name: format!("c{i}__speak"), + arguments: json!({ "text": format!("Member {i} speaks.") }).to_string(), + }) + .collect() +} + +fn one_cell(ticks: u32) -> VerseRunConfig { + VerseRunConfig { + ticks, + tick_span: TickMinutes::new(SPAN).unwrap(), + cover: CoverBudget { + cells: 1, + constituent_cap: 8, + urgency_slots: 0, + }, + } +} + +/// Runs the grouped cover for `ticks`, returning the verse, the outcome and +/// every report the operator saw. +async fn grouped_run(ticks: u32) -> (tempfile::TempDir, Harness, RunOutcome, Vec) { + let (dir, verse) = Harness::fresh().await; + let mut reports = Vec::new(); + let outcome = verse + .run_under( + ScriptedPort::new(grouped_reply), + u64::MAX, + one_cell(ticks), + |report| { + reports.push(report.clone()); + ControlFlow::Continue(()) + }, + ) + .await; + (dir, verse, outcome, reports) +} + +/// Every member of every frozen cell of a closed tick is, once, either a +/// subject whose command id the checkpoint holds or a subject it refused. +async fn assert_every_member_accounted_for(verse: &Harness, outcome: &RunOutcome) { + let world = verse.mailbox.snapshot().await.unwrap().world_id; + let closed: Vec<_> = outcome.record.checkpoints.iter().filter(|c| c.closed).collect(); + assert!(!closed.is_empty(), "a run with no closed tick proves nothing"); + for checkpoint in closed { + let tick = verse.runs.tick(world, checkpoint.tick).unwrap().unwrap(); + let mut members = 0; + for cell in &tick.cells { + for subject in &cell.members { + members += 1; + let command = CommandId::for_cell_constituent( + world, + cell.id, + *subject, + TickIndex(checkpoint.tick), + ); + let committed = checkpoint.committed.iter().filter(|c| **c == command).count(); + let refused = checkpoint + .refusals + .iter() + .filter(|r| r.subject == *subject) + .count(); + assert_eq!(committed + refused, 1, "tick {} member", checkpoint.tick); + } + } + assert_eq!(checkpoint.committed.len() + checkpoint.refusals.len(), members); + } +} + +#[tokio::test] +async fn a_grouped_cell_records_every_member_it_could_not_commit() { + let (_dir, verse, outcome, reports) = grouped_run(2).await; + assert_eq!(outcome.record.status, RunStatus::Completed); + assert_eq!(reports.len(), 2); + let world = verse.mailbox.snapshot().await.unwrap(); + for (checkpoint, report) in outcome.record.checkpoints.iter().zip(&reports) { + assert!(checkpoint.closed); + assert_eq!(checkpoint.cells, 1); + assert_eq!(checkpoint.committed.len(), 1); + let [refusal] = checkpoint.refusals.as_slice() else { + panic!("one member of two is refused") + }; + assert_eq!(refusal.reason, RefusalReason::ScopeMoved); + let label = &world + .subjects + .iter() + .find(|subject| subject.id == refusal.subject) + .unwrap() + .label; + assert!( + report + .to_string() + .contains(&format!("refused: cell 0 {label} scope_moved")), + "the operator reads who was refused" + ); + } + assert_every_member_accounted_for(&verse, &outcome).await; +} + +#[tokio::test] +async fn every_member_of_a_closed_tick_is_committed_or_refused_once() { + // Singletons, all committing. + let (_dir, verse) = Harness::fresh().await; + let outcome = verse.run_quietly(ScriptedPort::speaking(), u64::MAX, 2).await; + assert_every_member_accounted_for(&verse, &outcome).await; + + // Singletons, one refused by the world. + let (_dir, verse) = Harness::fresh().await; + let nothing = id_text(CommandId::new()); + let port = ScriptedPort::new(move |request| { + if who(request) == "Adrasteia" { + vec![InferenceEvent::ToolCall { + call_id: "act".into(), + name: "attack".into(), + arguments: json!({ "target": nothing, "text": "Adrasteia strikes." }).to_string(), + }] + } else { + vec![speak_call("Brannoch speaks.")] + } + }); + let outcome = verse.run_quietly(port, u64::MAX, 2).await; + assert!(outcome.record.checkpoints.iter().all(|c| c.refusals.len() == 1)); + assert_every_member_accounted_for(&verse, &outcome).await; + + // One grouped cell. + let (_dir, verse, outcome, _) = grouped_run(2).await; + assert_every_member_accounted_for(&verse, &outcome).await; +} + +#[tokio::test] +async fn a_refused_grouped_member_acts_next_tick() { + let (_dir, verse, outcome, _) = grouped_run(2).await; + let world = verse.mailbox.snapshot().await.unwrap().world_id; + let [first, second] = outcome.record.checkpoints.as_slice() else { + panic!("two ticks, two checkpoints") + }; + let refused_first = first.refusals[0].subject; + let tick = verse.runs.tick(world, second.tick).unwrap().unwrap(); + let again = CommandId::for_cell_constituent( + world, + tick.cells[0].id, + refused_first, + TickIndex(second.tick), + ); + assert!(second.committed.contains(&again)); + assert_ne!(second.refusals[0].subject, refused_first); +} + +// ---- Refusals are recorded by the tick's frozen cover ---- + +/// The cover of the fixture world at tick 0 under `cells` cells. +async fn cover_of(verse: &Harness, cells: u16) -> crate::cover::Cover { + let snapshot = verse.mailbox.snapshot().await.unwrap(); + let graph = verse.mailbox.agency_graph().await.unwrap(); + derive_cover( + snapshot.world_id, + TickIndex(0), + &snapshot.opportunities, + &graph, + CoverBudget { + cells, + constituent_cap: 8, + urgency_slots: 0, + }, + ) +} + +/// Opens tick 0's document with these frozen cells, as an earlier run would +/// have left it. +async fn open_tick_with(verse: &Harness, cells: Vec) { + let snapshot = verse.mailbox.snapshot().await.unwrap(); + verse + .runs + .open_tick(&VerseTick { + world_id: snapshot.world_id, + ordinal: 0, + opened_now: snapshot.now.0, + opened_revision: snapshot.revision, + span_minutes: SPAN, + cover_budget: RecordedCoverBudget { + cells: 2, + constituent_cap: 8, + urgency_slots: 0, + }, + oversubscribed: false, + cells, + opened_by_run: 1, + closed: None, + }) + .unwrap(); +} + +/// The fixture's one-cell cover: its group's id and the two real members. +async fn the_group(verse: &Harness) -> (crate::cover::CellId, Vec) { + let cover = cover_of(verse, 1).await; + let [Cell::Group { id, members, .. }] = cover.cells.as_slice() else { + panic!("one cell over the fixture is a group") + }; + (*id, members.iter().map(|member| member.subject).collect()) +} + +fn command_for(world: WorldId, cell: crate::cover::CellId, subject: SubjectId) -> CommandId { + CommandId::for_cell_constituent(world, cell, subject, TickIndex(0)) +} + +#[tokio::test] +async fn a_tick_refuses_the_frozen_members_that_hold_no_opportunity_by_cell_and_member() { + let (_dir, verse) = Harness::fresh().await; + let (group, real) = the_group(&verse).await; + let single = match &cover_of(&verse, 2).await.cells[1] { + Cell::Singleton { id, .. } => *id, + Cell::Group { .. } => panic!("the second cell of two is a singleton"), + }; + let (g1, g2, g3) = (SubjectId::issue(), SubjectId::issue(), SubjectId::issue()); + let mut members = real.clone(); + members.extend([g1, g2]); + open_tick_with( + &verse, + vec![ + FrozenCell { + id: group, + group: true, + members, + }, + FrozenCell { + id: single, + group: false, + members: vec![g3], + }, + ], + ) + .await; + let mut reports = Vec::new(); + let outcome = verse + .run_under(ScriptedPort::new(grouped_reply), u64::MAX, one_cell(1), |report| { + reports.push(report.clone()); + ControlFlow::Continue(()) + }) + .await; + let [checkpoint] = outcome.record.checkpoints.as_slice() else { + panic!("one tick, one checkpoint") + }; + assert!(checkpoint.closed); + let gone: Vec<(SubjectId, u32)> = checkpoint + .refusals + .iter() + .filter(|refusal| refusal.reason == RefusalReason::OpportunityGone) + .map(|refusal| (refusal.subject, refusal.cell)) + .collect(); + // A gone member of the first cell, a second one, and a cell gone + // entirely: each once, at its own cell. + assert_eq!(gone.len(), 3); + for expected in [(g1, 0), (g2, 0), (g3, 1)] { + assert_eq!(gone.iter().filter(|g| **g == expected).count(), 1); + } + // The two real members: one committed, one refused for the shared room. + assert_eq!(checkpoint.committed.len(), 1); + assert_eq!(checkpoint.refusals.len(), 4); + assert!( + reports[0] + .to_string() + .contains("refused: cell 1 - opportunity_gone"), + "a subject outside the snapshot prints without a label" + ); +} + +#[tokio::test] +async fn a_frozen_member_the_resumed_row_lacks_is_refused_not_lost() { + // The member the row lacks is the cell's second member, then its first, + // with the group first in the tick and then behind a cell that is gone. + for behind_a_gone_cell in [false, true] { + for lacking_first in [false, true] { + a_row_lacking_a_frozen_member(lacking_first, behind_a_gone_cell).await; + } + } +} + +async fn a_row_lacking_a_frozen_member(lacking_first: bool, behind_a_gone_cell: bool) { + let (_dir, verse) = Harness::fresh().await; + let (group, real) = the_group(&verse).await; + let (a, b) = (real[0], real[1]); + let single = match &cover_of(&verse, 2).await.cells[1] { + Cell::Singleton { id, .. } => *id, + Cell::Group { .. } => panic!("the second cell of two is a singleton"), + }; + let gone = SubjectId::issue(); + let leading = |cells: Vec| -> Vec { + if behind_a_gone_cell { + let mut all = vec![FrozenCell { + id: single, + group: false, + members: vec![gone], + }]; + all.extend(cells); + all + } else { + cells + } + }; + // The first attempt froze [a, ghost]: the row it persists holds only a. + open_tick_with( + &verse, + leading(vec![FrozenCell { + id: group, + group: true, + members: vec![a, SubjectId::issue()], + }]), + ) + .await; + let first = verse + .run_under(fault_at(0), u64::MAX, one_cell(1), |_| { + ControlFlow::Continue(()) + }) + .await; + assert!(matches!(first.record.status, RunStatus::Failed { .. })); + // On resume the frozen cell names a and b; the row still holds only a. + let world = verse.mailbox.snapshot().await.unwrap().world_id; + let mut tick = verse.runs.tick(world, 0).unwrap().unwrap(); + tick.cells = leading(vec![FrozenCell { + id: group, + group: true, + members: if lacking_first { vec![b, a] } else { vec![a, b] }, + }]); + verse.runs.put_tick(&tick).unwrap(); + let cell = u32::from(behind_a_gone_cell); + let second = verse + .run_under(ScriptedPort::new(grouped_reply), u64::MAX, one_cell(1), |_| { + ControlFlow::Continue(()) + }) + .await; + let checkpoint = &second.record.checkpoints[0]; + assert!(checkpoint.closed); + assert_eq!(checkpoint.committed, vec![command_for(world, group, a)]); + let mut expected = Vec::new(); + if behind_a_gone_cell { + expected.push(Refusal { + cell: 0, + subject: gone, + reason: RefusalReason::OpportunityGone, + }); + } + expected.push(Refusal { + cell, + subject: b, + reason: RefusalReason::RowDisagreesWithCell, + }); + assert_eq!(checkpoint.refusals, expected); +} + +#[tokio::test] +async fn a_row_member_outside_the_frozen_cell_is_never_listed() { + let (_dir, verse) = Harness::fresh().await; + let (group, real) = the_group(&verse).await; + let (a, b) = (real[0], real[1]); + open_tick_with( + &verse, + vec![FrozenCell { + id: group, + group: true, + members: vec![a, b], + }], + ) + .await; + let first = verse + .run_under(fault_at(0), u64::MAX, one_cell(1), |_| { + ControlFlow::Continue(()) + }) + .await; + assert!(matches!(first.record.status, RunStatus::Failed { .. })); + // On resume the frozen cell names only a; the row holds a and b. + let world = verse.mailbox.snapshot().await.unwrap().world_id; + let mut tick = verse.runs.tick(world, 0).unwrap().unwrap(); + tick.cells = vec![FrozenCell { + id: group, + group: true, + members: vec![a], + }]; + verse.runs.put_tick(&tick).unwrap(); + let second = verse + .run_under(ScriptedPort::new(grouped_reply), u64::MAX, one_cell(1), |_| { + ControlFlow::Continue(()) + }) + .await; + let checkpoint = &second.record.checkpoints[0]; + assert!(checkpoint.closed); + assert_eq!(checkpoint.committed, vec![command_for(world, group, a)]); + assert!(checkpoint.refusals.is_empty()); +} + +/// The cell folding is the one place a cell's outcome becomes the tick's +/// committed and refused lists, so it is driven with each outcome directly. +fn fold( + result: Result, + cell: &Cell, + position: u32, +) -> (Option, Vec, Vec) { + let (mut committed, mut refusals) = (Vec::new(), Vec::new()); + let status = absorb( + result, + cell, + position, + &mut committed, + &mut refusals, + &mut Vec::new(), + ); + (status, committed, refusals) +} + +fn grouped_refusing( + group: &Cell, + reasons: &[ConstituentRefusalReason], +) -> Result { + let Cell::Group { id, members, .. } = group else { + panic!("a group") + }; + Ok(CellRun::Grouped(crate::controllers::GroupedRun { + cell: *id, + resolution: crate::cover::Resolution::Coarse { + constituents: members.len(), + }, + submissions: Vec::new(), + needs: Vec::new(), + unattributed: Vec::new(), + refused: members + .iter() + .zip(reasons) + .map(|(member, reason)| crate::controllers::ConstituentRefusal { + subject: member.subject, + reason: *reason, + }) + .collect(), + pending: None, + })) +} + +#[tokio::test] +async fn a_cell_the_driver_folds_records_each_refused_member_at_its_own_position() { + let (_dir, verse) = Harness::fresh().await; + let cover = cover_of(&verse, 1).await; + let group = &cover.cells[0]; + let Cell::Group { members, .. } = group else { + panic!("a group") + }; + + // Every grouped reason maps to its own driver reason, for every member, + // at the position given (not cell 0). + for (grouped, driver) in [ + ( + ConstituentRefusalReason::LeftBeforeSelection, + RefusalReason::OpportunityGone, + ), + ( + ConstituentRefusalReason::ScopeChanged, + RefusalReason::ScopeMoved, + ), + ( + ConstituentRefusalReason::WorldRefused, + RefusalReason::WorldRejected, + ), + ( + ConstituentRefusalReason::ProposalUndecodable, + RefusalReason::ProposalUndecodable, + ), + ] { + let (status, committed, refusals) = + fold(grouped_refusing(group, &[grouped, grouped]), group, 3); + assert!(status.is_none() && committed.is_empty()); + let expected: Vec = members + .iter() + .map(|member| Refusal { + cell: 3, + subject: member.subject, + reason: driver, + }) + .collect(); + assert_eq!(refusals, expected, "{grouped:?}"); + } + + // A cell-level refusal of a grouped cell refuses every member. + let error = ControllerError::NoOpportunity { + expected: ControllerMode::OperationalAgent, + }; + let (status, _, refusals) = fold(Err(error), group, 2); + assert!(status.is_none()); + let expected: Vec = members + .iter() + .map(|member| Refusal { + cell: 2, + subject: member.subject, + reason: RefusalReason::OpportunityGone, + }) + .collect(); + assert_eq!(refusals, expected); + + // A narrative turn the world overtook refuses its member. + let single = &cover_of(&verse, 2).await.cells[1]; + let Cell::Singleton { member, .. } = single else { + panic!("a singleton") + }; + let overtaken = crate::controllers::overtaken_narrative_run(&member.opportunity); + let (status, committed, refusals) = fold(Ok(CellRun::Narrative(overtaken)), single, 1); + assert!(status.is_none() && committed.is_empty()); + assert_eq!( + refusals, + vec![Refusal { + cell: 1, + subject: member.subject, + reason: RefusalReason::Interrupted + }] + ); +} + +#[tokio::test] +async fn a_grouped_member_whose_proposal_fails_to_decode_is_refused_not_declined() { + for finished_anyway in [false, true] { + let (_dir, verse) = Harness::fresh().await; + let port = ScriptedPort::new(move |request| { + let mut events = grouped_reply(request); + if let InferenceEvent::ToolCall { arguments, .. } = &mut events[1] { + *arguments = json!({ "text": 5 }).to_string(); + } + if finished_anyway { + events.push(InferenceEvent::ToolCall { + call_id: "done".into(), + name: "c1__finish_without_proposal".into(), + arguments: "{}".into(), + }); + } + events + }); + let mut reports = Vec::new(); + let outcome = verse + .run_under(port, u64::MAX, one_cell(1), |report| { + reports.push(report.clone()); + ControlFlow::Continue(()) + }) + .await; + let checkpoint = &outcome.record.checkpoints[0]; + assert!(checkpoint.closed); + if finished_anyway { + // A later finish is the member's decision: it declines, committed. + assert_eq!(checkpoint.committed.len(), 2); + assert!(checkpoint.refusals.is_empty()); + } else { + assert_eq!(checkpoint.committed.len(), 1); + let [refusal] = checkpoint.refusals.as_slice() else { + panic!("the undecodable member is the one refusal") + }; + assert_eq!(refusal.reason, RefusalReason::ProposalUndecodable); + assert!(reports[0].to_string().contains("proposal_undecodable")); + assert_every_member_accounted_for(&verse, &outcome).await; + } + } +} + +#[tokio::test] +async fn a_grouped_cell_behind_another_is_accounted_at_its_own_position() { + let (_dir, verse) = Harness::fresh().await; + let (group, real) = the_group(&verse).await; + let single = match &cover_of(&verse, 2).await.cells[1] { + Cell::Singleton { id, .. } => *id, + Cell::Group { .. } => panic!("the second cell of two is a singleton"), + }; + let gone = SubjectId::issue(); + open_tick_with( + &verse, + vec![ + FrozenCell { + id: single, + group: false, + members: vec![gone], + }, + FrozenCell { + id: group, + group: true, + members: real.clone(), + }, + ], + ) + .await; + let outcome = verse + .run_under(ScriptedPort::new(grouped_reply), u64::MAX, one_cell(1), |_| { + ControlFlow::Continue(()) + }) + .await; + let [checkpoint] = outcome.record.checkpoints.as_slice() else { + panic!("one tick, one checkpoint") + }; + assert!(checkpoint.closed); + let world = verse.mailbox.snapshot().await.unwrap().world_id; + let [first, second] = checkpoint.refusals.as_slice() else { + panic!("the gone cell and the refused member: {:?}", checkpoint.refusals) + }; + assert_eq!( + *first, + Refusal { + cell: 0, + subject: gone, + reason: RefusalReason::OpportunityGone + } + ); + assert_eq!(second.cell, 1); + assert_eq!(second.reason, RefusalReason::ScopeMoved); + let acted: Vec = real.iter().copied().filter(|s| *s != second.subject).collect(); + assert_eq!(acted.len(), 1); + assert_eq!(checkpoint.committed, vec![command_for(world, group, acted[0])]); +} + +#[tokio::test] +async fn a_grouped_cell_whose_lists_do_not_match_its_members_ends_the_run() { + let (_dir, verse) = Harness::fresh().await; + let cover = cover_of(&verse, 1).await; + let group = &cover.cells[0]; + let Cell::Group { id, members, .. } = group else { + panic!("a group") + }; + let run = |refused: Vec, pending| { + Ok(CellRun::Grouped(crate::controllers::GroupedRun { + cell: *id, + resolution: crate::cover::Resolution::Coarse { + constituents: members.len(), + }, + submissions: Vec::new(), + needs: Vec::new(), + unattributed: Vec::new(), + refused, + pending, + })) + }; + let refusal = |subject, reason| crate::controllers::ConstituentRefusal { subject, reason }; + let left = ConstituentRefusalReason::LeftBeforeSelection; + let broken = Some(failed("cell_membership")); + + // A finished cell that lists one of its two members. + let (status, _, _) = fold(run(vec![refusal(members[0].subject, left)], None), group, 0); + assert_eq!(status, broken); + // One that lists a subject the cell does not hold, and misses one it does. + let (status, _, _) = fold( + run( + vec![refusal(members[0].subject, left), refusal(SubjectId::issue(), left)], + None, + ), + group, + 0, + ); + assert_eq!(status, broken); + // One that lists a member twice. + let (status, _, _) = fold( + run( + vec![refusal(members[0].subject, left), refusal(members[0].subject, left)], + None, + ), + group, + 0, + ); + assert_eq!(status, broken); + // A cell that stopped pending has not listed its unreached members, and + // that is not a broken invariant: the tick stays open and they resume. + let reason = crate::controllers::ControllerPendingReason::WorldUnavailable; + let (status, committed, refusals) = fold(run(Vec::new(), Some(reason)), group, 0); + assert_eq!(status, Some(failed(pending_code(reason)))); + assert!(committed.is_empty() && refusals.is_empty()); +} + +/// A cell of three members at a later position, so that the guard is held to +/// its length, to every member, and to the cell it is asked about rather than +/// to the two-member first cell the fixture covers. +async fn three_member_group(verse: &Harness) -> Cell { + let cover = cover_of(verse, 1).await; + let Cell::Group { id, tick, members } = &cover.cells[0] else { + panic!("a group") + }; + let mut members = members.clone(); + let mut third = members[0].clone(); + third.subject = SubjectId::issue(); + members.push(third); + Cell::Group { + id: *id, + tick: *tick, + members, + } +} + +#[tokio::test] +async fn a_finished_cell_of_three_must_list_each_member_exactly_once_wherever_it_sits() { + let (_dir, verse) = Harness::fresh().await; + let group = three_member_group(&verse).await; + let Cell::Group { id, members, .. } = &group else { + panic!("a group") + }; + let left = ConstituentRefusalReason::LeftBeforeSelection; + let run = |listed: Vec| { + Ok(CellRun::Grouped(crate::controllers::GroupedRun { + cell: *id, + resolution: crate::cover::Resolution::Coarse { + constituents: members.len(), + }, + submissions: Vec::new(), + needs: Vec::new(), + unattributed: Vec::new(), + refused: listed + .into_iter() + .map(|subject| crate::controllers::ConstituentRefusal { subject, reason: left }) + .collect(), + pending: None, + })) + }; + let [a, b, c] = [members[0].subject, members[1].subject, members[2].subject]; + let broken = Some(failed("cell_membership")); + for position in [0, 1, 2] { + let (status, _, _) = fold(run(vec![a, b, c]), &group, position); + assert_eq!(status, None, "every member listed once, at cell {position}"); + // Every member listed, and one subject the cell does not hold. + let (status, _, _) = fold(run(vec![a, b, c, SubjectId::issue()]), &group, position); + assert_eq!(status, broken, "a foreign extra at cell {position}"); + // Every member listed, and one of them twice. + let (status, _, _) = fold(run(vec![a, b, c, b]), &group, position); + assert_eq!(status, broken, "a duplicate at cell {position}"); + // The right number of entries, and the last member missing: only the + // member at the end of the cell can be the one that is not there. + let (status, _, _) = fold(run(vec![a, b, SubjectId::issue()]), &group, position); + assert_eq!(status, broken, "a missing last member at cell {position}"); + let (status, _, _) = fold(run(vec![a, c, a]), &group, position); + assert_eq!(status, broken, "a missing middle member at cell {position}"); + let (status, _, _) = fold(run(vec![a, b]), &group, position); + assert_eq!(status, broken, "a short list at cell {position}"); + } +} + +#[tokio::test] +async fn a_call_naming_no_handle_is_recorded_at_the_cell_it_was_made_in() { + let (_dir, verse) = Harness::fresh().await; + let group = three_member_group(&verse).await; + let Cell::Group { id, members, .. } = &group else { + panic!("a group") + }; + for position in [0, 1, 4] { + let run = CellRun::Grouped(crate::controllers::GroupedRun { + cell: *id, + resolution: crate::cover::Resolution::Coarse { + constituents: members.len(), + }, + submissions: Vec::new(), + needs: Vec::new(), + unattributed: vec!["c9__speak".to_string()], + refused: members + .iter() + .map(|member| crate::controllers::ConstituentRefusal { + subject: member.subject, + reason: ConstituentRefusalReason::CallUnattributed, + }) + .collect(), + pending: None, + }); + let mut unattributed = Vec::new(); + let status = absorb( + Ok(run), + &group, + position, + &mut Vec::new(), + &mut Vec::new(), + &mut unattributed, + ); + assert_eq!(status, None); + assert_eq!( + unattributed, + vec![UnattributedCall { + cell: position, + tool: "c9__speak".into() + }] + ); + } +} + +#[tokio::test] +async fn a_call_naming_no_handle_is_printed_and_its_quiet_member_is_refused_not_declined() { + let (_dir, verse) = Harness::fresh().await; + let rounds = AtomicUsize::new(0); + let port = ScriptedPort::new(move |request| { + // The first inference proposes for one member and strays once; the + // round after it says nothing more. + if rounds.fetch_add(1, Ordering::SeqCst) > 0 { + return Vec::new(); + } + let mut events = grouped_reply(request); + events.truncate(1); + events.push(InferenceEvent::ToolCall { + call_id: "stray".into(), + name: "c9__speak".into(), + arguments: json!({ "text": "SENTINEL-ARGUMENT" }).to_string(), + }); + events + }); + let mut reports = Vec::new(); + let outcome = verse + .run_under(port, u64::MAX, one_cell(1), |report| { + reports.push(report.clone()); + ControlFlow::Continue(()) + }) + .await; + let [checkpoint] = outcome.record.checkpoints.as_slice() else { + panic!("one tick, one checkpoint") + }; + assert!(checkpoint.closed); + assert_eq!(checkpoint.committed.len(), 1); + let [refusal] = checkpoint.refusals.as_slice() else { + panic!("the quiet member is the one refusal") + }; + assert_eq!(refusal.reason, RefusalReason::CallUnattributed); + assert_eq!( + checkpoint.unattributed, + vec![UnattributedCall { + cell: 0, + tool: "c9__speak".into() + }] + ); + let printed = reports[0].to_string(); + assert!(printed.contains("call_unattributed"), "{printed}"); + assert!( + printed.contains("note: cell 0 call names no handle: tool \"c9__speak\""), + "{printed}" + ); + assert!(!printed.contains("SENTINEL-ARGUMENT")); + assert_every_member_accounted_for(&verse, &outcome).await; +}