diff --git a/crates/ghostlight-dungeon/src/eve.rs b/crates/ghostlight-dungeon/src/eve.rs index 95ffe27..62e38d6 100644 --- a/crates/ghostlight-dungeon/src/eve.rs +++ b/crates/ghostlight-dungeon/src/eve.rs @@ -46,12 +46,13 @@ pub(crate) struct EveRouteHint { pub(crate) transport: Option, } -/// The `world_create.v4` payload, in one place, so the button that captures it -/// and the descriptor that advertises it cannot drift. -/// The card's whole account of a turn closed by a fault. +/// The card's whole account of a turn closed by a fault: a fixed line, so +/// the row can carry no detail text at all. const PLAY_FAULT_LINE: &str = "The storyteller lost the thread and your turn ended. Write again to start a new turn."; +/// The `world_create.v4` payload, in one place, so the button that captures it +/// and the descriptor that advertises it cannot drift. const CREATE_BINDINGS: [&str; 8] = [ "title", "brief", diff --git a/crates/ghostlight-dungeon/src/play.rs b/crates/ghostlight-dungeon/src/play.rs index 1586ce6..b8a340f 100644 --- a/crates/ghostlight-dungeon/src/play.rs +++ b/crates/ghostlight-dungeon/src/play.rs @@ -1758,8 +1758,11 @@ impl PlayTable { async fn close_with_fault(&self, turn: &mut PlayTurn, detail: String) -> Result<(), PlayError> { turn.state = PlayTurnState::Closed; // R2: the one place a fault closes a turn, so the one place it is - // logged. The detail is endpoint-free by construction (the inference - // port classifies without the request URL), so it is logged whole. + // logged, whole. The local inference port builds its detail from the + // fault kind and never from the transport error's text (tests pin + // that for each local failure class). The connector and SDK ports + // build details from their own error text, which this line does not + // scrub. tracing::warn!(turn_id = %turn.turn_id, detail = %detail, "play turn closed on a fault"); turn.fault = Some(detail); // PA.f173: a fault can close a turn that was `AwaitingPlayer`, and @@ -1822,8 +1825,9 @@ impl PlayTable { } /// One inference fault's own disposition decides its fate (PA.f94): only - /// `Retryable` retries at all, bounded by `ROUND_RETRY_BUDGET` and a - /// small capped exponential backoff (`retry_delay`, zero under test); + /// `Retryable` retries at all, bounded by `ROUND_RETRY_BUDGET` and the + /// capped exponential backoff of `retry_delay` (100 ms doubling to a 20 s + /// cap, 105.5 s summed over the budget; zero under test); /// `RecoveryRequired` and `IntegrityViolation` never retry, closing on /// their first occurrence — retrying either again cannot succeed /// differently. `Ok(())` means the caller's loop should try again; @@ -1844,8 +1848,10 @@ impl PlayTable { } } - /// A small capped exponential backoff between retries of a `Retryable` - /// inference fault (PA.f94). PA.f122: the base is an ordinary field on + /// The capped exponential backoff between retries of a `Retryable` + /// inference fault (PA.f94): 100 ms doubling to a 20 s cap, whose sum over + /// `ROUND_RETRY_BUDGET` retries must outlast a model server reload + /// (pinned by the summed-backoff test). PA.f122: the base is an ordinary field on /// the table, not a `#[cfg(test)]` swap over the formula itself — /// production and test both run `backoff_delay` (below, a free function, /// compiled and directly unit-testable in every build), over the exact @@ -8204,25 +8210,22 @@ pub(crate) mod tests { assert!(find_surface_node(&surface, "world.play.fault").is_none()); } - /// The whole path with a real local port and a dead endpoint: reqwest's - /// own error text carries the request URL, and the canary is that - /// endpoint. No byte of it may reach the stored fault, the warn line or - /// the card row; the error kind still does. - #[tokio::test] - async fn a_dead_local_endpoint_closes_the_turn_without_naming_the_endpoint() { - let (logs, _guard) = capture_logs(); - let fixture = play_world(None, "player-fault-canary").await; + /// A PlayTable over the real local port at `endpoint`, with the reply + /// timeout `timeout` (None is production's) and no retry sleep. + async fn local_play_table( + endpoint: std::net::SocketAddr, + timeout: Option, + account: &str, + directory: &std::path::Path, + ) -> (WorldFixture, PlayTable) { + let fixture = play_world(None, account).await; let personas = PersonaLane::new( ControllerPort::new(fixture.world.clone()), - ScriptedPort::new(vec![]), + ScriptedPort::new(vec![output("proj", vec![text_event("Fine.")])]), "gpt-5.6-sol".into(), "gpt-5.6-sol".into(), ) .unwrap(); - let endpoint = { - let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); - listener.local_addr().unwrap() - }; let inference = ghostlight::open_inference( None, None, @@ -8230,39 +8233,185 @@ pub(crate) mod tests { endpoint, model_prefix: "local/".into(), caller_runtime_id: "play-test-runtime".into(), + response_timeout: timeout, }), &["local/play"], ) .unwrap(); - let directory = tempfile::tempdir().unwrap(); let table = PlayTable::new( fixture.world.clone(), personas, inference, "local/play".into(), Arc::new(Semaphore::new(2)), - directory.path().join("play-turn-v1.cc"), + directory.join("play-turn-v1.cc"), ) .unwrap(); table.set_retry_delay_base_ms(0); - table.run(&fixture.principal, test_turn_id(431), "Hello?".into()).await.unwrap(); + (fixture, table) + } + + /// Runs one turn at `endpoint` through the real local port and asserts it + /// closed on a fault whose stored text contains `kind`, then that no byte + /// of the endpoint (the canary: reqwest's own error text carries the + /// request URL) reached the stored fault, the warn line, the card row, + /// or the Debug of the turn and its view. + async fn assert_faulted_turn_names_no_endpoint( + endpoint: std::net::SocketAddr, + timeout: Option, + account: &str, + turn_number: u128, + kind: &str, + ) { + let (logs, _guard) = capture_logs(); + let directory = tempfile::tempdir().unwrap(); + let (fixture, table) = local_play_table(endpoint, timeout, account, directory.path()).await; + table.run(&fixture.principal, test_turn_id(turn_number), "Hello?".into()).await.unwrap(); let world = fixture.world.snapshot().await.unwrap(); let view = table.current_turn_view().await.unwrap(); - assert!(view.fault, "the dead endpoint must close the turn on a fault"); - let surface = - crate::eve::authenticated_surface("player-fault-canary", Some(&world), Some(&view)).unwrap(); + assert!(view.fault, "the endpoint failure must close the turn on a fault"); + let surface = crate::eve::authenticated_surface(account, Some(&world), Some(&view)).unwrap(); let row = serde_json::to_string(find_surface_node(&surface, "world.play.fault").unwrap()).unwrap(); let stored = table.store.lock().await.current().unwrap().fault.clone().unwrap(); let log = logs.text(); - assert!(stored.contains("refused the connection"), "the error kind is kept: {stored}"); - for canary in [endpoint.to_string(), endpoint.ip().to_string(), "chat/completions".to_owned()] { - for (where_, text) in [("stored fault", &stored), ("warn line", &log), ("card row", &row)] { + let debug = format!("{view:?} {:?}", table.store.lock().await.current().unwrap()); + assert!(stored.contains(kind), "the error kind is kept: {stored}"); + for canary in [endpoint.to_string(), endpoint.ip().to_string(), endpoint.port().to_string(), "chat/completions".to_owned()] { + for (where_, text) in [("stored fault", &stored), ("warn line", &log), ("card row", &row), ("debug", &debug)] { assert!(!text.contains(&canary), "{canary} leaked into the {where_}: {text}"); } } } + /// The whole path with a dead endpoint (connect refused). + #[tokio::test] + async fn a_dead_local_endpoint_closes_the_turn_without_naming_the_endpoint() { + let endpoint = { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.local_addr().unwrap() + }; + assert_faulted_turn_names_no_endpoint(endpoint, None, "player-fault-canary", 431, "refused the connection").await; + } + + /// The whole path with a timeout, the leg a Bonsai stall takes: the + /// listener accepts and never answers, the client times out at 200 ms, + /// and the timeout is not retried. A timeout detail built from reqwest's + /// error text carries the request URL and fails the canary here. + #[tokio::test] + async fn a_timed_out_local_endpoint_closes_the_turn_without_naming_the_endpoint() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = listener.local_addr().unwrap(); + tokio::spawn(async move { + let mut held = Vec::new(); + while let Ok((stream, _)) = listener.accept().await { + held.push(stream); + } + }); + assert_faulted_turn_names_no_endpoint( + endpoint, + Some(Duration::from_millis(200)), + "player-fault-timeout", + 432, + "timed out", + ) + .await; + } + + /// Loopback server for the composed retry tests: closes the first + /// `drops` connections without a byte (a model server mid-reload), then + /// answers each later one with an `end_turn` tool call. Returns its + /// address and the count of connections seen. + fn closing_then_ending(drops: usize) -> (std::net::SocketAddr, Arc) { + use std::io::{Read, Write}; + use std::sync::atomic::{AtomicUsize, Ordering}; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let seen = Arc::new(AtomicUsize::new(0)); + let counter = seen.clone(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { return }; + let n = counter.fetch_add(1, Ordering::SeqCst); + if n < drops { + drop(stream); + continue; + } + let mut buf = Vec::new(); + let mut chunk = [0u8; 8192]; + loop { + let Ok(read) = stream.read(&mut chunk) else { break }; + if read == 0 { + break; + } + buf.extend_from_slice(&chunk[..read]); + let text = String::from_utf8_lossy(&buf).to_string(); + if let Some(split) = text.find("\r\n\r\n") { + let len = text[..split] + .to_ascii_lowercase() + .lines() + .find_map(|l| l.strip_prefix("content-length:").map(|v| v.trim().parse::().unwrap())) + .unwrap_or(0); + if buf.len() >= split + 4 + len { + break; + } + } + } + let body = serde_json::json!({ + "id": format!("resp-{n}"), + "choices": [{"message": {"role": "assistant", "content": null, + "tool_calls": [{"id": format!("c{n}"), "type": "function", + "function": {"name": END_TURN_TOOL, "arguments": "{}"}}]}, + "finish_reason": "tool_calls"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 2} + }) + .to_string(); + let _ = write!( + stream, + "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", + body.len(), + body + ); + let _ = stream.flush(); + } + }); + (addr, seen) + } + + /// R3 composed: a real closed send is `Retryable`, `absorb_or_close` + /// retries it, and the turn completes once the endpoint answers. It reads + /// the production budget, so the boundary moves with it: the turn + /// survives `ROUND_RETRY_BUDGET` empty replies and faults on one more. + #[tokio::test] + async fn a_turn_survives_empty_replies_up_to_the_retry_budget_and_faults_past_it() { + let (addr, seen) = closing_then_ending(3); + let directory = tempfile::tempdir().unwrap(); + let (fixture, table) = local_play_table(addr, None, "player-r3-three", directory.path()).await; + table.run(&fixture.principal, test_turn_id(900), "Hello?".into()).await.unwrap(); + let (state, fault) = { + let store = table.store.lock().await; + let turn = store.current().unwrap(); + (turn.state, turn.fault.clone()) + }; + assert_eq!(state, PlayTurnState::Closed); + assert!(fault.is_none(), "{fault:?}"); + assert_eq!(seen.load(std::sync::atomic::Ordering::SeqCst), 4, "three empty replies, then the answered call"); + + for (drops, account, id, survives) in [ + (ROUND_RETRY_BUDGET, "player-r3-edge", 901u128, true), + (ROUND_RETRY_BUDGET + 1, "player-r3-over", 902u128, false), + ] { + let (addr, _seen) = closing_then_ending(drops); + let directory = tempfile::tempdir().unwrap(); + let (fixture, table) = local_play_table(addr, None, account, directory.path()).await; + table.run(&fixture.principal, test_turn_id(id), "Hello?".into()).await.unwrap(); + let store = table.store.lock().await; + let turn = store.current().unwrap(); + assert_eq!(turn.state, PlayTurnState::Closed); + assert_eq!(turn.fault.is_none(), survives, "drops={drops} fault={:?}", turn.fault); + } + } + /// R4: a model server reload takes about 30-40 s, so the production /// backoff summed over the whole retry budget must outlast it with /// margin, while the first retry stays quick. Reads the production base, diff --git a/crates/ghostlight-dungeon/src/runtime.rs b/crates/ghostlight-dungeon/src/runtime.rs index 6f83252..dfbb870 100644 --- a/crates/ghostlight-dungeon/src/runtime.rs +++ b/crates/ghostlight-dungeon/src/runtime.rs @@ -576,6 +576,7 @@ fn open_controller( model_prefix: std::env::var("GHOSTLIGHT_LOCAL_MODEL_PREFIX") .unwrap_or_else(|_| DEFAULT_LOCAL_MODEL_PREFIX.into()), caller_runtime_id: runtime_id.to_owned(), + response_timeout: None, }), Err(_) => None, }; diff --git a/crates/ghostlight/src/controllers.rs b/crates/ghostlight/src/controllers.rs index 9b05a0d..f369bd6 100644 --- a/crates/ghostlight/src/controllers.rs +++ b/crates/ghostlight/src/controllers.rs @@ -11445,6 +11445,7 @@ mod tests { endpoint: local_endpoint, model_prefix: local_model_prefix, caller_runtime_id: runtime_id, + response_timeout: None, }), &models.each(), ) diff --git a/crates/ghostlight/src/local_inference.rs b/crates/ghostlight/src/local_inference.rs index b1ba2c3..ba68883 100644 --- a/crates/ghostlight/src/local_inference.rs +++ b/crates/ghostlight/src/local_inference.rs @@ -31,6 +31,10 @@ pub struct LocalBinding { pub endpoint: SocketAddr, pub model_prefix: String, pub caller_runtime_id: String, + /// How long one request may take, reply body included. `None` is + /// `RESPONSE_TIMEOUT`; a consumer or test sets it to reach the timeout + /// path without waiting that out. + pub response_timeout: Option, } const LOCAL_RECEIPT_SCHEMA: &str = "ghostlight.local_inference_receipt.v1"; @@ -193,30 +197,33 @@ impl LocalInferencePort { } .classed(class)); } - response.json::().await.map_err(|_| { - InferenceFault::integrity_violation( - "the local inference endpoint's reply was not the declared shape", - ) - .classed(InferenceFaultClass::BadReply) - }) + response.json::().await.map_err(send_fault) } } -/// The one classifier of a send that failed before any HTTP status arrived. -/// A request that got no response was never processed, so retrying it is -/// safe; only a timeout is not retried, because the endpoint may still be -/// generating. The fault carries the kind and never the error's own text: -/// `reqwest::Error`'s `Display` ends with the request URL, which would put -/// the endpoint into every log line and card that renders the detail. +/// The one classifier of a send that failed before a status arrived or while +/// its body was read. A timeout is not retried, because the endpoint may still +/// be generating. A body that arrived whole but does not decode is `BadReply`, +/// an integrity violation. Anything else means no complete reply came (a +/// refused or closed connection, a body cut off mid-stream), so retrying is +/// safe. The fault carries the kind and never the error's own text: +/// `reqwest::Error`'s `Display` ends with the request URL, so formatting it +/// would put the endpoint into every log line and card that renders the +/// detail. Tests pin this for each class. fn send_fault(error: reqwest::Error) -> InferenceFault { if error.is_timeout() { InferenceFault::new("the local inference request timed out") .classed(InferenceFaultClass::Timeout) + } else if error.is_decode() { + InferenceFault::integrity_violation( + "the local inference endpoint's reply was not the declared shape", + ) + .classed(InferenceFaultClass::BadReply) } else if error.is_connect() { InferenceFault::retryable("the local inference endpoint refused the connection") .classed(InferenceFaultClass::Connect) } else { - InferenceFault::retryable("the local inference endpoint closed the connection before replying") + InferenceFault::retryable("the local inference endpoint closed the connection before a complete reply") .classed(InferenceFaultClass::NoResponse) } } @@ -393,10 +400,11 @@ impl InferencePort for LocalInferencePort { pub(super) fn open_local_port( binding: LocalBinding, ) -> Result, ControllerOpenError> { - Ok(Arc::new(LocalInferencePort::new( + Ok(Arc::new(LocalInferencePort::with_timeout( binding.endpoint, binding.model_prefix, binding.caller_runtime_id, + binding.response_timeout.unwrap_or(RESPONSE_TIMEOUT), )?)) } @@ -791,6 +799,7 @@ mod tests { endpoint: "93.184.216.34:443".parse().unwrap(), model_prefix: DEFAULT_LOCAL_MODEL_PREFIX.into(), caller_runtime_id: TEST_RUNTIME.into(), + response_timeout: None, }), &[TEST_MODEL], ) @@ -897,6 +906,7 @@ mod tests { endpoint: "127.0.0.1:1".parse().unwrap(), model_prefix: "claude".into(), caller_runtime_id: TEST_RUNTIME.into(), + response_timeout: None, }), &["claude-opus-5"], ) @@ -1179,6 +1189,7 @@ mod tests { endpoint: "127.0.0.1:1".parse().unwrap(), model_prefix: String::new(), caller_runtime_id: TEST_RUNTIME.into(), + response_timeout: None, }), &[TEST_MODEL], ) @@ -1576,4 +1587,99 @@ mod tests { } } } + + fn port_with_timeout(endpoint: SocketAddr, timeout: std::time::Duration) -> LocalInferencePort { + let _guard = client_build_lock() + .lock() + .expect("the client-build lock is never poisoned"); + LocalInferencePort::with_timeout(endpoint, DEFAULT_LOCAL_MODEL_PREFIX, TEST_RUNTIME, timeout) + .expect("a loopback endpoint opens") + } + + /// A loopback server that answers every request with a `200` whose + /// header promises 1000 body bytes and whose body is a single `{`, then + /// either holds the connection open (`stall`) or closes it (a cut body). + async fn truncated_body_endpoint(stall: bool) -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("loopback binds"); + let addr = listener.local_addr().expect("a bound listener has an address"); + tokio::spawn(async move { + let mut held = Vec::new(); + while let Ok((mut stream, _)) = listener.accept().await { + let mut request = vec![0u8; 65536]; + let _ = stream.read(&mut request).await; + let _ = stream + .write_all(b"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: 1000\r\n\r\n{") + .await; + if stall { + held.push(stream); + } else { + let _ = stream.shutdown().await; + } + } + }); + addr + } + + /// One classifier: a timeout while the body streams is `Timeout`, not + /// retried. Mutants: classify the body-read error as `BadReply` again, or + /// treat a stall as retryable. + #[tokio::test] + async fn a_body_that_stalls_is_a_timeout_not_retried() { + let addr = truncated_body_endpoint(true).await; + let fault = infer_once(&port_with_timeout(addr, std::time::Duration::from_millis(300))) + .await + .expect_err("a stalled body produced an output"); + assert_eq!(fault.class(), InferenceFaultClass::Timeout, "{fault:?}"); + assert_eq!(fault.disposition(), InferenceFaultDisposition::RecoveryRequired, "{fault:?}"); + } + + /// A body cut off mid-stream (a reload dropping the connection during the + /// reply) is a reply that never completed: `NoResponse`, retryable. + #[tokio::test] + async fn a_body_cut_off_mid_stream_is_retryable() { + let addr = truncated_body_endpoint(false).await; + let fault = infer_once(&port(addr)).await.expect_err("a cut body produced an output"); + assert_eq!(fault.class(), InferenceFaultClass::NoResponse, "{fault:?}"); + assert_eq!(fault.disposition(), InferenceFaultDisposition::Retryable, "{fault:?}"); + } + + /// A body that arrived whole but is not the declared shape stays `BadReply` + /// and an integrity violation, never retried. + #[tokio::test] + async fn a_complete_malformed_body_is_a_bad_reply() { + let addr = ScriptedResponder::start(vec![(200, "not json".to_owned())]).await.endpoint(); + let fault = infer_once(&port(addr)).await.expect_err("a malformed body produced an output"); + assert_eq!(fault.class(), InferenceFaultClass::BadReply, "{fault:?}"); + assert_eq!(fault.disposition(), InferenceFaultDisposition::IntegrityViolation, "{fault:?}"); + } + + /// The timeout leg of the no-endpoint rule: reqwest's `Display` for a + /// timeout ends with the request URL, so a detail built from it names the + /// canary endpoint this test bound. Covers both a stalled send and a + /// stalled body. + #[tokio::test] + async fn a_timeout_names_no_endpoint() { + let silent = { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + let mut held = Vec::new(); + while let Ok((stream, _)) = listener.accept().await { + held.push(stream); + } + }); + addr + }; + let stalled_body = truncated_body_endpoint(true).await; + for addr in [silent, stalled_body] { + let fault = infer_once(&port_with_timeout(addr, std::time::Duration::from_millis(200))) + .await + .expect_err("a stalled endpoint produced an output"); + assert_eq!(fault.class(), InferenceFaultClass::Timeout, "{fault:?}"); + let rendered = format!("{fault} | {fault:?}"); + for canary in [addr.to_string(), addr.ip().to_string(), addr.port().to_string(), "chat/completions".to_owned()] { + assert!(!rendered.contains(&canary), "{canary} leaked into {rendered}"); + } + } + } }