From e1c2632f03b84d319a6e4de558da3f42be604683 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 9 Oct 2026 20:31:55 +0200 Subject: [PATCH] play-faults: classify send failures, show the fault on the card, log it, lengthen the retry window R3: every local send that fails before a status arrives is Retryable except a timeout; the fault carries a typed InferenceFaultClass (Connect, NoResponse, Timeout, Status, BadReply) and a detail without the request URL. R1: PlayTurnView.fault and a fixed world.play.fault card row. R2: close_with_fault logs the turn id and the (endpoint-free) detail at warn. R4: backoff cap 20 s, shift clamp 8: the retry budget now spans about 105 s. Not yet built or run when committed. Co-Authored-By: Claude Opus 5.5 --- crates/ghostlight-dungeon/src/eve.rs | 15 ++ crates/ghostlight-dungeon/src/play.rs | 192 ++++++++++++++++++++++- crates/ghostlight/src/controllers.rs | 36 +++++ crates/ghostlight/src/lib.rs | 2 +- crates/ghostlight/src/local_inference.rs | 185 ++++++++++++++++++++-- 5 files changed, 408 insertions(+), 22 deletions(-) diff --git a/crates/ghostlight-dungeon/src/eve.rs b/crates/ghostlight-dungeon/src/eve.rs index 5faf7e7..95ffe27 100644 --- a/crates/ghostlight-dungeon/src/eve.rs +++ b/crates/ghostlight-dungeon/src/eve.rs @@ -48,6 +48,10 @@ pub(crate) struct EveRouteHint { /// The `world_create.v4` payload, in one place, so the button that captures it /// and the descriptor that advertises it cannot drift. +/// The card's whole account of a turn closed by a fault. +const PLAY_FAULT_LINE: &str = + "The storyteller lost the thread and your turn ended. Write again to start a new turn."; + const CREATE_BINDINGS: [&str; 8] = [ "title", "brief", @@ -477,6 +481,17 @@ pub(crate) fn authenticated_surface( "children":[] })); } + // R1: a turn that died on its model says so. The line is + // fixed: the fault detail stays in the store and the log, + // never on the card (invariant 8, PA.f187). + if play.is_some_and(|view| view.fault) { + play_rows.push(json!({ + "id":"world.play.fault", + "kind":"text", + "props":{"value":PLAY_FAULT_LINE}, + "children":[] + })); + } // PA.f193-D ("the interrupted turn"; renumbered from // the PA.f188 duplicate this file's own comment used // to carry — PA.f188 names only `f295a3a`'s diff --git a/crates/ghostlight-dungeon/src/play.rs b/crates/ghostlight-dungeon/src/play.rs index a0f0f0d..a7dc346 100644 --- a/crates/ghostlight-dungeon/src/play.rs +++ b/crates/ghostlight-dungeon/src/play.rs @@ -1243,6 +1243,10 @@ pub(crate) struct PlayTurnView { /// the exact thing `admit`'s own `try_lock_owned` was about to refuse as /// `TurnBusy` anyway. pub(crate) run_in_progress: bool, + /// Whether the turn closed on a fault (`PlayTurn::fault` is set). The + /// detail stays in the stored row and the log; the card shows a fixed + /// player-safe line, never the detail (invariant 8, PA.f187). + pub(crate) fault: bool, } /// One outcome of executing a round's already-decided tool calls. @@ -1688,6 +1692,7 @@ impl PlayTable { refusal: turn.refusal.clone(), revision: store.state.revision, run_in_progress, + fault: turn.fault.is_some(), }) } @@ -1752,6 +1757,10 @@ impl PlayTable { /// this helper only ever writes the one turn's own record. async fn close_with_fault(&self, turn: &mut PlayTurn, detail: String) -> Result<(), PlayError> { turn.state = PlayTurnState::Closed; + // R2: the one place a fault closes a turn, so the one place it is + // logged. The detail is endpoint-free by construction (the inference + // port classifies without the request URL), so it is logged whole. + tracing::warn!(turn_id = %turn.turn_id, detail = %detail, "play turn closed on a fault"); turn.fault = Some(detail); // PA.f173: a fault can close a turn that was `AwaitingPlayer`, and // `answer_turn` is not the only door out of that state — clear the @@ -2968,8 +2977,8 @@ const RETRY_DELAY_BASE_MS: u64 = 100; /// whether that base is `RETRY_DELAY_BASE_MS` (production) or whatever a /// test set it to. fn backoff_delay(base_ms: u64, attempt: usize) -> Duration { - const CAP: u64 = 5_000; - Duration::from_millis(base_ms.saturating_mul(1u64 << attempt.min(6)).min(CAP)) + const CAP: u64 = 20_000; + Duration::from_millis(base_ms.saturating_mul(1u64 << attempt.min(8)).min(CAP)) } /// A round is incomplete when it holds a tool call this table has not yet @@ -3767,6 +3776,7 @@ pub(crate) mod tests { refusal: None, revision: 1, run_in_progress: false, + fault: false, }; let surface = crate::eve::authenticated_surface( "player-interrupted-turn", @@ -3791,6 +3801,7 @@ pub(crate) mod tests { refusal: None, revision: 1, run_in_progress: false, + fault: false, }; let awaiting_surface = crate::eve::authenticated_surface( "player-interrupted-turn", @@ -3830,6 +3841,7 @@ pub(crate) mod tests { refusal: None, revision: 1, run_in_progress: true, + fault: false, }; let surface = crate::eve::authenticated_surface( "player-interrupted-turn-busy", @@ -3871,6 +3883,7 @@ pub(crate) mod tests { refusal: None, revision: 1, run_in_progress: true, + fault: false, }; let surface = crate::eve::authenticated_surface( "player-round-in-flight", @@ -3892,6 +3905,7 @@ pub(crate) mod tests { refusal: None, revision: 1, run_in_progress: false, + fault: false, }; let idle_surface = crate::eve::authenticated_surface( "player-round-in-flight", @@ -3926,6 +3940,7 @@ pub(crate) mod tests { refusal: None, revision: 1, run_in_progress: false, + fault: false, }; let surface = crate::eve::authenticated_surface( "player-stalled-turn-text", @@ -8079,10 +8094,11 @@ pub(crate) mod tests { assert_eq!(backoff_delay(RETRY_DELAY_BASE_MS, 1), Duration::from_millis(200)); assert_eq!(backoff_delay(RETRY_DELAY_BASE_MS, 2), Duration::from_millis(400)); assert_eq!(backoff_delay(RETRY_DELAY_BASE_MS, 5), Duration::from_millis(3_200), "not yet capped"); - assert_eq!(backoff_delay(RETRY_DELAY_BASE_MS, 6), Duration::from_millis(5_000), "capped"); + assert_eq!(backoff_delay(RETRY_DELAY_BASE_MS, 7), Duration::from_millis(12_800), "not yet capped"); + assert_eq!(backoff_delay(RETRY_DELAY_BASE_MS, 8), Duration::from_millis(20_000), "capped"); assert_eq!( backoff_delay(RETRY_DELAY_BASE_MS, 1_000_000), - Duration::from_millis(5_000), + Duration::from_millis(20_000), "a very large attempt count must stay capped, never overflow or panic" ); assert_eq!( @@ -8092,6 +8108,174 @@ pub(crate) mod tests { ); } + /// Captures what a tracing subscriber writes, for the test that reads the + /// warn line a fault closes a turn with. + #[derive(Clone, Default)] + struct LogCapture(Arc>>); + + impl std::io::Write for LogCapture { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for LogCapture { + type Writer = LogCapture; + + fn make_writer(&'a self) -> Self::Writer { + self.clone() + } + } + + impl LogCapture { + fn text(&self) -> String { + String::from_utf8_lossy(&self.0.lock().unwrap()).into_owned() + } + } + + fn capture_logs() -> (LogCapture, tracing::subscriber::DefaultGuard) { + let capture = LogCapture::default(); + let guard = tracing::subscriber::set_default( + tracing_subscriber::fmt() + .with_writer(capture.clone()) + .with_ansi(false) + .finish(), + ); + (capture, guard) + } + + /// R1 and R2. A turn closed by a fault logs its id and the fault's + /// detail at warn, and the owner's card shows the fixed fault row until + /// the view stops reporting a fault. Mutations: dropping the `warn!` in + /// `close_with_fault` fails the log assertions; `fault: false` in + /// `current_turn_view` fails the row assertion. + #[tokio::test] + async fn a_fault_closed_turn_is_logged_and_shown_on_the_card() { + let (logs, _guard) = capture_logs(); + let fixture = play_world(None, "player-fault-card").await; + let personas = PersonaLane::new( + ControllerPort::new(fixture.world.clone()), + ScriptedPort::new(vec![]), + "gpt-5.6-sol".into(), + "gpt-5.6-sol".into(), + ) + .unwrap(); + let directory = tempfile::tempdir().unwrap(); + let table = PlayTable::new( + fixture.world.clone(), + personas, + Arc::new(AlwaysFaultyPort::new()), + "gpt-5.6-terra".into(), + Arc::new(Semaphore::new(2)), + directory.path().join("play-turn-v1.cc"), + ) + .unwrap(); + let turn_id = test_turn_id(430); + table.run(&fixture.principal, turn_id.clone(), "Hello?".into()).await.unwrap(); + + let line = logs + .text() + .lines() + .find(|line| line.contains("play turn closed on a fault")) + .map(str::to_owned) + .expect("closing a turn on a fault must log a line"); + assert!(line.contains("WARN"), "{line}"); + assert!(line.contains(&turn_id), "the line must name the turn: {line}"); + assert!(line.contains("provider unavailable"), "the line must carry the fault detail: {line}"); + + let world = fixture.world.snapshot().await.unwrap(); + let view = table.current_turn_view().await.unwrap(); + assert!(view.fault); + let surface = + crate::eve::authenticated_surface("player-fault-card", Some(&world), Some(&view)).unwrap(); + let row = find_surface_node(&surface, "world.play.fault") + .expect("a fault-closed turn must show the fault row"); + let encoded = serde_json::to_string(row).unwrap(); + assert!(!encoded.contains("provider unavailable"), "the row must not carry the detail: {encoded}"); + + let healthy = PlayTurnView { fault: false, ..view }; + let surface = + crate::eve::authenticated_surface("player-fault-card", Some(&world), Some(&healthy)).unwrap(); + assert!(find_surface_node(&surface, "world.play.fault").is_none()); + } + + /// The whole path with a real local port and a dead endpoint: reqwest's + /// own error text carries the request URL, and the canary is that + /// endpoint. No byte of it may reach the stored fault, the warn line or + /// the card row; the error kind still does. + #[tokio::test] + async fn a_dead_local_endpoint_closes_the_turn_without_naming_the_endpoint() { + let (logs, _guard) = capture_logs(); + let fixture = play_world(None, "player-fault-canary").await; + let personas = PersonaLane::new( + ControllerPort::new(fixture.world.clone()), + ScriptedPort::new(vec![]), + "gpt-5.6-sol".into(), + "gpt-5.6-sol".into(), + ) + .unwrap(); + let endpoint = { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.local_addr().unwrap() + }; + let inference = ghostlight::open_inference( + None, + None, + Some(ghostlight::LocalBinding { + endpoint, + model_prefix: "local/".into(), + caller_runtime_id: "play-test-runtime".into(), + }), + &["local/play"], + ) + .unwrap(); + let directory = tempfile::tempdir().unwrap(); + let table = PlayTable::new( + fixture.world.clone(), + personas, + inference, + "local/play".into(), + Arc::new(Semaphore::new(2)), + directory.path().join("play-turn-v1.cc"), + ) + .unwrap(); + table.set_retry_delay_base_ms(0); + table.run(&fixture.principal, test_turn_id(431), "Hello?".into()).await.unwrap(); + + let world = fixture.world.snapshot().await.unwrap(); + let view = table.current_turn_view().await.unwrap(); + assert!(view.fault, "the dead endpoint must close the turn on a fault"); + let surface = + crate::eve::authenticated_surface("player-fault-canary", Some(&world), Some(&view)).unwrap(); + let row = serde_json::to_string(find_surface_node(&surface, "world.play.fault").unwrap()).unwrap(); + let stored = table.store.lock().await.current().unwrap().fault.clone().unwrap(); + let log = logs.text(); + assert!(stored.contains("refused the connection"), "the error kind is kept: {stored}"); + for canary in [endpoint.to_string(), endpoint.ip().to_string(), "chat/completions".to_owned()] { + for (where_, text) in [("stored fault", &stored), ("warn line", &log), ("card row", &row)] { + assert!(!text.contains(&canary), "{canary} leaked into the {where_}: {text}"); + } + } + } + + /// R4: a model server reload takes about 30-40 s, so the production + /// backoff summed over the whole retry budget must outlast it with + /// margin, while the first retry stays quick. Reads the production base, + /// budget and curve; no copy of any of them. + #[test] + fn the_production_backoff_outlasts_a_model_reload() { + let total: Duration = (0..ROUND_RETRY_BUDGET) + .map(|attempt| backoff_delay(RETRY_DELAY_BASE_MS, attempt)) + .sum(); + assert!(total >= Duration::from_secs(90), "the retry window is only {total:?}"); + assert!(backoff_delay(RETRY_DELAY_BASE_MS, 0) <= Duration::from_millis(200)); + } + /// PA.f66a: exhausting the round budget closes the turn and narrates, /// rather than leaving it `Running` with nothing said. #[tokio::test] diff --git a/crates/ghostlight/src/controllers.rs b/crates/ghostlight/src/controllers.rs index 47a409d..9b05a0d 100644 --- a/crates/ghostlight/src/controllers.rs +++ b/crates/ghostlight/src/controllers.rs @@ -198,9 +198,31 @@ impl InferenceOutput { #[error("{detail}")] pub struct InferenceFault { disposition: InferenceFaultDisposition, + class: InferenceFaultClass, detail: String, } +/// What kind of trouble a fault is, as a typed value: the one place a reader +/// learns it without parsing `detail`. Disposition says what the owning flow +/// may do (retry, recover, quarantine); the class says what happened. A port +/// that can tell classifies at construction (`classed`), and every other +/// constructor leaves `Unclassified`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum InferenceFaultClass { + /// The connection to the endpoint could not be made. + Connect, + /// The connection was made but no HTTP status arrived: closed, reset or + /// an empty reply. + NoResponse, + /// The request ran out of time. + Timeout, + /// The endpoint answered with this non-success status. + Status(u16), + /// The endpoint's reply was not the declared shape. + BadReply, + Unclassified, +} + /// The three-way disposition an `InferenceFault` carries, readable but not /// independently constructible: `InferenceFault`'s own named constructors /// (`retryable`, `integrity_violation`, `recovery_required`) remain the only @@ -217,6 +239,7 @@ impl InferenceFault { pub(super) fn new(detail: impl Into) -> Self { Self { disposition: InferenceFaultDisposition::RecoveryRequired, + class: InferenceFaultClass::Unclassified, detail: detail.into(), } } @@ -225,6 +248,7 @@ impl InferenceFault { pub fn retryable(detail: impl Into) -> Self { Self { disposition: InferenceFaultDisposition::Retryable, + class: InferenceFaultClass::Unclassified, detail: detail.into(), } } @@ -235,6 +259,7 @@ impl InferenceFault { pub fn integrity_violation(detail: impl Into) -> Self { Self { disposition: InferenceFaultDisposition::IntegrityViolation, + class: InferenceFaultClass::Unclassified, detail: detail.into(), } } @@ -245,6 +270,17 @@ impl InferenceFault { Self::new(detail) } + /// Names what kind of trouble this fault is. + pub(super) fn classed(mut self, class: InferenceFaultClass) -> Self { + self.class = class; + self + } + + /// The fault's class, read-only. + pub fn class(&self) -> InferenceFaultClass { + self.class + } + pub(super) fn requires_recovery(&self) -> bool { self.disposition == InferenceFaultDisposition::RecoveryRequired } diff --git a/crates/ghostlight/src/lib.rs b/crates/ghostlight/src/lib.rs index 652c492..6576eab 100644 --- a/crates/ghostlight/src/lib.rs +++ b/crates/ghostlight/src/lib.rs @@ -83,7 +83,7 @@ pub use controllers::{ ControllerNeed, ControllerPendingReason, ControllerRunner, ControllerWork, ControllerWorkCustody, ControllerWorkLookup, ControllerWorkStore, ControllerWorkStoreError, ControllerWorkWrite, - GroupedCheckpoint, InferenceEvent, InferenceFault, InferenceFaultDisposition, InferenceOutput, + GroupedCheckpoint, InferenceEvent, InferenceFault, InferenceFaultClass, InferenceFaultDisposition, InferenceOutput, InferencePort, InferencePurpose, InferenceRequest, NarrativeCheckpoint, NarrativeRun, OperationalCheckpoint, OperationalRun, PersonaLane, PreparedInference, SubmissionDisposition, ToolResultOracle, diff --git a/crates/ghostlight/src/local_inference.rs b/crates/ghostlight/src/local_inference.rs index 8ff0111..b1ba2c3 100644 --- a/crates/ghostlight/src/local_inference.rs +++ b/crates/ghostlight/src/local_inference.rs @@ -9,7 +9,7 @@ //! flight through the same port at once. use super::controllers::{ - ControllerOpenError, InferenceEvent, InferenceFault, InferenceOutput, InferencePort, + ControllerOpenError, InferenceEvent, InferenceFault, InferenceFaultClass, InferenceOutput, InferencePort, InferenceRequest, PreparedInference, REQUEST_EXPIRY, RESPONSE_TIMEOUT, call_id_is_valid, tool_name_is_valid, unix_ms, }; @@ -146,6 +146,17 @@ impl LocalInferencePort { endpoint: SocketAddr, prefix: impl Into, caller_runtime_id: impl Into, + ) -> Result { + Self::with_timeout(endpoint, prefix, caller_runtime_id, RESPONSE_TIMEOUT) + } + + /// `new` with the response timeout named, so a test can reach the timeout + /// path without waiting out `RESPONSE_TIMEOUT`. + fn with_timeout( + endpoint: SocketAddr, + prefix: impl Into, + caller_runtime_id: impl Into, + timeout: std::time::Duration, ) -> Result { if !endpoint.ip().is_loopback() { return Err(ControllerOpenError::LocalEndpointNotLoopback { endpoint }); @@ -154,7 +165,7 @@ impl LocalInferencePort { client: reqwest::Client::builder() .no_proxy() .redirect(reqwest::redirect::Policy::none()) - .timeout(RESPONSE_TIMEOUT) + .timeout(timeout) .build() .expect("the local inference HTTP client builds with no custom TLS material"), endpoint, @@ -170,32 +181,46 @@ impl LocalInferencePort { .json(&body) .send() .await - .map_err(|error| { - if error.is_connect() { - InferenceFault::retryable(format!( - "the local inference endpoint refused the connection: {error}" - )) - } else { - InferenceFault::new(format!("the local inference request failed: {error}")) - } - })?; + .map_err(send_fault)?; let status = response.status(); if !status.is_success() { let detail = format!("the local inference endpoint returned {status}"); + let class = InferenceFaultClass::Status(status.as_u16()); return Err(if status.as_u16() == 429 || status.as_u16() == 503 { InferenceFault::retryable(detail) } else { InferenceFault::new(detail) - }); + } + .classed(class)); } - response.json::().await.map_err(|error| { - InferenceFault::integrity_violation(format!( - "the local inference endpoint's reply was not the declared shape: {error}" - )) + response.json::().await.map_err(|_| { + InferenceFault::integrity_violation( + "the local inference endpoint's reply was not the declared shape", + ) + .classed(InferenceFaultClass::BadReply) }) } } +/// The one classifier of a send that failed before any HTTP status arrived. +/// A request that got no response was never processed, so retrying it is +/// safe; only a timeout is not retried, because the endpoint may still be +/// generating. The fault carries the kind and never the error's own text: +/// `reqwest::Error`'s `Display` ends with the request URL, which would put +/// the endpoint into every log line and card that renders the detail. +fn send_fault(error: reqwest::Error) -> InferenceFault { + if error.is_timeout() { + InferenceFault::new("the local inference request timed out") + .classed(InferenceFaultClass::Timeout) + } else if error.is_connect() { + InferenceFault::retryable("the local inference endpoint refused the connection") + .classed(InferenceFaultClass::Connect) + } else { + InferenceFault::retryable("the local inference endpoint closed the connection before replying") + .classed(InferenceFaultClass::NoResponse) + } +} + /// Lowers one prepared request to the OpenAI chat-completions body: the /// system message is the request's instructions, every input item maps to /// one message in order, and every offered tool is declared `strict`. No @@ -378,7 +403,9 @@ pub(super) fn open_local_port( #[cfg(test)] mod tests { use super::*; - use crate::controllers::{InferencePurpose, RequestShape, open_inference, tool_request}; + use crate::controllers::{ + InferenceFaultDisposition, InferencePurpose, RequestShape, open_inference, tool_request, + }; use crate::elaboration::{ElaborationLoopEvaluation, SEED_ROUND_BUDGET, evaluate_elaboration_loop}; use crate::patch::{RECORD_GAP_PATCH_TOOL, SUBMIT_PATCH_TOOL, patch_tools}; use crate::sdk_inference::{DEFAULT_SDK_MODEL_PREFIX, RoutedInferencePort, SdkBinding}; @@ -1425,4 +1452,128 @@ mod tests { "a changed response_id did not change the receipt digest" ); } + + fn plain_request() -> InferenceRequest { + tool_request( + CommandId::new(), + 0, + InferencePurpose::Persona, + TEST_MODEL, + "Respond only in natural prose.", + vec![CodexInputItem::UserText { + text: "Say something true.".into(), + }], + Vec::::new(), + RequestShape { + max_output_tokens: 1_200, + parallel_tool_calls: false, + }, + ) + .expect("the request builds") + } + + async fn infer_once(port: &LocalInferencePort) -> Result { + let prepared = port.prepare(plain_request()).expect("the port prepares"); + port.infer(prepared).await + } + + /// A loopback server that accepts the first `drops` connections and + /// closes each without a byte (the shape of a model server reloading), + /// then answers every later connection with `reply`. + async fn closing_then_serving(drops: usize, reply: String) -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("loopback binds"); + let addr = listener.local_addr().expect("a bound listener has an address"); + let state = Arc::new(Mutex::new(ScriptedResponderState { + replies: VecDeque::from(vec![(200, reply)]), + ..Default::default() + })); + tokio::spawn(async move { + let mut seen = 0; + loop { + let Ok((mut stream, _)) = listener.accept().await else { + return; + }; + if seen < drops { + seen += 1; + let _ = stream.shutdown().await; + continue; + } + serve_one(stream, &state).await; + } + }); + addr + } + + /// R3. A send that dies after the connection was made but before any + /// status arrives is `Retryable` and classed `NoResponse`; the recovered + /// endpoint then answers the very next call from the same port. Mutation: + /// classify only `is_connect()` errors as retryable, everything else as + /// recovery-required (the pre-cut shape) and the first assertion fails. + #[tokio::test] + async fn an_empty_reply_is_retryable_and_the_recovered_endpoint_answers() { + let addr = closing_then_serving(1, text_reply("back")).await; + let port = port(addr); + let fault = infer_once(&port).await.expect_err("a closed connection produced an output"); + assert_eq!(fault.disposition(), InferenceFaultDisposition::Retryable, "{fault:?}"); + assert_eq!(fault.class(), InferenceFaultClass::NoResponse, "{fault:?}"); + infer_once(&port).await.expect("the recovered endpoint answers the retry"); + } + + /// R3's other half: a timeout is not retried, because the endpoint may + /// still be generating. The listener accepts and never answers. + #[tokio::test] + async fn a_timed_out_send_is_recovery_required_not_retried() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + let mut held = Vec::new(); + while let Ok((stream, _)) = listener.accept().await { + held.push(stream); + } + }); + let port = { + let _guard = client_build_lock().lock().expect("the client-build lock is never poisoned"); + LocalInferencePort::with_timeout( + addr, + DEFAULT_LOCAL_MODEL_PREFIX, + TEST_RUNTIME, + std::time::Duration::from_millis(150), + ) + .expect("a loopback endpoint opens") + }; + let fault = infer_once(&port).await.expect_err("a silent endpoint produced an output"); + assert_eq!(fault.disposition(), InferenceFaultDisposition::RecoveryRequired, "{fault:?}"); + assert_eq!(fault.class(), InferenceFaultClass::Timeout, "{fault:?}"); + } + + /// The class is a typed value for every way a send can fail, and the + /// endpoint appears in none of the fault's renderings: the canary is the + /// endpoint this test bound, so a `Display` that carries the request URL + /// (reqwest's does) fails here. + #[tokio::test] + async fn every_send_failure_is_classed_and_names_no_endpoint() { + let closed = { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + drop(listener); + addr + }; + let dropping = closing_then_serving(usize::MAX, String::new()).await; + let status = ScriptedResponder::start(vec![(400, "{}".to_owned())]).await.endpoint(); + let malformed = ScriptedResponder::start(vec![(200, "not json".to_owned())]).await.endpoint(); + let cases = [ + (closed, InferenceFaultClass::Connect), + (dropping, InferenceFaultClass::NoResponse), + (status, InferenceFaultClass::Status(400)), + (malformed, InferenceFaultClass::BadReply), + ]; + for (addr, expected) in cases { + let fault = infer_once(&port(addr)).await.expect_err("the send was expected to fail"); + assert_eq!(fault.class(), expected, "{fault:?}"); + let rendered = format!("{fault} | {fault:?}"); + for canary in [addr.to_string(), addr.ip().to_string(), "chat/completions".to_owned()] { + assert!(!rendered.contains(&canary), "{canary} leaked into {rendered}"); + } + } + } }