play-faults: classify send failures, show the fault on the card, log it, lengthen the retry window

R3: every local send that fails before a status arrives is Retryable except a timeout; the fault carries a typed InferenceFaultClass (Connect, NoResponse, Timeout, Status, BadReply) and a detail without the request URL.
R1: PlayTurnView.fault and a fixed world.play.fault card row.
R2: close_with_fault logs the turn id and the (endpoint-free) detail at warn.
R4: backoff cap 20 s, shift clamp 8: the retry budget now spans about 105 s.

Not yet built or run when committed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Meta Vens 2026-10-09 20:31:55 +02:00
commit e1c2632f03
5 changed files with 408 additions and 22 deletions

View file

@ -48,6 +48,10 @@ pub(crate) struct EveRouteHint {
/// The `world_create.v4` payload, in one place, so the button that captures it
/// and the descriptor that advertises it cannot drift.
/// The card's whole account of a turn closed by a fault.
const PLAY_FAULT_LINE: &str =
"The storyteller lost the thread and your turn ended. Write again to start a new turn.";
const CREATE_BINDINGS: [&str; 8] = [
"title",
"brief",
@ -477,6 +481,17 @@ pub(crate) fn authenticated_surface(
"children":[]
}));
}
// R1: a turn that died on its model says so. The line is
// fixed: the fault detail stays in the store and the log,
// never on the card (invariant 8, PA.f187).
if play.is_some_and(|view| view.fault) {
play_rows.push(json!({
"id":"world.play.fault",
"kind":"text",
"props":{"value":PLAY_FAULT_LINE},
"children":[]
}));
}
// PA.f193-D ("the interrupted turn"; renumbered from
// the PA.f188 duplicate this file's own comment used
// to carry — PA.f188 names only `f295a3a`'s

View file

@ -1243,6 +1243,10 @@ pub(crate) struct PlayTurnView {
/// the exact thing `admit`'s own `try_lock_owned` was about to refuse as
/// `TurnBusy` anyway.
pub(crate) run_in_progress: bool,
/// Whether the turn closed on a fault (`PlayTurn::fault` is set). The
/// detail stays in the stored row and the log; the card shows a fixed
/// player-safe line, never the detail (invariant 8, PA.f187).
pub(crate) fault: bool,
}
/// One outcome of executing a round's already-decided tool calls.
@ -1688,6 +1692,7 @@ impl PlayTable {
refusal: turn.refusal.clone(),
revision: store.state.revision,
run_in_progress,
fault: turn.fault.is_some(),
})
}
@ -1752,6 +1757,10 @@ impl PlayTable {
/// this helper only ever writes the one turn's own record.
async fn close_with_fault(&self, turn: &mut PlayTurn, detail: String) -> Result<(), PlayError> {
turn.state = PlayTurnState::Closed;
// R2: the one place a fault closes a turn, so the one place it is
// logged. The detail is endpoint-free by construction (the inference
// port classifies without the request URL), so it is logged whole.
tracing::warn!(turn_id = %turn.turn_id, detail = %detail, "play turn closed on a fault");
turn.fault = Some(detail);
// PA.f173: a fault can close a turn that was `AwaitingPlayer`, and
// `answer_turn` is not the only door out of that state — clear the
@ -2968,8 +2977,8 @@ const RETRY_DELAY_BASE_MS: u64 = 100;
/// whether that base is `RETRY_DELAY_BASE_MS` (production) or whatever a
/// test set it to.
fn backoff_delay(base_ms: u64, attempt: usize) -> Duration {
const CAP: u64 = 5_000;
Duration::from_millis(base_ms.saturating_mul(1u64 << attempt.min(6)).min(CAP))
const CAP: u64 = 20_000;
Duration::from_millis(base_ms.saturating_mul(1u64 << attempt.min(8)).min(CAP))
}
/// A round is incomplete when it holds a tool call this table has not yet
@ -3767,6 +3776,7 @@ pub(crate) mod tests {
refusal: None,
revision: 1,
run_in_progress: false,
fault: false,
};
let surface = crate::eve::authenticated_surface(
"player-interrupted-turn",
@ -3791,6 +3801,7 @@ pub(crate) mod tests {
refusal: None,
revision: 1,
run_in_progress: false,
fault: false,
};
let awaiting_surface = crate::eve::authenticated_surface(
"player-interrupted-turn",
@ -3830,6 +3841,7 @@ pub(crate) mod tests {
refusal: None,
revision: 1,
run_in_progress: true,
fault: false,
};
let surface = crate::eve::authenticated_surface(
"player-interrupted-turn-busy",
@ -3871,6 +3883,7 @@ pub(crate) mod tests {
refusal: None,
revision: 1,
run_in_progress: true,
fault: false,
};
let surface = crate::eve::authenticated_surface(
"player-round-in-flight",
@ -3892,6 +3905,7 @@ pub(crate) mod tests {
refusal: None,
revision: 1,
run_in_progress: false,
fault: false,
};
let idle_surface = crate::eve::authenticated_surface(
"player-round-in-flight",
@ -3926,6 +3940,7 @@ pub(crate) mod tests {
refusal: None,
revision: 1,
run_in_progress: false,
fault: false,
};
let surface = crate::eve::authenticated_surface(
"player-stalled-turn-text",
@ -8079,10 +8094,11 @@ pub(crate) mod tests {
assert_eq!(backoff_delay(RETRY_DELAY_BASE_MS, 1), Duration::from_millis(200));
assert_eq!(backoff_delay(RETRY_DELAY_BASE_MS, 2), Duration::from_millis(400));
assert_eq!(backoff_delay(RETRY_DELAY_BASE_MS, 5), Duration::from_millis(3_200), "not yet capped");
assert_eq!(backoff_delay(RETRY_DELAY_BASE_MS, 6), Duration::from_millis(5_000), "capped");
assert_eq!(backoff_delay(RETRY_DELAY_BASE_MS, 7), Duration::from_millis(12_800), "not yet capped");
assert_eq!(backoff_delay(RETRY_DELAY_BASE_MS, 8), Duration::from_millis(20_000), "capped");
assert_eq!(
backoff_delay(RETRY_DELAY_BASE_MS, 1_000_000),
Duration::from_millis(5_000),
Duration::from_millis(20_000),
"a very large attempt count must stay capped, never overflow or panic"
);
assert_eq!(
@ -8092,6 +8108,174 @@ pub(crate) mod tests {
);
}
/// Captures what a tracing subscriber writes, for the test that reads the
/// warn line a fault closes a turn with.
#[derive(Clone, Default)]
struct LogCapture(Arc<StdMutex<Vec<u8>>>);
impl std::io::Write for LogCapture {
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
self.0.lock().unwrap().extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for LogCapture {
type Writer = LogCapture;
fn make_writer(&'a self) -> Self::Writer {
self.clone()
}
}
impl LogCapture {
fn text(&self) -> String {
String::from_utf8_lossy(&self.0.lock().unwrap()).into_owned()
}
}
fn capture_logs() -> (LogCapture, tracing::subscriber::DefaultGuard) {
let capture = LogCapture::default();
let guard = tracing::subscriber::set_default(
tracing_subscriber::fmt()
.with_writer(capture.clone())
.with_ansi(false)
.finish(),
);
(capture, guard)
}
/// R1 and R2. A turn closed by a fault logs its id and the fault's
/// detail at warn, and the owner's card shows the fixed fault row until
/// the view stops reporting a fault. Mutations: dropping the `warn!` in
/// `close_with_fault` fails the log assertions; `fault: false` in
/// `current_turn_view` fails the row assertion.
#[tokio::test]
async fn a_fault_closed_turn_is_logged_and_shown_on_the_card() {
let (logs, _guard) = capture_logs();
let fixture = play_world(None, "player-fault-card").await;
let personas = PersonaLane::new(
ControllerPort::new(fixture.world.clone()),
ScriptedPort::new(vec![]),
"gpt-5.6-sol".into(),
"gpt-5.6-sol".into(),
)
.unwrap();
let directory = tempfile::tempdir().unwrap();
let table = PlayTable::new(
fixture.world.clone(),
personas,
Arc::new(AlwaysFaultyPort::new()),
"gpt-5.6-terra".into(),
Arc::new(Semaphore::new(2)),
directory.path().join("play-turn-v1.cc"),
)
.unwrap();
let turn_id = test_turn_id(430);
table.run(&fixture.principal, turn_id.clone(), "Hello?".into()).await.unwrap();
let line = logs
.text()
.lines()
.find(|line| line.contains("play turn closed on a fault"))
.map(str::to_owned)
.expect("closing a turn on a fault must log a line");
assert!(line.contains("WARN"), "{line}");
assert!(line.contains(&turn_id), "the line must name the turn: {line}");
assert!(line.contains("provider unavailable"), "the line must carry the fault detail: {line}");
let world = fixture.world.snapshot().await.unwrap();
let view = table.current_turn_view().await.unwrap();
assert!(view.fault);
let surface =
crate::eve::authenticated_surface("player-fault-card", Some(&world), Some(&view)).unwrap();
let row = find_surface_node(&surface, "world.play.fault")
.expect("a fault-closed turn must show the fault row");
let encoded = serde_json::to_string(row).unwrap();
assert!(!encoded.contains("provider unavailable"), "the row must not carry the detail: {encoded}");
let healthy = PlayTurnView { fault: false, ..view };
let surface =
crate::eve::authenticated_surface("player-fault-card", Some(&world), Some(&healthy)).unwrap();
assert!(find_surface_node(&surface, "world.play.fault").is_none());
}
/// The whole path with a real local port and a dead endpoint: reqwest's
/// own error text carries the request URL, and the canary is that
/// endpoint. No byte of it may reach the stored fault, the warn line or
/// the card row; the error kind still does.
#[tokio::test]
async fn a_dead_local_endpoint_closes_the_turn_without_naming_the_endpoint() {
let (logs, _guard) = capture_logs();
let fixture = play_world(None, "player-fault-canary").await;
let personas = PersonaLane::new(
ControllerPort::new(fixture.world.clone()),
ScriptedPort::new(vec![]),
"gpt-5.6-sol".into(),
"gpt-5.6-sol".into(),
)
.unwrap();
let endpoint = {
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
listener.local_addr().unwrap()
};
let inference = ghostlight::open_inference(
None,
None,
Some(ghostlight::LocalBinding {
endpoint,
model_prefix: "local/".into(),
caller_runtime_id: "play-test-runtime".into(),
}),
&["local/play"],
)
.unwrap();
let directory = tempfile::tempdir().unwrap();
let table = PlayTable::new(
fixture.world.clone(),
personas,
inference,
"local/play".into(),
Arc::new(Semaphore::new(2)),
directory.path().join("play-turn-v1.cc"),
)
.unwrap();
table.set_retry_delay_base_ms(0);
table.run(&fixture.principal, test_turn_id(431), "Hello?".into()).await.unwrap();
let world = fixture.world.snapshot().await.unwrap();
let view = table.current_turn_view().await.unwrap();
assert!(view.fault, "the dead endpoint must close the turn on a fault");
let surface =
crate::eve::authenticated_surface("player-fault-canary", Some(&world), Some(&view)).unwrap();
let row = serde_json::to_string(find_surface_node(&surface, "world.play.fault").unwrap()).unwrap();
let stored = table.store.lock().await.current().unwrap().fault.clone().unwrap();
let log = logs.text();
assert!(stored.contains("refused the connection"), "the error kind is kept: {stored}");
for canary in [endpoint.to_string(), endpoint.ip().to_string(), "chat/completions".to_owned()] {
for (where_, text) in [("stored fault", &stored), ("warn line", &log), ("card row", &row)] {
assert!(!text.contains(&canary), "{canary} leaked into the {where_}: {text}");
}
}
}
/// R4: a model server reload takes about 30-40 s, so the production
/// backoff summed over the whole retry budget must outlast it with
/// margin, while the first retry stays quick. Reads the production base,
/// budget and curve; no copy of any of them.
#[test]
fn the_production_backoff_outlasts_a_model_reload() {
let total: Duration = (0..ROUND_RETRY_BUDGET)
.map(|attempt| backoff_delay(RETRY_DELAY_BASE_MS, attempt))
.sum();
assert!(total >= Duration::from_secs(90), "the retry window is only {total:?}");
assert!(backoff_delay(RETRY_DELAY_BASE_MS, 0) <= Duration::from_millis(200));
}
/// PA.f66a: exhausting the round budget closes the turn and narrates,
/// rather than leaving it `Running` with nothing said.
#[tokio::test]

View file

@ -198,9 +198,31 @@ impl InferenceOutput {
#[error("{detail}")]
pub struct InferenceFault {
disposition: InferenceFaultDisposition,
class: InferenceFaultClass,
detail: String,
}
/// What kind of trouble a fault is, as a typed value: the one place a reader
/// learns it without parsing `detail`. Disposition says what the owning flow
/// may do (retry, recover, quarantine); the class says what happened. A port
/// that can tell classifies at construction (`classed`), and every other
/// constructor leaves `Unclassified`.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum InferenceFaultClass {
/// The connection to the endpoint could not be made.
Connect,
/// The connection was made but no HTTP status arrived: closed, reset or
/// an empty reply.
NoResponse,
/// The request ran out of time.
Timeout,
/// The endpoint answered with this non-success status.
Status(u16),
/// The endpoint's reply was not the declared shape.
BadReply,
Unclassified,
}
/// The three-way disposition an `InferenceFault` carries, readable but not
/// independently constructible: `InferenceFault`'s own named constructors
/// (`retryable`, `integrity_violation`, `recovery_required`) remain the only
@ -217,6 +239,7 @@ impl InferenceFault {
pub(super) fn new(detail: impl Into<String>) -> Self {
Self {
disposition: InferenceFaultDisposition::RecoveryRequired,
class: InferenceFaultClass::Unclassified,
detail: detail.into(),
}
}
@ -225,6 +248,7 @@ impl InferenceFault {
pub fn retryable(detail: impl Into<String>) -> Self {
Self {
disposition: InferenceFaultDisposition::Retryable,
class: InferenceFaultClass::Unclassified,
detail: detail.into(),
}
}
@ -235,6 +259,7 @@ impl InferenceFault {
pub fn integrity_violation(detail: impl Into<String>) -> Self {
Self {
disposition: InferenceFaultDisposition::IntegrityViolation,
class: InferenceFaultClass::Unclassified,
detail: detail.into(),
}
}
@ -245,6 +270,17 @@ impl InferenceFault {
Self::new(detail)
}
/// Names what kind of trouble this fault is.
pub(super) fn classed(mut self, class: InferenceFaultClass) -> Self {
self.class = class;
self
}
/// The fault's class, read-only.
pub fn class(&self) -> InferenceFaultClass {
self.class
}
pub(super) fn requires_recovery(&self) -> bool {
self.disposition == InferenceFaultDisposition::RecoveryRequired
}

View file

@ -83,7 +83,7 @@ pub use controllers::{
ControllerNeed, ControllerPendingReason, ControllerRunner, ControllerWork,
ControllerWorkCustody, ControllerWorkLookup, ControllerWorkStore, ControllerWorkStoreError,
ControllerWorkWrite,
GroupedCheckpoint, InferenceEvent, InferenceFault, InferenceFaultDisposition, InferenceOutput,
GroupedCheckpoint, InferenceEvent, InferenceFault, InferenceFaultClass, InferenceFaultDisposition, InferenceOutput,
InferencePort,
InferencePurpose, InferenceRequest, NarrativeCheckpoint, NarrativeRun, OperationalCheckpoint,
OperationalRun, PersonaLane, PreparedInference, SubmissionDisposition, ToolResultOracle,

View file

@ -9,7 +9,7 @@
//! flight through the same port at once.
use super::controllers::{
ControllerOpenError, InferenceEvent, InferenceFault, InferenceOutput, InferencePort,
ControllerOpenError, InferenceEvent, InferenceFault, InferenceFaultClass, InferenceOutput, InferencePort,
InferenceRequest, PreparedInference, REQUEST_EXPIRY, RESPONSE_TIMEOUT, call_id_is_valid,
tool_name_is_valid, unix_ms,
};
@ -146,6 +146,17 @@ impl LocalInferencePort {
endpoint: SocketAddr,
prefix: impl Into<String>,
caller_runtime_id: impl Into<String>,
) -> Result<Self, ControllerOpenError> {
Self::with_timeout(endpoint, prefix, caller_runtime_id, RESPONSE_TIMEOUT)
}
/// `new` with the response timeout named, so a test can reach the timeout
/// path without waiting out `RESPONSE_TIMEOUT`.
fn with_timeout(
endpoint: SocketAddr,
prefix: impl Into<String>,
caller_runtime_id: impl Into<String>,
timeout: std::time::Duration,
) -> Result<Self, ControllerOpenError> {
if !endpoint.ip().is_loopback() {
return Err(ControllerOpenError::LocalEndpointNotLoopback { endpoint });
@ -154,7 +165,7 @@ impl LocalInferencePort {
client: reqwest::Client::builder()
.no_proxy()
.redirect(reqwest::redirect::Policy::none())
.timeout(RESPONSE_TIMEOUT)
.timeout(timeout)
.build()
.expect("the local inference HTTP client builds with no custom TLS material"),
endpoint,
@ -170,32 +181,46 @@ impl LocalInferencePort {
.json(&body)
.send()
.await
.map_err(|error| {
if error.is_connect() {
InferenceFault::retryable(format!(
"the local inference endpoint refused the connection: {error}"
))
} else {
InferenceFault::new(format!("the local inference request failed: {error}"))
}
})?;
.map_err(send_fault)?;
let status = response.status();
if !status.is_success() {
let detail = format!("the local inference endpoint returned {status}");
let class = InferenceFaultClass::Status(status.as_u16());
return Err(if status.as_u16() == 429 || status.as_u16() == 503 {
InferenceFault::retryable(detail)
} else {
InferenceFault::new(detail)
});
}
.classed(class));
}
response.json::<LocalChatResponse>().await.map_err(|error| {
InferenceFault::integrity_violation(format!(
"the local inference endpoint's reply was not the declared shape: {error}"
))
response.json::<LocalChatResponse>().await.map_err(|_| {
InferenceFault::integrity_violation(
"the local inference endpoint's reply was not the declared shape",
)
.classed(InferenceFaultClass::BadReply)
})
}
}
/// The one classifier of a send that failed before any HTTP status arrived.
/// A request that got no response was never processed, so retrying it is
/// safe; only a timeout is not retried, because the endpoint may still be
/// generating. The fault carries the kind and never the error's own text:
/// `reqwest::Error`'s `Display` ends with the request URL, which would put
/// the endpoint into every log line and card that renders the detail.
fn send_fault(error: reqwest::Error) -> InferenceFault {
if error.is_timeout() {
InferenceFault::new("the local inference request timed out")
.classed(InferenceFaultClass::Timeout)
} else if error.is_connect() {
InferenceFault::retryable("the local inference endpoint refused the connection")
.classed(InferenceFaultClass::Connect)
} else {
InferenceFault::retryable("the local inference endpoint closed the connection before replying")
.classed(InferenceFaultClass::NoResponse)
}
}
/// Lowers one prepared request to the OpenAI chat-completions body: the
/// system message is the request's instructions, every input item maps to
/// one message in order, and every offered tool is declared `strict`. No
@ -378,7 +403,9 @@ pub(super) fn open_local_port(
#[cfg(test)]
mod tests {
use super::*;
use crate::controllers::{InferencePurpose, RequestShape, open_inference, tool_request};
use crate::controllers::{
InferenceFaultDisposition, InferencePurpose, RequestShape, open_inference, tool_request,
};
use crate::elaboration::{ElaborationLoopEvaluation, SEED_ROUND_BUDGET, evaluate_elaboration_loop};
use crate::patch::{RECORD_GAP_PATCH_TOOL, SUBMIT_PATCH_TOOL, patch_tools};
use crate::sdk_inference::{DEFAULT_SDK_MODEL_PREFIX, RoutedInferencePort, SdkBinding};
@ -1425,4 +1452,128 @@ mod tests {
"a changed response_id did not change the receipt digest"
);
}
fn plain_request() -> InferenceRequest {
tool_request(
CommandId::new(),
0,
InferencePurpose::Persona,
TEST_MODEL,
"Respond only in natural prose.",
vec![CodexInputItem::UserText {
text: "Say something true.".into(),
}],
Vec::<CodexToolDefinition>::new(),
RequestShape {
max_output_tokens: 1_200,
parallel_tool_calls: false,
},
)
.expect("the request builds")
}
async fn infer_once(port: &LocalInferencePort) -> Result<InferenceOutput, InferenceFault> {
let prepared = port.prepare(plain_request()).expect("the port prepares");
port.infer(prepared).await
}
/// A loopback server that accepts the first `drops` connections and
/// closes each without a byte (the shape of a model server reloading),
/// then answers every later connection with `reply`.
async fn closing_then_serving(drops: usize, reply: String) -> SocketAddr {
let listener = TcpListener::bind("127.0.0.1:0").await.expect("loopback binds");
let addr = listener.local_addr().expect("a bound listener has an address");
let state = Arc::new(Mutex::new(ScriptedResponderState {
replies: VecDeque::from(vec![(200, reply)]),
..Default::default()
}));
tokio::spawn(async move {
let mut seen = 0;
loop {
let Ok((mut stream, _)) = listener.accept().await else {
return;
};
if seen < drops {
seen += 1;
let _ = stream.shutdown().await;
continue;
}
serve_one(stream, &state).await;
}
});
addr
}
/// R3. A send that dies after the connection was made but before any
/// status arrives is `Retryable` and classed `NoResponse`; the recovered
/// endpoint then answers the very next call from the same port. Mutation:
/// classify only `is_connect()` errors as retryable, everything else as
/// recovery-required (the pre-cut shape) and the first assertion fails.
#[tokio::test]
async fn an_empty_reply_is_retryable_and_the_recovered_endpoint_answers() {
let addr = closing_then_serving(1, text_reply("back")).await;
let port = port(addr);
let fault = infer_once(&port).await.expect_err("a closed connection produced an output");
assert_eq!(fault.disposition(), InferenceFaultDisposition::Retryable, "{fault:?}");
assert_eq!(fault.class(), InferenceFaultClass::NoResponse, "{fault:?}");
infer_once(&port).await.expect("the recovered endpoint answers the retry");
}
/// R3's other half: a timeout is not retried, because the endpoint may
/// still be generating. The listener accepts and never answers.
#[tokio::test]
async fn a_timed_out_send_is_recovery_required_not_retried() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move {
let mut held = Vec::new();
while let Ok((stream, _)) = listener.accept().await {
held.push(stream);
}
});
let port = {
let _guard = client_build_lock().lock().expect("the client-build lock is never poisoned");
LocalInferencePort::with_timeout(
addr,
DEFAULT_LOCAL_MODEL_PREFIX,
TEST_RUNTIME,
std::time::Duration::from_millis(150),
)
.expect("a loopback endpoint opens")
};
let fault = infer_once(&port).await.expect_err("a silent endpoint produced an output");
assert_eq!(fault.disposition(), InferenceFaultDisposition::RecoveryRequired, "{fault:?}");
assert_eq!(fault.class(), InferenceFaultClass::Timeout, "{fault:?}");
}
/// The class is a typed value for every way a send can fail, and the
/// endpoint appears in none of the fault's renderings: the canary is the
/// endpoint this test bound, so a `Display` that carries the request URL
/// (reqwest's does) fails here.
#[tokio::test]
async fn every_send_failure_is_classed_and_names_no_endpoint() {
let closed = {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
drop(listener);
addr
};
let dropping = closing_then_serving(usize::MAX, String::new()).await;
let status = ScriptedResponder::start(vec![(400, "{}".to_owned())]).await.endpoint();
let malformed = ScriptedResponder::start(vec![(200, "not json".to_owned())]).await.endpoint();
let cases = [
(closed, InferenceFaultClass::Connect),
(dropping, InferenceFaultClass::NoResponse),
(status, InferenceFaultClass::Status(400)),
(malformed, InferenceFaultClass::BadReply),
];
for (addr, expected) in cases {
let fault = infer_once(&port(addr)).await.expect_err("the send was expected to fail");
assert_eq!(fault.class(), expected, "{fault:?}");
let rendered = format!("{fault} | {fault:?}");
for canary in [addr.to_string(), addr.ip().to_string(), "chat/completions".to_owned()] {
assert!(!rendered.contains(&canary), "{canary} leaked into {rendered}");
}
}
}
}