play-faults fix: one send classifier, timeout canary, composed retry test, honest comments

- send_fault now classifies the reply-body read too: a body-read timeout is
  Timeout, a body cut mid-stream is NoResponse (retryable), a whole body that
  does not decode stays BadReply. The response.json() map_err is gone.
- LocalBinding.response_timeout (None = RESPONSE_TIMEOUT) lets a test reach the
  timeout path through the full PlayTable path.
- Tests: stalled, cut and malformed bodies against loopback listeners; a
  timeout canary at the port and through PlayTable (stored fault, warn line,
  card row, Debug); a PlayTable turn that survives N empty replies and faults
  past ROUND_RETRY_BUDGET.
- Comments: CREATE_BINDINGS doc reattached, backoff comments state the curve,
  close_with_fault no longer claims "endpoint-free by construction".

Not yet built or tested at this commit; verified by the following job.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Meta Vens 2026-10-09 22:31:35 +02:00
commit 6db5d0873e
5 changed files with 303 additions and 45 deletions

View file

@ -46,12 +46,13 @@ pub(crate) struct EveRouteHint {
pub(crate) transport: Option<String>,
}
/// The `world_create.v4` payload, in one place, so the button that captures it
/// and the descriptor that advertises it cannot drift.
/// The card's whole account of a turn closed by a fault.
/// The card's whole account of a turn closed by a fault: a fixed line, so
/// the row can carry no detail text at all.
const PLAY_FAULT_LINE: &str =
"The storyteller lost the thread and your turn ended. Write again to start a new turn.";
/// The `world_create.v4` payload, in one place, so the button that captures it
/// and the descriptor that advertises it cannot drift.
const CREATE_BINDINGS: [&str; 8] = [
"title",
"brief",

View file

@ -1758,8 +1758,11 @@ impl PlayTable {
async fn close_with_fault(&self, turn: &mut PlayTurn, detail: String) -> Result<(), PlayError> {
turn.state = PlayTurnState::Closed;
// R2: the one place a fault closes a turn, so the one place it is
// logged. The detail is endpoint-free by construction (the inference
// port classifies without the request URL), so it is logged whole.
// logged, whole. The local inference port builds its detail from the
// fault kind and never from the transport error's text (tests pin
// that for each local failure class). The connector and SDK ports
// build details from their own error text, which this line does not
// scrub.
tracing::warn!(turn_id = %turn.turn_id, detail = %detail, "play turn closed on a fault");
turn.fault = Some(detail);
// PA.f173: a fault can close a turn that was `AwaitingPlayer`, and
@ -1822,8 +1825,9 @@ impl PlayTable {
}
/// One inference fault's own disposition decides its fate (PA.f94): only
/// `Retryable` retries at all, bounded by `ROUND_RETRY_BUDGET` and a
/// small capped exponential backoff (`retry_delay`, zero under test);
/// `Retryable` retries at all, bounded by `ROUND_RETRY_BUDGET` and the
/// capped exponential backoff of `retry_delay` (100 ms doubling to a 20 s
/// cap, 105.5 s summed over the budget; zero under test);
/// `RecoveryRequired` and `IntegrityViolation` never retry, closing on
/// their first occurrence — retrying either again cannot succeed
/// differently. `Ok(())` means the caller's loop should try again;
@ -1844,8 +1848,10 @@ impl PlayTable {
}
}
/// A small capped exponential backoff between retries of a `Retryable`
/// inference fault (PA.f94). PA.f122: the base is an ordinary field on
/// The capped exponential backoff between retries of a `Retryable`
/// inference fault (PA.f94): 100 ms doubling to a 20 s cap, whose sum over
/// `ROUND_RETRY_BUDGET` retries must outlast a model server reload
/// (pinned by the summed-backoff test). PA.f122: the base is an ordinary field on
/// the table, not a `#[cfg(test)]` swap over the formula itself —
/// production and test both run `backoff_delay` (below, a free function,
/// compiled and directly unit-testable in every build), over the exact
@ -8204,25 +8210,22 @@ pub(crate) mod tests {
assert!(find_surface_node(&surface, "world.play.fault").is_none());
}
/// The whole path with a real local port and a dead endpoint: reqwest's
/// own error text carries the request URL, and the canary is that
/// endpoint. No byte of it may reach the stored fault, the warn line or
/// the card row; the error kind still does.
#[tokio::test]
async fn a_dead_local_endpoint_closes_the_turn_without_naming_the_endpoint() {
let (logs, _guard) = capture_logs();
let fixture = play_world(None, "player-fault-canary").await;
/// A PlayTable over the real local port at `endpoint`, with the reply
/// timeout `timeout` (None is production's) and no retry sleep.
async fn local_play_table(
endpoint: std::net::SocketAddr,
timeout: Option<Duration>,
account: &str,
directory: &std::path::Path,
) -> (WorldFixture, PlayTable) {
let fixture = play_world(None, account).await;
let personas = PersonaLane::new(
ControllerPort::new(fixture.world.clone()),
ScriptedPort::new(vec![]),
ScriptedPort::new(vec![output("proj", vec![text_event("Fine.")])]),
"gpt-5.6-sol".into(),
"gpt-5.6-sol".into(),
)
.unwrap();
let endpoint = {
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
listener.local_addr().unwrap()
};
let inference = ghostlight::open_inference(
None,
None,
@ -8230,39 +8233,185 @@ pub(crate) mod tests {
endpoint,
model_prefix: "local/".into(),
caller_runtime_id: "play-test-runtime".into(),
response_timeout: timeout,
}),
&["local/play"],
)
.unwrap();
let directory = tempfile::tempdir().unwrap();
let table = PlayTable::new(
fixture.world.clone(),
personas,
inference,
"local/play".into(),
Arc::new(Semaphore::new(2)),
directory.path().join("play-turn-v1.cc"),
directory.join("play-turn-v1.cc"),
)
.unwrap();
table.set_retry_delay_base_ms(0);
table.run(&fixture.principal, test_turn_id(431), "Hello?".into()).await.unwrap();
(fixture, table)
}
/// Runs one turn at `endpoint` through the real local port and asserts it
/// closed on a fault whose stored text contains `kind`, then that no byte
/// of the endpoint (the canary: reqwest's own error text carries the
/// request URL) reached the stored fault, the warn line, the card row,
/// or the Debug of the turn and its view.
async fn assert_faulted_turn_names_no_endpoint(
endpoint: std::net::SocketAddr,
timeout: Option<Duration>,
account: &str,
turn_number: u128,
kind: &str,
) {
let (logs, _guard) = capture_logs();
let directory = tempfile::tempdir().unwrap();
let (fixture, table) = local_play_table(endpoint, timeout, account, directory.path()).await;
table.run(&fixture.principal, test_turn_id(turn_number), "Hello?".into()).await.unwrap();
let world = fixture.world.snapshot().await.unwrap();
let view = table.current_turn_view().await.unwrap();
assert!(view.fault, "the dead endpoint must close the turn on a fault");
let surface =
crate::eve::authenticated_surface("player-fault-canary", Some(&world), Some(&view)).unwrap();
assert!(view.fault, "the endpoint failure must close the turn on a fault");
let surface = crate::eve::authenticated_surface(account, Some(&world), Some(&view)).unwrap();
let row = serde_json::to_string(find_surface_node(&surface, "world.play.fault").unwrap()).unwrap();
let stored = table.store.lock().await.current().unwrap().fault.clone().unwrap();
let log = logs.text();
assert!(stored.contains("refused the connection"), "the error kind is kept: {stored}");
for canary in [endpoint.to_string(), endpoint.ip().to_string(), "chat/completions".to_owned()] {
for (where_, text) in [("stored fault", &stored), ("warn line", &log), ("card row", &row)] {
let debug = format!("{view:?} {:?}", table.store.lock().await.current().unwrap());
assert!(stored.contains(kind), "the error kind is kept: {stored}");
for canary in [endpoint.to_string(), endpoint.ip().to_string(), endpoint.port().to_string(), "chat/completions".to_owned()] {
for (where_, text) in [("stored fault", &stored), ("warn line", &log), ("card row", &row), ("debug", &debug)] {
assert!(!text.contains(&canary), "{canary} leaked into the {where_}: {text}");
}
}
}
/// The whole path with a dead endpoint (connect refused).
#[tokio::test]
async fn a_dead_local_endpoint_closes_the_turn_without_naming_the_endpoint() {
let endpoint = {
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
listener.local_addr().unwrap()
};
assert_faulted_turn_names_no_endpoint(endpoint, None, "player-fault-canary", 431, "refused the connection").await;
}
/// The whole path with a timeout, the leg a Bonsai stall takes: the
/// listener accepts and never answers, the client times out at 200 ms,
/// and the timeout is not retried. A timeout detail built from reqwest's
/// error text carries the request URL and fails the canary here.
#[tokio::test]
async fn a_timed_out_local_endpoint_closes_the_turn_without_naming_the_endpoint() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let endpoint = listener.local_addr().unwrap();
tokio::spawn(async move {
let mut held = Vec::new();
while let Ok((stream, _)) = listener.accept().await {
held.push(stream);
}
});
assert_faulted_turn_names_no_endpoint(
endpoint,
Some(Duration::from_millis(200)),
"player-fault-timeout",
432,
"timed out",
)
.await;
}
/// Loopback server for the composed retry tests: closes the first
/// `drops` connections without a byte (a model server mid-reload), then
/// answers each later one with an `end_turn` tool call. Returns its
/// address and the count of connections seen.
fn closing_then_ending(drops: usize) -> (std::net::SocketAddr, Arc<std::sync::atomic::AtomicUsize>) {
use std::io::{Read, Write};
use std::sync::atomic::{AtomicUsize, Ordering};
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let addr = listener.local_addr().unwrap();
let seen = Arc::new(AtomicUsize::new(0));
let counter = seen.clone();
std::thread::spawn(move || {
for stream in listener.incoming() {
let Ok(mut stream) = stream else { return };
let n = counter.fetch_add(1, Ordering::SeqCst);
if n < drops {
drop(stream);
continue;
}
let mut buf = Vec::new();
let mut chunk = [0u8; 8192];
loop {
let Ok(read) = stream.read(&mut chunk) else { break };
if read == 0 {
break;
}
buf.extend_from_slice(&chunk[..read]);
let text = String::from_utf8_lossy(&buf).to_string();
if let Some(split) = text.find("\r\n\r\n") {
let len = text[..split]
.to_ascii_lowercase()
.lines()
.find_map(|l| l.strip_prefix("content-length:").map(|v| v.trim().parse::<usize>().unwrap()))
.unwrap_or(0);
if buf.len() >= split + 4 + len {
break;
}
}
}
let body = serde_json::json!({
"id": format!("resp-{n}"),
"choices": [{"message": {"role": "assistant", "content": null,
"tool_calls": [{"id": format!("c{n}"), "type": "function",
"function": {"name": END_TURN_TOOL, "arguments": "{}"}}]},
"finish_reason": "tool_calls"}],
"usage": {"prompt_tokens": 3, "completion_tokens": 2}
})
.to_string();
let _ = write!(
stream,
"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
body.len(),
body
);
let _ = stream.flush();
}
});
(addr, seen)
}
/// R3 composed: a real closed send is `Retryable`, `absorb_or_close`
/// retries it, and the turn completes once the endpoint answers. It reads
/// the production budget, so the boundary moves with it: the turn
/// survives `ROUND_RETRY_BUDGET` empty replies and faults on one more.
#[tokio::test]
async fn a_turn_survives_empty_replies_up_to_the_retry_budget_and_faults_past_it() {
let (addr, seen) = closing_then_ending(3);
let directory = tempfile::tempdir().unwrap();
let (fixture, table) = local_play_table(addr, None, "player-r3-three", directory.path()).await;
table.run(&fixture.principal, test_turn_id(900), "Hello?".into()).await.unwrap();
let (state, fault) = {
let store = table.store.lock().await;
let turn = store.current().unwrap();
(turn.state, turn.fault.clone())
};
assert_eq!(state, PlayTurnState::Closed);
assert!(fault.is_none(), "{fault:?}");
assert_eq!(seen.load(std::sync::atomic::Ordering::SeqCst), 4, "three empty replies, then the answered call");
for (drops, account, id, survives) in [
(ROUND_RETRY_BUDGET, "player-r3-edge", 901u128, true),
(ROUND_RETRY_BUDGET + 1, "player-r3-over", 902u128, false),
] {
let (addr, _seen) = closing_then_ending(drops);
let directory = tempfile::tempdir().unwrap();
let (fixture, table) = local_play_table(addr, None, account, directory.path()).await;
table.run(&fixture.principal, test_turn_id(id), "Hello?".into()).await.unwrap();
let store = table.store.lock().await;
let turn = store.current().unwrap();
assert_eq!(turn.state, PlayTurnState::Closed);
assert_eq!(turn.fault.is_none(), survives, "drops={drops} fault={:?}", turn.fault);
}
}
/// R4: a model server reload takes about 30-40 s, so the production
/// backoff summed over the whole retry budget must outlast it with
/// margin, while the first retry stays quick. Reads the production base,

View file

@ -576,6 +576,7 @@ fn open_controller(
model_prefix: std::env::var("GHOSTLIGHT_LOCAL_MODEL_PREFIX")
.unwrap_or_else(|_| DEFAULT_LOCAL_MODEL_PREFIX.into()),
caller_runtime_id: runtime_id.to_owned(),
response_timeout: None,
}),
Err(_) => None,
};

View file

@ -11445,6 +11445,7 @@ mod tests {
endpoint: local_endpoint,
model_prefix: local_model_prefix,
caller_runtime_id: runtime_id,
response_timeout: None,
}),
&models.each(),
)

View file

@ -31,6 +31,10 @@ pub struct LocalBinding {
pub endpoint: SocketAddr,
pub model_prefix: String,
pub caller_runtime_id: String,
/// How long one request may take, reply body included. `None` is
/// `RESPONSE_TIMEOUT`; a consumer or test sets it to reach the timeout
/// path without waiting that out.
pub response_timeout: Option<std::time::Duration>,
}
const LOCAL_RECEIPT_SCHEMA: &str = "ghostlight.local_inference_receipt.v1";
@ -193,30 +197,33 @@ impl LocalInferencePort {
}
.classed(class));
}
response.json::<LocalChatResponse>().await.map_err(|_| {
InferenceFault::integrity_violation(
"the local inference endpoint's reply was not the declared shape",
)
.classed(InferenceFaultClass::BadReply)
})
response.json::<LocalChatResponse>().await.map_err(send_fault)
}
}
/// The one classifier of a send that failed before any HTTP status arrived.
/// A request that got no response was never processed, so retrying it is
/// safe; only a timeout is not retried, because the endpoint may still be
/// generating. The fault carries the kind and never the error's own text:
/// `reqwest::Error`'s `Display` ends with the request URL, which would put
/// the endpoint into every log line and card that renders the detail.
/// The one classifier of a send that failed before a status arrived or while
/// its body was read. A timeout is not retried, because the endpoint may still
/// be generating. A body that arrived whole but does not decode is `BadReply`,
/// an integrity violation. Anything else means no complete reply came (a
/// refused or closed connection, a body cut off mid-stream), so retrying is
/// safe. The fault carries the kind and never the error's own text:
/// `reqwest::Error`'s `Display` ends with the request URL, so formatting it
/// would put the endpoint into every log line and card that renders the
/// detail. Tests pin this for each class.
fn send_fault(error: reqwest::Error) -> InferenceFault {
if error.is_timeout() {
InferenceFault::new("the local inference request timed out")
.classed(InferenceFaultClass::Timeout)
} else if error.is_decode() {
InferenceFault::integrity_violation(
"the local inference endpoint's reply was not the declared shape",
)
.classed(InferenceFaultClass::BadReply)
} else if error.is_connect() {
InferenceFault::retryable("the local inference endpoint refused the connection")
.classed(InferenceFaultClass::Connect)
} else {
InferenceFault::retryable("the local inference endpoint closed the connection before replying")
InferenceFault::retryable("the local inference endpoint closed the connection before a complete reply")
.classed(InferenceFaultClass::NoResponse)
}
}
@ -393,10 +400,11 @@ impl InferencePort for LocalInferencePort {
pub(super) fn open_local_port(
binding: LocalBinding,
) -> Result<Arc<dyn InferencePort>, ControllerOpenError> {
Ok(Arc::new(LocalInferencePort::new(
Ok(Arc::new(LocalInferencePort::with_timeout(
binding.endpoint,
binding.model_prefix,
binding.caller_runtime_id,
binding.response_timeout.unwrap_or(RESPONSE_TIMEOUT),
)?))
}
@ -791,6 +799,7 @@ mod tests {
endpoint: "93.184.216.34:443".parse().unwrap(),
model_prefix: DEFAULT_LOCAL_MODEL_PREFIX.into(),
caller_runtime_id: TEST_RUNTIME.into(),
response_timeout: None,
}),
&[TEST_MODEL],
)
@ -897,6 +906,7 @@ mod tests {
endpoint: "127.0.0.1:1".parse().unwrap(),
model_prefix: "claude".into(),
caller_runtime_id: TEST_RUNTIME.into(),
response_timeout: None,
}),
&["claude-opus-5"],
)
@ -1179,6 +1189,7 @@ mod tests {
endpoint: "127.0.0.1:1".parse().unwrap(),
model_prefix: String::new(),
caller_runtime_id: TEST_RUNTIME.into(),
response_timeout: None,
}),
&[TEST_MODEL],
)
@ -1576,4 +1587,99 @@ mod tests {
}
}
}
fn port_with_timeout(endpoint: SocketAddr, timeout: std::time::Duration) -> LocalInferencePort {
let _guard = client_build_lock()
.lock()
.expect("the client-build lock is never poisoned");
LocalInferencePort::with_timeout(endpoint, DEFAULT_LOCAL_MODEL_PREFIX, TEST_RUNTIME, timeout)
.expect("a loopback endpoint opens")
}
/// A loopback server that answers every request with a `200` whose
/// header promises 1000 body bytes and whose body is a single `{`, then
/// either holds the connection open (`stall`) or closes it (a cut body).
async fn truncated_body_endpoint(stall: bool) -> SocketAddr {
let listener = TcpListener::bind("127.0.0.1:0").await.expect("loopback binds");
let addr = listener.local_addr().expect("a bound listener has an address");
tokio::spawn(async move {
let mut held = Vec::new();
while let Ok((mut stream, _)) = listener.accept().await {
let mut request = vec![0u8; 65536];
let _ = stream.read(&mut request).await;
let _ = stream
.write_all(b"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: 1000\r\n\r\n{")
.await;
if stall {
held.push(stream);
} else {
let _ = stream.shutdown().await;
}
}
});
addr
}
/// One classifier: a timeout while the body streams is `Timeout`, not
/// retried. Mutants: classify the body-read error as `BadReply` again, or
/// treat a stall as retryable.
#[tokio::test]
async fn a_body_that_stalls_is_a_timeout_not_retried() {
let addr = truncated_body_endpoint(true).await;
let fault = infer_once(&port_with_timeout(addr, std::time::Duration::from_millis(300)))
.await
.expect_err("a stalled body produced an output");
assert_eq!(fault.class(), InferenceFaultClass::Timeout, "{fault:?}");
assert_eq!(fault.disposition(), InferenceFaultDisposition::RecoveryRequired, "{fault:?}");
}
/// A body cut off mid-stream (a reload dropping the connection during the
/// reply) is a reply that never completed: `NoResponse`, retryable.
#[tokio::test]
async fn a_body_cut_off_mid_stream_is_retryable() {
let addr = truncated_body_endpoint(false).await;
let fault = infer_once(&port(addr)).await.expect_err("a cut body produced an output");
assert_eq!(fault.class(), InferenceFaultClass::NoResponse, "{fault:?}");
assert_eq!(fault.disposition(), InferenceFaultDisposition::Retryable, "{fault:?}");
}
/// A body that arrived whole but is not the declared shape stays `BadReply`
/// and an integrity violation, never retried.
#[tokio::test]
async fn a_complete_malformed_body_is_a_bad_reply() {
let addr = ScriptedResponder::start(vec![(200, "not json".to_owned())]).await.endpoint();
let fault = infer_once(&port(addr)).await.expect_err("a malformed body produced an output");
assert_eq!(fault.class(), InferenceFaultClass::BadReply, "{fault:?}");
assert_eq!(fault.disposition(), InferenceFaultDisposition::IntegrityViolation, "{fault:?}");
}
/// The timeout leg of the no-endpoint rule: reqwest's `Display` for a
/// timeout ends with the request URL, so a detail built from it names the
/// canary endpoint this test bound. Covers both a stalled send and a
/// stalled body.
#[tokio::test]
async fn a_timeout_names_no_endpoint() {
let silent = {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move {
let mut held = Vec::new();
while let Ok((stream, _)) = listener.accept().await {
held.push(stream);
}
});
addr
};
let stalled_body = truncated_body_endpoint(true).await;
for addr in [silent, stalled_body] {
let fault = infer_once(&port_with_timeout(addr, std::time::Duration::from_millis(200)))
.await
.expect_err("a stalled endpoint produced an output");
assert_eq!(fault.class(), InferenceFaultClass::Timeout, "{fault:?}");
let rendered = format!("{fault} | {fault:?}");
for canary in [addr.to_string(), addr.ip().to_string(), addr.port().to_string(), "chat/completions".to_owned()] {
assert!(!rendered.contains(&canary), "{canary} leaked into {rendered}");
}
}
}
}