play-faults: classify send failures, show the fault on the card, log it, lengthen the retry window
R3: every local send that fails before a status arrives is Retryable except a timeout; the fault carries a typed InferenceFaultClass (Connect, NoResponse, Timeout, Status, BadReply) and a detail without the request URL. R1: PlayTurnView.fault and a fixed world.play.fault card row. R2: close_with_fault logs the turn id and the (endpoint-free) detail at warn. R4: backoff cap 20 s, shift clamp 8: the retry budget now spans about 105 s. Not yet built or run when committed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
aa18e28925
commit
e1c2632f03
5 changed files with 408 additions and 22 deletions
|
|
@ -48,6 +48,10 @@ pub(crate) struct EveRouteHint {
|
|||
|
||||
/// The `world_create.v4` payload, in one place, so the button that captures it
|
||||
/// and the descriptor that advertises it cannot drift.
|
||||
/// The card's whole account of a turn closed by a fault.
|
||||
const PLAY_FAULT_LINE: &str =
|
||||
"The storyteller lost the thread and your turn ended. Write again to start a new turn.";
|
||||
|
||||
const CREATE_BINDINGS: [&str; 8] = [
|
||||
"title",
|
||||
"brief",
|
||||
|
|
@ -477,6 +481,17 @@ pub(crate) fn authenticated_surface(
|
|||
"children":[]
|
||||
}));
|
||||
}
|
||||
// R1: a turn that died on its model says so. The line is
|
||||
// fixed: the fault detail stays in the store and the log,
|
||||
// never on the card (invariant 8, PA.f187).
|
||||
if play.is_some_and(|view| view.fault) {
|
||||
play_rows.push(json!({
|
||||
"id":"world.play.fault",
|
||||
"kind":"text",
|
||||
"props":{"value":PLAY_FAULT_LINE},
|
||||
"children":[]
|
||||
}));
|
||||
}
|
||||
// PA.f193-D ("the interrupted turn"; renumbered from
|
||||
// the PA.f188 duplicate this file's own comment used
|
||||
// to carry — PA.f188 names only `f295a3a`'s
|
||||
|
|
|
|||
|
|
@ -1243,6 +1243,10 @@ pub(crate) struct PlayTurnView {
|
|||
/// the exact thing `admit`'s own `try_lock_owned` was about to refuse as
|
||||
/// `TurnBusy` anyway.
|
||||
pub(crate) run_in_progress: bool,
|
||||
/// Whether the turn closed on a fault (`PlayTurn::fault` is set). The
|
||||
/// detail stays in the stored row and the log; the card shows a fixed
|
||||
/// player-safe line, never the detail (invariant 8, PA.f187).
|
||||
pub(crate) fault: bool,
|
||||
}
|
||||
|
||||
/// One outcome of executing a round's already-decided tool calls.
|
||||
|
|
@ -1688,6 +1692,7 @@ impl PlayTable {
|
|||
refusal: turn.refusal.clone(),
|
||||
revision: store.state.revision,
|
||||
run_in_progress,
|
||||
fault: turn.fault.is_some(),
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -1752,6 +1757,10 @@ impl PlayTable {
|
|||
/// this helper only ever writes the one turn's own record.
|
||||
async fn close_with_fault(&self, turn: &mut PlayTurn, detail: String) -> Result<(), PlayError> {
|
||||
turn.state = PlayTurnState::Closed;
|
||||
// R2: the one place a fault closes a turn, so the one place it is
|
||||
// logged. The detail is endpoint-free by construction (the inference
|
||||
// port classifies without the request URL), so it is logged whole.
|
||||
tracing::warn!(turn_id = %turn.turn_id, detail = %detail, "play turn closed on a fault");
|
||||
turn.fault = Some(detail);
|
||||
// PA.f173: a fault can close a turn that was `AwaitingPlayer`, and
|
||||
// `answer_turn` is not the only door out of that state — clear the
|
||||
|
|
@ -2968,8 +2977,8 @@ const RETRY_DELAY_BASE_MS: u64 = 100;
|
|||
/// whether that base is `RETRY_DELAY_BASE_MS` (production) or whatever a
|
||||
/// test set it to.
|
||||
fn backoff_delay(base_ms: u64, attempt: usize) -> Duration {
|
||||
const CAP: u64 = 5_000;
|
||||
Duration::from_millis(base_ms.saturating_mul(1u64 << attempt.min(6)).min(CAP))
|
||||
const CAP: u64 = 20_000;
|
||||
Duration::from_millis(base_ms.saturating_mul(1u64 << attempt.min(8)).min(CAP))
|
||||
}
|
||||
|
||||
/// A round is incomplete when it holds a tool call this table has not yet
|
||||
|
|
@ -3767,6 +3776,7 @@ pub(crate) mod tests {
|
|||
refusal: None,
|
||||
revision: 1,
|
||||
run_in_progress: false,
|
||||
fault: false,
|
||||
};
|
||||
let surface = crate::eve::authenticated_surface(
|
||||
"player-interrupted-turn",
|
||||
|
|
@ -3791,6 +3801,7 @@ pub(crate) mod tests {
|
|||
refusal: None,
|
||||
revision: 1,
|
||||
run_in_progress: false,
|
||||
fault: false,
|
||||
};
|
||||
let awaiting_surface = crate::eve::authenticated_surface(
|
||||
"player-interrupted-turn",
|
||||
|
|
@ -3830,6 +3841,7 @@ pub(crate) mod tests {
|
|||
refusal: None,
|
||||
revision: 1,
|
||||
run_in_progress: true,
|
||||
fault: false,
|
||||
};
|
||||
let surface = crate::eve::authenticated_surface(
|
||||
"player-interrupted-turn-busy",
|
||||
|
|
@ -3871,6 +3883,7 @@ pub(crate) mod tests {
|
|||
refusal: None,
|
||||
revision: 1,
|
||||
run_in_progress: true,
|
||||
fault: false,
|
||||
};
|
||||
let surface = crate::eve::authenticated_surface(
|
||||
"player-round-in-flight",
|
||||
|
|
@ -3892,6 +3905,7 @@ pub(crate) mod tests {
|
|||
refusal: None,
|
||||
revision: 1,
|
||||
run_in_progress: false,
|
||||
fault: false,
|
||||
};
|
||||
let idle_surface = crate::eve::authenticated_surface(
|
||||
"player-round-in-flight",
|
||||
|
|
@ -3926,6 +3940,7 @@ pub(crate) mod tests {
|
|||
refusal: None,
|
||||
revision: 1,
|
||||
run_in_progress: false,
|
||||
fault: false,
|
||||
};
|
||||
let surface = crate::eve::authenticated_surface(
|
||||
"player-stalled-turn-text",
|
||||
|
|
@ -8079,10 +8094,11 @@ pub(crate) mod tests {
|
|||
assert_eq!(backoff_delay(RETRY_DELAY_BASE_MS, 1), Duration::from_millis(200));
|
||||
assert_eq!(backoff_delay(RETRY_DELAY_BASE_MS, 2), Duration::from_millis(400));
|
||||
assert_eq!(backoff_delay(RETRY_DELAY_BASE_MS, 5), Duration::from_millis(3_200), "not yet capped");
|
||||
assert_eq!(backoff_delay(RETRY_DELAY_BASE_MS, 6), Duration::from_millis(5_000), "capped");
|
||||
assert_eq!(backoff_delay(RETRY_DELAY_BASE_MS, 7), Duration::from_millis(12_800), "not yet capped");
|
||||
assert_eq!(backoff_delay(RETRY_DELAY_BASE_MS, 8), Duration::from_millis(20_000), "capped");
|
||||
assert_eq!(
|
||||
backoff_delay(RETRY_DELAY_BASE_MS, 1_000_000),
|
||||
Duration::from_millis(5_000),
|
||||
Duration::from_millis(20_000),
|
||||
"a very large attempt count must stay capped, never overflow or panic"
|
||||
);
|
||||
assert_eq!(
|
||||
|
|
@ -8092,6 +8108,174 @@ pub(crate) mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
/// Captures what a tracing subscriber writes, for the test that reads the
|
||||
/// warn line a fault closes a turn with.
|
||||
#[derive(Clone, Default)]
|
||||
struct LogCapture(Arc<StdMutex<Vec<u8>>>);
|
||||
|
||||
impl std::io::Write for LogCapture {
|
||||
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
|
||||
self.0.lock().unwrap().extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for LogCapture {
|
||||
type Writer = LogCapture;
|
||||
|
||||
fn make_writer(&'a self) -> Self::Writer {
|
||||
self.clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl LogCapture {
|
||||
fn text(&self) -> String {
|
||||
String::from_utf8_lossy(&self.0.lock().unwrap()).into_owned()
|
||||
}
|
||||
}
|
||||
|
||||
fn capture_logs() -> (LogCapture, tracing::subscriber::DefaultGuard) {
|
||||
let capture = LogCapture::default();
|
||||
let guard = tracing::subscriber::set_default(
|
||||
tracing_subscriber::fmt()
|
||||
.with_writer(capture.clone())
|
||||
.with_ansi(false)
|
||||
.finish(),
|
||||
);
|
||||
(capture, guard)
|
||||
}
|
||||
|
||||
/// R1 and R2. A turn closed by a fault logs its id and the fault's
|
||||
/// detail at warn, and the owner's card shows the fixed fault row until
|
||||
/// the view stops reporting a fault. Mutations: dropping the `warn!` in
|
||||
/// `close_with_fault` fails the log assertions; `fault: false` in
|
||||
/// `current_turn_view` fails the row assertion.
|
||||
#[tokio::test]
|
||||
async fn a_fault_closed_turn_is_logged_and_shown_on_the_card() {
|
||||
let (logs, _guard) = capture_logs();
|
||||
let fixture = play_world(None, "player-fault-card").await;
|
||||
let personas = PersonaLane::new(
|
||||
ControllerPort::new(fixture.world.clone()),
|
||||
ScriptedPort::new(vec![]),
|
||||
"gpt-5.6-sol".into(),
|
||||
"gpt-5.6-sol".into(),
|
||||
)
|
||||
.unwrap();
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let table = PlayTable::new(
|
||||
fixture.world.clone(),
|
||||
personas,
|
||||
Arc::new(AlwaysFaultyPort::new()),
|
||||
"gpt-5.6-terra".into(),
|
||||
Arc::new(Semaphore::new(2)),
|
||||
directory.path().join("play-turn-v1.cc"),
|
||||
)
|
||||
.unwrap();
|
||||
let turn_id = test_turn_id(430);
|
||||
table.run(&fixture.principal, turn_id.clone(), "Hello?".into()).await.unwrap();
|
||||
|
||||
let line = logs
|
||||
.text()
|
||||
.lines()
|
||||
.find(|line| line.contains("play turn closed on a fault"))
|
||||
.map(str::to_owned)
|
||||
.expect("closing a turn on a fault must log a line");
|
||||
assert!(line.contains("WARN"), "{line}");
|
||||
assert!(line.contains(&turn_id), "the line must name the turn: {line}");
|
||||
assert!(line.contains("provider unavailable"), "the line must carry the fault detail: {line}");
|
||||
|
||||
let world = fixture.world.snapshot().await.unwrap();
|
||||
let view = table.current_turn_view().await.unwrap();
|
||||
assert!(view.fault);
|
||||
let surface =
|
||||
crate::eve::authenticated_surface("player-fault-card", Some(&world), Some(&view)).unwrap();
|
||||
let row = find_surface_node(&surface, "world.play.fault")
|
||||
.expect("a fault-closed turn must show the fault row");
|
||||
let encoded = serde_json::to_string(row).unwrap();
|
||||
assert!(!encoded.contains("provider unavailable"), "the row must not carry the detail: {encoded}");
|
||||
|
||||
let healthy = PlayTurnView { fault: false, ..view };
|
||||
let surface =
|
||||
crate::eve::authenticated_surface("player-fault-card", Some(&world), Some(&healthy)).unwrap();
|
||||
assert!(find_surface_node(&surface, "world.play.fault").is_none());
|
||||
}
|
||||
|
||||
/// The whole path with a real local port and a dead endpoint: reqwest's
|
||||
/// own error text carries the request URL, and the canary is that
|
||||
/// endpoint. No byte of it may reach the stored fault, the warn line or
|
||||
/// the card row; the error kind still does.
|
||||
#[tokio::test]
|
||||
async fn a_dead_local_endpoint_closes_the_turn_without_naming_the_endpoint() {
|
||||
let (logs, _guard) = capture_logs();
|
||||
let fixture = play_world(None, "player-fault-canary").await;
|
||||
let personas = PersonaLane::new(
|
||||
ControllerPort::new(fixture.world.clone()),
|
||||
ScriptedPort::new(vec![]),
|
||||
"gpt-5.6-sol".into(),
|
||||
"gpt-5.6-sol".into(),
|
||||
)
|
||||
.unwrap();
|
||||
let endpoint = {
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
listener.local_addr().unwrap()
|
||||
};
|
||||
let inference = ghostlight::open_inference(
|
||||
None,
|
||||
None,
|
||||
Some(ghostlight::LocalBinding {
|
||||
endpoint,
|
||||
model_prefix: "local/".into(),
|
||||
caller_runtime_id: "play-test-runtime".into(),
|
||||
}),
|
||||
&["local/play"],
|
||||
)
|
||||
.unwrap();
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let table = PlayTable::new(
|
||||
fixture.world.clone(),
|
||||
personas,
|
||||
inference,
|
||||
"local/play".into(),
|
||||
Arc::new(Semaphore::new(2)),
|
||||
directory.path().join("play-turn-v1.cc"),
|
||||
)
|
||||
.unwrap();
|
||||
table.set_retry_delay_base_ms(0);
|
||||
table.run(&fixture.principal, test_turn_id(431), "Hello?".into()).await.unwrap();
|
||||
|
||||
let world = fixture.world.snapshot().await.unwrap();
|
||||
let view = table.current_turn_view().await.unwrap();
|
||||
assert!(view.fault, "the dead endpoint must close the turn on a fault");
|
||||
let surface =
|
||||
crate::eve::authenticated_surface("player-fault-canary", Some(&world), Some(&view)).unwrap();
|
||||
let row = serde_json::to_string(find_surface_node(&surface, "world.play.fault").unwrap()).unwrap();
|
||||
let stored = table.store.lock().await.current().unwrap().fault.clone().unwrap();
|
||||
let log = logs.text();
|
||||
assert!(stored.contains("refused the connection"), "the error kind is kept: {stored}");
|
||||
for canary in [endpoint.to_string(), endpoint.ip().to_string(), "chat/completions".to_owned()] {
|
||||
for (where_, text) in [("stored fault", &stored), ("warn line", &log), ("card row", &row)] {
|
||||
assert!(!text.contains(&canary), "{canary} leaked into the {where_}: {text}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// R4: a model server reload takes about 30-40 s, so the production
|
||||
/// backoff summed over the whole retry budget must outlast it with
|
||||
/// margin, while the first retry stays quick. Reads the production base,
|
||||
/// budget and curve; no copy of any of them.
|
||||
#[test]
|
||||
fn the_production_backoff_outlasts_a_model_reload() {
|
||||
let total: Duration = (0..ROUND_RETRY_BUDGET)
|
||||
.map(|attempt| backoff_delay(RETRY_DELAY_BASE_MS, attempt))
|
||||
.sum();
|
||||
assert!(total >= Duration::from_secs(90), "the retry window is only {total:?}");
|
||||
assert!(backoff_delay(RETRY_DELAY_BASE_MS, 0) <= Duration::from_millis(200));
|
||||
}
|
||||
|
||||
/// PA.f66a: exhausting the round budget closes the turn and narrates,
|
||||
/// rather than leaving it `Running` with nothing said.
|
||||
#[tokio::test]
|
||||
|
|
|
|||
|
|
@ -198,9 +198,31 @@ impl InferenceOutput {
|
|||
#[error("{detail}")]
|
||||
pub struct InferenceFault {
|
||||
disposition: InferenceFaultDisposition,
|
||||
class: InferenceFaultClass,
|
||||
detail: String,
|
||||
}
|
||||
|
||||
/// What kind of trouble a fault is, as a typed value: the one place a reader
|
||||
/// learns it without parsing `detail`. Disposition says what the owning flow
|
||||
/// may do (retry, recover, quarantine); the class says what happened. A port
|
||||
/// that can tell classifies at construction (`classed`), and every other
|
||||
/// constructor leaves `Unclassified`.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum InferenceFaultClass {
|
||||
/// The connection to the endpoint could not be made.
|
||||
Connect,
|
||||
/// The connection was made but no HTTP status arrived: closed, reset or
|
||||
/// an empty reply.
|
||||
NoResponse,
|
||||
/// The request ran out of time.
|
||||
Timeout,
|
||||
/// The endpoint answered with this non-success status.
|
||||
Status(u16),
|
||||
/// The endpoint's reply was not the declared shape.
|
||||
BadReply,
|
||||
Unclassified,
|
||||
}
|
||||
|
||||
/// The three-way disposition an `InferenceFault` carries, readable but not
|
||||
/// independently constructible: `InferenceFault`'s own named constructors
|
||||
/// (`retryable`, `integrity_violation`, `recovery_required`) remain the only
|
||||
|
|
@ -217,6 +239,7 @@ impl InferenceFault {
|
|||
pub(super) fn new(detail: impl Into<String>) -> Self {
|
||||
Self {
|
||||
disposition: InferenceFaultDisposition::RecoveryRequired,
|
||||
class: InferenceFaultClass::Unclassified,
|
||||
detail: detail.into(),
|
||||
}
|
||||
}
|
||||
|
|
@ -225,6 +248,7 @@ impl InferenceFault {
|
|||
pub fn retryable(detail: impl Into<String>) -> Self {
|
||||
Self {
|
||||
disposition: InferenceFaultDisposition::Retryable,
|
||||
class: InferenceFaultClass::Unclassified,
|
||||
detail: detail.into(),
|
||||
}
|
||||
}
|
||||
|
|
@ -235,6 +259,7 @@ impl InferenceFault {
|
|||
pub fn integrity_violation(detail: impl Into<String>) -> Self {
|
||||
Self {
|
||||
disposition: InferenceFaultDisposition::IntegrityViolation,
|
||||
class: InferenceFaultClass::Unclassified,
|
||||
detail: detail.into(),
|
||||
}
|
||||
}
|
||||
|
|
@ -245,6 +270,17 @@ impl InferenceFault {
|
|||
Self::new(detail)
|
||||
}
|
||||
|
||||
/// Names what kind of trouble this fault is.
|
||||
pub(super) fn classed(mut self, class: InferenceFaultClass) -> Self {
|
||||
self.class = class;
|
||||
self
|
||||
}
|
||||
|
||||
/// The fault's class, read-only.
|
||||
pub fn class(&self) -> InferenceFaultClass {
|
||||
self.class
|
||||
}
|
||||
|
||||
pub(super) fn requires_recovery(&self) -> bool {
|
||||
self.disposition == InferenceFaultDisposition::RecoveryRequired
|
||||
}
|
||||
|
|
|
|||
|
|
@ -83,7 +83,7 @@ pub use controllers::{
|
|||
ControllerNeed, ControllerPendingReason, ControllerRunner, ControllerWork,
|
||||
ControllerWorkCustody, ControllerWorkLookup, ControllerWorkStore, ControllerWorkStoreError,
|
||||
ControllerWorkWrite,
|
||||
GroupedCheckpoint, InferenceEvent, InferenceFault, InferenceFaultDisposition, InferenceOutput,
|
||||
GroupedCheckpoint, InferenceEvent, InferenceFault, InferenceFaultClass, InferenceFaultDisposition, InferenceOutput,
|
||||
InferencePort,
|
||||
InferencePurpose, InferenceRequest, NarrativeCheckpoint, NarrativeRun, OperationalCheckpoint,
|
||||
OperationalRun, PersonaLane, PreparedInference, SubmissionDisposition, ToolResultOracle,
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
//! flight through the same port at once.
|
||||
|
||||
use super::controllers::{
|
||||
ControllerOpenError, InferenceEvent, InferenceFault, InferenceOutput, InferencePort,
|
||||
ControllerOpenError, InferenceEvent, InferenceFault, InferenceFaultClass, InferenceOutput, InferencePort,
|
||||
InferenceRequest, PreparedInference, REQUEST_EXPIRY, RESPONSE_TIMEOUT, call_id_is_valid,
|
||||
tool_name_is_valid, unix_ms,
|
||||
};
|
||||
|
|
@ -146,6 +146,17 @@ impl LocalInferencePort {
|
|||
endpoint: SocketAddr,
|
||||
prefix: impl Into<String>,
|
||||
caller_runtime_id: impl Into<String>,
|
||||
) -> Result<Self, ControllerOpenError> {
|
||||
Self::with_timeout(endpoint, prefix, caller_runtime_id, RESPONSE_TIMEOUT)
|
||||
}
|
||||
|
||||
/// `new` with the response timeout named, so a test can reach the timeout
|
||||
/// path without waiting out `RESPONSE_TIMEOUT`.
|
||||
fn with_timeout(
|
||||
endpoint: SocketAddr,
|
||||
prefix: impl Into<String>,
|
||||
caller_runtime_id: impl Into<String>,
|
||||
timeout: std::time::Duration,
|
||||
) -> Result<Self, ControllerOpenError> {
|
||||
if !endpoint.ip().is_loopback() {
|
||||
return Err(ControllerOpenError::LocalEndpointNotLoopback { endpoint });
|
||||
|
|
@ -154,7 +165,7 @@ impl LocalInferencePort {
|
|||
client: reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.timeout(RESPONSE_TIMEOUT)
|
||||
.timeout(timeout)
|
||||
.build()
|
||||
.expect("the local inference HTTP client builds with no custom TLS material"),
|
||||
endpoint,
|
||||
|
|
@ -170,32 +181,46 @@ impl LocalInferencePort {
|
|||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| {
|
||||
if error.is_connect() {
|
||||
InferenceFault::retryable(format!(
|
||||
"the local inference endpoint refused the connection: {error}"
|
||||
))
|
||||
} else {
|
||||
InferenceFault::new(format!("the local inference request failed: {error}"))
|
||||
}
|
||||
})?;
|
||||
.map_err(send_fault)?;
|
||||
let status = response.status();
|
||||
if !status.is_success() {
|
||||
let detail = format!("the local inference endpoint returned {status}");
|
||||
let class = InferenceFaultClass::Status(status.as_u16());
|
||||
return Err(if status.as_u16() == 429 || status.as_u16() == 503 {
|
||||
InferenceFault::retryable(detail)
|
||||
} else {
|
||||
InferenceFault::new(detail)
|
||||
});
|
||||
}
|
||||
.classed(class));
|
||||
}
|
||||
response.json::<LocalChatResponse>().await.map_err(|error| {
|
||||
InferenceFault::integrity_violation(format!(
|
||||
"the local inference endpoint's reply was not the declared shape: {error}"
|
||||
))
|
||||
response.json::<LocalChatResponse>().await.map_err(|_| {
|
||||
InferenceFault::integrity_violation(
|
||||
"the local inference endpoint's reply was not the declared shape",
|
||||
)
|
||||
.classed(InferenceFaultClass::BadReply)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// The one classifier of a send that failed before any HTTP status arrived.
|
||||
/// A request that got no response was never processed, so retrying it is
|
||||
/// safe; only a timeout is not retried, because the endpoint may still be
|
||||
/// generating. The fault carries the kind and never the error's own text:
|
||||
/// `reqwest::Error`'s `Display` ends with the request URL, which would put
|
||||
/// the endpoint into every log line and card that renders the detail.
|
||||
fn send_fault(error: reqwest::Error) -> InferenceFault {
|
||||
if error.is_timeout() {
|
||||
InferenceFault::new("the local inference request timed out")
|
||||
.classed(InferenceFaultClass::Timeout)
|
||||
} else if error.is_connect() {
|
||||
InferenceFault::retryable("the local inference endpoint refused the connection")
|
||||
.classed(InferenceFaultClass::Connect)
|
||||
} else {
|
||||
InferenceFault::retryable("the local inference endpoint closed the connection before replying")
|
||||
.classed(InferenceFaultClass::NoResponse)
|
||||
}
|
||||
}
|
||||
|
||||
/// Lowers one prepared request to the OpenAI chat-completions body: the
|
||||
/// system message is the request's instructions, every input item maps to
|
||||
/// one message in order, and every offered tool is declared `strict`. No
|
||||
|
|
@ -378,7 +403,9 @@ pub(super) fn open_local_port(
|
|||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::controllers::{InferencePurpose, RequestShape, open_inference, tool_request};
|
||||
use crate::controllers::{
|
||||
InferenceFaultDisposition, InferencePurpose, RequestShape, open_inference, tool_request,
|
||||
};
|
||||
use crate::elaboration::{ElaborationLoopEvaluation, SEED_ROUND_BUDGET, evaluate_elaboration_loop};
|
||||
use crate::patch::{RECORD_GAP_PATCH_TOOL, SUBMIT_PATCH_TOOL, patch_tools};
|
||||
use crate::sdk_inference::{DEFAULT_SDK_MODEL_PREFIX, RoutedInferencePort, SdkBinding};
|
||||
|
|
@ -1425,4 +1452,128 @@ mod tests {
|
|||
"a changed response_id did not change the receipt digest"
|
||||
);
|
||||
}
|
||||
|
||||
fn plain_request() -> InferenceRequest {
|
||||
tool_request(
|
||||
CommandId::new(),
|
||||
0,
|
||||
InferencePurpose::Persona,
|
||||
TEST_MODEL,
|
||||
"Respond only in natural prose.",
|
||||
vec![CodexInputItem::UserText {
|
||||
text: "Say something true.".into(),
|
||||
}],
|
||||
Vec::<CodexToolDefinition>::new(),
|
||||
RequestShape {
|
||||
max_output_tokens: 1_200,
|
||||
parallel_tool_calls: false,
|
||||
},
|
||||
)
|
||||
.expect("the request builds")
|
||||
}
|
||||
|
||||
async fn infer_once(port: &LocalInferencePort) -> Result<InferenceOutput, InferenceFault> {
|
||||
let prepared = port.prepare(plain_request()).expect("the port prepares");
|
||||
port.infer(prepared).await
|
||||
}
|
||||
|
||||
/// A loopback server that accepts the first `drops` connections and
|
||||
/// closes each without a byte (the shape of a model server reloading),
|
||||
/// then answers every later connection with `reply`.
|
||||
async fn closing_then_serving(drops: usize, reply: String) -> SocketAddr {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("loopback binds");
|
||||
let addr = listener.local_addr().expect("a bound listener has an address");
|
||||
let state = Arc::new(Mutex::new(ScriptedResponderState {
|
||||
replies: VecDeque::from(vec![(200, reply)]),
|
||||
..Default::default()
|
||||
}));
|
||||
tokio::spawn(async move {
|
||||
let mut seen = 0;
|
||||
loop {
|
||||
let Ok((mut stream, _)) = listener.accept().await else {
|
||||
return;
|
||||
};
|
||||
if seen < drops {
|
||||
seen += 1;
|
||||
let _ = stream.shutdown().await;
|
||||
continue;
|
||||
}
|
||||
serve_one(stream, &state).await;
|
||||
}
|
||||
});
|
||||
addr
|
||||
}
|
||||
|
||||
/// R3. A send that dies after the connection was made but before any
|
||||
/// status arrives is `Retryable` and classed `NoResponse`; the recovered
|
||||
/// endpoint then answers the very next call from the same port. Mutation:
|
||||
/// classify only `is_connect()` errors as retryable, everything else as
|
||||
/// recovery-required (the pre-cut shape) and the first assertion fails.
|
||||
#[tokio::test]
|
||||
async fn an_empty_reply_is_retryable_and_the_recovered_endpoint_answers() {
|
||||
let addr = closing_then_serving(1, text_reply("back")).await;
|
||||
let port = port(addr);
|
||||
let fault = infer_once(&port).await.expect_err("a closed connection produced an output");
|
||||
assert_eq!(fault.disposition(), InferenceFaultDisposition::Retryable, "{fault:?}");
|
||||
assert_eq!(fault.class(), InferenceFaultClass::NoResponse, "{fault:?}");
|
||||
infer_once(&port).await.expect("the recovered endpoint answers the retry");
|
||||
}
|
||||
|
||||
/// R3's other half: a timeout is not retried, because the endpoint may
|
||||
/// still be generating. The listener accepts and never answers.
|
||||
#[tokio::test]
|
||||
async fn a_timed_out_send_is_recovery_required_not_retried() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
let mut held = Vec::new();
|
||||
while let Ok((stream, _)) = listener.accept().await {
|
||||
held.push(stream);
|
||||
}
|
||||
});
|
||||
let port = {
|
||||
let _guard = client_build_lock().lock().expect("the client-build lock is never poisoned");
|
||||
LocalInferencePort::with_timeout(
|
||||
addr,
|
||||
DEFAULT_LOCAL_MODEL_PREFIX,
|
||||
TEST_RUNTIME,
|
||||
std::time::Duration::from_millis(150),
|
||||
)
|
||||
.expect("a loopback endpoint opens")
|
||||
};
|
||||
let fault = infer_once(&port).await.expect_err("a silent endpoint produced an output");
|
||||
assert_eq!(fault.disposition(), InferenceFaultDisposition::RecoveryRequired, "{fault:?}");
|
||||
assert_eq!(fault.class(), InferenceFaultClass::Timeout, "{fault:?}");
|
||||
}
|
||||
|
||||
/// The class is a typed value for every way a send can fail, and the
|
||||
/// endpoint appears in none of the fault's renderings: the canary is the
|
||||
/// endpoint this test bound, so a `Display` that carries the request URL
|
||||
/// (reqwest's does) fails here.
|
||||
#[tokio::test]
|
||||
async fn every_send_failure_is_classed_and_names_no_endpoint() {
|
||||
let closed = {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
drop(listener);
|
||||
addr
|
||||
};
|
||||
let dropping = closing_then_serving(usize::MAX, String::new()).await;
|
||||
let status = ScriptedResponder::start(vec![(400, "{}".to_owned())]).await.endpoint();
|
||||
let malformed = ScriptedResponder::start(vec![(200, "not json".to_owned())]).await.endpoint();
|
||||
let cases = [
|
||||
(closed, InferenceFaultClass::Connect),
|
||||
(dropping, InferenceFaultClass::NoResponse),
|
||||
(status, InferenceFaultClass::Status(400)),
|
||||
(malformed, InferenceFaultClass::BadReply),
|
||||
];
|
||||
for (addr, expected) in cases {
|
||||
let fault = infer_once(&port(addr)).await.expect_err("the send was expected to fail");
|
||||
assert_eq!(fault.class(), expected, "{fault:?}");
|
||||
let rendered = format!("{fault} | {fault:?}");
|
||||
for canary in [addr.to_string(), addr.ip().to_string(), "chat/completions".to_owned()] {
|
||||
assert!(!rendered.contains(&canary), "{canary} leaked into {rendered}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue