play-faults fix: a fault's detail is fixed text by type; one timeout owner; pin the malformed-body and default-timeout legs

InferenceFault.detail is a &'static str, so no inference port (local,
connector, SDK) can put an endpoint, request or reply text into a fault. The
connector and SDK details that were built from error text are now fixed per
kind or reason. LocalBinding loses its public response_timeout field; the
short timeout the PlayTable canary needs comes from open_local_port_with_timeout,
which exists only under the ghostlight test-support feature (a dungeon
dev-dependency feature), so a release has one timeout, RESPONSE_TIMEOUT.
Canary tests on the malformed-body path (port and PlayTable), on the SDK and
connector ports, and on the default timeout at both resolving sites.

Not yet built or tested at this commit.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Meta Vens 2026-10-09 23:26:21 +02:00
commit 29efabe4d8
8 changed files with 377 additions and 120 deletions

View file

@ -42,5 +42,6 @@ uuid.workspace = true
zeroize.workspace = true
[dev-dependencies]
ghostlight = { path = "../ghostlight", features = ["test-support"] }
tempfile = "3"
tower = { version = "0.5", features = ["util"] }

View file

@ -1758,11 +1758,12 @@ impl PlayTable {
async fn close_with_fault(&self, turn: &mut PlayTurn, detail: String) -> Result<(), PlayError> {
turn.state = PlayTurnState::Closed;
// R2: the one place a fault closes a turn, so the one place it is
// logged, whole. The local inference port builds its detail from the
// fault kind and never from the transport error's text (tests pin
// that for each local failure class). The connector and SDK ports
// build details from their own error text, which this line does not
// scrub.
// logged, whole. An `InferenceFault`'s detail is a `&'static str`, so
// no inference port (local, connector or SDK) can have put an
// endpoint, request or reply text into it; the type guarantees that,
// not this line. The details of the other failures that reach here
// (kernel, request build, snapshot) are their callers' own error
// text, and nothing here constrains them.
tracing::warn!(turn_id = %turn.turn_id, detail = %detail, "play turn closed on a fault");
turn.fault = Some(detail);
// PA.f173: a fault can close a turn that was `AwaitingPlayer`, and
@ -3303,7 +3304,7 @@ pub(crate) mod tests {
/// exercise `world.play`'s own staleness comparison against a question
/// that actually opened, rather than one hand-built.
pub(crate) struct ScriptedPort {
queue: StdMutex<VecDeque<Result<InferenceOutput, String>>>,
queue: StdMutex<VecDeque<Result<InferenceOutput, &'static str>>>,
seen: StdMutex<Vec<InferenceRequest>>,
}
@ -8226,18 +8227,15 @@ pub(crate) mod tests {
"gpt-5.6-sol".into(),
)
.unwrap();
let inference = ghostlight::open_inference(
None,
None,
Some(ghostlight::LocalBinding {
endpoint,
model_prefix: "local/".into(),
caller_runtime_id: "play-test-runtime".into(),
response_timeout: timeout,
}),
&["local/play"],
)
.unwrap();
let binding = ghostlight::LocalBinding {
endpoint,
model_prefix: "local/".into(),
caller_runtime_id: "play-test-runtime".into(),
};
let inference = match timeout {
Some(timeout) => ghostlight::open_local_port_with_timeout(binding, timeout).unwrap(),
None => ghostlight::open_inference(None, None, Some(binding), &["local/play"]).unwrap(),
};
let table = PlayTable::new(
fixture.world.clone(),
personas,
@ -8262,6 +8260,7 @@ pub(crate) mod tests {
account: &str,
turn_number: u128,
kind: &str,
reply_canaries: &[&str],
) {
let (logs, _guard) = capture_logs();
let directory = tempfile::tempdir().unwrap();
@ -8277,7 +8276,10 @@ pub(crate) mod tests {
let log = logs.text();
let debug = format!("{view:?} {:?}", table.store.lock().await.current().unwrap());
assert!(stored.contains(kind), "the error kind is kept: {stored}");
for canary in [endpoint.to_string(), endpoint.ip().to_string(), endpoint.port().to_string(), "chat/completions".to_owned()] {
let canaries = [endpoint.to_string(), endpoint.ip().to_string(), endpoint.port().to_string(), "chat/completions".to_owned()]
.into_iter()
.chain(reply_canaries.iter().map(|canary| (*canary).to_owned()));
for canary in canaries {
for (where_, text) in [("stored fault", &stored), ("warn line", &log), ("card row", &row), ("debug", &debug)] {
assert!(!text.contains(&canary), "{canary} leaked into the {where_}: {text}");
}
@ -8291,7 +8293,7 @@ pub(crate) mod tests {
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
listener.local_addr().unwrap()
};
assert_faulted_turn_names_no_endpoint(endpoint, None, "player-fault-canary", 431, "refused the connection").await;
assert_faulted_turn_names_no_endpoint(endpoint, None, "player-fault-canary", 431, "refused the connection", &[]).await;
}
/// The whole path with a timeout, the leg a Bonsai stall takes: the
@ -8314,6 +8316,71 @@ pub(crate) mod tests {
"player-fault-timeout",
432,
"timed out",
&[],
)
.await;
}
/// Reads one HTTP request (headers and the declared body) off `stream`.
fn read_http_request(stream: &mut std::net::TcpStream) {
use std::io::Read;
let mut buf = Vec::new();
let mut chunk = [0u8; 8192];
loop {
let Ok(read) = stream.read(&mut chunk) else { break };
if read == 0 {
break;
}
buf.extend_from_slice(&chunk[..read]);
let text = String::from_utf8_lossy(&buf).to_string();
if let Some(split) = text.find("\r\n\r\n") {
let len = text[..split]
.to_ascii_lowercase()
.lines()
.find_map(|l| l.strip_prefix("content-length:").map(|v| v.trim().parse::<usize>().unwrap()))
.unwrap_or(0);
if buf.len() >= split + 4 + len {
break;
}
}
}
}
/// Loopback server that answers every request with a `200` whose whole
/// body is `body`.
fn replying_with(body: &'static str) -> std::net::SocketAddr {
use std::io::Write;
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let addr = listener.local_addr().unwrap();
std::thread::spawn(move || {
for stream in listener.incoming() {
let Ok(mut stream) = stream else { return };
read_http_request(&mut stream);
let _ = write!(
stream,
"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
body.len(),
body
);
}
});
addr
}
/// The malformed-reply leg through the whole path: a whole body that is
/// not the declared shape closes the turn on a fault, and serde's text
/// for it, which echoes the reply's own value, reaches the stored fault,
/// the warn line, the card row and Debug nowhere.
#[tokio::test]
async fn a_malformed_local_reply_closes_the_turn_without_echoing_it() {
let endpoint = replying_with("\\"CANARY-reply-5d02c8\\"");
assert_faulted_turn_names_no_endpoint(
endpoint,
None,
"player-fault-malformed",
433,
"not the declared shape",
&["CANARY-reply-5d02c8"],
)
.await;
}
@ -8323,7 +8390,7 @@ pub(crate) mod tests {
/// answers each later one with an `end_turn` tool call. Returns its
/// address and the count of connections seen.
fn closing_then_ending(drops: usize) -> (std::net::SocketAddr, Arc<std::sync::atomic::AtomicUsize>) {
use std::io::{Read, Write};
use std::io::Write;
use std::sync::atomic::{AtomicUsize, Ordering};
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let addr = listener.local_addr().unwrap();
@ -8337,26 +8404,7 @@ pub(crate) mod tests {
drop(stream);
continue;
}
let mut buf = Vec::new();
let mut chunk = [0u8; 8192];
loop {
let Ok(read) = stream.read(&mut chunk) else { break };
if read == 0 {
break;
}
buf.extend_from_slice(&chunk[..read]);
let text = String::from_utf8_lossy(&buf).to_string();
if let Some(split) = text.find("\r\n\r\n") {
let len = text[..split]
.to_ascii_lowercase()
.lines()
.find_map(|l| l.strip_prefix("content-length:").map(|v| v.trim().parse::<usize>().unwrap()))
.unwrap_or(0);
if buf.len() >= split + 4 + len {
break;
}
}
}
read_http_request(&mut stream);
let body = serde_json::json!({
"id": format!("resp-{n}"),
"choices": [{"message": {"role": "assistant", "content": null,

View file

@ -576,7 +576,6 @@ fn open_controller(
model_prefix: std::env::var("GHOSTLIGHT_LOCAL_MODEL_PREFIX")
.unwrap_or_else(|_| DEFAULT_LOCAL_MODEL_PREFIX.into()),
caller_runtime_id: runtime_id.to_owned(),
response_timeout: None,
}),
Err(_) => None,
};

View file

@ -22,5 +22,10 @@ tracing.workspace = true
uuid.workspace = true
zeroize.workspace = true
[features]
# Exposes `open_local_port_with_timeout` to a consumer's tests. Production
# builds never enable it.
test-support = []
[dev-dependencies]
tempfile = "3"

View file

@ -199,7 +199,12 @@ impl InferenceOutput {
pub struct InferenceFault {
disposition: InferenceFaultDisposition,
class: InferenceFaultClass,
detail: String,
/// Fixed text chosen by the port that built the fault. It is a
/// `&'static str`, so no constructor can be handed text that was
/// computed from a request, a reply, an endpoint or an error's own
/// `Display`: a fault names what kind of trouble it is and never carries
/// an input value. The type enforces this for every port.
detail: &'static str,
}
/// What kind of trouble a fault is, as a typed value: the one place a reader
@ -236,37 +241,37 @@ pub enum InferenceFaultDisposition {
}
impl InferenceFault {
pub(super) fn new(detail: impl Into<String>) -> Self {
pub(super) fn new(detail: &'static str) -> Self {
Self {
disposition: InferenceFaultDisposition::RecoveryRequired,
class: InferenceFaultClass::Unclassified,
detail: detail.into(),
detail: detail,
}
}
/// Transport trouble the owning flow may retry.
pub fn retryable(detail: impl Into<String>) -> Self {
pub fn retryable(detail: &'static str) -> Self {
Self {
disposition: InferenceFaultDisposition::Retryable,
class: InferenceFaultClass::Unclassified,
detail: detail.into(),
detail: detail,
}
}
/// The one disposition that makes `ControllerError::requires_quarantine`
/// true for an `Inference` fault: the provider's answer cannot be trusted,
/// so the tick driver stops rather than commits.
pub fn integrity_violation(detail: impl Into<String>) -> Self {
pub fn integrity_violation(detail: &'static str) -> Self {
Self {
disposition: InferenceFaultDisposition::IntegrityViolation,
class: InferenceFaultClass::Unclassified,
detail: detail.into(),
detail: detail,
}
}
/// The non-quarantining failure: this purpose cannot complete, but a
/// sibling cell in the same tick is unaffected.
pub fn recovery_required(detail: impl Into<String>) -> Self {
pub fn recovery_required(detail: &'static str) -> Self {
Self::new(detail)
}
@ -457,7 +462,8 @@ impl PreparedInference {
request: InferenceRequest,
) -> Result<PreparedInference, InferenceFault> {
let request_bytes =
serde_json::to_vec(&request).map_err(|error| InferenceFault::new(error.to_string()))?;
serde_json::to_vec(&request)
.map_err(|_| InferenceFault::new("an inference request could not be serialized"))?;
let purpose = request.purpose;
let invocation = CodexTransportInvocation::new(
caller_runtime_id,
@ -465,7 +471,7 @@ impl PreparedInference {
Sha256::digest(request_bytes).into(),
request.provider,
)
.map_err(|error| InferenceFault::new(error.to_string()))?;
.map_err(|_| InferenceFault::new("an inference invocation could not be built"))?;
Ok(PreparedInference {
purpose,
invocation,
@ -500,7 +506,7 @@ impl CodexConnectorInferencePort {
MAX_CONNECTOR_FRAME_BYTES,
Some(RESPONSE_TIMEOUT),
)
.map_err(|error| InferenceFault::new(error.to_string()))?;
.map_err(|_| InferenceFault::new("the CodexConnector client could not be built"))?;
Ok(Self {
client,
caller_runtime_id,
@ -513,10 +519,11 @@ impl CodexConnectorInferencePort {
caller_runtime_id: impl Into<String>,
) -> Result<Self, InferenceFault> {
let bytes = Zeroizing::new(
std::fs::read(path.as_ref()).map_err(|error| InferenceFault::new(error.to_string()))?,
std::fs::read(path.as_ref())
.map_err(|_| InferenceFault::new("the CodexConnector key file could not be read"))?,
);
let raw = std::str::from_utf8(bytes.as_slice())
.map_err(|error| InferenceFault::new(error.to_string()))?;
.map_err(|_| InferenceFault::new("the CodexConnector key file is not UTF-8"))?;
let key = raw.trim_end_matches(['\r', '\n']);
if key.is_empty() || key.len() != raw.trim().len() {
return Err(InferenceFault::new(
@ -537,42 +544,51 @@ impl CodexConnectorInferencePort {
.execute(&request.invocation)
.map_err(|error| match error {
CodexConnectorClientError::Connection(_) => {
InferenceFault::retryable(error.to_string())
InferenceFault::retryable("the CodexConnector connection failed")
}
CodexConnectorClientError::InvalidConfig
| CodexConnectorClientError::FrameSize
| CodexConnectorClientError::Encoding
| CodexConnectorClientError::Transport(_) => {
InferenceFault::integrity_violation(error.to_string())
InferenceFault::integrity_violation("the CodexConnector transport failed")
}
})?;
let (events, receipt) = match result.disposition {
CodexTransportDisposition::Refused(reason) => {
let detail = format!("CodexConnector refused request: {reason:?}");
return Err(match reason {
CodexRefusal::InFlight | CodexRefusal::Capacity => {
InferenceFault::retryable(detail)
CodexRefusal::InFlight => {
InferenceFault::retryable("CodexConnector refused request: in flight")
}
CodexRefusal::Expired | CodexRefusal::Indeterminate => {
InferenceFault::new(detail)
CodexRefusal::Capacity => {
InferenceFault::retryable("CodexConnector refused request: at capacity")
}
CodexRefusal::IdentitySubstitution
| CodexRefusal::ProviderDigestSubstitution
| CodexRefusal::Policy
| CodexRefusal::ReplayConflict
| CodexRefusal::Malformed => InferenceFault::integrity_violation(detail),
CodexRefusal::Expired => {
InferenceFault::new("CodexConnector refused request: expired")
}
CodexRefusal::Indeterminate => {
InferenceFault::new("CodexConnector refused request: indeterminate")
}
CodexRefusal::IdentitySubstitution => InferenceFault::integrity_violation(
"CodexConnector refused request: identity substitution",
),
CodexRefusal::ProviderDigestSubstitution => InferenceFault::integrity_violation(
"CodexConnector refused request: provider digest substitution",
),
CodexRefusal::Policy => {
InferenceFault::integrity_violation("CodexConnector refused request: policy")
}
CodexRefusal::ReplayConflict => InferenceFault::integrity_violation(
"CodexConnector refused request: replay conflict",
),
CodexRefusal::Malformed => InferenceFault::integrity_violation(
"CodexConnector refused request: malformed",
),
});
}
CodexTransportDisposition::Transported { events, receipt } => (events, receipt),
};
if let CodexTransportOutcome::Failed {
failure_kind,
message,
} = &receipt.outcome
{
return Err(InferenceFault::new(format!(
"Codex provider failed ({failure_kind}): {message}"
)));
if let CodexTransportOutcome::Failed { .. } = &receipt.outcome {
return Err(InferenceFault::new("the Codex provider reported a failed outcome"));
}
let events = events
.into_iter()
@ -590,7 +606,7 @@ impl CodexConnectorInferencePort {
})
.collect();
let receipt_bytes = rmp_serde::to_vec_named(&receipt)
.map_err(|error| InferenceFault::new(error.to_string()))?;
.map_err(|_| InferenceFault::new("the CodexConnector receipt could not be encoded"))?;
Ok(InferenceOutput {
events,
receipt_digest: format!("sha256:{:x}", Sha256::digest(&receipt_bytes)),
@ -612,14 +628,14 @@ impl InferencePort for CodexConnectorInferencePort {
let port = self.clone();
tokio::task::spawn_blocking(move || port.execute(request))
.await
.map_err(|error| InferenceFault::new(error.to_string()))?
.map_err(|_| InferenceFault::new("the CodexConnector worker task failed"))?
}
}
pub(super) fn unix_ms() -> Result<u64, InferenceFault> {
let millis = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_err(|error| InferenceFault::new(error.to_string()))?
.map_err(|_| InferenceFault::new("the system clock is before the Unix epoch"))?
.as_millis();
u64::try_from(millis).map_err(|_| InferenceFault::new("system time exceeds u64 milliseconds"))
}
@ -6833,6 +6849,34 @@ fn purpose_name(purpose: InferencePurpose) -> &'static str {
#[cfg(test)]
mod tests {
/// The connector port's share of the no-input-text rule: a key-file path
/// and a runtime identity each arrive as a canary, and the fault names
/// only what failed. (The legs that need a live connector, a refusal or a
/// failed provider outcome, are fixed text by the fault type.)
#[test]
fn a_connector_fault_carries_no_text_it_was_handed() {
const CANARY: &str = "CANARY-connector-2c4d7a";
let endpoint: SocketAddr = "127.0.0.1:9".parse().unwrap();
let directory = tempfile::tempdir().unwrap();
let not_utf8 = directory.path().join(CANARY);
std::fs::write(&not_utf8, [0xff_u8, 0xfe]).unwrap();
let faults = [
CodexConnectorInferencePort::from_secret_file(endpoint, directory.path().join("absent").join(CANARY), "runtime")
.err()
.expect("a missing key file opened"),
CodexConnectorInferencePort::from_secret_file(endpoint, &not_utf8, "runtime")
.err()
.expect("a key file that is not UTF-8 opened"),
CodexConnectorInferencePort::new(endpoint, "key".into(), format!(" {CANARY} "))
.err()
.expect("a runtime identity with whitespace opened"),
];
for fault in faults {
let rendered = format!("{fault} | {fault:?}");
assert!(!rendered.contains(CANARY), "{CANARY} leaked into {rendered}");
}
}
/// The decision lanes' tools go to the same provider under the same
/// strict rules as the patch catalog; the interpreter's fixed tools and a
/// granted catalog entry are checked offline here for the same reason.
@ -11445,7 +11489,6 @@ mod tests {
endpoint: local_endpoint,
model_prefix: local_model_prefix,
caller_runtime_id: runtime_id,
response_timeout: None,
}),
&models.each(),
)

View file

@ -97,6 +97,8 @@ pub use elaboration::{
};
pub use lens::{Lens, LensWeights};
pub use local_inference::{DEFAULT_LOCAL_MODEL_PREFIX, LocalBinding};
#[cfg(feature = "test-support")]
pub use local_inference::open_local_port_with_timeout;
pub(crate) use mailbox::ElaborationPort;
pub use mailbox::{ConsumerPort, ControllerPort, MailboxError, PlayPort, SeedPort, WorldMailbox};
pub use patch::{

View file

@ -31,10 +31,6 @@ pub struct LocalBinding {
pub endpoint: SocketAddr,
pub model_prefix: String,
pub caller_runtime_id: String,
/// How long one request may take, reply body included. `None` is
/// `RESPONSE_TIMEOUT`; a consumer or test sets it to reach the timeout
/// path without waiting that out.
pub response_timeout: Option<std::time::Duration>,
}
const LOCAL_RECEIPT_SCHEMA: &str = "ghostlight.local_inference_receipt.v1";
@ -135,6 +131,10 @@ pub(super) struct LocalInferencePort {
endpoint: SocketAddr,
prefix: String,
caller_runtime_id: String,
/// Read only by the tests that pin which timeout a constructor resolves
/// to; the live timeout is the client's.
#[cfg(test)]
response_timeout: std::time::Duration,
}
impl LocalInferencePort {
@ -154,8 +154,10 @@ impl LocalInferencePort {
Self::with_timeout(endpoint, prefix, caller_runtime_id, RESPONSE_TIMEOUT)
}
/// `new` with the response timeout named, so a test can reach the timeout
/// path without waiting out `RESPONSE_TIMEOUT`.
/// `new` with the response timeout named. `new` is the only production
/// caller and passes `RESPONSE_TIMEOUT`, so production has one timeout
/// owner; the other callers are tests that reach the timeout path without
/// waiting `RESPONSE_TIMEOUT` out.
fn with_timeout(
endpoint: SocketAddr,
prefix: impl Into<String>,
@ -175,6 +177,8 @@ impl LocalInferencePort {
endpoint,
prefix: prefix.into(),
caller_runtime_id: caller_runtime_id.into(),
#[cfg(test)]
response_timeout: timeout,
})
}
@ -188,7 +192,7 @@ impl LocalInferencePort {
.map_err(send_fault)?;
let status = response.status();
if !status.is_success() {
let detail = format!("the local inference endpoint returned {status}");
let detail = "the local inference endpoint returned a non-success status";
let class = InferenceFaultClass::Status(status.as_u16());
return Err(if status.as_u16() == 429 || status.as_u16() == 503 {
InferenceFault::retryable(detail)
@ -268,11 +272,8 @@ fn lower_request(prepared: &PreparedInference, prefix: &str) -> Result<Value, In
}
let mut tools = Vec::with_capacity(request.tools.len());
for tool in &request.tools {
let parameters: Value = serde_json::from_str(&tool.parameters_json).map_err(|error| {
InferenceFault::integrity_violation(format!(
"tool `{}` parameters are not valid JSON: {error}",
tool.name
))
let parameters: Value = serde_json::from_str(&tool.parameters_json).map_err(|_| {
InferenceFault::integrity_violation("a tool's parameters are not valid JSON")
})?;
tools.push(json!({
"type": "function",
@ -366,7 +367,7 @@ fn assemble_output(
.unwrap_or_default(),
};
let receipt_bytes = rmp_serde::to_vec_named(&receipt)
.map_err(|error| InferenceFault::new(error.to_string()))?;
.map_err(|_| InferenceFault::new("the local inference receipt could not be encoded"))?;
Ok(InferenceOutput::new(
events,
format!("sha256:{:x}", Sha256::digest(&receipt_bytes)),
@ -405,15 +406,32 @@ impl InferencePort for LocalInferencePort {
/// caller of this function, not only `open_inference`.
pub(super) fn open_local_port(
binding: LocalBinding,
) -> Result<Arc<dyn InferencePort>, ControllerOpenError> {
Ok(Arc::new(local_port(binding)?))
}
/// The same port with a named response timeout, for a consumer's tests that
/// reach the timeout path without waiting `RESPONSE_TIMEOUT` out. It exists
/// only under the `test-support` feature, which production builds never
/// enable, so `LocalBinding` carries no timeout and a release has exactly
/// one: `RESPONSE_TIMEOUT`.
#[cfg(feature = "test-support")]
pub fn open_local_port_with_timeout(
binding: LocalBinding,
timeout: std::time::Duration,
) -> Result<Arc<dyn InferencePort>, ControllerOpenError> {
Ok(Arc::new(LocalInferencePort::with_timeout(
binding.endpoint,
binding.model_prefix,
binding.caller_runtime_id,
binding.response_timeout.unwrap_or(RESPONSE_TIMEOUT),
timeout,
)?))
}
fn local_port(binding: LocalBinding) -> Result<LocalInferencePort, ControllerOpenError> {
LocalInferencePort::new(binding.endpoint, binding.model_prefix, binding.caller_runtime_id)
}
#[cfg(test)]
mod tests {
use super::*;
@ -805,7 +823,6 @@ mod tests {
endpoint: "93.184.216.34:443".parse().unwrap(),
model_prefix: DEFAULT_LOCAL_MODEL_PREFIX.into(),
caller_runtime_id: TEST_RUNTIME.into(),
response_timeout: None,
}),
&[TEST_MODEL],
)
@ -912,7 +929,6 @@ mod tests {
endpoint: "127.0.0.1:1".parse().unwrap(),
model_prefix: "claude".into(),
caller_runtime_id: TEST_RUNTIME.into(),
response_timeout: None,
}),
&["claude-opus-5"],
)
@ -1195,7 +1211,6 @@ mod tests {
endpoint: "127.0.0.1:1".parse().unwrap(),
model_prefix: String::new(),
caller_runtime_id: TEST_RUNTIME.into(),
response_timeout: None,
}),
&[TEST_MODEL],
)
@ -1659,6 +1674,48 @@ mod tests {
assert_eq!(fault.disposition(), InferenceFaultDisposition::IntegrityViolation, "{fault:?}");
}
/// The malformed-body leg of the no-input-text rule: serde's `Display`
/// for a type mismatch echoes the offending value, so a detail built from
/// it carries the reply's own text. The canary is a value inside a whole
/// body that fails to deserialize.
#[tokio::test]
async fn a_malformed_body_carries_neither_its_own_text_nor_the_endpoint() {
const CANARY: &str = "CANARY-reply-7f3e91";
let addr = ScriptedResponder::start(vec![(200, format!("\"{CANARY}\""))]).await.endpoint();
let fault = infer_once(&port(addr)).await.expect_err("a malformed body produced an output");
assert_eq!(fault.class(), InferenceFaultClass::BadReply, "{fault:?}");
let rendered = format!("{fault} | {fault:?}");
for canary in [CANARY.to_owned(), addr.to_string(), addr.ip().to_string()] {
assert!(!rendered.contains(&canary), "{canary} leaked into {rendered}");
}
}
/// The production default is `RESPONSE_TIMEOUT` (a cold model load lives
/// inside one request) at the constructor and at the port a binding
/// opens: a shorter default turns every cold start into a Timeout fault
/// that is never retried.
#[test]
fn the_production_default_timeout_is_response_timeout_at_both_resolving_sites() {
assert_eq!(RESPONSE_TIMEOUT, std::time::Duration::from_secs(900));
let endpoint: SocketAddr = "127.0.0.1:9".parse().unwrap();
let constructed = {
let _guard = client_build_lock().lock().expect("the client-build lock is never poisoned");
LocalInferencePort::new(endpoint, DEFAULT_LOCAL_MODEL_PREFIX, TEST_RUNTIME)
.expect("a loopback endpoint opens")
};
assert_eq!(constructed.response_timeout, RESPONSE_TIMEOUT);
let opened = {
let _guard = client_build_lock().lock().expect("the client-build lock is never poisoned");
local_port(LocalBinding {
endpoint,
model_prefix: DEFAULT_LOCAL_MODEL_PREFIX.into(),
caller_runtime_id: TEST_RUNTIME.into(),
})
.expect("a loopback binding opens")
};
assert_eq!(opened.response_timeout, RESPONSE_TIMEOUT);
}
/// The timeout leg of the no-endpoint rule: reqwest's `Display` for a
/// timeout ends with the request URL, so a detail built from it names the
/// canary endpoint this test bound. Covers both a stalled send and a

View file

@ -146,8 +146,27 @@ pub(super) enum SidecarFaultReason {
}
impl SidecarFaultReason {
fn into_fault(self, detail: String) -> InferenceFault {
let detail = format!("SDK sidecar reported {self:?}: {detail}");
/// The sidecar's own `detail` text is never carried: the reason names the
/// failure and the fault's text is fixed per reason.
fn into_fault(self) -> InferenceFault {
let detail = match self {
Self::RateLimited => "SDK sidecar reported rate limited",
Self::Overloaded => "SDK sidecar reported overloaded",
Self::ServerError => "SDK sidecar reported a server error",
Self::ApiTimeout => "SDK sidecar reported an API timeout",
Self::AuthenticationFailed => "SDK sidecar reported authentication failed",
Self::OrgNotAllowed => "SDK sidecar reported organization not allowed",
Self::BillingError => "SDK sidecar reported a billing error",
Self::InvalidRequest => "SDK sidecar reported an invalid request",
Self::ModelNotFound => "SDK sidecar reported model not found",
Self::MaxOutputTokens => "SDK sidecar reported max output tokens",
Self::MaxBudgetUsd => "SDK sidecar reported max budget",
Self::ExecutionError => "SDK sidecar reported an execution error",
Self::Unknown => "SDK sidecar reported an unknown failure",
Self::ProtocolViolation => "SDK sidecar reported a protocol violation",
Self::ToolRegistrationFailed => "SDK sidecar reported tool registration failed",
Self::TurnCapRefused => "SDK sidecar reported the turn cap refused",
};
match self {
Self::RateLimited | Self::Overloaded | Self::ServerError | Self::ApiTimeout => {
InferenceFault::retryable(detail)
@ -468,7 +487,7 @@ impl SdkInferencePort {
}
Err(error) => {
let _ = self.link.restart().await;
return Err(InferenceFault::new(error.to_string()));
return Err(InferenceFault::new("the tool-result owner refused a tool call"));
}
}
}
@ -482,10 +501,10 @@ impl SdkInferencePort {
SidecarFrame::Fault {
query_id: seen,
reason,
detail,
..
} if seen == query_id => {
let _ = self.link.restart().await;
return Err(reason.into_fault(detail));
return Err(reason.into_fault());
}
_ => {
let _ = self.link.restart().await;
@ -501,13 +520,11 @@ impl SdkInferencePort {
fn link_fault(error: SidecarLinkError) -> InferenceFault {
match error {
SidecarLinkError::Closed => {
InferenceFault::new("the SDK sidecar exited during a query".to_string())
InferenceFault::new("the SDK sidecar exited during a query")
}
SidecarLinkError::Spawn(detail) => {
InferenceFault::new(format!("the SDK sidecar could not start: {detail}"))
}
SidecarLinkError::Codec(detail) => {
InferenceFault::integrity_violation(format!("SDK sidecar framing: {detail}"))
SidecarLinkError::Spawn(_) => InferenceFault::new("the SDK sidecar could not start"),
SidecarLinkError::Codec(_) => {
InferenceFault::integrity_violation("the SDK sidecar's framing was invalid")
}
}
}
@ -526,9 +543,9 @@ pub(super) fn lower_query(
if let Some(effort) = request.reasoning_effort.as_deref()
&& !SDK_EFFORT_LEVELS.contains(&effort)
{
return Err(InferenceFault::integrity_violation(format!(
"reasoning effort `{effort}` has no SDK counterpart"
)));
return Err(InferenceFault::integrity_violation(
"the requested reasoning effort has no SDK counterpart",
));
}
let mut items = request.input.iter();
let Some(CodexInputItem::UserText { text: prompt }) = items.next() else {
@ -641,7 +658,7 @@ fn assemble_output(
total_cost_usd_estimate: material.total_cost_usd_estimate,
};
let receipt_bytes = rmp_serde::to_vec_named(&receipt)
.map_err(|error| InferenceFault::new(error.to_string()))?;
.map_err(|_| InferenceFault::new("the SDK receipt could not be encoded"))?;
Ok(InferenceOutput {
events: lowered,
receipt_digest: format!("sha256:{:x}", Sha256::digest(&receipt_bytes)),
@ -763,9 +780,9 @@ impl InferencePort for RoutedInferencePort {
let model = request.provider_model().to_owned();
match self.route(&model) {
Some(port) => port.prepare(request),
None => Err(InferenceFault::integrity_violation(format!(
"no configured inference backend claims the model `{model}`"
))),
None => Err(InferenceFault::integrity_violation(
"no configured inference backend claims the requested model",
)),
}
}
@ -773,9 +790,9 @@ impl InferencePort for RoutedInferencePort {
let model = request.invocation.request.model.clone();
match self.route(&model) {
Some(port) => port.infer(request).await,
None => Err(InferenceFault::integrity_violation(format!(
"no configured inference backend claims the model `{model}`"
))),
None => Err(InferenceFault::integrity_violation(
"no configured inference backend claims the requested model",
)),
}
}
@ -1482,7 +1499,7 @@ mod tests {
fn soul_every_sidecar_reason_carries_its_named_disposition() {
use SidecarFaultReason::*;
for reason in [RateLimited, Overloaded, ServerError, ApiTimeout] {
let fault = reason.into_fault("detail".into());
let fault = reason.into_fault();
assert!(
!fault.requires_recovery() && !fault.integrity_was_violated(),
"{reason:?} was not retryable"
@ -1500,13 +1517,13 @@ mod tests {
Unknown,
] {
assert!(
reason.into_fault("detail".into()).requires_recovery(),
reason.into_fault().requires_recovery(),
"{reason:?} was not recovery-required"
);
}
for reason in [ProtocolViolation, ToolRegistrationFailed, TurnCapRefused] {
assert!(
reason.into_fault("detail".into()).integrity_was_violated(),
reason.into_fault().integrity_was_violated(),
"{reason:?} did not violate integrity"
);
}
@ -1684,6 +1701,91 @@ mod tests {
assert!(fault.integrity_was_violated(), "{fault:?}");
}
fn assert_names_no(fault: &InferenceFault, canary: &str) {
let rendered = format!("{fault} | {fault:?}");
assert!(!rendered.contains(canary), "{canary} leaked into {rendered}");
}
/// The SDK port's share of the no-input-text rule: text the sidecar sent,
/// the link's own error text, a request's effort and a request's model
/// name each arrive as a canary, and none reaches the fault. The fault
/// type carries only fixed text, so this fails if a port ever routes a
/// computed string into it (through `Box::leak`, say).
#[tokio::test]
async fn an_sdk_fault_carries_no_text_it_was_handed() {
const CANARY: &str = "CANARY-sdk-91b6d0";
// The sidecar's own detail, for every reason it may name.
for reason in [
SidecarFaultReason::RateLimited,
SidecarFaultReason::Overloaded,
SidecarFaultReason::ServerError,
SidecarFaultReason::ApiTimeout,
SidecarFaultReason::AuthenticationFailed,
SidecarFaultReason::OrgNotAllowed,
SidecarFaultReason::BillingError,
SidecarFaultReason::InvalidRequest,
SidecarFaultReason::ModelNotFound,
SidecarFaultReason::MaxOutputTokens,
SidecarFaultReason::MaxBudgetUsd,
SidecarFaultReason::ExecutionError,
SidecarFaultReason::Unknown,
SidecarFaultReason::ProtocolViolation,
SidecarFaultReason::ToolRegistrationFailed,
SidecarFaultReason::TurnCapRefused,
] {
let link = ScriptedLink::new(vec![SidecarFrame::Fault {
query_id: 1,
reason,
detail: CANARY.into(),
}]);
let port = SdkInferencePort::new(link, TEST_RUNTIME);
let prepared = prose_prepared(&port, "Say something true.");
let fault = port.infer(prepared).await.expect_err("a fault frame produced an output");
assert_names_no(&fault, CANARY);
}
// The link's own error text.
assert_names_no(&link_fault(SidecarLinkError::Spawn(CANARY.into())), CANARY);
assert_names_no(&link_fault(SidecarLinkError::Codec(CANARY.into())), CANARY);
// An effort the SDK has no counterpart for.
let port = SdkInferencePort::new(ScriptedLink::new(Vec::new()), TEST_RUNTIME);
let mut prepared = prose_prepared(&port, "Say something true.");
prepared.invocation.request.reasoning_effort = Some(CANARY.into());
let fault = lower_query(1, &prepared, 1).expect_err("an unmapped effort lowered");
assert_names_no(&fault, CANARY);
// A model no backend claims, prepared and inferred through the router.
let directory = tempfile::tempdir().unwrap();
let entry = directory.path().join("main.js");
std::fs::write(&entry, "// sidecar").unwrap();
let routed = open_inference(
None,
Some(SdkBinding {
sidecar_entry: entry,
caller_runtime_id: TEST_RUNTIME.into(),
model_prefix: DEFAULT_SDK_MODEL_PREFIX.into(),
}),
None,
&models(TEST_MODEL).each(),
)
.expect("the SDK model routes");
let request = tool_request(
CommandId::new(),
0,
InferencePurpose::Persona,
CANARY,
"Respond only in natural prose.",
vec![CodexInputItem::UserText { text: "Say something true.".into() }],
Vec::<CodexToolDefinition>::new(),
RequestShape { max_output_tokens: 1_200, parallel_tool_calls: false },
)
.expect("the request builds");
let fault = routed.prepare(request).err().expect("an unroutable model prepared");
assert_names_no(&fault, CANARY);
}
/// Soul: the lowering's own gate table, every row.
#[test]
fn soul_lower_query_refuses_the_shapes_the_gate_table_names() {