play-faults fix: the warn line carries the fault's typed class, HTTP status included
close_with_fault now takes a TurnFault (detail plus InferenceFaultClass) and logs the class beside the fixed detail, so a 404 and a 502 read differently again. absorb_or_close hands the fault's class through; every non-inference caller is Unclassified. The &'static str comment now says what it does and does not guarantee (Box::leak defeats it; the per-port canary tests are the backstop). Not yet built or tested at this commit. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
189d1d3c2b
commit
b64b61c345
2 changed files with 87 additions and 20 deletions
|
|
@ -29,7 +29,7 @@ use chrono::Utc;
|
|||
use cultcache_rs::{CacheBackingStore, CultCacheEnvelope, OwnedRedbMessagePackBackingStore};
|
||||
use ghostlight::{
|
||||
CommandBody, CommandId, ControllerError, ControllerMode, ControllerPort, DecisionInvocation,
|
||||
DecisionOpportunity, InferenceEvent, InferenceFault, InferenceFaultDisposition,
|
||||
DecisionOpportunity, InferenceEvent, InferenceFault, InferenceFaultClass, InferenceFaultDisposition,
|
||||
InferenceOutput, InferencePort, InferenceRequest, KernelError, MailboxError, PersonaLane,
|
||||
PlayPort, PrincipalCommandIntent, Statement, SubjectId, TickMinutes, VerifiedPrincipalEvidence,
|
||||
WorldMailbox, WorldPatch, WorldSnapshot, actor_tools, authoring_tools, decode_actor_call,
|
||||
|
|
@ -37,6 +37,22 @@ use ghostlight::{
|
|||
table_view,
|
||||
};
|
||||
use ghostlight_persona_projection::{PersonaTurn, SourceSpan};
|
||||
|
||||
/// What closes a turn on a fault: the stored detail text and the typed class
|
||||
/// that says what happened. Every non-inference failure is `Unclassified`; an
|
||||
/// inference fault keeps the class its port gave it (`InferenceFaultClass`,
|
||||
/// the HTTP status included), so the warn line in `close_with_fault` can name
|
||||
/// it without the detail text carrying it.
|
||||
struct TurnFault {
|
||||
detail: String,
|
||||
class: InferenceFaultClass,
|
||||
}
|
||||
|
||||
impl From<String> for TurnFault {
|
||||
fn from(detail: String) -> Self {
|
||||
Self { detail, class: InferenceFaultClass::Unclassified }
|
||||
}
|
||||
}
|
||||
use codex_connector::{CodexInputItem, CodexToolDefinition};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
|
@ -1755,16 +1771,21 @@ impl PlayTable {
|
|||
/// `Invariant` fault. Distinct from poisoning the whole table: that
|
||||
/// stays each caller's own decision (`RoundOutcome::Poisoned`), because
|
||||
/// this helper only ever writes the one turn's own record.
|
||||
async fn close_with_fault(&self, turn: &mut PlayTurn, detail: String) -> Result<(), PlayError> {
|
||||
async fn close_with_fault(&self, turn: &mut PlayTurn, fault: impl Into<TurnFault>) -> Result<(), PlayError> {
|
||||
let TurnFault { detail, class } = fault.into();
|
||||
turn.state = PlayTurnState::Closed;
|
||||
// R2: the one place a fault closes a turn, so the one place it is
|
||||
// logged, whole. An `InferenceFault`'s detail is a `&'static str`, so
|
||||
// no inference port (local, connector or SDK) can have put an
|
||||
// endpoint, request or reply text into it; the type guarantees that,
|
||||
// not this line. The details of the other failures that reach here
|
||||
// (kernel, request build, snapshot) are their callers' own error
|
||||
// text, and nothing here constrains them.
|
||||
tracing::warn!(turn_id = %turn.turn_id, detail = %detail, "play turn closed on a fault");
|
||||
// logged, whole. The line names the typed class (for an HTTP reply,
|
||||
// its status code, which is the server's answer and not an input
|
||||
// value) beside the detail. An `InferenceFault`'s detail is a
|
||||
// `&'static str`, which stops a port from handing it computed text by
|
||||
// accident; it does not stop a deliberate `Box::leak` into the
|
||||
// `pub` constructors, and a new port's canary test is what actually
|
||||
// keeps endpoint, request and reply text out. The details of the
|
||||
// other failures that reach here (kernel, request build, snapshot)
|
||||
// are their callers' own error text, and nothing here constrains
|
||||
// them.
|
||||
tracing::warn!(turn_id = %turn.turn_id, class = ?class, detail = %detail, "play turn closed on a fault");
|
||||
turn.fault = Some(detail);
|
||||
// PA.f173: a fault can close a turn that was `AwaitingPlayer`, and
|
||||
// `answer_turn` is not the only door out of that state — clear the
|
||||
|
|
@ -1789,7 +1810,7 @@ impl PlayTable {
|
|||
turn: &PlayTurn,
|
||||
round: usize,
|
||||
principal: &VerifiedPrincipalEvidence,
|
||||
) -> Result<InferenceOutput, String> {
|
||||
) -> Result<InferenceOutput, TurnFault> {
|
||||
let snapshot = self.round_snapshot().await?;
|
||||
let dispatched = turn.dispatched_subjects();
|
||||
let tools = self.round_tools(&snapshot, &dispatched, principal)?;
|
||||
|
|
@ -1832,8 +1853,8 @@ impl PlayTable {
|
|||
/// `RecoveryRequired` and `IntegrityViolation` never retry, closing on
|
||||
/// their first occurrence — retrying either again cannot succeed
|
||||
/// differently. `Ok(())` means the caller's loop should try again;
|
||||
/// `Err(detail)` is the fault's own detail text, to close the turn with.
|
||||
async fn absorb_or_close(&self, fault: InferenceFault, attempt: &mut usize) -> Result<(), String> {
|
||||
/// `Err` carries the fault's own detail text and class, to close the turn with.
|
||||
async fn absorb_or_close(&self, fault: InferenceFault, attempt: &mut usize) -> Result<(), TurnFault> {
|
||||
match fault.disposition() {
|
||||
InferenceFaultDisposition::Retryable if *attempt < ROUND_RETRY_BUDGET => {
|
||||
let delay = self.retry_delay(*attempt);
|
||||
|
|
@ -1845,7 +1866,9 @@ impl PlayTable {
|
|||
}
|
||||
InferenceFaultDisposition::Retryable
|
||||
| InferenceFaultDisposition::RecoveryRequired
|
||||
| InferenceFaultDisposition::IntegrityViolation => Err(fault.to_string()),
|
||||
| InferenceFaultDisposition::IntegrityViolation => {
|
||||
Err(TurnFault { detail: fault.to_string(), class: fault.class() })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -4229,7 +4252,7 @@ pub(crate) mod tests {
|
|||
assert!(turn.question.is_some());
|
||||
assert!(turn.question_token.is_some());
|
||||
table
|
||||
.close_with_fault(&mut turn, "manufactured fault for PA.f173".into())
|
||||
.close_with_fault(&mut turn, "manufactured fault for PA.f173".to_owned())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
|
|
@ -8349,6 +8372,12 @@ pub(crate) mod tests {
|
|||
/// Loopback server that answers every request with a `200` whose whole
|
||||
/// body is `body`.
|
||||
fn replying_with(body: &'static str) -> std::net::SocketAddr {
|
||||
replying_with_status("200 OK", body)
|
||||
}
|
||||
|
||||
/// Loopback server that answers every request with `status` (a status
|
||||
/// line tail such as `404 Not Found`) and `body`.
|
||||
fn replying_with_status(status: &'static str, body: &'static str) -> std::net::SocketAddr {
|
||||
use std::io::Write;
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
|
|
@ -8358,7 +8387,8 @@ pub(crate) mod tests {
|
|||
read_http_request(&mut stream);
|
||||
let _ = write!(
|
||||
stream,
|
||||
"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
|
||||
"HTTP/1.1 {}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
|
||||
status,
|
||||
body.len(),
|
||||
body
|
||||
);
|
||||
|
|
@ -8367,6 +8397,42 @@ pub(crate) mod tests {
|
|||
addr
|
||||
}
|
||||
|
||||
/// The status leg through the whole path: an endpoint that answers 404
|
||||
/// (model not loaded) or 502 (a proxy in front of a dead server) closes
|
||||
/// the turn on a fault whose warn line carries that code as the typed
|
||||
/// class, and the reply body, which here is a canary, reaches nothing.
|
||||
/// Mutations: dropping `class` from the warn line, or logging a fixed
|
||||
/// status, fails one of the two.
|
||||
#[tokio::test]
|
||||
async fn a_non_success_status_reaches_the_warn_line_as_its_code() {
|
||||
for (number, status, code) in [(434u128, "404 Not Found", "404"), (435, "502 Bad Gateway", "502")] {
|
||||
let endpoint = replying_with_status(status, "CANARY-status-body-7e41");
|
||||
let account = format!("player-fault-status-{code}");
|
||||
assert_faulted_turn_names_no_endpoint(
|
||||
endpoint,
|
||||
None,
|
||||
&account,
|
||||
number,
|
||||
"non-success status",
|
||||
&["CANARY-status-body-7e41"],
|
||||
)
|
||||
.await;
|
||||
let (logs, _guard) = capture_logs();
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let (fixture, table) = local_play_table(endpoint, None, &account, directory.path()).await;
|
||||
table.run(&fixture.principal, test_turn_id(number + 100), "Hello?".into()).await.unwrap();
|
||||
let log = logs.text();
|
||||
let line = log
|
||||
.lines()
|
||||
.find(|line| line.contains("play turn closed on a fault"))
|
||||
.expect("the fault must be logged");
|
||||
let expected = format!("Status({code})");
|
||||
assert!(line.contains(&expected), "the warn line must carry {expected}: {line}");
|
||||
let other = if code == "404" { "Status(502)" } else { "Status(404)" };
|
||||
assert!(!line.contains(other), "{line}");
|
||||
}
|
||||
}
|
||||
|
||||
/// The malformed-reply leg through the whole path: a whole body that is
|
||||
/// not the declared shape closes the turn on a fault, and serde's text
|
||||
/// for it, which echoes the reply's own value, reaches the stored fault,
|
||||
|
|
|
|||
|
|
@ -199,11 +199,12 @@ impl InferenceOutput {
|
|||
pub struct InferenceFault {
|
||||
disposition: InferenceFaultDisposition,
|
||||
class: InferenceFaultClass,
|
||||
/// Fixed text chosen by the port that built the fault. It is a
|
||||
/// `&'static str`, so no constructor can be handed text that was
|
||||
/// computed from a request, a reply, an endpoint or an error's own
|
||||
/// `Display`: a fault names what kind of trouble it is and never carries
|
||||
/// an input value. The type enforces this for every port.
|
||||
/// Fixed text chosen by the port that built the fault. A `&'static str`
|
||||
/// stops a constructor from being handed a `String` computed from a
|
||||
/// request, reply, endpoint or error `Display` by accident. It is not a
|
||||
/// seal: the constructors are `pub`, and `Box::leak` turns any `String`
|
||||
/// into a `&'static str`. What keeps input values out of a fault is each
|
||||
/// port's canary test, so a new port needs one.
|
||||
detail: &'static str,
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue