2792 lines
96 KiB
Rust
2792 lines
96 KiB
Rust
use std::collections::VecDeque;
|
|
use std::path::{Path, PathBuf};
|
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
|
use std::sync::{Arc, Mutex};
|
|
use std::time::Duration;
|
|
|
|
use model_connector::{
|
|
CallerAdmission, PlanQuotaSnapshot, ReasoningEffort, StopReason, ToolDefinition,
|
|
TransportAdmission, TransportDisposition, TransportInvocation, TransportKey, TransportReceipt,
|
|
TransportService, decrypt_result, encrypt_invocation,
|
|
};
|
|
|
|
use super::*;
|
|
use crate::config::ClaudeCodeLimits;
|
|
use crate::handle::{FsSessionStore, SessionStore};
|
|
use crate::launch::ChildEnv;
|
|
|
|
const CANARY_TOKEN: &str = "canary-oauth-token-value";
|
|
/// The variable the connector's own environment must never hand on.
|
|
const TOKEN_ENV: &str = "CLAUDE_CODE_OAUTH_TOKEN";
|
|
const TOKEN_FD_ENV: &str = "CLAUDE_CODE_OAUTH_TOKEN_FILE_DESCRIPTOR";
|
|
const MODEL: &str = "claude-haiku-5-5";
|
|
const INSTANCE: &str = "persona-a";
|
|
const SESSION: &str = "11111111-2222-3333-4444-555555555555";
|
|
const HEX_SESSION: &str = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee";
|
|
const OTHER_MODEL: &str = "claude-opus-5";
|
|
const FORK: &str = "66666666-7777-8888-9999-000000000000";
|
|
const FRAME: usize = 1 << 20;
|
|
|
|
// ---- fixtures, written by hand from the fields the probes observed ----
|
|
|
|
fn init_line(model: &str, session: &str) -> String {
|
|
format!(
|
|
r#"{{"type":"system","subtype":"init","session_id":"{session}","model":"{model}","tools":[],"apiKeySource":"none"}}"#
|
|
)
|
|
}
|
|
|
|
fn text_line(id: &str, text: &str) -> String {
|
|
format!(
|
|
r#"{{"type":"assistant","message":{{"id":"{id}","role":"assistant","content":[{{"type":"text","text":"{text}"}}],"stop_reason":null}},"session_id":"{SESSION}"}}"#
|
|
)
|
|
}
|
|
|
|
fn thinking_line(id: &str) -> String {
|
|
format!(
|
|
r#"{{"type":"assistant","message":{{"id":"{id}","role":"assistant","content":[{{"type":"thinking","thinking":"","signature":"sig"}}]}}}}"#
|
|
)
|
|
}
|
|
|
|
fn tool_use_line() -> String {
|
|
r#"{"type":"assistant","message":{"id":"msg_t","content":[{"type":"tool_use","id":"toolu_1","name":"x","input":{}}]}}"#.to_string()
|
|
}
|
|
|
|
fn rate_limit_line(status: &str) -> String {
|
|
format!(
|
|
r#"{{"type":"rate_limit_event","rate_limit_info":{{"status":"{status}","rateLimitType":"five_hour"}}}}"#
|
|
)
|
|
}
|
|
|
|
const USAGE: &str = r#"{"claude-haiku-5-5":{"inputTokens":2,"outputTokens":4,"cacheReadInputTokens":7,"cacheCreationInputTokens":1043,"thinkingTokens":3,"costUSD":0.5}}"#;
|
|
|
|
/// What a resumed session's `modelUsage` says: every turn so far. Every result
|
|
/// fixture carries it beside the turn's own `model_usage_since_last_result`,
|
|
/// so reading the wrong field shows in a count.
|
|
const CUMULATIVE: &str = r#"{"claude-haiku-5-5":{"inputTokens":900000,"outputTokens":900000,"cacheReadInputTokens":900000,"cacheCreationInputTokens":900000,"thinkingTokens":900000,"costUSD":99.0}}"#;
|
|
|
|
fn result_line(stop_reason: &str, since_last: &str) -> String {
|
|
result_line_with(stop_reason, since_last, "")
|
|
}
|
|
|
|
/// A success result with `extra` (a leading comma and fields) appended.
|
|
fn result_line_with(stop_reason: &str, since_last: &str, extra: &str) -> String {
|
|
format!(
|
|
r#"{{"type":"result","subtype":"success","is_error":false,"stop_reason":"{stop_reason}","session_id":"{SESSION}","total_cost_usd":99.0,"modelUsage":{CUMULATIVE},"model_usage_since_last_result":{since_last}{extra}}}"#
|
|
)
|
|
}
|
|
|
|
// The shapes `claude -p --json-schema` produced on the pinned CLI (2.1.312):
|
|
// the answer is a call of the CLI's own StructuredOutput tool, the result
|
|
// carries it as `structured_output` with stop_reason `tool_use`, and a
|
|
// `rate_limit_event` reports the plan's windows under `unifiedWindows`.
|
|
const SCHEMA: &str = r#"{"type":"object","properties":{"answer":{"type":"string"},"n":{"type":"integer"}},"required":["answer","n"],"additionalProperties":false}"#;
|
|
const STRUCTURED: &str = r#"{"answer":"say hello","n":0}"#;
|
|
|
|
fn structured_call_line(name: &str) -> String {
|
|
format!(
|
|
r#"{{"type":"assistant","message":{{"id":"msg_s","role":"assistant","content":[{{"type":"tool_use","id":"toolu_01","name":"{name}","input":{STRUCTURED},"caller":{{"type":"direct"}}}}],"stop_reason":null}},"session_id":"{SESSION}"}}"#
|
|
)
|
|
}
|
|
|
|
fn structured_tool_result_line() -> String {
|
|
format!(
|
|
r#"{{"type":"user","message":{{"role":"user","content":[{{"tool_use_id":"toolu_01","type":"tool_result","content":"Structured output provided successfully"}}]}},"parent_tool_use_id":null,"session_id":"{SESSION}"}}"#
|
|
)
|
|
}
|
|
|
|
fn plan_rate_limit_line() -> String {
|
|
r#"{"type":"rate_limit_event","rate_limit_info":{"status":"allowed","resetsAt":4102444800,"rateLimitType":"five_hour","overageStatus":"rejected","isUsingOverage":false,"unifiedWindows":{"five_hour":{"utilization":0.08,"resetsAt":4102444800,"status":"allowed"},"seven_day":{"utilization":0.3,"resetsAt":4102531200,"status":"allowed"}}}}"#.to_string()
|
|
}
|
|
|
|
fn structured_result_line() -> String {
|
|
result_line_with(
|
|
"tool_use",
|
|
USAGE,
|
|
&format!(r#","structured_output":{STRUCTURED},"result":"ignored""#),
|
|
)
|
|
}
|
|
|
|
fn structured_script() -> Script {
|
|
Script::lines(vec![
|
|
init_line(MODEL, SESSION),
|
|
structured_call_line("StructuredOutput"),
|
|
structured_tool_result_line(),
|
|
plan_rate_limit_line(),
|
|
structured_result_line(),
|
|
])
|
|
}
|
|
|
|
fn schema_request() -> ProviderRequest {
|
|
let mut request = request(vec![user("hello")]);
|
|
request.output_schema_json = Some(SCHEMA.to_string());
|
|
request
|
|
}
|
|
|
|
fn a_tool() -> ToolDefinition {
|
|
ToolDefinition {
|
|
name: "lookup".into(),
|
|
description: "d".into(),
|
|
parameters_json: "{}".into(),
|
|
}
|
|
}
|
|
|
|
fn error_result_line(extra: &str) -> String {
|
|
format!(
|
|
r#"{{"type":"result","subtype":"error_during_execution","is_error":true,"session_id":"{SESSION}","total_cost_usd":99.0,"modelUsage":{CUMULATIVE},"model_usage_since_last_result":{USAGE}{extra}}}"#
|
|
)
|
|
}
|
|
|
|
fn ok_script(session: &str, text: &str) -> Script {
|
|
Script::lines(vec![
|
|
init_line(MODEL, session),
|
|
text_line("msg_1", text),
|
|
rate_limit_line("allowed"),
|
|
result_line("end_turn", USAGE),
|
|
])
|
|
}
|
|
|
|
fn usage(
|
|
input: u64,
|
|
cache_read: u64,
|
|
cache_write: u64,
|
|
output: u64,
|
|
reasoning: Option<u64>,
|
|
) -> Usage {
|
|
Usage {
|
|
input_tokens: input,
|
|
cache_read_tokens: cache_read,
|
|
cache_write_tokens: cache_write,
|
|
output_tokens: output,
|
|
reasoning_tokens: reasoning,
|
|
}
|
|
}
|
|
|
|
fn user(text: &str) -> InputItem {
|
|
InputItem::UserText { text: text.into() }
|
|
}
|
|
|
|
fn assistant(text: &str) -> InputItem {
|
|
InputItem::AssistantText { text: text.into() }
|
|
}
|
|
|
|
fn request(input: Vec<InputItem>) -> ProviderRequest {
|
|
let mut request = ProviderRequest::new("req", "conv", MODEL, "Be terse.", 1024);
|
|
request.input = input;
|
|
request
|
|
}
|
|
|
|
// ---- the scripted port: replays fixture lines and records what it was given ----
|
|
|
|
#[derive(Clone, Copy)]
|
|
enum Terminal {
|
|
Eof,
|
|
Timeout,
|
|
Breach,
|
|
}
|
|
|
|
struct Script {
|
|
lines: Vec<String>,
|
|
terminal: Terminal,
|
|
}
|
|
|
|
impl Script {
|
|
fn lines(lines: Vec<String>) -> Self {
|
|
Self {
|
|
lines,
|
|
terminal: Terminal::Eof,
|
|
}
|
|
}
|
|
|
|
fn ending(mut self, terminal: Terminal) -> Self {
|
|
self.terminal = terminal;
|
|
self
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Default)]
|
|
struct Seen {
|
|
argv: Vec<String>,
|
|
env: Vec<(String, String)>,
|
|
cwd: PathBuf,
|
|
stdin: Vec<u8>,
|
|
/// The token the backend handed the port, which a real port puts on a pipe.
|
|
token: String,
|
|
/// The file `--system-prompt-file` named, as it was when the child started.
|
|
prompt_file: Option<(PathBuf, String)>,
|
|
/// Every deadline the backend passed to a read of this child.
|
|
deadlines: Arc<Mutex<Vec<Instant>>>,
|
|
}
|
|
|
|
struct ScriptedPort {
|
|
scripts: Mutex<VecDeque<Script>>,
|
|
spawns: Arc<AtomicUsize>,
|
|
seen: Arc<Mutex<Seen>>,
|
|
stderr: Vec<u8>,
|
|
}
|
|
|
|
struct ScriptedChild {
|
|
lines: VecDeque<String>,
|
|
terminal: Terminal,
|
|
stderr: Vec<u8>,
|
|
deadlines: Arc<Mutex<Vec<Instant>>>,
|
|
}
|
|
|
|
impl ProcessPort for ScriptedPort {
|
|
fn spawn(
|
|
&self,
|
|
argv: &[String],
|
|
env: &ChildEnv,
|
|
cwd: &Path,
|
|
stdin: Vec<u8>,
|
|
token: &str,
|
|
) -> io::Result<Box<dyn RunningChild>> {
|
|
self.spawns.fetch_add(1, Ordering::SeqCst);
|
|
let deadlines = Arc::new(Mutex::new(Vec::new()));
|
|
*self.seen.lock().unwrap() = Seen {
|
|
prompt_file: arg_after(argv, "--system-prompt-file").map(|path| {
|
|
(
|
|
PathBuf::from(path),
|
|
std::fs::read_to_string(path).expect("the prompt file exists at spawn"),
|
|
)
|
|
}),
|
|
deadlines: deadlines.clone(),
|
|
argv: argv.to_vec(),
|
|
env: env
|
|
.iter()
|
|
.map(|(k, v)| (k.to_string(), v.to_string()))
|
|
.collect(),
|
|
cwd: cwd.to_path_buf(),
|
|
stdin,
|
|
token: token.to_string(),
|
|
};
|
|
let script = self
|
|
.scripts
|
|
.lock()
|
|
.unwrap()
|
|
.pop_front()
|
|
.expect("the test scripted no further spawn");
|
|
Ok(Box::new(ScriptedChild {
|
|
lines: script.lines.into(),
|
|
terminal: script.terminal,
|
|
stderr: self.stderr.clone(),
|
|
deadlines,
|
|
}))
|
|
}
|
|
}
|
|
|
|
impl RunningChild for ScriptedChild {
|
|
fn read_line(&mut self, deadline: Instant) -> StdoutRead {
|
|
self.deadlines.lock().unwrap().push(deadline);
|
|
match self.lines.pop_front() {
|
|
Some(line) => StdoutRead::Line(line.into_bytes()),
|
|
None => match self.terminal {
|
|
Terminal::Eof => StdoutRead::Eof,
|
|
Terminal::Timeout => StdoutRead::Timeout,
|
|
Terminal::Breach => StdoutRead::Breach,
|
|
},
|
|
}
|
|
}
|
|
fn stderr_tail(&self) -> Vec<u8> {
|
|
self.stderr.clone()
|
|
}
|
|
fn kill(&mut self) {}
|
|
fn wait(&mut self) {}
|
|
}
|
|
|
|
struct Sessions(Vec<String>);
|
|
|
|
impl SessionStore for Sessions {
|
|
fn exists(&self, session_id: &str) -> bool {
|
|
self.0.iter().any(|known| known == session_id)
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Default)]
|
|
struct Sink(Arc<Mutex<Vec<u8>>>);
|
|
|
|
impl StderrSink for Sink {
|
|
fn stderr_tail(&self, tail: &[u8]) {
|
|
self.0.lock().unwrap().extend_from_slice(tail);
|
|
}
|
|
}
|
|
|
|
/// The bytes of the stand-in `claude` the scripted rigs pin.
|
|
const PINNED: &[u8] = b"the pinned claude bytes";
|
|
|
|
fn pin_of(bytes: &[u8]) -> [u8; 32] {
|
|
Sha256::digest(bytes).into()
|
|
}
|
|
|
|
/// A directory holding a pinned `claude` file and an empty `prompts` directory.
|
|
fn sandbox() -> TempDir {
|
|
let dir = TempDir::new();
|
|
std::fs::write(dir.path().join("claude"), PINNED).unwrap();
|
|
std::fs::create_dir_all(dir.path().join("prompts")).unwrap();
|
|
dir
|
|
}
|
|
|
|
fn config(dir: &Path, instance: &str, limits: ClaudeCodeLimits) -> ClaudeCodeConfig {
|
|
ClaudeCodeConfig {
|
|
claude_binary: dir.join("claude"),
|
|
claude_binary_sha256: pin_of(PINNED),
|
|
work_dir: PathBuf::from("/work"),
|
|
config_dir: PathBuf::from("/cfg"),
|
|
prompt_dir: dir.join("prompts"),
|
|
token_credential: "claude-token".to_string(),
|
|
handle_secret_credential: "claude-handle-secret".to_string(),
|
|
instance: instance.to_string(),
|
|
limits,
|
|
}
|
|
}
|
|
|
|
/// A token source with a fixed answer.
|
|
struct FixedToken(Option<&'static str>);
|
|
|
|
impl TokenSource for FixedToken {
|
|
fn read(&self) -> Option<Zeroizing<String>> {
|
|
self.0.map(|token| Zeroizing::new(token.to_string()))
|
|
}
|
|
}
|
|
|
|
fn canary_token() -> Box<dyn TokenSource> {
|
|
Box::new(FixedToken(Some(CANARY_TOKEN)))
|
|
}
|
|
|
|
/// A handle secret source with a fixed answer; it names its own field.
|
|
struct FixedSecret(Option<&'static str>);
|
|
|
|
impl TokenSource for FixedSecret {
|
|
fn read(&self) -> Option<Zeroizing<String>> {
|
|
self.0.map(|secret| Zeroizing::new(secret.to_string()))
|
|
}
|
|
|
|
fn missing(&self) -> &'static str {
|
|
"handle_secret_credential"
|
|
}
|
|
}
|
|
|
|
const SECRET: &str = "handle-secret-0000000000000000000000000000000000000000000000";
|
|
const OTHER_SECRET: &str = "handle-secret-1111111111111111111111111111111111111111111111";
|
|
|
|
fn canary_secret() -> Box<dyn TokenSource> {
|
|
Box::new(FixedSecret(Some(SECRET)))
|
|
}
|
|
|
|
fn limits() -> ClaudeCodeLimits {
|
|
ClaudeCodeLimits {
|
|
deadline: Duration::from_secs(30),
|
|
max_line_bytes: 1 << 20,
|
|
max_total_bytes: 1 << 24,
|
|
stderr_tail_bytes: 4096,
|
|
}
|
|
}
|
|
|
|
struct Rig {
|
|
dir: TempDir,
|
|
backend: ClaudeCodeBackend,
|
|
spawns: Arc<AtomicUsize>,
|
|
seen: Arc<Mutex<Seen>>,
|
|
sink: Sink,
|
|
}
|
|
|
|
fn rig_for(instance: &str, scripts: Vec<Script>, sessions: &[&str], stderr: &[u8]) -> Rig {
|
|
rig_with_secret(instance, scripts, sessions, stderr, Some(SECRET))
|
|
}
|
|
|
|
fn rig_with_secret(
|
|
instance: &str,
|
|
scripts: Vec<Script>,
|
|
sessions: &[&str],
|
|
stderr: &[u8],
|
|
secret: Option<&'static str>,
|
|
) -> Rig {
|
|
let spawns = Arc::new(AtomicUsize::new(0));
|
|
let seen = Arc::new(Mutex::new(Seen::default()));
|
|
let sink = Sink::default();
|
|
let port = ScriptedPort {
|
|
scripts: Mutex::new(scripts.into()),
|
|
spawns: spawns.clone(),
|
|
seen: seen.clone(),
|
|
stderr: stderr.to_vec(),
|
|
};
|
|
let dir = sandbox();
|
|
let backend = ClaudeCodeBackend::with_ports(
|
|
config(dir.path(), instance, limits()),
|
|
Box::new(port),
|
|
Box::new(Sessions(sessions.iter().map(|s| s.to_string()).collect())),
|
|
Box::new(sink.clone()),
|
|
canary_token(),
|
|
Box::new(FixedSecret(secret)),
|
|
);
|
|
Rig {
|
|
dir,
|
|
backend,
|
|
spawns,
|
|
seen,
|
|
sink,
|
|
}
|
|
}
|
|
|
|
fn rig(scripts: Vec<Script>) -> Rig {
|
|
rig_for(INSTANCE, scripts, &[SESSION], b"")
|
|
}
|
|
|
|
impl Rig {
|
|
fn run(&self, request: ProviderRequest) -> TransportResult {
|
|
self.run_as(CALLER_1, request)
|
|
}
|
|
fn run_as(&self, caller: &str, request: ProviderRequest) -> TransportResult {
|
|
let mut door = Door::new();
|
|
let (claim, _) = door.begin_as(caller, request);
|
|
self.backend.execute(claim.call())
|
|
}
|
|
fn seen(&self) -> Seen {
|
|
self.seen.lock().unwrap().clone()
|
|
}
|
|
fn spawns(&self) -> usize {
|
|
self.spawns.load(Ordering::SeqCst)
|
|
}
|
|
}
|
|
|
|
// ---- the kit's door, so a call is what serve would hand the backend ----
|
|
|
|
static DOOR_SEQUENCE: AtomicUsize = AtomicUsize::new(0);
|
|
|
|
const CALLER_1: &str = "caller-1";
|
|
const CALLER_2: &str = "caller-2";
|
|
|
|
struct Door {
|
|
dir: PathBuf,
|
|
service: TransportService,
|
|
}
|
|
|
|
impl Door {
|
|
fn new() -> Self {
|
|
let dir = std::env::temp_dir().join(format!(
|
|
"claude-connector-door-{}-{}",
|
|
std::process::id(),
|
|
DOOR_SEQUENCE.fetch_add(1, Ordering::SeqCst)
|
|
));
|
|
std::fs::create_dir_all(&dir).unwrap();
|
|
let callers = [CALLER_1, CALLER_2].map(|id| {
|
|
CallerAdmission::new(
|
|
id,
|
|
format!("{id}-key"),
|
|
1,
|
|
[MODEL.to_string(), OTHER_MODEL.to_string()],
|
|
4,
|
|
FRAME / 2,
|
|
32_768,
|
|
)
|
|
.unwrap()
|
|
});
|
|
let service =
|
|
TransportService::open(&dir.join("replay.cc"), PROVIDER, callers, 5_000).unwrap();
|
|
Self { dir, service }
|
|
}
|
|
|
|
fn begin(
|
|
&mut self,
|
|
request: ProviderRequest,
|
|
) -> (model_connector::ExecutionClaim, TransportInvocation) {
|
|
self.begin_as(CALLER_1, request)
|
|
}
|
|
|
|
fn begin_as(
|
|
&mut self,
|
|
caller: &str,
|
|
request: ProviderRequest,
|
|
) -> (model_connector::ExecutionClaim, TransportInvocation) {
|
|
let key = TransportKey::from_connection_secret(&format!("{caller}-key")).unwrap();
|
|
let invocation = TransportInvocation::new(caller, 2_000, [7; 32], request).unwrap();
|
|
let envelope = encrypt_invocation(&invocation, &key).unwrap();
|
|
match self
|
|
.service
|
|
.begin(&envelope, 1_000, &PlanQuotaSnapshot::default(), FRAME)
|
|
.unwrap()
|
|
{
|
|
TransportAdmission::Execute(claim) => (*claim, invocation),
|
|
TransportAdmission::Reply(_) => panic!("the door answered instead of admitting"),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Drop for Door {
|
|
fn drop(&mut self) {
|
|
let _ = std::fs::remove_dir_all(&self.dir);
|
|
}
|
|
}
|
|
|
|
fn parts(result: &TransportResult) -> (&[TransportEvent], &TransportReceipt) {
|
|
match &result.disposition {
|
|
TransportDisposition::Transported { events, receipt } => (events, receipt),
|
|
TransportDisposition::Refused(refusal) => panic!("refused: {refusal:?}"),
|
|
}
|
|
}
|
|
|
|
fn outcome(result: &TransportResult) -> &TransportOutcome {
|
|
&parts(result).1.outcome
|
|
}
|
|
|
|
fn failure_of(result: &TransportResult) -> (FailureKind, String, Option<Usage>) {
|
|
match outcome(result) {
|
|
TransportOutcome::Failed {
|
|
kind,
|
|
message,
|
|
usage,
|
|
} => (*kind, message.clone(), usage.clone()),
|
|
other => panic!("expected a failure, got {other:?}"),
|
|
}
|
|
}
|
|
|
|
fn arg_after<'a>(argv: &'a [String], flag: &str) -> Option<&'a str> {
|
|
let at = argv.iter().position(|arg| arg == flag)?;
|
|
argv.get(at + 1).map(String::as_str)
|
|
}
|
|
|
|
/// A second request that continues `first` with the transcript the kit itself
|
|
/// derives from `result`.
|
|
fn continuing(first: &ProviderRequest, result: &TransportResult, next: &str) -> ProviderRequest {
|
|
let mut input = first.input.clone();
|
|
input.extend(result.transcript_items());
|
|
input.push(user(next));
|
|
request(input)
|
|
}
|
|
|
|
// ---- argv and environment ----
|
|
|
|
#[test]
|
|
fn argv_is_the_fixed_hygiene_set() {
|
|
let rig = rig(vec![ok_script(SESSION, "hi")]);
|
|
rig.run(request(vec![user("hello")]));
|
|
let argv = rig.seen().argv;
|
|
let prompt_path = arg_after(&argv, "--system-prompt-file")
|
|
.expect("the prompt is passed as a file")
|
|
.to_string();
|
|
let expected: Vec<String> = [
|
|
rig.dir.path().join("claude").to_str().unwrap(),
|
|
"-p",
|
|
"--output-format",
|
|
"stream-json",
|
|
"--verbose",
|
|
"--input-format",
|
|
"stream-json",
|
|
"--model",
|
|
MODEL,
|
|
"--system-prompt-file",
|
|
&prompt_path,
|
|
"--tools",
|
|
"",
|
|
"--strict-mcp-config",
|
|
"--setting-sources",
|
|
"",
|
|
"--disable-slash-commands",
|
|
"--permission-prompts",
|
|
"none",
|
|
]
|
|
.iter()
|
|
.map(|s| s.to_string())
|
|
.collect();
|
|
assert_eq!(argv, expected);
|
|
assert!(!argv.iter().any(|arg| arg == "--system-prompt"));
|
|
for forbidden in [
|
|
"--bare",
|
|
"--fallback-model",
|
|
"--no-session-persistence",
|
|
"--dangerously-skip-permissions",
|
|
"--mcp-config",
|
|
"--add-dir",
|
|
"--agents",
|
|
"--plugin-dir",
|
|
"--settings",
|
|
] {
|
|
assert!(!argv.iter().any(|arg| arg == forbidden), "{forbidden}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn reasoning_effort_is_passed_as_its_level() {
|
|
for (effort, level) in [
|
|
(ReasoningEffort::Low, "low"),
|
|
(ReasoningEffort::Medium, "medium"),
|
|
(ReasoningEffort::High, "high"),
|
|
(ReasoningEffort::Max, "max"),
|
|
] {
|
|
let rig = rig(vec![ok_script(SESSION, "hi")]);
|
|
let mut asked = request(vec![user("hello")]);
|
|
asked.reasoning_effort = Some(effort);
|
|
rig.run(asked);
|
|
assert_eq!(arg_after(&rig.seen().argv, "--effort"), Some(level));
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn resume_always_forks() {
|
|
let rig = rig(vec![ok_script(SESSION, "hi"), ok_script(FORK, "again")]);
|
|
let first = request(vec![user("hello")]);
|
|
let result = rig.run(first.clone());
|
|
let seed_argv = rig.seen().argv;
|
|
assert!(
|
|
!seed_argv
|
|
.iter()
|
|
.any(|a| a == "--resume" || a == "--fork-session")
|
|
);
|
|
|
|
rig.run(continuing(&first, &result, "and then?"));
|
|
let argv = rig.seen().argv;
|
|
let at = argv.iter().position(|a| a == "--resume").expect("resumes");
|
|
assert_eq!(argv[at + 1], SESSION);
|
|
assert_eq!(argv[at + 2], "--fork-session");
|
|
assert_eq!(argv.iter().filter(|a| *a == "--fork-session").count(), 1);
|
|
}
|
|
|
|
#[test]
|
|
fn child_env_is_cleared_and_names_the_token_descriptor_never_the_token() {
|
|
let rig = rig(vec![ok_script(SESSION, "hi")]);
|
|
let result = rig.run(request(vec![user("hello")]));
|
|
let seen = rig.seen();
|
|
let mut env = seen.env.clone();
|
|
env.sort();
|
|
assert_eq!(
|
|
env,
|
|
vec![
|
|
(
|
|
"CLAUDE_CODE_MAX_OUTPUT_TOKENS".to_string(),
|
|
"1024".to_string()
|
|
),
|
|
(TOKEN_FD_ENV.to_string(), "3".to_string()),
|
|
("CLAUDE_CONFIG_DIR".to_string(), "/cfg".to_string()),
|
|
("HOME".to_string(), "/cfg".to_string()),
|
|
(
|
|
"PATH".to_string(),
|
|
"/usr/local/bin:/usr/bin:/bin".to_string()
|
|
),
|
|
]
|
|
);
|
|
assert_eq!(seen.cwd, PathBuf::from("/work"));
|
|
// The credential travels beside the environment, to the port, alone.
|
|
assert_eq!(seen.token, CANARY_TOKEN);
|
|
assert!(
|
|
seen.env
|
|
.iter()
|
|
.all(|(name, value)| name != TOKEN_ENV && value != CANARY_TOKEN)
|
|
);
|
|
assert!(!format!("{result:?}").contains(CANARY_TOKEN));
|
|
}
|
|
|
|
#[test]
|
|
fn the_token_never_reaches_a_message_an_event_or_a_debug_form() {
|
|
// A failing run, so a message is built, and the env's own Debug.
|
|
let rig = rig_for(
|
|
INSTANCE,
|
|
vec![Script::lines(vec![init_line(MODEL, SESSION)])],
|
|
&[SESSION],
|
|
CANARY_TOKEN.as_bytes(),
|
|
);
|
|
let result = rig.run(request(vec![user("hello")]));
|
|
let (_, message, _) = failure_of(&result);
|
|
assert!(!message.contains(CANARY_TOKEN));
|
|
assert!(!format!("{result:?}").contains(CANARY_TOKEN));
|
|
// The tail is the sink's alone.
|
|
assert_eq!(
|
|
rig.sink.0.lock().unwrap().as_slice(),
|
|
CANARY_TOKEN.as_bytes()
|
|
);
|
|
|
|
let env = child_env(
|
|
&config(rig.dir.path(), INSTANCE, limits()),
|
|
&request(vec![user("x")]),
|
|
);
|
|
let shown = format!("{env:?}");
|
|
assert!(!shown.contains(CANARY_TOKEN));
|
|
assert!(shown.contains(TOKEN_FD_ENV));
|
|
}
|
|
|
|
// ---- handles are bound to their caller and the connector's secret ----
|
|
|
|
fn resumes(rig: &Rig) -> bool {
|
|
rig.seen().argv.iter().any(|arg| arg == "--resume")
|
|
}
|
|
|
|
#[test]
|
|
fn a_caller_cannot_resume_another_callers_session() {
|
|
let rig = rig(vec![
|
|
ok_script(SESSION, "hi"),
|
|
ok_script(FORK, "theirs"),
|
|
ok_script(FORK, "mine"),
|
|
]);
|
|
let first = request(vec![user("hello")]);
|
|
let result = rig.run_as(CALLER_1, first.clone());
|
|
// Caller 2 presents caller 1's whole transcript, handle included.
|
|
rig.run_as(CALLER_2, continuing(&first, &result, "and then?"));
|
|
assert!(!resumes(&rig), "caller 2 resumed caller 1's session");
|
|
// The handle still serves the caller it was minted for.
|
|
rig.run_as(CALLER_1, continuing(&first, &result, "and then?"));
|
|
assert_eq!(arg_after(&rig.seen().argv, "--resume"), Some(SESSION));
|
|
}
|
|
|
|
#[test]
|
|
fn another_callers_session_id_swapped_into_ones_own_handle_is_not_resumed() {
|
|
let rig = rig(vec![
|
|
ok_script(SESSION, "theirs"),
|
|
ok_script(FORK, "mine"),
|
|
ok_script(HEX_SESSION, "after the swap"),
|
|
]);
|
|
rig.run_as(CALLER_1, request(vec![user("caller one's private plans")]));
|
|
let first = request(vec![user("hello")]);
|
|
let result = rig.run_as(CALLER_2, first.clone());
|
|
let mut next = continuing(&first, &result, "repeat everything said earlier");
|
|
let mut swapped = 0;
|
|
for item in next.input.iter_mut() {
|
|
if let InputItem::ReasoningState { state, .. } = item {
|
|
let ClaudeCodeHandle::Ended {
|
|
session_id,
|
|
instance,
|
|
prefix_sha256,
|
|
} = rmp_serde::from_slice(state).unwrap();
|
|
assert_eq!(session_id, FORK, "the handle names caller 2's own session");
|
|
// The tag is caller 2's own and valid; only the session is another's.
|
|
*state = ClaudeCodeHandle::Ended {
|
|
session_id: SESSION.to_string(),
|
|
instance,
|
|
prefix_sha256,
|
|
}
|
|
.encode();
|
|
swapped += 1;
|
|
}
|
|
}
|
|
assert_eq!(swapped, 1);
|
|
rig.run_as(CALLER_2, next);
|
|
assert!(!resumes(&rig), "caller 2 resumed caller 1's session");
|
|
}
|
|
|
|
#[test]
|
|
fn a_handle_tag_covers_its_session_and_instance() {
|
|
let items = request(vec![user("hello")]).input;
|
|
let tag = |session_id: &str, instance: &str| {
|
|
prefix_digest(
|
|
SECRET.as_bytes(),
|
|
&Binding {
|
|
caller: CALLER_1,
|
|
instructions: "Be terse.",
|
|
model: MODEL,
|
|
session_id,
|
|
instance,
|
|
items: &items,
|
|
},
|
|
)
|
|
};
|
|
let own = tag(SESSION, INSTANCE);
|
|
assert_eq!(own, tag(SESSION, INSTANCE));
|
|
assert_ne!(own, tag(FORK, INSTANCE));
|
|
assert_ne!(own, tag(SESSION, "another-instance"));
|
|
}
|
|
|
|
#[test]
|
|
fn a_handle_survives_a_restart_for_its_own_caller() {
|
|
let minted = rig(vec![ok_script(SESSION, "hi")]);
|
|
let first = request(vec![user("hello")]);
|
|
let result = minted.run_as(CALLER_2, first.clone());
|
|
drop(minted);
|
|
let restarted = rig(vec![ok_script(FORK, "again")]);
|
|
restarted.run_as(CALLER_2, continuing(&first, &result, "next"));
|
|
assert_eq!(arg_after(&restarted.seen().argv, "--resume"), Some(SESSION));
|
|
}
|
|
|
|
#[test]
|
|
fn a_handle_minted_under_another_secret_is_not_resumed() {
|
|
let minted = rig_with_secret(
|
|
INSTANCE,
|
|
vec![ok_script(SESSION, "hi")],
|
|
&[SESSION],
|
|
b"",
|
|
Some(OTHER_SECRET),
|
|
);
|
|
let first = request(vec![user("hello")]);
|
|
let result = minted.run(first.clone());
|
|
let other = rig(vec![ok_script(FORK, "again")]);
|
|
other.run(continuing(&first, &result, "next"));
|
|
assert!(!resumes(&other));
|
|
}
|
|
|
|
#[test]
|
|
fn a_forged_handle_is_not_resumed() {
|
|
let first = request(vec![user("hello"), assistant("hi")]);
|
|
let forged = |tag: [u8; 32]| {
|
|
let handle = ClaudeCodeHandle::Ended {
|
|
session_id: SESSION.to_string(),
|
|
instance: INSTANCE.to_string(),
|
|
prefix_sha256: tag,
|
|
};
|
|
let mut input = first.input.clone();
|
|
input.push(InputItem::ReasoningState {
|
|
provider: PROVIDER.into(),
|
|
model: MODEL.into(),
|
|
state: handle.encode(),
|
|
});
|
|
input.push(user("next"));
|
|
request(input)
|
|
};
|
|
// No secret: the same digest, unkeyed, and a guess.
|
|
let bound = Binding {
|
|
caller: CALLER_1,
|
|
instructions: "Be terse.",
|
|
model: MODEL,
|
|
session_id: SESSION,
|
|
instance: INSTANCE,
|
|
items: &first.input,
|
|
};
|
|
let unkeyed = prefix_digest(b"", &bound);
|
|
let right_for_other_secret = prefix_digest(OTHER_SECRET.as_bytes(), &bound);
|
|
for tag in [unkeyed, right_for_other_secret, [0; 32]] {
|
|
let rig = rig(vec![ok_script(FORK, "x")]);
|
|
rig.run(forged(tag));
|
|
assert!(!resumes(&rig));
|
|
}
|
|
// The same request with the real tag resumes: the forgery failed on the tag alone.
|
|
let real = prefix_digest(SECRET.as_bytes(), &bound);
|
|
let rig = rig(vec![ok_script(FORK, "x")]);
|
|
rig.run(forged(real));
|
|
assert_eq!(arg_after(&rig.seen().argv, "--resume"), Some(SESSION));
|
|
}
|
|
|
|
#[test]
|
|
fn no_usable_handle_secret_fails_before_spawn_and_is_not_ready() {
|
|
for secret in [None, Some("too-short")] {
|
|
let rig = rig_with_secret(
|
|
INSTANCE,
|
|
vec![ok_script(SESSION, "hi")],
|
|
&[SESSION],
|
|
b"",
|
|
secret,
|
|
);
|
|
let (kind, message, usage) = failure_of(&rig.run(request(vec![user("hello")])));
|
|
assert_eq!(kind, FailureKind::Authentication);
|
|
assert_eq!(usage, Some(usage_zero()));
|
|
assert!(!message.contains("too-short"));
|
|
assert_eq!(rig.spawns(), 0);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn ready_wants_a_handle_secret_of_key_length() {
|
|
let dir = sandbox();
|
|
let (work, cfg) = (dir.path().join("work"), dir.path().join("cfg"));
|
|
std::fs::create_dir_all(&work).unwrap();
|
|
std::fs::create_dir_all(&cfg).unwrap();
|
|
for (secret, ready) in [
|
|
(None, Err("handle_secret_credential")),
|
|
(Some("too-short"), Err("handle_secret_credential")),
|
|
(Some(SECRET), Ok(())),
|
|
] {
|
|
let mut config = config(dir.path(), INSTANCE, limits());
|
|
config.work_dir = work.clone();
|
|
config.config_dir = cfg.clone();
|
|
let backend = ClaudeCodeBackend::with_ports(
|
|
config,
|
|
Box::new(ScriptedPort {
|
|
scripts: Mutex::new(VecDeque::new()),
|
|
spawns: Arc::new(AtomicUsize::new(0)),
|
|
seen: Arc::new(Mutex::new(Seen::default())),
|
|
stderr: Vec::new(),
|
|
}),
|
|
Box::new(Sessions(vec![])),
|
|
Box::new(Sink::default()),
|
|
canary_token(),
|
|
Box::new(FixedSecret(secret)),
|
|
);
|
|
assert_eq!(backend.ready(), ready);
|
|
}
|
|
}
|
|
|
|
// ---- ready ----
|
|
|
|
/// A backend over a scripted port that never has a script: `ready` runs nothing.
|
|
fn ready_backend(config: ClaudeCodeConfig, tokens: Box<dyn TokenSource>) -> ClaudeCodeBackend {
|
|
ClaudeCodeBackend::with_ports(
|
|
config,
|
|
Box::new(ScriptedPort {
|
|
scripts: Mutex::new(VecDeque::new()),
|
|
spawns: Arc::new(AtomicUsize::new(0)),
|
|
seen: Arc::new(Mutex::new(Seen::default())),
|
|
stderr: Vec::new(),
|
|
}),
|
|
Box::new(Sessions(vec![])),
|
|
Box::new(Sink::default()),
|
|
tokens,
|
|
canary_secret(),
|
|
)
|
|
}
|
|
|
|
#[test]
|
|
fn ready_checks_the_pinned_binary_digest() {
|
|
let dir = sandbox();
|
|
let (work, cfg) = (dir.path().join("work"), dir.path().join("cfg"));
|
|
std::fs::create_dir_all(&work).unwrap();
|
|
std::fs::create_dir_all(&cfg).unwrap();
|
|
let binary = dir.path().join("claude");
|
|
|
|
let ready_field = |sha: [u8; 32], binary: &Path, work: &Path, prompts: &Path, token: bool| {
|
|
let mut config = config(dir.path(), INSTANCE, limits());
|
|
config.claude_binary = binary.to_path_buf();
|
|
config.claude_binary_sha256 = sha;
|
|
config.work_dir = work.to_path_buf();
|
|
config.config_dir = cfg.clone();
|
|
config.prompt_dir = prompts.to_path_buf();
|
|
let tokens = Box::new(FixedToken(token.then_some("t")));
|
|
ready_backend(config, tokens).ready()
|
|
};
|
|
let ready = |sha: [u8; 32], binary: &Path, work: &Path, prompts: &Path, token: bool| {
|
|
ready_field(sha, binary, work, prompts, token).is_ok()
|
|
};
|
|
let prompts = dir.path().join("prompts");
|
|
let pin = pin_of(PINNED);
|
|
assert_eq!(
|
|
ready_field([0; 32], &binary, &work, &prompts, true),
|
|
Err("claude_binary_sha256")
|
|
);
|
|
assert_eq!(
|
|
ready_field(pin, &binary, &dir.path().join("absent"), &prompts, true),
|
|
Err("work_dir")
|
|
);
|
|
assert_eq!(
|
|
ready_field(pin, &binary, &work, &dir.path().join("absent"), true),
|
|
Err("prompt_dir")
|
|
);
|
|
assert_eq!(
|
|
ready_field(pin, &binary, &work, &prompts, false),
|
|
Err("token_credential")
|
|
);
|
|
// The credential is checked last: a document fault is never hidden by it.
|
|
assert_eq!(
|
|
ready_field([0; 32], &binary, &work, &prompts, false),
|
|
Err("claude_binary_sha256")
|
|
);
|
|
assert_eq!(
|
|
ready_field(pin, &binary, &dir.path().join("absent"), &prompts, false),
|
|
Err("work_dir")
|
|
);
|
|
|
|
assert!(ready(pin, &binary, &work, &prompts, true));
|
|
assert!(!ready([0; 32], &binary, &work, &prompts, true), "other pin");
|
|
assert!(
|
|
!ready(pin, &dir.path().join("absent"), &work, &prompts, true),
|
|
"no binary"
|
|
);
|
|
assert!(
|
|
!ready(pin, &binary, &dir.path().join("absent"), &prompts, true),
|
|
"no work dir"
|
|
);
|
|
assert!(
|
|
!ready(pin, &binary, &work, &dir.path().join("absent"), true),
|
|
"no prompt dir"
|
|
);
|
|
assert!(!ready(pin, &binary, &work, &prompts, false), "no token");
|
|
std::fs::write(&binary, b"other bytes").unwrap();
|
|
assert!(
|
|
!ready(pin, &binary, &work, &prompts, true),
|
|
"bytes differ from the pin"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn token_comes_from_the_credential_file_not_the_environment() {
|
|
static CANARY_PARENT: std::sync::Once = std::sync::Once::new();
|
|
CANARY_PARENT.call_once(|| {
|
|
// SAFETY: set once to a value no code path reads; the test proves none does.
|
|
unsafe { std::env::set_var(TOKEN_ENV, "canary-environment-token") };
|
|
});
|
|
let dir = sandbox();
|
|
let credentials = dir.path().join("credentials");
|
|
for sub in ["credentials", "work", "cfg"] {
|
|
std::fs::create_dir_all(dir.path().join(sub)).unwrap();
|
|
}
|
|
let seen = Arc::new(Mutex::new(Seen::default()));
|
|
let mut config = config(dir.path(), INSTANCE, limits());
|
|
config.work_dir = dir.path().join("work");
|
|
config.config_dir = dir.path().join("cfg");
|
|
let backend = ClaudeCodeBackend::with_ports(
|
|
config,
|
|
Box::new(ScriptedPort {
|
|
scripts: Mutex::new(vec![ok_script(SESSION, "x")].into()),
|
|
spawns: Arc::new(AtomicUsize::new(0)),
|
|
seen: seen.clone(),
|
|
stderr: Vec::new(),
|
|
}),
|
|
Box::new(Sessions(vec![])),
|
|
Box::new(Sink::default()),
|
|
Box::new(CredentialTokenSource::in_directory(
|
|
&credentials,
|
|
"claude-token",
|
|
)),
|
|
canary_secret(),
|
|
);
|
|
let mut door = Door::new();
|
|
|
|
// The variable is set and the file is absent: nothing to give the child.
|
|
assert_eq!(backend.ready(), Err("token_credential"));
|
|
let (claim, _) = door.begin(request(vec![user("hi")]));
|
|
let (kind, message, usage) = failure_of(&backend.execute(claim.call()));
|
|
assert_eq!(kind, FailureKind::Authentication);
|
|
assert_eq!(usage, Some(usage_zero()));
|
|
assert!(!message.contains("canary-environment-token"));
|
|
assert!(seen.lock().unwrap().argv.is_empty(), "no child was spawned");
|
|
|
|
// The file is read afresh at the call, its trailing newline trimmed, and
|
|
// the port is handed the file's value, which is in no environment.
|
|
std::fs::write(
|
|
credentials.join("claude-token"),
|
|
format!("{CANARY_TOKEN}\n"),
|
|
)
|
|
.unwrap();
|
|
assert_eq!(backend.ready(), Ok(()));
|
|
let mut door = Door::new();
|
|
let (claim, _) = door.begin(request(vec![user("hi")]));
|
|
let result = backend.execute(claim.call());
|
|
assert!(matches!(
|
|
outcome(&result),
|
|
TransportOutcome::Completed { .. }
|
|
));
|
|
let given = seen.lock().unwrap().clone();
|
|
assert_eq!(given.token, CANARY_TOKEN);
|
|
assert!(
|
|
given
|
|
.env
|
|
.iter()
|
|
.all(|(name, value)| name != TOKEN_ENV && !value.contains("canary-environment-token"))
|
|
);
|
|
|
|
// An empty credential is no token.
|
|
std::fs::write(credentials.join("claude-token"), "\n").unwrap();
|
|
assert_eq!(backend.ready(), Err("token_credential"));
|
|
}
|
|
|
|
#[test]
|
|
fn an_unreadable_credential_is_no_token() {
|
|
// A credential that is a directory, or a directory that is absent.
|
|
let dir = sandbox();
|
|
std::fs::create_dir_all(dir.path().join("claude-token")).unwrap();
|
|
let source = CredentialTokenSource::in_directory(dir.path(), "claude-token");
|
|
assert!(source.read().is_none());
|
|
assert!(
|
|
CredentialTokenSource::in_directory(dir.path().join("absent"), "x")
|
|
.read()
|
|
.is_none()
|
|
);
|
|
}
|
|
|
|
// ---- planning ----
|
|
|
|
#[test]
|
|
fn seed_renders_one_user_line_deterministically() {
|
|
let input = vec![
|
|
user("a"),
|
|
assistant("b"),
|
|
InputItem::ToolCall {
|
|
call_id: "call-1".into(),
|
|
name: "lookup".into(),
|
|
arguments: "{}".into(),
|
|
},
|
|
InputItem::ToolResult {
|
|
call_id: "call-1".into(),
|
|
output: "out".into(),
|
|
},
|
|
InputItem::ReasoningState {
|
|
provider: "other".into(),
|
|
model: "m".into(),
|
|
state: vec![1],
|
|
},
|
|
assistant("e"),
|
|
user("c"),
|
|
user("d"),
|
|
];
|
|
let rig = rig(vec![ok_script(SESSION, "x"), ok_script(SESSION, "x")]);
|
|
rig.run(request(input.clone()));
|
|
let first = rig.seen().stdin;
|
|
rig.run(request(input));
|
|
assert_eq!(rig.seen().stdin, first);
|
|
|
|
let transcript = "The conversation so far, replayed as text because the earlier session cannot be continued:\\n\\n[user]\\na\\n\\n[assistant]\\nb\\n\\n[tool_call name=lookup id=call-1]\\n{}\\n\\n[tool_result id=call-1]\\nout\\n\\n[assistant]\\ne";
|
|
let expected = format!(
|
|
r#"{{"type":"user","message":{{"role":"user","content":[{{"type":"text","text":"{transcript}"}},{{"type":"text","text":"c"}},{{"type":"text","text":"d"}}]}},"parent_tool_use_id":null}}"#
|
|
) + "\n";
|
|
assert_eq!(String::from_utf8(first).unwrap(), expected);
|
|
}
|
|
|
|
#[test]
|
|
fn a_first_message_is_one_text_block_with_no_transcript() {
|
|
let rig = rig(vec![ok_script(SESSION, "x")]);
|
|
rig.run(request(vec![user("hello")]));
|
|
let stdin = String::from_utf8(rig.seen().stdin).unwrap();
|
|
assert_eq!(
|
|
stdin,
|
|
"{\"type\":\"user\",\"message\":{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"hello\"}]},\"parent_tool_use_id\":null}\n"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn a_resume_sends_only_the_new_user_text() {
|
|
let rig = rig(vec![ok_script(SESSION, "hi"), ok_script(FORK, "again")]);
|
|
let first = request(vec![user("hello")]);
|
|
let result = rig.run(first.clone());
|
|
let mut next = continuing(&first, &result, "one");
|
|
next.input.push(user("two"));
|
|
rig.run(next);
|
|
let stdin = String::from_utf8(rig.seen().stdin).unwrap();
|
|
assert_eq!(
|
|
stdin,
|
|
"{\"type\":\"user\",\"message\":{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"one\"},{\"type\":\"text\",\"text\":\"two\"}]},\"parent_tool_use_id\":null}\n"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn handle_round_trips_through_transcript_items() {
|
|
let rig = rig(vec![ok_script(SESSION, "hi"), ok_script(FORK, "again")]);
|
|
let first = request(vec![user("hello")]);
|
|
let result = rig.run(first.clone());
|
|
let items = result.transcript_items();
|
|
assert_eq!(items.len(), 2);
|
|
assert_eq!(items[0], assistant("hi"));
|
|
assert!(
|
|
matches!(&items[1], InputItem::ReasoningState { provider, model, .. }
|
|
if provider == PROVIDER && model == MODEL)
|
|
);
|
|
|
|
rig.run(continuing(&first, &result, "next"));
|
|
assert_eq!(arg_after(&rig.seen().argv, "--resume"), Some(SESSION));
|
|
}
|
|
|
|
#[test]
|
|
fn a_result_with_no_text_still_resumes() {
|
|
let silent = Script::lines(vec![
|
|
init_line(MODEL, SESSION),
|
|
thinking_line("msg_1"),
|
|
result_line("end_turn", USAGE),
|
|
]);
|
|
let rig = rig(vec![silent, ok_script(FORK, "again")]);
|
|
let first = request(vec![user("hello")]);
|
|
let result = rig.run(first.clone());
|
|
assert_eq!(result.transcript_items().len(), 1, "only the handle");
|
|
rig.run(continuing(&first, &result, "next"));
|
|
assert_eq!(arg_after(&rig.seen().argv, "--resume"), Some(SESSION));
|
|
}
|
|
|
|
#[test]
|
|
fn changed_instructions_or_model_reseeds() {
|
|
let rig = rig(vec![
|
|
ok_script(SESSION, "hi"),
|
|
ok_script(FORK, "x"),
|
|
ok_script(FORK, "x"),
|
|
]);
|
|
let first = request(vec![user("hello")]);
|
|
let result = rig.run(first.clone());
|
|
|
|
let mut other_instructions = continuing(&first, &result, "next");
|
|
other_instructions.instructions = "Be verbose.".into();
|
|
rig.run(other_instructions);
|
|
assert_eq!(arg_after(&rig.seen().argv, "--resume"), None);
|
|
|
|
let mut other_model = continuing(&first, &result, "next");
|
|
other_model.model = OTHER_MODEL.into();
|
|
let script = Script::lines(vec![
|
|
init_line(OTHER_MODEL, FORK),
|
|
result_line("end_turn", USAGE),
|
|
]);
|
|
let rig2 = rig_for(INSTANCE, vec![script], &[SESSION], b"");
|
|
rig2.run(other_model);
|
|
assert_eq!(arg_after(&rig2.seen().argv, "--resume"), None);
|
|
}
|
|
|
|
#[test]
|
|
fn foreign_instance_missing_session_or_edited_prefix_reseeds() {
|
|
let first = request(vec![user("hello"), assistant("earlier"), user("more")]);
|
|
let make = |instance: &str, sessions: &[&str]| {
|
|
let rig = rig_for(
|
|
instance,
|
|
vec![ok_script(SESSION, "hi"), ok_script(FORK, "x")],
|
|
sessions,
|
|
b"",
|
|
);
|
|
let result = rig.run(first.clone());
|
|
(rig, result)
|
|
};
|
|
|
|
// Control: the same instance with its session resumes.
|
|
let (rig_ok, result) = make(INSTANCE, &[SESSION]);
|
|
rig_ok.run(continuing(&first, &result, "next"));
|
|
assert_eq!(arg_after(&rig_ok.seen().argv, "--resume"), Some(SESSION));
|
|
|
|
// Another instance wrote the handle.
|
|
let (rig_a, result) = make("persona-b", &[SESSION]);
|
|
let handle_request = continuing(&first, &result, "next");
|
|
let rig_b = rig_for(INSTANCE, vec![ok_script(FORK, "x")], &[SESSION], b"");
|
|
rig_b.run(handle_request);
|
|
assert_eq!(arg_after(&rig_b.seen().argv, "--resume"), None);
|
|
drop(rig_a);
|
|
|
|
// The session file is gone.
|
|
let (rig_c, result) = make(INSTANCE, &[]);
|
|
rig_c.run(continuing(&first, &result, "next"));
|
|
assert_eq!(arg_after(&rig_c.seen().argv, "--resume"), None);
|
|
|
|
// An item before the handle was edited.
|
|
let (rig_d, result) = make(INSTANCE, &[SESSION]);
|
|
let mut edited = continuing(&first, &result, "next");
|
|
edited.input[1] = assistant("edited");
|
|
rig_d.run(edited);
|
|
assert_eq!(arg_after(&rig_d.seen().argv, "--resume"), None);
|
|
}
|
|
|
|
#[test]
|
|
fn assistant_items_after_the_handle_reseed() {
|
|
for tail in [
|
|
vec![assistant("interjection"), user("next")],
|
|
vec![
|
|
InputItem::ToolCall {
|
|
call_id: "c".into(),
|
|
name: "n".into(),
|
|
arguments: "{}".into(),
|
|
},
|
|
user("next"),
|
|
],
|
|
] {
|
|
let rig = rig(vec![ok_script(SESSION, "hi"), ok_script(FORK, "x")]);
|
|
let first = request(vec![user("hello")]);
|
|
let result = rig.run(first.clone());
|
|
let mut input = first.input.clone();
|
|
input.extend(result.transcript_items());
|
|
input.extend(tail);
|
|
rig.run(request(input));
|
|
assert_eq!(arg_after(&rig.seen().argv, "--resume"), None);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn the_last_own_handle_decides() {
|
|
let rig = rig_for(
|
|
INSTANCE,
|
|
vec![
|
|
ok_script(SESSION, "one"),
|
|
ok_script(FORK, "two"),
|
|
ok_script(SESSION, "three"),
|
|
],
|
|
&[SESSION, FORK],
|
|
b"",
|
|
);
|
|
let first = request(vec![user("a")]);
|
|
let r1 = rig.run(first.clone());
|
|
let second = continuing(&first, &r1, "b");
|
|
let r2 = rig.run(second.clone());
|
|
let third = continuing(&second, &r2, "c");
|
|
rig.run(third);
|
|
// The newest handle names the fork of the first resume, not the first session.
|
|
assert_eq!(arg_after(&rig.seen().argv, "--resume"), Some(FORK));
|
|
}
|
|
|
|
#[test]
|
|
fn undecodable_own_handle_fails_before_spawn() {
|
|
let rig = rig(vec![]);
|
|
let bad = request(vec![
|
|
user("a"),
|
|
InputItem::ReasoningState {
|
|
provider: PROVIDER.into(),
|
|
model: MODEL.into(),
|
|
state: vec![1, 2, 3],
|
|
},
|
|
user("b"),
|
|
]);
|
|
let (kind, _, usage) = failure_of(&rig.run(bad));
|
|
assert_eq!(kind, FailureKind::InvalidRequest);
|
|
assert_eq!(usage, Some(usage_zero()));
|
|
assert_eq!(rig.spawns(), 0);
|
|
}
|
|
|
|
fn usage_zero() -> Usage {
|
|
usage(0, 0, 0, 0, None)
|
|
}
|
|
|
|
#[test]
|
|
fn tools_or_final_tool_result_fail_before_spawn() {
|
|
let mut with_tools = request(vec![user("a")]);
|
|
with_tools.tools.push(a_tool());
|
|
let final_tool_result = request(vec![
|
|
user("a"),
|
|
InputItem::ToolCall {
|
|
call_id: "c".into(),
|
|
name: "n".into(),
|
|
arguments: "{}".into(),
|
|
},
|
|
InputItem::ToolResult {
|
|
call_id: "c".into(),
|
|
output: "o".into(),
|
|
},
|
|
]);
|
|
let ends_without_user_text = request(vec![user("a"), assistant("b")]);
|
|
let tool_result_beside_text = request(vec![
|
|
assistant("b"),
|
|
InputItem::ToolResult {
|
|
call_id: "c".into(),
|
|
output: "o".into(),
|
|
},
|
|
user("a"),
|
|
]);
|
|
|
|
let rig = rig(vec![]);
|
|
let mut messages = Vec::new();
|
|
for refused in [
|
|
with_tools,
|
|
final_tool_result,
|
|
ends_without_user_text,
|
|
tool_result_beside_text,
|
|
] {
|
|
let (kind, message, usage) = failure_of(&rig.run(refused));
|
|
assert_eq!(kind, FailureKind::InvalidRequest);
|
|
assert_eq!(usage, Some(usage_zero()));
|
|
messages.push(message);
|
|
}
|
|
assert_eq!(rig.spawns(), 0);
|
|
// Each refusal names what lifts it.
|
|
assert_eq!(messages[1], messages[3]);
|
|
let distinct: std::collections::HashSet<_> = messages.iter().collect();
|
|
assert_eq!(distinct.len(), 3, "{messages:?}");
|
|
}
|
|
|
|
// ---- the stream ----
|
|
|
|
#[test]
|
|
fn stream_maps_text_in_order_and_ends_with_the_handle() {
|
|
let script = Script::lines(vec![
|
|
init_line(MODEL, SESSION),
|
|
thinking_line("msg_0"),
|
|
text_line("msg_1", "Hel"),
|
|
text_line("msg_2", ""),
|
|
text_line("msg_3", "lo"),
|
|
r#"{"type":"system","subtype":"post_turn_summary","summary":"x"}"#.to_string(),
|
|
r#"{"type":"unheard_of","field":1}"#.to_string(),
|
|
result_line("end_turn", USAGE),
|
|
]);
|
|
let rig = rig(vec![script]);
|
|
let result = rig.run(request(vec![user("hi")]));
|
|
let (events, receipt) = parts(&result);
|
|
let payloads: Vec<_> = events.iter().map(|e| e.payload.clone()).collect();
|
|
assert_eq!(payloads.len(), 3);
|
|
assert_eq!(
|
|
payloads[0],
|
|
TransportEventPayload::TextDelta { text: "Hel".into() }
|
|
);
|
|
assert_eq!(
|
|
payloads[1],
|
|
TransportEventPayload::TextDelta { text: "lo".into() }
|
|
);
|
|
assert!(matches!(
|
|
payloads[2],
|
|
TransportEventPayload::ReasoningState { .. }
|
|
));
|
|
assert_eq!(
|
|
events.iter().map(|e| e.sequence).collect::<Vec<_>>(),
|
|
vec![0, 1, 2]
|
|
);
|
|
let TransportOutcome::Completed {
|
|
provider_response_id,
|
|
stop_reason,
|
|
usage: _,
|
|
} = &receipt.outcome
|
|
else {
|
|
panic!("completed expected");
|
|
};
|
|
assert_eq!(provider_response_id.as_deref(), Some("msg_3"));
|
|
assert_eq!(*stop_reason, model_connector::StopReason::EndTurn);
|
|
}
|
|
|
|
#[test]
|
|
fn every_stop_reason_has_its_own_mapping() {
|
|
use model_connector::StopReason::*;
|
|
for (cli, expected) in [
|
|
("end_turn", EndTurn),
|
|
("max_tokens", MaxOutputTokens),
|
|
("stop_sequence", StopSequence),
|
|
("refusal", Refusal),
|
|
("model_context_window_exceeded", ContextWindow),
|
|
] {
|
|
let script = Script::lines(vec![init_line(MODEL, SESSION), result_line(cli, USAGE)]);
|
|
let result = rig(vec![script]).run(request(vec![user("hi")]));
|
|
let TransportOutcome::Completed { stop_reason, .. } = outcome(&result) else {
|
|
panic!("{cli} should complete");
|
|
};
|
|
assert_eq!(*stop_reason, expected, "{cli}");
|
|
}
|
|
for odd in ["tool_use", "pause_turn", ""] {
|
|
let script = Script::lines(vec![init_line(MODEL, SESSION), result_line(odd, USAGE)]);
|
|
let (kind, _, usage) = failure_of(&rig(vec![script]).run(request(vec![user("hi")])));
|
|
assert_eq!(kind, FailureKind::Provider);
|
|
assert!(usage.is_some());
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn usage_sums_model_usage_across_models() {
|
|
let two_models = r#"{"claude-haiku-5-5":{"inputTokens":2,"outputTokens":4,"cacheReadInputTokens":7,"cacheCreationInputTokens":1043,"thinkingTokens":3},"claude-sonnet-5":{"inputTokens":10,"outputTokens":20,"cacheReadInputTokens":30,"cacheCreationInputTokens":40,"thinkingTokens":5}}"#;
|
|
let run = |total_cost: &str| {
|
|
let result_line = format!(
|
|
r#"{{"type":"result","subtype":"success","is_error":false,"stop_reason":"end_turn","total_cost_usd":{total_cost},"model_usage_since_last_result":{two_models}}}"#
|
|
);
|
|
let script = Script::lines(vec![init_line(MODEL, SESSION), result_line]);
|
|
rig(vec![script]).run(request(vec![user("hi")]))
|
|
};
|
|
let (cheap, dear) = (run("0.000001"), run("99.5"));
|
|
let TransportOutcome::Completed { usage: summed, .. } = outcome(&cheap) else {
|
|
panic!("completed expected");
|
|
};
|
|
assert_eq!(*summed, usage(12, 37, 1083, 24, Some(8)));
|
|
// The receipt is the usage alone: the CLI's own total never reaches it.
|
|
assert_eq!(parts(&cheap).1, parts(&dear).1);
|
|
|
|
// No model reported thinking tokens: none is claimed.
|
|
let plain = r#"{"claude-haiku-5-5":{"inputTokens":1,"outputTokens":1,"cacheReadInputTokens":0,"cacheCreationInputTokens":0}}"#;
|
|
let script = Script::lines(vec![
|
|
init_line(MODEL, SESSION),
|
|
result_line("end_turn", plain),
|
|
]);
|
|
let result = rig(vec![script]).run(request(vec![user("hi")]));
|
|
let TransportOutcome::Completed { usage, .. } = outcome(&result) else {
|
|
panic!("completed expected");
|
|
};
|
|
assert_eq!(*usage, self::usage(1, 0, 0, 1, None));
|
|
}
|
|
|
|
#[test]
|
|
fn init_model_mismatch_is_a_provider_failure() {
|
|
let script = Script::lines(vec![
|
|
init_line("haiku", SESSION),
|
|
result_line("end_turn", USAGE),
|
|
]);
|
|
let (kind, _, usage) = failure_of(&rig(vec![script]).run(request(vec![user("hi")])));
|
|
assert_eq!(kind, FailureKind::Provider);
|
|
assert_eq!(usage, None);
|
|
}
|
|
|
|
#[test]
|
|
fn result_errors_map_to_failure_kinds_with_their_usage() {
|
|
let rate_limit_text = r#","result":"You have hit your rate limit""#;
|
|
let cases: Vec<(Vec<String>, FailureKind)> = vec![
|
|
(
|
|
vec![rate_limit_line("rejected"), error_result_line("")],
|
|
FailureKind::RateLimited,
|
|
),
|
|
// The rejection latches: a later event that allows does not undo it.
|
|
(
|
|
vec![
|
|
rate_limit_line("rejected"),
|
|
rate_limit_line("allowed"),
|
|
error_result_line(""),
|
|
],
|
|
FailureKind::RateLimited,
|
|
),
|
|
(
|
|
vec![error_result_line(r#","api_error_status":429"#)],
|
|
FailureKind::RateLimited,
|
|
),
|
|
(
|
|
vec![error_result_line(rate_limit_text)],
|
|
FailureKind::RateLimited,
|
|
),
|
|
(
|
|
vec![error_result_line(r#","errors":["rate_limit_error"]"#)],
|
|
FailureKind::RateLimited,
|
|
),
|
|
(
|
|
vec![error_result_line(r#","api_error_status":401"#)],
|
|
FailureKind::Authentication,
|
|
),
|
|
(
|
|
vec![error_result_line(r#","api_error_status":403"#)],
|
|
FailureKind::Authentication,
|
|
),
|
|
(
|
|
vec![error_result_line(r#","api_error_status":400"#)],
|
|
FailureKind::InvalidRequest,
|
|
),
|
|
(
|
|
vec![error_result_line(r#","api_error_status":500"#)],
|
|
FailureKind::Unavailable,
|
|
),
|
|
(
|
|
vec![error_result_line(r#","api_error_status":529"#)],
|
|
FailureKind::Unavailable,
|
|
),
|
|
(
|
|
vec![error_result_line(r#","api_error_status":null"#)],
|
|
FailureKind::Provider,
|
|
),
|
|
(
|
|
vec![rate_limit_line("allowed"), error_result_line("")],
|
|
FailureKind::Provider,
|
|
),
|
|
];
|
|
for (lines, expected) in cases {
|
|
let mut all = vec![init_line(MODEL, SESSION)];
|
|
all.extend(lines.clone());
|
|
let result = rig(vec![Script::lines(all)]).run(request(vec![user("hi")]));
|
|
let (kind, _, usage) = failure_of(&result);
|
|
assert_eq!(kind, expected, "{lines:?}");
|
|
assert!(
|
|
usage.is_some(),
|
|
"a result event carries its usage: {lines:?}"
|
|
);
|
|
}
|
|
// The usage of an error result is the sum the event reported.
|
|
let result = rig(vec![Script::lines(vec![
|
|
init_line(MODEL, SESSION),
|
|
error_result_line(""),
|
|
])])
|
|
.run(request(vec![user("hi")]));
|
|
assert_eq!(failure_of(&result).2, Some(usage(2, 7, 1043, 4, Some(3))));
|
|
}
|
|
|
|
#[test]
|
|
fn no_result_event_is_a_failure_with_unknown_usage() {
|
|
let ends: Vec<(Script, FailureKind)> = vec![
|
|
(
|
|
Script::lines(vec![init_line(MODEL, SESSION), text_line("m", "x")]),
|
|
FailureKind::Provider,
|
|
),
|
|
(
|
|
Script::lines(vec![
|
|
init_line(MODEL, SESSION),
|
|
"not json at all".to_string(),
|
|
]),
|
|
FailureKind::Provider,
|
|
),
|
|
(
|
|
Script::lines(vec![init_line(MODEL, SESSION)]).ending(Terminal::Breach),
|
|
FailureKind::Provider,
|
|
),
|
|
(
|
|
Script::lines(vec![init_line(MODEL, SESSION)]).ending(Terminal::Timeout),
|
|
FailureKind::Timeout,
|
|
),
|
|
];
|
|
let zero_billed = {
|
|
let (_, _, usage) = failure_of(&rig(vec![]).run({
|
|
let mut bad = request(vec![user("a")]);
|
|
bad.tools.push(a_tool());
|
|
bad
|
|
}));
|
|
usage
|
|
};
|
|
assert_eq!(zero_billed, Some(usage_zero()));
|
|
|
|
for (script, expected) in ends {
|
|
let rig = rig(vec![script]);
|
|
let result = rig.run(request(vec![user("hi")]));
|
|
let (kind, message, usage) = failure_of(&result);
|
|
assert_eq!(kind, expected);
|
|
assert_eq!(usage, None, "{message}");
|
|
assert!(result.transcript_items().is_empty());
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn tool_use_or_compaction_in_the_stream_is_a_provider_failure() {
|
|
for line in [
|
|
tool_use_line(),
|
|
r#"{"type":"system","subtype":"compact_boundary"}"#.to_string(),
|
|
] {
|
|
let script = Script::lines(vec![
|
|
init_line(MODEL, SESSION),
|
|
text_line("m", "partial"),
|
|
line.clone(),
|
|
result_line("end_turn", USAGE),
|
|
]);
|
|
let result = rig(vec![script]).run(request(vec![user("hi")]));
|
|
let (kind, _, usage) = failure_of(&result);
|
|
assert_eq!(kind, FailureKind::Provider, "{line}");
|
|
assert_eq!(usage, None);
|
|
assert!(
|
|
parts(&result).0.is_empty(),
|
|
"no partial reasoning is replayed"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn a_result_before_any_init_names_no_session() {
|
|
let script = Script::lines(vec![result_line("end_turn", USAGE)]);
|
|
let (kind, _, usage) = failure_of(&rig(vec![script]).run(request(vec![user("hi")])));
|
|
assert_eq!(kind, FailureKind::Provider);
|
|
assert!(usage.is_some());
|
|
}
|
|
|
|
// ---- spawns ----
|
|
|
|
#[test]
|
|
fn one_spawn_per_execute() {
|
|
let scripts: Vec<(Script, usize)> = vec![
|
|
(ok_script(SESSION, "x"), 1),
|
|
(
|
|
Script::lines(vec![init_line(MODEL, SESSION), error_result_line("")]),
|
|
1,
|
|
),
|
|
(Script::lines(vec![]).ending(Terminal::Eof), 1),
|
|
(Script::lines(vec![]).ending(Terminal::Timeout), 1),
|
|
(Script::lines(vec![]).ending(Terminal::Breach), 1),
|
|
];
|
|
for (script, expected) in scripts {
|
|
let rig = rig(vec![script]);
|
|
rig.run(request(vec![user("hi")]));
|
|
assert_eq!(rig.spawns(), expected);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn a_missing_token_fails_before_spawn() {
|
|
let mut rig = rig(vec![ok_script(SESSION, "x")]);
|
|
rig.backend.tokens = Box::new(FixedToken(None));
|
|
let (kind, _, usage) = failure_of(&rig.run(request(vec![user("hi")])));
|
|
assert_eq!(kind, FailureKind::Authentication);
|
|
assert_eq!(usage, Some(usage_zero()));
|
|
assert_eq!(rig.spawns(), 0);
|
|
}
|
|
|
|
#[test]
|
|
fn a_port_that_cannot_spawn_is_a_provider_failure() {
|
|
struct Refusing;
|
|
impl ProcessPort for Refusing {
|
|
fn spawn(
|
|
&self,
|
|
_: &[String],
|
|
_: &ChildEnv,
|
|
_: &Path,
|
|
_: Vec<u8>,
|
|
_: &str,
|
|
) -> io::Result<Box<dyn RunningChild>> {
|
|
Err(io::Error::from(io::ErrorKind::NotFound))
|
|
}
|
|
}
|
|
let dir = sandbox();
|
|
let backend = ClaudeCodeBackend::with_ports(
|
|
config(dir.path(), INSTANCE, limits()),
|
|
Box::new(Refusing),
|
|
Box::new(Sessions(vec![])),
|
|
Box::new(Sink::default()),
|
|
canary_token(),
|
|
canary_secret(),
|
|
);
|
|
let mut door = Door::new();
|
|
let (claim, _) = door.begin(request(vec![user("hi")]));
|
|
let result = backend.execute(claim.call());
|
|
let (kind, message, usage) = failure_of(&result);
|
|
assert_eq!(kind, FailureKind::Provider);
|
|
assert_eq!(
|
|
usage,
|
|
Some(usage_zero()),
|
|
"a child that never ran bills nothing"
|
|
);
|
|
assert!(
|
|
!message.contains(dir.path().to_str().unwrap()),
|
|
"no path in a message"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn backend_end_to_end_through_the_service() {
|
|
let rig = rig(vec![ok_script(SESSION, "hello back")]);
|
|
let mut door = Door::new();
|
|
let (claim, invocation) = door.begin(request(vec![user("hello")]));
|
|
let result = rig.backend.execute(claim.call());
|
|
let sealed = door.service.complete(claim, result, FRAME).unwrap();
|
|
let opened = decrypt_result(
|
|
&sealed,
|
|
&TransportKey::from_connection_secret("caller-1-key").unwrap(),
|
|
&invocation,
|
|
)
|
|
.unwrap();
|
|
let (events, receipt) = parts(&opened);
|
|
assert_eq!(receipt.provider, "claude-code");
|
|
assert_eq!(receipt.model, MODEL);
|
|
// USAGE counted: 2 input, 7 cache read, 1043 cache write, 4 output.
|
|
assert_eq!(events.len(), 2);
|
|
let items = opened.transcript_items();
|
|
assert_eq!(items[0], assistant("hello back"));
|
|
assert!(
|
|
matches!(&items[1], InputItem::ReasoningState { provider, .. } if provider == "claude-code")
|
|
);
|
|
}
|
|
|
|
// ---- usage: this turn's own, or unknown ----
|
|
|
|
const USAGE_TURN_2: &str = r#"{"claude-haiku-5-5":{"inputTokens":3,"outputTokens":5,"cacheReadInputTokens":1100,"cacheCreationInputTokens":9}}"#;
|
|
|
|
#[test]
|
|
fn a_resumed_turn_is_charged_only_its_own_usage() {
|
|
let rig = rig(vec![
|
|
ok_script(SESSION, "one"),
|
|
Script::lines(vec![
|
|
init_line(MODEL, FORK),
|
|
text_line("msg_2", "two"),
|
|
// The CLI's `modelUsage` here is the cumulative CUMULATIVE of
|
|
// result_line; only the turn's own field may be billed.
|
|
result_line("end_turn", USAGE_TURN_2),
|
|
]),
|
|
]);
|
|
let first_request = request(vec![user("hello")]);
|
|
let first = rig.run(first_request.clone());
|
|
let second = rig.run(continuing(&first_request, &first, "again"));
|
|
assert!(
|
|
rig.seen().argv.iter().any(|arg| arg == "--resume"),
|
|
"the second turn resumes"
|
|
);
|
|
let TransportOutcome::Completed { usage: billed, .. } = outcome(&second) else {
|
|
panic!("completed expected");
|
|
};
|
|
assert_eq!(*billed, usage(3, 1100, 9, 5, None));
|
|
// 3*3 + 1100*0.3 + 9*6 + 5*15 micro-USD.
|
|
}
|
|
|
|
#[test]
|
|
fn usage_the_result_does_not_report_stays_unknown() {
|
|
let success = |fields: &str| {
|
|
format!(
|
|
r#"{{"type":"result","subtype":"success","is_error":false,"stop_reason":"end_turn","modelUsage":{CUMULATIVE}{fields}}}"#
|
|
)
|
|
};
|
|
let error = |fields: &str| {
|
|
format!(
|
|
r#"{{"type":"result","subtype":"error_during_execution","is_error":true,"modelUsage":{CUMULATIVE}{fields}}}"#
|
|
)
|
|
};
|
|
let renamed = r#","model_usage_since_last_result":{"claude-haiku-5-5":{"input_tokens":5000,"output_tokens":9000}}"#;
|
|
let no_cache_counts = r#","model_usage_since_last_result":{"claude-haiku-5-5":{"inputTokens":1,"outputTokens":1}}"#;
|
|
let cases = [
|
|
success(""),
|
|
success(r#","model_usage_since_last_result":{}"#),
|
|
success(r#","model_usage_since_last_result":null"#),
|
|
success(r#","model_usage_since_last_result":"none""#),
|
|
success(renamed),
|
|
success(no_cache_counts),
|
|
error(""),
|
|
error(r#","model_usage_since_last_result":{}"#),
|
|
error(renamed),
|
|
];
|
|
// An empty report from a server error, or none from a refusal, is not a
|
|
// refusal at the door that reported nothing spent.
|
|
let by_status = [
|
|
(
|
|
r#","api_error_status":500,"model_usage_since_last_result":{}"#,
|
|
FailureKind::Unavailable,
|
|
),
|
|
(r#","api_error_status":401"#, FailureKind::Authentication),
|
|
];
|
|
for (fields, expected) in by_status {
|
|
let line = error(fields);
|
|
let result = rig(vec![Script::lines(vec![
|
|
init_line(MODEL, SESSION),
|
|
text_line("m", "work"),
|
|
line.clone(),
|
|
])])
|
|
.run(request(vec![user("hi")]));
|
|
let (kind, _, usage) = failure_of(&result);
|
|
assert_eq!(kind, expected, "{line}");
|
|
assert_eq!(usage, None, "{line}");
|
|
}
|
|
for line in cases {
|
|
let result = rig(vec![Script::lines(vec![
|
|
init_line(MODEL, SESSION),
|
|
text_line("m", "work"),
|
|
line.clone(),
|
|
])])
|
|
.run(request(vec![user("hi")]));
|
|
let (kind, _, usage) = failure_of(&result);
|
|
assert_eq!(kind, FailureKind::Provider, "{line}");
|
|
assert_eq!(usage, None, "{line}");
|
|
assert!(result.transcript_items().is_empty(), "{line}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn door_set_is_exactly_401_403_429() {
|
|
// Ruling claude-code-plan-quota-fuse-2 with its
|
|
// validity: a refusal at the door is charged nothing, for exactly the
|
|
// statuses the provider cannot have billed (401, 403, 429) and only when
|
|
// the result says its usage is explicitly empty. Every other shape is
|
|
// the usage is unknown. The 401 shape is a real claude 2.1.312 capture
|
|
// with a bogus token.
|
|
let retry = r#"{"type":"system","subtype":"api_retry","error":"authentication_failed"}"#;
|
|
let synthetic = r#"{"type":"assistant","api_error_status":401,"error":"authentication_failed","message":{"id":"e48001b7-cf4f-49b1-9810-db8fd7f73c73","model":"<synthetic>","stop_reason":"stop_sequence","content":[{"type":"text","text":"Failed to authenticate. API Error: 401 OAuth access token is invalid."}]}}"#;
|
|
let statuses: [Option<u16>; 9] = [
|
|
Some(400),
|
|
Some(401),
|
|
Some(402),
|
|
Some(403),
|
|
Some(404),
|
|
Some(429),
|
|
Some(500),
|
|
Some(529),
|
|
None,
|
|
];
|
|
let shapes = [
|
|
(Some("{}"), "empty"),
|
|
(None, "absent"),
|
|
(Some("null"), "null"),
|
|
];
|
|
for status in statuses {
|
|
for (since_last, shape) in shapes {
|
|
let status_field = status
|
|
.map(|status| format!(r#""api_error_status":{status},"#))
|
|
.unwrap_or_default();
|
|
let usage_field = since_last
|
|
.map(|usage| format!(r#","model_usage_since_last_result":{usage}"#))
|
|
.unwrap_or_default();
|
|
let result_line = format!(
|
|
r#"{{"type":"result","subtype":"success","is_error":true,{status_field}"stop_reason":"stop_sequence","result":"Failed.","modelUsage":{CUMULATIVE}{usage_field},"terminal_reason":"api_error"}}"#
|
|
);
|
|
let result = rig(vec![Script::lines(vec![
|
|
init_line(MODEL, SESSION),
|
|
retry.to_string(),
|
|
synthetic.to_string(),
|
|
result_line,
|
|
])])
|
|
.run(request(vec![user("hi")]));
|
|
let (_, _, usage) = failure_of(&result);
|
|
let case = format!("{status:?} {shape}");
|
|
if matches!(status, Some(401 | 403 | 429)) && shape == "empty" {
|
|
assert_eq!(usage, Some(usage_zero()), "{case}");
|
|
} else {
|
|
assert_eq!(usage, None, "{case}");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn the_deadline_is_the_configured_deadline() {
|
|
let rig = rig(vec![ok_script(SESSION, "x")]);
|
|
let (before, configured) = (Instant::now(), limits().deadline);
|
|
rig.run(request(vec![user("hi")]));
|
|
let after = Instant::now();
|
|
let deadlines = rig.seen().deadlines.lock().unwrap().clone();
|
|
assert!(deadlines.len() >= 2, "the result read and the drain read");
|
|
for deadline in deadlines {
|
|
// Neither clamped nor scaled: exactly `configured` after the call began.
|
|
assert!(
|
|
deadline >= before + configured,
|
|
"the deadline was shortened"
|
|
);
|
|
assert!(deadline <= after + configured, "the deadline was stretched");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn the_system_prompt_goes_in_a_file_however_large() {
|
|
let instructions = "p".repeat(300 * 1024);
|
|
let rig = rig(vec![ok_script(SESSION, "x")]);
|
|
let mut big = request(vec![user("hi")]);
|
|
big.instructions = instructions.clone();
|
|
rig.run(big);
|
|
let seen = rig.seen();
|
|
assert!(
|
|
seen.argv.iter().all(|arg| arg.len() < 4096),
|
|
"no argument carries the instructions"
|
|
);
|
|
let (path, contents) = seen.prompt_file.expect("the argv names a prompt file");
|
|
assert_eq!(contents, instructions);
|
|
assert!(!path.exists(), "the backend removes the file it staged");
|
|
}
|
|
|
|
#[test]
|
|
fn the_prompt_file_is_removed_when_the_call_fails() {
|
|
for script in [
|
|
Script::lines(vec![]).ending(Terminal::Eof),
|
|
Script::lines(vec![]).ending(Terminal::Timeout),
|
|
Script::lines(vec![init_line(MODEL, SESSION), error_result_line("")]),
|
|
] {
|
|
let rig = rig(vec![script]);
|
|
rig.run(request(vec![user("hi")]));
|
|
let (path, _) = rig.seen().prompt_file.expect("a prompt file");
|
|
assert!(!path.exists());
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn prompt_file_lives_in_prompt_dir_exclusively() {
|
|
let rig = rig(vec![ok_script(SESSION, "x")]);
|
|
rig.run(request(vec![user("hi")]));
|
|
let (path, _) = rig.seen().prompt_file.expect("a prompt file");
|
|
assert_eq!(
|
|
path.parent(),
|
|
Some(rig.dir.path().join("prompts").as_path())
|
|
);
|
|
|
|
// A prompt directory that cannot take the file fails staging: nothing
|
|
// reached the provider, so it bills nothing, and no child runs.
|
|
let rig = rig_for(INSTANCE, vec![ok_script(SESSION, "x")], &[SESSION], b"");
|
|
std::fs::remove_dir_all(rig.dir.path().join("prompts")).unwrap();
|
|
let result = rig.run(request(vec![user("hi")]));
|
|
let (kind, _, usage) = failure_of(&result);
|
|
assert_eq!((kind, usage), (FailureKind::Provider, Some(usage_zero())));
|
|
assert_eq!(rig.spawns(), 0);
|
|
}
|
|
|
|
#[test]
|
|
fn pin_is_checked_before_every_spawn() {
|
|
let rig = rig(vec![ok_script(SESSION, "x"), ok_script(SESSION, "y")]);
|
|
let binary = rig.dir.path().join("claude");
|
|
assert!(rig.backend.binary_matches_pin());
|
|
rig.run(request(vec![user("hi")]));
|
|
assert_eq!(rig.spawns(), 1);
|
|
|
|
// The binary is swapped after readiness: the next call never spawns.
|
|
std::fs::write(&binary, b"swapped bytes").unwrap();
|
|
let result = rig.run(request(vec![user("hi")]));
|
|
let (kind, message, usage) = failure_of(&result);
|
|
assert_eq!((kind, usage), (FailureKind::Provider, Some(usage_zero())));
|
|
assert!(!message.contains(rig.dir.path().to_str().unwrap()));
|
|
assert_eq!(rig.spawns(), 1, "no child for a binary off its pin");
|
|
assert!(!rig.backend.binary_matches_pin());
|
|
let staged = std::fs::read_dir(rig.dir.path().join("prompts")).unwrap();
|
|
assert_eq!(staged.count(), 0, "no prompt was staged for it");
|
|
|
|
// Restoring the pinned bytes is enough: nothing remembers the verdict.
|
|
std::fs::write(&binary, PINNED).unwrap();
|
|
rig.run(request(vec![user("hi")]));
|
|
assert_eq!(rig.spawns(), 2);
|
|
}
|
|
|
|
#[test]
|
|
fn the_provider_tag_is_claude_code() {
|
|
assert_eq!(rig(vec![]).backend.provider(), "claude-code");
|
|
}
|
|
|
|
// ---- the session store ----
|
|
|
|
#[test]
|
|
fn the_session_store_finds_only_canonical_session_files() {
|
|
let dir = TempDir::new();
|
|
let project = dir.path().join("projects").join("-work");
|
|
std::fs::create_dir_all(&project).unwrap();
|
|
std::fs::write(project.join(format!("{SESSION}.jsonl")), b"{}").unwrap();
|
|
std::fs::write(project.join(format!("{HEX_SESSION}.jsonl")), b"{}").unwrap();
|
|
// Files a loose check would find: one character short, one too long.
|
|
std::fs::write(project.join(format!("{}.jsonl", &SESSION[..35])), b"{}").unwrap();
|
|
std::fs::write(project.join(format!("{SESSION}0.jsonl")), b"{}").unwrap();
|
|
std::fs::write(dir.path().join("projects").join("secret.jsonl"), b"{}").unwrap();
|
|
let store = FsSessionStore::new(dir.path().to_path_buf());
|
|
assert!(store.exists(SESSION));
|
|
assert!(!store.exists(FORK));
|
|
assert!(!store.exists("secret"));
|
|
assert!(!store.exists("../secret"));
|
|
assert!(store.exists(HEX_SESSION));
|
|
assert!(!store.exists(&HEX_SESSION.to_uppercase()));
|
|
assert!(!store.exists("gggggggg-bbbb-cccc-dddd-eeeeeeeeeeee"));
|
|
assert!(!store.exists(&SESSION[..35]));
|
|
assert!(!store.exists(&format!("{SESSION}0")));
|
|
assert!(!store.exists("11111111_2222-3333-4444-555555555555"));
|
|
assert!(!FsSessionStore::new(dir.path().join("absent")).exists(SESSION));
|
|
}
|
|
|
|
// ---- real processes: a fake claude shell script (unix) ----
|
|
|
|
struct TempDir(PathBuf);
|
|
|
|
impl TempDir {
|
|
fn new() -> Self {
|
|
let path = std::env::temp_dir().join(format!(
|
|
"claude-connector-test-{}-{}",
|
|
std::process::id(),
|
|
DOOR_SEQUENCE.fetch_add(1, Ordering::SeqCst)
|
|
));
|
|
std::fs::create_dir_all(&path).unwrap();
|
|
Self(path)
|
|
}
|
|
fn path(&self) -> &Path {
|
|
&self.0
|
|
}
|
|
}
|
|
|
|
impl Drop for TempDir {
|
|
fn drop(&mut self) {
|
|
let _ = std::fs::remove_dir_all(&self.0);
|
|
}
|
|
}
|
|
|
|
// ---- structured output and the plan quota ----
|
|
|
|
#[test]
|
|
fn schema_request_lowers_to_json_schema() {
|
|
let rig = rig(vec![structured_script(), ok_script(SESSION, "hi")]);
|
|
rig.run(schema_request());
|
|
let argv = rig.seen().argv;
|
|
assert_eq!(arg_after(&argv, "--json-schema"), Some(SCHEMA));
|
|
assert_eq!(argv.iter().filter(|a| *a == "--json-schema").count(), 1);
|
|
// The tool-less hygiene stays with it.
|
|
assert_eq!(arg_after(&argv, "--tools"), Some(""));
|
|
assert!(argv.iter().any(|a| a == "--strict-mcp-config"));
|
|
assert!(!argv.iter().any(|a| a == "--mcp-config"));
|
|
|
|
rig.run(request(vec![user("hello")]));
|
|
assert!(!rig.seen().argv.iter().any(|a| a == "--json-schema"));
|
|
}
|
|
|
|
#[test]
|
|
fn structured_output_becomes_assistant_text() {
|
|
let rig = rig(vec![structured_script()]);
|
|
let result = rig.run(schema_request());
|
|
let TransportOutcome::Completed {
|
|
stop_reason, usage, ..
|
|
} = outcome(&result)
|
|
else {
|
|
panic!("expected completion, got {:?}", outcome(&result));
|
|
};
|
|
assert_eq!(*stop_reason, StopReason::EndTurn);
|
|
assert_eq!(*usage, usage_cost_fixture());
|
|
let (events, _) = parts(&result);
|
|
let texts: Vec<&str> = events
|
|
.iter()
|
|
.filter_map(|event| match &event.payload {
|
|
TransportEventPayload::TextDelta { text } => Some(text.as_str()),
|
|
_ => None,
|
|
})
|
|
.collect();
|
|
assert_eq!(texts, [STRUCTURED]);
|
|
let items = result.transcript_items();
|
|
assert_eq!(items[0], assistant(STRUCTURED));
|
|
// The answer is JSON the caller decodes with serde: it is the object.
|
|
let decoded: serde_json::Value = serde_json::from_str(STRUCTURED).unwrap();
|
|
assert_eq!(decoded["answer"], "say hello");
|
|
}
|
|
|
|
fn usage_cost_fixture() -> Usage {
|
|
usage(2, 7, 1043, 4, Some(3))
|
|
}
|
|
|
|
#[test]
|
|
fn a_schema_turn_without_structured_output_is_a_provider_failure_billed_as_reported() {
|
|
for line in [
|
|
result_line("tool_use", USAGE),
|
|
result_line_with("end_turn", USAGE, r#","structured_output":null"#),
|
|
] {
|
|
let rig = rig(vec![Script::lines(vec![
|
|
init_line(MODEL, SESSION),
|
|
text_line("msg_1", "prose, not the schema"),
|
|
line,
|
|
])]);
|
|
let result = rig.run(schema_request());
|
|
let (kind, _, usage) = failure_of(&result);
|
|
assert_eq!(kind, FailureKind::Provider);
|
|
assert_eq!(usage, Some(usage_cost_fixture()), "the turn was billed");
|
|
assert!(result.transcript_items().is_empty());
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn only_the_structured_output_tool_is_allowed_and_only_when_a_schema_was_sent() {
|
|
// Without a schema the same call is a tool in a tool-less turn.
|
|
let first = rig(vec![structured_script()]);
|
|
let (kind, message, _) = failure_of(&first.run(request(vec![user("hello")])));
|
|
assert_eq!(kind, FailureKind::Provider);
|
|
assert!(message.contains("tool"), "{message}");
|
|
|
|
// With a schema, any other tool is still refused.
|
|
let second = rig(vec![Script::lines(vec![
|
|
init_line(MODEL, SESSION),
|
|
structured_call_line("Bash"),
|
|
structured_result_line(),
|
|
])]);
|
|
let (kind, message, _) = failure_of(&second.run(schema_request()));
|
|
assert_eq!(kind, FailureKind::Provider);
|
|
assert!(message.contains("tool"), "{message}");
|
|
|
|
// `tool_use` is a stop reason only for a schema turn.
|
|
let third = rig(vec![Script::lines(vec![
|
|
init_line(MODEL, SESSION),
|
|
text_line("msg_1", "hi"),
|
|
result_line("tool_use", USAGE),
|
|
])]);
|
|
let (kind, message, _) = failure_of(&third.run(request(vec![user("hello")])));
|
|
assert_eq!(kind, FailureKind::Provider);
|
|
assert!(message.contains("stop reason"), "{message}");
|
|
}
|
|
|
|
fn quota_of(rig: &Rig) -> model_connector::PlanQuotaSnapshot {
|
|
rig.backend.plan_quota()
|
|
}
|
|
|
|
/// The windows with their observation stamps zeroed, for comparing readings.
|
|
fn readings(rig: &Rig) -> Vec<model_connector::PlanQuota> {
|
|
let mut windows = quota_of(rig).windows;
|
|
for window in &mut windows {
|
|
window.observed_at_unix_ms = 0;
|
|
}
|
|
windows
|
|
}
|
|
|
|
fn window(name: &str, permille: u32, resets_at_unix_ms: u64) -> model_connector::PlanQuota {
|
|
model_connector::PlanQuota {
|
|
window: name.to_string(),
|
|
utilization_permille: Some(permille),
|
|
resets_at_unix_ms,
|
|
status: "allowed".to_string(),
|
|
observed_at_unix_ms: 0,
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn rate_limit_event_updates_quota_cell() {
|
|
let rig = rig(vec![structured_script()]);
|
|
assert!(quota_of(&rig).windows.is_empty());
|
|
let before = std::time::SystemTime::now()
|
|
.duration_since(std::time::UNIX_EPOCH)
|
|
.unwrap()
|
|
.as_millis() as u64;
|
|
rig.run(schema_request());
|
|
let snapshot = quota_of(&rig);
|
|
assert_eq!(
|
|
readings(&rig),
|
|
vec![
|
|
window("five_hour", 80, 4_102_444_800_000),
|
|
window("seven_day", 300, 4_102_531_200_000),
|
|
]
|
|
);
|
|
let after = std::time::SystemTime::now()
|
|
.duration_since(std::time::UNIX_EPOCH)
|
|
.unwrap()
|
|
.as_millis() as u64;
|
|
for each in &snapshot.windows {
|
|
assert!((before..=after).contains(&each.observed_at_unix_ms));
|
|
}
|
|
// The cell is the one the backend hands out.
|
|
assert_eq!(rig.backend.quota().snapshot(), snapshot);
|
|
}
|
|
|
|
#[test]
|
|
fn a_rejected_turn_still_reports_its_windows_and_a_bad_window_costs_only_itself() {
|
|
let event = r#"{"type":"rate_limit_event","rate_limit_info":{"status":"rejected","unifiedWindows":{"five_hour":{"utilization":0.9996,"resetsAt":4102444800,"status":"rejected"},"seven_day":{"utilization":0,"resetsAt":4102531200,"status":"allowed"},"seven_day_overage_included":{"utilization":1,"resetsAt":4102531200,"status":"rejected"},"idle":{"utilization":0,"resetsAt":4102444800,"status":"allowed"}}}}"#;
|
|
let rig = rig(vec![Script::lines(vec![
|
|
init_line(MODEL, SESSION),
|
|
event.to_string(),
|
|
error_result_line(r#","api_error_status":429"#),
|
|
])]);
|
|
let (kind, _, _) = failure_of(&rig.run(request(vec![user("hello")])));
|
|
assert_eq!(kind, FailureKind::RateLimited);
|
|
let mut rejected = window("five_hour", 1000, 4_102_444_800_000);
|
|
rejected.status = "rejected".to_string();
|
|
// A window at exactly zero use is a reading, and is kept. Only the two
|
|
// windows pacing reads are taken: a per-model bucket that reads rejected
|
|
// must not close every lease, and no other name is a window of the plan.
|
|
assert_eq!(
|
|
readings(&rig),
|
|
vec![rejected, window("seven_day", 0, 4_102_531_200_000)]
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn a_later_event_replaces_only_the_windows_it_names() {
|
|
let only_five = r#"{"type":"rate_limit_event","rate_limit_info":{"status":"allowed","unifiedWindows":{"five_hour":{"utilization":0.5,"resetsAt":4102448400,"status":"allowed"}}}}"#;
|
|
let rig = rig(vec![
|
|
structured_script(),
|
|
Script::lines(vec![
|
|
init_line(MODEL, SESSION),
|
|
only_five.to_string(),
|
|
text_line("msg_1", "hi"),
|
|
result_line("end_turn", USAGE),
|
|
]),
|
|
]);
|
|
rig.run(schema_request());
|
|
rig.run(request(vec![user("again")]));
|
|
assert_eq!(
|
|
readings(&rig),
|
|
vec![
|
|
window("five_hour", 500, 4_102_448_400_000),
|
|
window("seven_day", 300, 4_102_531_200_000),
|
|
]
|
|
);
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
mod real {
|
|
use super::*;
|
|
use std::os::unix::fs::PermissionsExt;
|
|
|
|
/// Writing an executable and spawning it must not interleave with another
|
|
/// thread's fork (ETXTBSY), so every real-process test holds this.
|
|
static SPAWN_LOCK: Mutex<()> = Mutex::new(());
|
|
|
|
struct Fake {
|
|
_guard: std::sync::MutexGuard<'static, ()>,
|
|
dir: TempDir,
|
|
backend: ClaudeCodeBackend,
|
|
sink: Sink,
|
|
}
|
|
|
|
fn fake(body: &str, limits: ClaudeCodeLimits) -> Fake {
|
|
let guard = SPAWN_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
|
let dir = TempDir::new();
|
|
let (work, cfg) = (dir.path().join("work"), dir.path().join("cfg"));
|
|
std::fs::create_dir_all(&work).unwrap();
|
|
std::fs::create_dir_all(&cfg).unwrap();
|
|
let binary = dir.path().join("claude");
|
|
std::fs::write(&binary, format!("#!/bin/sh\n{body}\n")).unwrap();
|
|
std::fs::set_permissions(&binary, std::fs::Permissions::from_mode(0o755)).unwrap();
|
|
let prompts = dir.path().join("prompts");
|
|
std::fs::create_dir_all(&prompts).unwrap();
|
|
let mut config = config(dir.path(), INSTANCE, limits);
|
|
config.claude_binary_sha256 = pin_of(&std::fs::read(&binary).unwrap());
|
|
config.claude_binary = binary;
|
|
config.work_dir = work;
|
|
config.config_dir = cfg;
|
|
let sink = Sink::default();
|
|
let backend = ClaudeCodeBackend::with_ports(
|
|
config.clone(),
|
|
Box::new(crate::process::StdProcessPort::new(config.limits)),
|
|
Box::new(FsSessionStore::new(config.config_dir.clone())),
|
|
Box::new(sink.clone()),
|
|
canary_token(),
|
|
canary_secret(),
|
|
);
|
|
Fake {
|
|
_guard: guard,
|
|
dir,
|
|
backend,
|
|
sink,
|
|
}
|
|
}
|
|
|
|
impl Fake {
|
|
fn run(&self, request: ProviderRequest) -> TransportResult {
|
|
let mut door = Door::new();
|
|
let (claim, _) = door.begin(request);
|
|
self.backend.execute(claim.call())
|
|
}
|
|
}
|
|
|
|
fn printf_lines(lines: &[String]) -> String {
|
|
let quoted: Vec<String> = lines.iter().map(|l| format!("'{l}'")).collect();
|
|
format!("printf '%s\\n' {}", quoted.join(" "))
|
|
}
|
|
|
|
fn good_lines() -> Vec<String> {
|
|
vec![
|
|
init_line(MODEL, SESSION),
|
|
text_line("msg_1", "hi"),
|
|
result_line("end_turn", USAGE),
|
|
]
|
|
}
|
|
|
|
fn alive(pid: &str) -> bool {
|
|
match std::fs::read_to_string(format!("/proc/{pid}/stat")) {
|
|
Ok(stat) => stat
|
|
.rsplit(')')
|
|
.next()
|
|
.is_some_and(|rest| !rest.trim_start().starts_with('Z')),
|
|
Err(_) => false,
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn deadline_kills_the_process_group() {
|
|
let probe = TempDir::new();
|
|
let pidfile = probe.path().join("grandchild.pid");
|
|
let body = format!("sleep 300 &\necho $! > '{}'\nwait", pidfile.display());
|
|
let mut short = limits();
|
|
short.deadline = Duration::from_millis(700);
|
|
let fake = fake(&body, short);
|
|
|
|
let started = Instant::now();
|
|
let result = fake.run(request(vec![user("hi")]));
|
|
let (kind, _, usage) = failure_of(&result);
|
|
assert_eq!(kind, FailureKind::Timeout);
|
|
assert_eq!(usage, None);
|
|
assert!(started.elapsed() < Duration::from_secs(20));
|
|
|
|
let pid = std::fs::read_to_string(&pidfile)
|
|
.unwrap()
|
|
.trim()
|
|
.to_string();
|
|
assert!(!alive(&pid), "the grandchild {pid} outlived the call");
|
|
}
|
|
|
|
fn pid_in(file: &Path) -> String {
|
|
for _ in 0..200 {
|
|
if let Ok(pid) = std::fs::read_to_string(file)
|
|
&& pid.ends_with('\n')
|
|
{
|
|
return pid.trim().to_string();
|
|
}
|
|
std::thread::sleep(Duration::from_millis(10));
|
|
}
|
|
panic!("the fake claude never wrote its pid");
|
|
}
|
|
|
|
#[test]
|
|
fn the_child_is_reaped_when_the_call_ends() {
|
|
let probe = TempDir::new();
|
|
let pidfile = probe.path().join("child.pid");
|
|
let body = format!(
|
|
"echo $$ > '{}'\n{}",
|
|
pidfile.display(),
|
|
printf_lines(&good_lines())
|
|
);
|
|
let fake = fake(&body, limits());
|
|
let result = fake.run(request(vec![user("hi")]));
|
|
assert!(matches!(
|
|
outcome(&result),
|
|
TransportOutcome::Completed { .. }
|
|
));
|
|
let pid = pid_in(&pidfile);
|
|
assert!(
|
|
!Path::new(&format!("/proc/{pid}")).exists(),
|
|
"the child {pid} was left unreaped"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn a_dropped_child_takes_its_group_with_it() {
|
|
let probe = TempDir::new();
|
|
let (leader, grandchild) = (
|
|
probe.path().join("leader.pid"),
|
|
probe.path().join("grandchild.pid"),
|
|
);
|
|
let body = format!(
|
|
"sleep 300 &\necho $! > '{}'\necho $$ > '{}'\nwait",
|
|
grandchild.display(),
|
|
leader.display()
|
|
);
|
|
let fake = fake(&body, limits());
|
|
let config = config(fake.dir.path(), INSTANCE, limits());
|
|
let env = child_env(&config, &request(vec![user("x")]));
|
|
let argv = vec![
|
|
fake.dir
|
|
.path()
|
|
.join("claude")
|
|
.to_string_lossy()
|
|
.into_owned(),
|
|
];
|
|
let port = crate::process::StdProcessPort::new(limits());
|
|
let child = port
|
|
.spawn(&argv, &env, fake.dir.path(), Vec::new(), "t")
|
|
.unwrap();
|
|
let (leader_pid, grandchild_pid) = (pid_in(&leader), pid_in(&grandchild));
|
|
drop(child);
|
|
assert!(
|
|
!Path::new(&format!("/proc/{leader_pid}")).exists(),
|
|
"leader reaped"
|
|
);
|
|
assert!(
|
|
!alive(&grandchild_pid),
|
|
"the grandchild outlived the dropped child"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn stderr_stays_in_the_sink() {
|
|
let fake = fake("echo canary-stderr-text >&2\nexit 1", limits());
|
|
let result = fake.run(request(vec![user("hi")]));
|
|
let (kind, message, usage) = failure_of(&result);
|
|
assert_eq!(kind, FailureKind::Provider);
|
|
assert_eq!(usage, None);
|
|
assert!(!message.contains("canary-stderr-text"));
|
|
assert!(!format!("{result:?}").contains("canary-stderr-text"));
|
|
let sunk = String::from_utf8(fake.sink.0.lock().unwrap().clone()).unwrap();
|
|
assert!(sunk.contains("canary-stderr-text"), "{sunk:?}");
|
|
}
|
|
|
|
#[test]
|
|
fn the_real_port_clears_the_environment_and_delivers_stdin() {
|
|
static CANARY_PARENT: std::sync::Once = std::sync::Once::new();
|
|
CANARY_PARENT.call_once(|| {
|
|
// SAFETY: set once, before any child is spawned, to a value no test reads back.
|
|
unsafe { std::env::set_var("ANTHROPIC_API_KEY", "canary-api-key") };
|
|
});
|
|
let probe = TempDir::new();
|
|
let body = format!(
|
|
"env > '{d}/env'\npwd > '{d}/pwd'\ncat > '{d}/stdin'\n{}",
|
|
printf_lines(&good_lines()),
|
|
d = probe.path().display()
|
|
);
|
|
let fake = fake(&body, limits());
|
|
let result = fake.run(request(vec![user("hello")]));
|
|
assert!(matches!(
|
|
outcome(&result),
|
|
TransportOutcome::Completed { .. }
|
|
));
|
|
|
|
let env = std::fs::read_to_string(probe.path().join("env")).unwrap();
|
|
assert!(!env.contains("canary-api-key") && !env.contains("ANTHROPIC_API_KEY"));
|
|
let mut names: Vec<&str> = env
|
|
.lines()
|
|
.filter_map(|l| l.split_once('=').map(|(n, _)| n))
|
|
// a shell adds these three to its own environment
|
|
.filter(|n| !matches!(*n, "PWD" | "OLDPWD" | "SHLVL" | "_"))
|
|
.collect();
|
|
names.sort();
|
|
assert_eq!(
|
|
names,
|
|
[
|
|
"CLAUDE_CODE_MAX_OUTPUT_TOKENS",
|
|
TOKEN_FD_ENV,
|
|
"CLAUDE_CONFIG_DIR",
|
|
"HOME",
|
|
"PATH"
|
|
]
|
|
);
|
|
assert!(env.contains(&format!("{TOKEN_FD_ENV}=3")));
|
|
assert!(!env.contains(CANARY_TOKEN));
|
|
let cwd = std::fs::read_to_string(probe.path().join("pwd")).unwrap();
|
|
assert_eq!(cwd.trim(), fake.dir.path().join("work").to_str().unwrap());
|
|
let stdin = std::fs::read_to_string(probe.path().join("stdin")).unwrap();
|
|
assert!(stdin.ends_with('\n') && stdin.matches('\n').count() == 1);
|
|
assert!(stdin.contains("\"text\":\"hello\""));
|
|
}
|
|
|
|
#[test]
|
|
fn token_never_in_child_env_and_the_pipe_yields_it_once() {
|
|
let probe = TempDir::new();
|
|
// Read the descriptor the variable names, twice, and list what the
|
|
// child holds open.
|
|
let body = format!(
|
|
"fd=\"${TOKEN_FD_ENV}\"\neval \"cat <&$fd\" > '{d}/first'\neval \"cat <&$fd\" > '{d}/second'\nenv > '{d}/env'\nls /proc/$$/fd > '{d}/fds'\n{lines}",
|
|
d = probe.path().display(),
|
|
lines = printf_lines(&good_lines())
|
|
);
|
|
let fake = fake(&body, limits());
|
|
let result = fake.run(request(vec![user("hello")]));
|
|
assert!(matches!(
|
|
outcome(&result),
|
|
TransportOutcome::Completed { .. }
|
|
));
|
|
let read = |name: &str| std::fs::read_to_string(probe.path().join(name)).unwrap();
|
|
assert_eq!(read("first"), CANARY_TOKEN, "the pipe holds the token");
|
|
assert_eq!(read("second"), "", "and nothing the second time");
|
|
let env = read("env");
|
|
assert!(!env.contains(CANARY_TOKEN) && !env.contains(&format!("{TOKEN_ENV}=")));
|
|
assert!(env.contains(&format!("{TOKEN_FD_ENV}=3")));
|
|
// 0 1 2, the pipe at 3, and the shell's own working descriptors:
|
|
// no descriptor of the connector's other pipes survives into the child.
|
|
let held: Vec<u32> = read("fds")
|
|
.lines()
|
|
.filter_map(|l| l.trim().parse().ok())
|
|
.collect();
|
|
assert!(held.contains(&3));
|
|
}
|
|
|
|
#[test]
|
|
fn a_token_too_large_for_a_pipe_is_refused_at_its_bound_not_hung() {
|
|
let fake = fake("exit 0", limits());
|
|
let config = config(fake.dir.path(), INSTANCE, limits());
|
|
let env = child_env(&config, &request(vec![user("x")]));
|
|
let port = crate::process::StdProcessPort::new(limits());
|
|
let argv = vec![
|
|
fake.dir
|
|
.path()
|
|
.join("claude")
|
|
.to_string_lossy()
|
|
.into_owned(),
|
|
];
|
|
let spawn = |token: &str| port.spawn(&argv, &env, fake.dir.path(), Vec::new(), token);
|
|
let at_bound = spawn(&"t".repeat(4096)).expect("4096 bytes fit the bound");
|
|
drop(at_bound);
|
|
assert_eq!(
|
|
spawn(&"t".repeat(4097)).err().map(|e| e.kind()),
|
|
Some(io::ErrorKind::InvalidInput)
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn the_output_caps_hold_at_their_exact_sizes() {
|
|
let lines = good_lines();
|
|
let longest = lines.iter().map(String::len).max().unwrap();
|
|
let total: usize = lines.iter().map(|l| l.len() + 1).sum();
|
|
let body = printf_lines(&lines);
|
|
let with = |line: usize, all: usize| ClaudeCodeLimits {
|
|
max_line_bytes: line,
|
|
max_total_bytes: all,
|
|
..limits()
|
|
};
|
|
|
|
let at_both = fake(&body, with(longest, total)).run(request(vec![user("hi")]));
|
|
assert!(matches!(
|
|
outcome(&at_both),
|
|
TransportOutcome::Completed { .. }
|
|
));
|
|
|
|
for (line, all) in [(longest - 1, total), (longest, total - 1)] {
|
|
let result = fake(&body, with(line, all)).run(request(vec![user("hi")]));
|
|
let (kind, _, usage) = failure_of(&result);
|
|
assert_eq!((kind, usage), (FailureKind::Provider, None), "{line} {all}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn instructions_beyond_the_argument_limit_reach_the_child_in_a_private_file() {
|
|
// One argument is capped near 128 KiB on Linux; instructions are not.
|
|
let instructions = "i".repeat(300 * 1024);
|
|
let probe = TempDir::new();
|
|
let d = probe.path().display();
|
|
let body = format!(
|
|
"while [ $# -gt 0 ]; do\n\
|
|
if [ \"$1\" = --system-prompt-file ]; then\n\
|
|
cp \"$2\" '{d}/prompt'; stat -c %a \"$2\" > '{d}/mode'; echo \"$2\" > '{d}/path'\n\
|
|
fi\nshift\ndone\n{}",
|
|
printf_lines(&good_lines())
|
|
);
|
|
let fake = fake(&body, limits());
|
|
let mut big = request(vec![user("hi")]);
|
|
big.instructions = instructions.clone();
|
|
let result = fake.run(big);
|
|
assert!(matches!(
|
|
outcome(&result),
|
|
TransportOutcome::Completed { .. }
|
|
));
|
|
assert_eq!(
|
|
std::fs::read_to_string(probe.path().join("prompt")).unwrap(),
|
|
instructions
|
|
);
|
|
assert_eq!(
|
|
std::fs::read_to_string(probe.path().join("mode"))
|
|
.unwrap()
|
|
.trim(),
|
|
"600"
|
|
);
|
|
let path = std::fs::read_to_string(probe.path().join("path")).unwrap();
|
|
assert_eq!(
|
|
Path::new(path.trim()).parent(),
|
|
Some(fake.dir.path().join("prompts").as_path()),
|
|
"staged in the configured prompt directory"
|
|
);
|
|
assert!(!Path::new(path.trim()).exists(), "removed after the call");
|
|
}
|
|
|
|
#[test]
|
|
fn the_call_waits_for_the_child_to_close_stdout_after_its_result() {
|
|
// The CLI writes its session after the result; killing it at the
|
|
// result would lose the session a resume needs.
|
|
let probe = TempDir::new();
|
|
let body = format!(
|
|
"{}\nsleep 1\necho written > '{}/after-result'",
|
|
printf_lines(&good_lines()),
|
|
probe.path().display()
|
|
);
|
|
let result = fake(&body, limits()).run(request(vec![user("hi")]));
|
|
assert!(matches!(
|
|
outcome(&result),
|
|
TransportOutcome::Completed { .. }
|
|
));
|
|
assert!(
|
|
probe.path().join("after-result").exists(),
|
|
"the child was killed before it finished"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn the_stderr_tail_is_complete_when_the_call_ends() {
|
|
let body = format!(
|
|
"head -c 300000 /dev/zero | tr '\\0' x >&2\necho END-OF-STDERR >&2\n{}",
|
|
printf_lines(&good_lines())
|
|
);
|
|
let fake = fake(&body, limits());
|
|
fake.run(request(vec![user("hi")]));
|
|
let sunk = String::from_utf8(fake.sink.0.lock().unwrap().clone()).unwrap();
|
|
assert!(sunk.trim_end().ends_with("END-OF-STDERR"), "{}", sunk.len());
|
|
}
|
|
|
|
#[test]
|
|
fn stderr_join_survives_a_megabyte_after_result() {
|
|
// Soul s2's probe: the result line first, then a megabyte of stderr
|
|
// before the child exits. The call must wait for the stderr reader to
|
|
// finish, or the tail handed to the sink is cut short.
|
|
let body = format!(
|
|
"{}\nhead -c 1000000 /dev/zero | tr '\\0' x >&2\necho END-OF-STDERR >&2",
|
|
printf_lines(&good_lines())
|
|
);
|
|
let fake = fake(&body, limits());
|
|
let result = fake.run(request(vec![user("hi")]));
|
|
assert!(matches!(
|
|
outcome(&result),
|
|
TransportOutcome::Completed { .. }
|
|
));
|
|
let sunk = String::from_utf8(fake.sink.0.lock().unwrap().clone()).unwrap();
|
|
assert!(sunk.trim_end().ends_with("END-OF-STDERR"), "{}", sunk.len());
|
|
}
|
|
|
|
#[test]
|
|
fn a_result_is_enough_even_if_the_child_keeps_stdout_open() {
|
|
let body = format!("{}\nexec sleep 300", printf_lines(&good_lines()));
|
|
let mut short = limits();
|
|
short.deadline = Duration::from_millis(500);
|
|
let result = fake(&body, short).run(request(vec![user("hi")]));
|
|
assert!(matches!(
|
|
outcome(&result),
|
|
TransportOutcome::Completed { .. }
|
|
));
|
|
}
|
|
}
|
|
|
|
// ---- a call that never reached the provider ----
|
|
|
|
/// The shape claude 2.1.312 gave with no usable credential: an error result
|
|
/// with no status, no API time and no usage rows. `extra` replaces the
|
|
/// per-turn usage field.
|
|
fn not_logged_in_line(status: &str, api_ms: u64, extra: &str) -> String {
|
|
format!(
|
|
r#"{{"type":"result","subtype":"success","is_error":true,{status}"duration_ms":118,"duration_api_ms":{api_ms},"stop_reason":"stop_sequence","result":"Not logged in","total_cost_usd":0,"modelUsage":{{}}{extra}}}"#
|
|
)
|
|
}
|
|
|
|
#[test]
|
|
fn a_result_that_made_no_api_call_bills_nothing() {
|
|
let empty = r#","model_usage_since_last_result":{}"#;
|
|
let cases = [
|
|
// (status field, api ms, usage field, billed nothing)
|
|
(r#""api_error_status":null,"#, 0, empty, true),
|
|
("", 0, empty, true),
|
|
("", 0, "", true),
|
|
// An API call was made, or the status names a server fault, or the
|
|
// CLI reported rows: the usage stays unknown.
|
|
("", 2300, empty, false),
|
|
(r#""api_error_status":500,"#, 0, empty, false),
|
|
(
|
|
r#""api_error_status":null,"#,
|
|
0,
|
|
r#","model_usage_since_last_result":"?""#,
|
|
false,
|
|
),
|
|
];
|
|
for (status, api_ms, extra, free) in cases {
|
|
let line = not_logged_in_line(status, api_ms, extra);
|
|
let result = rig(vec![Script::lines(vec![
|
|
init_line(MODEL, SESSION),
|
|
line.clone(),
|
|
])])
|
|
.run(request(vec![user("hi")]));
|
|
let (_, _, usage) = failure_of(&result);
|
|
if free {
|
|
assert_eq!(usage, Some(usage_zero()), "{line}");
|
|
} else {
|
|
assert_eq!(usage, None, "{line}");
|
|
}
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn a_no_call_result_that_reports_rows_is_billed_those_rows() {
|
|
let line = not_logged_in_line(
|
|
"",
|
|
0,
|
|
&format!(r#","model_usage_since_last_result":{USAGE}"#),
|
|
);
|
|
let result = rig(vec![Script::lines(vec![init_line(MODEL, SESSION), line])])
|
|
.run(request(vec![user("hi")]));
|
|
let (_, _, billed) = failure_of(&result);
|
|
assert_eq!(billed, Some(usage(2, 7, 1043, 4, Some(3))));
|
|
}
|
|
|
|
// ---- the structured output is an object ----
|
|
|
|
#[test]
|
|
fn a_structured_output_that_is_not_an_object_fails_as_provider_with_usage() {
|
|
for output in ["\"text\"", "[1]", "42", "true", "null"] {
|
|
let result = rig(vec![Script::lines(vec![
|
|
init_line(MODEL, SESSION),
|
|
structured_call_line("StructuredOutput"),
|
|
result_line_with(
|
|
"tool_use",
|
|
USAGE,
|
|
&format!(r#","structured_output":{output}"#),
|
|
),
|
|
])])
|
|
.run(schema_request());
|
|
let (kind, message, usage) = failure_of(&result);
|
|
assert_eq!(kind, FailureKind::Provider, "{output}");
|
|
assert!(message.contains("structured output"), "{message}");
|
|
assert!(usage.is_some(), "{output}: the turn ran and is billed");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn an_object_answer_to_a_schema_whose_top_type_is_not_object_fails() {
|
|
for (schema, accepted) in [
|
|
(r#"{"type":"array"}"#, false),
|
|
(r#"{"type":"string"}"#, false),
|
|
(r#"{"type":["object","null"]}"#, false),
|
|
(r#"{"type":"object"}"#, true),
|
|
(r#"{"properties":{}}"#, true),
|
|
] {
|
|
let mut asked = request(vec![user("hello")]);
|
|
asked.output_schema_json = Some(schema.to_string());
|
|
let result = rig(vec![structured_script()]).run(asked);
|
|
assert_eq!(
|
|
matches!(outcome(&result), TransportOutcome::Completed { .. }),
|
|
accepted,
|
|
"{schema}"
|
|
);
|
|
}
|
|
}
|
|
|
|
// ---- a schema turn does not hand out a session to resume ----
|
|
|
|
#[test]
|
|
fn a_turn_after_a_schema_turn_is_seeded_not_resumed() {
|
|
let rig = rig(vec![structured_script(), ok_script(SESSION, "next")]);
|
|
let first = rig.run(schema_request());
|
|
assert!(
|
|
!parts(&first)
|
|
.0
|
|
.iter()
|
|
.any(|e| matches!(e.payload, TransportEventPayload::ReasoningState { .. })),
|
|
"a schema turn leaves no handle"
|
|
);
|
|
rig.run(request(vec![
|
|
user("hello"),
|
|
InputItem::AssistantText {
|
|
text: STRUCTURED.to_string(),
|
|
},
|
|
user("and now plainly"),
|
|
]));
|
|
let argv = rig.seen().argv;
|
|
assert!(!argv.iter().any(|a| a == "--resume"), "{argv:?}");
|
|
assert!(!argv.iter().any(|a| a == "--json-schema"));
|
|
}
|
|
|
|
// ---- the meter's call ----
|
|
|
|
/// One real call of the pinned CLI (2.1.312), captured on Yggdrasil on
|
|
/// 2026-11-03 with the connector's own argv: haiku, no tools, the trivial
|
|
/// prompt. Its `rate_limit_event` names only the week.
|
|
const METER_CALL: &str = include_str!("../../tests/fixtures/claude-2.1.312-meter-call.jsonl");
|
|
|
|
#[test]
|
|
fn the_meter_call_reads_the_week_from_a_real_captured_event() {
|
|
let rig = rig(vec![Script::lines(
|
|
METER_CALL.lines().map(str::to_string).collect(),
|
|
)]);
|
|
let usage = rig
|
|
.backend
|
|
.read_meter()
|
|
.expect("the captured call completes");
|
|
assert_eq!((usage.input_tokens, usage.output_tokens), (384, 4));
|
|
// 0.06 of the week used; it resets at 1794261600 (unix s).
|
|
assert_eq!(
|
|
readings(&rig),
|
|
vec![window("seven_day", 60, 1_794_261_600_000)]
|
|
);
|
|
let argv = rig.seen().argv;
|
|
let after = |flag: &str| argv[argv.iter().position(|a| a == flag).unwrap() + 1].clone();
|
|
assert_eq!(after("--model"), METER_MODEL);
|
|
assert_eq!(after("--tools"), "", "the call can use no tool");
|
|
assert!(
|
|
argv.iter().any(|a| a == "--no-session-persistence"),
|
|
"the meter's transcript is nobody's to keep: {argv:?}"
|
|
);
|
|
assert_eq!(rig.spawns(), 1, "one child, never run again");
|
|
}
|
|
|
|
#[test]
|
|
fn a_failed_meter_call_names_its_kind_and_leaves_the_cell_alone() {
|
|
let rig = rig(vec![Script::lines(vec![
|
|
init_line(MODEL, SESSION),
|
|
error_result_line(r#","api_error_status":500"#),
|
|
])]);
|
|
let failed = rig.backend.read_meter().expect_err("the call fails");
|
|
assert!(!failed.is_empty() && !failed.contains(' '), "{failed}");
|
|
assert!(quota_of(&rig).windows.is_empty());
|
|
}
|