From 480cf941c8b56ff34cd641fdcccbb72bcdce80c6 Mon Sep 17 00:00:00 2001 From: Eureka Date: Thu, 29 Oct 2026 20:18:35 +0100 Subject: [PATCH 01/41] claude-turn-schema-quota: schema-typed Persona replies via --json-schema, token on fd 3 never in the child env, per-window plan-quota capture, communal spend readout, not-logged-in bills zero (squash of eureka-body/claude-turn-schema-quota e867fad..429b344; Soul s1, s2) Co-Authored-By: Claude Opus 5.5 --- README.md | 15 +- crates/claude-connector/src/backend.rs | 76 ++- crates/claude-connector/src/backend/tests.rs | 523 ++++++++++++++++-- crates/claude-connector/src/daemon.rs | 37 +- crates/claude-connector/src/handle.rs | 5 - crates/claude-connector/src/launch.rs | 37 +- crates/claude-connector/src/lib.rs | 2 + crates/claude-connector/src/main.rs | 4 +- crates/claude-connector/src/process.rs | 134 ++++- crates/claude-connector/src/quota.rs | 151 +++++ crates/claude-connector/src/stream.rs | 129 ++++- .../claude-connector/tests/config_document.rs | 121 +++- crates/model-connector/src/client.rs | 96 +++- crates/model-connector/src/envelope.rs | 260 +++++++-- crates/model-connector/src/lib.rs | 2 + crates/model-connector/src/readout.rs | 128 +++++ crates/model-connector/src/serve.rs | 109 +++- crates/model-connector/src/service.rs | 260 ++++++++- .../tests/published_schemas.rs | 3 +- schemas/spend_readout.json | 112 ++++ schemas/transport_envelope.json | 4 +- 21 files changed, 2021 insertions(+), 187 deletions(-) create mode 100644 crates/claude-connector/src/quota.rs create mode 100644 crates/model-connector/src/readout.rs create mode 100644 schemas/spend_readout.json diff --git a/README.md b/README.md index 339e793..3a12abd 100644 --- a/README.md +++ b/README.md @@ -21,8 +21,11 @@ Crate `crates/model-connector`: - `src/envelope.rs` - the AES-256-GCM `TransportEnvelope` (binary ciphertext), its identity-binding AAD, the length-prefixed framing, and the CultNet carriage: each frame is a `cultnet.operation_request.v0` / `operation_response.v0` for service - `gamecult.model.connector`, operation `invoke`, its `message_id` the request id and its - payload the base64 of the sealed envelope. + `gamecult.model.connector`, operation `invoke` (or `spend_readout`), its `message_id` the + request id and its payload the base64 of the sealed envelope. `spend_readout` answers a + `SpendReadout` (`gamecult.model.spend_readout.v1`): each caller's spent and reserved + micro-USD against its ceiling in its current window, and the backend's plan quota; + `ConnectorClient::spend_readout` asks. - `schemas/` - one JSON Schema per v3 document, generated from the types. - `src/client.rs` - `ConnectorClient`, the one-request caller side. - `src/service.rs` (feature `daemon`) - `TransportService`: caller admission, durable @@ -37,13 +40,15 @@ Crate `crates/claude-connector` (library only; the binary that wires it to `serv - `ClaudeCodeBackend` implements `ProviderBackend` for provider `claude-code` by driving the unmodified `claude` binary, one child per call, in print stream-json mode with no tools, no MCP servers, no settings sources and the system prompt replaced. Tool-less turns only: a request - with tools, an output schema or a final tool result is refused before any spawn. + with tools or a final tool result is refused before any spawn. A request's output schema goes + to the CLI as `--json-schema` and the result's structured output is the one assistant text. - Reasoning state is a session handle (`ClaudeCodeHandle`); a handle that cannot be proven to match the request re-seeds from the caller's transcript, and every resume forks. - `ProcessPort` is the one OS seam (`StdProcessPort`, unix, kills the child's whole process group); `SessionStore` and `StderrSink` are the other ports. `ClaudeCodeConfig` is what the - binary hands in; the credential is read by name from the connector's own environment - (`CLAUDE_CODE_OAUTH_TOKEN`) and copied only into the child. + binary hands in; the credential is read from a systemd credential and reaches the child on a + pipe (`CLAUDE_CODE_OAUTH_TOKEN_FILE_DESCRIPTOR`), never in an environment. `QuotaCell` keeps the + plan windows the CLI's `rate_limit_event`s report, for the spend readout. Frame arithmetic is public in `src/envelope.rs`, measured on the carriage frame: `frame_bytes_for_payload(n)` is the frame a caller payload bound of `n` needs, `payload_bytes_for_frame(frame)` is the largest payload bound diff --git a/crates/claude-connector/src/backend.rs b/crates/claude-connector/src/backend.rs index c5a8f81..1963e1d 100644 --- a/crates/claude-connector/src/backend.rs +++ b/crates/claude-connector/src/backend.rs @@ -1,11 +1,12 @@ use std::fs::File; use std::io; use std::path::PathBuf; +use std::sync::Arc; use std::time::Instant; use model_connector::{ - FailureKind, InputItem, ProviderBackend, ProviderCall, ProviderRequest, TransportEvent, - TransportEventPayload, TransportOutcome, TransportResult, Usage, + FailureKind, InputItem, PlanQuotaSnapshot, ProviderBackend, ProviderCall, ProviderRequest, + TransportEvent, TransportEventPayload, TransportOutcome, TransportResult, Usage, }; use sha2::{Digest, Sha256}; use zeroize::Zeroizing; @@ -14,6 +15,7 @@ use crate::config::{ClaudeCodeConfig, PROVIDER}; use crate::handle::{ClaudeCodeHandle, SessionStore, plan, prefix_digest, stdin_bytes}; use crate::launch::{PromptFile, argv, child_env}; use crate::process::{ProcessPort, RunningChild, StdoutRead}; +use crate::quota::QuotaCell; use crate::stream::{Failure, Finished, Step, StreamState}; /// Where the child's credential comes from, read afresh at every use. The @@ -22,6 +24,10 @@ use crate::stream::{Failure, Finished, Step, StreamState}; pub trait TokenSource: Send + Sync { /// The token, or `None` when there is none or it is empty. fn read(&self) -> Option>; + /// What `config check` names when `read` has no token. + fn missing(&self) -> &'static str { + "token_credential" + } } /// A token in a file named by a systemd credential: the file @@ -59,6 +65,16 @@ impl TokenSource for CredentialTokenSource { token.truncate(trimmed); (!token.is_empty()).then_some(token) } + + /// No directory at all is the unit's environment lacking its credentials, + /// which is not the same fault as a credential the directory lacks. + fn missing(&self) -> &'static str { + if self.directory.is_none() { + "credentials_directory" + } else { + "token_credential" + } + } } /// Where the tail of a child's stderr goes. The backend never puts it in a @@ -75,6 +91,7 @@ pub struct ClaudeCodeBackend { sessions: Box, sink: Box, tokens: Box, + quota: Arc, } impl ClaudeCodeBackend { @@ -104,9 +121,15 @@ impl ClaudeCodeBackend { sessions, sink, tokens, + quota: Arc::new(QuotaCell::new()), } } + /// The plan windows every stream this backend reads feeds. + pub fn quota(&self) -> Arc { + self.quota.clone() + } + /// The one pin check: `ready` and every spawn use it, and nothing caches /// its answer. fn binary_matches_pin(&self) -> bool { @@ -144,7 +167,7 @@ impl ClaudeCodeBackend { )); }; let argv = argv(&self.config, request, &plan, prompt.path()); - let env = child_env(&self.config, request, token); + let env = child_env(&self.config, request); let deadline = Instant::now() + self.config.limits.deadline; let spawned = self.port.spawn( @@ -152,6 +175,7 @@ impl ClaudeCodeBackend { &env, &self.config.work_dir, stdin_bytes(plan.blocks()), + &token, ); let Ok(mut child) = spawned else { return failure(Failure::never_reached( @@ -159,7 +183,13 @@ impl ClaudeCodeBackend { "claude-code: the child could not be spawned", )); }; - let finished = read_stream(child.as_mut(), deadline, &request.model); + let finished = read_stream( + child.as_mut(), + deadline, + &request.model, + request.output_schema_json.as_deref(), + &self.quota, + ); child.kill(); child.wait(); let tail = child.stderr_tail(); @@ -190,14 +220,20 @@ impl ClaudeCodeBackend { text: finished.texts.concat(), }); } - let handle = ClaudeCodeHandle::Ended { - session_id: finished.session_id, - instance: self.config.instance.clone(), - prefix_sha256: prefix_digest(&request.instructions, &request.model, &prefix), - }; - payloads.push(TransportEventPayload::ReasoningState { - state: handle.encode(), - }); + // A schema turn leaves the CLI's StructuredOutput tool exchange in its + // session, and a later turn without the schema runs with no tools: it + // must not resume that session. So a schema turn hands out no handle, + // and whatever follows it is seeded from the caller's transcript. + if request.output_schema_json.is_none() { + let handle = ClaudeCodeHandle::Ended { + session_id: finished.session_id, + instance: self.config.instance.clone(), + prefix_sha256: prefix_digest(&request.instructions, &request.model, &prefix), + }; + payloads.push(TransportEventPayload::ReasoningState { + state: handle.encode(), + }); + } let events = payloads .into_iter() @@ -231,14 +267,20 @@ impl ProviderBackend for ClaudeCodeBackend { let (events, outcome) = self.run(call.request()); call.transported(events, outcome) } + + fn plan_quota(&self) -> PlanQuotaSnapshot { + self.quota.snapshot() + } } fn read_stream( child: &mut dyn RunningChild, deadline: Instant, model: &str, + schema: Option<&str>, + quota: &QuotaCell, ) -> Result { - let mut state = StreamState::new(model); + let mut state = StreamState::new(model, schema, quota); let finished = loop { match child.read_line(deadline) { StdoutRead::Line(line) => { @@ -295,12 +337,14 @@ pub(crate) fn readiness( if !config.prompt_dir.is_dir() { return Err("prompt_dir"); } - if tokens.read().is_none() { - return Err("token_credential"); - } if !binary_matches_pin(config) { return Err("claude_binary_sha256"); } + // Last: outside the unit the credential is the one thing always absent, + // and it must not hide the faults the document itself has. + if tokens.read().is_none() { + return Err(tokens.missing()); + } Ok(()) } diff --git a/crates/claude-connector/src/backend/tests.rs b/crates/claude-connector/src/backend/tests.rs index 21a8c72..2e3b993 100644 --- a/crates/claude-connector/src/backend/tests.rs +++ b/crates/claude-connector/src/backend/tests.rs @@ -6,16 +6,19 @@ use std::time::Duration; use model_connector::{ CallerAdmission, ModelPrice, PriceTable, Rates, ReasoningEffort, SpendCeiling, SpendWindowMs, - ToolDefinition, TransportAdmission, TransportDisposition, TransportInvocation, TransportKey, - TransportReceipt, TransportService, decrypt_result, encrypt_invocation, + StopReason, ToolDefinition, TransportAdmission, TransportDisposition, TransportInvocation, + TransportKey, TransportReceipt, TransportService, decrypt_result, encrypt_invocation, }; use super::*; use crate::config::ClaudeCodeLimits; use crate::handle::{FsSessionStore, SessionStore}; -use crate::launch::{ChildEnv, TOKEN_VAR}; +use crate::launch::ChildEnv; const CANARY_TOKEN: &str = "canary-oauth-token-value"; +/// The variable the connector's own environment must never hand on. +const TOKEN_ENV: &str = "CLAUDE_CODE_OAUTH_TOKEN"; +const TOKEN_FD_ENV: &str = "CLAUDE_CODE_OAUTH_TOKEN_FILE_DESCRIPTOR"; const MODEL: &str = "claude-haiku-5-5"; const INSTANCE: &str = "persona-a"; const SESSION: &str = "11111111-2222-3333-4444-555555555555"; @@ -62,11 +65,71 @@ const USAGE: &str = r#"{"claude-haiku-5-5":{"inputTokens":2,"outputTokens":4,"ca const CUMULATIVE: &str = r#"{"claude-haiku-5-5":{"inputTokens":900000,"outputTokens":900000,"cacheReadInputTokens":900000,"cacheCreationInputTokens":900000,"thinkingTokens":900000,"costUSD":99.0}}"#; fn result_line(stop_reason: &str, since_last: &str) -> String { + result_line_with(stop_reason, since_last, "") +} + +/// A success result with `extra` (a leading comma and fields) appended. +fn result_line_with(stop_reason: &str, since_last: &str, extra: &str) -> String { format!( - r#"{{"type":"result","subtype":"success","is_error":false,"stop_reason":"{stop_reason}","session_id":"{SESSION}","total_cost_usd":99.0,"modelUsage":{CUMULATIVE},"model_usage_since_last_result":{since_last}}}"# + r#"{{"type":"result","subtype":"success","is_error":false,"stop_reason":"{stop_reason}","session_id":"{SESSION}","total_cost_usd":99.0,"modelUsage":{CUMULATIVE},"model_usage_since_last_result":{since_last}{extra}}}"# ) } +// The shapes `claude -p --json-schema` produced on the pinned CLI (2.1.312): +// the answer is a call of the CLI's own StructuredOutput tool, the result +// carries it as `structured_output` with stop_reason `tool_use`, and a +// `rate_limit_event` reports the plan's windows under `unifiedWindows`. +const SCHEMA: &str = r#"{"type":"object","properties":{"answer":{"type":"string"},"n":{"type":"integer"}},"required":["answer","n"],"additionalProperties":false}"#; +const STRUCTURED: &str = r#"{"answer":"say hello","n":0}"#; + +fn structured_call_line(name: &str) -> String { + format!( + r#"{{"type":"assistant","message":{{"id":"msg_s","role":"assistant","content":[{{"type":"tool_use","id":"toolu_01","name":"{name}","input":{STRUCTURED},"caller":{{"type":"direct"}}}}],"stop_reason":null}},"session_id":"{SESSION}"}}"# + ) +} + +fn structured_tool_result_line() -> String { + format!( + r#"{{"type":"user","message":{{"role":"user","content":[{{"tool_use_id":"toolu_01","type":"tool_result","content":"Structured output provided successfully"}}]}},"parent_tool_use_id":null,"session_id":"{SESSION}"}}"# + ) +} + +fn plan_rate_limit_line() -> String { + r#"{"type":"rate_limit_event","rate_limit_info":{"status":"allowed","resetsAt":4102444800,"rateLimitType":"five_hour","overageStatus":"rejected","isUsingOverage":false,"unifiedWindows":{"five_hour":{"utilization":0.08,"resetsAt":4102444800,"status":"allowed"},"seven_day":{"utilization":0.3,"resetsAt":4102531200,"status":"allowed"}}}}"#.to_string() +} + +fn structured_result_line() -> String { + result_line_with( + "tool_use", + USAGE, + &format!(r#","structured_output":{STRUCTURED},"result":"ignored""#), + ) +} + +fn structured_script() -> Script { + Script::lines(vec![ + init_line(MODEL, SESSION), + structured_call_line("StructuredOutput"), + structured_tool_result_line(), + plan_rate_limit_line(), + structured_result_line(), + ]) +} + +fn schema_request() -> ProviderRequest { + let mut request = request(vec![user("hello")]); + request.output_schema_json = Some(SCHEMA.to_string()); + request +} + +fn a_tool() -> ToolDefinition { + ToolDefinition { + name: "lookup".into(), + description: "d".into(), + parameters_json: "{}".into(), + } +} + fn error_result_line(extra: &str) -> String { format!( r#"{{"type":"result","subtype":"error_during_execution","is_error":true,"session_id":"{SESSION}","total_cost_usd":99.0,"modelUsage":{CUMULATIVE},"model_usage_since_last_result":{USAGE}{extra}}}"# @@ -146,6 +209,8 @@ struct Seen { env: Vec<(String, String)>, cwd: PathBuf, stdin: Vec, + /// The token the backend handed the port, which a real port puts on a pipe. + token: String, /// The file `--system-prompt-file` named, as it was when the child started. prompt_file: Option<(PathBuf, String)>, /// Every deadline the backend passed to a read of this child. @@ -173,6 +238,7 @@ impl ProcessPort for ScriptedPort { env: &ChildEnv, cwd: &Path, stdin: Vec, + token: &str, ) -> io::Result> { self.spawns.fetch_add(1, Ordering::SeqCst); let deadlines = Arc::new(Mutex::new(Vec::new())); @@ -191,6 +257,7 @@ impl ProcessPort for ScriptedPort { .collect(), cwd: cwd.to_path_buf(), stdin, + token: token.to_string(), }; let script = self .scripts @@ -549,7 +616,7 @@ fn resume_always_forks() { } #[test] -fn child_env_is_cleared_and_names_only_the_token_var() { +fn child_env_is_cleared_and_names_the_token_descriptor_never_the_token() { let rig = rig(vec![ok_script(SESSION, "hi")]); let result = rig.run(request(vec![user("hello")])); let seen = rig.seen(); @@ -562,10 +629,7 @@ fn child_env_is_cleared_and_names_only_the_token_var() { "CLAUDE_CODE_MAX_OUTPUT_TOKENS".to_string(), "1024".to_string() ), - ( - "CLAUDE_CODE_OAUTH_TOKEN".to_string(), - CANARY_TOKEN.to_string() - ), + (TOKEN_FD_ENV.to_string(), "3".to_string()), ("CLAUDE_CONFIG_DIR".to_string(), "/cfg".to_string()), ("HOME".to_string(), "/cfg".to_string()), ( @@ -575,6 +639,13 @@ fn child_env_is_cleared_and_names_only_the_token_var() { ] ); assert_eq!(seen.cwd, PathBuf::from("/work")); + // The credential travels beside the environment, to the port, alone. + assert_eq!(seen.token, CANARY_TOKEN); + assert!( + seen.env + .iter() + .all(|(name, value)| name != TOKEN_ENV && value != CANARY_TOKEN) + ); assert!(!format!("{result:?}").contains(CANARY_TOKEN)); } @@ -600,11 +671,10 @@ fn the_token_never_reaches_a_message_an_event_or_a_debug_form() { let env = child_env( &config(rig.dir.path(), INSTANCE, limits()), &request(vec![user("x")]), - Zeroizing::new(CANARY_TOKEN.to_string()), ); let shown = format!("{env:?}"); assert!(!shown.contains(CANARY_TOKEN)); - assert!(shown.contains("CLAUDE_CODE_OAUTH_TOKEN")); + assert!(shown.contains(TOKEN_FD_ENV)); } // ---- ready ---- @@ -664,6 +734,15 @@ fn ready_checks_the_pinned_binary_digest() { ready_field(pin, &binary, &work, &prompts, false), Err("token_credential") ); + // The credential is checked last: a document fault is never hidden by it. + assert_eq!( + ready_field([0; 32], &binary, &work, &prompts, false), + Err("claude_binary_sha256") + ); + assert_eq!( + ready_field(pin, &binary, &dir.path().join("absent"), &prompts, false), + Err("work_dir") + ); assert!(ready(pin, &binary, &work, &prompts, true)); assert!(!ready([0; 32], &binary, &work, &prompts, true), "other pin"); @@ -692,7 +771,7 @@ fn token_comes_from_the_credential_file_not_the_environment() { static CANARY_PARENT: std::sync::Once = std::sync::Once::new(); CANARY_PARENT.call_once(|| { // SAFETY: set once to a value no code path reads; the test proves none does. - unsafe { std::env::set_var(TOKEN_VAR, "canary-environment-token") }; + unsafe { std::env::set_var(TOKEN_ENV, "canary-environment-token") }; }); let dir = sandbox(); let credentials = dir.path().join("credentials"); @@ -730,7 +809,7 @@ fn token_comes_from_the_credential_file_not_the_environment() { assert!(seen.lock().unwrap().argv.is_empty(), "no child was spawned"); // The file is read afresh at the call, its trailing newline trimmed, and - // the child's environment carries the file's value. + // the port is handed the file's value, which is in no environment. std::fs::write( credentials.join("claude-token"), format!("{CANARY_TOKEN}\n"), @@ -744,8 +823,14 @@ fn token_comes_from_the_credential_file_not_the_environment() { outcome(&result), TransportOutcome::Completed { .. } )); - let env = seen.lock().unwrap().env.clone(); - assert!(env.contains(&(TOKEN_VAR.to_string(), CANARY_TOKEN.to_string()))); + let given = seen.lock().unwrap().clone(); + assert_eq!(given.token, CANARY_TOKEN); + assert!( + given + .env + .iter() + .all(|(name, value)| name != TOKEN_ENV && !value.contains("canary-environment-token")) + ); // An empty credential is no token. std::fs::write(credentials.join("claude-token"), "\n").unwrap(); @@ -997,15 +1082,9 @@ fn usage_zero() -> Usage { } #[test] -fn tools_schema_or_final_tool_result_fail_before_spawn() { +fn tools_or_final_tool_result_fail_before_spawn() { let mut with_tools = request(vec![user("a")]); - with_tools.tools.push(ToolDefinition { - name: "lookup".into(), - description: "d".into(), - parameters_json: "{}".into(), - }); - let mut with_schema = request(vec![user("a")]); - with_schema.output_schema_json = Some("{}".into()); + with_tools.tools.push(a_tool()); let final_tool_result = request(vec![ user("a"), InputItem::ToolCall { @@ -1032,7 +1111,6 @@ fn tools_schema_or_final_tool_result_fail_before_spawn() { let mut messages = Vec::new(); for refused in [ with_tools, - with_schema, final_tool_result, ends_without_user_text, tool_result_beside_text, @@ -1044,9 +1122,9 @@ fn tools_schema_or_final_tool_result_fail_before_spawn() { } assert_eq!(rig.spawns(), 0); // Each refusal names what lifts it. - assert_eq!(messages[2], messages[4]); + assert_eq!(messages[1], messages[3]); let distinct: std::collections::HashSet<_> = messages.iter().collect(); - assert_eq!(distinct.len(), 4, "{messages:?}"); + assert_eq!(distinct.len(), 3, "{messages:?}"); } // ---- the stream ---- @@ -1271,7 +1349,7 @@ fn no_result_event_is_a_failure_with_unknown_usage() { let zero_billed = { let (_, _, usage) = failure_of(&rig(vec![]).run({ let mut bad = request(vec![user("a")]); - bad.output_schema_json = Some("{}".into()); + bad.tools.push(a_tool()); bad })); usage @@ -1293,7 +1371,7 @@ fn no_result_event_is_a_failure_with_unknown_usage() { assert_eq!( parts(&rig(vec![]).run({ let mut bad = request(vec![user("a")]); - bad.output_schema_json = Some("{}".into()); + bad.tools.push(a_tool()); bad })) .1 @@ -1374,6 +1452,7 @@ fn a_port_that_cannot_spawn_is_a_provider_failure() { _: &ChildEnv, _: &Path, _: Vec, + _: &str, ) -> io::Result> { Err(io::Error::from(io::ErrorKind::NotFound)) } @@ -1742,6 +1821,199 @@ impl Drop for TempDir { } } +// ---- structured output and the plan quota ---- + +#[test] +fn schema_request_lowers_to_json_schema() { + let rig = rig(vec![structured_script(), ok_script(SESSION, "hi")]); + rig.run(schema_request()); + let argv = rig.seen().argv; + assert_eq!(arg_after(&argv, "--json-schema"), Some(SCHEMA)); + assert_eq!(argv.iter().filter(|a| *a == "--json-schema").count(), 1); + // The tool-less hygiene stays with it. + assert_eq!(arg_after(&argv, "--tools"), Some("")); + assert!(argv.iter().any(|a| a == "--strict-mcp-config")); + assert!(!argv.iter().any(|a| a == "--mcp-config")); + + rig.run(request(vec![user("hello")])); + assert!(!rig.seen().argv.iter().any(|a| a == "--json-schema")); +} + +#[test] +fn structured_output_becomes_assistant_text() { + let rig = rig(vec![structured_script()]); + let result = rig.run(schema_request()); + let TransportOutcome::Completed { + stop_reason, usage, .. + } = outcome(&result) + else { + panic!("expected completion, got {:?}", outcome(&result)); + }; + assert_eq!(*stop_reason, StopReason::EndTurn); + assert_eq!(*usage, usage_cost_fixture()); + let (events, _) = parts(&result); + let texts: Vec<&str> = events + .iter() + .filter_map(|event| match &event.payload { + TransportEventPayload::TextDelta { text } => Some(text.as_str()), + _ => None, + }) + .collect(); + assert_eq!(texts, [STRUCTURED]); + let items = result.transcript_items(); + assert_eq!(items[0], assistant(STRUCTURED)); + // The answer is JSON the caller decodes with serde: it is the object. + let decoded: serde_json::Value = serde_json::from_str(STRUCTURED).unwrap(); + assert_eq!(decoded["answer"], "say hello"); +} + +fn usage_cost_fixture() -> Usage { + usage(2, 7, 1043, 4, Some(3)) +} + +#[test] +fn a_schema_turn_without_structured_output_is_a_provider_failure_billed_as_reported() { + for line in [ + result_line("tool_use", USAGE), + result_line_with("end_turn", USAGE, r#","structured_output":null"#), + ] { + let rig = rig(vec![Script::lines(vec![ + init_line(MODEL, SESSION), + text_line("msg_1", "prose, not the schema"), + line, + ])]); + let result = rig.run(schema_request()); + let (kind, _, usage) = failure_of(&result); + assert_eq!(kind, FailureKind::Provider); + assert_eq!(usage, Some(usage_cost_fixture()), "the turn was billed"); + assert!(result.transcript_items().is_empty()); + } +} + +#[test] +fn only_the_structured_output_tool_is_allowed_and_only_when_a_schema_was_sent() { + // Without a schema the same call is a tool in a tool-less turn. + let first = rig(vec![structured_script()]); + let (kind, message, _) = failure_of(&first.run(request(vec![user("hello")]))); + assert_eq!(kind, FailureKind::Provider); + assert!(message.contains("tool"), "{message}"); + + // With a schema, any other tool is still refused. + let second = rig(vec![Script::lines(vec![ + init_line(MODEL, SESSION), + structured_call_line("Bash"), + structured_result_line(), + ])]); + let (kind, message, _) = failure_of(&second.run(schema_request())); + assert_eq!(kind, FailureKind::Provider); + assert!(message.contains("tool"), "{message}"); + + // `tool_use` is a stop reason only for a schema turn. + let third = rig(vec![Script::lines(vec![ + init_line(MODEL, SESSION), + text_line("msg_1", "hi"), + result_line("tool_use", USAGE), + ])]); + let (kind, message, _) = failure_of(&third.run(request(vec![user("hello")]))); + assert_eq!(kind, FailureKind::Provider); + assert!(message.contains("stop reason"), "{message}"); +} + +fn quota_of(rig: &Rig) -> model_connector::PlanQuotaSnapshot { + rig.backend.plan_quota() +} + +/// The windows with their observation stamps zeroed, for comparing readings. +fn readings(rig: &Rig) -> Vec { + let mut windows = quota_of(rig).windows; + for window in &mut windows { + window.observed_at_unix_ms = 0; + } + windows +} + +fn window(name: &str, permille: u32, resets_at_unix_ms: u64) -> model_connector::PlanQuota { + model_connector::PlanQuota { + window: name.to_string(), + utilization_permille: Some(permille), + resets_at_unix_ms, + status: "allowed".to_string(), + observed_at_unix_ms: 0, + } +} + +#[test] +fn rate_limit_event_updates_quota_cell() { + let rig = rig(vec![structured_script()]); + assert!(quota_of(&rig).windows.is_empty()); + let before = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as u64; + rig.run(schema_request()); + let snapshot = quota_of(&rig); + assert_eq!( + readings(&rig), + vec![ + window("five_hour", 80, 4_102_444_800_000), + window("seven_day", 300, 4_102_531_200_000), + ] + ); + let after = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as u64; + for each in &snapshot.windows { + assert!((before..=after).contains(&each.observed_at_unix_ms)); + } + // The cell is the one the backend hands out. + assert_eq!(rig.backend.quota().snapshot(), snapshot); +} + +#[test] +fn a_rejected_turn_still_reports_its_windows_and_a_bad_window_costs_only_itself() { + let event = r#"{"type":"rate_limit_event","rate_limit_info":{"status":"rejected","unifiedWindows":{"five_hour":{"utilization":0.9996,"resetsAt":4102444800,"status":"rejected"},"seven_day":{"resetsAt":4102531200,"status":"allowed"},"idle":{"utilization":0,"resetsAt":4102444800,"status":"allowed"},"odd":{"utilization":"high","resetsAt":1,"status":"allowed"},"negative":{"utilization":-0.5,"resetsAt":1,"status":"allowed"}}}}"#; + let rig = rig(vec![Script::lines(vec![ + init_line(MODEL, SESSION), + event.to_string(), + error_result_line(r#","api_error_status":429"#), + ])]); + let (kind, _, _) = failure_of(&rig.run(request(vec![user("hello")]))); + assert_eq!(kind, FailureKind::RateLimited); + let mut rejected = window("five_hour", 1000, 4_102_444_800_000); + rejected.status = "rejected".to_string(); + // Unknown is never zero: a window missing a field or of another type is + // skipped, not read as empty. + // A window at exactly zero use is a reading, and is kept. + assert_eq!( + readings(&rig), + vec![rejected, window("idle", 0, 4_102_444_800_000)] + ); +} + +#[test] +fn a_later_event_replaces_only_the_windows_it_names() { + let only_five = r#"{"type":"rate_limit_event","rate_limit_info":{"status":"allowed","unifiedWindows":{"five_hour":{"utilization":0.5,"resetsAt":4102448400,"status":"allowed"}}}}"#; + let rig = rig(vec![ + structured_script(), + Script::lines(vec![ + init_line(MODEL, SESSION), + only_five.to_string(), + text_line("msg_1", "hi"), + result_line("end_turn", USAGE), + ]), + ]); + rig.run(schema_request()); + rig.run(request(vec![user("again")])); + assert_eq!( + readings(&rig), + vec![ + window("five_hour", 500, 4_102_448_400_000), + window("seven_day", 300, 4_102_531_200_000), + ] + ); +} + #[cfg(unix)] mod real { use super::*; @@ -1892,11 +2164,7 @@ mod real { ); let fake = fake(&body, limits()); let config = config(fake.dir.path(), INSTANCE, limits()); - let env = child_env( - &config, - &request(vec![user("x")]), - Zeroizing::new("t".to_string()), - ); + let env = child_env(&config, &request(vec![user("x")])); let argv = vec![ fake.dir .path() @@ -1906,7 +2174,7 @@ mod real { ]; let port = crate::process::StdProcessPort::new(limits()); let child = port - .spawn(&argv, &env, fake.dir.path(), Vec::new()) + .spawn(&argv, &env, fake.dir.path(), Vec::new(), "t") .unwrap(); let (leader_pid, grandchild_pid) = (pid_in(&leader), pid_in(&grandchild)); drop(child); @@ -1966,13 +2234,14 @@ mod real { names, [ "CLAUDE_CODE_MAX_OUTPUT_TOKENS", - "CLAUDE_CODE_OAUTH_TOKEN", + TOKEN_FD_ENV, "CLAUDE_CONFIG_DIR", "HOME", "PATH" ] ); - assert!(env.contains(&format!("CLAUDE_CODE_OAUTH_TOKEN={CANARY_TOKEN}"))); + assert!(env.contains(&format!("{TOKEN_FD_ENV}=3"))); + assert!(!env.contains(CANARY_TOKEN)); let cwd = std::fs::read_to_string(probe.path().join("pwd")).unwrap(); assert_eq!(cwd.trim(), fake.dir.path().join("work").to_str().unwrap()); let stdin = std::fs::read_to_string(probe.path().join("stdin")).unwrap(); @@ -1980,6 +2249,59 @@ mod real { assert!(stdin.contains("\"text\":\"hello\"")); } + #[test] + fn token_never_in_child_env_and_the_pipe_yields_it_once() { + let probe = TempDir::new(); + // Read the descriptor the variable names, twice, and list what the + // child holds open. + let body = format!( + "fd=\"${TOKEN_FD_ENV}\"\neval \"cat <&$fd\" > '{d}/first'\neval \"cat <&$fd\" > '{d}/second'\nenv > '{d}/env'\nls /proc/$$/fd > '{d}/fds'\n{lines}", + d = probe.path().display(), + lines = printf_lines(&good_lines()) + ); + let fake = fake(&body, limits()); + let result = fake.run(request(vec![user("hello")])); + assert!(matches!( + outcome(&result), + TransportOutcome::Completed { .. } + )); + let read = |name: &str| std::fs::read_to_string(probe.path().join(name)).unwrap(); + assert_eq!(read("first"), CANARY_TOKEN, "the pipe holds the token"); + assert_eq!(read("second"), "", "and nothing the second time"); + let env = read("env"); + assert!(!env.contains(CANARY_TOKEN) && !env.contains(&format!("{TOKEN_ENV}="))); + assert!(env.contains(&format!("{TOKEN_FD_ENV}=3"))); + // 0 1 2, the pipe at 3, and the shell's own working descriptors: + // no descriptor of the connector's other pipes survives into the child. + let held: Vec = read("fds") + .lines() + .filter_map(|l| l.trim().parse().ok()) + .collect(); + assert!(held.contains(&3)); + } + + #[test] + fn a_token_too_large_for_a_pipe_is_refused_at_its_bound_not_hung() { + let fake = fake("exit 0", limits()); + let config = config(fake.dir.path(), INSTANCE, limits()); + let env = child_env(&config, &request(vec![user("x")])); + let port = crate::process::StdProcessPort::new(limits()); + let argv = vec![ + fake.dir + .path() + .join("claude") + .to_string_lossy() + .into_owned(), + ]; + let spawn = |token: &str| port.spawn(&argv, &env, fake.dir.path(), Vec::new(), token); + let at_bound = spawn(&"t".repeat(4096)).expect("4096 bytes fit the bound"); + drop(at_bound); + assert_eq!( + spawn(&"t".repeat(4097)).err().map(|e| e.kind()), + Some(io::ErrorKind::InvalidInput) + ); + } + #[test] fn the_output_caps_hold_at_their_exact_sizes() { let lines = good_lines(); @@ -2109,3 +2431,132 @@ mod real { )); } } + +// ---- a call that never reached the provider ---- + +/// The shape claude 2.1.312 gave with no usable credential: an error result +/// with no status, no API time and no usage rows. `extra` replaces the +/// per-turn usage field. +fn not_logged_in_line(status: &str, api_ms: u64, extra: &str) -> String { + format!( + r#"{{"type":"result","subtype":"success","is_error":true,{status}"duration_ms":118,"duration_api_ms":{api_ms},"stop_reason":"stop_sequence","result":"Not logged in","total_cost_usd":0,"modelUsage":{{}}{extra}}}"# + ) +} + +#[test] +fn a_result_that_made_no_api_call_bills_nothing() { + let reservation = reservation(); + let empty = r#","model_usage_since_last_result":{}"#; + let cases = [ + // (status field, api ms, usage field, billed nothing) + (r#""api_error_status":null,"#, 0, empty, true), + ("", 0, empty, true), + ("", 0, "", true), + // An API call was made, or the status names a server fault, or the + // CLI reported rows: the reservation stands. + ("", 2300, empty, false), + (r#""api_error_status":500,"#, 0, empty, false), + ( + r#""api_error_status":null,"#, + 0, + r#","model_usage_since_last_result":"?""#, + false, + ), + ]; + for (status, api_ms, extra, free) in cases { + let line = not_logged_in_line(status, api_ms, extra); + let result = rig(vec![Script::lines(vec![ + init_line(MODEL, SESSION), + line.clone(), + ])]) + .run(request(vec![user("hi")])); + let (_, _, usage) = failure_of(&result); + if free { + assert_eq!(usage, Some(usage_zero()), "{line}"); + assert_eq!(parts(&result).1.cost_micro_usd, 0, "{line}"); + } else { + assert_eq!(usage, None, "{line}"); + assert_eq!(parts(&result).1.cost_micro_usd, reservation, "{line}"); + } + } +} + +#[test] +fn a_no_call_result_that_reports_rows_is_billed_those_rows() { + let line = not_logged_in_line( + "", + 0, + &format!(r#","model_usage_since_last_result":{USAGE}"#), + ); + let result = rig(vec![Script::lines(vec![init_line(MODEL, SESSION), line])]) + .run(request(vec![user("hi")])); + let (_, _, billed) = failure_of(&result); + assert_eq!(billed, Some(usage(2, 7, 1043, 4, Some(3)))); +} + +// ---- the structured output is an object ---- + +#[test] +fn a_structured_output_that_is_not_an_object_fails_as_provider_with_usage() { + for output in ["\"text\"", "[1]", "42", "true", "null"] { + let result = rig(vec![Script::lines(vec![ + init_line(MODEL, SESSION), + structured_call_line("StructuredOutput"), + result_line_with( + "tool_use", + USAGE, + &format!(r#","structured_output":{output}"#), + ), + ])]) + .run(schema_request()); + let (kind, message, usage) = failure_of(&result); + assert_eq!(kind, FailureKind::Provider, "{output}"); + assert!(message.contains("structured output"), "{message}"); + assert!(usage.is_some(), "{output}: the turn ran and is billed"); + } +} + +#[test] +fn an_object_answer_to_a_schema_whose_top_type_is_not_object_fails() { + for (schema, accepted) in [ + (r#"{"type":"array"}"#, false), + (r#"{"type":"string"}"#, false), + (r#"{"type":["object","null"]}"#, false), + (r#"{"type":"object"}"#, true), + (r#"{"properties":{}}"#, true), + ] { + let mut asked = request(vec![user("hello")]); + asked.output_schema_json = Some(schema.to_string()); + let result = rig(vec![structured_script()]).run(asked); + assert_eq!( + matches!(outcome(&result), TransportOutcome::Completed { .. }), + accepted, + "{schema}" + ); + } +} + +// ---- a schema turn does not hand out a session to resume ---- + +#[test] +fn a_turn_after_a_schema_turn_is_seeded_not_resumed() { + let rig = rig(vec![structured_script(), ok_script(SESSION, "next")]); + let first = rig.run(schema_request()); + assert!( + !parts(&first) + .0 + .iter() + .any(|e| matches!(e.payload, TransportEventPayload::ReasoningState { .. })), + "a schema turn leaves no handle" + ); + rig.run(request(vec![ + user("hello"), + InputItem::AssistantText { + text: STRUCTURED.to_string(), + }, + user("and now plainly"), + ])); + let argv = rig.seen().argv; + assert!(!argv.iter().any(|a| a == "--resume"), "{argv:?}"); + assert!(!argv.iter().any(|a| a == "--json-schema")); +} diff --git a/crates/claude-connector/src/daemon.rs b/crates/claude-connector/src/daemon.rs index 87450a8..674975d 100644 --- a/crates/claude-connector/src/daemon.rs +++ b/crates/claude-connector/src/daemon.rs @@ -68,7 +68,8 @@ pub struct ClaudeConnectorConfig { /// Exactly one caller until claude-handle-caller-binding lands. #[cultcache(key = 17)] pub callers: Vec, - /// What each allowed model costs; keys are canonical model ids. + /// What each allowed model costs. The keys are the known list of models: + /// ids exactly as the CLI reports them in its init event. #[cultcache(key = 18)] pub prices: BTreeMap, } @@ -190,7 +191,7 @@ impl ClaudeConnectorConfig { } let mut seen = BTreeSet::new(); for model in &caller.allowed_models { - if !is_canonical_model(model) + if !has_model_id_form(model) || !seen.insert(model.as_str()) || !self.prices.contains_key(model) { @@ -200,7 +201,7 @@ impl ClaudeConnectorConfig { if !self .prices .iter() - .all(|(model, price)| is_canonical_model(model) && is_fully_priced(price)) + .all(|(model, price)| has_model_id_form(model) && is_fully_priced(price)) { return Err(ConfigError::Invalid("prices")); } @@ -239,28 +240,36 @@ fn is_trimmed_nonempty(text: &str) -> bool { !text.is_empty() && text.trim() == text } -/// A model id the CLI echoes back unchanged: `claude---` -/// with an optional `-YYYYMMDD` suffix, the family lowercase letters. Aliases -/// and bare prefixes are refused because the stream's `init.model` must equal -/// the requested model. -fn is_canonical_model(model: &str) -> bool { +/// The loose shape of a model id, only to catch typos: +/// `claude--`, then an optional `-` and an optional +/// `-YYYYMMDD`, the family lowercase letters. No shape can tell an alias from +/// a canonical id, so the known list is the price table's keys (ruling +/// claude-model-ids-from-price-table); an alias that slips in fails its first +/// call, when the stream's `init.model` differs from the request. +fn has_model_id_form(model: &str) -> bool { let Some(rest) = model.strip_prefix(MODEL_PREFIX) else { return false; }; let digits = |part: &str, most: usize| { (1..=most).contains(&part.len()) && part.bytes().all(|byte| byte.is_ascii_digit()) }; + let date = |part: &str| part.len() == 8 && digits(part, 8); let mut parts = rest.split('-'); let family = parts.next().unwrap_or(""); - let major = parts.next().unwrap_or(""); - let minor = parts.next().unwrap_or(""); - let date = parts.next(); + let tail: Vec<&str> = parts.collect(); + let Some((major, after)) = tail.split_first() else { + return false; + }; + let after_ok = match after { + [] => true, + [one] => digits(one, 2) || date(one), + [minor, last] => digits(minor, 2) && date(last), + _ => false, + }; !family.is_empty() && family.bytes().all(|byte| byte.is_ascii_lowercase()) && digits(major, 2) - && digits(minor, 2) - && date.is_none_or(|date| date.len() == 8 && digits(date, 8)) - && parts.next().is_none() + && after_ok } /// A price is a fuse only if every component it bills is nonzero: a zero rate diff --git a/crates/claude-connector/src/handle.rs b/crates/claude-connector/src/handle.rs index c20fda0..222b74b 100644 --- a/crates/claude-connector/src/handle.rs +++ b/crates/claude-connector/src/handle.rs @@ -89,8 +89,6 @@ impl SessionStore for FsSessionStore { pub(crate) enum PlanError { #[error("claude-code: tools are not supported by this backend yet")] Tools, - #[error("claude-code: structured output is not supported by this backend yet")] - OutputSchema, #[error("claude-code: tool results are not supported by this backend yet")] ToolResult, #[error("claude-code: the request ends without user text")] @@ -128,9 +126,6 @@ pub(crate) fn plan( if !request.tools.is_empty() { return Err(PlanError::Tools); } - if request.output_schema_json.is_some() { - return Err(PlanError::OutputSchema); - } let input = &request.input; let run_start = input .iter() diff --git a/crates/claude-connector/src/launch.rs b/crates/claude-connector/src/launch.rs index c94134c..b233c09 100644 --- a/crates/claude-connector/src/launch.rs +++ b/crates/claude-connector/src/launch.rs @@ -9,19 +9,22 @@ use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; use model_connector::{ProviderRequest, ReasoningEffort}; -use zeroize::Zeroizing; use crate::config::ClaudeCodeConfig; use crate::handle::Plan; -/// The one credential variable: the child's environment is the only -/// environment the token ever enters. -pub(crate) const TOKEN_VAR: &str = "CLAUDE_CODE_OAUTH_TOKEN"; +/// The variable that tells the CLI which file descriptor holds its credential. +/// The token itself never enters the environment: the process port writes it +/// into a pipe and hands the child the read end at [`TOKEN_FD`]. +pub(crate) const TOKEN_FD_VAR: &str = "CLAUDE_CODE_OAUTH_TOKEN_FILE_DESCRIPTOR"; +/// The descriptor number the child finds the token pipe at: the first one +/// after stdin, stdout and stderr. +pub(crate) const TOKEN_FD: i32 = 3; const CHILD_PATH: &str = "/usr/local/bin:/usr/bin:/bin"; /// The environment of the child: exactly these names and nothing inherited. -/// Values are never formatted; `Debug` lists names only. -pub struct ChildEnv(Vec<(&'static str, Zeroizing)>); +/// `Debug` lists names only. +pub struct ChildEnv(Vec<(&'static str, String)>); impl ChildEnv { pub fn iter(&self) -> impl Iterator { @@ -37,21 +40,17 @@ impl fmt::Debug for ChildEnv { } } -pub(crate) fn child_env( - config: &ClaudeCodeConfig, - request: &ProviderRequest, - token: Zeroizing, -) -> ChildEnv { +pub(crate) fn child_env(config: &ClaudeCodeConfig, request: &ProviderRequest) -> ChildEnv { let config_dir = config.config_dir.to_string_lossy().into_owned(); ChildEnv(vec![ - ("PATH", Zeroizing::new(CHILD_PATH.to_string())), - ("HOME", Zeroizing::new(config_dir.clone())), - ("CLAUDE_CONFIG_DIR", Zeroizing::new(config_dir)), + ("PATH", CHILD_PATH.to_string()), + ("HOME", config_dir.clone()), + ("CLAUDE_CONFIG_DIR", config_dir), ( "CLAUDE_CODE_MAX_OUTPUT_TOKENS", - Zeroizing::new(request.max_output_tokens.to_string()), + request.max_output_tokens.to_string(), ), - (TOKEN_VAR, token), + (TOKEN_FD_VAR, TOKEN_FD.to_string()), ]) } @@ -108,7 +107,8 @@ fn effort(effort: ReasoningEffort) -> &'static str { /// rather than an argument because one argument is capped near 128 KiB /// (E2BIG) and instructions are not, the child has /// no tools, no MCP servers, no settings, no slash commands and nobody to ask -/// for permission, and a resume always forks. +/// for permission, and a resume always forks. A request's output schema goes +/// to the CLI as `--json-schema`; without one the flag is absent. pub(crate) fn argv( config: &ClaudeCodeConfig, request: &ProviderRequest, @@ -142,6 +142,9 @@ pub(crate) fn argv( if let Some(level) = request.reasoning_effort { argv.extend(["--effort".to_string(), effort(level).to_string()]); } + if let Some(schema) = &request.output_schema_json { + argv.extend(["--json-schema".to_string(), schema.clone()]); + } if let Plan::Resume { session_id, .. } = plan { argv.extend([ "--resume".to_string(), diff --git a/crates/claude-connector/src/lib.rs b/crates/claude-connector/src/lib.rs index ef3e404..327a078 100644 --- a/crates/claude-connector/src/lib.rs +++ b/crates/claude-connector/src/lib.rs @@ -9,6 +9,7 @@ mod daemon; mod handle; mod launch; mod process; +mod quota; mod stream; pub use backend::{ClaudeCodeBackend, CredentialTokenSource, StderrSink, TokenSource}; @@ -21,3 +22,4 @@ pub use launch::ChildEnv; #[cfg(unix)] pub use process::StdProcessPort; pub use process::{ProcessPort, RunningChild, StdoutRead}; +pub use quota::QuotaCell; diff --git a/crates/claude-connector/src/main.rs b/crates/claude-connector/src/main.rs index ece39cc..c58e7f5 100644 --- a/crates/claude-connector/src/main.rs +++ b/crates/claude-connector/src/main.rs @@ -37,7 +37,9 @@ fn main() -> ExitCode { #[derive(Debug, thiserror::Error)] enum Failure { - #[error("usage: serve --config PATH | config init PATH --flag value ... | config check PATH")] + #[error( + "usage: serve --config PATH | config init PATH --flag value ... | config check PATH (check reads the credential from the environment it runs in: run it in the unit's environment)" + )] Usage, #[error(transparent)] Config(#[from] ConfigError), diff --git a/crates/claude-connector/src/process.rs b/crates/claude-connector/src/process.rs index 450f388..2421e0e 100644 --- a/crates/claude-connector/src/process.rs +++ b/crates/claude-connector/src/process.rs @@ -20,7 +20,9 @@ pub enum StdoutRead { } /// Starts a `claude` child. `argv[0]` is the program; the environment is -/// exactly `env`; `stdin` is written and closed. +/// exactly `env`; `stdin` is written and closed. `token` is the child's +/// credential: it reaches the child on a pipe at `launch::TOKEN_FD`, once, and +/// is in no environment. pub trait ProcessPort: Send + Sync { fn spawn( &self, @@ -28,6 +30,7 @@ pub trait ProcessPort: Send + Sync { env: &ChildEnv, cwd: &Path, stdin: Vec, + token: &str, ) -> io::Result>; } @@ -47,6 +50,7 @@ pub use unix::StdProcessPort; #[cfg(unix)] mod unix { use std::io::{self, BufRead, BufReader, Read, Write}; + use std::os::fd::{AsRawFd, RawFd}; use std::os::unix::process::CommandExt; use std::path::Path; use std::process::{Child, Command, Stdio}; @@ -57,7 +61,40 @@ mod unix { use super::{ProcessPort, RunningChild, StdoutRead}; use crate::config::ClaudeCodeLimits; - use crate::launch::ChildEnv; + use crate::launch::{ChildEnv, TOKEN_FD}; + + /// A pipe holds a token without blocking the writer; a token is a few + /// hundred bytes, so anything near the pipe's capacity is not one. + const MAX_TOKEN_BYTES: usize = 4096; + + fn cvt(result: libc::c_int) -> io::Result { + if result < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(result) + } + } + + /// Makes `source` also descriptor `target`, open across exec. Runs between + /// fork and exec in the child, so it calls nothing but `dup2` and `fcntl`. + fn place_fd(source: RawFd, target: RawFd) -> io::Result<()> { + // SAFETY: dup2 and fcntl take integers only and are async-signal-safe. + unsafe { + if source == target { + // dup2 onto itself changes nothing, so the close-on-exec flag + // the pipe was created with must be cleared by hand. + let flags = cvt(libc::fcntl(target, libc::F_GETFD))?; + cvt(libc::fcntl( + target, + libc::F_SETFD, + flags & !libc::FD_CLOEXEC, + ))?; + } else { + cvt(libc::dup2(source, target))?; + } + } + Ok(()) + } /// Spawns each child in its own process group, so that killing the call /// kills what the child started too. @@ -78,7 +115,11 @@ mod unix { env: &ChildEnv, cwd: &Path, stdin: Vec, + token: &str, ) -> io::Result> { + if token.len() > MAX_TOKEN_BYTES { + return Err(io::Error::from(io::ErrorKind::InvalidInput)); + } let (program, args) = argv .split_first() .ok_or_else(|| io::Error::from(io::ErrorKind::InvalidInput))?; @@ -92,7 +133,17 @@ mod unix { .stdout(Stdio::piped()) .stderr(Stdio::piped()) .process_group(0); + // The write end closes here, so the child sees the token and then + // the end of the pipe. Both ends are close-on-exec; only the copy + // `place_fd` makes reaches the child. + let (reader, mut writer) = std::io::pipe()?; + writer.write_all(token.as_bytes())?; + drop(writer); + let source = reader.as_raw_fd(); + // SAFETY: the closure calls only async-signal-safe functions. + unsafe { command.pre_exec(move || place_fd(source, TOKEN_FD)) }; let mut child = command.spawn()?; + drop(reader); let (stdin_pipe, stdout, stderr) = (child.stdin.take(), child.stdout.take(), child.stderr.take()); @@ -229,4 +280,83 @@ mod unix { self.wait(); } } + + #[cfg(test)] + mod tests { + use super::*; + + fn cloexec(fd: RawFd) -> bool { + // SAFETY: fcntl on an integer descriptor. + let flags = unsafe { libc::fcntl(fd, libc::F_GETFD) }; + assert!(flags >= 0, "descriptor {fd} is not open"); + flags & libc::FD_CLOEXEC != 0 + } + + /// A descriptor at or above 300 on the same pipe, so a test never + /// touches a low number another thread may be using. + fn high_dup(source: RawFd, close_on_exec: bool) -> RawFd { + let command = if close_on_exec { + libc::F_DUPFD_CLOEXEC + } else { + libc::F_DUPFD + }; + // SAFETY: fcntl on an integer descriptor. + let fd = unsafe { libc::fcntl(source, command, 300) }; + assert!(fd >= 300); + fd + } + + /// The three tests below name descriptor numbers by value, and a number + /// one releases is the lowest free one for the next `high_dup`. They + /// hold this for their whole body, so none sees a number another + /// freed or places over one another owns. + static FDS: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + fn fds() -> std::sync::MutexGuard<'static, ()> { + FDS.lock().unwrap_or_else(|e| e.into_inner()) + } + + fn close(fd: RawFd) { + // SAFETY: closing a descriptor the test owns. + unsafe { libc::close(fd) }; + } + + #[test] + fn placing_a_descriptor_on_itself_clears_close_on_exec() { + let _held = fds(); + let (reader, _writer) = std::io::pipe().unwrap(); + let fd = high_dup(reader.as_raw_fd(), true); + assert!(cloexec(fd)); + place_fd(fd, fd).unwrap(); + assert!(!cloexec(fd), "it must survive exec"); + // One that already survives is left alone, not refused. + place_fd(fd, fd).unwrap(); + assert!(!cloexec(fd)); + close(fd); + } + + #[test] + fn placing_a_descriptor_elsewhere_copies_it_open_across_exec() { + let _held = fds(); + let (reader, _writer) = std::io::pipe().unwrap(); + let source = high_dup(reader.as_raw_fd(), true); + let target = high_dup(reader.as_raw_fd(), true); + assert!(cloexec(target)); + place_fd(source, target).unwrap(); + assert!(!cloexec(target), "the copy survives exec"); + assert!(cloexec(source), "the source does not"); + close(source); + close(target); + } + + #[test] + fn a_descriptor_that_is_not_open_is_refused_either_way() { + let _held = fds(); + let (reader, _writer) = std::io::pipe().unwrap(); + let closed = high_dup(reader.as_raw_fd(), true); + close(closed); + assert!(place_fd(closed, closed).is_err()); + assert!(place_fd(closed, closed + 1).is_err()); + } + } } diff --git a/crates/claude-connector/src/quota.rs b/crates/claude-connector/src/quota.rs new file mode 100644 index 0000000..76f0ce4 --- /dev/null +++ b/crates/claude-connector/src/quota.rs @@ -0,0 +1,151 @@ +//! The plan's own usage windows, as the CLI reports them. The subscription +//! decides how much a plan may burn; the CLI's `rate_limit_event` is the only +//! place that number appears, so the latest observation of each window is kept +//! here and read out beside the connector's own spend ledger. + +use std::collections::BTreeMap; +use std::sync::Mutex; +use std::time::{SystemTime, UNIX_EPOCH}; + +use model_connector::{PlanQuota, PlanQuotaSnapshot}; + +/// The latest observation of each plan window, shared by the backend that +/// feeds it and whatever reads it out. Each window carries the time of its +/// own observation: an event that names one window says nothing about the +/// others. +#[derive(Default)] +pub struct QuotaCell(Mutex>); + +impl QuotaCell { + pub fn new() -> Self { + Self::default() + } + + /// Records the windows one event reported, stamping each with now. Each + /// replaces that window's earlier observation, and windows the event did + /// not mention stay as they were, stamp included. + pub fn observe(&self, windows: impl IntoIterator) { + self.observe_at(unix_ms(), windows); + } + + fn observe_at(&self, now: u64, windows: impl IntoIterator) { + let mut held = self.0.lock().unwrap_or_else(|e| e.into_inner()); + for mut quota in windows { + quota.observed_at_unix_ms = now; + held.insert(quota.window.clone(), quota); + } + } + + pub fn snapshot(&self) -> PlanQuotaSnapshot { + self.snapshot_at(unix_ms()) + } + + /// The windows as of `now`. One whose reset time has passed is reported + /// `expired` with no utilization: the reading described a window that no + /// longer exists. + fn snapshot_at(&self, now: u64) -> PlanQuotaSnapshot { + let held = self.0.lock().unwrap_or_else(|e| e.into_inner()); + PlanQuotaSnapshot { + windows: held + .values() + .cloned() + .map(|mut quota| { + if now >= quota.resets_at_unix_ms { + quota.utilization_permille = None; + quota.status = EXPIRED.to_string(); + } + quota + }) + .collect(), + } + } +} + +/// The status of a window whose reset has passed since it was observed. +const EXPIRED: &str = "expired"; + +fn unix_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| { + u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + const RESET: u64 = 1_793_307_000_000; + + fn quota(window: &str, permille: u32, observed_at: u64) -> PlanQuota { + PlanQuota { + window: window.to_string(), + utilization_permille: Some(permille), + resets_at_unix_ms: RESET, + status: "allowed".to_string(), + observed_at_unix_ms: observed_at, + } + } + + #[test] + fn an_empty_cell_has_observed_nothing() { + assert!(QuotaCell::new().snapshot().windows.is_empty()); + } + + #[test] + fn each_window_keeps_its_own_latest_observation_and_its_own_time() { + let cell = QuotaCell::new(); + cell.observe_at(100, [quota("five_hour", 80, 0), quota("seven_day", 300, 0)]); + cell.observe_at(200, [quota("five_hour", 120, 0)]); + assert_eq!( + cell.snapshot_at(250).windows, + vec![quota("five_hour", 120, 200), quota("seven_day", 300, 100)], + "the window the later event did not name keeps its older stamp" + ); + } + + #[test] + fn an_event_with_no_windows_changes_nothing() { + let cell = QuotaCell::new(); + cell.observe_at(100, [quota("five_hour", 80, 0)]); + cell.observe_at(900, Vec::new()); + assert_eq!( + cell.snapshot_at(950).windows, + vec![quota("five_hour", 80, 100)] + ); + } + + #[test] + fn a_window_past_its_reset_is_expired_not_old_utilization() { + let cell = QuotaCell::new(); + cell.observe_at(100, [quota("five_hour", 80, 0)]); + let live = cell.snapshot_at(RESET - 1).windows; + assert_eq!(live, vec![quota("five_hour", 80, 100)]); + let expired = cell.snapshot_at(RESET).windows; + assert_eq!(expired.len(), 1); + assert_eq!(expired[0].status, "expired"); + assert_eq!(expired[0].utilization_permille, None); + assert_eq!( + expired[0].observed_at_unix_ms, 100, + "its age stays readable" + ); + assert_eq!(expired[0].resets_at_unix_ms, RESET); + } + + #[test] + fn the_live_clock_stamps_and_expires() { + let cell = QuotaCell::new(); + let before = unix_ms(); + let mut past = quota("old", 10, 0); + past.resets_at_unix_ms = 1; + cell.observe([past, quota("five_hour", 80, 0)]); + let after = unix_ms(); + let windows = cell.snapshot().windows; + let stamp = windows[0].observed_at_unix_ms; + assert!((before..=after).contains(&stamp)); + assert_eq!(windows[0].window, "five_hour"); + assert_eq!(windows[0].utilization_permille, Some(80)); + assert_eq!(windows[1].status, "expired"); + } +} diff --git a/crates/claude-connector/src/stream.rs b/crates/claude-connector/src/stream.rs index e99935f..f712892 100644 --- a/crates/claude-connector/src/stream.rs +++ b/crates/claude-connector/src/stream.rs @@ -3,10 +3,16 @@ use std::collections::BTreeMap; -use model_connector::{FailureKind, StopReason, Usage}; +use model_connector::{FailureKind, PlanQuota, StopReason, Usage}; use serde::Deserialize; use serde_json::Value; +use crate::quota::QuotaCell; + +/// The tool the CLI gives the model to deliver an answer that must match a +/// `--json-schema`. Its call is the structured output, not a tool exchange. +const STRUCTURED_OUTPUT_TOOL: &str = "StructuredOutput"; + /// A call that cannot complete, with the stage that failed. `usage` is `None` /// (the kit bills its reservation) unless a result event reported it or the /// call never reached the provider (zero). @@ -63,12 +69,18 @@ struct Line { session_id: Option, model: Option, message: Option, - rate_limit_info: Option, + /// Read by hand: the shape moves with the CLI, and a field that changes + /// type must cost the quota reading, never the call. + rate_limit_info: Option, is_error: Option, stop_reason: Option, api_error_status: Option, + /// Milliseconds spent in API calls; zero means no call was made. + duration_api_ms: Option, result: Option, errors: Option, + /// The answer to a `--json-schema` request, an object. + structured_output: Option, /// What this turn alone consumed. `modelUsage` is deliberately not read: /// on a resumed session it carries every earlier turn too. model_usage_since_last_result: Option, @@ -85,11 +97,7 @@ struct Block { #[serde(rename = "type")] kind: String, text: Option, -} - -#[derive(Deserialize)] -struct RateLimitInfo { - status: Option, + name: Option, } /// One model's entry. The four counts are required: an entry that lacks one @@ -107,6 +115,13 @@ struct ModelUsage { /// One call's stream, line by line. pub(crate) struct StreamState<'a> { model: &'a str, + /// Whether the request carried an output schema, so the answer is the + /// result's structured output. + structured: bool, + /// Whether that schema's top type admits an object (absent counts): the + /// structured output must be an object, never a bare scalar or array. + schema_top_is_object: bool, + quota: &'a QuotaCell, session_id: Option, texts: Vec, message_id: Option, @@ -114,9 +129,16 @@ pub(crate) struct StreamState<'a> { } impl<'a> StreamState<'a> { - pub(crate) fn new(model: &'a str) -> Self { + /// `schema` is the request's output schema text, when it carried one. + pub(crate) fn new(model: &'a str, schema: Option<&str>, quota: &'a QuotaCell) -> Self { + let top = schema + .and_then(|text| serde_json::from_str::(text).ok()) + .and_then(|schema| schema.get("type").cloned()); Self { model, + structured: schema.is_some(), + schema_top_is_object: top.is_none_or(|top| top.as_str() == Some("object")), + quota, session_id: None, texts: Vec::new(), message_id: None, @@ -151,10 +173,11 @@ impl<'a> StreamState<'a> { }, "assistant" => self.assistant(line.message), "rate_limit_event" => { - self.plan_rejected |= line - .rate_limit_info - .and_then(|info| info.status) - .is_some_and(|status| status == "rejected"); + if let Some(info) = &line.rate_limit_info { + self.plan_rejected |= + info.get("status").and_then(Value::as_str) == Some("rejected"); + self.quota.observe(plan_windows(info)); + } Step::More } "result" => Step::Done(Box::new(self.result(line))), @@ -176,6 +199,9 @@ impl<'a> StreamState<'a> { self.texts.push(text); } } + "tool_use" + if self.structured && block.name.as_deref() == Some(STRUCTURED_OUTPUT_TOOL) => { + } "tool_use" => { return fail( FailureKind::Provider, @@ -195,7 +221,16 @@ impl<'a> StreamState<'a> { line.api_error_status.as_ref().and_then(Value::as_u64), Some(401 | 403 | 429) ); - let usage = read_usage(line.model_usage_since_last_result, door_refusal); + // The CLI found no credential and never called the API: an error with + // no status and no API time at all. + let no_call = !success + && line.duration_api_ms == Some(0) + && line.api_error_status.as_ref().is_none_or(Value::is_null); + let usage = read_usage( + line.model_usage_since_last_result, + door_refusal || no_call, + no_call, + ); if !success { return Err(Failure { kind: self.error_kind(&line.api_error_status, &line.result, &line.errors), @@ -210,8 +245,26 @@ impl<'a> StreamState<'a> { usage: None, }); }; + let mut texts = std::mem::take(&mut self.texts); + if self.structured { + // The answer is the structured output alone; any prose around the + // tool call is not part of it. + let Some(output) = line + .structured_output + .filter(|value| value.is_object() && self.schema_top_is_object) + else { + return Err(Failure { + kind: FailureKind::Provider, + message: "claude-code: a schema was sent and the result carried no structured output object", + usage: Some(usage), + }); + }; + texts = vec![output.to_string()]; + } let stop_reason = match line.stop_reason.as_deref() { Some("end_turn") => StopReason::EndTurn, + // The structured-output tool call is how such a turn ends. + Some("tool_use") if self.structured => StopReason::EndTurn, Some("max_tokens") => StopReason::MaxOutputTokens, Some("stop_sequence") => StopReason::StopSequence, Some("refusal") => StopReason::Refusal, @@ -233,7 +286,7 @@ impl<'a> StreamState<'a> { }; Ok(Finished { session_id, - texts: std::mem::take(&mut self.texts), + texts, message_id: self.message_id.take(), stop_reason, usage, @@ -263,20 +316,54 @@ impl<'a> StreamState<'a> { } } +/// The plan windows one `rate_limit_event` reports under +/// `rate_limit_info.unifiedWindows`: utilization as a fraction of the window, +/// reset as unix seconds. A window missing any of its three fields is skipped; +/// an unknown reading is never zero. +fn plan_windows(info: &Value) -> Vec { + let Some(windows) = info.get("unifiedWindows").and_then(Value::as_object) else { + return Vec::new(); + }; + windows + .iter() + .filter_map(|(window, reading)| { + let utilization = reading.get("utilization")?.as_f64()?; + let resets_at = reading.get("resetsAt")?.as_u64()?; + let status = reading.get("status")?.as_str()?; + if !utilization.is_finite() || utilization < 0.0 { + return None; + } + Some(PlanQuota { + window: window.clone(), + // `as` saturates a float at the integer's bounds. + utilization_permille: Some((utilization * 1000.0).round() as u32), + resets_at_unix_ms: resets_at.saturating_mul(1000), + status: status.to_string(), + // Stamped by the cell that keeps it. + observed_at_unix_ms: 0, + }) + }) + .collect() +} + fn fail(kind: FailureKind, message: &'static str) -> Step { Step::Done(Box::new(Err(Failure::cost_unknown(kind, message)))) } /// What this turn consumed, summed over every model it used (harness side /// calls included), or `None` when the result does not say: the field absent, -/// of another shape, or empty. An empty report is believed only for a refusal -/// at the door (401, 403 or 429), which the provider cannot have billed; elsewhere -/// the CLI also zeroes usage on crashes after real work. `None` is billed the -/// reservation by the kit. -fn read_usage(reported: Option, door_refusal: bool) -> Option { - let models: BTreeMap = serde_json::from_value(reported?).ok()?; +/// of another shape, or empty. An empty report is believed only where the +/// provider cannot have billed (`empty_is_zero`: a refusal at the door, 401, +/// 403 or 429, or a call never made); elsewhere the CLI also zeroes usage on +/// crashes after real work. A call never made (`absent_is_zero`) may also +/// omit the field. `None` is billed the reservation by the kit. +fn read_usage(reported: Option, empty_is_zero: bool, absent_is_zero: bool) -> Option { + let Some(reported) = reported else { + return absent_is_zero.then(|| sum_usage(std::iter::empty())); + }; + let models: BTreeMap = serde_json::from_value(reported).ok()?; if models.is_empty() { - return door_refusal.then(|| sum_usage(std::iter::empty())); + return empty_is_zero.then(|| sum_usage(std::iter::empty())); } Some(sum_usage(models.into_values())) } diff --git a/crates/claude-connector/tests/config_document.rs b/crates/claude-connector/tests/config_document.rs index 4bc35d4..141f471 100644 --- a/crates/claude-connector/tests/config_document.rs +++ b/crates/claude-connector/tests/config_document.rs @@ -6,6 +6,7 @@ use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use std::process::{Command, Output}; use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; use claude_connector::{ClaudeConnectorConfig, ConfigError, load_config, write_config}; use cultcache_rs::{ @@ -276,13 +277,6 @@ fn config_refuses_aliases_unpriced_models_and_second_callers() { c.prices.insert("claude-sonnet-latest".to_string(), price()); }), ), - ( - "no minor", - Box::new(|c| { - c.callers[0].allowed_models = vec!["claude-opus-5".to_string()]; - c.prices.insert("claude-opus-5".to_string(), price()); - }), - ), ( "uppercase family", Box::new(|c| { @@ -343,7 +337,6 @@ fn config_refuses_aliases_unpriced_models_and_second_callers() { "allowed_models", "allowed_models", "allowed_models", - "allowed_models", "callers", "callers", ]; @@ -355,12 +348,23 @@ fn config_refuses_aliases_unpriced_models_and_second_callers() { } #[test] -fn a_dated_canonical_id_is_allowed() { - let dated = "claude-opus-5-1-20260101".to_string(); - let mut config = valid(); - config.callers[0].allowed_models = vec![dated.clone()]; - config.prices.insert(dated, price()); - config.validate().unwrap(); +fn a_priced_id_of_any_loose_form_is_allowed_and_the_price_table_decides() { + // Real ids come with a minor, without one, and with a date either way. + for id in [ + "claude-opus-5-1-20260101", + "claude-opus-5", + "claude-opus-5-20260101", + "claude-sonnet-4-5", + ] { + let mut config = valid(); + config.callers[0].allowed_models = vec![id.to_string()]; + config.prices.insert(id.to_string(), price()); + config.validate().unwrap_or_else(|e| panic!("{id}: {e}")); + + // The same id with no price is not on the known list. + config.prices.remove(id); + assert_eq!(refused(&config), "allowed_models", "{id}"); + } } #[test] @@ -543,13 +547,64 @@ fn no_refusal_echoes_a_value() { // ---- the binary ---- +/// How long any connector verb may take in these tests. `check` and `init` +/// return at once and `serve` is meant to refuse; a `serve` that starts +/// instead would run forever, so it fails a named test here, not the suite. +const VERB_DEADLINE: Duration = Duration::from_secs(20); + fn run(args: &[&str], envs: &[(&str, &Path)]) -> Output { let mut command = Command::new(env!("CARGO_BIN_EXE_claude-connector")); command.args(args).env_clear(); for (name, value) in envs { command.env(name, value); } - command.output().unwrap() + run_within(command, VERB_DEADLINE).unwrap_or_else(|| { + panic!( + "`{}` was still running after {VERB_DEADLINE:?}: it started", + args.join(" ") + ) + }) +} + +/// The command's output, or `None` when it outlived `deadline` and was killed. +fn run_within(mut command: Command, deadline: Duration) -> Option { + use std::process::Stdio; + use std::time::Instant; + + let mut child = command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let started = Instant::now(); + while child.try_wait().unwrap().is_none() { + if started.elapsed() > deadline { + child.kill().unwrap(); + child.wait().unwrap(); + return None; + } + std::thread::sleep(Duration::from_millis(20)); + } + Some(child.wait_with_output().unwrap()) +} + +#[cfg(unix)] +#[test] +fn a_process_past_its_deadline_is_killed_and_reported_as_none() { + let mut forever = Command::new("sleep"); + forever.arg("60"); + let started = std::time::Instant::now(); + assert!(run_within(forever, Duration::from_millis(200)).is_none()); + assert!(started.elapsed() < Duration::from_secs(10)); + let mut quick = Command::new("true"); + quick.env_clear(); + assert!( + run_within(quick, Duration::from_secs(10)) + .unwrap() + .status + .success() + ); } fn shown(output: &Output) -> String { @@ -756,8 +811,42 @@ fn check_refuses_exactly_what_serve_refuses_and_names_the_field() { assert!(init(&scratch, &flags).status.success()); let (checked, served) = check_and_serve(&scratch.path("connector.cc"), None); assert!(!checked.status.success() && !served.status.success()); - assert!(shown(&checked).contains("token_credential")); + assert!(shown(&checked).contains("credentials_directory")); + assert!(!shown(&checked).contains("token_credential")); assert_eq!(shown(&checked), shown(&served)); + + // A credential directory without the credential is the other fault, and + // the token is checked last: a broken pin is named first. + let scratch = Scratch::new(); + let (flags, credentials) = prepare(&scratch); + assert!(init(&scratch, &flags).status.success()); + std::fs::remove_file(credentials.join("claude-token")).unwrap(); + std::fs::write(scratch.path("claude"), b"other bytes").unwrap(); + let (checked, _) = check_and_serve(&scratch.path("connector.cc"), Some(&credentials)); + assert!( + shown(&checked).contains("claude_binary_sha256"), + "{}", + shown(&checked) + ); + std::fs::write(scratch.path("claude"), b"pinned claude").unwrap(); + let (checked, _) = check_and_serve(&scratch.path("connector.cc"), Some(&credentials)); + assert!( + shown(&checked).contains("token_credential"), + "{}", + shown(&checked) + ); + assert!(!shown(&checked).contains("credentials_directory")); +} + +#[test] +fn usage_says_check_runs_in_the_units_environment() { + let output = run(&["nonsense"], &[]); + assert!(!output.status.success()); + assert!( + shown(&output).contains("unit's environment"), + "{}", + shown(&output) + ); } #[cfg(unix)] diff --git a/crates/model-connector/src/client.rs b/crates/model-connector/src/client.rs index 1421a5b..47b6c94 100644 --- a/crates/model-connector/src/client.rs +++ b/crates/model-connector/src/client.rs @@ -4,11 +4,16 @@ use std::time::Duration; use thiserror::Error; use zeroize::Zeroizing; +use rand_core::{OsRng, RngCore}; + use crate::contract::{TransportInvocation, TransportKey, TransportResult}; use crate::envelope::{ - MAX_IDENTITY_BYTES, ServiceError, TransportFrameError, decode_result_frame, decrypt_result, - encode_invocation_frame, encrypt_invocation, read_transport_frame, write_transport_frame, + MAX_IDENTITY_BYTES, ServiceError, TransportFrameError, decode_result_frame, + decode_spend_readout_response_frame, decrypt_result, decrypt_spend_readout, + encode_invocation_frame, encode_spend_readout_request_frame, encrypt_invocation, + encrypt_spend_readout_request, read_transport_frame, write_transport_frame, }; +use crate::readout::SpendReadout; #[derive(Clone)] pub struct ConnectorClient { @@ -52,6 +57,34 @@ impl ConnectorClient { return Err(ConnectorClientError::RequestIdTooLong); } let request = encode_invocation_frame(&encrypt_invocation(invocation, &self.security)?)?; + let response = self.exchange(&request)?; + let envelope = decode_result_frame(&response)?; + decrypt_result(&envelope, &self.security, invocation).map_err(Into::into) + } + + /// Reads the connector's spend ledger and plan quota as `caller_runtime_id`, + /// which must be a caller whose connection key this client holds. + pub fn spend_readout( + &self, + caller_runtime_id: &str, + ) -> Result { + let mut nonce = [0_u8; 16]; + OsRng.fill_bytes(&mut nonce); + let request_id = format!( + "spend-readout-{}", + nonce.iter().map(|b| format!("{b:02x}")).collect::() + ); + let sealed = encrypt_spend_readout_request(caller_runtime_id, &request_id, &self.security)?; + let response = self.exchange(&encode_spend_readout_request_frame(&sealed)?)?; + let envelope = decode_spend_readout_response_frame(&response)?; + if envelope.caller_runtime_id != caller_runtime_id || envelope.request_id != request_id { + return Err(ServiceError::OuterIdentity.into()); + } + decrypt_spend_readout(&envelope, &self.security).map_err(Into::into) + } + + /// Sends one frame on a fresh connection and reads the one answer. + fn exchange(&self, request: &[u8]) -> Result, ConnectorClientError> { let connect_timeout = self .response_timeout .unwrap_or(Duration::from_secs(10)) @@ -62,12 +95,9 @@ impl ConnectorClient { .set_read_timeout(self.response_timeout) .and_then(|()| stream.set_write_timeout(self.response_timeout)) .map_err(ConnectorClientError::Connection)?; - write_transport_frame(&mut stream, &request, self.max_frame_bytes) + write_transport_frame(&mut stream, request, self.max_frame_bytes) .map_err(client_frame_error)?; - let response = - read_transport_frame(&mut stream, self.max_frame_bytes).map_err(client_frame_error)?; - let envelope = decode_result_frame(&response)?; - decrypt_result(&envelope, &self.security, invocation).map_err(Into::into) + read_transport_frame(&mut stream, self.max_frame_bytes).map_err(client_frame_error) } } @@ -226,4 +256,56 @@ mod fixture_tests { "the client must not even connect" ); } + + #[cfg(feature = "daemon")] + #[test] + fn a_readout_is_accepted_only_for_the_identity_that_asked() { + use crate::readout::PlanQuotaSnapshot; + // (caller the answer is sealed for, request id override, accepted) + let cases = [ + ("caller-1", None, true), + ("caller-2", None, false), + ("caller-1", Some("some-other-request"), false), + ]; + for (answer_caller, answer_request, accepted) in cases { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let endpoint = listener.local_addr().unwrap(); + let key = security("readout-client-key"); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let request = read_transport_frame(&mut stream, 64 * 1024).unwrap(); + let RequestFrame::SpendReadout(asked) = decode_request_frame(&request).unwrap() + else { + panic!("the client sent something other than a readout request"); + }; + let request_id = answer_request.unwrap_or(&asked.request_id); + let readout = SpendReadout::new(1, Vec::new(), PlanQuotaSnapshot::default()); + let sealed = + encrypt_spend_readout(answer_caller, request_id, &readout, &key).unwrap(); + write_transport_frame( + &mut stream, + &encode_spend_readout_response_frame(&sealed).unwrap(), + 64 * 1024, + ) + .unwrap(); + }); + let client = ConnectorClient::new( + endpoint, + "readout-client-key", + 64 * 1024, + Some(Duration::from_secs(1)), + ) + .unwrap(); + let answer = client.spend_readout("caller-1"); + if accepted { + assert_eq!(answer.unwrap().at_unix_ms, 1); + } else { + assert!(matches!( + answer, + Err(ConnectorClientError::Transport(ServiceError::OuterIdentity)) + )); + } + server.join().unwrap(); + } + } } diff --git a/crates/model-connector/src/envelope.rs b/crates/model-connector/src/envelope.rs index 40cfd09..70d8ed2 100644 --- a/crates/model-connector/src/envelope.rs +++ b/crates/model-connector/src/envelope.rs @@ -17,11 +17,14 @@ use crate::contract::{ ContractError, ENVELOPE_SCHEMA_ID, TransportInvocation, TransportKey, TransportResult, canonical_invocation_bytes, require_id, wire_decode, wire_encode, }; +use crate::readout::SpendReadout; #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub enum EnvelopeKind { Invocation, Result, + SpendReadoutRequest, + SpendReadoutResult, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] @@ -49,8 +52,10 @@ pub(crate) fn binary_schema(_: &mut schemars::SchemaGenerator) -> schemars::Sche /// The CultNet service every frame of this connector addresses. pub const CONNECTOR_SERVICE_ID: &str = "gamecult.model.connector"; -/// The one operation of that service: run one sealed invocation. +/// The operation that runs one sealed invocation. pub const INVOKE_OPERATION: &str = "invoke"; +/// The operation that reads the spend ledger and the plan's quota. +pub const SPEND_READOUT_OPERATION: &str = "spend_readout"; const PAYLOAD_ENCODING: &str = "messagepack-base64"; const OK_STATUS: &str = "ok"; @@ -69,16 +74,39 @@ fn seal_payload(envelope: &TransportEnvelope) -> Result { Ok(STANDARD.encode(wire_encode(envelope)?)) } -/// The CultNet frame an invocation travels in: a `cultnet.operation_request.v0` -/// for [`CONNECTOR_SERVICE_ID`] and [`INVOKE_OPERATION`], its `message_id` the +/// The two operations of the connector service. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Operation { + Invoke, + SpendReadout, +} + +impl Operation { + fn name(self) -> &'static str { + match self { + Operation::Invoke => INVOKE_OPERATION, + Operation::SpendReadout => SPEND_READOUT_OPERATION, + } + } + + fn from_name(name: &str) -> Option { + [Operation::Invoke, Operation::SpendReadout] + .into_iter() + .find(|operation| operation.name() == name) + } +} + +/// The CultNet frame a request travels in: a `cultnet.operation_request.v0` +/// for [`CONNECTOR_SERVICE_ID`] and the named operation, its `message_id` the /// request id and its payload the base64 of the named sealed envelope. -pub(crate) fn encode_invocation_frame( +fn request_frame( + operation: Operation, envelope: &TransportEnvelope, ) -> Result, ServiceError> { carriage_bytes(&CultNetMessage::OperationRequest { message_id: envelope.request_id.clone(), service_id: CONNECTOR_SERVICE_ID.to_string(), - operation: INVOKE_OPERATION.to_string(), + operation: operation.name().to_string(), payload_schema: ENVELOPE_SCHEMA_ID.to_string(), payload_encoding: PAYLOAD_ENCODING.to_string(), payload: seal_payload(envelope)?, @@ -87,14 +115,17 @@ pub(crate) fn encode_invocation_frame( }) } -/// The CultNet frame a result travels in: a `cultnet.operation_response.v0` -/// with status `ok`, otherwise shaped like the invocation's. +/// The CultNet frame a response travels in: a `cultnet.operation_response.v0` +/// with status `ok`, otherwise shaped like the request's. #[cfg(any(feature = "daemon", test))] -pub(crate) fn encode_result_frame(envelope: &TransportEnvelope) -> Result, ServiceError> { +fn response_frame( + operation: Operation, + envelope: &TransportEnvelope, +) -> Result, ServiceError> { carriage_bytes(&CultNetMessage::OperationResponse { message_id: envelope.request_id.clone(), service_id: CONNECTOR_SERVICE_ID.to_string(), - operation: INVOKE_OPERATION.to_string(), + operation: operation.name().to_string(), status: OK_STATUS.to_string(), payload_schema: ENVELOPE_SCHEMA_ID.to_string(), payload_encoding: PAYLOAD_ENCODING.to_string(), @@ -104,20 +135,43 @@ pub(crate) fn encode_result_frame(envelope: &TransportEnvelope) -> Result Result, ServiceError> { + request_frame(Operation::Invoke, envelope) +} + +pub(crate) fn encode_spend_readout_request_frame( + envelope: &TransportEnvelope, +) -> Result, ServiceError> { + request_frame(Operation::SpendReadout, envelope) +} + +#[cfg(any(feature = "daemon", test))] +pub(crate) fn encode_result_frame(envelope: &TransportEnvelope) -> Result, ServiceError> { + response_frame(Operation::Invoke, envelope) +} + +#[cfg(feature = "daemon")] +pub(crate) fn encode_spend_readout_response_frame( + envelope: &TransportEnvelope, +) -> Result, ServiceError> { + response_frame(Operation::SpendReadout, envelope) +} + /// Opens the carriage of one frame. Everything the carriage says is checked -/// here, before any decryption: the service, the operation, the payload schema, -/// and that the message id is the sealed envelope's request id. +/// here, before any decryption: the service, that the operation is one this +/// service has, the payload schema, and that the message id is the sealed +/// envelope's request id. fn open_carriage( message_id: &str, service_id: &str, operation: &str, payload_schema: &str, payload: &str, -) -> Result { - if service_id != CONNECTOR_SERVICE_ID - || operation != INVOKE_OPERATION - || payload_schema != ENVELOPE_SCHEMA_ID - { +) -> Result<(Operation, TransportEnvelope), ServiceError> { + let operation = Operation::from_name(operation).ok_or(ServiceError::Carriage)?; + if service_id != CONNECTOR_SERVICE_ID || payload_schema != ENVELOPE_SCHEMA_ID { return Err(ServiceError::Carriage); } let bytes = STANDARD @@ -127,7 +181,7 @@ fn open_carriage( if envelope.request_id != message_id { return Err(ServiceError::OuterIdentity); } - Ok(envelope) + Ok((operation, envelope)) } fn read_carriage(bytes: &[u8]) -> Result { @@ -135,8 +189,15 @@ fn read_carriage(bytes: &[u8]) -> Result { .map_err(|_| ServiceError::Carriage) } +/// A request frame, by the operation it names. #[cfg(any(feature = "daemon", test))] -pub(crate) fn decode_invocation_frame(bytes: &[u8]) -> Result { +pub(crate) enum RequestFrame { + Invoke(TransportEnvelope), + SpendReadout(TransportEnvelope), +} + +#[cfg(any(feature = "daemon", test))] +pub(crate) fn decode_request_frame(bytes: &[u8]) -> Result { match read_carriage(bytes)? { CultNetMessage::OperationRequest { message_id, @@ -145,18 +206,34 @@ pub(crate) fn decode_invocation_frame(bytes: &[u8]) -> Result open_carriage( - &message_id, - &service_id, - &operation, - &payload_schema, - &payload, - ), + } => { + match open_carriage( + &message_id, + &service_id, + &operation, + &payload_schema, + &payload, + )? { + (Operation::Invoke, envelope) => Ok(RequestFrame::Invoke(envelope)), + (Operation::SpendReadout, envelope) => Ok(RequestFrame::SpendReadout(envelope)), + } + } _ => Err(ServiceError::Carriage), } } -pub(crate) fn decode_result_frame(bytes: &[u8]) -> Result { +#[cfg(test)] +pub(crate) fn decode_invocation_frame(bytes: &[u8]) -> Result { + match decode_request_frame(bytes)? { + RequestFrame::Invoke(envelope) => Ok(envelope), + RequestFrame::SpendReadout(_) => Err(ServiceError::Carriage), + } +} + +fn decode_response_frame( + bytes: &[u8], + expected: Operation, +) -> Result { match read_carriage(bytes)? { CultNetMessage::OperationResponse { message_id, @@ -166,17 +243,32 @@ pub(crate) fn decode_result_frame(bytes: &[u8]) -> Result open_carriage( - &message_id, - &service_id, - &operation, - &payload_schema, - &payload, - ), + } if status == OK_STATUS => { + match open_carriage( + &message_id, + &service_id, + &operation, + &payload_schema, + &payload, + )? { + (found, envelope) if found == expected => Ok(envelope), + _ => Err(ServiceError::Carriage), + } + } _ => Err(ServiceError::Carriage), } } +pub(crate) fn decode_result_frame(bytes: &[u8]) -> Result { + decode_response_frame(bytes, Operation::Invoke) +} + +pub(crate) fn decode_spend_readout_response_frame( + bytes: &[u8], +) -> Result { + decode_response_frame(bytes, Operation::SpendReadout) +} + /// The most bytes one frame holding a sealed envelope of `ciphertext_len` /// ciphertext bytes can occupy, with ids at their longest: the frame a caller /// payload bound (`CallerAdmission`'s `max_payload_bytes`, a bound on @@ -324,6 +416,59 @@ pub fn decrypt_result( Ok(result) } +/// A spend readout request carries no content: the sealed envelope's +/// authenticated identity and kind are the whole request. +pub(crate) fn encrypt_spend_readout_request( + caller_runtime_id: &str, + request_id: &str, + security: &TransportKey, +) -> Result { + encrypt_message( + caller_runtime_id, + request_id, + EnvelopeKind::SpendReadoutRequest, + &[], + security, + ) +} + +#[cfg(any(feature = "daemon", test))] +pub(crate) fn decrypt_spend_readout_request( + envelope: &TransportEnvelope, + security: &TransportKey, +) -> Result<(), ServiceError> { + validate_envelope(envelope, EnvelopeKind::SpendReadoutRequest)?; + decrypt_message(envelope, security).map(drop) +} + +#[cfg(feature = "daemon")] +pub(crate) fn encrypt_spend_readout( + caller_runtime_id: &str, + request_id: &str, + readout: &SpendReadout, + security: &TransportKey, +) -> Result { + let plaintext = wire_encode(readout).map_err(|_| ServiceError::Encoding)?; + encrypt_message( + caller_runtime_id, + request_id, + EnvelopeKind::SpendReadoutResult, + &plaintext, + security, + ) +} + +pub(crate) fn decrypt_spend_readout( + envelope: &TransportEnvelope, + security: &TransportKey, +) -> Result { + validate_envelope(envelope, EnvelopeKind::SpendReadoutResult)?; + let plaintext = decrypt_message(envelope, security)?; + let readout: SpendReadout = wire_decode(&plaintext).map_err(|_| ServiceError::Encoding)?; + readout.validate()?; + Ok(readout) +} + #[derive(Debug)] pub(crate) enum TransportFrameError { Connection(std::io::Error), @@ -1138,4 +1283,53 @@ mod tests { ); } } + + #[test] + fn a_readout_request_is_not_an_invocation_and_names_its_own_operation() { + let key = TransportKey::from_connection_secret("readout-test-key").unwrap(); + let sealed = encrypt_spend_readout_request("caller-1", "readout-1", &key).unwrap(); + let frame = encode_spend_readout_request_frame(&sealed).unwrap(); + assert!(matches!( + decode_request_frame(&frame), + Ok(RequestFrame::SpendReadout(found)) if found == sealed + )); + assert_eq!(decode_invocation_frame(&frame), Err(ServiceError::Carriage)); + // Sealed as one kind, the authenticated data refuses it as another. + assert!(decrypt_spend_readout_request(&sealed, &key).is_ok()); + assert_eq!( + decrypt_spend_readout(&sealed, &key).unwrap_err(), + ServiceError::Envelope + ); + let mut relabelled = sealed.clone(); + relabelled.message_kind = EnvelopeKind::SpendReadoutResult; + assert_eq!( + decrypt_spend_readout(&relabelled, &key).unwrap_err(), + ServiceError::Encryption + ); + } + + #[cfg(feature = "daemon")] + #[test] + fn a_response_is_read_as_its_own_operation_only() { + use crate::readout::PlanQuotaSnapshot; + let key = TransportKey::from_connection_secret("readout-test-key").unwrap(); + let readout = SpendReadout::new(1, Vec::new(), PlanQuotaSnapshot::default()); + let sealed = encrypt_spend_readout("caller-1", "readout-1", &readout, &key).unwrap(); + let readout_frame = encode_spend_readout_response_frame(&sealed).unwrap(); + let invoke_frame = encode_result_frame(&sealed).unwrap(); + assert_eq!( + decode_spend_readout_response_frame(&readout_frame).unwrap(), + sealed + ); + assert_eq!(decode_result_frame(&invoke_frame).unwrap(), sealed); + assert_eq!( + decode_result_frame(&readout_frame), + Err(ServiceError::Carriage) + ); + assert_eq!( + decode_spend_readout_response_frame(&invoke_frame), + Err(ServiceError::Carriage) + ); + assert_eq!(decrypt_spend_readout(&sealed, &key).unwrap(), readout); + } } diff --git a/crates/model-connector/src/lib.rs b/crates/model-connector/src/lib.rs index ba1642b..c55202a 100644 --- a/crates/model-connector/src/lib.rs +++ b/crates/model-connector/src/lib.rs @@ -1,6 +1,7 @@ mod client; mod contract; mod envelope; +mod readout; #[cfg(test)] mod fixtures; @@ -13,6 +14,7 @@ mod service; pub use client::*; pub use contract::*; pub use envelope::*; +pub use readout::*; #[cfg(feature = "daemon")] pub use serve::*; diff --git a/crates/model-connector/src/readout.rs b/crates/model-connector/src/readout.rs new file mode 100644 index 0000000..36d4cb9 --- /dev/null +++ b/crates/model-connector/src/readout.rs @@ -0,0 +1,128 @@ +//! The spend readout: what each caller has spent and holds in its current +//! window, and the plan's own usage as the backend last saw it. The ledger is +//! the replay store's; a readout only reads it. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::contract::ContractError; + +pub const SPEND_READOUT_SCHEMA_ID: &str = "gamecult.model.spend_readout.v1"; + +/// One window of the subscription plan, as the provider reported it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct PlanQuota { + pub window: String, + /// How much of the window was used when it was observed, in thousandths; + /// `None` once the window has reset (`status` is then `expired`), because + /// the old reading says nothing about the new window. + pub utilization_permille: Option, + pub resets_at_unix_ms: u64, + pub status: String, + /// When this window was last reported. Its age in a readout is the + /// readout's `at_unix_ms` minus this. + pub observed_at_unix_ms: u64, +} + +/// What a backend last saw of the plan's windows, each stamped with its own +/// observation time. A backend with no plan to report returns the default. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct PlanQuotaSnapshot { + pub windows: Vec, +} + +/// One caller's current spend window in micro-USD. `spent` is settled charges +/// plus what a crash left counted at its full reservation; `reserved` is what +/// calls in flight hold, and both count against the ceiling. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CallerSpend { + pub caller_id: String, + pub window_ms: u64, + pub ceiling_micro_usd: u64, + pub spent_micro_usd: u64, + pub reserved_micro_usd: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SpendReadout { + pub schema_id: String, + pub at_unix_ms: u64, + pub callers: Vec, + pub plan_quota: Vec, +} + +impl SpendReadout { + pub fn new(at_unix_ms: u64, callers: Vec, plan: PlanQuotaSnapshot) -> Self { + Self { + schema_id: SPEND_READOUT_SCHEMA_ID.to_string(), + at_unix_ms, + callers, + plan_quota: plan.windows, + } + } + + pub(crate) fn validate(&self) -> Result<(), ContractError> { + if self.schema_id != SPEND_READOUT_SCHEMA_ID { + return Err(ContractError::Schema); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::contract::{wire_decode, wire_encode}; + + fn readout() -> SpendReadout { + SpendReadout::new( + 1_000, + vec![CallerSpend { + caller_id: "eureka".into(), + window_ms: 3_600_000, + ceiling_micro_usd: 50_000_000, + spent_micro_usd: 7, + reserved_micro_usd: 3, + }], + PlanQuotaSnapshot { + windows: vec![PlanQuota { + window: "five_hour".into(), + utilization_permille: Some(80), + resets_at_unix_ms: 1_793_307_000_000, + status: "allowed".into(), + observed_at_unix_ms: 999, + }], + }, + ) + } + + #[test] + fn a_readout_is_a_named_map_that_decodes_to_itself() { + let bytes = wire_encode(&readout()).unwrap(); + let value: rmpv::Value = rmp_serde::from_slice(&bytes).unwrap(); + let rmpv::Value::Map(fields) = value else { + panic!("a readout is a map, not a positional array"); + }; + let names: Vec<&str> = fields.iter().filter_map(|(k, _)| k.as_str()).collect(); + assert_eq!(names, ["schema_id", "at_unix_ms", "callers", "plan_quota"]); + assert_eq!(wire_decode::(&bytes).unwrap(), readout()); + } + + #[test] + fn an_unobserved_plan_is_none_not_zero() { + let bare = SpendReadout::new(1, Vec::new(), PlanQuotaSnapshot::default()); + let decoded: SpendReadout = wire_decode(&wire_encode(&bare).unwrap()).unwrap(); + assert!(decoded.plan_quota.is_empty()); + } + + #[test] + fn a_readout_of_another_schema_is_refused() { + let mut other = readout(); + other.schema_id = "gamecult.model.spend_readout.v2".into(); + assert!(other.validate().is_err()); + assert!(readout().validate().is_ok()); + } +} diff --git a/crates/model-connector/src/serve.rs b/crates/model-connector/src/serve.rs index d6723a4..26f5e2d 100644 --- a/crates/model-connector/src/serve.rs +++ b/crates/model-connector/src/serve.rs @@ -12,9 +12,11 @@ use zeroize::Zeroizing; use crate::contract::TransportResult; use crate::envelope::{ - ServiceError, TransportFrameError, decode_invocation_frame, encode_result_frame, - frame_bytes_for_payload, read_transport_frame, write_transport_frame, + RequestFrame, ServiceError, TransportFrameError, decode_request_frame, encode_result_frame, + encode_spend_readout_response_frame, frame_bytes_for_payload, read_transport_frame, + write_transport_frame, }; +use crate::readout::PlanQuotaSnapshot; use crate::service::{ CallerAdmission, PriceTable, ProviderCall, SpendCeiling, SpendWindowMs, TransportAdmission, TransportService, @@ -32,6 +34,11 @@ pub trait ProviderBackend: Send + Sync { /// `serve` on: it names the provider on every receipt and decides which /// reasoning state is foreign. fn provider(&self) -> &str; + /// What the provider's own plan allows and has used, for the spend + /// readout. A provider with no plan to report keeps the default. + fn plan_quota(&self) -> PlanQuotaSnapshot { + PlanQuotaSnapshot::default() + } /// Executes one admitted call. The service has already bound it to its /// caller and recorded its replay claim. The call cannot show a foreign /// reasoning item and builds the result's receipt itself. @@ -232,17 +239,28 @@ fn serve_connection( .and_then(|()| stream.set_write_timeout(Some(timeout))) .map_err(ServeError::Connection)?; let request = read_transport_frame(&mut stream, limits.max_frame_bytes).map_err(frame_error)?; - let response = execute_inference_frame(&request, service, backend, limits)?; + let response = answer_frame(&request, service, backend, limits)?; write_transport_frame(&mut stream, &response, limits.max_frame_bytes).map_err(frame_error) } -fn execute_inference_frame( +fn answer_frame( request: &[u8], service: &Mutex, backend: &dyn ProviderBackend, limits: &ServeLimits, ) -> Result, ServeError> { - let envelope = decode_invocation_frame(request).map_err(|_| ServeError::FrameEncoding)?; + let envelope = match decode_request_frame(request).map_err(|_| ServeError::FrameEncoding)? { + RequestFrame::Invoke(envelope) => envelope, + RequestFrame::SpendReadout(envelope) => { + let plan = backend.plan_quota(); + let response = service + .lock() + .map_err(|_| ServeError::ServicePoisoned)? + .spend_readout(&envelope, unix_ms()?, plan, limits.max_frame_bytes)?; + return encode_spend_readout_response_frame(&response) + .map_err(|_| ServeError::FrameEncoding); + } + }; let admission = service .lock() .map_err(|_| ServeError::ServicePoisoned)? @@ -495,6 +513,87 @@ mod tests { .collect() } + /// A counting backend that reports a plan, as a provider with one would. + struct PlanBackend { + inner: Arc, + plan: PlanQuotaSnapshot, + } + + impl ProviderBackend for PlanBackend { + fn ready(&self) -> Result<(), &'static str> { + self.inner.ready() + } + + fn provider(&self) -> &str { + self.inner.provider() + } + + fn plan_quota(&self) -> PlanQuotaSnapshot { + self.plan.clone() + } + + fn execute(&self, call: &ProviderCall) -> TransportResult { + self.inner.execute(call) + } + } + + #[test] + fn serve_reads_out_spend_and_plan_quota_without_executing_anything() { + let store = TestStore::new(); + let inner = CountingBackend::new(true); + let plan = PlanQuotaSnapshot { + windows: vec![crate::readout::PlanQuota { + window: "five_hour".into(), + utilization_permille: Some(80), + resets_at_unix_ms: 1_793_307_000_000, + status: "allowed".into(), + observed_at_unix_ms: 1_793_300_000_000, + }], + }; + let backend = Arc::new(PlanBackend { + inner: inner.clone(), + plan: plan.clone(), + }); + let (client, endpoint) = start_with(backend, &store, vec![caller(CALLER, KEY, 4)]); + client.execute(&fresh("request-1")).unwrap(); + assert_eq!(inner.executions.load(Ordering::SeqCst), 1); + + let before = now_ms(); + let readout = client.spend_readout(CALLER).unwrap(); + assert!(readout.at_unix_ms >= before && readout.at_unix_ms <= now_ms()); + assert_eq!(readout.plan_quota, plan.windows); + let [row] = readout.callers.as_slice() else { + panic!("one admitted caller"); + }; + assert_eq!(row.caller_id, CALLER); + assert_eq!((row.ceiling_micro_usd, row.window_ms), (CEILING, WINDOW_MS)); + assert!(row.spent_micro_usd > 0, "the settled call was charged"); + assert_eq!(row.reserved_micro_usd, 0); + assert_eq!( + inner.executions.load(Ordering::SeqCst), + 1, + "a readout runs nothing" + ); + + // A client without the caller's key is dropped, not answered. + let stranger = ConnectorClient::new( + endpoint, + "some-other-key", + FRAME, + Some(Duration::from_secs(5)), + ) + .unwrap(); + assert!(stranger.spend_readout(CALLER).is_err()); + } + + #[test] + fn a_backend_with_no_plan_reads_out_none() { + let store = TestStore::new(); + let (client, _) = start(CountingBackend::new(true), &store); + let readout = client.spend_readout(CALLER).unwrap(); + assert!(readout.plan_quota.is_empty()); + } + #[test] fn serve_refuses_a_request_over_its_max_cost_without_executing_it() { let store = TestStore::new(); diff --git a/crates/model-connector/src/service.rs b/crates/model-connector/src/service.rs index 6d10cb9..c269916 100644 --- a/crates/model-connector/src/service.rs +++ b/crates/model-connector/src/service.rs @@ -14,8 +14,10 @@ use crate::contract::{ }; use crate::envelope::{ EnvelopeKind, MAX_IDENTITY_BYTES, ServiceError, TransportEnvelope, decrypt_invocation, - encode_result_frame, encrypt_result, validate_envelope, + decrypt_spend_readout_request, encode_result_frame, encode_spend_readout_response_frame, + encrypt_result, encrypt_spend_readout, validate_envelope, }; +use crate::readout::{CallerSpend, PlanQuotaSnapshot, SpendReadout}; /// What a caller may spend in one window, in micro-USD. A distinct type from /// [`SpendWindowMs`] so the pair cannot be passed swapped: @@ -280,15 +282,15 @@ fn admission_clock( /// it was admitted in, so nothing older is kept. fn prune_windows( window_spend: &mut HashMap>, - live: &HashMap, + live: &HashMap, caller_runtime_id: &str, floor: u64, ) { if let Some(windows) = window_spend.get_mut(caller_runtime_id) { windows.retain(|window, _| { *window >= floor - || live.iter().any(|(identity, live_window)| { - live_window == window && identity.caller_runtime_id == caller_runtime_id + || live.iter().any(|(identity, request)| { + request.window == *window && identity.caller_runtime_id == caller_runtime_id }) }); } @@ -298,7 +300,7 @@ fn prune_windows( /// places that do not know the clock (open, settle). fn prune_to_latest( window_spend: &mut HashMap>, - live: &HashMap, + live: &HashMap, caller_runtime_id: &str, ) { let latest = window_spend @@ -309,6 +311,14 @@ fn prune_to_latest( } } +/// A request between claim and settlement: the window it was admitted in and +/// the reservation it holds there. +#[derive(Debug, Clone, Copy)] +struct LiveRequest { + window: u64, + reserved_micro_usd: u64, +} + #[derive(Debug, Clone, PartialEq, Eq, Hash)] struct RequestIdentity { caller_runtime_id: String, @@ -464,7 +474,7 @@ pub struct TransportService { callers: HashMap, /// The requests between claim and settlement, each with the window it was /// admitted in. - live: HashMap, + live: HashMap, replay_store: CultCache, max_expiry_skew_ms: u64, provider: String, @@ -611,6 +621,70 @@ impl TransportService { .unwrap_or(0) } + /// Answers one sealed spend readout request: for each admitted caller, what + /// its current window holds against its ceiling, with the backend's plan + /// quota beside it. Read-only: nothing is reserved, recorded or pruned. + /// Any admitted caller reads every caller's row; the spend is the + /// operator's and the callers are all hers. `max_frame_bytes` bounds the + /// reply like it bounds `complete`'s. + pub fn spend_readout( + &self, + envelope: &TransportEnvelope, + now_unix_ms: u64, + plan: PlanQuotaSnapshot, + max_frame_bytes: usize, + ) -> Result { + validate_envelope(envelope, EnvelopeKind::SpendReadoutRequest)?; + let asker = self + .callers + .get(&envelope.caller_runtime_id) + .ok_or(ServiceError::CallerNotAdmitted)?; + decrypt_spend_readout_request(envelope, &asker.security)?; + let mut callers: Vec = self + .callers + .values() + .map(|caller| { + // The window admission would use right now. + let window = admission_clock( + &self.window_spend, + &caller.caller_runtime_id, + now_unix_ms, + caller.spend_window_ms, + ) / caller.spend_window_ms; + let held = self.window_spend(&caller.caller_runtime_id, window); + let reserved: u64 = self + .live + .iter() + .filter(|(identity, request)| { + request.window == window + && identity.caller_runtime_id == caller.caller_runtime_id + }) + .fold(0, |sum, (_, request)| { + sum.saturating_add(request.reserved_micro_usd) + }); + CallerSpend { + caller_id: caller.caller_runtime_id.clone(), + window_ms: caller.spend_window_ms, + ceiling_micro_usd: caller.spend_ceiling_micro_usd, + spent_micro_usd: held.saturating_sub(reserved), + reserved_micro_usd: reserved, + } + }) + .collect(); + callers.sort_by(|a, b| a.caller_id.cmp(&b.caller_id)); + let readout = SpendReadout::new(now_unix_ms, callers, plan); + let sealed = encrypt_spend_readout( + &envelope.caller_runtime_id, + &envelope.request_id, + &readout, + &asker.security, + )?; + if !readout_fits_frame(&sealed, max_frame_bytes)? { + return Err(ServiceError::FrameBound); + } + Ok(sealed) + } + /// Admits one request. `max_frame_bytes` is the frame every reply must /// fit, the same bound `complete` is given: a recorded result that no /// longer fits it (the service reopened at a smaller frame) is answered @@ -794,7 +868,13 @@ impl TransportService { 0, reserved_micro_usd, ); - self.live.insert(identity, window); + self.live.insert( + identity, + LiveRequest { + window, + reserved_micro_usd, + }, + ); Ok(TransportAdmission::Execute(Box::new(ExecutionClaim { call: ProviderCall { receipt: TransportReceipt { @@ -1005,6 +1085,14 @@ fn fits_frame(envelope: &TransportEnvelope, max_frame_bytes: usize) -> Result Result { + encode_spend_readout_response_frame(envelope).map(|bytes| bytes.len() <= max_frame_bytes) +} + /// The provider named on the stand-in failure, fixed so its size is bounded. const STAND_IN_PROVIDER: &str = "model-connector"; @@ -1758,6 +1846,17 @@ mod tests { assert_eq!(fits_frame(&envelope, size - 1), Ok(false)); } + #[test] + fn a_readout_frame_check_is_inclusive_of_the_serialized_envelope() { + let readout = SpendReadout::new(1, Vec::new(), PlanQuotaSnapshot::default()); + let envelope = encrypt_spend_readout(CALLER, "sized", &readout, &security(KEY)).unwrap(); + let size = encode_spend_readout_response_frame(&envelope) + .unwrap() + .len(); + assert_eq!(readout_fits_frame(&envelope, size), Ok(true)); + assert_eq!(readout_fits_frame(&envelope, size - 1), Ok(false)); + } + fn claim_for( service: &mut TransportService, request_id: &str, @@ -3616,4 +3715,151 @@ mod tests { <= sized(Refusal::ProviderDigestSubstitution) ); } + + // ---- spend readout ---- + + fn readout_as( + service: &TransportService, + caller: &str, + key: &str, + now: u64, + frame: usize, + ) -> Result { + let sealed = encrypt_spend_readout_request(caller, "readout-1", &security(key)).unwrap(); + let answer = service.spend_readout(&sealed, now, PlanQuotaSnapshot::default(), frame)?; + assert_eq!(answer.request_id, "readout-1"); + Ok(decrypt_spend_readout(&answer, &security(key)).unwrap()) + } + + fn row<'a>(readout: &'a SpendReadout, caller: &str) -> &'a CallerSpend { + readout + .callers + .iter() + .find(|row| row.caller_id == caller) + .expect("the caller has a row") + } + + #[test] + fn spend_readout_reports_window() { + let store = TestStore::new(); + let mut service = open_service(&store, 4); + let claim = claim_at(&mut service, "request-1", 1_000); + + // In flight: the reservation is held, nothing is spent. + let readout = readout_as(&service, CALLER, KEY, 1_500, FRAME).unwrap(); + assert_eq!(readout.at_unix_ms, 1_500); + let mine = row(&readout, CALLER); + assert_eq!(mine.window_ms, WINDOW_MS); + assert_eq!(mine.ceiling_micro_usd, CEILING); + assert_eq!( + (mine.spent_micro_usd, mine.reserved_micro_usd), + (0, reservation()) + ); + + // Settled: the charge replaces the reservation. + let result = text_result(claim.call(), "done"); + service.complete(*claim, result, FRAME).unwrap(); + let readout = readout_as(&service, CALLER, KEY, 1_500, FRAME).unwrap(); + let mine = row(&readout, CALLER); + assert_eq!( + (mine.spent_micro_usd, mine.reserved_micro_usd), + (FIXTURE_USAGE_COST, 0) + ); + + // Every admitted caller has a row, in id order; the idle one is empty. + let ids: Vec<&str> = readout + .callers + .iter() + .map(|r| r.caller_id.as_str()) + .collect(); + assert_eq!(ids, ["epiphany-yggdrasil", "ghostlight-yggdrasil"]); + let idle = row(&readout, "ghostlight-yggdrasil"); + assert_eq!((idle.spent_micro_usd, idle.reserved_micro_usd), (0, 0)); + } + + #[test] + fn a_claim_a_crash_left_reads_as_spent_not_reserved() { + let store = TestStore::new(); + let mut service = open_service(&store, 4); + let _claim = claim_at(&mut service, "request-1", 1_000); + drop(service); + let service = open_service(&store, 4); + let readout = readout_as(&service, CALLER, KEY, 1_500, FRAME).unwrap(); + let mine = row(&readout, CALLER); + assert_eq!( + (mine.spent_micro_usd, mine.reserved_micro_usd), + (reservation(), 0) + ); + } + + #[test] + fn a_readout_reads_the_ledger_and_changes_nothing_in_it() { + let store = TestStore::new(); + let mut service = open_service(&store, 4); + let claim = claim_at(&mut service, "request-1", 1_000); + // Two windows later: the caller's current window is empty, and + // looking must not prune the window the live claim settles into. + let later = 2 * WINDOW_MS + 5; + let readout = readout_as(&service, CALLER, KEY, later, FRAME).unwrap(); + let mine = row(&readout, CALLER); + assert_eq!((mine.spent_micro_usd, mine.reserved_micro_usd), (0, 0)); + assert_eq!(service.window_spend(CALLER, 0), reservation()); + let result = text_result(claim.call(), "done"); + service.complete(*claim, result, FRAME).unwrap(); + assert_eq!(service.window_spend(CALLER, 0), FIXTURE_USAGE_COST); + } + + #[test] + fn a_readout_is_refused_to_a_stranger_a_wrong_key_and_an_invocation() { + let store = TestStore::new(); + let service = open_service(&store, 4); + let sealed = |caller: &str, key: &str| { + encrypt_spend_readout_request(caller, "readout-1", &security(key)).unwrap() + }; + assert_eq!( + service + .spend_readout( + &sealed("nobody", KEY), + 1, + PlanQuotaSnapshot::default(), + FRAME + ) + .unwrap_err(), + ServiceError::CallerNotAdmitted + ); + assert_eq!( + service + .spend_readout( + &sealed(CALLER, "another-key"), + 1, + PlanQuotaSnapshot::default(), + FRAME + ) + .unwrap_err(), + ServiceError::Encryption + ); + let invocation = invocation_at("request-1", 1_000, MAX_COST); + assert_eq!( + service + .spend_readout( + &raw_envelope(&invocation), + 1, + PlanQuotaSnapshot::default(), + FRAME + ) + .unwrap_err(), + ServiceError::Envelope + ); + } + + #[test] + fn a_readout_that_cannot_fit_the_frame_is_a_frame_bound_failure() { + let store = TestStore::new(); + let service = open_service(&store, 4); + assert_eq!( + readout_as(&service, CALLER, KEY, 1_000, 64).unwrap_err(), + ServiceError::FrameBound + ); + assert!(readout_as(&service, CALLER, KEY, 1_000, FRAME).is_ok()); + } } diff --git a/crates/model-connector/tests/published_schemas.rs b/crates/model-connector/tests/published_schemas.rs index 15ce197..cb21b01 100644 --- a/crates/model-connector/tests/published_schemas.rs +++ b/crates/model-connector/tests/published_schemas.rs @@ -28,6 +28,7 @@ fn published_schemas_are_the_types() { ("transport_result", render::()), ("transport_receipt", render::()), ("transport_envelope", render::()), + ("spend_readout", render::()), ]; let dir = schemas_dir(); if std::env::var_os(REGENERATE).is_some() { @@ -53,7 +54,7 @@ fn published_schemas_are_the_types() { expected.sort(); assert_eq!( on_disk, expected, - "schemas/ holds exactly the five documents" + "schemas/ holds exactly the six documents" ); } diff --git a/schemas/spend_readout.json b/schemas/spend_readout.json new file mode 100644 index 0000000..5a5d23d --- /dev/null +++ b/schemas/spend_readout.json @@ -0,0 +1,112 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "SpendReadout", + "type": "object", + "properties": { + "at_unix_ms": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "callers": { + "type": "array", + "items": { + "$ref": "#/$defs/CallerSpend" + } + }, + "plan_quota": { + "type": "array", + "items": { + "$ref": "#/$defs/PlanQuota" + } + }, + "schema_id": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "schema_id", + "at_unix_ms", + "callers", + "plan_quota" + ], + "$defs": { + "CallerSpend": { + "description": "One caller's current spend window in micro-USD. `spent` is settled charges\nplus what a crash left counted at its full reservation; `reserved` is what\ncalls in flight hold, and both count against the ceiling.", + "type": "object", + "properties": { + "caller_id": { + "type": "string" + }, + "ceiling_micro_usd": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "reserved_micro_usd": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "spent_micro_usd": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "window_ms": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "caller_id", + "window_ms", + "ceiling_micro_usd", + "spent_micro_usd", + "reserved_micro_usd" + ] + }, + "PlanQuota": { + "description": "One window of the subscription plan, as the provider reported it.", + "type": "object", + "properties": { + "observed_at_unix_ms": { + "description": "When this window was last reported. Its age in a readout is the\nreadout's `at_unix_ms` minus this.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "resets_at_unix_ms": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "status": { + "type": "string" + }, + "utilization_permille": { + "description": "How much of the window was used when it was observed, in thousandths;\n`None` once the window has reset (`status` is then `expired`), because\nthe old reading says nothing about the new window.", + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0 + }, + "window": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "window", + "resets_at_unix_ms", + "status", + "observed_at_unix_ms" + ] + } + } +} diff --git a/schemas/transport_envelope.json b/schemas/transport_envelope.json index 69a97d3..8b062e3 100644 --- a/schemas/transport_envelope.json +++ b/schemas/transport_envelope.json @@ -46,7 +46,9 @@ "type": "string", "enum": [ "Invocation", - "Result" + "Result", + "SpendReadoutRequest", + "SpendReadoutResult" ] } } From d1f25b5ae6d1c22c652aff3401e55f1a78773d5f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 29 Oct 2026 21:54:03 +0100 Subject: [PATCH 02/41] claude-self-run-lease: Self runs borrow the one Claude token over a unix socket bound to the peer's uid, its pidfd and its unit's InvocationID; the connector sets the reservation, settle bills reported per-model usage uncapped, no report bills the reservation (squash of eureka-body/claude-self-run-lease 480cf94..b5f26ef; Soul s1-s3) Co-Authored-By: Claude Opus 5.5 --- crates/claude-connector/src/daemon.rs | 171 ++- crates/claude-connector/src/lease.rs | 729 +++++++++++ crates/claude-connector/src/lib.rs | 16 +- crates/claude-connector/src/main.rs | 29 +- crates/claude-connector/src/unit.rs | 361 +++++ .../claude-connector/tests/config_document.rs | 219 +++- .../tests/fixtures/lease_v1/grant.bin | Bin 0 -> 109 bytes .../tests/fixtures/lease_v1/refusal.bin | Bin 0 -> 104 bytes .../tests/fixtures/lease_v1/request.bin | Bin 0 -> 66 bytes .../tests/fixtures/lease_v1/settle.bin | Bin 0 -> 336 bytes .../tests/fixtures/lease_v1/settled.bin | Bin 0 -> 81 bytes crates/claude-connector/tests/lease.rs | 1166 +++++++++++++++++ crates/model-connector/src/serve.rs | 38 +- crates/model-connector/src/service.rs | 29 +- crates/model-connector/src/service/lease.rs | 691 ++++++++++ 15 files changed, 3421 insertions(+), 28 deletions(-) create mode 100644 crates/claude-connector/src/lease.rs create mode 100644 crates/claude-connector/src/unit.rs create mode 100644 crates/claude-connector/tests/fixtures/lease_v1/grant.bin create mode 100644 crates/claude-connector/tests/fixtures/lease_v1/refusal.bin create mode 100644 crates/claude-connector/tests/fixtures/lease_v1/request.bin create mode 100644 crates/claude-connector/tests/fixtures/lease_v1/settle.bin create mode 100644 crates/claude-connector/tests/fixtures/lease_v1/settled.bin create mode 100644 crates/claude-connector/tests/lease.rs create mode 100644 crates/model-connector/src/service/lease.rs diff --git a/crates/claude-connector/src/daemon.rs b/crates/claude-connector/src/daemon.rs index 674975d..9f549fc 100644 --- a/crates/claude-connector/src/daemon.rs +++ b/crates/claude-connector/src/daemon.rs @@ -6,17 +6,22 @@ use std::collections::{BTreeMap, BTreeSet}; use std::net::{SocketAddr, TcpListener}; use std::path::{Path, PathBuf}; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use std::time::Duration; use cultcache_rs::{CultCache, DatabaseEntry, SingleFileMessagePackBackingStore}; use model_connector::{ - CallerAdmission, CallerConfig, MIN_PAYLOAD_BYTES, ModelPrice, ServeError, ServeLimits, - ServiceError, load_admissions, serve, + CallerAdmission, CallerConfig, MIN_PAYLOAD_BYTES, ModelPrice, ProviderBackend, ServeError, + ServeLimits, ServiceError, TransportService, load_admissions, serve_service, }; +use serde::{Deserialize, Serialize}; use thiserror::Error; use crate::backend::readiness; +#[cfg(target_os = "linux")] +use crate::lease::{LeaseCaller, LeaseLimits, serve_leases, user_uid}; +#[cfg(target_os = "linux")] +use crate::unit::SystemdUnits; use crate::{ ClaudeCodeBackend, ClaudeCodeConfig, ClaudeCodeLimits, CredentialTokenSource, StderrSink, }; @@ -24,6 +29,18 @@ use crate::{ const CONFIG_KEY: &str = "runtime"; const MODEL_PREFIX: &str = "claude-"; +/// A unix account that may lease the subscription token for a Self run, the +/// Self unit (`mind-self@.service`) its process must run in, +/// the caller whose spend window its leases reserve in, and what each lease +/// reserves there. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct LeaseCallerConfig { + pub unix_user: String, + pub caller_id: String, + pub unit_instance: String, + pub reservation_micro_usd: u64, +} + #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] #[cultcache(type = "gamecult.model_connector.claude_connector_config.v1")] pub struct ClaudeConnectorConfig { @@ -72,6 +89,17 @@ pub struct ClaudeConnectorConfig { /// ids exactly as the CLI reports them in its init event. #[cultcache(key = 18)] pub prices: BTreeMap, + /// Where the Self-run lease socket is bound (a systemd RuntimeDirectory), + /// or none when this connector lends no token. + #[cultcache(key = 19)] + pub lease_socket: Option, + /// The accounts allowed to lease, each bound to a configured caller. + /// Empty exactly when `lease_socket` is none. + #[cultcache(key = 20)] + pub lease_callers: Vec, + /// The furthest a lease's expiry may lie from now. + #[cultcache(key = 21)] + pub lease_max_ms: u64, } /// A refusal. It names the field or stage that failed and nothing the @@ -205,9 +233,39 @@ impl ClaudeConnectorConfig { { return Err(ConfigError::Invalid("prices")); } + self.validate_leases(&caller.caller_runtime_id, caller.spend_ceiling_micro_usd)?; Ok(listen) } + fn validate_leases(&self, caller_id: &str, spend_ceiling: u64) -> Result<(), ConfigError> { + let Some(socket) = &self.lease_socket else { + return if self.lease_callers.is_empty() { + Ok(()) + } else { + Err(ConfigError::Invalid("lease_callers")) + }; + }; + if !socket.is_absolute() { + return Err(ConfigError::Invalid("lease_socket")); + } + if self.lease_max_ms == 0 { + return Err(ConfigError::Invalid("lease_max_ms")); + } + let mut users = BTreeSet::new(); + if self.lease_callers.is_empty() + || !self.lease_callers.iter().all(|lease| { + is_trimmed_nonempty(&lease.unix_user) + && lease.caller_id == caller_id + && is_unit_instance(&lease.unit_instance) + && (1..=spend_ceiling).contains(&lease.reservation_micro_usd) + && users.insert(lease.unix_user.as_str()) + }) + { + return Err(ConfigError::Invalid("lease_callers")); + } + Ok(()) + } + fn backend_config(&self) -> ClaudeCodeConfig { ClaudeCodeConfig { claude_binary: self.claude_binary.clone(), @@ -236,6 +294,14 @@ impl ClaudeConnectorConfig { } } +/// The `` of a unit name, as plain as a systemd instance can be. +fn is_unit_instance(text: &str) -> bool { + !text.is_empty() + && text + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_')) +} + fn is_trimmed_nonempty(text: &str) -> bool { !text.is_empty() && text.trim() == text } @@ -394,9 +460,85 @@ fn readiness_of(config: &ClaudeConnectorConfig) -> Result, &CredentialTokenSource::from_environment(backend.token_credential.clone()), ) .map_err(ConfigError::Unusable)?; + if config.lease_socket.is_some() { + lease_callers_of(config)?; + } Ok(admissions) } +/// The accounts allowed to lease, resolved to the uids the kernel will report +/// for their connections. An account the system does not have is not ready. +#[cfg(target_os = "linux")] +fn lease_callers_of(config: &ClaudeConnectorConfig) -> Result, ConfigError> { + config + .lease_callers + .iter() + .map(|lease| { + user_uid(&lease.unix_user) + .map(|uid| LeaseCaller { + uid, + caller_id: lease.caller_id.clone(), + unit_instance: lease.unit_instance.clone(), + reservation_micro_usd: lease.reservation_micro_usd, + }) + .ok_or(ConfigError::Unusable("lease_callers")) + }) + .collect() +} + +/// Peer credentials identify a lease caller only on Linux. +#[cfg(not(target_os = "linux"))] +fn lease_callers_of(_: &ClaudeConnectorConfig) -> Result<(), ConfigError> { + Err(ConfigError::Unusable("lease_socket")) +} + +/// Binds the lease socket (mode 0660; the unit's RuntimeDirectory and group +/// decide who can reach it, and the kernel's peer uid decides who may lease) +/// and serves it beside the RUDP loop. The lease listener is half of the +/// connector: if it stops, so does the process. +#[cfg(target_os = "linux")] +fn start_leases( + config: &ClaudeConnectorConfig, + socket: &Path, + service: Arc>, +) -> Result<(), ConfigError> { + use std::os::unix::fs::{FileTypeExt, PermissionsExt}; + use std::os::unix::net::UnixListener; + + let callers = lease_callers_of(config)?; + // A socket a previous run left is replaced; anything else at the path is not ours. + if std::fs::symlink_metadata(socket).is_ok_and(|found| found.file_type().is_socket()) { + let _ = std::fs::remove_file(socket); + } + let listener = UnixListener::bind(socket).map_err(|_| ConfigError::Listen)?; + std::fs::set_permissions(socket, std::fs::Permissions::from_mode(0o660)) + .map_err(|_| ConfigError::Listen)?; + let limits = LeaseLimits { + socket_timeout: Duration::from_millis(config.socket_timeout_ms), + max_lease_ms: config.lease_max_ms, + max_connections: config.max_connections, + }; + let tokens = Arc::new(CredentialTokenSource::from_environment( + config.token_credential.clone(), + )); + std::thread::Builder::new() + .name("claude-lease-listener".to_string()) + .spawn(move || { + let _ = serve_leases( + &listener, + callers, + limits, + service, + tokens, + Arc::new(SystemdUnits::system()), + ); + eprintln!("claude-connector: the lease listener stopped"); + std::process::exit(1); + }) + .map_err(|_| ConfigError::Serving("lease listener"))?; + Ok(()) +} + /// Whether `serve` would start with this document and this environment. pub fn check_ready(config: &ClaudeConnectorConfig) -> Result<(), ConfigError> { readiness_of(config).map(drop) @@ -413,14 +555,21 @@ pub fn serve_config(path: &Path) -> Result<(), ConfigError> { Box::new(LogSink(std::sync::Mutex::new(std::io::stderr()))), )); let listener = TcpListener::bind(listen).map_err(|_| ConfigError::Listen)?; - serve( - listener, - config.serve_limits(), - admissions, - config.prices.clone(), - &config.replay_store, - backend, - )?; + let service = Arc::new(Mutex::new( + TransportService::open( + &config.replay_store, + backend.provider(), + config.prices.clone(), + admissions, + config.max_expiry_skew_ms, + ) + .map_err(|error| ConfigError::from(ServeError::Service(error)))?, + )); + #[cfg(target_os = "linux")] + if let Some(socket) = &config.lease_socket { + start_leases(&config, socket, service.clone())?; + } + serve_service(listener, config.serve_limits(), service, backend)?; Ok(()) } diff --git a/crates/claude-connector/src/lease.rs b/crates/claude-connector/src/lease.rs new file mode 100644 index 0000000..d8badf3 --- /dev/null +++ b/crates/claude-connector/src/lease.rs @@ -0,0 +1,729 @@ +//! The Self-run credential lease. The connector is the only holder of the +//! subscription token and the one owner of Self-run admission, reservation and +//! the spend ledger (rulings self-runs-through-connector, +//! self-run-credential-lease and self-run-lease-billing-and-binding). A Self +//! run asks over a unix socket for a lease. The connector decides who may ask +//! (the peer's uid, and the systemd unit invocation its process runs in), +//! reserves its own configured amount in the caller's spend window, and only +//! then hands back one pipe fd that holds the token. The run settles with the +//! per-model usage it reports, and the connector bills that at true cost, +//! uncapped; a run that vanishes or lets its lease expire is billed the whole +//! reservation. Nothing the run says about itself decides who it is, what it +//! reserves, or what the plan allows. +//! +//! The token exists in this process only as the value the token source returns +//! and in the pipe. It is not in an environment, an argument, a file, a log or +//! a reply frame. The connector spawns nothing here: the Self run's `claude` +//! stays in the Self run's own unit. + +use std::ffi::CString; +use std::io::{self, Read, Write}; +use std::mem::{size_of, zeroed}; +use std::os::fd::{AsFd, AsRawFd, BorrowedFd, FromRawFd, OwnedFd, RawFd}; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::ptr; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use model_connector::{ + LeaseAdmission, LeaseGrant, LeaseRefusal, LeaseSettlement, LeaseUse, ModelUsage, + TransportService, +}; +use serde::{Deserialize, Serialize}; + +use crate::TokenSource; + +/// The largest frame either side sends or accepts, in bytes. It is wire +/// contract: a Huginn client sizes against it, so it is pinned as a literal. +pub const MAX_LEASE_FRAME_BYTES: usize = 65_536; +const _: () = assert!(MAX_LEASE_FRAME_BYTES == 65_536); + +/// One message of the lease protocol: a u32 big-endian length, then this as a +/// MessagePack named map `{kind, value}`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", content = "value", rename_all = "snake_case")] +pub enum LeaseMessage { + ClaudeLeaseRequest(LeaseRequest), + /// Travels with exactly one SCM_RIGHTS fd: the read end of a pipe that + /// holds the token, its write end closed. + ClaudeLeaseGrant(LeaseGranted), + ClaudeLeaseRefusal(LeaseRefused), + ClaudeLeaseSettle(LeaseSettle), + ClaudeLeaseSettled(LeaseSettled), +} + +/// A request names only when the lease ends. Who asks, and what is reserved, +/// are the connector's to know. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct LeaseRequest { + pub expires_at_unix_ms: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct LeaseGranted { + pub lease_id: String, + pub reserved_micro_usd: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum LeaseRefusalKind { + SpendCeiling, + CostBound, + UnknownCaller, + Expired, + Busy, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct LeaseRefused { + pub kind: LeaseRefusalKind, + pub detail: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct LeaseSettle { + pub lease_id: String, + /// What the run used of each model, as the run reports it. + pub usage: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct LeaseSettled { + pub billed_micro_usd: u64, + /// How far the bill passed the reservation; zero when it did not. + pub overrun_micro_usd: u64, +} + +/// A frame: the length, then the named-map MessagePack of `message`. +pub fn encode_frame(message: &LeaseMessage) -> Vec { + // Every field of the message is plain data, so encoding cannot fail. + let body = rmp_serde::to_vec_named(message).unwrap_or_default(); + let mut frame = Vec::with_capacity(4 + body.len()); + frame.extend_from_slice(&(body.len() as u32).to_be_bytes()); + frame.extend_from_slice(&body); + frame +} + +/// The message in one whole frame, or `None` when the frame is not exactly one +/// well-formed message within the bound. +pub fn decode_frame(frame: &[u8]) -> Option { + let (length, body) = frame.split_first_chunk::<4>()?; + let length = u32::from_be_bytes(*length) as usize; + if length == 0 || length > MAX_LEASE_FRAME_BYTES || body.len() != length { + return None; + } + rmp_serde::from_slice(body).ok() +} + +/// A reader whose waiting can be bounded, so a frame has one deadline across +/// all of its reads. +trait Bounded: Read { + fn within(&mut self, left: Duration) -> io::Result<()>; +} + +impl Bounded for UnixStream { + fn within(&mut self, left: Duration) -> io::Result<()> { + self.set_read_timeout(Some(left)) + } +} + +impl Bounded for &[u8] { + fn within(&mut self, _: Duration) -> io::Result<()> { + Ok(()) + } +} + +/// Fills `buffer` or fails: when `deadline` passes, however the bytes trickle. +fn read_exact_by( + stream: &mut impl Bounded, + buffer: &mut [u8], + deadline: Instant, +) -> io::Result<()> { + let mut filled = 0; + while filled < buffer.len() { + let left = deadline + .checked_duration_since(Instant::now()) + .filter(|left| !left.is_zero()) + .ok_or_else(|| io::Error::from(io::ErrorKind::TimedOut))?; + stream.within(left)?; + match stream.read(&mut buffer[filled..]) { + Ok(0) => return Err(io::Error::from(io::ErrorKind::UnexpectedEof)), + Ok(read) => filled += read, + Err(error) + if matches!( + error.kind(), + io::ErrorKind::Interrupted | io::ErrorKind::WouldBlock + ) => {} + Err(error) => return Err(error), + } + } + Ok(()) +} + +/// One whole frame, complete by `deadline`. +fn read_message(stream: &mut impl Bounded, deadline: Instant) -> io::Result { + let invalid = || io::Error::from(io::ErrorKind::InvalidData); + let mut length = [0_u8; 4]; + read_exact_by(stream, &mut length, deadline)?; + let length = u32::from_be_bytes(length) as usize; + if length == 0 || length > MAX_LEASE_FRAME_BYTES { + return Err(invalid()); + } + let mut body = vec![0_u8; length]; + read_exact_by(stream, &mut body, deadline)?; + rmp_serde::from_slice(&body).map_err(|_| invalid()) +} + +/// The most a pipe takes in one atomic write. +const PIPE_BUF: usize = 4096; + +/// A unix account allowed to lease, the Self unit its process must run in, +/// and the caller whose spend window its leases reserve in. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LeaseCaller { + pub uid: u32, + pub caller_id: String, + /// The `` of `mind-self@.service`. + pub unit_instance: String, + /// What each of this caller's leases reserves, whatever the run asks. + pub reservation_micro_usd: u64, +} + +impl LeaseCaller { + /// The one unit a lease to this caller may be requested from. + pub fn unit(&self) -> String { + format!("mind-self@{}.service", self.unit_instance) + } +} + +/// The systemd unit invocation a process runs in. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PeerUnit { + pub unit: String, + pub invocation_id: String, +} + +/// Which invocation of `unit` a process is running, as the system records it. +/// The lease listener asks this port; `SystemdUnits` answers it in production. +/// `None` when the process is not in `unit` or the system cannot say. The +/// process is named by the handle the kernel gave for the peer, never by a pid. +pub trait PeerUnits: Send + Sync { + fn invocation_in(&self, process: BorrowedFd<'_>, unit: &str) -> Option; +} + +/// The bounds of the lease listener. +#[derive(Debug, Clone, Copy)] +pub struct LeaseLimits { + /// How long a peer has to send its request frame, whole. + pub socket_timeout: Duration, + /// The furthest an expiry may lie from now. + pub max_lease_ms: u64, + pub max_connections: usize, +} + +/// Who is on the other end of a unix stream, as the kernel recorded them at +/// connect. +#[derive(Debug)] +pub struct PeerCredentials { + pub uid: u32, + /// A handle on the peer's process (SO_PEERPIDFD, Linux 6.5 and later). It + /// pins the process's identity: a recycled pid cannot be read through it. + /// `None` when the kernel cannot give one, and then no unit is vouched + /// for. + pub process: Option, +} + +/// The uid and process handle of the process on the other end of `stream`. +pub fn peer_credentials(stream: &UnixStream) -> io::Result { + // SAFETY: `credentials` and `length` are valid for the call, and `length` + // is the size of the buffer getsockopt writes into. + let (result, credentials) = unsafe { + let mut credentials: libc::ucred = zeroed(); + let mut length = size_of::() as libc::socklen_t; + let result = libc::getsockopt( + stream.as_raw_fd(), + libc::SOL_SOCKET, + libc::SO_PEERCRED, + ptr::from_mut(&mut credentials).cast(), + &mut length, + ); + (result, credentials) + }; + if result != 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: as above, with a `c_int` the kernel fills with a new descriptor + // that this function then owns. + let (result, pidfd) = unsafe { + let mut pidfd: libc::c_int = -1; + let mut length = size_of::() as libc::socklen_t; + let result = libc::getsockopt( + stream.as_raw_fd(), + libc::SOL_SOCKET, + libc::SO_PEERPIDFD, + ptr::from_mut(&mut pidfd).cast(), + &mut length, + ); + (result, pidfd) + }; + Ok(PeerCredentials { + uid: credentials.uid, + // SAFETY: on success `pidfd` is a descriptor nothing else owns. + process: (result == 0 && pidfd >= 0).then(|| unsafe { OwnedFd::from_raw_fd(pidfd) }), + }) +} + +/// The uid of the account `name`, if the system has one. +pub fn user_uid(name: &str) -> Option { + let name = CString::new(name).ok()?; + let mut buffer = vec![0 as libc::c_char; 16 * 1024]; + // SAFETY: every pointer is valid for the call and `buffer` is as long as + // the length passed; `found` is read only when the call reports success. + unsafe { + let mut entry: libc::passwd = zeroed(); + let mut found: *mut libc::passwd = ptr::null_mut(); + let result = libc::getpwnam_r( + name.as_ptr(), + &mut entry, + buffer.as_mut_ptr(), + buffer.len(), + &mut found, + ); + (result == 0 && !found.is_null()).then_some(entry.pw_uid) + } +} + +/// A pipe whose read end holds `token` and whose write end is closed, so a +/// reader gets the token and then end of file. The token is smaller than +/// PIPE_BUF, so the write is whole and cannot block. +fn token_pipe(token: &[u8]) -> io::Result { + if token.len() > PIPE_BUF { + return Err(io::Error::from(io::ErrorKind::InvalidInput)); + } + let mut ends = [0 as RawFd; 2]; + // SAFETY: `ends` has room for the two descriptors pipe2 writes. + if unsafe { libc::pipe2(ends.as_mut_ptr(), libc::O_CLOEXEC) } != 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: pipe2 just returned these two descriptors and nothing else owns them. + let (read, write) = unsafe { (OwnedFd::from_raw_fd(ends[0]), OwnedFd::from_raw_fd(ends[1])) }; + // SAFETY: the pointer and length describe `token`. + let written = unsafe { libc::write(write.as_raw_fd(), token.as_ptr().cast(), token.len()) }; + if usize::try_from(written).ok() != Some(token.len()) { + return Err(io::Error::last_os_error()); + } + drop(write); + Ok(read) +} + +/// Writes `frame` to `stream` with `fd` attached as SCM_RIGHTS. +fn send_with_fd(stream: &UnixStream, frame: &[u8], fd: RawFd) -> io::Result<()> { + let fd_size = size_of::() as u32; + // SAFETY: CMSG_SPACE is a pure size computation. + let space = unsafe { libc::CMSG_SPACE(fd_size) } as usize; + // u64 elements align the control buffer for a cmsghdr. + let mut control = vec![0_u64; space.div_ceil(size_of::())]; + let mut vector = libc::iovec { + iov_base: frame.as_ptr().cast_mut().cast(), + iov_len: frame.len(), + }; + // SAFETY: msghdr is plain data for which all-zero is valid; the pointers + // set below outlive the sendmsg call, and the cmsg header written lies + // inside `control`, which CMSG_SPACE sized for one descriptor. + let sent = unsafe { + let mut message: libc::msghdr = zeroed(); + message.msg_iov = &mut vector; + message.msg_iovlen = 1; + message.msg_control = control.as_mut_ptr().cast(); + message.msg_controllen = space as _; + let header = libc::CMSG_FIRSTHDR(&message); + (*header).cmsg_level = libc::SOL_SOCKET; + (*header).cmsg_type = libc::SCM_RIGHTS; + (*header).cmsg_len = libc::CMSG_LEN(fd_size) as _; + ptr::copy_nonoverlapping( + ptr::from_ref(&fd).cast::(), + libc::CMSG_DATA(header), + size_of::(), + ); + loop { + let sent = libc::sendmsg(stream.as_raw_fd(), &message, libc::MSG_NOSIGNAL); + if sent >= 0 || io::Error::last_os_error().kind() != io::ErrorKind::Interrupted { + break sent; + } + } + }; + let Ok(sent) = usize::try_from(sent) else { + return Err(io::Error::last_os_error()); + }; + // The descriptor travels with the first byte; any rest is plain data. + let mut stream = stream; + stream.write_all(&frame[sent..]) +} + +fn unix_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| { + u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX) + }) +} + +struct Permit(Arc); + +impl Permit { + fn acquire(active: &Arc, limit: usize) -> Option { + active + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |held| { + (held < limit).then_some(held + 1) + }) + .ok() + .map(|_| Self(active.clone())) + } +} + +impl Drop for Permit { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::AcqRel); + } +} + +struct Lender { + callers: Vec, + limits: LeaseLimits, + service: Arc>, + tokens: Arc, + units: Arc, +} + +/// Serves leases on `listener` until it fails. Each connection is one lease: +/// request, then grant or refusal, then (after a grant) settle. A caller is +/// the account whose uid the kernel reports for the connection, and only a +/// process in that caller's Self unit may lease, once per unit invocation; no +/// field of any message names either. +pub fn serve_leases( + listener: &UnixListener, + callers: Vec, + limits: LeaseLimits, + service: Arc>, + tokens: Arc, + units: Arc, +) -> io::Result<()> { + let lender = Arc::new(Lender { + callers, + limits, + service, + tokens, + units, + }); + let active = Arc::new(AtomicUsize::new(0)); + loop { + let (stream, _) = listener.accept()?; + let Some(permit) = Permit::acquire(&active, limits.max_connections) else { + continue; + }; + let lender = lender.clone(); + thread::Builder::new() + .name("claude-lease".to_string()) + .spawn(move || { + let _permit = permit; + // A connection that fails is dropped; the lease it may have + // held is settled inside `lend`. + let _ = lender.lend(stream); + })?; + } +} + +impl Lender { + fn lend(&self, mut stream: UnixStream) -> io::Result<()> { + stream.set_write_timeout(Some(self.limits.socket_timeout))?; + let peer = peer_credentials(&stream)?; + let request_deadline = Instant::now() + self.limits.socket_timeout; + let LeaseMessage::ClaudeLeaseRequest(request) = + read_message(&mut stream, request_deadline)? + else { + return Err(io::Error::from(io::ErrorKind::InvalidData)); + }; + let Some(caller) = self.callers.iter().find(|caller| caller.uid == peer.uid) else { + return refuse( + &mut stream, + LeaseRefusalKind::UnknownCaller, + "this account may not lease", + ); + }; + // The unit and its invocation come from the system's record of the + // peer's process, never from the peer. + let Some(invocation) = peer + .process + .as_ref() + .and_then(|process| self.units.invocation_in(process.as_fd(), &caller.unit())) + .filter(|found| found.unit == caller.unit()) + else { + return refuse( + &mut stream, + LeaseRefusalKind::UnknownCaller, + "this process is not in the Self unit", + ); + }; + let now = unix_ms(); + if request.expires_at_unix_ms <= now + || request.expires_at_unix_ms > now.saturating_add(self.limits.max_lease_ms) + { + return refuse( + &mut stream, + LeaseRefusalKind::Expired, + "the expiry is not in the future or is past the lease bound", + ); + } + // The reservation is claimed in the ledger before the token is read, + // let alone lent. A ledger that fails closes the connection with no + // answer and no lease. + let admission = self + .service + .lock() + .map_err(|_| io::Error::other("service lock"))? + .begin_lease( + &caller.caller_id, + &invocation.invocation_id, + caller.reservation_micro_usd, + request.expires_at_unix_ms, + now, + ) + .map_err(|_| io::Error::other("ledger"))?; + let grant = match admission { + LeaseAdmission::Granted(grant) => grant, + LeaseAdmission::Refused(why) => { + let (kind, detail) = match why { + LeaseRefusal::SpendCeiling => ( + LeaseRefusalKind::SpendCeiling, + "the window cannot hold this reservation", + ), + LeaseRefusal::CostBound => ( + LeaseRefusalKind::CostBound, + "the reservation must be above zero", + ), + LeaseRefusal::UnknownCaller => ( + LeaseRefusalKind::UnknownCaller, + "this account is bound to no admitted caller", + ), + LeaseRefusal::Busy => ( + LeaseRefusalKind::Busy, + "a lease is open for this caller or this unit invocation leased", + ), + }; + return refuse(&mut stream, kind, detail); + } + }; + self.hold(stream, caller, &grant, request.expires_at_unix_ms) + } + + /// Everything after a reservation exists: lend the token, then settle the + /// lease exactly once whatever happens. + fn hold( + &self, + mut stream: UnixStream, + caller: &LeaseCaller, + grant: &LeaseGrant, + expires_at_unix_ms: u64, + ) -> io::Result<()> { + // A token that cannot be read or piped was never lent: nothing was + // used, so nothing is billed. + let Some(read_end) = self + .tokens + .read() + .and_then(|token| token_pipe(token.as_bytes()).ok()) + else { + self.settle(caller, grant, LeaseUse::NeverLent)?; + return Ok(()); + }; + let frame = encode_frame(&LeaseMessage::ClaudeLeaseGrant(LeaseGranted { + lease_id: grant.lease_id.clone(), + reserved_micro_usd: grant.reserved_micro_usd, + })); + let sent = send_with_fd(&stream, &frame, read_end.as_raw_fd()); + drop(read_end); + if sent.is_err() { + // The descriptor may have been delivered: the reservation is billed. + self.settle(caller, grant, LeaseUse::Reservation)?; + return Ok(()); + } + let outcome = self.await_settle(&mut stream, grant, expires_at_unix_ms); + match outcome { + Some(settle) => { + let settlement = self.settle(caller, grant, LeaseUse::Reported(&settle.usage))?; + let _ = stream.write_all(&encode_frame(&LeaseMessage::ClaudeLeaseSettled( + LeaseSettled { + billed_micro_usd: settlement.billed_micro_usd, + overrun_micro_usd: settlement.overrun_micro_usd, + }, + ))); + } + None => { + self.settle(caller, grant, LeaseUse::Reservation)?; + } + } + Ok(()) + } + + /// The run's settle, if its whole frame arrives for this lease before the + /// lease expires. Anything else (end of file, silence, a frame still + /// trickling at the expiry, a malformed frame, the wrong lease) is no + /// report. + fn await_settle( + &self, + stream: &mut UnixStream, + grant: &LeaseGrant, + expires_at_unix_ms: u64, + ) -> Option { + let remaining = expires_at_unix_ms.saturating_sub(unix_ms()); + let deadline = Instant::now() + Duration::from_millis(remaining); + match read_message(stream, deadline).ok()? { + LeaseMessage::ClaudeLeaseSettle(settle) if settle.lease_id == grant.lease_id => { + Some(settle) + } + _ => None, + } + } + + fn settle( + &self, + caller: &LeaseCaller, + grant: &LeaseGrant, + usage: LeaseUse<'_>, + ) -> io::Result { + self.service + .lock() + .map_err(|_| io::Error::other("service lock"))? + .settle_lease(&caller.caller_id, &grant.lease_id, usage) + .map_err(|_| io::Error::other("ledger")) + } +} + +fn refuse(stream: &mut UnixStream, kind: LeaseRefusalKind, detail: &str) -> io::Result<()> { + stream.write_all(&encode_frame(&LeaseMessage::ClaudeLeaseRefusal( + LeaseRefused { + kind, + detail: detail.to_string(), + }, + ))) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A settle frame whose body is exactly `body_bytes` long. + fn settle_frame_of(body_bytes: usize) -> Vec { + let frame_with = |pad: usize| { + encode_frame(&LeaseMessage::ClaudeLeaseSettle(LeaseSettle { + lease_id: "x".repeat(pad), + usage: Vec::new(), + })) + }; + // The string header is the same width between 256 and 65535 bytes. + let base = frame_with(300).len() - 4 - 300; + frame_with(body_bytes - base) + } + + fn within(frame: &[u8]) -> io::Result { + read_message(&mut &frame[..], Instant::now() + Duration::from_secs(5)) + } + + #[test] + fn the_frame_bound_is_sixty_five_thousand_five_hundred_thirty_six() { + assert_eq!(MAX_LEASE_FRAME_BYTES, 65_536); + } + + #[test] + fn a_frame_is_one_whole_message_within_the_bound() { + let at_bound = settle_frame_of(MAX_LEASE_FRAME_BYTES); + assert_eq!(at_bound.len(), 4 + MAX_LEASE_FRAME_BYTES); + assert!(decode_frame(&at_bound).is_some()); + let past_bound = settle_frame_of(MAX_LEASE_FRAME_BYTES + 1); + assert!(decode_frame(&past_bound).is_none()); + let mut trailing = at_bound.clone(); + trailing.push(0); + assert!(decode_frame(&trailing).is_none()); + assert!(decode_frame(&at_bound[..at_bound.len() - 1]).is_none()); + assert!(decode_frame(&[0, 0, 0, 0]).is_none()); + assert!(decode_frame(&[0, 0]).is_none()); + } + + #[test] + fn a_stream_frame_over_the_bound_is_refused_before_it_is_read() { + let past_bound = settle_frame_of(MAX_LEASE_FRAME_BYTES + 1); + assert!(within(&past_bound).is_err()); + let at_bound = settle_frame_of(MAX_LEASE_FRAME_BYTES); + assert!(within(&at_bound).is_ok()); + assert!(within(&[0_u8, 0, 0, 0]).is_err()); + // Short of the length it announced is end of file, not a message. + assert!(within(&at_bound[..at_bound.len() - 1]).is_err()); + } + + #[test] + fn a_frame_that_trickles_past_its_deadline_is_refused_however_small_each_read() { + let (mut reader, mut writer) = UnixStream::pair().unwrap(); + let frame = encode_frame(&LeaseMessage::ClaudeLeaseSettle(LeaseSettle { + lease_id: "lease:x".to_string(), + usage: Vec::new(), + })); + let trickle = thread::spawn(move || { + for byte in frame { + if writer.write_all(&[byte]).is_err() { + return; + } + thread::sleep(Duration::from_millis(60)); + } + }); + // Every read lands well inside any per-read timeout; the frame as a + // whole does not. + let started = Instant::now(); + let outcome = read_message(&mut reader, started + Duration::from_millis(400)); + assert_eq!(outcome.unwrap_err().kind(), io::ErrorKind::TimedOut); + assert!(started.elapsed() < Duration::from_millis(1_000)); + drop(reader); + trickle.join().unwrap(); + } + + /// A reader that fails the way a reset connection does. + struct Reset; + + impl Read for Reset { + fn read(&mut self, _: &mut [u8]) -> io::Result { + Err(io::Error::from(io::ErrorKind::ConnectionReset)) + } + } + + impl Bounded for Reset { + fn within(&mut self, _: Duration) -> io::Result<()> { + Ok(()) + } + } + + #[test] + fn a_read_that_fails_ends_the_frame_at_once_not_at_the_deadline() { + let started = Instant::now(); + let outcome = read_message(&mut Reset, started + Duration::from_secs(30)); + assert_eq!(outcome.unwrap_err().kind(), io::ErrorKind::ConnectionReset); + assert!(started.elapsed() < Duration::from_secs(1)); + } + + #[test] + fn a_frame_that_arrives_in_pieces_within_its_deadline_is_read() { + let (mut reader, mut writer) = UnixStream::pair().unwrap(); + let frame = encode_frame(&LeaseMessage::ClaudeLeaseRequest(LeaseRequest { + expires_at_unix_ms: 7, + })); + let expected = decode_frame(&frame); + let pieces = thread::spawn(move || { + for part in frame.chunks(3) { + writer.write_all(part).unwrap(); + thread::sleep(Duration::from_millis(10)); + } + }); + let read = read_message(&mut reader, Instant::now() + Duration::from_secs(5)).unwrap(); + assert_eq!(Some(read), expected); + pieces.join().unwrap(); + } +} diff --git a/crates/claude-connector/src/lib.rs b/crates/claude-connector/src/lib.rs index 327a078..22ff46f 100644 --- a/crates/claude-connector/src/lib.rs +++ b/crates/claude-connector/src/lib.rs @@ -8,18 +8,32 @@ mod config; mod daemon; mod handle; mod launch; +#[cfg(target_os = "linux")] +mod lease; mod process; mod quota; mod stream; +#[cfg(target_os = "linux")] +mod unit; pub use backend::{ClaudeCodeBackend, CredentialTokenSource, StderrSink, TokenSource}; pub use config::{ClaudeCodeConfig, ClaudeCodeLimits}; -pub use daemon::{ClaudeConnectorConfig, ConfigError, check_ready, load_config, write_config}; +pub use daemon::{ + ClaudeConnectorConfig, ConfigError, LeaseCallerConfig, check_ready, load_config, write_config, +}; #[cfg(unix)] pub use daemon::{check_prompt_dir, serve_config}; pub use handle::{FsSessionStore, SessionStore}; pub use launch::ChildEnv; +#[cfg(target_os = "linux")] +pub use lease::{ + LeaseCaller, LeaseGranted, LeaseLimits, LeaseMessage, LeaseRefusalKind, LeaseRefused, + LeaseRequest, LeaseSettle, LeaseSettled, MAX_LEASE_FRAME_BYTES, PeerCredentials, PeerUnit, + PeerUnits, decode_frame, encode_frame, peer_credentials, serve_leases, user_uid, +}; #[cfg(unix)] pub use process::StdProcessPort; pub use process::{ProcessPort, RunningChild, StdoutRead}; pub use quota::QuotaCell; +#[cfg(target_os = "linux")] +pub use unit::SystemdUnits; diff --git a/crates/claude-connector/src/main.rs b/crates/claude-connector/src/main.rs index c58e7f5..0ea296f 100644 --- a/crates/claude-connector/src/main.rs +++ b/crates/claude-connector/src/main.rs @@ -13,7 +13,7 @@ use std::path::PathBuf; use std::process::ExitCode; use claude_connector::{ - ClaudeConnectorConfig, ConfigError, check_ready, load_config, write_config, + ClaudeConnectorConfig, ConfigError, LeaseCallerConfig, check_ready, load_config, write_config, }; use model_connector::{CallerConfig, ModelPrice, Rates, frame_bytes_for_payload}; @@ -76,9 +76,10 @@ struct Flags { single: BTreeMap<&'static str, String>, models: Vec, prices: Vec, + lease_users: Vec, } -const SINGLE: [&str; 22] = [ +const SINGLE: [&str; 26] = [ "--listen", "--replay-store", "--claude-binary", @@ -101,6 +102,10 @@ const SINGLE: [&str; 22] = [ "--max-payload-bytes", "--max-output-tokens", "--socket-timeout-ms", + "--lease-socket", + "--lease-max-ms", + "--lease-unit-instance", + "--lease-reservation-micro-usd", ]; impl Flags { @@ -109,6 +114,7 @@ impl Flags { single: BTreeMap::new(), models: Vec::new(), prices: Vec::new(), + lease_users: Vec::new(), }; let mut args = args.iter(); while let Some(&name) = args.next() { @@ -118,6 +124,7 @@ impl Flags { match name { "--allowed-model" => flags.models.push(value.to_string()), "--price" => flags.prices.push(value.to_string()), + "--lease-user" => flags.lease_users.push(value.to_string()), _ => { let Some(known) = SINGLE.iter().find(|known| **known == name) else { return Err(Failure::Flags); @@ -206,6 +213,19 @@ fn init(path: &str, args: &[&str]) -> Result<(), Failure> { return Err(Failure::Malformed("--price")); } } + let caller_id = flags.text("--caller-id")?; + let lease_callers = flags + .lease_users + .iter() + .map(|unix_user| { + Ok(LeaseCallerConfig { + unix_user: unix_user.clone(), + caller_id: caller_id.clone(), + unit_instance: flags.text("--lease-unit-instance")?, + reservation_micro_usd: flags.number("--lease-reservation-micro-usd")?, + }) + }) + .collect::, Failure>>()?; let config = ClaudeConnectorConfig { listen: flags.text("--listen")?, replay_store: flags.path("--replay-store")?, @@ -226,7 +246,7 @@ fn init(path: &str, args: &[&str]) -> Result<(), Failure> { socket_timeout_ms: flags.number_or("--socket-timeout-ms", 10_000)?, max_expiry_skew_ms: 300_000, callers: vec![CallerConfig { - caller_runtime_id: flags.text("--caller-id")?, + caller_runtime_id: caller_id.clone(), connection_key_file: flags.path("--connection-key-file")?, connection_key_epoch: flags.number("--connection-key-epoch")?, allowed_models: flags.models.clone(), @@ -237,6 +257,9 @@ fn init(path: &str, args: &[&str]) -> Result<(), Failure> { spend_window_ms: flags.number("--spend-window-ms")?, }], prices, + lease_socket: flags.single.get("--lease-socket").map(PathBuf::from), + lease_callers, + lease_max_ms: flags.number_or("--lease-max-ms", 3_600_000)?, }; write_config(path.as_ref(), &config)?; Ok(()) diff --git a/crates/claude-connector/src/unit.rs b/crates/claude-connector/src/unit.rs new file mode 100644 index 0000000..1ab8dc9 --- /dev/null +++ b/crates/claude-connector/src/unit.rs @@ -0,0 +1,361 @@ +//! Which systemd unit invocation a process runs in, from the system's own +//! record: the process's cgroup says the unit, and the service manager says +//! that unit's current InvocationID. The lease listener asks this through the +//! `PeerUnits` port; the answer never comes from the peer. +//! +//! This is the one place the connector asks systemd anything. It runs +//! `systemctl show` for one read-only property of one unit and takes nothing +//! from the output but a 32-digit hex id. + +use std::fs; +use std::io::Read; +use std::os::fd::{AsRawFd, BorrowedFd}; +use std::path::PathBuf; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +use crate::lease::{PeerUnit, PeerUnits}; + +/// Answers `PeerUnits` from `//cgroup` and `systemctl show`. +pub struct SystemdUnits { + proc_root: PathBuf, + systemctl: PathBuf, + /// How long the manager has to answer before the lease is refused. + deadline: Duration, +} + +impl SystemdUnits { + /// The running system's `/proc` and `systemctl`. + pub fn system() -> Self { + Self { + proc_root: PathBuf::from("/proc"), + systemctl: PathBuf::from("/usr/bin/systemctl"), + deadline: Duration::from_secs(5), + } + } + + /// The pid a process handle names now, from the handle's own record. The + /// handle holds the process's identity, so the number cannot have been + /// recycled; a process that has exited reads as `-1`. + fn pid_of(&self, process: BorrowedFd<'_>) -> Option { + let info = fs::read_to_string( + self.proc_root + .join("self/fdinfo") + .join(process.as_raw_fd().to_string()), + ) + .ok()?; + info.lines() + .find_map(|line| line.strip_prefix("Pid:"))? + .trim() + .parse() + .ok() + } + + fn unit_in_cgroup(&self, pid: u32) -> Option { + let cgroup = fs::read_to_string(self.proc_root.join(pid.to_string()).join("cgroup")); + unit_of_cgroup(&cgroup.ok()?) + } + + /// `systemctl show` of `unit`'s InvocationID, killed and refused if it has + /// not exited by the deadline. + fn invocation_of(&self, unit: &str) -> Option { + let mut child = Command::new(&self.systemctl) + .env_clear() + .args(["show", "--property=InvocationID", "--value", "--", unit]) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .ok()?; + let give_up = Instant::now() + self.deadline; + let status = loop { + match child.try_wait() { + Ok(Some(status)) => break status, + Ok(None) if Instant::now() < give_up => { + std::thread::sleep(Duration::from_millis(10)); + } + _ => { + let _ = child.kill(); + let _ = child.wait(); + return None; + } + } + }; + if !status.success() { + return None; + } + // The id is 33 bytes; anything past a small bound is not an answer. + let mut output = Vec::new(); + child + .stdout + .take()? + .take(256) + .read_to_end(&mut output) + .ok()?; + invocation_id_of(std::str::from_utf8(&output).ok()?) + } +} + +/// The service unit a cgroup file puts its process in: the last component of +/// the unified-hierarchy path, if it names a `.service`. A process in a +/// sub-cgroup, a scope, a slice or the root is in no service unit this +/// recognises. +pub fn unit_of_cgroup(cgroup: &str) -> Option { + let path = cgroup.lines().find_map(|line| line.strip_prefix("0::"))?; + let unit = path.rsplit('/').next()?; + (unit.len() > ".service".len() && unit.ends_with(".service")).then(|| unit.to_string()) +} + +/// An InvocationID as `systemctl show --value` prints it: 32 lowercase hex +/// digits and a newline. Anything else (including the empty answer of a unit +/// that is not running) is not an invocation. +pub fn invocation_id_of(output: &str) -> Option { + let id = output.strip_suffix('\n').unwrap_or(output); + (id.len() == 32 && id.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f'))) + .then(|| id.to_string()) +} + +impl PeerUnits for SystemdUnits { + fn invocation_in(&self, process: BorrowedFd<'_>, unit: &str) -> Option { + let pid = self.pid_of(process)?; + // A process in any other unit is not asked about. + if self.unit_in_cgroup(pid).as_deref() != Some(unit) { + return None; + } + let invocation_id = self.invocation_of(unit)?; + // The same process must still be in that unit now: one that left (or + // died) while the manager was asked is not vouched for. + (self.pid_of(process) == Some(pid) && self.unit_in_cgroup(pid).as_deref() == Some(unit)) + .then(|| PeerUnit { + unit: unit.to_string(), + invocation_id, + }) + } +} + +#[cfg(test)] +mod tests { + use std::os::fd::{AsFd, AsRawFd, OwnedFd}; + + use super::*; + + #[test] + fn a_service_unit_is_the_last_cgroup_component() { + let service = "0::/system.slice/system-mind\\x2dself.slice/mind-self@huginn.service\n"; + assert_eq!( + unit_of_cgroup(service).as_deref(), + Some("mind-self@huginn.service") + ); + // Several hierarchies listed: the unified one decides. + let hybrid = "12:cpu:/foo\n0::/system.slice/mind-self@a.service\n"; + assert_eq!( + unit_of_cgroup(hybrid).as_deref(), + Some("mind-self@a.service") + ); + } + + #[test] + fn a_process_outside_a_service_unit_has_none() { + for cgroup in [ + "0::/\n", + "0::/user.slice/user-1000.slice/session-3.scope\n", + "0::/user.slice/user-1000.slice/user@1000.service/app.slice/x.service/child\n", + "0::/system.slice/mind-self@huginn.service/sub\n", + "0::/system.slice/.service\n", + "1:name=systemd:/system.slice/a.service\n", + "", + ] { + assert_eq!(unit_of_cgroup(cgroup), None, "{cgroup:?}"); + } + } + + #[test] + fn an_invocation_id_is_exactly_32_lowercase_hex_digits() { + let id = "0123456789abcdef0123456789abcdef"; + assert_eq!(invocation_id_of(&format!("{id}\n")).as_deref(), Some(id)); + assert_eq!(invocation_id_of(id).as_deref(), Some(id)); + for bad in [ + "", + "\n", + "0123456789ABCDEF0123456789ABCDEF\n", + "0123456789abcdef0123456789abcde\n", + "0123456789abcdef0123456789abcdef0\n", + "0123456789abcdef0123456789abcdeg\n", + "0123456789abcdef0123456789abcdef\n\n", + ] { + assert_eq!(invocation_id_of(bad), None, "{bad:?}"); + } + } + + const ID: &str = "0123456789abcdef0123456789abcdef"; + const UNIT: &str = "mind-self@eureka.service"; + + /// A fake `/proc` with one process in `cgroup`, and a fake `systemctl` + /// that records its arguments, runs `side_effect`, and exits `status` + /// after printing `output`. + struct Fake { + dir: PathBuf, + units: SystemdUnits, + // A script being written while another test forks is "text file + // busy" when it is run: fakes take turns. + _turn: std::sync::MutexGuard<'static, ()>, + } + + static TURN: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + impl Fake { + fn new(name: &str, cgroup: &str, output: &str, status: i32, side_effect: &str) -> Self { + use std::os::unix::fs::PermissionsExt; + let turn = TURN + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let dir = std::env::temp_dir().join(format!("unit-{}-{name}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let side_effect = side_effect.replace("{d}", &dir.display().to_string()); + fs::create_dir_all(dir.join("proc/42")).unwrap(); + fs::write(dir.join("proc/42/cgroup"), cgroup).unwrap(); + let script = dir.join("systemctl"); + fs::write( + &script, + format!( + "#!/bin/sh\nprintf '%s' \"$*\" > {d}/args\n{side_effect}\nprintf '%s' '{output}'\nexit {status}\n", + d = dir.display() + ), + ) + .unwrap(); + fs::set_permissions(&script, fs::Permissions::from_mode(0o755)).unwrap(); + let units = SystemdUnits { + proc_root: dir.join("proc"), + systemctl: script, + deadline: Duration::from_secs(5), + }; + Self { + dir, + units, + _turn: turn, + } + } + + /// A process handle that names process `pid`, as the kernel's own + /// record of the handle says (`-1` once the process has exited). + fn process(&self, pid: i64) -> OwnedFd { + let handle: OwnedFd = fs::File::open("/dev/null").unwrap().into(); + let info = self.dir.join("proc/self/fdinfo"); + fs::create_dir_all(&info).unwrap(); + fs::write( + info.join(handle.as_raw_fd().to_string()), + format!("pos:\t0\nflags:\t02000002\nPid:\t{pid}\nNSpid:\t{pid}\n"), + ) + .unwrap(); + handle + } + + fn invocation_in(&self, pid: i64, unit: &str) -> Option { + self.units.invocation_in(self.process(pid).as_fd(), unit) + } + + fn args(&self) -> String { + fs::read_to_string(self.dir.join("args")).unwrap() + } + } + + impl Drop for Fake { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.dir); + } + } + + fn eureka_cgroup() -> String { + format!("0::/system.slice/{UNIT}\n") + } + + #[test] + fn a_process_in_the_unit_is_that_unit_at_the_invocation_the_manager_reports() { + let fake = Fake::new("ok", &eureka_cgroup(), &format!("{ID}\n"), 0, ""); + assert_eq!( + fake.invocation_in(42, UNIT), + Some(PeerUnit { + unit: UNIT.to_string(), + invocation_id: ID.to_string() + }) + ); + // The manager is asked for exactly the invocation of that unit. + assert_eq!( + fake.args(), + format!("show --property=InvocationID --value -- {UNIT}") + ); + // A handle on a process that has exited, or on one with no cgroup + // record, is in no unit. + assert_eq!(fake.invocation_in(-1, UNIT), None); + assert_eq!(fake.invocation_in(43, UNIT), None); + } + + #[test] + fn a_process_outside_the_unit_is_not_asked_about() { + let scope = Fake::new("scope", "0::/user.slice/session-3.scope\n", ID, 0, ""); + assert_eq!(scope.invocation_in(42, UNIT), None); + assert!(!scope.dir.join("args").exists()); + drop(scope); + // Another service, even a real one, is not the caller's unit: the + // manager is never spawned for it. + let other = Fake::new("other", "0::/system.slice/ssh.service\n", ID, 0, ""); + assert_eq!(other.invocation_in(42, UNIT), None); + assert!(!other.dir.join("args").exists()); + } + + #[test] + fn a_manager_that_fails_or_names_no_invocation_vouches_for_nothing() { + let failed = Fake::new("fail", &eureka_cgroup(), ID, 1, ""); + assert_eq!(failed.invocation_in(42, UNIT), None); + drop(failed); + let not_running = Fake::new("idle", &eureka_cgroup(), "\n", 0, ""); + assert_eq!(not_running.invocation_in(42, UNIT), None); + let missing = SystemdUnits { + proc_root: not_running.dir.join("proc"), + systemctl: not_running.dir.join("no-such-systemctl"), + deadline: Duration::from_secs(5), + }; + assert_eq!( + missing.invocation_in(not_running.process(42).as_fd(), UNIT), + None + ); + } + + #[test] + fn a_manager_that_does_not_answer_in_time_is_killed_and_the_lease_refused() { + let mut fake = Fake::new("hung", &eureka_cgroup(), ID, 0, "exec sleep 30"); + fake.units.deadline = Duration::from_millis(200); + let started = Instant::now(); + assert_eq!(fake.invocation_in(42, UNIT), None); + assert!(started.elapsed() < Duration::from_secs(5)); + } + + #[test] + fn a_process_that_left_its_unit_while_the_manager_was_asked_is_not_vouched_for() { + let fake = Fake::new( + "moves", + &eureka_cgroup(), + ID, + 0, + "printf '0::/system.slice/other.service\n' > {d}/proc/42/cgroup", + ); + assert_eq!(fake.invocation_in(42, UNIT), None); + } + + #[test] + fn a_process_that_exited_while_the_manager_was_asked_is_not_vouched_for() { + let fake = Fake::new("dies", &eureka_cgroup(), ID, 0, ""); + let handle = fake.process(42); + let side = fake + .dir + .join("proc/self/fdinfo") + .join(handle.as_raw_fd().to_string()); + let mut script = fs::read_to_string(fake.dir.join("systemctl")).unwrap(); + script = script.replace( + "exit 0", + &format!("printf 'Pid:\t-1\n' > {}\nexit 0", side.display()), + ); + fs::write(fake.dir.join("systemctl"), script).unwrap(); + assert_eq!(fake.units.invocation_in(handle.as_fd(), UNIT), None); + } +} diff --git a/crates/claude-connector/tests/config_document.rs b/crates/claude-connector/tests/config_document.rs index 141f471..80285a2 100644 --- a/crates/claude-connector/tests/config_document.rs +++ b/crates/claude-connector/tests/config_document.rs @@ -8,7 +8,9 @@ use std::process::{Command, Output}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; -use claude_connector::{ClaudeConnectorConfig, ConfigError, load_config, write_config}; +use claude_connector::{ + ClaudeConnectorConfig, ConfigError, LeaseCallerConfig, load_config, write_config, +}; use cultcache_rs::{ CacheBackingStore, CultCache, CultCacheEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, @@ -92,6 +94,14 @@ fn valid() -> ClaudeConnectorConfig { spend_window_ms: 3_600_000, }], prices: BTreeMap::from([(HAIKU.to_string(), price()), (OPUS.to_string(), price())]), + lease_socket: Some("/run/claude-connector/lease.sock".into()), + lease_callers: vec![LeaseCallerConfig { + unix_user: "mind-eureka-self".to_string(), + caller_id: "persona-a-caller".to_string(), + unit_instance: "eureka".to_string(), + reservation_micro_usd: 5_000_000, + }], + lease_max_ms: 3_600_000, } } @@ -989,3 +999,210 @@ fn a_private_regular_file_is_not_a_prompt_dir() { assert!(claude_connector::check_prompt_dir(&file).is_err()); assert!(claude_connector::check_prompt_dir(&scratch.path("absent")).is_err()); } + +#[test] +fn leases_are_configured_together_or_not_at_all() { + let cases: Vec<(&str, Change)> = vec![ + ( + "lease_socket", + Box::new(|c| c.lease_socket = Some("lease.sock".into())), + ), + ("lease_callers", Box::new(|c| c.lease_callers.clear())), + ("lease_callers", Box::new(|c| c.lease_socket = None)), + ( + "lease_callers", + Box::new(|c| c.lease_callers[0].caller_id = "another-caller".to_string()), + ), + ( + "lease_callers", + Box::new(|c| c.lease_callers[0].unix_user = " ".to_string()), + ), + ( + "lease_callers", + Box::new(|c| { + let again = c.lease_callers[0].clone(); + c.lease_callers.push(again); + }), + ), + ("lease_max_ms", Box::new(|c| c.lease_max_ms = 0)), + ( + "lease_callers", + Box::new(|c| c.lease_callers[0].reservation_micro_usd = 0), + ), + ( + "lease_callers", + Box::new(|c| c.lease_callers[0].reservation_micro_usd = 50_000_001), + ), + ( + "lease_callers", + Box::new(|c| c.lease_callers[0].unit_instance = String::new()), + ), + ( + "lease_callers", + Box::new(|c| c.lease_callers[0].unit_instance = "a/b".to_string()), + ), + ]; + for (field, change) in cases { + let mut config = valid(); + change(&mut config); + assert_eq!(refused(&config), field); + } + // The whole ceiling is a reservation the caller can hold. + let mut whole = valid(); + whole.lease_callers[0].reservation_micro_usd = 50_000_000; + whole.validate().unwrap(); + let mut none = valid(); + none.lease_socket = None; + none.lease_callers.clear(); + none.validate().unwrap(); +} + +#[test] +fn config_init_binds_each_lease_user_to_the_configured_caller() { + let scratch = Scratch::new(); + let made = init( + &scratch, + &init_flags( + &scratch, + &[ + "--lease-socket", + "/run/claude-connector/lease.sock", + "--lease-user", + "mind-eureka-self", + "--lease-max-ms", + "120000", + "--lease-unit-instance", + "eureka", + "--lease-reservation-micro-usd", + "5000000", + ], + ), + ); + assert!(made.status.success(), "{}", shown(&made)); + let config = load_config(&scratch.path("connector.cc")).unwrap(); + assert_eq!( + config.lease_callers, + [LeaseCallerConfig { + unix_user: "mind-eureka-self".to_string(), + caller_id: "persona-a-caller".to_string(), + unit_instance: "eureka".to_string(), + reservation_micro_usd: 5_000_000, + }] + ); + assert_eq!(config.lease_max_ms, 120_000); +} + +#[cfg(target_os = "linux")] +#[test] +fn config_check_refuses_a_lease_account_the_system_does_not_have() { + let scratch = Scratch::new(); + let (mut flags, credentials) = prepare(&scratch); + let socket = scratch.path("lease.sock"); + flags.extend( + [ + "--lease-socket", + socket.to_str().unwrap(), + "--lease-user", + "no-such-account-4f1c", + "--lease-unit-instance", + "eureka", + "--lease-reservation-micro-usd", + "5000000", + ] + .map(String::from), + ); + let made = init(&scratch, &flags); + assert!(made.status.success(), "{}", shown(&made)); + let config = scratch.path("connector.cc"); + let checked = run( + &["config", "check", config.to_str().unwrap()], + &[("CREDENTIALS_DIRECTORY", &credentials)], + ); + assert!(!checked.status.success()); + assert!( + shown(&checked).contains("lease_callers"), + "{}", + shown(&checked) + ); + assert!(!shown(&checked).contains("no-such-account")); +} + +#[cfg(target_os = "linux")] +#[test] +fn serve_binds_the_lease_socket_and_answers_a_stranger_with_a_refusal() { + use std::io::{Read, Write}; + use std::os::unix::fs::PermissionsExt; + use std::os::unix::net::UnixStream; + use std::process::Stdio; + use std::time::Instant; + + use claude_connector::{ + LeaseMessage, LeaseRefusalKind, LeaseRefused, LeaseRequest, decode_frame, encode_frame, + }; + + let scratch = Scratch::new(); + let (mut flags, credentials) = prepare(&scratch); + let socket = scratch.path("lease.sock"); + // The test runs as someone other than `nobody`, so it is a stranger. + flags.extend( + [ + "--lease-socket", + socket.to_str().unwrap(), + "--lease-user", + "nobody", + "--lease-unit-instance", + "eureka", + "--lease-reservation-micro-usd", + "5000000", + ] + .map(String::from), + ); + let made = init(&scratch, &flags); + assert!(made.status.success(), "{}", shown(&made)); + let config = scratch.path("connector.cc"); + let mut server = Command::new(env!("CARGO_BIN_EXE_claude-connector")) + .args(["serve", "--config", config.to_str().unwrap()]) + .env_clear() + .env("CREDENTIALS_DIRECTORY", &credentials) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let started = Instant::now(); + while !socket.exists() && started.elapsed() < Duration::from_secs(10) { + std::thread::sleep(Duration::from_millis(50)); + } + let outcome = std::panic::catch_unwind(|| { + assert_eq!( + std::fs::metadata(&socket).unwrap().permissions().mode() & 0o777, + 0o660 + ); + let mut stream = UnixStream::connect(&socket).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + stream + .write_all(&encode_frame(&LeaseMessage::ClaudeLeaseRequest( + LeaseRequest { + expires_at_unix_ms: u64::MAX / 2, + }, + ))) + .unwrap(); + let mut reply = vec![0_u8; 4096]; + let read = stream.read(&mut reply).unwrap(); + assert!(matches!( + decode_frame(&reply[..read]), + Some(LeaseMessage::ClaudeLeaseRefusal(LeaseRefused { + kind: LeaseRefusalKind::UnknownCaller, + .. + })) + )); + }); + // The server is still serving: it did not return when it started. + let running = server.try_wait().unwrap().is_none(); + server.kill().unwrap(); + server.wait().unwrap(); + outcome.unwrap(); + assert!(running); +} diff --git a/crates/claude-connector/tests/fixtures/lease_v1/grant.bin b/crates/claude-connector/tests/fixtures/lease_v1/grant.bin new file mode 100644 index 0000000000000000000000000000000000000000..554e3a64cb35edbf81585cd63ecbe95744d580df GIT binary patch literal 109 zcmZQzV90D*lAW2CvMD(yu{0$$J|{J?I5j@KC^4^OX<1@UX=>97h)`zAO*Jsf%D~Xb r*u>P#+`=+3DLExI4V}NKD784Xs4O)lJ~uPDC_lcmIOQAzr}_Z^$PFl0 literal 0 HcmV?d00001 diff --git a/crates/claude-connector/tests/fixtures/lease_v1/refusal.bin b/crates/claude-connector/tests/fixtures/lease_v1/refusal.bin new file mode 100644 index 0000000000000000000000000000000000000000..22a4a25e3ceba32fc73bbfaf38dd854feff2550d GIT binary patch literal 104 zcmXBL!3~2j3KM{AsLOPL()t?<|E=k6otQfW-a1l#;wZPj&fC#$7NP yxM8fAdt=)cwvG#YhZkJ#P*Z>Gf?&1(Ei!05ds%5>_Y#zg2}RaM6%BRa(BcQ7qADT) literal 0 HcmV?d00001 diff --git a/crates/claude-connector/tests/fixtures/lease_v1/request.bin b/crates/claude-connector/tests/fixtures/lease_v1/request.bin new file mode 100644 index 0000000000000000000000000000000000000000..fddaa3785fe2e36c3d51f2ec7bb63649d2decda3 GIT binary patch literal 66 zcmZQzV6bailAW2CvL!huu{0$$J|{J?I5oZ~wXig`xMXQrVoqsl3W!){%1t#e%gVse z$k@cx%-q51Y}lBbn4FOsUzD1d0uwAX*aQ(QFUl-Qg^4nLS({&4f~1#OVFOTCaeiKA lUOH4!3e@fmP`fh{GqX!^TFy8D({ctZRx_YlY=dF#0RUJliN*i` literal 0 HcmV?d00001 diff --git a/crates/claude-connector/tests/fixtures/lease_v1/settled.bin b/crates/claude-connector/tests/fixtures/lease_v1/settled.bin new file mode 100644 index 0000000000000000000000000000000000000000..710d4b817e73b57863b3e0adaf6b682b017db04c GIT binary patch literal 81 zcmZQzVDN2PlAW2CvL!huu{0$$J|{J?I5oaFwWK5`HDzg8Voqsl(}twX93U?~H#4~? bKfbg$ u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis() as u64 +} + +fn own_uid() -> u32 { + // SAFETY: geteuid has no preconditions and cannot fail. + unsafe { libc::geteuid() } +} + +/// The 32-digit invocation id of the test's `n`th unit invocation. +fn invocation(n: u64) -> String { + format!("{n:032x}") +} + +type Probe = Box; + +/// A token source that counts its reads and can run a probe at the moment of +/// each one. +struct FakeTokens { + token: Option<&'static str>, + reads: AtomicUsize, + probe: Mutex>, +} + +impl TokenSource for FakeTokens { + fn read(&self) -> Option> { + self.reads.fetch_add(1, Ordering::SeqCst); + if let Some(probe) = self.probe.lock().unwrap().as_ref() { + probe(); + } + self.token.map(|token| Zeroizing::new(token.to_string())) + } +} + +/// The system's record of where every peer runs, as the test sets it, and the +/// pid each handle it was asked about names (by the kernel's own record). +struct FakeUnits { + peer: Mutex>, + asked_about: Mutex>, +} + +impl PeerUnits for FakeUnits { + fn invocation_in(&self, process: BorrowedFd<'_>, unit: &str) -> Option { + let info = + std::fs::read_to_string(format!("/proc/self/fdinfo/{}", process.as_raw_fd())).unwrap(); + let pid = info + .lines() + .find_map(|line| line.strip_prefix("Pid:")) + .unwrap() + .trim() + .to_string(); + self.asked_about + .lock() + .unwrap() + .push((pid, unit.to_string())); + self.peer.lock().unwrap().clone() + } +} + +struct Harness { + dir: PathBuf, + socket: PathBuf, + service: Arc>, + tokens: Arc, + /// The fake behind the listener, when it has one. + units: Option>, +} + +impl Harness { + /// The peers connect from invocation `n` of the Self unit. + fn enter(&self, n: u64) { + self.run_in(UNIT, n); + } + + fn run_in(&self, unit: &str, n: u64) { + *self.units.as_ref().unwrap().peer.lock().unwrap() = Some(PeerUnit { + unit: unit.to_string(), + invocation_id: invocation(n), + }); + } +} + +impl Drop for Harness { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.dir); + } +} + +fn price() -> ModelPrice { + // One micro-USD per token in every class. + ModelPrice { + rates: Rates { + input: 1_000_000, + cache_write_short: 1_000_000, + cache_write_long: 1_000_000, + cache_read: 1_000_000, + output: 1_000_000, + }, + long_context: None, + overhead_tokens: 0, + } +} + +/// What a harness is built from. +struct Setup { + ceiling: u64, + reservation: u64, + uid: u32, + token: Option<&'static str>, + max_connections: usize, +} + +fn setup(ceiling: u64, reservation: u64) -> Setup { + Setup { + ceiling, + reservation, + uid: own_uid(), + token: Some(TOKEN), + max_connections: 8, + } +} + +/// A listener serving `uid` as the caller `eureka`, each of whose leases +/// reserves `reservation`, with a spend window so long that it never rolls +/// over during a test. Peers start in invocation 1 of the Self unit. +fn start(setup: Setup) -> Harness { + let units = Arc::new(FakeUnits { + peer: Mutex::new(None), + asked_about: Mutex::new(Vec::new()), + }); + let harness = start_with(setup, units.clone(), Some(units), INSTANCE); + harness.enter(1); + harness +} + +fn start_with( + setup: Setup, + units: Arc, + fake: Option>, + instance: &str, +) -> Harness { + let dir = std::env::temp_dir().join(format!( + "claude-lease-{}-{}", + std::process::id(), + SEQUENCE.fetch_add(1, Ordering::SeqCst) + )); + std::fs::create_dir_all(&dir).unwrap(); + let socket = dir.join("lease.sock"); + let service = Arc::new(Mutex::new( + TransportService::open( + &dir.join("replay.cc"), + "claude-code", + BTreeMap::from([(MODEL.to_string(), price())]), + [CallerAdmission::new( + CALLER, + "lease-test-connection-key", + 1, + [MODEL.to_string()], + 1, + 65_536, + 32_768, + SpendCeiling(setup.ceiling), + SpendWindowMs(1 << 50), + ) + .unwrap()], + 300_000, + ) + .unwrap(), + )); + let tokens = Arc::new(FakeTokens { + token: setup.token, + reads: AtomicUsize::new(0), + probe: Mutex::new(None), + }); + let listener = UnixListener::bind(&socket).unwrap(); + let serving = (service.clone(), tokens.clone(), units); + let caller = LeaseCaller { + uid: setup.uid, + caller_id: CALLER.to_string(), + unit_instance: instance.to_string(), + reservation_micro_usd: setup.reservation, + }; + std::thread::spawn(move || { + let _ = serve_leases( + &listener, + vec![caller], + LeaseLimits { + socket_timeout: Duration::from_secs(2), + max_lease_ms: 3_600_000, + max_connections: setup.max_connections, + }, + serving.0, + serving.1, + serving.2, + ); + }); + Harness { + dir, + socket, + service, + tokens, + units: fake, + } +} + +/// One frame, and the descriptors that arrived with it. An empty frame is end +/// of file. +fn receive(stream: &UnixStream) -> (Vec, Vec) { + use std::os::fd::AsRawFd; + let mut buffer = vec![0_u8; 70_000]; + let mut control = vec![0_u64; 32]; + let mut vector = libc::iovec { + iov_base: buffer.as_mut_ptr().cast(), + iov_len: buffer.len(), + }; + // SAFETY: the buffers outlive the call and the lengths describe them; the + // descriptors read out of the control messages were just installed in this + // process by the kernel and are owned by nothing else. + unsafe { + let mut message: libc::msghdr = std::mem::zeroed(); + message.msg_iov = &mut vector; + message.msg_iovlen = 1; + message.msg_control = control.as_mut_ptr().cast(); + message.msg_controllen = (control.len() * 8) as _; + let received = libc::recvmsg(stream.as_raw_fd(), &mut message, libc::MSG_CMSG_CLOEXEC); + if received < 0 { + // A listener that drops a connection with our request unread + // resets it: that is end of file for this test's purposes. + let error = std::io::Error::last_os_error(); + assert_eq!(error.kind(), std::io::ErrorKind::ConnectionReset, "{error}"); + return (Vec::new(), Vec::new()); + } + let mut fds = Vec::new(); + let mut header = libc::CMSG_FIRSTHDR(&message); + while !header.is_null() { + if (*header).cmsg_level == libc::SOL_SOCKET && (*header).cmsg_type == libc::SCM_RIGHTS { + let bytes = (*header).cmsg_len as usize - libc::CMSG_LEN(0) as usize; + let data = libc::CMSG_DATA(header).cast::(); + for index in 0..bytes / std::mem::size_of::() { + fds.push(OwnedFd::from_raw_fd(data.add(index).read_unaligned())); + } + } + header = libc::CMSG_NXTHDR(&message, header); + } + buffer.truncate(received as usize); + (buffer, fds) + } +} + +fn connect(harness: &Harness) -> UnixStream { + let stream = UnixStream::connect(&harness.socket).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + stream +} + +fn send(stream: &mut UnixStream, message: &LeaseMessage) { + stream.write_all(&encode_frame(message)).unwrap(); +} + +fn request(ttl_ms: u64) -> LeaseMessage { + LeaseMessage::ClaudeLeaseRequest(LeaseRequest { + expires_at_unix_ms: now_ms() + ttl_ms, + }) +} + +/// What one request to the listener came back with. +struct Answer { + stream: UnixStream, + raw: Vec, + message: Option, + fds: Vec, +} + +fn ask(harness: &Harness, ttl_ms: u64) -> Answer { + ask_at(&harness.socket, ttl_ms) +} + +fn ask_at(socket: &std::path::Path, ttl_ms: u64) -> Answer { + let mut stream = UnixStream::connect(socket).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(10))) + .unwrap(); + // A listener at its limit has already dropped us: the write may fail. + let _ = stream.write_all(&encode_frame(&request(ttl_ms))); + let (raw, fds) = receive(&stream); + let message = decode_frame(&raw); + Answer { + stream, + raw, + message, + fds, + } +} + +fn refusal_of(answer: &Answer) -> Option { + match &answer.message { + Some(LeaseMessage::ClaudeLeaseRefusal(LeaseRefused { kind, .. })) => Some(*kind), + _ => None, + } +} + +fn granted(answer: &Answer) -> LeaseGranted { + match &answer.message { + Some(LeaseMessage::ClaudeLeaseGrant(grant)) => grant.clone(), + other => panic!("expected a grant, got {other:?}"), + } +} + +fn granted_ok(answer: &Answer) -> bool { + matches!(answer.message, Some(LeaseMessage::ClaudeLeaseGrant(_))) && answer.fds.len() == 1 +} + +/// Asks, as a fresh unit invocation each time, until the caller's previous +/// lease has been settled by the listener. +fn ask_after_busy(harness: &Harness, first_invocation: u64) -> Answer { + let deadline = Instant::now() + Duration::from_secs(5); + for attempt in 0.. { + harness.enter(first_invocation + attempt); + let answer = ask(harness, 60_000); + if refusal_of(&answer) != Some(LeaseRefusalKind::Busy) || Instant::now() > deadline { + return answer; + } + std::thread::sleep(Duration::from_millis(20)); + } + unreachable!() +} + +fn used(model: &str, tokens: u64) -> ModelUsage { + ModelUsage { + model: model.to_string(), + usage: Usage { + input_tokens: 0, + cache_read_tokens: 0, + cache_write_tokens: 0, + output_tokens: tokens, + reasoning_tokens: None, + }, + } +} + +fn settle_frame(answer: &Answer, usage: Vec) -> Vec { + encode_frame(&LeaseMessage::ClaudeLeaseSettle(LeaseSettle { + lease_id: granted(answer).lease_id, + usage, + })) +} + +fn settle(answer: &mut Answer, usage: Vec) -> LeaseSettled { + let frame = settle_frame(answer, usage); + answer.stream.write_all(&frame).unwrap(); + let (raw, _) = receive(&answer.stream); + match decode_frame(&raw) { + Some(LeaseMessage::ClaudeLeaseSettled(settled)) => settled, + other => panic!("expected a settled reply, got {other:?}"), + } +} + +fn contains(haystack: &[u8], needle: &str) -> bool { + haystack + .windows(needle.len()) + .any(|window| window == needle.as_bytes()) +} + +/// Asks the ledger directly for a reservation of `cost`, as an invocation +/// nobody else uses. A grant is left open. +fn probe_reserve(harness: &Harness, name: &str, cost: u64) -> LeaseAdmission { + harness + .service + .lock() + .unwrap() + .begin_lease(CALLER, name, cost, FAR, now_ms()) + .unwrap() +} + +fn ceiling_refused() -> LeaseAdmission { + LeaseAdmission::Refused(LeaseRefusal::SpendCeiling) +} + +#[test] +fn a_grant_passes_one_fd_holding_the_token_then_end_of_file() { + let harness = start(setup(1_000, 600)); + let mut answer = ask(&harness, 60_000); + assert_eq!(granted(&answer).reserved_micro_usd, 600); + assert_eq!(answer.fds.len(), 1); + // Reading to end of file proves the write end is closed: nobody holds + // the pipe open for another write. + let mut read = Vec::new(); + File::from(answer.fds.remove(0)) + .read_to_end(&mut read) + .unwrap(); + assert_eq!(read, TOKEN.as_bytes()); + assert_eq!(harness.tokens.reads.load(Ordering::SeqCst), 1); +} + +#[test] +fn the_lease_is_named_for_the_unit_invocation_not_for_anything_the_peer_says() { + let harness = start(setup(1_000, 600)); + harness.enter(0xabc); + let answer = ask(&harness, 60_000); + assert_eq!( + granted(&answer).lease_id, + format!("lease:{}", invocation(0xabc)) + ); +} + +#[test] +fn the_token_is_in_the_pipe_and_nowhere_else() { + let harness = start(setup(1_000, 600)); + let mut answer = ask(&harness, 60_000); + let mut frames = vec![answer.raw.clone()]; + let settled = settle(&mut answer, vec![used(MODEL, 10)]); + frames.push(encode_frame(&LeaseMessage::ClaudeLeaseSettled(settled))); + harness.enter(2); + let refused = ask(&harness, 1); + frames.push(refused.raw.clone()); + for frame in &frames { + assert!(!contains(frame, TOKEN)); + } + assert!(std::env::vars().all(|(name, value)| !name.contains(TOKEN) && !value.contains(TOKEN))); + assert!(std::env::args().all(|argument| !argument.contains(TOKEN))); + // The one place it is: the pipe. + let mut read = String::new(); + File::from(answer.fds.remove(0)) + .read_to_string(&mut read) + .unwrap(); + assert_eq!(read, TOKEN); +} + +#[test] +fn a_peer_uid_outside_the_lease_callers_is_refused_and_nothing_is_reserved_or_read() { + let harness = start(Setup { + uid: own_uid() + 1, + ..setup(1_000, 600) + }); + let answer = ask(&harness, 60_000); + assert_eq!(refusal_of(&answer), Some(LeaseRefusalKind::UnknownCaller)); + assert!(answer.fds.is_empty()); + assert!(!contains(&answer.raw, TOKEN)); + assert_eq!(harness.tokens.reads.load(Ordering::SeqCst), 0); + // The ledger was not touched: the caller's whole ceiling is still free. + assert!(matches!( + probe_reserve(&harness, "probe", 1_000), + LeaseAdmission::Granted(_) + )); +} + +#[test] +fn the_configured_uid_outside_the_self_unit_is_refused_and_nothing_is_reserved_or_read() { + let harness = start(setup(1_000, 600)); + for (label, peer) in [ + ("another instance", Some(("mind-self@other.service", 1))), + ("a user service", Some(("huginn.service", 1))), + ("no unit", None), + ] { + *harness.units.as_ref().unwrap().peer.lock().unwrap() = peer.map(|(unit, n)| PeerUnit { + unit: unit.to_string(), + invocation_id: invocation(n), + }); + let answer = ask(&harness, 60_000); + assert_eq!( + refusal_of(&answer), + Some(LeaseRefusalKind::UnknownCaller), + "{label}" + ); + assert!(answer.fds.is_empty(), "{label}"); + } + assert_eq!(harness.tokens.reads.load(Ordering::SeqCst), 0); + assert!(matches!( + probe_reserve(&harness, "probe", 1_000), + LeaseAdmission::Granted(_) + )); +} + +#[test] +fn one_invocation_of_the_self_unit_leases_once_and_the_next_invocation_leases_again() { + let harness = start(setup(1_000_000, 600)); + let mut first = ask(&harness, 60_000); + assert!(granted_ok(&first)); + settle(&mut first, vec![used(MODEL, 10)]); + // Settled, the caller holds nothing; the invocation has still leased. + let again = ask(&harness, 60_000); + assert_eq!(refusal_of(&again), Some(LeaseRefusalKind::Busy)); + assert!(again.fds.is_empty()); + harness.enter(2); + assert!(granted_ok(&ask(&harness, 60_000))); + assert_eq!(harness.tokens.reads.load(Ordering::SeqCst), 2); +} + +#[test] +fn the_reservation_is_claimed_before_the_token_is_read_or_lent() { + let harness = start(setup(1_000, 600)); + let seen = Arc::new(Mutex::new(None)); + let (service, sink) = (harness.service.clone(), seen.clone()); + *harness.tokens.probe.lock().unwrap() = Some(Box::new(move || { + // From inside the token read: the caller already holds a lease. + let admission = service + .lock() + .unwrap() + .begin_lease(CALLER, "probe", 1, FAR, now_ms()) + .unwrap(); + *sink.lock().unwrap() = Some(admission); + })); + let answer = ask(&harness, 60_000); + assert_eq!(answer.fds.len(), 1); + assert_eq!( + *seen.lock().unwrap(), + Some(LeaseAdmission::Refused(LeaseRefusal::Busy)) + ); +} + +#[test] +fn a_reservation_the_window_cannot_hold_is_refused_and_lends_nothing() { + let harness = start(setup(1_000, 1_001)); + let answer = ask(&harness, 60_000); + assert_eq!(refusal_of(&answer), Some(LeaseRefusalKind::SpendCeiling)); + assert!(answer.fds.is_empty()); + assert_eq!(harness.tokens.reads.load(Ordering::SeqCst), 0); + // Exactly the ceiling fits. + let harness = start(setup(1_000, 1_000)); + assert!(granted_ok(&ask(&harness, 60_000))); +} + +#[test] +fn a_free_lease_and_a_bad_expiry_are_refused() { + let harness = start(setup(1_000, 0)); + assert_eq!( + refusal_of(&ask(&harness, 60_000)), + Some(LeaseRefusalKind::CostBound) + ); + let harness = start(setup(1_000, 100)); + for ttl in [0, 3_600_001 + 1_000] { + let mut stream = connect(&harness); + let expires_at_unix_ms = if ttl == 0 { 1 } else { now_ms() + ttl }; + send( + &mut stream, + &LeaseMessage::ClaudeLeaseRequest(LeaseRequest { expires_at_unix_ms }), + ); + let (raw, fds) = receive(&stream); + assert!(fds.is_empty()); + assert!(matches!( + decode_frame(&raw), + Some(LeaseMessage::ClaudeLeaseRefusal(LeaseRefused { + kind: LeaseRefusalKind::Expired, + .. + })) + )); + } + assert_eq!(harness.tokens.reads.load(Ordering::SeqCst), 0); +} + +#[test] +fn a_second_open_lease_for_the_caller_is_busy_even_from_another_invocation() { + let harness = start(setup(1_000_000, 100)); + let first = ask(&harness, 60_000); + assert!(granted_ok(&first)); + harness.enter(2); + let second = ask(&harness, 60_000); + assert_eq!(refusal_of(&second), Some(LeaseRefusalKind::Busy)); + assert!(second.fds.is_empty()); + assert_eq!(harness.tokens.reads.load(Ordering::SeqCst), 1); +} + +#[test] +fn settlement_bills_the_true_cost_of_each_models_use_and_the_window_holds_it() { + let harness = start(setup(1_000, 600)); + let mut first = ask(&harness, 60_000); + // One micro-USD a token: 250 of the priced model, and 100 of a model the + // table has never heard of, which bills at the dearest known price. + let settled = settle( + &mut first, + vec![used(MODEL, 250), used("claude-new-9", 100)], + ); + assert_eq!( + (settled.billed_micro_usd, settled.overrun_micro_usd), + (350, 0) + ); + // 350 spent: 650 remain, not 651. + assert_eq!(probe_reserve(&harness, "over", 651), ceiling_refused()); + assert!(matches!( + probe_reserve(&harness, "fits", 650), + LeaseAdmission::Granted(_) + )); +} + +#[test] +fn usage_above_the_reservation_is_billed_in_full_and_the_overrun_is_reported_and_counted() { + let harness = start(setup(1_000, 100)); + let mut first = ask(&harness, 60_000); + let settled = settle(&mut first, vec![used(MODEL, 700), used(MODEL, 5_000)]); + assert_eq!(settled.billed_micro_usd, 5_700); + assert_eq!(settled.overrun_micro_usd, 5_600); + // The window is past its ceiling: nothing more reserves in it. + assert_eq!(probe_reserve(&harness, "next", 1), ceiling_refused()); +} + +#[test] +fn a_plan_quota_the_run_reports_is_not_part_of_the_protocol() { + // A settle in the shape the first protocol used, with a plan_quota the + // run vouches for: the extra key is ignored and the usage is billed. + let harness = start(setup(1_000, 600)); + let mut first = ask(&harness, 60_000); + let legacy = serde_json::json!({ + "kind": "claude_lease_settle", + "value": { + "lease_id": granted(&first).lease_id, + "usage": [{"model": MODEL, "usage": { + "input_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, + "output_tokens": 40, "reasoning_tokens": null}}], + "plan_quota": [{"window": "five_hour", "utilization_permille": 0, + "resets_at_unix_ms": 1, "status": "allowed", "observed_at_unix_ms": 1}] + } + }); + let body = rmp_serde::to_vec_named(&legacy).unwrap(); + let mut frame = (body.len() as u32).to_be_bytes().to_vec(); + frame.extend_from_slice(&body); + first.stream.write_all(&frame).unwrap(); + let (raw, _) = receive(&first.stream); + assert!(matches!( + decode_frame(&raw), + Some(LeaseMessage::ClaudeLeaseSettled(LeaseSettled { + billed_micro_usd: 40, + overrun_micro_usd: 0 + })) + )); +} + +#[test] +fn a_connection_dropped_without_settling_is_billed_the_reservation() { + let harness = start(setup(1_000, 600)); + let first = ask(&harness, 60_000); + assert!(granted_ok(&first)); + drop(first); + let over = ask_after_busy(&harness, 2); + assert_eq!(refusal_of(&over), Some(LeaseRefusalKind::SpendCeiling)); + assert_eq!(probe_reserve(&harness, "over", 401), ceiling_refused()); + assert!(matches!( + probe_reserve(&harness, "fits", 400), + LeaseAdmission::Granted(_) + )); +} + +#[test] +fn a_lease_that_expires_unsettled_is_closed_and_billed_the_reservation() { + let harness = start(setup(1_000, 600)); + let first = ask(&harness, 400); + assert!(granted_ok(&first)); + let started = Instant::now(); + // The listener, not the client, ends the lease at its expiry. + let (raw, fds) = receive(&first.stream); + assert!(raw.is_empty() && fds.is_empty()); + assert!(started.elapsed() >= Duration::from_millis(250)); + harness.enter(2); + let over = ask(&harness, 60_000); + assert_eq!(refusal_of(&over), Some(LeaseRefusalKind::SpendCeiling)); +} + +#[test] +fn a_settle_trickled_past_the_expiry_is_no_report_and_bills_the_reservation() { + let harness = start(setup(1_000, 600)); + let mut answer = ask(&harness, 1_500); + let frame = settle_frame(&answer, vec![used(MODEL, 10)]); + let chunk = frame.len() / 4 + 1; + // Each write lands inside any per-read wait; the frame does not land + // inside the lease. + for (i, part) in frame.chunks(chunk).enumerate() { + if i > 0 { + std::thread::sleep(Duration::from_millis(1_200)); + } + let _ = answer.stream.write_all(part); + } + let (raw, _) = receive(&answer.stream); + assert!(raw.is_empty(), "{:?}", decode_frame(&raw)); + assert_eq!(probe_reserve(&harness, "over", 401), ceiling_refused()); +} + +#[test] +fn a_request_trickled_past_the_socket_timeout_is_dropped_without_a_reservation() { + let harness = start(setup(1_000, 600)); + let mut stream = connect(&harness); + let started = Instant::now(); + // Three bytes of a length, 0.9 s apart, then silence: every wait is inside + // the 2 s socket timeout, the frame is not. + for byte in [0_u8, 0, 0] { + stream.write_all(&[byte]).unwrap(); + std::thread::sleep(Duration::from_millis(900)); + } + let (raw, fds) = receive(&stream); + assert!(raw.is_empty() && fds.is_empty()); + assert!( + started.elapsed() < Duration::from_millis(3_300), + "{:?}", + started.elapsed() + ); + assert_eq!(harness.tokens.reads.load(Ordering::SeqCst), 0); + assert!(matches!( + probe_reserve(&harness, "probe", 1_000), + LeaseAdmission::Granted(_) + )); +} + +#[test] +fn a_settle_for_another_lease_is_no_report_and_bills_the_reservation() { + let harness = start(setup(1_000, 600)); + let mut first = ask(&harness, 60_000); + send( + &mut first.stream, + &LeaseMessage::ClaudeLeaseSettle(LeaseSettle { + lease_id: "lease:someone-else".to_string(), + usage: vec![used(MODEL, 1)], + }), + ); + let (raw, _) = receive(&first.stream); + assert!(raw.is_empty()); + harness.enter(2); + let over = ask(&harness, 60_000); + assert_eq!(refusal_of(&over), Some(LeaseRefusalKind::SpendCeiling)); +} + +#[test] +fn a_settle_that_reports_nothing_is_billed_the_reservation() { + let harness = start(setup(1_000, 600)); + let mut first = ask(&harness, 60_000); + let settled = settle(&mut first, vec![]); + assert_eq!( + (settled.billed_micro_usd, settled.overrun_micro_usd), + (600, 0) + ); + // 600 spent: the whole ceiling no longer reserves. + assert_eq!(probe_reserve(&harness, "all", 1_000), ceiling_refused()); +} + +#[test] +fn a_token_that_cannot_be_read_is_never_lent_and_bills_nothing() { + let harness = start(Setup { + token: None, + ..setup(1_000, 1_000) + }); + let first = ask(&harness, 60_000); + assert!(first.raw.is_empty() && first.fds.is_empty()); + assert_eq!(harness.tokens.reads.load(Ordering::SeqCst), 1); + // Nothing was billed: the whole ceiling reserves again and gets as far + // as the token read, not a refusal. + let again = ask_after_busy(&harness, 2); + assert!(again.message.is_none() && again.fds.is_empty()); + assert!(harness.tokens.reads.load(Ordering::SeqCst) >= 2); +} + +#[test] +fn a_malformed_or_oversized_frame_is_dropped_without_a_reservation() { + let harness = start(setup(1_000, 1_000)); + for bytes in [ + vec![0xff, 0xff, 0xff, 0xff], + vec![0, 0, 0, 0], + vec![0, 0, 0, 3, 0xc1, 0xc1, 0xc1], + ] { + let mut stream = connect(&harness); + stream.write_all(&bytes).unwrap(); + let started = Instant::now(); + let (raw, fds) = receive(&stream); + assert!(raw.is_empty() && fds.is_empty()); + // Refused at once, not after the socket timeout of a read that was + // never going to complete. + assert!(started.elapsed() < Duration::from_millis(1_000)); + } + assert_eq!(harness.tokens.reads.load(Ordering::SeqCst), 0); + assert!(granted_ok(&ask(&harness, 60_000))); +} + +#[test] +fn every_message_kind_is_the_pinned_bytes_of_its_fixture() { + let expires = 1_793_307_000_000; + let lease = format!("lease:{}", "0123456789abcdef0123456789abcdef"); + let messages = [ + ( + &include_bytes!("fixtures/lease_v1/request.bin")[..], + LeaseMessage::ClaudeLeaseRequest(LeaseRequest { + expires_at_unix_ms: expires, + }), + ), + ( + &include_bytes!("fixtures/lease_v1/grant.bin")[..], + LeaseMessage::ClaudeLeaseGrant(LeaseGranted { + lease_id: lease.clone(), + reserved_micro_usd: 600_000, + }), + ), + ( + &include_bytes!("fixtures/lease_v1/refusal.bin")[..], + LeaseMessage::ClaudeLeaseRefusal(LeaseRefused { + kind: LeaseRefusalKind::SpendCeiling, + detail: "the window cannot hold this reservation".to_string(), + }), + ), + ( + &include_bytes!("fixtures/lease_v1/settle.bin")[..], + LeaseMessage::ClaudeLeaseSettle(LeaseSettle { + lease_id: lease, + usage: vec![ + ModelUsage { + model: "claude-opus-5-1".to_string(), + usage: Usage { + input_tokens: 1_200, + cache_read_tokens: 30_000, + cache_write_tokens: 500, + output_tokens: 800, + reasoning_tokens: Some(100), + }, + }, + ModelUsage { + model: "claude-haiku-5-1".to_string(), + usage: Usage { + input_tokens: 400, + cache_read_tokens: 0, + cache_write_tokens: 0, + output_tokens: 60, + reasoning_tokens: None, + }, + }, + ], + }), + ), + ( + &include_bytes!("fixtures/lease_v1/settled.bin")[..], + LeaseMessage::ClaudeLeaseSettled(LeaseSettled { + billed_micro_usd: 123_456, + overrun_micro_usd: 23_456, + }), + ), + ]; + for (bytes, message) in messages { + assert_eq!(encode_frame(&message), bytes, "{message:?}"); + assert_eq!(decode_frame(bytes), Some(message)); + } +} + +#[test] +fn connections_past_the_limit_are_dropped_and_finished_ones_free_their_place() { + let harness = start(Setup { + max_connections: 2, + ..setup(1_000, 0) + }); + let held: Vec = (0..2).map(|_| connect(&harness)).collect(); + // Let the listener take both before the third arrives. + std::thread::sleep(Duration::from_millis(300)); + let third = ask(&harness, 60_000); + assert!(third.raw.is_empty() && third.fds.is_empty()); + drop(held); + // Finished connections give their places back: more refusals in a row + // than the limit holds, each answered. + let first = ask_until_answered(&harness); + assert_eq!(refusal_of(&first), Some(LeaseRefusalKind::CostBound)); + for _ in 0..4 { + let again = ask(&harness, 60_000); + assert_eq!(refusal_of(&again), Some(LeaseRefusalKind::CostBound)); + } +} + +/// A free lease, asked until the listener answers instead of dropping. +fn ask_until_answered(harness: &Harness) -> Answer { + let deadline = Instant::now() + Duration::from_secs(5); + loop { + let answer = ask(harness, 60_000); + if !answer.raw.is_empty() || Instant::now() > deadline { + return answer; + } + std::thread::sleep(Duration::from_millis(20)); + } +} + +#[test] +fn an_account_resolves_to_its_uid_and_a_missing_one_to_none() { + use claude_connector::user_uid; + assert_eq!(user_uid("root"), Some(0)); + assert_eq!(user_uid("nobody"), Some(65534)); + assert_eq!(user_uid("no-such-account-4f1c"), None); + assert_eq!(user_uid("bad name"), None); +} + +#[test] +fn a_token_up_to_one_atomic_pipe_write_is_lent_and_a_longer_one_is_not() { + let fits: &'static str = Box::leak("t".repeat(4096).into_boxed_str()); + let harness = start(Setup { + token: Some(fits), + ..setup(1_000, 100) + }); + let mut answer = ask(&harness, 60_000); + let mut read = Vec::new(); + File::from(answer.fds.remove(0)) + .read_to_end(&mut read) + .unwrap(); + assert_eq!(read.len(), 4096); + + let too_long: &'static str = Box::leak("t".repeat(4097).into_boxed_str()); + let harness = start(Setup { + token: Some(too_long), + ..setup(1_000, 100) + }); + let answer = ask(&harness, 60_000); + assert!(answer.raw.is_empty() && answer.fds.is_empty()); +} + +// ---- a second real uid (Soul s1, kept as a regression) ---- + +fn describe(answer: &Answer) -> String { + let kind = match &answer.message { + Some(LeaseMessage::ClaudeLeaseGrant(_)) => "grant".to_string(), + Some(LeaseMessage::ClaudeLeaseRefusal(r)) => format!("refusal:{:?}", r.kind), + Some(other) => format!("other:{other:?}"), + None => format!("none(raw={})", answer.raw.len()), + }; + format!("kind={kind} fds={}", answer.fds.len()) +} + +/// The child half of the second-uid test: connects to `LEASE_SOCK` as whoever +/// it was started as and prints one line saying what came back. The fake +/// canary token is never printed, only whether it matched. +#[test] +#[ignore = "run as a child of a_second_real_uid_cannot_lease"] +fn lease_probe_client() { + let socket = PathBuf::from(std::env::var("LEASE_SOCK").unwrap()); + let mut answer = ask_at(&socket, 60_000); + let shown = describe(&answer); + let mut token_matches = false; + if !answer.fds.is_empty() { + let mut read = Vec::new(); + File::from(answer.fds.remove(0)) + .read_to_end(&mut read) + .unwrap(); + token_matches = read == TOKEN.as_bytes(); + } + println!( + "LEASEPROBE uid={} {} token_matches_canary={token_matches}", + own_uid(), + shown + ); +} + +/// Runs the probe client as `reuid`, with primary group `regid` and +/// supplementary `groups`, and returns its LEASEPROBE line. The marker is +/// found anywhere on a line: the test harness prints it after its own prefix. +fn probe_as(socket: &std::path::Path, reuid: u32, regid: u32, groups: &str) -> String { + let exe = std::env::current_exe().unwrap(); + let out = std::process::Command::new("setpriv") + .arg(format!("--reuid={reuid}")) + .arg(format!("--regid={regid}")) + .arg(format!("--groups={groups}")) + .arg(&exe) + .args([ + "--exact", + "lease_probe_client", + "--ignored", + "--nocapture", + "--test-threads=1", + ]) + .env("LEASE_SOCK", socket) + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&out.stdout).to_string(); + stdout + .lines() + .find_map(|line| line.find("LEASEPROBE").map(|at| line[at..].to_string())) + .unwrap_or_else(|| { + format!( + "NO-LINE status={:?} stdout={stdout:?} stderr={}", + out.status, + String::from_utf8_lossy(&out.stderr) + ) + }) +} + +fn have(tool: &str) -> bool { + std::process::Command::new(tool) + .arg("--version") + .output() + .is_ok() +} + +#[test] +fn a_second_real_uid_cannot_lease() { + use std::os::unix::fs::PermissionsExt; + if own_uid() != 0 || !have("setpriv") { + eprintln!("SKIPPED a_second_real_uid_cannot_lease: needs root and setpriv"); + return; + } + // Numeric uids need no account: 4242 is the configured Self user, 4243 a + // second account that shares its groups. + let harness = start(Setup { + uid: 4242, + ..setup(1_000_000, 600) + }); + std::fs::set_permissions(&harness.dir, std::fs::Permissions::from_mode(0o755)).unwrap(); + std::fs::set_permissions(&harness.socket, std::fs::Permissions::from_mode(0o666)).unwrap(); + // Another uid, in the caller's group and the lease group. + let a = probe_as(&harness.socket, 4243, 4243, "4242,4244"); + // Another uid whose primary gid is the caller's. + let b = probe_as(&harness.socket, 4243, 4242, "4242"); + // Root, which is not configured. + let root = describe(&ask(&harness, 60_000)); + assert_eq!(harness.tokens.reads.load(Ordering::SeqCst), 0); + // The configured uid does lease. + let c = probe_as(&harness.socket, 4242, 4242, "4242"); + assert!(a.contains("kind=refusal:UnknownCaller fds=0"), "{a}"); + assert!(b.contains("kind=refusal:UnknownCaller fds=0"), "{b}"); + assert!(root.contains("kind=refusal:UnknownCaller fds=0"), "{root}"); + assert!( + c.contains("kind=grant fds=1") && c.contains("token_matches_canary=true"), + "{c}" + ); + assert_eq!(harness.tokens.reads.load(Ordering::SeqCst), 1); +} + +// ---- a real systemd, where there is one ---- + +/// The child half of the systemd test, run inside a transient service: leases, +/// settles, and leases again from the same invocation. +#[test] +#[ignore = "run inside a transient unit by a_real_systemd_unit_invocation_leases_once"] +fn lease_in_unit_client() { + let socket = PathBuf::from(std::env::var("LEASE_SOCK").unwrap()); + let mut first = ask_at(&socket, 60_000); + let first_kind = describe(&first); + if granted_ok(&first) { + settle(&mut first, Vec::new()); + } + let second = ask_at(&socket, 60_000); + println!( + "LEASEPROBE first[{first_kind}] second[{}]", + describe(&second) + ); +} + +fn run_in_unit(socket: &std::path::Path) -> String { + let exe = std::env::current_exe().unwrap(); + let out = std::process::Command::new("systemd-run") + .args([ + "--unit=mind-self@leasetest.service", + "--pipe", + "--wait", + "--collect", + "--quiet", + ]) + .arg(format!("--setenv=LEASE_SOCK={}", socket.display())) + .arg(&exe) + .args([ + "--exact", + "lease_in_unit_client", + "--ignored", + "--nocapture", + "--test-threads=1", + ]) + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&out.stdout).to_string(); + stdout + .lines() + .find_map(|line| line.find("LEASEPROBE").map(|at| line[at..].to_string())) + .unwrap_or_else(|| { + format!( + "NO-LINE status={:?} stdout={stdout:?} stderr={}", + out.status, + String::from_utf8_lossy(&out.stderr) + ) + }) +} + +#[test] +fn a_real_systemd_unit_invocation_leases_once() { + if own_uid() != 0 + || !std::path::Path::new("/run/systemd/system").exists() + || !have("systemd-run") + { + eprintln!( + "SKIPPED a_real_systemd_unit_invocation_leases_once: needs root, systemd as init and systemd-run" + ); + return; + } + // The listener asks the production resolver, not the fake. + let harness = start_with( + setup(1_000_000, 600), + Arc::new(SystemdUnits::system()), + None, + "leasetest", + ); + let leased_then_busy = "first[kind=grant fds=1] second[kind=refusal:Busy fds=0]"; + // Two invocations of the same unit: each leases once, as its own. + let one = run_in_unit(&harness.socket); + let two = run_in_unit(&harness.socket); + assert!(one.contains(leased_then_busy), "{one}"); + assert!(two.contains(leased_then_busy), "{two}"); + assert_eq!(harness.tokens.reads.load(Ordering::SeqCst), 2); + // A process outside any such unit is not a Self run, though its uid is + // the configured one. + let outside = ask(&harness, 60_000); + assert_eq!(refusal_of(&outside), Some(LeaseRefusalKind::UnknownCaller)); + assert_eq!(harness.tokens.reads.load(Ordering::SeqCst), 2); +} + +#[test] +fn the_peer_is_named_by_a_process_handle_not_a_pid_number() { + let (one, _other) = UnixStream::pair().unwrap(); + let peer = claude_connector::peer_credentials(&one).unwrap(); + assert_eq!(peer.uid, own_uid()); + let process = peer.process.expect("SO_PEERPIDFD on this kernel"); + let info = + std::fs::read_to_string(format!("/proc/self/fdinfo/{}", process.as_raw_fd())).unwrap(); + assert!( + info.lines() + .any(|line| line.strip_prefix("Pid:").map(str::trim) + == Some(&std::process::id().to_string())), + "{info}" + ); +} + +#[test] +fn the_unit_is_asked_of_the_callers_unit_through_the_peers_handle() { + let harness = start(setup(1_000, 600)); + let answer = ask(&harness, 60_000); + assert!(refusal_of(&answer).is_none() && answer.fds.len() == 1); + assert_eq!( + *harness.units.as_ref().unwrap().asked_about.lock().unwrap(), + vec![(std::process::id().to_string(), UNIT.to_string())] + ); +} diff --git a/crates/model-connector/src/serve.rs b/crates/model-connector/src/serve.rs index 26f5e2d..cf76afc 100644 --- a/crates/model-connector/src/serve.rs +++ b/crates/model-connector/src/serve.rs @@ -171,15 +171,7 @@ pub fn serve( backend: Arc, ) -> Result<(), ServeError> { limits.validate(&admissions)?; - if !listener - .local_addr() - .map_err(ServeError::Listen)? - .ip() - .is_loopback() - { - return Err(ServeError::InvalidConfig("listener address")); - } - backend.ready().map_err(ServeError::BackendUnready)?; + admit_listener(&listener, backend.as_ref())?; let service = Arc::new(Mutex::new(TransportService::open( replay_store, backend.provider(), @@ -190,6 +182,34 @@ pub fn serve( accept_loop(|| listener.accept(), limits, service, backend) } +/// `serve` over a service the caller opened and keeps a handle to, so another +/// front of the same ledger (the Self-run credential lease) reserves in the +/// same windows. The limits must already have been validated against the +/// service's admissions. +pub fn serve_service( + listener: TcpListener, + limits: ServeLimits, + service: Arc>, + backend: Arc, +) -> Result<(), ServeError> { + admit_listener(&listener, backend.as_ref())?; + accept_loop(|| listener.accept(), limits, service, backend) +} + +/// What must be true before a serve loop touches the ledger: the listener is +/// on loopback and the backend can take work. +fn admit_listener(listener: &TcpListener, backend: &dyn ProviderBackend) -> Result<(), ServeError> { + if !listener + .local_addr() + .map_err(ServeError::Listen)? + .ip() + .is_loopback() + { + return Err(ServeError::InvalidConfig("listener address")); + } + backend.ready().map_err(ServeError::BackendUnready) +} + /// The accept loop over any source of connections, so a test can present a /// peer address that a loopback listener never reports. fn accept_loop( diff --git a/crates/model-connector/src/service.rs b/crates/model-connector/src/service.rs index c269916..16853dc 100644 --- a/crates/model-connector/src/service.rs +++ b/crates/model-connector/src/service.rs @@ -317,6 +317,9 @@ fn prune_to_latest( struct LiveRequest { window: u64, reserved_micro_usd: u64, + /// A Self run's credential lease, not a provider call: it holds its + /// reservation in the window like a call does but takes no request slot. + lease: bool, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -338,6 +341,12 @@ enum ReplayState { charged_micro_usd: u64, overrun_micro_usd: u64, }, + /// A credential lease that settled: it has no answer to replay, only the + /// charge it left in its window and how far that passed the reservation. + LeaseSettled { + charged_micro_usd: u64, + overrun_micro_usd: u64, + }, } #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] @@ -586,6 +595,9 @@ impl TransportService { ReplayState::Active { reserved_micro_usd } => *reserved_micro_usd, ReplayState::Completed { charged_micro_usd, .. + } + | ReplayState::LeaseSettled { + charged_micro_usd, .. } => *charged_micro_usd, }; add_spend( @@ -787,6 +799,9 @@ impl TransportService { .map_err(replay_store_error)?; Ok(TransportAdmission::Reply(stand_in)) } + ReplayState::LeaseSettled { .. } => { + self.reply_refusal(&invocation, Refusal::ReplayConflict, &security) + } ReplayState::Active { .. } => { let refusal = if self.live.contains_key(&identity) { Refusal::InFlight @@ -810,8 +825,10 @@ impl TransportService { }; let active_for_caller = self .live - .keys() - .filter(|key| key.caller_runtime_id == invocation.caller_runtime_id) + .iter() + .filter(|(key, request)| { + !request.lease && key.caller_runtime_id == invocation.caller_runtime_id + }) .count(); if active_for_caller >= max_concurrent_requests { return self.reply_refusal(&invocation, Refusal::Capacity, &security); @@ -873,6 +890,7 @@ impl TransportService { LiveRequest { window, reserved_micro_usd, + lease: false, }, ); Ok(TransportAdmission::Execute(Box::new(ExecutionClaim { @@ -1234,6 +1252,9 @@ fn replay_store_error(error: impl std::fmt::Display) -> ServiceError { ServiceError::ReplayStore(error.to_string()) } +mod lease; +pub use lease::{LeaseAdmission, LeaseGrant, LeaseRefusal, LeaseSettlement, LeaseUse, ModelUsage}; + fn contract_refusal(error: &ContractError) -> Refusal { match error { ContractError::Expiry => Refusal::Expired, @@ -2525,7 +2546,9 @@ mod tests { ReplayState::Completed { charged_micro_usd, .. } => charged_micro_usd, - ReplayState::Active { .. } => panic!("{request_id} is not completed"), + ReplayState::Active { .. } | ReplayState::LeaseSettled { .. } => { + panic!("{request_id} is not completed") + } } } diff --git a/crates/model-connector/src/service/lease.rs b/crates/model-connector/src/service/lease.rs new file mode 100644 index 0000000..6e9be71 --- /dev/null +++ b/crates/model-connector/src/service/lease.rs @@ -0,0 +1,691 @@ +//! A credential lease: a Self run's reservation in the caller's spend window. +//! The lease is one record of the same ledger a provider call reserves in, not +//! a second ledger: it is claimed before anything is lent, counts against the +//! window's ceiling while open, and settles to the true cost of what the run +//! reports it used, per model and uncapped: a charge above the reservation is +//! kept as the lease's overrun, and still counts against the window. A lease +//! nothing settled is billed the whole reservation, as a call a crash left +//! behind is. + +use super::*; + +/// The request-id namespace of leases in a caller's replay records. +const LEASE_PREFIX: &str = "lease:"; + +/// What an admitted lease holds. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LeaseGrant { + pub lease_id: String, + pub reserved_micro_usd: u64, +} + +/// What a run used of one model. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ModelUsage { + pub model: String, + pub usage: Usage, +} + +/// What a settled lease was billed. `overrun_micro_usd` is how far the charge +/// passed the reservation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct LeaseSettlement { + pub billed_micro_usd: u64, + pub overrun_micro_usd: u64, +} + +/// What a lease's settlement is billed on. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LeaseUse<'a> { + /// No report: the whole reservation. A run that cannot say what it used + /// pays what was set aside for it. + Reservation, + /// The run's report. A report that names no use (an empty list) is no + /// report, and bills the reservation. + Reported(&'a [ModelUsage]), + /// The token was never lent, so nothing was used and nothing is billed. + NeverLent, +} + +/// What `usage` costs at the table's prices: each entry at its model's price, +/// and a model with no price at the dearest price the table holds, so an +/// unknown name never bills less than any known one. `None` when the table +/// prices nothing. +fn lease_cost(prices: &PriceTable, usage: &[ModelUsage]) -> Option { + let dearest = |usage: &Usage| { + prices + .values() + .map(|price| cost_micro_usd(price, usage, None)) + .max() + }; + usage + .iter() + .map(|entry| match prices.get(&entry.model) { + Some(price) => Some(cost_micro_usd(price, &entry.usage, None)), + None => dearest(&entry.usage), + }) + .try_fold(0_u64, |total, cost| Some(total.saturating_add(cost?))) +} + +/// Why no lease was reserved. Nothing was written to the ledger. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LeaseRefusal { + /// The window's spend plus open reservations plus this one passes the + /// caller's ceiling. + SpendCeiling, + /// The reservation asked for is zero: a lease that bills nothing is a + /// free pass. + CostBound, + UnknownCaller, + /// The caller holds a lease already, or this run id was leased before. + Busy, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LeaseAdmission { + Granted(LeaseGrant), + Refused(LeaseRefusal), +} + +impl TransportService { + /// Reserves `max_cost_micro_usd` in `caller_runtime_id`'s current window + /// for the unit invocation `invocation_id`, which leases once. A ledger + /// that cannot be written answers `Err` and reserves nothing. A lease takes no request slot. + pub fn begin_lease( + &mut self, + caller_runtime_id: &str, + invocation_id: &str, + max_cost_micro_usd: u64, + expires_at_unix_ms: u64, + now_unix_ms: u64, + ) -> Result { + let Some(caller) = self.callers.get(caller_runtime_id) else { + return Ok(LeaseAdmission::Refused(LeaseRefusal::UnknownCaller)); + }; + let spend_ceiling = caller.spend_ceiling_micro_usd; + let spend_window_ms = caller.spend_window_ms; + let connection_key_epoch = caller.connection_key_epoch; + require_id(invocation_id, "invocation_id")?; + if invocation_id.len() + LEASE_PREFIX.len() > MAX_IDENTITY_BYTES { + return Err(ContractError::Invalid("invocation_id").into()); + } + if max_cost_micro_usd == 0 { + return Ok(LeaseAdmission::Refused(LeaseRefusal::CostBound)); + } + let identity = RequestIdentity { + caller_runtime_id: caller_runtime_id.to_string(), + request_id: format!("{LEASE_PREFIX}{invocation_id}"), + }; + let key = replay_key(&identity); + let holding = self + .live + .iter() + .any(|(held, request)| request.lease && held.caller_runtime_id == caller_runtime_id); + if holding + || self + .replay_store + .get::(&key) + .map_err(replay_store_error)? + .is_some() + { + return Ok(LeaseAdmission::Refused(LeaseRefusal::Busy)); + } + let admitted_at_unix_ms = admission_clock( + &self.window_spend, + caller_runtime_id, + now_unix_ms, + spend_window_ms, + ); + let window = admitted_at_unix_ms / spend_window_ms; + prune_windows( + &mut self.window_spend, + &self.live, + caller_runtime_id, + window, + ); + if self + .window_spend(caller_runtime_id, window) + .saturating_add(max_cost_micro_usd) + > spend_ceiling + { + return Ok(LeaseAdmission::Refused(LeaseRefusal::SpendCeiling)); + } + let mut digest = Sha256::new(); + digest.update(b"claude_lease_request"); + digest.update((invocation_id.len() as u64).to_be_bytes()); + digest.update(invocation_id.as_bytes()); + digest.update(max_cost_micro_usd.to_be_bytes()); + digest.update(expires_at_unix_ms.to_be_bytes()); + let record = ReplayRecord { + caller_runtime_id: caller_runtime_id.to_string(), + request_id: identity.request_id.clone(), + connection_key_epoch, + invocation_sha256: digest.finalize().into(), + expires_at_unix_ms, + state: ReplayState::Active { + reserved_micro_usd: max_cost_micro_usd, + }, + admitted_at_unix_ms, + }; + // Reserving and claiming are one write, as for a call: a ledger that + // cannot be written lends nothing. + self.replay_store + .put(key, &record) + .map_err(replay_store_error)?; + add_spend( + &mut self.window_spend, + caller_runtime_id, + window, + 0, + max_cost_micro_usd, + ); + let lease_id = identity.request_id.clone(); + self.live.insert( + identity, + LiveRequest { + window, + reserved_micro_usd: max_cost_micro_usd, + lease: true, + }, + ); + Ok(LeaseAdmission::Granted(LeaseGrant { + lease_id, + reserved_micro_usd: max_cost_micro_usd, + })) + } + + /// Settles an open lease and returns what it was billed. A report bills + /// the true cost of every model's reported use, uncapped (the overrun is + /// recorded); no report, or an empty one, bills the whole reservation; + /// only a token never lent bills nothing. A failed ledger + /// write frees the caller's lease and keeps the reservation counted, as a + /// call's does. + pub fn settle_lease( + &mut self, + caller_runtime_id: &str, + lease_id: &str, + usage: LeaseUse<'_>, + ) -> Result { + let identity = RequestIdentity { + caller_runtime_id: caller_runtime_id.to_string(), + request_id: lease_id.to_string(), + }; + if !self + .live + .get(&identity) + .is_some_and(|request| request.lease) + { + return Err(ServiceError::NoActiveRequest); + } + let key = replay_key(&identity); + let active = self + .replay_store + .get::(&key) + .map_err(replay_store_error)? + .ok_or(ServiceError::NoActiveRequest)?; + let ReplayState::Active { reserved_micro_usd } = active.state else { + return Err(ServiceError::ActiveRequestMismatch); + }; + let caller = self + .callers + .get(caller_runtime_id) + .ok_or(ServiceError::CallerNotAdmitted)?; + let spend_window_ms = caller.spend_window_ms; + let charged_micro_usd = match usage { + LeaseUse::Reservation | LeaseUse::Reported([]) => reserved_micro_usd, + LeaseUse::NeverLent => 0, + LeaseUse::Reported(usage) => { + lease_cost(&self.prices, usage).unwrap_or(reserved_micro_usd) + } + }; + let overrun_micro_usd = charged_micro_usd.saturating_sub(reserved_micro_usd); + let settled = ReplayRecord { + state: ReplayState::LeaseSettled { + charged_micro_usd, + overrun_micro_usd, + }, + ..active.clone() + }; + self.replay_store + .put(key, &settled) + .map_err(replay_store_error) + .inspect_err(|_| { + self.live.remove(&identity); + })?; + add_spend( + &mut self.window_spend, + caller_runtime_id, + active.admitted_at_unix_ms / spend_window_ms, + reserved_micro_usd, + charged_micro_usd, + ); + self.live.remove(&identity); + prune_to_latest(&mut self.window_spend, &self.live, caller_runtime_id); + Ok(LeaseSettlement { + billed_micro_usd: charged_micro_usd, + overrun_micro_usd, + }) + } +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::Ordering; + + use super::*; + use crate::fixtures::*; + + const CALLER: &str = "eureka"; + const KEY: &str = "lease-distinct-test-key"; + const NOW: u64 = 7_200_500; + + fn open_with(store: impl CacheBackingStore + 'static, ceiling: u64) -> TransportService { + TransportService::with_store( + store, + "fixture-sse", + fixture_prices(), + [CallerAdmission::new( + CALLER, + KEY, + 1, + ["gpt-5.4".to_string()], + 1, + 65_536, + 32_768, + SpendCeiling(ceiling), + SpendWindowMs(WINDOW_MS), + ) + .unwrap()], + 5_000, + ) + .unwrap() + } + + fn open(store: &TestStore, ceiling: u64) -> TransportService { + TransportService::open( + &store.path, + "fixture-sse", + fixture_prices(), + [CallerAdmission::new( + CALLER, + KEY, + 1, + ["gpt-5.4".to_string()], + 1, + 65_536, + 32_768, + SpendCeiling(ceiling), + SpendWindowMs(WINDOW_MS), + ) + .unwrap()], + 5_000, + ) + .unwrap() + } + + fn lease(service: &mut TransportService, run: &str, cost: u64) -> LeaseAdmission { + service + .begin_lease(CALLER, run, cost, NOW + 60_000, NOW) + .unwrap() + } + + fn granted(admission: LeaseAdmission) -> LeaseGrant { + match admission { + LeaseAdmission::Granted(grant) => grant, + LeaseAdmission::Refused(why) => panic!("refused {why:?}"), + } + } + + fn held(service: &TransportService) -> u64 { + service.window_spend(CALLER, NOW / WINDOW_MS) + } + + fn output(tokens: u64) -> Vec { + used("gpt-5.4", tokens) + } + + fn used(model: &str, tokens: u64) -> Vec { + vec![ModelUsage { + model: model.to_string(), + usage: Usage { + input_tokens: 0, + cache_read_tokens: 0, + cache_write_tokens: 0, + output_tokens: tokens, + reasoning_tokens: None, + }, + }] + } + + fn billed(settlement: LeaseSettlement) -> u64 { + settlement.billed_micro_usd + } + + #[test] + fn a_lease_reserves_in_the_callers_window_and_names_what_it_holds() { + let store = TestStore::new(); + let mut service = open(&store, 1_000); + let grant = granted(lease(&mut service, "run-1", 600)); + assert_eq!(grant.reserved_micro_usd, 600); + assert_eq!(held(&service), 600); + assert_eq!( + billed( + service + .settle_lease(CALLER, &grant.lease_id, LeaseUse::Reservation) + .unwrap() + ), + 600 + ); + } + + #[test] + fn the_window_ceiling_counts_open_and_settled_reservations_inclusively() { + let store = TestStore::new(); + let mut service = open(&store, 1_000); + let first = granted(lease(&mut service, "run-1", 600)); + service + .settle_lease(CALLER, &first.lease_id, LeaseUse::Reservation) + .unwrap(); + assert_eq!( + lease(&mut service, "run-2", 401), + LeaseAdmission::Refused(LeaseRefusal::SpendCeiling) + ); + assert_eq!(held(&service), 600); + granted(lease(&mut service, "run-3", 400)); + assert_eq!(held(&service), 1_000); + } + + #[test] + fn a_caller_holds_one_lease_at_a_time_and_an_invocation_id_is_leased_once() { + let store = TestStore::new(); + let mut service = open(&store, 1_000_000); + let first = granted(lease(&mut service, "run-1", 100)); + assert_eq!( + lease(&mut service, "run-2", 100), + LeaseAdmission::Refused(LeaseRefusal::Busy) + ); + service + .settle_lease(CALLER, &first.lease_id, LeaseUse::Reservation) + .unwrap(); + assert_eq!( + lease(&mut service, "run-1", 100), + LeaseAdmission::Refused(LeaseRefusal::Busy) + ); + granted(lease(&mut service, "run-2", 100)); + } + + #[test] + fn a_stranger_and_a_free_lease_are_refused_without_a_ledger_write() { + let store = TestStore::new(); + let mut service = open(&store, 1_000); + assert_eq!( + service + .begin_lease("stranger", "run-1", 100, NOW + 1, NOW) + .unwrap(), + LeaseAdmission::Refused(LeaseRefusal::UnknownCaller) + ); + assert_eq!( + lease(&mut service, "run-1", 0), + LeaseAdmission::Refused(LeaseRefusal::CostBound) + ); + assert_eq!(held(&service), 0); + // Nothing was claimed: the same run id leases once it asks sensibly. + granted(lease(&mut service, "run-1", 100)); + } + + #[test] + fn settlement_bills_the_true_cost_uncapped_and_records_the_overrun() { + let store = TestStore::new(); + let mut service = open(&store, 1_000_000); + // Output at 15 USD per million tokens: 10 tokens bill 150 micro-USD. + let small = granted(lease(&mut service, "run-1", 10_000)); + let settled = service + .settle_lease(CALLER, &small.lease_id, LeaseUse::Reported(&output(10)[..])) + .unwrap(); + assert_eq!( + (settled.billed_micro_usd, settled.overrun_micro_usd), + (150, 0) + ); + assert_eq!(held(&service), 150); + // 1000 tokens bill 15_000: the reservation does not cap it, and the + // 5_000 past it is recorded as the overrun. + let big = granted(lease(&mut service, "run-2", 10_000)); + let settled = service + .settle_lease( + CALLER, + &big.lease_id, + LeaseUse::Reported(&output(1_000)[..]), + ) + .unwrap(); + assert_eq!( + (settled.billed_micro_usd, settled.overrun_micro_usd), + (15_000, 5_000) + ); + assert_eq!(held(&service), 15_150); + } + + #[test] + fn settlement_prices_each_model_at_its_own_rate_and_an_unknown_model_at_the_dearest() { + let store = TestStore::new(); + let cheap = ModelPrice { + rates: Rates { + input: 1_000_000, + cache_write_short: 1_000_000, + cache_write_long: 1_000_000, + cache_read: 1_000_000, + output: 2_000_000, + }, + ..fixture_price() + }; + let prices = PriceTable::from([ + ("gpt-5.4".to_string(), fixture_price()), + ("cheap".to_string(), cheap), + ]); + let mut service = TransportService::open( + &store.path, + "fixture-sse", + prices, + [CallerAdmission::new( + CALLER, + KEY, + 1, + ["gpt-5.4".to_string()], + 1, + 65_536, + 32_768, + SpendCeiling(1_000_000), + SpendWindowMs(WINDOW_MS), + ) + .unwrap()], + 5_000, + ) + .unwrap(); + let mixed: Vec = [("gpt-5.4", 10), ("cheap", 10), ("unheard-of", 10)] + .into_iter() + .flat_map(|(model, tokens)| used(model, tokens)) + .collect(); + let grant = granted(lease(&mut service, "run-1", 10_000)); + // 10 tokens at 15, 10 at 2, and 10 at the dearest (15) per million. + assert_eq!( + billed( + service + .settle_lease(CALLER, &grant.lease_id, LeaseUse::Reported(&mixed[..])) + .unwrap() + ), + 150 + 20 + 150 + ); + // A run that reports no use pays the reservation, not nothing. + let grant = granted(lease(&mut service, "run-2", 10_000)); + assert_eq!( + billed( + service + .settle_lease(CALLER, &grant.lease_id, LeaseUse::Reported(&[])) + .unwrap() + ), + 10_000 + ); + } + + #[test] + fn settling_an_empty_report_cannot_free_the_reservation() { + let store = TestStore::new(); + let mut service = open(&store, 1_000); + let grant = granted(lease(&mut service, "run-1", 600)); + assert_eq!( + billed( + service + .settle_lease(CALLER, &grant.lease_id, LeaseUse::Reported(&[])) + .unwrap() + ), + 600 + ); + assert_eq!(held(&service), 600); + assert!(matches!( + lease(&mut service, "run-2", 1_000), + LeaseAdmission::Refused(LeaseRefusal::SpendCeiling) + )); + } + + #[test] + fn a_token_never_lent_bills_nothing_and_frees_the_window() { + let store = TestStore::new(); + let mut service = open(&store, 1_000); + let grant = granted(lease(&mut service, "run-1", 600)); + let settled = service + .settle_lease(CALLER, &grant.lease_id, LeaseUse::NeverLent) + .unwrap(); + assert_eq!( + (settled.billed_micro_usd, settled.overrun_micro_usd), + (0, 0) + ); + assert_eq!(held(&service), 0); + granted(lease(&mut service, "run-2", 1_000)); + } + + #[test] + fn a_lease_settles_once() { + let store = TestStore::new(); + let mut service = open(&store, 1_000_000); + let grant = granted(lease(&mut service, "run-1", 100)); + service + .settle_lease(CALLER, &grant.lease_id, LeaseUse::Reservation) + .unwrap(); + assert!(matches!( + service.settle_lease(CALLER, &grant.lease_id, LeaseUse::Reservation), + Err(ServiceError::NoActiveRequest) + )); + assert_eq!(held(&service), 100); + } + + #[test] + fn a_lease_left_open_by_a_crash_counts_at_its_reservation_after_restart() { + let store = TestStore::new(); + let mut service = open(&store, 1_000_000); + granted(lease(&mut service, "run-1", 700)); + drop(service); + let service = open(&store, 1_000_000); + assert_eq!(held(&service), 700); + } + + #[test] + fn a_ledger_that_cannot_be_written_lends_nothing() { + let flaky = FlakyStore::new(1_000); + let allowance = flaky.allowance(); + let mut service = open_with(flaky, 1_000_000); + allowance.store(0, Ordering::SeqCst); + assert!( + service + .begin_lease(CALLER, "run-1", 100, NOW + 1, NOW) + .is_err() + ); + assert_eq!(held(&service), 0); + allowance.store(1_000, Ordering::SeqCst); + granted(lease(&mut service, "run-1", 100)); + } + + #[test] + fn a_failed_settle_write_frees_the_lease_and_keeps_the_reservation_counted() { + let flaky = FlakyStore::new(1_000); + let allowance = flaky.allowance(); + let mut service = open_with(flaky, 1_000_000); + let grant = granted(lease(&mut service, "run-1", 100)); + allowance.store(0, Ordering::SeqCst); + assert!( + service + .settle_lease(CALLER, &grant.lease_id, LeaseUse::Reported(&output(1)[..])) + .is_err() + ); + assert_eq!(held(&service), 100); + allowance.store(1_000, Ordering::SeqCst); + granted(lease(&mut service, "run-2", 100)); + } + + fn begin_call(service: &mut TransportService, id: &str) -> TransportAdmission { + let mut call = request(); + call.request_id = id.to_string(); + let invocation = + TransportInvocation::new(CALLER, NOW + 1_000, [7; 32], MAX_COST, call).unwrap(); + let sealed = crate::envelope::encrypt_message( + CALLER, + id, + crate::envelope::EnvelopeKind::Invocation, + &crate::contract::wire_encode(&invocation).unwrap(), + &security(KEY), + ) + .unwrap(); + service.begin(&sealed, NOW, 1 << 20).unwrap() + } + + #[test] + fn a_lease_takes_no_request_slot_from_the_callers_calls() { + // The caller admits one concurrent call; an open lease must not use it. + let store = TestStore::new(); + let mut service = open(&store, 1 << 40); + granted(lease(&mut service, "run-1", 100)); + match begin_call(&mut service, "call-1") { + TransportAdmission::Execute(_) => {} + TransportAdmission::Reply(_) => panic!("the lease held the call's slot"), + } + } + + #[test] + fn a_call_in_flight_does_not_make_the_caller_busy_for_a_lease() { + let store = TestStore::new(); + let mut service = open(&store, 1 << 40); + assert!(matches!( + begin_call(&mut service, "call-1"), + TransportAdmission::Execute(_) + )); + granted(lease(&mut service, "run-1", 100)); + } + + #[test] + fn an_invocation_id_is_bounded_so_its_lease_id_fits_an_identity() { + let store = TestStore::new(); + let mut service = open(&store, 1 << 40); + let longest = "r".repeat(MAX_IDENTITY_BYTES - LEASE_PREFIX.len()); + let grant = granted(lease(&mut service, &longest, 100)); + assert_eq!(grant.lease_id.len(), MAX_IDENTITY_BYTES); + service + .settle_lease(CALLER, &grant.lease_id, LeaseUse::Reservation) + .unwrap(); + assert!( + service + .begin_lease(CALLER, &format!("{longest}r"), 100, NOW + 1, NOW) + .is_err() + ); + } + + #[test] + fn a_settled_lease_leaves_its_charge_not_its_reservation_across_a_restart() { + let store = TestStore::new(); + let mut service = open(&store, 1_000_000); + let grant = granted(lease(&mut service, "run-1", 10_000)); + service + .settle_lease(CALLER, &grant.lease_id, LeaseUse::Reported(&output(10)[..])) + .unwrap(); + drop(service); + let service = open(&store, 1_000_000); + assert_eq!(held(&service), 150); + } +} From fdcfa063c7da6de1b0f3f21a677619cb2633bada Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 30 Oct 2026 00:11:53 +0100 Subject: [PATCH 03/41] connector-many-callers: one connector serves every Mind as its own caller; session handles are an HMAC under a connector secret over caller, session, instance, instructions, model and transcript prefix; config add-caller/add-lease-caller; one owner for distinct admissions (squash of eureka-body/connector-many-callers d1f25b5..a96b276; Soul s1-s2) Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + README.md | 5 +- crates/claude-connector/Cargo.toml | 1 + crates/claude-connector/src/backend.rs | 83 ++- crates/claude-connector/src/backend/tests.rs | 296 ++++++++++- crates/claude-connector/src/config.rs | 3 + crates/claude-connector/src/daemon.rs | 92 ++-- crates/claude-connector/src/handle.rs | 69 ++- crates/claude-connector/src/lib.rs | 2 +- crates/claude-connector/src/main.rs | 264 ++++++++-- crates/claude-connector/src/quota.rs | 2 +- .../claude-connector/tests/config_document.rs | 474 +++++++++++++++++- crates/model-connector/src/serve.rs | 23 +- crates/model-connector/src/service.rs | 63 ++- 14 files changed, 1228 insertions(+), 150 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 524654e..5291d19 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -140,6 +140,7 @@ version = "0.1.0" dependencies = [ "anyhow", "cultcache-rs", + "hmac", "libc", "model-connector", "rmp-serde", diff --git a/README.md b/README.md index 3a12abd..c2e23c7 100644 --- a/README.md +++ b/README.md @@ -42,8 +42,9 @@ Crate `crates/claude-connector` (library only; the binary that wires it to `serv servers, no settings sources and the system prompt replaced. Tool-less turns only: a request with tools or a final tool result is refused before any spawn. A request's output schema goes to the CLI as `--json-schema` and the result's structured output is the one assistant text. -- Reasoning state is a session handle (`ClaudeCodeHandle`); a handle that cannot be proven to - match the request re-seeds from the caller's transcript, and every resume forks. +- Reasoning state is a session handle (`ClaudeCodeHandle`) authenticated with a connector secret + and bound to the caller it was minted for; a handle that cannot be proven to match the request + and its caller re-seeds from the caller's transcript, and every resume forks. - `ProcessPort` is the one OS seam (`StdProcessPort`, unix, kills the child's whole process group); `SessionStore` and `StderrSink` are the other ports. `ClaudeCodeConfig` is what the binary hands in; the credential is read from a systemd credential and reaches the child on a diff --git a/crates/claude-connector/Cargo.toml b/crates/claude-connector/Cargo.toml index 7f9b042..6592252 100644 --- a/crates/claude-connector/Cargo.toml +++ b/crates/claude-connector/Cargo.toml @@ -13,6 +13,7 @@ path = "src/main.rs" [dependencies] anyhow = "1" +hmac = "0.12" cultcache-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "a4eae9056152335662f1115f97bf35fb00ab6d23", package = "cultcache-rs" } model-connector = { path = "../model-connector", features = ["daemon"] } rmp-serde = "1" diff --git a/crates/claude-connector/src/backend.rs b/crates/claude-connector/src/backend.rs index 1963e1d..4833521 100644 --- a/crates/claude-connector/src/backend.rs +++ b/crates/claude-connector/src/backend.rs @@ -12,7 +12,9 @@ use sha2::{Digest, Sha256}; use zeroize::Zeroizing; use crate::config::{ClaudeCodeConfig, PROVIDER}; -use crate::handle::{ClaudeCodeHandle, SessionStore, plan, prefix_digest, stdin_bytes}; +use crate::handle::{ + Binding, ClaudeCodeHandle, MIN_SECRET_BYTES, SessionStore, plan, prefix_digest, stdin_bytes, +}; use crate::launch::{PromptFile, argv, child_env}; use crate::process::{ProcessPort, RunningChild, StdoutRead}; use crate::quota::QuotaCell; @@ -36,6 +38,8 @@ pub trait TokenSource: Send + Sync { pub struct CredentialTokenSource { directory: Option, name: String, + /// The configuration field `missing` names when the credential is absent. + field: &'static str, } impl CredentialTokenSource { @@ -46,13 +50,21 @@ impl CredentialTokenSource { Self { directory: std::env::var_os("CREDENTIALS_DIRECTORY").map(PathBuf::from), name: name.into(), + field: "token_credential", } } + /// The same source, naming `field` when its credential is absent. + pub fn for_field(mut self, field: &'static str) -> Self { + self.field = field; + self + } + pub fn in_directory(directory: impl Into, name: impl Into) -> Self { Self { directory: Some(directory.into()), name: name.into(), + field: "token_credential", } } } @@ -72,7 +84,7 @@ impl TokenSource for CredentialTokenSource { if self.directory.is_none() { "credentials_directory" } else { - "token_credential" + self.field } } } @@ -91,9 +103,17 @@ pub struct ClaudeCodeBackend { sessions: Box, sink: Box, tokens: Box, + handle_secret: Box, quota: Arc, } +/// The handle secret, or `None` when it is absent or shorter than a key. +fn usable_secret(source: &dyn TokenSource) -> Option> { + source + .read() + .filter(|secret| secret.len() >= MIN_SECRET_BYTES) +} + impl ClaudeCodeBackend { /// The backend over real processes and the CLI's real session store. #[cfg(unix)] @@ -105,7 +125,11 @@ impl ClaudeCodeBackend { let tokens = Box::new(CredentialTokenSource::from_environment( config.token_credential.clone(), )); - Self::with_ports(config, port, sessions, sink, tokens) + let handle_secret = Box::new( + CredentialTokenSource::from_environment(config.handle_secret_credential.clone()) + .for_field("handle_secret_credential"), + ); + Self::with_ports(config, port, sessions, sink, tokens, handle_secret) } pub fn with_ports( @@ -114,6 +138,7 @@ impl ClaudeCodeBackend { sessions: Box, sink: Box, tokens: Box, + handle_secret: Box, ) -> Self { Self { config, @@ -121,6 +146,7 @@ impl ClaudeCodeBackend { sessions, sink, tokens, + handle_secret, quota: Arc::new(QuotaCell::new()), } } @@ -136,8 +162,25 @@ impl ClaudeCodeBackend { binary_matches_pin(&self.config) } - fn run(&self, request: &ProviderRequest) -> (Vec, TransportOutcome) { - let plan = match plan(request, &self.config, self.sessions.as_ref()) { + fn run( + &self, + request: &ProviderRequest, + caller: &str, + ) -> (Vec, TransportOutcome) { + let Some(secret) = usable_secret(self.handle_secret.as_ref()) else { + return failed( + FailureKind::Authentication, + "claude-code: the connector has no handle secret".to_string(), + Some(zero_usage()), + ); + }; + let plan = match plan( + request, + &self.config, + secret.as_bytes(), + caller, + self.sessions.as_ref(), + ) { Ok(plan) => plan, Err(error) => { return failed( @@ -198,7 +241,7 @@ impl ClaudeCodeBackend { } match finished { - Ok(finished) => self.completed(request, finished), + Ok(finished) => self.completed(request, caller, secret.as_bytes(), finished), Err(failure_) => failure(failure_), } } @@ -206,6 +249,8 @@ impl ClaudeCodeBackend { fn completed( &self, request: &ProviderRequest, + caller: &str, + secret: &[u8], finished: Finished, ) -> (Vec, TransportOutcome) { let mut payloads: Vec = finished @@ -226,9 +271,19 @@ impl ClaudeCodeBackend { // and whatever follows it is seeded from the caller's transcript. if request.output_schema_json.is_none() { let handle = ClaudeCodeHandle::Ended { - session_id: finished.session_id, + session_id: finished.session_id.clone(), instance: self.config.instance.clone(), - prefix_sha256: prefix_digest(&request.instructions, &request.model, &prefix), + prefix_sha256: prefix_digest( + secret, + &Binding { + caller, + instructions: &request.instructions, + model: &request.model, + session_id: &finished.session_id, + instance: &self.config.instance, + items: &prefix, + }, + ), }; payloads.push(TransportEventPayload::ReasoningState { state: handle.encode(), @@ -256,7 +311,11 @@ impl ClaudeCodeBackend { impl ProviderBackend for ClaudeCodeBackend { fn ready(&self) -> Result<(), &'static str> { - readiness(&self.config, self.tokens.as_ref()) + readiness( + &self.config, + self.tokens.as_ref(), + self.handle_secret.as_ref(), + ) } fn provider(&self) -> &str { @@ -264,7 +323,7 @@ impl ProviderBackend for ClaudeCodeBackend { } fn execute(&self, call: &ProviderCall) -> TransportResult { - let (events, outcome) = self.run(call.request()); + let (events, outcome) = self.run(call.request(), call.caller_runtime_id()); call.transported(events, outcome) } @@ -327,6 +386,7 @@ fn binary_matches_pin(config: &ClaudeCodeConfig) -> bool { pub(crate) fn readiness( config: &ClaudeCodeConfig, tokens: &dyn TokenSource, + handle_secret: &dyn TokenSource, ) -> Result<(), &'static str> { if !config.work_dir.is_dir() { return Err("work_dir"); @@ -342,6 +402,9 @@ pub(crate) fn readiness( } // Last: outside the unit the credential is the one thing always absent, // and it must not hide the faults the document itself has. + if usable_secret(handle_secret).is_none() { + return Err(handle_secret.missing()); + } if tokens.read().is_none() { return Err(tokens.missing()); } diff --git a/crates/claude-connector/src/backend/tests.rs b/crates/claude-connector/src/backend/tests.rs index 2e3b993..fc2be99 100644 --- a/crates/claude-connector/src/backend/tests.rs +++ b/crates/claude-connector/src/backend/tests.rs @@ -333,6 +333,7 @@ fn config(dir: &Path, instance: &str, limits: ClaudeCodeLimits) -> ClaudeCodeCon config_dir: PathBuf::from("/cfg"), prompt_dir: dir.join("prompts"), token_credential: "claude-token".to_string(), + handle_secret_credential: "claude-handle-secret".to_string(), instance: instance.to_string(), limits, } @@ -351,6 +352,26 @@ fn canary_token() -> Box { Box::new(FixedToken(Some(CANARY_TOKEN))) } +/// A handle secret source with a fixed answer; it names its own field. +struct FixedSecret(Option<&'static str>); + +impl TokenSource for FixedSecret { + fn read(&self) -> Option> { + self.0.map(|secret| Zeroizing::new(secret.to_string())) + } + + fn missing(&self) -> &'static str { + "handle_secret_credential" + } +} + +const SECRET: &str = "handle-secret-0000000000000000000000000000000000000000000000"; +const OTHER_SECRET: &str = "handle-secret-1111111111111111111111111111111111111111111111"; + +fn canary_secret() -> Box { + Box::new(FixedSecret(Some(SECRET))) +} + fn limits() -> ClaudeCodeLimits { ClaudeCodeLimits { deadline: Duration::from_secs(30), @@ -369,6 +390,16 @@ struct Rig { } fn rig_for(instance: &str, scripts: Vec