Hosted inference lane and token usage on every model reply (WIP, builds unverified)
The OpenAI-compatible port gains a hosted binding (https endpoint, bearer key read once from a file); InferenceOutput carries Option<TokenUsage>. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
367e2eb50f
commit
7262e72a0d
8 changed files with 567 additions and 73 deletions
|
|
@ -8265,7 +8265,7 @@ pub(crate) mod tests {
|
|||
};
|
||||
let inference = match timeout {
|
||||
Some(timeout) => ghostlight::open_local_port_with_timeout(binding, timeout).unwrap(),
|
||||
None => ghostlight::open_inference(None, None, Some(binding), &["local/play"]).unwrap(),
|
||||
None => ghostlight::open_inference(None, None, Some(binding), None, &["local/play"]).unwrap(),
|
||||
};
|
||||
let table = PlayTable::new(
|
||||
fixture.world.clone(),
|
||||
|
|
|
|||
|
|
@ -587,7 +587,7 @@ fn open_controller(
|
|||
};
|
||||
let mut all_models: Vec<&str> = models.each().to_vec();
|
||||
all_models.push(&play_model);
|
||||
let inference = open_inference(connector, sdk, local, &all_models)?;
|
||||
let inference = open_inference(connector, sdk, local, None, &all_models)?;
|
||||
let work = open_controller_work(service_root.join("controller-work.cc"))?;
|
||||
let runner = ControllerRunner::open(world.clone(), inference.clone(), work, models.clone())?;
|
||||
Ok(OpenedControllers {
|
||||
|
|
@ -2513,7 +2513,7 @@ mod tests {
|
|||
key_path: controller_key,
|
||||
caller_runtime_id: "ghostlight-runtime-test".into(),
|
||||
};
|
||||
let inference = open_inference(Some(connector), None, None, &models.each()).unwrap();
|
||||
let inference = open_inference(Some(connector), None, None, None, &models.each()).unwrap();
|
||||
let work = open_controller_work(directory.path().join("controller-work.cc")).unwrap();
|
||||
let controllers =
|
||||
ControllerRunner::open(world.clone(), inference.clone(), work, models.clone()).unwrap();
|
||||
|
|
|
|||
|
|
@ -11,7 +11,9 @@ use super::elaboration::{
|
|||
NullEvidenceSource, SeedCheckpoint, SeedRunner, valid_elaboration_progression,
|
||||
valid_seed_progression,
|
||||
};
|
||||
use super::local_inference::{DEFAULT_LOCAL_MODEL_PREFIX, LocalBinding, open_local_port};
|
||||
use super::local_inference::{
|
||||
DEFAULT_LOCAL_MODEL_PREFIX, HostedBinding, LocalBinding, open_hosted_port, open_local_port,
|
||||
};
|
||||
use super::sdk_inference::{
|
||||
ChildProcessLink, DEFAULT_SDK_MODEL_PREFIX, RoutedInferencePort, SdkBinding, SdkInferencePort,
|
||||
};
|
||||
|
|
@ -156,6 +158,21 @@ pub enum InferenceEvent {
|
|||
pub struct InferenceOutput {
|
||||
pub(super) events: Vec<InferenceEvent>,
|
||||
pub(super) receipt_digest: String,
|
||||
/// What the provider reported spending on this reply. `None` is a reply
|
||||
/// that reported none (the connector lane's receipt carries no counts);
|
||||
/// it is never a zero.
|
||||
#[serde(default)]
|
||||
pub(super) usage: Option<TokenUsage>,
|
||||
}
|
||||
|
||||
/// Tokens one reply cost, as the provider counted them. `prompt` is the
|
||||
/// whole prompt, cache hits included; `cached_prompt` is the part the
|
||||
/// provider served from its cache, `None` when it reported no cache figure.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct TokenUsage {
|
||||
pub prompt: u64,
|
||||
pub completion: u64,
|
||||
pub cached_prompt: Option<u64>,
|
||||
}
|
||||
|
||||
impl InferenceOutput {
|
||||
|
|
@ -165,9 +182,16 @@ impl InferenceOutput {
|
|||
Self {
|
||||
events,
|
||||
receipt_digest: receipt_digest.into(),
|
||||
usage: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The same output carrying the provider's reported token usage.
|
||||
pub fn with_usage(mut self, usage: TokenUsage) -> Self {
|
||||
self.usage = Some(usage);
|
||||
self
|
||||
}
|
||||
|
||||
pub(super) fn prose_only(
|
||||
self,
|
||||
purpose: InferencePurpose,
|
||||
|
|
@ -611,6 +635,7 @@ impl CodexConnectorInferencePort {
|
|||
Ok(InferenceOutput {
|
||||
events,
|
||||
receipt_digest: format!("sha256:{:x}", Sha256::digest(&receipt_bytes)),
|
||||
usage: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -2485,7 +2510,11 @@ pub enum ControllerOpenError {
|
|||
SdkSidecarMissing { path: String },
|
||||
#[error("the local inference endpoint `{endpoint}` is not loopback")]
|
||||
LocalEndpointNotLoopback { endpoint: SocketAddr },
|
||||
#[error("the local and SDK inference transports both claim model prefix `{prefix}`")]
|
||||
#[error("the hosted inference endpoint must be an https URL")]
|
||||
HostedEndpointNotHttps,
|
||||
#[error("the hosted inference key file is unreadable, empty, or has surrounding whitespace")]
|
||||
HostedKeyUnreadable,
|
||||
#[error("two inference transports both claim model prefix `{prefix}`")]
|
||||
SharedModelPrefix { prefix: String },
|
||||
#[error("the {transport} inference transport's model prefix must not be empty")]
|
||||
EmptyModelPrefix { transport: &'static str },
|
||||
|
|
@ -2508,6 +2537,7 @@ pub fn open_inference(
|
|||
connector: Option<ConnectorBinding>,
|
||||
sdk: Option<SdkBinding>,
|
||||
local: Option<LocalBinding>,
|
||||
hosted: Option<HostedBinding>,
|
||||
models: &[&str],
|
||||
) -> Result<Arc<dyn InferencePort>, ControllerOpenError> {
|
||||
let connector: Option<Arc<dyn InferencePort>> = match connector {
|
||||
|
|
@ -2551,15 +2581,31 @@ pub fn open_inference(
|
|||
}
|
||||
None => None,
|
||||
};
|
||||
if let Some((prefix, _)) = &local
|
||||
&& sdk.is_some()
|
||||
&& *prefix == sdk_model_prefix
|
||||
{
|
||||
// The hosted lane is the local port's second binding: the same
|
||||
// OpenAI-compatible port, an https endpoint and a bearer key read once
|
||||
// from a file. It claims its own prefix beside the local one.
|
||||
let hosted: Option<(String, Arc<dyn InferencePort>)> = match hosted {
|
||||
Some(binding) => {
|
||||
if binding.model_prefix.is_empty() {
|
||||
return Err(ControllerOpenError::EmptyModelPrefix { transport: "hosted" });
|
||||
}
|
||||
let prefix = binding.model_prefix.clone();
|
||||
Some((prefix, open_hosted_port(binding)?))
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
let claims: Vec<(String, Arc<dyn InferencePort>)> = local.into_iter().chain(hosted).collect();
|
||||
let mut claimed: Vec<&str> = claims.iter().map(|(prefix, _)| prefix.as_str()).collect();
|
||||
if sdk.is_some() {
|
||||
claimed.push(&sdk_model_prefix);
|
||||
}
|
||||
claimed.sort_unstable();
|
||||
if let Some(pair) = claimed.windows(2).find(|pair| pair[0] == pair[1]) {
|
||||
return Err(ControllerOpenError::SharedModelPrefix {
|
||||
prefix: prefix.clone(),
|
||||
prefix: pair[0].to_owned(),
|
||||
});
|
||||
}
|
||||
let routed = RoutedInferencePort::new(connector, sdk, sdk_model_prefix, local);
|
||||
let routed = RoutedInferencePort::new(connector, sdk, sdk_model_prefix, claims);
|
||||
for &model in models {
|
||||
if routed.route(model).is_none() {
|
||||
return Err(ControllerOpenError::UnroutableModel {
|
||||
|
|
@ -7092,6 +7138,7 @@ mod tests {
|
|||
arguments: json!({"detail":"The wind reading is missing."}).to_string(),
|
||||
}],
|
||||
receipt_digest: "sha256:operational-round-zero".into(),
|
||||
usage: None,
|
||||
}];
|
||||
let substituted_operational = operational_in_flight(
|
||||
operational_command,
|
||||
|
|
@ -7130,6 +7177,7 @@ mod tests {
|
|||
.to_string(),
|
||||
}],
|
||||
receipt_digest: "sha256:narrative-round-zero".into(),
|
||||
usage: None,
|
||||
}];
|
||||
let substituted_narrative = narrative_interpreter_in_flight(
|
||||
narrative_command,
|
||||
|
|
@ -9931,6 +9979,7 @@ mod tests {
|
|||
.map(|(id, name, arguments)| oracle_call(id, name, arguments))
|
||||
.collect(),
|
||||
receipt_digest: format!("sha256:{receipt}"),
|
||||
usage: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -10807,6 +10856,7 @@ mod tests {
|
|||
Ok(InferenceOutput {
|
||||
events,
|
||||
receipt_digest: format!("sha256:{receipt}"),
|
||||
usage: None,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -11491,6 +11541,7 @@ mod tests {
|
|||
model_prefix: local_model_prefix,
|
||||
caller_runtime_id: runtime_id,
|
||||
}),
|
||||
None,
|
||||
&models.each(),
|
||||
)
|
||||
.unwrap();
|
||||
|
|
@ -14763,12 +14814,14 @@ mod tests {
|
|||
InferenceOutput {
|
||||
events: vec![speak_call("call", "c0__notgranted", "Anything.")],
|
||||
receipt_digest: "sha256:grouped".into(),
|
||||
usage: None,
|
||||
},
|
||||
// The repair round the grouped budget buys. It calls nothing, so the
|
||||
// handle finishes silent.
|
||||
InferenceOutput {
|
||||
events: vec![InferenceEvent::Text("Nothing further.".into())],
|
||||
receipt_digest: "sha256:grouped-repair".into(),
|
||||
usage: None,
|
||||
},
|
||||
];
|
||||
let GroupedLoopEvaluation::Complete { capture } =
|
||||
|
|
|
|||
|
|
@ -2158,6 +2158,7 @@ mod tests {
|
|||
})
|
||||
.collect(),
|
||||
receipt_digest: "sha256:receipt".into(),
|
||||
usage: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -86,7 +86,7 @@ pub use controllers::{
|
|||
GroupedCheckpoint, InferenceEvent, InferenceFault, InferenceFaultClass, InferenceFaultDisposition, InferenceOutput,
|
||||
InferencePort,
|
||||
InferencePurpose, InferenceRequest, NarrativeCheckpoint, NarrativeRun, OperationalCheckpoint,
|
||||
OperationalRun, PersonaLane, PreparedInference, SubmissionDisposition, ToolResultOracle,
|
||||
OperationalRun, PersonaLane, PreparedInference, SubmissionDisposition, TokenUsage, ToolResultOracle,
|
||||
TracingInferencePort, open_controller_work, open_inference,
|
||||
};
|
||||
pub(crate) use cover::{CellId, Constituent, Resolution};
|
||||
|
|
@ -96,7 +96,7 @@ pub use elaboration::{
|
|||
EvidenceReceipt, EvidenceSource, SeedCheckpoint, SeedOutcome, SeedRunner, select_row,
|
||||
};
|
||||
pub use lens::{Lens, LensWeights};
|
||||
pub use local_inference::{DEFAULT_LOCAL_MODEL_PREFIX, LocalBinding};
|
||||
pub use local_inference::{DEFAULT_HOSTED_MODEL_PREFIX, DEFAULT_LOCAL_MODEL_PREFIX, HostedBinding, LocalBinding};
|
||||
#[cfg(feature = "test-support")]
|
||||
pub use local_inference::open_local_port_with_timeout;
|
||||
pub(crate) use mailbox::ElaborationPort;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
//! A local OpenAI-compatible inference port, for a loopback model server this
|
||||
//! process does not authenticate to and does not need a credential for.
|
||||
//! An OpenAI-compatible inference port with two bindings. The local binding
|
||||
//! reaches a loopback model server this process does not authenticate to and
|
||||
//! needs no credential for. The hosted binding reaches an https endpoint
|
||||
//! (a cheap hosted model for the Aetheria verse) with a bearer key read once
|
||||
//! from a file.
|
||||
//!
|
||||
//! This module owns one authority: lowering a prepared request to one
|
||||
//! `POST /v1/chat/completions` call and lifting the reply back into the same
|
||||
|
|
@ -10,8 +13,8 @@
|
|||
|
||||
use super::controllers::{
|
||||
ControllerOpenError, InferenceEvent, InferenceFault, InferenceFaultClass, InferenceOutput, InferencePort,
|
||||
InferenceRequest, PreparedInference, REQUEST_EXPIRY, RESPONSE_TIMEOUT, call_id_is_valid,
|
||||
tool_name_is_valid, unix_ms,
|
||||
InferenceRequest, PreparedInference, REQUEST_EXPIRY, RESPONSE_TIMEOUT, TokenUsage,
|
||||
call_id_is_valid, tool_name_is_valid, unix_ms,
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use codex_connector::CodexInputItem;
|
||||
|
|
@ -19,7 +22,9 @@ use serde::{Deserialize, Serialize};
|
|||
use serde_json::{Value, json};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::net::SocketAddr;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use zeroize::Zeroizing;
|
||||
|
||||
/// The model-name prefix that names the local transport when nothing
|
||||
/// configures one.
|
||||
|
|
@ -33,6 +38,41 @@ pub struct LocalBinding {
|
|||
pub caller_runtime_id: String,
|
||||
}
|
||||
|
||||
/// The model-name prefix that names the hosted transport when nothing
|
||||
/// configures one.
|
||||
pub const DEFAULT_HOSTED_MODEL_PREFIX: &str = "hosted/";
|
||||
|
||||
/// Everything the hosted binding needs to open. `base_url` is the provider's
|
||||
/// origin (`https://api.deepseek.com`); the port appends
|
||||
/// `/v1/chat/completions`. The key is read from `key_path` once, at open.
|
||||
pub struct HostedBinding {
|
||||
pub base_url: String,
|
||||
pub key_path: PathBuf,
|
||||
pub model_prefix: String,
|
||||
pub caller_runtime_id: String,
|
||||
}
|
||||
|
||||
/// Where the port sends its one request. The credential lives only in the
|
||||
/// `Hosted` variant, so the loopback binding cannot carry one.
|
||||
enum Endpoint {
|
||||
Loopback(SocketAddr),
|
||||
Hosted {
|
||||
base_url: String,
|
||||
key: Zeroizing<String>,
|
||||
},
|
||||
}
|
||||
|
||||
impl Endpoint {
|
||||
fn url(&self) -> String {
|
||||
match self {
|
||||
Self::Loopback(address) => format!("http://{address}/v1/chat/completions"),
|
||||
Self::Hosted { base_url, .. } => {
|
||||
format!("{}/v1/chat/completions", base_url.trim_end_matches('/'))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const LOCAL_RECEIPT_SCHEMA: &str = "ghostlight.local_inference_receipt.v1";
|
||||
|
||||
/// The identity half is computed here from the invocation, exactly as the SDK
|
||||
|
|
@ -113,22 +153,37 @@ struct LocalChatFunctionCall {
|
|||
arguments: String,
|
||||
}
|
||||
|
||||
/// Every count is optional on the wire: a reply missing the prompt or
|
||||
/// completion count reports no usage rather than a zero.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct LocalChatUsage {
|
||||
#[serde(default)]
|
||||
prompt_tokens: u64,
|
||||
prompt_tokens: Option<u64>,
|
||||
#[serde(default)]
|
||||
completion_tokens: u64,
|
||||
completion_tokens: Option<u64>,
|
||||
/// DeepSeek's cache-hit count, part of `prompt_tokens`.
|
||||
#[serde(default)]
|
||||
prompt_cache_hit_tokens: Option<u64>,
|
||||
}
|
||||
|
||||
/// The local OpenAI-compatible backend behind Ghostlight's inference seam.
|
||||
impl LocalChatUsage {
|
||||
fn reported(&self) -> Option<TokenUsage> {
|
||||
Some(TokenUsage {
|
||||
prompt: self.prompt_tokens?,
|
||||
completion: self.completion_tokens?,
|
||||
cached_prompt: self.prompt_cache_hit_tokens,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// The OpenAI-compatible backend behind Ghostlight's inference seam.
|
||||
/// Unlike the SDK port, it runs no query loop: one prepared request lowers to
|
||||
/// one HTTP call, and any tool call in the reply comes back inert for Rust's
|
||||
/// own evaluator to drive the next round, exactly as the connector lane
|
||||
/// already does.
|
||||
pub(super) struct LocalInferencePort {
|
||||
client: reqwest::Client,
|
||||
endpoint: SocketAddr,
|
||||
endpoint: Endpoint,
|
||||
prefix: String,
|
||||
caller_runtime_id: String,
|
||||
/// Read only by the tests that pin which timeout a constructor resolves
|
||||
|
|
@ -138,16 +193,17 @@ pub(super) struct LocalInferencePort {
|
|||
}
|
||||
|
||||
impl LocalInferencePort {
|
||||
/// The loopback check happens here, in the constructor, so a non-loopback
|
||||
/// endpoint cannot reach this port through any path that builds one —
|
||||
/// not only the one `open_inference` exercises today. The client is
|
||||
/// built with `.no_proxy()` so an environment proxy variable can never
|
||||
/// carry this request, or a `proxy-authorization` header derived from a
|
||||
/// proxy URL's userinfo, off this loopback endpoint; and with redirects
|
||||
/// disabled, so a 3xx reply cannot re-POST the request's contents to a
|
||||
/// second endpoint this port never opened.
|
||||
/// The endpoint checks happen here, in the constructor, so neither a
|
||||
/// non-loopback local endpoint nor a non-https hosted one can reach this
|
||||
/// port through any path that builds one — not only the one
|
||||
/// `open_inference` exercises today. The client is built with
|
||||
/// `.no_proxy()` so an environment proxy variable can never carry this
|
||||
/// request, or a `proxy-authorization` header derived from a proxy URL's
|
||||
/// userinfo, off the endpoint; and with redirects disabled, so a 3xx
|
||||
/// reply cannot re-POST the request's contents (or the hosted bearer) to
|
||||
/// a second endpoint this port never opened.
|
||||
fn new(
|
||||
endpoint: SocketAddr,
|
||||
endpoint: Endpoint,
|
||||
prefix: impl Into<String>,
|
||||
caller_runtime_id: impl Into<String>,
|
||||
) -> Result<Self, ControllerOpenError> {
|
||||
|
|
@ -159,13 +215,34 @@ impl LocalInferencePort {
|
|||
/// owner; the other callers are tests that reach the timeout path without
|
||||
/// waiting `RESPONSE_TIMEOUT` out.
|
||||
fn with_timeout(
|
||||
endpoint: SocketAddr,
|
||||
endpoint: Endpoint,
|
||||
prefix: impl Into<String>,
|
||||
caller_runtime_id: impl Into<String>,
|
||||
timeout: std::time::Duration,
|
||||
) -> Result<Self, ControllerOpenError> {
|
||||
if !endpoint.ip().is_loopback() {
|
||||
return Err(ControllerOpenError::LocalEndpointNotLoopback { endpoint });
|
||||
Self::build(endpoint, prefix, caller_runtime_id, timeout, false)
|
||||
}
|
||||
|
||||
/// The one builder. `allow_plain_http` is false on every production path;
|
||||
/// the tests' hosted seam passes true so a hosted endpoint can be a
|
||||
/// loopback responder with no TLS.
|
||||
fn build(
|
||||
endpoint: Endpoint,
|
||||
prefix: impl Into<String>,
|
||||
caller_runtime_id: impl Into<String>,
|
||||
timeout: std::time::Duration,
|
||||
allow_plain_http: bool,
|
||||
) -> Result<Self, ControllerOpenError> {
|
||||
match &endpoint {
|
||||
Endpoint::Loopback(address) if !address.ip().is_loopback() => {
|
||||
return Err(ControllerOpenError::LocalEndpointNotLoopback { endpoint: *address });
|
||||
}
|
||||
Endpoint::Hosted { base_url, .. }
|
||||
if !allow_plain_http && !base_url.starts_with("https://") =>
|
||||
{
|
||||
return Err(ControllerOpenError::HostedEndpointNotHttps);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
Ok(Self {
|
||||
client: reqwest::Client::builder()
|
||||
|
|
@ -183,13 +260,11 @@ impl LocalInferencePort {
|
|||
}
|
||||
|
||||
async fn send(&self, body: Value) -> Result<LocalChatResponse, InferenceFault> {
|
||||
let response = self
|
||||
.client
|
||||
.post(format!("http://{}/v1/chat/completions", self.endpoint))
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(send_fault)?;
|
||||
let mut request = self.client.post(self.endpoint.url()).json(&body);
|
||||
if let Endpoint::Hosted { key, .. } = &self.endpoint {
|
||||
request = request.bearer_auth(key.as_str());
|
||||
}
|
||||
let response = request.send().await.map_err(send_fault)?;
|
||||
let status = response.status();
|
||||
if !status.is_success() {
|
||||
let detail = "the local inference endpoint returned a non-success status";
|
||||
|
|
@ -240,8 +315,9 @@ fn bad_reply() -> InferenceFault {
|
|||
|
||||
/// Lowers one prepared request to the OpenAI chat-completions body: the
|
||||
/// system message is the request's instructions, every input item maps to
|
||||
/// one message in order, and every offered tool is declared `strict`. No
|
||||
/// header but content type is ever set on the request this builds.
|
||||
/// one message in order, and every offered tool is declared `strict`. The
|
||||
/// body carries no credential; the hosted binding's bearer is the one header
|
||||
/// `send` adds.
|
||||
fn lower_request(prepared: &PreparedInference, prefix: &str) -> Result<Value, InferenceFault> {
|
||||
let request = &prepared.invocation.request;
|
||||
let model = request.model.strip_prefix(prefix).unwrap_or(&request.model);
|
||||
|
|
@ -345,6 +421,7 @@ fn assemble_output(
|
|||
arguments: call.function.arguments,
|
||||
});
|
||||
}
|
||||
let usage = response.usage.as_ref().and_then(LocalChatUsage::reported);
|
||||
let receipt = LocalInferenceReceipt {
|
||||
schema_id: LOCAL_RECEIPT_SCHEMA.to_owned(),
|
||||
request_id: request.request_id.clone(),
|
||||
|
|
@ -358,20 +435,21 @@ fn assemble_output(
|
|||
prompt_tokens: response
|
||||
.usage
|
||||
.as_ref()
|
||||
.map(|usage| usage.prompt_tokens)
|
||||
.and_then(|usage| usage.prompt_tokens)
|
||||
.unwrap_or_default(),
|
||||
completion_tokens: response
|
||||
.usage
|
||||
.as_ref()
|
||||
.map(|usage| usage.completion_tokens)
|
||||
.and_then(|usage| usage.completion_tokens)
|
||||
.unwrap_or_default(),
|
||||
};
|
||||
let receipt_bytes = rmp_serde::to_vec_named(&receipt)
|
||||
.map_err(|_| InferenceFault::new("the local inference receipt could not be encoded"))?;
|
||||
Ok(InferenceOutput::new(
|
||||
events,
|
||||
format!("sha256:{:x}", Sha256::digest(&receipt_bytes)),
|
||||
))
|
||||
let output = InferenceOutput::new(events, format!("sha256:{:x}", Sha256::digest(&receipt_bytes)));
|
||||
Ok(match usage {
|
||||
Some(usage) => output.with_usage(usage),
|
||||
None => output,
|
||||
})
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
|
|
@ -421,7 +499,7 @@ pub fn open_local_port_with_timeout(
|
|||
timeout: std::time::Duration,
|
||||
) -> Result<Arc<dyn InferencePort>, ControllerOpenError> {
|
||||
Ok(Arc::new(LocalInferencePort::with_timeout(
|
||||
binding.endpoint,
|
||||
Endpoint::Loopback(binding.endpoint),
|
||||
binding.model_prefix,
|
||||
binding.caller_runtime_id,
|
||||
timeout,
|
||||
|
|
@ -429,7 +507,44 @@ pub fn open_local_port_with_timeout(
|
|||
}
|
||||
|
||||
fn local_port(binding: LocalBinding) -> Result<LocalInferencePort, ControllerOpenError> {
|
||||
LocalInferencePort::new(binding.endpoint, binding.model_prefix, binding.caller_runtime_id)
|
||||
LocalInferencePort::new(
|
||||
Endpoint::Loopback(binding.endpoint),
|
||||
binding.model_prefix,
|
||||
binding.caller_runtime_id,
|
||||
)
|
||||
}
|
||||
|
||||
/// Builds the hosted port from its binding: reads the key file once, then the
|
||||
/// constructor checks the URL is https. Every failure names the field's
|
||||
/// error and never the path, the URL or the key.
|
||||
pub(super) fn open_hosted_port(
|
||||
binding: HostedBinding,
|
||||
) -> Result<Arc<dyn InferencePort>, ControllerOpenError> {
|
||||
Ok(Arc::new(hosted_port(binding)?))
|
||||
}
|
||||
|
||||
fn hosted_port(binding: HostedBinding) -> Result<LocalInferencePort, ControllerOpenError> {
|
||||
let key = read_key_file(&binding.key_path)?;
|
||||
LocalInferencePort::new(
|
||||
Endpoint::Hosted {
|
||||
base_url: binding.base_url,
|
||||
key,
|
||||
},
|
||||
binding.model_prefix,
|
||||
binding.caller_runtime_id,
|
||||
)
|
||||
}
|
||||
|
||||
/// The key file's whole content, minus one trailing newline run. Empty, not
|
||||
/// UTF-8, or padded with whitespace is unreadable.
|
||||
fn read_key_file(path: &std::path::Path) -> Result<Zeroizing<String>, ControllerOpenError> {
|
||||
let bytes = Zeroizing::new(std::fs::read(path).map_err(|_| ControllerOpenError::HostedKeyUnreadable)?);
|
||||
let raw = std::str::from_utf8(bytes.as_slice()).map_err(|_| ControllerOpenError::HostedKeyUnreadable)?;
|
||||
let key = raw.trim_end_matches(['\r', '\n']);
|
||||
if key.is_empty() || key.len() != key.trim().len() {
|
||||
return Err(ControllerOpenError::HostedKeyUnreadable);
|
||||
}
|
||||
Ok(Zeroizing::new(key.to_owned()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
@ -663,7 +778,7 @@ mod tests {
|
|||
let _guard = client_build_lock()
|
||||
.lock()
|
||||
.expect("the client-build lock is never poisoned");
|
||||
LocalInferencePort::new(endpoint, DEFAULT_LOCAL_MODEL_PREFIX, TEST_RUNTIME)
|
||||
LocalInferencePort::new(Endpoint::Loopback(endpoint), DEFAULT_LOCAL_MODEL_PREFIX, TEST_RUNTIME)
|
||||
.expect("a loopback endpoint opens")
|
||||
}
|
||||
|
||||
|
|
@ -824,6 +939,7 @@ mod tests {
|
|||
model_prefix: DEFAULT_LOCAL_MODEL_PREFIX.into(),
|
||||
caller_runtime_id: TEST_RUNTIME.into(),
|
||||
}),
|
||||
None,
|
||||
&[TEST_MODEL],
|
||||
)
|
||||
.err()
|
||||
|
|
@ -903,7 +1019,7 @@ mod tests {
|
|||
Some(Arc::clone(&connector_port)),
|
||||
Some(Arc::clone(&sdk_port)),
|
||||
"claude",
|
||||
Some(("claude-local/".to_owned(), Arc::clone(&local_port))),
|
||||
vec![("claude-local/".to_owned(), Arc::clone(&local_port))],
|
||||
);
|
||||
assert!(Arc::ptr_eq(
|
||||
routed.route("claude-local/llama-8b").unwrap(),
|
||||
|
|
@ -930,6 +1046,7 @@ mod tests {
|
|||
model_prefix: "claude".into(),
|
||||
caller_runtime_id: TEST_RUNTIME.into(),
|
||||
}),
|
||||
None,
|
||||
&["claude-opus-5"],
|
||||
)
|
||||
.err()
|
||||
|
|
@ -1212,6 +1329,7 @@ mod tests {
|
|||
model_prefix: String::new(),
|
||||
caller_runtime_id: TEST_RUNTIME.into(),
|
||||
}),
|
||||
None,
|
||||
&[TEST_MODEL],
|
||||
)
|
||||
.err()
|
||||
|
|
@ -1237,6 +1355,7 @@ mod tests {
|
|||
model_prefix: String::new(),
|
||||
}),
|
||||
None,
|
||||
None,
|
||||
&[TEST_MODEL],
|
||||
)
|
||||
.err()
|
||||
|
|
@ -1613,7 +1732,7 @@ mod tests {
|
|||
let _guard = client_build_lock()
|
||||
.lock()
|
||||
.expect("the client-build lock is never poisoned");
|
||||
LocalInferencePort::with_timeout(endpoint, DEFAULT_LOCAL_MODEL_PREFIX, TEST_RUNTIME, timeout)
|
||||
LocalInferencePort::with_timeout(Endpoint::Loopback(endpoint), DEFAULT_LOCAL_MODEL_PREFIX, TEST_RUNTIME, timeout)
|
||||
.expect("a loopback endpoint opens")
|
||||
}
|
||||
|
||||
|
|
@ -1706,7 +1825,7 @@ mod tests {
|
|||
let endpoint: SocketAddr = "127.0.0.1:9".parse().unwrap();
|
||||
let constructed = {
|
||||
let _guard = client_build_lock().lock().expect("the client-build lock is never poisoned");
|
||||
LocalInferencePort::new(endpoint, DEFAULT_LOCAL_MODEL_PREFIX, TEST_RUNTIME)
|
||||
LocalInferencePort::new(Endpoint::Loopback(endpoint), DEFAULT_LOCAL_MODEL_PREFIX, TEST_RUNTIME)
|
||||
.expect("a loopback endpoint opens")
|
||||
};
|
||||
assert_eq!(constructed.response_timeout, RESPONSE_TIMEOUT);
|
||||
|
|
@ -1751,4 +1870,298 @@ mod tests {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- The hosted binding and token usage (cut hosted-lane) ----
|
||||
|
||||
/// A synthetic bearer, generated for the test and written to a temp key
|
||||
/// file, so the production reader is the path under test.
|
||||
const SYNTHETIC_KEY: &str = "synthetic-bearer-0123456789abcdef";
|
||||
|
||||
fn write_key_file(directory: &tempfile::TempDir, content: &[u8]) -> PathBuf {
|
||||
let path = directory.path().join("provider.key");
|
||||
std::fs::write(&path, content).unwrap();
|
||||
path
|
||||
}
|
||||
|
||||
fn hosted_binding(base_url: &str, key_path: PathBuf) -> HostedBinding {
|
||||
HostedBinding {
|
||||
base_url: base_url.into(),
|
||||
key_path,
|
||||
model_prefix: DEFAULT_HOSTED_MODEL_PREFIX.into(),
|
||||
caller_runtime_id: TEST_RUNTIME.into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The test-only seam: the production key reader and the production
|
||||
/// builder, with the https requirement lifted so the endpoint can be a
|
||||
/// loopback responder with no TLS.
|
||||
fn hosted_port_over_plain_http(base_url: String, key_path: &std::path::Path) -> LocalInferencePort {
|
||||
let _guard = client_build_lock()
|
||||
.lock()
|
||||
.expect("the client-build lock is never poisoned");
|
||||
LocalInferencePort::build(
|
||||
Endpoint::Hosted {
|
||||
base_url,
|
||||
key: read_key_file(key_path).expect("the synthetic key file reads"),
|
||||
},
|
||||
DEFAULT_HOSTED_MODEL_PREFIX,
|
||||
TEST_RUNTIME,
|
||||
RESPONSE_TIMEOUT,
|
||||
true,
|
||||
)
|
||||
.expect("the hosted seam opens")
|
||||
}
|
||||
|
||||
fn usage_reply(usage: Value) -> String {
|
||||
json!({
|
||||
"id": "resp-usage",
|
||||
"choices": [{
|
||||
"message": {"role": "assistant", "content": "ok", "tool_calls": []},
|
||||
"finish_reason": "stop",
|
||||
}],
|
||||
"usage": usage,
|
||||
})
|
||||
.to_string()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hosted_binding_refuses_plain_http() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let key_path = write_key_file(&directory, SYNTHETIC_KEY.as_bytes());
|
||||
for url in ["http://api.example.test", "ftp://api.example.test", "api.example.test", ""] {
|
||||
let error = open_hosted_port(hosted_binding(url, key_path.clone()))
|
||||
.err()
|
||||
.expect("a non-https hosted endpoint opened");
|
||||
assert!(matches!(error, ControllerOpenError::HostedEndpointNotHttps), "{url}");
|
||||
assert!(!error.to_string().contains("example"), "the refusal echoed the URL");
|
||||
}
|
||||
let through_open_inference = open_inference(
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
Some(hosted_binding("http://api.example.test", key_path.clone())),
|
||||
&["hosted/deepseek-chat"],
|
||||
)
|
||||
.err()
|
||||
.expect("open_inference opened a plain-http hosted endpoint");
|
||||
assert!(matches!(through_open_inference, ControllerOpenError::HostedEndpointNotHttps));
|
||||
assert!(open_hosted_port(hosted_binding("https://api.example.test", key_path)).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hosted_binding_refuses_an_empty_key_file() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let absent = directory.path().join("absent.key");
|
||||
let cases: Vec<(&str, PathBuf)> = vec![
|
||||
("empty", write_key_file(&directory, b"")),
|
||||
("only a newline", write_key_file(&directory, b"\n")),
|
||||
("padded", write_key_file(&directory, b" key-with-padding \n")),
|
||||
("not utf-8", write_key_file(&directory, &[0xff, 0xfe, 0xfd])),
|
||||
("absent", absent),
|
||||
];
|
||||
for (label, path) in cases {
|
||||
let canary = path.display().to_string();
|
||||
let error = open_hosted_port(hosted_binding("https://api.example.test", path))
|
||||
.err()
|
||||
.unwrap_or_else(|| panic!("{label}: a bad key file opened"));
|
||||
assert!(matches!(error, ControllerOpenError::HostedKeyUnreadable), "{label}");
|
||||
let rendered = format!("{error} | {error:?}");
|
||||
assert!(!rendered.contains(&canary), "{label}: the path leaked");
|
||||
assert!(!rendered.contains("provider.key"), "{label}: the file name leaked");
|
||||
}
|
||||
// A key file that ends in the usual newline is the key without it.
|
||||
let ok = write_key_file(&directory, b"fine-key\r\n");
|
||||
assert_eq!(read_key_file(&ok).unwrap().as_str(), "fine-key");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn hosted_request_carries_the_bearer_and_loopback_carries_none() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let key_path = write_key_file(&directory, format!("{SYNTHETIC_KEY}\n").as_bytes());
|
||||
|
||||
let hosted_responder = ScriptedResponder::start(vec![(200, text_reply("ok"))]).await;
|
||||
let hosted = hosted_port_over_plain_http(format!("http://{}", hosted_responder.endpoint()), &key_path);
|
||||
let output = infer_once(&hosted).await.expect("the hosted port infers");
|
||||
assert_eq!(output.events, vec![InferenceEvent::Text("ok".into())]);
|
||||
let header_block = hosted_responder.headers()[0].to_ascii_lowercase();
|
||||
assert!(
|
||||
header_block.contains(&format!("authorization: bearer {}", SYNTHETIC_KEY.to_ascii_lowercase())),
|
||||
"{header_block}"
|
||||
);
|
||||
|
||||
let local_responder = ScriptedResponder::start(vec![(200, text_reply("ok"))]).await;
|
||||
infer_once(&port(local_responder.endpoint())).await.expect("the local port infers");
|
||||
assert!(!local_responder.headers()[0].to_ascii_lowercase().contains("authorization"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_hosted_request_strips_its_prefix_and_names_the_v1_path() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let key_path = write_key_file(&directory, SYNTHETIC_KEY.as_bytes());
|
||||
let responder = ScriptedResponder::start(vec![(200, text_reply("ok"))]).await;
|
||||
// A trailing slash on the origin must not double the path separator.
|
||||
let hosted = hosted_port_over_plain_http(format!("http://{}/", responder.endpoint()), &key_path);
|
||||
let request = tool_request(
|
||||
CommandId::new(),
|
||||
0,
|
||||
InferencePurpose::Persona,
|
||||
"hosted/deepseek-chat",
|
||||
"Respond only in natural prose.",
|
||||
vec![CodexInputItem::UserText { text: "Say something true.".into() }],
|
||||
Vec::<CodexToolDefinition>::new(),
|
||||
RequestShape { max_output_tokens: 1_200, parallel_tool_calls: false },
|
||||
)
|
||||
.expect("the request builds");
|
||||
let prepared = hosted.prepare(request).unwrap();
|
||||
hosted.infer(prepared).await.expect("the hosted port infers");
|
||||
assert!(responder.headers()[0].starts_with("POST /v1/chat/completions HTTP/1.1"));
|
||||
let body: Value = serde_json::from_str(&responder.bodies()[0]).unwrap();
|
||||
assert_eq!(body["model"], "deepseek-chat");
|
||||
}
|
||||
|
||||
/// The bearer must not follow a redirect, and no fault, log line or
|
||||
/// receipt may carry the key or the endpoint.
|
||||
#[tokio::test]
|
||||
async fn the_hosted_bearer_is_never_sent_to_a_redirect_target_or_echoed() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let key_path = write_key_file(&directory, SYNTHETIC_KEY.as_bytes());
|
||||
let target = ScriptedResponder::start(vec![(200, text_reply("reached"))]).await;
|
||||
let redirecting = start_redirecting_to(target.endpoint()).await;
|
||||
let hosted = hosted_port_over_plain_http(format!("http://{redirecting}"), &key_path);
|
||||
let fault = infer_once(&hosted).await.expect_err("a redirect produced an output");
|
||||
assert!(target.headers().is_empty(), "the redirect target was reached");
|
||||
|
||||
let refusing = ScriptedResponder::start(vec![(401, "{}".to_owned())]).await;
|
||||
let hosted = hosted_port_over_plain_http(format!("http://{}", refusing.endpoint()), &key_path);
|
||||
let unauthorised = infer_once(&hosted).await.expect_err("a 401 produced an output");
|
||||
let ok_responder = ScriptedResponder::start(vec![(200, text_reply("ok"))]).await;
|
||||
let hosted = hosted_port_over_plain_http(format!("http://{}", ok_responder.endpoint()), &key_path);
|
||||
let output = infer_once(&hosted).await.expect("the hosted port infers");
|
||||
|
||||
let rendered = format!(
|
||||
"{fault} | {fault:?} | {unauthorised} | {unauthorised:?} | {output:?}"
|
||||
);
|
||||
assert!(!rendered.contains(SYNTHETIC_KEY), "the key leaked: {rendered}");
|
||||
for addr in [redirecting, refusing.endpoint(), ok_responder.endpoint()] {
|
||||
assert!(!rendered.contains(&addr.port().to_string()), "an endpoint leaked: {rendered}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_reaches_inference_output() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let key_path = write_key_file(&directory, SYNTHETIC_KEY.as_bytes());
|
||||
let responder = ScriptedResponder::start(vec![
|
||||
(
|
||||
200,
|
||||
usage_reply(json!({
|
||||
"prompt_tokens": 120,
|
||||
"completion_tokens": 7,
|
||||
"prompt_cache_hit_tokens": 100,
|
||||
"prompt_cache_miss_tokens": 20,
|
||||
})),
|
||||
),
|
||||
(200, usage_reply(json!({"prompt_tokens": 31, "completion_tokens": 4}))),
|
||||
])
|
||||
.await;
|
||||
for (hosted, expected) in [
|
||||
(
|
||||
true,
|
||||
TokenUsage { prompt: 120, completion: 7, cached_prompt: Some(100) },
|
||||
),
|
||||
(
|
||||
false,
|
||||
TokenUsage { prompt: 31, completion: 4, cached_prompt: None },
|
||||
),
|
||||
] {
|
||||
let output = if hosted {
|
||||
let port =
|
||||
hosted_port_over_plain_http(format!("http://{}", responder.endpoint()), &key_path);
|
||||
infer_once(&port).await.unwrap()
|
||||
} else {
|
||||
infer_once(&port(responder.endpoint())).await.unwrap()
|
||||
};
|
||||
assert_eq!(output.usage(), Some(expected));
|
||||
}
|
||||
}
|
||||
|
||||
/// A reply without usage is a typed absence. The receipt keeps the zero
|
||||
/// counts it always carried (the local lane's receipt is unchanged), so
|
||||
/// the digest equals the digest of a reply that reported zeroes, while the
|
||||
/// output tells the two apart.
|
||||
#[tokio::test]
|
||||
async fn a_reply_without_usage_is_none_and_the_local_receipt_is_unchanged() {
|
||||
let absent = json!({
|
||||
"id": "resp-usage",
|
||||
"choices": [{
|
||||
"message": {"role": "assistant", "content": "ok", "tool_calls": []},
|
||||
"finish_reason": "stop",
|
||||
}],
|
||||
})
|
||||
.to_string();
|
||||
let responder = ScriptedResponder::start(vec![
|
||||
(200, absent),
|
||||
(200, usage_reply(json!({"prompt_tokens": 0, "completion_tokens": 0}))),
|
||||
(200, usage_reply(json!({"prompt_tokens": 9}))),
|
||||
(200, usage_reply(json!({"prompt_tokens": null, "completion_tokens": 3}))),
|
||||
])
|
||||
.await;
|
||||
let port = port(responder.endpoint());
|
||||
let none = infer_once(&port).await.unwrap();
|
||||
let zeroes = infer_once(&port).await.unwrap();
|
||||
let half = infer_once(&port).await.unwrap();
|
||||
let null = infer_once(&port).await.unwrap();
|
||||
assert_eq!(none.usage(), None);
|
||||
assert_eq!(zeroes.usage(), Some(TokenUsage { prompt: 0, completion: 0, cached_prompt: None }));
|
||||
assert_eq!(half.usage(), None, "a reply missing a count reported a zero");
|
||||
assert_eq!(null.usage(), None);
|
||||
assert_eq!(none.receipt_digest(), zeroes.receipt_digest());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_hosted_prefix_routes_to_the_hosted_port_and_shares_no_prefix() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let key_path = write_key_file(&directory, SYNTHETIC_KEY.as_bytes());
|
||||
let local_binding = || LocalBinding {
|
||||
endpoint: "127.0.0.1:1".parse().unwrap(),
|
||||
model_prefix: DEFAULT_LOCAL_MODEL_PREFIX.into(),
|
||||
caller_runtime_id: TEST_RUNTIME.into(),
|
||||
};
|
||||
let hosted = || hosted_binding("https://api.example.test", key_path.clone());
|
||||
let models = ["local/llama-8b", "hosted/deepseek-chat"];
|
||||
assert!(open_inference(None, None, Some(local_binding()), Some(hosted()), &models).is_ok());
|
||||
// Without the hosted binding its model has no owner: loud, not a fallback.
|
||||
assert!(matches!(
|
||||
open_inference(None, None, Some(local_binding()), None, &models),
|
||||
Err(ControllerOpenError::UnroutableModel { .. })
|
||||
));
|
||||
let mut clashing = hosted();
|
||||
clashing.model_prefix = DEFAULT_LOCAL_MODEL_PREFIX.into();
|
||||
assert!(matches!(
|
||||
open_inference(None, None, Some(local_binding()), Some(clashing), &["local/x"]),
|
||||
Err(ControllerOpenError::SharedModelPrefix { .. })
|
||||
));
|
||||
let mut empty = hosted();
|
||||
empty.model_prefix = String::new();
|
||||
assert!(matches!(
|
||||
open_inference(None, None, None, Some(empty), &["x"]),
|
||||
Err(ControllerOpenError::EmptyModelPrefix { transport: "hosted" })
|
||||
));
|
||||
|
||||
// Routing itself: each prefix reaches its own port.
|
||||
let local_port = Arc::new(port("127.0.0.1:1".parse().unwrap())) as Arc<dyn InferencePort>;
|
||||
let hosted_port = Arc::new(port("127.0.0.1:1".parse().unwrap())) as Arc<dyn InferencePort>;
|
||||
let routed = RoutedInferencePort::new(
|
||||
None,
|
||||
None,
|
||||
"claude",
|
||||
vec![
|
||||
("local/".to_owned(), Arc::clone(&local_port)),
|
||||
("hosted/".to_owned(), Arc::clone(&hosted_port)),
|
||||
],
|
||||
);
|
||||
assert!(Arc::ptr_eq(routed.route("local/a").unwrap(), &local_port));
|
||||
assert!(Arc::ptr_eq(routed.route("hosted/a").unwrap(), &hosted_port));
|
||||
assert!(routed.route("other").is_none());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
//! its own, computes no tool result, and holds no credential.
|
||||
|
||||
use super::controllers::{
|
||||
InferenceEvent, InferenceFault, InferenceOutput, InferencePort, InferenceRequest,
|
||||
InferenceEvent, InferenceFault, InferenceOutput, InferencePort, InferenceRequest, TokenUsage,
|
||||
PreparedInference, REQUEST_EXPIRY, RESPONSE_TIMEOUT, ToolResultOracle, call_id_is_valid,
|
||||
tool_name_is_valid, unix_ms,
|
||||
};
|
||||
|
|
@ -121,6 +121,22 @@ pub(super) struct SdkModelUsage {
|
|||
pub(super) cache_creation_input_tokens: u64,
|
||||
}
|
||||
|
||||
/// The reply's token usage summed over the models the run used. The whole
|
||||
/// prompt is the fresh input plus both cache figures (Anthropic reports them
|
||||
/// apart); the cached part is what was read from cache. A run that reported
|
||||
/// no usage at all is `None`, never a zero.
|
||||
fn sdk_usage(usage: &[SdkModelUsage]) -> Option<TokenUsage> {
|
||||
if usage.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let sum = |field: fn(&SdkModelUsage) -> u64| usage.iter().map(field).sum::<u64>();
|
||||
Some(TokenUsage {
|
||||
prompt: sum(|u| u.input_tokens) + sum(|u| u.cache_read_input_tokens) + sum(|u| u.cache_creation_input_tokens),
|
||||
completion: sum(|u| u.output_tokens),
|
||||
cached_prompt: Some(sum(|u| u.cache_read_input_tokens)),
|
||||
})
|
||||
}
|
||||
|
||||
/// The failures the sidecar is allowed to name. It reports a reason; this
|
||||
/// module assigns the disposition, so TypeScript can never quarantine the
|
||||
/// world's cognition by naming an integrity violation directly.
|
||||
|
|
@ -654,9 +670,10 @@ fn assemble_output(
|
|||
};
|
||||
let receipt_bytes = rmp_serde::to_vec_named(&receipt)
|
||||
.map_err(|_| InferenceFault::new("the SDK receipt could not be encoded"))?;
|
||||
Ok(InferenceOutput {
|
||||
events: lowered,
|
||||
receipt_digest: format!("sha256:{:x}", Sha256::digest(&receipt_bytes)),
|
||||
let output = InferenceOutput::new(lowered, format!("sha256:{:x}", Sha256::digest(&receipt_bytes)));
|
||||
Ok(match sdk_usage(&receipt.usage) {
|
||||
Some(usage) => output.with_usage(usage),
|
||||
None => output,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -713,10 +730,10 @@ pub(super) struct RoutedInferencePort {
|
|||
connector: Option<Arc<dyn InferencePort>>,
|
||||
sdk: Option<Arc<dyn InferencePort>>,
|
||||
sdk_model_prefix: String,
|
||||
/// The local port's own claimed prefix, carried beside it rather than in a
|
||||
/// third bare `String` field, so `route` can compare its length against
|
||||
/// the SDK's claim without a third parallel field to keep in sync.
|
||||
local: Option<(String, Arc<dyn InferencePort>)>,
|
||||
/// The OpenAI-compatible ports (the loopback one and the hosted one), each
|
||||
/// with its own claimed prefix carried beside it, so `route` can compare
|
||||
/// lengths without a parallel field per port to keep in sync.
|
||||
claims: Vec<(String, Arc<dyn InferencePort>)>,
|
||||
}
|
||||
|
||||
impl RoutedInferencePort {
|
||||
|
|
@ -724,13 +741,13 @@ impl RoutedInferencePort {
|
|||
connector: Option<Arc<dyn InferencePort>>,
|
||||
sdk: Option<Arc<dyn InferencePort>>,
|
||||
sdk_model_prefix: impl Into<String>,
|
||||
local: Option<(String, Arc<dyn InferencePort>)>,
|
||||
claims: Vec<(String, Arc<dyn InferencePort>)>,
|
||||
) -> Self {
|
||||
Self {
|
||||
connector,
|
||||
sdk,
|
||||
sdk_model_prefix: sdk_model_prefix.into(),
|
||||
local,
|
||||
claims,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -747,9 +764,10 @@ impl RoutedInferencePort {
|
|||
// the connector, which is what keeps a typo'd model prefix loud
|
||||
// instead of silently reaching the wrong backend.
|
||||
let local_claim = self
|
||||
.local
|
||||
.as_ref()
|
||||
.claims
|
||||
.iter()
|
||||
.filter(|(prefix, _)| model.starts_with(prefix.as_str()))
|
||||
.max_by_key(|(prefix, _)| prefix.len())
|
||||
.map(|(prefix, port)| (prefix.len(), port));
|
||||
let sdk_claim = model
|
||||
.starts_with(&self.sdk_model_prefix)
|
||||
|
|
@ -1335,7 +1353,7 @@ mod tests {
|
|||
Some(Arc::clone(&connector)),
|
||||
Some(Arc::clone(&sdk)),
|
||||
DEFAULT_SDK_MODEL_PREFIX,
|
||||
None,
|
||||
Vec::new(),
|
||||
);
|
||||
assert!(Arc::ptr_eq(routed.route("claude-opus-5").unwrap(), &sdk));
|
||||
assert!(Arc::ptr_eq(
|
||||
|
|
@ -1346,7 +1364,7 @@ mod tests {
|
|||
Some(Arc::clone(&connector)),
|
||||
Some(Arc::clone(&sdk)),
|
||||
"gpt-",
|
||||
None,
|
||||
Vec::new(),
|
||||
);
|
||||
assert!(Arc::ptr_eq(moved.route("gpt-5.6-terra").unwrap(), &sdk));
|
||||
assert!(Arc::ptr_eq(
|
||||
|
|
@ -1385,11 +1403,11 @@ mod tests {
|
|||
};
|
||||
|
||||
assert!(matches!(
|
||||
open_inference(None, Some(sdk()), None, &models("gpt-5.6-terra").each()),
|
||||
open_inference(None, Some(sdk()), None, None, &models("gpt-5.6-terra").each()),
|
||||
Err(ControllerOpenError::UnroutableModel { .. })
|
||||
));
|
||||
assert!(matches!(
|
||||
open_inference(Some(connector()), None, None, &models("claude-opus-5").each()),
|
||||
open_inference(Some(connector()), None, None, None, &models("claude-opus-5").each()),
|
||||
Err(ControllerOpenError::UnroutableModel { .. })
|
||||
));
|
||||
assert!(matches!(
|
||||
|
|
@ -1401,12 +1419,13 @@ mod tests {
|
|||
model_prefix: DEFAULT_SDK_MODEL_PREFIX.into(),
|
||||
}),
|
||||
None,
|
||||
None,
|
||||
&models(TEST_MODEL).each(),
|
||||
),
|
||||
Err(ControllerOpenError::SdkSidecarMissing { .. })
|
||||
));
|
||||
assert!(
|
||||
open_inference(Some(connector()), Some(sdk()), None, &models(TEST_MODEL).each()).is_ok(),
|
||||
open_inference(Some(connector()), Some(sdk()), None, None, &models(TEST_MODEL).each()).is_ok(),
|
||||
"a configuration where every model routes did not open"
|
||||
);
|
||||
}
|
||||
|
|
@ -1759,6 +1778,7 @@ mod tests {
|
|||
model_prefix: DEFAULT_SDK_MODEL_PREFIX.into(),
|
||||
}),
|
||||
None,
|
||||
None,
|
||||
&models(TEST_MODEL).each(),
|
||||
)
|
||||
.expect("the SDK model routes");
|
||||
|
|
@ -1881,7 +1901,7 @@ mod tests {
|
|||
)) as Arc<dyn InferencePort>),
|
||||
None,
|
||||
DEFAULT_SDK_MODEL_PREFIX,
|
||||
None,
|
||||
Vec::new(),
|
||||
);
|
||||
assert!(connector_only.route(TEST_MODEL).is_none());
|
||||
let prepared =
|
||||
|
|
@ -1913,6 +1933,7 @@ mod tests {
|
|||
model_prefix: DEFAULT_SDK_MODEL_PREFIX.into(),
|
||||
}),
|
||||
None,
|
||||
None,
|
||||
&models(TEST_MODEL).each(),
|
||||
)
|
||||
.is_ok()
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@
|
|||
//! Projector may call it.
|
||||
|
||||
use super::controllers::{
|
||||
self, ControllerError, InferenceEvent, InferenceOutput, InferencePurpose, InferenceRequest,
|
||||
self, ControllerError, InferenceEvent, InferenceOutput, InferencePurpose, InferenceRequest, TokenUsage,
|
||||
RequestShape, tool_request,
|
||||
};
|
||||
use super::patch::{
|
||||
|
|
@ -1070,6 +1070,12 @@ impl InferenceOutput {
|
|||
pub fn receipt_digest(&self) -> &str {
|
||||
&self.receipt_digest
|
||||
}
|
||||
|
||||
/// The tokens the provider reported for this reply; `None` when it
|
||||
/// reported none.
|
||||
pub fn usage(&self) -> Option<TokenUsage> {
|
||||
self.usage
|
||||
}
|
||||
}
|
||||
|
||||
/// The canonical id as the patch vocabulary spells it: the bare UUID text,
|
||||
|
|
|
|||
Loading…
Reference in a new issue