Projector: an overrun ends on the last whole sentence inside the budget; every render and judge call reports the connector's usage
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
33b42226f5
commit
432d76e994
3 changed files with 108 additions and 18 deletions
|
|
@ -30,7 +30,7 @@ use crate::address::{Addressee, is_for, localpart};
|
|||
use crate::chat_client::{Chat, ChatPending, Form};
|
||||
use crate::heartbeat::HeartbeatView;
|
||||
|
||||
pub use narrate::{KitModel, Model, connector_key};
|
||||
pub use narrate::{Answer, KitModel, Model, Spent, connector_key};
|
||||
pub use select::TurnKind;
|
||||
pub use tuning::{TUNING_FILE, TUNING_ROOT};
|
||||
|
||||
|
|
@ -197,7 +197,8 @@ fn packet_json(packet: &Packet) -> Value {
|
|||
/// store answers a repeated id with its first answer.
|
||||
fn narrate_one(model: &dyn Model, instance: &str, packet: &Packet, tuning: &tuning::Tuning, tag: &str, now: DateTime<Utc>) -> Result<Value> {
|
||||
let request_id = format!("{AGENT}:{instance}:project:{tag}:{}:{}", packet.seed, now.timestamp_millis());
|
||||
let raw = model.ask(&request_id, &tuning.projector.model, &projector_prompt(packet, tuning))?;
|
||||
let answer = model.ask(&request_id, &tuning.projector.model, &projector_prompt(packet, tuning))?;
|
||||
let raw = answer.text;
|
||||
let budget = word_budget(packet.kind, tuning);
|
||||
let narration = cap_words(&raw, budget);
|
||||
let ids: Vec<String> = packet.selected.iter().map(|pick| pick.id.clone()).collect();
|
||||
|
|
@ -209,6 +210,7 @@ fn narrate_one(model: &dyn Model, instance: &str, packet: &Packet, tuning: &tuni
|
|||
"words": words(&narration),
|
||||
"cut_at_budget": words(&raw) > budget,
|
||||
"field_names": field_names(&narration, &named),
|
||||
"usage": answer.usage,
|
||||
"narration": narration,
|
||||
}))
|
||||
}
|
||||
|
|
@ -287,10 +289,10 @@ pub fn project(organs: &Organs, request: &Request, now: DateTime<Utc>) -> Result
|
|||
let (first, second) = if swapped { (b, a) } else { (a, b) };
|
||||
let text = narration["narration"].as_str().unwrap_or_default();
|
||||
let answer = model.ask(&format!("{AGENT}:{}:judge:{at}:{}", request.instance, now.timestamp_millis()), &tuning.projector.model, &judge_prompt(first, second, text))?;
|
||||
let letter = judged(&answer).map(|letter| if swapped { if letter == 'A' { 'B' } else { 'A' } } else { letter });
|
||||
let letter = judged(&answer.text).map(|letter| if swapped { if letter == 'A' { 'B' } else { 'A' } } else { letter });
|
||||
let correct = letter == Some(truth);
|
||||
right += usize::from(correct);
|
||||
verdicts.push(json!({ "from": truth.to_string(), "judged": letter.map(String::from), "correct": correct }));
|
||||
verdicts.push(json!({ "from": truth.to_string(), "judged": letter.map(String::from), "correct": correct, "usage": answer.usage }));
|
||||
}
|
||||
let accuracy = right as f64 / verdicts.len().max(1) as f64;
|
||||
report["against"] = json!({ "packet": packet_json(&others[0]), "renders": other_narrations, "distinct_2": distinct_2(&texts(&other_narrations)) });
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ use anyhow::{Result, anyhow, bail};
|
|||
use chrono::Utc;
|
||||
use ghostlight_persona_projection::{ProjectorPrompt, build_projector_prompt};
|
||||
use model_connector::{ConnectorClient, InputItem, ProviderRequest, ReasoningEffort, StopReason, TransportDisposition, TransportEventPayload, TransportInvocation, TransportOutcome};
|
||||
use serde::Serialize;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::select::{Packet, TurnKind};
|
||||
|
|
@ -20,7 +21,25 @@ use super::tuning::Tuning;
|
|||
/// One tool-less model turn. `KitModel` is the one live owner; a test answers with recorded text.
|
||||
pub trait Model {
|
||||
/// One turn of `model`: `prompt` is its one user text and `request_id` its identity on the connector.
|
||||
fn ask(&self, request_id: &str, model: &str, prompt: &str) -> Result<String>;
|
||||
fn ask(&self, request_id: &str, model: &str, prompt: &str) -> Result<Answer>;
|
||||
}
|
||||
|
||||
/// A model's answer, and what the turn cost as the connector counted it: none when the answerer counts nothing.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Answer {
|
||||
pub text: String,
|
||||
pub usage: Option<Spent>,
|
||||
}
|
||||
|
||||
/// One turn's tokens, by the model that ran it, as the connector's receipt reports them.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||||
pub struct Spent {
|
||||
pub model: String,
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub cache_read_tokens: u64,
|
||||
pub cache_write_tokens: u64,
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
}
|
||||
|
||||
/// What the connector's request asks of the model beside the user text. It names no membrane text: that is Ghostlight's.
|
||||
|
|
@ -56,7 +75,7 @@ impl KitModel {
|
|||
}
|
||||
|
||||
impl Model for KitModel {
|
||||
fn ask(&self, request_id: &str, model: &str, prompt: &str) -> Result<String> {
|
||||
fn ask(&self, request_id: &str, model: &str, prompt: &str) -> Result<Answer> {
|
||||
let mut request = ProviderRequest::new(request_id, request_id, model, INSTRUCTIONS, MAX_OUTPUT_TOKENS);
|
||||
request.input = vec![InputItem::UserText { text: prompt.to_owned() }];
|
||||
request.reasoning_effort = Some(ReasoningEffort::Low);
|
||||
|
|
@ -66,13 +85,23 @@ impl Model for KitModel {
|
|||
match result.disposition {
|
||||
TransportDisposition::Refused(reason) => bail!("the connector refused the call: {reason:?}"),
|
||||
TransportDisposition::Transported { events, receipt } => match receipt.outcome {
|
||||
TransportOutcome::Completed { stop_reason: StopReason::EndTurn, .. } => Ok(events
|
||||
.into_iter()
|
||||
.filter_map(|event| match event.payload {
|
||||
TransportEventPayload::TextDelta { text } => Some(text),
|
||||
_ => None,
|
||||
})
|
||||
.collect()),
|
||||
TransportOutcome::Completed { stop_reason: StopReason::EndTurn, usage, .. } => Ok(Answer {
|
||||
text: events
|
||||
.into_iter()
|
||||
.filter_map(|event| match event.payload {
|
||||
TransportEventPayload::TextDelta { text } => Some(text),
|
||||
_ => None,
|
||||
})
|
||||
.collect(),
|
||||
usage: Some(Spent {
|
||||
model: model.to_owned(),
|
||||
input_tokens: usage.input_tokens,
|
||||
output_tokens: usage.output_tokens,
|
||||
cache_read_tokens: usage.cache_read_tokens,
|
||||
cache_write_tokens: usage.cache_write_tokens,
|
||||
reasoning_tokens: usage.reasoning_tokens,
|
||||
}),
|
||||
}),
|
||||
TransportOutcome::Completed { stop_reason, .. } => bail!("the model's turn ended early: {stop_reason:?}"),
|
||||
TransportOutcome::Failed { kind, .. } => bail!("the provider failed the call: {kind:?}"),
|
||||
},
|
||||
|
|
@ -93,7 +122,9 @@ pub(crate) fn projector_prompt(packet: &Packet, tuning: &Tuning) -> String {
|
|||
build_projector_prompt(&ProjectorPrompt { identity: &packet.identity, typed_context: &packet.context, visible_stimulus: &packet.stimulus, domain_guidance: &guidance, word_budget: word_budget(packet.kind, tuning) })
|
||||
}
|
||||
|
||||
/// `text` cut after its `max`-th word; the model's count is not trusted.
|
||||
/// `text` held to `max` whitespace-separated words; the model's count is not trusted. An overrun is cut at the last
|
||||
/// sentence end inside the budget, so the stream ends on a whole sentence; only when the budget holds no sentence end
|
||||
/// is it cut after the `max`-th word.
|
||||
pub(crate) fn cap_words(text: &str, max: usize) -> String {
|
||||
let text = text.trim();
|
||||
let mut words = 0;
|
||||
|
|
@ -105,13 +136,36 @@ pub(crate) fn cap_words(text: &str, max: usize) -> String {
|
|||
in_word = true;
|
||||
words += 1;
|
||||
if words > max {
|
||||
return text[..at].trim_end().to_owned();
|
||||
let within = text[..at].trim_end();
|
||||
return last_sentence_end(within).map_or_else(|| within.to_owned(), |end| within[..end].to_owned());
|
||||
}
|
||||
}
|
||||
}
|
||||
text.to_owned()
|
||||
}
|
||||
|
||||
/// The byte just past the last sentence end in `text`: a `.`, `!`, `?` or `…`, with any closing quotes or brackets after
|
||||
/// it, followed by whitespace or the end of the text.
|
||||
fn last_sentence_end(text: &str) -> Option<usize> {
|
||||
let chars: Vec<(usize, char)> = text.char_indices().collect();
|
||||
let mut found = None;
|
||||
for (index, (at, c)) in chars.iter().enumerate() {
|
||||
if !matches!(c, '.' | '!' | '?' | '…') {
|
||||
continue;
|
||||
}
|
||||
let mut end = at + c.len_utf8();
|
||||
let mut next = index + 1;
|
||||
while let Some((close_at, close)) = chars.get(next).filter(|(_, close)| matches!(close, '"' | '\'' | '”' | '’' | ')' | ']')) {
|
||||
end = close_at + close.len_utf8();
|
||||
next += 1;
|
||||
}
|
||||
if chars.get(next).is_none_or(|(_, after)| after.is_whitespace()) {
|
||||
found = Some(end);
|
||||
}
|
||||
}
|
||||
found
|
||||
}
|
||||
|
||||
pub(crate) fn words(text: &str) -> usize {
|
||||
text.split_whitespace().count()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -457,6 +457,16 @@ fn the_narration_is_cut_at_its_word_budget() {
|
|||
assert_eq!(words(&cut), 150);
|
||||
assert!(cut.ends_with("w149"));
|
||||
assert_eq!(cap_words(" a b\n c ", 150), "a b\n c");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_overrun_is_cut_at_the_last_sentence_end_inside_the_budget() {
|
||||
assert_eq!(cap_words("One two three. Four five six! Seven eight nine ten", 8), "One two three. Four five six!");
|
||||
assert_eq!(cap_words("She said \"go.\" Then she left the room for good", 6), "She said \"go.\"");
|
||||
assert_eq!(cap_words("Is it late? It is… and quiet now", 5), "Is it late? It is…");
|
||||
assert_eq!(cap_words("Version 1.2 is out and running", 3), "Version 1.2 is", "a dot inside a token is no sentence end");
|
||||
assert_eq!(cap_words("one two three four five six", 4), "one two three four", "no sentence end inside the budget: the word cut");
|
||||
assert_eq!(cap_words("Short. Whole.", 4), "Short. Whole.", "inside the budget nothing is cut");
|
||||
let tuning = Tuning::default();
|
||||
assert_eq!((word_budget(TurnKind::Chat, &tuning), word_budget(TurnKind::Beat, &tuning)), (150, 90));
|
||||
}
|
||||
|
|
@ -575,9 +585,11 @@ impl Recorded {
|
|||
}
|
||||
|
||||
impl Model for Recorded {
|
||||
fn ask(&self, request_id: &str, model: &str, prompt: &str) -> Result<String> {
|
||||
fn ask(&self, request_id: &str, model: &str, prompt: &str) -> Result<Answer> {
|
||||
self.calls.borrow_mut().push((request_id.to_owned(), model.to_owned(), prompt.to_owned()));
|
||||
Ok(if prompt.starts_with("<!-- membrane:") { (self.narration)(prompt) } else { (self.judge)(prompt) })
|
||||
let text = if prompt.starts_with("<!-- membrane:") { (self.narration)(prompt) } else { (self.judge)(prompt) };
|
||||
let usage = Spent { model: model.to_owned(), input_tokens: prompt.len() as u64, output_tokens: text.len() as u64, cache_read_tokens: 7, cache_write_tokens: 3, reasoning_tokens: None };
|
||||
Ok(Answer { text, usage: Some(usage) })
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -802,7 +814,9 @@ fn a_kit_call_is_one_tool_less_turn_by_her_instance() {
|
|||
let backend = Scripted::ending(StopReason::EndTurn, &["I turn ", "it over."]);
|
||||
let (model, _root) = kit(Arc::clone(&backend), KIT_MODEL);
|
||||
let id = "mind-persona:eureka:project:a:1:1";
|
||||
assert_eq!(model.ask(id, KIT_MODEL, "The projector prompt.").unwrap(), "I turn it over.");
|
||||
let answer = model.ask(id, KIT_MODEL, "The projector prompt.").unwrap();
|
||||
assert_eq!(answer.text, "I turn it over.");
|
||||
assert_eq!(answer.usage, Some(Spent { model: KIT_MODEL.into(), input_tokens: 1, output_tokens: 1, cache_read_tokens: 0, cache_write_tokens: 0, reasoning_tokens: None }), "the receipt's usage, by the model that ran");
|
||||
let seen = backend.seen.lock().unwrap();
|
||||
let (caller, request) = &seen[0];
|
||||
assert_eq!(caller, INSTANCE);
|
||||
|
|
@ -1116,3 +1130,23 @@ fn every_range_is_validated_on_its_own() {
|
|||
edge.select.addressed_bytes = 1024;
|
||||
assert!(edge.validate().is_ok(), "the edges themselves are allowed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_render_and_every_judge_call_carries_its_usage() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let stored = Stored(state());
|
||||
let mut tense = state();
|
||||
tense["affect"]["socialBonds"][0]["tension"] = json!(0.9);
|
||||
let chat = mouth();
|
||||
let model = Recorded::discriminating();
|
||||
let organs = Organs { persona: Some(&stored), chat: Some(&chat), addressing: None, model: Some(&model) };
|
||||
let report = project(&organs, &request(dir.path(), Some(2), Some(tense)), now()).unwrap();
|
||||
for render in report["renders"].as_array().unwrap().iter().chain(report["against"]["renders"].as_array().unwrap()) {
|
||||
let usage = &render["usage"];
|
||||
assert_eq!((usage["model"].as_str(), usage["cache_read_tokens"].as_u64(), usage["cache_write_tokens"].as_u64()), (Some("claude-sonnet-5-5"), Some(7), Some(3)), "{render}");
|
||||
assert!(usage["input_tokens"].as_u64().unwrap() > 0 && usage["output_tokens"].as_u64().unwrap() > 0);
|
||||
}
|
||||
for verdict in report["judge"]["verdicts"].as_array().unwrap() {
|
||||
assert_eq!(verdict["usage"]["output_tokens"].as_u64(), Some(1), "the judge answered one letter: {verdict}");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue