usage-per-turn: every result writes one usage line to the unit's journal, from its per-turn usage
usage_line reads model_usage_since_last_result (never the cumulative modelUsage); write_result writes it after the result line, so Self runs and Persona turns share it. Unreported usage says "not reported", never zeros. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
f1d5c4039b
commit
e719aa3766
2 changed files with 109 additions and 1 deletions
|
|
@ -1051,14 +1051,46 @@ fn describe_result(event: &serde_json::Map<String, Value>) -> String {
|
|||
format!("{}: {}", result_head(event), result_text(event))
|
||||
}
|
||||
|
||||
/// What a result spent, as one journal line: its tokens by model for this turn alone, read from
|
||||
/// `model_usage_since_last_result`. `modelUsage` is the session's running total on a resumed
|
||||
/// session, so it is never the source. Usage the result did not report, or reported
|
||||
/// incompletely, is said so and never written as zeros. `subagent_stats.spawned` is the
|
||||
/// session's count so far, not the turn's (the fixture's second result spawns none and still
|
||||
/// reads 1), so it is named for what it is. No text of the result enters the line. This line is
|
||||
/// observation for the journal: nothing in the run reads it.
|
||||
fn usage_line(event: &serde_json::Map<String, Value>) -> String {
|
||||
let turn = event.get("result_index").and_then(Value::as_u64).map_or_else(|| "?".to_string(), |index| index.to_string());
|
||||
let segments: Option<Vec<String>> = event.get("model_usage_since_last_result").and_then(Value::as_object).filter(|models| !models.is_empty()).and_then(|models| {
|
||||
models
|
||||
.iter()
|
||||
.map(|(model, counts)| {
|
||||
let count = |name: &str| counts.get(name).and_then(Value::as_u64);
|
||||
Some(format!(
|
||||
"{model} input {} cache_read {} cache_creation {} output {}",
|
||||
count("inputTokens")?,
|
||||
count("cacheReadInputTokens")?,
|
||||
count("cacheCreationInputTokens")?,
|
||||
count("outputTokens")?
|
||||
))
|
||||
})
|
||||
.collect()
|
||||
});
|
||||
let Some(segments) = segments else { return format!("usage turn {turn}: not reported") };
|
||||
let spawned = event.get("subagent_stats").and_then(|stats| stats.get("spawned")).and_then(Value::as_u64);
|
||||
let tail = spawned.map(|spawned| format!(" subagents_spawned_total {spawned}")).unwrap_or_default();
|
||||
format!("usage turn {turn}: {}{tail}", segments.join("; "))
|
||||
}
|
||||
|
||||
/// What `claude` concluded, written to the unit's stdout so it reaches the
|
||||
/// journal: the transcript is a cache, and the result's words are kept here.
|
||||
/// The result's per-turn usage follows as its own line.
|
||||
fn write_result(line: &str, out: &mut dyn Write) -> std::io::Result<()> {
|
||||
let Ok(Value::Object(event)) = serde_json::from_str::<Value>(line) else { return Ok(()) };
|
||||
if event.get("type").and_then(Value::as_str) != Some("result") {
|
||||
return Ok(());
|
||||
}
|
||||
writeln!(out, "{}", describe_result(&event))
|
||||
writeln!(out, "{}", describe_result(&event))?;
|
||||
writeln!(out, "{}", usage_line(&event))
|
||||
}
|
||||
|
||||
/// The real spawner: the `claude` on the unit's `PATH`, its stderr in the
|
||||
|
|
|
|||
|
|
@ -496,6 +496,82 @@ fn a_result_events_text_reaches_the_journal_with_its_subtype_and_error_flag() {
|
|||
assert!(ran.out.contains("result subtype=success is_error=false: all done\n"), "{}", ran.out);
|
||||
}
|
||||
|
||||
/// What the journal holds for one stream line.
|
||||
fn journal_of(line: &str) -> String {
|
||||
let mut out = Vec::new();
|
||||
write_result(line, &mut out).unwrap();
|
||||
String::from_utf8(out).unwrap()
|
||||
}
|
||||
|
||||
/// The result envelopes of the real two-turn background-agent capture.
|
||||
fn bg_agent_results() -> Vec<String> {
|
||||
include_str!("claude-2.1.312-live-session-bg-agent.jsonl")
|
||||
.lines()
|
||||
.filter_map(|entry| serde_json::from_str::<Value>(entry).ok()?.get("line")?.as_str().map(str::to_string))
|
||||
.filter(|line| serde_json::from_str::<Value>(line).is_ok_and(|event| event["type"] == "result"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn usage_lines(journal: &str) -> Vec<&str> {
|
||||
journal.lines().filter(|line| line.starts_with("usage turn")).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_result_writes_its_per_turn_usage_and_not_the_sessions_total() {
|
||||
let results = bg_agent_results();
|
||||
assert_eq!(results.len(), 2);
|
||||
let lines: Vec<String> = results.iter().map(|line| journal_of(line)).collect();
|
||||
// The capture's modelUsage runs 20/10 then 40/20; the turn's own usage is 20/10 both times.
|
||||
assert_eq!(usage_lines(&lines[0]), ["usage turn 0: claude-opus-5-5 input 20 cache_read 0 cache_creation 0 output 10 subagents_spawned_total 1"]);
|
||||
assert_eq!(usage_lines(&lines[1]), ["usage turn 1: claude-opus-5-5 input 20 cache_read 0 cache_creation 0 output 10 subagents_spawned_total 1"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn each_model_in_the_turns_usage_is_its_own_segment() {
|
||||
let line = json!({
|
||||
"type": "result", "result_index": 3,
|
||||
"model_usage_since_last_result": {
|
||||
"claude-haiku-4-5": {"inputTokens": 3, "cacheReadInputTokens": 4, "cacheCreationInputTokens": 5, "outputTokens": 6},
|
||||
"claude-opus-5-5": {"inputTokens": 7, "cacheReadInputTokens": 8, "cacheCreationInputTokens": 9, "outputTokens": 10}
|
||||
}
|
||||
})
|
||||
.to_string();
|
||||
assert_eq!(
|
||||
usage_lines(&journal_of(&line)),
|
||||
["usage turn 3: claude-haiku-4-5 input 3 cache_read 4 cache_creation 5 output 6; claude-opus-5-5 input 7 cache_read 8 cache_creation 9 output 10"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unreported_usage_is_said_never_zero() {
|
||||
let whole = json!({"inputTokens": 1, "cacheReadInputTokens": 2, "cacheCreationInputTokens": 3, "outputTokens": 4});
|
||||
let mut cases = vec![json!({"type": "result", "result_index": 2}), json!({"type": "result", "result_index": 2, "model_usage_since_last_result": {}})];
|
||||
for missing in ["inputTokens", "cacheReadInputTokens", "cacheCreationInputTokens", "outputTokens"] {
|
||||
let mut entry = whole.clone();
|
||||
entry.as_object_mut().unwrap().remove(missing);
|
||||
// The whole entry is unknown, even beside a model that reported everything.
|
||||
cases.push(json!({"type": "result", "result_index": 2, "model_usage_since_last_result": {"a": whole, "b": entry}}));
|
||||
}
|
||||
for case in cases {
|
||||
let journal = journal_of(&case.to_string());
|
||||
assert_eq!(usage_lines(&journal), ["usage turn 2: not reported"], "{case}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_result_line_is_unchanged_and_first_and_the_usage_line_carries_none_of_its_text() {
|
||||
let line = json!({
|
||||
"type": "result", "subtype": "success", "is_error": false, "result": "all done", "result_index": 0,
|
||||
"model_usage_since_last_result": {"m": {"inputTokens": 1, "cacheReadInputTokens": 2, "cacheCreationInputTokens": 3, "outputTokens": 4}}
|
||||
})
|
||||
.to_string();
|
||||
let journal = journal_of(&line);
|
||||
let lines: Vec<&str> = journal.lines().collect();
|
||||
assert_eq!(lines, ["result subtype=success is_error=false: all done", "usage turn 0: m input 1 cache_read 2 cache_creation 3 output 4"]);
|
||||
// A line that is not a result writes nothing.
|
||||
assert_eq!(journal_of(&json!({"type": "assistant"}).to_string()), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_long_result_reaches_the_journal_whole() {
|
||||
let text = format!("a{}", "é".repeat(5000));
|
||||
|
|
|
|||
Loading…
Reference in a new issue