From f295a3a4fbf66eb8588e5bfd0ce1a50083c94262 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 23 Sep 2026 05:31:39 +0200 Subject: [PATCH] Cut 15: retire every unreadable play-turn row, not just legacy ones (PA.f184, PA.f185, PA.f188, PA.f189, PA.f192) play::PlayTurnStore::open's current-schema arm and its final catch-all still bail!ed unconditionally on a row it could not read, so a damaged or future-schema row (any PlayTurn field-set change that lands without moving STORE_SCHEMA) left the play card gone forever, exactly the PA.f177 gap but for the arms PA.f177 itself did not touch. Both now retire through the same retire_unreadable_row path the legacy arms use. upgrade_legacy_row's own three non-decode exits (an inner schema tag that disagrees with the envelope, and two compare-and-swap failure shapes) used to be indistinguishable from a genuine decode failure once open caught the Err, so a readable turn (Soul's probe: envelope v1, inner schema v2, decodes cleanly) was destroyed under a sidecar reason that falsely claimed it "did not decode". LegacyRowError now splits Decode from Other; only Decode retires, Other propagates as an ordinary open failure that leaves the row untouched. Adds test coverage at the two construction sites PA.f179 fixed but never pinned (open_question_of, and run's own StaleAnswer arm) proving a missing question_token surfaces as None, not a fabricated Some(""); both fail under Soul's exact mutation. The retirement sidecar is now named *.msgpack instead of *.cc (PA.f192): it is a one-shot forensic dump via std::fs::write, never a CultCache store, and should not read as one. retire_unreadable_row's own doc comment now states the write-before-destroy ordering and the orphan-sidecar risk explicitly (PA.f189): a failed compare-and-swap after a successful sidecar write leaves that sidecar in place with no corresponding successful retirement. PlayTurnStore and PlayTable now remember the sidecar path a startup retirement wrote (PA.f186's own plumbing; runtime.rs wiring in the next commit). Verified: cargo test -p ghostlight-dungeon --bin ghostlight-dungeon, 177 passed. Ran the mutations quoted in every new doc comment by hand and reverted them; each failed the test it names (see the Cut 15 report for the transcript). Co-Authored-By: Claude Sonnet 5 --- crates/ghostlight-dungeon/src/play.rs | 598 +++++++++++++++++++++++--- 1 file changed, 534 insertions(+), 64 deletions(-) diff --git a/crates/ghostlight-dungeon/src/play.rs b/crates/ghostlight-dungeon/src/play.rs index abc3a9f..499f10d 100644 --- a/crates/ghostlight-dungeon/src/play.rs +++ b/crates/ghostlight-dungeon/src/play.rs @@ -23,7 +23,7 @@ use std::{ time::Duration, }; -use anyhow::{Context, bail}; +use anyhow::{Context, anyhow, bail}; use chrono::Utc; use cultcache_rs::{CacheBackingStore, CultCacheEnvelope, OwnedRedbMessagePackBackingStore}; use ghostlight::{ @@ -483,6 +483,13 @@ struct PlayTurnStore { row: CultCacheEnvelope, state: PlayTurnStoreState, healthy: bool, + /// PA.f186: `Some` when this store's own `open` retired an unreadable + /// row this process, naming the sidecar it wrote. Set once, at `open`, + /// and never cleared for the life of this `PlayTurnStore` — a later + /// commit does not touch it, since a retirement is a fact about how this + /// process's own custody of the row began, not an ongoing health signal + /// that could clear itself. + retired_sidecar: Option, } impl PlayTurnStore { @@ -492,7 +499,12 @@ impl PlayTurnStore { .validate_path_identity() .context("play-turn store path identity is invalid")?; let rows = store.pull_all()?; - let (row, state) = match rows.as_slice() { + // PA.f186: the third element is the sidecar path a retirement wrote + // this call, `None` on every ordinary path (fresh init, a row that + // decoded/upgraded cleanly) — carried up into `PlayTurnStore` itself + // so `runtime_readiness` can report a retirement structurally, + // without re-deriving it from `tracing::warn!` output. + let (row, state, retired_sidecar) = match rows.as_slice() { [] => { let state = PlayTurnStoreState { schema: STORE_SCHEMA.into(), @@ -504,22 +516,34 @@ impl PlayTurnStore { if !store.compare_and_swap_batch(&[], vec![row.clone()])? { bail!("play-turn store changed during initialization"); } - (row, state) + (row, state, None) } + // PA.f184: a row already at the current schema id can still fail + // to read — genuinely corrupt bytes, or (the realistic trigger) a + // future change to `PlayTurn`'s field set or order that lands + // without moving `STORE_SCHEMA`, which flips the canonical + // re-encode check below without touching `schema_id` at all. + // Before this fix every one of those three checks `bail!`ed + // unconditionally, so `open` failed the identical way on every + // future restart and the play card stayed gone until an operator + // deleted the store by hand — exactly the PA.f177 gap, but for a + // row this store itself is supposed to already understand. Now it + // retires through the same `retire_unreadable_row` path the + // legacy arms use, so a damaged or future-schema row loses its + // one turn instead of the whole play surface. [row] if row.r#type == STORE_TYPE && row.key == STORE_KEY && row.schema_id.as_deref() == Some(STORE_SCHEMA) => { - let state: PlayTurnStoreState = - rmp_serde::from_slice(&row.payload).context("play-turn state is corrupt")?; - if rmp_serde::to_vec_named(&state)? != row.payload { - bail!("play-turn state is not canonical"); + match read_current_schema_row(row) { + Ok(state) => (row.clone(), state, None), + Err(error) => { + let (row, state, sidecar) = + retire_unreadable_row(&store, path.as_ref(), row, STORE_SCHEMA, &error)?; + (row, state, Some(sidecar)) + } } - if state.schema != STORE_SCHEMA { - bail!("play-turn state schema is invalid"); - } - (row.clone(), state) } // PA.f172/PA.f170: a row written before `question_token` existed // — named legacy, not corrupt, because it is neither: it is a @@ -549,9 +573,24 @@ impl PlayTurnStore { && row.schema_id.as_deref() == Some(LEGACY_STORE_SCHEMA_V1) => { match upgrade_legacy_row(&store, row, LEGACY_STORE_SCHEMA_V1) { - Ok(upgraded) => upgraded, - Err(error) => { - retire_unreadable_row(&store, path.as_ref(), row, LEGACY_STORE_SCHEMA_V1, &error)? + Ok((row, state)) => (row, state, None), + Err(LegacyRowError::Decode(error)) => { + let (row, state, sidecar) = retire_unreadable_row( + &store, + path.as_ref(), + row, + LEGACY_STORE_SCHEMA_V1, + &error, + )?; + (row, state, Some(sidecar)) + } + // PA.f185: not a decode failure — an inner `schema` tag + // that disagrees with the envelope, or a lost + // compare-and-swap race — so the row is left untouched + // and `open` fails loudly instead of destroying a turn + // that decoded just fine. + Err(LegacyRowError::Other(error)) => { + return Err(error).context("upgrading a legacy play-turn row failed"); } } } @@ -566,12 +605,43 @@ impl PlayTurnStore { && row.schema_id.as_deref() == Some(LEGACY_STORE_SCHEMA_V2) => { match upgrade_legacy_row(&store, row, LEGACY_STORE_SCHEMA_V2) { - Ok(upgraded) => upgraded, - Err(error) => { - retire_unreadable_row(&store, path.as_ref(), row, LEGACY_STORE_SCHEMA_V2, &error)? + Ok((row, state)) => (row, state, None), + Err(LegacyRowError::Decode(error)) => { + let (row, state, sidecar) = retire_unreadable_row( + &store, + path.as_ref(), + row, + LEGACY_STORE_SCHEMA_V2, + &error, + )?; + (row, state, Some(sidecar)) + } + Err(LegacyRowError::Other(error)) => { + return Err(error).context("upgrading a legacy play-turn row failed"); } } } + // PA.f184: our own key and type, but a schema id none of the arms + // above recognize — a future schema this build has never seen, + // not foreign data — so it is retired the same way a damaged row + // is, rather than leaving `open` refusing forever. Multiple rows, + // or a row under a different `type`/`key` entirely, still bail + // below: that is genuinely foreign content this store must not + // guess about destroying. + [row] if row.r#type == STORE_TYPE && row.key == STORE_KEY => { + let reason = anyhow!( + "play-turn row declares unrecognized schema {:?}", + row.schema_id + ); + let (row, state, sidecar) = retire_unreadable_row( + &store, + path.as_ref(), + row, + row.schema_id.as_deref().unwrap_or(""), + &reason, + )?; + (row, state, Some(sidecar)) + } _ => bail!("play-turn store contains foreign or otherwise-unreadable records; start with a fresh store"), }; Ok(Self { @@ -579,9 +649,19 @@ impl PlayTurnStore { row, state, healthy: true, + retired_sidecar, }) } + /// PA.f186: the sidecar path this store's own `open` wrote, if this + /// process's own open retired a row — `None` on every ordinary open. + /// Read once by `PlayTable::new` into its own plain field; retirement + /// only ever happens during `open`, so nothing after startup needs to + /// re-check this. + fn retired_sidecar(&self) -> Option<&Path> { + self.retired_sidecar.as_deref() + } + fn current(&self) -> Option<&PlayTurn> { self.state.turn.as_ref() } @@ -700,6 +780,25 @@ fn mint_question_token() -> String { uuid::Uuid::new_v4().to_string() } +/// `upgrade_legacy_row`'s own error, split by whether the row was actually +/// unreadable (PA.f185). Soul's probe: a row whose envelope declares +/// `LEGACY_STORE_SCHEMA_V1` and whose inner `schema` field says +/// `LEGACY_STORE_SCHEMA_V2` decodes its turn perfectly — `opening_prompt`, +/// `question`, every field reads back intact — but the inner-schema +/// consistency check below, and a lost `compare_and_swap_batch` race while +/// persisting the upgrade, both used to be indistinguishable from "the bytes +/// would not parse at all" once `PlayTurnStore::open` caught the `Err`: every +/// exit landed in `retire_unreadable_row`, destroying a readable turn under a +/// sidecar reason ("did not decode even leniently") that was false for +/// either case. Only `Decode` is that genuine case; `Other` propagates as an +/// ordinary `open` failure instead, so a schema-tag mismatch is diagnosable +/// and a transient CAS race can be retried, rather than both being treated as +/// permanent corruption and quietly erased. +enum LegacyRowError { + Decode(anyhow::Error), + Other(anyhow::Error), +} + /// Shared by `PlayTurnStore::open`'s `LEGACY_STORE_SCHEMA_V1` and /// `LEGACY_STORE_SCHEMA_V2` branches (PA.f170): decodes `row` under the /// current `PlayTurnStoreState`/`PlayTurn` structs — both legacy schemas @@ -712,11 +811,15 @@ fn upgrade_legacy_row( store: &OwnedRedbMessagePackBackingStore, row: &CultCacheEnvelope, legacy_schema: &str, -) -> anyhow::Result<(CultCacheEnvelope, PlayTurnStoreState)> { +) -> Result<(CultCacheEnvelope, PlayTurnStoreState), LegacyRowError> { let mut state: PlayTurnStoreState = rmp_serde::from_slice(&row.payload) - .with_context(|| format!("play-turn legacy {legacy_schema} state is corrupt"))?; + .with_context(|| format!("play-turn legacy {legacy_schema} state is corrupt")) + .map_err(LegacyRowError::Decode)?; if state.schema != legacy_schema { - bail!("play-turn legacy {legacy_schema} state schema is invalid"); + return Err(LegacyRowError::Other(anyhow!( + "play-turn legacy {legacy_schema} state schema is invalid: payload itself declares {:?}", + state.schema + ))); } if let Some(turn) = state.turn.as_mut() && turn.state == PlayTurnState::AwaitingPlayer @@ -726,11 +829,34 @@ fn upgrade_legacy_row( turn.question_token = Some(mint_question_token()); } state.schema = STORE_SCHEMA.into(); - let upgraded_row = envelope(&state)?; - if !store.compare_and_swap_batch(std::slice::from_ref(row), vec![upgraded_row.clone()])? { - bail!("play-turn store changed while upgrading a legacy {legacy_schema} row"); + let upgraded_row = envelope(&state).map_err(LegacyRowError::Other)?; + match store.compare_and_swap_batch(std::slice::from_ref(row), vec![upgraded_row.clone()]) { + Ok(true) => Ok((upgraded_row, state)), + Ok(false) => Err(LegacyRowError::Other(anyhow!( + "play-turn store changed while upgrading a legacy {legacy_schema} row" + ))), + Err(error) => Err(LegacyRowError::Other(error)), } - Ok((upgraded_row, state)) +} + +/// `PlayTurnStore::open`'s current-schema arm (PA.f184): a row already +/// declaring `STORE_SCHEMA` can still fail to read — genuinely corrupt bytes, +/// or a future change to `PlayTurn`'s own field set or order that lands +/// without moving `STORE_SCHEMA`, which the canonical re-encode check below +/// exists to catch. All three checks here are read failures, not merely +/// validity concerns: unlike `upgrade_legacy_row` (PA.f185), there is no +/// upward-migration path for a current-schema row that fails one of these — +/// nothing upgrades it in place — so every exit here is retirement-worthy. +fn read_current_schema_row(row: &CultCacheEnvelope) -> anyhow::Result { + let state: PlayTurnStoreState = + rmp_serde::from_slice(&row.payload).context("play-turn state is corrupt")?; + if rmp_serde::to_vec_named(&state)? != row.payload { + bail!("play-turn state is not canonical"); + } + if state.schema != STORE_SCHEMA { + bail!("play-turn state schema is invalid"); + } + Ok(state) } /// A legacy row's payload, `type`, `key` and `schema_id`, preserved verbatim @@ -748,10 +874,12 @@ struct RetiredPlayTurnRow<'a> { payload: &'a [u8], } -/// `PlayTurnStore::open`'s fallback when a row recognized as legacy by its -/// `schema_id` still cannot be decoded, even leniently, by `upgrade_legacy_row` -/// (PA.f177) — a row older than `applied_keys`/`fault` themselves, or one that -/// is genuinely damaged past what a schema-shape upgrade can recover. Before +/// `PlayTurnStore::open`'s fallback when a row this build recognizes by its +/// `schema_id` (legacy or current) still cannot be turned into a usable +/// `PlayTurnStoreState` — a row older than `applied_keys`/`fault` themselves +/// (PA.f177), one genuinely damaged past what a schema-shape upgrade can +/// recover, a current-schema row whose bytes no longer round-trip +/// canonically (PA.f184), or a schema id no arm above recognizes at all. Before /// PA.f177 this row made `open` return `Err` unconditionally: `open_play` /// caught that, logged it, and left `state.play` `None` — every future `open` /// hit the identical row and failed the identical way, so the play card @@ -759,23 +887,43 @@ struct RetiredPlayTurnRow<'a> { /// that a daemon never answers an unreadable row by quietly having no play /// surface, so this writes the row, byte for byte, to a sidecar file beside /// the store (named for what it is, so it reads as forensic evidence and not -/// as another store), clears the row through the store's own -/// compare-and-swap (the same guard `swap_in` uses for every other write — -/// this is not a second, unguarded writer), and returns a fresh, empty -/// state so `open` succeeds and play starts fresh. Logged at `warn`, so the -/// fact reaches the operator through the same log stream `runtime.rs`'s own -/// `open_play` failure warning already uses, next to the `playStatus` the -/// operator already checks. +/// as another store — PA.f192: the extension is `.msgpack`, not `.cc`, since +/// this sidecar is a one-shot forensic dump, never a CultCache store some +/// future process might mistake it for and try to open), clears the row +/// through the store's own compare-and-swap (the same guard `swap_in` uses +/// for every other write — this is not a second, unguarded writer), and +/// returns a fresh, empty state so `open` succeeds and play starts fresh. +/// Logged at `warn`, so the fact reaches the operator through the same log +/// stream `runtime.rs`'s own `open_play` failure warning already uses, next +/// to the `playStatus` the operator already checks (PA.f186: `PlayTurnStore` +/// itself also remembers the sidecar path this call, if any, wrote, so +/// `runtime_readiness` can report it structurally, not only in the log). +/// +/// PA.f189: two orderings matter here and are both deliberate. +/// - The sidecar write happens *before* the destructive compare-and-swap. +/// If the write fails, this returns `Err` without ever touching the row — +/// `open` fails loudly and the original row is exactly where it was, so a +/// disk-full or permission failure never destroys evidence it could not +/// also preserve. +/// - If the write succeeds but the compare-and-swap below then fails — the +/// store changed identity mid-retirement, another writer raced this one, +/// or custody was lost — this still returns `Err`. The sidecar file it +/// already wrote is *not* cleaned up: nothing here can tell whether the +/// race means the original row is still there (safe to retry retirement, +/// which would then write a second, differently-timestamped sidecar) or +/// already gone some other way. An orphaned sidecar from a failed +/// retirement is therefore an expected, if rare, shape — a forensic +/// leftover with no corresponding successful retirement — and an operator +/// clearing them out should confirm the row it names is actually resolved +/// (upgraded, retired again, or the store rebuilt) before deleting one. fn retire_unreadable_row( store: &OwnedRedbMessagePackBackingStore, store_path: &Path, row: &CultCacheEnvelope, - legacy_schema: &str, + declared_schema: &str, decode_error: &anyhow::Error, -) -> anyhow::Result<(CultCacheEnvelope, PlayTurnStoreState)> { - let reason = format!( - "row declared legacy schema {legacy_schema} but did not decode even leniently: {decode_error:#}" - ); +) -> anyhow::Result<(CultCacheEnvelope, PlayTurnStoreState, PathBuf)> { + let reason = format!("row declared schema {declared_schema} but could not be read: {decode_error:#}"); let sidecar = retired_row_sidecar_path(store_path); let record = RetiredPlayTurnRow { retired_at: Utc::now().to_rfc3339(), @@ -798,28 +946,32 @@ fn retire_unreadable_row( }; let fresh_row = envelope(&fresh)?; if !store.compare_and_swap_batch(std::slice::from_ref(row), vec![fresh_row.clone()])? { - bail!("play-turn store changed while retiring an unreadable legacy row"); + bail!("play-turn store changed while retiring an unreadable row (the sidecar at {} was already written and is left in place)", sidecar.display()); } tracing::warn!( sidecar = %sidecar.display(), reason = %reason, - "a play-turn row could not be read even by the legacy reader; it was retired to a sidecar file and play starts fresh" + "a play-turn row could not be read; it was retired to a sidecar file and play starts fresh" ); - Ok((fresh_row, fresh)) + Ok((fresh_row, fresh, sidecar)) } -/// Beside the store, named for what it is: `.retired-unreadable-row..cc`. +/// Beside the store, named for what it is: +/// `.retired-unreadable-row..msgpack` (PA.f192). /// Timestamped so a second unreadable row (a different corruption, on a later /// run) never collides with or silently overwrites the first one's evidence. +/// `.msgpack`, not `.cc`: this file is a one-shot forensic dump written +/// through `std::fs::write`, never through a `CacheBackingStore`, and never +/// meant to be opened as one. fn retired_row_sidecar_path(store_path: &Path) -> PathBuf { let stamp = Utc::now().format("%Y%m%dT%H%M%S%.6fZ"); let stem = store_path .file_stem() .and_then(|stem| stem.to_str()) .unwrap_or("play-turn"); - store_path.with_file_name(format!("{stem}.retired-unreadable-row.{stamp}.cc")) + store_path.with_file_name(format!("{stem}.retired-unreadable-row.{stamp}.msgpack")) } #[derive(Debug, thiserror::Error)] @@ -991,6 +1143,12 @@ pub(crate) struct PlayTable { /// itself. permits: Arc, store: Mutex, + /// PA.f186: the sidecar path `store`'s own `open` wrote during this + /// table's construction, if it retired a row — read once, here, before + /// `store` moves behind its `Mutex`, so `runtime_readiness` can report it + /// without taking that lock. `None` for the overwhelming majority of + /// opens, where nothing was retired. + retired_sidecar: Option, /// One turn at a time for this table (PA.f68/PA.f112): held for the /// whole body of `run`, so the key check and the turn open happen under /// one acquisition and two concurrent `run` calls can never interleave. @@ -1040,6 +1198,7 @@ impl PlayTable { ) -> anyhow::Result { let play = PlayPort::new(world.clone()); let store = PlayTurnStore::open(store_path)?; + let retired_sidecar = store.retired_sidecar().map(Path::to_path_buf); Ok(Self { play, world, @@ -1048,6 +1207,7 @@ impl PlayTable { model, permits, store: Mutex::new(store), + retired_sidecar, run_lock: Arc::new(Mutex::new(())), poisoned: AtomicBool::new(false), retry_delay_base_ms: std::sync::atomic::AtomicU64::new(RETRY_DELAY_BASE_MS), @@ -1058,6 +1218,14 @@ impl PlayTable { }) } + /// PA.f186: the sidecar path this table's own store wrote if opening it + /// retired an unreadable row, so `runtime.rs`'s `/health` surface can + /// report the retirement structurally instead of only through the + /// `tracing::warn!` `retire_unreadable_row` already emits. + pub(crate) fn retired_row_sidecar(&self) -> Option<&Path> { + self.retired_sidecar.as_deref() + } + /// Installs PA.f99's own before-submit test hook (see the field's own /// doc comment). Test-only: no production call site exists. #[cfg(test)] @@ -2822,6 +2990,31 @@ pub(crate) mod tests { }; use tokio::task::JoinHandle; + /// PA.f186's own cross-module seam: `runtime.rs`'s readiness tests need + /// to prove a retirement during `PlayTable::new` reaches `/health` + /// end-to-end, but `STORE_TYPE`/`STORE_KEY`/`STORE_SCHEMA` and + /// `CultCacheEnvelope` are this module's own store internals. Rather than + /// re-importing `cultcache_rs` and duplicating this shape in + /// `runtime.rs`, this writes one row `PlayTurnStore::open` cannot read — + /// the exact same undecodable-bytes shape + /// `a_row_with_the_current_schema_id_and_undecodable_bytes_is_retired_to_a_sidecar_and_play_starts_fresh` + /// seeds below — at `store_path`, before a caller opens a `PlayTable` + /// over it. + pub(crate) fn seed_unreadable_row_for_test(store_path: &std::path::Path) { + let corrupt_row = CultCacheEnvelope { + key: STORE_KEY.into(), + r#type: STORE_TYPE.into(), + payload: vec![0xC1], // msgpack's own reserved "never used" marker byte + stored_at: Utc::now().to_rfc3339(), + schema_id: Some(STORE_SCHEMA.into()), + }; + let raw_store = OwnedRedbMessagePackBackingStore::new(store_path).unwrap(); + assert!( + raw_store.compare_and_swap_batch(&[], vec![corrupt_row]).unwrap(), + "seeding the corrupt row must succeed against an empty store" + ); + } + /// Every `.rs` file under `src`, with everything a `#[cfg(test)]` item /// removed, the same extent-by-indent scan `app_session.rs`'s own /// forbidden-writer test uses. @@ -3953,44 +4146,189 @@ pub(crate) mod tests { assert!(!token.is_empty(), "a minted token must not be the empty-string fallback"); } - /// PA.f172's own negative: a row that matches the current schema id but - /// whose bytes do not decode at all is still refused as corrupt — the - /// legacy path recognizes only the one schema id every pre-Cut-11 - /// process actually wrote, never a general-purpose "try to read it - /// anyway" escape hatch for a row that is genuinely damaged. + /// PA.f184: a row that matches the *current* schema id but whose bytes do + /// not decode at all used to leave `open` refusing forever — `open_play` + /// caught the `Err`, logged it, and `state.play` stayed `None` across + /// every future restart, exactly the PA.f177 gap this fix closes for the + /// arm PA.f177 itself did not touch. It is now retired to a sidecar file + /// beside the store, the same path a damaged legacy row already took, so + /// the play card comes back (empty) instead of staying gone. + /// + /// Mutation: change `read_current_schema_row`'s decode line back to a + /// bare `rmp_serde::from_slice(&row.payload)?` (or otherwise make this + /// arm `bail!` instead of calling `retire_unreadable_row`) — `open` then + /// returns `Err` for this row and `store.expect(...)` below panics. #[tokio::test] - async fn a_row_with_the_current_schema_id_and_undecodable_bytes_is_still_refused_as_corrupt() { + async fn a_row_with_the_current_schema_id_and_undecodable_bytes_is_retired_to_a_sidecar_and_play_starts_fresh() { + let directory = tempfile::tempdir().unwrap(); + let store_path = directory.path().join("play-turn-v1.cc"); + seed_unreadable_row_for_test(&store_path); + + let store = PlayTurnStore::open(&store_path) + .expect("a damaged current-schema row must be retired, not fail open outright"); + assert!( + store.current().is_none(), + "play must start fresh with no current turn once the old row is retired" + ); + assert!( + store + .retired_sidecar() + .is_some_and(|sidecar| sidecar.file_name().unwrap().to_string_lossy().contains("retired-unreadable-row")), + "the store must remember the sidecar path its own open wrote" + ); + + let sidecar = std::fs::read_dir(directory.path()) + .unwrap() + .filter_map(|entry| entry.ok()) + .find(|entry| entry.file_name().to_string_lossy().contains("retired-unreadable-row")) + .expect("the damaged row must be written to a sidecar file beside the store"); + assert!( + sidecar.file_name().to_string_lossy().ends_with(".msgpack"), + "the sidecar must be named for what it is (PA.f192), not `.cc`: {:?}", + sidecar.file_name() + ); + let sidecar_bytes = std::fs::read(sidecar.path()).unwrap(); + assert!(!sidecar_bytes.is_empty(), "the sidecar must actually carry the retired row's bytes"); + + // Reopening must not hit the same row again: it was cleared from the + // store as part of retiring it, not merely skipped this once. + drop(store); + let reopened = PlayTurnStore::open(&store_path) + .expect("reopening after retirement must not refuse the store again"); + assert!(reopened.current().is_none()); + assert!( + reopened.retired_sidecar().is_none(), + "a clean reopen after retirement must not itself report a fresh retirement" + ); + } + + /// PA.f184: a single row under this store's own `type`/`key` but a + /// schema id no arm recognizes — a future schema this build has never + /// seen — is retired exactly like a damaged row, not left bailing + /// forever with "foreign or otherwise-unreadable records". + /// + /// Mutation: delete the catch-all `[row] if row.r#type == STORE_TYPE && + /// row.key == STORE_KEY => { ... }` arm from `PlayTurnStore::open` — this + /// row then falls through to the final `_ => bail!(...)` and + /// `store.expect(...)` below panics. + #[tokio::test] + async fn a_row_declaring_an_unrecognized_future_schema_is_retired_not_refused_forever() { let directory = tempfile::tempdir().unwrap(); let store_path = directory.path().join("play-turn-v1.cc"); - let corrupt_row = CultCacheEnvelope { + let future_row = CultCacheEnvelope { key: STORE_KEY.into(), r#type: STORE_TYPE.into(), - payload: vec![0xC1], // msgpack's own reserved "never used" marker byte + payload: vec![1, 2, 3], stored_at: Utc::now().to_rfc3339(), - schema_id: Some(STORE_SCHEMA.into()), + schema_id: Some("ghostlight.play_turn_store.v4".into()), }; { let raw_store = OwnedRedbMessagePackBackingStore::new(&store_path).unwrap(); assert!( - raw_store - .compare_and_swap_batch(&[], vec![corrupt_row]) - .unwrap(), - "seeding the corrupt row must succeed against an empty store" + raw_store.compare_and_swap_batch(&[], vec![future_row]).unwrap(), + "seeding the future-schema row must succeed against an empty store" + ); + } + + let store = PlayTurnStore::open(&store_path) + .expect("an unrecognized future schema must be retired, not fail open outright"); + assert!(store.current().is_none()); + assert!( + std::fs::read_dir(directory.path()) + .unwrap() + .filter_map(|entry| entry.ok()) + .any(|entry| entry.file_name().to_string_lossy().contains("retired-unreadable-row")), + "the future-schema row must be written to a sidecar file beside the store" + ); + } + + /// PA.f185: Soul's own probe — a row whose envelope declares + /// `LEGACY_STORE_SCHEMA_V1` but whose inner `schema` field says + /// `LEGACY_STORE_SCHEMA_V2` decodes its turn perfectly + /// (`opening_prompt` reads back intact), yet before this fix `open` + /// routed the inner-schema mismatch into `retire_unreadable_row` anyway, + /// destroying a readable turn under a sidecar reason ("did not decode + /// even leniently") that was false. Now `open` returns `Err` and leaves + /// the row untouched: no sidecar, no destructive compare-and-swap, + /// because an inconsistent inner tag is not the same failure as bytes + /// that will not parse at all. + /// + /// Mutation: change `Err(LegacyRowError::Other(error)) => return + /// Err(error)...` (in either legacy arm) back to + /// `retire_unreadable_row(&store, path.as_ref(), row, + /// LEGACY_STORE_SCHEMA_V1, &error)?` — the row is then retired, a + /// sidecar appears, and the assertions below fail. + #[tokio::test] + async fn a_legacy_rows_inner_schema_disagreeing_with_its_envelope_is_not_retired() { + let directory = tempfile::tempdir().unwrap(); + let store_path = directory.path().join("play-turn-v1.cc"); + + let legacy_turn = LegacyPlayTurnV1 { + turn_id: "55555555-5555-5555-5555-555555555555".into(), + applied_keys: Vec::new(), + opening_prompt: "You stand at the gate.".into(), + player_prose: vec!["I knock.".into()], + rounds: Vec::new(), + calls: Vec::new(), + persona_turns: Vec::new(), + question: None, + refusal: None, + narration: None, + fault: None, + state: PlayTurnState::Running, + }; + // Envelope's own `schema_id` says v1 (handled by this arm); the + // payload's own inner `schema` field says v2 — the exact disagreement + // Soul's probe found, constructed by hand since no production writer + // ever mints this shape. + let legacy_state = LegacyPlayTurnStoreStateV1 { + schema: LEGACY_STORE_SCHEMA_V2.into(), + turn: Some(legacy_turn), + ledger: KeyLedger::default(), + revision: 3, + }; + let legacy_row = CultCacheEnvelope { + key: STORE_KEY.into(), + r#type: STORE_TYPE.into(), + payload: rmp_serde::to_vec_named(&legacy_state).unwrap(), + stored_at: Utc::now().to_rfc3339(), + schema_id: Some(LEGACY_STORE_SCHEMA_V1.into()), + }; + { + let raw_store = OwnedRedbMessagePackBackingStore::new(&store_path).unwrap(); + assert!( + raw_store.compare_and_swap_batch(&[], vec![legacy_row]).unwrap(), + "seeding the mismatched-inner-schema row must succeed against an empty store" ); } - // `.expect_err` needs `T: Debug`; `PlayTurnStore` does not derive it - // (its own row holds turn text nobody wants dumped by a failed - // assertion), so this matches directly instead. let error = match PlayTurnStore::open(&store_path) { - Ok(_) => panic!("undecodable bytes must still refuse to open"), + Ok(_) => panic!("an inner-schema mismatch must not silently succeed either"), Err(error) => error, }; + // `{error:#}` (anyhow's alternate form) walks the whole `.context(...)` + // chain; `{error}` alone would only show `open`'s own outer wrapper + // ("upgrading a legacy play-turn row failed") and never actually + // check that the underlying cause names the schema mismatch at all. + let full_chain = format!("{error:#}"); assert!( - error.to_string().contains("corrupt"), - "a genuinely undecodable row must still be named corrupt, not legacy: {error}" + full_chain.contains("schema"), + "the failure must name the schema mismatch, not read as a plain decode failure: {full_chain}" ); + assert!( + std::fs::read_dir(directory.path()) + .unwrap() + .filter_map(|entry| entry.ok()) + .all(|entry| !entry.file_name().to_string_lossy().contains("retired-unreadable-row")), + "a readable turn must not be retired over an inner-schema tag mismatch alone" + ); + + // The row itself is untouched: no destructive compare-and-swap ran. + let raw_store = OwnedRedbMessagePackBackingStore::new(&store_path).unwrap(); + let rows = raw_store.pull_all().unwrap(); + assert_eq!(rows.len(), 1, "the original row must still be the only row in the store"); + assert_eq!(rows[0].schema_id.as_deref(), Some(LEGACY_STORE_SCHEMA_V1)); } /// PA.f177: a row older than `applied_keys`/`fault` themselves — written @@ -8536,6 +8874,138 @@ pub(crate) mod tests { ); } + /// PA.f188: `open_question_of`'s own `token` field must carry + /// `question_token` exactly — `None` stays `None` — never fabricated into + /// a comparable `Some("")` (PA.f179's own invariant, at this construction + /// site specifically). Production never actually reaches `question_token: + /// None` with a genuinely open question (`execute_round`'s + /// `ASK_PLAYER_TOOL` arm always sets both together), but the type system + /// alone does not forbid it, and this construction site had no direct + /// test of its own before Soul's mutation below. + /// + /// Mutation (Soul's own probe, PA.f188): + /// ```text + /// - token: turn.question_token.clone(), + /// + token: Some(turn.question_token.clone().unwrap_or_default()), + /// ``` + /// survived the whole suite before this test existed. + #[test] + fn open_question_of_never_fabricates_a_token_when_none_was_minted() { + let turn = PlayTurn { + turn_id: test_turn_id(9100), + applied_keys: Vec::new(), + opening_prompt: "opening".into(), + player_prose: vec!["I try the act.".into()], + rounds: vec![output( + "r0", + vec![call_event("c0", ASK_PLAYER_TOOL, serde_json::json!({"question": "Which way?"}))], + )], + calls: vec![CallRecord { + call_id: "c0".into(), + round: 0, + slot: 0, + body: Some(RecordedCall::AskPlayer("Which way?".into())), + result: None, + }], + persona_turns: Vec::new(), + question: Some("Which way?".into()), + question_token: None, + refusal: None, + narration: None, + fault: None, + state: PlayTurnState::AwaitingPlayer, + }; + let open = open_question_of(&turn).expect("the turn's own open question must still be found"); + assert_eq!( + open.token, None, + "a turn whose question_token was never minted must surface as an absent token, not a \ + comparable empty string" + ); + } + + /// PA.f188: the `StaleAnswer` refusal's own construction site + /// (`run`'s `_ =>` arm) must carry the same guarantee as + /// `open_question_of` — `existing.question_token.clone()` verbatim, never + /// fabricated into `Some("")`. Seeds a turn directly into the store with + /// a genuinely open question but `question_token: None` (a shape + /// production never produces, since `execute_round` always mints both + /// together, but one this refusal arm must still not degrade), then + /// answers it with a deliberately stale question id so the `_ =>` arm + /// actually runs. + /// + /// Mutation: the same shape as `open_question_of`'s own probe, applied to + /// this arm's `token: existing.question_token.clone()` instead. + #[tokio::test] + async fn a_stale_answer_refusal_never_fabricates_a_token_the_turn_never_minted() { + let fixture = play_world(None, "player-stale-no-token").await; + let directory = tempfile::tempdir().unwrap(); + let store_path = directory.path().join("play-turn-v1.cc"); + let personas = PersonaLane::new( + ControllerPort::new(fixture.world.clone()), + ScriptedPort::new(vec![]), + "gpt-5.6-sol".into(), + "gpt-5.6-sol".into(), + ) + .unwrap(); + let table = PlayTable::new( + fixture.world.clone(), + personas, + ScriptedPort::new(vec![]), + "gpt-5.6-terra".into(), + Arc::new(Semaphore::new(2)), + &store_path, + ) + .unwrap(); + + let turn = PlayTurn { + turn_id: test_turn_id(9200), + applied_keys: Vec::new(), + opening_prompt: "opening".into(), + player_prose: vec!["I try the act.".into()], + rounds: vec![output( + "r0", + vec![call_event("c0", ASK_PLAYER_TOOL, serde_json::json!({"question": "Which way?"}))], + )], + calls: vec![CallRecord { + call_id: "c0".into(), + round: 0, + slot: 0, + body: Some(RecordedCall::AskPlayer("Which way?".into())), + result: None, + }], + persona_turns: Vec::new(), + question: Some("Which way?".into()), + question_token: None, + refusal: None, + narration: None, + fault: None, + state: PlayTurnState::AwaitingPlayer, + }; + table.store.lock().await.open_new_turn(turn).unwrap(); + + let stale_question = QuestionId { + turn_id: test_turn_id(9200), + round: 99, + slot: 99, + }; + let error = table + .run( + &fixture.principal, + test_turn_id(9201), + PlayRequest { text: "left".into(), answers: Some(stale_question) }, + ) + .await + .unwrap_err(); + let PlayError::StaleAnswer { question: Some(open), .. } = error else { + panic!("expected a StaleAnswer refusal carrying the turn's own still-open question: {error:?}"); + }; + assert_eq!( + open.token, None, + "a turn whose question_token was never minted must surface as an absent token in the \ + stale refusal too, not a comparable empty string" + ); + } + /// PA.f101: `answers: None` while a question genuinely is open is /// refused exactly like a stale, wrongly-named one — the match at /// `run`'s own open-question arm only ever applies the `(Some(given),