From d3f8dea75ac0e8fe2fb807e28cc72434b5fcd70c Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 14 Oct 2026 02:38:23 +0200 Subject: [PATCH 01/13] cultcache-rs: decode_payload grows its stack through serde_stacker A payload inside the 256 nesting bound no longer depends on the caller's thread stack: the rmp-serde deserializer runs under serde_stacker with a 256 KiB red zone and 2 MiB segments. Adds tests/decode_stack.rs (child process per test, sized decode thread) and renames the 2 MiB union_parity stack test to a 1 MiB pin. Not yet built or run at this commit. Co-Authored-By: Claude Opus 5.5 --- packages/cultcache-rs/Cargo.toml | 1 + packages/cultcache-rs/README.md | 10 + packages/cultcache-rs/src/lib.rs | 20 +- packages/cultcache-rs/tests/decode_stack.rs | 239 ++++++++++++++++++++ packages/cultcache-rs/tests/union_parity.rs | 8 +- 5 files changed, 273 insertions(+), 5 deletions(-) create mode 100644 packages/cultcache-rs/tests/decode_stack.rs diff --git a/packages/cultcache-rs/Cargo.toml b/packages/cultcache-rs/Cargo.toml index 23d53801..624ae887 100644 --- a/packages/cultcache-rs/Cargo.toml +++ b/packages/cultcache-rs/Cargo.toml @@ -22,6 +22,7 @@ redb = "4.1.0" serde = { version = "1", features = ["derive"] } serde_bytes = "0.11" serde_json = "1" +serde_stacker = "0.1.14" uuid = { version = "1", features = ["v4"] } [dev-dependencies] diff --git a/packages/cultcache-rs/README.md b/packages/cultcache-rs/README.md index 7191eeed..35999e5c 100644 --- a/packages/cultcache-rs/README.md +++ b/packages/cultcache-rs/README.md @@ -456,6 +456,16 @@ That is the Rust version of the original trick: The derive/registry work should preserve that shape. Its job is to remove manual registration ceremony, not to loosen the closed-world boundary. +Payload nesting is bounded at 256 MessagePack containers, checked before any +recursion, and typed decode then runs under `serde_stacker`, which moves onto a +fresh stack segment whenever less than 256 KiB of stack remains. A payload inside +the bound therefore decodes on a small thread instead of aborting the process; +past the bound the refusal is a typed `PayloadTooDeep`. The residual is a single +stack frame larger than the 256 KiB red zone. Stack growth on Windows is not yet +verified. On wasm32 the shadow stack grows the same way, but the engine's own +call stack remains the host's limit. Building the crate needs a C toolchain +(`psm`, `stacker`). + ## License Private GameCult infrastructure for now. Public packaging can wait until the diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs index 5332e8a4..4eccfd73 100644 --- a/packages/cultcache-rs/src/lib.rs +++ b/packages/cultcache-rs/src/lib.rs @@ -595,12 +595,30 @@ fn element_header(bytes: &[u8], position: usize) -> Option<(usize, Option }) } +/// The stack left below which a decode moves to a fresh segment. Measured (map D4): the worst +/// distance between two stack checks is 128-144 KiB in a debug build and under 32 KiB in release, +/// so 256 KiB leaves a margin in both. A frame larger than this between two checks still overruns. +const DECODE_RED_ZONE: usize = 256 * 1024; + +/// The size of each segment a decode grows onto. It does not change the cost of a decode (map D6). +const DECODE_STACK_SEGMENT: usize = 2 * 1024 * 1024; + /// The one decode of a document payload: the nesting bound first, then the deserializer, whose own /// depth counter (1024) is only a backstop. A failure keeps its kind and never quotes the bytes. /// Readers of stored envelope payloads outside this crate decode through it too. +/// +/// The deserializer runs under `serde_stacker`, which moves onto a fresh stack segment whenever less +/// than the red zone remains, at every container level serde's impls descend through, skipped values +/// included. The calling thread's stack size therefore never decides whether a payload inside the +/// bound decodes. The one residual is a single frame larger than the red zone. On wasm32 the shadow +/// stack grows the same way, while the engine's own call stack stays the host's limit. pub fn decode_payload(payload: &[u8]) -> Result { check_payload_depth(payload)?; - rmp_serde::from_slice(payload).map_err(opaque_decode_error) + let mut reader = rmp_serde::Deserializer::from_read_ref(payload); + let mut grown = serde_stacker::Deserializer::new(&mut reader); + grown.red_zone = DECODE_RED_ZONE; + grown.stack_size = DECODE_STACK_SEGMENT; + ::deserialize(grown).map_err(opaque_decode_error) } /// The nesting bound for an envelope admitted without a typed decode, from the same walk that diff --git a/packages/cultcache-rs/tests/decode_stack.rs b/packages/cultcache-rs/tests/decode_stack.rs new file mode 100644 index 00000000..14546519 --- /dev/null +++ b/packages/cultcache-rs/tests/decode_stack.rs @@ -0,0 +1,239 @@ +//! Typed decode never aborts a payload inside the nesting bound, whatever the caller's stack. +//! Each test runs its decode in a child test process on a thread of a stated size, so an abort +//! fails one named test. Values are built, encoded, compared and dropped on a large thread; only +//! the decode runs on the small one. + +use cultcache_rs::CultSlots; +use cultcache_rs::MAX_PAYLOAD_DEPTH; +use cultcache_rs::PayloadTooDeep; +use cultcache_rs::decode_payload; +use serde::de::DeserializeOwned; + +const CHILD_MARKER: &str = "CULTCACHE_DECODE_STACK_CHILD"; +const KIB: usize = 1024; + +#[derive(Clone, Debug, PartialEq, CultSlots)] +struct Wide32 { + #[cultcache(key = 0)] + a0: [u64; 32], + #[cultcache(key = 1)] + a1: [u64; 32], + #[cultcache(key = 2)] + a2: [u64; 32], + #[cultcache(key = 3)] + a3: [u64; 32], + #[cultcache(key = 4)] + a4: [u64; 32], + #[cultcache(key = 5)] + a5: [u64; 32], + #[cultcache(key = 6)] + a6: [u64; 32], + #[cultcache(key = 7)] + a7: [u64; 32], + #[cultcache(key = 8)] + a8: [u64; 32], + #[cultcache(key = 9)] + a9: [u64; 32], + #[cultcache(key = 10)] + a10: [u64; 32], + #[cultcache(key = 11)] + a11: [u64; 32], + #[cultcache(key = 12)] + a12: [u64; 32], + #[cultcache(key = 13)] + a13: [u64; 32], + #[cultcache(key = 14)] + a14: [u64; 32], + #[cultcache(key = 15)] + a15: [u64; 32], + #[cultcache(key = 16)] + a16: [u64; 32], + #[cultcache(key = 17)] + a17: [u64; 32], + #[cultcache(key = 18)] + a18: [u64; 32], + #[cultcache(key = 19)] + a19: [u64; 32], + #[cultcache(key = 20)] + a20: [u64; 32], + #[cultcache(key = 21)] + a21: [u64; 32], + #[cultcache(key = 22)] + a22: [u64; 32], + #[cultcache(key = 23)] + a23: [u64; 32], + #[cultcache(key = 24)] + a24: [u64; 32], + #[cultcache(key = 25)] + a25: [u64; 32], + #[cultcache(key = 26)] + a26: [u64; 32], + #[cultcache(key = 27)] + a27: [u64; 32], + #[cultcache(key = 28)] + a28: [u64; 32], + #[cultcache(key = 29)] + a29: [u64; 32], + #[cultcache(key = 30)] + a30: [u64; 32], + #[cultcache(key = 31)] + a31: [u64; 32], + #[cultcache(key = 32)] + next: Option>, +} + +#[derive(Clone, Debug, PartialEq, CultSlots)] +struct One { + #[cultcache(key = 0)] + a: u8, +} + +#[derive(Clone, Debug, PartialEq, CultSlots)] +struct DynamicSlot { + #[cultcache(key = 0)] + v: serde_json::Value, +} + +/// Runs `body` on a thread with a stack of `bytes`, returning what it returns. +fn on_stack(bytes: usize, body: impl FnOnce() -> R + Send + 'static) -> R { + std::thread::Builder::new().stack_size(bytes).spawn(body).unwrap().join().unwrap() +} + +/// Runs `body` in a child process that runs only the test `name`; the parent passes when the +/// child exits cleanly having run exactly that one test. An abort in the child fails the parent. +fn in_child(name: &str, body: impl FnOnce()) { + if std::env::var_os(CHILD_MARKER).is_some() { + body(); + return; + } + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([name, "--exact", "--nocapture", "--test-threads=1"]) + .env(CHILD_MARKER, "1") + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success() && stdout.contains("1 passed"), + "child {name} failed: {:?} +{stdout} +{stderr}", + output.status + ); +} + +/// A chain of `levels` Wide32 records: `levels` containers deep, plus one for its last arrays. +fn wide_chain(levels: usize) -> Wide32 { + let mut chain: Option> = None; + for level in (0..levels).rev() { + let mut node = Wide32 { next: chain, ..empty_wide() }; + node.a0[0] = level as u64; + chain = Some(Box::new(node)); + } + *chain.unwrap() +} + +fn empty_wide() -> Wide32 { + Wide32 { + a0: [0; 32], + a1: [0; 32], + a2: [0; 32], + a3: [0; 32], + a4: [0; 32], + a5: [0; 32], + a6: [0; 32], + a7: [0; 32], + a8: [0; 32], + a9: [0; 32], + a10: [0; 32], + a11: [0; 32], + a12: [0; 32], + a13: [0; 32], + a14: [0; 32], + a15: [0; 32], + a16: [0; 32], + a17: [0; 32], + a18: [0; 32], + a19: [0; 32], + a20: [0; 32], + a21: [0; 32], + a22: [0; 32], + a23: [0; 32], + a24: [0; 32], + a25: [0; 32], + a26: [0; 32], + a27: [0; 32], + a28: [0; 32], + a29: [0; 32], + a30: [0; 32], + a31: [0; 32], + next: None, + } +} + +fn decode_on(stack: usize, bytes: Vec) -> anyhow::Result { + on_stack(stack, move || decode_payload::(&bytes)) +} + +#[test] +fn a_wide_record_at_the_limit_decodes_on_a_1mib_thread() { + in_child("a_wide_record_at_the_limit_decodes_on_a_1mib_thread", || { + let bytes = on_stack(64 * KIB * KIB, || { + rmp_serde::to_vec(&wide_chain(MAX_PAYLOAD_DEPTH - 1)).unwrap() + }); + let decoded = decode_on::(1024 * KIB, bytes).unwrap(); + on_stack(64 * KIB * KIB, move || { + assert!(decoded == wide_chain(MAX_PAYLOAD_DEPTH - 1)); + }); + }); +} + +#[test] +fn one_past_the_limit_is_refused_typed_on_a_1mib_thread() { + in_child("one_past_the_limit_is_refused_typed_on_a_1mib_thread", || { + let bytes = on_stack(64 * KIB * KIB, || { + rmp_serde::to_vec(&wide_chain(MAX_PAYLOAD_DEPTH)).unwrap() + }); + let error = decode_on::(1024 * KIB, bytes).unwrap_err(); + assert_eq!( + error.downcast_ref::(), + Some(&PayloadTooDeep { depth: MAX_PAYLOAD_DEPTH + 1, limit: MAX_PAYLOAD_DEPTH }) + ); + }); +} + +#[test] +fn a_foreign_value_at_the_limit_decodes_on_a_64kib_thread() { + in_child("a_foreign_value_at_the_limit_decodes_on_a_64kib_thread", || { + // The slot's array is depth 1, so 255 nested arrays under it reach depth 256. + let value = on_stack(64 * KIB * KIB, || { + let mut value = serde_json::Value::Null; + for _ in 0..MAX_PAYLOAD_DEPTH - 1 { + value = serde_json::Value::Array(vec![value]); + } + value + }); + let expected = DynamicSlot { v: value }; + let bytes = on_stack(64 * KIB * KIB, { + let expected = expected.clone(); + move || rmp_serde::to_vec(&expected).unwrap() + }); + let decoded = decode_on::(64 * KIB, bytes).unwrap(); + on_stack(64 * KIB * KIB, move || { + assert!(decoded == expected); + drop(expected); + }); + }); +} + +#[test] +fn a_skipped_slot_at_the_limit_decodes_on_a_64kib_thread() { + in_child("a_skipped_slot_at_the_limit_decodes_on_a_64kib_thread", || { + // [1, [[...]]]: slot 1 is unknown to One and is skipped; the whole payload is 256 deep. + let mut bytes = vec![0x92u8, 0x01]; + bytes.extend(std::iter::repeat(0x91u8).take(MAX_PAYLOAD_DEPTH - 2)); + bytes.push(0x90); + let decoded = decode_on::(64 * KIB, bytes).unwrap(); + assert_eq!(decoded, One { a: 1 }); + }); +} diff --git a/packages/cultcache-rs/tests/union_parity.rs b/packages/cultcache-rs/tests/union_parity.rs index da2b1a08..a566e627 100644 --- a/packages/cultcache-rs/tests/union_parity.rs +++ b/packages/cultcache-rs/tests/union_parity.rs @@ -781,12 +781,12 @@ fn the_raw_doors_hold_the_bound_for_arms_maps_and_arrays() { } } -// A debug build on a 2 MiB thread carries 405 containers of this document; the bound keeps every -// accepted payload below that, and every refusal is an error, not an abort. +// The bound counts containers, not stack: decode grows its own stack, so the document at the limit +// decodes on a 1 MiB thread in both profiles, and every refusal is an error, not an abort. #[test] -fn stack_budget_debug_2mib() { +fn the_union_chain_at_the_limit_decodes_on_a_1mib_thread() { std::thread::Builder::new() - .stack_size(2 * 1024 * 1024) + .stack_size(1024 * 1024) .spawn(|| { assert_eq!( through_cache(&vector("depth-at-limit.msgpack")).unwrap(), From 2505c0c45f4c5ae89762a59e66fa4e6ebf3b3084 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 14 Oct 2026 02:55:21 +0200 Subject: [PATCH 02/13] decode_stack test: repeat_n for the clippy lint Co-Authored-By: Claude Opus 5.5 --- packages/cultcache-rs/tests/decode_stack.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cultcache-rs/tests/decode_stack.rs b/packages/cultcache-rs/tests/decode_stack.rs index 14546519..e92da265 100644 --- a/packages/cultcache-rs/tests/decode_stack.rs +++ b/packages/cultcache-rs/tests/decode_stack.rs @@ -231,7 +231,7 @@ fn a_skipped_slot_at_the_limit_decodes_on_a_64kib_thread() { in_child("a_skipped_slot_at_the_limit_decodes_on_a_64kib_thread", || { // [1, [[...]]]: slot 1 is unknown to One and is skipped; the whole payload is 256 deep. let mut bytes = vec![0x92u8, 0x01]; - bytes.extend(std::iter::repeat(0x91u8).take(MAX_PAYLOAD_DEPTH - 2)); + bytes.extend(std::iter::repeat_n(0x91u8, MAX_PAYLOAD_DEPTH - 2)); bytes.push(0x90); let decoded = decode_on::(64 * KIB, bytes).unwrap(); assert_eq!(decoded, One { a: 1 }); From fc630d3b85dacbf095da4eae303c0350e11d74ed Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 14 Oct 2026 19:48:57 +0200 Subject: [PATCH 03/13] cultcache-rs 0.3.0: one grown decoder for stored bytes, derives refuse records over 16 KiB decode_grown checks the stack before any frame that holds T, then serde_stacker at every container; decode_payload walks the nesting bound and calls it; the legacy envelope array, redb reads and v1 snapshots decode through it too. Red zone 512 KiB against MAX_RECORD_INLINE_BYTES = 16 KiB, which the DatabaseEntry, CultSlots and CultUnion derives assert at compile time. cultcache-rs-derive 0.2.0. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-rs/Cargo.toml | 3 +- .../cultcache-rs-derive/Cargo.toml | 2 +- .../cultcache-rs-derive/src/lib.rs | 10 ++ packages/cultcache-rs/src/lib.rs | 108 ++++++++++++++---- 4 files changed, 99 insertions(+), 24 deletions(-) diff --git a/packages/cultcache-rs/Cargo.toml b/packages/cultcache-rs/Cargo.toml index 624ae887..9610aeb6 100644 --- a/packages/cultcache-rs/Cargo.toml +++ b/packages/cultcache-rs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cultcache-rs" -version = "0.2.0" +version = "0.3.0" edition = "2024" license = "MIT" publish = false @@ -23,6 +23,7 @@ serde = { version = "1", features = ["derive"] } serde_bytes = "0.11" serde_json = "1" serde_stacker = "0.1.14" +stacker = "0.1.25" uuid = { version = "1", features = ["v4"] } [dev-dependencies] diff --git a/packages/cultcache-rs/cultcache-rs-derive/Cargo.toml b/packages/cultcache-rs/cultcache-rs-derive/Cargo.toml index 74d52a4f..de299ff7 100644 --- a/packages/cultcache-rs/cultcache-rs-derive/Cargo.toml +++ b/packages/cultcache-rs/cultcache-rs-derive/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cultcache-rs-derive" -version = "0.1.0" +version = "0.2.0" edition = "2024" publish = false diff --git a/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs b/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs index d172e4fd..65e41b92 100644 --- a/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs +++ b/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs @@ -275,6 +275,11 @@ fn slot_codec(ident: &syn::Ident, slots: &[CultCacheSlot]) -> proc_macro2::Token }); quote! { + const _: () = ::core::assert!( + ::core::mem::size_of::<#ident>() <= ::cultcache_rs::MAX_RECORD_INLINE_BYTES, + "a CultCache record is at most cultcache_rs::MAX_RECORD_INLINE_BYTES bytes inline; box its largest slot" + ); + impl ::cultcache_rs::SlotRecord for #ident { fn __slots_array() {} } @@ -509,6 +514,11 @@ fn union_codec(input: DeriveInput) -> syn::Result { }); Ok(quote! { + const _: () = ::core::assert!( + ::core::mem::size_of::<#ident>() <= ::cultcache_rs::MAX_RECORD_INLINE_BYTES, + "a CultCache record is at most cultcache_rs::MAX_RECORD_INLINE_BYTES bytes inline; box its largest slot" + ); + // An arm is a slots array: a newtype over anything else is refused here, by the // trait the CultSlots and DatabaseEntry derives implement. const _: fn() = || { diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs index 4eccfd73..98258845 100644 --- a/packages/cultcache-rs/src/lib.rs +++ b/packages/cultcache-rs/src/lib.rs @@ -595,30 +595,94 @@ fn element_header(bytes: &[u8], position: usize) -> Option<(usize, Option }) } -/// The stack left below which a decode moves to a fresh segment. Measured (map D4): the worst -/// distance between two stack checks is 128-144 KiB in a debug build and under 32 KiB in release, -/// so 256 KiB leaves a margin in both. A frame larger than this between two checks still overruns. -const DECODE_RED_ZONE: usize = 256 * 1024; +/// The stack left below which a decode moves to a fresh segment. Measured (map D10, D12): the +/// distance between two stack checks is 14 times the inline size of the record entered plus 3 KiB in +/// a debug build and 6 times in release, so at [`MAX_RECORD_INLINE_BYTES`] it is 220 KiB and 93 KiB; +/// 512 KiB is 2.3 times the larger. A record the derives produced can never exceed that bound. +const DECODE_RED_ZONE: usize = 512 * 1024; -/// The size of each segment a decode grows onto. It does not change the cost of a decode (map D6). +/// The size of each segment a decode grows onto, four red zones. It does not change the cost of a +/// decode (map D6); it only has to exceed the red zone plus one gap. const DECODE_STACK_SEGMENT: usize = 2 * 1024 * 1024; -/// The one decode of a document payload: the nesting bound first, then the deserializer, whose own -/// depth counter (1024) is only a backstop. A failure keeps its kind and never quotes the bytes. -/// Readers of stored envelope payloads outside this crate decode through it too. +/// The largest `size_of` a record the derives produce may have. `DatabaseEntry`, `CultSlots` and +/// `CultUnion` refuse a larger one at compile time: put its large data behind a `Vec` or a `Box`. /// -/// The deserializer runs under `serde_stacker`, which moves onto a fresh stack segment whenever less -/// than the red zone remains, at every container level serde's impls descend through, skipped values -/// included. The calling thread's stack size therefore never decides whether a payload inside the -/// bound decodes. The one residual is a single frame larger than the red zone. On wasm32 the shadow -/// stack grows the same way, while the engine's own call stack stays the host's limit. +/// Decoding builds a record in stack frames that hold it, and the distance between two stack checks +/// grows with its size, so the decode's red zone is set against a record of exactly this size. A +/// field of a type the derives did not produce (a plain serde struct, a large tuple) that exceeds +/// this size behind a `Box` or `Vec` is outside the guarantee. +/// +/// ```compile_fail,E0080 +/// use cultcache_rs::CultSlots; +/// +/// #[derive(CultSlots)] +/// struct TooLarge { +/// #[cultcache(key = 0)] +/// bytes: [u8; 16 * 1024 + 1], +/// } +/// ``` +/// +/// A union is refused by its own size, though every arm is within the limit: +/// +/// ```compile_fail,E0080 +/// use cultcache_rs::{CultSlots, CultUnion}; +/// +/// #[derive(CultSlots)] +/// struct Arm { +/// #[cultcache(key = 0)] +/// a: [[u64; 32]; 32], +/// #[cultcache(key = 1)] +/// b: [[u64; 32]; 32], +/// } +/// +/// #[derive(CultUnion)] +/// enum TooLarge { +/// #[cultcache(key = 0)] +/// Arm(Arm), +/// #[cultcache(key = 1)] +/// Leaf, +/// } +/// ``` +/// +/// Just under the limit compiles: +/// +/// ``` +/// use cultcache_rs::CultSlots; +/// +/// #[derive(CultSlots)] +/// struct JustUnder { +/// #[cultcache(key = 0)] +/// bytes: Vec, +/// #[cultcache(key = 1)] +/// header: [u8; 16 * 1024 - 64], +/// } +/// ``` +pub const MAX_RECORD_INLINE_BYTES: usize = 16 * 1024; + +/// The one grown decoder of stored bytes: the stack check comes first, before any frame that holds +/// a `T`, then `serde_stacker` re-checks at every container serde's impls descend through, skipped +/// values included. No nesting bound is applied here; [`decode_payload`] walks first. +fn decode_grown(bytes: &[u8]) -> std::result::Result { + stacker::maybe_grow(DECODE_RED_ZONE, DECODE_STACK_SEGMENT, || { + let mut reader = rmp_serde::Deserializer::from_read_ref(bytes); + let mut grown = serde_stacker::Deserializer::new(&mut reader); + grown.red_zone = DECODE_RED_ZONE; + grown.stack_size = DECODE_STACK_SEGMENT; + T::deserialize(grown) + }) +} + +/// The one decode of a document payload: the nesting bound first, then [`decode_grown`], whose +/// rmp-serde depth counter (1024) is only a backstop. A failure keeps its kind and never quotes the +/// bytes. Readers of stored envelope payloads outside this crate decode through it too. +/// +/// A payload inside the bound decodes whatever the calling thread's stack: the thread needs only its +/// own frames that hold the returned value. That holds for every record the derives produce, which +/// [`MAX_RECORD_INLINE_BYTES`] bounds; Windows stack growth is not yet verified. pub fn decode_payload(payload: &[u8]) -> Result { check_payload_depth(payload)?; - let mut reader = rmp_serde::Deserializer::from_read_ref(payload); - let mut grown = serde_stacker::Deserializer::new(&mut reader); - grown.red_zone = DECODE_RED_ZONE; - grown.stack_size = DECODE_STACK_SEGMENT; - ::deserialize(grown).map_err(opaque_decode_error) + decode_grown(payload).map_err(opaque_decode_error) } /// The nesting bound for an envelope admitted without a typed decode, from the same walk that @@ -1158,7 +1222,7 @@ impl SingleFileMessagePackBackingStore { CultCacheStoreUnreadableKind::UnsupportedFormat, anyhow!("its header is {}; this runtime reads {STORE_FORMAT_V1:?} only", describe_header(&header)), )), - _ => rmp_serde::from_slice(&bytes) + _ => decode_grown(&bytes) .context("failed to decode legacy CultCache envelope array") .map_err(|error| unreadable(CultCacheStoreUnreadableKind::Undecodable, error)), } @@ -2311,7 +2375,7 @@ fn read_redb_entry( table .get(key.as_slice())? .map(|value| { - rmp_serde::from_slice(value.value()) + decode_grown(value.value()) .map_err(opaque_decode_error) .context("failed to decode redb CultCache envelope") }) @@ -2335,7 +2399,7 @@ fn read_all_redb( .iter()? .map(|row| { let (_, value) = row?; - rmp_serde::from_slice(value.value()) + decode_grown(value.value()) .map_err(opaque_decode_error) .context("failed to decode redb CultCache envelope") }) @@ -3025,7 +3089,7 @@ fn encode_store_snapshot(entries: &[CultCacheEnvelope]) -> Result Result> { let snapshot: PersistedStoreSnapshot = - rmp_serde::from_slice(bytes).context("failed to decode CultCache v1 snapshot")?; + decode_grown(bytes).context("failed to decode CultCache v1 snapshot")?; let catalog = snapshot .1 From 40a089ff9acd6bb65f0eb5fd4aaebc2ef4b83396 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 14 Oct 2026 19:48:58 +0200 Subject: [PATCH 04/13] decode_stack tests: shallow depths, 64 KiB door, largest record on every thread size, legacy envelope Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-rs/tests/decode_stack.rs | 139 +++++++++++++++++++- 1 file changed, 138 insertions(+), 1 deletion(-) diff --git a/packages/cultcache-rs/tests/decode_stack.rs b/packages/cultcache-rs/tests/decode_stack.rs index e92da265..819e9b79 100644 --- a/packages/cultcache-rs/tests/decode_stack.rs +++ b/packages/cultcache-rs/tests/decode_stack.rs @@ -1,11 +1,17 @@ -//! Typed decode never aborts a payload inside the nesting bound, whatever the caller's stack. +//! Typed decode never aborts a payload inside the nesting bound, whatever the caller's stack, for +//! every record the derives accept (up to `MAX_RECORD_INLINE_BYTES` inline); envelope reads of a +//! store go through the same grown decoder. //! Each test runs its decode in a child test process on a thread of a stated size, so an abort //! fails one named test. Values are built, encoded, compared and dropped on a large thread; only //! the decode runs on the small one. use cultcache_rs::CultSlots; +use cultcache_rs::CultUnion; use cultcache_rs::MAX_PAYLOAD_DEPTH; +use cultcache_rs::MAX_RECORD_INLINE_BYTES; use cultcache_rs::PayloadTooDeep; +use cultcache_rs::SingleFileMessagePackBackingStore; +use cultcache_rs::check_payload_depth; use cultcache_rs::decode_payload; use serde::de::DeserializeOwned; @@ -237,3 +243,134 @@ fn a_skipped_slot_at_the_limit_decodes_on_a_64kib_thread() { assert_eq!(decoded, One { a: 1 }); }); } + +// A record the derives accept at the bound: two slots of 31 arrays of 32 words, 15,880 bytes inline. +#[derive(Clone, Debug, PartialEq, CultSlots)] +struct Largest { + #[cultcache(key = 0)] + a: [[u64; 32]; 31], + #[cultcache(key = 1)] + b: [[u64; 32]; 31], + #[cultcache(key = 2)] + next: Option>, +} + +#[derive(Clone, Debug, PartialEq, CultSlots)] +struct LargestUnionArm { + #[cultcache(key = 0)] + a: [[u64; 32]; 31], + #[cultcache(key = 1)] + b: [[u64; 32]; 31], + #[cultcache(key = 2)] + next: Option>, +} + +#[derive(Clone, Debug, PartialEq, CultUnion)] +enum LargestUnion { + #[cultcache(key = 0)] + Arm(LargestUnionArm), + #[cultcache(key = 1)] + Leaf, +} + +// The fixtures sit within a KiB under the bound, so a decode that survives them survives any record. +const _: () = assert!( + std::mem::size_of::() <= MAX_RECORD_INLINE_BYTES + && std::mem::size_of::() + KIB >= MAX_RECORD_INLINE_BYTES + && std::mem::size_of::() <= MAX_RECORD_INLINE_BYTES + && std::mem::size_of::() + KIB >= MAX_RECORD_INLINE_BYTES +); + +fn largest_chain(levels: usize) -> Largest { + let mut chain: Option> = None; + for level in (0..levels).rev() { + let mut a = [[0u64; 32]; 31]; + a[0][0] = level as u64; + chain = Some(Box::new(Largest { a, b: [[0; 32]; 31], next: chain })); + } + *chain.unwrap() +} + +fn union_chain(levels: usize) -> LargestUnion { + let mut chain = LargestUnion::Leaf; + for level in (0..levels).rev() { + let mut a = [[0u64; 32]; 31]; + a[0][0] = level as u64; + chain = LargestUnion::Arm(LargestUnionArm { a, b: [[0; 32]; 31], next: Some(Box::new(chain)) }); + } + chain +} + +#[test] +fn a_wide_record_20_levels_deep_decodes_on_a_1mib_thread() { + in_child("a_wide_record_20_levels_deep_decodes_on_a_1mib_thread", || decode_wide_chain(20, 1024 * KIB)); +} + +#[test] +fn a_wide_record_100_levels_deep_decodes_on_a_1mib_thread() { + in_child("a_wide_record_100_levels_deep_decodes_on_a_1mib_thread", || decode_wide_chain(100, 1024 * KIB)); +} + +#[test] +fn a_wide_record_decodes_on_a_64kib_thread_at_one_level_and_at_the_limit() { + in_child("a_wide_record_decodes_on_a_64kib_thread_at_one_level_and_at_the_limit", || { + decode_wide_chain(1, 64 * KIB); + decode_wide_chain(MAX_PAYLOAD_DEPTH - 1, 64 * KIB); + }); +} + +fn decode_wide_chain(levels: usize, stack: usize) { + let bytes = on_stack(64 * KIB * KIB, move || rmp_serde::to_vec(&wide_chain(levels)).unwrap()); + let decoded = decode_on::(stack, bytes).unwrap(); + on_stack(64 * KIB * KIB, move || assert!(decoded == wide_chain(levels))); +} + +#[test] +fn the_largest_record_at_the_limit_decodes_on_every_thread_size() { + in_child("the_largest_record_at_the_limit_decodes_on_every_thread_size", || { + // 254 records of Largest are 256 containers deep (each is one, its slots' arrays two more); + // 127 union levels are the same (a union and its arm are two each, the leaf's arrays two). + let (records, unions) = on_stack(64 * KIB * KIB, || { + let records = rmp_serde::to_vec(&largest_chain(254)).unwrap(); + let unions = rmp_serde::to_vec(&union_chain(127)).unwrap(); + assert!(check_payload_depth(&records).is_ok()); + assert!(check_payload_depth(&unions).is_ok()); + assert!(check_payload_depth(&rmp_serde::to_vec(&largest_chain(255)).unwrap()).is_err()); + assert!(check_payload_depth(&rmp_serde::to_vec(&union_chain(128)).unwrap()).is_err()); + (records, unions) + }); + let sizes = (128..=1024).step_by(32).map(|kib| kib * KIB).chain([1536, 2048, 3072, 4096, 8192].map(|kib| kib * KIB)); + for size in sizes { + println!("thread of {} KiB", size / KIB); + let decoded = decode_on::(size, records.clone()).unwrap(); + let decoded_union = decode_on::(size, unions.clone()).unwrap(); + on_stack(64 * KIB * KIB, move || { + assert!(decoded == largest_chain(254)); + assert!(decoded_union == union_chain(127)); + }); + } + }); +} + +#[test] +fn a_legacy_store_with_a_deep_unknown_envelope_key_reads_on_a_256kib_thread() { + in_child("a_legacy_store_with_a_deep_unknown_envelope_key_reads_on_a_256kib_thread", || { + // [ {key: "k", type: "t", payload: bin[1], storedAt: "s", extra: [[[...]]]} ]: the envelope + // skips the unknown key, 1000 containers deep, with no payload walk (the bound is the payload's). + let mut bytes = vec![0x91, 0x85]; + for (name, value) in [("key", &[0xa1, b'k'][..]), ("type", &[0xa1, b't']), ("payload", &[0xc4, 1, 1]), ("storedAt", &[0xa1, b's'])] { + bytes.push(0xa0 | name.len() as u8); + bytes.extend(name.as_bytes()); + bytes.extend(value); + } + bytes.extend([0xa5, b'e', b'x', b't', b'r', b'a']); + bytes.extend(std::iter::repeat_n(0x91u8, 999)); + bytes.push(0x90); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("legacy.cc"); + std::fs::write(&path, bytes).unwrap(); + let entries = on_stack(256 * KIB, move || SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()).unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].key, "k"); + }); +} From 95b98e588bf2b215d1f10d6f4119ae69a9c84586 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 14 Oct 2026 19:48:58 +0200 Subject: [PATCH 05/13] cultcache-rs README: security model without the wasm claim, the record bound, build dependencies Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-rs/README.md | 34 +++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/packages/cultcache-rs/README.md b/packages/cultcache-rs/README.md index 35999e5c..9b017ce1 100644 --- a/packages/cultcache-rs/README.md +++ b/packages/cultcache-rs/README.md @@ -457,14 +457,32 @@ The derive/registry work should preserve that shape. Its job is to remove manual registration ceremony, not to loosen the closed-world boundary. Payload nesting is bounded at 256 MessagePack containers, checked before any -recursion, and typed decode then runs under `serde_stacker`, which moves onto a -fresh stack segment whenever less than 256 KiB of stack remains. A payload inside -the bound therefore decodes on a small thread instead of aborting the process; -past the bound the refusal is a typed `PayloadTooDeep`. The residual is a single -stack frame larger than the 256 KiB red zone. Stack growth on Windows is not yet -verified. On wasm32 the shadow stack grows the same way, but the engine's own -call stack remains the host's limit. Building the crate needs a C toolchain -(`psm`, `stacker`). +recursion; past the bound the refusal is a typed `PayloadTooDeep`. Typed decode +of stored bytes then grows its stack: the decoder checks the remaining stack +before its first frame and `serde_stacker` checks it at every container, moving +onto a fresh segment whenever less than 512 KiB remains, so a payload inside the +bound decodes on a small thread instead of aborting the process. Envelope and +store reads (single-file, redb, v1 snapshots) use the same grown decoder and +keep accepting and refusing what they did, without the payload walk. + +The guarantee holds for records the derives produce. `MAX_RECORD_INLINE_BYTES` +(16 KiB) is the largest `size_of` a `DatabaseEntry`, `CultSlots` or `CultUnion` +record may have, and the derives refuse a larger one at compile time: put its +large data behind a `Vec` or a `Box`. A field of a type the derives did not +produce that is larger than that, behind a `Box` or `Vec`, is outside the +guarantee. The caller's thread needs only its own frames, which hold the +returned value. Stack growth on Windows is not yet verified. The crate does not +build for wasm32 (`fs2`), and no wasm behaviour is claimed. + +### Build dependencies + +Stack growth adds these crates to every consumer's build, and a C toolchain +(`psm`, `stacker`). On all targets: `serde_stacker` 0.1.14, `stacker` 0.1.25, +`psm` 0.1.32, `cc` 1.8.0, `find-msvc-tools` 0.1.14, `shlex` 2.0.1 (MIT OR +Apache-2.0), `object` 0.39.1 (Apache-2.0 OR MIT) and `ar_archive_writer` 0.5.3 +(Apache-2.0 WITH LLVM-exception, build-only). On `x86_64-pc-windows-msvc` also +`libc` 0.2.190 and `windows-sys` 0.61.2 (MIT OR Apache-2.0), beside the 0.59 the +crate uses itself. ## License From f7c0902d8d846a3b048a1d37df3c8567ea380619 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 14 Oct 2026 19:56:57 +0200 Subject: [PATCH 06/13] decode_stack tests: box the decoded result on the small thread, measured thread sizes; doctest fixture serde-able Measured on Yggdrasil: the decode needs 56 KiB (Wide32) and 80 KiB (Largest) in debug on a thread that returns a bool; returning the record through std::thread's own frames adds several copies of it, so the tests box it and size threads at 96 KiB and from 160 KiB. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-rs/src/lib.rs | 2 +- packages/cultcache-rs/tests/decode_stack.rs | 15 +++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs index 98258845..ba40b72a 100644 --- a/packages/cultcache-rs/src/lib.rs +++ b/packages/cultcache-rs/src/lib.rs @@ -655,7 +655,7 @@ const DECODE_STACK_SEGMENT: usize = 2 * 1024 * 1024; /// #[cultcache(key = 0)] /// bytes: Vec, /// #[cultcache(key = 1)] -/// header: [u8; 16 * 1024 - 64], +/// header: [[u64; 32]; 63], /// } /// ``` pub const MAX_RECORD_INLINE_BYTES: usize = 16 * 1024; diff --git a/packages/cultcache-rs/tests/decode_stack.rs b/packages/cultcache-rs/tests/decode_stack.rs index 819e9b79..444667d6 100644 --- a/packages/cultcache-rs/tests/decode_stack.rs +++ b/packages/cultcache-rs/tests/decode_stack.rs @@ -177,8 +177,11 @@ fn empty_wide() -> Wide32 { } } +/// The decode alone runs on the small thread. Its result is boxed there so that the thread's own +/// return machinery, which would hold several more copies of a large record in a debug build, does +/// not count against the stack under test; what the caller's frames hold is the caller's to size. fn decode_on(stack: usize, bytes: Vec) -> anyhow::Result { - on_stack(stack, move || decode_payload::(&bytes)) + *on_stack(stack, move || Box::new(decode_payload::(&bytes))) } #[test] @@ -312,10 +315,10 @@ fn a_wide_record_100_levels_deep_decodes_on_a_1mib_thread() { } #[test] -fn a_wide_record_decodes_on_a_64kib_thread_at_one_level_and_at_the_limit() { - in_child("a_wide_record_decodes_on_a_64kib_thread_at_one_level_and_at_the_limit", || { - decode_wide_chain(1, 64 * KIB); - decode_wide_chain(MAX_PAYLOAD_DEPTH - 1, 64 * KIB); +fn a_wide_record_decodes_on_a_96kib_thread_at_one_level_and_at_the_limit() { + in_child("a_wide_record_decodes_on_a_96kib_thread_at_one_level_and_at_the_limit", || { + decode_wide_chain(1, 96 * KIB); + decode_wide_chain(MAX_PAYLOAD_DEPTH - 1, 96 * KIB); }); } @@ -339,7 +342,7 @@ fn the_largest_record_at_the_limit_decodes_on_every_thread_size() { assert!(check_payload_depth(&rmp_serde::to_vec(&union_chain(128)).unwrap()).is_err()); (records, unions) }); - let sizes = (128..=1024).step_by(32).map(|kib| kib * KIB).chain([1536, 2048, 3072, 4096, 8192].map(|kib| kib * KIB)); + let sizes = (160..=1024).step_by(32).map(|kib| kib * KIB).chain([1536, 2048, 3072, 4096, 8192].map(|kib| kib * KIB)); for size in sizes { println!("thread of {} KiB", size / KIB); let decoded = decode_on::(size, records.clone()).unwrap(); From 57243090ec7a0e40f8ae9b0ad9350db74d92ce5a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 14 Oct 2026 20:02:52 +0200 Subject: [PATCH 07/13] MAX_RECORD_INLINE_BYTES doctests: serde-able fixtures, positive twins of each refusal Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-rs/src/lib.rs | 39 +++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs index ba40b72a..2c34d260 100644 --- a/packages/cultcache-rs/src/lib.rs +++ b/packages/cultcache-rs/src/lib.rs @@ -619,7 +619,11 @@ const DECODE_STACK_SEGMENT: usize = 2 * 1024 * 1024; /// #[derive(CultSlots)] /// struct TooLarge { /// #[cultcache(key = 0)] -/// bytes: [u8; 16 * 1024 + 1], +/// a: [[u64; 32]; 32], +/// #[cultcache(key = 1)] +/// b: [[u64; 32]; 32], +/// #[cultcache(key = 2)] +/// one_more: u8, /// } /// ``` /// @@ -645,17 +649,40 @@ const DECODE_STACK_SEGMENT: usize = 2 * 1024 * 1024; /// } /// ``` /// -/// Just under the limit compiles: +/// A record of exactly the limit, a union over a record of exactly the limit's half, and a record +/// under it with a `Vec` for its large data compile: /// /// ``` -/// use cultcache_rs::CultSlots; +/// use cultcache_rs::{CultSlots, CultUnion}; /// /// #[derive(CultSlots)] -/// struct JustUnder { +/// struct AtTheLimit { /// #[cultcache(key = 0)] -/// bytes: Vec, +/// a: [[u64; 32]; 32], /// #[cultcache(key = 1)] -/// header: [[u64; 32]; 63], +/// b: [[u64; 32]; 32], +/// } +/// +/// #[derive(CultSlots)] +/// struct Half { +/// #[cultcache(key = 0)] +/// a: [[u64; 32]; 32], +/// } +/// +/// #[derive(CultUnion)] +/// enum Fits { +/// #[cultcache(key = 0)] +/// Half(Half), +/// #[cultcache(key = 1)] +/// Leaf, +/// } +/// +/// #[derive(CultSlots)] +/// struct Boxed { +/// #[cultcache(key = 0)] +/// large: Vec, +/// #[cultcache(key = 1)] +/// header: [[u64; 32]; 31], /// } /// ``` pub const MAX_RECORD_INLINE_BYTES: usize = 16 * 1024; From b43c4dfc98548a5a4d2b6573aacc9fb532eef8bc Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 14 Oct 2026 20:08:16 +0200 Subject: [PATCH 08/13] decode_stack tests: v1 snapshot and redb reads through the grown decoder Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-rs/tests/decode_stack.rs | 109 +++++++++++++++++--- 1 file changed, 97 insertions(+), 12 deletions(-) diff --git a/packages/cultcache-rs/tests/decode_stack.rs b/packages/cultcache-rs/tests/decode_stack.rs index 444667d6..8ecd0708 100644 --- a/packages/cultcache-rs/tests/decode_stack.rs +++ b/packages/cultcache-rs/tests/decode_stack.rs @@ -5,11 +5,16 @@ //! fails one named test. Values are built, encoded, compared and dropped on a large thread; only //! the decode runs on the small one. +use cultcache_rs::CacheBackingStore; +use cultcache_rs::CultCacheEnvelope; +use cultcache_rs::CultCacheStoreUnreadable; +use cultcache_rs::CultCacheStoreUnreadableKind; use cultcache_rs::CultSlots; use cultcache_rs::CultUnion; use cultcache_rs::MAX_PAYLOAD_DEPTH; use cultcache_rs::MAX_RECORD_INLINE_BYTES; use cultcache_rs::PayloadTooDeep; +use cultcache_rs::RedbMessagePackBackingStore; use cultcache_rs::SingleFileMessagePackBackingStore; use cultcache_rs::check_payload_depth; use cultcache_rs::decode_payload; @@ -355,25 +360,105 @@ fn the_largest_record_at_the_limit_decodes_on_every_thread_size() { }); } +/// One stored envelope, `{key: "k", type: "t", payload: bin[1], storedAt: "s", extra: [[[...]]]}`, +/// whose unknown key nests 1000 containers deep and is skipped without a payload walk. +fn deep_envelope() -> Vec { + let mut bytes = vec![0x85]; + for (name, value) in [("key", &[0xa1, b'k'][..]), ("type", &[0xa1, b't']), ("payload", &[0xc4, 1, 1]), ("storedAt", &[0xa1, b's'])] { + bytes.push(0xa0 | name.len() as u8); + bytes.extend(name.as_bytes()); + bytes.extend(value); + } + bytes.extend([0xa5, b'e', b'x', b't', b'r', b'a']); + bytes.extend(std::iter::repeat_n(0x91u8, 999)); + bytes.push(0x90); + bytes +} + +fn fixstr(text: &str) -> Vec { + let mut bytes = vec![0xa0 | text.len() as u8]; + bytes.extend(text.as_bytes()); + bytes +} + #[test] fn a_legacy_store_with_a_deep_unknown_envelope_key_reads_on_a_256kib_thread() { in_child("a_legacy_store_with_a_deep_unknown_envelope_key_reads_on_a_256kib_thread", || { - // [ {key: "k", type: "t", payload: bin[1], storedAt: "s", extra: [[[...]]]} ]: the envelope - // skips the unknown key, 1000 containers deep, with no payload walk (the bound is the payload's). - let mut bytes = vec![0x91, 0x85]; - for (name, value) in [("key", &[0xa1, b'k'][..]), ("type", &[0xa1, b't']), ("payload", &[0xc4, 1, 1]), ("storedAt", &[0xa1, b's'])] { - bytes.push(0xa0 | name.len() as u8); - bytes.extend(name.as_bytes()); - bytes.extend(value); - } - bytes.extend([0xa5, b'e', b'x', b't', b'r', b'a']); - bytes.extend(std::iter::repeat_n(0x91u8, 999)); - bytes.push(0x90); let directory = tempfile::tempdir().unwrap(); let path = directory.path().join("legacy.cc"); - std::fs::write(&path, bytes).unwrap(); + std::fs::write(&path, [vec![0x91], deep_envelope()].concat()).unwrap(); let entries = on_stack(256 * KIB, move || SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()).unwrap(); assert_eq!(entries.len(), 1); assert_eq!(entries[0].key, "k"); }); } + +#[test] +fn a_v1_store_with_a_deep_fifth_record_slot_is_refused_typed_on_a_256kib_thread() { + in_child("a_v1_store_with_a_deep_fifth_record_slot_is_refused_typed_on_a_256kib_thread", || { + // [header, [], [[key, schema, storedAt, payload, [[[...]]]]]]: the record's unknown fifth slot is + // skipped 1000 containers deep, then the record is refused. + let mut bytes = vec![0x93]; + bytes.extend(fixstr("cultcache.store.v1")); + bytes.extend([0x90, 0x91, 0x95]); + for text in ["k", "s", "t"] { + bytes.extend(fixstr(text)); + } + bytes.extend([0xc4, 1, 1]); + bytes.extend(std::iter::repeat_n(0x91u8, 999)); + bytes.push(0x90); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("v1.cc"); + std::fs::write(&path, bytes).unwrap(); + let error = on_stack(256 * KIB, move || SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()).unwrap_err(); + let unreadable = error.downcast_ref::().expect("a typed refusal"); + assert!(matches!(unreadable.kind, CultCacheStoreUnreadableKind::Undecodable)); + }); +} + +/// A redb store holding the one deep envelope under its identity, `["t", "k"]`. +fn redb_with_deep_row(path: &std::path::Path) { + let database = redb::Database::create(path).unwrap(); + let write = database.begin_write().unwrap(); + { + let mut table = write + .open_table(redb::TableDefinition::<&[u8], &[u8]>::new("cultcache_envelopes")) + .unwrap(); + table.insert([0x92, 0xa1, b't', 0xa1, b'k'].as_slice(), deep_envelope().as_slice()).unwrap(); + } + write.commit().unwrap(); +} + +#[test] +fn a_redb_store_with_a_deep_unknown_envelope_key_reads_on_a_1mib_thread() { + in_child("a_redb_store_with_a_deep_unknown_envelope_key_reads_on_a_1mib_thread", || { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("rows.redb"); + redb_with_deep_row(&path); + let entries = on_stack(1024 * KIB, move || RedbMessagePackBackingStore::new(path).unwrap().pull_all()).unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].key, "k"); + }); +} + +#[test] +fn a_redb_compare_and_swap_reads_a_deep_unknown_envelope_key_on_a_1mib_thread() { + in_child("a_redb_compare_and_swap_reads_a_deep_unknown_envelope_key_on_a_1mib_thread", || { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("rows.redb"); + redb_with_deep_row(&path); + let expected = CultCacheEnvelope { + key: "k".into(), + r#type: "t".into(), + payload: vec![1], + stored_at: "s".into(), + schema_id: None, + }; + let replacement = CultCacheEnvelope { payload: vec![2], ..expected.clone() }; + let swapped = on_stack(1024 * KIB, move || { + RedbMessagePackBackingStore::new(path).unwrap().compare_and_swap_entry(&expected, replacement) + }) + .unwrap(); + assert!(swapped); + }); +} From 74ecbb2e0c179e783dc50f66d6bb8a048e10ba55 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 14 Oct 2026 20:22:34 +0200 Subject: [PATCH 09/13] decode_stack tests: the at-bound union fixture is allowed its large variant Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-rs/tests/decode_stack.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/cultcache-rs/tests/decode_stack.rs b/packages/cultcache-rs/tests/decode_stack.rs index 8ecd0708..c2e61a3d 100644 --- a/packages/cultcache-rs/tests/decode_stack.rs +++ b/packages/cultcache-rs/tests/decode_stack.rs @@ -273,6 +273,8 @@ struct LargestUnionArm { next: Option>, } +// The fixture is the largest arm the derives accept next to a unit variant, which is the point. +#[allow(clippy::large_enum_variant)] #[derive(Clone, Debug, PartialEq, CultUnion)] enum LargestUnion { #[cultcache(key = 0)] From dbafbf52bc778cd131e37bd8eee60a06a7cf8902 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 14 Oct 2026 20:57:58 +0200 Subject: [PATCH 10/13] cultcache-rs docs: state the decoder's fixed stack floor, record the Windows run The README and decode_payload claimed the calling thread needs only its own frames. Measured, the door has a payload-independent floor (release 56 KiB vs 16 for a same-signature non-decoding function; debug 104 vs 88). Say so, and replace "Windows not yet verified" with the one by-hand Starfire run of decode_stack on x86_64-pc-windows-msvc. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-rs/README.md | 7 +++++-- packages/cultcache-rs/src/lib.rs | 8 +++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/packages/cultcache-rs/README.md b/packages/cultcache-rs/README.md index 9b017ce1..2d1fa671 100644 --- a/packages/cultcache-rs/README.md +++ b/packages/cultcache-rs/README.md @@ -470,8 +470,11 @@ The guarantee holds for records the derives produce. `MAX_RECORD_INLINE_BYTES` record may have, and the derives refuse a larger one at compile time: put its large data behind a `Vec` or a `Box`. A field of a type the derives did not produce that is larger than that, behind a `Box` or `Vec`, is outside the -guarantee. The caller's thread needs only its own frames, which hold the -returned value. Stack growth on Windows is not yet verified. The crate does not +guarantee. The decoder's own frames have a fixed floor that no payload changes: at the bound a thread decodes a record of +this size from about 56 KiB in a release build and 104 KiB in a debug build, against 16 and 88 KiB for a function +of the same signature that decodes nothing, plus the caller's own frames. Stack growth was verified on +x86_64-pc-windows-msvc once, by hand, on Starfire (`decode_stack`, debug and release); later `stacker` bumps repeat it. +The crate does not build for wasm32 (`fs2`), and no wasm behaviour is claimed. ### Build dependencies diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs index 2c34d260..4714ab79 100644 --- a/packages/cultcache-rs/src/lib.rs +++ b/packages/cultcache-rs/src/lib.rs @@ -704,9 +704,11 @@ fn decode_grown(bytes: &[u8]) -> std::result::Re /// rmp-serde depth counter (1024) is only a backstop. A failure keeps its kind and never quotes the /// bytes. Readers of stored envelope payloads outside this crate decode through it too. /// -/// A payload inside the bound decodes whatever the calling thread's stack: the thread needs only its -/// own frames that hold the returned value. That holds for every record the derives produce, which -/// [`MAX_RECORD_INLINE_BYTES`] bounds; Windows stack growth is not yet verified. +/// A payload inside the bound decodes on any thread that has the decoder's fixed floor, which no payload +/// changes: at the bound, about 56 KiB in release and 104 KiB in debug, against 16 and 88 KiB for a +/// same-signature function that decodes nothing, plus the caller's own frames. That holds for every +/// record the derives produce, which [`MAX_RECORD_INLINE_BYTES`] bounds. Stack growth was verified on +/// Windows once, by hand; a `stacker` bump repeats it. pub fn decode_payload(payload: &[u8]) -> Result { check_payload_depth(payload)?; decode_grown(payload).map_err(opaque_decode_error) From 4b55c7d30b063a4166afe7b8e3245b34103c36ff Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 14 Oct 2026 20:57:59 +0200 Subject: [PATCH 11/13] decode_stack tests: store reads and the outer check pinned at release-reachable sizes Legacy and v1 reads on 64 KiB, redb pull_all and compare-and-swap on 104 KiB, and one Largest level on 88 KiB, the sizes where a plain from_slice or a missing outer maybe_grow dies in release as well as debug. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-rs/tests/decode_stack.rs | 36 ++++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/packages/cultcache-rs/tests/decode_stack.rs b/packages/cultcache-rs/tests/decode_stack.rs index c2e61a3d..5e31df49 100644 --- a/packages/cultcache-rs/tests/decode_stack.rs +++ b/packages/cultcache-rs/tests/decode_stack.rs @@ -329,6 +329,18 @@ fn a_wide_record_decodes_on_a_96kib_thread_at_one_level_and_at_the_limit() { }); } +#[test] +fn the_largest_record_decodes_on_an_88kib_thread_at_one_level() { + in_child("the_largest_record_decodes_on_an_88kib_thread_at_one_level", || { + // The door's own floor, not the payload, decides this size: one Largest level, boxed as decode_on + // does, needs 72 KiB in debug and 56 in release with the outer stack check, and 104 in release + // without it. + let bytes = on_stack(64 * KIB * KIB, || rmp_serde::to_vec(&largest_chain(1)).unwrap()); + let decoded = decode_on::(88 * KIB, bytes).unwrap(); + on_stack(64 * KIB * KIB, move || assert!(decoded == largest_chain(1))); + }); +} + fn decode_wide_chain(levels: usize, stack: usize) { let bytes = on_stack(64 * KIB * KIB, move || rmp_serde::to_vec(&wide_chain(levels)).unwrap()); let decoded = decode_on::(stack, bytes).unwrap(); @@ -384,20 +396,20 @@ fn fixstr(text: &str) -> Vec { } #[test] -fn a_legacy_store_with_a_deep_unknown_envelope_key_reads_on_a_256kib_thread() { - in_child("a_legacy_store_with_a_deep_unknown_envelope_key_reads_on_a_256kib_thread", || { +fn a_legacy_store_with_a_deep_unknown_envelope_key_reads_on_a_64kib_thread() { + in_child("a_legacy_store_with_a_deep_unknown_envelope_key_reads_on_a_64kib_thread", || { let directory = tempfile::tempdir().unwrap(); let path = directory.path().join("legacy.cc"); std::fs::write(&path, [vec![0x91], deep_envelope()].concat()).unwrap(); - let entries = on_stack(256 * KIB, move || SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()).unwrap(); + let entries = on_stack(64 * KIB, move || SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()).unwrap(); assert_eq!(entries.len(), 1); assert_eq!(entries[0].key, "k"); }); } #[test] -fn a_v1_store_with_a_deep_fifth_record_slot_is_refused_typed_on_a_256kib_thread() { - in_child("a_v1_store_with_a_deep_fifth_record_slot_is_refused_typed_on_a_256kib_thread", || { +fn a_v1_store_with_a_deep_fifth_record_slot_is_refused_typed_on_a_64kib_thread() { + in_child("a_v1_store_with_a_deep_fifth_record_slot_is_refused_typed_on_a_64kib_thread", || { // [header, [], [[key, schema, storedAt, payload, [[[...]]]]]]: the record's unknown fifth slot is // skipped 1000 containers deep, then the record is refused. let mut bytes = vec![0x93]; @@ -412,7 +424,7 @@ fn a_v1_store_with_a_deep_fifth_record_slot_is_refused_typed_on_a_256kib_thread( let directory = tempfile::tempdir().unwrap(); let path = directory.path().join("v1.cc"); std::fs::write(&path, bytes).unwrap(); - let error = on_stack(256 * KIB, move || SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()).unwrap_err(); + let error = on_stack(64 * KIB, move || SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()).unwrap_err(); let unreadable = error.downcast_ref::().expect("a typed refusal"); assert!(matches!(unreadable.kind, CultCacheStoreUnreadableKind::Undecodable)); }); @@ -432,20 +444,20 @@ fn redb_with_deep_row(path: &std::path::Path) { } #[test] -fn a_redb_store_with_a_deep_unknown_envelope_key_reads_on_a_1mib_thread() { - in_child("a_redb_store_with_a_deep_unknown_envelope_key_reads_on_a_1mib_thread", || { +fn a_redb_store_with_a_deep_unknown_envelope_key_reads_on_a_104kib_thread() { + in_child("a_redb_store_with_a_deep_unknown_envelope_key_reads_on_a_104kib_thread", || { let directory = tempfile::tempdir().unwrap(); let path = directory.path().join("rows.redb"); redb_with_deep_row(&path); - let entries = on_stack(1024 * KIB, move || RedbMessagePackBackingStore::new(path).unwrap().pull_all()).unwrap(); + let entries = on_stack(104 * KIB, move || RedbMessagePackBackingStore::new(path).unwrap().pull_all()).unwrap(); assert_eq!(entries.len(), 1); assert_eq!(entries[0].key, "k"); }); } #[test] -fn a_redb_compare_and_swap_reads_a_deep_unknown_envelope_key_on_a_1mib_thread() { - in_child("a_redb_compare_and_swap_reads_a_deep_unknown_envelope_key_on_a_1mib_thread", || { +fn a_redb_compare_and_swap_reads_a_deep_unknown_envelope_key_on_a_104kib_thread() { + in_child("a_redb_compare_and_swap_reads_a_deep_unknown_envelope_key_on_a_104kib_thread", || { let directory = tempfile::tempdir().unwrap(); let path = directory.path().join("rows.redb"); redb_with_deep_row(&path); @@ -457,7 +469,7 @@ fn a_redb_compare_and_swap_reads_a_deep_unknown_envelope_key_on_a_1mib_thread() schema_id: None, }; let replacement = CultCacheEnvelope { payload: vec![2], ..expected.clone() }; - let swapped = on_stack(1024 * KIB, move || { + let swapped = on_stack(104 * KIB, move || { RedbMessagePackBackingStore::new(path).unwrap().compare_and_swap_entry(&expected, replacement) }) .unwrap(); From 10a39dac983b5342290d3e3f981e0061295f7ade Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 14 Oct 2026 21:19:13 +0200 Subject: [PATCH 12/13] cultcache-rs docs: debug floor stated for a plain caller; first-frame sentences removed; Windows line names its run Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-rs/README.md | 10 ++++++---- packages/cultcache-rs/src/lib.rs | 15 ++++++++------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/packages/cultcache-rs/README.md b/packages/cultcache-rs/README.md index 2d1fa671..7bdbd99b 100644 --- a/packages/cultcache-rs/README.md +++ b/packages/cultcache-rs/README.md @@ -459,7 +459,7 @@ manual registration ceremony, not to loosen the closed-world boundary. Payload nesting is bounded at 256 MessagePack containers, checked before any recursion; past the bound the refusal is a typed `PayloadTooDeep`. Typed decode of stored bytes then grows its stack: the decoder checks the remaining stack -before its first frame and `serde_stacker` checks it at every container, moving +at entry and `serde_stacker` checks it at every container, moving onto a fresh segment whenever less than 512 KiB remains, so a payload inside the bound decodes on a small thread instead of aborting the process. Envelope and store reads (single-file, redb, v1 snapshots) use the same grown decoder and @@ -471,9 +471,11 @@ record may have, and the derives refuse a larger one at compile time: put its large data behind a `Vec` or a `Box`. A field of a type the derives did not produce that is larger than that, behind a `Box` or `Vec`, is outside the guarantee. The decoder's own frames have a fixed floor that no payload changes: at the bound a thread decodes a record of -this size from about 56 KiB in a release build and 104 KiB in a debug build, against 16 and 88 KiB for a function -of the same signature that decodes nothing, plus the caller's own frames. Stack growth was verified on -x86_64-pc-windows-msvc once, by hand, on Starfire (`decode_stack`, debug and release); later `stacker` bumps repeat it. +this size from about 56 KiB in a release build and 72 KiB in a debug build for a caller that only tests the result +(`is_ok`), plus the caller's own frames. The figure depends on how the caller holds the result: one that moves the +record out holds a second copy and needs more (about 104 KiB in debug). Stack growth was verified on +x86_64-pc-windows-msvc by hand (`decode_stack`, WINREV, 13 tests, debug and release); a `stacker` bump or a change +to the stack-size tests repeats it. The crate does not build for wasm32 (`fs2`), and no wasm behaviour is claimed. diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs index 4714ab79..3f895fb4 100644 --- a/packages/cultcache-rs/src/lib.rs +++ b/packages/cultcache-rs/src/lib.rs @@ -687,9 +687,8 @@ const DECODE_STACK_SEGMENT: usize = 2 * 1024 * 1024; /// ``` pub const MAX_RECORD_INLINE_BYTES: usize = 16 * 1024; -/// The one grown decoder of stored bytes: the stack check comes first, before any frame that holds -/// a `T`, then `serde_stacker` re-checks at every container serde's impls descend through, skipped -/// values included. No nesting bound is applied here; [`decode_payload`] walks first. +/// The one grown decoder of stored bytes: the outer stack check, then `serde_stacker` re-checks at +/// every container serde's impls descend through, skipped values included. No nesting bound is applied here; [`decode_payload`] walks first. fn decode_grown(bytes: &[u8]) -> std::result::Result { stacker::maybe_grow(DECODE_RED_ZONE, DECODE_STACK_SEGMENT, || { let mut reader = rmp_serde::Deserializer::from_read_ref(bytes); @@ -705,10 +704,12 @@ fn decode_grown(bytes: &[u8]) -> std::result::Re /// bytes. Readers of stored envelope payloads outside this crate decode through it too. /// /// A payload inside the bound decodes on any thread that has the decoder's fixed floor, which no payload -/// changes: at the bound, about 56 KiB in release and 104 KiB in debug, against 16 and 88 KiB for a -/// same-signature function that decodes nothing, plus the caller's own frames. That holds for every -/// record the derives produce, which [`MAX_RECORD_INLINE_BYTES`] bounds. Stack growth was verified on -/// Windows once, by hand; a `stacker` bump repeats it. +/// changes: at the bound, about 56 KiB in release and 72 KiB in debug for a caller that only tests the +/// result (`is_ok`), plus the caller's own frames. The figure depends on how the caller holds the result: +/// one that moves the record out holds a second copy and needs more (about 104 KiB in debug). That holds +/// for every record the derives produce, which [`MAX_RECORD_INLINE_BYTES`] bounds. Stack growth was +/// verified on x86_64-pc-windows-msvc by hand (`decode_stack`, WINREV, 13 tests, debug and release); a +/// `stacker` bump or a change to the stack-size tests repeats it. pub fn decode_payload(payload: &[u8]) -> Result { check_payload_depth(payload)?; decode_grown(payload).map_err(opaque_decode_error) From 9d2b64bd8840d66e26a953a00ff3fffffd2120cf Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 14 Oct 2026 21:19:19 +0200 Subject: [PATCH 13/13] cultcache-rs docs: Windows line names the revision of the run it describes The decode_stack Windows run is at 10a39dac (tests unchanged since 4b55c7d3); the head after this commit awaits a Soul rerun. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-rs/README.md | 2 +- packages/cultcache-rs/src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cultcache-rs/README.md b/packages/cultcache-rs/README.md index 7bdbd99b..afae7285 100644 --- a/packages/cultcache-rs/README.md +++ b/packages/cultcache-rs/README.md @@ -474,7 +474,7 @@ guarantee. The decoder's own frames have a fixed floor that no payload changes: this size from about 56 KiB in a release build and 72 KiB in a debug build for a caller that only tests the result (`is_ok`), plus the caller's own frames. The figure depends on how the caller holds the result: one that moves the record out holds a second copy and needs more (about 104 KiB in debug). Stack growth was verified on -x86_64-pc-windows-msvc by hand (`decode_stack`, WINREV, 13 tests, debug and release); a `stacker` bump or a change +x86_64-pc-windows-msvc by hand (`decode_stack`, revision 10a39dac, 13 tests, debug and release); a `stacker` bump or a change to the stack-size tests repeats it. The crate does not build for wasm32 (`fs2`), and no wasm behaviour is claimed. diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs index 3f895fb4..d58f1242 100644 --- a/packages/cultcache-rs/src/lib.rs +++ b/packages/cultcache-rs/src/lib.rs @@ -708,7 +708,7 @@ fn decode_grown(bytes: &[u8]) -> std::result::Re /// result (`is_ok`), plus the caller's own frames. The figure depends on how the caller holds the result: /// one that moves the record out holds a second copy and needs more (about 104 KiB in debug). That holds /// for every record the derives produce, which [`MAX_RECORD_INLINE_BYTES`] bounds. Stack growth was -/// verified on x86_64-pc-windows-msvc by hand (`decode_stack`, WINREV, 13 tests, debug and release); a +/// verified on x86_64-pc-windows-msvc by hand (`decode_stack`, revision 10a39dac, 13 tests, debug and release); a /// `stacker` bump or a change to the stack-size tests repeats it. pub fn decode_payload(payload: &[u8]) -> Result { check_payload_depth(payload)?;