diff --git a/packages/cultcache-rs/Cargo.toml b/packages/cultcache-rs/Cargo.toml index 23d53801..9610aeb6 100644 --- a/packages/cultcache-rs/Cargo.toml +++ b/packages/cultcache-rs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cultcache-rs" -version = "0.2.0" +version = "0.3.0" edition = "2024" license = "MIT" publish = false @@ -22,6 +22,8 @@ redb = "4.1.0" serde = { version = "1", features = ["derive"] } serde_bytes = "0.11" serde_json = "1" +serde_stacker = "0.1.14" +stacker = "0.1.25" uuid = { version = "1", features = ["v4"] } [dev-dependencies] diff --git a/packages/cultcache-rs/README.md b/packages/cultcache-rs/README.md index 7191eeed..afae7285 100644 --- a/packages/cultcache-rs/README.md +++ b/packages/cultcache-rs/README.md @@ -456,6 +456,39 @@ That is the Rust version of the original trick: The derive/registry work should preserve that shape. Its job is to remove manual registration ceremony, not to loosen the closed-world boundary. +Payload nesting is bounded at 256 MessagePack containers, checked before any +recursion; past the bound the refusal is a typed `PayloadTooDeep`. Typed decode +of stored bytes then grows its stack: the decoder checks the remaining stack +at entry and `serde_stacker` checks it at every container, moving +onto a fresh segment whenever less than 512 KiB remains, so a payload inside the +bound decodes on a small thread instead of aborting the process. Envelope and +store reads (single-file, redb, v1 snapshots) use the same grown decoder and +keep accepting and refusing what they did, without the payload walk. + +The guarantee holds for records the derives produce. `MAX_RECORD_INLINE_BYTES` +(16 KiB) is the largest `size_of` a `DatabaseEntry`, `CultSlots` or `CultUnion` +record may have, and the derives refuse a larger one at compile time: put its +large data behind a `Vec` or a `Box`. A field of a type the derives did not +produce that is larger than that, behind a `Box` or `Vec`, is outside the +guarantee. The decoder's own frames have a fixed floor that no payload changes: at the bound a thread decodes a record of +this size from about 56 KiB in a release build and 72 KiB in a debug build for a caller that only tests the result +(`is_ok`), plus the caller's own frames. The figure depends on how the caller holds the result: one that moves the +record out holds a second copy and needs more (about 104 KiB in debug). Stack growth was verified on +x86_64-pc-windows-msvc by hand (`decode_stack`, revision 10a39dac, 13 tests, debug and release); a `stacker` bump or a change +to the stack-size tests repeats it. +The crate does not +build for wasm32 (`fs2`), and no wasm behaviour is claimed. + +### Build dependencies + +Stack growth adds these crates to every consumer's build, and a C toolchain +(`psm`, `stacker`). On all targets: `serde_stacker` 0.1.14, `stacker` 0.1.25, +`psm` 0.1.32, `cc` 1.8.0, `find-msvc-tools` 0.1.14, `shlex` 2.0.1 (MIT OR +Apache-2.0), `object` 0.39.1 (Apache-2.0 OR MIT) and `ar_archive_writer` 0.5.3 +(Apache-2.0 WITH LLVM-exception, build-only). On `x86_64-pc-windows-msvc` also +`libc` 0.2.190 and `windows-sys` 0.61.2 (MIT OR Apache-2.0), beside the 0.59 the +crate uses itself. + ## License Private GameCult infrastructure for now. Public packaging can wait until the diff --git a/packages/cultcache-rs/cultcache-rs-derive/Cargo.toml b/packages/cultcache-rs/cultcache-rs-derive/Cargo.toml index 74d52a4f..de299ff7 100644 --- a/packages/cultcache-rs/cultcache-rs-derive/Cargo.toml +++ b/packages/cultcache-rs/cultcache-rs-derive/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cultcache-rs-derive" -version = "0.1.0" +version = "0.2.0" edition = "2024" publish = false diff --git a/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs b/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs index d172e4fd..65e41b92 100644 --- a/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs +++ b/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs @@ -275,6 +275,11 @@ fn slot_codec(ident: &syn::Ident, slots: &[CultCacheSlot]) -> proc_macro2::Token }); quote! { + const _: () = ::core::assert!( + ::core::mem::size_of::<#ident>() <= ::cultcache_rs::MAX_RECORD_INLINE_BYTES, + "a CultCache record is at most cultcache_rs::MAX_RECORD_INLINE_BYTES bytes inline; box its largest slot" + ); + impl ::cultcache_rs::SlotRecord for #ident { fn __slots_array() {} } @@ -509,6 +514,11 @@ fn union_codec(input: DeriveInput) -> syn::Result { }); Ok(quote! { + const _: () = ::core::assert!( + ::core::mem::size_of::<#ident>() <= ::cultcache_rs::MAX_RECORD_INLINE_BYTES, + "a CultCache record is at most cultcache_rs::MAX_RECORD_INLINE_BYTES bytes inline; box its largest slot" + ); + // An arm is a slots array: a newtype over anything else is refused here, by the // trait the CultSlots and DatabaseEntry derives implement. const _: fn() = || { diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs index 5332e8a4..d58f1242 100644 --- a/packages/cultcache-rs/src/lib.rs +++ b/packages/cultcache-rs/src/lib.rs @@ -595,12 +595,124 @@ fn element_header(bytes: &[u8], position: usize) -> Option<(usize, Option }) } -/// The one decode of a document payload: the nesting bound first, then the deserializer, whose own -/// depth counter (1024) is only a backstop. A failure keeps its kind and never quotes the bytes. -/// Readers of stored envelope payloads outside this crate decode through it too. +/// The stack left below which a decode moves to a fresh segment. Measured (map D10, D12): the +/// distance between two stack checks is 14 times the inline size of the record entered plus 3 KiB in +/// a debug build and 6 times in release, so at [`MAX_RECORD_INLINE_BYTES`] it is 220 KiB and 93 KiB; +/// 512 KiB is 2.3 times the larger. A record the derives produced can never exceed that bound. +const DECODE_RED_ZONE: usize = 512 * 1024; + +/// The size of each segment a decode grows onto, four red zones. It does not change the cost of a +/// decode (map D6); it only has to exceed the red zone plus one gap. +const DECODE_STACK_SEGMENT: usize = 2 * 1024 * 1024; + +/// The largest `size_of` a record the derives produce may have. `DatabaseEntry`, `CultSlots` and +/// `CultUnion` refuse a larger one at compile time: put its large data behind a `Vec` or a `Box`. +/// +/// Decoding builds a record in stack frames that hold it, and the distance between two stack checks +/// grows with its size, so the decode's red zone is set against a record of exactly this size. A +/// field of a type the derives did not produce (a plain serde struct, a large tuple) that exceeds +/// this size behind a `Box` or `Vec` is outside the guarantee. +/// +/// ```compile_fail,E0080 +/// use cultcache_rs::CultSlots; +/// +/// #[derive(CultSlots)] +/// struct TooLarge { +/// #[cultcache(key = 0)] +/// a: [[u64; 32]; 32], +/// #[cultcache(key = 1)] +/// b: [[u64; 32]; 32], +/// #[cultcache(key = 2)] +/// one_more: u8, +/// } +/// ``` +/// +/// A union is refused by its own size, though every arm is within the limit: +/// +/// ```compile_fail,E0080 +/// use cultcache_rs::{CultSlots, CultUnion}; +/// +/// #[derive(CultSlots)] +/// struct Arm { +/// #[cultcache(key = 0)] +/// a: [[u64; 32]; 32], +/// #[cultcache(key = 1)] +/// b: [[u64; 32]; 32], +/// } +/// +/// #[derive(CultUnion)] +/// enum TooLarge { +/// #[cultcache(key = 0)] +/// Arm(Arm), +/// #[cultcache(key = 1)] +/// Leaf, +/// } +/// ``` +/// +/// A record of exactly the limit, a union over a record of exactly the limit's half, and a record +/// under it with a `Vec` for its large data compile: +/// +/// ``` +/// use cultcache_rs::{CultSlots, CultUnion}; +/// +/// #[derive(CultSlots)] +/// struct AtTheLimit { +/// #[cultcache(key = 0)] +/// a: [[u64; 32]; 32], +/// #[cultcache(key = 1)] +/// b: [[u64; 32]; 32], +/// } +/// +/// #[derive(CultSlots)] +/// struct Half { +/// #[cultcache(key = 0)] +/// a: [[u64; 32]; 32], +/// } +/// +/// #[derive(CultUnion)] +/// enum Fits { +/// #[cultcache(key = 0)] +/// Half(Half), +/// #[cultcache(key = 1)] +/// Leaf, +/// } +/// +/// #[derive(CultSlots)] +/// struct Boxed { +/// #[cultcache(key = 0)] +/// large: Vec, +/// #[cultcache(key = 1)] +/// header: [[u64; 32]; 31], +/// } +/// ``` +pub const MAX_RECORD_INLINE_BYTES: usize = 16 * 1024; + +/// The one grown decoder of stored bytes: the outer stack check, then `serde_stacker` re-checks at +/// every container serde's impls descend through, skipped values included. No nesting bound is applied here; [`decode_payload`] walks first. +fn decode_grown(bytes: &[u8]) -> std::result::Result { + stacker::maybe_grow(DECODE_RED_ZONE, DECODE_STACK_SEGMENT, || { + let mut reader = rmp_serde::Deserializer::from_read_ref(bytes); + let mut grown = serde_stacker::Deserializer::new(&mut reader); + grown.red_zone = DECODE_RED_ZONE; + grown.stack_size = DECODE_STACK_SEGMENT; + T::deserialize(grown) + }) +} + +/// The one decode of a document payload: the nesting bound first, then [`decode_grown`], whose +/// rmp-serde depth counter (1024) is only a backstop. A failure keeps its kind and never quotes the +/// bytes. Readers of stored envelope payloads outside this crate decode through it too. +/// +/// A payload inside the bound decodes on any thread that has the decoder's fixed floor, which no payload +/// changes: at the bound, about 56 KiB in release and 72 KiB in debug for a caller that only tests the +/// result (`is_ok`), plus the caller's own frames. The figure depends on how the caller holds the result: +/// one that moves the record out holds a second copy and needs more (about 104 KiB in debug). That holds +/// for every record the derives produce, which [`MAX_RECORD_INLINE_BYTES`] bounds. Stack growth was +/// verified on x86_64-pc-windows-msvc by hand (`decode_stack`, revision 10a39dac, 13 tests, debug and release); a +/// `stacker` bump or a change to the stack-size tests repeats it. pub fn decode_payload(payload: &[u8]) -> Result { check_payload_depth(payload)?; - rmp_serde::from_slice(payload).map_err(opaque_decode_error) + decode_grown(payload).map_err(opaque_decode_error) } /// The nesting bound for an envelope admitted without a typed decode, from the same walk that @@ -1140,7 +1252,7 @@ impl SingleFileMessagePackBackingStore { CultCacheStoreUnreadableKind::UnsupportedFormat, anyhow!("its header is {}; this runtime reads {STORE_FORMAT_V1:?} only", describe_header(&header)), )), - _ => rmp_serde::from_slice(&bytes) + _ => decode_grown(&bytes) .context("failed to decode legacy CultCache envelope array") .map_err(|error| unreadable(CultCacheStoreUnreadableKind::Undecodable, error)), } @@ -2293,7 +2405,7 @@ fn read_redb_entry( table .get(key.as_slice())? .map(|value| { - rmp_serde::from_slice(value.value()) + decode_grown(value.value()) .map_err(opaque_decode_error) .context("failed to decode redb CultCache envelope") }) @@ -2317,7 +2429,7 @@ fn read_all_redb( .iter()? .map(|row| { let (_, value) = row?; - rmp_serde::from_slice(value.value()) + decode_grown(value.value()) .map_err(opaque_decode_error) .context("failed to decode redb CultCache envelope") }) @@ -3007,7 +3119,7 @@ fn encode_store_snapshot(entries: &[CultCacheEnvelope]) -> Result Result> { let snapshot: PersistedStoreSnapshot = - rmp_serde::from_slice(bytes).context("failed to decode CultCache v1 snapshot")?; + decode_grown(bytes).context("failed to decode CultCache v1 snapshot")?; let catalog = snapshot .1 diff --git a/packages/cultcache-rs/tests/decode_stack.rs b/packages/cultcache-rs/tests/decode_stack.rs new file mode 100644 index 00000000..5e31df49 --- /dev/null +++ b/packages/cultcache-rs/tests/decode_stack.rs @@ -0,0 +1,478 @@ +//! Typed decode never aborts a payload inside the nesting bound, whatever the caller's stack, for +//! every record the derives accept (up to `MAX_RECORD_INLINE_BYTES` inline); envelope reads of a +//! store go through the same grown decoder. +//! Each test runs its decode in a child test process on a thread of a stated size, so an abort +//! fails one named test. Values are built, encoded, compared and dropped on a large thread; only +//! the decode runs on the small one. + +use cultcache_rs::CacheBackingStore; +use cultcache_rs::CultCacheEnvelope; +use cultcache_rs::CultCacheStoreUnreadable; +use cultcache_rs::CultCacheStoreUnreadableKind; +use cultcache_rs::CultSlots; +use cultcache_rs::CultUnion; +use cultcache_rs::MAX_PAYLOAD_DEPTH; +use cultcache_rs::MAX_RECORD_INLINE_BYTES; +use cultcache_rs::PayloadTooDeep; +use cultcache_rs::RedbMessagePackBackingStore; +use cultcache_rs::SingleFileMessagePackBackingStore; +use cultcache_rs::check_payload_depth; +use cultcache_rs::decode_payload; +use serde::de::DeserializeOwned; + +const CHILD_MARKER: &str = "CULTCACHE_DECODE_STACK_CHILD"; +const KIB: usize = 1024; + +#[derive(Clone, Debug, PartialEq, CultSlots)] +struct Wide32 { + #[cultcache(key = 0)] + a0: [u64; 32], + #[cultcache(key = 1)] + a1: [u64; 32], + #[cultcache(key = 2)] + a2: [u64; 32], + #[cultcache(key = 3)] + a3: [u64; 32], + #[cultcache(key = 4)] + a4: [u64; 32], + #[cultcache(key = 5)] + a5: [u64; 32], + #[cultcache(key = 6)] + a6: [u64; 32], + #[cultcache(key = 7)] + a7: [u64; 32], + #[cultcache(key = 8)] + a8: [u64; 32], + #[cultcache(key = 9)] + a9: [u64; 32], + #[cultcache(key = 10)] + a10: [u64; 32], + #[cultcache(key = 11)] + a11: [u64; 32], + #[cultcache(key = 12)] + a12: [u64; 32], + #[cultcache(key = 13)] + a13: [u64; 32], + #[cultcache(key = 14)] + a14: [u64; 32], + #[cultcache(key = 15)] + a15: [u64; 32], + #[cultcache(key = 16)] + a16: [u64; 32], + #[cultcache(key = 17)] + a17: [u64; 32], + #[cultcache(key = 18)] + a18: [u64; 32], + #[cultcache(key = 19)] + a19: [u64; 32], + #[cultcache(key = 20)] + a20: [u64; 32], + #[cultcache(key = 21)] + a21: [u64; 32], + #[cultcache(key = 22)] + a22: [u64; 32], + #[cultcache(key = 23)] + a23: [u64; 32], + #[cultcache(key = 24)] + a24: [u64; 32], + #[cultcache(key = 25)] + a25: [u64; 32], + #[cultcache(key = 26)] + a26: [u64; 32], + #[cultcache(key = 27)] + a27: [u64; 32], + #[cultcache(key = 28)] + a28: [u64; 32], + #[cultcache(key = 29)] + a29: [u64; 32], + #[cultcache(key = 30)] + a30: [u64; 32], + #[cultcache(key = 31)] + a31: [u64; 32], + #[cultcache(key = 32)] + next: Option>, +} + +#[derive(Clone, Debug, PartialEq, CultSlots)] +struct One { + #[cultcache(key = 0)] + a: u8, +} + +#[derive(Clone, Debug, PartialEq, CultSlots)] +struct DynamicSlot { + #[cultcache(key = 0)] + v: serde_json::Value, +} + +/// Runs `body` on a thread with a stack of `bytes`, returning what it returns. +fn on_stack(bytes: usize, body: impl FnOnce() -> R + Send + 'static) -> R { + std::thread::Builder::new().stack_size(bytes).spawn(body).unwrap().join().unwrap() +} + +/// Runs `body` in a child process that runs only the test `name`; the parent passes when the +/// child exits cleanly having run exactly that one test. An abort in the child fails the parent. +fn in_child(name: &str, body: impl FnOnce()) { + if std::env::var_os(CHILD_MARKER).is_some() { + body(); + return; + } + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([name, "--exact", "--nocapture", "--test-threads=1"]) + .env(CHILD_MARKER, "1") + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success() && stdout.contains("1 passed"), + "child {name} failed: {:?} +{stdout} +{stderr}", + output.status + ); +} + +/// A chain of `levels` Wide32 records: `levels` containers deep, plus one for its last arrays. +fn wide_chain(levels: usize) -> Wide32 { + let mut chain: Option> = None; + for level in (0..levels).rev() { + let mut node = Wide32 { next: chain, ..empty_wide() }; + node.a0[0] = level as u64; + chain = Some(Box::new(node)); + } + *chain.unwrap() +} + +fn empty_wide() -> Wide32 { + Wide32 { + a0: [0; 32], + a1: [0; 32], + a2: [0; 32], + a3: [0; 32], + a4: [0; 32], + a5: [0; 32], + a6: [0; 32], + a7: [0; 32], + a8: [0; 32], + a9: [0; 32], + a10: [0; 32], + a11: [0; 32], + a12: [0; 32], + a13: [0; 32], + a14: [0; 32], + a15: [0; 32], + a16: [0; 32], + a17: [0; 32], + a18: [0; 32], + a19: [0; 32], + a20: [0; 32], + a21: [0; 32], + a22: [0; 32], + a23: [0; 32], + a24: [0; 32], + a25: [0; 32], + a26: [0; 32], + a27: [0; 32], + a28: [0; 32], + a29: [0; 32], + a30: [0; 32], + a31: [0; 32], + next: None, + } +} + +/// The decode alone runs on the small thread. Its result is boxed there so that the thread's own +/// return machinery, which would hold several more copies of a large record in a debug build, does +/// not count against the stack under test; what the caller's frames hold is the caller's to size. +fn decode_on(stack: usize, bytes: Vec) -> anyhow::Result { + *on_stack(stack, move || Box::new(decode_payload::(&bytes))) +} + +#[test] +fn a_wide_record_at_the_limit_decodes_on_a_1mib_thread() { + in_child("a_wide_record_at_the_limit_decodes_on_a_1mib_thread", || { + let bytes = on_stack(64 * KIB * KIB, || { + rmp_serde::to_vec(&wide_chain(MAX_PAYLOAD_DEPTH - 1)).unwrap() + }); + let decoded = decode_on::(1024 * KIB, bytes).unwrap(); + on_stack(64 * KIB * KIB, move || { + assert!(decoded == wide_chain(MAX_PAYLOAD_DEPTH - 1)); + }); + }); +} + +#[test] +fn one_past_the_limit_is_refused_typed_on_a_1mib_thread() { + in_child("one_past_the_limit_is_refused_typed_on_a_1mib_thread", || { + let bytes = on_stack(64 * KIB * KIB, || { + rmp_serde::to_vec(&wide_chain(MAX_PAYLOAD_DEPTH)).unwrap() + }); + let error = decode_on::(1024 * KIB, bytes).unwrap_err(); + assert_eq!( + error.downcast_ref::(), + Some(&PayloadTooDeep { depth: MAX_PAYLOAD_DEPTH + 1, limit: MAX_PAYLOAD_DEPTH }) + ); + }); +} + +#[test] +fn a_foreign_value_at_the_limit_decodes_on_a_64kib_thread() { + in_child("a_foreign_value_at_the_limit_decodes_on_a_64kib_thread", || { + // The slot's array is depth 1, so 255 nested arrays under it reach depth 256. + let value = on_stack(64 * KIB * KIB, || { + let mut value = serde_json::Value::Null; + for _ in 0..MAX_PAYLOAD_DEPTH - 1 { + value = serde_json::Value::Array(vec![value]); + } + value + }); + let expected = DynamicSlot { v: value }; + let bytes = on_stack(64 * KIB * KIB, { + let expected = expected.clone(); + move || rmp_serde::to_vec(&expected).unwrap() + }); + let decoded = decode_on::(64 * KIB, bytes).unwrap(); + on_stack(64 * KIB * KIB, move || { + assert!(decoded == expected); + drop(expected); + }); + }); +} + +#[test] +fn a_skipped_slot_at_the_limit_decodes_on_a_64kib_thread() { + in_child("a_skipped_slot_at_the_limit_decodes_on_a_64kib_thread", || { + // [1, [[...]]]: slot 1 is unknown to One and is skipped; the whole payload is 256 deep. + let mut bytes = vec![0x92u8, 0x01]; + bytes.extend(std::iter::repeat_n(0x91u8, MAX_PAYLOAD_DEPTH - 2)); + bytes.push(0x90); + let decoded = decode_on::(64 * KIB, bytes).unwrap(); + assert_eq!(decoded, One { a: 1 }); + }); +} + +// A record the derives accept at the bound: two slots of 31 arrays of 32 words, 15,880 bytes inline. +#[derive(Clone, Debug, PartialEq, CultSlots)] +struct Largest { + #[cultcache(key = 0)] + a: [[u64; 32]; 31], + #[cultcache(key = 1)] + b: [[u64; 32]; 31], + #[cultcache(key = 2)] + next: Option>, +} + +#[derive(Clone, Debug, PartialEq, CultSlots)] +struct LargestUnionArm { + #[cultcache(key = 0)] + a: [[u64; 32]; 31], + #[cultcache(key = 1)] + b: [[u64; 32]; 31], + #[cultcache(key = 2)] + next: Option>, +} + +// The fixture is the largest arm the derives accept next to a unit variant, which is the point. +#[allow(clippy::large_enum_variant)] +#[derive(Clone, Debug, PartialEq, CultUnion)] +enum LargestUnion { + #[cultcache(key = 0)] + Arm(LargestUnionArm), + #[cultcache(key = 1)] + Leaf, +} + +// The fixtures sit within a KiB under the bound, so a decode that survives them survives any record. +const _: () = assert!( + std::mem::size_of::() <= MAX_RECORD_INLINE_BYTES + && std::mem::size_of::() + KIB >= MAX_RECORD_INLINE_BYTES + && std::mem::size_of::() <= MAX_RECORD_INLINE_BYTES + && std::mem::size_of::() + KIB >= MAX_RECORD_INLINE_BYTES +); + +fn largest_chain(levels: usize) -> Largest { + let mut chain: Option> = None; + for level in (0..levels).rev() { + let mut a = [[0u64; 32]; 31]; + a[0][0] = level as u64; + chain = Some(Box::new(Largest { a, b: [[0; 32]; 31], next: chain })); + } + *chain.unwrap() +} + +fn union_chain(levels: usize) -> LargestUnion { + let mut chain = LargestUnion::Leaf; + for level in (0..levels).rev() { + let mut a = [[0u64; 32]; 31]; + a[0][0] = level as u64; + chain = LargestUnion::Arm(LargestUnionArm { a, b: [[0; 32]; 31], next: Some(Box::new(chain)) }); + } + chain +} + +#[test] +fn a_wide_record_20_levels_deep_decodes_on_a_1mib_thread() { + in_child("a_wide_record_20_levels_deep_decodes_on_a_1mib_thread", || decode_wide_chain(20, 1024 * KIB)); +} + +#[test] +fn a_wide_record_100_levels_deep_decodes_on_a_1mib_thread() { + in_child("a_wide_record_100_levels_deep_decodes_on_a_1mib_thread", || decode_wide_chain(100, 1024 * KIB)); +} + +#[test] +fn a_wide_record_decodes_on_a_96kib_thread_at_one_level_and_at_the_limit() { + in_child("a_wide_record_decodes_on_a_96kib_thread_at_one_level_and_at_the_limit", || { + decode_wide_chain(1, 96 * KIB); + decode_wide_chain(MAX_PAYLOAD_DEPTH - 1, 96 * KIB); + }); +} + +#[test] +fn the_largest_record_decodes_on_an_88kib_thread_at_one_level() { + in_child("the_largest_record_decodes_on_an_88kib_thread_at_one_level", || { + // The door's own floor, not the payload, decides this size: one Largest level, boxed as decode_on + // does, needs 72 KiB in debug and 56 in release with the outer stack check, and 104 in release + // without it. + let bytes = on_stack(64 * KIB * KIB, || rmp_serde::to_vec(&largest_chain(1)).unwrap()); + let decoded = decode_on::(88 * KIB, bytes).unwrap(); + on_stack(64 * KIB * KIB, move || assert!(decoded == largest_chain(1))); + }); +} + +fn decode_wide_chain(levels: usize, stack: usize) { + let bytes = on_stack(64 * KIB * KIB, move || rmp_serde::to_vec(&wide_chain(levels)).unwrap()); + let decoded = decode_on::(stack, bytes).unwrap(); + on_stack(64 * KIB * KIB, move || assert!(decoded == wide_chain(levels))); +} + +#[test] +fn the_largest_record_at_the_limit_decodes_on_every_thread_size() { + in_child("the_largest_record_at_the_limit_decodes_on_every_thread_size", || { + // 254 records of Largest are 256 containers deep (each is one, its slots' arrays two more); + // 127 union levels are the same (a union and its arm are two each, the leaf's arrays two). + let (records, unions) = on_stack(64 * KIB * KIB, || { + let records = rmp_serde::to_vec(&largest_chain(254)).unwrap(); + let unions = rmp_serde::to_vec(&union_chain(127)).unwrap(); + assert!(check_payload_depth(&records).is_ok()); + assert!(check_payload_depth(&unions).is_ok()); + assert!(check_payload_depth(&rmp_serde::to_vec(&largest_chain(255)).unwrap()).is_err()); + assert!(check_payload_depth(&rmp_serde::to_vec(&union_chain(128)).unwrap()).is_err()); + (records, unions) + }); + let sizes = (160..=1024).step_by(32).map(|kib| kib * KIB).chain([1536, 2048, 3072, 4096, 8192].map(|kib| kib * KIB)); + for size in sizes { + println!("thread of {} KiB", size / KIB); + let decoded = decode_on::(size, records.clone()).unwrap(); + let decoded_union = decode_on::(size, unions.clone()).unwrap(); + on_stack(64 * KIB * KIB, move || { + assert!(decoded == largest_chain(254)); + assert!(decoded_union == union_chain(127)); + }); + } + }); +} + +/// One stored envelope, `{key: "k", type: "t", payload: bin[1], storedAt: "s", extra: [[[...]]]}`, +/// whose unknown key nests 1000 containers deep and is skipped without a payload walk. +fn deep_envelope() -> Vec { + let mut bytes = vec![0x85]; + for (name, value) in [("key", &[0xa1, b'k'][..]), ("type", &[0xa1, b't']), ("payload", &[0xc4, 1, 1]), ("storedAt", &[0xa1, b's'])] { + bytes.push(0xa0 | name.len() as u8); + bytes.extend(name.as_bytes()); + bytes.extend(value); + } + bytes.extend([0xa5, b'e', b'x', b't', b'r', b'a']); + bytes.extend(std::iter::repeat_n(0x91u8, 999)); + bytes.push(0x90); + bytes +} + +fn fixstr(text: &str) -> Vec { + let mut bytes = vec![0xa0 | text.len() as u8]; + bytes.extend(text.as_bytes()); + bytes +} + +#[test] +fn a_legacy_store_with_a_deep_unknown_envelope_key_reads_on_a_64kib_thread() { + in_child("a_legacy_store_with_a_deep_unknown_envelope_key_reads_on_a_64kib_thread", || { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("legacy.cc"); + std::fs::write(&path, [vec![0x91], deep_envelope()].concat()).unwrap(); + let entries = on_stack(64 * KIB, move || SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()).unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].key, "k"); + }); +} + +#[test] +fn a_v1_store_with_a_deep_fifth_record_slot_is_refused_typed_on_a_64kib_thread() { + in_child("a_v1_store_with_a_deep_fifth_record_slot_is_refused_typed_on_a_64kib_thread", || { + // [header, [], [[key, schema, storedAt, payload, [[[...]]]]]]: the record's unknown fifth slot is + // skipped 1000 containers deep, then the record is refused. + let mut bytes = vec![0x93]; + bytes.extend(fixstr("cultcache.store.v1")); + bytes.extend([0x90, 0x91, 0x95]); + for text in ["k", "s", "t"] { + bytes.extend(fixstr(text)); + } + bytes.extend([0xc4, 1, 1]); + bytes.extend(std::iter::repeat_n(0x91u8, 999)); + bytes.push(0x90); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("v1.cc"); + std::fs::write(&path, bytes).unwrap(); + let error = on_stack(64 * KIB, move || SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()).unwrap_err(); + let unreadable = error.downcast_ref::().expect("a typed refusal"); + assert!(matches!(unreadable.kind, CultCacheStoreUnreadableKind::Undecodable)); + }); +} + +/// A redb store holding the one deep envelope under its identity, `["t", "k"]`. +fn redb_with_deep_row(path: &std::path::Path) { + let database = redb::Database::create(path).unwrap(); + let write = database.begin_write().unwrap(); + { + let mut table = write + .open_table(redb::TableDefinition::<&[u8], &[u8]>::new("cultcache_envelopes")) + .unwrap(); + table.insert([0x92, 0xa1, b't', 0xa1, b'k'].as_slice(), deep_envelope().as_slice()).unwrap(); + } + write.commit().unwrap(); +} + +#[test] +fn a_redb_store_with_a_deep_unknown_envelope_key_reads_on_a_104kib_thread() { + in_child("a_redb_store_with_a_deep_unknown_envelope_key_reads_on_a_104kib_thread", || { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("rows.redb"); + redb_with_deep_row(&path); + let entries = on_stack(104 * KIB, move || RedbMessagePackBackingStore::new(path).unwrap().pull_all()).unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].key, "k"); + }); +} + +#[test] +fn a_redb_compare_and_swap_reads_a_deep_unknown_envelope_key_on_a_104kib_thread() { + in_child("a_redb_compare_and_swap_reads_a_deep_unknown_envelope_key_on_a_104kib_thread", || { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("rows.redb"); + redb_with_deep_row(&path); + let expected = CultCacheEnvelope { + key: "k".into(), + r#type: "t".into(), + payload: vec![1], + stored_at: "s".into(), + schema_id: None, + }; + let replacement = CultCacheEnvelope { payload: vec![2], ..expected.clone() }; + let swapped = on_stack(104 * KIB, move || { + RedbMessagePackBackingStore::new(path).unwrap().compare_and_swap_entry(&expected, replacement) + }) + .unwrap(); + assert!(swapped); + }); +} diff --git a/packages/cultcache-rs/tests/union_parity.rs b/packages/cultcache-rs/tests/union_parity.rs index da2b1a08..a566e627 100644 --- a/packages/cultcache-rs/tests/union_parity.rs +++ b/packages/cultcache-rs/tests/union_parity.rs @@ -781,12 +781,12 @@ fn the_raw_doors_hold_the_bound_for_arms_maps_and_arrays() { } } -// A debug build on a 2 MiB thread carries 405 containers of this document; the bound keeps every -// accepted payload below that, and every refusal is an error, not an abort. +// The bound counts containers, not stack: decode grows its own stack, so the document at the limit +// decodes on a 1 MiB thread in both profiles, and every refusal is an error, not an abort. #[test] -fn stack_budget_debug_2mib() { +fn the_union_chain_at_the_limit_decodes_on_a_1mib_thread() { std::thread::Builder::new() - .stack_size(2 * 1024 * 1024) + .stack_size(1024 * 1024) .spawn(|| { assert_eq!( through_cache(&vector("depth-at-limit.msgpack")).unwrap(),