Merge hands/decode-stack-safety
This commit is contained in:
Meta Vens
commit
bbb98eaf66
7 changed files with 649 additions and 14 deletions
|
|
@ -1,6 +1,6 @@
|
|||
[package]
|
||||
name = "cultcache-rs"
|
||||
version = "0.2.0"
|
||||
version = "0.3.0"
|
||||
edition = "2024"
|
||||
license = "MIT"
|
||||
publish = false
|
||||
|
|
@ -22,6 +22,8 @@ redb = "4.1.0"
|
|||
serde = { version = "1", features = ["derive"] }
|
||||
serde_bytes = "0.11"
|
||||
serde_json = "1"
|
||||
serde_stacker = "0.1.14"
|
||||
stacker = "0.1.25"
|
||||
uuid = { version = "1", features = ["v4"] }
|
||||
|
||||
[dev-dependencies]
|
||||
|
|
|
|||
|
|
@ -456,6 +456,39 @@ That is the Rust version of the original trick:
|
|||
The derive/registry work should preserve that shape. Its job is to remove
|
||||
manual registration ceremony, not to loosen the closed-world boundary.
|
||||
|
||||
Payload nesting is bounded at 256 MessagePack containers, checked before any
|
||||
recursion; past the bound the refusal is a typed `PayloadTooDeep`. Typed decode
|
||||
of stored bytes then grows its stack: the decoder checks the remaining stack
|
||||
at entry and `serde_stacker` checks it at every container, moving
|
||||
onto a fresh segment whenever less than 512 KiB remains, so a payload inside the
|
||||
bound decodes on a small thread instead of aborting the process. Envelope and
|
||||
store reads (single-file, redb, v1 snapshots) use the same grown decoder and
|
||||
keep accepting and refusing what they did, without the payload walk.
|
||||
|
||||
The guarantee holds for records the derives produce. `MAX_RECORD_INLINE_BYTES`
|
||||
(16 KiB) is the largest `size_of` a `DatabaseEntry`, `CultSlots` or `CultUnion`
|
||||
record may have, and the derives refuse a larger one at compile time: put its
|
||||
large data behind a `Vec` or a `Box`. A field of a type the derives did not
|
||||
produce that is larger than that, behind a `Box` or `Vec`, is outside the
|
||||
guarantee. The decoder's own frames have a fixed floor that no payload changes: at the bound a thread decodes a record of
|
||||
this size from about 56 KiB in a release build and 72 KiB in a debug build for a caller that only tests the result
|
||||
(`is_ok`), plus the caller's own frames. The figure depends on how the caller holds the result: one that moves the
|
||||
record out holds a second copy and needs more (about 104 KiB in debug). Stack growth was verified on
|
||||
x86_64-pc-windows-msvc by hand (`decode_stack`, revision 10a39dac, 13 tests, debug and release); a `stacker` bump or a change
|
||||
to the stack-size tests repeats it.
|
||||
The crate does not
|
||||
build for wasm32 (`fs2`), and no wasm behaviour is claimed.
|
||||
|
||||
### Build dependencies
|
||||
|
||||
Stack growth adds these crates to every consumer's build, and a C toolchain
|
||||
(`psm`, `stacker`). On all targets: `serde_stacker` 0.1.14, `stacker` 0.1.25,
|
||||
`psm` 0.1.32, `cc` 1.8.0, `find-msvc-tools` 0.1.14, `shlex` 2.0.1 (MIT OR
|
||||
Apache-2.0), `object` 0.39.1 (Apache-2.0 OR MIT) and `ar_archive_writer` 0.5.3
|
||||
(Apache-2.0 WITH LLVM-exception, build-only). On `x86_64-pc-windows-msvc` also
|
||||
`libc` 0.2.190 and `windows-sys` 0.61.2 (MIT OR Apache-2.0), beside the 0.59 the
|
||||
crate uses itself.
|
||||
|
||||
## License
|
||||
|
||||
Private GameCult infrastructure for now. Public packaging can wait until the
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[package]
|
||||
name = "cultcache-rs-derive"
|
||||
version = "0.1.0"
|
||||
version = "0.2.0"
|
||||
edition = "2024"
|
||||
publish = false
|
||||
|
||||
|
|
|
|||
|
|
@ -275,6 +275,11 @@ fn slot_codec(ident: &syn::Ident, slots: &[CultCacheSlot]) -> proc_macro2::Token
|
|||
});
|
||||
|
||||
quote! {
|
||||
const _: () = ::core::assert!(
|
||||
::core::mem::size_of::<#ident>() <= ::cultcache_rs::MAX_RECORD_INLINE_BYTES,
|
||||
"a CultCache record is at most cultcache_rs::MAX_RECORD_INLINE_BYTES bytes inline; box its largest slot"
|
||||
);
|
||||
|
||||
impl ::cultcache_rs::SlotRecord for #ident {
|
||||
fn __slots_array() {}
|
||||
}
|
||||
|
|
@ -509,6 +514,11 @@ fn union_codec(input: DeriveInput) -> syn::Result<proc_macro2::TokenStream> {
|
|||
});
|
||||
|
||||
Ok(quote! {
|
||||
const _: () = ::core::assert!(
|
||||
::core::mem::size_of::<#ident>() <= ::cultcache_rs::MAX_RECORD_INLINE_BYTES,
|
||||
"a CultCache record is at most cultcache_rs::MAX_RECORD_INLINE_BYTES bytes inline; box its largest slot"
|
||||
);
|
||||
|
||||
// An arm is a slots array: a newtype over anything else is refused here, by the
|
||||
// trait the CultSlots and DatabaseEntry derives implement.
|
||||
const _: fn() = || {
|
||||
|
|
|
|||
|
|
@ -595,12 +595,124 @@ fn element_header(bytes: &[u8], position: usize) -> Option<(usize, Option<usize>
|
|||
})
|
||||
}
|
||||
|
||||
/// The one decode of a document payload: the nesting bound first, then the deserializer, whose own
|
||||
/// depth counter (1024) is only a backstop. A failure keeps its kind and never quotes the bytes.
|
||||
/// Readers of stored envelope payloads outside this crate decode through it too.
|
||||
/// The stack left below which a decode moves to a fresh segment. Measured (map D10, D12): the
|
||||
/// distance between two stack checks is 14 times the inline size of the record entered plus 3 KiB in
|
||||
/// a debug build and 6 times in release, so at [`MAX_RECORD_INLINE_BYTES`] it is 220 KiB and 93 KiB;
|
||||
/// 512 KiB is 2.3 times the larger. A record the derives produced can never exceed that bound.
|
||||
const DECODE_RED_ZONE: usize = 512 * 1024;
|
||||
|
||||
/// The size of each segment a decode grows onto, four red zones. It does not change the cost of a
|
||||
/// decode (map D6); it only has to exceed the red zone plus one gap.
|
||||
const DECODE_STACK_SEGMENT: usize = 2 * 1024 * 1024;
|
||||
|
||||
/// The largest `size_of` a record the derives produce may have. `DatabaseEntry`, `CultSlots` and
|
||||
/// `CultUnion` refuse a larger one at compile time: put its large data behind a `Vec` or a `Box`.
|
||||
///
|
||||
/// Decoding builds a record in stack frames that hold it, and the distance between two stack checks
|
||||
/// grows with its size, so the decode's red zone is set against a record of exactly this size. A
|
||||
/// field of a type the derives did not produce (a plain serde struct, a large tuple) that exceeds
|
||||
/// this size behind a `Box` or `Vec` is outside the guarantee.
|
||||
///
|
||||
/// ```compile_fail,E0080
|
||||
/// use cultcache_rs::CultSlots;
|
||||
///
|
||||
/// #[derive(CultSlots)]
|
||||
/// struct TooLarge {
|
||||
/// #[cultcache(key = 0)]
|
||||
/// a: [[u64; 32]; 32],
|
||||
/// #[cultcache(key = 1)]
|
||||
/// b: [[u64; 32]; 32],
|
||||
/// #[cultcache(key = 2)]
|
||||
/// one_more: u8,
|
||||
/// }
|
||||
/// ```
|
||||
///
|
||||
/// A union is refused by its own size, though every arm is within the limit:
|
||||
///
|
||||
/// ```compile_fail,E0080
|
||||
/// use cultcache_rs::{CultSlots, CultUnion};
|
||||
///
|
||||
/// #[derive(CultSlots)]
|
||||
/// struct Arm {
|
||||
/// #[cultcache(key = 0)]
|
||||
/// a: [[u64; 32]; 32],
|
||||
/// #[cultcache(key = 1)]
|
||||
/// b: [[u64; 32]; 32],
|
||||
/// }
|
||||
///
|
||||
/// #[derive(CultUnion)]
|
||||
/// enum TooLarge {
|
||||
/// #[cultcache(key = 0)]
|
||||
/// Arm(Arm),
|
||||
/// #[cultcache(key = 1)]
|
||||
/// Leaf,
|
||||
/// }
|
||||
/// ```
|
||||
///
|
||||
/// A record of exactly the limit, a union over a record of exactly the limit's half, and a record
|
||||
/// under it with a `Vec` for its large data compile:
|
||||
///
|
||||
/// ```
|
||||
/// use cultcache_rs::{CultSlots, CultUnion};
|
||||
///
|
||||
/// #[derive(CultSlots)]
|
||||
/// struct AtTheLimit {
|
||||
/// #[cultcache(key = 0)]
|
||||
/// a: [[u64; 32]; 32],
|
||||
/// #[cultcache(key = 1)]
|
||||
/// b: [[u64; 32]; 32],
|
||||
/// }
|
||||
///
|
||||
/// #[derive(CultSlots)]
|
||||
/// struct Half {
|
||||
/// #[cultcache(key = 0)]
|
||||
/// a: [[u64; 32]; 32],
|
||||
/// }
|
||||
///
|
||||
/// #[derive(CultUnion)]
|
||||
/// enum Fits {
|
||||
/// #[cultcache(key = 0)]
|
||||
/// Half(Half),
|
||||
/// #[cultcache(key = 1)]
|
||||
/// Leaf,
|
||||
/// }
|
||||
///
|
||||
/// #[derive(CultSlots)]
|
||||
/// struct Boxed {
|
||||
/// #[cultcache(key = 0)]
|
||||
/// large: Vec<u8>,
|
||||
/// #[cultcache(key = 1)]
|
||||
/// header: [[u64; 32]; 31],
|
||||
/// }
|
||||
/// ```
|
||||
pub const MAX_RECORD_INLINE_BYTES: usize = 16 * 1024;
|
||||
|
||||
/// The one grown decoder of stored bytes: the outer stack check, then `serde_stacker` re-checks at
|
||||
/// every container serde's impls descend through, skipped values included. No nesting bound is applied here; [`decode_payload`] walks first.
|
||||
fn decode_grown<T: serde::de::DeserializeOwned>(bytes: &[u8]) -> std::result::Result<T, rmp_serde::decode::Error> {
|
||||
stacker::maybe_grow(DECODE_RED_ZONE, DECODE_STACK_SEGMENT, || {
|
||||
let mut reader = rmp_serde::Deserializer::from_read_ref(bytes);
|
||||
let mut grown = serde_stacker::Deserializer::new(&mut reader);
|
||||
grown.red_zone = DECODE_RED_ZONE;
|
||||
grown.stack_size = DECODE_STACK_SEGMENT;
|
||||
T::deserialize(grown)
|
||||
})
|
||||
}
|
||||
|
||||
/// The one decode of a document payload: the nesting bound first, then [`decode_grown`], whose
|
||||
/// rmp-serde depth counter (1024) is only a backstop. A failure keeps its kind and never quotes the
|
||||
/// bytes. Readers of stored envelope payloads outside this crate decode through it too.
|
||||
///
|
||||
/// A payload inside the bound decodes on any thread that has the decoder's fixed floor, which no payload
|
||||
/// changes: at the bound, about 56 KiB in release and 72 KiB in debug for a caller that only tests the
|
||||
/// result (`is_ok`), plus the caller's own frames. The figure depends on how the caller holds the result:
|
||||
/// one that moves the record out holds a second copy and needs more (about 104 KiB in debug). That holds
|
||||
/// for every record the derives produce, which [`MAX_RECORD_INLINE_BYTES`] bounds. Stack growth was
|
||||
/// verified on x86_64-pc-windows-msvc by hand (`decode_stack`, revision 10a39dac, 13 tests, debug and release); a
|
||||
/// `stacker` bump or a change to the stack-size tests repeats it.
|
||||
pub fn decode_payload<T: serde::de::DeserializeOwned>(payload: &[u8]) -> Result<T> {
|
||||
check_payload_depth(payload)?;
|
||||
rmp_serde::from_slice(payload).map_err(opaque_decode_error)
|
||||
decode_grown(payload).map_err(opaque_decode_error)
|
||||
}
|
||||
|
||||
/// The nesting bound for an envelope admitted without a typed decode, from the same walk that
|
||||
|
|
@ -1140,7 +1252,7 @@ impl SingleFileMessagePackBackingStore {
|
|||
CultCacheStoreUnreadableKind::UnsupportedFormat,
|
||||
anyhow!("its header is {}; this runtime reads {STORE_FORMAT_V1:?} only", describe_header(&header)),
|
||||
)),
|
||||
_ => rmp_serde::from_slice(&bytes)
|
||||
_ => decode_grown(&bytes)
|
||||
.context("failed to decode legacy CultCache envelope array")
|
||||
.map_err(|error| unreadable(CultCacheStoreUnreadableKind::Undecodable, error)),
|
||||
}
|
||||
|
|
@ -2293,7 +2405,7 @@ fn read_redb_entry(
|
|||
table
|
||||
.get(key.as_slice())?
|
||||
.map(|value| {
|
||||
rmp_serde::from_slice(value.value())
|
||||
decode_grown(value.value())
|
||||
.map_err(opaque_decode_error)
|
||||
.context("failed to decode redb CultCache envelope")
|
||||
})
|
||||
|
|
@ -2317,7 +2429,7 @@ fn read_all_redb(
|
|||
.iter()?
|
||||
.map(|row| {
|
||||
let (_, value) = row?;
|
||||
rmp_serde::from_slice(value.value())
|
||||
decode_grown(value.value())
|
||||
.map_err(opaque_decode_error)
|
||||
.context("failed to decode redb CultCache envelope")
|
||||
})
|
||||
|
|
@ -3007,7 +3119,7 @@ fn encode_store_snapshot(entries: &[CultCacheEnvelope]) -> Result<PersistedStore
|
|||
/// Decodes a store whose header the caller has already read as [`STORE_FORMAT_V1`].
|
||||
fn decode_store_snapshot(bytes: &[u8]) -> Result<Vec<CultCacheEnvelope>> {
|
||||
let snapshot: PersistedStoreSnapshot =
|
||||
rmp_serde::from_slice(bytes).context("failed to decode CultCache v1 snapshot")?;
|
||||
decode_grown(bytes).context("failed to decode CultCache v1 snapshot")?;
|
||||
|
||||
let catalog = snapshot
|
||||
.1
|
||||
|
|
|
|||
478
packages/cultcache-rs/tests/decode_stack.rs
Normal file
478
packages/cultcache-rs/tests/decode_stack.rs
Normal file
|
|
@ -0,0 +1,478 @@
|
|||
//! Typed decode never aborts a payload inside the nesting bound, whatever the caller's stack, for
|
||||
//! every record the derives accept (up to `MAX_RECORD_INLINE_BYTES` inline); envelope reads of a
|
||||
//! store go through the same grown decoder.
|
||||
//! Each test runs its decode in a child test process on a thread of a stated size, so an abort
|
||||
//! fails one named test. Values are built, encoded, compared and dropped on a large thread; only
|
||||
//! the decode runs on the small one.
|
||||
|
||||
use cultcache_rs::CacheBackingStore;
|
||||
use cultcache_rs::CultCacheEnvelope;
|
||||
use cultcache_rs::CultCacheStoreUnreadable;
|
||||
use cultcache_rs::CultCacheStoreUnreadableKind;
|
||||
use cultcache_rs::CultSlots;
|
||||
use cultcache_rs::CultUnion;
|
||||
use cultcache_rs::MAX_PAYLOAD_DEPTH;
|
||||
use cultcache_rs::MAX_RECORD_INLINE_BYTES;
|
||||
use cultcache_rs::PayloadTooDeep;
|
||||
use cultcache_rs::RedbMessagePackBackingStore;
|
||||
use cultcache_rs::SingleFileMessagePackBackingStore;
|
||||
use cultcache_rs::check_payload_depth;
|
||||
use cultcache_rs::decode_payload;
|
||||
use serde::de::DeserializeOwned;
|
||||
|
||||
const CHILD_MARKER: &str = "CULTCACHE_DECODE_STACK_CHILD";
|
||||
const KIB: usize = 1024;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, CultSlots)]
|
||||
struct Wide32 {
|
||||
#[cultcache(key = 0)]
|
||||
a0: [u64; 32],
|
||||
#[cultcache(key = 1)]
|
||||
a1: [u64; 32],
|
||||
#[cultcache(key = 2)]
|
||||
a2: [u64; 32],
|
||||
#[cultcache(key = 3)]
|
||||
a3: [u64; 32],
|
||||
#[cultcache(key = 4)]
|
||||
a4: [u64; 32],
|
||||
#[cultcache(key = 5)]
|
||||
a5: [u64; 32],
|
||||
#[cultcache(key = 6)]
|
||||
a6: [u64; 32],
|
||||
#[cultcache(key = 7)]
|
||||
a7: [u64; 32],
|
||||
#[cultcache(key = 8)]
|
||||
a8: [u64; 32],
|
||||
#[cultcache(key = 9)]
|
||||
a9: [u64; 32],
|
||||
#[cultcache(key = 10)]
|
||||
a10: [u64; 32],
|
||||
#[cultcache(key = 11)]
|
||||
a11: [u64; 32],
|
||||
#[cultcache(key = 12)]
|
||||
a12: [u64; 32],
|
||||
#[cultcache(key = 13)]
|
||||
a13: [u64; 32],
|
||||
#[cultcache(key = 14)]
|
||||
a14: [u64; 32],
|
||||
#[cultcache(key = 15)]
|
||||
a15: [u64; 32],
|
||||
#[cultcache(key = 16)]
|
||||
a16: [u64; 32],
|
||||
#[cultcache(key = 17)]
|
||||
a17: [u64; 32],
|
||||
#[cultcache(key = 18)]
|
||||
a18: [u64; 32],
|
||||
#[cultcache(key = 19)]
|
||||
a19: [u64; 32],
|
||||
#[cultcache(key = 20)]
|
||||
a20: [u64; 32],
|
||||
#[cultcache(key = 21)]
|
||||
a21: [u64; 32],
|
||||
#[cultcache(key = 22)]
|
||||
a22: [u64; 32],
|
||||
#[cultcache(key = 23)]
|
||||
a23: [u64; 32],
|
||||
#[cultcache(key = 24)]
|
||||
a24: [u64; 32],
|
||||
#[cultcache(key = 25)]
|
||||
a25: [u64; 32],
|
||||
#[cultcache(key = 26)]
|
||||
a26: [u64; 32],
|
||||
#[cultcache(key = 27)]
|
||||
a27: [u64; 32],
|
||||
#[cultcache(key = 28)]
|
||||
a28: [u64; 32],
|
||||
#[cultcache(key = 29)]
|
||||
a29: [u64; 32],
|
||||
#[cultcache(key = 30)]
|
||||
a30: [u64; 32],
|
||||
#[cultcache(key = 31)]
|
||||
a31: [u64; 32],
|
||||
#[cultcache(key = 32)]
|
||||
next: Option<Box<Wide32>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, CultSlots)]
|
||||
struct One {
|
||||
#[cultcache(key = 0)]
|
||||
a: u8,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, CultSlots)]
|
||||
struct DynamicSlot {
|
||||
#[cultcache(key = 0)]
|
||||
v: serde_json::Value,
|
||||
}
|
||||
|
||||
/// Runs `body` on a thread with a stack of `bytes`, returning what it returns.
|
||||
fn on_stack<R: Send + 'static>(bytes: usize, body: impl FnOnce() -> R + Send + 'static) -> R {
|
||||
std::thread::Builder::new().stack_size(bytes).spawn(body).unwrap().join().unwrap()
|
||||
}
|
||||
|
||||
/// Runs `body` in a child process that runs only the test `name`; the parent passes when the
|
||||
/// child exits cleanly having run exactly that one test. An abort in the child fails the parent.
|
||||
fn in_child(name: &str, body: impl FnOnce()) {
|
||||
if std::env::var_os(CHILD_MARKER).is_some() {
|
||||
body();
|
||||
return;
|
||||
}
|
||||
let output = std::process::Command::new(std::env::current_exe().unwrap())
|
||||
.args([name, "--exact", "--nocapture", "--test-threads=1"])
|
||||
.env(CHILD_MARKER, "1")
|
||||
.output()
|
||||
.unwrap();
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
assert!(
|
||||
output.status.success() && stdout.contains("1 passed"),
|
||||
"child {name} failed: {:?}
|
||||
{stdout}
|
||||
{stderr}",
|
||||
output.status
|
||||
);
|
||||
}
|
||||
|
||||
/// A chain of `levels` Wide32 records: `levels` containers deep, plus one for its last arrays.
|
||||
fn wide_chain(levels: usize) -> Wide32 {
|
||||
let mut chain: Option<Box<Wide32>> = None;
|
||||
for level in (0..levels).rev() {
|
||||
let mut node = Wide32 { next: chain, ..empty_wide() };
|
||||
node.a0[0] = level as u64;
|
||||
chain = Some(Box::new(node));
|
||||
}
|
||||
*chain.unwrap()
|
||||
}
|
||||
|
||||
fn empty_wide() -> Wide32 {
|
||||
Wide32 {
|
||||
a0: [0; 32],
|
||||
a1: [0; 32],
|
||||
a2: [0; 32],
|
||||
a3: [0; 32],
|
||||
a4: [0; 32],
|
||||
a5: [0; 32],
|
||||
a6: [0; 32],
|
||||
a7: [0; 32],
|
||||
a8: [0; 32],
|
||||
a9: [0; 32],
|
||||
a10: [0; 32],
|
||||
a11: [0; 32],
|
||||
a12: [0; 32],
|
||||
a13: [0; 32],
|
||||
a14: [0; 32],
|
||||
a15: [0; 32],
|
||||
a16: [0; 32],
|
||||
a17: [0; 32],
|
||||
a18: [0; 32],
|
||||
a19: [0; 32],
|
||||
a20: [0; 32],
|
||||
a21: [0; 32],
|
||||
a22: [0; 32],
|
||||
a23: [0; 32],
|
||||
a24: [0; 32],
|
||||
a25: [0; 32],
|
||||
a26: [0; 32],
|
||||
a27: [0; 32],
|
||||
a28: [0; 32],
|
||||
a29: [0; 32],
|
||||
a30: [0; 32],
|
||||
a31: [0; 32],
|
||||
next: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The decode alone runs on the small thread. Its result is boxed there so that the thread's own
|
||||
/// return machinery, which would hold several more copies of a large record in a debug build, does
|
||||
/// not count against the stack under test; what the caller's frames hold is the caller's to size.
|
||||
fn decode_on<T: DeserializeOwned + Send + 'static>(stack: usize, bytes: Vec<u8>) -> anyhow::Result<T> {
|
||||
*on_stack(stack, move || Box::new(decode_payload::<T>(&bytes)))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_wide_record_at_the_limit_decodes_on_a_1mib_thread() {
|
||||
in_child("a_wide_record_at_the_limit_decodes_on_a_1mib_thread", || {
|
||||
let bytes = on_stack(64 * KIB * KIB, || {
|
||||
rmp_serde::to_vec(&wide_chain(MAX_PAYLOAD_DEPTH - 1)).unwrap()
|
||||
});
|
||||
let decoded = decode_on::<Wide32>(1024 * KIB, bytes).unwrap();
|
||||
on_stack(64 * KIB * KIB, move || {
|
||||
assert!(decoded == wide_chain(MAX_PAYLOAD_DEPTH - 1));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_past_the_limit_is_refused_typed_on_a_1mib_thread() {
|
||||
in_child("one_past_the_limit_is_refused_typed_on_a_1mib_thread", || {
|
||||
let bytes = on_stack(64 * KIB * KIB, || {
|
||||
rmp_serde::to_vec(&wide_chain(MAX_PAYLOAD_DEPTH)).unwrap()
|
||||
});
|
||||
let error = decode_on::<Wide32>(1024 * KIB, bytes).unwrap_err();
|
||||
assert_eq!(
|
||||
error.downcast_ref::<PayloadTooDeep>(),
|
||||
Some(&PayloadTooDeep { depth: MAX_PAYLOAD_DEPTH + 1, limit: MAX_PAYLOAD_DEPTH })
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_foreign_value_at_the_limit_decodes_on_a_64kib_thread() {
|
||||
in_child("a_foreign_value_at_the_limit_decodes_on_a_64kib_thread", || {
|
||||
// The slot's array is depth 1, so 255 nested arrays under it reach depth 256.
|
||||
let value = on_stack(64 * KIB * KIB, || {
|
||||
let mut value = serde_json::Value::Null;
|
||||
for _ in 0..MAX_PAYLOAD_DEPTH - 1 {
|
||||
value = serde_json::Value::Array(vec![value]);
|
||||
}
|
||||
value
|
||||
});
|
||||
let expected = DynamicSlot { v: value };
|
||||
let bytes = on_stack(64 * KIB * KIB, {
|
||||
let expected = expected.clone();
|
||||
move || rmp_serde::to_vec(&expected).unwrap()
|
||||
});
|
||||
let decoded = decode_on::<DynamicSlot>(64 * KIB, bytes).unwrap();
|
||||
on_stack(64 * KIB * KIB, move || {
|
||||
assert!(decoded == expected);
|
||||
drop(expected);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_skipped_slot_at_the_limit_decodes_on_a_64kib_thread() {
|
||||
in_child("a_skipped_slot_at_the_limit_decodes_on_a_64kib_thread", || {
|
||||
// [1, [[...]]]: slot 1 is unknown to One and is skipped; the whole payload is 256 deep.
|
||||
let mut bytes = vec![0x92u8, 0x01];
|
||||
bytes.extend(std::iter::repeat_n(0x91u8, MAX_PAYLOAD_DEPTH - 2));
|
||||
bytes.push(0x90);
|
||||
let decoded = decode_on::<One>(64 * KIB, bytes).unwrap();
|
||||
assert_eq!(decoded, One { a: 1 });
|
||||
});
|
||||
}
|
||||
|
||||
// A record the derives accept at the bound: two slots of 31 arrays of 32 words, 15,880 bytes inline.
|
||||
#[derive(Clone, Debug, PartialEq, CultSlots)]
|
||||
struct Largest {
|
||||
#[cultcache(key = 0)]
|
||||
a: [[u64; 32]; 31],
|
||||
#[cultcache(key = 1)]
|
||||
b: [[u64; 32]; 31],
|
||||
#[cultcache(key = 2)]
|
||||
next: Option<Box<Largest>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, CultSlots)]
|
||||
struct LargestUnionArm {
|
||||
#[cultcache(key = 0)]
|
||||
a: [[u64; 32]; 31],
|
||||
#[cultcache(key = 1)]
|
||||
b: [[u64; 32]; 31],
|
||||
#[cultcache(key = 2)]
|
||||
next: Option<Box<LargestUnion>>,
|
||||
}
|
||||
|
||||
// The fixture is the largest arm the derives accept next to a unit variant, which is the point.
|
||||
#[allow(clippy::large_enum_variant)]
|
||||
#[derive(Clone, Debug, PartialEq, CultUnion)]
|
||||
enum LargestUnion {
|
||||
#[cultcache(key = 0)]
|
||||
Arm(LargestUnionArm),
|
||||
#[cultcache(key = 1)]
|
||||
Leaf,
|
||||
}
|
||||
|
||||
// The fixtures sit within a KiB under the bound, so a decode that survives them survives any record.
|
||||
const _: () = assert!(
|
||||
std::mem::size_of::<Largest>() <= MAX_RECORD_INLINE_BYTES
|
||||
&& std::mem::size_of::<Largest>() + KIB >= MAX_RECORD_INLINE_BYTES
|
||||
&& std::mem::size_of::<LargestUnion>() <= MAX_RECORD_INLINE_BYTES
|
||||
&& std::mem::size_of::<LargestUnion>() + KIB >= MAX_RECORD_INLINE_BYTES
|
||||
);
|
||||
|
||||
fn largest_chain(levels: usize) -> Largest {
|
||||
let mut chain: Option<Box<Largest>> = None;
|
||||
for level in (0..levels).rev() {
|
||||
let mut a = [[0u64; 32]; 31];
|
||||
a[0][0] = level as u64;
|
||||
chain = Some(Box::new(Largest { a, b: [[0; 32]; 31], next: chain }));
|
||||
}
|
||||
*chain.unwrap()
|
||||
}
|
||||
|
||||
fn union_chain(levels: usize) -> LargestUnion {
|
||||
let mut chain = LargestUnion::Leaf;
|
||||
for level in (0..levels).rev() {
|
||||
let mut a = [[0u64; 32]; 31];
|
||||
a[0][0] = level as u64;
|
||||
chain = LargestUnion::Arm(LargestUnionArm { a, b: [[0; 32]; 31], next: Some(Box::new(chain)) });
|
||||
}
|
||||
chain
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_wide_record_20_levels_deep_decodes_on_a_1mib_thread() {
|
||||
in_child("a_wide_record_20_levels_deep_decodes_on_a_1mib_thread", || decode_wide_chain(20, 1024 * KIB));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_wide_record_100_levels_deep_decodes_on_a_1mib_thread() {
|
||||
in_child("a_wide_record_100_levels_deep_decodes_on_a_1mib_thread", || decode_wide_chain(100, 1024 * KIB));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_wide_record_decodes_on_a_96kib_thread_at_one_level_and_at_the_limit() {
|
||||
in_child("a_wide_record_decodes_on_a_96kib_thread_at_one_level_and_at_the_limit", || {
|
||||
decode_wide_chain(1, 96 * KIB);
|
||||
decode_wide_chain(MAX_PAYLOAD_DEPTH - 1, 96 * KIB);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_largest_record_decodes_on_an_88kib_thread_at_one_level() {
|
||||
in_child("the_largest_record_decodes_on_an_88kib_thread_at_one_level", || {
|
||||
// The door's own floor, not the payload, decides this size: one Largest level, boxed as decode_on
|
||||
// does, needs 72 KiB in debug and 56 in release with the outer stack check, and 104 in release
|
||||
// without it.
|
||||
let bytes = on_stack(64 * KIB * KIB, || rmp_serde::to_vec(&largest_chain(1)).unwrap());
|
||||
let decoded = decode_on::<Largest>(88 * KIB, bytes).unwrap();
|
||||
on_stack(64 * KIB * KIB, move || assert!(decoded == largest_chain(1)));
|
||||
});
|
||||
}
|
||||
|
||||
fn decode_wide_chain(levels: usize, stack: usize) {
|
||||
let bytes = on_stack(64 * KIB * KIB, move || rmp_serde::to_vec(&wide_chain(levels)).unwrap());
|
||||
let decoded = decode_on::<Wide32>(stack, bytes).unwrap();
|
||||
on_stack(64 * KIB * KIB, move || assert!(decoded == wide_chain(levels)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_largest_record_at_the_limit_decodes_on_every_thread_size() {
|
||||
in_child("the_largest_record_at_the_limit_decodes_on_every_thread_size", || {
|
||||
// 254 records of Largest are 256 containers deep (each is one, its slots' arrays two more);
|
||||
// 127 union levels are the same (a union and its arm are two each, the leaf's arrays two).
|
||||
let (records, unions) = on_stack(64 * KIB * KIB, || {
|
||||
let records = rmp_serde::to_vec(&largest_chain(254)).unwrap();
|
||||
let unions = rmp_serde::to_vec(&union_chain(127)).unwrap();
|
||||
assert!(check_payload_depth(&records).is_ok());
|
||||
assert!(check_payload_depth(&unions).is_ok());
|
||||
assert!(check_payload_depth(&rmp_serde::to_vec(&largest_chain(255)).unwrap()).is_err());
|
||||
assert!(check_payload_depth(&rmp_serde::to_vec(&union_chain(128)).unwrap()).is_err());
|
||||
(records, unions)
|
||||
});
|
||||
let sizes = (160..=1024).step_by(32).map(|kib| kib * KIB).chain([1536, 2048, 3072, 4096, 8192].map(|kib| kib * KIB));
|
||||
for size in sizes {
|
||||
println!("thread of {} KiB", size / KIB);
|
||||
let decoded = decode_on::<Largest>(size, records.clone()).unwrap();
|
||||
let decoded_union = decode_on::<LargestUnion>(size, unions.clone()).unwrap();
|
||||
on_stack(64 * KIB * KIB, move || {
|
||||
assert!(decoded == largest_chain(254));
|
||||
assert!(decoded_union == union_chain(127));
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// One stored envelope, `{key: "k", type: "t", payload: bin[1], storedAt: "s", extra: [[[...]]]}`,
|
||||
/// whose unknown key nests 1000 containers deep and is skipped without a payload walk.
|
||||
fn deep_envelope() -> Vec<u8> {
|
||||
let mut bytes = vec![0x85];
|
||||
for (name, value) in [("key", &[0xa1, b'k'][..]), ("type", &[0xa1, b't']), ("payload", &[0xc4, 1, 1]), ("storedAt", &[0xa1, b's'])] {
|
||||
bytes.push(0xa0 | name.len() as u8);
|
||||
bytes.extend(name.as_bytes());
|
||||
bytes.extend(value);
|
||||
}
|
||||
bytes.extend([0xa5, b'e', b'x', b't', b'r', b'a']);
|
||||
bytes.extend(std::iter::repeat_n(0x91u8, 999));
|
||||
bytes.push(0x90);
|
||||
bytes
|
||||
}
|
||||
|
||||
fn fixstr(text: &str) -> Vec<u8> {
|
||||
let mut bytes = vec![0xa0 | text.len() as u8];
|
||||
bytes.extend(text.as_bytes());
|
||||
bytes
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_legacy_store_with_a_deep_unknown_envelope_key_reads_on_a_64kib_thread() {
|
||||
in_child("a_legacy_store_with_a_deep_unknown_envelope_key_reads_on_a_64kib_thread", || {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let path = directory.path().join("legacy.cc");
|
||||
std::fs::write(&path, [vec![0x91], deep_envelope()].concat()).unwrap();
|
||||
let entries = on_stack(64 * KIB, move || SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()).unwrap();
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert_eq!(entries[0].key, "k");
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_v1_store_with_a_deep_fifth_record_slot_is_refused_typed_on_a_64kib_thread() {
|
||||
in_child("a_v1_store_with_a_deep_fifth_record_slot_is_refused_typed_on_a_64kib_thread", || {
|
||||
// [header, [], [[key, schema, storedAt, payload, [[[...]]]]]]: the record's unknown fifth slot is
|
||||
// skipped 1000 containers deep, then the record is refused.
|
||||
let mut bytes = vec![0x93];
|
||||
bytes.extend(fixstr("cultcache.store.v1"));
|
||||
bytes.extend([0x90, 0x91, 0x95]);
|
||||
for text in ["k", "s", "t"] {
|
||||
bytes.extend(fixstr(text));
|
||||
}
|
||||
bytes.extend([0xc4, 1, 1]);
|
||||
bytes.extend(std::iter::repeat_n(0x91u8, 999));
|
||||
bytes.push(0x90);
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let path = directory.path().join("v1.cc");
|
||||
std::fs::write(&path, bytes).unwrap();
|
||||
let error = on_stack(64 * KIB, move || SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()).unwrap_err();
|
||||
let unreadable = error.downcast_ref::<CultCacheStoreUnreadable>().expect("a typed refusal");
|
||||
assert!(matches!(unreadable.kind, CultCacheStoreUnreadableKind::Undecodable));
|
||||
});
|
||||
}
|
||||
|
||||
/// A redb store holding the one deep envelope under its identity, `["t", "k"]`.
|
||||
fn redb_with_deep_row(path: &std::path::Path) {
|
||||
let database = redb::Database::create(path).unwrap();
|
||||
let write = database.begin_write().unwrap();
|
||||
{
|
||||
let mut table = write
|
||||
.open_table(redb::TableDefinition::<&[u8], &[u8]>::new("cultcache_envelopes"))
|
||||
.unwrap();
|
||||
table.insert([0x92, 0xa1, b't', 0xa1, b'k'].as_slice(), deep_envelope().as_slice()).unwrap();
|
||||
}
|
||||
write.commit().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_redb_store_with_a_deep_unknown_envelope_key_reads_on_a_104kib_thread() {
|
||||
in_child("a_redb_store_with_a_deep_unknown_envelope_key_reads_on_a_104kib_thread", || {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let path = directory.path().join("rows.redb");
|
||||
redb_with_deep_row(&path);
|
||||
let entries = on_stack(104 * KIB, move || RedbMessagePackBackingStore::new(path).unwrap().pull_all()).unwrap();
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert_eq!(entries[0].key, "k");
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_redb_compare_and_swap_reads_a_deep_unknown_envelope_key_on_a_104kib_thread() {
|
||||
in_child("a_redb_compare_and_swap_reads_a_deep_unknown_envelope_key_on_a_104kib_thread", || {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let path = directory.path().join("rows.redb");
|
||||
redb_with_deep_row(&path);
|
||||
let expected = CultCacheEnvelope {
|
||||
key: "k".into(),
|
||||
r#type: "t".into(),
|
||||
payload: vec![1],
|
||||
stored_at: "s".into(),
|
||||
schema_id: None,
|
||||
};
|
||||
let replacement = CultCacheEnvelope { payload: vec![2], ..expected.clone() };
|
||||
let swapped = on_stack(104 * KIB, move || {
|
||||
RedbMessagePackBackingStore::new(path).unwrap().compare_and_swap_entry(&expected, replacement)
|
||||
})
|
||||
.unwrap();
|
||||
assert!(swapped);
|
||||
});
|
||||
}
|
||||
|
|
@ -781,12 +781,12 @@ fn the_raw_doors_hold_the_bound_for_arms_maps_and_arrays() {
|
|||
}
|
||||
}
|
||||
|
||||
// A debug build on a 2 MiB thread carries 405 containers of this document; the bound keeps every
|
||||
// accepted payload below that, and every refusal is an error, not an abort.
|
||||
// The bound counts containers, not stack: decode grows its own stack, so the document at the limit
|
||||
// decodes on a 1 MiB thread in both profiles, and every refusal is an error, not an abort.
|
||||
#[test]
|
||||
fn stack_budget_debug_2mib() {
|
||||
fn the_union_chain_at_the_limit_decodes_on_a_1mib_thread() {
|
||||
std::thread::Builder::new()
|
||||
.stack_size(2 * 1024 * 1024)
|
||||
.stack_size(1024 * 1024)
|
||||
.spawn(|| {
|
||||
assert_eq!(
|
||||
through_cache(&vector("depth-at-limit.msgpack")).unwrap(),
|
||||
|
|
|
|||
Loading…
Reference in a new issue