From 8e218b04e87b9030426bf52c5a3dc4fa8e3776b1 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 04:17:42 +0000 Subject: [PATCH] Bytes after the value: a payload is one MessagePack value (Rust) One private payload check (depth, then one value) behind decode_payload and raw-envelope/batch admission; refuses a tail by count, never walks or quotes it. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-rs/CHANGELOG.md | 1 + packages/cultcache-rs/src/lib.rs | 68 +++++++++++++++++---- packages/cultcache-rs/tests/union_parity.rs | 62 +++++++++++++++++++ packages/cultnet-rs/CHANGELOG.md | 1 + packages/cultnet-rs/tests/decode_door.rs | 10 +++ 5 files changed, 131 insertions(+), 11 deletions(-) diff --git a/packages/cultcache-rs/CHANGELOG.md b/packages/cultcache-rs/CHANGELOG.md index ed7cd60a..4630994a 100644 --- a/packages/cultcache-rs/CHANGELOG.md +++ b/packages/cultcache-rs/CHANGELOG.md @@ -8,6 +8,7 @@ only in git history. ### Breaking +- `decode_payload`, `put_raw_envelope` and prepared batches refuse a payload with bytes after its first MessagePack value (`payload has {n} bytes after its first MessagePack value`), by count; the tail is never walked or quoted. They used to read the first value and keep or drop the tail, so a record with a stray tail was served and rewritten without it. A store holding such a record loads, and its read refuses, as for a payload past the nesting bound. Behaviour change in a public API: the next release is a minor bump (0.x breaking). - A single-file store written as `cultcache.store.v3` is read and written: a store the C# runtime marked v3 (it holds a record with an element id: an owned list element with a `[CultElementId]` member) keeps its header here, and is carried through every rewrite with its ids; this runtime never drops the mark. A runtime from before this change refuses a v3 store by name. Rolling back to an older release after a store has been written as v3 is one-way: there is no carry-back, so restore from a copy taken before upgrading. - A write over a file the reader refuses (a zero-byte file, a truncated store, a stamp this runtime does not read) refuses with `CultCacheStoreUnreadable` and leaves the file as it is. A whole-store write used to replace such a file. This reverses 0.3.1, where `push_all` replaced a file with no readable stamp (healing it); the stamp rule now has one owner, the reader every write asks under the lock, and it accepts v1 and v3 (0.3.1 refused v3). - `#[cultcache(index)]` means unique, as in C# `RefuseIndexSharing`: within one type and one alias no two records hold one non-blank value. A put, batch, push or load that would make two records hold a value is refused, naming both keys and the alias, and a store that already holds a duplicate does not open. A grouping index is no longer expressible; a consumer that relied on shared values must drop the `index` and look up by another way. diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs index 686547f4..4f00ea59 100644 --- a/packages/cultcache-rs/src/lib.rs +++ b/packages/cultcache-rs/src/lib.rs @@ -600,14 +600,21 @@ impl std::error::Error for PayloadTooDeep {} /// Walks the MessagePack headers of one payload without recursion, counting every array and map and /// refusing the first container past [`MAX_PAYLOAD_DEPTH`]. Scalars are skipped by length and nothing -/// is allocated per element. A payload that ends early is the decoder's to refuse. +/// is allocated per element. Only the first value is walked; a payload that ends early is the +/// decoder's to refuse. A depth-only probe: [`decode_payload`] also refuses bytes after the value. pub fn check_payload_depth(bytes: &[u8]) -> std::result::Result<(), PayloadTooDeep> { + first_value_end(bytes).map(|_| ()) +} + +/// Where the first MessagePack value of `bytes` ends, walking only that value (its tail is never +/// read). `None` when the payload ends before the value does: the decoder's to refuse. +fn first_value_end(bytes: &[u8]) -> std::result::Result, PayloadTooDeep> { let mut remaining = [0usize; MAX_PAYLOAD_DEPTH]; let mut depth = 0usize; let mut position = 0usize; while position < bytes.len() { let Some((advance, children)) = element_header(bytes, position) else { - return Ok(()); + return Ok(None); }; position = position.saturating_add(advance); if let Some(children) = children { @@ -629,7 +636,18 @@ pub fn check_payload_depth(bytes: &[u8]) -> std::result::Result<(), PayloadTooDe depth -= 1; } if depth == 0 { - return Ok(()); + return Ok((position <= bytes.len()).then_some(position)); + } + } + Ok(None) +} + +/// The one rule for "is this payload one value": the nesting bound, then no bytes after the first +/// value. The count is all it reports; the tail is never walked or quoted. +fn check_record_payload(bytes: &[u8]) -> Result<()> { + if let Some(end) = first_value_end(bytes)? { + if end < bytes.len() { + bail!("payload has {} bytes after its first MessagePack value", bytes.len() - end); } } Ok(()) @@ -784,7 +802,7 @@ fn decode_grown(bytes: &[u8]) -> std::result::Re grown(&mut reader, |grown| T::deserialize(grown)) } -/// The one decode of a document payload: the nesting bound first, then [`decode_grown`], whose +/// The one decode of a document payload: the payload check first (nesting bound, one value), then [`decode_grown`], whose /// rmp-serde depth counter (1024) is only a backstop. A failure keeps its kind and never quotes the /// bytes. Readers of stored envelope payloads outside this crate decode through it too. /// @@ -796,14 +814,14 @@ fn decode_grown(bytes: &[u8]) -> std::result::Re /// verified on x86_64-pc-windows-msvc by hand (`decode_stack`, revision 10a39dac, 13 tests, debug and release); a /// `stacker` bump or a change to the stack-size tests repeats it. pub fn decode_payload(payload: &[u8]) -> Result { - check_payload_depth(payload)?; + check_record_payload(payload)?; decode_grown(payload).map_err(opaque_decode_error) } -/// The nesting bound for an envelope admitted without a typed decode, from the same walk that -/// [`decode_payload`] runs first. Names the entry's type and key, never the payload. -fn require_payload_depth(entry: &CultCacheEnvelope) -> Result<()> { - check_payload_depth(&entry.payload).with_context(|| { +/// The payload check for an envelope admitted without a typed decode, the one [`decode_payload`] +/// runs first. Names the entry's type and key, never the payload. +fn require_record_payload(entry: &CultCacheEnvelope) -> Result<()> { + check_record_payload(&entry.payload).with_context(|| { format!( "refused CultCache envelope {:?} at key {:?}", entry.r#type, entry.key @@ -3265,7 +3283,7 @@ impl CultCache { entry.r#type )); } - require_payload_depth(entry)?; + require_record_payload(entry)?; self.refuse_claimed_id(entry)?; let Some(home) = self.home_index(&entry.r#type)? else { continue; @@ -3344,7 +3362,7 @@ impl CultCache { entry.r#type )); } - require_payload_depth(&entry)?; + require_record_payload(&entry)?; self.refuse_claimed_id(&entry)?; self.refuse_second_global(&self.entries, &entry.r#type, &entry.key)?; self.refuse_retype(&self.entries, &self.foreign, &entry)?; @@ -6060,6 +6078,34 @@ mod tests { Ok(()) } + #[test] + fn put_raw_envelope_refuses_bytes_after_the_value() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("tail.cc"); + let mut cache = CultCache::new(); + cache.register_entry_type::()?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; + let tailed = CultCacheEnvelope { + key: "app".to_string(), + r#type: Settings::TYPE.to_string(), + payload: vec![0x91, 0x01, 0xc0], + stored_at: now_utc_second(), + schema_id: None, + }; + let error = format!("{:#}", cache.put_raw_envelope(tailed.clone()).unwrap_err()); + assert!(error.contains("payload has 1 bytes after its first MessagePack value"), "{error}"); + assert!(error.contains("app") && error.contains(Settings::TYPE), "{error}"); + assert!(cache.snapshot().is_empty()); + assert!(!store_path.exists() || SingleFileMessagePackBackingStore::new(&store_path).pull_all()?.is_empty()); + let error = format!("{:#}", cache.put_prepared_batch(vec![tailed.clone()]).unwrap_err()); + assert!(error.contains("payload has 1 bytes after its first MessagePack value"), "{error}"); + assert!(cache.snapshot().is_empty()); + // The same bytes without the tail are admitted: the refusal is the count, not the shape. + cache.put_raw_envelope(CultCacheEnvelope { payload: vec![0x91, 0x01], ..tailed })?; + assert_eq!(cache.snapshot().len(), 1); + Ok(()) + } + #[test] fn soa_snapshot_round_trips_registered_entries() -> Result<()> { let temp = tempfile::tempdir()?; diff --git a/packages/cultcache-rs/tests/union_parity.rs b/packages/cultcache-rs/tests/union_parity.rs index 24fe999a..9bc0c4c7 100644 --- a/packages/cultcache-rs/tests/union_parity.rs +++ b/packages/cultcache-rs/tests/union_parity.rs @@ -9,6 +9,8 @@ use cultcache_rs::CultCacheEnvelope; use cultcache_rs::MAX_PAYLOAD_DEPTH; use cultcache_rs::PayloadTooDeep; use cultcache_rs::check_payload_depth; +use cultcache_rs::decode_payload; +use cultcache_rs::RedbMessagePackBackingStore; use cultcache_rs::CultSlots; use cultcache_rs::CultUnion; use cultcache_rs::DatabaseEntry; @@ -757,6 +759,66 @@ fn bounds_vectors() { assert_eq!((accepted, refused, read_only), (4, 7, 1)); } +// A payload is one MessagePack value (bytes-after-outcomes.msgpack): every door refuses a tail by +// its count, names the clause, never the depth limit (the tail is not walked) and never a value. +#[test] +fn bytes_after_vectors() { + let rows: Vec = rmp_serde::from_slice(&vector("bytes-after-outcomes.msgpack")).unwrap(); + assert_eq!(rows.len(), 4); + let surface = vector("union-surface.msgpack"); + for row in &rows { + let fields = row.as_array().unwrap(); + let (file, outcome, union, detail) = (fields[0].as_str().unwrap(), fields[1].as_str().unwrap(), fields[2].as_str().unwrap(), fields[4].as_str().unwrap()); + assert_eq!((outcome, union, detail), ("refused", "UnionSurface", "bytes-after"), "{file}"); + let bytes = vector(file); + assert_eq!(&bytes[..surface.len()], &surface[..], "{file}"); + let clause = format!("payload has {} bytes after its first MessagePack value", bytes.len() - surface.len()); + let check = |door: &str, message: &str| { + assert!(message.contains(&clause), "{file} {door}: {message}"); + assert!(!message.contains("the limit is"), "{file} {door} walked the tail: {message}"); + assert!(!message.contains("CANARY-bytes-after-7d1e"), "{file} {door} echoed the tail: {message}"); + assert!(!message.contains(CANARY), "{file} {door} echoed a value: {message}"); + }; + let error = decode_payload::(&bytes).unwrap_err(); + assert!(error.downcast_ref::().is_none(), "{file}"); + check("decode_payload", &format!("{error:#}")); + // A store written behind the cache's back loads (the bound is the read's, as for depth) and + // its read refuses, naming the key and the type. + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("foreign.cc"); + SingleFileMessagePackBackingStore::new(&path).push(&envelope("k", &bytes)).unwrap(); + let mut single = CultCache::new(); + single.register_entry_type::().unwrap(); + single.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap(); + single.pull_all_backing_stores().unwrap(); + let message = format!("{:#}", single.get::("k").unwrap_err()); + check("single-file read", &message); + assert!(message.contains("\"k\"") && message.contains(UnionSurface::TYPE), "{file}: {message}"); + let redb_path = temp.path().join("foreign.redb"); + RedbMessagePackBackingStore::new(&redb_path).unwrap().push(&envelope("k", &bytes)).unwrap(); + let mut redb = CultCache::new(); + redb.register_entry_type::().unwrap(); + redb.add_generic_backing_store(RedbMessagePackBackingStore::new(&redb_path).unwrap()).unwrap(); + redb.pull_all_backing_stores().unwrap(); + check("redb read", &format!("{:#}", redb.get::("k").unwrap_err())); + // Raw admission refuses and stages nothing. + let mut raw = CultCache::new(); + raw.register_entry_type::().unwrap(); + let message = format!("{:#}", raw.put_raw_envelope(envelope("k", &bytes)).unwrap_err()); + check("put_raw_envelope", &message); + assert!(raw.snapshot().is_empty(), "{file}"); + } +} + +// A payload that ends early is not bytes after the value: the decoder refuses it, not the check. +#[test] +fn a_truncated_payload_is_the_decoders_not_bytes_after() { + let mut truncated = vec![0x91, 0xd9, 0x40]; + truncated.extend_from_slice(b"short"); + let message = format!("{:#}", decode_payload::(&truncated).unwrap_err()); + assert!(!message.contains("bytes after"), "{message}"); +} + fn wrap_chain(wraps: usize, leaf: Node) -> UnionSurface { let mut node = leaf; for _ in 0..wraps { diff --git a/packages/cultnet-rs/CHANGELOG.md b/packages/cultnet-rs/CHANGELOG.md index 384395ec..1a01dbe1 100644 --- a/packages/cultnet-rs/CHANGELOG.md +++ b/packages/cultnet-rs/CHANGELOG.md @@ -15,5 +15,6 @@ only in git history. ### Breaking +- A CultNet frame with bytes after its message is refused, by count (through `cultcache_rs::decode_payload`); the decoder used to read the message and drop the tail. No producer emits one. Behaviour change: the next release is a minor bump (0.x breaking). - `CultNetDocumentRegistry::register` returns `Result<&mut Self>` where it returned `&mut Self`. A binding that claims a document type or a schema id another binding already holds is refused with `cultcache_rs::SchemaConflictError` (naming the new binding's schema id and both document types) and nothing is bound; binding a type to the schema id it already holds still replaces that binding. Callers chain `register(...)?`. - `CultMeshCdnChunkRef` gains a public `id` field, the decimal text of `offset` (the reference's element id, slot 4). A manifest written before ids reads `""`; encoding writes the slot. diff --git a/packages/cultnet-rs/tests/decode_door.rs b/packages/cultnet-rs/tests/decode_door.rs index fd5bab6a..c55a6404 100644 --- a/packages/cultnet-rs/tests/decode_door.rs +++ b/packages/cultnet-rs/tests/decode_door.rs @@ -190,6 +190,16 @@ fn message_past_the_bound_is_refused_typed() { }); } +// A frame is one message: cultcache's payload rule refuses bytes after it, by count. +#[test] +fn frame_with_bytes_after_is_refused() { + let mut bytes = hello_with(&nested(0)); + assert!(decode_message(bytes.clone(), BIG).is_ok()); + bytes.push(0xc0); + let error = format!("{:#}", decode_message(bytes, BIG).unwrap_err()); + assert!(error.contains("1 bytes after its first MessagePack value"), "{error}"); +} + fn raw_v1_bytes(header_unknown: Option<&[u8]>) -> Vec { let mut header: Vec<(rmpv::Value, rmpv::Value)> = vec![ ("schemaId".into(), "s".into()),