Research: how JSON became cultcache-py's payload default, and the MessagePack-default probes

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

docs/research/py-msgpack-default/README.md

@@ -0,0 +1,22 @@
+# Python MessagePack default payload: history and probes (2026-10-11)
+
+Serves `cultlib-gaps:ruling:py-msgpack-default` and `cultlib-gaps:cut_spec:cut-py-msgpack-default.r1`.
+
+## How a JSON payload default got into .cc
+
+1. Ghostlight `2531b42` (2026-05-06), the first consumer: "The JSONL store is deliberate bootstrap plumbing", msgpack optional.
+2. CultLib `b214daab` (2026-05-24) squashed in `python/cultcache-py` from GameCult/cultcache-py@9b4bbb8a with
+   `define_document_type` already defaulting to `_json_payload_encoder`; msgpack was an optional extra and the README
+   promised a standard-library-only bootstrap. JSON was the default because it was the only codec without msgpack.
+3. CultLib `a1ef58cb` (2026-06-13) made msgpack a hard dependency but kept the JSON default and the optional-import
+   guards, so the default outlived its reason.
+4. Earlier history is unreachable (GameCult/cultcache-py no longer exists; CultLib holds only the squash). voidbot
+   `search_history` found no discussion of the choice.
+
+## Probes
+
+- `probe1.py`, `probe2.py`: no runtime records a payload's format in the store, so a reader cannot tell JSON from
+  MessagePack except by trying; a one-byte JSON payload `0`-`9` is valid MessagePack (48-57); a MessagePack map with
+  a non-str/bytes key packs but `unpackb` refuses it.
+- `rmpprobe/`: `rmp_serde::from_slice` reads a JSON object's bytes as `Number(123)`, which is why Brokkr's daemon must
+  read exactly one MessagePack value with a trailing-bytes check.

docs/research/py-msgpack-default/probe1.py

@@ -0,0 +1,47 @@
+import json, sys, msgpack
+print(sys.version.split()[0], msgpack.version)
+def pk(v): return msgpack.packb(v, use_bin_type=True)
+def up(b): return msgpack.unpackb(b, raw=False)
+def jd(v): return json.dumps(v, ensure_ascii=False, sort_keys=True).encode()
+cases = {"dict": {"schema":"x","n":1}, "list":[1,2], "str":"hi", "digit":5, "neg":-3, "true":True, "null":None, "float":1.5, "bigdigit":12}
+print("== msgpack.unpackb on JSON payloads")
+for k,v in cases.items():
+    b = jd(v)
+    try: r = up(b); print(k, b, "->ACCEPTED", repr(r))
+    except Exception as e: print(k, b, "->", type(e).__module__+"."+type(e).__name__, isinstance(e, ValueError), str(e)[:60])
+print("== json.loads on msgpack payloads")
+for k,v in cases.items():
+    b = pk(v)
+    try: r = json.loads(b.decode("utf-8")); print(k, b, "->ACCEPTED", repr(r))
+    except Exception as e: print(k, b, "->", type(e).__name__, str(e)[:50])
+print("== semantic differences, value -> json roundtrip vs msgpack roundtrip")
+diffs = {"intkeys": {1:"a"}, "tuple": (1,2), "bytes": b"ab", "bigint": 2**70, "nan": float("nan"), "boolkey": {True:1}, "nested_intkey": {"a":{2:3}}, "set": {1,2}, "order": {"b":1,"a":2}}
+for k,v in diffs.items():
+    out=[]
+    try: out.append("json:"+repr(json.loads(jd(v))))
+    except Exception as e: out.append("json:"+type(e).__name__)
+    try: b=pk(v)
+    except Exception as e: out.append("pack:"+type(e).__name__+":"+str(e)[:40]); print(k, out); continue
+    try: out.append("mp:"+repr(up(b)))
+    except Exception as e: out.append("unpack:"+type(e).__name__+":"+str(e)[:60])
+    try: out.append("mp_nonstrict:"+repr(msgpack.unpackb(b, raw=False, strict_map_key=False)))
+    except Exception as e: out.append("ns:"+type(e).__name__)
+    print(k, out)
+print("== custom node classes")
+import collections, types
+class OnlyIter:
+    def __iter__(self): return iter([1])
+for k,v in {"deque":collections.deque([1]), "mappingproxy":types.MappingProxyType({"a":1}), "userlist":collections.UserList([1]), "onlyiter":OnlyIter(), "frozenset":frozenset([1]), "userdict":collections.UserDict({"a":1})}.items():
+    r=[]
+    for name,f in (("pack",pk),("json",jd)):
+        try: f(v); r.append(name+":ok")
+        except Exception as e: r.append(name+":"+type(e).__name__)
+    print(k, r)
+print("== JSON read recursion threshold")
+for d in (900, 1000, 3000, 5000, 10000, 100000):
+    try: json.loads("["*d + "]"*d); print(d, "ok")
+    except RecursionError: print(d, "RecursionError")
+print("== msgpack read deep")
+for d in (300, 1000, 2000, 100000):
+    try: up(b"\x91"*d + b"\xc0"); print(d,"ok")
+    except Exception as e: print(d, type(e).__module__, type(e).__name__)

docs/research/py-msgpack-default/probe2.py

@@ -0,0 +1,33 @@
+# Brokkr-shaped flow at CultLib 9d7b5842: a store written by today's JSON default, then read by a MessagePack-default definition.
+import sys, tempfile, os, json, msgpack, timeit, dataclasses
+sys.path.insert(0, r"F:/Projects/CultLib/packages/cultcache-py/src")
+import cultcache_py as cc
+from cultcache_py.stores import SingleFileMessagePackBackingStore
+d = tempfile.mkdtemp(); path = os.path.join(d, "blender-editor.ccmp")
+def open_cache(defn):
+    c = cc.CultCache.builder().register_document_type(defn).add_generic_store(SingleFileMessagePackBackingStore(path)).build()
+    c.pull_all_backing_stores(); return c
+old = cc.define_document_type("brokkr.sync.var.v0")
+value = {"schema": "brokkr.sync.var.v0", "syncVarId": "v1", "enabled": True, "kind": "transform", "updatedAt": "2026-10-11T00:00:00Z"}
+c = open_cache(old); c.put(old, "sync/vars/v1", value)
+raw = msgpack.unpackb(open(path, "rb").read(), raw=False)
+print("store header", raw[0], "catalog entry", raw[1][0][:5] if raw[1] else None)
+print("record", raw[2][0][:3], "payload", raw[2][0][3][:40])
+# MessagePack-default reader, today's _messagepack pair standing in for the proposed default
+new = cc.define_document_type("brokkr.sync.var.v0", payload_encoder=lambda t: msgpack.packb(t, use_bin_type=True), payload_decoder=lambda b: msgpack.unpackb(b, raw=False))
+try: open_cache(new); print("new reader: ACCEPTED")
+except Exception as e: print("new reader refused:", type(e).__module__, type(e).__name__, str(e)[:80])
+# opt-in JSON pair reads it
+print("json opt-in reads:", open_cache(old).get(old, "sync/vars/v1") == value)
+# migration: read with JSON definition, write with MessagePack definition, same key
+v = open_cache(old).get(old, "sync/vars/v1"); c2 = cc.CultCache.builder().register_document_type(new).add_generic_store(SingleFileMessagePackBackingStore(path)).build()
+c2._state  # not pulled: pull would fail on the JSON doc
+try:
+    c2.put(new, "sync/vars/v1", v); print("put without pull ok")
+except Exception as e: print("put without pull:", type(e).__name__, str(e)[:80])
+print("after migration new reader reads:", open_cache(new).get(new, "sync/vars/v1") == value)
+# size and speed, one Brokkr-shaped document
+jb = json.dumps(value, ensure_ascii=False, sort_keys=True).encode(); mb = msgpack.packb(value, use_bin_type=True)
+print("bytes json", len(jb), "msgpack", len(mb))
+print("encode us json %.2f msgpack %.2f" % (timeit.timeit(lambda: json.dumps(value, ensure_ascii=False, sort_keys=True).encode(), number=100000)*10, timeit.timeit(lambda: msgpack.packb(value, use_bin_type=True), number=100000)*10))
+print("decode us json %.2f msgpack %.2f" % (timeit.timeit(lambda: json.loads(jb.decode()), number=100000)*10, timeit.timeit(lambda: msgpack.unpackb(mb, raw=False), number=100000)*10))

docs/research/py-msgpack-default/rmpprobe/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "rmpprobe"
+version = "0.1.0"
+edition = "2021"
+[dependencies]
+rmp-serde = "=1.3.1"
+serde_json = "1"
+serde = "1"

docs/research/py-msgpack-default/rmpprobe/src/main.rs

@@ -0,0 +1,21 @@
+fn main() {
+    let json_obj = br#"{"enabled":true,"kind":"transform","schema":"brokkr.sync.var.v0"}"#;
+    let r: Result<serde_json::Value, _> = rmp_serde::from_slice(json_obj);
+    println!("rmp_serde on a JSON object payload: {:?}", r);
+    let mp = rmp_serde::to_vec(&serde_json::json!({"schema":"x","n":1})).unwrap();
+    println!("rmp_serde::to_vec of a json! object, first byte {:#04x} len {}", mp[0], mp.len());
+    let j: Result<serde_json::Map<String, serde_json::Value>, _> = serde_json::from_slice(&mp);
+    println!("serde_json on that MessagePack map: {:?}", j.map(|_| ()).map_err(|e| e.to_string()));
+    let mut de = rmp_serde::Deserializer::from_read_ref(&json_obj[..]);
+    let v: serde_json::Value = serde::Deserialize::deserialize(&mut de).unwrap();
+    println!("one_value(json obj) = {:?}; one_value(mp) = {:?}", one_value(json_obj), one_value(&mp).map(|_| ()));
+    println!("consumed {} of {} bytes, value {:?}", de.get_ref().len() as usize, json_obj.len(), v);
+}
+#[allow(dead_code)]
+fn one_value(bytes: &[u8]) -> Result<serde_json::Value, String> {
+    let mut de = rmp_serde::Deserializer::new(std::io::Cursor::new(bytes));
+    let v: serde_json::Value = serde::Deserialize::deserialize(&mut de).map_err(|e| e.to_string())?;
+    let used = de.get_ref().position() as usize;
+    if used != bytes.len() { return Err(format!("{} trailing bytes after the first MessagePack value", bytes.len() - used)); }
+    Ok(v)
+}