diff --git a/docs/research/py-msgpack-default/README.md b/docs/research/py-msgpack-default/README.md new file mode 100644 index 00000000..03388625 --- /dev/null +++ b/docs/research/py-msgpack-default/README.md @@ -0,0 +1,22 @@ +# Python MessagePack default payload: history and probes (2026-10-11) + +Serves `cultlib-gaps:ruling:py-msgpack-default` and `cultlib-gaps:cut_spec:cut-py-msgpack-default.r1`. + +## How a JSON payload default got into .cc + +1. Ghostlight `2531b42` (2026-05-06), the first consumer: "The JSONL store is deliberate bootstrap plumbing", msgpack optional. +2. CultLib `b214daab` (2026-05-24) squashed in `python/cultcache-py` from GameCult/cultcache-py@9b4bbb8a with + `define_document_type` already defaulting to `_json_payload_encoder`; msgpack was an optional extra and the README + promised a standard-library-only bootstrap. JSON was the default because it was the only codec without msgpack. +3. CultLib `a1ef58cb` (2026-06-13) made msgpack a hard dependency but kept the JSON default and the optional-import + guards, so the default outlived its reason. +4. Earlier history is unreachable (GameCult/cultcache-py no longer exists; CultLib holds only the squash). voidbot + `search_history` found no discussion of the choice. + +## Probes + +- `probe1.py`, `probe2.py`: no runtime records a payload's format in the store, so a reader cannot tell JSON from + MessagePack except by trying; a one-byte JSON payload `0`-`9` is valid MessagePack (48-57); a MessagePack map with + a non-str/bytes key packs but `unpackb` refuses it. +- `rmpprobe/`: `rmp_serde::from_slice` reads a JSON object's bytes as `Number(123)`, which is why Brokkr's daemon must + read exactly one MessagePack value with a trailing-bytes check. diff --git a/docs/research/py-msgpack-default/probe1.py b/docs/research/py-msgpack-default/probe1.py new file mode 100644 index 00000000..7276ab25 --- /dev/null +++ b/docs/research/py-msgpack-default/probe1.py @@ -0,0 +1,47 @@ +import json, sys, msgpack +print(sys.version.split()[0], msgpack.version) +def pk(v): return msgpack.packb(v, use_bin_type=True) +def up(b): return msgpack.unpackb(b, raw=False) +def jd(v): return json.dumps(v, ensure_ascii=False, sort_keys=True).encode() +cases = {"dict": {"schema":"x","n":1}, "list":[1,2], "str":"hi", "digit":5, "neg":-3, "true":True, "null":None, "float":1.5, "bigdigit":12} +print("== msgpack.unpackb on JSON payloads") +for k,v in cases.items(): + b = jd(v) + try: r = up(b); print(k, b, "->ACCEPTED", repr(r)) + except Exception as e: print(k, b, "->", type(e).__module__+"."+type(e).__name__, isinstance(e, ValueError), str(e)[:60]) +print("== json.loads on msgpack payloads") +for k,v in cases.items(): + b = pk(v) + try: r = json.loads(b.decode("utf-8")); print(k, b, "->ACCEPTED", repr(r)) + except Exception as e: print(k, b, "->", type(e).__name__, str(e)[:50]) +print("== semantic differences, value -> json roundtrip vs msgpack roundtrip") +diffs = {"intkeys": {1:"a"}, "tuple": (1,2), "bytes": b"ab", "bigint": 2**70, "nan": float("nan"), "boolkey": {True:1}, "nested_intkey": {"a":{2:3}}, "set": {1,2}, "order": {"b":1,"a":2}} +for k,v in diffs.items(): + out=[] + try: out.append("json:"+repr(json.loads(jd(v)))) + except Exception as e: out.append("json:"+type(e).__name__) + try: b=pk(v) + except Exception as e: out.append("pack:"+type(e).__name__+":"+str(e)[:40]); print(k, out); continue + try: out.append("mp:"+repr(up(b))) + except Exception as e: out.append("unpack:"+type(e).__name__+":"+str(e)[:60]) + try: out.append("mp_nonstrict:"+repr(msgpack.unpackb(b, raw=False, strict_map_key=False))) + except Exception as e: out.append("ns:"+type(e).__name__) + print(k, out) +print("== custom node classes") +import collections, types +class OnlyIter: + def __iter__(self): return iter([1]) +for k,v in {"deque":collections.deque([1]), "mappingproxy":types.MappingProxyType({"a":1}), "userlist":collections.UserList([1]), "onlyiter":OnlyIter(), "frozenset":frozenset([1]), "userdict":collections.UserDict({"a":1})}.items(): + r=[] + for name,f in (("pack",pk),("json",jd)): + try: f(v); r.append(name+":ok") + except Exception as e: r.append(name+":"+type(e).__name__) + print(k, r) +print("== JSON read recursion threshold") +for d in (900, 1000, 3000, 5000, 10000, 100000): + try: json.loads("["*d + "]"*d); print(d, "ok") + except RecursionError: print(d, "RecursionError") +print("== msgpack read deep") +for d in (300, 1000, 2000, 100000): + try: up(b"\x91"*d + b"\xc0"); print(d,"ok") + except Exception as e: print(d, type(e).__module__, type(e).__name__) diff --git a/docs/research/py-msgpack-default/probe2.py b/docs/research/py-msgpack-default/probe2.py new file mode 100644 index 00000000..259320eb --- /dev/null +++ b/docs/research/py-msgpack-default/probe2.py @@ -0,0 +1,33 @@ +# Brokkr-shaped flow at CultLib 9d7b5842: a store written by today's JSON default, then read by a MessagePack-default definition. +import sys, tempfile, os, json, msgpack, timeit, dataclasses +sys.path.insert(0, r"F:/Projects/CultLib/packages/cultcache-py/src") +import cultcache_py as cc +from cultcache_py.stores import SingleFileMessagePackBackingStore +d = tempfile.mkdtemp(); path = os.path.join(d, "blender-editor.ccmp") +def open_cache(defn): + c = cc.CultCache.builder().register_document_type(defn).add_generic_store(SingleFileMessagePackBackingStore(path)).build() + c.pull_all_backing_stores(); return c +old = cc.define_document_type("brokkr.sync.var.v0") +value = {"schema": "brokkr.sync.var.v0", "syncVarId": "v1", "enabled": True, "kind": "transform", "updatedAt": "2026-10-11T00:00:00Z"} +c = open_cache(old); c.put(old, "sync/vars/v1", value) +raw = msgpack.unpackb(open(path, "rb").read(), raw=False) +print("store header", raw[0], "catalog entry", raw[1][0][:5] if raw[1] else None) +print("record", raw[2][0][:3], "payload", raw[2][0][3][:40]) +# MessagePack-default reader, today's _messagepack pair standing in for the proposed default +new = cc.define_document_type("brokkr.sync.var.v0", payload_encoder=lambda t: msgpack.packb(t, use_bin_type=True), payload_decoder=lambda b: msgpack.unpackb(b, raw=False)) +try: open_cache(new); print("new reader: ACCEPTED") +except Exception as e: print("new reader refused:", type(e).__module__, type(e).__name__, str(e)[:80]) +# opt-in JSON pair reads it +print("json opt-in reads:", open_cache(old).get(old, "sync/vars/v1") == value) +# migration: read with JSON definition, write with MessagePack definition, same key +v = open_cache(old).get(old, "sync/vars/v1"); c2 = cc.CultCache.builder().register_document_type(new).add_generic_store(SingleFileMessagePackBackingStore(path)).build() +c2._state # not pulled: pull would fail on the JSON doc +try: + c2.put(new, "sync/vars/v1", v); print("put without pull ok") +except Exception as e: print("put without pull:", type(e).__name__, str(e)[:80]) +print("after migration new reader reads:", open_cache(new).get(new, "sync/vars/v1") == value) +# size and speed, one Brokkr-shaped document +jb = json.dumps(value, ensure_ascii=False, sort_keys=True).encode(); mb = msgpack.packb(value, use_bin_type=True) +print("bytes json", len(jb), "msgpack", len(mb)) +print("encode us json %.2f msgpack %.2f" % (timeit.timeit(lambda: json.dumps(value, ensure_ascii=False, sort_keys=True).encode(), number=100000)*10, timeit.timeit(lambda: msgpack.packb(value, use_bin_type=True), number=100000)*10)) +print("decode us json %.2f msgpack %.2f" % (timeit.timeit(lambda: json.loads(jb.decode()), number=100000)*10, timeit.timeit(lambda: msgpack.unpackb(mb, raw=False), number=100000)*10)) diff --git a/docs/research/py-msgpack-default/rmpprobe/Cargo.toml b/docs/research/py-msgpack-default/rmpprobe/Cargo.toml new file mode 100644 index 00000000..05ae7ca1 --- /dev/null +++ b/docs/research/py-msgpack-default/rmpprobe/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "rmpprobe" +version = "0.1.0" +edition = "2021" +[dependencies] +rmp-serde = "=1.3.1" +serde_json = "1" +serde = "1" diff --git a/docs/research/py-msgpack-default/rmpprobe/src/main.rs b/docs/research/py-msgpack-default/rmpprobe/src/main.rs new file mode 100644 index 00000000..198c4760 --- /dev/null +++ b/docs/research/py-msgpack-default/rmpprobe/src/main.rs @@ -0,0 +1,21 @@ +fn main() { + let json_obj = br#"{"enabled":true,"kind":"transform","schema":"brokkr.sync.var.v0"}"#; + let r: Result = rmp_serde::from_slice(json_obj); + println!("rmp_serde on a JSON object payload: {:?}", r); + let mp = rmp_serde::to_vec(&serde_json::json!({"schema":"x","n":1})).unwrap(); + println!("rmp_serde::to_vec of a json! object, first byte {:#04x} len {}", mp[0], mp.len()); + let j: Result, _> = serde_json::from_slice(&mp); + println!("serde_json on that MessagePack map: {:?}", j.map(|_| ()).map_err(|e| e.to_string())); + let mut de = rmp_serde::Deserializer::from_read_ref(&json_obj[..]); + let v: serde_json::Value = serde::Deserialize::deserialize(&mut de).unwrap(); + println!("one_value(json obj) = {:?}; one_value(mp) = {:?}", one_value(json_obj), one_value(&mp).map(|_| ())); + println!("consumed {} of {} bytes, value {:?}", de.get_ref().len() as usize, json_obj.len(), v); +} +#[allow(dead_code)] +fn one_value(bytes: &[u8]) -> Result { + let mut de = rmp_serde::Deserializer::new(std::io::Cursor::new(bytes)); + let v: serde_json::Value = serde::Deserialize::deserialize(&mut de).map_err(|e| e.to_string())?; + let used = de.get_ref().position() as usize; + if used != bytes.len() { return Err(format!("{} trailing bytes after the first MessagePack value", bytes.len() - used)); } + Ok(v) +}