From 76ab8308eb1e9664ee3d857fcd6d4c30379487ff Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 22 Oct 2026 21:10:01 +0200 Subject: [PATCH] Vili: text-embedding cache and one-shot GPU encoder child text_encoding.py holds the CultCache embedding store (gamecult.vili.text_embedding.v0 through cultcache-py), the encoder identity and key, plan_placement with its typed gpu-busy refusal, and CachedTextEncoder, which sends a request's misses to one child call. kimodo-text-encode.py is that child: it assembles Kimodo's LLM2Vec encoder in bf16 with its own loader (operator ruling encoder-own-loader; peft's offload path raises TypeError in Kimodo's loader), encodes each text alone, writes a MessagePack answer and exits. GPU reserve 3 GiB: four cold runs on Raven's 16 GB card produced identical vectors with no CUDA error. The worker and daemon are not wired yet. Co-Authored-By: Claude Opus 5.5 --- scripts/kimodo-text-encode.py | 117 ++++++++++++++++++++ scripts/test_text_encoding.py | 182 ++++++++++++++++++++++++++++++ scripts/text_encoding.py | 203 ++++++++++++++++++++++++++++++++++ 3 files changed, 502 insertions(+) create mode 100644 scripts/kimodo-text-encode.py create mode 100644 scripts/test_text_encoding.py create mode 100644 scripts/text_encoding.py diff --git a/scripts/kimodo-text-encode.py b/scripts/kimodo-text-encode.py new file mode 100644 index 0000000..ce91f18 --- /dev/null +++ b/scripts/kimodo-text-encode.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""One-shot Kimodo text encoder: texts on stdin, vectors in --out, then exit so the OS reclaims RAM and VRAM. + +Reads {"texts": [...]} as JSON on stdin. Loads Kimodo's LLM2Vec preset in bfloat16 with the layers that do not +fit in free VRAM held in RAM, encodes each text alone as LLM2VecEncoder does, and writes a MessagePack answer. +There is no CPU fallback; --device cpu exists only for the reference probe. +""" +import argparse +import json +import resource +import sys +import time +from pathlib import Path + +import msgpack +import torch + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from text_encoding import CHILD_EXIT_GPU_BUSY, EMBEDDING_DIM, GpuBusy, plan_placement # noqa: E402 + +from peft import PeftModel # noqa: E402 +from transformers import AutoConfig, AutoTokenizer # noqa: E402 + +from kimodo.model.llm2vec.llm2vec import LLM2Vec, batch_to_device # noqa: E402 +from kimodo.model.load_model import TEXT_ENCODER_PRESETS # noqa: E402 + + +def load_encoder(base_id, peft_id, dtype, peft_kwargs, **model_kwargs): + """Assemble the encoder in the order and with the ids of LLM2Vec.from_pretrained. + + MUST TRACK Kimodo's loader: kimodo/model/llm2vec/llm2vec.py, LLM2Vec.from_pretrained, Kimodo commit 6bb5848 + (nv-tlabs/kimodo). It is mirrored, not called, because that method hands PeftModel.from_pretrained no kwargs, and + with layers offloaded to RAM peft then takes a path that raises TypeError in accelerate's get_balanced_memory + (peft 0.18.1, accelerate 1.13.0, transformers 5.1.0). Here every PeftModel.from_pretrained gets peft_kwargs + (device_map='sequential'). Operator ruling motion-pipeline:ruling:encoder-own-loader. The ids' llm2vec_config.json + is not a local path, so Kimodo's loader applies no encoder config; neither does this. + """ + tokenizer = AutoTokenizer.from_pretrained(base_id) + tokenizer.pad_token = tokenizer.eos_token + tokenizer.padding_side = "left" + model_class = LLM2Vec._get_model_class(AutoConfig.from_pretrained(base_id).__class__.__name__, enable_bidirectional=True) + model = model_class.from_pretrained(base_id, torch_dtype=dtype, **model_kwargs) + if hasattr(model, "peft_config"): + model = PeftModel.from_pretrained(model, base_id, **peft_kwargs).merge_and_unload() + model = PeftModel.from_pretrained(model, peft_id, **peft_kwargs) + return LLM2Vec(model=model, tokenizer=tokenizer) + + +def mem_available() -> int: + for line in Path("/proc/meminfo").read_text().splitlines(): + if line.startswith("MemAvailable:"): + return int(line.split()[1]) * 1024 + raise RuntimeError("MemAvailable is not in /proc/meminfo") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--out", required=True) + parser.add_argument("--device", choices=["gpu", "cpu"], default="gpu") + args = parser.parse_args() + texts = json.load(sys.stdin)["texts"] + preset = TEXT_ENCODER_PRESETS["llm2vec"]["kwargs"] + + load_kwargs, peft_kwargs = {}, {} + if args.device == "gpu": + free_vram, _ = torch.cuda.mem_get_info(0) + try: + max_memory = plan_placement(free_vram, mem_available()) + load_kwargs = {"device_map": "auto", "max_memory": max_memory} + peft_kwargs = {"device_map": "sequential", "max_memory": max_memory} + except GpuBusy as busy: + Path(args.out).write_bytes(msgpack.packb({"error": busy.code, "freeVramBytes": busy.free_vram}, use_bin_type=True)) + return CHILD_EXIT_GPU_BUSY + + started = time.perf_counter() + model = load_encoder( + preset["base_model_name_or_path"], + preset["peft_model_name_or_path"], + getattr(torch, preset["dtype"]), + peft_kwargs, + **load_kwargs, + ) + model.eval() + for p in model.parameters(): + p.requires_grad = False + load_seconds = time.perf_counter() - started + + placement = {"gpuBytes": 0, "cpuBytes": 0} + for p in model.parameters(): + placement["gpuBytes" if p.device.type == "cuda" else "cpuBytes"] += p.numel() * p.element_size() + input_device = "cuda:0" if args.device == "gpu" else "cpu" + + vectors, encode_seconds = [], [] + for text in texts: + began = time.perf_counter() + # The path LLM2Vec.encode takes for one text, without its .to(device) that would undo the device map. + features = model.tokenize([model.prepare_for_tokenization(model._convert_to_str("", text))]) + features = batch_to_device(features, input_device) + with torch.no_grad(): + vector = model.forward(features).detach().cpu().to(torch.float32)[0] + assert vector.shape == (EMBEDDING_DIM,) + vectors.append(vector.numpy().tobytes()) + encode_seconds.append(time.perf_counter() - began) + + Path(args.out).write_bytes(msgpack.packb({ + "vectors": vectors, + "encodeSeconds": encode_seconds, + "loadSeconds": load_seconds, + "placement": placement, + "peakVramBytes": torch.cuda.max_memory_allocated(0) if torch.cuda.is_available() else 0, + "peakRssBytes": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024, + "device": args.device, + }, use_bin_type=True)) + return 0 + + +sys.exit(main()) diff --git a/scripts/test_text_encoding.py b/scripts/test_text_encoding.py new file mode 100644 index 0000000..0f4ed53 --- /dev/null +++ b/scripts/test_text_encoding.py @@ -0,0 +1,182 @@ +"""Run inside the Kimodo image: python -m unittest discover -s /opt/vili -p test_text_encoding.py""" +import sys +import tempfile +import textwrap +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import text_encoding as te # noqa: E402 + +GIB = te.GIB +SECRET = "a person whispers the launch code" + + +def vector(seed: int) -> bytes: + return bytes([seed % 251]) * te.EMBEDDING_BYTES + + +class FakeChild: + """Stands in for the encoder child: one call per request, vectors derived from the text.""" + + def __init__(self): + self.calls = [] + + def __call__(self, texts): + self.calls.append(list(texts)) + return { + "vectors": [vector(len(text)) for text in texts], + "encodeSeconds": [0.5] * len(texts), + "placement": {"gpuBytes": 11 * GIB, "cpuBytes": 5 * GIB}, + } + + +def open_encoder(directory, child, identity="id-a"): + store = te.open_store(Path(directory) / "embeddings.cc") + return te.CachedTextEncoder(store, identity, child) + + +class PlanPlacement(unittest.TestCase): + def test_refuses_below_two_gib_with_a_typed_error_and_no_cpu_placement(self): + for free in (0, 1 * GIB, int(1.5 * GIB), 2 * GIB - 1): + with self.assertRaises(te.GpuBusy) as raised: + te.plan_placement(free, 64 * GIB) + self.assertEqual(raised.exception.code, "gpu-busy") + self.assertEqual(raised.exception.free_vram, free) + + def test_splits_free_vram_and_ram_each_less_the_reserve(self): + self.assertEqual(te.plan_placement(12 * GIB, 7 * GIB), {0: 12 * GIB - te.GPU_RESERVE, "cpu": 7 * GIB - te.CPU_RESERVE}) + self.assertEqual(te.plan_placement(14 * GIB, 10 * GIB), {0: 14 * GIB - te.GPU_RESERVE, "cpu": 10 * GIB - te.CPU_RESERVE}) + + def test_the_gpu_reserve_exceeds_the_old_one_that_crashed_at_the_card_ceiling(self): + self.assertGreater(te.GPU_RESERVE, int(1.5 * GIB)) + + def test_ram_below_the_reserve_never_goes_negative(self): + self.assertEqual(te.plan_placement(12 * GIB, GIB)["cpu"], 0) + + +class Identity(unittest.TestCase): + def test_key_changes_with_identity_and_text_and_is_stable(self): + self.assertEqual(te.embedding_key("a", "x"), te.embedding_key("a", "x")) + self.assertNotEqual(te.embedding_key("a", "x"), te.embedding_key("b", "x")) + self.assertNotEqual(te.embedding_key("a", "x"), te.embedding_key("a", "y")) + self.assertNotEqual(te.embedding_key("ab", "x"), te.embedding_key("a", "bx")) + + def test_identity_names_every_input(self): + base = dict(revisions={"r/one": "111", "r/two": "222"}, dtype="bfloat16", transformers_version="5.1.0", peft_version="0.18.1", kimodo_commit="6bb5848") + identity = te.encoder_identity(**base) + for field, other in (("dtype", "float32"), ("transformers_version", "5.2.0"), ("peft_version", "0.19.0"), ("kimodo_commit", "abcdef0"), + ("revisions", {"r/one": "111", "r/two": "333"})): + self.assertNotEqual(identity, te.encoder_identity(**{**base, field: other}), field) + + def test_snapshot_revision_is_read_from_the_mounted_cache(self): + with tempfile.TemporaryDirectory() as home: + refs = Path(home) / "hub" / "models--org--name" / "refs" + refs.mkdir(parents=True) + (refs / "main").write_text("deadbeef\n") + self.assertEqual(te.read_snapshot_revision(Path(home), "org/name"), "deadbeef") + + +class CachedEncoder(unittest.TestCase): + def test_one_child_call_for_a_requests_misses_and_none_for_hits(self): + with tempfile.TemporaryDirectory() as directory: + child = FakeChild() + encoder = open_encoder(directory, child) + features, lengths = encoder(["aa", "bbb", "aa", ""]) + self.assertEqual(child.calls, [["aa", "bbb", ""]]) + self.assertEqual(tuple(features.shape), (4, 1, te.EMBEDDING_DIM)) + self.assertEqual(lengths, [1, 1, 1, 1]) + self.assertEqual((encoder.hits, encoder.misses), (0, 3)) + encoder.reset() + again, _ = encoder(["bbb", "aa"]) + self.assertEqual(len(child.calls), 1) + self.assertEqual((encoder.hits, encoder.misses, encoder.encode_seconds), (2, 0, 0.0)) + self.assertTrue((again[0] == features[1]).all() and (again[1] == features[0]).all()) + + def test_a_new_encoder_on_the_same_file_hits(self): + with tempfile.TemporaryDirectory() as directory: + open_encoder(directory, FakeChild())(["aa"]) + second = FakeChild() + encoder = open_encoder(directory, second) + encoder(["aa"]) + self.assertEqual(second.calls, []) + self.assertEqual(encoder.count(), 1) + + def test_a_different_identity_misses(self): + with tempfile.TemporaryDirectory() as directory: + open_encoder(directory, FakeChild(), "id-a")(["aa"]) + other = FakeChild() + open_encoder(directory, other, "id-b")(["aa"]) + self.assertEqual(other.calls, [["aa"]]) + + def test_same_text_yields_the_same_vector_hash_every_time(self): + with tempfile.TemporaryDirectory() as directory: + encoder = open_encoder(directory, FakeChild()) + encoder(["aa"]) + encoder(["aa"]) + self.assertEqual(len(encoder.embeddings), 2) + self.assertEqual(len({row["vectorSha256"] for row in encoder.embeddings}), 1) + + def test_a_wrong_sized_vector_is_refused_and_not_stored(self): + with tempfile.TemporaryDirectory() as directory: + encoder = open_encoder(directory, lambda texts: {"vectors": [b"short"] * len(texts), "placement": {}}) + with self.assertRaises(RuntimeError) as raised: + encoder([SECRET]) + self.assertNotIn(SECRET, str(raised.exception)) + self.assertEqual(encoder.count(), 0) + + def test_gpu_busy_propagates_without_a_cpu_path_and_stores_nothing(self): + def busy(texts): + raise te.GpuBusy(GIB) + + with tempfile.TemporaryDirectory() as directory: + encoder = open_encoder(directory, busy) + with self.assertRaises(te.GpuBusy) as raised: + encoder([SECRET]) + self.assertNotIn(SECRET, str(raised.exception)) + self.assertEqual(encoder.count(), 0) + + def test_the_contract_methods_return_self_or_the_device(self): + with tempfile.TemporaryDirectory() as directory: + encoder = open_encoder(directory, FakeChild()) + self.assertIs(encoder.to("cpu"), encoder) + self.assertIs(encoder.eval(), encoder) + self.assertEqual(encoder.get_device(), "cpu") + + +class ChildBoundary(unittest.TestCase): + def run_stub(self, body, texts=("aa",)): + with tempfile.TemporaryDirectory() as directory: + stub = Path(directory) / "child.py" + stub.write_text(textwrap.dedent(body)) + return te.run_child(list(texts), Path(directory) / "out.mc", child_path=stub, timeout=60) + + def test_exit_three_is_gpu_busy_with_the_measured_free_vram(self): + with self.assertRaises(te.GpuBusy) as raised: + self.run_stub(""" + import sys, msgpack + out = sys.argv[sys.argv.index("--out") + 1] + open(out, "wb").write(msgpack.packb({"error": "gpu-busy", "freeVramBytes": 1234}, use_bin_type=True)) + sys.exit(3) + """) + self.assertEqual(raised.exception.free_vram, 1234) + + def test_other_failures_name_the_exit_code_and_never_the_text(self): + with self.assertRaises(RuntimeError) as raised: + self.run_stub("import sys; sys.exit(1)", texts=(SECRET,)) + self.assertIn("exit code 1", str(raised.exception)) + self.assertNotIn(SECRET, str(raised.exception)) + + def test_success_returns_the_answer_file_and_the_child_gets_the_texts_on_stdin(self): + answer = self.run_stub(""" + import json, sys, msgpack + texts = json.load(sys.stdin)["texts"] + out = sys.argv[sys.argv.index("--out") + 1] + open(out, "wb").write(msgpack.packb({"vectors": [b"x" * 16384 for _ in texts], "texts_seen": texts}, use_bin_type=True)) + """, texts=("aa", "bb")) + self.assertEqual(answer["texts_seen"], ["aa", "bb"]) + self.assertEqual(len(answer["vectors"]), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/text_encoding.py b/scripts/text_encoding.py new file mode 100644 index 0000000..668e529 --- /dev/null +++ b/scripts/text_encoding.py @@ -0,0 +1,203 @@ +"""Text-embedding cache and transient-encoder contract for Vili's resident Kimodo worker. + +The worker never holds the 8B text encoder. CachedTextEncoder answers Kimodo's +encoder call from a CultCache store of float32 vectors and sends only the +missing texts to kimodo-text-encode.py, a one-shot child that loads the encoder, +encodes, writes its answer to a file and exits. Nothing here logs or raises with +a prompt in it: an error names a count, a size or a rule, never a text. +""" +import hashlib +import json +import subprocess +import sys +import tempfile +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path + +import msgpack +import torch + +from cultcache_py import CultCache, SingleFileMessagePackBackingStore, define_database_entry_type, database_entry_field, leaves as t + +GIB = 1024 ** 3 +MIN_FREE_VRAM = 2 * GIB +GPU_RESERVE = int(3 * GIB) # headroom for activations and offloaded layers streamed through the card +CPU_RESERVE = int(1.5 * GIB) +EMBEDDING_DIM = 4096 +EMBEDDING_BYTES = EMBEDDING_DIM * 4 +CHILD_EXIT_GPU_BUSY = 3 +CHILD_PATH = Path(__file__).with_name("kimodo-text-encode.py") +LLAMA_BASE = "meta-llama/Meta-Llama-3-8B-Instruct" # the base model the mntp adapter repo points at + + +class GpuBusy(RuntimeError): + """The card has too little free VRAM to host the encoder; there is no CPU fallback.""" + + code = "gpu-busy" + + def __init__(self, free_vram: int) -> None: + self.free_vram = free_vram + super().__init__(f"gpu-busy: {free_vram / GIB:.2f} GiB of VRAM free, {MIN_FREE_VRAM / GIB:.0f} GiB needed") + + +def plan_placement(free_vram: int, mem_available: int) -> dict: + """The accelerate max_memory for one child call: free VRAM and available RAM, each less its reserve.""" + if free_vram < MIN_FREE_VRAM: + raise GpuBusy(free_vram) + return {0: free_vram - GPU_RESERVE, "cpu": max(mem_available - CPU_RESERVE, 0)} + + +def read_snapshot_revision(hf_home: Path, repo_id: str) -> str: + """The commit the mounted Hugging Face cache resolves a repo's main ref to.""" + ref = Path(hf_home) / "hub" / ("models--" + repo_id.replace("/", "--")) / "refs" / "main" + return ref.read_text().strip() + + +def encoder_identity(*, revisions: dict, dtype: str, transformers_version: str, peft_version: str, kimodo_commit: str) -> str: + """The canonical string a vector is keyed under: every input that can change it.""" + parts = [f"{repo}@{revision}" for repo, revision in sorted(revisions.items())] + parts += [f"dtype={dtype}", f"transformers={transformers_version}", f"peft={peft_version}", f"kimodo={kimodo_commit}"] + return ";".join(parts) + + +def embedding_key(identity: str, text: str) -> str: + return hashlib.sha256((identity + "\n" + text).encode("utf-8")).hexdigest() + + +@dataclass +class TextEmbedding: + encoderIdentity: str + text: str + dtype: str + dim: int + vector: bytes + vectorSha256: str + gpuBytes: int + cpuBytes: int + encodeSeconds: float + createdAt: str + + +TEXT_EMBEDDING = define_database_entry_type( + "gamecult.vili.text_embedding.v0", + [ + database_entry_field("encoderIdentity", 0, codec=t.string), + database_entry_field("text", 1, codec=t.string), + database_entry_field("dtype", 2, codec=t.string), + database_entry_field("dim", 3, codec=t.int32), + database_entry_field("vector", 4, codec=t.bytes), + database_entry_field("vectorSha256", 5, codec=t.string), + database_entry_field("gpuBytes", 6, codec=t.int64), + database_entry_field("cpuBytes", 7, codec=t.int64), + database_entry_field("encodeSeconds", 8, codec=t.float64), + database_entry_field("createdAt", 9, codec=t.string), + ], + cls=TextEmbedding, +) + + +def open_store(path) -> CultCache: + cache = ( + CultCache.builder() + .register_document_type(TEXT_EMBEDDING) + .add_generic_store(SingleFileMessagePackBackingStore(path)) + .build() + ) + cache.pull_all_backing_stores() + return cache + + +def run_child(texts: list, out_path, child_path=CHILD_PATH, timeout: float = 1800.0) -> dict: + """Run the one-shot encoder child for `texts` and return its answer file. Raises GpuBusy or RuntimeError.""" + process = subprocess.run( + [sys.executable, str(child_path), "--out", str(out_path)], + input=json.dumps({"texts": texts}).encode("utf-8"), + stdout=subprocess.DEVNULL, + timeout=timeout, + ) + answer = msgpack.unpackb(Path(out_path).read_bytes(), raw=False) if Path(out_path).exists() else {} + if process.returncode == CHILD_EXIT_GPU_BUSY: + raise GpuBusy(int(answer.get("freeVramBytes", 0))) + if process.returncode != 0: + raise RuntimeError(f"text encoder child failed with exit code {process.returncode}") + return answer + + +def encode_misses_in_child(texts: list) -> dict: + with tempfile.TemporaryDirectory() as directory: + return run_child(texts, Path(directory) / "answer.mp") + + +class CachedTextEncoder: + """Kimodo's encoder contract, answered from the store; misses go to one child call per request.""" + + def __init__(self, store: CultCache, identity: str, encode_misses, device="cpu", dtype_name="float32") -> None: + self.store = store + self.identity = identity + self._encode_misses = encode_misses + self._device = device + self._dtype_name = dtype_name + self.reset() + + def reset(self) -> None: + self.hits = 0 + self.misses = 0 + self.encode_seconds = 0.0 + self.embeddings = [] + + def to(self, device): + self._device = device + return self + + def eval(self): + return self + + def get_device(self): + return self._device + + def count(self) -> int: + return len(self.store.get_all(TEXT_EMBEDDING)) + + def __call__(self, texts): + single = isinstance(texts, str) + texts = [texts] if single else list(texts) + keys = [embedding_key(self.identity, text) for text in texts] + present = [self.store.get(TEXT_EMBEDDING, key) is not None for key in keys] + missing = list(dict.fromkeys(text for text, found in zip(texts, present) if not found)) + self.hits += sum(present) + self.misses += len(missing) + if missing: + started = time.perf_counter() + answer = self._encode_misses(missing) + self.encode_seconds += time.perf_counter() - started + self._admit(missing, answer) + rows = [] + for text, key in zip(texts, keys): + record = self.store.get_required(TEXT_EMBEDDING, key) + self.embeddings.append({"text": text, "vectorSha256": record.vectorSha256}) + rows.append(torch.frombuffer(bytearray(record.vector), dtype=torch.float32)) + features = torch.stack(rows)[:, None].to(self._device) + return features, [1] * len(texts) + + def _admit(self, missing, answer) -> None: + vectors = answer.get("vectors", []) + if len(vectors) != len(missing) or any(len(v) != EMBEDDING_BYTES for v in vectors): + raise RuntimeError(f"text encoder child returned {len(vectors)} vectors for {len(missing)} texts, each {EMBEDDING_BYTES} bytes expected") + placement = answer.get("placement", {}) + seconds = answer.get("encodeSeconds", [0.0] * len(missing)) + for text, vector, took in zip(missing, vectors, seconds): + vector = bytes(vector) + self.store.put(TEXT_EMBEDDING, embedding_key(self.identity, text), TextEmbedding( + encoderIdentity=self.identity, + text=text, + dtype="float32", + dim=EMBEDDING_DIM, + vector=vector, + vectorSha256=hashlib.sha256(vector).hexdigest(), + gpuBytes=int(placement.get("gpuBytes", 0)), + cpuBytes=int(placement.get("cpuBytes", 0)), + encodeSeconds=float(took), + createdAt=datetime.now(timezone.utc).isoformat(), + ))