From 02ca1cd6d8d4c2aa37b45173c20374b7cf8c1915 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 22 Oct 2026 21:40:42 +0200 Subject: [PATCH] Vili README and tests: describe the embedding cache and the transient encoder Replaces the 2026-06-12 timing paragraph with the cache, the child, and the numbers measured on Raven on 2026-10-22. Adds the tests hand mutation showed missing: the exact 2 GiB boundary, a child answer with too few vectors, and the 4096 dimension. Co-Authored-By: Claude Opus 5.5 --- README.md | 30 ++++++++++++++++++++++++------ scripts/test_text_encoding.py | 19 +++++++++++++++++++ 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index da6a095..33920af 100644 --- a/README.md +++ b/README.md @@ -140,9 +140,27 @@ substrate does not disappear underneath the model. `GET /health` observes worker state without launching model load. `GET /smoke` verifies the backend and starts the resident worker when it is absent. -Verified on `2026-06-12`: the resident worker loaded -`Kimodo-SOMA-RP-v1.1` on Raven's RTX 4060 Ti in about 100 seconds. Five warmed -service-path generations of a 1 second clip at 20 diffusion steps averaged -about 54.4 seconds each, roughly `0.0184x` realtime. The old per-request -`docker run --rm kimodo_gen` path measured about 195 seconds per 1 second clip -and is no longer the generation authority. +Kimodo's 8B text encoder (LLM2Vec, Llama 3) never lives in the worker. The +worker answers Kimodo's encoder call from a CultCache store of float32 vectors, +`.vili/embeddings/text-embeddings.cc` (`gamecult.vili.text_embedding.v0`, written +through the cultcache-py checkout mounted from `E:\Projects\CultLib`). A vector's +key is `sha256(encoderIdentity + newline + text)`, and the identity names the three +Hugging Face snapshot revisions, bfloat16, the transformers and peft versions and +the Kimodo commit, so a changed input misses instead of serving a stale vector. +Kimodo normalizes the prompt before encoding (`A person shrugs sadly.`), and the +cache keys on that text. A request's misses go to one transient child, +`scripts/kimodo-text-encode.py`: it loads the encoder in bf16 on the GPU, with the +layers that do not fit in free VRAM held in RAM, encodes, writes its answer and +exits, so the OS reclaims its RAM and VRAM. It refuses with a typed `gpu-busy` +below 2 GiB of free VRAM and has no CPU fallback; hits never touch it. Each +`/motion/generate` job carries the encoder record (hits, misses, seconds, vector +hashes) and the worker identity. `GET /health` shows the identity and the stored +count. + +Measured on Raven on `2026-10-22`, for a 2 second clip at 20 diffusion steps: +the worker is healthy 29 seconds after `docker rm` (was 179) and rests at 2.3 GiB +resident (was 10.4 GiB, with 5.8 GB swapped). A cache miss takes 59 to 103 +seconds, the encode being nearly all of it (the 103 was with the model files cold +in the page cache); the CPU encoder took 316 to 376. A hit takes under one +second. The child peaks at 14.1 GiB of VRAM, so a miss needs almost the whole +16 GB card. diff --git a/scripts/test_text_encoding.py b/scripts/test_text_encoding.py index 0f4ed53..65aa1e3 100644 --- a/scripts/test_text_encoding.py +++ b/scripts/test_text_encoding.py @@ -51,6 +51,9 @@ class PlanPlacement(unittest.TestCase): def test_the_gpu_reserve_exceeds_the_old_one_that_crashed_at_the_card_ceiling(self): self.assertGreater(te.GPU_RESERVE, int(1.5 * GIB)) + def test_exactly_two_gib_free_is_admitted(self): + self.assertEqual(te.plan_placement(2 * GIB, 10 * GIB)[0], 2 * GIB - te.GPU_RESERVE) + def test_ram_below_the_reserve_never_goes_negative(self): self.assertEqual(te.plan_placement(12 * GIB, GIB)["cpu"], 0) @@ -125,6 +128,22 @@ class CachedEncoder(unittest.TestCase): self.assertNotIn(SECRET, str(raised.exception)) self.assertEqual(encoder.count(), 0) + def test_a_child_answer_with_too_few_vectors_is_refused_and_stores_nothing(self): + with tempfile.TemporaryDirectory() as directory: + encoder = open_encoder(directory, lambda texts: {"vectors": [vector(1)], "placement": {}}) + with self.assertRaises(RuntimeError): + encoder(["aa", "bb"]) + self.assertEqual(encoder.count(), 0) + + def test_kimodos_text_dimension_is_4096(self): + self.assertEqual(te.EMBEDDING_DIM, 4096) + self.assertEqual(te.EMBEDDING_BYTES, 4096 * 4) + with tempfile.TemporaryDirectory() as directory: + encoder = open_encoder(directory, FakeChild()) + features, _ = encoder(["aa"]) + self.assertEqual(tuple(features.shape), (1, 1, 4096)) + self.assertEqual(encoder.store.get_all(te.TEXT_EMBEDDING)[0].dim, 4096) + def test_gpu_busy_propagates_without_a_cpu_path_and_stores_nothing(self): def busy(texts): raise te.GpuBusy(GIB)