GameCult/Vili
Watch
9
0
Fork
You've already forked Vili
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Encoder B loads from snapshot directories, as the reference vector did

Browse source 
Measured on Yggdrasil: loading by repo id leaves the model's _name_or_path
equal to the Llama repo id, so prepare_for_tokenization wraps the text in
the chat header and the vector matches assembly A (cosine 1.0, 0.909 to the
reference B). Loading from the snapshot paths reproduces the reference.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Meta Vens 2026-10-23 12:22:25 +02:00
commit ffa2c7bf58
3 changed files with 31 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

13
scripts/test_text_encoding.py
View file

@ -72,6 +72,17 @@ class Identity(unittest.TestCase):
self.assertEqual(te.read_snapshot_revision(Path(home), "org/name"), "deadbeef")
class SnapshotPath(unittest.TestCase):
def test_resolves_the_revision_the_mounted_cache_names(self):
with tempfile.TemporaryDirectory() as home:
repo = Path(home) / "hub" / "models--org--name"
(repo / "refs").mkdir(parents=True)
(repo / "refs" / "main").write_text("deadbeef
")
(repo / "snapshots" / "deadbeef").mkdir(parents=True)
self.assertEqual(te.snapshot_path(Path(home), "org/name"), repo / "snapshots" / "deadbeef")
class CachedEncoder(unittest.TestCase):
def test_one_child_call_for_a_requests_misses_and_none_for_hits(self):
with tempfile.TemporaryDirectory() as directory:
@ -167,7 +178,7 @@ class ThreadCap(unittest.TestCase):
def test_the_child_applies_the_cap_before_it_loads_anything(self):
source = (Path(te.__file__).with_name("kimodo-text-encode.py")).read_text()
self.assertLess(source.index("apply_thread_cap(os.environ)"), source.index("load_encoder(preset"))
self.assertLess(source.index("apply_thread_cap(os.environ)"), source.index("load_encoder(os.environ"))
class ChildBoundary(unittest.TestCase):