Emoji font script: pin Unicode emoji-test.txt, refuse outside the pinned container

The corpus now derives from emoji-test.txt (sha256 pinned), keeps single code points of any digit
count, and lists the emoji Twemoji lacks. The script runs only on linux, Python 3.13, image
python:3.13-slim and records those in the lock; identical pins with a different font exit 1 and
leave the committed font and lock untouched.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Meta Vens 2026-10-08 22:05:09 +02:00
commit 851476bc23

View file

@ -1,14 +1,19 @@
"""Build Assets/Fonts/Emoji/Twemoji.ttf from the pinned Twemoji release.
Run with the venv that has tools/emoji/requirements.txt installed:
py -3 tools/emoji/fetch_emoji_font.py [--cache DIR] [--jobs N]
The font reproduces ONLY inside the pinned Linux container (PINNED_IMAGE, Python PINNED_PYTHON); on any
other platform the script refuses to run and writes nothing. It does not run under `py -3` on Windows.
Run it on Yggdrasil (or any docker host) from the repository root:
docker run --rm -e EMOJI_FONT_IMAGE=python:3.13-slim -v "$PWD":/src -w /src python:3.13-slim sh -c \\
"pip install -r tools/emoji/requirements.txt && python tools/emoji/fetch_emoji_font.py [--jobs N]"
Downloads the jdecked/twemoji release source archive (sha256 pinned below), runs nanoemoji
(glyf_colr_0) over its SVGs, and writes the font, the licence text, the single-code-point corpus
the editor smoke reads, and emoji-font.lock. The font is the only artifact Unity consumes; nothing
here is hand-made art. A second run with the same pins must reproduce the font's sha256.
Downloads the jdecked/twemoji release source archive and Unicode's emoji-test.txt (both sha256 pinned
below), runs nanoemoji (glyf_colr_0) over the SVGs, and writes the font, the licence text, the smoke's
corpus (emoji-singles.txt, from emoji-test.txt), the list of emoji Twemoji lacks (emoji-unsupported.txt)
and emoji-font.lock. The font is the only artifact Unity consumes; nothing here is hand-made art.
A run whose font-determining pins match the lock must reproduce the font's sha256, or the script exits 1
and leaves the committed font and lock untouched; the lock never follows the machine.
"""
import argparse, hashlib, os, re, shutil, subprocess, sys, tarfile, tempfile, urllib.request
import argparse, hashlib, os, platform, re, shutil, subprocess, sys, tarfile, tempfile, urllib.request
from importlib import metadata
from pathlib import Path
@ -18,10 +23,20 @@ TWEMOJI_SHA256 = "a0855654b633045ae2337537e77f1bb4361162f7fcd910e613eaab1d6d9c5f
# 2026-06-01T22:21:37Z, the release's publication time: fixes head.created/modified for reproducibility.
SOURCE_DATE_EPOCH = "1780352497"
FAMILY = "Twemoji"
# Unicode's emoji-test.txt for the Emoji version Twemoji v17.0.3 targets: the smoke's corpus.
EMOJI_TEST_VERSION = "17.0"
EMOJI_TEST_URL = "https://unicode.org/Public/17.0.0/emoji/emoji-test.txt"
EMOJI_TEST_SHA256 = "1d8a944f88d7952f7ef7c5167fef3c67995bcae24543949710231b03a201acda"
# The one environment the font reproduces in. The docker command passes the image tag in; a container
# cannot read its own image name.
PINNED_PLATFORM = "linux"
PINNED_PYTHON = "3.13"
PINNED_IMAGE = "python:3.13-slim"
ROOT = Path(__file__).resolve().parents[2]
FONT_DIR = ROOT / "Assets" / "Fonts" / "Emoji"
SINGLES = ROOT / "tools" / "emoji" / "emoji-singles.txt"
UNSUPPORTED = ROOT / "tools" / "emoji" / "emoji-unsupported.txt"
def sha256(path):
@ -32,16 +47,36 @@ def sha256(path):
return h.hexdigest()
def download(cache):
archive = cache / f"twemoji-{TWEMOJI_VERSION}.tar.gz"
if not archive.exists():
def download(cache, url, name, pinned):
path = cache / name
if not path.exists():
cache.mkdir(parents=True, exist_ok=True)
urllib.request.urlretrieve(TWEMOJI_URL, archive)
got = sha256(archive)
if got != TWEMOJI_SHA256:
archive.unlink()
sys.exit(f"archive sha256 {got} != pinned {TWEMOJI_SHA256}")
return archive
urllib.request.urlretrieve(url, path)
got = sha256(path)
if got != pinned:
path.unlink()
sys.exit(f"{name} sha256 {got} != pinned {pinned}")
return path
def single_code_points(emoji_test):
"""Every emoji of the Emoji version that is one code point, bare or with its FE0F presentation
selector (so (c) and (r) are in), of any qualification status including components."""
found = set()
for line in emoji_test.read_text(encoding="utf-8").splitlines():
if not line or line.startswith("#"):
continue
points = [int(p, 16) for p in line.split(";", 1)[0].split() if int(p, 16) != 0xFE0F]
if len(points) == 1:
found.add(points[0])
return sorted(found)
def refuse_unless_pinned_environment():
here = (sys.platform, f"{sys.version_info.major}.{sys.version_info.minor}", os.environ.get("EMOJI_FONT_IMAGE"))
if here != (PINNED_PLATFORM, PINNED_PYTHON, PINNED_IMAGE):
sys.exit(f"refusing to run: the font reproduces only on {PINNED_PLATFORM}, Python {PINNED_PYTHON}, image "
f"{PINNED_IMAGE} (EMOJI_FONT_IMAGE); this is {here}. Nothing was written.")
def licence_text(root):
@ -66,8 +101,10 @@ def main():
ap.add_argument("--cache", type=Path, default=Path(tempfile.gettempdir()) / "aetheria-emoji-cache")
ap.add_argument("--jobs", type=int, default=max(1, min(4, (os.cpu_count() or 2) // 2)))
args = ap.parse_args()
refuse_unless_pinned_environment()
archive = download(args.cache)
archive = download(args.cache, TWEMOJI_URL, f"twemoji-{TWEMOJI_VERSION}.tar.gz", TWEMOJI_SHA256)
emoji_test = download(args.cache, EMOJI_TEST_URL, f"emoji-test-{EMOJI_TEST_VERSION}.txt", EMOJI_TEST_SHA256)
work = args.cache / f"build-{TWEMOJI_VERSION}"
if work.exists():
shutil.rmtree(work)
@ -90,31 +127,48 @@ def main():
subprocess.run(run, check=True, env=env, cwd=work)
subprocess.run(["ninja", "-j", str(args.jobs), "-C", str(build)], check=True, env=env)
built = build / f"{FAMILY}.ttf"
font_sha = sha256(built)
tools = {n: metadata.version(n) for n in ("nanoemoji", "fonttools", "picosvg", "ninja")}
font_pins = {
"source_sha256": TWEMOJI_SHA256, **{f"tool_{n}": v for n, v in tools.items()},
"source_date_epoch": SOURCE_DATE_EPOCH, "color_format": "glyf_colr_0",
"python_version": platform.python_version(), "platform": PINNED_PLATFORM, "container_image": PINNED_IMAGE,
}
lock_path = FONT_DIR / "emoji-font.lock"
old = dict(l.split("=", 1) for l in lock_path.read_text().splitlines() if "=" in l) if lock_path.exists() else {}
if old and all(old.get(k) == v for k, v in font_pins.items()):
if old.get("font_sha256") != font_sha:
sys.exit(f"NOT REPRODUCED: font_sha256 {font_sha} != lock {old.get('font_sha256')} with identical pins. "
"Nothing was written.")
print("reproduced: font_sha256 equals the lock")
# The smoke's corpus is Unicode's list of single-code-point emoji, not the supplier's file names;
# what Twemoji does not draw is reported, not hidden.
drawn = {int(p.stem, 16) for p in svgs if re.fullmatch(r"[0-9a-f]+", p.stem)}
singles = single_code_points(emoji_test)
unsupported = [cp for cp in singles if cp not in drawn]
FONT_DIR.mkdir(parents=True, exist_ok=True)
out = FONT_DIR / f"{FAMILY}.ttf"
shutil.copyfile(build / f"{FAMILY}.ttf", out)
shutil.copyfile(built, out)
(FONT_DIR / "LICENSE.txt").write_text(licence_text(src), encoding="utf-8", newline="\n")
SINGLES.write_text("".join(f"{cp:X}\n" for cp in singles), encoding="utf-8", newline="\n")
UNSUPPORTED.write_text("".join(f"{cp:X}\n" for cp in unsupported), encoding="utf-8", newline="\n")
print(f"Twemoji lacks {len(unsupported)} of {len(singles)} single-code-point emoji: "
+ " ".join(f"U+{cp:04X}" for cp in unsupported))
# The smoke's corpus: every single-code-point glyph the pinned release draws.
singles = sorted(p.stem for p in svgs if re.fullmatch(r"[0-9a-f]{4,6}", p.stem))
SINGLES.write_text("".join(f"{s}\n" for s in singles), encoding="utf-8", newline="\n")
tools = {n: metadata.version(n) for n in ("nanoemoji", "fonttools", "picosvg", "ninja")}
lock = [
f"source_url={TWEMOJI_URL}", f"source_version={TWEMOJI_VERSION}", f"source_sha256={TWEMOJI_SHA256}",
f"svg_count={len(svgs)}", f"single_codepoint_count={len(singles)}",
f"emoji_test_version={EMOJI_TEST_VERSION}", f"emoji_test_url={EMOJI_TEST_URL}", f"emoji_test_sha256={EMOJI_TEST_SHA256}",
f"svg_count={len(svgs)}", f"single_codepoint_count={len(singles)}", f"unsupported_count={len(unsupported)}",
*(f"tool_{n}={v}" for n, v in tools.items()),
f"source_date_epoch={SOURCE_DATE_EPOCH}", "color_format=glyf_colr_0",
f"font_file={out.name}", f"font_sha256={sha256(out)}", f"singles_sha256={sha256(SINGLES)}",
f"python_version={font_pins['python_version']}", f"platform={PINNED_PLATFORM}", f"container_image={PINNED_IMAGE}",
f"font_file={out.name}", f"font_sha256={font_sha}",
f"singles_sha256={sha256(SINGLES)}", f"unsupported_sha256={sha256(UNSUPPORTED)}",
]
previous = (FONT_DIR / "emoji-font.lock")
if previous.exists():
old = dict(l.split("=", 1) for l in previous.read_text().splitlines() if "=" in l)
if old.get("font_sha256") != sha256(out):
print(f"NOT REPRODUCED: font_sha256 {sha256(out)} != lock {old.get('font_sha256')}")
else:
print("reproduced: font_sha256 equals the lock")
previous.write_text("\n".join(lock) + "\n", encoding="utf-8", newline="\n")
lock_path.write_text("\n".join(lock) + "\n", encoding="utf-8", newline="\n")
print("\n".join(lock))