Idunn recipe, release assembler and the hash locks

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

deployment/idunn/assemble-release.mjs

@@ -0,0 +1,52 @@
+// Assembles the node half of Vili's sealed release (run by the recipe's node-build runner):
+//   node deployment/idunn/assemble-release.mjs release/vili release/node
+// release/vili gets the daemon, the Idunn entry, the worker and child sources, the installed node_modules (installed
+// with --install-links, so cultcache-ts and cultnet-ts are real copies with their dependencies and no link reaches
+// into vendor/), cultcache-py's sources and the lock files the entry hashes; release/node gets the node binary.
+// It refuses to finish if any path it wrote is a symlink whose target is absolute: the seal would refuse the tree.
+import { chmodSync, copyFileSync, cpSync, lstatSync, mkdirSync, readdirSync, readlinkSync, rmSync } from "node:fs";
+import path from "node:path";
+
+const [viliOut, nodeOut] = process.argv.slice(2);
+if (!viliOut || !nodeOut || process.argv.length !== 4) {
+  console.error("usage: assemble-release.mjs <vili output dir> <node output dir>");
+  process.exit(2);
+}
+
+const copy = (from, to) => cpSync(from, to, { recursive: true, verbatimSymlinks: true, errorOnExist: true });
+
+rmSync(viliOut, { recursive: true, force: true });
+rmSync(nodeOut, { recursive: true, force: true });
+mkdirSync(path.join(viliOut, "scripts"), { recursive: true });
+mkdirSync(path.join(viliOut, "deployment", "idunn"), { recursive: true });
+mkdirSync(nodeOut, { recursive: true });
+
+// Runtime sources only: the tests and their fixtures stay out of the release.
+for (const entry of readdirSync("scripts", { withFileTypes: true })) {
+  if (!entry.isFile() || !/\.(mjs|py)$/u.test(entry.name) || /^test[-_]/u.test(entry.name)) continue;
+  copyFileSync(path.join("scripts", entry.name), path.join(viliOut, "scripts", entry.name));
+}
+copyFileSync("package.json", path.join(viliOut, "package.json"));
+copy("node_modules", path.join(viliOut, "node_modules"));
+copy(path.join("vendor", "CultLib", "packages", "cultcache-py", "src"), path.join(viliOut, "vendor", "CultLib", "packages", "cultcache-py", "src"));
+for (const lock of ["requirements-build.lock", "requirements.lock"]) {
+  copyFileSync(path.join("deployment", "idunn", lock), path.join(viliOut, "deployment", "idunn", lock));
+}
+copyFileSync(process.execPath, path.join(nodeOut, "node"));
+chmodSync(path.join(nodeOut, "node"), 0o755);
+
+function refuseAbsoluteLinks(directory) {
+  for (const entry of readdirSync(directory, { withFileTypes: true })) {
+    const full = path.join(directory, entry.name);
+    if (lstatSync(full).isSymbolicLink()) {
+      if (path.isAbsolute(readlinkSync(full))) {
+        console.error(`absolute symlink in the release: ${path.relative(process.cwd(), full)}`);
+        process.exit(1);
+      }
+    } else if (entry.isDirectory()) {
+      refuseAbsoluteLinks(full);
+    }
+  }
+}
+refuseAbsoluteLinks(viliOut);
+refuseAbsoluteLinks(nodeOut);

deployment/idunn/recipe.toml

@@ -0,0 +1,222 @@
+schema = "gamecult.idunn.target_declaration.v1"
+target = "vili"
+source_stamp_environment = "VILI_BUILD_COMMIT"
+required_gitlinks = ["vendor/CultLib", "vendor/kimodo", "vendor/SOMA-X"]
+
+# The one external input: a relocatable CPython. The locked site-packages are installed into this tree, never into
+# a venv (a venv's bin/python is an absolute link the seal refuses and its pyvenv.cfg names the build workspace).
+[[external_inputs]]
+id = "cpython"
+url = "https://github.com/astral-sh/python-build-standalone/releases/download/20261021/cpython-3.12.15+20261021-x86_64-unknown-linux-gnu-install_only.tar.gz"
+sha256 = "5ff9f0602ce964a529ed1e8555d3abab6341926d724402ba9a15e9e877191124"
+runner = "python-build"
+destination = "release/cpython.tar.gz"
+
+# ---- node-build: the daemon, the Idunn entry and its dependencies
+
+[[steps]]
+id = "install-cultlib-build-dependencies"
+phase = "prepare"
+runner = "node-build"
+working_directory = "vendor/CultLib"
+required_environment = ["HOME", "NPM_CONFIG_CACHE"]
+argv = ["npm", "ci", "--ignore-scripts"]
+
+[[steps]]
+id = "build-cultcache-ts"
+phase = "prepare"
+runner = "node-build"
+working_directory = "vendor/CultLib"
+required_environment = ["HOME", "NPM_CONFIG_CACHE"]
+argv = ["npm", "run", "build", "--workspace=packages/cultcache-ts"]
+
+[[steps]]
+id = "build-cultnet-ts"
+phase = "prepare"
+runner = "node-build"
+working_directory = "vendor/CultLib"
+required_environment = ["HOME", "NPM_CONFIG_CACHE"]
+argv = ["npm", "run", "build", "--workspace=packages/cultnet-ts"]
+
+[[steps]]
+id = "install-vili-dependencies"
+phase = "prepare"
+runner = "node-build"
+required_environment = ["HOME", "NPM_CONFIG_CACHE"]
+argv = ["npm", "ci", "--ignore-scripts", "--install-links"]
+
+[[steps]]
+id = "test-daemon"
+phase = "test"
+runner = "node-build"
+required_environment = ["HOME", "NPM_CONFIG_CACHE"]
+argv = ["node", "--test", "--test-timeout=60000", "scripts/test-vili-daemon.mjs"]
+
+[[steps]]
+id = "assemble-vili-release"
+phase = "build"
+runner = "node-build"
+required_environment = ["HOME", "NPM_CONFIG_CACHE"]
+argv = ["node", "deployment/idunn/assemble-release.mjs", "release/vili", "release/node"]
+
+# ---- python-build: the interpreter and its locked packages
+
+[[steps]]
+id = "unpack-cpython"
+phase = "prepare"
+runner = "python-build"
+argv = ["tar", "-xzf", "release/cpython.tar.gz", "-C", "release"]
+
+[[steps]]
+id = "install-build-tools"
+phase = "prepare"
+runner = "python-build"
+required_environment = ["HOME", "PIP_CACHE_DIR", "TMPDIR"]
+argv = ["release/python/bin/python3.12", "-m", "pip", "install", "--no-deps", "--require-hashes", "-r", "deployment/idunn/requirements-build.lock"]
+
+[[steps]]
+id = "install-locked-packages"
+phase = "prepare"
+runner = "python-build"
+required_environment = ["HOME", "PIP_CACHE_DIR", "TMPDIR"]
+argv = ["release/python/bin/python3.12", "-m", "pip", "install", "--no-deps", "--require-hashes", "--no-build-isolation", "-r", "deployment/idunn/requirements.lock"]
+
+[[steps]]
+id = "install-vendored-packages"
+phase = "prepare"
+runner = "python-build"
+required_environment = ["HOME", "PIP_CACHE_DIR", "TMPDIR"]
+argv = ["release/python/bin/python3.12", "-m", "pip", "install", "--no-deps", "--no-build-isolation", "--no-index", "vendor/SOMA-X", "vendor/kimodo/MotionCorrection", "vendor/kimodo"]
+
+[[steps]]
+id = "pip-check"
+phase = "test"
+runner = "python-build"
+required_environment = ["HOME", "PIP_CACHE_DIR", "TMPDIR"]
+argv = ["release/python/bin/python3.12", "-m", "pip", "check"]
+
+[[steps]]
+id = "test-worker"
+phase = "test"
+runner = "python-build"
+working_directory = "."
+required_environment = ["HOME", "TMPDIR", "PYTHONPATH"]
+argv = ["release/python/bin/python3.12", "-m", "unittest", "discover", "-s", "scripts", "-p", "test_*.py"]
+
+# Every module the worker and the encoder child import at top level, kimodo.scripts.generate included.
+[[steps]]
+id = "import-smoke"
+phase = "acceptance"
+runner = "python-build"
+required_environment = ["HOME", "TMPDIR", "PYTHONPATH"]
+argv = [
+  "release/python/bin/python3.12", "-c",
+  "import accelerate, cultcache_py, huggingface_hub.constants, kimodo, kimodo.constraints, kimodo.exports.motion_io, kimodo.meta, kimodo.model.llm2vec.llm2vec, kimodo.model.load_model, kimodo.model.loading, kimodo.model.registry, kimodo.scripts.generate, kimodo.tools, msgpack, peft, torch, transformers",
+]
+
+[[steps]]
+id = "compile-python"
+phase = "build"
+runner = "python-build"
+argv = ["release/python/bin/python3.12", "-m", "compileall", "-q", "-j", "4", "--invalidation-mode", "unchecked-hash", "release/python/lib/python3.12"]
+
+# ---- the sealed release: node, vili, python
+
+[[artifacts]]
+id = "node"
+source_kind = "runner-output"
+runner = "node-build"
+source = "release/node/node"
+destination = "node"
+executable = true
+
+[[artifacts]]
+id = "vili"
+source_kind = "runner-output"
+runner = "node-build"
+source = "release/vili"
+destination = "vili"
+
+[[artifacts]]
+id = "python"
+source_kind = "runner-output"
+runner = "python-build"
+source = "release/python"
+destination = "python"
+executable_members = ["bin/python3.12"]
+
+[service]
+executable_artifact = "node"
+required_adjacent_artifacts = ["vili", "python"]
+arguments = [
+  { kind = "literal", value = "vili/scripts/idunn-entry.mjs" },
+  { kind = "literal", value = "--state-root" },
+  { kind = "binding", name = "state_root" },
+  { kind = "literal", value = "--kimodo-commit" },
+  { kind = "literal", value = "6bb58488037dd65360ff0c5d1692b403a23309f7" },
+]
+transport = "http"
+route_required = true
+required_environment = [
+  "GAMECULT_IDUNN_CANDIDATE_BIND",
+  "GAMECULT_IDUNN_PROCESS_WRITE_LEASE",
+  "GAMECULT_IDUNN_RUNTIME_BUNDLE",
+  "VILI_MODELS_ROOT",
+  "VILI_THREADS",
+]
+
+[service.health]
+contract = "gamecult.runtime_presence_health.v2"
+
+[state]
+schema_generation = "vili-v1"
+
+[[state.slots]]
+id = "service-store"
+relative_path = "vili.service.cc"
+kind = "cultcache-file"
+schema = "gamecult.vili.operator_state.v1"
+writer = "process-bound-single-writer"
+recovery = "rebuildable"
+startup = "create-or-open-after-write-lease"
+
+[[state.slots]]
+id = "text-embeddings"
+relative_path = "embeddings/text-embeddings.cc"
+kind = "cultcache-file"
+schema = "gamecult.vili.text_embedding.v0"
+writer = "process-bound-single-writer"
+recovery = "rebuildable"
+startup = "create-or-open-after-write-lease"
+
+[[state.slots]]
+id = "jobs"
+relative_path = "jobs"
+kind = "external-directory"
+schema = "gamecult.vili.motion_job.v0"
+writer = "process-bound-single-writer"
+recovery = "preserve"
+startup = "create-or-open-after-write-lease"
+
+[[state.slots]]
+id = "artifacts"
+relative_path = "artifacts"
+kind = "external-directory"
+schema = "gamecult.vili.motion_artifact.v0"
+writer = "process-bound-single-writer"
+recovery = "preserve"
+startup = "create-or-open-after-write-lease"
+
+[[state.slots]]
+id = "logs"
+relative_path = "logs"
+kind = "external-directory"
+schema = "text.log"
+writer = "process-bound-single-writer"
+recovery = "rebuildable"
+startup = "create-or-open-after-write-lease"
+
+[[provides]]
+capability = "vili.motion.api"
+schema = "http.v1"
+compatibility = "v1"

deployment/idunn/requirements-build.lock

@@ -0,0 +1,39 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile --generate-hashes --python-version 3.12 --python-platform x86_64-manylinux_2_28 requirements-build.in -o requirements-build.lock
+cmake==4.4.4 \
+    --hash=sha256:02e9599cba932b386dc5c7e29db5dbda0610d13263798ff1521298c18014357d \
+    --hash=sha256:3c3f40c4656ec1ad1b873dd502bb31b3bba75295ae66b2623e515f2931eddbcb \
+    --hash=sha256:45a5c5e058e1e456f8da99e8bb83c98df079f385afd0bbea222a7eab6fdc1a8f \
+    --hash=sha256:6b914a7d9492d6134d43d484500cacd3a70466f6e31fcaf5145cf87128c28114 \
+    --hash=sha256:7b4a9c0d8146691d91b094633e4cd4f0cc4cdb8e04db0e8d805056db064396e0 \
+    --hash=sha256:883962a72b1b3a16ff445427f24c8caca9797dd9e563bbe208cbcd89e877f9d3 \
+    --hash=sha256:959fdcb277c63180928fe2c388063aa26dcfa1968033f7904eed1948ed44283b \
+    --hash=sha256:9ca4d839ced82f92b69319f50a65addf3051fae5d34464d9ac5b0296aba6ccb3 \
+    --hash=sha256:a3b4d4a35ba69360e630d2ce33020f42a61e09af28fe81122697989d11be8b39 \
+    --hash=sha256:ad8e0a38b5707e27882701146bdfffecacd80e7703fa0fc36c77528518d545af \
+    --hash=sha256:b3029f586853e01ddf2824c1ff124a1c5284f8ae897994758e9c4eeb45903b58 \
+    --hash=sha256:b4ed2c1a8f9c90e37a1f3095f7ed94979ebf3f94d0e90e63d50f4e72d419773a \
+    --hash=sha256:c7def2c08d9d0d31bab239209946e45a97520518ae63c0401772206f28b81523 \
+    --hash=sha256:d25e1d6baad137ddf21e576bf5296cd941bdd0da85baedbf78f673261fd2fca3 \
+    --hash=sha256:e2d98233f16b5d6b9d4673a0a724ce801fc413811befb8f0b008cbffb7a2ad63 \
+    --hash=sha256:eb0ff21b4309828f8fc0f0542022768f24b15aa1af3284e16c1eb01cdbe7d3e5 \
+    --hash=sha256:ed50c732a82ac04f6e5606a0677ddfb49dda9ee130f9d4163c5df217e478417e \
+    --hash=sha256:f337d18e33b116cabae9a7a8ff423fc0bc3f389da0acefeb724e1397128dcc34 \
+    --hash=sha256:f773a0544c66370408f451acd487c2c643b0dfebb48c5d64f3638bc1bb820238
+    # via -r requirements-build.in
+packaging==26.3 \
+    --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \
+    --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c
+    # via wheel
+pybind11==3.1.0 \
+    --hash=sha256:a1cc06b524ab3edca51f8ad3895f9c4fa20b8b19283173dff4ae781449dc9639 \
+    --hash=sha256:b8488090f8acffbcb6b5d6a85571a6827a0a2981ffb75e5a0b27b87c4a6b7dd0
+    # via -r requirements-build.in
+setuptools==84.2.1 \
+    --hash=sha256:3ff8b3b5492a0aa66960f4419365a86b84e013070d4313e4d4a6894f5d64d6bc \
+    --hash=sha256:ca140b45d5325a9478f880e351484f62fef15f336462e449ca0c03c873d589c2
+    # via -r requirements-build.in
+wheel==0.48.0 \
+    --hash=sha256:3217dcc807155e45db462d7ef2431f5ddda0d7273b700d05a67b271ceb1287ab \
+    --hash=sha256:94800765601e9171bf5d58d066e640662842bcedcbab982b2c90787a2c987322
+    # via -r requirements-build.in