diff --git a/README.md b/README.md index 6a59398..477f91e 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,7 @@ Supported request fields: - `cfg`, `cfgType` / `cfg_type`, `cfgWeight` / `cfg_weight` - `bvh`, `bvhStandardTpose` / `bvh_standard_tpose` - `fbx` and `clipName` (see FBX Export) +- `loop` (a boolean, valid only with `fbx: true`; see FBX Export) - `noPostprocess` / `no_postprocess` - `saveExampleDir` / `save_example_dir` - `timeoutMs`, `workerTimeoutMs`, `exportTimeoutMs` (at most 120000) @@ -166,6 +167,18 @@ timeout, and its output in `/logs/export.log`. It is killed with the work SIGTERM. A failed export fails the job (`fbx-export-refused` or `fbx-export-failed (...)`, 502) and the answer never carries Blender's output. +`"loop": true` (with `fbx: true`; otherwise a 400, as is a non-boolean) exports the take's lowest-cost loop window instead +of the whole take. The worker chooses it (`scripts/loop_window.py`) from Kimodo's own posed joints: every joint relative +to its Hips and turned by its own heading (`compute_global_heading`), and the seam from frame `b` back to frame `a` +costs the pose RMS plus 0.1 s times the velocity RMS (central differences, in m and m/s), over `b - a >= ceil(F / 2)`. +The lowest cost wins; a tie goes to the longer window, then the earlier. The worker's answer carries `loops`, one +`{first, last, poseCost, velocityCost}` per sample (BVH frame indices, `last` inclusive), and the daemon hands +sample k's window to `bvh-to-fbx.py` as `--frames ` (Blender counts from 1); the daemon never +computes a window, and an answer without one for a sample fails the job (`fbx-export-failed (no loop window)`). +The BVH and NPZ stay the whole generated take. The job record keeps `request.loop`, `worker.loops` and the clip's +`frames` as Blender reports them, and keeps inline `constraints` as sent (a path otherwise), so a committed record +reproduces its clip. Without `loop` the FBX is the whole take, as before. + One BVH gives the same FBX bytes on one host body (a fixed hash seed and a fixed time stamp); FBX bytes differ between glibc versions in the last float digits, so the BVH is the cross-body reproducibility unit. diff --git a/scripts/bvh-to-fbx.py b/scripts/bvh-to-fbx.py index c15b5a5..3dad2c3 100644 --- a/scripts/bvh-to-fbx.py +++ b/scripts/bvh-to-fbx.py @@ -1,6 +1,9 @@ # Converts one Kimodo standard-T-pose BVH to one FBX. Blender converts formats and nothing else: no retarget, no -# rename, no key reduction, no resample, no root-motion bake. Run by the daemon (scripts/vili-daemon.mjs exportClip): -# blender -b --factory-startup --python-use-system-env --python-exit-code 1 --python bvh-to-fbx.py -- +# rename, no key reduction, no resample, no root-motion bake. It applies the frame range it is handed and chooses +# none: the whole take unless the daemon passes --frames. Run by the daemon (scripts/vili-daemon.mjs exportClip): +# blender -b --factory-startup --python-use-system-env --python-exit-code 1 --python bvh-to-fbx.py -- [--frames ] +# Frames are Blender's, counted from 1: the BVH's frame k is Blender's k + 1. Exit 3 is an argument or a hash seed it +# will not take, exit 4 a range outside the take. # with PYTHONHASHSEED=0 in the environment. The settings are Blender API parameters, cross-checked against the # 'unity' preset of wifimetas/kimodo-mcp blender_export.py (Apache-2.0, 83b5361); no code is copied. import datetime @@ -15,7 +18,9 @@ import bpy if sys.flags.hash_randomization != 0: sys.exit(3) -bvh, fbx, clip = sys.argv[sys.argv.index("--") + 1:] +bvh, fbx, clip, *window = sys.argv[sys.argv.index("--") + 1:] +if window and not (len(window) == 3 and window[0] == "--frames" and all(arg.isdigit() for arg in window[1:])): + sys.exit(3) bpy.ops.wm.read_factory_settings(use_empty=True) scene = bpy.context.scene @@ -38,6 +43,10 @@ action = armature.animation_data.action # The FBX take takes the scene's name. armature.name = armature.data.name = action.name = scene.name = clip first, last = (int(frame) for frame in action.frame_range) +if window: + first, last = int(window[1]), int(window[2]) + if not (int(action.frame_range[0]) <= first < last <= int(action.frame_range[1])): + sys.exit(4) scene.frame_start, scene.frame_end = first, last bpy.ops.object.select_all(action="DESELECT") diff --git a/scripts/fixtures/stub-worker.mjs b/scripts/fixtures/stub-worker.mjs index 1ad2ccc..e138201 100644 --- a/scripts/fixtures/stub-worker.mjs +++ b/scripts/fixtures/stub-worker.mjs @@ -1,7 +1,7 @@ // Stands in for kimodo-resident-worker.py in the daemon's tests. It binds port 0, prints the ready line the // daemon waits for, and answers /health and /generate with what the daemon gave it. The --model value picks a // misbehaviour: crash-before-ready, hang-before-ready, sick, slow-health, stubborn, fail-generate, -// refuse-generate, drop-generate, slow-generate, writer (appends to .writes every 50 ms, never answers, ignores SIGTERM), slow-to-stop (no answer; prints a line and exits 1.5 s after SIGTERM), chatty (prints a line the moment it is healthy), encoder (a grandchild, then no answer), graceful (exits 0.5 s after SIGTERM), die-after-ready, bvh-outside (lists a BVH outside the job's directory), bvh-link (lists a symlink as its BVH), bvh-fifo (lists a FIFO as its BVH), plant-link (a normal answer with a file symlink and a directory symlink planted beside the BVH). A request with bvh writes .bvh (motion/motion_NN.bvh for several samples) and lists it. Any other value is just the model the worker serves. +// refuse-generate, drop-generate, slow-generate, writer (appends to .writes every 50 ms, never answers, ignores SIGTERM), slow-to-stop (no answer; prints a line and exits 1.5 s after SIGTERM), chatty (prints a line the moment it is healthy), encoder (a grandchild, then no answer), graceful (exits 0.5 s after SIGTERM), die-after-ready, bvh-outside (lists a BVH outside the job's directory), bvh-link (lists a symlink as its BVH), bvh-fifo (lists a FIFO as its BVH), plant-link (a normal answer with a file symlink and a directory symlink planted beside the BVH), no-loops (a normal answer that leaves out the loops a loop request asked for), bad-loops, negative-loops, empty-loops, fractional-last-loops (loops whose window is not a frame range: a fractional first, a negative first, first equal to last, a fractional last), short-loops (one window fewer than samples). A loop request is answered with loops, sample k as {first 3 + k, last 223 + k}. A request with bvh writes .bvh (motion/motion_NN.bvh for several samples) and lists it. Any other value is just the model the worker serves. import http from "node:http"; import os from "node:os"; import path from "node:path"; @@ -14,6 +14,12 @@ const flag = (name) => { return index < 0 ? undefined : argv[index + 1]; }; const model = flag("--model"); +const BAD_WINDOWS = { + "bad-loops": { first: 5.5, last: 9 }, + "negative-loops": { first: -1, last: 9 }, + "empty-loops": { first: 7, last: 7 }, + "fractional-last-loops": { first: 5, last: 9.5 }, +}; const spawnLog = path.join(process.env.VILI_EMBEDDINGS_DIR, "stub-spawns.log"); let requests = 0; let healthCalls = 0; @@ -117,6 +123,10 @@ if (model === "hang-before-ready") { seed: payload.seed, metaSeed: payload.meta?.seed, bvh: payload.bvh, + ...(payload.loop && model !== "no-loops" ? { + loops: Array.from({ length: payload.numSamples - (model === "short-loops" ? 1 : 0) }, (_, k) => (BAD_WINDOWS[model] + ?? { first: 3 + k, last: 223 + k, poseCost: 0.03, velocityCost: 0.2 })), + } : {}), bvhStandardTpose: payload.bvhStandardTpose, model: model || "stub-model", pid: process.pid, diff --git a/scripts/fixtures/take.npz b/scripts/fixtures/take.npz new file mode 100644 index 0000000..d1dd66d Binary files /dev/null and b/scripts/fixtures/take.npz differ diff --git a/scripts/kimodo-resident-worker.py b/scripts/kimodo-resident-worker.py index e855a32..307f994 100644 --- a/scripts/kimodo-resident-worker.py +++ b/scripts/kimodo-resident-worker.py @@ -28,6 +28,7 @@ from kimodo.scripts.generate import ( ) from kimodo.tools import save_json, seed_everything +import loop_window import text_encoding @@ -319,6 +320,7 @@ def generate(payload): request_bool(payload, "bvh"), request_bool(payload, "bvhStandardTpose") or request_bool(payload, "bvh_standard_tpose"), ) + loops = loop_window.loop_windows(output, model) if request_bool(payload, "loop") else None if request_bool(payload, "saveExampleDir") or request_bool(payload, "save_example_dir"): files.extend(save_example_dirs(output_base, texts, num_frames, num_samples, constraints_path)) generation_count += 1 @@ -347,6 +349,7 @@ def generate(payload): "constraints": constraints_path, "seed": seed, "files": files, + **({"loops": loops} if loops is not None else {}), "encoder": encoder_record, "worker": worker_identity, "timing": { diff --git a/scripts/loop_window.py b/scripts/loop_window.py new file mode 100644 index 0000000..8d3c542 --- /dev/null +++ b/scripts/loop_window.py @@ -0,0 +1,60 @@ +"""The frame window of a generated take that loops with the smallest seam. Called by the resident worker, nowhere else. + +A seam is the jump from frame b back to frame a. Its cost is the pose difference plus 0.1 s times the velocity +difference (Kovar, Gleicher and Pighin 2002, motion-graph transitions), both as RMS over joints, with each frame +taken relative to its own Hips and turned to face a fixed way, so where the take wanders and which way it faces +do not count. loop_window is numpy only; loop_windows needs torch and kimodo's heading, and no model weights. +""" +import math + +import numpy as np +import torch +from kimodo.constraints import compute_global_heading + +VELOCITY_WEIGHT_SECONDS = 0.1 + + +def loop_window(joints, heading, fps, min_frames, hips=0): + """joints [F, J, 3] world positions, heading [F] radians about Y (Kimodo's hip-vector angle), the joint index of + Hips. Returns (first, last, pose_cost, velocity_cost): frame indices into the take, last inclusive, last - first + at least min_frames, the pair with the lowest cost. A tie goes to the longer window, then to the earlier first.""" + joints = np.asarray(joints, dtype=np.float64) + heading = np.asarray(heading, dtype=np.float64) + frames = joints.shape[0] + if joints.ndim != 3 or joints.shape[2] != 3 or heading.shape != (frames,) or not 1 <= min_frames < frames: + raise ValueError("loop_window: shapes or min_frames out of range") + relative = joints - joints[:, hips:hips + 1] + cos, sin = np.cos(-heading), np.sin(-heading) + x, z = relative[..., 0], relative[..., 2] + pose = relative.copy() + pose[..., 0] = x * cos[:, None] + z * sin[:, None] + pose[..., 2] = -x * sin[:, None] + z * cos[:, None] + velocity = np.gradient(pose, axis=0) * fps # central differences, one-sided at the two ends + + firsts, lasts = np.triu_indices(frames, k=min_frames) + pose_cost = np.empty(len(firsts)) + velocity_cost = np.empty(len(firsts)) + for first in range(frames - min_frames): # one row of seams at a time: memory stays O(frames * joints) + row = slice(*np.searchsorted(firsts, [first, first + 1])) + later = lasts[row] + pose_cost[row] = np.sqrt(((pose[later] - pose[first]) ** 2).sum(-1).mean(-1)) + velocity_cost[row] = np.sqrt(((velocity[later] - velocity[first]) ** 2).sum(-1).mean(-1)) + cost = pose_cost + VELOCITY_WEIGHT_SECONDS * velocity_cost + best = np.flatnonzero(cost == cost.min()) + pick = best[np.lexsort((firsts[best], -(lasts[best] - firsts[best])))[0]] + return int(firsts[pick]), int(lasts[pick]), float(pose_cost[pick]), float(velocity_cost[pick]) + + +def loop_windows(output, model): + """One loop window per sample of a Kimodo output, as the worker answers them: {first, last, poseCost, velocityCost}, + frame indices into the take, last inclusive. The joints are posed_joints, always in model.output_skeleton (the + somaskel77, not model.skeleton), and the heading is Kimodo's own from those joints.""" + skeleton = model.output_skeleton + hips = skeleton.bone_index["Hips"] + windows = [] + for joints in output["posed_joints"]: + heading = compute_global_heading(torch.from_numpy(joints[None]), skeleton)[0] + angle = torch.atan2(heading[:, 1], heading[:, 0]).numpy() + first, last, pose_cost, velocity_cost = loop_window(joints, angle, model.fps, math.ceil(len(joints) / 2), hips) + windows.append({"first": first, "last": last, "poseCost": pose_cost, "velocityCost": velocity_cost}) + return windows diff --git a/scripts/test-vili-daemon.mjs b/scripts/test-vili-daemon.mjs index 66f9819..b1fd271 100644 --- a/scripts/test-vili-daemon.mjs +++ b/scripts/test-vili-daemon.mjs @@ -1441,6 +1441,124 @@ test("fbx refusals happen before anything is written or spawned", async () => { } }); +test("loop asks for the seamless window, and the window is the FBX's frame range", async () => { + const daemon = await startDaemon(); + try { + const constraints = [{ type: "root2d", frame_indices: [0, 10], smooth_root_2d: [[0, 0], [0, 0.5]] }]; + const answer = await generate(daemon, exportRequest({ clipName: "walk", seed: 7, loop: true, constraints })); + assert.equal(answer.status, 200); + const job = answer.body; + assert.equal(job.request.loop, true); + assert.deepEqual(job.worker.loops, [{ first: 3, last: 223, poseCost: 0.03, velocityCost: 0.2 }]); + assert.deepEqual(job.request.constraints, constraints, "the record keeps inline constraints as sent"); + assert.deepEqual(JSON.parse(readFileSync(path.join(job.artifactDir, "constraints.json"), "utf8")), constraints); + assert.equal(JSON.parse(readFileSync(path.join(job.artifactDir, "request.json"), "utf8")).loop, true, "the worker is asked for the loop"); + assert.equal(job.worker.constraints, path.join(job.artifactDir, "constraints.json"), "the worker still gets the path"); + const [run] = exportStarts(daemon); + assert.deepEqual(run.argv.slice(run.argv.indexOf("--") + 3), ["walk", "--frames", "4", "224"], "Blender's frames count from 1"); + assert.deepEqual(job.export.clips.map((clip) => clip.frames), [[1, 60]], "the clip's frames are what Blender reports"); + + const several = await generate(daemon, exportRequest({ clipName: "pair", numSamples: 2, loop: true })); + assert.equal(several.status, 200); + assert.deepEqual(several.body.worker.loops.map((loop) => [loop.first, loop.last]), [[3, 223], [4, 224]]); + const frames = Object.fromEntries(exportStarts(daemon).slice(1).map((start) => { + const [, , clip, ...rest] = start.argv.slice(start.argv.indexOf("--") + 1); + return [clip, rest]; + })); + assert.deepEqual(frames, { pair_00: ["--frames", "4", "224"], pair_01: ["--frames", "5", "225"] }, "each sample is exported over its own window"); + + const plain = await generate(daemon, exportRequest({ clipName: "plain" })); + assert.equal(plain.status, 200); + assert.equal(plain.body.worker.loops, undefined); + assert.equal(plain.body.request.loop, undefined); + assert.equal(JSON.parse(readFileSync(path.join(plain.body.artifactDir, "request.json"), "utf8")).loop, undefined); + assert.equal(exportStarts(daemon).at(-1).argv.at(-1), "plain", "no window, no --frames"); + const kept = await generate(daemon, exportRequest({ constraints_path: "/some/where.json", constraints: undefined })); + assert.equal(kept.body.request.constraints, "/some/where.json", "a path stays a path"); + } finally { + await daemon.stop(); + } +}); + +test("past 100 samples each clip is exported over the window of its own sample", async () => { + const daemon = await startDaemon(); + try { + const answer = await generate(daemon, exportRequest({ clipName: "m", loop: true, numSamples: 101, exportTimeoutMs: 60000 })); + assert.equal(answer.status, 200); + const starts = exportStarts(daemon); + assert.equal(answer.body.export.clips.length, 101); + answer.body.export.clips.forEach((clip, i) => { + const sample = Number(/motion_(\d+)\.fbx$/.exec(clip.file)[1]); + const argv = starts[i].argv.slice(starts[i].argv.indexOf("--") + 1); + assert.equal(argv[2], `m_${String(sample).padStart(2, "0")}`, `${clip.file} is named for its sample`); + assert.deepEqual(argv.slice(3), ["--frames", String(4 + sample), String(224 + sample)], `${clip.file} gets its own sample's window`); + }); + assert.ok(answer.body.export.clips.some((clip) => clip.file.endsWith("motion_100.fbx"))); + } finally { + await daemon.stop(); + } +}); + +test("constraints given twice are refused before anything is written or spawned", async () => { + const daemon = await startDaemon(); + try { + const inline = [{ type: "root2d" }]; + for (const bad of [ + { constraints: inline, constraints_path: "/canary/a.json" }, + { constraints: inline, constraintsPath: "/canary/a.json" }, + { constraints: "/canary/a.json", constraints_path: "/canary/b.json" }, + { constraints_path: "/canary/a.json", constraintsPath: "/canary/b.json" }, + ]) { + const answer = await generate(daemon, exportRequest(bad)); + assert.equal(answer.status, 400, JSON.stringify(bad)); + assert.deepEqual(answer.body, { ok: false, error: "Invalid constraints request." }); + assert.ok(!answer.raw.includes("canary"), "the refusal repeated the input"); + } + assert.equal(existsSync(path.join(daemon.stateRoot, "artifacts")), false, "no artifact directory was made"); + assert.deepEqual(jobFiles(daemon), []); + assert.deepEqual(daemon.spawns(), [], "no worker was started"); + assert.equal((await generate(daemon, exportRequest({ constraints_path: "/some/where.json" }))).status, 200, "one input is fine"); + } finally { + await daemon.stop(); + } +}); + +test("loop is a boolean and needs fbx, refused before anything is written or spawned", async () => { + const daemon = await startDaemon(); + try { + for (const bad of [{ fbx: false, loop: true }, { fbx: undefined, loop: true }, { loop: "yes canary" }, { loop: 1 }, { loop: null }, { loop: "true" }]) { + const answer = await generate(daemon, exportRequest(bad)); + assert.equal(answer.status, 400, JSON.stringify(bad)); + assert.deepEqual(answer.body, { ok: false, error: "Invalid fbx request." }); + assert.ok(!answer.raw.includes("canary"), "the refusal repeated the input"); + } + assert.equal(existsSync(path.join(daemon.stateRoot, "artifacts")), false, "no artifact directory was made"); + assert.deepEqual(jobFiles(daemon), []); + assert.deepEqual(daemon.spawns(), [], "no worker was started"); + assert.equal(exportStarts(daemon).length, 0); + assert.equal((await generate(daemon, exportRequest({ loop: false }))).status, 200, "loop false is no loop"); + assert.equal((await generate(daemon, { prompt: "a person waves", duration: 2, loop: false })).status, 200); + } finally { + await daemon.stop(); + } +}); + +test("a loop request whose answer has no usable window for a sample fails without exporting that sample", async () => { + for (const [model, exported] of [["no-loops", 0], ["bad-loops", 0], ["negative-loops", 0], ["empty-loops", 0], ["fractional-last-loops", 0], ["short-loops", 1]]) { + const daemon = await startDaemon(); + try { + const answer = await generate(daemon, exportRequest({ model, loop: true, numSamples: 2 })); + assert.equal(answer.status, 502, model); + assert.equal(answer.body.ok, false); + assert.equal(answer.body.error, "fbx-export-failed (no loop window)"); + assert.equal(exportStarts(daemon).length, exported, `${model}: only a sample with a window is exported`); + assert.equal(answer.body.export.clips.length, exported); + } finally { + await daemon.stop(); + } + } +}); + test("the BVH is never given to Blender unless it is a regular file inside the job's directory", async () => { const daemon = await startDaemon(); try { diff --git a/scripts/test_bvh_to_fbx.py b/scripts/test_bvh_to_fbx.py index b8ea8e5..5ce081d 100644 --- a/scripts/test_bvh_to_fbx.py +++ b/scripts/test_bvh_to_fbx.py @@ -155,6 +155,21 @@ class BvhToFbx(unittest.TestCase): line = next(line for line in run.stdout.splitlines() if line.startswith("INSPECT ")) return json.loads(line[len("INSPECT "):]) + def walk(self, name, frames): + """A BVH of the rest pose's skeleton whose Hips travel one metre, accelerating, and whose Spine1 is turned.""" + text = self.rest.read_text() + header = text[:text.index("MOTION")] + channels = sum(int(n) for n in re.findall(r"CHANNELS\s+(\d+)", header)) + rows = [] + for frame in range(frames): + row = ["0"] * channels + row[8] = repr(100.0 * (frame / (frames - 1)) ** 2) # Hips Zposition, cm: one metre, accelerating + row[12], row[13], row[14] = "30", "20", "40" # Spine1 Z, Y, X rotation: the Euler order matters + rows.append(" ".join(row)) + bvh = self.dir / f"{name}.bvh" + bvh.write_text(f"{header}MOTION\nFrames: {frames}\nFrame Time: 0.0333333\n" + "\n".join(rows) + "\n") + return bvh + def test_rest_pose_keeps_the_bvh_joint_names_and_height(self): fbx = self.dir / "rest.fbx" run = convert(self.rest, fbx, "SOMA77_Rest") @@ -189,18 +204,8 @@ class BvhToFbx(unittest.TestCase): self.assertFalse(target.exists()) def test_a_clip_keeps_its_travel_timing_pose_and_name(self): - text = self.rest.read_text() - header = text[:text.index("MOTION")] - channels = sum(int(n) for n in re.findall(r"CHANNELS\s+(\d+)", header)) frames = 31 - rows = [] - for frame in range(frames): - row = ["0"] * channels - row[8] = repr(100.0 * (frame / (frames - 1)) ** 2) # Hips Zposition, cm: one metre, accelerating - row[12], row[13], row[14] = "30", "20", "40" # Spine1 Z, Y, X rotation: the Euler order matters - rows.append(" ".join(row)) - bvh = self.dir / "walk.bvh" - bvh.write_text(f"{header}MOTION\nFrames: {frames}\nFrame Time: 0.0333333\n" + "\n".join(rows) + "\n") + bvh = self.walk("walk", frames) fbx = self.dir / "walk.fbx" run = convert(bvh, fbx, "walk") self.assertEqual(run.returncode, 0, run.stderr[-2000:]) @@ -222,6 +227,32 @@ class BvhToFbx(unittest.TestCase): for got, want in zip(seen["head_from_hips"], seen["reference_head_from_hips"]): self.assertAlmostEqual(got, want, delta=0.002, msg="the rotated pose is Blender's own native reading of the BVH") + def test_frames_exports_that_range_and_nothing_else(self): + bvh = self.walk("long", 61) + fbx = self.dir / "window.fbx" + run = blender("--python", str(SCRIPT), "--", str(bvh), str(fbx), "window", "--frames", "5", "40") + self.assertEqual(run.returncode, 0, run.stderr[-2000:]) + self.assertEqual(result(run), {"bones": 78, "frames": [5, 40], "fps": 30}) + seen = self.inspect(fbx, bvh) + # The BVH's frame k is Blender's k + 1, so frames 5..40 are BVH rows 4..39 and Hips z is 100 (k/60)^2 cm. + self.assertAlmostEqual(seen["hips_travel"], (39 ** 2 - 4 ** 2) / 60 ** 2, delta=0.002, msg=json.dumps(seen)) + self.assertEqual(set(fbx_key_counts(fbx)), {36}, "one key per frame of the window") + whole = self.dir / "whole.fbx" + self.assertEqual(result(convert(bvh, whole, "whole"))["frames"], [1, 61], "without --frames the take is whole") + + def test_frames_outside_the_take_or_malformed_exit_without_a_file(self): + bvh = self.walk("short", 31) + for index, (arguments, exit_code) in enumerate(( + (("--frames", "0", "10"), 4), (("--frames", "5", "32"), 4), (("--frames", "10", "10"), 4), + (("--frames", "20", "10"), 4), + (("--frames", "5"), 3), (("--frames", "a", "9"), 3), (("--frames", "-1", "9"), 3), + (("--frames", "4", "20", "9"), 3), (("--range", "4", "20"), 3), (("4", "20"), 3), + )): + fbx = self.dir / f"refused{index}.fbx" + run = blender("--python", str(SCRIPT), "--", str(bvh), str(fbx), "refused", *arguments) + self.assertEqual(run.returncode, exit_code, arguments) + self.assertFalse(fbx.exists(), arguments) + if __name__ == "__main__": unittest.main() diff --git a/scripts/test_loop_window.py b/scripts/test_loop_window.py new file mode 100644 index 0000000..6065729 --- /dev/null +++ b/scripts/test_loop_window.py @@ -0,0 +1,189 @@ +"""python3 -m unittest scripts/test_loop_window.py (numpy, torch and kimodo; no model weights)""" +import math +import pathlib +import sys +import unittest + +import numpy as np + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) +from loop_window import loop_window, loop_windows # noqa: E402 +from types import SimpleNamespace # noqa: E402 + +FPS = 30 + + +def turn(points, angle): + """Rotate points [..., 3] about Y by angle, so that +Z turns toward +X (Kimodo's heading: 0 faces +Z).""" + x, z = points[..., 0], points[..., 2] + out = points.copy() + out[..., 0] = x * math.cos(angle) + z * math.sin(angle) + out[..., 2] = -x * math.sin(angle) + z * math.cos(angle) + return out + + +def stride(frames=240, period=37, start=40, end=200): + """A stride cycle between two different rests. Hips travels; every other joint swings.""" + rng = np.random.default_rng(3) + rest_a, rest_b = rng.normal(size=(2, 6, 3)) * 0.3 + phase, direction = rng.uniform(0, 6.28, 6), rng.normal(size=(6, 3)) + out = np.zeros((frames, 7, 3)) + for f in range(frames): + weight = min(1.0, f / start, (frames - 1 - f) / (frames - 1 - end)) + rest = rest_a if f < frames // 2 else rest_b + cycle = 0.2 * np.sin(2 * math.pi * f / period + phase)[:, None] * direction + out[f, 1:] = weight * (rest_a + cycle) + (1 - weight) * rest + out[f, 0] = (0.03 * f, 0.9, 0.0) # Hips + out[f, 1:] += out[f, 0] + return out + + +def reference(joints, heading, fps, min_frames): + """The cost written out frame by frame and joint by joint, to check the vectorised search against.""" + frames, count = joints.shape[0], joints.shape[1] + pose = np.zeros((frames, count, 3)) + for f in range(frames): + pose[f] = turn(joints[f] - joints[f, 0], -heading[f]) + velocity = np.zeros_like(pose) + for f in range(frames): + lo, hi = max(f - 1, 0), min(f + 1, frames - 1) + velocity[f] = (pose[hi] - pose[lo]) / (hi - lo) * fps + best = None + for a in range(frames): + for b in range(a + min_frames, frames): + p = math.sqrt(sum(np.sum((pose[b, j] - pose[a, j]) ** 2) for j in range(count)) / count) + v = math.sqrt(sum(np.sum((velocity[b, j] - velocity[a, j]) ** 2) for j in range(count)) / count) + cost = p + 0.1 * v + key = (cost, -(b - a), a) + if best is None or key < best[0]: + best = (key, a, b, p, v) + return best[1:] + + +def wandering(frames=44, count=4, seed=0): + """A smooth, unrepeating motion that turns as it goes, so the best window is unique and pose, velocity and + heading each move it.""" + rng = np.random.default_rng(seed) + time = np.arange(frames) + joints = sum(rng.normal(size=(1, count, 3)) * 0.3 * np.sin(time[:, None, None] * rng.uniform(0.05, 0.5) + rng.uniform(0, 6.28)) + for _ in range(4)) + heading = sum(0.4 * np.sin(time * rng.uniform(0.03, 0.2) + rng.uniform(0, 6.28)) for _ in range(2)) + return joints, heading + + +class LoopWindow(unittest.TestCase): + def test_a_periodic_motion_between_rest_ramps(self): + joints = stride() + first, last, pose_cost, _ = loop_window(joints, np.zeros(240), FPS, 120) + self.assertGreaterEqual(first, 40) + self.assertLessEqual(last, 200) + self.assertEqual((last - first) % 37, 0, (first, last)) + self.assertGreaterEqual(last - first, 120) + self.assertLess(pose_cost, 1e-3) + self.assertNotEqual((first, last), (0, 239), "the whole take is not the answer") + + def test_the_search_is_the_stated_cost(self): + joints, heading = wandering() + want_first, want_last, want_pose, want_velocity = reference(joints, heading, FPS, 22) + got = loop_window(joints, heading, FPS, 22) + self.assertEqual(got[:2], (want_first, want_last)) + self.assertAlmostEqual(got[2], want_pose, places=9) + self.assertAlmostEqual(got[3], want_velocity, places=9) + # The velocity term decides here: matching poses alone would pick another window. + pose_only = min( + ((np.sqrt(((turn(joints[b] - joints[b, 0], -heading[b]) - turn(joints[a] - joints[a, 0], -heading[a])) ** 2).sum(-1).mean()), a, b) + for a in range(44) for b in range(a + 22, 44))) + self.assertNotEqual(pose_only[1:], (want_first, want_last)) + + def test_heading_is_normalised_away(self): + joints, heading = wandering() + upright = np.stack([turn(joints[f] - joints[f, :1], -heading[f]) + joints[f, :1] for f in range(44)]) + turned = np.stack([turn(upright[f] - upright[f, :1], heading[f]) + upright[f, :1] for f in range(44)]) + want = loop_window(upright, np.zeros(44), FPS, 22) + got = loop_window(turned, heading, FPS, 22) + self.assertEqual(got[:2], want[:2]) + self.assertAlmostEqual(got[2], want[2], places=9) + self.assertNotEqual(loop_window(turned, np.zeros(44), FPS, 22)[:2], want[:2], "the turning would otherwise change the answer") + + def test_a_tie_goes_to_the_longer_window_then_the_earlier(self): + cycle = np.array([[0, 0, 0], [0.5, 0.2, 0.1], [0.2, 0.7, 0.3], [0.4, 0.1, 0.9]], dtype=float) + joints = np.stack([np.stack([np.zeros(3), cycle[f % 4]]) for f in range(30)]) + # Exact repeats every 4 frames: every a, b with b - a a multiple of 4 inside 1..28 costs zero. + self.assertEqual(loop_window(joints, np.zeros(30), FPS, 15)[:2], (1, 25)) + still = np.zeros((30, 3, 3)) + self.assertEqual(loop_window(still, np.zeros(30), FPS, 15)[:2], (0, 29)) + + def test_no_window_is_shorter_than_the_minimum(self): + joints, heading = wandering(frames=31) + for minimum in (5, 16, 25): + first, last, _, _ = loop_window(joints, heading, FPS, minimum) + self.assertGreaterEqual(last - first, minimum) + self.assertEqual(loop_window(joints, heading, FPS, 30)[:2], (0, 30)) + + def test_hips_is_the_joint_it_is_told(self): + joints, heading = wandering() + moved = joints[:, [2, 0, 1, 3]] + self.assertEqual(loop_window(moved, heading, FPS, 22, hips=1), loop_window(joints, heading, FPS, 22, hips=0)) + + def test_shapes_and_minimum_are_checked_without_echoing_them(self): + joints, heading = wandering() + for bad in ((joints, heading[:-1], 22), (joints, heading, 0), (joints, heading, 4321), (joints[..., :2], heading, 22)): + with self.assertRaises(ValueError) as caught: + loop_window(bad[0], bad[1], FPS, bad[2]) + self.assertNotIn("4321", str(caught.exception)) + + +class StandInSkeleton: + """What kimodo's heading and the worker read of a skeleton: the names and parents of the real one, its Hips index + and the [right, left] hip joints (RightLeg, LeftLeg) that heading is taken from.""" + + def __init__(self, names, parents): + self.bone_order_names = [str(name) for name in names] + self.bone_index = {name: index for index, name in enumerate(self.bone_order_names)} + self.bone_parents = {name: (None if parent < 0 else self.bone_order_names[parent]) for name, parent in zip(self.bone_order_names, parents)} + self.hip_joint_idx = [self.bone_index["RightLeg"], self.bone_index["LeftLeg"]] + + +class WorkerLoopWindowsTest(unittest.TestCase): + """loop_windows on a real generated take (the somaskel77 joints of a live Vili take, 240 frames at 30 fps), with + stand-ins for the model's two skeletons: the 77-joint output skeleton and the 30-joint one the model also carries.""" + + @classmethod + def setUpClass(cls): + data = np.load(pathlib.Path(__file__).resolve().parent / "fixtures" / "take.npz") + cls.joints = data["joints"] + cls.model = SimpleNamespace( + output_skeleton=StandInSkeleton(data["names77"], data["parents77"]), + skeleton=StandInSkeleton(data["names30"], data["parents30"]), + fps=FPS, + ) + + def window(self, joints): + (answer,) = loop_windows({"posed_joints": [joints]}, self.model) + return answer["first"], answer["last"] + + def test_the_real_take_gets_the_window_of_its_own_heading_in_the_output_skeleton(self): + self.assertEqual(self.joints.shape, (240, 77, 3)) + self.assertEqual(self.model.output_skeleton.hip_joint_idx, [72, 67]) + self.assertEqual(self.model.skeleton.hip_joint_idx, [26, 22]) + self.assertEqual(self.window(self.joints), (35, 222)) + # The 30-joint skeleton's hip indices read the wrong joints of a 77-joint take and move the window. + wrong = SimpleNamespace(output_skeleton=self.model.skeleton, fps=FPS) + self.assertNotEqual(loop_windows({"posed_joints": [self.joints]}, wrong)[0]["first"], 35) + + def test_turning_the_body_about_its_hips_by_a_smooth_angle_leaves_the_window(self): + turned = np.stack([ + turn(self.joints[f] - self.joints[f, 0], 0.8 * math.sin(0.05 * f)) + self.joints[f, 0] for f in range(len(self.joints)) + ]).astype(np.float32) + self.assertEqual(self.window(turned), (35, 222)) + + def test_each_sample_gets_its_own_window_with_its_costs(self): + still = np.repeat(self.joints[:1], 240, axis=0) + answers = loop_windows({"posed_joints": [self.joints, still]}, self.model) + self.assertEqual([(a["first"], a["last"]) for a in answers], [(35, 222), (0, 239)]) + self.assertEqual(set(answers[0]), {"first", "last", "poseCost", "velocityCost"}) + self.assertAlmostEqual(answers[1]["poseCost"], 0.0, places=6) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/vili-daemon.mjs b/scripts/vili-daemon.mjs index 908ca98..298ac71 100644 --- a/scripts/vili-daemon.mjs +++ b/scripts/vili-daemon.mjs @@ -592,8 +592,9 @@ async function readInside(file, directory) { // the BVH and writes a private FBX in a directory made for this export alone, and the FBX is then created beside the // BVH exclusively, so nothing planted at either path is read, written through or replaced. The FBX counts only if // Blender exits 0, prints its result line and its file exists. A failure names the exit or the signal and never -// Blender's output. -async function exportClip(bvh, fbx, clipName, timeoutMs, artifactDir) { +// Blender's output. A window {first, last} (BVH frame indices, last inclusive) is handed to Blender as the frame +// range to export, in Blender's frames, which count from 1; the daemon never chooses one. +async function exportClip(bvh, fbx, clipName, timeoutMs, artifactDir, window = null) { const input = await readInside(bvh, artifactDir); if (input === null) return { error: "fbx-export-refused" }; guard(); @@ -606,6 +607,7 @@ async function exportClip(bvh, fbx, clipName, timeoutMs, artifactDir) { const run = await runBlender([ "-b", "--factory-startup", "--python-use-system-env", "--python-exit-code", "1", "--python", exportScript, "--", privateBvh, privateFbx, clipName, + ...(window ? ["--frames", String(window.first + 1), String(window.last + 1)] : []), ], timeoutMs); let info = null; if (run.code === 0 && run.result !== null) { @@ -648,19 +650,28 @@ async function sweepWork() { // Every BVH the worker listed becomes an FBX, or the job fails. A listed BVH must be a regular file inside the job's // artifact directory, and that is decided when its export begins (exportClip), never earlier. Sample N of several is -// _NN. -async function exportListed(listed, artifactDir, clipName, timeoutMs) { +// _NN, N being its own index. With loops (the worker's answer, an array when the request asked for a loop, +// else undefined), sample k is exported over loops[k], and a job whose answer has no usable window for a sample fails. +async function exportListed(listed, artifactDir, clipName, timeoutMs, loops = undefined) { const bvhs = []; for (const file of Array.isArray(listed) ? listed : []) { if (typeof file !== "string" || !file.endsWith(".bvh")) continue; bvhs.push(path.resolve(artifactDir, file)); } if (bvhs.length === 0) return { clips: [], error: "fbx-export-failed (no bvh)" }; - bvhs.sort(); + // A sample's own index is in its file name (motion_NN.bvh, more digits past 99), so the order and the window come + // from that number and never from sorting the names as text. + const sampleOf = (bvh) => Number(/_(\d+)\.bvh$/.exec(bvh)?.[1] ?? 0); + bvhs.sort((a, b) => sampleOf(a) - sampleOf(b) || (a < b ? -1 : a > b ? 1 : 0)); const clips = []; - for (const [index, bvh] of bvhs.entries()) { + for (const bvh of bvhs) { + const index = sampleOf(bvh); const name = bvhs.length === 1 ? clipName : `${clipName}_${String(index).padStart(2, "0")}`; - const done = await exportClip(bvh, `${bvh.slice(0, -".bvh".length)}.fbx`, name, timeoutMs, artifactDir); + const window = loops === undefined ? null : loops[index]; + if (window !== null && !(Number.isInteger(window?.first) && Number.isInteger(window?.last) && window.first >= 0 && window.last > window.first)) { + return { clips, error: "fbx-export-failed (no loop window)" }; + } + const done = await exportClip(bvh, `${bvh.slice(0, -".bvh".length)}.fbx`, name, timeoutMs, artifactDir, window); if (done.error) return { clips, error: done.error }; clips.push({ ...done.clip, file: path.relative(artifactDir, done.clip.file).split(path.sep).join("/") }); } @@ -864,8 +875,10 @@ async function generate(request, response) { // One export plan: fbx implies the BVH and its standard T-pose, which the FBX is made from. const fbx = boolValue(body.fbx); const askedTpose = firstDefined(body.bvhStandardTpose, body.bvh_standard_tpose); - if (fbx && ((askedTpose !== undefined && !boolValue(askedTpose)) - || (body.clipName !== undefined && !(typeof body.clipName === "string" && CLIP_NAME.test(body.clipName))))) { + const loop = body.loop === true; + if ((body.loop !== undefined && typeof body.loop !== "boolean") || (loop && !fbx) + || (fbx && ((askedTpose !== undefined && !boolValue(askedTpose)) + || (body.clipName !== undefined && !(typeof body.clipName === "string" && CLIP_NAME.test(body.clipName)))))) { json(response, 400, { ok: false, error: "Invalid fbx request." }); return; } @@ -878,6 +891,13 @@ async function generate(request, response) { bvhStandardTpose: fbx || boolValue(askedTpose), }; + // One way to give constraints: the worker reads exactly one path, so two inputs would leave the record naming + // constraints that did not make the clip. + if ([body.constraints, body.constraints_path, body.constraintsPath].filter((given) => given !== undefined).length > 1) { + json(response, 400, { ok: false, error: "Invalid constraints request." }); + return; + } + const bodyMeta = typeof body.meta === "object" && body.meta ? body.meta : {}; const askedSeed = firstDefined(body.seed, bodyMeta.seed); if (askedSeed !== undefined && !(Number.isInteger(askedSeed) && askedSeed >= 0 && askedSeed < SEED_LIMIT)) { @@ -934,9 +954,10 @@ async function generate(request, response) { numSamples, diffusion_steps: diffusionSteps, numTransitionFrames: firstDefined(body.numTransitionFrames, body.num_transition_frames), - constraints: constraintsPath, + constraints: body.constraints !== undefined && typeof body.constraints !== "string" ? body.constraints : constraintsPath, seed, fbx: fbx || undefined, + loop: loop || undefined, clipName, cfg: body.cfg, cfgType: firstDefined(body.cfgType, body.cfg_type), @@ -967,6 +988,7 @@ async function generate(request, response) { numTransitionFrames: firstDefined(body.numTransitionFrames, body.num_transition_frames), constraints: constraintsPath, seed, + loop: loop || undefined, cfg: body.cfg, cfgType: firstDefined(body.cfgType, body.cfg_type), cfgWeight: firstDefined(body.cfgWeight, body.cfg_weight), @@ -993,7 +1015,7 @@ async function generate(request, response) { let exported = null; if (fbx && result.ok) { const timeoutMs = Math.min(EXPORT_TIMEOUT_MS, Math.max(1, Number(firstDefined(body.exportTimeoutMs, EXPORT_TIMEOUT_MS)) || EXPORT_TIMEOUT_MS)); - exported = await exportListed(result.body?.files, artifactDir, clipName, timeoutMs); + exported = await exportListed(result.body?.files, artifactDir, clipName, timeoutMs, loop ? (Array.isArray(result.body?.loops) ? result.body.loops : []) : undefined); if (exported.error) result = { ok: false, error: exported.error, body: result.body }; }