Merge cut/vili-loop-export: Vili exports a seam-matched loop window on request (Soul s1 fixed, s2 holds)
The worker picks the pose- and velocity-matched window from the generated joints; the FBX is that window; the record keeps request.loop, worker.loops and the constraints as sent. Constraints given more than once are refused. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Meta Vens
commit
3066c375d6
10 changed files with 481 additions and 26 deletions
13
README.md
13
README.md
|
|
@ -132,6 +132,7 @@ Supported request fields:
|
|||
- `cfg`, `cfgType` / `cfg_type`, `cfgWeight` / `cfg_weight`
|
||||
- `bvh`, `bvhStandardTpose` / `bvh_standard_tpose`
|
||||
- `fbx` and `clipName` (see FBX Export)
|
||||
- `loop` (a boolean, valid only with `fbx: true`; see FBX Export)
|
||||
- `noPostprocess` / `no_postprocess`
|
||||
- `saveExampleDir` / `save_example_dir`
|
||||
- `timeoutMs`, `workerTimeoutMs`, `exportTimeoutMs` (at most 120000)
|
||||
|
|
@ -166,6 +167,18 @@ timeout, and its output in `<state>/logs/export.log`. It is killed with the work
|
|||
SIGTERM. A failed export fails the job (`fbx-export-refused` or `fbx-export-failed (...)`, 502) and the answer never
|
||||
carries Blender's output.
|
||||
|
||||
`"loop": true` (with `fbx: true`; otherwise a 400, as is a non-boolean) exports the take's lowest-cost loop window instead
|
||||
of the whole take. The worker chooses it (`scripts/loop_window.py`) from Kimodo's own posed joints: every joint relative
|
||||
to its Hips and turned by its own heading (`compute_global_heading`), and the seam from frame `b` back to frame `a`
|
||||
costs the pose RMS plus 0.1 s times the velocity RMS (central differences, in m and m/s), over `b - a >= ceil(F / 2)`.
|
||||
The lowest cost wins; a tie goes to the longer window, then the earlier. The worker's answer carries `loops`, one
|
||||
`{first, last, poseCost, velocityCost}` per sample (BVH frame indices, `last` inclusive), and the daemon hands
|
||||
sample k's window to `bvh-to-fbx.py` as `--frames <first+1> <last+1>` (Blender counts from 1); the daemon never
|
||||
computes a window, and an answer without one for a sample fails the job (`fbx-export-failed (no loop window)`).
|
||||
The BVH and NPZ stay the whole generated take. The job record keeps `request.loop`, `worker.loops` and the clip's
|
||||
`frames` as Blender reports them, and keeps inline `constraints` as sent (a path otherwise), so a committed record
|
||||
reproduces its clip. Without `loop` the FBX is the whole take, as before.
|
||||
|
||||
One BVH gives the same FBX bytes on one host body (a fixed hash seed and a fixed time stamp); FBX bytes differ
|
||||
between glibc versions in the last float digits, so the BVH is the cross-body reproducibility unit.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
# Converts one Kimodo standard-T-pose BVH to one FBX. Blender converts formats and nothing else: no retarget, no
|
||||
# rename, no key reduction, no resample, no root-motion bake. Run by the daemon (scripts/vili-daemon.mjs exportClip):
|
||||
# blender -b --factory-startup --python-use-system-env --python-exit-code 1 --python bvh-to-fbx.py -- <bvh> <fbx> <clipName>
|
||||
# rename, no key reduction, no resample, no root-motion bake. It applies the frame range it is handed and chooses
|
||||
# none: the whole take unless the daemon passes --frames. Run by the daemon (scripts/vili-daemon.mjs exportClip):
|
||||
# blender -b --factory-startup --python-use-system-env --python-exit-code 1 --python bvh-to-fbx.py -- <bvh> <fbx> <clipName> [--frames <A> <B>]
|
||||
# Frames are Blender's, counted from 1: the BVH's frame k is Blender's k + 1. Exit 3 is an argument or a hash seed it
|
||||
# will not take, exit 4 a range outside the take.
|
||||
# with PYTHONHASHSEED=0 in the environment. The settings are Blender API parameters, cross-checked against the
|
||||
# 'unity' preset of wifimetas/kimodo-mcp blender_export.py (Apache-2.0, 83b5361); no code is copied.
|
||||
import datetime
|
||||
|
|
@ -15,7 +18,9 @@ import bpy
|
|||
if sys.flags.hash_randomization != 0:
|
||||
sys.exit(3)
|
||||
|
||||
bvh, fbx, clip = sys.argv[sys.argv.index("--") + 1:]
|
||||
bvh, fbx, clip, *window = sys.argv[sys.argv.index("--") + 1:]
|
||||
if window and not (len(window) == 3 and window[0] == "--frames" and all(arg.isdigit() for arg in window[1:])):
|
||||
sys.exit(3)
|
||||
|
||||
bpy.ops.wm.read_factory_settings(use_empty=True)
|
||||
scene = bpy.context.scene
|
||||
|
|
@ -38,6 +43,10 @@ action = armature.animation_data.action
|
|||
# The FBX take takes the scene's name.
|
||||
armature.name = armature.data.name = action.name = scene.name = clip
|
||||
first, last = (int(frame) for frame in action.frame_range)
|
||||
if window:
|
||||
first, last = int(window[1]), int(window[2])
|
||||
if not (int(action.frame_range[0]) <= first < last <= int(action.frame_range[1])):
|
||||
sys.exit(4)
|
||||
scene.frame_start, scene.frame_end = first, last
|
||||
|
||||
bpy.ops.object.select_all(action="DESELECT")
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
// Stands in for kimodo-resident-worker.py in the daemon's tests. It binds port 0, prints the ready line the
|
||||
// daemon waits for, and answers /health and /generate with what the daemon gave it. The --model value picks a
|
||||
// misbehaviour: crash-before-ready, hang-before-ready, sick, slow-health, stubborn, fail-generate,
|
||||
// refuse-generate, drop-generate, slow-generate, writer (appends to <output>.writes every 50 ms, never answers, ignores SIGTERM), slow-to-stop (no answer; prints a line and exits 1.5 s after SIGTERM), chatty (prints a line the moment it is healthy), encoder (a grandchild, then no answer), graceful (exits 0.5 s after SIGTERM), die-after-ready, bvh-outside (lists a BVH outside the job's directory), bvh-link (lists a symlink as its BVH), bvh-fifo (lists a FIFO as its BVH), plant-link (a normal answer with a file symlink and a directory symlink planted beside the BVH). A request with bvh writes <output>.bvh (motion/motion_NN.bvh for several samples) and lists it. Any other value is just the model the worker serves.
|
||||
// refuse-generate, drop-generate, slow-generate, writer (appends to <output>.writes every 50 ms, never answers, ignores SIGTERM), slow-to-stop (no answer; prints a line and exits 1.5 s after SIGTERM), chatty (prints a line the moment it is healthy), encoder (a grandchild, then no answer), graceful (exits 0.5 s after SIGTERM), die-after-ready, bvh-outside (lists a BVH outside the job's directory), bvh-link (lists a symlink as its BVH), bvh-fifo (lists a FIFO as its BVH), plant-link (a normal answer with a file symlink and a directory symlink planted beside the BVH), no-loops (a normal answer that leaves out the loops a loop request asked for), bad-loops, negative-loops, empty-loops, fractional-last-loops (loops whose window is not a frame range: a fractional first, a negative first, first equal to last, a fractional last), short-loops (one window fewer than samples). A loop request is answered with loops, sample k as {first 3 + k, last 223 + k}. A request with bvh writes <output>.bvh (motion/motion_NN.bvh for several samples) and lists it. Any other value is just the model the worker serves.
|
||||
import http from "node:http";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
|
@ -14,6 +14,12 @@ const flag = (name) => {
|
|||
return index < 0 ? undefined : argv[index + 1];
|
||||
};
|
||||
const model = flag("--model");
|
||||
const BAD_WINDOWS = {
|
||||
"bad-loops": { first: 5.5, last: 9 },
|
||||
"negative-loops": { first: -1, last: 9 },
|
||||
"empty-loops": { first: 7, last: 7 },
|
||||
"fractional-last-loops": { first: 5, last: 9.5 },
|
||||
};
|
||||
const spawnLog = path.join(process.env.VILI_EMBEDDINGS_DIR, "stub-spawns.log");
|
||||
let requests = 0;
|
||||
let healthCalls = 0;
|
||||
|
|
@ -117,6 +123,10 @@ if (model === "hang-before-ready") {
|
|||
seed: payload.seed,
|
||||
metaSeed: payload.meta?.seed,
|
||||
bvh: payload.bvh,
|
||||
...(payload.loop && model !== "no-loops" ? {
|
||||
loops: Array.from({ length: payload.numSamples - (model === "short-loops" ? 1 : 0) }, (_, k) => (BAD_WINDOWS[model]
|
||||
?? { first: 3 + k, last: 223 + k, poseCost: 0.03, velocityCost: 0.2 })),
|
||||
} : {}),
|
||||
bvhStandardTpose: payload.bvhStandardTpose,
|
||||
model: model || "stub-model",
|
||||
pid: process.pid,
|
||||
|
|
|
|||
BIN
scripts/fixtures/take.npz
Normal file
BIN
scripts/fixtures/take.npz
Normal file
Binary file not shown.
|
|
@ -28,6 +28,7 @@ from kimodo.scripts.generate import (
|
|||
)
|
||||
from kimodo.tools import save_json, seed_everything
|
||||
|
||||
import loop_window
|
||||
import text_encoding
|
||||
|
||||
|
||||
|
|
@ -319,6 +320,7 @@ def generate(payload):
|
|||
request_bool(payload, "bvh"),
|
||||
request_bool(payload, "bvhStandardTpose") or request_bool(payload, "bvh_standard_tpose"),
|
||||
)
|
||||
loops = loop_window.loop_windows(output, model) if request_bool(payload, "loop") else None
|
||||
if request_bool(payload, "saveExampleDir") or request_bool(payload, "save_example_dir"):
|
||||
files.extend(save_example_dirs(output_base, texts, num_frames, num_samples, constraints_path))
|
||||
generation_count += 1
|
||||
|
|
@ -347,6 +349,7 @@ def generate(payload):
|
|||
"constraints": constraints_path,
|
||||
"seed": seed,
|
||||
"files": files,
|
||||
**({"loops": loops} if loops is not None else {}),
|
||||
"encoder": encoder_record,
|
||||
"worker": worker_identity,
|
||||
"timing": {
|
||||
|
|
|
|||
60
scripts/loop_window.py
Normal file
60
scripts/loop_window.py
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
"""The frame window of a generated take that loops with the smallest seam. Called by the resident worker, nowhere else.
|
||||
|
||||
A seam is the jump from frame b back to frame a. Its cost is the pose difference plus 0.1 s times the velocity
|
||||
difference (Kovar, Gleicher and Pighin 2002, motion-graph transitions), both as RMS over joints, with each frame
|
||||
taken relative to its own Hips and turned to face a fixed way, so where the take wanders and which way it faces
|
||||
do not count. loop_window is numpy only; loop_windows needs torch and kimodo's heading, and no model weights.
|
||||
"""
|
||||
import math
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from kimodo.constraints import compute_global_heading
|
||||
|
||||
VELOCITY_WEIGHT_SECONDS = 0.1
|
||||
|
||||
|
||||
def loop_window(joints, heading, fps, min_frames, hips=0):
|
||||
"""joints [F, J, 3] world positions, heading [F] radians about Y (Kimodo's hip-vector angle), the joint index of
|
||||
Hips. Returns (first, last, pose_cost, velocity_cost): frame indices into the take, last inclusive, last - first
|
||||
at least min_frames, the pair with the lowest cost. A tie goes to the longer window, then to the earlier first."""
|
||||
joints = np.asarray(joints, dtype=np.float64)
|
||||
heading = np.asarray(heading, dtype=np.float64)
|
||||
frames = joints.shape[0]
|
||||
if joints.ndim != 3 or joints.shape[2] != 3 or heading.shape != (frames,) or not 1 <= min_frames < frames:
|
||||
raise ValueError("loop_window: shapes or min_frames out of range")
|
||||
relative = joints - joints[:, hips:hips + 1]
|
||||
cos, sin = np.cos(-heading), np.sin(-heading)
|
||||
x, z = relative[..., 0], relative[..., 2]
|
||||
pose = relative.copy()
|
||||
pose[..., 0] = x * cos[:, None] + z * sin[:, None]
|
||||
pose[..., 2] = -x * sin[:, None] + z * cos[:, None]
|
||||
velocity = np.gradient(pose, axis=0) * fps # central differences, one-sided at the two ends
|
||||
|
||||
firsts, lasts = np.triu_indices(frames, k=min_frames)
|
||||
pose_cost = np.empty(len(firsts))
|
||||
velocity_cost = np.empty(len(firsts))
|
||||
for first in range(frames - min_frames): # one row of seams at a time: memory stays O(frames * joints)
|
||||
row = slice(*np.searchsorted(firsts, [first, first + 1]))
|
||||
later = lasts[row]
|
||||
pose_cost[row] = np.sqrt(((pose[later] - pose[first]) ** 2).sum(-1).mean(-1))
|
||||
velocity_cost[row] = np.sqrt(((velocity[later] - velocity[first]) ** 2).sum(-1).mean(-1))
|
||||
cost = pose_cost + VELOCITY_WEIGHT_SECONDS * velocity_cost
|
||||
best = np.flatnonzero(cost == cost.min())
|
||||
pick = best[np.lexsort((firsts[best], -(lasts[best] - firsts[best])))[0]]
|
||||
return int(firsts[pick]), int(lasts[pick]), float(pose_cost[pick]), float(velocity_cost[pick])
|
||||
|
||||
|
||||
def loop_windows(output, model):
|
||||
"""One loop window per sample of a Kimodo output, as the worker answers them: {first, last, poseCost, velocityCost},
|
||||
frame indices into the take, last inclusive. The joints are posed_joints, always in model.output_skeleton (the
|
||||
somaskel77, not model.skeleton), and the heading is Kimodo's own from those joints."""
|
||||
skeleton = model.output_skeleton
|
||||
hips = skeleton.bone_index["Hips"]
|
||||
windows = []
|
||||
for joints in output["posed_joints"]:
|
||||
heading = compute_global_heading(torch.from_numpy(joints[None]), skeleton)[0]
|
||||
angle = torch.atan2(heading[:, 1], heading[:, 0]).numpy()
|
||||
first, last, pose_cost, velocity_cost = loop_window(joints, angle, model.fps, math.ceil(len(joints) / 2), hips)
|
||||
windows.append({"first": first, "last": last, "poseCost": pose_cost, "velocityCost": velocity_cost})
|
||||
return windows
|
||||
|
|
@ -1441,6 +1441,124 @@ test("fbx refusals happen before anything is written or spawned", async () => {
|
|||
}
|
||||
});
|
||||
|
||||
test("loop asks for the seamless window, and the window is the FBX's frame range", async () => {
|
||||
const daemon = await startDaemon();
|
||||
try {
|
||||
const constraints = [{ type: "root2d", frame_indices: [0, 10], smooth_root_2d: [[0, 0], [0, 0.5]] }];
|
||||
const answer = await generate(daemon, exportRequest({ clipName: "walk", seed: 7, loop: true, constraints }));
|
||||
assert.equal(answer.status, 200);
|
||||
const job = answer.body;
|
||||
assert.equal(job.request.loop, true);
|
||||
assert.deepEqual(job.worker.loops, [{ first: 3, last: 223, poseCost: 0.03, velocityCost: 0.2 }]);
|
||||
assert.deepEqual(job.request.constraints, constraints, "the record keeps inline constraints as sent");
|
||||
assert.deepEqual(JSON.parse(readFileSync(path.join(job.artifactDir, "constraints.json"), "utf8")), constraints);
|
||||
assert.equal(JSON.parse(readFileSync(path.join(job.artifactDir, "request.json"), "utf8")).loop, true, "the worker is asked for the loop");
|
||||
assert.equal(job.worker.constraints, path.join(job.artifactDir, "constraints.json"), "the worker still gets the path");
|
||||
const [run] = exportStarts(daemon);
|
||||
assert.deepEqual(run.argv.slice(run.argv.indexOf("--") + 3), ["walk", "--frames", "4", "224"], "Blender's frames count from 1");
|
||||
assert.deepEqual(job.export.clips.map((clip) => clip.frames), [[1, 60]], "the clip's frames are what Blender reports");
|
||||
|
||||
const several = await generate(daemon, exportRequest({ clipName: "pair", numSamples: 2, loop: true }));
|
||||
assert.equal(several.status, 200);
|
||||
assert.deepEqual(several.body.worker.loops.map((loop) => [loop.first, loop.last]), [[3, 223], [4, 224]]);
|
||||
const frames = Object.fromEntries(exportStarts(daemon).slice(1).map((start) => {
|
||||
const [, , clip, ...rest] = start.argv.slice(start.argv.indexOf("--") + 1);
|
||||
return [clip, rest];
|
||||
}));
|
||||
assert.deepEqual(frames, { pair_00: ["--frames", "4", "224"], pair_01: ["--frames", "5", "225"] }, "each sample is exported over its own window");
|
||||
|
||||
const plain = await generate(daemon, exportRequest({ clipName: "plain" }));
|
||||
assert.equal(plain.status, 200);
|
||||
assert.equal(plain.body.worker.loops, undefined);
|
||||
assert.equal(plain.body.request.loop, undefined);
|
||||
assert.equal(JSON.parse(readFileSync(path.join(plain.body.artifactDir, "request.json"), "utf8")).loop, undefined);
|
||||
assert.equal(exportStarts(daemon).at(-1).argv.at(-1), "plain", "no window, no --frames");
|
||||
const kept = await generate(daemon, exportRequest({ constraints_path: "/some/where.json", constraints: undefined }));
|
||||
assert.equal(kept.body.request.constraints, "/some/where.json", "a path stays a path");
|
||||
} finally {
|
||||
await daemon.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test("past 100 samples each clip is exported over the window of its own sample", async () => {
|
||||
const daemon = await startDaemon();
|
||||
try {
|
||||
const answer = await generate(daemon, exportRequest({ clipName: "m", loop: true, numSamples: 101, exportTimeoutMs: 60000 }));
|
||||
assert.equal(answer.status, 200);
|
||||
const starts = exportStarts(daemon);
|
||||
assert.equal(answer.body.export.clips.length, 101);
|
||||
answer.body.export.clips.forEach((clip, i) => {
|
||||
const sample = Number(/motion_(\d+)\.fbx$/.exec(clip.file)[1]);
|
||||
const argv = starts[i].argv.slice(starts[i].argv.indexOf("--") + 1);
|
||||
assert.equal(argv[2], `m_${String(sample).padStart(2, "0")}`, `${clip.file} is named for its sample`);
|
||||
assert.deepEqual(argv.slice(3), ["--frames", String(4 + sample), String(224 + sample)], `${clip.file} gets its own sample's window`);
|
||||
});
|
||||
assert.ok(answer.body.export.clips.some((clip) => clip.file.endsWith("motion_100.fbx")));
|
||||
} finally {
|
||||
await daemon.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test("constraints given twice are refused before anything is written or spawned", async () => {
|
||||
const daemon = await startDaemon();
|
||||
try {
|
||||
const inline = [{ type: "root2d" }];
|
||||
for (const bad of [
|
||||
{ constraints: inline, constraints_path: "/canary/a.json" },
|
||||
{ constraints: inline, constraintsPath: "/canary/a.json" },
|
||||
{ constraints: "/canary/a.json", constraints_path: "/canary/b.json" },
|
||||
{ constraints_path: "/canary/a.json", constraintsPath: "/canary/b.json" },
|
||||
]) {
|
||||
const answer = await generate(daemon, exportRequest(bad));
|
||||
assert.equal(answer.status, 400, JSON.stringify(bad));
|
||||
assert.deepEqual(answer.body, { ok: false, error: "Invalid constraints request." });
|
||||
assert.ok(!answer.raw.includes("canary"), "the refusal repeated the input");
|
||||
}
|
||||
assert.equal(existsSync(path.join(daemon.stateRoot, "artifacts")), false, "no artifact directory was made");
|
||||
assert.deepEqual(jobFiles(daemon), []);
|
||||
assert.deepEqual(daemon.spawns(), [], "no worker was started");
|
||||
assert.equal((await generate(daemon, exportRequest({ constraints_path: "/some/where.json" }))).status, 200, "one input is fine");
|
||||
} finally {
|
||||
await daemon.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test("loop is a boolean and needs fbx, refused before anything is written or spawned", async () => {
|
||||
const daemon = await startDaemon();
|
||||
try {
|
||||
for (const bad of [{ fbx: false, loop: true }, { fbx: undefined, loop: true }, { loop: "yes canary" }, { loop: 1 }, { loop: null }, { loop: "true" }]) {
|
||||
const answer = await generate(daemon, exportRequest(bad));
|
||||
assert.equal(answer.status, 400, JSON.stringify(bad));
|
||||
assert.deepEqual(answer.body, { ok: false, error: "Invalid fbx request." });
|
||||
assert.ok(!answer.raw.includes("canary"), "the refusal repeated the input");
|
||||
}
|
||||
assert.equal(existsSync(path.join(daemon.stateRoot, "artifacts")), false, "no artifact directory was made");
|
||||
assert.deepEqual(jobFiles(daemon), []);
|
||||
assert.deepEqual(daemon.spawns(), [], "no worker was started");
|
||||
assert.equal(exportStarts(daemon).length, 0);
|
||||
assert.equal((await generate(daemon, exportRequest({ loop: false }))).status, 200, "loop false is no loop");
|
||||
assert.equal((await generate(daemon, { prompt: "a person waves", duration: 2, loop: false })).status, 200);
|
||||
} finally {
|
||||
await daemon.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test("a loop request whose answer has no usable window for a sample fails without exporting that sample", async () => {
|
||||
for (const [model, exported] of [["no-loops", 0], ["bad-loops", 0], ["negative-loops", 0], ["empty-loops", 0], ["fractional-last-loops", 0], ["short-loops", 1]]) {
|
||||
const daemon = await startDaemon();
|
||||
try {
|
||||
const answer = await generate(daemon, exportRequest({ model, loop: true, numSamples: 2 }));
|
||||
assert.equal(answer.status, 502, model);
|
||||
assert.equal(answer.body.ok, false);
|
||||
assert.equal(answer.body.error, "fbx-export-failed (no loop window)");
|
||||
assert.equal(exportStarts(daemon).length, exported, `${model}: only a sample with a window is exported`);
|
||||
assert.equal(answer.body.export.clips.length, exported);
|
||||
} finally {
|
||||
await daemon.stop();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test("the BVH is never given to Blender unless it is a regular file inside the job's directory", async () => {
|
||||
const daemon = await startDaemon();
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -155,6 +155,21 @@ class BvhToFbx(unittest.TestCase):
|
|||
line = next(line for line in run.stdout.splitlines() if line.startswith("INSPECT "))
|
||||
return json.loads(line[len("INSPECT "):])
|
||||
|
||||
def walk(self, name, frames):
|
||||
"""A BVH of the rest pose's skeleton whose Hips travel one metre, accelerating, and whose Spine1 is turned."""
|
||||
text = self.rest.read_text()
|
||||
header = text[:text.index("MOTION")]
|
||||
channels = sum(int(n) for n in re.findall(r"CHANNELS\s+(\d+)", header))
|
||||
rows = []
|
||||
for frame in range(frames):
|
||||
row = ["0"] * channels
|
||||
row[8] = repr(100.0 * (frame / (frames - 1)) ** 2) # Hips Zposition, cm: one metre, accelerating
|
||||
row[12], row[13], row[14] = "30", "20", "40" # Spine1 Z, Y, X rotation: the Euler order matters
|
||||
rows.append(" ".join(row))
|
||||
bvh = self.dir / f"{name}.bvh"
|
||||
bvh.write_text(f"{header}MOTION\nFrames: {frames}\nFrame Time: 0.0333333\n" + "\n".join(rows) + "\n")
|
||||
return bvh
|
||||
|
||||
def test_rest_pose_keeps_the_bvh_joint_names_and_height(self):
|
||||
fbx = self.dir / "rest.fbx"
|
||||
run = convert(self.rest, fbx, "SOMA77_Rest")
|
||||
|
|
@ -189,18 +204,8 @@ class BvhToFbx(unittest.TestCase):
|
|||
self.assertFalse(target.exists())
|
||||
|
||||
def test_a_clip_keeps_its_travel_timing_pose_and_name(self):
|
||||
text = self.rest.read_text()
|
||||
header = text[:text.index("MOTION")]
|
||||
channels = sum(int(n) for n in re.findall(r"CHANNELS\s+(\d+)", header))
|
||||
frames = 31
|
||||
rows = []
|
||||
for frame in range(frames):
|
||||
row = ["0"] * channels
|
||||
row[8] = repr(100.0 * (frame / (frames - 1)) ** 2) # Hips Zposition, cm: one metre, accelerating
|
||||
row[12], row[13], row[14] = "30", "20", "40" # Spine1 Z, Y, X rotation: the Euler order matters
|
||||
rows.append(" ".join(row))
|
||||
bvh = self.dir / "walk.bvh"
|
||||
bvh.write_text(f"{header}MOTION\nFrames: {frames}\nFrame Time: 0.0333333\n" + "\n".join(rows) + "\n")
|
||||
bvh = self.walk("walk", frames)
|
||||
fbx = self.dir / "walk.fbx"
|
||||
run = convert(bvh, fbx, "walk")
|
||||
self.assertEqual(run.returncode, 0, run.stderr[-2000:])
|
||||
|
|
@ -222,6 +227,32 @@ class BvhToFbx(unittest.TestCase):
|
|||
for got, want in zip(seen["head_from_hips"], seen["reference_head_from_hips"]):
|
||||
self.assertAlmostEqual(got, want, delta=0.002, msg="the rotated pose is Blender's own native reading of the BVH")
|
||||
|
||||
def test_frames_exports_that_range_and_nothing_else(self):
|
||||
bvh = self.walk("long", 61)
|
||||
fbx = self.dir / "window.fbx"
|
||||
run = blender("--python", str(SCRIPT), "--", str(bvh), str(fbx), "window", "--frames", "5", "40")
|
||||
self.assertEqual(run.returncode, 0, run.stderr[-2000:])
|
||||
self.assertEqual(result(run), {"bones": 78, "frames": [5, 40], "fps": 30})
|
||||
seen = self.inspect(fbx, bvh)
|
||||
# The BVH's frame k is Blender's k + 1, so frames 5..40 are BVH rows 4..39 and Hips z is 100 (k/60)^2 cm.
|
||||
self.assertAlmostEqual(seen["hips_travel"], (39 ** 2 - 4 ** 2) / 60 ** 2, delta=0.002, msg=json.dumps(seen))
|
||||
self.assertEqual(set(fbx_key_counts(fbx)), {36}, "one key per frame of the window")
|
||||
whole = self.dir / "whole.fbx"
|
||||
self.assertEqual(result(convert(bvh, whole, "whole"))["frames"], [1, 61], "without --frames the take is whole")
|
||||
|
||||
def test_frames_outside_the_take_or_malformed_exit_without_a_file(self):
|
||||
bvh = self.walk("short", 31)
|
||||
for index, (arguments, exit_code) in enumerate((
|
||||
(("--frames", "0", "10"), 4), (("--frames", "5", "32"), 4), (("--frames", "10", "10"), 4),
|
||||
(("--frames", "20", "10"), 4),
|
||||
(("--frames", "5"), 3), (("--frames", "a", "9"), 3), (("--frames", "-1", "9"), 3),
|
||||
(("--frames", "4", "20", "9"), 3), (("--range", "4", "20"), 3), (("4", "20"), 3),
|
||||
)):
|
||||
fbx = self.dir / f"refused{index}.fbx"
|
||||
run = blender("--python", str(SCRIPT), "--", str(bvh), str(fbx), "refused", *arguments)
|
||||
self.assertEqual(run.returncode, exit_code, arguments)
|
||||
self.assertFalse(fbx.exists(), arguments)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
189
scripts/test_loop_window.py
Normal file
189
scripts/test_loop_window.py
Normal file
|
|
@ -0,0 +1,189 @@
|
|||
"""python3 -m unittest scripts/test_loop_window.py (numpy, torch and kimodo; no model weights)"""
|
||||
import math
|
||||
import pathlib
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
|
||||
from loop_window import loop_window, loop_windows # noqa: E402
|
||||
from types import SimpleNamespace # noqa: E402
|
||||
|
||||
FPS = 30
|
||||
|
||||
|
||||
def turn(points, angle):
|
||||
"""Rotate points [..., 3] about Y by angle, so that +Z turns toward +X (Kimodo's heading: 0 faces +Z)."""
|
||||
x, z = points[..., 0], points[..., 2]
|
||||
out = points.copy()
|
||||
out[..., 0] = x * math.cos(angle) + z * math.sin(angle)
|
||||
out[..., 2] = -x * math.sin(angle) + z * math.cos(angle)
|
||||
return out
|
||||
|
||||
|
||||
def stride(frames=240, period=37, start=40, end=200):
|
||||
"""A stride cycle between two different rests. Hips travels; every other joint swings."""
|
||||
rng = np.random.default_rng(3)
|
||||
rest_a, rest_b = rng.normal(size=(2, 6, 3)) * 0.3
|
||||
phase, direction = rng.uniform(0, 6.28, 6), rng.normal(size=(6, 3))
|
||||
out = np.zeros((frames, 7, 3))
|
||||
for f in range(frames):
|
||||
weight = min(1.0, f / start, (frames - 1 - f) / (frames - 1 - end))
|
||||
rest = rest_a if f < frames // 2 else rest_b
|
||||
cycle = 0.2 * np.sin(2 * math.pi * f / period + phase)[:, None] * direction
|
||||
out[f, 1:] = weight * (rest_a + cycle) + (1 - weight) * rest
|
||||
out[f, 0] = (0.03 * f, 0.9, 0.0) # Hips
|
||||
out[f, 1:] += out[f, 0]
|
||||
return out
|
||||
|
||||
|
||||
def reference(joints, heading, fps, min_frames):
|
||||
"""The cost written out frame by frame and joint by joint, to check the vectorised search against."""
|
||||
frames, count = joints.shape[0], joints.shape[1]
|
||||
pose = np.zeros((frames, count, 3))
|
||||
for f in range(frames):
|
||||
pose[f] = turn(joints[f] - joints[f, 0], -heading[f])
|
||||
velocity = np.zeros_like(pose)
|
||||
for f in range(frames):
|
||||
lo, hi = max(f - 1, 0), min(f + 1, frames - 1)
|
||||
velocity[f] = (pose[hi] - pose[lo]) / (hi - lo) * fps
|
||||
best = None
|
||||
for a in range(frames):
|
||||
for b in range(a + min_frames, frames):
|
||||
p = math.sqrt(sum(np.sum((pose[b, j] - pose[a, j]) ** 2) for j in range(count)) / count)
|
||||
v = math.sqrt(sum(np.sum((velocity[b, j] - velocity[a, j]) ** 2) for j in range(count)) / count)
|
||||
cost = p + 0.1 * v
|
||||
key = (cost, -(b - a), a)
|
||||
if best is None or key < best[0]:
|
||||
best = (key, a, b, p, v)
|
||||
return best[1:]
|
||||
|
||||
|
||||
def wandering(frames=44, count=4, seed=0):
|
||||
"""A smooth, unrepeating motion that turns as it goes, so the best window is unique and pose, velocity and
|
||||
heading each move it."""
|
||||
rng = np.random.default_rng(seed)
|
||||
time = np.arange(frames)
|
||||
joints = sum(rng.normal(size=(1, count, 3)) * 0.3 * np.sin(time[:, None, None] * rng.uniform(0.05, 0.5) + rng.uniform(0, 6.28))
|
||||
for _ in range(4))
|
||||
heading = sum(0.4 * np.sin(time * rng.uniform(0.03, 0.2) + rng.uniform(0, 6.28)) for _ in range(2))
|
||||
return joints, heading
|
||||
|
||||
|
||||
class LoopWindow(unittest.TestCase):
|
||||
def test_a_periodic_motion_between_rest_ramps(self):
|
||||
joints = stride()
|
||||
first, last, pose_cost, _ = loop_window(joints, np.zeros(240), FPS, 120)
|
||||
self.assertGreaterEqual(first, 40)
|
||||
self.assertLessEqual(last, 200)
|
||||
self.assertEqual((last - first) % 37, 0, (first, last))
|
||||
self.assertGreaterEqual(last - first, 120)
|
||||
self.assertLess(pose_cost, 1e-3)
|
||||
self.assertNotEqual((first, last), (0, 239), "the whole take is not the answer")
|
||||
|
||||
def test_the_search_is_the_stated_cost(self):
|
||||
joints, heading = wandering()
|
||||
want_first, want_last, want_pose, want_velocity = reference(joints, heading, FPS, 22)
|
||||
got = loop_window(joints, heading, FPS, 22)
|
||||
self.assertEqual(got[:2], (want_first, want_last))
|
||||
self.assertAlmostEqual(got[2], want_pose, places=9)
|
||||
self.assertAlmostEqual(got[3], want_velocity, places=9)
|
||||
# The velocity term decides here: matching poses alone would pick another window.
|
||||
pose_only = min(
|
||||
((np.sqrt(((turn(joints[b] - joints[b, 0], -heading[b]) - turn(joints[a] - joints[a, 0], -heading[a])) ** 2).sum(-1).mean()), a, b)
|
||||
for a in range(44) for b in range(a + 22, 44)))
|
||||
self.assertNotEqual(pose_only[1:], (want_first, want_last))
|
||||
|
||||
def test_heading_is_normalised_away(self):
|
||||
joints, heading = wandering()
|
||||
upright = np.stack([turn(joints[f] - joints[f, :1], -heading[f]) + joints[f, :1] for f in range(44)])
|
||||
turned = np.stack([turn(upright[f] - upright[f, :1], heading[f]) + upright[f, :1] for f in range(44)])
|
||||
want = loop_window(upright, np.zeros(44), FPS, 22)
|
||||
got = loop_window(turned, heading, FPS, 22)
|
||||
self.assertEqual(got[:2], want[:2])
|
||||
self.assertAlmostEqual(got[2], want[2], places=9)
|
||||
self.assertNotEqual(loop_window(turned, np.zeros(44), FPS, 22)[:2], want[:2], "the turning would otherwise change the answer")
|
||||
|
||||
def test_a_tie_goes_to_the_longer_window_then_the_earlier(self):
|
||||
cycle = np.array([[0, 0, 0], [0.5, 0.2, 0.1], [0.2, 0.7, 0.3], [0.4, 0.1, 0.9]], dtype=float)
|
||||
joints = np.stack([np.stack([np.zeros(3), cycle[f % 4]]) for f in range(30)])
|
||||
# Exact repeats every 4 frames: every a, b with b - a a multiple of 4 inside 1..28 costs zero.
|
||||
self.assertEqual(loop_window(joints, np.zeros(30), FPS, 15)[:2], (1, 25))
|
||||
still = np.zeros((30, 3, 3))
|
||||
self.assertEqual(loop_window(still, np.zeros(30), FPS, 15)[:2], (0, 29))
|
||||
|
||||
def test_no_window_is_shorter_than_the_minimum(self):
|
||||
joints, heading = wandering(frames=31)
|
||||
for minimum in (5, 16, 25):
|
||||
first, last, _, _ = loop_window(joints, heading, FPS, minimum)
|
||||
self.assertGreaterEqual(last - first, minimum)
|
||||
self.assertEqual(loop_window(joints, heading, FPS, 30)[:2], (0, 30))
|
||||
|
||||
def test_hips_is_the_joint_it_is_told(self):
|
||||
joints, heading = wandering()
|
||||
moved = joints[:, [2, 0, 1, 3]]
|
||||
self.assertEqual(loop_window(moved, heading, FPS, 22, hips=1), loop_window(joints, heading, FPS, 22, hips=0))
|
||||
|
||||
def test_shapes_and_minimum_are_checked_without_echoing_them(self):
|
||||
joints, heading = wandering()
|
||||
for bad in ((joints, heading[:-1], 22), (joints, heading, 0), (joints, heading, 4321), (joints[..., :2], heading, 22)):
|
||||
with self.assertRaises(ValueError) as caught:
|
||||
loop_window(bad[0], bad[1], FPS, bad[2])
|
||||
self.assertNotIn("4321", str(caught.exception))
|
||||
|
||||
|
||||
class StandInSkeleton:
|
||||
"""What kimodo's heading and the worker read of a skeleton: the names and parents of the real one, its Hips index
|
||||
and the [right, left] hip joints (RightLeg, LeftLeg) that heading is taken from."""
|
||||
|
||||
def __init__(self, names, parents):
|
||||
self.bone_order_names = [str(name) for name in names]
|
||||
self.bone_index = {name: index for index, name in enumerate(self.bone_order_names)}
|
||||
self.bone_parents = {name: (None if parent < 0 else self.bone_order_names[parent]) for name, parent in zip(self.bone_order_names, parents)}
|
||||
self.hip_joint_idx = [self.bone_index["RightLeg"], self.bone_index["LeftLeg"]]
|
||||
|
||||
|
||||
class WorkerLoopWindowsTest(unittest.TestCase):
|
||||
"""loop_windows on a real generated take (the somaskel77 joints of a live Vili take, 240 frames at 30 fps), with
|
||||
stand-ins for the model's two skeletons: the 77-joint output skeleton and the 30-joint one the model also carries."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
data = np.load(pathlib.Path(__file__).resolve().parent / "fixtures" / "take.npz")
|
||||
cls.joints = data["joints"]
|
||||
cls.model = SimpleNamespace(
|
||||
output_skeleton=StandInSkeleton(data["names77"], data["parents77"]),
|
||||
skeleton=StandInSkeleton(data["names30"], data["parents30"]),
|
||||
fps=FPS,
|
||||
)
|
||||
|
||||
def window(self, joints):
|
||||
(answer,) = loop_windows({"posed_joints": [joints]}, self.model)
|
||||
return answer["first"], answer["last"]
|
||||
|
||||
def test_the_real_take_gets_the_window_of_its_own_heading_in_the_output_skeleton(self):
|
||||
self.assertEqual(self.joints.shape, (240, 77, 3))
|
||||
self.assertEqual(self.model.output_skeleton.hip_joint_idx, [72, 67])
|
||||
self.assertEqual(self.model.skeleton.hip_joint_idx, [26, 22])
|
||||
self.assertEqual(self.window(self.joints), (35, 222))
|
||||
# The 30-joint skeleton's hip indices read the wrong joints of a 77-joint take and move the window.
|
||||
wrong = SimpleNamespace(output_skeleton=self.model.skeleton, fps=FPS)
|
||||
self.assertNotEqual(loop_windows({"posed_joints": [self.joints]}, wrong)[0]["first"], 35)
|
||||
|
||||
def test_turning_the_body_about_its_hips_by_a_smooth_angle_leaves_the_window(self):
|
||||
turned = np.stack([
|
||||
turn(self.joints[f] - self.joints[f, 0], 0.8 * math.sin(0.05 * f)) + self.joints[f, 0] for f in range(len(self.joints))
|
||||
]).astype(np.float32)
|
||||
self.assertEqual(self.window(turned), (35, 222))
|
||||
|
||||
def test_each_sample_gets_its_own_window_with_its_costs(self):
|
||||
still = np.repeat(self.joints[:1], 240, axis=0)
|
||||
answers = loop_windows({"posed_joints": [self.joints, still]}, self.model)
|
||||
self.assertEqual([(a["first"], a["last"]) for a in answers], [(35, 222), (0, 239)])
|
||||
self.assertEqual(set(answers[0]), {"first", "last", "poseCost", "velocityCost"})
|
||||
self.assertAlmostEqual(answers[1]["poseCost"], 0.0, places=6)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -592,8 +592,9 @@ async function readInside(file, directory) {
|
|||
// the BVH and writes a private FBX in a directory made for this export alone, and the FBX is then created beside the
|
||||
// BVH exclusively, so nothing planted at either path is read, written through or replaced. The FBX counts only if
|
||||
// Blender exits 0, prints its result line and its file exists. A failure names the exit or the signal and never
|
||||
// Blender's output.
|
||||
async function exportClip(bvh, fbx, clipName, timeoutMs, artifactDir) {
|
||||
// Blender's output. A window {first, last} (BVH frame indices, last inclusive) is handed to Blender as the frame
|
||||
// range to export, in Blender's frames, which count from 1; the daemon never chooses one.
|
||||
async function exportClip(bvh, fbx, clipName, timeoutMs, artifactDir, window = null) {
|
||||
const input = await readInside(bvh, artifactDir);
|
||||
if (input === null) return { error: "fbx-export-refused" };
|
||||
guard();
|
||||
|
|
@ -606,6 +607,7 @@ async function exportClip(bvh, fbx, clipName, timeoutMs, artifactDir) {
|
|||
const run = await runBlender([
|
||||
"-b", "--factory-startup", "--python-use-system-env", "--python-exit-code", "1",
|
||||
"--python", exportScript, "--", privateBvh, privateFbx, clipName,
|
||||
...(window ? ["--frames", String(window.first + 1), String(window.last + 1)] : []),
|
||||
], timeoutMs);
|
||||
let info = null;
|
||||
if (run.code === 0 && run.result !== null) {
|
||||
|
|
@ -648,19 +650,28 @@ async function sweepWork() {
|
|||
|
||||
// Every BVH the worker listed becomes an FBX, or the job fails. A listed BVH must be a regular file inside the job's
|
||||
// artifact directory, and that is decided when its export begins (exportClip), never earlier. Sample N of several is
|
||||
// <clipName>_NN.
|
||||
async function exportListed(listed, artifactDir, clipName, timeoutMs) {
|
||||
// <clipName>_NN, N being its own index. With loops (the worker's answer, an array when the request asked for a loop,
|
||||
// else undefined), sample k is exported over loops[k], and a job whose answer has no usable window for a sample fails.
|
||||
async function exportListed(listed, artifactDir, clipName, timeoutMs, loops = undefined) {
|
||||
const bvhs = [];
|
||||
for (const file of Array.isArray(listed) ? listed : []) {
|
||||
if (typeof file !== "string" || !file.endsWith(".bvh")) continue;
|
||||
bvhs.push(path.resolve(artifactDir, file));
|
||||
}
|
||||
if (bvhs.length === 0) return { clips: [], error: "fbx-export-failed (no bvh)" };
|
||||
bvhs.sort();
|
||||
// A sample's own index is in its file name (motion_NN.bvh, more digits past 99), so the order and the window come
|
||||
// from that number and never from sorting the names as text.
|
||||
const sampleOf = (bvh) => Number(/_(\d+)\.bvh$/.exec(bvh)?.[1] ?? 0);
|
||||
bvhs.sort((a, b) => sampleOf(a) - sampleOf(b) || (a < b ? -1 : a > b ? 1 : 0));
|
||||
const clips = [];
|
||||
for (const [index, bvh] of bvhs.entries()) {
|
||||
for (const bvh of bvhs) {
|
||||
const index = sampleOf(bvh);
|
||||
const name = bvhs.length === 1 ? clipName : `${clipName}_${String(index).padStart(2, "0")}`;
|
||||
const done = await exportClip(bvh, `${bvh.slice(0, -".bvh".length)}.fbx`, name, timeoutMs, artifactDir);
|
||||
const window = loops === undefined ? null : loops[index];
|
||||
if (window !== null && !(Number.isInteger(window?.first) && Number.isInteger(window?.last) && window.first >= 0 && window.last > window.first)) {
|
||||
return { clips, error: "fbx-export-failed (no loop window)" };
|
||||
}
|
||||
const done = await exportClip(bvh, `${bvh.slice(0, -".bvh".length)}.fbx`, name, timeoutMs, artifactDir, window);
|
||||
if (done.error) return { clips, error: done.error };
|
||||
clips.push({ ...done.clip, file: path.relative(artifactDir, done.clip.file).split(path.sep).join("/") });
|
||||
}
|
||||
|
|
@ -864,8 +875,10 @@ async function generate(request, response) {
|
|||
// One export plan: fbx implies the BVH and its standard T-pose, which the FBX is made from.
|
||||
const fbx = boolValue(body.fbx);
|
||||
const askedTpose = firstDefined(body.bvhStandardTpose, body.bvh_standard_tpose);
|
||||
if (fbx && ((askedTpose !== undefined && !boolValue(askedTpose))
|
||||
|| (body.clipName !== undefined && !(typeof body.clipName === "string" && CLIP_NAME.test(body.clipName))))) {
|
||||
const loop = body.loop === true;
|
||||
if ((body.loop !== undefined && typeof body.loop !== "boolean") || (loop && !fbx)
|
||||
|| (fbx && ((askedTpose !== undefined && !boolValue(askedTpose))
|
||||
|| (body.clipName !== undefined && !(typeof body.clipName === "string" && CLIP_NAME.test(body.clipName)))))) {
|
||||
json(response, 400, { ok: false, error: "Invalid fbx request." });
|
||||
return;
|
||||
}
|
||||
|
|
@ -878,6 +891,13 @@ async function generate(request, response) {
|
|||
bvhStandardTpose: fbx || boolValue(askedTpose),
|
||||
};
|
||||
|
||||
// One way to give constraints: the worker reads exactly one path, so two inputs would leave the record naming
|
||||
// constraints that did not make the clip.
|
||||
if ([body.constraints, body.constraints_path, body.constraintsPath].filter((given) => given !== undefined).length > 1) {
|
||||
json(response, 400, { ok: false, error: "Invalid constraints request." });
|
||||
return;
|
||||
}
|
||||
|
||||
const bodyMeta = typeof body.meta === "object" && body.meta ? body.meta : {};
|
||||
const askedSeed = firstDefined(body.seed, bodyMeta.seed);
|
||||
if (askedSeed !== undefined && !(Number.isInteger(askedSeed) && askedSeed >= 0 && askedSeed < SEED_LIMIT)) {
|
||||
|
|
@ -934,9 +954,10 @@ async function generate(request, response) {
|
|||
numSamples,
|
||||
diffusion_steps: diffusionSteps,
|
||||
numTransitionFrames: firstDefined(body.numTransitionFrames, body.num_transition_frames),
|
||||
constraints: constraintsPath,
|
||||
constraints: body.constraints !== undefined && typeof body.constraints !== "string" ? body.constraints : constraintsPath,
|
||||
seed,
|
||||
fbx: fbx || undefined,
|
||||
loop: loop || undefined,
|
||||
clipName,
|
||||
cfg: body.cfg,
|
||||
cfgType: firstDefined(body.cfgType, body.cfg_type),
|
||||
|
|
@ -967,6 +988,7 @@ async function generate(request, response) {
|
|||
numTransitionFrames: firstDefined(body.numTransitionFrames, body.num_transition_frames),
|
||||
constraints: constraintsPath,
|
||||
seed,
|
||||
loop: loop || undefined,
|
||||
cfg: body.cfg,
|
||||
cfgType: firstDefined(body.cfgType, body.cfg_type),
|
||||
cfgWeight: firstDefined(body.cfgWeight, body.cfg_weight),
|
||||
|
|
@ -993,7 +1015,7 @@ async function generate(request, response) {
|
|||
let exported = null;
|
||||
if (fbx && result.ok) {
|
||||
const timeoutMs = Math.min(EXPORT_TIMEOUT_MS, Math.max(1, Number(firstDefined(body.exportTimeoutMs, EXPORT_TIMEOUT_MS)) || EXPORT_TIMEOUT_MS));
|
||||
exported = await exportListed(result.body?.files, artifactDir, clipName, timeoutMs);
|
||||
exported = await exportListed(result.body?.files, artifactDir, clipName, timeoutMs, loop ? (Array.isArray(result.body?.loops) ? result.body.loops : []) : undefined);
|
||||
if (exported.error) result = { ok: false, error: exported.error, body: result.body };
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue