diff --git a/packages/cultmath/README.md b/packages/cultmath/README.md index 086dd222..381b194a 100644 --- a/packages/cultmath/README.md +++ b/packages/cultmath/README.md @@ -190,6 +190,8 @@ CultMath keeps portable DXC outside git under `packages/cultmath/.tools/`: -OutputPath E:\Projects\Odin\crates\muninn-move-tracker\artifacts\shader\MoveSphereCandidate.comp.metal ``` +`tools/glsl/build-glsl.sh ` (Linux, repository root) builds the library GLSL from the HLSL with dxc, SPIRV-Cross and SPIRV-Tools and writes `core.glsl`, `cycles.glsl` and `phacelle.glsl` bodies into ``. It does not yet own the committed GLSL; `GlslLowering` still does. `GLSL_DIR= tools/device/run-lavapipe.sh` runs that output on lavapipe and compares it with the HLSL results bit for bit instead of judging the committed files. That comparison is in strict form (dxc `-Gis` and a `NoContraction` decoration on every float result of both modules), and a consumer overrides `CULTMATH_MAX_SITES` in the built GLSL the same way as in HLSL: define it before including `core.glsl`. + `compile-hlsl.ps1` compiles both `CultMath.hlsl` copies as a `lib_6_3` library with the x64 DXC; a warning fails it too (`-WX`). DXC emits Vulkan SPIR-V with `-spirv`. `-TargetEnv vulkan1.2` is exposed by the diff --git a/packages/cultmath/tools/device/gen.py b/packages/cultmath/tools/device/gen.py index b408a4ea..52e074c7 100644 --- a/packages/cultmath/tools/device/gen.py +++ b/packages/cultmath/tools/device/gen.py @@ -115,7 +115,9 @@ def body(sig, n, win, wout, defs, glsl): def main(work, root): defs = structs(root) shaders = os.path.join(root, 'shaders') - library = ''.join(open(os.path.join(shaders, f)).read() for f in ('CultMath.glsl', 'CultMath.Cycles.glsl', 'CultMath.Phacelle.glsl')) + built = os.environ.get('GLSL_DIR') + parts = [os.path.join(built, f) for f in ('core.glsl', 'cycles.glsl', 'phacelle.glsl')] if built else [os.path.join(shaders, f) for f in ('CultMath.glsl', 'CultMath.Cycles.glsl', 'CultMath.Phacelle.glsl')] + library = ''.join(open(p).read() for p in parts) count = 0 for line in open(os.path.join(work, 'manifest.txt')): tag, _, sig, n, win, wout = line.rstrip('\n').split('\t') diff --git a/packages/cultmath/tools/device/nocontract.py b/packages/cultmath/tools/device/nocontract.py new file mode 100644 index 00000000..26a200a7 --- /dev/null +++ b/packages/cultmath/tools/device/nocontract.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""The strict-form harness pass of the GLSL equality gate: decorates the result of every OpFMul, OpFAdd, OpFSub, +OpVectorTimesScalar and OpDot NoContraction, so the driver may not fuse them into a multiply-add. Binary, instruction +level, reusing spvedit.py's load and mk. It lives in the gate's job-local modules only, never in shipped GLSL. + +usage: nocontract.py prints the count of decorated results""" +import importlib.util, os, struct, sys + +here = os.path.dirname(os.path.abspath(__file__)) +spec = importlib.util.spec_from_file_location('spvedit', os.path.join(here, '..', 'glsl', 'spvedit.py')) +spvedit = importlib.util.module_from_spec(spec) +spec.loader.exec_module(spvedit) + +OP_DOT, OP_FADD, OP_FSUB, OP_FMUL, OP_VECTOR_TIMES_SCALAR = 148, 129, 131, 133, 142 +FLOAT_OPS = (OP_FADD, OP_FSUB, OP_FMUL, OP_VECTOR_TIMES_SCALAR, OP_DOT) +DECORATION_NO_CONTRACTION = 42 +FIRST_TYPE_OP, LAST_TYPE_OP = 19, 39 # OpTypeVoid .. OpTypeForwardPointer: the annotations end where types begin + + +def main(src, dst): + h, ins = spvedit.load(src) + done = {x[1] for x in ins if spvedit.op(x) == spvedit.OP_DECORATE and x[2] == DECORATION_NO_CONTRACTION} + ids = [x[2] for x in ins if spvedit.op(x) in FLOAT_OPS and x[2] not in done] + at = next((k for k, x in enumerate(ins) if FIRST_TYPE_OP <= spvedit.op(x) <= LAST_TYPE_OP), None) + if at is None: + sys.exit('refuse: no type section to place the decorations before') + ins[at:at] = [spvedit.mk(spvedit.OP_DECORATE, i, DECORATION_NO_CONTRACTION) for i in ids] + words = h + [v for x in ins for v in x] + open(dst, 'wb').write(struct.pack('<%dI' % len(words), *words)) + print('nocontract: %d results decorated' % len(ids)) + + +if __name__ == '__main__': + main(*sys.argv[1:3]) diff --git a/packages/cultmath/tools/device/run-lavapipe.sh b/packages/cultmath/tools/device/run-lavapipe.sh index 22b04ece..2cbb1aaa 100644 --- a/packages/cultmath/tools/device/run-lavapipe.sh +++ b/packages/cultmath/tools/device/run-lavapipe.sh @@ -5,6 +5,11 @@ # # Run from the CultLib root inside mcr.microsoft.com/dotnet/sdk:10.0 (Ubuntu 24.04), for example # SUMMARY= tools/stopgap/ygg-verify.sh mcr.microsoft.com/dotnet/sdk:10.0 'bash packages/cultmath/tools/device/run-lavapipe.sh' +# GLSL_DIR= (the output of tools/glsl/build-glsl.sh) runs the built +# route instead of the committed one and skips the bounds judge: every lavapipe-glsl/.out must equal its +# lavapipe-hlsl/.out byte for byte, and a difference exits 1. It records no bounds. That compare is in strict form: +# dxc gets -Gis and nocontract.py decorates every float result of both modules NoContraction, so the two routes cannot +# differ by a fused multiply-add. # WRITE_BOUNDS=1 re-records packages/cultmath/tests/CultMath.Tests/fixtures/device-bounds-lavapipe.txt and prints it # between BOUNDS-BEGIN and BOUNDS-END, because the job's clone is discarded. set -euo pipefail @@ -30,20 +35,40 @@ export LD_LIBRARY_PATH=$(dirname "$DXC")/../lib:$(dirname "$DXC")${LD_LIBRARY_PA gcc -O2 -o "$W/vkrun" "$DEV/vkrun.c" -lvulkan dotnet build "$TESTS" -nologo -v q >/dev/null +GLSL_DIR=${GLSL_DIR:+$(realpath "$GLSL_DIR")}; export GLSL_DIR export CULTMATH_DEVICE_WORK=$W/work dotnet test "$TESTS" --no-build --filter "FullyQualifiedName~GlslMirrorTests.WriteDeviceWork" python3 "$DEV/gen.py" "$W/work" "$CM" +STRICT=(); [ -z "${GLSL_DIR:-}" ] || STRICT=(-Gis) while IFS=$'\t' read -r tag _ _ n _ wout; do in=$W/work/$tag.in - "$DXC" -T cs_6_0 -E main -HV 2021 -spirv -fspv-target-env=vulkan1.2 -I "$CM/shaders" "$W/work/$tag.hlsl" -Fo "$W/work/$tag.h.spv" >"$W/work/$tag.h.log" 2>&1 \ + "$DXC" -T cs_6_0 -E main -HV 2021 "${STRICT[@]}" -spirv -fspv-target-env=vulkan1.2 -I "$CM/shaders" "$W/work/$tag.hlsl" -Fo "$W/work/$tag.h.spv" >"$W/work/$tag.h.log" 2>&1 \ || { echo "dxc failed for $tag"; head -5 "$W/work/$tag.h.log"; exit 1; } glslangValidator -V --target-env vulkan1.2 -o "$W/work/$tag.g.spv" "$W/work/$tag.comp" >"$W/work/$tag.g.log" 2>&1 \ || { echo "glslang failed for $tag"; head -5 "$W/work/$tag.g.log"; exit 1; } + if [ -n "${GLSL_DIR:-}" ]; then + for m in h g; do + python3 "$DEV/nocontract.py" "$W/work/$tag.$m.spv" "$W/work/$tag.$m.nc.spv" >>"$W/nocontract.txt" + mv "$W/work/$tag.$m.nc.spv" "$W/work/$tag.$m.spv" + done + fi "$W/vkrun" "$W/work/$tag.h.spv" "$in" "$W/results/lavapipe-hlsl/$tag.out" "$n" "$wout" >>"$W/device.txt" "$W/vkrun" "$W/work/$tag.g.spv" "$in" "$W/results/lavapipe-glsl/$tag.out" "$n" "$wout" >>"$W/device.txt" done <"$W/work/manifest.txt" +if [ -n "${GLSL_DIR:-}" ]; then + total=0; equal=0 + for h in "$W"/results/lavapipe-hlsl/*.out; do + tag=$(basename "$h" .out); total=$((total + 1)) + if cmp -s "$h" "$W/results/lavapipe-glsl/$tag.out"; then equal=$((equal + 1)); else echo "differs: $tag"; fi + done + echo "strict form: $(wc -l <"$W/nocontract.txt") modules rewritten, $(awk '{n += $2} END {print n}' "$W/nocontract.txt") results decorated NoContraction" + echo "lavapipe-glsl equals lavapipe-hlsl: $equal/$total" + [ "$equal" = "$total" ] && [ "$total" -gt 0 ] + exit +fi + RENDERER=$(sort -u "$W/device.txt") [ "$(echo "$RENDERER" | wc -l)" = 1 ] || { echo "the runs saw more than one device: $RENDERER"; exit 1; } export CULTMATH_DEVICE_NAME=${RENDERER#device=} diff --git a/packages/cultmath/tools/glsl/build-glsl.sh b/packages/cultmath/tools/glsl/build-glsl.sh new file mode 100644 index 00000000..3281d545 --- /dev/null +++ b/packages/cultmath/tools/glsl/build-glsl.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +# Builds the library GLSL from the HLSL: dxc lib_6_3 -spirv -Od, one SPIR-V edit, a fixed spirv-opt pair, SPIRV-Cross, +# partition. Linux, run from the repository root: build-glsl.sh +# Writes core.glsl, cycles.glsl and phacelle.glsl bodies into ; the committed GLSL is untouched. +# Tools are cached in GLSL_TOOLS (default packages/cultmath/.tools/glsl, gitignored): the pinned dxc tarball and +# SPIRV-Cross, SPIRV-Tools, SPIRV-Headers at TAG built with cmake. +set -euo pipefail +OUT=$(realpath -m "${1:?usage: build-glsl.sh }") +ROOT=$(pwd) +CM=$ROOT/packages/cultmath +G=$CM/tools/glsl +TOOLS=$(realpath -m "${GLSL_TOOLS:-$CM/.tools/glsl}") +TAG=vulkan-sdk-1.4.363.0 +DXC_URL=https://github.com/microsoft/DirectXShaderCompiler/releases/download/v1.9.2607/linux_dxc_2026_07_29.x86_x64.tar.gz +DXC_SHA=55665c87824051ed4774ff3280a79ccbbb7d39243b9736ca5e98222134112d54 +W=$(mktemp -d) +mkdir -p "$TOOLS" "$OUT" + +export DEBIAN_FRONTEND=noninteractive +apt-get update -qq >/dev/null +apt-get install -y -qq --no-install-recommends ca-certificates curl git cmake g++ make python3 glslang-tools >/dev/null + +if [ ! -x "$TOOLS/dxc.ok" ]; then + curl -sL --fail -o "$TOOLS/dxc.tgz" "$DXC_URL" + echo "$DXC_SHA $TOOLS/dxc.tgz" | sha256sum -c - >/dev/null || { echo "dxc tarball sha256 differs from the pinned one"; rm -f "$TOOLS/dxc.tgz"; exit 1; } + rm -rf "$TOOLS/dxc"; mkdir -p "$TOOLS/dxc"; tar xzf "$TOOLS/dxc.tgz" -C "$TOOLS/dxc"; : >"$TOOLS/dxc.ok" +fi +DXC=$(find "$TOOLS/dxc" -name dxc -type f | head -1) +export LD_LIBRARY_PATH=$(dirname "$DXC")/../lib:$(dirname "$DXC")${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} + +if [ ! -x "$TOOLS/bin/spirv-cross" ]; then + git clone -q --depth 1 --branch $TAG https://github.com/KhronosGroup/SPIRV-Cross.git "$TOOLS/sc-src" + cmake -S "$TOOLS/sc-src" -B "$TOOLS/sc-build" -DCMAKE_BUILD_TYPE=Release -DSPIRV_CROSS_ENABLE_TESTS=OFF -DSPIRV_CROSS_CLI=ON >/dev/null + cmake --build "$TOOLS/sc-build" --target spirv-cross -j8 >/dev/null + mkdir -p "$TOOLS/bin"; cp "$TOOLS/sc-build/spirv-cross" "$TOOLS/bin/" +fi +if [ ! -x "$TOOLS/bin/spirv-opt" ]; then + git clone -q --depth 1 --branch $TAG https://github.com/KhronosGroup/SPIRV-Tools.git "$TOOLS/st-src" + git clone -q --depth 1 --branch $TAG https://github.com/KhronosGroup/SPIRV-Headers.git "$TOOLS/st-src/external/spirv-headers" + cmake -S "$TOOLS/st-src" -B "$TOOLS/st-build" -DCMAKE_BUILD_TYPE=Release -DSPIRV_SKIP_TESTS=ON -DSPIRV_WERROR=OFF >/dev/null + cmake --build "$TOOLS/st-build" --target spirv-opt spirv-val -j8 >/dev/null + mkdir -p "$TOOLS/bin"; cp "$TOOLS/st-build/tools/spirv-opt" "$TOOLS/st-build/tools/spirv-val" "$TOOLS/bin/" +fi +export PATH=$TOOLS/bin:$PATH +echo "dxc: $("$DXC" --version | head -1)"; echo "spirv-opt: $(spirv-opt --version | head -1)" + +pipeline() { # shaders dir, work dir, out dir: HLSL to partitioned GLSL; returns 1 at the first refusal + local sh=$1 w=$2 out=$3 + mkdir -p "$w" "$out" + python3 "$G/wrappers.py" "$sh" "$w/shw" || return 1 + "$DXC" -T lib_6_3 -HV 2021 -spirv -fspv-target-env=universal1.5 -Od -I "$w/shw" "$w/shw/cm_wrappers.hlsl" -Fo "$w/lib.spv" >"$w/dxc.log" 2>&1 || { head -20 "$w/dxc.log"; return 1; } + python3 "$G/spvedit.py" "$w/lib.spv" "$w/edited.spv" "$w/shw/wrappers.tsv" "$w/shw/capacity.tsv" || return 1 + spirv-val --target-env spv1.5 "$w/edited.spv" || return 1 + spirv-opt --target-env=spv1.5 --eliminate-local-single-store --eliminate-dead-code-aggressive "$w/edited.spv" -o "$w/opt.spv" || return 1 + spirv-val --target-env spv1.5 "$w/opt.spv" || return 1 + spirv-cross "$w/opt.spv" --es --version 310 --relax-nan-checks --output "$w/cross.glsl" || return 1 + python3 "$G/partition.py" "$sh" "$w/cross.glsl" "$out" || return 1 +} +pipeline "$CM/shaders" "$W" "$OUT" || exit 1 + +# Consumer checks: glslang ES 3.00 fragment compiles of every include combination, and a consumer calling overloads +# with an r-value argument. +CONSUMER='out highp vec4 c; +void main() { vec3 p = gl_FragCoord.xyz; c = vec4(cultmath_lerp(p, p.yzx, vec3(0.5)), cultmath_lerp(1.0, 2.0, p.x)) + cultmath_saturate(vec4(p, 1.0)) + vec4(cultmath_smootherstep(gl_FragCoord.x + 1.0)); }' +for combo in "core" "core cycles" "core phacelle" "core cycles phacelle"; do + f=$W/check.frag; printf '#version 300 es\nprecision highp float;\nprecision highp int;\n' >"$f" + for part in $combo; do cat "$OUT/$part.glsl" >>"$f"; done + printf '%s\n' "$CONSUMER" >>"$f" + glslangValidator "$f" >"$W/check.log" 2>&1 || { echo "ES 3.00 compile failed for [$combo]"; head -8 "$W/check.log"; exit 1; } + echo "ES 3.00 ok: $combo" +done + +# Capacity checks: a consumer's prior #define CULTMATH_MAX_SITES changes the array type through dxc, the committed GLSL +# and the built GLSL, and 128 still works with no define. +printf '#define CULTMATH_MAX_SITES 4\n#include "CultMath.hlsl"\nfloat4 cap4(float3 p, float4 s[4]) { return cultmath_power_border(p, s, 4); }\n' >"$W/shw/cap4.hlsl" +"$DXC" -T lib_6_3 -HV 2021 -Od -I "$W/shw" "$W/shw/cap4.hlsl" -Fo "$W/cap4.dxil" >"$W/cap4.log" 2>&1 || { echo "capacity (a) failed: dxc with CULTMATH_MAX_SITES 4"; head -8 "$W/cap4.log"; exit 1; } +echo "capacity (a) ok: dxc lib_6_3 with a prior define 4 takes float4 s[4]" +consume() { # label, expected (ok|fail), define lines, GLSL library file, main body + local f=$W/cap.frag + printf '#version 300 es +precision highp float; +precision highp int; +%s +' "$3" >"$f"; cat "$4" >>"$f" + printf 'out highp vec4 c; +void main() { %s } +' "$5" >>"$f" + local got=fail; glslangValidator "$f" >"$W/cap.log" 2>&1 && got=ok + [ "$got" = "$2" ] || { echo "capacity $1 failed: expected $2, got $got"; head -6 "$W/cap.log"; exit 1; } + echo "capacity $1 ok: $2" +} +cap_check() { # label, expected, define lines, GLSL library file, site array length + consume "$1" "$2" "$3" "$4" "vec4 s[$5]; for (int i = 0; i < $5; i++) s[i] = vec4(float(i)); c = cultmath_power_border(gl_FragCoord.xyz, s, 4);" +} +D4='#define CULTMATH_MAX_SITES 4' +cap_check "(b) built, define 4, vec4 s[4]" ok "$D4" "$OUT/core.glsl" 4 +cap_check "(c) built, define 4, vec4 s[128]" fail "$D4" "$OUT/core.glsl" 128 +cap_check "(d) built, no define, vec4 s[128]" ok "" "$OUT/core.glsl" 128 +cap_check "(e) committed, define 4, vec4 s[4]" ok "$D4" "$CM/shaders/CultMath.glsl" 4 +cap_check "(e) committed, define 4, vec4 s[128]" fail "$D4" "$CM/shaders/CultMath.glsl" 128 + +for part in core cycles phacelle; do + g=$OUT/$part.glsl + fn=$(grep -cE '^[A-Za-z_]\w* [A-Za-z_]\w*\(.*\)$' "$g" || true) + names=$(grep -oE '^[A-Za-z_]\w* [A-Za-z_]\w*\(' "$g" | awk '{print $2}' | sort -u | wc -l) + echo "$part: functions $fn, distinct names $names, lines $(wc -l <"$g"), bytes $(wc -c <"$g"), sha256 $(sha256sum "$g" | cut -c1-64)" +done +echo "suffixed ids: $(cat "$OUT"/*.glsl | grep -oE '\bcultmath_\w+_[0-9]+\(' | sort -u | wc -l); inout: $(cat "$OUT"/*.glsl | grep -c inout || true)" +# Build shape: the lowering leaves none of these, and the NaN-check option leaves no mix(mix( expansions. A literal +# [128] is not among them: the capacity is selected by name, so a library array of length 128 is legitimate output. +SHAPE='inout|param_var_|cmw[0-9]{3}_|export |mix\(mix\(|\b_cos\b|\b_sin\b' +shape_check() { # output directory; returns 1 when the shape fails + if cat "$1"/*.glsl | grep -Eq "$SHAPE"; then + echo "build shape failed:"; cat "$1"/*.glsl | grep -Eo "$SHAPE" | sort | uniq -c; return 1 + fi +} +shape_check "$OUT" || exit 1 +echo "build shape ok" + +# Tool refusals, and the capacity's reach: each variant appends one probe function to a scratch copy of CultMath.hlsl and +# runs the whole pipeline on it. +variant() { # label, probe HLSL; leaves the variant directory in VD + VD=$W/v/$1; rm -rf "$VD"; mkdir -p "$VD"; cp -r "$CM/shaders" "$VD/sh" + printf ' +%s +' "$2" >>"$VD/sh/CultMath.hlsl" + pipeline "$VD/sh" "$VD/w" "$VD/out" >"$VD/log" 2>&1 +} +expect_refused() { # label, probe HLSL, pattern the refusal prints + if variant "$1" "$2"; then echo "refusal $1 failed: the build passed"; exit 1; fi + grep -q "$3" "$VD/log" || { echo "refusal $1 failed: no '$3' in the log"; tail -5 "$VD/log"; exit 1; } + echo "refusal $1 ok" +} +SENTINEL=$(cut -f2 "$W/shw/capacity.tsv") +expect_refused macro-in-expression 'int cultmath_zz_probe(int n) { return n + CULTMATH_MAX_SITES; }' 'outside its guard' +# the sentinel spelled in hex: wrappers.py's scan reads decimal tokens, so only the length check can see it +expect_refused length-in-expression "uint cultmath_zz_probe(uint n) { return n + $(printf '0x%xu' "$SENTINEL"); }" 'also used by opcode' +# the sentinel as another element type's array length: not a capacity parameter type +expect_refused other-element-same-length "float3 cultmath_zz_probe(float3 x[$(printf '0x%x' "$SENTINEL")], int n) { return x[n]; }" 'not a capacity parameter type' +variant array-128 'float4 cultmath_zz_probe(float4 x[128], int n) { return x[n]; }' || { echo "array-128 failed: a library array of length 128 broke the build"; tail -5 "$VD/log"; exit 1; } +grep -Eq 'cultmath_zz_probe\(.*\[128\]' "$VD/out/core.glsl" || { echo "array-128 failed: the 128 array is not literal in core.glsl"; exit 1; } +shape_check "$VD/out" || { echo "array-128 failed: the shape check refused a library array of length 128"; exit 1; } +PROBE='vec4 x[128]; c = cultmath_zz_probe(x, 1);' +consume "(f) non-site array, define 4, vec4 x[128]" ok "$D4" "$VD/out/core.glsl" "$PROBE" +consume "(f) non-site array, define 4, vec4 x[4]" fail "$D4" "$VD/out/core.glsl" 'vec4 x[4]; c = cultmath_zz_probe(x, 1);' +consume "(g) consumer's own SPIRV_CROSS_CONSTANT_ID_0" ok '#define SPIRV_CROSS_CONSTANT_ID_0 7u' "$OUT/core.glsl" 'c = vec4(float(cultmath_max_sites));' diff --git a/packages/cultmath/tools/glsl/partition.py b/packages/cultmath/tools/glsl/partition.py new file mode 100644 index 00000000..415c9b2b --- /dev/null +++ b/packages/cultmath/tools/glsl/partition.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""Partition the SPIRV-Cross GLSL of the whole library into the three files a consumer includes. + +usage: partition.py +Each top-level struct and function chunk goes to the file whose HLSL defines it (CultMath, Interval and Affine to core; +Cycles; Phacelle). The library-header prologue and any main are dropped. Writes core.glsl, cycles.glsl and +phacelle.glsl bodies, with the 13 `static const float` constants of the HLSL as `const float NAME = ;` at the +head of the file that defines them. core.glsl opens with the HLSL's own CULTMATH_MAX_SITES guard lines, one line mapping +SPIRV_CROSS_CONSTANT_ID_ (CultMath's own, 'CMS1') to uint(CULTMATH_MAX_SITES), and SPIRV-Cross's one spec-constant block. Licence lines, include guards and the #version contract belong to the caller. +Refuses: a static const line that is not `static const float CULTMATH_X = ;`, an unknown-origin chunk, a +duplicate chunk, an unexpected prologue line, a missing or other SPIRV_CROSS_CONSTANT_ID block, and a call from a non-core chunk into a different non-core origin.""" +import collections, os, re, sys +from spvedit import SPEC_ID + +OWNER = {'CultMath.hlsl': 'core', 'CultMath.Interval.hlsl': 'core', 'CultMath.Affine.hlsl': 'core', + 'CultMath.Cycles.hlsl': 'cycles', 'CultMath.Phacelle.hlsl': 'phacelle'} +CONST = re.compile(r'^static const float (CULTMATH_\w+) = (-?\d+(?:\.\d*)?(?:[eE][+-]?\d+)?);$') +PROLOGUE = re.compile(r'^(#version |#extension |precision |#ifdef SPIRV_CROSS|#define SPIRV_CROSS|#endif)') +GUARD = re.compile(r'#ifndef CULTMATH_MAX_SITES\n#define CULTMATH_MAX_SITES (\d+)\n#endif\n') +ID = 'SPIRV_CROSS_CONSTANT_ID_%d' % SPEC_ID +SPEC = re.compile(r'#ifndef %s\n#define %s (\d+)u\n#endif\n' + r'const uint cultmath_max_sites = %s;\n?' % (ID, ID, ID)) +STRUCT, FUNC = re.compile(r'^struct (\w+)$'), re.compile(r'^\w+ (\w+)\(.*\)$') + + +def origins(shaders): + origin, consts = {}, collections.defaultdict(list) + for fn, o in OWNER.items(): + text = open(os.path.join(shaders, fn)).read() + for m in re.finditer(r'(?m)^\w+\s+(cultmath_\w+)\s*\(', text): + origin.setdefault(m.group(1), o) + for m in re.finditer(r'(?m)^struct\s+(\w+)', text): + origin.setdefault(m.group(1), o) + for line in text.splitlines(): + if line.startswith('static const'): + m = CONST.match(line) + if not m: + sys.exit('refuse: %s has a static const line that is not `static const float CULTMATH_X = ;`' % fn) + consts[o].append('const float %s = %s;' % m.groups()) + return origin, consts + + +def chunks(lines): + out, prologue, i = [], [], 0 + while i < len(lines): + m = STRUCT.match(lines[i]) or FUNC.match(lines[i]) + if not m: + if lines[i].strip(): + prologue.append(lines[i]) + i += 1 + continue + j = i + while lines[j] not in ('};', '}'): + j += 1 + if m.group(1) != 'main': + out.append((m.group(1), '\n'.join(lines[i:j + 1]))) + i = j + 1 + return out, prologue + + +def main(shaders, glsl, outdir): + origin, consts = origins(shaders) + guard = GUARD.search(open(os.path.join(shaders, 'CultMath.hlsl')).read().replace('\r\n', '\n')) + if not guard: + sys.exit('refuse: CultMath.hlsl has no CULTMATH_MAX_SITES guard') + parts, prologue = chunks(open(glsl).read().split('\n')) + joined = '\n'.join(prologue) + '\n' + spec = SPEC.search(joined) + text, found = SPEC.subn('', joined) + if found != 1 or spec.group(1) != guard.group(1): + sys.exit('refuse: expected exactly one spec-constant block defaulting to the guard value, found %d' % found) + odd = [l for l in text.split('\n') if l and (not PROLOGUE.match(l) or 'SPIRV_CROSS_CONSTANT_ID' in l)] + if odd: + sys.exit('refuse: %d unexpected prologue lines, first at %r' % (len(odd), odd[0][:60])) + seen = collections.Counter(b.split(chr(10))[0] for _, b in parts) # overloads share a name, never a signature + dup = sorted(n for n, c in seen.items() if c > 1) + if dup: + sys.exit('refuse: duplicate chunks %s' % dup[:6]) + unknown = sorted(n for n, _ in parts if n not in origin) + if unknown: + sys.exit('refuse: %d unknown-origin chunks %s' % (len(unknown), unknown[:8])) + files = collections.defaultdict(list) + for name, body in parts: + o = origin[name] + for callee in set(re.findall(r'\b(cultmath_\w+|[A-Z]\w+)\b', body)) - {name}: + co = origin.get(callee) + if o != 'core' and co and co != 'core' and co != o: + sys.exit('refuse: %s (%s) calls %s (%s)' % (name, o, callee, co)) + files[o].append(body) + os.makedirs(outdir, exist_ok=True) + for o in OWNER.values(): + head = [guard.group(0).rstrip('\n'), '#define %s uint(CULTMATH_MAX_SITES)' % ID, spec.group(0).rstrip('\n')] if o == 'core' else [] + text = '\n'.join(head + consts[o] + files[o]) + '\n' + open(os.path.join(outdir, o + '.glsl'), 'w').write(text) + print('partition:', {o: len(files[o]) for o in sorted(files)}, 'constants', sum(len(v) for v in consts.values())) + + +if __name__ == '__main__': + main(*sys.argv[1:4]) diff --git a/packages/cultmath/tools/glsl/spvedit.py b/packages/cultmath/tools/glsl/spvedit.py new file mode 100644 index 00000000..a5707518 --- /dev/null +++ b/packages/cultmath/tools/glsl/spvedit.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 +"""The one instruction-level SPIR-V edit of the GLSL build. Binary, never text. + +usage: spvedit.py +1. Keeps only the capabilities Shader and Linkage and drops every OpExtension. +2. Deletes each wrapper function with its OpName and decorations, moves its LinkageAttributes Export to its callee, and + sets the callee's OpName to the source name. Refuses a wrapper that is not exactly one OpFunctionCall. +3. By-value lowering: dxc -Od passes every parameter as a Function pointer and the caller copies in, so a callee that + assigns its parameter comes out of SPIRV-Cross as `inout`. Each Function-pointer OpFunctionParameter becomes a value + parameter with a new id; the old id stays as an entry-block OpVariable stored from it (OpName copied to the new id); + OpTypeFunction is reused if the signature exists, else added; each OpFunctionCall loads the pointer it passed. +The header bound is last id + 1.""" +import struct, sys + +OP_NAME, OP_MEMBER_NAME, OP_CAPABILITY, OP_EXTENSION, OP_DECORATE, OP_MEMBER_DECORATE = 5, 6, 17, 10, 71, 72 +OP_TYPE_ARRAY, OP_CONSTANT, OP_SPEC_CONSTANT, DECORATION_SPEC_ID = 28, 43, 50, 1 +SPEC_ID = int.from_bytes(b'CMS1', 'big') # CultMath's own id: a consumer's SPIRV_CROSS_CONSTANT_ID_n macro cannot meet it +OP_TYPE_POINTER, OP_TYPE_FUNCTION, OP_FUNCTION, OP_PARAMETER, OP_FUNCTION_END = 32, 33, 54, 55, 56 +OP_CALL, OP_LABEL, OP_VARIABLE, OP_LOAD, OP_STORE = 57, 248, 59, 61, 62 +CAP_SHADER, CAP_LINKAGE, DECORATION_LINKAGE, LINKAGE_EXPORT, FUNCTION_STORAGE = 1, 5, 41, 41, 7 +TERMINATORS = (8, 317, 62, 246, 247, 249, 250, 253, 254, 252) + + +def load(path): + raw = open(path, 'rb').read() + words = list(struct.unpack('<%dI' % (len(raw) // 4), raw)) + ins, i = [], 5 + while i < len(words): + n = words[i] >> 16 + ins.append(words[i:i + n]) + i += n + return words[:5], ins + + +def op(x): return x[0] & 0xffff +def mk(o, *ws): return [((len(ws) + 1) << 16) | o, *ws] + + +def decode(ws): + b = struct.pack('<%dI' % len(ws), *ws) + return b[:b.index(0)].decode() + + +def encode(s): + b = s.encode() + b'\0' + b += b'\0' * (-len(b) % 4) + return list(struct.unpack('<%dI' % (len(b) // 4), b)) + + +def functions(ins): + """function id -> [first index, last index, callee ids, ids defined inside]""" + fs, cur = {}, None + for k, x in enumerate(ins): + o = op(x) + if o == OP_FUNCTION: + cur = x[2] + fs[cur] = [k, None, [], set()] + if cur is None: + continue + if o == OP_CALL: + fs[cur][2].append(x[3]) + if o == OP_LABEL: + fs[cur][3].add(x[1]) + elif o not in TERMINATORS and len(x) > 2: + fs[cur][3].add(x[2]) + if o == OP_FUNCTION_END: + fs[cur][1] = k + cur = None + return fs + + +def export_edit(h, ins, rows): + names = {x[1]: decode(x[2:]) for x in ins if op(x) == OP_NAME} + fs = functions(ins) + by_name = {v: k for k, v in names.items() if k in fs} + drop_k, drop_ids, set_name, retarget = set(), set(), {}, {} + callee_of = [] + for wn, name, _, _, _ in rows: + fid = by_name.get(wn) + callees = fs[fid][2] if fid else None + if not callees or len(callees) != 1: + sys.exit('refuse: wrapper %s is not exactly one OpFunctionCall' % wn) + a, b, _, ids = fs[fid] + drop_k.update(range(a, b + 1)) + drop_ids |= ids | {fid} + set_name[callees[0]] = name + retarget[fid] = callees[0] + callee_of.append(callees[0]) + out, named = [], set() + for k, x in enumerate(ins): + if k in drop_k: + continue + o = op(x) + if o == OP_CAPABILITY and x[1] not in (CAP_SHADER, CAP_LINKAGE): + continue + if o == OP_EXTENSION: + continue + if o == OP_NAME and x[1] in set_name: + x = mk(OP_NAME, x[1], *encode(set_name[x[1]])) + named.add(x[1]) + elif o in (OP_NAME, OP_MEMBER_NAME) and x[1] in drop_ids: + continue + if o == OP_DECORATE and x[1] in drop_ids: + if x[1] in retarget and x[2] == DECORATION_LINKAGE: + x = [x[0], retarget[x[1]]] + x[2:] + else: + continue + out.append(x) + for fid, name in set_name.items(): + if fid not in named: + out.insert(next(i for i, x in enumerate(out) if op(x) == OP_NAME), mk(OP_NAME, fid, *encode(name))) + return out, callee_of + + +def by_value(h, ins): + bound = h[3] + ptr = {x[1]: (x[2], x[3]) for x in ins if op(x) == OP_TYPE_POINTER} # id -> (storage, pointee) + ftype = {x[1]: x for x in ins if op(x) == OP_TYPE_FUNCTION} + fnparams, fntype, cur = {}, {}, None + for x in ins: # per function: which parameters are Function pointers + if op(x) == OP_FUNCTION: + cur = x[2] + fnparams[cur], fntype[cur] = [], x[4] + elif op(x) == OP_PARAMETER and cur is not None: + t = x[1] + conv = t in ptr and ptr[t][0] == FUNCTION_STORAGE + fnparams[cur].append((x[2], t, ptr[t][1] if conv else None, bound if conv else None)) + bound += conv + newid = {p[0]: p[3] for ps in fnparams.values() for p in ps if p[2]} + newtype, typeins = {}, [] + sig = {tuple(x[2:]): x[1] for x in ins if op(x) == OP_TYPE_FUNCTION} + for f, ps in fnparams.items(): + if any(p[2] for p in ps): + key = (ftype[fntype[f]][2], *[p[2] if p[2] else p[1] for p in ps]) + if key not in sig: + bound += 1 + sig[key] = bound + typeins.append(mk(OP_TYPE_FUNCTION, bound, *key)) + newtype[f] = sig[key] + out, cur, pend, need_stores, first, pc = [], None, [], False, True, 0 + for x in ins: + o = op(x) + if o == OP_FUNCTION and first: + out += typeins + first = False + if need_stores and o != OP_VARIABLE: + out += [mk(OP_STORE, p, q) for p, _, q in pend] + pend, need_stores = [], False + if o == OP_FUNCTION: + cur, pc = x[2], 0 + if cur in newtype: + x = x[:4] + [newtype[cur]] + x[5:] + elif o == OP_PARAMETER: + _, t, pointee, q = fnparams[cur][pc] + pc += 1 + if pointee: + out.append(mk(OP_PARAMETER, pointee, q)) + pend.append((x[2], t, q)) + continue + elif o == OP_LABEL and pend: + out.append(x) + out += [mk(OP_VARIABLE, t, p, FUNCTION_STORAGE) for p, t, _ in pend] + need_stores = True + continue + elif o == OP_CALL: + args = [] + for a, (_, _, pointee, _) in zip(x[4:], fnparams[x[3]]): + if pointee: + bound += 1 + out.append(mk(OP_LOAD, pointee, bound, a)) + args.append(bound) + else: + args.append(a) + x = mk(OP_CALL, *x[1:4], *args) + out.append(x) + if o == OP_NAME and x[1] in newid: + out.append([x[0], newid[x[1]]] + x[2:]) + h[3] = bound + 1 + return out, len(newtype), len(newid) + + +def capacity_edit(h, ins, rows, callee_of, default): + """Each capacity parameter (wrappers.tsv column 5) of each callee has an OpTypeArray type. Exactly one length id may + serve all of them, and nothing but OpTypeArray, OpName and decorations may use it. That OpConstant becomes an + OpSpecConstant with SpecId SPEC_ID named cultmath_max_sites, defaulting to the HLSL guard value. The constant is the + sentinel wrappers.py compiled the library with, so no array outside the capacity parameters shares its length id. + Returns the instructions and the spec constant id.""" + ftype = {x[1]: x for x in ins if op(x) == OP_TYPE_FUNCTION} + arrays = {x[1]: x for x in ins if op(x) == OP_TYPE_ARRAY} + params, cur = {}, None + for x in ins: + if op(x) == OP_FUNCTION: + cur = x[2] + params[cur] = [] + elif op(x) == OP_PARAMETER: + params[cur].append(x[1]) + lengths = set() + for row, fid in zip(rows, callee_of): + for i in [int(c) for c in row[4].split(',') if c]: + t = params[fid][i] + if t not in arrays: + sys.exit('refuse: capacity parameter %d of %s is not an array type' % (i, row[1])) + lengths.add(arrays[t][3]) + if len(lengths) != 1: + sys.exit('refuse: %d array length ids serve the capacity parameters, expected exactly one' % len(lengths)) + cap = lengths.pop() + site_types = {params[fid][int(c)] for row, fid in zip(rows, callee_of) for c in row[4].split(',') if c} + if {t for t, a in arrays.items() if a[3] == cap} != site_types: + sys.exit('refuse: an array type of the capacity length is not a capacity parameter type') + for x in ins: + o = op(x) + defines = o in (OP_CONSTANT, OP_SPEC_CONSTANT) and x[2] == cap + allowed = o in (OP_TYPE_ARRAY, OP_NAME, OP_DECORATE, OP_MEMBER_DECORATE) or defines + if cap in x[1:] and not allowed: + sys.exit('refuse: the capacity length id %d is also used by opcode %d' % (cap, o)) + out = [] + for x in ins: + if op(x) == OP_CONSTANT and x[2] == cap: + x = [(x[0] & 0xffff0000) | OP_SPEC_CONSTANT, x[1], x[2], default] + out.append(x) + first = next((k for k, x in enumerate(out) if op(x) in (OP_DECORATE, OP_MEMBER_DECORATE)), None) + if first is None: + sys.exit('refuse: no annotation section to decorate the capacity constant in') + out[first:first] = [mk(OP_NAME, cap, *encode('cultmath_max_sites')), mk(OP_DECORATE, cap, DECORATION_SPEC_ID, SPEC_ID)] + return out, cap + + +def main(src, dst, tsv, capacity): + h, ins = load(src) + rows = [l.split('\t') for l in open(tsv).read().splitlines() if l] + ins, callee_of = export_edit(h, ins, rows) + ins, changed, params = by_value(h, ins) + default = int(open(capacity).read().split()[0]) + ins, spec = capacity_edit(h, ins, rows, callee_of, default) + words = h + [v for x in ins for v in x] + open(dst, 'wb').write(struct.pack('<%dI' % len(words), *words)) + print('spvedit: %d wrappers dropped; by-value: %d functions changed, %d parameters; capacity spec constant id %d' % (len(rows), changed, params, spec)) + + +if __name__ == '__main__': + main(*sys.argv[1:5]) diff --git a/packages/cultmath/tools/glsl/wrappers.py b/packages/cultmath/tools/glsl/wrappers.py new file mode 100644 index 00000000..742881ac --- /dev/null +++ b/packages/cultmath/tools/glsl/wrappers.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Build-only export wrappers: one exported, uniquely named wrapper per top-level cultmath_ definition, so dxc keeps +every overload as its own function. The shaders are copied unchanged; the unique names exist only in . + +usage: wrappers.py +Writes /cm_wrappers.hlsl, /wrappers.tsv (wrapper, name, file, params, capacity) and +/capacity.tsv (the guard default, the sentinel). cm_wrappers.hlsl defines CULTMATH_MAX_SITES as the sentinel, +a number no shader file spells, so the site arrays get an array length id of their own and no other array shares it. An argument is the +parameter's declarator name with any [..] suffix removed; capacity lists the indices of the parameters declared +[CULTMATH_MAX_SITES]. Definitions with a Texture or Sampler parameter get no wrapper. Refuses a file where the +definition-looking lines differ from the parsed ones, and a file that names CULTMATH_MAX_SITES anywhere but its +#ifndef/#define/#endif guard and array declarators.""" +import os, re, shutil, sys + +FILES = ['CultMath.hlsl', 'CultMath.Interval.hlsl', 'CultMath.Affine.hlsl', 'CultMath.Cycles.hlsl', 'CultMath.Phacelle.hlsl'] +SIG = re.compile(r'(?m)^(\w+)[ \t]+(cultmath_\w+)[ \t]*\(([^)]*)\)') +LOOKS = re.compile(r'(?m)^\w+ cultmath_\w+\(') +GUARD = re.compile(r'#ifndef CULTMATH_MAX_SITES\n#define CULTMATH_MAX_SITES (\d+)\n#endif\n') +COMMENT = re.compile(r'//[^\n]*|/\*.*?\*/', re.S) +CAP = '[CULTMATH_MAX_SITES]' + + +def main(shaders, out): + shutil.copytree(shaders, out, dirs_exist_ok=True) + wrappers, rows, spelled, default = [], [], '', None + for fn in FILES: + text = open(os.path.join(shaders, fn)).read().replace('\r\n', '\n') + spelled += COMMENT.sub('', text) + default = default or GUARD.search(text) + stray = GUARD.sub('', COMMENT.sub('', text)).replace(CAP, '') + if 'CULTMATH_MAX_SITES' in stray: + sys.exit('refuse: %s names CULTMATH_MAX_SITES outside its guard and array declarators' % fn) + parsed = list(SIG.finditer(text)) + looks = len(LOOKS.findall(text)) + if looks != len(parsed): + sys.exit('refuse: %s has %d definition-looking lines but %d parsed' % (fn, looks, len(parsed))) + for m in parsed: + ret, name, ps = m.groups() + if re.search(r'Texture|Sampler', ps): + continue + params = [p.strip() for p in ps.split(',') if p.strip()] + args = [re.split(r'\s+', p)[-1].split('[')[0] for p in params] + cap = ','.join(str(i) for i, p in enumerate(params) if p.endswith(CAP)) + wn = 'cmw%03d_%s' % (len(rows), name) + call = '%s(%s)' % (name, ', '.join(args)) + wrappers.append('export %s %s(%s) { %s%s; }' % (ret, wn, ', '.join(params), '' if ret == 'void' else 'return ', call)) + rows.append('\t'.join([wn, name, fn, ', '.join(params), cap])) + if not default: + sys.exit('refuse: no shader file carries the CULTMATH_MAX_SITES guard') + sentinel = int(default.group(1)) + 1 + while re.search(r'(?