diff --git a/packages/cultmath/tools/glsl/build-glsl.sh b/packages/cultmath/tools/glsl/build-glsl.sh index 0bef11c2..4f2a1a66 100644 --- a/packages/cultmath/tools/glsl/build-glsl.sh +++ b/packages/cultmath/tools/glsl/build-glsl.sh @@ -44,14 +44,19 @@ fi export PATH=$TOOLS/bin:$PATH echo "dxc: $("$DXC" --version | head -1)"; echo "spirv-opt: $(spirv-opt --version | head -1)" -python3 "$G/wrappers.py" "$CM/shaders" "$W/shw" -"$DXC" -T lib_6_3 -HV 2021 -spirv -fspv-target-env=universal1.5 -Od -I "$W/shw" "$W/shw/cm_wrappers.hlsl" -Fo "$W/lib.spv" >"$W/dxc.log" 2>&1 || { head -20 "$W/dxc.log"; exit 1; } -python3 "$G/spvedit.py" "$W/lib.spv" "$W/edited.spv" "$W/shw/wrappers.tsv" -spirv-val --target-env spv1.5 "$W/edited.spv" -spirv-opt --target-env=spv1.5 --eliminate-local-single-store --eliminate-dead-code-aggressive "$W/edited.spv" -o "$W/opt.spv" -spirv-val --target-env spv1.5 "$W/opt.spv" -spirv-cross "$W/opt.spv" --es --version 310 --relax-nan-checks --output "$W/cross.glsl" -python3 "$G/partition.py" "$CM/shaders" "$W/cross.glsl" "$OUT" +pipeline() { # shaders dir, work dir, out dir: HLSL to partitioned GLSL; returns 1 at the first refusal + local sh=$1 w=$2 out=$3 + mkdir -p "$w" "$out" + python3 "$G/wrappers.py" "$sh" "$w/shw" || return 1 + "$DXC" -T lib_6_3 -HV 2021 -spirv -fspv-target-env=universal1.5 -Od -I "$w/shw" "$w/shw/cm_wrappers.hlsl" -Fo "$w/lib.spv" >"$w/dxc.log" 2>&1 || { head -20 "$w/dxc.log"; return 1; } + python3 "$G/spvedit.py" "$w/lib.spv" "$w/edited.spv" "$w/shw/wrappers.tsv" "$w/shw/capacity.tsv" || return 1 + spirv-val --target-env spv1.5 "$w/edited.spv" || return 1 + spirv-opt --target-env=spv1.5 --eliminate-local-single-store --eliminate-dead-code-aggressive "$w/edited.spv" -o "$w/opt.spv" || return 1 + spirv-val --target-env spv1.5 "$w/opt.spv" || return 1 + spirv-cross "$w/opt.spv" --es --version 310 --relax-nan-checks --output "$w/cross.glsl" || return 1 + python3 "$G/partition.py" "$sh" "$w/cross.glsl" "$out" || return 1 +} +pipeline "$CM/shaders" "$W" "$OUT" || exit 1 # Consumer checks: glslang ES 3.00 fragment compiles of every include combination, and a consumer calling overloads # with an r-value argument. @@ -70,14 +75,23 @@ done printf '#define CULTMATH_MAX_SITES 4\n#include "CultMath.hlsl"\nfloat4 cap4(float3 p, float4 s[4]) { return cultmath_power_border(p, s, 4); }\n' >"$W/shw/cap4.hlsl" "$DXC" -T lib_6_3 -HV 2021 -Od -I "$W/shw" "$W/shw/cap4.hlsl" -Fo "$W/cap4.dxil" >"$W/cap4.log" 2>&1 || { echo "capacity (a) failed: dxc with CULTMATH_MAX_SITES 4"; head -8 "$W/cap4.log"; exit 1; } echo "capacity (a) ok: dxc lib_6_3 with a prior define 4 takes float4 s[4]" -cap_check() { # label, expected (ok|fail), define lines, GLSL library file, array length +consume() { # label, expected (ok|fail), define lines, GLSL library file, main body local f=$W/cap.frag - printf '#version 300 es\nprecision highp float;\nprecision highp int;\n%s\n' "$3" >"$f"; cat "$4" >>"$f" - printf 'out highp vec4 c;\nvoid main() { vec4 s[%s]; for (int i = 0; i < %s; i++) s[i] = vec4(float(i)); c = cultmath_power_border(gl_FragCoord.xyz, s, 4); }\n' "$5" "$5" >>"$f" + printf '#version 300 es +precision highp float; +precision highp int; +%s +' "$3" >"$f"; cat "$4" >>"$f" + printf 'out highp vec4 c; +void main() { %s } +' "$5" >>"$f" local got=fail; glslangValidator "$f" >"$W/cap.log" 2>&1 && got=ok [ "$got" = "$2" ] || { echo "capacity $1 failed: expected $2, got $got"; head -6 "$W/cap.log"; exit 1; } echo "capacity $1 ok: $2" } +cap_check() { # label, expected, define lines, GLSL library file, site array length + consume "$1" "$2" "$3" "$4" "vec4 s[$5]; for (int i = 0; i < $5; i++) s[i] = vec4(float(i)); c = cultmath_power_border(gl_FragCoord.xyz, s, 4);" +} D4='#define CULTMATH_MAX_SITES 4' cap_check "(b) built, define 4, vec4 s[4]" ok "$D4" "$OUT/core.glsl" 4 cap_check "(c) built, define 4, vec4 s[128]" fail "$D4" "$OUT/core.glsl" 128 @@ -98,3 +112,26 @@ if cat "$OUT"/*.glsl | grep -Eq "$SHAPE"; then echo "build shape failed:"; cat "$OUT"/*.glsl | grep -Eo "$SHAPE" | sort | uniq -c; exit 1 fi echo "build shape ok" + +# Tool refusals, and the capacity's reach: each variant appends one probe function to a scratch copy of CultMath.hlsl and +# runs the whole pipeline on it. +variant() { # label, probe HLSL; leaves the variant directory in VD + VD=$W/v/$1; rm -rf "$VD"; mkdir -p "$VD"; cp -r "$CM/shaders" "$VD/sh" + printf ' +%s +' "$2" >>"$VD/sh/CultMath.hlsl" + pipeline "$VD/sh" "$VD/w" "$VD/out" >"$VD/log" 2>&1 +} +expect_refused() { # label, probe HLSL, pattern the refusal prints + if variant "$1" "$2"; then echo "refusal $1 failed: the build passed"; exit 1; fi + grep -q "$3" "$VD/log" || { echo "refusal $1 failed: no '$3' in the log"; tail -5 "$VD/log"; exit 1; } + echo "refusal $1 ok" +} +SENTINEL=$(cut -f2 "$W/shw/capacity.tsv") +expect_refused macro-in-expression 'int cultmath_zz_probe(int n) { return n + CULTMATH_MAX_SITES; }' 'outside its guard' +expect_refused length-in-expression "uint cultmath_zz_probe(uint n) { return n + (100u + $((SENTINEL - 100))u); }" 'also used by opcode' +variant array-128 'float4 cultmath_zz_probe(float4 x[128], int n) { return x[n]; }' || { echo "array-128 failed: a library array of length 128 broke the build"; tail -5 "$VD/log"; exit 1; } +grep -Eq 'cultmath_zz_probe\(.*\[128\]' "$VD/out/core.glsl" || { echo "array-128 failed: the 128 array is not literal in core.glsl"; exit 1; } +PROBE='vec4 x[128]; c = cultmath_zz_probe(x, 1);' +consume "(f) non-site array, define 4, vec4 x[128]" ok "$D4" "$VD/out/core.glsl" "$PROBE" +consume "(f) non-site array, define 4, vec4 x[4]" fail "$D4" "$VD/out/core.glsl" 'vec4 x[4]; c = cultmath_zz_probe(x, 1);' diff --git a/packages/cultmath/tools/glsl/partition.py b/packages/cultmath/tools/glsl/partition.py index b4d25312..415c9b2b 100644 --- a/packages/cultmath/tools/glsl/partition.py +++ b/packages/cultmath/tools/glsl/partition.py @@ -6,18 +6,20 @@ Each top-level struct and function chunk goes to the file whose HLSL defines it Cycles; Phacelle). The library-header prologue and any main are dropped. Writes core.glsl, cycles.glsl and phacelle.glsl bodies, with the 13 `static const float` constants of the HLSL as `const float NAME = ;` at the head of the file that defines them. core.glsl opens with the HLSL's own CULTMATH_MAX_SITES guard lines, one line mapping -SPIRV_CROSS_CONSTANT_ID_0 to uint(CULTMATH_MAX_SITES), and SPIRV-Cross's one spec-constant block. Licence lines, include guards and the #version contract belong to the caller. +SPIRV_CROSS_CONSTANT_ID_ (CultMath's own, 'CMS1') to uint(CULTMATH_MAX_SITES), and SPIRV-Cross's one spec-constant block. Licence lines, include guards and the #version contract belong to the caller. Refuses: a static const line that is not `static const float CULTMATH_X = ;`, an unknown-origin chunk, a duplicate chunk, an unexpected prologue line, a missing or other SPIRV_CROSS_CONSTANT_ID block, and a call from a non-core chunk into a different non-core origin.""" import collections, os, re, sys +from spvedit import SPEC_ID OWNER = {'CultMath.hlsl': 'core', 'CultMath.Interval.hlsl': 'core', 'CultMath.Affine.hlsl': 'core', 'CultMath.Cycles.hlsl': 'cycles', 'CultMath.Phacelle.hlsl': 'phacelle'} CONST = re.compile(r'^static const float (CULTMATH_\w+) = (-?\d+(?:\.\d*)?(?:[eE][+-]?\d+)?);$') PROLOGUE = re.compile(r'^(#version |#extension |precision |#ifdef SPIRV_CROSS|#define SPIRV_CROSS|#endif)') GUARD = re.compile(r'#ifndef CULTMATH_MAX_SITES\n#define CULTMATH_MAX_SITES (\d+)\n#endif\n') -SPEC = re.compile(r'#ifndef SPIRV_CROSS_CONSTANT_ID_0\n#define SPIRV_CROSS_CONSTANT_ID_0 (\d+)u\n#endif\n' - r'const uint cultmath_max_sites = SPIRV_CROSS_CONSTANT_ID_0;\n?') +ID = 'SPIRV_CROSS_CONSTANT_ID_%d' % SPEC_ID +SPEC = re.compile(r'#ifndef %s\n#define %s (\d+)u\n#endif\n' + r'const uint cultmath_max_sites = %s;\n?' % (ID, ID, ID)) STRUCT, FUNC = re.compile(r'^struct (\w+)$'), re.compile(r'^\w+ (\w+)\(.*\)$') @@ -87,7 +89,7 @@ def main(shaders, glsl, outdir): files[o].append(body) os.makedirs(outdir, exist_ok=True) for o in OWNER.values(): - head = [guard.group(0).rstrip('\n'), '#define SPIRV_CROSS_CONSTANT_ID_0 uint(CULTMATH_MAX_SITES)', spec.group(0).rstrip('\n')] if o == 'core' else [] + head = [guard.group(0).rstrip('\n'), '#define %s uint(CULTMATH_MAX_SITES)' % ID, spec.group(0).rstrip('\n')] if o == 'core' else [] text = '\n'.join(head + consts[o] + files[o]) + '\n' open(os.path.join(outdir, o + '.glsl'), 'w').write(text) print('partition:', {o: len(files[o]) for o in sorted(files)}, 'constants', sum(len(v) for v in consts.values())) diff --git a/packages/cultmath/tools/glsl/spvedit.py b/packages/cultmath/tools/glsl/spvedit.py index 560ca870..d1b3e169 100644 --- a/packages/cultmath/tools/glsl/spvedit.py +++ b/packages/cultmath/tools/glsl/spvedit.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """The one instruction-level SPIR-V edit of the GLSL build. Binary, never text. -usage: spvedit.py +usage: spvedit.py 1. Keeps only the capabilities Shader and Linkage and drops every OpExtension. 2. Deletes each wrapper function with its OpName and decorations, moves its LinkageAttributes Export to its callee, and sets the callee's OpName to the source name. Refuses a wrapper that is not exactly one OpFunctionCall. @@ -14,6 +14,7 @@ import struct, sys OP_NAME, OP_MEMBER_NAME, OP_CAPABILITY, OP_EXTENSION, OP_DECORATE, OP_MEMBER_DECORATE = 5, 6, 17, 10, 71, 72 OP_TYPE_ARRAY, OP_CONSTANT, OP_SPEC_CONSTANT, DECORATION_SPEC_ID = 28, 43, 50, 1 +SPEC_ID = 0 OP_TYPE_POINTER, OP_TYPE_FUNCTION, OP_FUNCTION, OP_PARAMETER, OP_FUNCTION_END = 32, 33, 54, 55, 56 OP_CALL, OP_LABEL, OP_VARIABLE, OP_LOAD, OP_STORE = 57, 248, 59, 61, 62 CAP_SHADER, CAP_LINKAGE, DECORATION_LINKAGE, LINKAGE_EXPORT, FUNCTION_STORAGE = 1, 5, 41, 41, 7 @@ -178,10 +179,12 @@ def by_value(h, ins): return out, len(newtype), len(newid) -def capacity_edit(h, ins, rows, callee_of): +def capacity_edit(h, ins, rows, callee_of, default, sentinel): """Each capacity parameter (wrappers.tsv column 5) of each callee has an OpTypeArray type. Exactly one length id may serve all of them, and nothing but OpTypeArray, OpName and decorations may use it. That OpConstant becomes an - OpSpecConstant with SpecId 0 named cultmath_max_sites. Returns the instructions and the spec constant id.""" + OpSpecConstant with SpecId SPEC_ID named cultmath_max_sites, defaulting to the HLSL guard value. The constant is the + sentinel wrappers.py compiled the library with, so no array outside the capacity parameters shares its length id. + Returns the instructions and the spec constant id.""" ftype = {x[1]: x for x in ins if op(x) == OP_TYPE_FUNCTION} arrays = {x[1]: x for x in ins if op(x) == OP_TYPE_ARRAY} params, cur = {}, None @@ -201,6 +204,12 @@ def capacity_edit(h, ins, rows, callee_of): if len(lengths) != 1: sys.exit('refuse: %d array length ids serve the capacity parameters, expected exactly one' % len(lengths)) cap = lengths.pop() + value = next(x[3] for x in ins if op(x) == OP_CONSTANT and x[2] == cap) + if value != sentinel: + sys.exit('refuse: the capacity length constant is %d, expected the sentinel %d' % (value, sentinel)) + site_types = {params[fid][int(c)] for row, fid in zip(rows, callee_of) for c in row[4].split(',') if c} + if {t for t, a in arrays.items() if a[3] == cap} != site_types: + sys.exit('refuse: an array type of the capacity length is not a capacity parameter type') for x in ins: o = op(x) defines = o in (OP_CONSTANT, OP_SPEC_CONSTANT) and x[2] == cap @@ -210,25 +219,26 @@ def capacity_edit(h, ins, rows, callee_of): out = [] for x in ins: if op(x) == OP_CONSTANT and x[2] == cap: - x = [(x[0] & 0xffff0000) | OP_SPEC_CONSTANT] + x[1:] + x = [(x[0] & 0xffff0000) | OP_SPEC_CONSTANT, x[1], x[2], default] out.append(x) first = next((k for k, x in enumerate(out) if op(x) in (OP_DECORATE, OP_MEMBER_DECORATE)), None) if first is None: sys.exit('refuse: no annotation section to decorate the capacity constant in') - out[first:first] = [mk(OP_NAME, cap, *encode('cultmath_max_sites')), mk(OP_DECORATE, cap, DECORATION_SPEC_ID, 0)] + out[first:first] = [mk(OP_NAME, cap, *encode('cultmath_max_sites')), mk(OP_DECORATE, cap, DECORATION_SPEC_ID, SPEC_ID)] return out, cap -def main(src, dst, tsv): +def main(src, dst, tsv, capacity): h, ins = load(src) rows = [l.split('\t') for l in open(tsv).read().splitlines() if l] ins, callee_of = export_edit(h, ins, rows) ins, changed, params = by_value(h, ins) - ins, spec = capacity_edit(h, ins, rows, callee_of) + default, sentinel = [int(v) for v in open(capacity).read().split()] + ins, spec = capacity_edit(h, ins, rows, callee_of, default, sentinel) words = h + [v for x in ins for v in x] open(dst, 'wb').write(struct.pack('<%dI' % len(words), *words)) print('spvedit: %d wrappers dropped; by-value: %d functions changed, %d parameters; capacity spec constant id %d' % (len(rows), changed, params, spec)) if __name__ == '__main__': - main(*sys.argv[1:4]) + main(*sys.argv[1:5]) diff --git a/packages/cultmath/tools/glsl/wrappers.py b/packages/cultmath/tools/glsl/wrappers.py index 96c1c7b5..742881ac 100644 --- a/packages/cultmath/tools/glsl/wrappers.py +++ b/packages/cultmath/tools/glsl/wrappers.py @@ -3,7 +3,9 @@ every overload as its own function. The shaders are copied unchanged; the unique names exist only in . usage: wrappers.py -Writes /cm_wrappers.hlsl and /wrappers.tsv (wrapper, name, file, params, capacity). An argument is the +Writes /cm_wrappers.hlsl, /wrappers.tsv (wrapper, name, file, params, capacity) and +/capacity.tsv (the guard default, the sentinel). cm_wrappers.hlsl defines CULTMATH_MAX_SITES as the sentinel, +a number no shader file spells, so the site arrays get an array length id of their own and no other array shares it. An argument is the parameter's declarator name with any [..] suffix removed; capacity lists the indices of the parameters declared [CULTMATH_MAX_SITES]. Definitions with a Texture or Sampler parameter get no wrapper. Refuses a file where the definition-looking lines differ from the parsed ones, and a file that names CULTMATH_MAX_SITES anywhere but its @@ -13,17 +15,18 @@ import os, re, shutil, sys FILES = ['CultMath.hlsl', 'CultMath.Interval.hlsl', 'CultMath.Affine.hlsl', 'CultMath.Cycles.hlsl', 'CultMath.Phacelle.hlsl'] SIG = re.compile(r'(?m)^(\w+)[ \t]+(cultmath_\w+)[ \t]*\(([^)]*)\)') LOOKS = re.compile(r'(?m)^\w+ cultmath_\w+\(') -GUARD = re.compile(r'#ifndef CULTMATH_MAX_SITES\n#define CULTMATH_MAX_SITES \d+\n#endif\n') +GUARD = re.compile(r'#ifndef CULTMATH_MAX_SITES\n#define CULTMATH_MAX_SITES (\d+)\n#endif\n') COMMENT = re.compile(r'//[^\n]*|/\*.*?\*/', re.S) CAP = '[CULTMATH_MAX_SITES]' def main(shaders, out): shutil.copytree(shaders, out, dirs_exist_ok=True) - wrappers = ['#include "CultMath.hlsl"'] - rows = [] + wrappers, rows, spelled, default = [], [], '', None for fn in FILES: text = open(os.path.join(shaders, fn)).read().replace('\r\n', '\n') + spelled += COMMENT.sub('', text) + default = default or GUARD.search(text) stray = GUARD.sub('', COMMENT.sub('', text)).replace(CAP, '') if 'CULTMATH_MAX_SITES' in stray: sys.exit('refuse: %s names CULTMATH_MAX_SITES outside its guard and array declarators' % fn) @@ -42,6 +45,13 @@ def main(shaders, out): call = '%s(%s)' % (name, ', '.join(args)) wrappers.append('export %s %s(%s) { %s%s; }' % (ret, wn, ', '.join(params), '' if ret == 'void' else 'return ', call)) rows.append('\t'.join([wn, name, fn, ', '.join(params), cap])) + if not default: + sys.exit('refuse: no shader file carries the CULTMATH_MAX_SITES guard') + sentinel = int(default.group(1)) + 1 + while re.search(r'(?