From e598c4433398d1ad22e7bf0394ab5b8ba1079d96 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 16 Oct 2026 17:24:47 +0200 Subject: [PATCH] Research: shader runner probes (dxc SPIR-V library + SPIRV-Cross, lavapipe on Yggdrasil, native ANGLE D3D11 on Starfire) for the three questions replacing device-run-driver Co-Authored-By: Claude Opus 5.5 --- docs/research/native-shader-probes.md | 143 ++++++++++++++++++ .../native-shader-probes/angle_native.py | 96 ++++++++++++ docs/research/native-shader-probes/fam.py | 112 ++++++++++++++ docs/research/native-shader-probes/lib.sh | 52 +++++++ docs/research/native-shader-probes/probe.sh | 81 ++++++++++ docs/research/native-shader-probes/vkrun.c | 87 +++++++++++ 6 files changed, 571 insertions(+) create mode 100644 docs/research/native-shader-probes.md create mode 100644 docs/research/native-shader-probes/angle_native.py create mode 100644 docs/research/native-shader-probes/fam.py create mode 100644 docs/research/native-shader-probes/lib.sh create mode 100644 docs/research/native-shader-probes/probe.sh create mode 100644 docs/research/native-shader-probes/vkrun.c diff --git a/docs/research/native-shader-probes.md b/docs/research/native-shader-probes.md new file mode 100644 index 00000000..087a87e6 --- /dev/null +++ b/docs/research/native-shader-probes.md @@ -0,0 +1,143 @@ +# Shader runner probes: the HLSL-to-GLSL ES toolchain, lavapipe on Yggdrasil, native ANGLE on Starfire + +For the cultmath-tapes questions that replace `device-run-driver`: `glsl-es-translation-owner`, +`conforming-shader-run` and `real-device-shader-run`. These are body facts measured by +imagination-ag4-shader-runner (session self-2026-10-16-ag4) on 2026-10-16, between about 14:10 and +15:20 UTC (16:10 to 17:20 CEST). Every probe read CultLib at f697e03f (origin/main). Companion to +`docs/research/native-shader-testing.md` (f697e03f) and `docs/research/device-capture-runners.md` (c9d3e672). + +The probe scripts are in this directory under `p/`: `probe.sh` and `lib.sh` (Yggdrasil jobs), `fam.py` +(the per-family shader generator and bit judge), `vkrun.c` (an 87-line Vulkan compute host) and +`angle_native.py` (a 96-line native ANGLE host). They are throwaway probe code, not a proposed +implementation. The logs are `q12-run3.log` (lavapipe) and `lib-run4.log` (library route). The earlier +runs are superseded: run1 used a raw-ulp metric, run2 died on CRLF, and lib-run1 to lib-run3 failed on +the capability and grep problems recorded below. + +## Metric + +Each fixture family has 256 cases, and the fixture records the C# result bits. "Exact" means all 256 +cases are bit-identical to the C# fixture. Otherwise the worst error is given in units of +2^-23 * max(|expected|, 1) (written "u1" below), the scale the fixture's ulp-bounded checks use. NaN and +Inf cases are compared only as NaN-versus-not-NaN. These figures describe the data. They are not the +fixture's own judge, which is C# and per family: GlslMirrorTests tolerances, and the `check` text on 10 +families. + +## The fixture surface (glsl-parity.json at f697e03f) + +- 109 families, each with 256 cases, encoded `args -> result` as hex 32-bit words. There are at most 32 + words per case. +- 105 families use only float, int, vecN, ivecN and mat3 (mat3 appears as a return only). The other 4 + take or return structs: `CultCellular cultmath_cellular(vec3)`, `CultPhasor cultmath_phacelle(vec3, + vec3, float, float)`, and `SpanNoise cultmath_span_fbm/ridged(SpanPath, SpanStage, int, float, float)`. + Any runner needs per-struct flatteners in field order for these 4. +- Tolerance classes: 89 are `ulp-bounded`, 2 are `exact`, 7 are `platform` (exp, pow, log, asin, acos, + atan, and phacelle's sin and cos), and 10 carry a `check` (enclosure). +- What a harness needs: generate one shader per family that reads the argument words, bit-casts them to + the signature's types, calls the function and writes the result words. The probe generator does this + for both HLSL and GLSL in 112 lines of Python, judge included. + +## Q1. Can dxc emit a SPIR-V library that SPIRV-Cross lowers to GLSL ES for CultMath's real surface? + +Yggdrasil, ubuntu:24.04. Tools: dxc v1.9.2607 Linux (`linux_dxc_2026_07_29`, the version get-dxc.ps1 +pins on Windows), and SPIRV-Cross built from HEAD 300681bb (2026-10-14). + +| Step | Result | +|---|---| +| `dxc -T lib_6_6 -spirv -fspv-target-env=vulkan1.2 CultMath.hlsl`, no change to the source | fails: "No OpEntryPoint ... only allowed if the Linkage ... capability is being used" | +| `-default-linkage external` (lib_6_3, universal1.5), no `export` | fails the same way: no exports in SPIR-V | +| `export` added to every top-level `cultmath_` definition, vulkan1.2 | fails: "Capability Linkage is not allowed by Vulkan 1.2" | +| same, `-fspv-target-env=universal1.5` | compiles, but declares 14 capabilities, ray query, interlock, quad control and partitioned subgroups among them | +| SPIRV-Cross `--es --version 310` on that module | fails: "RayQuery requires Vulkan GLSL 460" | +| ray query stripped | fails: "Cannot find mapping for combined sampler parameter". The Texture2D/SamplerState functions were exported | +| `export` only on the 173 non-texture definitions, all capabilities but Shader and Linkage stripped, every OpExtension stripped, then `spirv-as` | assembles; SPIRV-Cross `--es --version 310` exits 0 | +| SPIRV-Cross `--es --version 300` | fails: "At least ESSL 3.10 required for compute shaders" (a library gets a GLCompute placeholder entry) | +| ES 3.10 output, `#version` guarded by `#ifdef SPIRV_CROSS_LIBRARY_HEADER`, placeholder main removed, compiled by glslang as a `#version 300 es` fragment shader | compiles | + +Shape of the output against the committed lowering: + +| | committed (CultMath.glsl, .Cycles, .Phacelle) | dxc + SPIRV-Cross | +|---|---|---| +| lines | 1035 + Cycles + Phacelle | 3458, one file (the MPL and Apache files are merged) | +| function definitions | 173 (121 + 51 + 1) | 127 | +| overloads | kept | **46 dropped**: one definition per name survives (for example `cultmath_snoise(vec2)` and `cultmath_iv_snoise_ball(vec2, float)` are gone) | +| SSA temporaries `_NNN` | 0 | 5632 | +| min and max | as written | NaN-guarded `mix(mix(min(a,b),b,isnan(a)),a,isnan(b))` | + +In `cultmath_snoise(vec3)` as emitted, the body is one temporary per SSA value (`vec3 _1157 = floor(...)`) +and constants are spelt to 20 digits. + +**Answer:** yes, but only with three owned workarounds: an `export` annotation on 173 definitions, +which FXC (Unity's D3D11 compiler) does not accept, so it would have to be a macro; a capability and +extension strip pass; and unique names for 46 overloads, which breaks the public API for both the HLSL +and the GLSL consumers. The output is generated text, not a readable library. + +## Q2. Does a conforming Vulkan implementation run the families in a Linux container on Yggdrasil? + +Yggdrasil, ubuntu:24.04, apt packages: mesa-vulkan-drivers 25.2.8 (lavapipe), glslang-tools 15.1.0, +spirv-tools 2025.1~rc1, libvulkan-dev, and the same dxc. There is no GPU. + +- Device: `llvmpipe (LLVM 20.1.2, 256 bits)`, Vulkan 1.4. Float controls for f32: denormPreserve 0, + flushToZero 0, RTE 1, RTZ 0, signedZeroInfNanPreserve 1. +- HLSL route: a generated `cs_6_0` compute shader that includes CultMath.hlsl, through `dxc -spirv + -fspv-target-env=vulkan1.2`. 105 of 105 families compile and run. Fma instructions in the SPIR-V: 0. +- GLSL route: the committed CultMath.glsl, .Cycles.glsl and .Phacelle.glsl concatenated after `#version + 310 es` and highp precision, as a compute shader through `glslangValidator -V`. 105 of 105 compile and + run. +- Library route: the dxc + SPIRV-Cross library from Q1 through glslang. 103 of 105 run; the 2 missing + ones are the dropped overloads. Against the fixture: 75 exact, 4 within 4 u1, 7 within 64, 17 above + 64. Its worst rows are the same figures as the HLSL route's. +- Time: all 105 dispatches together take 8 s on a re-run. Each job runs about 4 minutes, almost all of + it apt, the dxc download and the SPIRV-Cross build. +- Determinism: re-running all 105 SPIR-V modules gave bit-identical outputs. +- The HLSL and GLSL routes are bit-identical to each other in 85 of 105 families. +- Against the C# fixture, HLSL route: 75 exact, 6 within 4 u1, 7 within 64, 17 above 64. GLSL route: + 75, 4, 8 and 18. The largest errors are in the `coordinate bands 1e2 to 4e6` families of snoise_grad + and snoise_hessian (13e6 and 1.6e8 u1), then fbm_grad and ridged_grad (about 3.5e3 to 5e3 u1), Cycles + arcsine (2.7e3) and arccosine (1.0e3), and the other hessian families (1e3 to 2.4e3). + +**Answer:** yes. The image needs nothing beyond distro packages and the pinned dxc. It is +deterministic and fast, and it runs both the HLSL and the lowered GLSL on the same implementation. + +## Q3. A real-device run on Starfire without a browser: native ANGLE + +Starfire. The host is `angle_native.py`, Python ctypes over the libEGL.dll and libGLESv2.dll that ship +with Docker Desktop 4.71.0 (an Electron app): ANGLE 2.1.26626, git hash a4490148bd3d. Display: +`EGL_PLATFORM_ANGLE_TYPE_D3D11`. Context: ES 3.0, optionally `EGL_CONTEXT_WEBGL_COMPATIBILITY_ANGLE`. +Each family is a `#version 300 es` fragment shader, with arguments read by texelFetch from an RGBA32UI +texture and results written to up to 3 RGBA32UI targets over a 16x16 grid, then read back with +glReadPixels. The same RGBA32UI readback is used by the scratchpad browser pages (gpu.js). + +- `GL_RENDERER`: `ANGLE (NVIDIA, NVIDIA GeForce GTX 1070 (0x00001B81) Direct3D11 vs_5_0 ps_5_0, D3D11)`. + `GL_VERSION`: `OpenGL ES 3.0 (ANGLE 2.1.26626 git hash: a4490148bd3d)`. +- 105 of 105 families compile and run. All 105 together take 6.3 s wall, process start included. The GL + error at the end is 0. +- The WebGL-compatibility context and the plain ES 3.0 context are bit-identical in 105 of 105 families. +- Against the C# fixture: 38 exact, 31 within 4 u1, 17 within 64, 19 above 64. +- It agrees with the earlier browser measurements on the same GPU. `cultmath_iv_exp`: 19 of 256 exact, + worst 12 ulp; the browser run of finding iv-exp-gpu-not-enclosed measured "up to 12 ulp". + `cultmath_iv_frustum_ball`: 213 of 256 exact, so 43 off, by at most 2 ulp; finding + frustum-exact-unsound-on-webgl2 measured "43/256 cases by up to 2 ulp". +- The dxc + SPIRV-Cross library from Q1 also runs on this ANGLE, as a WebGL-compatibility ES 3.00 + fragment shader: 103 of 105 families (the 2 dropped overloads fail to compile), with 41 exact. It is + bit-identical to the committed lowering's ANGLE output in 71 of 103 families. +- Largest differences: the `coordinate bands 1e2 to 4e6` families again. 173 cases of snoise_grad and + 415 component values of snoise_hessian differ by more than 1e-3 relative, up to 2 and 20. These + families exist for the span allowance's device margin (follow-up span-allowance-gpu-margin, cut + stage-shaders-device). Their judge is the allowance check, not an ulp bound. + +**Answer:** yes. A pinned ANGLE build, loaded natively, runs the site's real translation path +(ES 3.00, then ANGLE, then HLSL, then D3DCompile, then the NVIDIA D3D11 driver) with no browser and no +SwiftShader fallback, because the backend is named explicitly and the renderer string is recorded. The +result agrees bit for bit with the WebGL-compatibility context, and it reproduces the two browser +measurements on record. + +## What was not probed + +- SwiftShader as a second conforming implementation. +- Amber as the dispatch host instead of a small owned one. +- WARP and native D3D11 HLSL on Starfire, which is the Unity consumer path. +- naga with dxc SPIR-V as input. +- Chrome's own ANGLE feature flags, which the native host does not reproduce. +- Mobile or mediump devices. +- An Electron release's ANGLE pinned by download. The probe used the copy Docker Desktop happens to + ship. diff --git a/docs/research/native-shader-probes/angle_native.py b/docs/research/native-shader-probes/angle_native.py new file mode 100644 index 00000000..f47578b8 --- /dev/null +++ b/docs/research/native-shader-probes/angle_native.py @@ -0,0 +1,96 @@ +"""Probe: run the GLSL fixture families through a native ANGLE (DLLs shipped with an Electron app), D3D11 +backend, ES 3.0 context, fragment-shader readback through RGBA32UI targets. No browser. +usage: py -3 -I angle_native.py [webgl]""" +import ctypes as C, json, os, struct, sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import fam + +dll, fx, sh, out = sys.argv[1:5]; webgl = len(sys.argv) > 5 +os.add_dll_directory(dll) +egl = C.CDLL(os.path.join(dll, 'libEGL.dll')); gl = C.CDLL(os.path.join(dll, 'libGLESv2.dll')) +egl.eglGetProcAddress.restype = C.c_void_p +for n in ['eglInitialize', 'eglChooseConfig', 'eglCreatePbufferSurface', 'eglCreateContext', 'eglMakeCurrent', 'eglBindAPI', 'eglGetError']: + getattr(egl, n).restype = C.c_void_p if n.startswith('eglCreate') else C.c_uint +egl.eglCreatePbufferSurface.argtypes = [C.c_void_p, C.c_void_p, C.POINTER(C.c_int)] +egl.eglCreateContext.argtypes = [C.c_void_p, C.c_void_p, C.c_void_p, C.POINTER(C.c_int)] +egl.eglMakeCurrent.argtypes = [C.c_void_p, C.c_void_p, C.c_void_p, C.c_void_p] +egl.eglInitialize.argtypes = [C.c_void_p, C.c_void_p, C.c_void_p] +egl.eglChooseConfig.argtypes = [C.c_void_p, C.POINTER(C.c_int), C.POINTER(C.c_void_p), C.c_int, C.POINTER(C.c_int)] +GPD = C.CFUNCTYPE(C.c_void_p, C.c_uint, C.c_void_p, C.POINTER(C.c_int))(egl.eglGetProcAddress(b'eglGetPlatformDisplayEXT')) +def ia(*v): return (C.c_int * (len(v) + 1))(*v, 0x3038) +disp = GPD(0x3202, None, ia(0x3203, 0x3208)) # EGL_PLATFORM_ANGLE_TYPE_ANGLE = D3D11 +assert egl.eglInitialize(disp, None, None), 'eglInitialize %x' % egl.eglGetError() +cfg = C.c_void_p(); ncfg = C.c_int() +egl.eglChooseConfig(disp, ia(0x3033, 0x1, 0x3040, 0x40), C.byref(cfg), 1, C.byref(ncfg)) # PBUFFER, ES3 +surf = egl.eglCreatePbufferSurface(disp, cfg, ia(0x3057, 16, 0x3056, 16)) +egl.eglBindAPI(0x30A0) +ctx_attr = (0x3098, 3) + ((0x33AC, 1) if webgl else ()) # CLIENT_VERSION 3 [, WEBGL_COMPATIBILITY_ANGLE] +ctx = egl.eglCreateContext(disp, cfg, None, ia(*ctx_attr)) +assert ctx and egl.eglMakeCurrent(disp, surf, surf, ctx), 'context %x' % egl.eglGetError() +gl.glGetString.restype = C.c_char_p +print('GL_RENDERER', gl.glGetString(0x1F01).decode()); print('GL_VERSION', gl.glGetString(0x1F02).decode()) +print('webgl compatibility context', webgl) + +gl.glCreateShader.restype = C.c_uint; gl.glCreateProgram.restype = C.c_uint +def shader(kind, src): + s = gl.glCreateShader(kind); b = C.c_char_p(src.encode()); gl.glShaderSource(s, 1, C.byref(b), None); gl.glCompileShader(s) + ok = C.c_int(); gl.glGetShaderiv(s, 0x8B81, C.byref(ok)) + if not ok.value: + buf = C.create_string_buffer(4000); gl.glGetShaderInfoLog(s, 4000, None, buf); raise RuntimeError(buf.value.decode()[:300]) + return s +VS = '#version 300 es\nvoid main() { vec2 p = vec2((gl_VertexID << 1) & 2, gl_VertexID & 2); gl_Position = vec4(p * 2.0 - 1.0, 0.0, 1.0); }\n' +lib = open(os.environ['LIBGLSL']).read() if os.environ.get('LIBGLSL') else ''.join(open(os.path.join(sh, f)).read() for f in ['CultMath.glsl', 'CultMath.Cycles.glsl', 'CultMath.Phacelle.glsl']) +vao = C.c_uint(); gl.glGenVertexArrays(1, C.byref(vao)); gl.glBindVertexArray(vao) +d = json.load(open(fx)); man = []; skipped = 0 +for idx, f in enumerate(d['functions']): + r, name, ps = fam.parse(f['glsl']); rt = fam.ty(r); pts = [fam.ty(p) for p in ps] + if rt is None or any(p is None for p in pts): skipped += 1; continue + nin = sum(p[1] for p in pts); nout = rt[1]; tw = (nin + 3) // 4; nt = (nout + 3) // 4; tag = 'f%03d' % idx + words = [int(x, 16) for c in f['cases'] for x in c.split('->')[0].split()] + tex = [] + for ci in range(256): + row = words[ci * nin:(ci + 1) * nin]; tex += row + [0] * (tw * 4 - nin) + lines, off = [], 0 + for j, (p, (ht, n, k)) in enumerate(zip(ps, pts)): + src = ', '.join('w[%d]' % (off + t) for t in range(n)); v = ('uvec%d(%s)' % (n, src)) if n > 1 else src + conv = {'f': 'uintBitsToFloat', 'i': ('ivec%d' % n if n > 1 else 'int'), 'u': ''}[k] + lines.append(' %s a%d = %s(%s);' % (p, j, conv, v)); off += n + lines.append(' %s r = %s(%s);' % (r, name, ', '.join('a%d' % j for j in range(len(ps))))) + comps = ['r[%d][%d]' % (a, b) for a in range(3) for b in range(3)] if r == 'mat3' else ['r'] if nout == 1 else ['r[%d]' % i for i in range(nout)] + for i, c in enumerate(comps): + lines.append(' ow[%d] = %s(%s);' % (i, {'f': 'floatBitsToUint', 'i': 'uint', 'u': ''}[rt[2]], c)) + outs = ''.join('layout(location = %d) out highp uvec4 o%d;\n' % (t, t) for t in range(nt)) + wr = ''.join(' o%d = uvec4(ow[%d], ow[%d], ow[%d], ow[%d]);\n' % (t, 4 * t, 4 * t + 1, 4 * t + 2, 4 * t + 3) for t in range(nt)) + FS = ('#version 300 es\nprecision highp float;\nprecision highp int;\n' + lib + '\nuniform highp usampler2D In;\n' + outs + + 'void main() {\n int i = int(gl_FragCoord.y) * 16 + int(gl_FragCoord.x);\n uint w[%d];\n for (int k = 0; k < %d; k++) w[k] = texelFetch(In, ivec2(k / 4, i), 0)[k %% 4];\n' + ' uint ow[%d];\n' % (tw * 4, nin, nt * 4) + '\n'.join(lines) + '\n' + ''.join(' ow[%d] = 0u;\n' % z for z in range(nout, nt * 4)) + wr + '}\n') + try: + prog = gl.glCreateProgram(); gl.glAttachShader(prog, shader(0x8B31, VS)); gl.glAttachShader(prog, shader(0x8B30, FS)); gl.glLinkProgram(prog) + except RuntimeError as e: + print('compile fail', f['glsl'], str(e).replace('\n', ' ')[:200]); continue + gl.glUseProgram(prog) + t_in = C.c_uint(); gl.glGenTextures(1, C.byref(t_in)); gl.glActiveTexture(0x84C0); gl.glBindTexture(0x0DE1, t_in) + gl.glTexParameteri(0x0DE1, 0x2801, 0x2600); gl.glTexParameteri(0x0DE1, 0x2800, 0x2600) + buf = (C.c_uint * len(tex))(*tex); gl.glTexImage2D(0x0DE1, 0, 0x8D70, tw, 256, 0, 0x8D99, 0x1405, buf) + gl.glUniform1i(gl.glGetUniformLocation(prog, b'In'), 0) + fbo = C.c_uint(); gl.glGenFramebuffers(1, C.byref(fbo)); gl.glBindFramebuffer(0x8D40, fbo) + tos = (C.c_uint * nt)(); gl.glGenTextures(nt, tos) + for t in range(nt): + gl.glBindTexture(0x0DE1, tos[t]); gl.glTexImage2D(0x0DE1, 0, 0x8D70, 16, 16, 0, 0x8D99, 0x1405, None) + gl.glFramebufferTexture2D(0x8D40, 0x8CE0 + t, 0x0DE1, tos[t], 0) + gl.glDrawBuffers(nt, (C.c_uint * nt)(*[0x8CE0 + t for t in range(nt)])) + gl.glActiveTexture(0x84C0); gl.glBindTexture(0x0DE1, t_in) + gl.glViewport(0, 0, 16, 16); gl.glDrawArrays(0x0004, 0, 3) + res = [[0] * 1024 for _ in range(nt)] + for t in range(nt): + gl.glReadBuffer(0x8CE0 + t); px = (C.c_uint * 1024)(); gl.glReadPixels(0, 0, 16, 16, 0x8D99, 0x1405, px); res[t] = list(px) + flat = [] + for ci in range(256): + allw = sum((res[t][ci * 4:ci * 4 + 4] for t in range(nt)), []); flat += allw[:nout] + open(os.path.join(out, tag + '.in'), 'wb').write(b'\0') + open(os.path.join(out, tag + '.angle.out'), 'wb').write(struct.pack('<%dI' % len(flat), *flat)) + man.append({'tag': tag, 'glsl': f['glsl'], 'n': 256, 'nout': nout, 'kind': rt[2], 'tol': f['tolerance'], 'platform': f.get('platform')}) + gl.glDeleteFramebuffers(1, C.byref(fbo)); gl.glDeleteTextures(nt, tos); gl.glDeleteTextures(1, C.byref(t_in)); gl.glDeleteProgram(prog) +print('gl error at end 0x%x' % gl.glGetError(), '; ran', len(man), 'skipped struct', skipped) +json.dump(man, open(os.path.join(out, 'manifest.json'), 'w')) +fam.cmp(fx, out, 'angle') diff --git a/docs/research/native-shader-probes/fam.py b/docs/research/native-shader-probes/fam.py new file mode 100644 index 00000000..28489f10 --- /dev/null +++ b/docs/research/native-shader-probes/fam.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Probe: per fixture family, generate an HLSL and a GLSL ES 3.10 compute shader that call the family's +function on the fixture's argument bits, and compare device results with the fixture's result bits. +usage: fam.py gen | fam.py cmp """ +import json, re, struct, sys, os + +SCAL = {'float': ('float', 1, 'f'), 'int': ('int', 1, 'i'), 'uint': ('uint', 1, 'u')} +def ty(t): + if t in SCAL: return SCAL[t] + m = re.fullmatch(r'(i|u|)vec([234])', t) + if m: + k = {'': 'f', 'i': 'i', 'u': 'u'}[m.group(1)]; n = int(m.group(2)) + return ({'f': 'float', 'i': 'int', 'u': 'uint'}[k] + str(n), n, k) + if t == 'mat3': return ('float3x3', 9, 'f') + return None + +def parse(sig): + m = re.fullmatch(r'(\w+)\s+(\w+)\((.*)\)', sig) + ps = [p.strip() for p in m.group(3).split(',') if p.strip()] + return m.group(1), m.group(2), ps + +def hlsl_read(k, words): + return {'f': 'asfloat', 'i': 'asint', 'u': ''}[k] + '(' + words + ')' + +def gen(fx, out): + d = json.load(open(fx)); man = []; skipped = [] + for idx, f in enumerate(d['functions']): + r, name, ps = parse(f['glsl']) + rt = ty(r); pts = [ty(p) for p in ps] + if rt is None or any(p is None for p in pts): + skipped.append(f['glsl']); continue + nin = sum(p[1] for p in pts); nout = rt[1] + tag = 'f%03d' % idx + words = [] + for c in f['cases']: + a, b = c.split('->'); aw = a.split(); assert len(aw) == nin, (f['glsl'], len(aw), nin) + words += [int(x, 16) for x in aw] + open(os.path.join(out, tag + '.in'), 'wb').write(struct.pack('<%dI' % len(words), *words) if words else b'\0\0\0\0') + # HLSL + hl, gl, off = [], [], 0 + for j, (p, (ht, n, k)) in enumerate(zip(ps, pts)): + idxs = ', '.join('In[b + %d]' % (off + t) for t in range(n)) + gidx = ', '.join('In.a[b + %du]' % (off + t) for t in range(n)) + if ht == 'float3x3': raise SystemExit('mat3 arg unsupported') + vec = ('uint%d(%s)' % (n, idxs)) if n > 1 else idxs + hl.append(' %s a%d = %s;' % (ht, j, hlsl_read(k, vec) if k != 'u' else vec)) + gvec = ('uvec%d(%s)' % (n, gidx)) if n > 1 else gidx + gfun = {'f': 'uintBitsToFloat', 'i': ('ivec%d' % n if n > 1 else 'int'), 'u': ''}[k] + gl.append(' %s a%d = %s(%s);' % (p, j, gfun, gvec)) + off += n + args = ', '.join('a%d' % j for j in range(len(ps))) + hl.append(' %s r = %s(%s);' % (rt[0], name, args)) + gl.append(' %s r = %s(%s);' % (r, name, args)) + if rt[0] == 'float3x3': + comps = ['r[%d][%d]' % (i, j) for i in range(3) for j in range(3)] + elif rt[1] == 1: comps = ['r'] + else: comps = ['r[%d]' % i for i in range(rt[1])] + for i, c in enumerate(comps): + hl.append(' Out[o + %d] = %s;' % (i, {'f': 'asuint', 'i': 'asuint', 'u': ''}[rt[2]] + '(' + c + ')')) + gl.append(' Out.a[o + %du] = %s;' % (i, {'f': 'floatBitsToUint', 'i': 'uint', 'u': ''}[rt[2]] + '(' + c + ')')) + n = len(f['cases']) + open(os.path.join(out, tag + '.hlsl'), 'w').write( + '#include "CultMath.hlsl"\n[[vk::binding(0)]] StructuredBuffer In;\n[[vk::binding(1)]] RWStructuredBuffer Out;\n' + '[numthreads(64, 1, 1)] void main(uint3 id : SV_DispatchThreadID) {\n uint i = id.x; if (i >= %du) return; uint b = i * %du; uint o = i * %du;\n%s\n}\n' + % (n, nin, nout, '\n'.join(hl))) + open(os.path.join(out, tag + '.body.glsl'), 'w').write( + 'layout(local_size_x = 64) in;\nlayout(std430, binding = 0) readonly buffer IB { uint a[]; } In;\nlayout(std430, binding = 1) buffer OB { uint a[]; } Out;\n' + 'void main() {\n uint i = gl_GlobalInvocationID.x; if (i >= %du) return; uint b = i * %du; uint o = i * %du;\n%s\n}\n' + % (n, nin, nout, '\n'.join(gl))) + man.append({'tag': tag, 'glsl': f['glsl'], 'n': n, 'nout': nout, 'kind': rt[2], 'tol': f['tolerance'], 'platform': f.get('platform')}) + json.dump(man, open(os.path.join(out, 'manifest.json'), 'w')) + print('generated', len(man), 'families; skipped (struct types)', len(skipped), ':', '; '.join(skipped)) + +def ordered(u): # float bits -> monotonic int for ulp distance + return u if u < 0x80000000 else 0x80000000 - u + +def cmp(fx, out, tag): + d = json.load(open(fx)); man = json.load(open(os.path.join(out, 'manifest.json'))) + rows = []; buckets = {'exact': 0, '<=4': 0, '<=64': 0, '>64': 0, 'missing': 0} + for m in man: + p = os.path.join(out, m['tag'] + '.' + tag + '.out') + if not os.path.exists(p): buckets['missing'] += 1; rows.append((m['glsl'], 'MISSING', '')); continue + got = struct.unpack('<%dI' % (m['n'] * m['nout']), open(p, 'rb').read()) + f = d['functions'][int(m['tag'][1:])] + worst = 0; exact = 0; nanmis = 0 + for ci, c in enumerate(f['cases']): + exp = [int(x, 16) for x in c.split('->')[1].split()] + g = got[ci * m['nout']:(ci + 1) * m['nout']] + if list(g) == exp: exact += 1; continue + for e, h in zip(exp, g): + if m['kind'] != 'f': worst = max(worst, 1 << 40); continue + en = (e & 0x7f800000) == 0x7f800000 and e & 0x7fffff; hn = (h & 0x7f800000) == 0x7f800000 and h & 0x7fffff + if en or hn: + if not (en and hn): nanmis += 1 + continue + fe = struct.unpack('64' + if worst >= 1 << 40: b = '>64' + buckets[b] += 1 + rows.append((m['glsl'], '%d/%d exact' % (exact, m['n']), 'max err %s ulp(max(|v|,1))%s' % (('%.1f' % worst) if worst < 1 << 40 else 'int-mismatch', ', nan-mismatch %d' % nanmis if nanmis else ''))) + print(tag, 'buckets', buckets) + for r in sorted(rows, key=lambda r: r[1])[:0]: pass + rows.sort(key=lambda r: -float(re.search(r'max err ([0-9.]+)', r[2]).group(1)) if re.search(r'max err ([0-9.]+)', r[2]) else -1e18) + for r in rows[:int(os.environ.get('CMP_TOP', '25'))]: + if not r[1].startswith('256/256'): print(' ', r[0], '|', r[1], '|', r[2]) + +if __name__ == '__main__': + {'gen': lambda: gen(sys.argv[2], sys.argv[3]), 'cmp': lambda: cmp(sys.argv[2], sys.argv[3], sys.argv[4])}[sys.argv[1]]() diff --git a/docs/research/native-shader-probes/lib.sh b/docs/research/native-shader-probes/lib.sh new file mode 100644 index 00000000..39b62a70 --- /dev/null +++ b/docs/research/native-shader-probes/lib.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Probe job, runs in ubuntu:24.04 with /src = CultLib at the probed revision. Q1 library path, Q2 lavapipe runs. +set -u +P=/tmp/p; W=/tmp/w; mkdir -p $W/fam $W/tools; cd $W +step() { echo; echo "=== $*"; } +step apt +export DEBIAN_FRONTEND=noninteractive +apt-get update -qq >/dev/null && apt-get install -y -qq --no-install-recommends ca-certificates curl git cmake g++ make python3 \ + mesa-vulkan-drivers libvulkan-dev vulkan-tools glslang-tools spirv-tools >/dev/null 2>&1; echo "apt exit $?" +dpkg-query -W -f='${Package} ${Version}\n' mesa-vulkan-drivers glslang-tools spirv-tools libvulkan-dev +step dxc +url=$(curl -s https://api.github.com/repos/microsoft/DirectXShaderCompiler/releases/tags/v1.9.2607 | python3 -c "import json,sys;print([a['browser_download_url'] for a in json.load(sys.stdin)['assets'] if 'linux' in a['name'] and a['name'].endswith('.tar.gz')][0])") +echo "$url"; curl -sL --fail -o dxc.tgz "$url" && mkdir -p tools/dxc && tar xzf dxc.tgz -C tools/dxc; DXC=$(find tools/dxc -name dxc -type f | head -1); echo "DXC=$DXC"; $DXC --version 2>&1 | head -2 +step spirv-cross +git clone -q --depth 1 https://github.com/KhronosGroup/SPIRV-Cross.git tools/sc && git -C tools/sc log -1 --format='SPIRV-Cross %H %cs' && \ + cmake -S tools/sc -B tools/scb -DCMAKE_BUILD_TYPE=Release -DSPIRV_CROSS_ENABLE_TESTS=OFF >/dev/null && cmake --build tools/scb --target spirv-cross -j4 >/dev/null 2>&1; SC=tools/scb/spirv-cross; ls -la $SC | cut -c1-120 +SH=/src/packages/cultmath/shaders +python3 - "$SH" <<'PY' +import re,sys,os,shutil +src=sys.argv[1]; dst='/tmp/w/shx'; shutil.copytree(src,dst,dirs_exist_ok=True); n=0 +for fn in os.listdir(dst): + if fn.endswith('.hlsl'): + t=open(os.path.join(dst,fn)).read(); t2,k=re.subn(r'(?m)^(?!static|export)(\w+(?:\dx\d)?\s+cultmath_\w+\s*\(([^)]*)\))', lambda m: m.group(1) if re.search('Texture2D|SamplerState', m.group(2)) else 'export ' + m.group(1), t); k=len(re.findall(r'(?m)^export ', t2)); n+=k; open(os.path.join(dst,fn),'w').write(t2) +print('export keywords added', n) +PY +step "L1 dxc lib_6_3 universal1.5, export on non-texture defs" +$DXC -T lib_6_3 -HV 2021 -spirv -fspv-target-env=universal1.5 -I shx shx/CultMath.hlsl -Fo lib.spv 2>&1 | head -3; echo "exit ${PIPESTATUS[0]}" +spirv-dis lib.spv -o lib.spvasm; echo "exports $(grep -c 'LinkageAttributes.*Export' lib.spvasm); capabilities $(grep -c OpCapability lib.spvasm)" +step "L1b -default-linkage external, no export keyword" +$DXC -T lib_6_3 -HV 2021 -spirv -fspv-target-env=universal1.5 -default-linkage external -I $SH $SH/CultMath.hlsl -Fo libd.spv 2>&1 | head -3; echo "exit ${PIPESTATUS[0]}" +[ -s libd.spv ] && { spirv-dis libd.spv -o libd.spvasm; echo "exports $(grep -c 'LinkageAttributes.*Export' libd.spvasm)"; } +step "L2 strip every capability but Shader and Linkage, every extension; SPIRV-Cross ES 310 and 300" +grep -vP '^\s*OpCapability (?!Shader$|Linkage$)|^\s*OpExtension ' lib.spvasm > s.spvasm; spirv-as --target-env spv1.5 s.spvasm -o s.spv; echo "as exit $?" +for v in 310 300; do $SC s.spv --es --version $v --output s.$v.glsl 2>&1 | head -2; echo "es$v exit ${PIPESTATUS[0]}"; done +G=s.310.glsl; echo "G=$G"; [ -s "$G" ] || { echo PROBE-DONE; exit 0; } +step "L3 shape of $G" +echo "lines $(wc -l < $G); defs cultmath_: $(grep -cE '^\w+ cultmath_\w+\(' $G); distinct: $(grep -oE '^\w+ cultmath_\w+\(' $G | sed -E 's/^\w+ //' | sort -u | wc -l); renamed _N: $(grep -oE '^\w+ cultmath_\w+_[0-9]+\(' $G | wc -l); _NNN temporaries: $(grep -oE '\b_[0-9]+\b' $G | wc -l); non-cultmath defs: $(grep -cE '^\w+ (?!cultmath_)\w+\(' -P $G)" +grep -nE '^#|^layout|precision|^struct|^void main' $G | head -12 +echo "--- renamed defs sample:"; grep -oE '^\w+ cultmath_\w+_[0-9]+\([^)]*\)' $G | head -6 +echo "--- snoise(vec3):"; awk '/^(highp )?float cultmath_snoise\((highp )?vec3/{p=1} p{print; if(++n>=22) exit}' $G +sed -E '/^#version/d; /^layout\(local_size/d' $G | awk 'BEGIN{s=0} /^void main\(\)/{s=1} s==0{print} s==1&&/^\}/{s=0}' > lib.body.glsl +{ printf '#version 300 es\nprecision highp float;\nprecision highp int;\n'; cat lib.body.glsl; printf 'out highp vec4 c;\nvoid main() { c = vec4(cultmath_snoise(gl_FragCoord.xyz)); }\n'; } > es300.frag +glslangValidator es300.frag > es300.txt 2>&1; echo "glslang ES 3.00 fragment with the library: exit $? $(grep -m3 ERROR es300.txt | tr '\n' ' ' | cut -c1-300)" +step "L4 library through glslang on lavapipe" +gcc -O2 -o vkrun $P/vkrun.c -lvulkan 2>/dev/null; python3 $P/fam.py gen /src/packages/cultmath/tests/CultMath.Tests/fixtures/glsl-parity.json fam >/dev/null +okl=0; for b in fam/*.body.glsl; do t=${b%.body.glsl}; tag=$(basename $t); n=$(python3 -c "import json;m={x['tag']:x for x in json.load(open('fam/manifest.json'))}['$tag'];print(m['n'],m['nout'])") + { printf '#version 310 es\nprecision highp float;\nprecision highp int;\n'; cat lib.body.glsl $b; } > $t.l.comp + glslangValidator -V --target-env vulkan1.2 -o $t.l.spv $t.l.comp >$t.l.log 2>&1 && ./vkrun $t.l.spv $t.in $t.lib.out $n && okl=$((okl+1)); done +echo "library path ran $okl of 105"; grep -h -m1 ERROR fam/*.l.log | sed -E 's/[0-9]+:/N:/g' | sort | uniq -c | sort -rn | head -6 +CMP_TOP=8 python3 $P/fam.py cmp /src/packages/cultmath/tests/CultMath.Tests/fixtures/glsl-parity.json fam lib +echo "LIBGZ-BEGIN"; gzip -9c lib.body.glsl | base64 -w0; echo; echo "LIBGZ-END" +echo PROBE-DONE diff --git a/docs/research/native-shader-probes/probe.sh b/docs/research/native-shader-probes/probe.sh new file mode 100644 index 00000000..74a65d18 --- /dev/null +++ b/docs/research/native-shader-probes/probe.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Probe job, runs in ubuntu:24.04 with /src = CultLib at the probed revision. Q1 library path, Q2 lavapipe runs. +set -u +P=/tmp/p; W=/tmp/w; mkdir -p $W/fam $W/tools; cd $W +step() { echo; echo "=== $*"; } +step apt +export DEBIAN_FRONTEND=noninteractive +apt-get update -qq >/dev/null && apt-get install -y -qq --no-install-recommends ca-certificates curl git cmake g++ make python3 \ + mesa-vulkan-drivers libvulkan-dev vulkan-tools glslang-tools spirv-tools >/dev/null 2>&1; echo "apt exit $?" +dpkg-query -W -f='${Package} ${Version}\n' mesa-vulkan-drivers glslang-tools spirv-tools libvulkan-dev +step dxc +url=$(curl -s https://api.github.com/repos/microsoft/DirectXShaderCompiler/releases/tags/v1.9.2607 | python3 -c "import json,sys;print([a['browser_download_url'] for a in json.load(sys.stdin)['assets'] if 'linux' in a['name'] and a['name'].endswith('.tar.gz')][0])") +echo "$url"; curl -sL --fail -o dxc.tgz "$url" && mkdir -p tools/dxc && tar xzf dxc.tgz -C tools/dxc; DXC=$(find tools/dxc -name dxc -type f | head -1); echo "DXC=$DXC"; $DXC --version 2>&1 | head -2 +step spirv-cross +git clone -q --depth 1 https://github.com/KhronosGroup/SPIRV-Cross.git tools/sc && git -C tools/sc log -1 --format='SPIRV-Cross %H %cs' && \ + cmake -S tools/sc -B tools/scb -DCMAKE_BUILD_TYPE=Release -DSPIRV_CROSS_ENABLE_TESTS=OFF >/dev/null && cmake --build tools/scb --target spirv-cross -j4 >/dev/null 2>&1; SC=tools/scb/spirv-cross; ls -la $SC | cut -c1-120 +SH=/src/packages/cultmath/shaders +step "Q1a dxc library SPIR-V, no wrapper (lib_6_6, vulkan1.2)" +$DXC -T lib_6_6 -HV 2021 -spirv -fspv-target-env=vulkan1.2 -I $SH $SH/CultMath.hlsl -Fo lib.spv 2>&1 | head -8; echo "dxc exit ${PIPESTATUS[0]}" +[ -f lib.spv ] && { spirv-dis lib.spv -o lib.spvasm; echo "functions: $(grep -c ' = OpFunction ' lib.spvasm) exports: $(grep -c 'LinkageAttributes.*Export' lib.spvasm) entrypoints: $(grep -c OpEntryPoint lib.spvasm)"; } +step "Q1b dxc library with 'export' added to every top-level cultmath_ definition" +python3 - "$SH" <<'PY' +import re,sys,os,shutil +src=sys.argv[1]; dst='/tmp/w/shx'; shutil.copytree(src,dst,dirs_exist_ok=True) +n=0 +for fn in os.listdir(dst): + if not fn.endswith('.hlsl'): continue + t=open(os.path.join(dst,fn)).read() + t2,k=re.subn(r'(?m)^(?!static|export)(\w+(?:\dx\d)?\s+cultmath_\w+\s*\()', r'export \1', t); n+=k + open(os.path.join(dst,fn),'w').write(t2) +print('export keywords added', n) +PY +for v in "universal1.5" "vulkan1.2 -Vd" "vulkan1.3 -Vd"; do echo "--- variant: -fspv-target-env=$v"; rm -f libx.spv + $DXC -T lib_6_6 -HV 2021 -spirv -fspv-target-env=$v -I shx shx/CultMath.hlsl -Fo libx.spv 2>&1 | head -4; echo "dxc exit ${PIPESTATUS[0]}" + [ -s libx.spv ] && { spirv-dis libx.spv -o libx.spvasm; echo "functions: $(grep -c ' = OpFunction ' libx.spvasm) exports: $(grep -c 'LinkageAttributes.*Export' libx.spvasm) OpName cultmath_: $(grep -c 'OpName.*"cultmath_' libx.spvasm) Fma: $(grep -c ' Fma ' libx.spvasm) NoContraction: $(grep -c NoContraction libx.spvasm)"; spirv-val --target-env vulkan1.2 libx.spv 2>&1 | head -2; break; } +done +LIB=libx.spv +step "Q1c SPIRV-Cross ES output of $LIB" +for v in 300 310; do $SC $LIB --es --version $v --output lib$v.glsl 2>&1 | head -3; echo "spirv-cross --es --version $v exit ${PIPESTATUS[0]}"; done +for v in 300 310; do [ -s lib$v.glsl ] || continue + echo "lib$v.glsl: $(wc -l < lib$v.glsl) lines; defs named cultmath_: $(grep -cE '^\w+ cultmath_\w+\(' lib$v.glsl); distinct names: $(grep -oE '^\w+ cultmath_\w+\(' lib$v.glsl | sed -E 's/^\w+ //' | sort -u | wc -l); suffixed _N names: $(grep -oE '^\w+ cultmath_\w+_[0-9]+\(' lib$v.glsl | wc -l); _NNN temporaries: $(grep -oE '\b_[0-9]+\b' lib$v.glsl | wc -l); precision lines: $(grep -c precision lib$v.glsl); main: $(grep -c 'void main' lib$v.glsl)"; done +echo "committed CultMath.glsl: $(wc -l < $SH/CultMath.glsl) lines; defs $(grep -cE '^\w+ cultmath_\w+\(' $SH/CultMath.glsl); distinct names $(grep -oE '^\w+ cultmath_\w+\(' $SH/CultMath.glsl | sed -E 's/^\w+ //' | sort -u | wc -l)" +F=$(ls lib310.glsl lib300.glsl 2>/dev/null | head -1); [ -n "$F" ] && { echo "--- head of $F"; head -12 $F; echo "--- cultmath_snoise(vec3) as emitted (first 25 lines)"; awk '/^float cultmath_snoise\(/{p=1} p{print; if(++n>=25) exit}' $F; } +for v in 300 310; do [ -s lib$v.glsl ] && { glslangValidator -S comp lib$v.glsl >gv$v.txt 2>&1; echo "glslang -S comp lib$v.glsl exit $? : $(head -3 gv$v.txt | tr '\n' ' ' | cut -c1-300)"; }; done +step "Q2 lavapipe runner" +gcc -O2 -o vkrun $P/vkrun.c -lvulkan && echo "vkrun built" +python3 $P/fam.py gen /src/packages/cultmath/tests/CultMath.Tests/fixtures/glsl-parity.json fam +cat $SH/CultMath.glsl $SH/CultMath.Cycles.glsl $SH/CultMath.Phacelle.glsl > all.glsl +python3 - <<'PY' +import json; print('families', len(json.load(open('/tmp/w/fam/manifest.json')))) +PY +ok_h=0; ok_g=0; fail=0; first=1 +for h in fam/*.hlsl; do t=${h%.hlsl}; tag=$(basename $t) + n=$(python3 -c "import json;m={x['tag']:x for x in json.load(open('fam/manifest.json'))}['$tag'];print(m['n'],m['nout'])") + if $DXC -T cs_6_0 -E main -HV 2021 -spirv -fspv-target-env=vulkan1.2 -I $SH $h -Fo $t.h.spv >$t.h.log 2>&1; then + if [ $first = 1 ]; then VKRUN_INFO=1 ./vkrun $t.h.spv $t.in $t.hlsl.out $n; first=0; else ./vkrun $t.h.spv $t.in $t.hlsl.out $n; fi && ok_h=$((ok_h+1)) || echo "run fail hlsl $tag" + else fail=$((fail+1)); echo "dxc fail $tag: $(head -2 $t.h.log | tr '\n' ' ' | cut -c1-200)"; fi + { printf '#version 310 es\nprecision highp float;\nprecision highp int;\n'; cat all.glsl $t.body.glsl; } > $t.comp + if glslangValidator -V --target-env vulkan1.2 -o $t.g.spv $t.comp >$t.g.log 2>&1; then ./vkrun $t.g.spv $t.in $t.glsl.out $n && ok_g=$((ok_g+1)) || echo "run fail glsl $tag" + else echo "glslang fail $tag: $(grep -m2 ERROR $t.g.log | tr '\n' ' ' | cut -c1-200)"; fi +done +echo "ran: hlsl->dxc->lavapipe $ok_h, committed glsl->glslang->lavapipe $ok_g, dxc compile failures $fail" +echo "dxc-compute Fma total: $(for s in fam/*.h.spv; do spirv-dis $s | grep -c ' Fma '; done | python3 -c "import sys;print(sum(int(l) for l in sys.stdin))")" +python3 $P/fam.py cmp /src/packages/cultmath/tests/CultMath.Tests/fixtures/glsl-parity.json fam hlsl +python3 $P/fam.py cmp /src/packages/cultmath/tests/CultMath.Tests/fixtures/glsl-parity.json fam glsl +python3 - <<'PY2' +import os,filecmp +fs=sorted(f[:-9] for f in os.listdir('/tmp/w/fam') if f.endswith('.hlsl.out')) +same=sum(filecmp.cmp('/tmp/w/fam/%s.hlsl.out'%f,'/tmp/w/fam/%s.glsl.out'%f,shallow=False) for f in fs if os.path.exists('/tmp/w/fam/%s.glsl.out'%f)) +print('lavapipe: HLSL-path and GLSL-path outputs bit-identical in',same,'of',len(fs),'families') +PY2 +t0=$(date +%s); for h in fam/*.h.spv; do t=${h%.h.spv}; n=$(python3 -c "import json;m={x['tag']:x for x in json.load(open('fam/manifest.json'))}['$(basename $t)'];print(m['n'],m['nout'])"); ./vkrun $h $t.in $t.rerun.out $n; cmp -s $t.rerun.out $t.hlsl.out || echo "NONDETERMINISTIC $(basename $t)"; done; echo "rerun of 105 SPIR-V modules on lavapipe: $(( $(date +%s) - t0 ))s, determinism check done" +step "Q2b SPIRV-Cross ES 3.10 library through glslang on lavapipe (only if Q1 produced one)" +if [ -s lib310.glsl ]; then sed -E '/^#version/d; /^void main\(\)/,/^\}/d' lib310.glsl > lib.body.glsl; okl=0 + for b in fam/*.body.glsl; do t=${b%.body.glsl}; tag=$(basename $t); n=$(python3 -c "import json;m={x['tag']:x for x in json.load(open('fam/manifest.json'))}['$tag'];print(m['n'],m['nout'])") + { printf '#version 310 es\nprecision highp float;\nprecision highp int;\n'; cat lib.body.glsl $b; } > $t.l.comp + glslangValidator -V --target-env vulkan1.2 -o $t.l.spv $t.l.comp >$t.l.log 2>&1 && ./vkrun $t.l.spv $t.in $t.lib.out $n && okl=$((okl+1)) + done; echo "lib path ran $okl"; grep -h -m1 ERROR fam/*.l.log | sort | uniq -c | sort -rn | head -5 + python3 $P/fam.py cmp /src/packages/cultmath/tests/CultMath.Tests/fixtures/glsl-parity.json fam lib | head -15 +fi +echo PROBE-DONE diff --git a/docs/research/native-shader-probes/vkrun.c b/docs/research/native-shader-probes/vkrun.c new file mode 100644 index 00000000..8cd8d3f8 --- /dev/null +++ b/docs/research/native-shader-probes/vkrun.c @@ -0,0 +1,87 @@ +// Probe runner: one compute dispatch, binding 0 = input uints, binding 1 = output uints. +// usage: vkrun [entry] +#include +#include +#include +#include +#define CK(x) do { VkResult r_ = (x); if (r_ != VK_SUCCESS) { fprintf(stderr, "FAIL %s = %d\n", #x, r_); exit(2); } } while (0) + +static void *slurp(const char *p, size_t *n) { + FILE *f = fopen(p, "rb"); if (!f) { perror(p); exit(2); } + fseek(f, 0, SEEK_END); *n = ftell(f); fseek(f, 0, SEEK_SET); + void *b = malloc(*n ? *n : 4); fread(b, 1, *n, f); fclose(f); return b; +} + +int main(int argc, char **argv) { + if (argc < 6) { fprintf(stderr, "usage\n"); return 2; } + size_t spvn, inn; void *spv = slurp(argv[1], &spvn); void *in = slurp(argv[2], &inn); + unsigned ncases = atoi(argv[4]), ow = atoi(argv[5]); const char *entry = argc > 6 ? argv[6] : "main"; + size_t outn = (size_t)ncases * ow * 4; + + VkApplicationInfo ai = { VK_STRUCTURE_TYPE_APPLICATION_INFO, 0, "probe", 1, "probe", 1, VK_API_VERSION_1_2 }; + VkInstanceCreateInfo ici = { VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO, 0, 0, &ai }; + VkInstance inst; CK(vkCreateInstance(&ici, 0, &inst)); + uint32_t np = 1; VkPhysicalDevice pd; vkEnumeratePhysicalDevices(inst, &np, &pd); if (!np) { fprintf(stderr, "no device\n"); return 2; } + VkPhysicalDeviceFloatControlsProperties fc = { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FLOAT_CONTROLS_PROPERTIES }; + VkPhysicalDeviceProperties2 pp = { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &fc }; + vkGetPhysicalDeviceProperties2(pd, &pp); + if (getenv("VKRUN_INFO")) + printf("device=%s api=%u.%u driver=%u f32: denormPreserve=%u ftz=%u rte=%u rtz=%u signedZeroInfNan=%u\n", + pp.properties.deviceName, VK_VERSION_MAJOR(pp.properties.apiVersion), VK_VERSION_MINOR(pp.properties.apiVersion), + pp.properties.driverVersion, fc.shaderDenormPreserveFloat32, fc.shaderDenormFlushToZeroFloat32, + fc.shaderRoundingModeRTEFloat32, fc.shaderRoundingModeRTZFloat32, fc.shaderSignedZeroInfNanPreserveFloat32); + + float prio = 1; VkDeviceQueueCreateInfo qci = { VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO, 0, 0, 0, 1, &prio }; + VkDeviceCreateInfo dci = { VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO, 0, 0, 1, &qci }; + VkDevice dev; CK(vkCreateDevice(pd, &dci, 0, &dev)); VkQueue q; vkGetDeviceQueue(dev, 0, 0, &q); + + VkPhysicalDeviceMemoryProperties mp; vkGetPhysicalDeviceMemoryProperties(pd, &mp); + VkBuffer buf[2]; VkDeviceMemory mem[2]; size_t sz[2] = { inn, outn }; void *map[2]; + for (int i = 0; i < 2; i++) { + VkBufferCreateInfo bci = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, 0, 0, sz[i], VK_BUFFER_USAGE_STORAGE_BUFFER_BIT }; + CK(vkCreateBuffer(dev, &bci, 0, &buf[i])); VkMemoryRequirements mr; vkGetBufferMemoryRequirements(dev, buf[i], &mr); + uint32_t t = 0; VkMemoryPropertyFlags want = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + for (; t < mp.memoryTypeCount; t++) if ((mr.memoryTypeBits & (1u << t)) && (mp.memoryTypes[t].propertyFlags & want) == want) break; + VkMemoryAllocateInfo mai = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, 0, mr.size, t }; + CK(vkAllocateMemory(dev, &mai, 0, &mem[i])); CK(vkBindBufferMemory(dev, buf[i], mem[i], 0)); + CK(vkMapMemory(dev, mem[i], 0, sz[i], 0, &map[i])); + } + memcpy(map[0], in, inn); memset(map[1], 0xEE, outn); + + VkDescriptorSetLayoutBinding b[2] = { { 0, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT }, + { 1, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT } }; + VkDescriptorSetLayoutCreateInfo lci = { VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, 0, 0, 2, b }; + VkDescriptorSetLayout dsl; CK(vkCreateDescriptorSetLayout(dev, &lci, 0, &dsl)); + VkPipelineLayoutCreateInfo plci = { VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, 0, 0, 1, &dsl }; + VkPipelineLayout pl; CK(vkCreatePipelineLayout(dev, &plci, 0, &pl)); + VkShaderModuleCreateInfo smci = { VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, 0, 0, spvn, spv }; + VkShaderModule sm; CK(vkCreateShaderModule(dev, &smci, 0, &sm)); + VkComputePipelineCreateInfo cpci = { VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, 0, 0, + { VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, 0, 0, VK_SHADER_STAGE_COMPUTE_BIT, sm, entry }, pl }; + VkPipeline pipe; CK(vkCreateComputePipelines(dev, 0, 1, &cpci, 0, &pipe)); + + VkDescriptorPoolSize ps = { VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2 }; + VkDescriptorPoolCreateInfo dpci = { VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO, 0, 0, 1, 1, &ps }; + VkDescriptorPool dp; CK(vkCreateDescriptorPool(dev, &dpci, 0, &dp)); + VkDescriptorSetAllocateInfo dsai = { VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO, 0, dp, 1, &dsl }; + VkDescriptorSet ds; CK(vkAllocateDescriptorSets(dev, &dsai, &ds)); + VkDescriptorBufferInfo dbi[2] = { { buf[0], 0, VK_WHOLE_SIZE }, { buf[1], 0, VK_WHOLE_SIZE } }; + VkWriteDescriptorSet w[2]; + for (int i = 0; i < 2; i++) { VkWriteDescriptorSet x = { VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, 0, ds, i, 0, 1, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 0, &dbi[i] }; w[i] = x; } + vkUpdateDescriptorSets(dev, 2, w, 0, 0); + + VkCommandPoolCreateInfo cpi = { VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO, 0, 0, 0 }; + VkCommandPool cp; CK(vkCreateCommandPool(dev, &cpi, 0, &cp)); + VkCommandBufferAllocateInfo cbai = { VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, 0, cp, VK_COMMAND_BUFFER_LEVEL_PRIMARY, 1 }; + VkCommandBuffer cb; CK(vkAllocateCommandBuffers(dev, &cbai, &cb)); + VkCommandBufferBeginInfo cbbi = { VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO }; + CK(vkBeginCommandBuffer(cb, &cbbi)); + vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, pipe); + vkCmdBindDescriptorSets(cb, VK_PIPELINE_BIND_POINT_COMPUTE, pl, 0, 1, &ds, 0, 0); + vkCmdDispatch(cb, (ncases + 63) / 64, 1, 1); + CK(vkEndCommandBuffer(cb)); + VkSubmitInfo si = { VK_STRUCTURE_TYPE_SUBMIT_INFO, 0, 0, 0, 0, 1, &cb }; + CK(vkQueueSubmit(q, 1, &si, 0)); CK(vkQueueWaitIdle(q)); + FILE *f = fopen(argv[3], "wb"); fwrite(map[1], 1, outn, f); fclose(f); + return 0; +}