diff --git a/packages/cultmath/docs/design.md b/packages/cultmath/docs/design.md index c66aa8e9..4c9cb4ed 100644 --- a/packages/cultmath/docs/design.md +++ b/packages/cultmath/docs/design.md @@ -265,10 +265,13 @@ name the platform that generated them and the test compares them case by case against the running platform, printing the largest distance. `iv_exp` ("ulp-bounded, platform exp") must agree within one ulp. `phacelle` ("ulp-bounded, platform exp, sin, cos; across platforms |diff| <= 2^-20 -max(|v|, 1)") must agree within `2^-20 max(|v|, 1)` per component, `v` being +max(|v|, 1); on a device |diff| <= 2^-18 max(|v|, 1)") must agree across +platforms within `2^-20 max(|v|, 1)` per component, `v` being the fixture's value: its sums of `exp`, `sin` and `cos` cancel near zero, where ulps mean nothing, and Windows differs from the Linux fixture by up to -`3.6e-7 max(|v|, 1)`. Every other family is compared as text. `phacelle` is in +`3.6e-7 max(|v|, 1)`. The device clause is `2^-18` because the devices measured +are farther: lavapipe is `2.2e-6 max(|v|, 1)` and the GTX 1070 `3.1e-6`; the conforming +run judges it. Every other family is compared as text. `phacelle` is in `CultMath.Phacelle.glsl`, so the evaluator concatenates `CultMath.Phacelle.glsl` after `CultMath.glsl` for that family, with normalization 0.5. The site does not vendor the MPL file and skips the family. diff --git a/packages/cultmath/tests/CultMath.Tests/GlslMirrorTests.Device.cs b/packages/cultmath/tests/CultMath.Tests/GlslMirrorTests.Device.cs index 59d3e2e1..51186a07 100644 --- a/packages/cultmath/tests/CultMath.Tests/GlslMirrorTests.Device.cs +++ b/packages/cultmath/tests/CultMath.Tests/GlslMirrorTests.Device.cs @@ -112,7 +112,7 @@ public sealed partial class GlslMirrorTests // ---- Whether a device's results hold ---- - private sealed record BoundRow(string Route, int Index, string Glsl, string Tolerance, int Exact, double Worst, int NanClass); + private sealed record BoundRow(string Route, int Index, string Glsl, string Tolerance, int Exact, double Worst, int NanClass, string Bits); private sealed record Measured(BoundRow Row, string Class); @@ -183,6 +183,23 @@ public sealed partial class GlslMirrorTests }; } + // sha256 of the family's result bytes, and of its reference result when it is judged on one. + private static string ResultBits(string path, string? reference) + { + var bytes = File.ReadAllBytes(path); + if (reference is not null) + bytes = bytes.Concat(File.ReadAllBytes(reference)).ToArray(); + return Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(bytes)).ToLowerInvariant(); + } + + // A tolerance text may state what a device is allowed: "on a device |diff| <= 2^-N max(|v|, 1)", which in + // u1 units (2^-23) is 2^(23-N). The judge reads it here and nowhere else. + private static double? StatedDeviceLimit(string tolerance) + { + var m = System.Text.RegularExpressions.Regex.Match(tolerance, @"on a device \|diff\| <= 2\^-(\d+) max\(\|v\|, 1\)"); + return m.Success ? Math.ScaleB(1.0, 23 - int.Parse(m.Groups[1].Value)) : null; + } + private static string FirstLine(Exception e) { var line = e.Message.Split('\n')[0]; @@ -229,9 +246,10 @@ public sealed partial class GlslMirrorTests } var (exact, worst, nanClass) = Measure(cases, floatWord, got); + var bits = ResultBits(path, IsSampledForm(family) ? Path.Combine(results, route, RefTag(f) + ".out") : null); var exactClass = family.Tolerance == "exact" || !floatWord.Any(w => w); var kind = exactClass ? "exact" : IsSampledForm(family) ? "sampled" : family.Verify is not null ? "check" : "bound"; - measured.Add(new Measured(new BoundRow(route, f, family.Glsl, family.Tolerance, exact, worst, nanClass), kind)); + measured.Add(new Measured(new BoundRow(route, f, family.Glsl, family.Tolerance, exact, worst, nanClass, bits), kind)); try { if (exactClass) @@ -268,6 +286,10 @@ public sealed partial class GlslMirrorTests failures.Add($"{name}: the recorded row is for another family; re-record with CULTMATH_WRITE_DEVICE_BOUNDS=1"); else if (m.Class == "bound" && (m.Row.Worst > bound.Worst || m.Row.NanClass > bound.NanClass)) failures.Add($"{name}: worst u1 {m.Row.Worst:R} and {m.Row.NanClass} NaN-class mismatches, past the recorded {bound.Worst:R} and {bound.NanClass}"); + else if (m.Class == "bound" && StatedDeviceLimit(m.Row.Tolerance) is { } limit && m.Row.Worst > limit) + failures.Add($"{name}: worst u1 {m.Row.Worst:R}, past the {limit:R} its stated tolerance allows a device"); + else if (m.Row.Bits != bound.Bits && !failures.Any(x => x.StartsWith(name + ":"))) + failures.Add($"{name}: the result bits differ from the recorded run's; if the change is meant, re-record with CULTMATH_WRITE_DEVICE_BOUNDS=1"); } return new Judgement(failures, measured); @@ -281,10 +303,10 @@ public sealed partial class GlslMirrorTests { "# renderer\t" + renderer, "# identity\t" + identity, - "# route\tindex\tglsl\ttolerance\texact cases\tworst u1\tNaN-class mismatches; written only by CULTMATH_WRITE_DEVICE_BOUNDS=1", + "# route\tindex\tglsl\ttolerance\texact cases\tworst u1\tNaN-class mismatches result sha256; written only by CULTMATH_WRITE_DEVICE_BOUNDS=1", }; lines.AddRange(rows.OrderBy(r => r.Route, StringComparer.Ordinal).ThenBy(r => r.Index) - .Select(r => string.Join('\t', r.Route, r.Index, r.Glsl, r.Tolerance, r.Exact, r.Worst.ToString("R", CultureInfo.InvariantCulture), r.NanClass))); + .Select(r => string.Join('\t', r.Route, r.Index, r.Glsl, r.Tolerance, r.Exact, r.Worst.ToString("R", CultureInfo.InvariantCulture), r.NanClass, r.Bits))); File.WriteAllText(path, string.Join('\n', lines) + "\n"); } @@ -298,7 +320,7 @@ public sealed partial class GlslMirrorTests if (line.StartsWith("# renderer\t") || line.StartsWith("# identity\t")) header[c[0][2..]] = c[1]; else if (!line.StartsWith('#') && line.Length > 0) - rows[(c[0], int.Parse(c[1]))] = new BoundRow(c[0], int.Parse(c[1]), c[2], c[3], int.Parse(c[4]), double.Parse(c[5], CultureInfo.InvariantCulture), int.Parse(c[6])); + rows[(c[0], int.Parse(c[1]))] = new BoundRow(c[0], int.Parse(c[1]), c[2], c[3], int.Parse(c[4]), double.Parse(c[5], CultureInfo.InvariantCulture), int.Parse(c[6]), c[7]); } return (header, rows); @@ -604,4 +626,30 @@ public sealed partial class GlslMirrorTests Assert.Equal(DeviceRoutes.Length, failures.Count); Assert.All(failures, line => Assert.Contains($"#{f} ", line)); } + + [Fact] + public void AChangeInResultBitsFailsEvenWithinTheRecordedBound() + { + // Recorded on a device off in every case; judged on one off in a single case. The bound holds, the bits do not. + var f = BoundFamily(); + using var recorded = Simulate((_, item, words) => item.Index == f.ToString() ? Alter(item, words, w => w ^ 4u) : words); + Record(recorded); + using var fewer = Simulate((_, item, words) => item.Index == f.ToString() ? Alter(item, words, w => w ^ 4u, 1) : words); + var failures = Judge(fewer, recorded).Failures; + Assert.Equal(DeviceRoutes.Length, failures.Count); + Assert.All(failures, line => { Assert.Contains($"#{f} ", line); Assert.Contains("result bits differ", line); }); + } + + [Fact] + public void AFamilyIsHeldToTheDeviceToleranceItsOwnTextStates() + { + // Recorded on the very device that violates it: only the stated tolerance can fail the family. + var f = Array.FindIndex(Families, fam => StatedDeviceLimit(fam.Tolerance) is not null); + Assert.True(f >= 0, "no family states a device tolerance"); + using var far = Simulate((_, item, words) => item.Index == f.ToString() ? Alter(item, words, w => w ^ 0x00100000u) : words); + Record(far); + var failures = Judge(far, far).Failures; + Assert.Equal(DeviceRoutes.Length, failures.Count); + Assert.All(failures, line => Assert.Contains("stated tolerance", line)); + } } diff --git a/packages/cultmath/tests/CultMath.Tests/GlslMirrorTests.Precision.cs b/packages/cultmath/tests/CultMath.Tests/GlslMirrorTests.Precision.cs new file mode 100644 index 00000000..0815b1f5 --- /dev/null +++ b/packages/cultmath/tests/CultMath.Tests/GlslMirrorTests.Precision.cs @@ -0,0 +1,56 @@ +using System.Text.RegularExpressions; +using Xunit; + +namespace CultMath.Tests; + +public sealed partial class GlslMirrorTests +{ + // Tripwire: lavapipe and desktop ANGLE ignore mediump and min16float, so a reduced-precision qualifier in the + // committed shaders would change the computation on a phone GPU and nowhere the conforming run can see it. + // Stated limits: a syntax guardrail only. It reads whole words in the committed .glsl and .hlsl under shaders/ + // with comments removed; it does not see a qualifier spelled through a macro, an include or a generated + // shader, and it says nothing about a driver that relaxes highp itself. It names file and line, never text. + private static readonly Regex ReducedPrecision = new( + @"\b(mediump|lowp|min16float|min10float|min16int|min12int|min16uint|half([1-4](x[1-4])?)?)\b|\bprecision\s+(?!highp\b)\w+", RegexOptions.Compiled); + + private static IEnumerable ReducedPrecisionLines(string source) + { + var text = Regex.Replace(source, @"/\*.*?\*/", m => new string('\n', m.Value.Count(c => c == '\n')), RegexOptions.Singleline); + var lines = text.Split('\n'); + for (var i = 0; i < lines.Length; i++) + { + var code = lines[i].Split("//")[0]; + if (ReducedPrecision.IsMatch(code)) + yield return i + 1; + } + } + + [Fact] + public void CommittedShadersCarryNoReducedPrecisionQualifier() + { + var found = new List(); + foreach (var file in Directory.EnumerateFiles(Path.Combine(Root, "shaders")).Where(f => f.EndsWith(".glsl") || f.EndsWith(".hlsl")).OrderBy(f => f, StringComparer.Ordinal)) + found.AddRange(ReducedPrecisionLines(File.ReadAllText(file)).Select(line => $"{Path.GetFileName(file)}:{line}")); + Assert.True(found.Count == 0, "reduced-precision qualifier at " + string.Join(", ", found)); + } + + [Theory] + [InlineData("mediump float x;", true)] + [InlineData("lowp vec3 c;", true)] + [InlineData("precision mediump float;", true)] + [InlineData("precision highp float;", false)] + [InlineData("min16float x;", true)] + [InlineData("min10float2 x;", true)] + [InlineData("min16int i;", true)] + [InlineData("half3x3 m;", true)] + [InlineData("float half_width;", false)] + [InlineData("float x; // mediump is not used", false)] + [InlineData("/* half\n min16float */ float x;", false)] + public void TheTripwireReadsQualifiersAndNotComments(string source, bool trips) => Assert.Equal(trips, ReducedPrecisionLines(source).Any()); + + [Fact] + public void TheTripwireNamesTheLine() + { + Assert.Equal(new[] { 3 }, ReducedPrecisionLines("float a;\n/* c */\nmediump float b;\n").ToArray()); + } +} diff --git a/packages/cultmath/tests/CultMath.Tests/GlslMirrorTests.cs b/packages/cultmath/tests/CultMath.Tests/GlslMirrorTests.cs index d642d66e..120e06fb 100644 --- a/packages/cultmath/tests/CultMath.Tests/GlslMirrorTests.cs +++ b/packages/cultmath/tests/CultMath.Tests/GlslMirrorTests.cs @@ -684,7 +684,7 @@ static int4 clamp(int4 x, int4 a, int4 b) { if (a.x > b.x || a.y > b.y || a.z > return new object[] { slope, z0, z0 + Uniform(r, 0.01f, 50.0f), Uniform(r, 0.0001f, 0.05f), Uniform(r, 0.0f, 2.0f) }; }, FrustumBall, Verify: AssertFrustumBall), new("vec2 cultmath_iv_exp(vec2)", "ulp-bounded, platform exp", r => new object[] { Interval(r) }, Platform: Platform.Ulp), - new("CultPhasor cultmath_phacelle(vec3, vec3, float, float)", "ulp-bounded, platform exp, sin, cos; across platforms |diff| <= 2^-20 max(|v|, 1)", + new("CultPhasor cultmath_phacelle(vec3, vec3, float, float)", "ulp-bounded, platform exp, sin, cos; across platforms |diff| <= 2^-20 max(|v|, 1); on a device |diff| <= 2^-18 max(|v|, 1)", r => new object[] { Point3(r), Side(r), Uniform(r, 0.0f, 1.0f), 0.5f }, Platform: Platform.Scaled), new("float cultmath_snoise(vec3)", "ulp-bounded, on the simplex tie set", r => new object[] { TiePoint3(r) }), new("vec4 cultmath_snoise_grad(vec3)", "ulp-bounded, on the simplex tie set", r => new object[] { TiePoint3(r) }), diff --git a/packages/cultmath/tests/CultMath.Tests/fixtures/device-bounds-lavapipe.txt b/packages/cultmath/tests/CultMath.Tests/fixtures/device-bounds-lavapipe.txt index 305f1cf4..5ca286d6 100644 --- a/packages/cultmath/tests/CultMath.Tests/fixtures/device-bounds-lavapipe.txt +++ b/packages/cultmath/tests/CultMath.Tests/fixtures/device-bounds-lavapipe.txt @@ -28,7 +28,7 @@ lavapipe-glsl 23 vec2 cultmath_iv_snoise_ball(vec3, float) ulp-bounded 242 54 0 lavapipe-glsl 24 vec2 cultmath_iv_fbm_ball(vec3, float, int, float, float) ulp-bounded 250 56 0 lavapipe-glsl 25 vec4 cultmath_iv_frustum_ball(vec2, float, float, float, float) ulp-bounded 198 1.6457063576593662 0 lavapipe-glsl 26 vec2 cultmath_iv_exp(vec2) ulp-bounded, platform exp 7 7.526057804152758 0 -lavapipe-glsl 27 CultPhasor cultmath_phacelle(vec3, vec3, float, float) ulp-bounded, platform exp, sin, cos; across platforms |diff| <= 2^-20 max(|v|, 1) 0 18.5 0 +lavapipe-glsl 27 CultPhasor cultmath_phacelle(vec3, vec3, float, float) ulp-bounded, platform exp, sin, cos; across platforms |diff| <= 2^-20 max(|v|, 1); on a device |diff| <= 2^-18 max(|v|, 1) 0 18.5 0 lavapipe-glsl 28 float cultmath_snoise(vec3) ulp-bounded, on the simplex tie set 127 112 0 lavapipe-glsl 29 vec4 cultmath_snoise_grad(vec3) ulp-bounded, on the simplex tie set 102 546.5 0 lavapipe-glsl 30 vec3 cultmath_af_point(float) ulp-bounded 256 0 0 @@ -137,7 +137,7 @@ lavapipe-hlsl 23 vec2 cultmath_iv_snoise_ball(vec3, float) ulp-bounded 244 54 0 lavapipe-hlsl 24 vec2 cultmath_iv_fbm_ball(vec3, float, int, float, float) ulp-bounded 252 75 0 lavapipe-hlsl 25 vec4 cultmath_iv_frustum_ball(vec2, float, float, float, float) ulp-bounded 198 1.6457063576593662 0 lavapipe-hlsl 26 vec2 cultmath_iv_exp(vec2) ulp-bounded, platform exp 7 7.526057804152758 0 -lavapipe-hlsl 27 CultPhasor cultmath_phacelle(vec3, vec3, float, float) ulp-bounded, platform exp, sin, cos; across platforms |diff| <= 2^-20 max(|v|, 1) 0 18.5 0 +lavapipe-hlsl 27 CultPhasor cultmath_phacelle(vec3, vec3, float, float) ulp-bounded, platform exp, sin, cos; across platforms |diff| <= 2^-20 max(|v|, 1); on a device |diff| <= 2^-18 max(|v|, 1) 0 18.5 0 lavapipe-hlsl 28 float cultmath_snoise(vec3) ulp-bounded, on the simplex tie set 144 99 0 lavapipe-hlsl 29 vec4 cultmath_snoise_grad(vec3) ulp-bounded, on the simplex tie set 111 546.5 0 lavapipe-hlsl 30 vec3 cultmath_af_point(float) ulp-bounded 256 0 0 diff --git a/packages/cultmath/tests/CultMath.Tests/fixtures/glsl-parity.json b/packages/cultmath/tests/CultMath.Tests/fixtures/glsl-parity.json index 7d8b83ca..e0971399 100644 --- a/packages/cultmath/tests/CultMath.Tests/fixtures/glsl-parity.json +++ b/packages/cultmath/tests/CultMath.Tests/fixtures/glsl-parity.json @@ -7082,7 +7082,7 @@ }, { "glsl": "CultPhasor cultmath_phacelle(vec3, vec3, float, float)", - "tolerance": "ulp-bounded, platform exp, sin, cos; across platforms |diff| <= 2^-20 max(|v|, 1)", + "tolerance": "ulp-bounded, platform exp, sin, cos; across platforms |diff| <= 2^-20 max(|v|, 1); on a device |diff| <= 2^-18 max(|v|, 1)", "platform": "Ubuntu 24.04.5 LTS; .NET 10.0.12", "cases": [ "c01d8630 41b2cd8c 41ea274c bfa19894 40aeb6ce 40918978 3e9be843 3f000000 -> bf74d9fb 3fb71956 bf35fe4e 3f21280a c01277e2 403b5fe4 40464bbd bd5b4455", diff --git a/packages/cultmath/tools/device/run-lavapipe.sh b/packages/cultmath/tools/device/run-lavapipe.sh index 052bad66..22b04ece 100644 --- a/packages/cultmath/tools/device/run-lavapipe.sh +++ b/packages/cultmath/tools/device/run-lavapipe.sh @@ -57,4 +57,4 @@ if [ "${WRITE_BOUNDS:-0}" = 1 ]; then export CULTMATH_WRITE_DEVICE_BOUNDS=1 trap 'echo BOUNDS-BEGIN; cat "$TESTS/fixtures/device-bounds-lavapipe.txt"; echo BOUNDS-END' EXIT fi -dotnet test "$TESTS" --no-build --filter "FullyQualifiedName~GlslMirrorTests.DeviceResultsHoldFixture" --logger "console;verbosity=detailed" +dotnet test "$TESTS" --no-build --filter "FullyQualifiedName~GlslMirrorTests.DeviceResultsHoldFixture|FullyQualifiedName~GlslMirrorTests.CommittedShadersCarryNoReducedPrecisionQualifier" --logger "console;verbosity=detailed"