cultmath: lavapipe bounds carry exact result bits, a reduced-precision tripwire, and the judge reads the stated device tolerance
Per verdict s1: the judge keeps the fixture tolerance as correctness and adds a recorded sha256 of each family's result per route (a change in the computation fails until re-recorded); a test refuses mediump/lowp/precision other than highp and min16float/min10float/min16int/half in the committed shaders; phacelle's tolerance text states a device clause (2^-18) that the judge parses and enforces. The committed bounds file lacks the new column until the next WRITE_BOUNDS=1 job, so this commit's device run fails and it does not pass DeviceResultsHoldFixture. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Meta Vens
parent
ea7e665e6c
commit
9a3ad78c11
7 changed files with 119 additions and 12 deletions
|
|
@ -265,10 +265,13 @@ name the platform that generated them and the test compares them case by case
|
|||
against the running platform, printing the largest distance. `iv_exp`
|
||||
("ulp-bounded, platform exp") must agree within one ulp. `phacelle`
|
||||
("ulp-bounded, platform exp, sin, cos; across platforms |diff| <= 2^-20
|
||||
max(|v|, 1)") must agree within `2^-20 max(|v|, 1)` per component, `v` being
|
||||
max(|v|, 1); on a device |diff| <= 2^-18 max(|v|, 1)") must agree across
|
||||
platforms within `2^-20 max(|v|, 1)` per component, `v` being
|
||||
the fixture's value: its sums of `exp`, `sin` and `cos` cancel near zero, where
|
||||
ulps mean nothing, and Windows differs from the Linux fixture by up to
|
||||
`3.6e-7 max(|v|, 1)`. Every other family is compared as text. `phacelle` is in
|
||||
`3.6e-7 max(|v|, 1)`. The device clause is `2^-18` because the devices measured
|
||||
are farther: lavapipe is `2.2e-6 max(|v|, 1)` and the GTX 1070 `3.1e-6`; the conforming
|
||||
run judges it. Every other family is compared as text. `phacelle` is in
|
||||
`CultMath.Phacelle.glsl`, so the evaluator concatenates
|
||||
`CultMath.Phacelle.glsl` after `CultMath.glsl` for that family, with
|
||||
normalization 0.5. The site does not vendor the MPL file and skips the family.
|
||||
|
|
|
|||
|
|
@ -112,7 +112,7 @@ public sealed partial class GlslMirrorTests
|
|||
|
||||
// ---- Whether a device's results hold ----
|
||||
|
||||
private sealed record BoundRow(string Route, int Index, string Glsl, string Tolerance, int Exact, double Worst, int NanClass);
|
||||
private sealed record BoundRow(string Route, int Index, string Glsl, string Tolerance, int Exact, double Worst, int NanClass, string Bits);
|
||||
|
||||
private sealed record Measured(BoundRow Row, string Class);
|
||||
|
||||
|
|
@ -183,6 +183,23 @@ public sealed partial class GlslMirrorTests
|
|||
};
|
||||
}
|
||||
|
||||
// sha256 of the family's result bytes, and of its reference result when it is judged on one.
|
||||
private static string ResultBits(string path, string? reference)
|
||||
{
|
||||
var bytes = File.ReadAllBytes(path);
|
||||
if (reference is not null)
|
||||
bytes = bytes.Concat(File.ReadAllBytes(reference)).ToArray();
|
||||
return Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(bytes)).ToLowerInvariant();
|
||||
}
|
||||
|
||||
// A tolerance text may state what a device is allowed: "on a device |diff| <= 2^-N max(|v|, 1)", which in
|
||||
// u1 units (2^-23) is 2^(23-N). The judge reads it here and nowhere else.
|
||||
private static double? StatedDeviceLimit(string tolerance)
|
||||
{
|
||||
var m = System.Text.RegularExpressions.Regex.Match(tolerance, @"on a device \|diff\| <= 2\^-(\d+) max\(\|v\|, 1\)");
|
||||
return m.Success ? Math.ScaleB(1.0, 23 - int.Parse(m.Groups[1].Value)) : null;
|
||||
}
|
||||
|
||||
private static string FirstLine(Exception e)
|
||||
{
|
||||
var line = e.Message.Split('\n')[0];
|
||||
|
|
@ -229,9 +246,10 @@ public sealed partial class GlslMirrorTests
|
|||
}
|
||||
|
||||
var (exact, worst, nanClass) = Measure(cases, floatWord, got);
|
||||
var bits = ResultBits(path, IsSampledForm(family) ? Path.Combine(results, route, RefTag(f) + ".out") : null);
|
||||
var exactClass = family.Tolerance == "exact" || !floatWord.Any(w => w);
|
||||
var kind = exactClass ? "exact" : IsSampledForm(family) ? "sampled" : family.Verify is not null ? "check" : "bound";
|
||||
measured.Add(new Measured(new BoundRow(route, f, family.Glsl, family.Tolerance, exact, worst, nanClass), kind));
|
||||
measured.Add(new Measured(new BoundRow(route, f, family.Glsl, family.Tolerance, exact, worst, nanClass, bits), kind));
|
||||
try
|
||||
{
|
||||
if (exactClass)
|
||||
|
|
@ -268,6 +286,10 @@ public sealed partial class GlslMirrorTests
|
|||
failures.Add($"{name}: the recorded row is for another family; re-record with CULTMATH_WRITE_DEVICE_BOUNDS=1");
|
||||
else if (m.Class == "bound" && (m.Row.Worst > bound.Worst || m.Row.NanClass > bound.NanClass))
|
||||
failures.Add($"{name}: worst u1 {m.Row.Worst:R} and {m.Row.NanClass} NaN-class mismatches, past the recorded {bound.Worst:R} and {bound.NanClass}");
|
||||
else if (m.Class == "bound" && StatedDeviceLimit(m.Row.Tolerance) is { } limit && m.Row.Worst > limit)
|
||||
failures.Add($"{name}: worst u1 {m.Row.Worst:R}, past the {limit:R} its stated tolerance allows a device");
|
||||
else if (m.Row.Bits != bound.Bits && !failures.Any(x => x.StartsWith(name + ":")))
|
||||
failures.Add($"{name}: the result bits differ from the recorded run's; if the change is meant, re-record with CULTMATH_WRITE_DEVICE_BOUNDS=1");
|
||||
}
|
||||
|
||||
return new Judgement(failures, measured);
|
||||
|
|
@ -281,10 +303,10 @@ public sealed partial class GlslMirrorTests
|
|||
{
|
||||
"# renderer\t" + renderer,
|
||||
"# identity\t" + identity,
|
||||
"# route\tindex\tglsl\ttolerance\texact cases\tworst u1\tNaN-class mismatches; written only by CULTMATH_WRITE_DEVICE_BOUNDS=1",
|
||||
"# route\tindex\tglsl\ttolerance\texact cases\tworst u1\tNaN-class mismatches result sha256; written only by CULTMATH_WRITE_DEVICE_BOUNDS=1",
|
||||
};
|
||||
lines.AddRange(rows.OrderBy(r => r.Route, StringComparer.Ordinal).ThenBy(r => r.Index)
|
||||
.Select(r => string.Join('\t', r.Route, r.Index, r.Glsl, r.Tolerance, r.Exact, r.Worst.ToString("R", CultureInfo.InvariantCulture), r.NanClass)));
|
||||
.Select(r => string.Join('\t', r.Route, r.Index, r.Glsl, r.Tolerance, r.Exact, r.Worst.ToString("R", CultureInfo.InvariantCulture), r.NanClass, r.Bits)));
|
||||
File.WriteAllText(path, string.Join('\n', lines) + "\n");
|
||||
}
|
||||
|
||||
|
|
@ -298,7 +320,7 @@ public sealed partial class GlslMirrorTests
|
|||
if (line.StartsWith("# renderer\t") || line.StartsWith("# identity\t"))
|
||||
header[c[0][2..]] = c[1];
|
||||
else if (!line.StartsWith('#') && line.Length > 0)
|
||||
rows[(c[0], int.Parse(c[1]))] = new BoundRow(c[0], int.Parse(c[1]), c[2], c[3], int.Parse(c[4]), double.Parse(c[5], CultureInfo.InvariantCulture), int.Parse(c[6]));
|
||||
rows[(c[0], int.Parse(c[1]))] = new BoundRow(c[0], int.Parse(c[1]), c[2], c[3], int.Parse(c[4]), double.Parse(c[5], CultureInfo.InvariantCulture), int.Parse(c[6]), c[7]);
|
||||
}
|
||||
|
||||
return (header, rows);
|
||||
|
|
@ -604,4 +626,30 @@ public sealed partial class GlslMirrorTests
|
|||
Assert.Equal(DeviceRoutes.Length, failures.Count);
|
||||
Assert.All(failures, line => Assert.Contains($"#{f} ", line));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AChangeInResultBitsFailsEvenWithinTheRecordedBound()
|
||||
{
|
||||
// Recorded on a device off in every case; judged on one off in a single case. The bound holds, the bits do not.
|
||||
var f = BoundFamily();
|
||||
using var recorded = Simulate((_, item, words) => item.Index == f.ToString() ? Alter(item, words, w => w ^ 4u) : words);
|
||||
Record(recorded);
|
||||
using var fewer = Simulate((_, item, words) => item.Index == f.ToString() ? Alter(item, words, w => w ^ 4u, 1) : words);
|
||||
var failures = Judge(fewer, recorded).Failures;
|
||||
Assert.Equal(DeviceRoutes.Length, failures.Count);
|
||||
Assert.All(failures, line => { Assert.Contains($"#{f} ", line); Assert.Contains("result bits differ", line); });
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AFamilyIsHeldToTheDeviceToleranceItsOwnTextStates()
|
||||
{
|
||||
// Recorded on the very device that violates it: only the stated tolerance can fail the family.
|
||||
var f = Array.FindIndex(Families, fam => StatedDeviceLimit(fam.Tolerance) is not null);
|
||||
Assert.True(f >= 0, "no family states a device tolerance");
|
||||
using var far = Simulate((_, item, words) => item.Index == f.ToString() ? Alter(item, words, w => w ^ 0x00100000u) : words);
|
||||
Record(far);
|
||||
var failures = Judge(far, far).Failures;
|
||||
Assert.Equal(DeviceRoutes.Length, failures.Count);
|
||||
Assert.All(failures, line => Assert.Contains("stated tolerance", line));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,56 @@
|
|||
using System.Text.RegularExpressions;
|
||||
using Xunit;
|
||||
|
||||
namespace CultMath.Tests;
|
||||
|
||||
public sealed partial class GlslMirrorTests
|
||||
{
|
||||
// Tripwire: lavapipe and desktop ANGLE ignore mediump and min16float, so a reduced-precision qualifier in the
|
||||
// committed shaders would change the computation on a phone GPU and nowhere the conforming run can see it.
|
||||
// Stated limits: a syntax guardrail only. It reads whole words in the committed .glsl and .hlsl under shaders/
|
||||
// with comments removed; it does not see a qualifier spelled through a macro, an include or a generated
|
||||
// shader, and it says nothing about a driver that relaxes highp itself. It names file and line, never text.
|
||||
private static readonly Regex ReducedPrecision = new(
|
||||
@"\b(mediump|lowp|min16float|min10float|min16int|min12int|min16uint|half([1-4](x[1-4])?)?)\b|\bprecision\s+(?!highp\b)\w+", RegexOptions.Compiled);
|
||||
|
||||
private static IEnumerable<int> ReducedPrecisionLines(string source)
|
||||
{
|
||||
var text = Regex.Replace(source, @"/\*.*?\*/", m => new string('\n', m.Value.Count(c => c == '\n')), RegexOptions.Singleline);
|
||||
var lines = text.Split('\n');
|
||||
for (var i = 0; i < lines.Length; i++)
|
||||
{
|
||||
var code = lines[i].Split("//")[0];
|
||||
if (ReducedPrecision.IsMatch(code))
|
||||
yield return i + 1;
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CommittedShadersCarryNoReducedPrecisionQualifier()
|
||||
{
|
||||
var found = new List<string>();
|
||||
foreach (var file in Directory.EnumerateFiles(Path.Combine(Root, "shaders")).Where(f => f.EndsWith(".glsl") || f.EndsWith(".hlsl")).OrderBy(f => f, StringComparer.Ordinal))
|
||||
found.AddRange(ReducedPrecisionLines(File.ReadAllText(file)).Select(line => $"{Path.GetFileName(file)}:{line}"));
|
||||
Assert.True(found.Count == 0, "reduced-precision qualifier at " + string.Join(", ", found));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("mediump float x;", true)]
|
||||
[InlineData("lowp vec3 c;", true)]
|
||||
[InlineData("precision mediump float;", true)]
|
||||
[InlineData("precision highp float;", false)]
|
||||
[InlineData("min16float x;", true)]
|
||||
[InlineData("min10float2 x;", true)]
|
||||
[InlineData("min16int i;", true)]
|
||||
[InlineData("half3x3 m;", true)]
|
||||
[InlineData("float half_width;", false)]
|
||||
[InlineData("float x; // mediump is not used", false)]
|
||||
[InlineData("/* half\n min16float */ float x;", false)]
|
||||
public void TheTripwireReadsQualifiersAndNotComments(string source, bool trips) => Assert.Equal(trips, ReducedPrecisionLines(source).Any());
|
||||
|
||||
[Fact]
|
||||
public void TheTripwireNamesTheLine()
|
||||
{
|
||||
Assert.Equal(new[] { 3 }, ReducedPrecisionLines("float a;\n/* c */\nmediump float b;\n").ToArray());
|
||||
}
|
||||
}
|
||||
|
|
@ -684,7 +684,7 @@ static int4 clamp(int4 x, int4 a, int4 b) { if (a.x > b.x || a.y > b.y || a.z >
|
|||
return new object[] { slope, z0, z0 + Uniform(r, 0.01f, 50.0f), Uniform(r, 0.0001f, 0.05f), Uniform(r, 0.0f, 2.0f) };
|
||||
}, FrustumBall, Verify: AssertFrustumBall),
|
||||
new("vec2 cultmath_iv_exp(vec2)", "ulp-bounded, platform exp", r => new object[] { Interval(r) }, Platform: Platform.Ulp),
|
||||
new("CultPhasor cultmath_phacelle(vec3, vec3, float, float)", "ulp-bounded, platform exp, sin, cos; across platforms |diff| <= 2^-20 max(|v|, 1)",
|
||||
new("CultPhasor cultmath_phacelle(vec3, vec3, float, float)", "ulp-bounded, platform exp, sin, cos; across platforms |diff| <= 2^-20 max(|v|, 1); on a device |diff| <= 2^-18 max(|v|, 1)",
|
||||
r => new object[] { Point3(r), Side(r), Uniform(r, 0.0f, 1.0f), 0.5f }, Platform: Platform.Scaled),
|
||||
new("float cultmath_snoise(vec3)", "ulp-bounded, on the simplex tie set", r => new object[] { TiePoint3(r) }),
|
||||
new("vec4 cultmath_snoise_grad(vec3)", "ulp-bounded, on the simplex tie set", r => new object[] { TiePoint3(r) }),
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ lavapipe-glsl 23 vec2 cultmath_iv_snoise_ball(vec3, float) ulp-bounded 242 54 0
|
|||
lavapipe-glsl 24 vec2 cultmath_iv_fbm_ball(vec3, float, int, float, float) ulp-bounded 250 56 0
|
||||
lavapipe-glsl 25 vec4 cultmath_iv_frustum_ball(vec2, float, float, float, float) ulp-bounded 198 1.6457063576593662 0
|
||||
lavapipe-glsl 26 vec2 cultmath_iv_exp(vec2) ulp-bounded, platform exp 7 7.526057804152758 0
|
||||
lavapipe-glsl 27 CultPhasor cultmath_phacelle(vec3, vec3, float, float) ulp-bounded, platform exp, sin, cos; across platforms |diff| <= 2^-20 max(|v|, 1) 0 18.5 0
|
||||
lavapipe-glsl 27 CultPhasor cultmath_phacelle(vec3, vec3, float, float) ulp-bounded, platform exp, sin, cos; across platforms |diff| <= 2^-20 max(|v|, 1); on a device |diff| <= 2^-18 max(|v|, 1) 0 18.5 0
|
||||
lavapipe-glsl 28 float cultmath_snoise(vec3) ulp-bounded, on the simplex tie set 127 112 0
|
||||
lavapipe-glsl 29 vec4 cultmath_snoise_grad(vec3) ulp-bounded, on the simplex tie set 102 546.5 0
|
||||
lavapipe-glsl 30 vec3 cultmath_af_point(float) ulp-bounded 256 0 0
|
||||
|
|
@ -137,7 +137,7 @@ lavapipe-hlsl 23 vec2 cultmath_iv_snoise_ball(vec3, float) ulp-bounded 244 54 0
|
|||
lavapipe-hlsl 24 vec2 cultmath_iv_fbm_ball(vec3, float, int, float, float) ulp-bounded 252 75 0
|
||||
lavapipe-hlsl 25 vec4 cultmath_iv_frustum_ball(vec2, float, float, float, float) ulp-bounded 198 1.6457063576593662 0
|
||||
lavapipe-hlsl 26 vec2 cultmath_iv_exp(vec2) ulp-bounded, platform exp 7 7.526057804152758 0
|
||||
lavapipe-hlsl 27 CultPhasor cultmath_phacelle(vec3, vec3, float, float) ulp-bounded, platform exp, sin, cos; across platforms |diff| <= 2^-20 max(|v|, 1) 0 18.5 0
|
||||
lavapipe-hlsl 27 CultPhasor cultmath_phacelle(vec3, vec3, float, float) ulp-bounded, platform exp, sin, cos; across platforms |diff| <= 2^-20 max(|v|, 1); on a device |diff| <= 2^-18 max(|v|, 1) 0 18.5 0
|
||||
lavapipe-hlsl 28 float cultmath_snoise(vec3) ulp-bounded, on the simplex tie set 144 99 0
|
||||
lavapipe-hlsl 29 vec4 cultmath_snoise_grad(vec3) ulp-bounded, on the simplex tie set 111 546.5 0
|
||||
lavapipe-hlsl 30 vec3 cultmath_af_point(float) ulp-bounded 256 0 0
|
||||
|
|
|
|||
|
|
@ -7082,7 +7082,7 @@
|
|||
},
|
||||
{
|
||||
"glsl": "CultPhasor cultmath_phacelle(vec3, vec3, float, float)",
|
||||
"tolerance": "ulp-bounded, platform exp, sin, cos; across platforms |diff| <= 2^-20 max(|v|, 1)",
|
||||
"tolerance": "ulp-bounded, platform exp, sin, cos; across platforms |diff| <= 2^-20 max(|v|, 1); on a device |diff| <= 2^-18 max(|v|, 1)",
|
||||
"platform": "Ubuntu 24.04.5 LTS; .NET 10.0.12",
|
||||
"cases": [
|
||||
"c01d8630 41b2cd8c 41ea274c bfa19894 40aeb6ce 40918978 3e9be843 3f000000 -> bf74d9fb 3fb71956 bf35fe4e 3f21280a c01277e2 403b5fe4 40464bbd bd5b4455",
|
||||
|
|
|
|||
|
|
@ -57,4 +57,4 @@ if [ "${WRITE_BOUNDS:-0}" = 1 ]; then
|
|||
export CULTMATH_WRITE_DEVICE_BOUNDS=1
|
||||
trap 'echo BOUNDS-BEGIN; cat "$TESTS/fixtures/device-bounds-lavapipe.txt"; echo BOUNDS-END' EXIT
|
||||
fi
|
||||
dotnet test "$TESTS" --no-build --filter "FullyQualifiedName~GlslMirrorTests.DeviceResultsHoldFixture" --logger "console;verbosity=detailed"
|
||||
dotnet test "$TESTS" --no-build --filter "FullyQualifiedName~GlslMirrorTests.DeviceResultsHoldFixture|FullyQualifiedName~GlslMirrorTests.CommittedShadersCarryNoReducedPrecisionQualifier" --logger "console;verbosity=detailed"
|
||||
|
|
|
|||
Loading…
Reference in a new issue