af_range finds a NaN endpoint by its bit pattern, which dxc's fast-math compare cannot fold [af-range-nan-guard-folded-by-dxc]

dxc marks compares fast, so `lo == lo ? lo : -inf` was deleted under default flags and an
overflowed form ranged to NaN on HLSL targets. The guard now tests (asuint(lo) & 0x7fffffff)
> 0x7f800000, in HLSL, the Unity copy, math.Affine.cs and the regenerated CultMath.glsl.
dxc 1.9.0.5402 -T ps_6_0 default flags: before, no fcmp/select/and in the af_range body;
after, and/icmp ugt/select to +-inf for both endpoints (also under -Gis).
AffineRangeMirrorMatchesCSharpBitForBitAtTheFloatExtremes pins the mirror at zero to the
float maximum, infinities and NaN components.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

packages/cultmath/docs/design.md

@@ -743,7 +743,8 @@ add `2^-126`, the most an underflow to a subnormal can lose (`TinyOperandsEnclos
 `af_from_iv` halves its endpoints before it sums and differences them, and `af_range`
 scales its terms before it sums them, so endpoints near the float maximum do not overflow
 in the middle of the op; a form that overflows anyway gives `af_range` infinite endpoints,
-never NaN (`FromIntervalAndRangeHoldAtTheExtremes`). `AbsorptionsArePinned` and
+never NaN (`FromIntervalAndRangeHoldAtTheExtremes`). `af_range` finds a NaN endpoint by its bit pattern, because
+`lo == lo` is a compare dxc marks `fast` and deletes under its default flags. `AbsorptionsArePinned` and
 `SnoiseFormCarriesItsDocumentedWidth` pin each absorption and `af_snoise`'s float32 allowance both
 ways: the enclosure tests alone cannot see them on a CPU, where float32 is exact.
 

packages/cultmath/shaders/CultMath.Affine.hlsl

@@ -22,14 +22,16 @@ float3 cultmath_af_from_iv(float2 interval)
     return float3(x0, 0.0, e + (abs(x0) + e) * 9.5367431640625e-7 + 1.17549435e-38);
 }
 
-// An endpoint that would be NaN is the infinity on its side.
+// An endpoint that would be NaN is the infinity on its side. NaN is found by its bit pattern: dxc marks
+// compares fast, so a test that needs NaN to make `lo == lo` false is deleted under its default flags.
 float2 cultmath_af_range(float3 x)
 {
     float r = abs(x.y) + x.z;
     float w = abs(x.x) * 9.5367431640625e-7 + r * 9.5367431640625e-7;
     float lo = x.x - r - w;
     float hi = x.x + r + w;
-    return float2(lo == lo ? lo : asfloat(0xff800000u), hi == hi ? hi : asfloat(0x7f800000u));
+    return float2((asuint(lo) & 0x7fffffffu) > 0x7f800000u ? asfloat(0xff800000u) : lo,
+        (asuint(hi) & 0x7fffffffu) > 0x7f800000u ? asfloat(0x7f800000u) : hi);
 }
 
 float3 cultmath_af_add(float3 x, float3 y)

packages/cultmath/shaders/CultMath.glsl

@@ -665,14 +665,16 @@ vec3 cultmath_af_from_iv(vec2 interval)
     return vec3(x0, 0.0, e + (abs(x0) + e) * 9.5367431640625e-7 + 1.17549435e-38);
 }
 
-// An endpoint that would be NaN is the infinity on its side.
+// An endpoint that would be NaN is the infinity on its side. NaN is found by its bit pattern: dxc marks
+// compares fast, so a test that needs NaN to make `lo == lo` false is deleted under its default flags.
 vec2 cultmath_af_range(vec3 x)
 {
     float r = abs(x.y) + x.z;
     float w = abs(x.x) * 9.5367431640625e-7 + r * 9.5367431640625e-7;
     float lo = x.x - r - w;
     float hi = x.x + r + w;
-    return vec2(lo == lo ? lo : uintBitsToFloat(0xff800000u), hi == hi ? hi : uintBitsToFloat(0x7f800000u));
+    return vec2((floatBitsToUint(lo) & 0x7fffffffu) > 0x7f800000u ? uintBitsToFloat(0xff800000u) : lo,
+        (floatBitsToUint(hi) & 0x7fffffffu) > 0x7f800000u ? uintBitsToFloat(0x7f800000u) : hi);
 }
 
 vec3 cultmath_af_add(vec3 x, vec3 y)

packages/cultmath/src/CultMath/math.Affine.cs

@@ -66,7 +66,8 @@ public static partial class math
     /// (|x0| + |a| + e) 2^-20, formed term by term so that the sum never passes the float maximum before it
     /// is scaled. A form with an infinite component (an overflowed op) gives infinite
     /// endpoints, and an endpoint that would be NaN (infinity minus infinity, or a NaN component) is
-    /// the infinity on its side, so the result is never NaN and still encloses.
+    /// the infinity on its side, so the result is never NaN and still encloses. NaN is found by its bit
+    /// pattern, not by <c>lo == lo</c>, which a compiler that assumes no NaNs (dxc's default) deletes.
     /// </summary>
     public static float2 af_range(float3 x)
     {
@@ -74,7 +75,8 @@ public static partial class math
         var w = abs(x.x) * 9.5367431640625e-7f + r * 9.5367431640625e-7f;
         var lo = x.x - r - w;
         var hi = x.x + r + w;
-        return new float2(lo == lo ? lo : float.NegativeInfinity, hi == hi ? hi : float.PositiveInfinity);
+        return new float2((asuint(lo) & 0x7fffffffu) > 0x7f800000u ? float.NegativeInfinity : lo,
+            (asuint(hi) & 0x7fffffffu) > 0x7f800000u ? float.PositiveInfinity : hi);
     }
 
     /// <summary>

packages/cultmath/tests/CultMath.Tests/HlslSourceCompatibilityTests.cs

@@ -169,6 +169,36 @@ public sealed class HlslSourceCompatibilityTests
         }
     }
 
+
+    /// <summary>
+    /// The generic comparison stops at 1e7, so an HLSL edit of <c>cultmath_af_range</c>'s NaN test that
+    /// misreads the finite values near the float maximum (or an overflowed endpoint) survives it. Forms
+    /// from zero to the float maximum, infinite and NaN components, both signs, in every slot: the mirror
+    /// returns C#'s bits, and neither endpoint is ever NaN.
+    /// </summary>
+    [Fact]
+    public void AffineRangeMirrorMatchesCSharpBitForBitAtTheFloatExtremes()
+    {
+        var (assembly, errors) = CompileShaderMirror();
+        Assert.True(assembly is not null, string.Join(Environment.NewLine, errors));
+        var shaderType = assembly!.GetType("CultMathHlsl.HlslShader")!;
+        var shader = Activator.CreateInstance(shaderType);
+        var mirror = ShaderFunction(shaderType, "cultmath_af_range", typeof(float3))!;
+
+        var magnitudes = new[] { 0.0f, 1.0f, 1.0e30f, 1.0e38f, float.MaxValue / 2.0f, float.MaxValue, float.PositiveInfinity, float.NaN };
+        var values = magnitudes.Concat(magnitudes.Where(m => !float.IsNaN(m)).Select(m => -m)).ToArray();
+        foreach (var x0 in values)
+        foreach (var a in values)
+        foreach (var e in values)
+        {
+            var form = new float3(x0, a, e);
+            var expected = af_range(form);
+            var actual = Values(mirror.Invoke(shader, new object[] { form })!);
+            Assert.True(BitwiseEqual(Values(expected), actual), $"cultmath_af_range differs from C# at {form}");
+            Assert.False(float.IsNaN(expected.x) || float.IsNaN(expected.y), $"af_range({form}) is {expected}, a NaN endpoint");
+        }
+    }
+
     /// <summary>
     /// The dense sweep almost never meets a cell whose box bound is just under the 2.25 cut and whose
     /// weight is still nonzero: a mirror cut of 2.245, 2.24 or 2.2 survives it. These points sit next

packages/cultmath/unity/org.gamecult.cultmath/Shaders/CultMath.Affine.hlsl

@@ -22,14 +22,16 @@ float3 cultmath_af_from_iv(float2 interval)
     return float3(x0, 0.0, e + (abs(x0) + e) * 9.5367431640625e-7 + 1.17549435e-38);
 }
 
-// An endpoint that would be NaN is the infinity on its side.
+// An endpoint that would be NaN is the infinity on its side. NaN is found by its bit pattern: dxc marks
+// compares fast, so a test that needs NaN to make `lo == lo` false is deleted under its default flags.
 float2 cultmath_af_range(float3 x)
 {
     float r = abs(x.y) + x.z;
     float w = abs(x.x) * 9.5367431640625e-7 + r * 9.5367431640625e-7;
     float lo = x.x - r - w;
     float hi = x.x + r + w;
-    return float2(lo == lo ? lo : asfloat(0xff800000u), hi == hi ? hi : asfloat(0x7f800000u));
+    return float2((asuint(lo) & 0x7fffffffu) > 0x7f800000u ? asfloat(0xff800000u) : lo,
+        (asuint(hi) & 0x7fffffffu) > 0x7f800000u ? asfloat(0x7f800000u) : hi);
 }
 
 float3 cultmath_af_add(float3 x, float3 y)