701 lines
35 KiB
C#
701 lines
35 KiB
C#
using System.Globalization;
|
|
using System.Text.Json.Nodes;
|
|
using CultMath;
|
|
using Xunit;
|
|
using Xunit.Sdk;
|
|
|
|
namespace CultMath.Tests;
|
|
|
|
/// <summary>
|
|
/// The conforming run's C# half. <see cref="WriteWork"/> says what a device must compute: every fixture
|
|
/// family on the fixture's own arguments, plus one reference family for each sampled-form family, whose
|
|
/// points are the ones the form check evaluates. <see cref="JudgeDevice"/> says whether a device's results
|
|
/// hold: exact and integer families bit for bit, check families by their Verify, sampled-form families by
|
|
/// the form check on the device's own reference values, every other family within its recorded bound.
|
|
/// The generator, the Vulkan host and the job script only spell, dispatch and run; they compare nothing.
|
|
/// </summary>
|
|
public sealed partial class GlslMirrorTests
|
|
{
|
|
private static readonly string[] DeviceRoutes = { "lavapipe-hlsl", "lavapipe-glsl" };
|
|
|
|
private static string BoundsPath => Path.Combine(Root, "tests", "CultMath.Tests", "fixtures", "device-bounds-lavapipe.txt");
|
|
|
|
private static string? Env(string name) => Environment.GetEnvironmentVariable(name) is { Length: > 0 } value ? value : null;
|
|
|
|
private static JsonArray FixtureFamilies() => JsonNode.Parse(File.ReadAllText(FixturePath))!["functions"]!.AsArray();
|
|
|
|
private static string[] Sides(JsonNode? entry) => ((string)entry!).Split(" -> ");
|
|
|
|
private static uint[] DevWords(string hex) => hex.Split(' ').Select(h => Convert.ToUInt32(h, 16)).ToArray();
|
|
|
|
private static string DevHex(IEnumerable<uint> words) => string.Join(' ', words.Select(w => w.ToString("x8")));
|
|
|
|
private static uint[] ReadWords(string path)
|
|
{
|
|
var bytes = File.ReadAllBytes(path);
|
|
var words = new uint[bytes.Length / 4];
|
|
for (var i = 0; i < words.Length; i++)
|
|
words[i] = BitConverter.ToUInt32(bytes, i * 4);
|
|
return words;
|
|
}
|
|
|
|
private static void WriteWords(string path, IEnumerable<uint> words)
|
|
{
|
|
using var writer = new BinaryWriter(File.Create(path));
|
|
foreach (var word in words)
|
|
writer.Write(word);
|
|
}
|
|
|
|
private static bool IsSampledForm(Family family) =>
|
|
family.Check is not null && (family.Glsl.Contains("cultmath_af_snoise(") || family.Glsl.Contains("cultmath_af_fbm("));
|
|
|
|
private static string Tag(int family) => $"f{family:D3}";
|
|
|
|
private static string RefTag(int family) => $"r{family:D3}";
|
|
|
|
private sealed record WorkItem(string Tag, string Index, string Glsl, int N, int WordsIn, int WordsOut);
|
|
|
|
// ---- What a device must compute ----
|
|
|
|
private static List<WorkItem> WriteWork(string dir)
|
|
{
|
|
Directory.CreateDirectory(dir);
|
|
var fixture = FixtureFamilies();
|
|
Assert.Equal(Families.Length, fixture.Count);
|
|
var items = new List<WorkItem>();
|
|
for (var f = 0; f < Families.Length; f++)
|
|
{
|
|
Assert.Equal(Families[f].Glsl, (string?)fixture[f]!["glsl"]);
|
|
var cases = fixture[f]!["cases"]!.AsArray();
|
|
WriteWords(Path.Combine(dir, Tag(f) + ".in"), cases.SelectMany(c => DevWords(Sides(c)[0])));
|
|
items.Add(new WorkItem(Tag(f), f.ToString(CultureInfo.InvariantCulture), Families[f].Glsl, cases.Count, DevWords(Sides(cases[0])[0]).Length, DevWords(Sides(cases[0])[1]).Length));
|
|
if (IsSampledForm(Families[f]))
|
|
items.Add(WriteReference(dir, f, cases));
|
|
}
|
|
|
|
File.WriteAllLines(Path.Combine(dir, "manifest.txt"), items.Select(i => string.Join('\t', i.Tag, i.Index, i.Glsl, i.N, i.WordsIn, i.WordsOut)));
|
|
return items;
|
|
}
|
|
|
|
// The points the form check evaluates the reference at, in the order it evaluates them: the reference
|
|
// func records p, and for fBm the case's octaves, lacunarity and gain, then answers as C# does.
|
|
private static WorkItem WriteReference(string dir, int family, JsonArray cases)
|
|
{
|
|
var fbm = Families[family].Glsl.Contains("cultmath_af_fbm(");
|
|
var recorded = new List<uint>();
|
|
Func<string[], float3, float> recorder = (t, p) =>
|
|
{
|
|
recorded.AddRange(new[] { BitConverter.SingleToUInt32Bits(p.x), BitConverter.SingleToUInt32Bits(p.y), BitConverter.SingleToUInt32Bits(p.z) });
|
|
if (fbm)
|
|
recorded.AddRange(t[7..10].Select(h => Convert.ToUInt32(h, 16)));
|
|
return fbm ? FbmReference(t, p) : SnoiseReference(t, p);
|
|
};
|
|
if (fbm)
|
|
AssertFbmForm(cases, false, recorder);
|
|
else
|
|
AssertSnoiseForm(cases, false, recorder);
|
|
|
|
var wordsIn = fbm ? 6 : 3;
|
|
WriteWords(Path.Combine(dir, RefTag(family) + ".in"), recorded);
|
|
return new WorkItem(RefTag(family), "ref:" + family.ToString(CultureInfo.InvariantCulture),
|
|
fbm ? "vec4 cultmath_fbm_grad(vec3, int, float, float)" : "float cultmath_snoise(vec3)", recorded.Count / wordsIn, wordsIn, fbm ? 4 : 1);
|
|
}
|
|
|
|
[Fact]
|
|
public void WriteDeviceWork()
|
|
{
|
|
var dir = Env("CULTMATH_DEVICE_WORK");
|
|
Assert.SkipUnless(dir is not null, "CULTMATH_DEVICE_WORK=<dir> writes the work list a device runs");
|
|
var items = WriteWork(dir!);
|
|
output.WriteLine($"{items.Count} work items ({items.Count(i => i.Index.StartsWith("ref:"))} reference families) in {dir}");
|
|
}
|
|
|
|
// ---- Whether a device's results hold ----
|
|
|
|
private sealed record BoundRow(string Route, int Index, string Glsl, string Tolerance, int Exact, double Worst, int NanClass, string Bits);
|
|
|
|
private sealed record Measured(BoundRow Row, string Class);
|
|
|
|
private sealed record Judgement(List<string> Failures, List<Measured> Rows);
|
|
|
|
// Which words of a family's result are floats; the rest are integers and must match bit for bit.
|
|
private static IEnumerable<bool> FloatWords(Type type) =>
|
|
type == typeof(float) ? new[] { true }
|
|
: type == typeof(int) || type == typeof(uint) ? new[] { false }
|
|
: type == typeof(float3x3) ? Enumerable.Repeat(true, 9)
|
|
: type.GetFields().Where(f => !f.IsStatic).SelectMany(f => FloatWords(f.FieldType));
|
|
|
|
// u1 = |device - fixture| / max(|fixture|, 1) / 2^-23 per float component, worst over the family; NaN and
|
|
// Inf are compared as classes (NaN against not-NaN, an Inf against anything else), never by value.
|
|
private static (int Exact, double Worst, int NanClass) Measure(JsonArray cases, bool[] floatWord, uint[] got)
|
|
{
|
|
int exact = 0, nanClass = 0;
|
|
var worst = 0.0;
|
|
var wordsOut = floatWord.Length;
|
|
for (var c = 0; c < cases.Count; c++)
|
|
{
|
|
var want = DevWords(Sides(cases[c])[1]);
|
|
var same = true;
|
|
for (var w = 0; w < wordsOut; w++)
|
|
{
|
|
var have = got[c * wordsOut + w];
|
|
if (have == want[w])
|
|
continue;
|
|
same = false;
|
|
if (!floatWord[w])
|
|
{
|
|
nanClass++;
|
|
continue;
|
|
}
|
|
|
|
var e = BitConverter.UInt32BitsToSingle(want[w]);
|
|
var h = BitConverter.UInt32BitsToSingle(have);
|
|
if (float.IsNaN(e) || float.IsNaN(h))
|
|
nanClass += float.IsNaN(e) == float.IsNaN(h) ? 0 : 1;
|
|
else if (float.IsInfinity(e) || float.IsInfinity(h))
|
|
nanClass += e == h ? 0 : 1;
|
|
else
|
|
worst = Math.Max(worst, Math.Abs((double)e - h) / Math.Max(Math.Abs((double)e), 1.0) / Math.ScaleB(1.0, -23));
|
|
}
|
|
|
|
if (same)
|
|
exact++;
|
|
}
|
|
|
|
return (exact, worst, nanClass);
|
|
}
|
|
|
|
// The device's own reference values, in the order WriteReference recorded their points. The check asks
|
|
// for the reference at a point; the point it asks for must be the one the device was given at that place.
|
|
private static Func<string[], float3, float> DeviceReference(string work, string routeResults, WorkItem reference)
|
|
{
|
|
var points = ReadWords(Path.Combine(work, reference.Tag + ".in"));
|
|
var values = ReadWords(Path.Combine(routeResults, reference.Tag + ".out"));
|
|
Assert.Equal(reference.N * reference.WordsOut, values.Length);
|
|
var next = 0;
|
|
return (_, p) =>
|
|
{
|
|
Assert.True(next < reference.N, "reference point asked for past the recorded work");
|
|
var at = next * reference.WordsIn;
|
|
Assert.True(points[at] == BitConverter.SingleToUInt32Bits(p.x) && points[at + 1] == BitConverter.SingleToUInt32Bits(p.y) && points[at + 2] == BitConverter.SingleToUInt32Bits(p.z),
|
|
$"reference point {next} is not the point the device was given");
|
|
return BitConverter.UInt32BitsToSingle(values[next++ * reference.WordsOut + reference.WordsOut - 1]);
|
|
};
|
|
}
|
|
|
|
// sha256 of the family's result bytes, and of its reference result when it is judged on one.
|
|
private static string ResultBits(string path, string? reference)
|
|
{
|
|
var bytes = File.ReadAllBytes(path);
|
|
if (reference is not null)
|
|
bytes = bytes.Concat(File.ReadAllBytes(reference)).ToArray();
|
|
return Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(bytes)).ToLowerInvariant();
|
|
}
|
|
|
|
// A tolerance text may state what a device is allowed: "on a device |diff| <= 2^-N max(|v|, 1)", which in
|
|
// u1 units (2^-23) is 2^(23-N). The judge reads it here and nowhere else.
|
|
private static double? StatedDeviceLimit(string tolerance)
|
|
{
|
|
var m = System.Text.RegularExpressions.Regex.Match(tolerance, @"on a device \|diff\| <= 2\^-(\d+) max\(\|v\|, 1\)");
|
|
return m.Success ? Math.ScaleB(1.0, 23 - int.Parse(m.Groups[1].Value)) : null;
|
|
}
|
|
|
|
private static string FirstLine(Exception e)
|
|
{
|
|
var line = e.Message.Split('\n')[0];
|
|
return line.Length > 300 ? line[..300] : line;
|
|
}
|
|
|
|
private static Judgement JudgeDevice(string work, string results, string renderer, string identity, string boundsPath, bool write)
|
|
{
|
|
var failures = new List<string>();
|
|
var measured = new List<Measured>();
|
|
Dictionary<(string, int), BoundRow>? recorded = null;
|
|
if (!write)
|
|
{
|
|
if (!File.Exists(boundsPath))
|
|
return new Judgement(new List<string> { "no recorded device bounds; record them with CULTMATH_WRITE_DEVICE_BOUNDS=1" }, measured);
|
|
var (header, rows) = ReadBounds(boundsPath);
|
|
if (header.GetValueOrDefault("renderer") != renderer || header.GetValueOrDefault("identity") != identity)
|
|
return new Judgement(new List<string> { "the device or its toolchain is not the one the bounds were recorded on; re-record them with CULTMATH_WRITE_DEVICE_BOUNDS=1" }, measured);
|
|
recorded = rows;
|
|
}
|
|
|
|
var fixture = FixtureFamilies();
|
|
var items = File.ReadAllLines(Path.Combine(work, "manifest.txt")).Select(l => l.Split('\t'))
|
|
.ToDictionary(c => c[1], c => new WorkItem(c[0], c[1], c[2], int.Parse(c[3]), int.Parse(c[4]), int.Parse(c[5])));
|
|
foreach (var route in DeviceRoutes)
|
|
for (var f = 0; f < Families.Length; f++)
|
|
{
|
|
var family = Families[f];
|
|
var cases = fixture[f]!["cases"]!.AsArray();
|
|
var name = $"{route} #{f} {family.Glsl}";
|
|
var path = Path.Combine(results, route, Tag(f) + ".out");
|
|
if (!File.Exists(path))
|
|
{
|
|
failures.Add($"{name}: missing {Tag(f)}.out");
|
|
continue;
|
|
}
|
|
|
|
var floatWord = FloatWords(Resolve(family.Glsl).ReturnType).ToArray();
|
|
var got = ReadWords(path);
|
|
if (got.Length != cases.Count * floatWord.Length)
|
|
{
|
|
failures.Add($"{name}: {Tag(f)}.out holds {got.Length} words, not {cases.Count * floatWord.Length}");
|
|
continue;
|
|
}
|
|
|
|
var (exact, worst, nanClass) = Measure(cases, floatWord, got);
|
|
var bits = ResultBits(path, IsSampledForm(family) ? Path.Combine(results, route, RefTag(f) + ".out") : null);
|
|
var exactClass = family.Tolerance == "exact" || !floatWord.Any(w => w);
|
|
var kind = exactClass ? "exact" : IsSampledForm(family) ? "sampled" : family.Verify is not null ? "check" : "bound";
|
|
measured.Add(new Measured(new BoundRow(route, f, family.Glsl, family.Tolerance, exact, worst, nanClass, bits), kind));
|
|
try
|
|
{
|
|
if (exactClass)
|
|
Assert.True(exact == cases.Count, $"{exact} of {cases.Count} cases are bit-identical to the fixture");
|
|
else if (kind == "sampled")
|
|
{
|
|
var reference = DeviceReference(work, Path.Combine(results, route), items["ref:" + f]);
|
|
if (family.Glsl.Contains("cultmath_af_fbm("))
|
|
AssertFbmForm(cases, false, reference);
|
|
else
|
|
AssertSnoiseForm(cases, false, reference);
|
|
}
|
|
else if (kind == "check")
|
|
family.Verify!(new JsonArray(cases.Select((c, i) => (JsonNode)JsonValue.Create(Sides(c)[0] + " -> " + DevHex(got.Skip(i * floatWord.Length).Take(floatWord.Length)))!).ToArray()));
|
|
}
|
|
catch (XunitException e)
|
|
{
|
|
failures.Add($"{name}: {FirstLine(e)}");
|
|
}
|
|
}
|
|
|
|
if (write)
|
|
{
|
|
WriteBounds(boundsPath, renderer, identity, measured.Select(m => m.Row));
|
|
recorded = measured.ToDictionary(m => (m.Row.Route, m.Row.Index), m => m.Row);
|
|
}
|
|
|
|
foreach (var m in measured)
|
|
{
|
|
var name = $"{m.Row.Route} #{m.Row.Index} {m.Row.Glsl}";
|
|
if (!recorded!.TryGetValue((m.Row.Route, m.Row.Index), out var bound))
|
|
failures.Add($"{name}: no recorded row; re-record with CULTMATH_WRITE_DEVICE_BOUNDS=1");
|
|
else if (bound.Glsl != m.Row.Glsl || bound.Tolerance != m.Row.Tolerance)
|
|
failures.Add($"{name}: the recorded row is for another family; re-record with CULTMATH_WRITE_DEVICE_BOUNDS=1");
|
|
else if (m.Class == "bound" && (m.Row.Worst > bound.Worst || m.Row.NanClass > bound.NanClass))
|
|
failures.Add($"{name}: worst u1 {m.Row.Worst:R} and {m.Row.NanClass} NaN-class mismatches, past the recorded {bound.Worst:R} and {bound.NanClass}");
|
|
else if (m.Class == "bound" && StatedDeviceLimit(m.Row.Tolerance) is { } limit && m.Row.Worst > limit)
|
|
failures.Add($"{name}: worst u1 {m.Row.Worst:R}, past the {limit:R} its stated tolerance allows a device");
|
|
else if (m.Row.Bits != bound.Bits && !failures.Any(x => x.StartsWith(name + ":")))
|
|
failures.Add($"{name}: the result bits differ from the recorded run's; if the change is meant, re-record with CULTMATH_WRITE_DEVICE_BOUNDS=1");
|
|
}
|
|
|
|
return new Judgement(failures, measured);
|
|
}
|
|
|
|
// ---- The recorded bounds: plain text, tab-separated, one row per route and family ----
|
|
|
|
private static void WriteBounds(string path, string renderer, string identity, IEnumerable<BoundRow> rows)
|
|
{
|
|
var lines = new List<string>
|
|
{
|
|
"# renderer\t" + renderer,
|
|
"# identity\t" + identity,
|
|
"# route\tindex\tglsl\ttolerance\texact cases\tworst u1\tNaN-class mismatches result sha256; written only by CULTMATH_WRITE_DEVICE_BOUNDS=1",
|
|
};
|
|
lines.AddRange(rows.OrderBy(r => r.Route, StringComparer.Ordinal).ThenBy(r => r.Index)
|
|
.Select(r => string.Join('\t', r.Route, r.Index, r.Glsl, r.Tolerance, r.Exact, r.Worst.ToString("R", CultureInfo.InvariantCulture), r.NanClass, r.Bits)));
|
|
File.WriteAllText(path, string.Join('\n', lines) + "\n");
|
|
}
|
|
|
|
private static (Dictionary<string, string> Header, Dictionary<(string, int), BoundRow> Rows) ReadBounds(string path)
|
|
{
|
|
var header = new Dictionary<string, string>();
|
|
var rows = new Dictionary<(string, int), BoundRow>();
|
|
foreach (var line in File.ReadAllLines(path))
|
|
{
|
|
var c = line.Split('\t');
|
|
if (line.StartsWith("# renderer\t") || line.StartsWith("# identity\t"))
|
|
header[c[0][2..]] = c[1];
|
|
else if (!line.StartsWith('#') && line.Length > 0)
|
|
rows[(c[0], int.Parse(c[1]))] = new BoundRow(c[0], int.Parse(c[1]), c[2], c[3], int.Parse(c[4]), double.Parse(c[5], CultureInfo.InvariantCulture), int.Parse(c[6]), c[7]);
|
|
}
|
|
|
|
return (header, rows);
|
|
}
|
|
|
|
[Fact]
|
|
public void DeviceResultsHoldFixture()
|
|
{
|
|
var results = Env("CULTMATH_DEVICE_RESULTS");
|
|
Assert.SkipUnless(results is not null, "CULTMATH_DEVICE_RESULTS=<dir> (holding <route>/<tag>.out) judges a device's results");
|
|
var work = Env("CULTMATH_DEVICE_WORK");
|
|
var renderer = Env("CULTMATH_DEVICE_NAME");
|
|
var identity = Env("CULTMATH_DEVICE_IDENTITY");
|
|
Assert.True(work is not null && renderer is not null && identity is not null, "CULTMATH_DEVICE_WORK, CULTMATH_DEVICE_NAME and CULTMATH_DEVICE_IDENTITY are required beside CULTMATH_DEVICE_RESULTS");
|
|
var judgement = JudgeDevice(work!, results!, renderer!, identity!, BoundsPath, Env("CULTMATH_WRITE_DEVICE_BOUNDS") == "1");
|
|
foreach (var route in DeviceRoutes)
|
|
{
|
|
var rows = judgement.Rows.Where(m => m.Row.Route == route).ToList();
|
|
output.WriteLine($"{route}: {rows.Count} families judged: " + string.Join(", ", rows.GroupBy(m => m.Class).OrderBy(g => g.Key).Select(g => $"{g.Count()} {g.Key}")));
|
|
foreach (var m in rows.OrderByDescending(m => m.Row.Worst).Take(5))
|
|
output.WriteLine($" worst u1 {m.Row.Worst:R}, {m.Row.NanClass} NaN-class: #{m.Row.Index} {m.Row.Glsl} ({m.Class})");
|
|
}
|
|
|
|
foreach (var failure in judgement.Failures.Take(60))
|
|
output.WriteLine("FAIL " + failure);
|
|
Assert.True(judgement.Failures.Count == 0, $"{judgement.Failures.Count} device results do not hold the fixture; first: {judgement.Failures.FirstOrDefault()}");
|
|
}
|
|
|
|
// ---- The judge against simulated devices ----
|
|
// A faithful device answers the fixture's own result bits and, for a reference family, C#'s snoise or
|
|
// fBm at the recorded points. A test alters what one family or one route answers.
|
|
|
|
private sealed record Simulated(string Root, string Work, string Results, string Bounds) : IDisposable
|
|
{
|
|
public void Dispose() => Directory.Delete(Root, true);
|
|
}
|
|
|
|
private static Simulated Simulate(Func<string, WorkItem, uint[], uint[]>? alter = null)
|
|
{
|
|
var root = Path.Combine(Path.GetTempPath(), "cultmath-device-" + Guid.NewGuid().ToString("N"));
|
|
var work = Path.Combine(root, "work");
|
|
var items = WriteWork(work);
|
|
var fixture = FixtureFamilies();
|
|
foreach (var route in DeviceRoutes)
|
|
{
|
|
Directory.CreateDirectory(Path.Combine(root, "results", route));
|
|
foreach (var item in items)
|
|
{
|
|
uint[] words;
|
|
if (item.Index.StartsWith("ref:"))
|
|
{
|
|
var points = ReadWords(Path.Combine(work, item.Tag + ".in"));
|
|
words = new uint[item.N * item.WordsOut];
|
|
for (var k = 0; k < item.N; k++)
|
|
{
|
|
var at = k * item.WordsIn;
|
|
var p = new float3(BitConverter.UInt32BitsToSingle(points[at]), BitConverter.UInt32BitsToSingle(points[at + 1]), BitConverter.UInt32BitsToSingle(points[at + 2]));
|
|
var tokens = new string[10];
|
|
if (item.WordsIn == 6)
|
|
for (var t = 0; t < 3; t++)
|
|
tokens[7 + t] = points[at + 3 + t].ToString("x8");
|
|
words[k * item.WordsOut + item.WordsOut - 1] = BitConverter.SingleToUInt32Bits(item.WordsIn == 6 ? FbmReference(tokens, p) : SnoiseReference(tokens, p));
|
|
}
|
|
}
|
|
else
|
|
words = fixture[int.Parse(item.Index)]!["cases"]!.AsArray().SelectMany(c => DevWords(Sides(c)[1])).ToArray();
|
|
|
|
WriteWords(Path.Combine(root, "results", route, item.Tag + ".out"), alter?.Invoke(route, item, words) ?? words);
|
|
}
|
|
}
|
|
|
|
return new Simulated(root, work, Path.Combine(root, "results"), Path.Combine(root, "bounds.txt"));
|
|
}
|
|
|
|
private static Judgement Record(Simulated device) => JudgeDevice(device.Work, device.Results, "test renderer", "test identity", device.Bounds, true);
|
|
|
|
private static Judgement Judge(Simulated device, Simulated recordedOn, string renderer = "test renderer", string identity = "test identity") =>
|
|
JudgeDevice(device.Work, device.Results, renderer, identity, recordedOn.Bounds, false);
|
|
|
|
// Every case's first result word, altered.
|
|
private static uint[] Alter(WorkItem item, uint[] words, Func<uint, uint> change, int cases = int.MaxValue)
|
|
{
|
|
var altered = (uint[])words.Clone();
|
|
for (var c = 0; c < Math.Min(item.N, cases); c++)
|
|
altered[c * item.WordsOut] = change(altered[c * item.WordsOut]);
|
|
return altered;
|
|
}
|
|
|
|
// A family judged by its recorded bound alone: not exact, not checked, not a sampled form, a float first word.
|
|
private static int BoundFamily() =>
|
|
Array.FindIndex(Families, f => f.Tolerance == "ulp-bounded" && f.Verify is null && FloatWords(Resolve(f.Glsl).ReturnType).First());
|
|
|
|
[Fact]
|
|
public void AFaithfulDeviceHoldsEveryFamilyOnBothRoutes()
|
|
{
|
|
using var device = Simulate();
|
|
var recorded = Record(device);
|
|
Assert.Empty(recorded.Failures);
|
|
Assert.Equal(DeviceRoutes.Length * Families.Length, recorded.Rows.Count);
|
|
Assert.All(recorded.Rows, m => Assert.Equal(0.0, m.Row.Worst));
|
|
Assert.Equal(DeviceRoutes.Length * Families.Count(IsSampledForm), recorded.Rows.Count(m => m.Class == "sampled"));
|
|
Assert.Equal(DeviceRoutes.Length * Families.Count(f => f.Tolerance == "exact"), recorded.Rows.Count(m => m.Class == "exact"));
|
|
Assert.Empty(Judge(device, device).Failures);
|
|
}
|
|
|
|
[Fact]
|
|
public void AnExactFamilyAllowsNoDifferenceAtAll()
|
|
{
|
|
// Bounds recorded on the very device that is off by one ulp: a bound would pass it, the exact rule must not.
|
|
var exact = Array.FindIndex(Families, f => f.Tolerance == "exact");
|
|
using var off = Simulate((route, item, words) => route == "lavapipe-glsl" && item.Index == exact.ToString() ? Alter(item, words, w => w ^ 1u) : words);
|
|
Record(off);
|
|
var failures = Judge(off, off).Failures;
|
|
Assert.Single(failures);
|
|
Assert.Contains($"lavapipe-glsl #{exact} ", failures[0]);
|
|
|
|
// One case in 256 is enough.
|
|
using var oneCase = Simulate((_, item, words) => item.Index == exact.ToString() ? Alter(item, words, w => w ^ 1u, 1) : words);
|
|
Record(oneCase);
|
|
Assert.Equal(DeviceRoutes.Length, Judge(oneCase, oneCase).Failures.Count);
|
|
}
|
|
|
|
[Fact]
|
|
public void AFamilyIsHeldToItsRecordedWorstAndNotBeyond()
|
|
{
|
|
var f = BoundFamily();
|
|
using var near = Simulate((_, item, words) => item.Index == f.ToString() ? Alter(item, words, w => w ^ 4u) : words);
|
|
Record(near);
|
|
Assert.Empty(Judge(near, near).Failures);
|
|
using var farther = Simulate((_, item, words) => item.Index == f.ToString() ? Alter(item, words, w => w ^ 7u) : words);
|
|
var failures = Judge(farther, near).Failures;
|
|
Assert.Equal(DeviceRoutes.Length, failures.Count);
|
|
Assert.All(failures, line => Assert.Contains($"#{f} ", line));
|
|
|
|
var lines = File.ReadAllLines(near.Bounds).ToList();
|
|
var at = lines.FindIndex(l => l.StartsWith($"lavapipe-hlsl\t{f}\t"));
|
|
var row = lines[at].Split('\t');
|
|
Assert.NotEqual(0.0, double.Parse(row[5], CultureInfo.InvariantCulture));
|
|
row[5] = "0";
|
|
lines[at] = string.Join('\t', row);
|
|
File.WriteAllLines(near.Bounds, lines);
|
|
Assert.Single(Judge(near, near).Failures);
|
|
|
|
lines.RemoveAt(at);
|
|
File.WriteAllLines(near.Bounds, lines);
|
|
var missing = Judge(near, near).Failures;
|
|
Assert.Single(missing);
|
|
Assert.Contains("no recorded row", missing[0]);
|
|
}
|
|
|
|
[Fact]
|
|
public void ABoundMeasuresTheErrorAgainstTheFixturesMagnitude()
|
|
{
|
|
// The metric divides by max(|fixture|, 1): a result of twice a fixture value of at least 2 is 2^23 away,
|
|
// and of twice a fixture value under 1 is |fixture| 2^23 away.
|
|
var fixture = FixtureFamilies();
|
|
float Fixture(int family, int c) => BitConverter.UInt32BitsToSingle(DevWords(Sides(fixture[family]!["cases"]![c])[1])[0]);
|
|
int Find(int family, Func<float, bool> wanted) => Enumerable.Range(0, Points).FirstOrDefault(c => wanted(Math.Abs(Fixture(family, c))), -1);
|
|
bool Large(float v) => v is >= 2.0f and < 1e30f;
|
|
bool Small(float v) => v is > 1e-3f and < 1.0f;
|
|
var f = Enumerable.Range(0, Families.Length).First(g => Families[g].Tolerance == "ulp-bounded" && Families[g].Verify is null
|
|
&& FloatWords(Resolve(Families[g].Glsl).ReturnType).First() && Find(g, Large) >= 0 && Find(g, Small) >= 0);
|
|
var large = Find(f, Large);
|
|
var small = Find(f, Small);
|
|
foreach (var (c, expected) in new[] { (large, Math.ScaleB(1.0, 23)), (small, Math.Abs((double)Fixture(f, small)) * Math.ScaleB(1.0, 23)) })
|
|
{
|
|
using var device = Simulate((_, item, words) =>
|
|
{
|
|
if (item.Index != f.ToString())
|
|
return words;
|
|
var doubled = (uint[])words.Clone();
|
|
doubled[c * item.WordsOut] = BitConverter.SingleToUInt32Bits(2.0f * BitConverter.UInt32BitsToSingle(words[c * item.WordsOut]));
|
|
return doubled;
|
|
});
|
|
Assert.Equal(expected, Record(device).Rows.First(m => m.Row.Route == "lavapipe-hlsl" && m.Row.Index == f).Row.Worst);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void ANaNWhereTheFixtureHasANumberFailsAFamilyRecordedWithout()
|
|
{
|
|
var f = BoundFamily();
|
|
using var faithful = Simulate();
|
|
Record(faithful);
|
|
using var nan = Simulate((_, item, words) => item.Index == f.ToString() ? Alter(item, words, _ => 0x7fc00000u) : words);
|
|
var failures = Judge(nan, faithful).Failures;
|
|
Assert.Equal(DeviceRoutes.Length, failures.Count);
|
|
Assert.All(failures, line => Assert.Contains("NaN-class", line));
|
|
}
|
|
|
|
[Fact]
|
|
public void AMissingOutputFailsOnlyItsFamily()
|
|
{
|
|
using var device = Simulate();
|
|
Record(device);
|
|
File.Delete(Path.Combine(device.Results, "lavapipe-hlsl", Tag(3) + ".out"));
|
|
var failures = Judge(device, device).Failures;
|
|
Assert.Single(failures);
|
|
Assert.Contains("missing", failures[0]);
|
|
Assert.Contains("lavapipe-hlsl #3 ", failures[0]);
|
|
}
|
|
|
|
[Fact]
|
|
public void ADeviceOrToolchainOtherThanTheRecordedOneIsRefusedBeforeJudging()
|
|
{
|
|
using var device = Simulate();
|
|
Record(device);
|
|
foreach (var (renderer, identity) in new[] { ("another renderer", "test identity"), ("test renderer", "another toolchain") })
|
|
{
|
|
var judgement = Judge(device, device, renderer, identity);
|
|
Assert.Single(judgement.Failures);
|
|
Assert.Empty(judgement.Rows);
|
|
}
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData("cultmath_af_snoise(")]
|
|
[InlineData("cultmath_af_fbm(")]
|
|
public void ASampledFormIsJudgedOnTheDevicesOwnReferenceAtTheRecordedPoints(string function)
|
|
{
|
|
var sampled = Array.FindIndex(Families, f => IsSampledForm(f) && f.Glsl.Contains(function));
|
|
using var faithful = Simulate();
|
|
Record(faithful);
|
|
|
|
// A reference that answers wrongly puts the form's enclosure out of reach.
|
|
using var wrong = Simulate((_, item, words) => item.Index == "ref:" + sampled
|
|
? words.Select(w => BitConverter.SingleToUInt32Bits(BitConverter.UInt32BitsToSingle(w) + 100.0f)).ToArray() : words);
|
|
var failures = Judge(wrong, faithful).Failures;
|
|
Assert.Equal(DeviceRoutes.Length, failures.Count);
|
|
Assert.All(failures, line => Assert.Contains($"#{sampled} ", line));
|
|
|
|
// Two points in the other order: the device's i-th answer is not for the point the check asks about.
|
|
using var swapped = Simulate();
|
|
var path = Path.Combine(swapped.Work, RefTag(sampled) + ".in");
|
|
var stride = function.Contains("fbm") ? 6 : 3;
|
|
var points = ReadWords(path);
|
|
(points[0], points[stride]) = (points[stride], points[0]);
|
|
WriteWords(path, points);
|
|
var order = Judge(swapped, faithful).Failures;
|
|
Assert.Equal(DeviceRoutes.Length, order.Count);
|
|
Assert.All(order, line => Assert.Contains("not the point the device was given", line));
|
|
}
|
|
|
|
[Fact]
|
|
public void AnInfinityWhereTheFixtureHasAnotherValueIsAMismatchOfClass()
|
|
{
|
|
var fixture = FixtureFamilies();
|
|
int f = -1, c = 0, w = 0;
|
|
for (var g = 0; g < Families.Length && f < 0; g++)
|
|
{
|
|
if (Families[g].Tolerance == "exact" || Families[g].Verify is not null)
|
|
continue;
|
|
var floatWord = FloatWords(Resolve(Families[g].Glsl).ReturnType).ToArray();
|
|
for (var k = 0; k < Points && f < 0; k++)
|
|
{
|
|
var want = DevWords(Sides(fixture[g]!["cases"]![k])[1]);
|
|
for (var at = 0; at < want.Length && f < 0; at++)
|
|
if (floatWord[at] && float.IsInfinity(BitConverter.UInt32BitsToSingle(want[at])))
|
|
(f, c, w) = (g, k, at);
|
|
}
|
|
}
|
|
|
|
Assert.True(f >= 0, "no fixture family holds an infinite result");
|
|
using var faithful = Simulate();
|
|
Record(faithful);
|
|
using var finite = Simulate((_, item, words) =>
|
|
{
|
|
if (item.Index != f.ToString())
|
|
return words;
|
|
var altered = (uint[])words.Clone();
|
|
altered[c * item.WordsOut + w] = BitConverter.SingleToUInt32Bits(1.0f);
|
|
return altered;
|
|
});
|
|
var failures = Judge(finite, faithful).Failures;
|
|
Assert.Equal(DeviceRoutes.Length, failures.Count);
|
|
Assert.All(failures, line => Assert.Contains("NaN-class", line));
|
|
}
|
|
|
|
[Fact]
|
|
public void ARecordedRowForAnotherFamilyIsRefused()
|
|
{
|
|
using var device = Simulate();
|
|
Record(device);
|
|
var lines = File.ReadAllLines(device.Bounds).ToList();
|
|
var at = lines.FindIndex(l => l.StartsWith("lavapipe-glsl 5 "));
|
|
var row = lines[at].Split(' ');
|
|
row[2] = "float cultmath_other(float)";
|
|
lines[at] = string.Join(' ', row);
|
|
File.WriteAllLines(device.Bounds, lines);
|
|
var failures = Judge(device, device).Failures;
|
|
Assert.Single(failures);
|
|
Assert.Contains("another family", failures[0]);
|
|
}
|
|
|
|
[Fact]
|
|
public void ACheckFamilyIsJudgedByItsVerify()
|
|
{
|
|
var f = Array.FindIndex(Families, fam => fam.Verify is not null && !IsSampledForm(fam));
|
|
using var faithful = Simulate();
|
|
Record(faithful);
|
|
using var broken = Simulate((_, item, words) => item.Index == f.ToString() ? words.Select(_ => 0x7fc00000u).ToArray() : words);
|
|
var failures = Judge(broken, faithful).Failures;
|
|
Assert.Equal(DeviceRoutes.Length, failures.Count);
|
|
Assert.All(failures, line => Assert.Contains($"#{f} ", line));
|
|
}
|
|
|
|
[Fact]
|
|
public void AChangeInResultBitsFailsEvenWithinTheRecordedBound()
|
|
{
|
|
// Recorded on a device off in every case; judged on one off in a single case. The bound holds, the bits do not.
|
|
var f = BoundFamily();
|
|
using var recorded = Simulate((_, item, words) => item.Index == f.ToString() ? Alter(item, words, w => w ^ 4u) : words);
|
|
Record(recorded);
|
|
using var fewer = Simulate((_, item, words) => item.Index == f.ToString() ? Alter(item, words, w => w ^ 4u, 1) : words);
|
|
var failures = Judge(fewer, recorded).Failures;
|
|
Assert.Equal(DeviceRoutes.Length, failures.Count);
|
|
Assert.All(failures, line => { Assert.Contains($"#{f} ", line); Assert.Contains("result bits differ", line); });
|
|
}
|
|
|
|
[Fact]
|
|
public void AFamilyIsHeldToTheDeviceToleranceItsOwnTextStates()
|
|
{
|
|
// Recorded on the very device that violates it: only the stated tolerance can fail the family.
|
|
var f = Array.FindIndex(Families, fam => StatedDeviceLimit(fam.Tolerance) is not null);
|
|
Assert.True(f >= 0, "no family states a device tolerance");
|
|
using var far = Simulate((_, item, words) => item.Index == f.ToString() ? Alter(item, words, w => w ^ 0x00100000u) : words);
|
|
Record(far);
|
|
var failures = Judge(far, far).Failures;
|
|
Assert.Equal(DeviceRoutes.Length, failures.Count);
|
|
Assert.All(failures, line => Assert.Contains("stated tolerance", line));
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData(24, true)]
|
|
[InlineData(40, false)]
|
|
public void TheStatedDeviceToleranceIsTheLimitInUlpsOfTheFixturesMagnitude(int ulps, bool holds)
|
|
{
|
|
// 2^-18 max(|v|, 1) is 32 u1. A fixture value in [1, 1.1) moved by k ulps is k/|v| u1 away, 0.91 k to k.
|
|
var f = Array.FindIndex(Families, fam => StatedDeviceLimit(fam.Tolerance) is not null);
|
|
Assert.Equal(32.0, StatedDeviceLimit(Families[f].Tolerance));
|
|
var fixture = FixtureFamilies()[f]!["cases"]!.AsArray();
|
|
var floatWord = FloatWords(Resolve(Families[f].Glsl).ReturnType).ToArray();
|
|
var at = -1;
|
|
for (var c = 0; c < fixture.Count && at < 0; c++)
|
|
{
|
|
var want = DevWords(Sides(fixture[c])[1]);
|
|
for (var w = 0; w < want.Length && at < 0; w++)
|
|
if (floatWord[w] && Math.Abs(BitConverter.UInt32BitsToSingle(want[w])) is >= 1.0f and < 1.1f)
|
|
at = c * want.Length + w;
|
|
}
|
|
|
|
Assert.True(at >= 0, "no fixture value of the family lies in [1, 1.1)");
|
|
using var device = Simulate((_, item, words) =>
|
|
{
|
|
if (item.Index != f.ToString())
|
|
return words;
|
|
var moved = (uint[])words.Clone();
|
|
moved[at] += (uint)ulps; // magnitude grows by k ulps whatever the sign
|
|
return moved;
|
|
});
|
|
Record(device);
|
|
var failures = Judge(device, device).Failures;
|
|
Assert.Equal(holds ? 0 : DeviceRoutes.Length, failures.Count);
|
|
}
|
|
|
|
[Fact]
|
|
public void AChangeInTheReferenceBitsOfASampledFormFailsItsFamily()
|
|
{
|
|
// One ulp in one reference value stays inside the form's enclosure, so only the recorded bits can see it.
|
|
var sampled = Array.FindIndex(Families, IsSampledForm);
|
|
using var faithful = Simulate();
|
|
Record(faithful);
|
|
using var moved = Simulate((_, item, words) => item.Index == "ref:" + sampled ? Alter(item, words, w => w + 1u, 1) : words);
|
|
var failures = Judge(moved, faithful).Failures;
|
|
Assert.Equal(DeviceRoutes.Length, failures.Count);
|
|
Assert.All(failures, line => { Assert.Contains($"#{sampled} ", line); Assert.Contains("result bits differ", line); });
|
|
}
|
|
}
|