using System.Collections.Generic; using System; using CultMath; using Xunit; namespace CultMath.Tests; // Fixtures are deliberately non-degenerate: A is non-square and non-identity, and b is outside A's range, // so the unconstrained optimum has a real residual and the box actually bites. public sealed class BoundedLeastSquaresTests { private static double Cost(int m, int n, float[] a, float[] b, float[] x) { var c = 0.0; for (var r = 0; r < m; r++) { var s = -(double)b[r]; for (var j = 0; j < n; j++) s += (double)a[r * n + j] * x[j]; c += s * s; } return c; } // Gradient of 0.5*||Ax-b||^2, in double. private static double[] Gradient(int m, int n, float[] a, float[] b, float[] x) { var g = new double[n]; for (var r = 0; r < m; r++) { var s = -(double)b[r]; for (var j = 0; j < n; j++) s += (double)a[r * n + j] * x[j]; for (var j = 0; j < n; j++) g[j] += a[r * n + j] * s; } return g; } private static double AtbScale(int m, int n, float[] a, float[] b) { var mx = 0.0; for (var j = 0; j < n; j++) { var s = 0.0; for (var r = 0; r < m; r++) s += (double)a[r * n + j] * b[r]; mx = Math.Max(mx, Math.Abs(s)); } return 1 + mx; } // Asserts feasibility and the KKT sign rule: free columns have ~zero gradient, columns at lo have // gradient >= 0, columns at hi have gradient <= 0. private static void AssertKkt(int m, int n, float[] a, float[] b, float[] lo, float[] hi, float[] x) { var g = Gradient(m, n, a, b, x); var tol = 1e-3 * AtbScale(m, n, a, b); for (var j = 0; j < n; j++) { Assert.True(x[j] >= lo[j] && x[j] <= hi[j], $"column {j} outside its box: {x[j]}"); if (lo[j] == hi[j]) continue; if (x[j] == lo[j]) Assert.True(g[j] >= -tol, $"column {j} at lo with gradient {g[j]}"); else if (x[j] == hi[j]) Assert.True(g[j] <= tol, $"column {j} at hi with gradient {g[j]}"); else Assert.True(Math.Abs(g[j]) <= tol, $"column {j} free with gradient {g[j]}"); } } // Accelerated projected gradient (FISTA) with step 1/L, L bounded by the Frobenius norm squared of A. // Independent of the solver's active-set logic; run to a fixed point. private static double ReferenceCost(int m, int n, float[] a, float[] b, float[] lo, float[] hi) { var frob = 0.0; foreach (var v in a) frob += (double)v * v; var step = 1.0 / Math.Max(frob, 1e-12); var x = new double[n]; var y = new double[n]; var xf = new float[n]; for (var j = 0; j < n; j++) x[j] = y[j] = Math.Clamp(0.0, lo[j], hi[j]); var t = 1.0; for (var it = 0; it < 100000; it++) { for (var j = 0; j < n; j++) xf[j] = (float)y[j]; var g = Gradient(m, n, a, b, xf); var moved = 0.0; var tNext = (1.0 + Math.Sqrt(1.0 + 4.0 * t * t)) / 2.0; for (var j = 0; j < n; j++) { var next = Math.Clamp(y[j] - step * g[j], lo[j], hi[j]); moved = Math.Max(moved, Math.Abs(next - x[j])); y[j] = next + (t - 1.0) / tNext * (next - x[j]); x[j] = next; } t = tNext; if (moved < 1e-10) break; } for (var j = 0; j < n; j++) xf[j] = (float)x[j]; return Cost(m, n, a, b, xf); } private static BoundedLeastSquaresStatus Solve(int m, int n, float[] a, float[] b, float[] lo, float[] hi, float[] x, out int iterations, int maxIterations = BoundedLeastSquares.DefaultMaxIterations) => BoundedLeastSquares.Solve(m, n, a, b, lo, hi, x, new float[BoundedLeastSquares.WorkspaceLength(n)], out iterations, maxIterations); private static float[] Fill(float value, int n) { var v = new float[n]; Array.Fill(v, value); return v; } private static double[] NormalEquationSolution(int m, int n, float[] a, float[] b) { var mat = new double[n, n + 1]; for (var i = 0; i < n; i++) { for (var j = 0; j < n; j++) for (var r = 0; r < m; r++) mat[i, j] += (double)a[r * n + i] * a[r * n + j]; for (var r = 0; r < m; r++) mat[i, n] += (double)a[r * n + i] * b[r]; } for (var c = 0; c < n; c++) { var piv = mat[c, c]; for (var j = 0; j <= n; j++) mat[c, j] /= piv; for (var i = 0; i < n; i++) { if (i == c) continue; var f = mat[i, c]; for (var j = 0; j <= n; j++) mat[i, j] -= f * mat[c, j]; } } var x = new double[n]; for (var i = 0; i < n; i++) x[i] = mat[i, n]; return x; } private static (int m, int n, float[] a, float[] b, float[] lo, float[] hi) RandomProblem(uint seed) { var rng = new CultMath.Random(seed * 7919u); var n = rng.NextInt(2, 17); var m = rng.NextInt(n, 25); var a = new float[m * n]; var b = new float[m]; for (var i = 0; i < a.Length; i++) a[i] = rng.NextFloat(-1f, 1f); for (var i = 0; i < m; i++) b[i] = rng.NextFloat(-3f, 3f); var lo = new float[n]; var hi = new float[n]; for (var j = 0; j < n; j++) { switch (rng.NextInt(3)) { case 0: lo[j] = -1f; hi[j] = 1f; break; case 1: lo[j] = 0f; hi[j] = 1f; break; default: lo[j] = -0.25f; hi[j] = 0.1f; break; } } return (m, n, a, b, lo, hi); } private static readonly float[] A54 = { 1.0f, 0.3f, -0.2f, 0.2f, 1.1f, 0.4f, -0.5f, 0.6f, 0.9f, 0.7f, -0.4f, 0.5f, 0.1f, 0.8f, -0.6f, }; private static readonly float[] B5 = { 0.9f, -0.4f, 1.3f, 0.2f, -0.7f }; [Fact] public void InteriorOptimumEqualsNormalEquations() { const int m = 5, n = 3; var x = new float[n]; var status = Solve(m, n, A54, B5, Fill(-100f, n), Fill(100f, n), x, out _); Assert.Equal(BoundedLeastSquaresStatus.Converged, status); var expected = NormalEquationSolution(m, n, A54, B5); for (var i = 0; i < n; i++) Assert.Equal(expected[i], x[i], 1e-4); } [Fact] public void KktHoldsOnRandomBoxedProblems() { var anyActive = false; for (uint seed = 1; seed <= 150; seed++) { var (m, n, a, b, lo, hi) = RandomProblem(seed); var x = new float[n]; var status = Solve(m, n, a, b, lo, hi, x, out _); Assert.True(status == BoundedLeastSquaresStatus.Converged, $"seed {seed} ({m}x{n}) hit the cap"); AssertKkt(m, n, a, b, lo, hi, x); for (var j = 0; j < n; j++) anyActive |= x[j] == lo[j] || x[j] == hi[j]; var cs = Cost(m, n, a, b, x); var cr = ReferenceCost(m, n, a, b, lo, hi); Assert.True(Math.Abs(cs - cr) <= 1e-4 * cr + 1e-6, $"seed {seed} ({m}x{n}): solver cost {cs}, reference {cr}"); } Assert.True(anyActive); } [Fact] public void AllBoundsActive() { const int m = 4, n = 3; var a = new[] { 1.0f, 0.2f, 0.1f, 0.1f, 1.0f, 0.3f, -0.2f, 0.2f, 1.0f, 0.5f, 0.5f, 0.4f }; var b = new[] { 10f, -10f, 10f, 1f }; var lo = Fill(-1f, n); var hi = Fill(1f, n); var x = new float[n]; var status = Solve(m, n, a, b, lo, hi, x, out var iterations); Assert.Equal(BoundedLeastSquaresStatus.Converged, status); Assert.Equal(new[] { 1f, -1f, 1f }, x); Assert.Equal(3, iterations); AssertKkt(m, n, a, b, lo, hi, x); } [Fact] public void ZeroColumn() { const int m = 5, n = 3; var a = (float[])A54.Clone(); for (var r = 0; r < m; r++) a[r * n + 1] = 0f; var lo = Fill(-1f, n); var hi = Fill(1f, n); var x = new float[n]; var status = Solve(m, n, a, B5, lo, hi, x, out _); Assert.Equal(BoundedLeastSquaresStatus.Converged, status); AssertKkt(m, n, a, B5, lo, hi, x); var cs = Cost(m, n, a, B5, x); var cr = ReferenceCost(m, n, a, B5, lo, hi); Assert.True(Math.Abs(cs - cr) <= 1e-4 * cr + 1e-6); } [Fact] public void DuplicateColumns() { const int m = 5, n = 3; var a = (float[])A54.Clone(); for (var r = 0; r < m; r++) a[r * n + 2] = a[r * n + 0]; var lo = Fill(-1f, n); var hi = new[] { 0.3f, 1f, 0.3f }; var x = new float[n]; var status = Solve(m, n, a, B5, lo, hi, x, out _); Assert.Equal(BoundedLeastSquaresStatus.Converged, status); AssertKkt(m, n, a, B5, lo, hi, x); var cs = Cost(m, n, a, B5, x); var cr = ReferenceCost(m, n, a, B5, lo, hi); Assert.True(Math.Abs(cs - cr) <= 1e-4 * cr + 1e-6); } [Fact] public void WarmStartMatchesColdStart() { const int m = 5, n = 3; var lo = new[] { -0.2f, -1f, 0f }; var hi = new[] { 0.5f, 0.3f, 1f }; var cold = new float[n]; Solve(m, n, A54, B5, lo, hi, cold, out _); AssertKkt(m, n, A54, B5, lo, hi, cold); var starts = new[] { new[] { 0.5f, -1f, 1f }, new[] { 9f, 9f, -9f }, new[] { 0.1f, 0.1f, 0.5f }, (float[])cold.Clone() }; foreach (var start in starts) { var status = Solve(m, n, A54, B5, lo, hi, start, out var iterations); Assert.Equal(BoundedLeastSquaresStatus.Converged, status); for (var j = 0; j < n; j++) Assert.Equal(cold[j], start[j], 1e-4); if (ReferenceEquals(start, starts[^1])) Assert.Equal(0, iterations); } } [Fact] public void AsymmetricBounds() { const int m = 5, n = 3; var lo = new[] { -1f, 0f, -1f }; var hi = new[] { 1f, 1f, 1f }; // Column 1 wants to go negative (b has a strong component against it) but its floor is 0. var b = new[] { 0.2f, -2.5f, 0.4f, -1.5f, -2.0f }; var x = new float[n]; var status = Solve(m, n, A54, b, lo, hi, x, out _); Assert.Equal(BoundedLeastSquaresStatus.Converged, status); AssertKkt(m, n, A54, b, lo, hi, x); Assert.Equal(0f, x[1]); Assert.True(x[0] < 0f || x[2] < 0f, "a symmetric-bounds column should use its negative side"); var cs = Cost(m, n, A54, b, x); var cr = ReferenceCost(m, n, A54, b, lo, hi); Assert.True(Math.Abs(cs - cr) <= 1e-4 * cr + 1e-6); } [Fact] public void IterationCapReturnsFeasibleIterateAndLimitStatus() { const int m = 4, n = 3; var a = new[] { 1.0f, 0.2f, 0.1f, 0.1f, 1.0f, 0.3f, -0.2f, 0.2f, 1.0f, 0.5f, 0.5f, 0.4f }; var b = new[] { 10f, -10f, 10f, 1f }; var lo = Fill(-1f, n); var hi = Fill(1f, n); var x = new float[n]; var start = Cost(m, n, a, b, x); var status = Solve(m, n, a, b, lo, hi, x, out var iterations, maxIterations: 1); Assert.Equal(BoundedLeastSquaresStatus.IterationLimit, status); Assert.Equal(1, iterations); for (var j = 0; j < n; j++) Assert.InRange(x[j], lo[j], hi[j]); Assert.True(Cost(m, n, a, b, x) < start); var zero = new float[n]; Assert.Equal(BoundedLeastSquaresStatus.IterationLimit, Solve(m, n, a, b, lo, hi, zero, out iterations, maxIterations: 0)); Assert.Equal(0, iterations); Assert.Equal(new float[n], zero); } [Fact] public void StartOutsideTheBoxIsClampedFirst() { const int m = 5, n = 3; var lo = Fill(-0.1f, n); var hi = Fill(0.1f, n); var x = new[] { 50f, -50f, float.NaN }; Solve(m, n, A54, B5, lo, hi, x, out _, maxIterations: 0); Assert.Equal(new[] { 0.1f, -0.1f, -0.1f }, x); } [Fact] public void FixedColumnStaysPutEvenWhenItsGradientWantsOut() { const int m = 5, n = 3; var lo = new[] { -100f, -1f, -100f }; var hi = new[] { 100f, -1f, 100f }; var x = new float[n]; var status = Solve(m, n, A54, B5, lo, hi, x, out var iterations); Assert.Equal(BoundedLeastSquaresStatus.Converged, status); Assert.Equal(-1f, x[1]); Assert.True(Gradient(m, n, A54, B5, x)[1] < -1e-3, "fixture must pull the fixed column inward"); Assert.Equal(0, iterations); AssertKkt(m, n, A54, B5, lo, hi, x); } [Fact] public void WarmCallAllocatesNothing() { const int m = 5, n = 3; var lo = new[] { -0.2f, -1f, 0f }; var hi = new[] { 0.5f, 0.3f, 1f }; var x = new float[n]; var workspace = new float[BoundedLeastSquares.WorkspaceLength(n)]; BoundedLeastSquares.Solve(m, n, A54, B5, lo, hi, x, workspace, out _); var before = GC.GetAllocatedBytesForCurrentThread(); var status = BoundedLeastSquares.Solve(m, n, A54, B5, lo, hi, x, workspace, out _); var after = GC.GetAllocatedBytesForCurrentThread(); Assert.Equal(BoundedLeastSquaresStatus.Converged, status); Assert.Equal(0L, after - before); } [Fact] public void LargeInteriorOptimumEqualsNormalEquations() { const int m = 24, n = 16; var rng = new CultMath.Random(4242u); var a = new float[m * n]; var b = new float[m]; for (var i = 0; i < a.Length; i++) a[i] = rng.NextFloat(-1f, 1f); for (var i = 0; i < m; i++) b[i] = rng.NextFloat(-3f, 3f); var x = new float[n]; var status = Solve(m, n, a, b, Fill(-1000f, n), Fill(1000f, n), x, out var iterations); Assert.Equal(BoundedLeastSquaresStatus.Converged, status); Assert.Equal(0, iterations); var expected = NormalEquationSolution(m, n, a, b); for (var i = 0; i < n; i++) Assert.Equal(expected[i], x[i], 1e-3); } [Fact] public void ScaledAndNearDuplicateColumnsAreOptimal() { const int m = 5, n = 3; foreach (var noise in new[] { 0f, 1e-5f }) { var a = (float[])A54.Clone(); for (var r = 0; r < m; r++) a[r * n + 2] = 0.37f * a[r * n + 0] + noise * (r - 2); var lo = Fill(-1f, n); var hi = new[] { 0.6f, 1f, 0.6f }; var x = new float[n]; var status = Solve(m, n, a, B5, lo, hi, x, out _); Assert.Equal(BoundedLeastSquaresStatus.Converged, status); AssertKkt(m, n, a, B5, lo, hi, x); var cs = Cost(m, n, a, B5, x); var cr = ReferenceCost(m, n, a, B5, lo, hi); Assert.True(Math.Abs(cs - cr) <= 1e-4 * cr + 1e-6, $"noise {noise}: solver cost {cs}, reference {cr}"); } } [Fact] public void WorkspaceContentsOnEntryAreIgnored() { const int m = 5, n = 3; var zeroCol = (float[])A54.Clone(); for (var r = 0; r < m; r++) zeroCol[r * n + 1] = 0f; var dupCol = (float[])A54.Clone(); for (var r = 0; r < m; r++) dupCol[r * n + 2] = dupCol[r * n + 0]; foreach (var a in new[] { A54, zeroCol, dupCol }) { var lo = new[] { -1f, -1f, -1f }; var hi = new[] { 0.3f, 1f, 0.3f }; var clean = new float[n]; Solve(m, n, a, B5, lo, hi, clean, out var cleanIterations); foreach (var junk in new[] { float.NaN, 7f, -1e30f }) { var workspace = new float[BoundedLeastSquares.WorkspaceLength(n)]; Array.Fill(workspace, junk); var x = new float[n]; BoundedLeastSquares.Solve(m, n, a, B5, lo, hi, x, workspace, out var iterations); Assert.Equal(clean, x); Assert.Equal(cleanIterations, iterations); } } } // Junk in the workspace, including NaN doubles, must not reach any decision: random boxed problems (which release // bounds and use the tolerance) give bit-identical x and the same iteration count. [Fact] public void WorkspaceJunkNeverReachesTheSolveOnRandomProblems() { for (uint seed = 1; seed <= 25; seed++) { var (m, n, a, b, lo, hi) = RandomProblem(seed); var clean = new float[n]; Solve(m, n, a, b, lo, hi, clean, out var cleanIterations); foreach (var junk in new[] { float.NaN, 7f, -1e30f }) { var workspace = new float[BoundedLeastSquares.WorkspaceLength(n)]; Array.Fill(workspace, junk); var x = new float[n]; BoundedLeastSquares.Solve(m, n, a, b, lo, hi, x, workspace, out var iterations); Assert.Equal(cleanIterations, iterations); for (var j = 0; j < n; j++) Assert.Equal(BitConverter.SingleToInt32Bits(clean[j]), BitConverter.SingleToInt32Bits(x[j])); } } } [Fact] public void WarmStartAtAVertexTakesNoIterations() { const int m = 4, n = 3; var a = new[] { 1.0f, 0.2f, 0.1f, 0.1f, 1.0f, 0.3f, -0.2f, 0.2f, 1.0f, 0.5f, 0.5f, 0.4f }; var b = new[] { 10f, -10f, 10f, 1f }; var x = new[] { 1f, -1f, 1f }; var status = Solve(m, n, a, b, Fill(-1f, n), Fill(1f, n), x, out var iterations); Assert.Equal(BoundedLeastSquaresStatus.Converged, status); Assert.Equal(0, iterations); Assert.Equal(new[] { 1f, -1f, 1f }, x); } [Fact] public void IterationCountFollowsTheCapExactly() { var sawRelease = false; for (uint seed = 1; seed <= 40; seed++) { var (m, n, a, b, lo, hi) = RandomProblem(seed); var full = new float[n]; Solve(m, n, a, b, lo, hi, full, out var needed); var previousCost = double.MaxValue; for (var cap = 0; cap <= needed; cap++) { var x = new float[n]; var status = Solve(m, n, a, b, lo, hi, x, out var iterations, cap); Assert.Equal(cap, iterations); Assert.Equal(cap < needed ? BoundedLeastSquaresStatus.IterationLimit : BoundedLeastSquaresStatus.Converged, status); for (var j = 0; j < n; j++) Assert.InRange(x[j], lo[j], hi[j]); var cost = Cost(m, n, a, b, x); Assert.True(cost <= previousCost * (1 + 1e-6) + 1e-9, $"seed {seed} cap {cap}: cost rose from {previousCost} to {cost}"); previousCost = cost; } sawRelease |= needed > n; } Assert.True(sawRelease, "no random problem needed more iterations than columns, so no bound was ever released"); } [Fact] public void EachShortSpanIsRejected() { const int m = 5, n = 3; var ws = new float[BoundedLeastSquares.WorkspaceLength(n)]; var lo = Fill(-1f, n); var hi = Fill(1f, n); void Expect(Action call) => Assert.Contains("BoundedLeastSquares", Assert.Throws(call).Message); Expect(() => BoundedLeastSquares.Solve(m, n, new float[m * n - 1], B5, lo, hi, new float[n], ws, out _)); Expect(() => BoundedLeastSquares.Solve(m, n, A54, new float[m - 1], lo, hi, new float[n], ws, out _)); Expect(() => BoundedLeastSquares.Solve(m, n, A54, B5, new float[n - 1], hi, new float[n], ws, out _)); Expect(() => BoundedLeastSquares.Solve(m, n, A54, B5, lo, new float[n - 1], new float[n], ws, out _)); Expect(() => BoundedLeastSquares.Solve(m, n, A54, B5, lo, hi, new float[n - 1], ws, out _)); Expect(() => BoundedLeastSquares.Solve(m, n, A54, B5, lo, hi, new float[n], new float[ws.Length - 1], out _)); Expect(() => BoundedLeastSquares.Solve(-1, n, A54, B5, lo, hi, new float[n], ws, out _)); Expect(() => BoundedLeastSquares.Solve(m, -1, A54, B5, lo, hi, new float[n], ws, out _)); // Exactly sized spans, and empty problems, are accepted. BoundedLeastSquares.Solve(m, n, A54, B5, lo, hi, new float[n], ws, out _); Assert.Equal(BoundedLeastSquaresStatus.Converged, BoundedLeastSquares.Solve(0, 0, default, default, default, default, default, default, out _)); Assert.Equal(BoundedLeastSquaresStatus.Converged, BoundedLeastSquares.Solve(0, n, default, default, lo, hi, new float[n], ws, out _)); } // Well-conditioned enough (m = n + 6) that x is comparable across scales. private static (int m, int n, float[] a, float[] b, float[] lo, float[] hi) ScaleProblem(uint seed) { var rng = new CultMath.Random(seed * 104729u); var n = rng.NextInt(3, 11); var m = n + 6; var a = new float[m * n]; var b = new float[m]; for (var i = 0; i < a.Length; i++) a[i] = rng.NextFloat(-1f, 1f); for (var i = 0; i < m; i++) b[i] = rng.NextFloat(-3f, 3f); var lo = new float[n]; var hi = new float[n]; for (var j = 0; j < n; j++) { lo[j] = rng.NextFloat(-0.6f, -0.05f); hi[j] = rng.NextFloat(0.05f, 0.6f); } return (m, n, a, b, lo, hi); } private static float[] Scaled(float[] v, float k) { var r = new float[v.Length]; for (var i = 0; i < v.Length; i++) r[i] = v[i] * k; return r; } [Fact] public void SolutionIsScaleFree() { var sawActive = false; for (uint seed = 1; seed <= 40; seed++) { var (m, n, a, b, lo, hi) = ScaleProblem(seed); var x = (float[])lo.Clone(); Assert.Equal(BoundedLeastSquaresStatus.Converged, Solve(m, n, a, b, lo, hi, x, out _)); AssertKkt(m, n, a, b, lo, hi, x); for (var j = 0; j < n; j++) sawActive |= x[j] == lo[j] || x[j] == hi[j]; foreach (var k in new[] { 1e-6f, 1e3f, 1e6f }) { // b and the bounds scaled: x scales with them. var xb = Scaled(lo, k); var status = Solve(m, n, a, Scaled(b, k), Scaled(lo, k), Scaled(hi, k), xb, out _); Assert.True(status == BoundedLeastSquaresStatus.Converged, $"seed {seed} k {k}: {status}"); for (var j = 0; j < n; j++) Assert.True(Math.Abs(xb[j] - x[j] * k) <= 1e-3 * Math.Abs(k) * (1 + Math.Abs(x[j])), $"seed {seed} b*{k} col {j}: {xb[j]} vs {x[j] * k}"); // A scaled: x scales inversely, so the bounds stay put and only b is scaled with A to keep the same box. var xa = (float[])lo.Clone(); var lo2 = Scaled(lo, 1f / k); var hi2 = Scaled(hi, 1f / k); for (var j = 0; j < n; j++) xa[j] = lo2[j]; status = Solve(m, n, Scaled(a, k), b, lo2, hi2, xa, out _); Assert.True(status == BoundedLeastSquaresStatus.Converged, $"seed {seed} A*{k}: {status}"); for (var j = 0; j < n; j++) Assert.True(Math.Abs(xa[j] - x[j] / k) <= 1e-3 * Math.Abs(1 / k) * (1 + Math.Abs(x[j])), $"seed {seed} A*{k} col {j}: {xa[j]} vs {x[j] / k}"); } } Assert.True(sawActive); } [Fact] public void NonFiniteOrInvertedInputIsRejectedAndLeavesXUntouched() { const int m = 5, n = 3; var nan = float.NaN; var inf = float.PositiveInfinity; var cases = new (string name, Action corrupt)[] { ("A NaN", (a, b, lo, hi) => a[4] = nan), ("A +Inf", (a, b, lo, hi) => a[7] = inf), ("A -Inf", (a, b, lo, hi) => a[1] = -inf), ("b NaN", (a, b, lo, hi) => b[2] = nan), ("b +Inf", (a, b, lo, hi) => b[0] = inf), ("lo NaN", (a, b, lo, hi) => lo[1] = nan), ("hi NaN", (a, b, lo, hi) => hi[2] = nan), ("lo > hi", (a, b, lo, hi) => { lo[1] = 0.5f; hi[1] = 0.25f; }), ("lo = +Inf", (a, b, lo, hi) => { lo[0] = inf; hi[0] = inf; }), ("hi = -Inf", (a, b, lo, hi) => { lo[0] = -inf; hi[0] = -inf; }), }; foreach (var (name, corrupt) in cases) { var a = (float[])A54.Clone(); var b = (float[])B5.Clone(); var lo = Fill(-1f, n); var hi = Fill(1f, n); corrupt(a, b, lo, hi); foreach (var start in new[] { new[] { 0.5f, -3f, 9f }, new[] { nan, 0f, 0f } }) { var x = (float[])start.Clone(); var status = Solve(m, n, a, b, lo, hi, x, out var iterations); Assert.True(status == BoundedLeastSquaresStatus.InvalidInput, $"{name}: {status}"); Assert.Equal(0, iterations); for (var j = 0; j < n; j++) Assert.Equal(BitConverter.SingleToInt32Bits(start[j]), BitConverter.SingleToInt32Bits(x[j])); } } } [Fact] public void ValidateNamesTheFirstInvalidInput() { const int m = 5, n = 3; var nan = float.NaN; var inf = float.PositiveInfinity; const double goodTol = 1e-6; var cases = new (string name, BoundedLeastSquaresInput expected, double tol, Action corrupt)[] { ("valid", BoundedLeastSquaresInput.Valid, goodTol, (a, b, lo, hi) => { }), ("infinite bounds are unbounded", BoundedLeastSquaresInput.Valid, goodTol, (a, b, lo, hi) => { lo[0] = -inf; hi[1] = inf; }), ("lo = hi", BoundedLeastSquaresInput.Valid, goodTol, (a, b, lo, hi) => { lo[2] = 0.5f; hi[2] = 0.5f; }), ("tolerance 0", BoundedLeastSquaresInput.InvalidTolerance, 0.0, (a, b, lo, hi) => { }), ("tolerance negative", BoundedLeastSquaresInput.InvalidTolerance, -1e-6, (a, b, lo, hi) => { }), ("tolerance NaN", BoundedLeastSquaresInput.InvalidTolerance, double.NaN, (a, b, lo, hi) => { }), ("tolerance +Inf", BoundedLeastSquaresInput.InvalidTolerance, double.PositiveInfinity, (a, b, lo, hi) => { }), ("tolerance tiny but positive", BoundedLeastSquaresInput.Valid, double.Epsilon, (a, b, lo, hi) => { }), ("lo NaN", BoundedLeastSquaresInput.InvalidBounds, goodTol, (a, b, lo, hi) => lo[1] = nan), ("hi NaN", BoundedLeastSquaresInput.InvalidBounds, goodTol, (a, b, lo, hi) => hi[2] = nan), ("lo > hi", BoundedLeastSquaresInput.InvalidBounds, goodTol, (a, b, lo, hi) => { lo[1] = 0.5f; hi[1] = 0.25f; }), ("lo = +Inf", BoundedLeastSquaresInput.InvalidBounds, goodTol, (a, b, lo, hi) => { lo[0] = inf; hi[0] = inf; }), ("hi = -Inf", BoundedLeastSquaresInput.InvalidBounds, goodTol, (a, b, lo, hi) => { lo[0] = -inf; hi[0] = -inf; }), ("A NaN", BoundedLeastSquaresInput.NonFiniteMatrix, goodTol, (a, b, lo, hi) => a[4] = nan), ("A +Inf", BoundedLeastSquaresInput.NonFiniteMatrix, goodTol, (a, b, lo, hi) => a[7] = inf), ("A -Inf in the last entry", BoundedLeastSquaresInput.NonFiniteMatrix, goodTol, (a, b, lo, hi) => a[m * n - 1] = -inf), ("b NaN", BoundedLeastSquaresInput.NonFiniteTarget, goodTol, (a, b, lo, hi) => b[2] = nan), ("b +Inf in the first entry", BoundedLeastSquaresInput.NonFiniteTarget, goodTol, (a, b, lo, hi) => b[0] = inf), ("b -Inf in the last entry", BoundedLeastSquaresInput.NonFiniteTarget, goodTol, (a, b, lo, hi) => b[m - 1] = -inf), // Two faults at once: the earlier member in the order wins, whichever input comes first in the argument list. ("tolerance and bounds", BoundedLeastSquaresInput.InvalidTolerance, 0.0, (a, b, lo, hi) => lo[0] = nan), ("tolerance and A", BoundedLeastSquaresInput.InvalidTolerance, double.NaN, (a, b, lo, hi) => a[0] = nan), ("tolerance and b", BoundedLeastSquaresInput.InvalidTolerance, -1.0, (a, b, lo, hi) => b[0] = nan), ("bounds and A", BoundedLeastSquaresInput.InvalidBounds, goodTol, (a, b, lo, hi) => { hi[0] = nan; a[0] = nan; }), ("bounds and b", BoundedLeastSquaresInput.InvalidBounds, goodTol, (a, b, lo, hi) => { lo[2] = 1f; hi[2] = -1f; b[4] = inf; }), ("A and b", BoundedLeastSquaresInput.NonFiniteMatrix, goodTol, (a, b, lo, hi) => { a[14] = inf; b[0] = nan; }), }; foreach (var (name, expected, tol, corrupt) in cases) { var a = (float[])A54.Clone(); var b = (float[])B5.Clone(); var lo = Fill(-1f, n); var hi = Fill(1f, n); corrupt(a, b, lo, hi); var verdict = BoundedLeastSquares.Validate(m, n, a, b, lo, hi, tol); Assert.True(expected == verdict, $"{name}: {verdict}"); var x = new float[n]; var status = BoundedLeastSquares.Solve(m, n, a, b, lo, hi, x, new float[BoundedLeastSquares.WorkspaceLength(n)], out _, tol); Assert.True((status == BoundedLeastSquaresStatus.InvalidInput) == (expected != BoundedLeastSquaresInput.Valid), $"{name}: Solve {status}"); } // The overload without a tolerance validates against the default one. Assert.Equal(BoundedLeastSquaresInput.Valid, BoundedLeastSquares.Validate(m, n, A54, B5, Fill(-1f, n), Fill(1f, n))); Assert.Equal(BoundedLeastSquaresInput.InvalidBounds, BoundedLeastSquares.Validate(m, n, A54, B5, Fill(2f, n), Fill(1f, n))); } [Fact] public void ValidateReadsOnlyTheStatedDimensions() { // Entries beyond m * n, m and n are not inputs: a NaN there is not a fault. const int m = 2, n = 2; var a = new[] { 1f, 0f, 0f, 1f, float.NaN }; var b = new[] { 1f, 1f, float.NaN }; var lo = new[] { -1f, -1f, float.NaN }; var hi = new[] { 1f, 1f, float.NaN }; Assert.Equal(BoundedLeastSquaresInput.Valid, BoundedLeastSquares.Validate(m, n, a, b, lo, hi)); // A problem with no unknown has no use for b, so a non-finite b is not an invalid input there. var nanB = new[] { float.NaN, 1f }; Assert.Equal(BoundedLeastSquaresInput.Valid, BoundedLeastSquares.Validate(2, 0, ReadOnlySpan.Empty, nanB, ReadOnlySpan.Empty, ReadOnlySpan.Empty)); Assert.Equal(BoundedLeastSquaresStatus.Converged, BoundedLeastSquares.Solve(2, 0, ReadOnlySpan.Empty, nanB, ReadOnlySpan.Empty, ReadOnlySpan.Empty, Span.Empty, Span.Empty, out _)); } [Fact] public void ValidateThrowsOnAShortSpanLikeSolve() { const int m = 5, n = 3; var lo = Fill(-1f, n); var hi = Fill(1f, n); // The caller-bug contract holds ahead of any value check: a NaN tolerance does not hide it. Assert.Throws(() => BoundedLeastSquares.Validate(m, n, A54.AsSpan(1), B5, lo, hi, double.NaN)); Assert.Throws(() => BoundedLeastSquares.Validate(m, n, A54, B5.AsSpan(1), lo, hi)); Assert.Throws(() => BoundedLeastSquares.Validate(m, n, A54, B5, lo.AsSpan(1), hi)); Assert.Throws(() => BoundedLeastSquares.Validate(m, n, A54, B5, lo, hi.AsSpan(1))); Assert.Throws(() => BoundedLeastSquares.Validate(-1, n, A54, B5, lo, hi)); Assert.Throws(() => BoundedLeastSquares.Validate(m, -1, A54, B5, lo, hi)); } [Fact] public void ValidateAllocatesNothing() { const int m = 5, n = 3; var bad = (float[])A54.Clone(); bad[14] = float.NaN; var lo = Fill(-1f, n); var hi = Fill(1f, n); foreach (var (name, a, tol) in new[] { ("valid", A54, 1e-6), ("invalid matrix", bad, 1e-6), ("invalid tolerance", A54, 0.0) }) { var bytes = new long[2]; for (var call = 0; call < 2; call++) { var before = GC.GetAllocatedBytesForCurrentThread(); BoundedLeastSquares.Validate(m, n, a, B5, lo, hi, tol); bytes[call] = GC.GetAllocatedBytesForCurrentThread() - before; } Assert.True(bytes[0] == 0 && bytes[1] == 0, $"{name}: first call {bytes[0]} B, second {bytes[1]} B"); } } [Fact] public void SolveReturnsInvalidInputExactlyWhenValidateIsNotValid() { var invalid = 0; for (uint seed = 1; seed <= 300; seed++) { var (m, n, a, b, lo, hi) = RandomProblem(seed); var rng = new CultMath.Random(seed * 104729u); var tol = 1e-6; // Corrupt zero to three inputs, so some problems stay clean. for (var k = rng.NextInt(4); k > 0; k--) { switch (rng.NextInt(5)) { case 0: tol = rng.NextInt(2) == 0 ? 0.0 : double.NaN; break; case 1: lo[rng.NextInt(n)] = float.NaN; break; case 2: { var j = rng.NextInt(n); lo[j] = hi[j] + 1f; } break; case 3: a[rng.NextInt(m * n)] = rng.NextInt(2) == 0 ? float.NaN : float.NegativeInfinity; break; default: b[rng.NextInt(m)] = float.PositiveInfinity; break; } } var verdict = BoundedLeastSquares.Validate(m, n, a, b, lo, hi, tol); var start = Fill(0.5f, n); var x = (float[])start.Clone(); var status = BoundedLeastSquares.Solve(m, n, a, b, lo, hi, x, new float[BoundedLeastSquares.WorkspaceLength(n)], out var iterations, tol); Assert.True((status == BoundedLeastSquaresStatus.InvalidInput) == (verdict != BoundedLeastSquaresInput.Valid), $"seed {seed}: {verdict} but {status}"); if (verdict != BoundedLeastSquaresInput.Valid) { invalid++; Assert.Equal(0, iterations); Assert.Equal(start, x); } } Assert.InRange(invalid, 100, 280); } // ANoiseLevelViolationIsNeverReleased with the columns rescaled by S (x scales by 1/S, so A x is unchanged): the gate is // the rounding bound of the released column's own gradient, whatever the scale of that column or of its neighbours. A gate // read from another column's row, or one that shrinks as the terms grow, releases rounding noise and spins. [Fact] public void ANoiseLevelViolationIsNeverReleasedAtAnyColumnScale() { const int m = 2, n = 6; var a0 = new[] { 1.0213068f, 0.97530603f, -0.5673449f, -0.64136946f, -0.07432885f, -0.35571396f, 0.4326824f, 1.1969734f, -0.1343108f, -0.23431005f, -1.4009786f, 0.10656489f, }; var lo0 = new[] { -0.44782114f, 0f, -0.008422324f, -0.03878026f, 0f, -6.6434336f }; var hi0 = new[] { 0.0448007f, 0f, 0.005942123f, 0.0785512f, 0f, 0.23079813f }; var x0 = new[] { -0.28347927f, 0f, 0.0051532285f, -0.03878026f, 0f, 0.23079813f }; var patterns = new (string name, float[] scale)[] { ("uniform 1e3", Fill(1e3f, n)), ("uniform 1e-3", Fill(1e-3f, n)), ("ascending", new[] { 1f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f }), ("descending", new[] { 1e5f, 1e4f, 1e3f, 1e2f, 1e1f, 1f }), ("one heavy column", new[] { 1f, 1f, 1e4f, 1f, 1f, 1f }), ("one light column", new[] { 1e3f, 1e3f, 1e-3f, 1e3f, 1e3f, 1e3f }), }; foreach (var (name, scale) in patterns) { var a = new float[m * n]; var lo = new float[n]; var hi = new float[n]; var x = new float[n]; for (var j = 0; j < n; j++) { for (var i = 0; i < m; i++) a[i * n + j] = a0[i * n + j] * scale[j]; lo[j] = lo0[j] / scale[j]; hi[j] = hi0[j] / scale[j]; x[j] = Math.Clamp(x0[j] / scale[j], lo[j], hi[j]); } var status = Solve(m, n, a, new float[m], lo, hi, x, out var iterations); Assert.True(status == BoundedLeastSquaresStatus.Converged, $"{name}: {status}"); Assert.True(iterations <= 8, $"{name}: {iterations} iterations"); } } // Soul's stall-after-a-release family (cut bls-release-stop, s1): after a release the next stationarity miss is not yet // a stall, so the solve takes one more pass to confirm before it stops. Two small problems whose release is followed by // exactly such a miss; ending that first miss at once (a stall mark that starts below every cost) stops one pass early. [Theory] [InlineData( new[] { -0.0067761745f, -207.21786f, -0.004115388f, -751.1994f, -8.1700025E-05f, -481.7794f, 0.0038280683f, 480.12833f }, new[] { 1.7477255f, 2.4914713f, -2.3861952f, 2.187128f }, -1f, 1f, -0.25f, 0.1f)] [InlineData( new[] { 70.07762f, 5.7015634f, 111.790436f, -12.586686f, 75.29796f, 12.872999f, -402.25464f, -1.8093401f, -524.51056f, -8.649797f }, new[] { -1.8312539f, 2.9556413f, 0.7550676f, -2.1365998f, 2.30717f }, -1f, 1f, 0f, 1f)] public void AReleaseIsConfirmedByAPassBeforeTheSolveStops(float[] a, float[] b, float lo0, float hi0, float lo1, float hi1) { var m = b.Length; const int n = 2; var lo = new[] { lo0, lo1 }; var hi = new[] { hi0, hi1 }; var x = (float[])lo.Clone(); Assert.Equal(BoundedLeastSquaresStatus.Converged, SolveAt(m, n, a, b, lo, hi, x, out var iterations, 1e-12)); Assert.Equal(2, iterations); AssertKkt(m, n, a, b, lo, hi, x); } [Fact] public void InfiniteBoundsMeanUnbounded() { const int m = 5, n = 3; var inf = float.PositiveInfinity; var x = new float[n]; Assert.Equal(BoundedLeastSquaresStatus.Converged, Solve(m, n, A54, B5, Fill(-inf, n), Fill(inf, n), x, out _)); var expected = NormalEquationSolution(m, n, A54, B5); for (var j = 0; j < n; j++) Assert.Equal(expected[j], x[j], 1e-4); // One-sided, with an infinite warm start: the start is clamped to a finite point first. var y = new[] { -inf, inf, float.NaN }; var lo = new[] { -inf, 0.2f, -inf }; var hi = new[] { inf, inf, 0.1f }; Assert.Equal(BoundedLeastSquaresStatus.Converged, Solve(m, n, A54, B5, lo, hi, y, out _)); AssertKkt(m, n, A54, B5, lo, hi, y); Assert.All(y, v => Assert.True(float.IsFinite(v))); } // Exact reference: every assignment of each column to {lo, hi, free}, free set solved in double. private static double ExactBoxedCost(int m, int n, float[] a, float[] b, float[] lo, float[] hi, double pivotTol = 1e-9, double ridge = 0.0) { var maxDiag = 0.0; for (var j = 0; j < n; j++) { var d = 0.0; for (var r = 0; r < m; r++) d += (double)a[r * n + j] * a[r * n + j]; maxDiag = Math.Max(maxDiag, d); } var best = double.MaxValue; var total = 1; for (var j = 0; j < n; j++) total *= 3; var x = new double[n]; for (var code = 0; code < total; code++) { var c = code; var free = new System.Collections.Generic.List(); for (var j = 0; j < n; j++) { var d = c % 3; c /= 3; if (d == 0) x[j] = lo[j]; else if (d == 1) x[j] = hi[j]; else free.Add(j); } var k = free.Count; if (k > 0) { var mat = new double[k, k + 1]; for (var i = 0; i < k; i++) { for (var l = 0; l < k; l++) for (var r = 0; r < m; r++) mat[i, l] += (double)a[r * n + free[i]] * a[r * n + free[l]]; for (var r = 0; r < m; r++) { var fixedPart = 0.0; for (var j = 0; j < n; j++) if (!free.Contains(j)) fixedPart += (double)a[r * n + j] * x[j]; mat[i, k] += (double)a[r * n + free[i]] * (b[r] - fixedPart); } } for (var i = 0; i < k; i++) mat[i, i] += ridge * maxDiag; var ok = true; for (var col = 0; col < k && ok; col++) { var piv = col; for (var r = col + 1; r < k; r++) if (Math.Abs(mat[r, col]) > Math.Abs(mat[piv, col])) piv = r; if (Math.Abs(mat[piv, col]) < pivotTol) { ok = false; break; } for (var l = 0; l <= k; l++) (mat[col, l], mat[piv, l]) = (mat[piv, l], mat[col, l]); for (var r = 0; r < k; r++) { if (r == col) continue; var f = mat[r, col] / mat[col, col]; for (var l = col; l <= k; l++) mat[r, l] -= f * mat[col, l]; } } if (!ok) continue; for (var i = 0; i < k; i++) { x[free[i]] = mat[i, k] / mat[i, i]; if (x[free[i]] < lo[free[i]] - 1e-12 || x[free[i]] > hi[free[i]] + 1e-12) ok = false; } if (!ok) continue; } var cost = 0.0; for (var r = 0; r < m; r++) { var s = -(double)b[r]; for (var j = 0; j < n; j++) s += (double)a[r * n + j] * x[j]; cost += s * s; } best = Math.Min(best, cost); } return best; } // m < n with columns that are nearly parallel (condition number of A around 1e2), b outside the reachable set. // The absolute scale of A must not matter: the KKT tolerance is relative, and the working iterate is double, so a // 1e6 A with untouched bounds reaches the same optimum to the same allowance. [Theory] [InlineData(1f)] [InlineData(1e6f)] public void NearCollinearColumnsReachTheExactOptimum(float scale) { const int m = 2, n = 7; for (var family = 0; family < 4; family++) { for (uint seed = 1; seed <= 120; seed++) { var rng = new CultMath.Random(seed * 15485863u + (uint)family * 977u); var a = new float[m * n]; for (var j = 0; j < n; j++) { switch (family) { case 0: a[j] = rng.NextFloat(0.5f, 1f); a[n + j] = 0.01f * rng.NextFloat(-1f, 1f); break; case 1: a[j] = 1f + 0.02f * rng.NextFloat(-1f, 1f); a[n + j] = 0.01f * rng.NextFloat(-1f, 1f); break; case 2: a[j] = rng.NextFloat(-1f, 1f); a[n + j] = 0.01f * a[j] + 0.01f * rng.NextFloat(-1f, 1f); break; default: a[j] = rng.NextFloat(0.5f, 1f); a[n + j] = a[j] * 0.9f + 0.01f * rng.NextFloat(-1f, 1f); break; } } for (var i = 0; i < a.Length; i++) a[i] *= scale; var b = new[] { rng.NextFloat(-8f, 8f), rng.NextFloat(-8f, 8f) }; var lo = new float[n]; var hi = new float[n]; for (var j = 0; j < n; j++) { lo[j] = rng.NextFloat() < 0.5f ? -1f : 0f; hi[j] = 1f; } var x = new float[n]; var status = Solve(m, n, a, b, lo, hi, x, out _); Assert.True(status == BoundedLeastSquaresStatus.Converged, $"family {family} seed {seed}: {status}"); var gap = Cost(m, n, a, b, x) - ExactBoxedCost(m, n, a, b, lo, hi); var allowed = 1e-5 * (b[0] * b[0] + b[1] * b[1]); Assert.True(gap <= allowed, $"family {family} seed {seed}: cost {gap} above the exact optimum (allowed {allowed})"); } } } // A pass can end with the KKT test still failing and the cost exactly where it was: a dependent free column keeps a // gradient above the tolerance, or a bound release lands on a column the factor drops. The solver must call that // converged; if the stall rule mis-reads the cost scale or the progress, it spins to the iteration cap instead. // Each kind is a fixed-seed batch of small boxed problems with warm starts on and off the bounds and pinned columns: // rank-deficient (duplicate and zero columns), nearly collinear (1e-2 to 1e-5), integer-tied, wide, cond(A) 1e5 to // 3e5, and "weak", where b lies along the weakest singular direction of a cond(A) 1e6 to 1e7 matrix so the gradient // tolerance sits below the rounding noise with interior columns nonzero. Every kind must converge; all but "weak" // (past the documented condition range) must also land within the documented cost gap, relative to |b|^2, of an // exact enumeration of the active sets (a 1e-15 ridge keeps singular free sets solvable). [Theory] [InlineData("plain", 300)] [InlineData("rankdef", 300)] [InlineData("collinear", 300)] [InlineData("integer-tie", 300)] [InlineData("wide", 300)] [InlineData("cond", 300)] [InlineData("weak", 6000)] public void StalledPassesConvergeInsteadOfSpinningToTheCap(string kind, int seeds) { var failures = new System.Collections.Generic.List(); for (uint seed = 1; seed <= seeds; seed++) { var rng = new CultMath.Random(seed * 2246822519u + (uint)kind.Length * 3266489917u); double Gauss() => Math.Sqrt(-2.0 * Math.Log(1.0 - rng.NextFloat() * 0.999999)) * Math.Cos(2.0 * Math.PI * rng.NextFloat()); var n = rng.NextInt(2, 8); var m = rng.NextInt(2, 12); if (kind == "wide") m = Math.Max(1, n - rng.NextInt(1, 4)); if (kind == "cond" || kind == "weak") m = n + rng.NextInt(0, 5); float[] a; float[] bWeak = null; if (kind == "weak") { (a, bWeak) = BlsTrace.ConditionedProblem(m, n, Math.Pow(10.0, 6.0 + rng.NextFloat()), seed, 1.0, true); } else if (kind == "cond") { a = BlsTrace.ConditionedProblem(m, n, Math.Pow(10.0, 5.0 + 0.5 * rng.NextFloat()), seed).a; } else { a = new float[m * n]; for (var i = 0; i < m; i++) { var row = new double[n]; for (var j = 0; j < n; j++) row[j] = kind == "integer-tie" ? rng.NextInt(-2, 3) : Gauss(); for (var j = 1; j < n; j++) { if (kind == "rankdef" && j % 3 == 1) row[j] = row[j - 1]; if (kind == "rankdef" && j % 3 == 2) row[j] = 0.0; if (kind == "collinear" && j % 2 == 1) row[j] = row[j - 1] + Math.Pow(10.0, -rng.NextInt(2, 6)) * Gauss(); } for (var j = 0; j < n; j++) a[i * n + j] = (float)row[j]; } } var lo = new float[n]; var hi = new float[n]; var x = new float[n]; var xt = new double[n]; for (var j = 0; j < n; j++) { var w = Math.Pow(10.0, rng.NextInt(-2, 2)); lo[j] = (float)(-w * rng.NextFloat()); hi[j] = (float)(w * rng.NextFloat()); if (rng.NextInt(6) == 0) hi[j] = lo[j]; else if (rng.NextInt(8) == 0) lo[j] = hi[j] = 0f; else if (rng.NextInt(8) == 0) { lo[j] = (float)(0.5 + rng.NextFloat()); hi[j] = lo[j] + 0.5f; } if (kind == "weak") { var c = (float)Math.Pow(10.0, rng.NextInt(3, 9)); lo[j] = rng.NextInt(2) == 0 ? -c : 0f; hi[j] = c; } if (kind == "integer-tie") { lo[j] = -1f; hi[j] = rng.NextInt(4) == 0 ? -1f : 1f; } x[j] = rng.NextInt(3) == 0 ? lo[j] : rng.NextInt(2) == 0 ? hi[j] : lo[j] + (hi[j] - lo[j]) * rng.NextFloat(); xt[j] = (lo[j] + hi[j]) / 2.0 + (hi[j] - lo[j]) * 3.0 * (rng.NextFloat() - 0.5); } var noise = Math.Pow(10.0, rng.NextInt(-2, 4)); var mode = rng.NextInt(4); var b = new float[m]; for (var i = 0; i < m; i++) { var s = 0.0; for (var j = 0; j < n; j++) s += (double)a[i * n + j] * xt[j]; b[i] = bWeak != null ? bWeak[i] : mode == 0 ? 0f : (float)(s + noise * Gauss() * (mode == 3 ? 1.0 : 0.01)); } var status = Solve(m, n, a, b, lo, hi, x, out var iterations); if (status != BoundedLeastSquaresStatus.Converged) { failures.Add($"seed {seed}: {status} after {iterations}"); continue; } if (kind == "weak") continue; var bb = 0.0; foreach (var v in b) bb += (double)v * v; if (bb == 0.0) bb = 1.0; var gap = Cost(m, n, a, b, x) - ExactBoxedCost(m, n, a, b, lo, hi, 1e-300, 1e-15); if (gap > 1e-5 * bb) failures.Add($"seed {seed}: cost gap {gap / bb:E2} of |b|^2"); } Assert.True(failures.Count == 0, $"{kind}: {failures.Count} of {seeds} failed; first: {string.Join("; ", failures.GetRange(0, Math.Min(3, failures.Count)))}"); } // Unsaturated allocator-shaped ticks: 24 columns, every column free at the optimum. The solve is one Newton step; // at most one more may confirm it. Before the working iterate moved to double, the float gradient's noise floor // exceeded the tolerance from cond ~1e3 and the solver spun to the 100-iteration cap on optimal points. [Theory] [InlineData(1e2)] [InlineData(1e3)] [InlineData(1e4)] public void UnsaturatedIllConditionedProblemsConvergeInAtMostTwoIterations(double cond) { const int m = 28, n = 24; for (uint seed = 1; seed <= 30; seed++) { var (a, b) = BlsTrace.ConditionedProblem(m, n, cond, seed); var x = new float[n]; var status = Solve(m, n, a, b, Fill(-1e9f, n), Fill(1e9f, n), x, out var iterations); Assert.True(status == BoundedLeastSquaresStatus.Converged, $"cond {cond} seed {seed}: {status} after {iterations}"); Assert.True(iterations <= 2, $"cond {cond} seed {seed}: {iterations} iterations"); var bb = 0.0; foreach (var v in b) bb += (double)v * v; var expected = NormalEquationSolution(m, n, a, b); var xs = new float[n]; for (var j = 0; j < n; j++) xs[j] = (float)expected[j]; Assert.True(Cost(m, n, a, b, x) - Cost(m, n, a, b, xs) <= 1e-6 * bb, $"cond {cond} seed {seed}: cost above the exact optimum"); } } // Near the pivot cutoff (cond(A) ~ 3e7, cond(A^T A) ~ 1e15) a Newton step leaves a free gradient above the // tolerance on some problems. Those take one confirming iteration and stop because it lowers nothing; the rest // are stationary at once. Never the cap, and never spinning. [Fact] public void NearSingularFreeSetsConfirmOnceAndStop() { const int m = 28, n = 24; var confirmed = 0; for (uint seed = 1; seed <= 60; seed++) { var (a, b) = BlsTrace.ConditionedProblem(m, n, 3e7, seed); var x = new float[n]; var status = Solve(m, n, a, b, Fill(-1e9f, n), Fill(1e9f, n), x, out var iterations); Assert.True(status == BoundedLeastSquaresStatus.Converged, $"seed {seed}: {status}"); Assert.True(iterations <= 3, $"seed {seed}: {iterations} iterations"); if (iterations > 0) confirmed++; } Assert.InRange(confirmed, 1, 20); } // b along the smallest left singular vector makes the optimum lie along the weakest direction: |A^T b| is tiny, so // the tolerance (1e-6 of it) sits below the rounding noise of the gradient (~1e-16 |A^T A| |x|) for cond(A) >= 1e6. // The tolerance is then unreachable, and the solver must stop on the progress rule, after the handful of Newton // passes that still lower the cost, never at the 100-iteration cap. [Theory] [InlineData(1e6)] [InlineData(1e7)] public void ToleranceBelowTheNoiseFloorStopsOnProgressNotOnTheCap(double cond) { const int m = 28, n = 24; var confirmed = 0; for (uint seed = 1; seed <= 30; seed++) { var (a, b) = BlsTrace.ConditionedProblem(m, n, cond, seed, 1.0, bAlongSmallestSingularVector: true); var x = new float[n]; var status = Solve(m, n, a, b, Fill(-1e9f, n), Fill(1e9f, n), x, out var iterations); Assert.True(status == BoundedLeastSquaresStatus.Converged, $"cond {cond} seed {seed}: {status} after {iterations}"); Assert.True(iterations <= 6, $"cond {cond} seed {seed}: {iterations} iterations"); if (iterations >= 1) confirmed++; } Assert.True(confirmed >= 15, $"cond {cond}: only {confirmed} of 30 reached the tolerance floor"); } // Scale-freeness was only claimed for scaling A, b and the bounds together; A alone at 1e6 with the bounds left // alone must also converge, and as fast. [Theory] [InlineData(1e2)] [InlineData(1e4)] public void LargeAWithUnscaledBoundsConvergesInAtMostTwoIterations(double cond) { const int m = 28, n = 24; for (uint seed = 1; seed <= 30; seed++) { var (a, b) = BlsTrace.ConditionedProblem(m, n, cond, seed, 1e6); var x = new float[n]; var status = Solve(m, n, a, b, Fill(-1e3f, n), Fill(1e3f, n), x, out var iterations); Assert.True(status == BoundedLeastSquaresStatus.Converged, $"cond {cond} seed {seed}: {status} after {iterations}"); Assert.True(iterations <= 2, $"cond {cond} seed {seed}: {iterations} iterations"); } } // b = 0 must not borrow its tolerance from the start point: a far start's own gradient is enormous, so that // scale judged it converged after one iteration at cost 5001. The optimum here is ~1e-9. [Fact] public void ZeroTargetFromAFarStartIsNotJudgedConvergedByItsOwnSteepness() { var a = new[] { 1f, 1f, 1f, 1.0001f }; var b = new float[2]; var lo = new[] { 0.5f, -1e6f }; var hi = new[] { 1e6f, 1e6f }; var x = new[] { 1e6f, 1e6f }; var status = Solve(2, 2, a, b, lo, hi, x, out _); Assert.Equal(BoundedLeastSquaresStatus.Converged, status); AssertKkt(2, 2, a, b, lo, hi, x); Assert.True(Cost(2, 2, a, b, x) < 1e-6, $"cost {Cost(2, 2, a, b, x)}"); Assert.True(x[0] >= 0.5f); // The origin inside the box: the answer is the origin, from any start. var lo2 = new[] { -1e6f, -1e6f }; var y = new[] { 1e6f, -1e6f }; Assert.Equal(BoundedLeastSquaresStatus.Converged, Solve(2, 2, a, b, lo2, hi, y, out _)); Assert.True(Cost(2, 2, a, b, y) < 1e-9, $"cost {Cost(2, 2, a, b, y)}"); } // The tolerance scale is the gradient at the box point nearest the origin. Here that point is 1000 and the // wanted release is a gradient of about 6e-5 against a 1000-scale problem: only a scale taken from the box point // (about 6e-5, tolerance 6e-11) releases it; a scale taken from A^T b (about 1000, tolerance 1e-3) would not. // Run with the excluded column in each position so the scale must come from the right entry. [Fact] public void ToleranceScaleIsTheGradientAtTheBoxPointNearestTheOrigin() { var a = new[] { 1f, 0f, 0f, 1f }; foreach (var excluded in new[] { 0, 1 }) { var b = new float[2]; var lo = new[] { -1e6f, -1e6f }; var hi = new[] { 1e6f, 1e6f }; b[excluded] = 1000.0001f; lo[excluded] = 1000f; hi[excluded] = 2000f; var x = new float[2]; x[excluded] = 1000f; var status = Solve(2, 2, a, b, lo, hi, x, out var iterations); Assert.Equal(BoundedLeastSquaresStatus.Converged, status); Assert.True(iterations >= 1, $"column {excluded}: no release"); Assert.Equal(b[excluded], x[excluded]); Assert.Equal(0f, x[1 - excluded]); } } // b = 0 with the origin in the box and the start at the origin: the gradient is exactly zero and so is the // tolerance. Nothing is violated, so the solve is over without an iteration. [Fact] public void ZeroTargetAtTheOriginTakesNoIterations() { const int m = 5, n = 3; var x = new float[n]; var status = Solve(m, n, A54, new float[m], Fill(-1f, n), Fill(1f, n), x, out var iterations); Assert.Equal(BoundedLeastSquaresStatus.Converged, status); Assert.Equal(0, iterations); Assert.Equal(new float[n], x); } // One column, A = [1], with b beyond the box: the first Newton step would cross a bound, so with a cap of 0 the // solver returns the clamped start untouched. That makes the clamp itself the thing under test, for every // combination of a finite, infinite or NaN start with finite and infinite bounds. [Fact] public void WarmStartClampIsExact() { var inf = float.PositiveInfinity; var cases = new (float lo, float hi, float start, float expected)[] { (-1f, 1f, 0.25f, 0.25f), (-1f, 1f, 5f, 1f), (-1f, 1f, -5f, -1f), (-1f, 1f, float.NaN, -1f), (0.5f, 2f, inf, 2f), (0.5f, 2f, -inf, 0.5f), (0.2f, inf, inf, 0.2f), (0.2f, inf, -inf, 0.2f), (0.2f, inf, float.NaN, 0.2f), (-inf, -2f, -inf, -2f), (-inf, -2f, 3f, -2f), (-inf, 3f, float.NaN, 0f), (-inf, 3f, -inf, 0f), (-2f, inf, float.NaN, -2f), (-inf, -2f, float.NaN, -2f), (-inf, 3f, 1f, 1f), }; foreach (var (l, h, start, expected) in cases) { // b beyond the box on the side the start is not on the bound of, so the step crosses a bound (or the // start already sits on one). var b = new[] { float.IsInfinity(h) ? l - 10f : h + 10f }; if (float.IsInfinity(l) && float.IsInfinity(h)) b[0] = 0f; var x = new[] { start }; Solve(1, 1, new[] { 1f }, b, new[] { l }, new[] { h }, x, out var iterations, maxIterations: 0); Assert.Equal(0, iterations); Assert.True(x[0] == expected, $"box [{l}, {h}] start {start}: {x[0]}, expected {expected}"); } } // A duplicate column that is free and interior gets no step: the columns share one gradient, and moving both // along it would overshoot. [Fact] public void FreeDuplicateColumnGetsNoStep() { const int m = 5, n = 3; var a = (float[])A54.Clone(); for (var r = 0; r < m; r++) a[r * n + 2] = a[r * n + 0]; var x = new float[n]; var status = Solve(m, n, a, B5, Fill(-100f, n), Fill(100f, n), x, out var iterations); Assert.Equal(BoundedLeastSquaresStatus.Converged, status); Assert.Equal(0, iterations); Assert.Equal(0f, x[2]); var reduced = new float[m * 2]; for (var r = 0; r < m; r++) { reduced[r * 2] = a[r * n]; reduced[r * 2 + 1] = a[r * n + 1]; } var xr = NormalEquationSolution(m, 2, reduced, B5); Assert.Equal(xr[0], x[0], 1e-4); Assert.Equal(xr[1], x[1], 1e-4); } // Columns 1000x apart in scale and exactly parallel: the pivot tolerance is relative to the column's own // diagonal, so it catches the duplicate at any absolute scale. [Theory] [InlineData(1e3f)] [InlineData(1e-3f)] public void ParallelColumnsAreDependentAtAnyAbsoluteScale(float scale) { const int m = 5, n = 3; var a = (float[])A54.Clone(); for (var r = 0; r < m; r++) a[r * n + 2] = 0.37f * a[r * n + 0]; for (var i = 0; i < a.Length; i++) a[i] *= scale; var lo = Fill(-1000f / scale, n); var hi = Fill(1000f / scale, n); var x = new float[n]; var status = Solve(m, n, a, B5, lo, hi, x, out var iterations); Assert.Equal(BoundedLeastSquaresStatus.Converged, status); Assert.True(iterations <= 2, $"{iterations} iterations"); var reduced = new float[m * 2]; for (var r = 0; r < m; r++) { reduced[r * 2] = a[r * n]; reduced[r * 2 + 1] = a[r * n + 1]; } var xr = NormalEquationSolution(m, 2, reduced, B5); var xs = new float[n]; xs[0] = (float)xr[0]; xs[1] = (float)xr[1]; var bb = 0.0; foreach (var v in B5) bb += (double)v * v; Assert.True(Cost(m, n, a, B5, x) - Cost(m, n, a, B5, xs) <= 1e-6 * bb, "cost above the reduced optimum"); } // No path allocates: not the first InvalidInput, not a cap hit, not a solve that releases bounds. Each path is // measured on its first call and again after, so a one-time runtime cost shows up as a difference between the two // rather than as a solver allocation. [Fact] public void EveryPathAllocatesNothing() { const int m = 4, n = 3; var a = new[] { 1.0f, 0.2f, 0.1f, 0.1f, 1.0f, 0.3f, -0.2f, 0.2f, 1.0f, 0.5f, 0.5f, 0.4f }; var b = new[] { 10f, -10f, 10f, 1f }; var lo = Fill(-1f, n); var hi = Fill(1f, n); var workspace = new float[BoundedLeastSquares.WorkspaceLength(n)]; var x = new float[n]; var bad = (float[])a.Clone(); bad[4] = float.NaN; var paths = new (string name, float[] a, int cap, BoundedLeastSquaresStatus expected)[] { ("invalid", bad, 100, BoundedLeastSquaresStatus.InvalidInput), ("converged", a, 100, BoundedLeastSquaresStatus.Converged), ("limit", a, 1, BoundedLeastSquaresStatus.IterationLimit), }; foreach (var (name, matrix, cap, expected) in paths) { var bytes = new long[2]; for (var call = 0; call < 2; call++) { Array.Clear(x); var before = GC.GetAllocatedBytesForCurrentThread(); var status = BoundedLeastSquares.Solve(m, n, matrix, b, lo, hi, x, workspace, out _, cap); bytes[call] = GC.GetAllocatedBytesForCurrentThread() - before; Assert.Equal(expected, status); } Assert.True(bytes[0] == 0 && bytes[1] == 0, $"{name}: first call {bytes[0]} B, second {bytes[1]} B"); } } // The allocator problem of the Aetheria thrust allocator: four throttles u in [0,1] and four slack pairs // e+x e-x e+y e-y in [0,2]. Rows: x and y translation, yaw, four slack pulls, four throttle regularisers. Duel columns. private static readonly float[] DuelX = { 0f, 0f, 29.08f, -21.80f }; private static readonly float[] DuelY = { 53.94f, 61.19f, 0f, 0f }; private static readonly float[] DuelYaw = { 49.24f, -55.86f, 166.26f, -124.63f }; private static (int m, int n, float[] a, float[] b, float[] lo, float[] hi) DuelProblem(float ix, float iy, float iz) { const int m = 11, n = 8; const float h = 100f, wy = 1000f, w = 0.3f, kappa = 10f, rho = 0.1f; var a = new float[m * n]; var b = new float[m]; var rows = new[] { DuelX, DuelY, DuelYaw }; var intent = new[] { ix, iy, iz }; for (var r = 0; r < 3; r++) { var pos = 0f; var neg = 0f; foreach (var v in rows[r]) { if (v > 0f) pos += v; else neg -= v; } var s = Math.Max(pos, neg); var d = intent[r] * (intent[r] >= 0f ? pos : neg); var weight = r == 2 ? wy : h; for (var k = 0; k < 4; k++) a[r * n + k] = weight * rows[r][k] / s; if (r < 2) { a[r * n + 4 + 2 * r] = -weight; a[r * n + 5 + 2 * r] = weight; } b[r] = weight * d / s; } for (var k = 0; k < 4; k++) { a[(3 + k) * n + 4 + k] = w; b[3 + k] = -w * kappa; a[(7 + k) * n + k] = rho; } var lo = new float[n]; var hi = new float[n]; for (var j = 0; j < n; j++) hi[j] = j < 4 ? 1f : 2f; return (m, n, a, b, lo, hi); } private static string Bits(BoundedLeastSquaresStatus status, int iterations, float[] x) { var sb = new System.Text.StringBuilder().Append(status).Append(' ').Append(iterations); foreach (var v in x) sb.Append(' ').Append(BitConverter.SingleToInt32Bits(v).ToString("x8")); return sb.ToString(); } // The before/after pin of the cut that added a tolerance argument: literals recorded by running the pre-cut solver. [Fact] public void DefaultSolvesAreBitIdenticalToTheirGoldenBits() { List Run(bool passTheDefault) { var cases = new List(); BoundedLeastSquaresStatus Go(int m, int n, float[] a, float[] b, float[] lo, float[] hi, float[] x, out int it) => passTheDefault ? SolveAt(m, n, a, b, lo, hi, x, out it, BoundedLeastSquares.DefaultKktRelativeTolerance) : Solve(m, n, a, b, lo, hi, x, out it); { const int m = 5, n = 3; var x = new float[n]; var status = Go(m, n, A54, B5, Fill(-1f, n), Fill(1f, n), x, out var it); cases.Add(Bits(status, it, x)); } foreach (var seed in new uint[] { 3, 11, 29 }) { var (m, n, a, b, lo, hi) = RandomProblem(seed); var x = new float[n]; var status = Go(m, n, a, b, lo, hi, x, out var it); cases.Add(Bits(status, it, x)); } { var (m, n, a, b, lo, hi) = DuelProblem(0f, 0.5f, 0.25f); var x = new float[n]; var status = Go(m, n, a, b, lo, hi, x, out var it); cases.Add(Bits(status, it, x)); } return cases; } var golden = new[] { "Converged 0 3e8cfe03 bdd096e8 3f65616f", "Converged 7 3f800000 00000000 be800000 bdb5b48d 00000000 3f800000 bf3f85df 3dca36f5 3dd187b0 3dcccccd 3f469416 3ea87652 3f4e1a2a", "Converged 2 be800000 be2481b8 bf1f1f78 3eb37f7c 3ec24a46 bdbe3cf6 3dcccccd", "Converged 3 00000000 3d0d206f 3dcccccd 3f800000 3ef2bfd6 3ebe0bf2 bf600698 bf800000 3f528e24 bddf446c 3c173499 00000000", "Converged 5 3f800000 00000000 3ce45d43 00000000 3ce39ffe 00000000 00000000 3d0098ef", }; Assert.Equal(string.Join("\n", golden), string.Join("\n", Run(false))); Assert.Equal(string.Join("\n", golden), string.Join("\n", Run(true))); } private static BoundedLeastSquaresStatus SolveAt(int m, int n, float[] a, float[] b, float[] lo, float[] hi, float[] x, out int iterations, double kktRelativeTolerance, int maxIterations = BoundedLeastSquares.DefaultMaxIterations) => BoundedLeastSquares.Solve(m, n, a, b, lo, hi, x, new float[BoundedLeastSquares.WorkspaceLength(n)], out iterations, kktRelativeTolerance, maxIterations); private static float[] Start(int n, float throttles) { var x = new float[n]; for (var k = 0; k < 4; k++) x[k] = throttles; return x; } [Fact] public void TheDefaultToleranceConstantIsOneEMinusSix() { Assert.Equal(1e-6, BoundedLeastSquares.DefaultKktRelativeTolerance); for (uint seed = 1; seed <= 50; seed++) { var (m, n, a, b, lo, hi) = RandomProblem(seed); var omitted = new float[n]; var passed = new float[n]; var s1 = Solve(m, n, a, b, lo, hi, omitted, out var i1); var s2 = SolveAt(m, n, a, b, lo, hi, passed, out var i2, BoundedLeastSquares.DefaultKktRelativeTolerance); Assert.Equal(Bits(s1, i1, omitted), Bits(s2, i2, passed)); } } // Row 0 is heavy (gradient scale 1e4); row 1 carries a light term whose gradient w^2 = 9e-6 sits under 1e-6 of // that scale, so the default calls x0 stationary at 0 although cost falls by 90 when x0 rises to 1. private static readonly float[] HeavyA = { 10f, 100f, 0.003f, 0f }; private static readonly float[] HeavyB = { 100f, 0.003f }; [Fact] public void ALightTermUnderAHeavyRowIsResolvedOnlyByATightEnoughTolerance() { const int m = 2, n = 2; var lo = new float[n]; var hi = Fill(1f, n); foreach (var tol in new[] { BoundedLeastSquares.DefaultKktRelativeTolerance, 1e-9 }) { var x = new float[n]; Assert.Equal(BoundedLeastSquaresStatus.Converged, SolveAt(m, n, HeavyA, HeavyB, lo, hi, x, out _, tol)); Assert.Equal(0f, x[0]); } var tight = new float[n]; Assert.Equal(BoundedLeastSquaresStatus.Converged, SolveAt(m, n, HeavyA, HeavyB, lo, hi, tight, out _, 1e-12)); Assert.Equal(1f, tight[0], 1e-5); Assert.Equal(0.9f, tight[1], 1e-5); } [Fact] public void ATightToleranceDoesNotDependOnTheStart() { const int m = 2, n = 2; var lo = new float[n]; var hi = Fill(1f, n); foreach (var (s0, s1) in new[] { (0f, 0f), (1f, 0f), (0.5f, 0.5f), (1f, 1f) }) { var x = new[] { s0, s1 }; Assert.Equal(BoundedLeastSquaresStatus.Converged, SolveAt(m, n, HeavyA, HeavyB, lo, hi, x, out _, 1e-12)); Assert.Equal(1f, x[0], 1e-5); Assert.Equal(0.9f, x[1], 1e-5); } var cold = new float[n]; var warm = new[] { 1f, 0f }; Solve(m, n, HeavyA, HeavyB, lo, hi, cold, out _); Solve(m, n, HeavyA, HeavyB, lo, hi, warm, out _); Assert.Equal(0f, cold[0]); Assert.Equal(1f, warm[0]); } [Theory] [InlineData(0f, 0.5f, 0.25f)] [InlineData(-1f, 0f, 0.25f)] public void AnAllocatorShapedProblemIsStartIndependentAtATightTolerance(float ix, float iy, float iz) { var (m, n, a, b, lo, hi) = DuelProblem(ix, iy, iz); var cold = Start(n, 0f); Assert.Equal(BoundedLeastSquaresStatus.Converged, SolveAt(m, n, a, b, lo, hi, cold, out var coldIterations, 1e-12)); Assert.True(coldIterations <= 20, $"cold {coldIterations} iterations"); foreach (var throttles in new[] { 1f, 0.5f }) { var x = Start(n, throttles); var status = SolveAt(m, n, a, b, lo, hi, x, out var iterations, 1e-12); Assert.Equal(BoundedLeastSquaresStatus.Converged, status); Assert.True(iterations <= 20, $"start {throttles}: {iterations} iterations"); for (var k = 0; k < 4; k++) Assert.Equal(cold[k], x[k], 1e-4); } var defaultCold = Start(n, 0f); var defaultOnes = Start(n, 1f); Solve(m, n, a, b, lo, hi, defaultCold, out _); Solve(m, n, a, b, lo, hi, defaultOnes, out _); var spread = 0.0; for (var k = 0; k < 4; k++) spread = Math.Max(spread, Math.Abs(defaultOnes[k] - defaultCold[k])); Assert.True(spread >= 0.5, $"default tolerance: starts differ by only {spread}"); } // Soul's W5 (finding aetheria-release:finding:cut-thrust-allocator-core.s1.solver-converged-early): an allocator // matrix (3 force rows, 4 slack pulls, 8 ridge rows; 8 throttles in [0,1], 4 slacks in [0,2]). From x = 0 the // last release lowered the cost by less than the stall band while throttle 7 still sat at its upper bound // with a gradient ten thousand times the tolerance, and the solver reported Converged there. [Theory] [InlineData(1e-12)] [InlineData(1e-9)] public void TheStopAfterAReleaseIsOnTheGradientNotTheCost(double tolerance) { var (m, n, a, b, lo, hi) = BlsTrace.AllocatorW5(0); var cold = new float[n]; Assert.Equal(BoundedLeastSquaresStatus.Converged, SolveAt(m, n, a, b, lo, hi, cold, out _, tolerance)); Assert.Equal(0.92221445f, cold[7], 1e-4); var again = (float[])cold.Clone(); Assert.Equal(BoundedLeastSquaresStatus.Converged, SolveAt(m, n, a, b, lo, hi, again, out var iterations, tolerance)); Assert.Equal(0, iterations); for (var j = 0; j < n; j++) Assert.Equal(cold[j], again[j], 1e-6); } // The same problem with its columns rotated by `shift`, so that throttle 7 sits in every column position in turn and the // release the old rule misjudged is a release of column 0, of the last column and of every one between. The stop after a // release reads the gradient of whichever column was released, never a fixed one. [Theory] [InlineData(1e-12)] [InlineData(1e-9)] public void TheStopAfterAReleaseIsOnTheGradientWhicheverColumnWasReleased(double tolerance) { const int n = 12; for (var shift = 0; shift < n; shift++) { var (m, _, a, b, lo, hi) = BlsTrace.AllocatorW5(shift); var cold = new float[n]; Assert.True(BoundedLeastSquaresStatus.Converged == SolveAt(m, n, a, b, lo, hi, cold, out _, tolerance), $"shift {shift}"); Assert.True(Math.Abs(cold[(7 + shift) % n] - 0.92221445f) <= 1e-4, $"shift {shift}: throttle 7 is {cold[(7 + shift) % n]}"); var again = (float[])cold.Clone(); Assert.True(BoundedLeastSquaresStatus.Converged == SolveAt(m, n, a, b, lo, hi, again, out var iterations, tolerance), $"shift {shift}: re-solve"); Assert.True(iterations == 0, $"shift {shift}: re-solve took {iterations} iterations"); } } // b = 0 with the origin inside the box: the optimum is any x with Ax = 0, the gradient scale is 0 and so is the tolerance, and // every gradient component the solve leaves behind is rounding. Releasing on such a violation cycles release / // stationarity miss / re-bind until the iteration cap (the plain-batch problem at seed 141 in miniature). [Fact] public void ANoiseLevelViolationIsNeverReleased() { const int m = 2, n = 6; var a = new[] { 1.0213068f, 0.97530603f, -0.5673449f, -0.64136946f, -0.07432885f, -0.35571396f, 0.4326824f, 1.1969734f, -0.1343108f, -0.23431005f, -1.4009786f, 0.10656489f, }; var b = new float[m]; var lo = new[] { -0.44782114f, 0f, -0.008422324f, -0.03878026f, 0f, -6.6434336f }; var hi = new[] { 0.0448007f, 0f, 0.005942123f, 0.0785512f, 0f, 0.23079813f }; var x = new[] { -0.28347927f, 0f, 0.0051532285f, -0.03878026f, 0f, 0.23079813f }; Assert.Equal(BoundedLeastSquaresStatus.Converged, Solve(m, n, a, b, lo, hi, x, out var iterations)); Assert.True(iterations <= 4, $"{iterations} iterations"); for (var i = 0; i < m; i++) { var r = 0.0; for (var j = 0; j < n; j++) r += a[i * n + j] * x[j]; Assert.Equal(0.0, r, 1e-5); } } [Fact] public void AnInvalidToleranceIsRejectedAndLeavesXUntouched() { const int m = 5, n = 3; var lo = Fill(-1f, n); var hi = Fill(1f, n); foreach (var tol in new[] { 0.0, -1e-6, double.NaN, double.PositiveInfinity, double.NegativeInfinity, double.MinValue }) { var x = new[] { 0.25f, -0.5f, 2f }; var before = (float[])x.Clone(); var status = SolveAt(m, n, A54, B5, lo, hi, x, out var iterations, tol); Assert.True(status == BoundedLeastSquaresStatus.InvalidInput, $"tolerance {tol}: {status}"); Assert.Equal(0, iterations); for (var j = 0; j < n; j++) Assert.Equal(BitConverter.SingleToInt32Bits(before[j]), BitConverter.SingleToInt32Bits(x[j])); } var tiny = new float[n]; Assert.Equal(BoundedLeastSquaresStatus.Converged, SolveAt(m, n, A54, B5, lo, hi, tiny, out _, double.Epsilon)); } [Theory] [InlineData(1e6, 1e-12)] [InlineData(1e6, 1e-30)] [InlineData(1e7, 1e-12)] [InlineData(1e7, 1e-30)] public void ATightToleranceStopsOnProgressNotOnTheCap(double cond, double tol) { const int m = 28, n = 24; var worst = 0; for (uint seed = 1; seed <= 30; seed++) { var (a, b) = BlsTrace.ConditionedProblem(m, n, cond, seed, 1.0, bAlongSmallestSingularVector: true); var x = new float[n]; var status = SolveAt(m, n, a, b, Fill(-1e9f, n), Fill(1e9f, n), x, out var iterations, tol); Assert.True(status == BoundedLeastSquaresStatus.Converged, $"cond {cond} tol {tol} seed {seed}: {status} after {iterations}"); worst = Math.Max(worst, iterations); } Assert.True(worst <= 30, $"cond {cond} tol {tol}: {worst} iterations"); } [Fact] public void TheTolerancePathAllocatesNothing() { const int m = 4, n = 3; var a = new[] { 1.0f, 0.2f, 0.1f, 0.1f, 1.0f, 0.3f, -0.2f, 0.2f, 1.0f, 0.5f, 0.5f, 0.4f }; var b = new[] { 10f, -10f, 10f, 1f }; var lo = Fill(-1f, n); var hi = Fill(1f, n); var workspace = new float[BoundedLeastSquares.WorkspaceLength(n)]; var x = new float[n]; var paths = new (string name, double tol, BoundedLeastSquaresStatus expected)[] { ("tight", 1e-12, BoundedLeastSquaresStatus.Converged), ("invalid", double.NaN, BoundedLeastSquaresStatus.InvalidInput), }; foreach (var (name, tol, expected) in paths) { var bytes = new long[2]; for (var call = 0; call < 2; call++) { Array.Clear(x); var before = GC.GetAllocatedBytesForCurrentThread(); var status = BoundedLeastSquares.Solve(m, n, a, b, lo, hi, x, workspace, out _, tol); bytes[call] = GC.GetAllocatedBytesForCurrentThread() - before; Assert.Equal(expected, status); } Assert.True(bytes[0] == 0 && bytes[1] == 0, $"{name}: first call {bytes[0]} B, second {bytes[1]} B"); } } [Fact] public void TheOriginalSolveSignatureStillExists() { var original = typeof(BoundedLeastSquares).GetMethod(nameof(BoundedLeastSquares.Solve), new[] { typeof(int), typeof(int), typeof(ReadOnlySpan), typeof(ReadOnlySpan), typeof(ReadOnlySpan), typeof(ReadOnlySpan), typeof(Span), typeof(Span), typeof(int).MakeByRefType(), typeof(int), }); Assert.NotNull(original); } // The heavy-row shape with a free light weight w: at (x0, x1) = (0, 1) the gradient of x0 is w^2 against a // gradient scale of 1e4, so x0's release flips where the caller's tolerance crosses the relative gradient // computed here from the inputs alone. private static float[] ShapeA(float w) => new[] { 10f, 100f, w, 0f }; private static float[] ShapeB(float w) => new[] { 100f, w }; private static double ShapeRelativeGradient(float w) { var a = ShapeA(w); var b = ShapeB(w); double atb0 = (double)a[0] * b[0] + (double)a[2] * b[1]; double atb1 = (double)a[1] * b[0] + (double)a[3] * b[1]; var gScale = Math.Max(Math.Abs(atb0), Math.Abs(atb1)); double ata01 = (double)a[0] * a[1] + (double)a[2] * a[3]; return Math.Abs(ata01 - atb0) / gScale; } private static float ShapeX0(float w, double? tolerance) { var x = new float[2]; var lo = new float[2]; var hi = Fill(1f, 2); var status = tolerance is { } t ? SolveAt(2, 2, ShapeA(w), ShapeB(w), lo, hi, x, out _, t) : Solve(2, 2, ShapeA(w), ShapeB(w), lo, hi, x, out _); Assert.Equal(BoundedLeastSquaresStatus.Converged, status); return x[0]; } // Weights put the relative gradient at about 1e-5 (above the default tolerance) and 9e-10, 9e-12, 9e-14 (below it). [Theory] [InlineData(0.316f)] [InlineData(0.03f)] [InlineData(0.003f)] [InlineData(0.0003f)] public void TheReleaseFlipsAtTheCallersToleranceTimesTheGradientScale(float w) { var rel = ShapeRelativeGradient(w); Assert.True(ShapeX0(w, rel * 1.05) == 0f, $"w {w} rel {rel:R}: above the threshold x0 stays at 0"); var below = ShapeX0(w, rel * 0.95); Assert.True(Math.Abs(below - 1f) < 1e-4, $"w {w} rel {rel:R}: below the threshold x0 is {below}"); } // A relative gradient 10% either side of 1e-6: the solve that omits the tolerance releases the one above and // not the one below, and equals the solve that passes the default, so the forwarded default is 1e-6 and no // other value within 10%. [Theory] [InlineData(0.104881f, 1f)] [InlineData(0.094868f, 0f)] public void TheOmittedToleranceReleasesAtOneEMinusSixAndNowhereNear(float w, float expectedX0) { var rel = ShapeRelativeGradient(w); Assert.InRange(rel, 0.9e-6 * 0.99, 1.1e-6 * 1.01); Assert.Equal(expectedX0, ShapeX0(w, null), 1e-4); Assert.Equal(expectedX0, ShapeX0(w, BoundedLeastSquares.DefaultKktRelativeTolerance), 1e-4); } // The box is wide enough that no bound is ever released, so any difference between the default and a tight // tolerance is decided by the stationarity test of the free set alone. [Fact] public void AllFreeConditionedProblemsHonourTheToleranceAtTheFreeSetTest() { const int m = 28, n = 24; int differ = 0, total = 0; foreach (var cond in new[] { 1e2, 1e3, 1e4, 1e5, 1e6 }) foreach (var along in new[] { true, false }) for (uint seed = 1; seed <= 30; seed++) { var (a, b) = BlsTrace.ConditionedProblem(m, n, cond, seed, 1.0, bAlongSmallestSingularVector: along); var byDefault = new float[n]; var tight = new float[n]; var s1 = Solve(m, n, a, b, Fill(-1e9f, n), Fill(1e9f, n), byDefault, out var i1); var s2 = SolveAt(m, n, a, b, Fill(-1e9f, n), Fill(1e9f, n), tight, out var i2, 1e-12); total++; if (Bits(s1, i1, byDefault) != Bits(s2, i2, tight)) differ++; } Assert.True(differ >= total / 10, $"only {differ} of {total} all-free solves changed between the default and 1e-12"); } }