StoreReplace: unresolved kernel32 imports degrade, claims say rare not never, killing tests

ReplaceWindows owns the unresolved-import decision: a POSIX call that does not
resolve (DllNotFound/EntryPointNotFound) is "unsupported"; a MoveFileExW that
does not resolve degrades to the portable File.Replace/Move. Tests for each
exception type, the last-attempt same-attempt fallback and the retried-87 case.
Doc and Mesh test comments no longer claim a reader never finds the path absent;
the Mesh race test tolerates a brief not-found on Windows.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

src/GameCult.Caching/StoreReplace.cs

@@ -19,8 +19,10 @@ namespace GameCult.Caching
 
     /// <summary>
     /// Owns how a staged file becomes the store, for the single-file and directory stores. Windows tries a POSIX-semantics
-    /// rename first (a lock-free reader never sees the path empty), falls back to write-through MoveFileExW where that is
-    /// unsupported, and retries transient sharing errors; other OSes keep rename. Mirrors cultcache-ts store-file.ts
+    /// rename first (which makes a moment with the path absent rare for a lock-free reader, not impossible: a stalled rename was
+    /// measured leaving it absent for 5-47 ms), falls back to write-through MoveFileExW where that is unsupported or does not
+    /// resolve, falls back to the portable File.Replace if MoveFileExW itself does not resolve, and retries transient sharing
+    /// errors; other OSes keep rename. Mirrors cultcache-ts store-file.ts
     /// replaceStoreFile (ruling variants:ruling:windows-replace-posix-first).
     /// </summary>
     internal static class StoreReplace
@@ -44,7 +46,15 @@ namespace GameCult.Caching
             {
                 ReplaceWindows(staged, destination, WindowsKernel);
             }
-            else if (File.Exists(destination))
+            else
+            {
+                ReplacePortable(staged, destination);
+            }
+        }
+
+        private static void ReplacePortable(string staged, string destination)
+        {
+            if (File.Exists(destination))
             {
                 File.Replace(staged, destination, null);
             }
@@ -54,20 +64,30 @@ namespace GameCult.Caching
             }
         }
 
+        private static bool Unresolved(Exception ex) => ex is DllNotFoundException || ex is EntryPointNotFoundException;
+
         internal static void ReplaceWindows(string staged, string destination, IReplaceKernel kernel)
         {
             var posix = true;
             var lastCode = 0;
             for (var attempt = 1; attempt <= ReplaceAttempts; attempt++)
             {
-                var code = posix
-                    ? kernel.RenamePosix(staged, destination)
-                    : kernel.MoveFileEx(staged, destination, MoveFileReplaceWriteThrough);
-                if (posix && (code == 1 || code == ErrorNotSupported || code == ErrorInvalidParameter))
+                int code;
+                try
                 {
-                    // POSIX semantics are unsupported here (FAT, exFAT, before Windows 10 1607): the rest of this call is write-through.
-                    posix = false;
-                    code = kernel.MoveFileEx(staged, destination, MoveFileReplaceWriteThrough);
+                    code = posix ? RenamePosixOrUnsupported(kernel, staged, destination) : kernel.MoveFileEx(staged, destination, MoveFileReplaceWriteThrough);
+                    if (posix && (code == 1 || code == ErrorNotSupported || code == ErrorInvalidParameter))
+                    {
+                        // POSIX semantics are unsupported here (FAT, exFAT, before Windows 10 1607): the rest of this call is write-through.
+                        posix = false;
+                        code = kernel.MoveFileEx(staged, destination, MoveFileReplaceWriteThrough);
+                    }
+                }
+                catch (Exception ex) when (Unresolved(ex))
+                {
+                    // kernel32's MoveFileExW does not resolve here (a sandboxed player): the runtime-owned replace is all that is left.
+                    ReplacePortable(staged, destination);
+                    return;
                 }
 
                 if (code == 0)
@@ -87,6 +107,19 @@ namespace GameCult.Caching
             throw new IOException($"CultCache could not replace the store after {ReplaceAttempts} attempts: last Win32 error {lastCode}.");
         }
 
+        private static int RenamePosixOrUnsupported(IReplaceKernel kernel, string staged, string destination)
+        {
+            try
+            {
+                return kernel.RenamePosix(staged, destination);
+            }
+            catch (Exception ex) when (Unresolved(ex))
+            {
+                // The POSIX call does not resolve: the same answer as "unsupported", so the replace falls back to MoveFileExW.
+                return ErrorNotSupported;
+            }
+        }
+
         private sealed class Kernel32Replace : IReplaceKernel
         {
             private const uint DeleteSynchronize = 0x00110000;
@@ -98,39 +131,31 @@ namespace GameCult.Caching
 
             public int RenamePosix(string staged, string destination)
             {
+                var handle = CreateFileW(staged, DeleteSynchronize, ShareAll, IntPtr.Zero, OpenExisting, FileAttributeNormal, IntPtr.Zero);
+                if (handle == new IntPtr(-1))
+                {
+                    return Marshal.GetLastWin32Error();
+                }
+
                 try
                 {
-                    var handle = CreateFileW(staged, DeleteSynchronize, ShareAll, IntPtr.Zero, OpenExisting, FileAttributeNormal, IntPtr.Zero);
-                    if (handle == new IntPtr(-1))
-                    {
-                        return Marshal.GetLastWin32Error();
-                    }
-
+                    // FILE_RENAME_INFO: Flags u32 at 0, RootDirectory handle, FileNameLength u32, FileName (UTF-16, offsets for this pointer size).
+                    var name = System.Text.Encoding.Unicode.GetBytes(Path.GetFullPath(destination));
+                    var lengthOffset = 2 * IntPtr.Size;
+                    var nameOffset = lengthOffset + 4;
+                    var size = nameOffset + name.Length + 2;
+                    var info = Marshal.AllocHGlobal(size);
                     try
                     {
-                        // FILE_RENAME_INFO: Flags u32 at 0, RootDirectory handle, FileNameLength u32, FileName (UTF-16, offsets for this pointer size).
-                        var name = System.Text.Encoding.Unicode.GetBytes(Path.GetFullPath(destination));
-                        var lengthOffset = 2 * IntPtr.Size;
-                        var nameOffset = lengthOffset + 4;
-                        var size = nameOffset + name.Length + 2;
-                        var info = Marshal.AllocHGlobal(size);
-                        try
-                        {
-                            for (var i = 0; i < size; i++) Marshal.WriteByte(info, i, 0);
-                            Marshal.WriteInt32(info, 0, (int)RenameReplacePosix);
-                            Marshal.WriteInt32(info, lengthOffset, name.Length);
-                            Marshal.Copy(name, 0, info + nameOffset, name.Length);
-                            return SetFileInformationByHandle(handle, FileRenameInfoEx, info, (uint)size) ? 0 : Marshal.GetLastWin32Error();
-                        }
-                        finally { Marshal.FreeHGlobal(info); }
+                        for (var i = 0; i < size; i++) Marshal.WriteByte(info, i, 0);
+                        Marshal.WriteInt32(info, 0, (int)RenameReplacePosix);
+                        Marshal.WriteInt32(info, lengthOffset, name.Length);
+                        Marshal.Copy(name, 0, info + nameOffset, name.Length);
+                        return SetFileInformationByHandle(handle, FileRenameInfoEx, info, (uint)size) ? 0 : Marshal.GetLastWin32Error();
                     }
-                    finally { CloseHandle(handle); }
-                }
-                catch (Exception ex) when (ex is DllNotFoundException || ex is EntryPointNotFoundException)
-                {
-                    // The call does not resolve here: report "unsupported" so the replace falls back to MoveFileExW.
-                    return ErrorNotSupported;
+                    finally { Marshal.FreeHGlobal(info); }
                 }
+                finally { CloseHandle(handle); }
             }
 
             public int MoveFileEx(string staged, string destination, uint flags)

tests/GameCult.Caching.Tests/StoreReplaceTests.cs

@@ -16,6 +16,8 @@ namespace GameCult.Caching.Tests
             private readonly Queue<int> _moveEx;
             public readonly List<string> Log = new List<string>();
             public readonly List<int> Sleeps = new List<int>();
+            public Exception? PosixThrows;
+            public Exception? MoveExThrows;
 
             public FakeKernel(int[] posix, int[] moveEx)
             {
@@ -26,12 +28,14 @@ namespace GameCult.Caching.Tests
             public int RenamePosix(string staged, string destination)
             {
                 Log.Add("posix");
+                if (PosixThrows != null) throw PosixThrows;
                 return _posix.Count > 0 ? _posix.Dequeue() : 0;
             }
 
             public int MoveFileEx(string staged, string destination, uint flags)
             {
                 Log.Add("moveex:" + flags);
+                if (MoveExThrows != null) throw MoveExThrows;
                 return _moveEx.Count > 0 ? _moveEx.Dequeue() : 0;
             }
 
@@ -91,6 +95,67 @@ namespace GameCult.Caching.Tests
             Assert.That(kernel.Log, Is.EqualTo(new[] { "posix", "moveex:9" }));
         }
 
+        private static IEnumerable<Exception> Unresolved()
+        {
+            yield return new DllNotFoundException("kernel32.dll");
+            yield return new EntryPointNotFoundException("SetFileInformationByHandle");
+        }
+
+        [TestCaseSource(nameof(Unresolved))]
+        public void APosixCallThatDoesNotResolveTakesTheUnsupportedFallback(Exception unresolved)
+        {
+            var kernel = new FakeKernel(new int[0], new[] { 0 }) { PosixThrows = unresolved };
+            StoreReplace.ReplaceWindows("a", "b", kernel);
+            Assert.That(kernel.Log, Is.EqualTo(new[] { "posix", "moveex:9" }));
+        }
+
+        [TestCaseSource(nameof(Unresolved))]
+        public void AMoveFileExThatDoesNotResolveDegradesToThePortableReplace(Exception unresolved)
+        {
+            var dir = Path.Combine(TestContext.CurrentContext.WorkDirectory, "storereplace-" + Guid.NewGuid().ToString("N"));
+            Directory.CreateDirectory(dir);
+            try
+            {
+                var destination = Path.Combine(dir, "store.cc");
+                var staged = Path.Combine(dir, "staged");
+                File.WriteAllBytes(destination, new byte[] { 1 });
+                File.WriteAllBytes(staged, new byte[] { 2 });
+                var kernel = new FakeKernel(new[] { 87 }, new int[0]) { MoveExThrows = unresolved };
+                StoreReplace.ReplaceWindows(staged, destination, kernel);
+                Assert.That(kernel.Log, Is.EqualTo(new[] { "posix", "moveex:9" }));
+                Assert.That(File.ReadAllBytes(destination), Is.EqualTo(new byte[] { 2 }));
+                Assert.That(File.Exists(staged), Is.False);
+
+                // Both calls unresolved: the same portable replace, here onto an absent destination.
+                File.Delete(destination);
+                File.WriteAllBytes(staged, new byte[] { 3 });
+                var both = new FakeKernel(new int[0], new int[0]) { PosixThrows = unresolved, MoveExThrows = unresolved };
+                StoreReplace.ReplaceWindows(staged, destination, both);
+                Assert.That(File.ReadAllBytes(destination), Is.EqualTo(new byte[] { 3 }));
+            }
+            finally { Directory.Delete(dir, true); }
+        }
+
+        [Test]
+        public void TheFallbackRunsInTheSameAttemptEvenOnTheLastOne()
+        {
+            var posix = new int[25].Fill(32);
+            posix[24] = 87;
+            var kernel = new FakeKernel(posix, new[] { 0 });
+            StoreReplace.ReplaceWindows("a", "b", kernel);
+            Assert.That(kernel.Log.Count, Is.EqualTo(26));
+            Assert.That(kernel.Log[24], Is.EqualTo("posix"));
+            Assert.That(kernel.Log[25], Is.EqualTo("moveex:9"));
+        }
+
+        [Test]
+        public void ARetriedMoveFileExThatAnswersUnsupportedFailsAfterOneCallNotTwo()
+        {
+            var kernel = new FakeKernel(new[] { 87 }, new[] { 32, 87 });
+            Assert.That(Fails(kernel).Message, Does.Contain("Win32 error 87"));
+            Assert.That(kernel.Log, Is.EqualTo(new[] { "posix", "moveex:9", "moveex:9" }));
+        }
+
         [Test]
         public void ARetryThatNeverSucceedsGivesUpAfterTwentyFiveAttemptsNamingTheCode()
         {

tests/GameCult.Mesh.Tests/CultMeshSingleFileStoreTests.cs

@@ -4,6 +4,7 @@ using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
+using System.Runtime.InteropServices;
 using System.Threading;
 using System.Threading.Tasks;
 using FluentAssertions;
@@ -364,7 +365,11 @@ public sealed class CultMeshSingleFileStoreTests
     [Test]
     public void ACacheOpeningDuringMeshWritesNeverSeesAnEmptyStore()
     {
-        // Windows replaces with a POSIX-semantics rename (StoreReplace), so a lock-free reader never finds the path empty.
+        // Windows replaces with a POSIX-semantics rename first (StoreReplace), which makes a lock-free reader finding the path
+        // absent rare, not impossible: a stalled rename was measured leaving it absent for 5-47 ms on Windows. So on Windows a
+        // brief not-found (a read throwing, or a cache opening onto an absent path) is tolerated and reported; elsewhere rename
+        // is atomic and none is allowed.
+        var windows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
 
         var path = Path.Combine(_root, "race.cc");
         CultMesh.WriteSingleFileDocument(path, Key, Publication());
@@ -382,7 +387,7 @@ public sealed class CultMeshSingleFileStoreTests
             Volatile.Write(ref stop, 1);
         });
 
-        int opens = 0, empty = 0, reads = 0;
+        int opens = 0, empty = 0, reads = 0, absent = 0;
         var clock = Stopwatch.StartNew();
         while (Volatile.Read(ref stop) == 0 && clock.Elapsed < TimeSpan.FromSeconds(60))
         {
@@ -393,6 +398,10 @@ public sealed class CultMeshSingleFileStoreTests
                 CultMesh.ReadSingleFileDocumentPayload(path, Key, SchemaId());
                 reads++;
             }
+            catch (Exception ex) when (windows && (ex is FileNotFoundException || ex is DirectoryNotFoundException))
+            {
+                absent++;
+            }
             catch (Exception ex)
             {
                 errors.Enqueue("read: " + ex.GetType().Name + ": " + ex.Message);
@@ -400,9 +409,10 @@ public sealed class CultMeshSingleFileStoreTests
         }
 
         writer.Wait(TimeSpan.FromSeconds(60)).Should().BeTrue();
-        TestContext.Out.WriteLine($"mesh writes={writes} cache opens={opens} empty stores={empty} mesh reads ok={reads} errors={errors.Count}");
+        TestContext.Out.WriteLine($"mesh writes={writes} cache opens={opens} empty stores={empty} not-found reads={absent} mesh reads ok={reads} errors={errors.Count}");
         errors.Take(3).Should().BeEmpty();
-        empty.Should().Be(0, "a cache opened in the window where nothing was at the path");
+        if (!windows)
+            empty.Should().Be(0, "a cache opened in the window where nothing was at the path");
         opens.Should().BeGreaterThan(0);
     }