C#: store replace on Windows tries POSIX-semantics rename first (store-windows-cs) #152

Merged
libby merged 4 commits from store-replace/windows-cs into main 2026-11-13 00:38:08 +00:00
5 changed files with 466 additions and 46 deletions

View file

@ -681,14 +681,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
stream.Flush(true);
}
if (File.Exists(path))
{
ReplaceExistingFile(tempPath, path);
}
else
{
File.Move(tempPath, path);
}
StoreReplace.Replace(tempPath, path);
}
finally
{
@ -746,26 +739,4 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
}
}
}
private static void ReplaceExistingFile(string sourcePath, string destinationPath)
{
for (var attempt = 0; attempt < 3; attempt++)
{
try
{
File.Replace(sourcePath, destinationPath, null);
return;
}
catch (IOException) when (attempt < 2)
{
Thread.Sleep(10 * (attempt + 1));
}
catch (UnauthorizedAccessException) when (attempt < 2)
{
Thread.Sleep(10 * (attempt + 1));
}
}
File.Replace(sourcePath, destinationPath, null);
}
}

View file

@ -5204,14 +5204,7 @@ namespace GameCult.Caching
stream.Flush(true);
}
if (File.Exists(path))
{
File.Replace(tempPath, path, null);
}
else
{
File.Move(tempPath, path);
}
StoreReplace.Replace(tempPath, path);
}
finally
{

View file

@ -0,0 +1,182 @@
using System;
using System.IO;
using System.Runtime.InteropServices;
using System.Threading;
namespace GameCult.Caching
{
/// <summary>The kernel32 calls a Windows replace needs, each answering 0 or a Win32 error code.</summary>
internal interface IReplaceKernel
{
/// <summary>CreateFileW(staged, DELETE|SYNCHRONIZE), FileRenameInfoEx with REPLACE_IF_EXISTS|POSIX_SEMANTICS, CloseHandle.</summary>
int RenamePosix(string staged, string destination);
/// <summary>MoveFileExW(staged, destination, flags).</summary>
int MoveFileEx(string staged, string destination, uint flags);
void Sleep(int milliseconds);
}
/// <summary>
/// Owns how a staged file becomes the store, for the single-file and directory stores. Windows tries a POSIX-semantics
/// rename first (which makes a moment with the path absent rare for a lock-free reader, not impossible: a stalled rename was
/// measured leaving it absent for 5-47 ms), falls back to write-through MoveFileExW where that is unsupported or does not
/// resolve, falls back to the portable File.Replace if MoveFileExW itself does not resolve, and retries transient sharing
/// errors; other OSes keep rename. Mirrors cultcache-ts store-file.ts
/// replaceStoreFile (ruling variants:ruling:windows-replace-posix-first).
/// </summary>
internal static class StoreReplace
{
private const int ErrorAccessDenied = 5;
private const int ErrorSharingViolation = 32;
private const int ErrorLockViolation = 33;
private const int ErrorNotSupported = 50;
private const int ErrorInvalidParameter = 87;
private const int ReplaceAttempts = 25;
private const int ReplaceRetryMs = 20;
/// <summary>MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH.</summary>
internal const uint MoveFileReplaceWriteThrough = 0x9;
internal static readonly IReplaceKernel WindowsKernel = new Kernel32Replace();
public static void Replace(string staged, string destination)
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
ReplaceWindows(staged, destination, WindowsKernel);
}
else
{
ReplacePortable(staged, destination);
}
}
private static void ReplacePortable(string staged, string destination)
{
if (File.Exists(destination))
{
File.Replace(staged, destination, null);
}
else
{
File.Move(staged, destination);
}
}
private static bool Unresolved(Exception ex) => ex is DllNotFoundException || ex is EntryPointNotFoundException;
internal static void ReplaceWindows(string staged, string destination, IReplaceKernel kernel)
{
var posix = true;
var lastCode = 0;
for (var attempt = 1; attempt <= ReplaceAttempts; attempt++)
{
int code;
try
{
code = posix ? RenamePosixOrUnsupported(kernel, staged, destination) : kernel.MoveFileEx(staged, destination, MoveFileReplaceWriteThrough);
if (posix && (code == 1 || code == ErrorNotSupported || code == ErrorInvalidParameter))
{
// POSIX semantics are unsupported here (FAT, exFAT, before Windows 10 1607): the rest of this call is write-through.
posix = false;
code = kernel.MoveFileEx(staged, destination, MoveFileReplaceWriteThrough);
}
}
catch (Exception ex) when (Unresolved(ex))
{
// kernel32's MoveFileExW does not resolve here (a sandboxed player): the runtime-owned replace is all that is left.
ReplacePortable(staged, destination);
return;
}
if (code == 0)
{
return;
}
if (code != ErrorAccessDenied && code != ErrorSharingViolation && code != ErrorLockViolation)
{
throw new IOException($"CultCache could not replace the store: Win32 error {code}.");
}
lastCode = code;
kernel.Sleep(ReplaceRetryMs);
}
throw new IOException($"CultCache could not replace the store after {ReplaceAttempts} attempts: last Win32 error {lastCode}.");
}
private static int RenamePosixOrUnsupported(IReplaceKernel kernel, string staged, string destination)
{
try
{
return kernel.RenamePosix(staged, destination);
}
catch (Exception ex) when (Unresolved(ex))
{
// The POSIX call does not resolve: the same answer as "unsupported", so the replace falls back to MoveFileExW.
return ErrorNotSupported;
}
}
private sealed class Kernel32Replace : IReplaceKernel
{
private const uint DeleteSynchronize = 0x00110000;
private const uint ShareAll = 0x7;
private const uint OpenExisting = 3;
private const uint FileAttributeNormal = 0x80;
private const int FileRenameInfoEx = 22;
private const uint RenameReplacePosix = 0x3;
public int RenamePosix(string staged, string destination)
{
var handle = CreateFileW(staged, DeleteSynchronize, ShareAll, IntPtr.Zero, OpenExisting, FileAttributeNormal, IntPtr.Zero);
if (handle == new IntPtr(-1))
{
return Marshal.GetLastWin32Error();
}
try
{
// FILE_RENAME_INFO: Flags u32 at 0, RootDirectory handle, FileNameLength u32, FileName (UTF-16, offsets for this pointer size).
var name = System.Text.Encoding.Unicode.GetBytes(Path.GetFullPath(destination));
var lengthOffset = 2 * IntPtr.Size;
var nameOffset = lengthOffset + 4;
var size = nameOffset + name.Length + 2;
var info = Marshal.AllocHGlobal(size);
try
{
for (var i = 0; i < size; i++) Marshal.WriteByte(info, i, 0);
Marshal.WriteInt32(info, 0, (int)RenameReplacePosix);
Marshal.WriteInt32(info, lengthOffset, name.Length);
Marshal.Copy(name, 0, info + nameOffset, name.Length);
return SetFileInformationByHandle(handle, FileRenameInfoEx, info, (uint)size) ? 0 : Marshal.GetLastWin32Error();
}
finally { Marshal.FreeHGlobal(info); }
}
finally { CloseHandle(handle); }
}
public int MoveFileEx(string staged, string destination, uint flags)
=> MoveFileExW(staged, destination, flags) ? 0 : Marshal.GetLastWin32Error();
public void Sleep(int milliseconds) => Thread.Sleep(milliseconds);
[DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)]
private static extern IntPtr CreateFileW(string fileName, uint access, uint share, IntPtr security, uint disposition, uint flags, IntPtr template);
[DllImport("kernel32.dll", SetLastError = true)]
[return: MarshalAs(UnmanagedType.Bool)]
private static extern bool SetFileInformationByHandle(IntPtr handle, int informationClass, IntPtr information, uint size);
[DllImport("kernel32.dll", SetLastError = true)]
[return: MarshalAs(UnmanagedType.Bool)]
private static extern bool CloseHandle(IntPtr handle);
[DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)]
[return: MarshalAs(UnmanagedType.Bool)]
private static extern bool MoveFileExW(string existing, string target, uint flags);
}
}
}

View file

@ -0,0 +1,268 @@
#nullable enable
using System;
using System.Collections.Generic;
using System.IO;
using System.Runtime.InteropServices;
using System.Threading;
using NUnit.Framework;
namespace GameCult.Caching.Tests
{
public class StoreReplaceTests
{
private sealed class FakeKernel : IReplaceKernel
{
private readonly Queue<int> _posix;
private readonly Queue<int> _moveEx;
public readonly List<string> Log = new List<string>();
public readonly List<int> Sleeps = new List<int>();
public Exception? PosixThrows;
public Exception? MoveExThrows;
public FakeKernel(int[] posix, int[] moveEx)
{
_posix = new Queue<int>(posix);
_moveEx = new Queue<int>(moveEx);
}
public int RenamePosix(string staged, string destination)
{
Log.Add("posix");
if (PosixThrows != null) throw PosixThrows;
return _posix.Count > 0 ? _posix.Dequeue() : 0;
}
public int MoveFileEx(string staged, string destination, uint flags)
{
Log.Add("moveex:" + flags);
if (MoveExThrows != null) throw MoveExThrows;
return _moveEx.Count > 0 ? _moveEx.Dequeue() : 0;
}
public void Sleep(int milliseconds) => Sleeps.Add(milliseconds);
}
private static IOException Fails(FakeKernel kernel)
=> Assert.Throws<IOException>(() => StoreReplace.ReplaceWindows("a", "b", kernel))!;
[Test]
public void APosixRenameThatWorksIsTheWholeReplace()
{
var kernel = new FakeKernel(new[] { 0 }, new int[0]);
StoreReplace.ReplaceWindows("a", "b", kernel);
Assert.That(kernel.Log, Is.EqualTo(new[] { "posix" }));
Assert.That(kernel.Sleeps, Is.Empty);
}
[TestCase(1)]
[TestCase(50)]
[TestCase(87)]
public void WhereAPosixRenameIsUnsupportedTheReplaceFallsBackToWriteThroughMoveFileEx(int unsupported)
{
var kernel = new FakeKernel(new[] { unsupported }, new[] { 0 });
StoreReplace.ReplaceWindows("a", "b", kernel);
Assert.That(kernel.Log, Is.EqualTo(new[] { "posix", "moveex:9" }));
}
[Test]
public void OnceThePosixRenameIsUnsupportedTheRestOfTheCallStaysOnMoveFileEx()
{
var kernel = new FakeKernel(new[] { 87 }, new[] { 32, 0 });
StoreReplace.ReplaceWindows("a", "b", kernel);
Assert.That(kernel.Log, Is.EqualTo(new[] { "posix", "moveex:9", "moveex:9" }));
Assert.That(kernel.Sleeps, Is.EqualTo(new[] { 20 }));
}
[TestCase(5)]
[TestCase(32)]
[TestCase(33)]
public void ATransientErrorRetriesTheSamePrimitiveAfterTwentyMilliseconds(int transient)
{
var kernel = new FakeKernel(new[] { transient, transient, 0 }, new int[0]);
StoreReplace.ReplaceWindows("a", "b", kernel);
Assert.That(kernel.Log, Is.EqualTo(new[] { "posix", "posix", "posix" }));
Assert.That(kernel.Sleeps, Is.EqualTo(new[] { 20, 20 }));
}
[TestCase(1)]
[TestCase(50)]
[TestCase(87)]
public void AnUnsupportedCodeAfterTheFallbackIsAnErrorNotAnotherFallback(int unsupported)
{
var kernel = new FakeKernel(new[] { 87 }, new[] { unsupported });
var error = Fails(kernel);
Assert.That(error.Message, Does.Contain("Win32 error " + unsupported));
Assert.That(kernel.Log, Is.EqualTo(new[] { "posix", "moveex:9" }));
}
private static IEnumerable<Exception> Unresolved()
{
yield return new DllNotFoundException("kernel32.dll");
yield return new EntryPointNotFoundException("SetFileInformationByHandle");
}
[TestCaseSource(nameof(Unresolved))]
public void APosixCallThatDoesNotResolveTakesTheUnsupportedFallback(Exception unresolved)
{
var kernel = new FakeKernel(new int[0], new[] { 0 }) { PosixThrows = unresolved };
StoreReplace.ReplaceWindows("a", "b", kernel);
Assert.That(kernel.Log, Is.EqualTo(new[] { "posix", "moveex:9" }));
}
[TestCaseSource(nameof(Unresolved))]
public void AMoveFileExThatDoesNotResolveDegradesToThePortableReplace(Exception unresolved)
{
var dir = Path.Combine(TestContext.CurrentContext.WorkDirectory, "storereplace-" + Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(dir);
try
{
var destination = Path.Combine(dir, "store.cc");
var staged = Path.Combine(dir, "staged");
File.WriteAllBytes(destination, new byte[] { 1 });
File.WriteAllBytes(staged, new byte[] { 2 });
var kernel = new FakeKernel(new[] { 87 }, new int[0]) { MoveExThrows = unresolved };
StoreReplace.ReplaceWindows(staged, destination, kernel);
Assert.That(kernel.Log, Is.EqualTo(new[] { "posix", "moveex:9" }));
Assert.That(File.ReadAllBytes(destination), Is.EqualTo(new byte[] { 2 }));
Assert.That(File.Exists(staged), Is.False);
// Both calls unresolved: the same portable replace, here onto an absent destination.
File.Delete(destination);
File.WriteAllBytes(staged, new byte[] { 3 });
var both = new FakeKernel(new int[0], new int[0]) { PosixThrows = unresolved, MoveExThrows = unresolved };
StoreReplace.ReplaceWindows(staged, destination, both);
Assert.That(File.ReadAllBytes(destination), Is.EqualTo(new byte[] { 3 }));
}
finally { Directory.Delete(dir, true); }
}
[Test]
public void TheFallbackRunsInTheSameAttemptEvenOnTheLastOne()
{
var posix = new int[25].Fill(32);
posix[24] = 87;
var kernel = new FakeKernel(posix, new[] { 0 });
StoreReplace.ReplaceWindows("a", "b", kernel);
Assert.That(kernel.Log.Count, Is.EqualTo(26));
Assert.That(kernel.Log[24], Is.EqualTo("posix"));
Assert.That(kernel.Log[25], Is.EqualTo("moveex:9"));
}
[Test]
public void ARetriedMoveFileExThatAnswersUnsupportedFailsAfterOneCallNotTwo()
{
var kernel = new FakeKernel(new[] { 87 }, new[] { 32, 87 });
Assert.That(Fails(kernel).Message, Does.Contain("Win32 error 87"));
Assert.That(kernel.Log, Is.EqualTo(new[] { "posix", "moveex:9", "moveex:9" }));
}
[Test]
public void ARetryThatNeverSucceedsGivesUpAfterTwentyFiveAttemptsNamingTheCode()
{
var kernel = new FakeKernel(new int[30].Fill(32), new int[0]);
var error = Fails(kernel);
Assert.That(error.Message, Does.Contain("25 attempts").And.Contain("last Win32 error 32"));
Assert.That(kernel.Log.Count, Is.EqualTo(25));
Assert.That(kernel.Sleeps.Count, Is.EqualTo(25));
}
[Test]
public void AnyOtherCodeThrowsAtOnceNamingTheCode()
{
var kernel = new FakeKernel(new[] { 2 }, new int[0]);
var error = Fails(kernel);
Assert.That(error.Message, Does.Contain("Win32 error 2"));
Assert.That(kernel.Log.Count, Is.EqualTo(1));
Assert.That(kernel.Sleeps, Is.Empty);
}
[Test]
public void AFallbackThatFailsOtherwiseThrowsItsCode()
{
var kernel = new FakeKernel(new[] { 87 }, new[] { 3 });
Assert.That(Fails(kernel).Message, Does.Contain("Win32 error 3"));
}
[Test]
public void ReplaceOverAnExistingAndThenAnAbsentDestinationLeavesTheStagedBytes()
{
var dir = Path.Combine(TestContext.CurrentContext.WorkDirectory, "storereplace-" + Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(dir);
try
{
var destination = Path.Combine(dir, "store.cc");
var staged = Path.Combine(dir, "staged");
File.WriteAllBytes(destination, new byte[] { 1 });
File.WriteAllBytes(staged, new byte[] { 2 });
StoreReplace.Replace(staged, destination);
Assert.That(File.ReadAllBytes(destination), Is.EqualTo(new byte[] { 2 }));
Assert.That(File.Exists(staged), Is.False);
File.Delete(destination);
File.WriteAllBytes(staged, new byte[] { 3 });
StoreReplace.Replace(staged, destination);
Assert.That(File.ReadAllBytes(destination), Is.EqualTo(new byte[] { 3 }));
}
finally { Directory.Delete(dir, true); }
}
[Test]
public void OnWindowsAReplaceOverADestinationAReaderHoldsOpenSucceedsOnTheFirstAttempt()
{
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
Assert.Ignore("The real kernel runs on Windows only.");
var dir = Path.Combine(TestContext.CurrentContext.WorkDirectory, "storereplace-" + Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(dir);
try
{
var destination = Path.Combine(dir, "store.cc");
var staged = Path.Combine(dir, "staged");
File.WriteAllBytes(destination, new byte[] { 1 });
File.WriteAllBytes(staged, new byte[] { 2 });
using (new FileStream(destination, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete))
{
var kernel = new CountingKernel(StoreReplace.WindowsKernel);
StoreReplace.ReplaceWindows(staged, destination, kernel);
Assert.That(kernel.PosixCalls, Is.EqualTo(1));
Assert.That(kernel.MoveExCalls, Is.EqualTo(0));
Assert.That(kernel.Sleeps, Is.EqualTo(0));
Assert.That(kernel.LastCode, Is.EqualTo(0));
}
Assert.That(File.ReadAllBytes(destination), Is.EqualTo(new byte[] { 2 }));
}
finally { Directory.Delete(dir, true); }
}
private sealed class CountingKernel : IReplaceKernel
{
private readonly IReplaceKernel _inner;
public int PosixCalls, MoveExCalls, Sleeps, LastCode;
public CountingKernel(IReplaceKernel inner) { _inner = inner; }
public int RenamePosix(string staged, string destination)
{
PosixCalls++;
return LastCode = _inner.RenamePosix(staged, destination);
}
public int MoveFileEx(string staged, string destination, uint flags)
{
MoveExCalls++;
return LastCode = _inner.MoveFileEx(staged, destination, flags);
}
public void Sleep(int milliseconds) { Sleeps++; _inner.Sleep(milliseconds); }
}
}
internal static class ArrayFill
{
public static int[] Fill(this int[] array, int value)
{
for (var i = 0; i < array.Length; i++) array[i] = value;
return array;
}
}
}

View file

@ -4,6 +4,7 @@ using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;
using FluentAssertions;
@ -364,11 +365,11 @@ public sealed class CultMeshSingleFileStoreTests
[Test]
public void ACacheOpeningDuringMeshWritesNeverSeesAnEmptyStore()
{
// Windows replaces with ReplaceFile, which leaves a moment with nothing at the path for a lock-free reader: 1,722 empty
// stores in 17,313 opens at b1ff2dcb. That is the store's replace, not Mesh's, and it is the same for a cache flush beside a
// cache open; the fork is in the cut report.
if (OperatingSystem.IsWindows())
Assert.Ignore("Windows File.Replace is not atomic to a lock-free reader; see cut report fork mesh-reader-windows-replace-window.");
// Windows replaces with a POSIX-semantics rename first (StoreReplace), which makes a lock-free reader finding the path
// absent rare, not impossible: a stalled rename was measured leaving it absent for 5-47 ms on Windows. So on Windows a
// brief not-found (a read throwing, or a cache opening onto an absent path) is tolerated and reported; elsewhere rename
// is atomic and none is allowed.
var windows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
var path = Path.Combine(_root, "race.cc");
CultMesh.WriteSingleFileDocument(path, Key, Publication());
@ -386,7 +387,7 @@ public sealed class CultMeshSingleFileStoreTests
Volatile.Write(ref stop, 1);
});
int opens = 0, empty = 0, reads = 0;
int opens = 0, empty = 0, reads = 0, absent = 0;
var clock = Stopwatch.StartNew();
while (Volatile.Read(ref stop) == 0 && clock.Elapsed < TimeSpan.FromSeconds(60))
{
@ -397,6 +398,10 @@ public sealed class CultMeshSingleFileStoreTests
CultMesh.ReadSingleFileDocumentPayload(path, Key, SchemaId());
reads++;
}
catch (Exception ex) when (windows && (ex is FileNotFoundException || ex is DirectoryNotFoundException))
{
absent++;
}
catch (Exception ex)
{
errors.Enqueue("read: " + ex.GetType().Name + ": " + ex.Message);
@ -404,9 +409,10 @@ public sealed class CultMeshSingleFileStoreTests
}
writer.Wait(TimeSpan.FromSeconds(60)).Should().BeTrue();
TestContext.Out.WriteLine($"mesh writes={writes} cache opens={opens} empty stores={empty} mesh reads ok={reads} errors={errors.Count}");
TestContext.Out.WriteLine($"mesh writes={writes} cache opens={opens} empty stores={empty} not-found reads={absent} mesh reads ok={reads} errors={errors.Count}");
errors.Take(3).Should().BeEmpty();
empty.Should().Be(0, "a cache opened in the window where nothing was at the path");
if (!windows)
empty.Should().Be(0, "a cache opened in the window where nothing was at the path");
opens.Should().BeGreaterThan(0);
}