Retry transient Eve asset presentation failures

This commit is contained in:
Meta Vens 2026-08-19 10:04:30 +02:00
commit f3b5076c6d
2 changed files with 80 additions and 18 deletions

View file

@ -24,6 +24,8 @@ namespace GameCult.Eve.UnityScene
IEveUnityGameObjectAssetProvider,
IDisposable
{
private const int AssetPresentationAttemptLimit = 3;
private readonly string _cachePath;
private readonly string _rendezvousEndpoint;
private readonly CultMeshSessionTarget _target;
@ -75,6 +77,7 @@ namespace GameCult.Eve.UnityScene
private long _highestObservedBaseSurfaceVersion = -1;
private long _activeBaseSurfaceCandidateVersion = -1;
private long _activeBaseSurfaceCandidateGeneration = -1;
private int _lastAssetPresentationAttemptCount;
private long _mountedBaseSurfaceVersion;
private long _presentationBarrierVersion = -1;
private long _lastQueuedEntityViewEpoch = -1;
@ -1342,36 +1345,59 @@ namespace GameCult.Eve.UnityScene
long generation,
CancellationToken cancellationToken)
{
try
Exception? finalError = null;
Interlocked.Exchange(ref _lastAssetPresentationAttemptCount, 0);
for (var attempt = 1; attempt <= AssetPresentationAttemptLimit; attempt++)
{
if (generation != Interlocked.Read(ref _surfaceAssetGeneration))
return;
await _assetGenerationGate.WaitAsync(cancellationToken);
Interlocked.Exchange(ref _lastAssetPresentationAttemptCount, attempt);
try
{
if (generation != Interlocked.Read(ref _surfaceAssetGeneration))
return;
var candidate = await BuildAssetGenerationAsync(source, cancellationToken);
if (generation != Interlocked.Read(ref _surfaceAssetGeneration))
await _assetGenerationGate.WaitAsync(cancellationToken);
try
{
candidate.Dispose();
if (generation != Interlocked.Read(ref _surfaceAssetGeneration))
return;
var candidate = await BuildAssetGenerationAsync(source, cancellationToken);
if (generation != Interlocked.Read(ref _surfaceAssetGeneration))
{
candidate.Dispose();
return;
}
CommitAssetGeneration(candidate, surface);
return;
}
CommitAssetGeneration(candidate, surface);
finally
{
_assetGenerationGate.Release();
}
}
finally
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
_assetGenerationGate.Release();
return;
}
catch (Exception error)
{
finalError = error;
if (generation != Interlocked.Read(ref _surfaceAssetGeneration))
return;
if (attempt < AssetPresentationAttemptLimit)
{
try
{
await Task.Delay(TimeSpan.FromMilliseconds(100 * attempt), cancellationToken);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
return;
}
}
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
}
catch (Exception error)
{
RetireFailedBaseSurfaceCandidate(surface.Version, generation);
_liveDocuments.Enqueue(error);
}
RetireFailedBaseSurfaceCandidate(surface.Version, generation);
_liveDocuments.Enqueue(finalError ?? new InvalidOperationException(
$"Eve surface {surface.Version} asset preparation exhausted its retry policy."));
}
private AssetSource ResolveAssetSource(EveSurfaceDocument surface)

View file

@ -386,6 +386,42 @@ namespace GameCult.Eve.UnityScene.Tests
Assert.That(begin.Invoke(transport, new object[] { 9L, -1L }), Is.False);
}
[Test]
public async Task FailedAssetPreparationRunsBoundedRetryPolicyAndRetiresCandidate()
{
using var transport = new EveUnityCultMeshLiveProviderTransport(
"test-cache",
"cultnet+tcp://127.0.0.1:1",
"test.verse",
"test.runtime",
"test.provider",
"test.surface");
var type = typeof(EveUnityCultMeshLiveProviderTransport);
var begin = type.GetMethod(
"TryBeginBaseSurfaceCandidate",
BindingFlags.NonPublic | BindingFlags.Instance)!;
var refresh = type.GetMethod(
"RefreshAssetsForSurfaceAsync",
BindingFlags.NonPublic | BindingFlags.Instance)!;
var attempts = type.GetField(
"_lastAssetPresentationAttemptCount",
BindingFlags.NonPublic | BindingFlags.Instance)!;
var first = new object[] { 10L, -1L };
Assert.That(begin.Invoke(transport, first), Is.True);
var task = (Task)refresh.Invoke(transport, new object[]
{
SceneSurface(10).SurfaceDocument,
CreateAssetSource("test.verse", "test.runtime", "test.provider", "catalog", 41),
(long)first[1],
CancellationToken.None
})!;
await task;
Assert.That(attempts.GetValue(transport), Is.EqualTo(3));
Assert.That(begin.Invoke(transport, new object[] { 10L, -1L }), Is.True);
}
private static EveUnitySceneProviderSurfaceDocument SceneSurface(long version)
{
var surface = new EveSurfaceDocument(