diff --git a/src/GameCult.Mesh/CultMeshCdn.cs b/src/GameCult.Mesh/CultMeshCdn.cs index 72f38c6a..7aaf7eba 100644 --- a/src/GameCult.Mesh/CultMeshCdn.cs +++ b/src/GameCult.Mesh/CultMeshCdn.cs @@ -364,6 +364,10 @@ namespace GameCult.Mesh if (manifest == null) throw new ArgumentNullException(nameof(manifest)); ValidateManifestShape(manifest); + if (manifest.SizeBytes > int.MaxValue) + { + throw new InvalidDataException("CDN artifact is too large to materialize into one byte array."); + } var output = new byte[checked((int)manifest.SizeBytes)]; foreach (var reference in manifest.Chunks.OrderBy(chunk => chunk.Offset)) { @@ -395,26 +399,21 @@ namespace GameCult.Mesh if (database == null) throw new ArgumentNullException(nameof(database)); if (manifest == null) throw new ArgumentNullException(nameof(manifest)); - ValidateManifestShape(manifest); - var output = new byte[checked((int)manifest.SizeBytes)]; - foreach (var reference in manifest.Chunks.OrderBy(chunk => chunk.Offset)) + var directory = Path.Combine(Path.GetTempPath(), "cultmesh-content", Guid.NewGuid().ToString("N")); + try { - var recordKey = string.IsNullOrWhiteSpace(reference.RecordKey) - ? CultMeshCdnArtifactChunk.CreateRecordKey(reference.ChunkHash) - : new CultRecordKey(reference.RecordKey); - var chunk = await database.GetAsync(recordKey).ConfigureAwait(false) - ?? throw new FileNotFoundException("CDN artifact chunk is missing from the database.", recordKey.Value); - ValidateChunk(reference, chunk); - Buffer.BlockCopy(chunk.Payload, 0, output, checked((int)reference.Offset), reference.SizeBytes); + using var state = new CultCache(); + var service = new CultMeshContentTransferService( + state, + new[] { new CultMeshDatabaseContentProvider("legacy-database", database) }, + new CultMeshContentTransferOptions(directory)); + var path = await service.FetchAsync(manifest).ConfigureAwait(false); + return await File.ReadAllBytesAsync(path).ConfigureAwait(false); } - - var actualHash = HashBytes(output); - if (!string.Equals(NormalizeHash(manifest.ContentHash, nameof(manifest.ContentHash)), actualHash, StringComparison.Ordinal)) + finally { - throw new InvalidDataException("CDN artifact content hash does not match its manifest."); + if (Directory.Exists(directory)) Directory.Delete(directory, true); } - - return output; } /// @@ -492,6 +491,11 @@ namespace GameCult.Mesh } } + internal static void ValidateManifest(CultMeshCdnArtifactManifest manifest) + { + ValidateManifestShape(manifest); + } + private static void ValidateManifestShape(CultMeshCdnArtifactManifest manifest) { if (manifest.SizeBytes < 0) @@ -499,11 +503,6 @@ namespace GameCult.Mesh throw new InvalidDataException("CDN artifact manifest has a negative size."); } - if (manifest.SizeBytes > int.MaxValue) - { - throw new InvalidDataException("CDN artifact is too large to materialize into one byte array."); - } - var expectedOffset = 0L; foreach (var reference in manifest.Chunks.OrderBy(chunk => chunk.Offset)) { @@ -526,6 +525,11 @@ namespace GameCult.Mesh } } + internal static void ValidateChunkPayload(CultMeshCdnChunkRef reference, CultMeshCdnArtifactChunk chunk) + { + ValidateChunk(reference, chunk); + } + private static void ValidateChunk(CultMeshCdnChunkRef reference, CultMeshCdnArtifactChunk chunk) { if (chunk.Payload.Length != reference.SizeBytes || diff --git a/src/GameCult.Mesh/CultMeshContentTransfer.cs b/src/GameCult.Mesh/CultMeshContentTransfer.cs new file mode 100644 index 00000000..02abebaf --- /dev/null +++ b/src/GameCult.Mesh/CultMeshContentTransfer.cs @@ -0,0 +1,274 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Threading; +using System.Threading.Tasks; +using GameCult.Caching; +using GameCult.Networking; +using MessagePack; + +namespace GameCult.Mesh +{ + public static class CultMeshContentTransferSchemaVersions + { + public const string State = "gamecult.mesh.content_transfer_state.v1"; + } + + [MessagePackObject] + [CultDocument("gamecult.mesh.content_transfer_state", CultMeshContentTransferSchemaVersions.State)] + public sealed class CultMeshContentTransferStateDocument + { + [Key(0), CultName] public string ContentHash { get; set; } = string.Empty; + [Key(1)] public string ManifestFingerprint { get; set; } = string.Empty; + [Key(2)] public long SizeBytes { get; set; } + [Key(3)] public int[] VerifiedChunkIndexes { get; set; } = Array.Empty(); + [Key(4)] public string UpdatedAtUtc { get; set; } = string.Empty; + } + + public interface ICultMeshContentProvider + { + string ProviderId { get; } + Task GetChunkAsync(CultMeshCdnChunkRef chunk, CancellationToken cancellationToken = default); + } + + public sealed class CultMeshDatabaseContentProvider : ICultMeshContentProvider + { + private readonly CultNetDatabase _database; + + public CultMeshDatabaseContentProvider(string providerId, CultNetDatabase database) + { + ProviderId = string.IsNullOrWhiteSpace(providerId) ? throw new ArgumentException("Provider id must be non-empty.", nameof(providerId)) : providerId; + _database = database ?? throw new ArgumentNullException(nameof(database)); + } + + public string ProviderId { get; } + + public async Task GetChunkAsync(CultMeshCdnChunkRef chunk, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + var key = string.IsNullOrWhiteSpace(chunk.RecordKey) + ? CultMeshCdnArtifactChunk.CreateRecordKey(chunk.ChunkHash) + : new CultRecordKey(chunk.RecordKey); + var result = await _database.GetAsync(key).ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); + return result; + } + } + + public sealed class CultMeshContentTransferOptions + { + public CultMeshContentTransferOptions(string cacheDirectory) + { + CacheDirectory = string.IsNullOrWhiteSpace(cacheDirectory) + ? throw new ArgumentException("Cache directory must be non-empty.", nameof(cacheDirectory)) + : Path.GetFullPath(cacheDirectory); + } + + public string CacheDirectory { get; } + } + + public sealed class CultMeshContentTransferService + { + private readonly CultCache _stateCache; + private readonly ICultMeshContentProvider[] _providers; + private readonly string _cacheDirectory; + private readonly ConcurrentDictionary _contentLocks = new(StringComparer.Ordinal); + + public CultMeshContentTransferService( + CultCache stateCache, + IEnumerable providers, + CultMeshContentTransferOptions options) + { + _stateCache = stateCache ?? throw new ArgumentNullException(nameof(stateCache)); + _providers = providers?.ToArray() ?? throw new ArgumentNullException(nameof(providers)); + if (_providers.Length == 0) throw new ArgumentException("At least one content provider is required.", nameof(providers)); + if (_providers.Any(provider => provider == null)) throw new ArgumentException("Content providers cannot contain null entries.", nameof(providers)); + _cacheDirectory = (options ?? throw new ArgumentNullException(nameof(options))).CacheDirectory; + } + + public async Task FetchAsync( + CultMeshCdnArtifactManifest manifest, + CancellationToken cancellationToken = default) + { + if (manifest == null) throw new ArgumentNullException(nameof(manifest)); + CultMeshCdn.ValidateManifest(manifest); + + var contentHash = CultMeshCdn.NormalizeHash(manifest.ContentHash, nameof(manifest.ContentHash)); + var contentLock = _contentLocks.GetOrAdd(contentHash, _ => new SemaphoreSlim(1, 1)); + await contentLock.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + return await FetchOwnedAsync(manifest, contentHash, cancellationToken).ConfigureAwait(false); + } + finally + { + contentLock.Release(); + } + } + + private async Task FetchOwnedAsync( + CultMeshCdnArtifactManifest manifest, + string contentHash, + CancellationToken cancellationToken) + { + Directory.CreateDirectory(_cacheDirectory); + var finalPath = Path.Combine(_cacheDirectory, contentHash + ".body"); + var partialPath = Path.Combine(_cacheDirectory, "." + contentHash + ".partial"); + var stateKey = StateKey(contentHash); + + if (File.Exists(finalPath)) + { + if (await HashFileAsync(finalPath, cancellationToken).ConfigureAwait(false) == contentHash) + return finalPath; + File.Delete(finalPath); + } + + var fingerprint = ManifestFingerprint(manifest); + var state = _stateCache.Get(stateKey); + var verified = RestoreVerifiedState(state, manifest, fingerprint, partialPath); + + using (var stream = new FileStream(partialPath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read, 81920, FileOptions.Asynchronous)) + { + stream.SetLength(manifest.SizeBytes); + var ordered = manifest.Chunks + .Select((chunk, index) => new { Chunk = chunk, Index = index }) + .OrderBy(item => item.Chunk.Offset) + .ToArray(); + + foreach (var item in ordered.Where(item => !verified.Contains(item.Index))) + { + cancellationToken.ThrowIfCancellationRequested(); + var chunk = await FetchVerifiedChunkAsync(item.Chunk, cancellationToken).ConfigureAwait(false); + stream.Position = item.Chunk.Offset; + await stream.WriteAsync(chunk.Payload, 0, chunk.Payload.Length, cancellationToken).ConfigureAwait(false); + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + verified.Add(item.Index); + await SaveStateAsync(contentHash, fingerprint, manifest.SizeBytes, verified, stateKey).ConfigureAwait(false); + } + } + + cancellationToken.ThrowIfCancellationRequested(); + var actualHash = await HashFileAsync(partialPath, cancellationToken).ConfigureAwait(false); + if (!string.Equals(actualHash, contentHash, StringComparison.Ordinal)) + { + File.Delete(partialPath); + _stateCache.Remove(stateKey); + await _stateCache.FlushAsync().ConfigureAwait(false); + throw new InvalidDataException("CDN artifact content hash does not match its manifest."); + } + + PromoteAtomically(partialPath, finalPath); + _stateCache.Remove(stateKey); + await _stateCache.FlushAsync().ConfigureAwait(false); + return finalPath; + } + + private HashSet RestoreVerifiedState( + CultMeshContentTransferStateDocument? state, + CultMeshCdnArtifactManifest manifest, + string fingerprint, + string partialPath) + { + if (state == null || !File.Exists(partialPath) || + state.SizeBytes != manifest.SizeBytes || + !string.Equals(state.ManifestFingerprint, fingerprint, StringComparison.Ordinal) || + new FileInfo(partialPath).Length != manifest.SizeBytes) + { + if (File.Exists(partialPath)) File.Delete(partialPath); + return new HashSet(); + } + + var verified = new HashSet(); + using var stream = new FileStream(partialPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + foreach (var index in state.VerifiedChunkIndexes.Distinct().Where(index => index >= 0 && index < manifest.Chunks.Length)) + { + var reference = manifest.Chunks[index]; + var bytes = new byte[reference.SizeBytes]; + stream.Position = reference.Offset; + if (stream.Read(bytes, 0, bytes.Length) == bytes.Length && + string.Equals(CultMeshCdn.HashBytes(bytes), CultMeshCdn.NormalizeHash(reference.ChunkHash, nameof(reference.ChunkHash)), StringComparison.Ordinal)) + { + verified.Add(index); + } + } + + return verified; + } + + private async Task FetchVerifiedChunkAsync( + CultMeshCdnChunkRef reference, + CancellationToken cancellationToken) + { + var failures = new List(); + foreach (var provider in _providers) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + var chunk = await provider.GetChunkAsync(reference, cancellationToken).ConfigureAwait(false) + ?? throw new FileNotFoundException("CDN artifact chunk is missing from the provider.", reference.RecordKey); + CultMeshCdn.ValidateChunkPayload(reference, chunk); + return chunk; + } + catch (Exception error) when (!(error is OperationCanceledException && cancellationToken.IsCancellationRequested)) + { + failures.Add(new InvalidDataException("Content provider '" + provider.ProviderId + "' could not supply a valid chunk.", error)); + } + } + + throw new AggregateException("No content provider supplied a valid CDN chunk.", failures); + } + + private async Task SaveStateAsync( + string contentHash, + string fingerprint, + long sizeBytes, + HashSet verified, + CultRecordKey stateKey) + { + var state = new CultMeshContentTransferStateDocument + { + ContentHash = contentHash, + ManifestFingerprint = fingerprint, + SizeBytes = sizeBytes, + VerifiedChunkIndexes = verified.OrderBy(value => value).ToArray(), + UpdatedAtUtc = DateTimeOffset.UtcNow.ToString("O", CultureInfo.InvariantCulture) + }; + await _stateCache.UpsertAsync(typeof(CultMeshContentTransferStateDocument), state, stateKey).ConfigureAwait(false); + await _stateCache.FlushAsync().ConfigureAwait(false); + } + + private static CultRecordKey StateKey(string contentHash) => new CultRecordKey("mesh:content-transfer:" + contentHash); + + private static string ManifestFingerprint(CultMeshCdnArtifactManifest manifest) + { + var shape = string.Join("|", manifest.Chunks.Select(chunk => + chunk.Offset.ToString(CultureInfo.InvariantCulture) + ":" + + chunk.SizeBytes.ToString(CultureInfo.InvariantCulture) + ":" + + CultMeshCdn.NormalizeHash(chunk.ChunkHash, nameof(chunk.ChunkHash)))); + return CultMeshCdn.HashBytes(System.Text.Encoding.UTF8.GetBytes(manifest.SizeBytes.ToString(CultureInfo.InvariantCulture) + "|" + shape)); + } + + private static async Task HashFileAsync(string path, CancellationToken cancellationToken) + { + using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 81920, FileOptions.Asynchronous | FileOptions.SequentialScan); + using var sha = SHA256.Create(); + var buffer = new byte[81920]; + int read; + while ((read = await stream.ReadAsync(buffer, 0, buffer.Length, cancellationToken).ConfigureAwait(false)) > 0) + sha.TransformBlock(buffer, 0, read, null, 0); + sha.TransformFinalBlock(Array.Empty(), 0, 0); + return string.Concat(sha.Hash!.Select(value => value.ToString("x2", CultureInfo.InvariantCulture))); + } + + private static void PromoteAtomically(string partialPath, string finalPath) + { + if (File.Exists(finalPath)) File.Replace(partialPath, finalPath, null); + else File.Move(partialPath, finalPath); + } + } +} diff --git a/src/GameCult.Mesh/docs/reliability-migration-plan.md b/src/GameCult.Mesh/docs/reliability-migration-plan.md index 0144524e..61cf9095 100644 --- a/src/GameCult.Mesh/docs/reliability-migration-plan.md +++ b/src/GameCult.Mesh/docs/reliability-migration-plan.md @@ -521,6 +521,51 @@ session path. ## Phase 4: Verified Content Transfer +### Phase 4 first-slice body map + +**Owner:** `CultMeshContentTransferService` owns verified partial state and the +only promotion path into the completed content cache. Work is serialized and +shared by normalized content hash; provider identity is never part of content +identity or the destination name. + +**Inputs:** a validated CDN manifest, ordered `ICultMeshContentProvider` +observations, a typed CultCache checkpoint store, a canonical cache directory, +and caller cancellation. + +**Outputs:** a complete read-only-by-convention body path named by SHA-256, or +a failure with no completed body. `gamecult.mesh.content_transfer_state.v1` +records verified chunk indexes keyed by content hash. + +**Derived state:** missing chunks, provider failover order, deterministic +partial-file path, manifest fingerprint, and completed cache path. A provider +success is evidence only for the returned chunk; it does not confer publisher +authority. + +**Forbidden writers:** callers, renderers, providers, manifests, and the legacy +async CDN reader cannot publish a final body or mark a range verified. CultCache +checkpoint claims are rehashed against the partial body before reuse. + +**Shared paths:** direct transfers and the legacy distributed-database reader +both pass through the same chunk verification, full-hash verification, and +atomic promotion primitive. Concurrent requests for one hash share the same +per-content transfer lock and verified work. + +**Cut line:** the legacy distributed reader no longer owns a sequential chunk +loop or whole-artifact allocation. The first slice intentionally leaves +adaptive parallelism, quotas, pinning, signer policy, arbitrary streaming +destinations, and cleanup scheduling to later Phase 4 work. + +**State transition:** each verified chunk flushes a typed checkpoint. Restart +rehydrates that record, rehashes every claimed range, and fetches corrupt or +unclaimed chunks again. Completion verifies the entire partial file, atomically +promotes it, and removes the checkpoint. Cancellation leaves only re-verifiable +partial state. + +**Verification layer:** focused tests cover durable cancellation/restart, +provider corruption and failover, forged checkpoint rejection, final-hash +failure, atomic visibility, and concurrent request coalescing. Existing CDN +tests cover the compatibility reader. + ### Cut first - Delete the first consumer-owned CDN chunk loop and partial-file convention. diff --git a/tests/GameCult.Mesh.Tests/CultMeshContentTransferTests.cs b/tests/GameCult.Mesh.Tests/CultMeshContentTransferTests.cs new file mode 100644 index 00000000..a23d9e60 --- /dev/null +++ b/tests/GameCult.Mesh.Tests/CultMeshContentTransferTests.cs @@ -0,0 +1,212 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using NUnit.Framework; + +namespace GameCult.Mesh.Tests; + +[TestFixture] +public sealed class CultMeshContentTransferTests +{ + private string _directory = null!; + + [SetUp] + public void SetUp() + { + _directory = Path.Combine(Path.GetTempPath(), "cultmesh-transfer-tests", Guid.NewGuid().ToString("N")); + } + + [TearDown] + public void TearDown() + { + if (Directory.Exists(_directory)) Directory.Delete(_directory, true); + } + + [Test] + public async Task FetchAsync_VerifiesChunksAndAtomicallyPublishesFinalBody() + { + var artifact = Artifact("verified", 4); + using var cache = Cache(); + var provider = new FakeProvider("source", artifact.Chunks); + var service = Service(cache, provider); + + var path = await service.FetchAsync(artifact.Manifest); + + File.ReadAllBytes(path).Should().Equal(Payload()); + Directory.GetFiles(_directory, "*.partial").Should().BeEmpty(); + cache.Get(StateKey(artifact.Manifest)).Should().BeNull(); + } + + [Test] + public async Task FetchAsync_FailsOverAfterProviderCorruption() + { + var artifact = Artifact("failover", 4); + using var cache = Cache(); + var corrupt = artifact.Chunks.Select(Clone).ToArray(); + corrupt[0].Payload[0] ^= 0xff; + var first = new FakeProvider("corrupt", corrupt); + var second = new FakeProvider("good", artifact.Chunks); + + var path = await Service(cache, first, second).FetchAsync(artifact.Manifest); + + File.ReadAllBytes(path).Should().Equal(Payload()); + first.Requests.Should().ContainKey(artifact.Manifest.Chunks[0].ChunkHash); + second.Requests.Should().ContainKey(artifact.Manifest.Chunks[0].ChunkHash); + } + + [Test] + public async Task FetchAsync_CancellationLeavesVerifiedCheckpointAndResumeSkipsIt() + { + var artifact = Artifact("resume", 4); + var statePath = Path.Combine(_directory, "transfer-state.cc"); + using var cache = DurableCache(statePath); + using var cancellation = new CancellationTokenSource(); + var provider = new FakeProvider("source", artifact.Chunks) { BlockFromRequest = 2 }; + var service = Service(cache, provider); + var fetch = service.FetchAsync(artifact.Manifest, cancellation.Token); + + await WaitUntilAsync(() => cache.Get(StateKey(artifact.Manifest))?.VerifiedChunkIndexes.Length == 1); + cancellation.Cancel(); + await FluentActions.Awaiting(() => fetch).Should().ThrowAsync(); + File.Exists(Path.Combine(_directory, artifact.Manifest.ContentHash + ".body")).Should().BeFalse(); + cache.Dispose(); + + using var restoredCache = DurableCache(statePath); + await restoredCache.PullAllBackingStoresAsync(); + restoredCache.Get(StateKey(artifact.Manifest)) + .Should().NotBeNull("the verified checkpoint must survive a cache restart"); + var resumedProvider = new FakeProvider("resume", artifact.Chunks); + var path = await Service(restoredCache, resumedProvider).FetchAsync(artifact.Manifest); + + File.ReadAllBytes(path).Should().Equal(Payload()); + resumedProvider.Requests.Should().NotContainKey(artifact.Manifest.Chunks[0].ChunkHash); + } + + [Test] + public async Task FetchAsync_RejectsForgedCheckpointAfterRehashingClaimedChunks() + { + var artifact = Artifact("forged", 4); + using var cache = Cache(); + using var cancellation = new CancellationTokenSource(); + var firstProvider = new FakeProvider("first", artifact.Chunks) { BlockFromRequest = 2 }; + var firstFetch = Service(cache, firstProvider).FetchAsync(artifact.Manifest, cancellation.Token); + await WaitUntilAsync(() => cache.Get(StateKey(artifact.Manifest))?.VerifiedChunkIndexes.Length == 1); + cancellation.Cancel(); + await FluentActions.Awaiting(() => firstFetch).Should().ThrowAsync(); + + var partial = Path.Combine(_directory, "." + artifact.Manifest.ContentHash + ".partial"); + var bytes = File.ReadAllBytes(partial); + bytes[0] ^= 0xff; + File.WriteAllBytes(partial, bytes); + + var provider = new FakeProvider("source", artifact.Chunks); + await Service(cache, provider).FetchAsync(artifact.Manifest); + + provider.Requests.Values.Sum().Should().Be(artifact.Chunks.Count); + File.ReadAllBytes(Path.Combine(_directory, artifact.Manifest.ContentHash + ".body")).Should().Equal(Payload()); + } + + [Test] + public async Task FetchAsync_FinalHashFailurePublishesNoBodyOrTrustedCheckpoint() + { + var artifact = Artifact("wrong-final", 4); + artifact.Manifest.ContentHash = new string('0', 64); + using var cache = Cache(); + + await FluentActions.Awaiting(() => Service(cache, new FakeProvider("source", artifact.Chunks)).FetchAsync(artifact.Manifest)) + .Should().ThrowAsync(); + + File.Exists(Path.Combine(_directory, new string('0', 64) + ".body")).Should().BeFalse(); + cache.Get(new CultRecordKey("mesh:content-transfer:" + new string('0', 64))).Should().BeNull(); + } + + [Test] + public async Task FetchAsync_ConcurrentRequestsShareVerifiedWork() + { + var artifact = Artifact("shared", 4); + using var cache = Cache(); + var provider = new FakeProvider("source", artifact.Chunks) { DelayMilliseconds = 20 }; + var service = Service(cache, provider); + + var paths = await Task.WhenAll(service.FetchAsync(artifact.Manifest), service.FetchAsync(artifact.Manifest)); + + paths[0].Should().Be(paths[1]); + provider.Requests.Values.Sum().Should().Be(artifact.Chunks.Count); + } + + private CultMeshContentTransferService Service(CultCache cache, params ICultMeshContentProvider[] providers) => + new(cache, providers, new CultMeshContentTransferOptions(_directory)); + + private static CultCache Cache() + { + var registry = CultMesh.CreateCultCacheDocumentRegistry(typeof(CultMeshContentTransferStateDocument)); + return new CultCache(registry); + } + + private static CultCache DurableCache(string path) + { + var cache = Cache(); + cache.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + return cache; + } + + private static CultMeshCdnArtifact Artifact(string id, int chunkSize) => + CultMesh.PackCdnArtifact(id, Payload(), new CultMeshCdnPackOptions { ChunkSizeBytes = chunkSize }); + + private static byte[] Payload() => Enumerable.Range(0, 19).Select(value => (byte)(value + 1)).ToArray(); + + private static CultRecordKey StateKey(CultMeshCdnArtifactManifest manifest) => + new("mesh:content-transfer:" + manifest.ContentHash); + + private static CultMeshCdnArtifactChunk Clone(CultMeshCdnArtifactChunk chunk) => new() + { + ChunkHash = chunk.ChunkHash, + SizeBytes = chunk.SizeBytes, + Payload = chunk.Payload.ToArray() + }; + + private static async Task WaitUntilAsync(Func predicate) + { + var deadline = DateTime.UtcNow.AddSeconds(5); + while (!predicate()) + { + if (DateTime.UtcNow >= deadline) throw new TimeoutException("Condition was not observed."); + await Task.Delay(10); + } + } + + private sealed class FakeProvider : ICultMeshContentProvider + { + private readonly Dictionary _chunks; + private int _requestCount; + + public FakeProvider(string providerId, IEnumerable chunks) + { + ProviderId = providerId; + _chunks = chunks.ToDictionary(chunk => chunk.ChunkHash, Clone, StringComparer.Ordinal); + } + + public string ProviderId { get; } + public ConcurrentDictionary Requests { get; } = new(StringComparer.Ordinal); + public int BlockFromRequest { get; set; } = int.MaxValue; + public int DelayMilliseconds { get; set; } + + public async Task GetChunkAsync(CultMeshCdnChunkRef chunk, CancellationToken cancellationToken = default) + { + Requests.AddOrUpdate(chunk.ChunkHash, 1, (_, count) => count + 1); + var request = Interlocked.Increment(ref _requestCount); + if (request >= BlockFromRequest) + await Task.Delay(Timeout.Infinite, cancellationToken); + if (DelayMilliseconds > 0) + await Task.Delay(DelayMilliseconds, cancellationToken); + return _chunks.TryGetValue(chunk.ChunkHash, out var value) ? Clone(value) : null; + } + } +}