Add verified resumable CultMesh transfers
This commit is contained in:
parent
fb24e30ab6
commit
b9f17a7126
4 changed files with 556 additions and 21 deletions
|
|
@ -364,6 +364,10 @@ namespace GameCult.Mesh
|
|||
if (manifest == null) throw new ArgumentNullException(nameof(manifest));
|
||||
|
||||
ValidateManifestShape(manifest);
|
||||
if (manifest.SizeBytes > int.MaxValue)
|
||||
{
|
||||
throw new InvalidDataException("CDN artifact is too large to materialize into one byte array.");
|
||||
}
|
||||
var output = new byte[checked((int)manifest.SizeBytes)];
|
||||
foreach (var reference in manifest.Chunks.OrderBy(chunk => chunk.Offset))
|
||||
{
|
||||
|
|
@ -395,26 +399,21 @@ namespace GameCult.Mesh
|
|||
if (database == null) throw new ArgumentNullException(nameof(database));
|
||||
if (manifest == null) throw new ArgumentNullException(nameof(manifest));
|
||||
|
||||
ValidateManifestShape(manifest);
|
||||
var output = new byte[checked((int)manifest.SizeBytes)];
|
||||
foreach (var reference in manifest.Chunks.OrderBy(chunk => chunk.Offset))
|
||||
var directory = Path.Combine(Path.GetTempPath(), "cultmesh-content", Guid.NewGuid().ToString("N"));
|
||||
try
|
||||
{
|
||||
var recordKey = string.IsNullOrWhiteSpace(reference.RecordKey)
|
||||
? CultMeshCdnArtifactChunk.CreateRecordKey(reference.ChunkHash)
|
||||
: new CultRecordKey(reference.RecordKey);
|
||||
var chunk = await database.GetAsync<CultMeshCdnArtifactChunk>(recordKey).ConfigureAwait(false)
|
||||
?? throw new FileNotFoundException("CDN artifact chunk is missing from the database.", recordKey.Value);
|
||||
ValidateChunk(reference, chunk);
|
||||
Buffer.BlockCopy(chunk.Payload, 0, output, checked((int)reference.Offset), reference.SizeBytes);
|
||||
using var state = new CultCache();
|
||||
var service = new CultMeshContentTransferService(
|
||||
state,
|
||||
new[] { new CultMeshDatabaseContentProvider("legacy-database", database) },
|
||||
new CultMeshContentTransferOptions(directory));
|
||||
var path = await service.FetchAsync(manifest).ConfigureAwait(false);
|
||||
return await File.ReadAllBytesAsync(path).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var actualHash = HashBytes(output);
|
||||
if (!string.Equals(NormalizeHash(manifest.ContentHash, nameof(manifest.ContentHash)), actualHash, StringComparison.Ordinal))
|
||||
finally
|
||||
{
|
||||
throw new InvalidDataException("CDN artifact content hash does not match its manifest.");
|
||||
if (Directory.Exists(directory)) Directory.Delete(directory, true);
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -492,6 +491,11 @@ namespace GameCult.Mesh
|
|||
}
|
||||
}
|
||||
|
||||
internal static void ValidateManifest(CultMeshCdnArtifactManifest manifest)
|
||||
{
|
||||
ValidateManifestShape(manifest);
|
||||
}
|
||||
|
||||
private static void ValidateManifestShape(CultMeshCdnArtifactManifest manifest)
|
||||
{
|
||||
if (manifest.SizeBytes < 0)
|
||||
|
|
@ -499,11 +503,6 @@ namespace GameCult.Mesh
|
|||
throw new InvalidDataException("CDN artifact manifest has a negative size.");
|
||||
}
|
||||
|
||||
if (manifest.SizeBytes > int.MaxValue)
|
||||
{
|
||||
throw new InvalidDataException("CDN artifact is too large to materialize into one byte array.");
|
||||
}
|
||||
|
||||
var expectedOffset = 0L;
|
||||
foreach (var reference in manifest.Chunks.OrderBy(chunk => chunk.Offset))
|
||||
{
|
||||
|
|
@ -526,6 +525,11 @@ namespace GameCult.Mesh
|
|||
}
|
||||
}
|
||||
|
||||
internal static void ValidateChunkPayload(CultMeshCdnChunkRef reference, CultMeshCdnArtifactChunk chunk)
|
||||
{
|
||||
ValidateChunk(reference, chunk);
|
||||
}
|
||||
|
||||
private static void ValidateChunk(CultMeshCdnChunkRef reference, CultMeshCdnArtifactChunk chunk)
|
||||
{
|
||||
if (chunk.Payload.Length != reference.SizeBytes ||
|
||||
|
|
|
|||
274
src/GameCult.Mesh/CultMeshContentTransfer.cs
Normal file
274
src/GameCult.Mesh/CultMeshContentTransfer.cs
Normal file
|
|
@ -0,0 +1,274 @@
|
|||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using GameCult.Caching;
|
||||
using GameCult.Networking;
|
||||
using MessagePack;
|
||||
|
||||
namespace GameCult.Mesh
|
||||
{
|
||||
public static class CultMeshContentTransferSchemaVersions
|
||||
{
|
||||
public const string State = "gamecult.mesh.content_transfer_state.v1";
|
||||
}
|
||||
|
||||
[MessagePackObject]
|
||||
[CultDocument("gamecult.mesh.content_transfer_state", CultMeshContentTransferSchemaVersions.State)]
|
||||
public sealed class CultMeshContentTransferStateDocument
|
||||
{
|
||||
[Key(0), CultName] public string ContentHash { get; set; } = string.Empty;
|
||||
[Key(1)] public string ManifestFingerprint { get; set; } = string.Empty;
|
||||
[Key(2)] public long SizeBytes { get; set; }
|
||||
[Key(3)] public int[] VerifiedChunkIndexes { get; set; } = Array.Empty<int>();
|
||||
[Key(4)] public string UpdatedAtUtc { get; set; } = string.Empty;
|
||||
}
|
||||
|
||||
public interface ICultMeshContentProvider
|
||||
{
|
||||
string ProviderId { get; }
|
||||
Task<CultMeshCdnArtifactChunk?> GetChunkAsync(CultMeshCdnChunkRef chunk, CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
public sealed class CultMeshDatabaseContentProvider : ICultMeshContentProvider
|
||||
{
|
||||
private readonly CultNetDatabase _database;
|
||||
|
||||
public CultMeshDatabaseContentProvider(string providerId, CultNetDatabase database)
|
||||
{
|
||||
ProviderId = string.IsNullOrWhiteSpace(providerId) ? throw new ArgumentException("Provider id must be non-empty.", nameof(providerId)) : providerId;
|
||||
_database = database ?? throw new ArgumentNullException(nameof(database));
|
||||
}
|
||||
|
||||
public string ProviderId { get; }
|
||||
|
||||
public async Task<CultMeshCdnArtifactChunk?> GetChunkAsync(CultMeshCdnChunkRef chunk, CancellationToken cancellationToken = default)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var key = string.IsNullOrWhiteSpace(chunk.RecordKey)
|
||||
? CultMeshCdnArtifactChunk.CreateRecordKey(chunk.ChunkHash)
|
||||
: new CultRecordKey(chunk.RecordKey);
|
||||
var result = await _database.GetAsync<CultMeshCdnArtifactChunk>(key).ConfigureAwait(false);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class CultMeshContentTransferOptions
|
||||
{
|
||||
public CultMeshContentTransferOptions(string cacheDirectory)
|
||||
{
|
||||
CacheDirectory = string.IsNullOrWhiteSpace(cacheDirectory)
|
||||
? throw new ArgumentException("Cache directory must be non-empty.", nameof(cacheDirectory))
|
||||
: Path.GetFullPath(cacheDirectory);
|
||||
}
|
||||
|
||||
public string CacheDirectory { get; }
|
||||
}
|
||||
|
||||
public sealed class CultMeshContentTransferService
|
||||
{
|
||||
private readonly CultCache _stateCache;
|
||||
private readonly ICultMeshContentProvider[] _providers;
|
||||
private readonly string _cacheDirectory;
|
||||
private readonly ConcurrentDictionary<string, SemaphoreSlim> _contentLocks = new(StringComparer.Ordinal);
|
||||
|
||||
public CultMeshContentTransferService(
|
||||
CultCache stateCache,
|
||||
IEnumerable<ICultMeshContentProvider> providers,
|
||||
CultMeshContentTransferOptions options)
|
||||
{
|
||||
_stateCache = stateCache ?? throw new ArgumentNullException(nameof(stateCache));
|
||||
_providers = providers?.ToArray() ?? throw new ArgumentNullException(nameof(providers));
|
||||
if (_providers.Length == 0) throw new ArgumentException("At least one content provider is required.", nameof(providers));
|
||||
if (_providers.Any(provider => provider == null)) throw new ArgumentException("Content providers cannot contain null entries.", nameof(providers));
|
||||
_cacheDirectory = (options ?? throw new ArgumentNullException(nameof(options))).CacheDirectory;
|
||||
}
|
||||
|
||||
public async Task<string> FetchAsync(
|
||||
CultMeshCdnArtifactManifest manifest,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (manifest == null) throw new ArgumentNullException(nameof(manifest));
|
||||
CultMeshCdn.ValidateManifest(manifest);
|
||||
|
||||
var contentHash = CultMeshCdn.NormalizeHash(manifest.ContentHash, nameof(manifest.ContentHash));
|
||||
var contentLock = _contentLocks.GetOrAdd(contentHash, _ => new SemaphoreSlim(1, 1));
|
||||
await contentLock.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
return await FetchOwnedAsync(manifest, contentHash, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
contentLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<string> FetchOwnedAsync(
|
||||
CultMeshCdnArtifactManifest manifest,
|
||||
string contentHash,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
Directory.CreateDirectory(_cacheDirectory);
|
||||
var finalPath = Path.Combine(_cacheDirectory, contentHash + ".body");
|
||||
var partialPath = Path.Combine(_cacheDirectory, "." + contentHash + ".partial");
|
||||
var stateKey = StateKey(contentHash);
|
||||
|
||||
if (File.Exists(finalPath))
|
||||
{
|
||||
if (await HashFileAsync(finalPath, cancellationToken).ConfigureAwait(false) == contentHash)
|
||||
return finalPath;
|
||||
File.Delete(finalPath);
|
||||
}
|
||||
|
||||
var fingerprint = ManifestFingerprint(manifest);
|
||||
var state = _stateCache.Get<CultMeshContentTransferStateDocument>(stateKey);
|
||||
var verified = RestoreVerifiedState(state, manifest, fingerprint, partialPath);
|
||||
|
||||
using (var stream = new FileStream(partialPath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read, 81920, FileOptions.Asynchronous))
|
||||
{
|
||||
stream.SetLength(manifest.SizeBytes);
|
||||
var ordered = manifest.Chunks
|
||||
.Select((chunk, index) => new { Chunk = chunk, Index = index })
|
||||
.OrderBy(item => item.Chunk.Offset)
|
||||
.ToArray();
|
||||
|
||||
foreach (var item in ordered.Where(item => !verified.Contains(item.Index)))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var chunk = await FetchVerifiedChunkAsync(item.Chunk, cancellationToken).ConfigureAwait(false);
|
||||
stream.Position = item.Chunk.Offset;
|
||||
await stream.WriteAsync(chunk.Payload, 0, chunk.Payload.Length, cancellationToken).ConfigureAwait(false);
|
||||
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||||
verified.Add(item.Index);
|
||||
await SaveStateAsync(contentHash, fingerprint, manifest.SizeBytes, verified, stateKey).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var actualHash = await HashFileAsync(partialPath, cancellationToken).ConfigureAwait(false);
|
||||
if (!string.Equals(actualHash, contentHash, StringComparison.Ordinal))
|
||||
{
|
||||
File.Delete(partialPath);
|
||||
_stateCache.Remove(stateKey);
|
||||
await _stateCache.FlushAsync().ConfigureAwait(false);
|
||||
throw new InvalidDataException("CDN artifact content hash does not match its manifest.");
|
||||
}
|
||||
|
||||
PromoteAtomically(partialPath, finalPath);
|
||||
_stateCache.Remove(stateKey);
|
||||
await _stateCache.FlushAsync().ConfigureAwait(false);
|
||||
return finalPath;
|
||||
}
|
||||
|
||||
private HashSet<int> RestoreVerifiedState(
|
||||
CultMeshContentTransferStateDocument? state,
|
||||
CultMeshCdnArtifactManifest manifest,
|
||||
string fingerprint,
|
||||
string partialPath)
|
||||
{
|
||||
if (state == null || !File.Exists(partialPath) ||
|
||||
state.SizeBytes != manifest.SizeBytes ||
|
||||
!string.Equals(state.ManifestFingerprint, fingerprint, StringComparison.Ordinal) ||
|
||||
new FileInfo(partialPath).Length != manifest.SizeBytes)
|
||||
{
|
||||
if (File.Exists(partialPath)) File.Delete(partialPath);
|
||||
return new HashSet<int>();
|
||||
}
|
||||
|
||||
var verified = new HashSet<int>();
|
||||
using var stream = new FileStream(partialPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
|
||||
foreach (var index in state.VerifiedChunkIndexes.Distinct().Where(index => index >= 0 && index < manifest.Chunks.Length))
|
||||
{
|
||||
var reference = manifest.Chunks[index];
|
||||
var bytes = new byte[reference.SizeBytes];
|
||||
stream.Position = reference.Offset;
|
||||
if (stream.Read(bytes, 0, bytes.Length) == bytes.Length &&
|
||||
string.Equals(CultMeshCdn.HashBytes(bytes), CultMeshCdn.NormalizeHash(reference.ChunkHash, nameof(reference.ChunkHash)), StringComparison.Ordinal))
|
||||
{
|
||||
verified.Add(index);
|
||||
}
|
||||
}
|
||||
|
||||
return verified;
|
||||
}
|
||||
|
||||
private async Task<CultMeshCdnArtifactChunk> FetchVerifiedChunkAsync(
|
||||
CultMeshCdnChunkRef reference,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var failures = new List<Exception>();
|
||||
foreach (var provider in _providers)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
try
|
||||
{
|
||||
var chunk = await provider.GetChunkAsync(reference, cancellationToken).ConfigureAwait(false)
|
||||
?? throw new FileNotFoundException("CDN artifact chunk is missing from the provider.", reference.RecordKey);
|
||||
CultMeshCdn.ValidateChunkPayload(reference, chunk);
|
||||
return chunk;
|
||||
}
|
||||
catch (Exception error) when (!(error is OperationCanceledException && cancellationToken.IsCancellationRequested))
|
||||
{
|
||||
failures.Add(new InvalidDataException("Content provider '" + provider.ProviderId + "' could not supply a valid chunk.", error));
|
||||
}
|
||||
}
|
||||
|
||||
throw new AggregateException("No content provider supplied a valid CDN chunk.", failures);
|
||||
}
|
||||
|
||||
private async Task SaveStateAsync(
|
||||
string contentHash,
|
||||
string fingerprint,
|
||||
long sizeBytes,
|
||||
HashSet<int> verified,
|
||||
CultRecordKey stateKey)
|
||||
{
|
||||
var state = new CultMeshContentTransferStateDocument
|
||||
{
|
||||
ContentHash = contentHash,
|
||||
ManifestFingerprint = fingerprint,
|
||||
SizeBytes = sizeBytes,
|
||||
VerifiedChunkIndexes = verified.OrderBy(value => value).ToArray(),
|
||||
UpdatedAtUtc = DateTimeOffset.UtcNow.ToString("O", CultureInfo.InvariantCulture)
|
||||
};
|
||||
await _stateCache.UpsertAsync(typeof(CultMeshContentTransferStateDocument), state, stateKey).ConfigureAwait(false);
|
||||
await _stateCache.FlushAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static CultRecordKey StateKey(string contentHash) => new CultRecordKey("mesh:content-transfer:" + contentHash);
|
||||
|
||||
private static string ManifestFingerprint(CultMeshCdnArtifactManifest manifest)
|
||||
{
|
||||
var shape = string.Join("|", manifest.Chunks.Select(chunk =>
|
||||
chunk.Offset.ToString(CultureInfo.InvariantCulture) + ":" +
|
||||
chunk.SizeBytes.ToString(CultureInfo.InvariantCulture) + ":" +
|
||||
CultMeshCdn.NormalizeHash(chunk.ChunkHash, nameof(chunk.ChunkHash))));
|
||||
return CultMeshCdn.HashBytes(System.Text.Encoding.UTF8.GetBytes(manifest.SizeBytes.ToString(CultureInfo.InvariantCulture) + "|" + shape));
|
||||
}
|
||||
|
||||
private static async Task<string> HashFileAsync(string path, CancellationToken cancellationToken)
|
||||
{
|
||||
using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 81920, FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
using var sha = SHA256.Create();
|
||||
var buffer = new byte[81920];
|
||||
int read;
|
||||
while ((read = await stream.ReadAsync(buffer, 0, buffer.Length, cancellationToken).ConfigureAwait(false)) > 0)
|
||||
sha.TransformBlock(buffer, 0, read, null, 0);
|
||||
sha.TransformFinalBlock(Array.Empty<byte>(), 0, 0);
|
||||
return string.Concat(sha.Hash!.Select(value => value.ToString("x2", CultureInfo.InvariantCulture)));
|
||||
}
|
||||
|
||||
private static void PromoteAtomically(string partialPath, string finalPath)
|
||||
{
|
||||
if (File.Exists(finalPath)) File.Replace(partialPath, finalPath, null);
|
||||
else File.Move(partialPath, finalPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -521,6 +521,51 @@ session path.
|
|||
|
||||
## Phase 4: Verified Content Transfer
|
||||
|
||||
### Phase 4 first-slice body map
|
||||
|
||||
**Owner:** `CultMeshContentTransferService` owns verified partial state and the
|
||||
only promotion path into the completed content cache. Work is serialized and
|
||||
shared by normalized content hash; provider identity is never part of content
|
||||
identity or the destination name.
|
||||
|
||||
**Inputs:** a validated CDN manifest, ordered `ICultMeshContentProvider`
|
||||
observations, a typed CultCache checkpoint store, a canonical cache directory,
|
||||
and caller cancellation.
|
||||
|
||||
**Outputs:** a complete read-only-by-convention body path named by SHA-256, or
|
||||
a failure with no completed body. `gamecult.mesh.content_transfer_state.v1`
|
||||
records verified chunk indexes keyed by content hash.
|
||||
|
||||
**Derived state:** missing chunks, provider failover order, deterministic
|
||||
partial-file path, manifest fingerprint, and completed cache path. A provider
|
||||
success is evidence only for the returned chunk; it does not confer publisher
|
||||
authority.
|
||||
|
||||
**Forbidden writers:** callers, renderers, providers, manifests, and the legacy
|
||||
async CDN reader cannot publish a final body or mark a range verified. CultCache
|
||||
checkpoint claims are rehashed against the partial body before reuse.
|
||||
|
||||
**Shared paths:** direct transfers and the legacy distributed-database reader
|
||||
both pass through the same chunk verification, full-hash verification, and
|
||||
atomic promotion primitive. Concurrent requests for one hash share the same
|
||||
per-content transfer lock and verified work.
|
||||
|
||||
**Cut line:** the legacy distributed reader no longer owns a sequential chunk
|
||||
loop or whole-artifact allocation. The first slice intentionally leaves
|
||||
adaptive parallelism, quotas, pinning, signer policy, arbitrary streaming
|
||||
destinations, and cleanup scheduling to later Phase 4 work.
|
||||
|
||||
**State transition:** each verified chunk flushes a typed checkpoint. Restart
|
||||
rehydrates that record, rehashes every claimed range, and fetches corrupt or
|
||||
unclaimed chunks again. Completion verifies the entire partial file, atomically
|
||||
promotes it, and removes the checkpoint. Cancellation leaves only re-verifiable
|
||||
partial state.
|
||||
|
||||
**Verification layer:** focused tests cover durable cancellation/restart,
|
||||
provider corruption and failover, forged checkpoint rejection, final-hash
|
||||
failure, atomic visibility, and concurrent request coalescing. Existing CDN
|
||||
tests cover the compatibility reader.
|
||||
|
||||
### Cut first
|
||||
|
||||
- Delete the first consumer-owned CDN chunk loop and partial-file convention.
|
||||
|
|
|
|||
212
tests/GameCult.Mesh.Tests/CultMeshContentTransferTests.cs
Normal file
212
tests/GameCult.Mesh.Tests/CultMeshContentTransferTests.cs
Normal file
|
|
@ -0,0 +1,212 @@
|
|||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FluentAssertions;
|
||||
using GameCult.Caching;
|
||||
using GameCult.Caching.MessagePack;
|
||||
using NUnit.Framework;
|
||||
|
||||
namespace GameCult.Mesh.Tests;
|
||||
|
||||
[TestFixture]
|
||||
public sealed class CultMeshContentTransferTests
|
||||
{
|
||||
private string _directory = null!;
|
||||
|
||||
[SetUp]
|
||||
public void SetUp()
|
||||
{
|
||||
_directory = Path.Combine(Path.GetTempPath(), "cultmesh-transfer-tests", Guid.NewGuid().ToString("N"));
|
||||
}
|
||||
|
||||
[TearDown]
|
||||
public void TearDown()
|
||||
{
|
||||
if (Directory.Exists(_directory)) Directory.Delete(_directory, true);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task FetchAsync_VerifiesChunksAndAtomicallyPublishesFinalBody()
|
||||
{
|
||||
var artifact = Artifact("verified", 4);
|
||||
using var cache = Cache();
|
||||
var provider = new FakeProvider("source", artifact.Chunks);
|
||||
var service = Service(cache, provider);
|
||||
|
||||
var path = await service.FetchAsync(artifact.Manifest);
|
||||
|
||||
File.ReadAllBytes(path).Should().Equal(Payload());
|
||||
Directory.GetFiles(_directory, "*.partial").Should().BeEmpty();
|
||||
cache.Get<CultMeshContentTransferStateDocument>(StateKey(artifact.Manifest)).Should().BeNull();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task FetchAsync_FailsOverAfterProviderCorruption()
|
||||
{
|
||||
var artifact = Artifact("failover", 4);
|
||||
using var cache = Cache();
|
||||
var corrupt = artifact.Chunks.Select(Clone).ToArray();
|
||||
corrupt[0].Payload[0] ^= 0xff;
|
||||
var first = new FakeProvider("corrupt", corrupt);
|
||||
var second = new FakeProvider("good", artifact.Chunks);
|
||||
|
||||
var path = await Service(cache, first, second).FetchAsync(artifact.Manifest);
|
||||
|
||||
File.ReadAllBytes(path).Should().Equal(Payload());
|
||||
first.Requests.Should().ContainKey(artifact.Manifest.Chunks[0].ChunkHash);
|
||||
second.Requests.Should().ContainKey(artifact.Manifest.Chunks[0].ChunkHash);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task FetchAsync_CancellationLeavesVerifiedCheckpointAndResumeSkipsIt()
|
||||
{
|
||||
var artifact = Artifact("resume", 4);
|
||||
var statePath = Path.Combine(_directory, "transfer-state.cc");
|
||||
using var cache = DurableCache(statePath);
|
||||
using var cancellation = new CancellationTokenSource();
|
||||
var provider = new FakeProvider("source", artifact.Chunks) { BlockFromRequest = 2 };
|
||||
var service = Service(cache, provider);
|
||||
var fetch = service.FetchAsync(artifact.Manifest, cancellation.Token);
|
||||
|
||||
await WaitUntilAsync(() => cache.Get<CultMeshContentTransferStateDocument>(StateKey(artifact.Manifest))?.VerifiedChunkIndexes.Length == 1);
|
||||
cancellation.Cancel();
|
||||
await FluentActions.Awaiting(() => fetch).Should().ThrowAsync<OperationCanceledException>();
|
||||
File.Exists(Path.Combine(_directory, artifact.Manifest.ContentHash + ".body")).Should().BeFalse();
|
||||
cache.Dispose();
|
||||
|
||||
using var restoredCache = DurableCache(statePath);
|
||||
await restoredCache.PullAllBackingStoresAsync();
|
||||
restoredCache.Get<CultMeshContentTransferStateDocument>(StateKey(artifact.Manifest))
|
||||
.Should().NotBeNull("the verified checkpoint must survive a cache restart");
|
||||
var resumedProvider = new FakeProvider("resume", artifact.Chunks);
|
||||
var path = await Service(restoredCache, resumedProvider).FetchAsync(artifact.Manifest);
|
||||
|
||||
File.ReadAllBytes(path).Should().Equal(Payload());
|
||||
resumedProvider.Requests.Should().NotContainKey(artifact.Manifest.Chunks[0].ChunkHash);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task FetchAsync_RejectsForgedCheckpointAfterRehashingClaimedChunks()
|
||||
{
|
||||
var artifact = Artifact("forged", 4);
|
||||
using var cache = Cache();
|
||||
using var cancellation = new CancellationTokenSource();
|
||||
var firstProvider = new FakeProvider("first", artifact.Chunks) { BlockFromRequest = 2 };
|
||||
var firstFetch = Service(cache, firstProvider).FetchAsync(artifact.Manifest, cancellation.Token);
|
||||
await WaitUntilAsync(() => cache.Get<CultMeshContentTransferStateDocument>(StateKey(artifact.Manifest))?.VerifiedChunkIndexes.Length == 1);
|
||||
cancellation.Cancel();
|
||||
await FluentActions.Awaiting(() => firstFetch).Should().ThrowAsync<OperationCanceledException>();
|
||||
|
||||
var partial = Path.Combine(_directory, "." + artifact.Manifest.ContentHash + ".partial");
|
||||
var bytes = File.ReadAllBytes(partial);
|
||||
bytes[0] ^= 0xff;
|
||||
File.WriteAllBytes(partial, bytes);
|
||||
|
||||
var provider = new FakeProvider("source", artifact.Chunks);
|
||||
await Service(cache, provider).FetchAsync(artifact.Manifest);
|
||||
|
||||
provider.Requests.Values.Sum().Should().Be(artifact.Chunks.Count);
|
||||
File.ReadAllBytes(Path.Combine(_directory, artifact.Manifest.ContentHash + ".body")).Should().Equal(Payload());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task FetchAsync_FinalHashFailurePublishesNoBodyOrTrustedCheckpoint()
|
||||
{
|
||||
var artifact = Artifact("wrong-final", 4);
|
||||
artifact.Manifest.ContentHash = new string('0', 64);
|
||||
using var cache = Cache();
|
||||
|
||||
await FluentActions.Awaiting(() => Service(cache, new FakeProvider("source", artifact.Chunks)).FetchAsync(artifact.Manifest))
|
||||
.Should().ThrowAsync<InvalidDataException>();
|
||||
|
||||
File.Exists(Path.Combine(_directory, new string('0', 64) + ".body")).Should().BeFalse();
|
||||
cache.Get<CultMeshContentTransferStateDocument>(new CultRecordKey("mesh:content-transfer:" + new string('0', 64))).Should().BeNull();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task FetchAsync_ConcurrentRequestsShareVerifiedWork()
|
||||
{
|
||||
var artifact = Artifact("shared", 4);
|
||||
using var cache = Cache();
|
||||
var provider = new FakeProvider("source", artifact.Chunks) { DelayMilliseconds = 20 };
|
||||
var service = Service(cache, provider);
|
||||
|
||||
var paths = await Task.WhenAll(service.FetchAsync(artifact.Manifest), service.FetchAsync(artifact.Manifest));
|
||||
|
||||
paths[0].Should().Be(paths[1]);
|
||||
provider.Requests.Values.Sum().Should().Be(artifact.Chunks.Count);
|
||||
}
|
||||
|
||||
private CultMeshContentTransferService Service(CultCache cache, params ICultMeshContentProvider[] providers) =>
|
||||
new(cache, providers, new CultMeshContentTransferOptions(_directory));
|
||||
|
||||
private static CultCache Cache()
|
||||
{
|
||||
var registry = CultMesh.CreateCultCacheDocumentRegistry(typeof(CultMeshContentTransferStateDocument));
|
||||
return new CultCache(registry);
|
||||
}
|
||||
|
||||
private static CultCache DurableCache(string path)
|
||||
{
|
||||
var cache = Cache();
|
||||
cache.AddBackingStore(new SingleFileMessagePackBackingStore(path));
|
||||
return cache;
|
||||
}
|
||||
|
||||
private static CultMeshCdnArtifact Artifact(string id, int chunkSize) =>
|
||||
CultMesh.PackCdnArtifact(id, Payload(), new CultMeshCdnPackOptions { ChunkSizeBytes = chunkSize });
|
||||
|
||||
private static byte[] Payload() => Enumerable.Range(0, 19).Select(value => (byte)(value + 1)).ToArray();
|
||||
|
||||
private static CultRecordKey StateKey(CultMeshCdnArtifactManifest manifest) =>
|
||||
new("mesh:content-transfer:" + manifest.ContentHash);
|
||||
|
||||
private static CultMeshCdnArtifactChunk Clone(CultMeshCdnArtifactChunk chunk) => new()
|
||||
{
|
||||
ChunkHash = chunk.ChunkHash,
|
||||
SizeBytes = chunk.SizeBytes,
|
||||
Payload = chunk.Payload.ToArray()
|
||||
};
|
||||
|
||||
private static async Task WaitUntilAsync(Func<bool> predicate)
|
||||
{
|
||||
var deadline = DateTime.UtcNow.AddSeconds(5);
|
||||
while (!predicate())
|
||||
{
|
||||
if (DateTime.UtcNow >= deadline) throw new TimeoutException("Condition was not observed.");
|
||||
await Task.Delay(10);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeProvider : ICultMeshContentProvider
|
||||
{
|
||||
private readonly Dictionary<string, CultMeshCdnArtifactChunk> _chunks;
|
||||
private int _requestCount;
|
||||
|
||||
public FakeProvider(string providerId, IEnumerable<CultMeshCdnArtifactChunk> chunks)
|
||||
{
|
||||
ProviderId = providerId;
|
||||
_chunks = chunks.ToDictionary(chunk => chunk.ChunkHash, Clone, StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
public string ProviderId { get; }
|
||||
public ConcurrentDictionary<string, int> Requests { get; } = new(StringComparer.Ordinal);
|
||||
public int BlockFromRequest { get; set; } = int.MaxValue;
|
||||
public int DelayMilliseconds { get; set; }
|
||||
|
||||
public async Task<CultMeshCdnArtifactChunk?> GetChunkAsync(CultMeshCdnChunkRef chunk, CancellationToken cancellationToken = default)
|
||||
{
|
||||
Requests.AddOrUpdate(chunk.ChunkHash, 1, (_, count) => count + 1);
|
||||
var request = Interlocked.Increment(ref _requestCount);
|
||||
if (request >= BlockFromRequest)
|
||||
await Task.Delay(Timeout.Infinite, cancellationToken);
|
||||
if (DelayMilliseconds > 0)
|
||||
await Task.Delay(DelayMilliseconds, cancellationToken);
|
||||
return _chunks.TryGetValue(chunk.ChunkHash, out var value) ? Clone(value) : null;
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue