From 2727e325bbc9f5fa948c94f2c8f59b618fcadfd2 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 13 Jul 2026 15:36:50 +0100 Subject: [PATCH] Map verified CultMesh CDN bodies without copies --- src/GameCult.Mesh/CultMeshBodies.cs | 96 +++++++++++++++++-- src/GameCult.Mesh/CultMeshContentTransfer.cs | 31 +++++- .../docs/reliability-migration-plan.md | 29 ++++++ .../CultMeshContentTransferTests.cs | 78 +++++++++++++++ 4 files changed, 226 insertions(+), 8 deletions(-) diff --git a/src/GameCult.Mesh/CultMeshBodies.cs b/src/GameCult.Mesh/CultMeshBodies.cs index 5b76e347..f673a3ca 100644 --- a/src/GameCult.Mesh/CultMeshBodies.cs +++ b/src/GameCult.Mesh/CultMeshBodies.cs @@ -3,6 +3,7 @@ using System.IO; using System.IO.MemoryMappedFiles; using System.Security.Cryptography; using System.Collections.Generic; +using System.Collections.Concurrent; using System.Linq; using MessagePack; @@ -244,13 +245,17 @@ namespace GameCult.Mesh public sealed class CultMeshMappedBodyAdapter : ICultMeshBodyTransportAdapter { - private readonly string _rootPath; + private readonly string? _rootPath; + private readonly CultMeshVerifiedBodyMappingBroker? _verifiedBodies; public CultMeshMappedBodyAdapter(string rootPath) => _rootPath = Path.GetFullPath(string.IsNullOrWhiteSpace(rootPath) ? throw new ArgumentException("Mapped body root is required.", nameof(rootPath)) : rootPath); + public CultMeshMappedBodyAdapter(CultMeshVerifiedBodyMappingBroker verifiedBodies) => + _verifiedBodies = verifiedBodies ?? throw new ArgumentNullException(nameof(verifiedBodies)); + public CultMeshBodyTransportKind TransportKind => CultMeshBodyTransportKind.SharedFileMapping; public bool CanOpen(CultMeshBodyDescriptor descriptor) => @@ -263,17 +268,94 @@ namespace GameCult.Mesh CultMeshBodyDescriptorValidator.Validate(descriptor, request); if (descriptor.TransportKind != CultMeshBodyTransportKind.SharedFileMapping) throw new NotSupportedException($"CultMesh body transport '{descriptor.TransportKind}' is not a file mapping."); - var token = descriptor.CapabilityToken; - if (token.IndexOfAny(Path.GetInvalidFileNameChars()) >= 0 || token.Contains("..")) - throw new UnauthorizedAccessException("Invalid CultMesh body capability token."); - var path = Path.GetFullPath(Path.Combine(_rootPath, token + ".body")); - if (!path.StartsWith(_rootPath + Path.DirectorySeparatorChar, StringComparison.OrdinalIgnoreCase)) - throw new UnauthorizedAccessException("CultMesh body token escaped the broker root."); + var path = _verifiedBodies != null + ? _verifiedBodies.Resolve(descriptor.CapabilityToken, request.NowUtc) + : ResolvePublisherPath(descriptor.CapabilityToken); var info = new FileInfo(path); if (!info.Exists || info.Length != descriptor.ByteSize) throw new InvalidDataException("Mapped CultMesh body size does not match its descriptor."); return new CultMeshMappedBodyLease(descriptor, path); } + + private string ResolvePublisherPath(string token) + { + if (token.IndexOfAny(Path.GetInvalidFileNameChars()) >= 0 || token.Contains("..")) + throw new UnauthorizedAccessException("Invalid CultMesh body capability token."); + var path = Path.GetFullPath(Path.Combine(_rootPath!, token + ".body")); + if (!path.StartsWith(_rootPath + Path.DirectorySeparatorChar, StringComparison.OrdinalIgnoreCase)) + throw new UnauthorizedAccessException("CultMesh body token escaped the broker root."); + return path; + } + } + + public sealed class CultMeshVerifiedBodyMappingBroker + { + private sealed class Grant + { + public string Path { get; set; } = string.Empty; + public long ExpiresAtUnixMs { get; set; } + } + + private readonly string _cacheDirectory; + private readonly ConcurrentDictionary _grants = new(StringComparer.Ordinal); + + public CultMeshVerifiedBodyMappingBroker(string cacheDirectory) => + _cacheDirectory = Path.GetFullPath(string.IsNullOrWhiteSpace(cacheDirectory) + ? throw new ArgumentException("Verified body cache directory is required.", nameof(cacheDirectory)) + : cacheDirectory); + + internal CultMeshBodyDescriptor GrantVerified( + string contentHash, + string verifiedPath, + CultMeshBodyDescriptor networkDescriptor, + DateTimeOffset expiresAtUtc) + { + if (networkDescriptor == null) throw new ArgumentNullException(nameof(networkDescriptor)); + var expectedPath = Path.Combine(_cacheDirectory, contentHash + ".body"); + if (!string.Equals(Path.GetFullPath(verifiedPath), expectedPath, StringComparison.OrdinalIgnoreCase)) + throw new InvalidOperationException("Only the transfer-owned verified body may be mapped."); + var info = new FileInfo(expectedPath); + if (!info.Exists || info.Length != networkDescriptor.ByteSize) + throw new InvalidDataException("Verified CultMesh body size does not match its network descriptor."); + + var token = CreateCapabilityToken(); + var expiresAtUnixMs = expiresAtUtc.ToUnixTimeMilliseconds(); + _grants[token] = new Grant { Path = expectedPath, ExpiresAtUnixMs = expiresAtUnixMs }; + return new CultMeshBodyDescriptor + { + BodyId = networkDescriptor.BodyId, + SchemaId = networkDescriptor.SchemaId, + LayoutVersion = networkDescriptor.LayoutVersion, + ByteSize = networkDescriptor.ByteSize, + Capacity = networkDescriptor.Capacity, + ProducerEpoch = networkDescriptor.ProducerEpoch, + Sequence = networkDescriptor.Sequence, + AccessMode = CultMeshBodyAccessMode.ReadOnly, + Synchronization = networkDescriptor.Synchronization, + LeaseExpiresAtUnixMs = expiresAtUnixMs, + TransportKind = CultMeshBodyTransportKind.SharedFileMapping, + CapabilityToken = token + }; + } + + internal string Resolve(string token, DateTimeOffset nowUtc) + { + if (!_grants.TryGetValue(token, out var grant)) + throw new UnauthorizedAccessException("Unknown verified body capability token."); + if (grant.ExpiresAtUnixMs <= nowUtc.ToUnixTimeMilliseconds()) + { + _grants.TryRemove(token, out _); + throw new InvalidOperationException("Verified body capability has expired."); + } + return grant.Path; + } + + private static string CreateCapabilityToken() + { + var bytes = new byte[32]; + using (var random = RandomNumberGenerator.Create()) random.GetBytes(bytes); + return BitConverter.ToString(bytes).Replace("-", string.Empty).ToLowerInvariant(); + } } public sealed class CultMeshSharedMemoryBodyAdapter : ICultMeshBodyTransportAdapter diff --git a/src/GameCult.Mesh/CultMeshContentTransfer.cs b/src/GameCult.Mesh/CultMeshContentTransfer.cs index 02abebaf..349c8736 100644 --- a/src/GameCult.Mesh/CultMeshContentTransfer.cs +++ b/src/GameCult.Mesh/CultMeshContentTransfer.cs @@ -76,18 +76,47 @@ namespace GameCult.Mesh private readonly CultCache _stateCache; private readonly ICultMeshContentProvider[] _providers; private readonly string _cacheDirectory; + private readonly CultMeshVerifiedBodyMappingBroker? _mappedBodies; private readonly ConcurrentDictionary _contentLocks = new(StringComparer.Ordinal); public CultMeshContentTransferService( CultCache stateCache, IEnumerable providers, - CultMeshContentTransferOptions options) + CultMeshContentTransferOptions options, + CultMeshVerifiedBodyMappingBroker? mappedBodies = null) { _stateCache = stateCache ?? throw new ArgumentNullException(nameof(stateCache)); _providers = providers?.ToArray() ?? throw new ArgumentNullException(nameof(providers)); if (_providers.Length == 0) throw new ArgumentException("At least one content provider is required.", nameof(providers)); if (_providers.Any(provider => provider == null)) throw new ArgumentException("Content providers cannot contain null entries.", nameof(providers)); _cacheDirectory = (options ?? throw new ArgumentNullException(nameof(options))).CacheDirectory; + _mappedBodies = mappedBodies; + } + + public async Task FetchMappedBodyAsync( + CultMeshCdnArtifactManifest manifest, + CultMeshBodyDescriptor networkDescriptor, + DateTimeOffset nowUtc, + TimeSpan leaseDuration, + CancellationToken cancellationToken = default) + { + if (_mappedBodies == null) + throw new InvalidOperationException("Verified body mapping is not configured for this transfer service."); + if (networkDescriptor == null) throw new ArgumentNullException(nameof(networkDescriptor)); + if (networkDescriptor.TransportKind != CultMeshBodyTransportKind.Network) + throw new ArgumentException("The fallback descriptor must use the network body transport.", nameof(networkDescriptor)); + if (leaseDuration <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(leaseDuration)); + CultMeshCdn.ValidateManifest(manifest); + if (networkDescriptor.ByteSize != manifest.SizeBytes) + throw new InvalidDataException("CDN manifest size does not match its network body descriptor."); + + var path = await FetchAsync(manifest, cancellationToken).ConfigureAwait(false); + var contentHash = CultMeshCdn.NormalizeHash(manifest.ContentHash, nameof(manifest.ContentHash)); + var requestedExpiry = nowUtc.Add(leaseDuration); + var networkExpiry = DateTimeOffset.FromUnixTimeMilliseconds(networkDescriptor.LeaseExpiresAtUnixMs); + var expiry = requestedExpiry <= networkExpiry ? requestedExpiry : networkExpiry; + if (expiry <= nowUtc) throw new InvalidOperationException("Network body lease has expired."); + return _mappedBodies.GrantVerified(contentHash, path, networkDescriptor, expiry); } public async Task FetchAsync( diff --git a/src/GameCult.Mesh/docs/reliability-migration-plan.md b/src/GameCult.Mesh/docs/reliability-migration-plan.md index 61cf9095..531e4b17 100644 --- a/src/GameCult.Mesh/docs/reliability-migration-plan.md +++ b/src/GameCult.Mesh/docs/reliability-migration-plan.md @@ -566,6 +566,35 @@ provider corruption and failover, forged checkpoint rejection, final-hash failure, atomic visibility, and concurrent request coalescing. Existing CDN tests cover the compatibility reader. +### Phase 5 verified CDN mapped-body slice + +**Owner:** `CultMeshContentTransferService` remains the sole writer and promoter +of completed CDN bodies. `CultMeshVerifiedBodyMappingBroker` owns only ephemeral, +opaque capability-to-file grants for those completed bodies. Body negotiation +and producer authorization remain owned by `CultMeshBodyTransportService`. + +**Inputs:** the atomically committed `.body` path returned by transfer, +the equivalent network body descriptor, and a bounded lease. + +**Outputs:** a read-only file-mapping descriptor for the same logical generation. +Multiple consumers map the same completed cache file and therefore share the OS +page cache; no process-sized byte array or republished body file is created. + +**Derived state:** opaque capability tokens and their expiry. Tokens are neither +content hashes nor paths and do not confer producer authority or content trust. + +**Forbidden writers:** the mapping broker and adapter cannot create, promote, +rename, repair, or map partial/checkpoint files. `CultMeshMappedBodyPublisher` +is not used for verified CDN content. + +**Shared paths:** local and network descriptors preserve body identity, +schema/layout, capacity, producer epoch, sequence, and byte size. Local open +failure is observed by body negotiation and falls back to that network +representation. + +**Cut line:** there is no second CDN publication path. Mapping is granted only +after the transfer owner returns its verified final file. + ### Cut first - Delete the first consumer-owned CDN chunk loop and partial-file convention. diff --git a/tests/GameCult.Mesh.Tests/CultMeshContentTransferTests.cs b/tests/GameCult.Mesh.Tests/CultMeshContentTransferTests.cs index a23d9e60..396137ef 100644 --- a/tests/GameCult.Mesh.Tests/CultMeshContentTransferTests.cs +++ b/tests/GameCult.Mesh.Tests/CultMeshContentTransferTests.cs @@ -141,6 +141,59 @@ public sealed class CultMeshContentTransferTests provider.Requests.Values.Sum().Should().Be(artifact.Chunks.Count); } + [Test] + public async Task FetchMappedBodyAsync_MapsOnlyCommittedBodyForMultipleConsumersAndFallsBackToNetwork() + { + var artifact = Artifact("mapped", 4); + using var cache = Cache(); + var broker = new CultMeshVerifiedBodyMappingBroker(_directory); + var service = new CultMeshContentTransferService( + cache, + new[] { new FakeProvider("source", artifact.Chunks) }, + new CultMeshContentTransferOptions(_directory), + broker); + var now = DateTimeOffset.UtcNow; + var network = NetworkDescriptor(artifact, now); + + var first = await service.FetchMappedBodyAsync(artifact.Manifest, network, now, TimeSpan.FromMinutes(1)); + var second = await service.FetchMappedBodyAsync(artifact.Manifest, network, now, TimeSpan.FromMinutes(1)); + + first.Should().BeEquivalentTo(second, options => options.Excluding(value => value.CapabilityToken)); + first.CapabilityToken.Should().NotBe(artifact.Manifest.ContentHash); + first.CapabilityToken.Should().NotContain(Path.DirectorySeparatorChar.ToString()); + first.CapabilityToken.Should().NotContain(Path.AltDirectorySeparatorChar.ToString()); + Directory.GetFiles(_directory, "*.body").Should().ContainSingle() + .Which.Should().Be(Path.Combine(_directory, artifact.Manifest.ContentHash + ".body")); + Directory.GetFiles(_directory, "*.partial").Should().BeEmpty(); + + var adapter = new CultMeshMappedBodyAdapter(broker); + using (var firstLease = adapter.OpenReadOnly(first, Request(first, now))) + using (var secondLease = adapter.OpenReadOnly(second, Request(second, now))) + { + var firstBytes = new byte[first.ByteSize]; + var secondBytes = new byte[second.ByteSize]; + firstLease.CopyTo(0, firstBytes, 0, firstBytes.Length); + secondLease.CopyTo(0, secondBytes, 0, secondBytes.Length); + firstBytes.Should().Equal(Payload()); + secondBytes.Should().Equal(Payload()); + } + + File.Delete(Path.Combine(_directory, artifact.Manifest.ContentHash + ".body")); + var transport = new CultMeshBodyTransportService( + new ICultMeshBodyTransportAdapter[] + { + adapter, + new CultMeshNetworkBodyAdapter(_ => Payload()) + }, + _ => true); + + using var fallback = transport.OpenReadOnly(first, network, Request(first, now), out var localFailure); + + localFailure.Should().BeOfType(); + fallback.TransportKind.Should().Be(CultMeshBodyTransportKind.Network); + fallback.Descriptor.BodyId.Should().Be(first.BodyId); + } + private CultMeshContentTransferService Service(CultCache cache, params ICultMeshContentProvider[] providers) => new(cache, providers, new CultMeshContentTransferOptions(_directory)); @@ -165,6 +218,31 @@ public sealed class CultMeshContentTransferTests private static CultRecordKey StateKey(CultMeshCdnArtifactManifest manifest) => new("mesh:content-transfer:" + manifest.ContentHash); + private static CultMeshBodyDescriptor NetworkDescriptor(CultMeshCdnArtifact artifact, DateTimeOffset now) => new() + { + BodyId = artifact.Manifest.ArtifactId, + SchemaId = "gamecult.mesh.cdn-artifact.v1", + LayoutVersion = 1, + ByteSize = artifact.Manifest.SizeBytes, + Capacity = checked((int)artifact.Manifest.SizeBytes), + ProducerEpoch = 4, + Sequence = 7, + AccessMode = CultMeshBodyAccessMode.ReadOnly, + Synchronization = CultMeshBodySynchronization.ImmutableSequence, + LeaseExpiresAtUnixMs = now.AddMinutes(2).ToUnixTimeMilliseconds(), + TransportKind = CultMeshBodyTransportKind.Network + }; + + private static CultMeshBodyValidationRequest Request(CultMeshBodyDescriptor descriptor, DateTimeOffset now) => new() + { + BodyId = descriptor.BodyId, + SchemaId = descriptor.SchemaId, + LayoutVersion = descriptor.LayoutVersion, + ProducerEpoch = descriptor.ProducerEpoch, + AccessMode = CultMeshBodyAccessMode.ReadOnly, + NowUtc = now + }; + private static CultMeshCdnArtifactChunk Clone(CultMeshCdnArtifactChunk chunk) => new() { ChunkHash = chunk.ChunkHash,