GameCult/CultLib
Watch
9
0
Fork
You've already forked CultLib
 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Map verified CultMesh CDN bodies without copies

Browse source
This commit is contained in:
Meta Vens 2026-07-13 15:36:50 +01:00
commit 2727e325bb
4 changed files with 226 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

96
src/GameCult.Mesh/CultMeshBodies.cs
View file

@ -3,6 +3,7 @@ using System.IO;
using System.IO.MemoryMappedFiles;
using System.Security.Cryptography;
using System.Collections.Generic;
using System.Collections.Concurrent;
using System.Linq;
using MessagePack;
@ -244,13 +245,17 @@ namespace GameCult.Mesh
public sealed class CultMeshMappedBodyAdapter : ICultMeshBodyTransportAdapter
{
private readonly string _rootPath;
private readonly string? _rootPath;
private readonly CultMeshVerifiedBodyMappingBroker? _verifiedBodies;
public CultMeshMappedBodyAdapter(string rootPath) =>
_rootPath = Path.GetFullPath(string.IsNullOrWhiteSpace(rootPath)
? throw new ArgumentException("Mapped body root is required.", nameof(rootPath))
: rootPath);
public CultMeshMappedBodyAdapter(CultMeshVerifiedBodyMappingBroker verifiedBodies) =>
_verifiedBodies = verifiedBodies ?? throw new ArgumentNullException(nameof(verifiedBodies));
public CultMeshBodyTransportKind TransportKind => CultMeshBodyTransportKind.SharedFileMapping;
public bool CanOpen(CultMeshBodyDescriptor descriptor) =>
@ -263,17 +268,94 @@ namespace GameCult.Mesh
CultMeshBodyDescriptorValidator.Validate(descriptor, request);
if (descriptor.TransportKind != CultMeshBodyTransportKind.SharedFileMapping)
throw new NotSupportedException($"CultMesh body transport '{descriptor.TransportKind}' is not a file mapping.");
var token = descriptor.CapabilityToken;
if (token.IndexOfAny(Path.GetInvalidFileNameChars()) >= 0 || token.Contains(".."))
throw new UnauthorizedAccessException("Invalid CultMesh body capability token.");
var path = Path.GetFullPath(Path.Combine(_rootPath, token + ".body"));
if (!path.StartsWith(_rootPath + Path.DirectorySeparatorChar, StringComparison.OrdinalIgnoreCase))
throw new UnauthorizedAccessException("CultMesh body token escaped the broker root.");
var path = _verifiedBodies != null
? _verifiedBodies.Resolve(descriptor.CapabilityToken, request.NowUtc)
: ResolvePublisherPath(descriptor.CapabilityToken);
var info = new FileInfo(path);
if (!info.Exists || info.Length != descriptor.ByteSize)
throw new InvalidDataException("Mapped CultMesh body size does not match its descriptor.");
return new CultMeshMappedBodyLease(descriptor, path);
}
private string ResolvePublisherPath(string token)
{
if (token.IndexOfAny(Path.GetInvalidFileNameChars()) >= 0 || token.Contains(".."))
throw new UnauthorizedAccessException("Invalid CultMesh body capability token.");
var path = Path.GetFullPath(Path.Combine(_rootPath!, token + ".body"));
if (!path.StartsWith(_rootPath + Path.DirectorySeparatorChar, StringComparison.OrdinalIgnoreCase))
throw new UnauthorizedAccessException("CultMesh body token escaped the broker root.");
return path;
}
}
public sealed class CultMeshVerifiedBodyMappingBroker
{
private sealed class Grant
{
public string Path { get; set; } = string.Empty;
public long ExpiresAtUnixMs { get; set; }
}
private readonly string _cacheDirectory;
private readonly ConcurrentDictionary<string, Grant> _grants = new(StringComparer.Ordinal);
public CultMeshVerifiedBodyMappingBroker(string cacheDirectory) =>
_cacheDirectory = Path.GetFullPath(string.IsNullOrWhiteSpace(cacheDirectory)
? throw new ArgumentException("Verified body cache directory is required.", nameof(cacheDirectory))
: cacheDirectory);
internal CultMeshBodyDescriptor GrantVerified(
string contentHash,
string verifiedPath,
CultMeshBodyDescriptor networkDescriptor,
DateTimeOffset expiresAtUtc)
{
if (networkDescriptor == null) throw new ArgumentNullException(nameof(networkDescriptor));
var expectedPath = Path.Combine(_cacheDirectory, contentHash + ".body");
if (!string.Equals(Path.GetFullPath(verifiedPath), expectedPath, StringComparison.OrdinalIgnoreCase))
throw new InvalidOperationException("Only the transfer-owned verified body may be mapped.");
var info = new FileInfo(expectedPath);
if (!info.Exists || info.Length != networkDescriptor.ByteSize)
throw new InvalidDataException("Verified CultMesh body size does not match its network descriptor.");
var token = CreateCapabilityToken();
var expiresAtUnixMs = expiresAtUtc.ToUnixTimeMilliseconds();
_grants[token] = new Grant { Path = expectedPath, ExpiresAtUnixMs = expiresAtUnixMs };
return new CultMeshBodyDescriptor
{
BodyId = networkDescriptor.BodyId,
SchemaId = networkDescriptor.SchemaId,
LayoutVersion = networkDescriptor.LayoutVersion,
ByteSize = networkDescriptor.ByteSize,
Capacity = networkDescriptor.Capacity,
ProducerEpoch = networkDescriptor.ProducerEpoch,
Sequence = networkDescriptor.Sequence,
AccessMode = CultMeshBodyAccessMode.ReadOnly,
Synchronization = networkDescriptor.Synchronization,
LeaseExpiresAtUnixMs = expiresAtUnixMs,
TransportKind = CultMeshBodyTransportKind.SharedFileMapping,
CapabilityToken = token
};
}
internal string Resolve(string token, DateTimeOffset nowUtc)
{
if (!_grants.TryGetValue(token, out var grant))
throw new UnauthorizedAccessException("Unknown verified body capability token.");
if (grant.ExpiresAtUnixMs <= nowUtc.ToUnixTimeMilliseconds())
{
_grants.TryRemove(token, out _);
throw new InvalidOperationException("Verified body capability has expired.");
}
return grant.Path;
}
private static string CreateCapabilityToken()
{
var bytes = new byte[32];
using (var random = RandomNumberGenerator.Create()) random.GetBytes(bytes);
return BitConverter.ToString(bytes).Replace("-", string.Empty).ToLowerInvariant();
}
}
public sealed class CultMeshSharedMemoryBodyAdapter : ICultMeshBodyTransportAdapter

31
src/GameCult.Mesh/CultMeshContentTransfer.cs
View file

@ -76,18 +76,47 @@ namespace GameCult.Mesh
private readonly CultCache _stateCache;
private readonly ICultMeshContentProvider[] _providers;
private readonly string _cacheDirectory;
private readonly CultMeshVerifiedBodyMappingBroker? _mappedBodies;
private readonly ConcurrentDictionary<string, SemaphoreSlim> _contentLocks = new(StringComparer.Ordinal);
public CultMeshContentTransferService(
CultCache stateCache,
IEnumerable<ICultMeshContentProvider> providers,
CultMeshContentTransferOptions options)
CultMeshContentTransferOptions options,
CultMeshVerifiedBodyMappingBroker? mappedBodies = null)
{
_stateCache = stateCache ?? throw new ArgumentNullException(nameof(stateCache));
_providers = providers?.ToArray() ?? throw new ArgumentNullException(nameof(providers));
if (_providers.Length == 0) throw new ArgumentException("At least one content provider is required.", nameof(providers));
if (_providers.Any(provider => provider == null)) throw new ArgumentException("Content providers cannot contain null entries.", nameof(providers));
_cacheDirectory = (options ?? throw new ArgumentNullException(nameof(options))).CacheDirectory;
_mappedBodies = mappedBodies;
}
public async Task<CultMeshBodyDescriptor> FetchMappedBodyAsync(
CultMeshCdnArtifactManifest manifest,
CultMeshBodyDescriptor networkDescriptor,
DateTimeOffset nowUtc,
TimeSpan leaseDuration,
CancellationToken cancellationToken = default)
{
if (_mappedBodies == null)
throw new InvalidOperationException("Verified body mapping is not configured for this transfer service.");
if (networkDescriptor == null) throw new ArgumentNullException(nameof(networkDescriptor));
if (networkDescriptor.TransportKind != CultMeshBodyTransportKind.Network)
throw new ArgumentException("The fallback descriptor must use the network body transport.", nameof(networkDescriptor));
if (leaseDuration <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(leaseDuration));
CultMeshCdn.ValidateManifest(manifest);
if (networkDescriptor.ByteSize != manifest.SizeBytes)
throw new InvalidDataException("CDN manifest size does not match its network body descriptor.");
var path = await FetchAsync(manifest, cancellationToken).ConfigureAwait(false);
var contentHash = CultMeshCdn.NormalizeHash(manifest.ContentHash, nameof(manifest.ContentHash));
var requestedExpiry = nowUtc.Add(leaseDuration);
var networkExpiry = DateTimeOffset.FromUnixTimeMilliseconds(networkDescriptor.LeaseExpiresAtUnixMs);
var expiry = requestedExpiry <= networkExpiry ? requestedExpiry : networkExpiry;
if (expiry <= nowUtc) throw new InvalidOperationException("Network body lease has expired.");
return _mappedBodies.GrantVerified(contentHash, path, networkDescriptor, expiry);
}
public async Task<string> FetchAsync(

29
src/GameCult.Mesh/docs/reliability-migration-plan.md
View file

@ -566,6 +566,35 @@ provider corruption and failover, forged checkpoint rejection, final-hash
failure, atomic visibility, and concurrent request coalescing. Existing CDN
tests cover the compatibility reader.
### Phase 5 verified CDN mapped-body slice
**Owner:** `CultMeshContentTransferService` remains the sole writer and promoter
of completed CDN bodies. `CultMeshVerifiedBodyMappingBroker` owns only ephemeral,
opaque capability-to-file grants for those completed bodies. Body negotiation
and producer authorization remain owned by `CultMeshBodyTransportService`.
**Inputs:** the atomically committed `<sha256>.body` path returned by transfer,
the equivalent network body descriptor, and a bounded lease.
**Outputs:** a read-only file-mapping descriptor for the same logical generation.
Multiple consumers map the same completed cache file and therefore share the OS
page cache; no process-sized byte array or republished body file is created.
**Derived state:** opaque capability tokens and their expiry. Tokens are neither
content hashes nor paths and do not confer producer authority or content trust.
**Forbidden writers:** the mapping broker and adapter cannot create, promote,
rename, repair, or map partial/checkpoint files. `CultMeshMappedBodyPublisher`
is not used for verified CDN content.
**Shared paths:** local and network descriptors preserve body identity,
schema/layout, capacity, producer epoch, sequence, and byte size. Local open
failure is observed by body negotiation and falls back to that network
representation.
**Cut line:** there is no second CDN publication path. Mapping is granted only
after the transfer owner returns its verified final file.
### Cut first
- Delete the first consumer-owned CDN chunk loop and partial-file convention.

78
tests/GameCult.Mesh.Tests/CultMeshContentTransferTests.cs
View file

@ -141,6 +141,59 @@ public sealed class CultMeshContentTransferTests
provider.Requests.Values.Sum().Should().Be(artifact.Chunks.Count);
}
[Test]
public async Task FetchMappedBodyAsync_MapsOnlyCommittedBodyForMultipleConsumersAndFallsBackToNetwork()
{
var artifact = Artifact("mapped", 4);
using var cache = Cache();
var broker = new CultMeshVerifiedBodyMappingBroker(_directory);
var service = new CultMeshContentTransferService(
cache,
new[] { new FakeProvider("source", artifact.Chunks) },
new CultMeshContentTransferOptions(_directory),
broker);
var now = DateTimeOffset.UtcNow;
var network = NetworkDescriptor(artifact, now);
var first = await service.FetchMappedBodyAsync(artifact.Manifest, network, now, TimeSpan.FromMinutes(1));
var second = await service.FetchMappedBodyAsync(artifact.Manifest, network, now, TimeSpan.FromMinutes(1));
first.Should().BeEquivalentTo(second, options => options.Excluding(value => value.CapabilityToken));
first.CapabilityToken.Should().NotBe(artifact.Manifest.ContentHash);
first.CapabilityToken.Should().NotContain(Path.DirectorySeparatorChar.ToString());
first.CapabilityToken.Should().NotContain(Path.AltDirectorySeparatorChar.ToString());
Directory.GetFiles(_directory, "*.body").Should().ContainSingle()
.Which.Should().Be(Path.Combine(_directory, artifact.Manifest.ContentHash + ".body"));
Directory.GetFiles(_directory, "*.partial").Should().BeEmpty();
var adapter = new CultMeshMappedBodyAdapter(broker);
using (var firstLease = adapter.OpenReadOnly(first, Request(first, now)))
using (var secondLease = adapter.OpenReadOnly(second, Request(second, now)))
{
var firstBytes = new byte[first.ByteSize];
var secondBytes = new byte[second.ByteSize];
firstLease.CopyTo(0, firstBytes, 0, firstBytes.Length);
secondLease.CopyTo(0, secondBytes, 0, secondBytes.Length);
firstBytes.Should().Equal(Payload());
secondBytes.Should().Equal(Payload());
}
File.Delete(Path.Combine(_directory, artifact.Manifest.ContentHash + ".body"));
var transport = new CultMeshBodyTransportService(
new ICultMeshBodyTransportAdapter[]
{
adapter,
new CultMeshNetworkBodyAdapter(_ => Payload())
},
_ => true);
using var fallback = transport.OpenReadOnly(first, network, Request(first, now), out var localFailure);
localFailure.Should().BeOfType<InvalidDataException>();
fallback.TransportKind.Should().Be(CultMeshBodyTransportKind.Network);
fallback.Descriptor.BodyId.Should().Be(first.BodyId);
}
private CultMeshContentTransferService Service(CultCache cache, params ICultMeshContentProvider[] providers) =>
new(cache, providers, new CultMeshContentTransferOptions(_directory));
@ -165,6 +218,31 @@ public sealed class CultMeshContentTransferTests
private static CultRecordKey StateKey(CultMeshCdnArtifactManifest manifest) =>
new("mesh:content-transfer:" + manifest.ContentHash);
private static CultMeshBodyDescriptor NetworkDescriptor(CultMeshCdnArtifact artifact, DateTimeOffset now) => new()
{
BodyId = artifact.Manifest.ArtifactId,
SchemaId = "gamecult.mesh.cdn-artifact.v1",
LayoutVersion = 1,
ByteSize = artifact.Manifest.SizeBytes,
Capacity = checked((int)artifact.Manifest.SizeBytes),
ProducerEpoch = 4,
Sequence = 7,
AccessMode = CultMeshBodyAccessMode.ReadOnly,
Synchronization = CultMeshBodySynchronization.ImmutableSequence,
LeaseExpiresAtUnixMs = now.AddMinutes(2).ToUnixTimeMilliseconds(),
TransportKind = CultMeshBodyTransportKind.Network
};
private static CultMeshBodyValidationRequest Request(CultMeshBodyDescriptor descriptor, DateTimeOffset now) => new()
{
BodyId = descriptor.BodyId,
SchemaId = descriptor.SchemaId,
LayoutVersion = descriptor.LayoutVersion,
ProducerEpoch = descriptor.ProducerEpoch,
AccessMode = CultMeshBodyAccessMode.ReadOnly,
NowUtc = now
};
private static CultMeshCdnArtifactChunk Clone(CultMeshCdnArtifactChunk chunk) => new()
{
ChunkHash = chunk.ChunkHash,