using System; using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; using System.Security.Cryptography; using System.Text; using System.Threading.Tasks; using GameCult.Caching; using GameCult.Networking; using MessagePack; namespace GameCult.Mesh { /// /// Schema versions for CultMesh distributed CDN documents. /// public static class CultMeshCdnSchemaVersions { /// Manifest document schema id. public const string ArtifactManifest = "gamecult.mesh.cdn_artifact_manifest.v1"; /// Chunk document schema id. public const string ArtifactChunk = "gamecult.mesh.cdn_artifact_chunk.v1"; } /// /// Well-known CultMesh CDN artifact kinds. /// public static class CultMeshCdnArtifactKinds { /// Game, UI, shader, texture, audio, or data asset payload. public const string Asset = "asset"; /// Executable build or install/update payload. public const string Build = "build"; /// Generic packaged payload. public const string Package = "package"; } /// /// Options for packing bytes into CultMesh CDN documents. /// public sealed class CultMeshCdnPackOptions { /// Default CDN chunk size. public const int DefaultChunkSizeBytes = 4 * 1024 * 1024; /// Maximum bytes per content chunk. public int ChunkSizeBytes { get; set; } = DefaultChunkSizeBytes; /// Artifact kind, such as asset or build. public string Kind { get; set; } = CultMeshCdnArtifactKinds.Asset; /// Caller-defined artifact version. public string Version { get; set; } = string.Empty; /// Artifact MIME type. public string MimeType { get; set; } = "application/octet-stream"; /// Search and routing tags copied to the manifest. public string[] Tags { get; set; } = Array.Empty(); /// Caller metadata copied to the manifest. public Dictionary Metadata { get; set; } = new(); /// Override manifest creation timestamp in round-trip UTC format. public string? CreatedAtUtc { get; set; } } /// /// Reference from an artifact manifest to one content-addressed chunk document. /// [MessagePackObject] public sealed class CultMeshCdnChunkRef { /// SHA-256 hash of the referenced chunk payload. [Key(0)] public string ChunkHash { get; set; } = string.Empty; /// Byte offset of this chunk inside the materialized artifact. [Key(1)] public long Offset { get; set; } /// Number of payload bytes contributed by this chunk. [Key(2)] public int SizeBytes { get; set; } /// CultCache record key for the referenced chunk document. [Key(3)] public string RecordKey { get; set; } = string.Empty; /// Element id: the decimal text of , which is unique within a manifest and never random. [Key(4)] [CultElementId(nameof(Offset))] public string Id { get; set; } = string.Empty; } /// /// Content-addressed artifact chunk stored in CultCache and replicated through CultNet. /// [MessagePackObject] [CultDocument("gamecult.mesh.cdn_artifact_chunk", CultMeshCdnSchemaVersions.ArtifactChunk)] public sealed class CultMeshCdnArtifactChunk { /// SHA-256 hash of the chunk payload. [Key(0)] [CultName] public string ChunkHash { get; set; } = string.Empty; /// Payload byte count. [Key(1)] public int SizeBytes { get; set; } /// Raw chunk payload bytes. [Key(2)] public byte[] Payload { get; set; } = Array.Empty(); /// Builds the deterministic content-addressed chunk record key. public static CultRecordKey CreateRecordKey(CultMeshCdnArtifactChunk chunk) { if (chunk == null) throw new ArgumentNullException(nameof(chunk)); return CreateRecordKey(chunk.ChunkHash); } /// Builds the deterministic content-addressed chunk record key. public static CultRecordKey CreateRecordKey(string chunkHash) { return new CultRecordKey("mesh:cdn:chunk:" + CultMeshCdn.NormalizeHash(chunkHash, nameof(chunkHash))); } } /// /// Manifest for one versioned CDN artifact assembled from content-addressed chunks. /// [MessagePackObject] [CultDocument("gamecult.mesh.cdn_artifact_manifest", CultMeshCdnSchemaVersions.ArtifactManifest)] public sealed class CultMeshCdnArtifactManifest { /// Stable artifact id, such as a logical asset or build path. [Key(0)] [CultName] public string ArtifactId { get; set; } = string.Empty; /// Artifact kind, such as asset, build, or package. [Key(1)] [CultRole] public string Kind { get; set; } = CultMeshCdnArtifactKinds.Asset; /// Caller-defined artifact version. [Key(2)] [CultRole] public string Version { get; set; } = string.Empty; /// SHA-256 hash of the full materialized artifact. [Key(3)] [CultRole] public string ContentHash { get; set; } = string.Empty; /// Full materialized artifact size. [Key(4)] public long SizeBytes { get; set; } /// Artifact MIME type. [Key(5)] public string MimeType { get; set; } = "application/octet-stream"; /// Manifest creation timestamp in round-trip UTC format. [Key(6)] public string CreatedAtUtc { get; set; } = string.Empty; /// Ordered chunk references needed to materialize the artifact. [Key(7)] public CultMeshCdnChunkRef[] Chunks { get; set; } = Array.Empty(); /// Search and routing tags. [Key(8)] public string[] Tags { get; set; } = Array.Empty(); /// Caller metadata, such as platform or texture channel layout. [Key(9)] public Dictionary Metadata { get; set; } = new(); /// Builds the deterministic versioned artifact manifest record key. public static CultRecordKey CreateRecordKey(CultMeshCdnArtifactManifest manifest) { if (manifest == null) throw new ArgumentNullException(nameof(manifest)); return CreateRecordKey(manifest.ArtifactId, manifest.Version, manifest.ContentHash); } /// Builds the deterministic versioned artifact manifest record key. public static CultRecordKey CreateRecordKey(string artifactId, string version, string contentHash) { var artifactToken = CultMeshCdn.StableToken(artifactId, nameof(artifactId)); var versionToken = string.IsNullOrWhiteSpace(version) ? "unversioned" : CultMeshCdn.StableToken(version, nameof(version)); return new CultRecordKey( "mesh:cdn:artifact:" + artifactToken + ":" + versionToken + ":" + CultMeshCdn.NormalizeHash(contentHash, nameof(contentHash))); } } /// /// Packed CDN artifact ready to publish into a cache or database. /// public sealed class CultMeshCdnArtifact { /// Creates a packed artifact container. public CultMeshCdnArtifact( CultMeshCdnArtifactManifest manifest, IReadOnlyList chunks) { Manifest = manifest ?? throw new ArgumentNullException(nameof(manifest)); Chunks = chunks ?? throw new ArgumentNullException(nameof(chunks)); } /// Artifact manifest document. public CultMeshCdnArtifactManifest Manifest { get; } /// Content-addressed chunk documents. public IReadOnlyList Chunks { get; } /// Deterministic manifest record key. public CultRecordKey ManifestKey => CultMeshCdnArtifactManifest.CreateRecordKey(Manifest); /// All unique record keys required to replicate this artifact. public IReadOnlyList RecordKeys => new[] { ManifestKey }.Concat(Chunks.Select(CultMeshCdnArtifactChunk.CreateRecordKey)).Distinct().ToArray(); } /// /// Content-addressed byte artifact helpers for distributing assets and builds through CultMesh clients. /// public static class CultMeshCdn { /// /// Splits a payload into content-addressed chunk documents and a versioned manifest. /// public static CultMeshCdnArtifact PackArtifact( string artifactId, byte[] payload, CultMeshCdnPackOptions? options = null) { if (string.IsNullOrWhiteSpace(artifactId)) { throw new ArgumentException("Artifact id must be non-empty.", nameof(artifactId)); } if (payload == null) throw new ArgumentNullException(nameof(payload)); options ??= new CultMeshCdnPackOptions(); if (options.ChunkSizeBytes <= 0) { throw new ArgumentOutOfRangeException(nameof(options), "ChunkSizeBytes must be greater than zero."); } var chunks = new List(); var refs = new List(); for (var offset = 0; offset < payload.Length; offset += options.ChunkSizeBytes) { var count = Math.Min(options.ChunkSizeBytes, payload.Length - offset); var chunkPayload = new byte[count]; Buffer.BlockCopy(payload, offset, chunkPayload, 0, count); var chunkHash = HashBytes(chunkPayload); var chunk = new CultMeshCdnArtifactChunk { ChunkHash = chunkHash, SizeBytes = count, Payload = chunkPayload }; chunks.Add(chunk); refs.Add(new CultMeshCdnChunkRef { ChunkHash = chunkHash, Offset = offset, SizeBytes = count, RecordKey = CultMeshCdnArtifactChunk.CreateRecordKey(chunk).Value, Id = offset.ToString(CultureInfo.InvariantCulture) }); } if (payload.Length == 0) { var chunkHash = HashBytes(Array.Empty()); var chunk = new CultMeshCdnArtifactChunk { ChunkHash = chunkHash, SizeBytes = 0, Payload = Array.Empty() }; chunks.Add(chunk); refs.Add(new CultMeshCdnChunkRef { ChunkHash = chunkHash, Offset = 0, SizeBytes = 0, RecordKey = CultMeshCdnArtifactChunk.CreateRecordKey(chunk).Value, Id = "0" }); } var manifest = new CultMeshCdnArtifactManifest { ArtifactId = artifactId, Kind = string.IsNullOrWhiteSpace(options.Kind) ? CultMeshCdnArtifactKinds.Asset : options.Kind, Version = options.Version ?? string.Empty, ContentHash = HashBytes(payload), SizeBytes = payload.LongLength, MimeType = string.IsNullOrWhiteSpace(options.MimeType) ? "application/octet-stream" : options.MimeType, CreatedAtUtc = options.CreatedAtUtc ?? DateTimeOffset.UtcNow.ToString("O", CultureInfo.InvariantCulture), Chunks = refs.ToArray(), Tags = options.Tags ?? Array.Empty(), Metadata = options.Metadata ?? new Dictionary() }; return new CultMeshCdnArtifact(manifest, chunks); } /// /// Publishes all chunk documents and the manifest into a local CultCache. /// public static async Task PublishAsync( CultCache cache, CultMeshCdnArtifact artifact) { if (cache == null) throw new ArgumentNullException(nameof(cache)); if (artifact == null) throw new ArgumentNullException(nameof(artifact)); ValidateArtifact(artifact); foreach (var chunk in artifact.Chunks) { await cache.UpsertAsync( chunk, new CultRecordHandle( CultMeshCdnArtifactChunk.CreateRecordKey(chunk))).ConfigureAwait(false); } await cache.UpsertAsync( artifact.Manifest, new CultRecordHandle( CultMeshCdnArtifactManifest.CreateRecordKey(artifact.Manifest))).ConfigureAwait(false); return artifact.Manifest; } /// /// Publishes all chunk documents and the manifest into a distributed CultNet database. /// public static async Task PublishAsync( CultNetDatabase database, CultMeshCdnArtifact artifact) { if (database == null) throw new ArgumentNullException(nameof(database)); if (artifact == null) throw new ArgumentNullException(nameof(artifact)); ValidateArtifact(artifact); foreach (var chunk in artifact.Chunks) { await database.PutAsync(CultMeshCdnArtifactChunk.CreateRecordKey(chunk), chunk) .ConfigureAwait(false); } await database.PutAsync(CultMeshCdnArtifactManifest.CreateRecordKey(artifact.Manifest), artifact.Manifest) .ConfigureAwait(false); return artifact.Manifest; } /// /// Reassembles and verifies an artifact from a local CultCache. /// public static byte[] ReadArtifact(CultCache cache, CultMeshCdnArtifactManifest manifest) { if (cache == null) throw new ArgumentNullException(nameof(cache)); if (manifest == null) throw new ArgumentNullException(nameof(manifest)); ValidateManifestShape(manifest); if (manifest.SizeBytes > int.MaxValue) { throw new InvalidDataException("CDN artifact is too large to materialize into one byte array."); } var output = new byte[checked((int)manifest.SizeBytes)]; foreach (var reference in manifest.Chunks.OrderBy(chunk => chunk.Offset)) { var recordKey = string.IsNullOrWhiteSpace(reference.RecordKey) ? CultMeshCdnArtifactChunk.CreateRecordKey(reference.ChunkHash) : new CultRecordKey(reference.RecordKey); var chunk = cache.Get(recordKey) ?? throw new FileNotFoundException("CDN artifact chunk is missing from the cache.", recordKey.Value); ValidateChunk(reference, chunk); Buffer.BlockCopy(chunk.Payload, 0, output, checked((int)reference.Offset), reference.SizeBytes); } var actualHash = HashBytes(output); if (!string.Equals(NormalizeHash(manifest.ContentHash, nameof(manifest.ContentHash)), actualHash, StringComparison.Ordinal)) { throw new InvalidDataException("CDN artifact content hash does not match its manifest."); } return output; } /// /// Reassembles and verifies an artifact from a distributed CultNet database. /// public static async Task ReadArtifactAsync( CultNetDatabase database, CultMeshCdnArtifactManifest manifest) { if (database == null) throw new ArgumentNullException(nameof(database)); if (manifest == null) throw new ArgumentNullException(nameof(manifest)); var directory = Path.Combine(Path.GetTempPath(), "cultmesh-content", Guid.NewGuid().ToString("N")); try { using var state = new CultCache(); var service = new CultMeshContentTransferService( state, new[] { new CultMeshDatabaseContentProvider("legacy-database", database) }, new CultMeshContentTransferOptions(directory)); var path = await service.FetchAsync(manifest).ConfigureAwait(false); return await File.ReadAllBytesAsync(path).ConfigureAwait(false); } finally { if (Directory.Exists(directory)) Directory.Delete(directory, true); } } /// /// Creates a raw CultNet document registry for CDN manifests and chunks. /// public static CultNetDocumentRegistry CreateDocumentRegistry(CultDocumentRegistry? documents = null) { documents ??= CultDocumentRegistry.Shared; return new CultNetDocumentRegistry(documents) .Register(CultNetDocumentBinding.ForDocument(documents)) .Register(CultNetDocumentBinding.ForDocument(documents)); } internal static string HashBytes(byte[] bytes) { using var sha256 = SHA256.Create(); var hash = sha256.ComputeHash(bytes); var builder = new StringBuilder(hash.Length * 2); foreach (var value in hash) { builder.Append(value.ToString("x2", CultureInfo.InvariantCulture)); } return builder.ToString(); } private static void ValidateArtifact(CultMeshCdnArtifact artifact) { ValidateManifestShape(artifact.Manifest); var expectedRefs = artifact.Chunks .Select(chunk => new { Hash = HashBytes(chunk.Payload), ChunkHash = NormalizeHash(chunk.ChunkHash, nameof(chunk.ChunkHash)), chunk.SizeBytes, Key = CultMeshCdnArtifactChunk.CreateRecordKey(chunk).Value }) .ToArray(); if (artifact.Manifest.Chunks.Length != expectedRefs.Length) { throw new InvalidDataException("CDN artifact manifest chunk count does not match chunk payloads."); } for (var i = 0; i < expectedRefs.Length; i++) { var reference = artifact.Manifest.Chunks[i]; var expected = expectedRefs[i]; if (!string.Equals(NormalizeHash(reference.ChunkHash, nameof(reference.ChunkHash)), expected.Hash, StringComparison.Ordinal) || !string.Equals(expected.ChunkHash, expected.Hash, StringComparison.Ordinal) || reference.SizeBytes != expected.SizeBytes || !string.Equals(reference.RecordKey, expected.Key, StringComparison.Ordinal)) { throw new InvalidDataException("CDN artifact manifest chunk reference does not match its chunk payload."); } } var materialized = new byte[checked((int)artifact.Manifest.SizeBytes)]; foreach (var reference in artifact.Manifest.Chunks.OrderBy(chunk => chunk.Offset)) { var chunk = artifact.Chunks.First(candidate => string.Equals( CultMeshCdnArtifactChunk.CreateRecordKey(candidate).Value, reference.RecordKey, StringComparison.Ordinal)); Buffer.BlockCopy(chunk.Payload, 0, materialized, checked((int)reference.Offset), reference.SizeBytes); } if (!string.Equals( NormalizeHash(artifact.Manifest.ContentHash, nameof(artifact.Manifest.ContentHash)), HashBytes(materialized), StringComparison.Ordinal)) { throw new InvalidDataException("CDN artifact content hash does not match its manifest."); } } internal static void ValidateManifest(CultMeshCdnArtifactManifest manifest) { ValidateManifestShape(manifest); } private static void ValidateManifestShape(CultMeshCdnArtifactManifest manifest) { if (manifest.SizeBytes < 0) { throw new InvalidDataException("CDN artifact manifest has a negative size."); } var expectedOffset = 0L; foreach (var reference in manifest.Chunks.OrderBy(chunk => chunk.Offset)) { if (reference.Offset != expectedOffset) { throw new InvalidDataException("CDN artifact chunks are not contiguous."); } if (reference.SizeBytes < 0) { throw new InvalidDataException("CDN artifact chunk has a negative size."); } expectedOffset += reference.SizeBytes; } if (expectedOffset != manifest.SizeBytes) { throw new InvalidDataException("CDN artifact chunk sizes do not sum to the manifest size."); } } internal static void ValidateChunkPayload(CultMeshCdnChunkRef reference, CultMeshCdnArtifactChunk chunk) { ValidateChunk(reference, chunk); } private static void ValidateChunk(CultMeshCdnChunkRef reference, CultMeshCdnArtifactChunk chunk) { if (chunk.Payload.Length != reference.SizeBytes || chunk.SizeBytes != reference.SizeBytes) { throw new InvalidDataException("CDN artifact chunk payload metadata does not match its manifest reference."); } var actualHash = HashBytes(chunk.Payload); if (!string.Equals(NormalizeHash(reference.ChunkHash, nameof(reference.ChunkHash)), actualHash, StringComparison.Ordinal) || !string.Equals(NormalizeHash(chunk.ChunkHash, nameof(chunk.ChunkHash)), actualHash, StringComparison.Ordinal)) { throw new InvalidDataException("CDN artifact chunk hash does not match its payload."); } } internal static string NormalizeHash(string hash, string parameterName) { if (string.IsNullOrWhiteSpace(hash)) { throw new ArgumentException("Hash must be non-empty.", parameterName); } var value = hash.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase) ? hash.Substring("sha256:".Length) : hash; return value.Trim().ToLowerInvariant(); } internal static string StableToken(string value, string parameterName) { if (string.IsNullOrWhiteSpace(value)) { throw new ArgumentException("Value must be non-empty.", parameterName); } var normalized = value.Trim().ToLowerInvariant(); var builder = new StringBuilder(normalized.Length); foreach (var c in normalized) { builder.Append(char.IsLetterOrDigit(c) || c == '-' || c == '_' || c == '.' ? c : '-'); } var token = builder.ToString().Trim('-'); return string.IsNullOrWhiteSpace(token) ? HashBytes(Encoding.UTF8.GetBytes(value)) : token; } } }