using System; using System.Collections.Generic; using System.Linq; using System.Reflection; using System.Threading.Tasks; using GameCult.Caching; using R3; namespace GameCult.Networking { /// /// Describes the kind of domain change observed through a distributed CultCache surface. /// public enum CultNetDatabaseChangeKind { /// /// A document was added. /// Added, /// /// A document was updated. /// Updated, /// /// A document was removed. /// Removed, /// /// A document was accepted through compatible schema migration. /// SchemaMigrated, /// /// A document change was rejected. /// Rejected, /// /// A document was applied locally before authoritative commit. /// Predicted, /// /// A predicted document was replaced by the authoritative committed state. /// Reconciled } /// /// Declares which documents this runtime may predict locally before authoritative commit. /// public sealed class CultNetClientAuthorityScope { /// /// Creates a client authority scope. /// public CultNetClientAuthorityScope( string ownerRuntimeId, IEnumerable? schemaIds = null, string? keyPrefix = null) { OwnerRuntimeId = string.IsNullOrWhiteSpace(ownerRuntimeId) ? throw new ArgumentException("Value must be non-empty.", nameof(ownerRuntimeId)) : ownerRuntimeId; SchemaIds = schemaIds?.Where(value => !string.IsNullOrWhiteSpace(value)).Distinct(StringComparer.Ordinal).ToArray() ?? Array.Empty(); KeyPrefix = keyPrefix; } /// /// Gets the runtime that owns authoring for this scope. /// public string OwnerRuntimeId { get; } /// /// Gets the schema ids governed by this scope. Empty means all schemas. /// public IReadOnlyList SchemaIds { get; } /// /// Gets the optional record-key prefix governed by this scope. /// public string? KeyPrefix { get; } internal bool Matches(string runtimeId, string schemaId, CultRecordKey key) { return string.Equals(OwnerRuntimeId, runtimeId, StringComparison.Ordinal) && (SchemaIds.Count == 0 || SchemaIds.Contains(schemaId, StringComparer.Ordinal)) && (string.IsNullOrEmpty(KeyPrefix) || key.Value.StartsWith(KeyPrefix!, StringComparison.Ordinal)); } } /// /// Describes one shard owned or observed by a CultNet database surface. /// public sealed class CultNetShardDescriptor { /// /// Creates a shard descriptor. /// public CultNetShardDescriptor( string shardId, string ownerRuntimeId, long epoch, bool isPrimary, IEnumerable? schemaIds = null, string? keyPrefix = null, IEnumerable? primaryEndpoints = null, IEnumerable? replicaEndpoints = null, IEnumerable? readReplicaEndpoints = null, string? region = null) { ShardId = RequireNonEmpty(shardId, nameof(shardId)); OwnerRuntimeId = RequireNonEmpty(ownerRuntimeId, nameof(ownerRuntimeId)); Epoch = epoch; IsPrimary = isPrimary; SchemaIds = schemaIds?.Where(value => !string.IsNullOrWhiteSpace(value)).Distinct(StringComparer.Ordinal).ToArray() ?? Array.Empty(); KeyPrefix = keyPrefix; PrimaryEndpoints = CleanEndpoints(primaryEndpoints); ReplicaEndpoints = CleanEndpoints(replicaEndpoints); ReadReplicaEndpoints = CleanEndpoints(readReplicaEndpoints); Region = region; } /// /// Gets the stable shard identifier. /// public string ShardId { get; } /// /// Gets the runtime that currently owns writes for this shard. /// public string OwnerRuntimeId { get; } /// /// Gets the monotonic shard authority epoch. /// public long Epoch { get; } /// /// Gets whether the local database instance is authoritative for writes to this shard. /// public bool IsPrimary { get; } /// /// Gets the schema ids governed by this shard. Empty means all schemas. /// public IReadOnlyList SchemaIds { get; } /// /// Gets the optional record-key prefix governed by this shard. /// public string? KeyPrefix { get; } /// /// Gets endpoints that can accept authoritative writes for this shard. /// public IReadOnlyList PrimaryEndpoints { get; } /// /// Gets endpoints that replicate authoritative shard mutations. /// public IReadOnlyList ReplicaEndpoints { get; } /// /// Gets endpoints intended for low-latency read and subscription traffic. /// public IReadOnlyList ReadReplicaEndpoints { get; } /// /// Gets an optional locality label for this shard owner. /// public string? Region { get; } /// /// Creates a primary shard that accepts every schema and key. /// public static CultNetShardDescriptor PrimaryAll(string ownerRuntimeId = "local", string shardId = "primary") { return new CultNetShardDescriptor(shardId, ownerRuntimeId, epoch: 1, isPrimary: true); } /// /// Creates a read-only shard descriptor. /// public static CultNetShardDescriptor ReadOnly( string shardId, string ownerRuntimeId, long epoch = 1, IEnumerable? schemaIds = null, string? keyPrefix = null) { return new CultNetShardDescriptor(shardId, ownerRuntimeId, epoch, isPrimary: false, schemaIds, keyPrefix); } internal bool Matches(string schemaId, CultRecordKey key) { var schemaMatches = SchemaIds.Count == 0 || SchemaIds.Contains(schemaId, StringComparer.Ordinal); var keyMatches = string.IsNullOrEmpty(KeyPrefix) || key.Value.StartsWith(KeyPrefix!, StringComparison.Ordinal); return schemaMatches && keyMatches; } private static string RequireNonEmpty(string value, string paramName) { if (string.IsNullOrWhiteSpace(value)) { throw new ArgumentException("Value must be non-empty.", paramName); } return value; } private static string[] CleanEndpoints(IEnumerable? endpoints) { return endpoints?.Where(value => !string.IsNullOrWhiteSpace(value)).Distinct(StringComparer.Ordinal).ToArray() ?? Array.Empty(); } } /// /// Options for creating a distributed CultCache database surface. /// public sealed class CultNetDatabaseOptions { /// /// Gets or sets the local runtime id advertised as shard owner for default primary shards. /// public string RuntimeId { get; set; } = "local"; /// /// Gets or sets explicit shard descriptors. When omitted, this instance owns one primary shard for all records. /// public IReadOnlyList? Shards { get; set; } /// /// Gets or sets document scopes this runtime may predict before authoritative commit. /// public IReadOnlyList? ClientAuthorityScopes { get; set; } /// /// Gets or sets the document registry used to create and apply raw CultNet document messages. /// public CultNetDocumentRegistry? DocumentRegistry { get; set; } /// /// Gets or sets the durable store for accepted shard mutation logs. /// public ICultNetShardMutationLogStore? MutationLogStore { get; set; } } /// /// A typed domain change emitted by a CultNet database surface. /// public sealed class CultNetDatabaseChange where T : class { /// /// Creates a database change. /// public CultNetDatabaseChange( CultNetDatabaseChangeKind kind, CultRecordKey key, string schemaId, CultNetShardDescriptor shard, T? document, T? previousDocument, string? message = null) { Kind = kind; Key = key; SchemaId = schemaId; Shard = shard; Document = document; PreviousDocument = previousDocument; Message = message; } /// /// Gets the change kind. /// public CultNetDatabaseChangeKind Kind { get; } /// /// Gets the changed record key. /// public CultRecordKey Key { get; } /// /// Gets the schema id of the changed document. /// public string SchemaId { get; } /// /// Gets the shard that accepted or rejected the change. /// public CultNetShardDescriptor Shard { get; } /// /// Gets the current document for added and updated events. /// public T? Document { get; } /// /// Gets the previous document for updated and removed events. /// public T? PreviousDocument { get; } /// /// Gets an optional diagnostic message. /// public string? Message { get; } } /// /// One ordered mutation accepted for a shard. /// public sealed class CultNetShardMutationLogEntry { /// /// Creates a shard mutation log entry. /// public CultNetShardMutationLogEntry( string shardId, long shardEpoch, long sequence, string committedAt, CultNetDatabaseChangeKind kind, string schemaId, CultRecordKey key, object? document, object? previousDocument) { ShardId = shardId; ShardEpoch = shardEpoch; Sequence = sequence; CommittedAt = committedAt; Kind = kind; SchemaId = schemaId; Key = key; Document = document; PreviousDocument = previousDocument; } /// /// Gets the shard id. /// public string ShardId { get; } /// /// Gets the shard epoch. /// public long ShardEpoch { get; } /// /// Gets the per-shard mutation sequence. /// public long Sequence { get; } /// /// Gets the commit timestamp. /// public string CommittedAt { get; } /// /// Gets the mutation kind. /// public CultNetDatabaseChangeKind Kind { get; } /// /// Gets the schema id. /// public string SchemaId { get; } /// /// Gets the record key. /// public CultRecordKey Key { get; } /// /// Gets the current document, when present. /// public object? Document { get; } /// /// Gets the previous document, when present. /// public object? PreviousDocument { get; } } /// /// Raised when a database write targets a shard this node does not own. /// public sealed class CultNetShardAuthorityException : InvalidOperationException { /// /// Creates a shard authority exception. /// public CultNetShardAuthorityException(CultNetShardDescriptor shard, string message, string reason = "not_primary") : base(message) { Shard = shard; Reason = reason; } /// /// Gets the shard that rejected the write. /// public CultNetShardDescriptor Shard { get; } /// /// Gets the machine-readable authority rejection reason. /// public string Reason { get; } } /// /// Database-style CultNet facade over a CultCache shard set. /// public sealed class CultNetDatabase : IDisposable { private readonly CultCache _cache; private readonly CultNetDocumentRegistry _documents; private readonly string _runtimeId; private readonly List _shards; private readonly List _clientAuthorityScopes; private readonly ICultNetShardMutationLogStore? _mutationLogStore; private readonly Dictionary _predictedDocuments = new(StringComparer.Ordinal); private readonly Dictionary> _mutationLogs = new(StringComparer.Ordinal); private readonly Dictionary _nextLogSequences = new(StringComparer.Ordinal); private readonly Dictionary _appliedShardSequences = new(StringComparer.Ordinal); private readonly Subject _changes = new(); private bool _disposed; /// /// Creates a database surface over a CultCache instance. /// public CultNetDatabase(CultCache cache, CultNetDatabaseOptions? options = null) { _cache = cache ?? throw new ArgumentNullException(nameof(cache)); options ??= new CultNetDatabaseOptions(); _runtimeId = options.RuntimeId; _documents = options.DocumentRegistry ?? new CultNetDocumentRegistry(cache.Registry); _mutationLogStore = options.MutationLogStore; _shards = (options.Shards == null || options.Shards.Count == 0 ? new[] { CultNetShardDescriptor.PrimaryAll(options.RuntimeId) } : options.Shards) .ToList(); _clientAuthorityScopes = (options.ClientAuthorityScopes ?? Array.Empty()).ToList(); InitializeLogSequencesFromStore(); _cache.OnUpdate += PublishCacheUpdate; } /// /// Gets the local cache backing this database surface. /// public CultCache Cache => _cache; /// /// Gets the document registry used by the raw CultNet lane. /// public CultNetDocumentRegistry Documents => _documents; /// /// Gets the shard map known to this database surface. /// public IReadOnlyList Shards => _shards; /// /// Gets the document scopes this runtime may predict before authoritative commit. /// public IReadOnlyList ClientAuthorityScopes => _clientAuthorityScopes; /// /// Gets mutation log entries for a shard after the supplied sequence. /// public IReadOnlyList GetMutationLog( string shardId, long afterSequence = 0, int? limit = null) { ThrowIfDisposed(); if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId)); if (!_mutationLogs.TryGetValue(shardId, out var entries)) { return Array.Empty(); } var query = entries.Where(entry => entry.Sequence > afterSequence); if (limit.HasValue) { query = query.Take(limit.Value); } return query.ToArray(); } /// /// Gets replica catch-up log entries for a shard after the supplied sequence. /// public IReadOnlyList GetMutationLogMessages( string shardId, long afterSequence = 0, int? limit = null) { ThrowIfDisposed(); if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId)); if (_mutationLogStore != null) { return _mutationLogStore.Read(shardId, afterSequence, limit); } return GetMutationLog(shardId, afterSequence, limit) .Select(ToLogEntryMessage) .ToArray(); } /// /// Gets the highest retained-log sequence that has been compacted away for a shard. /// public long GetCompactedMutationLogSequence(string shardId) { ThrowIfDisposed(); if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId)); return _mutationLogStore?.GetCompactedThrough(shardId) ?? 0; } /// /// Gets the latest known shard-log sequence for a shard. /// public long GetLatestMutationLogSequence(string shardId) { ThrowIfDisposed(); if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId)); var compactedThrough = GetCompactedMutationLogSequence(shardId); var retained = _mutationLogStore != null ? _mutationLogStore.Read(shardId).Select(entry => entry.Sequence) : GetMutationLog(shardId).Select(entry => entry.Sequence); return retained.DefaultIfEmpty(compactedThrough).Max(); } /// /// Creates a raw document snapshot bounded to one shard. /// public CultNetSnapshotResponseRawMessage CreateShardSnapshotResponse( CultNetShardDescriptor shard, string messageId, CultNetSnapshotRequestMessage? filter = null) { ThrowIfDisposed(); if (shard == null) throw new ArgumentNullException(nameof(shard)); var requestedSchemaIds = filter?.SchemaIds != null ? new HashSet(filter.SchemaIds, StringComparer.Ordinal) : null; var requestedRecordKeys = filter?.RecordKeys != null ? new HashSet(filter.RecordKeys, StringComparer.Ordinal) : null; var documents = new List(); foreach (var document in _cache.AllEntries) { var documentType = document.GetType(); var descriptor = _cache.Registry.GetRequired(documentType); var key = GetTrackedKey(document, documentType); if (string.IsNullOrWhiteSpace(key.Value) || !shard.Matches(descriptor.SchemaId, key) || (requestedSchemaIds != null && !requestedSchemaIds.Contains(descriptor.SchemaId)) || (requestedRecordKeys != null && !requestedRecordKeys.Contains(key.Value))) { continue; } documents.Add(CreateRawDocumentRecord(key, document)); } return new CultNetSnapshotResponseRawMessage { MessageId = string.IsNullOrWhiteSpace(messageId) ? Guid.NewGuid().ToString("N") : messageId, Documents = documents.ToArray(), ShardId = shard.ShardId, ShardEpoch = shard.Epoch, ShardLogSequence = GetLatestMutationLogSequence(shard.ShardId) }; } /// /// Applies a shard-bounded snapshot and advances the replica cursor to its represented sequence. /// public async Task ApplyShardSnapshotResponseAsync( CultNetShardDescriptor shard, CultNetSnapshotResponseRawMessage response) { ThrowIfDisposed(); if (shard == null) throw new ArgumentNullException(nameof(shard)); if (response == null) throw new ArgumentNullException(nameof(response)); if (!string.IsNullOrWhiteSpace(response.ShardId) && !string.Equals(response.ShardId, shard.ShardId, StringComparison.Ordinal)) { throw new InvalidOperationException($"Snapshot for shard '{response.ShardId}' cannot be applied to shard '{shard.ShardId}'."); } if (response.ShardEpoch.HasValue && response.ShardEpoch.Value != shard.Epoch) { throw new CultNetShardAuthorityException( shard, $"Shard '{shard.ShardId}' is at epoch {shard.Epoch}, not snapshot epoch {response.ShardEpoch.Value}.", "stale_epoch"); } var incomingKeys = response.Documents .Where(document => ShardMatchesRawDocument(shard, document)) .Select(document => document.RecordKey) .ToHashSet(StringComparer.Ordinal); var local = GetLocalShardDocuments(shard).ToArray(); foreach (var existing in local.Where(item => !incomingKeys.Contains(item.Key.Value))) { RemoveTrackedDocument(existing.DocumentType, existing.Key); PublishUntyped( existing.DocumentType, CultNetDatabaseChangeKind.Removed, existing.Key, existing.SchemaId, shard, document: null, previousDocument: existing.Document); } foreach (var document in response.Documents.Where(document => ShardMatchesRawDocument(shard, document))) { var key = new CultRecordKey(document.RecordKey); var descriptor = _cache.Registry.GetRequiredBySchemaId(document.SchemaId); var previous = _cache.Get(key); var applied = await _documents.ApplyRawDocumentPutMessageAsync( _cache, new CultNetDocumentPutRawMessage { MessageId = response.MessageId, Document = document, ShardId = shard.ShardId, ShardEpoch = shard.Epoch }).ConfigureAwait(false); PublishUntyped( descriptor.DocumentType, previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated, key, document.SchemaId, shard, applied, previous); } var sequence = response.ShardLogSequence ?? 0; SetAppliedShardSequence(shard.ShardId, sequence); return sequence; } /// /// Gets the last replicated sequence applied for a shard. /// public long GetAppliedShardSequence(string shardId) { ThrowIfDisposed(); if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId)); return _appliedShardSequences.TryGetValue(shardId, out var sequence) ? sequence : 0; } internal void SetAppliedShardSequence(string shardId, long sequence) { ThrowIfDisposed(); if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId)); if (sequence < 0) throw new ArgumentOutOfRangeException(nameof(sequence), "Sequence must be non-negative."); _appliedShardSequences[shardId] = sequence; } /// /// Creates a shard catalog response for optional schema/key filters. /// public CultNetShardCatalogResponseMessage CreateShardCatalogResponse(CultNetShardCatalogRequestMessage request) { ThrowIfDisposed(); if (request == null) throw new ArgumentNullException(nameof(request)); var schemaIds = request.SchemaIds == null || request.SchemaIds.Length == 0 ? null : request.SchemaIds; var recordKeys = request.RecordKeys == null || request.RecordKeys.Length == 0 ? null : request.RecordKeys.Select(key => new CultRecordKey(key)).ToArray(); var descriptors = _shards .Where(shard => MatchesCatalogFilter(shard, schemaIds, recordKeys)) .Select(ToMessage) .ToArray(); return new CultNetShardCatalogResponseMessage { MessageId = string.IsNullOrWhiteSpace(request.MessageId) ? Guid.NewGuid().ToString("N") : request.MessageId, Shards = descriptors }; } /// /// Resolves the shard that governs the supplied schema and key. /// public CultNetShardDescriptor ResolveShard(string schemaId, CultRecordKey key) { ThrowIfDisposed(); return ResolveShardInternal(schemaId, key); } /// /// Gets a document by key. /// public Task GetAsync(CultRecordKey key) where T : class { ThrowIfDisposed(); return Task.FromResult(_cache.Get(key)); } /// /// Opens a reactive POCO presentation for one distributed CultCache document. /// public CultManagedDocument Document(CultRecordKey key) where T : class { ThrowIfDisposed(); return new CultManagedDocument( key, () => _cache.Get(key), async value => { await PutAsync(key, value).ConfigureAwait(false); }, WatchRecord(key) .Where(change => change.Document != null) .Select(change => change.Document!)); } /// /// Adds or replaces a document at a specific key. /// public async Task> PutAsync(CultRecordKey key, T document) where T : class { ThrowIfDisposed(); if (document == null) throw new ArgumentNullException(nameof(document)); var descriptor = _cache.Registry.GetRequired(); var shard = ResolveShardInternal(descriptor.SchemaId, key); EnsurePrimary(shard, descriptor.SchemaId, key); var previous = _cache.Get(key); var handle = await _cache.UpsertAsync(document, new CultRecordHandle(key)).ConfigureAwait(false); AppendMutationLogEntry( shard, previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated, key, descriptor.SchemaId, document, previous); Publish(new CultNetDatabaseChange( previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated, key, descriptor.SchemaId, shard, document, previous)); return handle; } /// /// Applies a locally predicted document for a client-owned input scope. /// public async Task> PutPredictedAsync(CultRecordKey key, T document) where T : class { ThrowIfDisposed(); if (document == null) throw new ArgumentNullException(nameof(document)); var descriptor = _cache.Registry.GetRequired(); var shard = ResolveShardInternal(descriptor.SchemaId, key); EnsureClientAuthority(descriptor.SchemaId, key); var previous = _cache.Get(key); var handle = await _cache.UpsertAsync(document, new CultRecordHandle(key)).ConfigureAwait(false); _predictedDocuments[PredictionKey(descriptor.SchemaId, key)] = document; Publish(new CultNetDatabaseChange( CultNetDatabaseChangeKind.Predicted, key, descriptor.SchemaId, shard, document, previous)); return handle; } /// /// Deletes a document by key. /// public Task DeleteAsync(CultRecordKey key) where T : class { ThrowIfDisposed(); var descriptor = _cache.Registry.GetRequired(); var shard = ResolveShardInternal(descriptor.SchemaId, key); EnsurePrimary(shard, descriptor.SchemaId, key); var previous = _cache.Get(key); if (previous != null) { _cache.Remove(new CultRecordHandle(key)); AppendMutationLogEntry( shard, CultNetDatabaseChangeKind.Removed, key, descriptor.SchemaId, document: null, previousDocument: previous); Publish(new CultNetDatabaseChange( CultNetDatabaseChangeKind.Removed, key, descriptor.SchemaId, shard, document: null, previousDocument: previous)); } return Task.CompletedTask; } /// /// Applies a raw document mutation after checking shard authority. /// public async Task ApplyPutAsync(CultNetDocumentPutRawMessage message) { ThrowIfDisposed(); if (message == null) throw new ArgumentNullException(nameof(message)); if (message.Document == null) { throw new ArgumentException("CultNet raw document message is missing its document payload.", nameof(message)); } var key = new CultRecordKey(message.Document.RecordKey); var shard = ResolveShardInternal(message.Document.SchemaId, key); EnsurePrimary(shard, message.Document.SchemaId, key, message.ShardEpoch); var descriptor = _cache.Registry.GetRequiredBySchemaId(message.Document.SchemaId); var previous = _cache.Get(key); var document = await _documents.ApplyRawDocumentPutMessageAsync(_cache, message).ConfigureAwait(false); var kind = previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated; var predictionKey = PredictionKey(descriptor.SchemaId, key); if (_predictedDocuments.Remove(predictionKey)) { kind = CultNetDatabaseChangeKind.Reconciled; } AppendMutationLogEntry( shard, kind == CultNetDatabaseChangeKind.Reconciled ? CultNetDatabaseChangeKind.Updated : kind, key, descriptor.SchemaId, document, previous, new CultNetShardLogEntryMessage { ChangeKind = previous == null ? "added" : "updated", Put = message }); PublishUntyped( descriptor.DocumentType, kind, key, descriptor.SchemaId, shard, document, previous); return document; } /// /// Applies a raw document delete after checking shard authority. /// public Task ApplyDeleteAsync(CultNetDocumentDeleteMessage message) { ThrowIfDisposed(); if (message == null) throw new ArgumentNullException(nameof(message)); var key = new CultRecordKey(message.RecordKey); var shard = ResolveShardInternal(message.SchemaId, key); EnsurePrimary(shard, message.SchemaId, key, message.ShardEpoch); var descriptor = _cache.Registry.GetRequiredBySchemaId(message.SchemaId); var previous = _cache.Get(key); if (previous == null) { return Task.CompletedTask; } _cache.Remove(key); AppendMutationLogEntry( shard, CultNetDatabaseChangeKind.Removed, key, descriptor.SchemaId, document: null, previousDocument: previous, wireEntry: new CultNetShardLogEntryMessage { ChangeKind = "removed", Delete = message }); PublishUntyped( descriptor.DocumentType, CultNetDatabaseChangeKind.Removed, key, descriptor.SchemaId, shard, document: null, previousDocument: previous); return Task.CompletedTask; } /// /// Applies committed shard-log entries from an authoritative peer. /// public async Task ApplyShardLogResponseAsync(CultNetShardLogResponseMessage response) { ThrowIfDisposed(); if (response == null) throw new ArgumentNullException(nameof(response)); if (response.ResyncRequired) { throw new InvalidOperationException( $"Shard '{response.ShardId}' requires snapshot resync before log application: {response.Reason ?? "unspecified"}."); } var shard = _shards.FirstOrDefault(candidate => string.Equals(candidate.ShardId, response.ShardId, StringComparison.Ordinal)); if (shard == null) { throw new InvalidOperationException($"Shard '{response.ShardId}' is not known by this database."); } if (response.ShardEpoch != shard.Epoch) { throw new CultNetShardAuthorityException( shard, $"Shard '{shard.ShardId}' is at epoch {shard.Epoch}, not response epoch {response.ShardEpoch}.", "stale_epoch"); } var lastApplied = GetAppliedShardSequence(response.ShardId); var orderedEntries = response.Entries.OrderBy(entry => entry.Sequence).ToArray(); foreach (var entry in orderedEntries) { if (entry.Sequence <= lastApplied) { continue; } if (entry.Sequence != lastApplied + 1) { throw new InvalidOperationException( $"Shard '{response.ShardId}' log has a gap. Expected sequence {lastApplied + 1}, received {entry.Sequence}."); } await ApplyCommittedShardLogEntryAsync(shard, entry).ConfigureAwait(false); lastApplied = entry.Sequence; _appliedShardSequences[response.ShardId] = lastApplied; } return lastApplied; } /// /// Watches all changes assignable to the requested document type. /// public Observable> Watch() where T : class { ThrowIfDisposed(); return _changes .Where(change => change is CultNetDatabaseChange) .Select(change => (CultNetDatabaseChange)change); } /// /// Watches all typed domain changes as boxed change objects. /// public Observable WatchAllChanges() { ThrowIfDisposed(); return _changes; } /// /// Watches one record. /// public Observable> WatchRecord(CultRecordKey key) where T : class { return Watch().Where(change => change.Key.Equals(key)); } /// /// Watches the global record for a document type. /// public Observable> WatchGlobal() where T : class { var descriptor = _cache.Registry.GetRequired(); return WatchRecord(new CultRecordKey($"global:{descriptor.SchemaId}")); } /// /// Watches changes for a named document. /// public Observable> WatchByName(string name) where T : class { if (string.IsNullOrWhiteSpace(name)) throw new ArgumentException("Value must be non-empty.", nameof(name)); return Watch().Where(change => (change.Document != null && ReferenceEquals(_cache.GetByName(name), change.Document)) || (change.PreviousDocument != null && ReferenceEquals(_cache.GetByName(name), change.PreviousDocument))); } /// /// Watches changes for the current document mapped by an indexed value. /// public Observable> WatchByIndex(string alias, string value) where T : class { if (string.IsNullOrWhiteSpace(alias)) throw new ArgumentException("Value must be non-empty.", nameof(alias)); if (string.IsNullOrWhiteSpace(value)) throw new ArgumentException("Value must be non-empty.", nameof(value)); return Watch().Where(change => (change.Document != null && ReferenceEquals(_cache.GetByIndex(alias, value), change.Document)) || (change.PreviousDocument != null && ReferenceEquals(_cache.GetByIndex(alias, value), change.PreviousDocument))); } /// /// Releases database subscriptions. /// public void Dispose() { if (_disposed) { return; } _disposed = true; _cache.OnUpdate -= PublishCacheUpdate; _changes.Dispose(); } private void PublishCacheUpdate(object? previous, object? current) { var document = current ?? previous; if (document == null) { return; } var documentType = document.GetType(); var descriptor = _cache.Registry.GetRequired(documentType); var key = current != null ? GetTrackedKey(current, documentType) : new CultRecordKey(string.Empty); var shard = ResolveShardInternal(descriptor.SchemaId, key); var changeType = typeof(CultNetDatabaseChange<>).MakeGenericType(documentType); var change = Activator.CreateInstance( changeType, current == null ? CultNetDatabaseChangeKind.Removed : previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated, key, descriptor.SchemaId, shard, current, previous, null); if (change != null) { _changes.OnNext(change); } } private CultRecordKey GetTrackedKey(object document, Type documentType) { var method = typeof(CultCache).GetMethod(nameof(CultCache.TryGetHandle))! .MakeGenericMethod(documentType); var handleObject = method.Invoke(_cache, new[] { document }); if (handleObject == null) { return new CultRecordKey(string.Empty); } var keyProperty = handleObject.GetType().GetProperty("Key"); return keyProperty?.GetValue(handleObject) is CultRecordKey key ? key : new CultRecordKey(string.Empty); } private IEnumerable GetLocalShardDocuments(CultNetShardDescriptor shard) { foreach (var document in _cache.AllEntries) { var documentType = document.GetType(); var descriptor = _cache.Registry.GetRequired(documentType); var key = GetTrackedKey(document, documentType); if (!string.IsNullOrWhiteSpace(key.Value) && shard.Matches(descriptor.SchemaId, key)) { yield return new TrackedShardDocument(documentType, descriptor.SchemaId, key, document); } } } private bool ShardMatchesRawDocument(CultNetShardDescriptor shard, CultNetRawDocumentRecord document) { return document != null && !string.IsNullOrWhiteSpace(document.SchemaId) && !string.IsNullOrWhiteSpace(document.RecordKey) && shard.Matches(document.SchemaId, new CultRecordKey(document.RecordKey)); } private void RemoveTrackedDocument(Type documentType, CultRecordKey key) { _cache.Remove(key); } private void Publish(CultNetDatabaseChange change) where T : class { _changes.OnNext(change); } private sealed class TrackedShardDocument { public TrackedShardDocument(Type documentType, string schemaId, CultRecordKey key, object document) { DocumentType = documentType; SchemaId = schemaId; Key = key; Document = document; } public Type DocumentType { get; } public string SchemaId { get; } public CultRecordKey Key { get; } public object Document { get; } } private void AppendMutationLogEntry( CultNetShardDescriptor shard, CultNetDatabaseChangeKind kind, CultRecordKey key, string schemaId, object? document, object? previousDocument, CultNetShardLogEntryMessage? wireEntry = null) { var sequence = NextMutationLogSequence(shard.ShardId); var committedAt = DateTimeOffset.UtcNow.ToString("O"); var entry = new CultNetShardMutationLogEntry( shard.ShardId, shard.Epoch, sequence, committedAt, kind, schemaId, key, document, previousDocument); var storedWireEntry = _mutationLogStore == null ? null : wireEntry ?? ToLogEntryMessage(entry); RecordMutationLogEntry(entry, storedWireEntry); } private void RecordMutationLogEntry( CultNetShardMutationLogEntry entry, CultNetShardLogEntryMessage? wireEntry = null) { if (!_mutationLogs.TryGetValue(entry.ShardId, out var entries)) { entries = new List(); _mutationLogs[entry.ShardId] = entries; } entries.Add(entry); if (!_nextLogSequences.TryGetValue(entry.ShardId, out var next) || next <= entry.Sequence) { _nextLogSequences[entry.ShardId] = entry.Sequence + 1; } if (_mutationLogStore != null && wireEntry != null) { _mutationLogStore.Append(entry.ShardId, NormalizeWireLogEntry(entry, wireEntry)); } } private long NextMutationLogSequence(string shardId) { var sequence = _nextLogSequences.TryGetValue(shardId, out var next) ? next : 1; _nextLogSequences[shardId] = sequence + 1; return sequence; } private void InitializeLogSequencesFromStore() { if (_mutationLogStore == null) { return; } foreach (var shard in _shards) { var highest = _mutationLogStore.Read(shard.ShardId) .Select(entry => entry.Sequence) .DefaultIfEmpty(0) .Max(); if (highest > 0) { _nextLogSequences[shard.ShardId] = highest + 1; } } } private async Task ApplyCommittedShardLogEntryAsync( CultNetShardDescriptor shard, CultNetShardLogEntryMessage entry) { if (entry.Put != null) { await ApplyCommittedPutAsync(shard, entry).ConfigureAwait(false); return; } if (entry.Delete != null) { ApplyCommittedDelete(shard, entry); return; } throw new InvalidOperationException( $"Shard '{shard.ShardId}' log entry {entry.Sequence} has no put or delete payload."); } private async Task ApplyCommittedPutAsync( CultNetShardDescriptor shard, CultNetShardLogEntryMessage entry) { var message = entry.Put!; if (message.Document == null) { throw new InvalidOperationException( $"Shard '{shard.ShardId}' log entry {entry.Sequence} put is missing its document payload."); } if (!string.Equals(message.ShardId, shard.ShardId, StringComparison.Ordinal) || message.ShardEpoch != shard.Epoch) { throw new CultNetShardAuthorityException( shard, $"Shard '{shard.ShardId}' log entry {entry.Sequence} targets a different shard or epoch.", "stale_epoch"); } var key = new CultRecordKey(message.Document.RecordKey); var descriptor = _cache.Registry.GetRequiredBySchemaId(message.Document.SchemaId); var previous = _cache.Get(key); var document = await _documents.ApplyRawDocumentPutMessageAsync(_cache, message).ConfigureAwait(false); var kind = ChangeKindFromWire(entry.ChangeKind); if (kind == CultNetDatabaseChangeKind.Removed) { throw new InvalidOperationException( $"Shard '{shard.ShardId}' log entry {entry.Sequence} has a removed kind with a put payload."); } var predictionKey = PredictionKey(descriptor.SchemaId, key); if (_predictedDocuments.Remove(predictionKey)) { kind = CultNetDatabaseChangeKind.Reconciled; } RecordMutationLogEntry(new CultNetShardMutationLogEntry( shard.ShardId, shard.Epoch, entry.Sequence, entry.CommittedAt, kind == CultNetDatabaseChangeKind.Reconciled ? CultNetDatabaseChangeKind.Updated : kind, descriptor.SchemaId, key, document, previous), entry); PublishUntyped( descriptor.DocumentType, kind, key, descriptor.SchemaId, shard, document, previous); } private void ApplyCommittedDelete( CultNetShardDescriptor shard, CultNetShardLogEntryMessage entry) { var message = entry.Delete!; if (!string.Equals(message.ShardId, shard.ShardId, StringComparison.Ordinal) || message.ShardEpoch != shard.Epoch) { throw new CultNetShardAuthorityException( shard, $"Shard '{shard.ShardId}' log entry {entry.Sequence} targets a different shard or epoch.", "stale_epoch"); } var key = new CultRecordKey(message.RecordKey); var descriptor = _cache.Registry.GetRequiredBySchemaId(message.SchemaId); var previous = _cache.Get(key); if (previous != null) { _cache.Remove(key); } RecordMutationLogEntry(new CultNetShardMutationLogEntry( shard.ShardId, shard.Epoch, entry.Sequence, entry.CommittedAt, CultNetDatabaseChangeKind.Removed, descriptor.SchemaId, key, document: null, previousDocument: previous), entry); PublishUntyped( descriptor.DocumentType, CultNetDatabaseChangeKind.Removed, key, descriptor.SchemaId, shard, document: null, previousDocument: previous); } private CultNetShardLogEntryMessage ToLogEntryMessage(CultNetShardMutationLogEntry entry) { if (entry.Kind == CultNetDatabaseChangeKind.Removed || entry.Document == null) { return new CultNetShardLogEntryMessage { Sequence = entry.Sequence, CommittedAt = entry.CommittedAt, ChangeKind = "removed", Delete = new CultNetDocumentDeleteMessage { MessageId = Guid.NewGuid().ToString("N"), SchemaId = entry.SchemaId, RecordKey = entry.Key.Value, ShardId = entry.ShardId, ShardEpoch = entry.ShardEpoch } }; } return new CultNetShardLogEntryMessage { Sequence = entry.Sequence, CommittedAt = entry.CommittedAt, ChangeKind = entry.Kind == CultNetDatabaseChangeKind.Added ? "added" : "updated", Put = new CultNetDocumentPutRawMessage { MessageId = Guid.NewGuid().ToString("N"), Document = CreateRawDocumentRecord(entry.Key, entry.Document), ShardId = entry.ShardId, ShardEpoch = entry.ShardEpoch } }; } private CultNetShardLogEntryMessage NormalizeWireLogEntry( CultNetShardMutationLogEntry entry, CultNetShardLogEntryMessage wireEntry) { return new CultNetShardLogEntryMessage { Sequence = entry.Sequence, CommittedAt = string.IsNullOrWhiteSpace(entry.CommittedAt) ? wireEntry.CommittedAt : entry.CommittedAt, ChangeKind = entry.Kind == CultNetDatabaseChangeKind.Removed ? "removed" : entry.Kind == CultNetDatabaseChangeKind.Added ? "added" : "updated", Put = wireEntry.Put == null ? null : new CultNetDocumentPutRawMessage { MessageId = string.IsNullOrWhiteSpace(wireEntry.Put.MessageId) ? Guid.NewGuid().ToString("N") : wireEntry.Put.MessageId, Document = wireEntry.Put.Document, ShardId = entry.ShardId, ShardEpoch = entry.ShardEpoch }, Delete = wireEntry.Delete == null ? null : new CultNetDocumentDeleteMessage { MessageId = string.IsNullOrWhiteSpace(wireEntry.Delete.MessageId) ? Guid.NewGuid().ToString("N") : wireEntry.Delete.MessageId, SchemaId = wireEntry.Delete.SchemaId, RecordKey = wireEntry.Delete.RecordKey, ShardId = entry.ShardId, ShardEpoch = entry.ShardEpoch } }; } private CultNetRawDocumentRecord CreateRawDocumentRecord(CultRecordKey key, object document) { var method = typeof(CultNetDocumentRegistry) .GetMethods(BindingFlags.Public | BindingFlags.Instance) .Single(candidate => candidate.Name == nameof(CultNetDocumentRegistry.CreateRawDocumentPutMessage) && candidate.IsGenericMethodDefinition); var documentType = document.GetType(); var handleType = typeof(CultRecordHandle<>).MakeGenericType(documentType); var handle = Activator.CreateInstance(handleType, new object[] { key }); var message = method .MakeGenericMethod(documentType) .Invoke(_documents, new[] { Guid.NewGuid().ToString("N"), handle, document, null }); return ((CultNetDocumentPutRawMessage)message!).Document; } private static CultNetDatabaseChangeKind ChangeKindFromWire(string changeKind) { return changeKind switch { "added" => CultNetDatabaseChangeKind.Added, "updated" => CultNetDatabaseChangeKind.Updated, "removed" => CultNetDatabaseChangeKind.Removed, _ => throw new InvalidOperationException($"Unsupported shard log change kind '{changeKind}'.") }; } private void EnsureClientAuthority(string schemaId, CultRecordKey key) { if (_clientAuthorityScopes.Any(scope => scope.Matches(_runtimeId, schemaId, key))) { return; } var shard = ResolveShardInternal(schemaId, key); throw new CultNetShardAuthorityException( shard, $"Runtime '{_runtimeId}' does not have client prediction authority for schema '{schemaId}' key '{key.Value}'.", "not_client_authority"); } private static string PredictionKey(string schemaId, CultRecordKey key) { return $"{schemaId}:{key.Value}"; } private void PublishUntyped( Type documentType, CultNetDatabaseChangeKind kind, CultRecordKey key, string schemaId, CultNetShardDescriptor shard, object? document, object? previousDocument) { var changeType = typeof(CultNetDatabaseChange<>).MakeGenericType(documentType); var change = Activator.CreateInstance( changeType, kind, key, schemaId, shard, document, previousDocument, null); if (change != null) { _changes.OnNext(change); } } private CultNetShardDescriptor ResolveShardInternal(string schemaId, CultRecordKey key) { return _shards.FirstOrDefault(shard => shard.Matches(schemaId, key)) ?? _shards[0]; } private static void EnsurePrimary(CultNetShardDescriptor shard, string schemaId, CultRecordKey key, long? expectedEpoch = null) { if (expectedEpoch.HasValue && expectedEpoch.Value != shard.Epoch) { throw new CultNetShardAuthorityException( shard, $"Shard '{shard.ShardId}' is at epoch {shard.Epoch}, not requested epoch {expectedEpoch.Value}.", "stale_epoch"); } if (shard.IsPrimary) { return; } throw new CultNetShardAuthorityException( shard, $"Shard '{shard.ShardId}' owned by '{shard.OwnerRuntimeId}' does not accept local writes for schema '{schemaId}' key '{key.Value}'.", "not_primary"); } private static bool MatchesCatalogFilter( CultNetShardDescriptor shard, IReadOnlyList? schemaIds, IReadOnlyList? recordKeys) { var schemaMatches = schemaIds == null || schemaIds.Count == 0 || shard.SchemaIds.Count == 0 || schemaIds.Any(schemaId => shard.SchemaIds.Contains(schemaId, StringComparer.Ordinal)); var keyMatches = recordKeys == null || recordKeys.Count == 0 || recordKeys.Any(key => string.IsNullOrEmpty(shard.KeyPrefix) || key.Value.StartsWith(shard.KeyPrefix!, StringComparison.Ordinal)); return schemaMatches && keyMatches; } internal static CultNetShardDescriptorMessage ToMessage(CultNetShardDescriptor shard) { return new CultNetShardDescriptorMessage { ShardId = shard.ShardId, OwnerRuntimeId = shard.OwnerRuntimeId, Epoch = shard.Epoch, IsPrimary = shard.IsPrimary, SchemaIds = shard.SchemaIds.ToArray(), KeyPrefix = shard.KeyPrefix, PrimaryEndpoints = shard.PrimaryEndpoints.ToArray(), ReplicaEndpoints = shard.ReplicaEndpoints.ToArray(), ReadReplicaEndpoints = shard.ReadReplicaEndpoints.ToArray(), Region = shard.Region }; } private void ThrowIfDisposed() { if (_disposed) { throw new ObjectDisposedException(nameof(CultNetDatabase)); } } } }