using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using MessagePack;
namespace GameCult.Networking
{
///
/// Fetches committed shard-log entries from an authoritative shard owner.
///
public interface ICultNetShardLogFetcher
{
///
/// Fetches shard-log entries after the supplied sequence.
///
Task FetchAsync(
CultNetShardDescriptor shard,
long afterSequence,
int? limit = null);
}
///
/// Fetches shard-bounded snapshots from an authoritative shard owner.
///
public interface ICultNetShardSnapshotFetcher
{
///
/// Fetches a shard snapshot.
///
Task FetchAsync(CultNetShardDescriptor shard);
}
///
/// Persisted replica cursor for one shard.
///
[MessagePackObject]
public sealed class CultNetShardReplicaCursor
{
///
/// Gets or sets the shard id.
///
[Key("shardId")] public string ShardId { get; set; } = string.Empty;
///
/// Gets or sets the shard epoch.
///
[Key("shardEpoch")] public long ShardEpoch { get; set; }
///
/// Gets or sets the last applied shard-log sequence.
///
[Key("lastAppliedSequence")] public long LastAppliedSequence { get; set; }
///
/// Gets or sets the cursor update timestamp.
///
[Key("updatedAt")] public string UpdatedAt { get; set; } = string.Empty;
}
///
/// Stores restart-safe replica cursors.
///
public interface ICultNetShardReplicaCursorStore
{
///
/// Reads the cursor for a shard, if one exists.
///
Task ReadAsync(string shardId);
///
/// Writes the cursor for a shard.
///
Task WriteAsync(CultNetShardReplicaCursor cursor);
}
///
/// Options for background shard-log replication.
///
public sealed class CultNetShardReplicatorOptions
{
///
/// Gets or sets the interval used by the background pull loop.
///
public TimeSpan PollInterval { get; set; } = TimeSpan.FromSeconds(1);
///
/// Gets or sets the maximum number of log entries requested per pull.
///
public int? BatchSize { get; set; } = 256;
///
/// Gets or sets the fetcher used to read from shard primaries.
///
public ICultNetShardLogFetcher? Fetcher { get; set; }
///
/// Gets or sets the fetcher used when log history has been compacted.
///
public ICultNetShardSnapshotFetcher? SnapshotFetcher { get; set; }
///
/// Gets or sets a callback for background pull failures.
///
public Action? OnError { get; set; }
///
/// Gets or sets optional restart-safe cursor storage.
///
public ICultNetShardReplicaCursorStore? CursorStore { get; set; }
}
///
/// Pulls committed shard logs from primaries and applies them to a local replica database.
///
public sealed class CultNetShardReplicator : IDisposable
{
private readonly CultNetDatabase _database;
private readonly CultNetShardReplicatorOptions _options;
private readonly List _subscriptions = new();
private readonly ConcurrentDictionary _pullsInFlight = new(StringComparer.Ordinal);
private bool _disposed;
///
/// Creates a shard replicator.
///
public CultNetShardReplicator(
CultNetDatabase database,
CultNetShardReplicatorOptions? options = null)
{
_database = database ?? throw new ArgumentNullException(nameof(database));
_options = options ?? new CultNetShardReplicatorOptions();
}
///
/// Starts background pull loops for non-primary shards with advertised primaries.
///
public void Start()
{
ThrowIfDisposed();
if (_options.Fetcher == null)
{
throw new InvalidOperationException("A shard log fetcher is required before starting replication.");
}
if (_subscriptions.Count > 0)
{
return;
}
foreach (var shard in _database.Shards.Where(shard => !shard.IsPrimary && shard.PrimaryEndpoints.Count > 0))
{
_subscriptions.Add(new Timer(
_ => _ = PullIfIdleAsync(shard),
state: null,
dueTime: TimeSpan.Zero,
period: _options.PollInterval));
}
}
///
/// Pulls and applies one batch for a shard id.
///
public Task PullOnceAsync(string shardId)
{
ThrowIfDisposed();
if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId));
var shard = _database.Shards.FirstOrDefault(candidate =>
string.Equals(candidate.ShardId, shardId, StringComparison.Ordinal));
if (shard == null)
{
throw new InvalidOperationException($"Shard '{shardId}' is not known by this database.");
}
return PullOnceAsync(shard);
}
///
/// Pulls and applies one batch for a shard.
///
public async Task PullOnceAsync(CultNetShardDescriptor shard)
{
ThrowIfDisposed();
if (shard == null) throw new ArgumentNullException(nameof(shard));
if (_options.Fetcher == null)
{
throw new InvalidOperationException("A shard log fetcher is required before pulling replication.");
}
if (shard.IsPrimary)
{
throw new InvalidOperationException($"Shard '{shard.ShardId}' is primary on this node and does not need replica pulling.");
}
var afterSequence = await GetAfterSequenceAsync(shard).ConfigureAwait(false);
var response = await _options.Fetcher
.FetchAsync(shard, afterSequence, _options.BatchSize)
.ConfigureAwait(false);
if (response.ResyncRequired &&
string.Equals(response.Reason, "compacted_history", StringComparison.Ordinal))
{
if (_options.SnapshotFetcher == null)
{
throw new InvalidOperationException(
$"Shard '{shard.ShardId}' requires snapshot resync, but no shard snapshot fetcher is configured.");
}
var snapshot = await _options.SnapshotFetcher.FetchAsync(shard).ConfigureAwait(false);
var snapshotSequence = await _database.ApplyShardSnapshotResponseAsync(shard, snapshot).ConfigureAwait(false);
await WriteCursorAsync(shard, snapshotSequence).ConfigureAwait(false);
return snapshotSequence;
}
var appliedSequence = await _database.ApplyShardLogResponseAsync(response).ConfigureAwait(false);
await WriteCursorAsync(shard, appliedSequence).ConfigureAwait(false);
return appliedSequence;
}
///
public void Dispose()
{
if (_disposed)
{
return;
}
_disposed = true;
foreach (var subscription in _subscriptions)
{
subscription.Dispose();
}
_subscriptions.Clear();
_pullsInFlight.Clear();
}
private async Task PullIfIdleAsync(CultNetShardDescriptor shard)
{
if (!_pullsInFlight.TryAdd(shard.ShardId, 0))
{
return;
}
try
{
await PullOnceAsync(shard).ConfigureAwait(false);
}
catch (Exception ex)
{
_options.OnError?.Invoke(ex);
}
finally
{
_pullsInFlight.TryRemove(shard.ShardId, out _);
}
}
private async Task GetAfterSequenceAsync(CultNetShardDescriptor shard)
{
var afterSequence = _database.GetAppliedShardSequence(shard.ShardId);
if (afterSequence > 0 || _options.CursorStore == null)
{
return afterSequence;
}
var cursor = await _options.CursorStore.ReadAsync(shard.ShardId).ConfigureAwait(false);
if (cursor == null || cursor.ShardEpoch != shard.Epoch)
{
return afterSequence;
}
_database.SetAppliedShardSequence(shard.ShardId, cursor.LastAppliedSequence);
return cursor.LastAppliedSequence;
}
private async Task WriteCursorAsync(CultNetShardDescriptor shard, long appliedSequence)
{
if (_options.CursorStore == null)
{
return;
}
await _options.CursorStore.WriteAsync(new CultNetShardReplicaCursor
{
ShardId = shard.ShardId,
ShardEpoch = shard.Epoch,
LastAppliedSequence = appliedSequence,
UpdatedAt = DateTimeOffset.UtcNow.ToString("O")
}).ConfigureAwait(false);
}
private void ThrowIfDisposed()
{
if (_disposed)
{
throw new ObjectDisposedException(nameof(CultNetShardReplicator));
}
}
}
///
/// Stores replica cursors in one local MessagePack file.
///
public sealed class CultNetFileShardReplicaCursorStore : ICultNetShardReplicaCursorStore
{
private readonly string _filePath;
private readonly object _gate = new();
///
/// Creates a file-backed cursor store.
///
public CultNetFileShardReplicaCursorStore(string filePath)
{
_filePath = string.IsNullOrWhiteSpace(filePath)
? throw new ArgumentException("Value must be non-empty.", nameof(filePath))
: filePath;
}
///
public Task ReadAsync(string shardId)
{
if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId));
lock (_gate)
{
CultNetShardReplicaCursor? cursor = ReadAll().FirstOrDefault(candidate =>
string.Equals(candidate.ShardId, shardId, StringComparison.Ordinal));
return Task.FromResult(cursor);
}
}
///
public Task WriteAsync(CultNetShardReplicaCursor cursor)
{
if (cursor == null) throw new ArgumentNullException(nameof(cursor));
if (string.IsNullOrWhiteSpace(cursor.ShardId))
{
throw new ArgumentException("Cursor requires a shardId.", nameof(cursor));
}
lock (_gate)
{
var cursors = ReadAll()
.Where(existing => !string.Equals(existing.ShardId, cursor.ShardId, StringComparison.Ordinal))
.Concat(new[] { cursor })
.OrderBy(existing => existing.ShardId, StringComparer.Ordinal)
.ToArray();
var directory = Path.GetDirectoryName(Path.GetFullPath(_filePath));
if (!string.IsNullOrWhiteSpace(directory))
{
Directory.CreateDirectory(directory);
}
File.WriteAllBytes(_filePath, MessagePackSerializer.Serialize(cursors));
return Task.CompletedTask;
}
}
private CultNetShardReplicaCursor[] ReadAll()
{
if (!File.Exists(_filePath))
{
return Array.Empty();
}
var bytes = File.ReadAllBytes(_filePath);
return bytes.Length == 0
? Array.Empty()
: MessagePackSerializer.Deserialize(bytes);
}
}
///
/// Options for the schema-v0 client-based shard-log fetcher.
///
public sealed class CultNetSchemaShardLogFetcherOptions
{
///
/// Gets or sets client security options used to connect to primary endpoints.
///
public ClientSecurityOptions? Security { get; set; }
///
/// Gets or sets how long to wait for a connection before failing.
///
public TimeSpan ConnectTimeout { get; set; } = TimeSpan.FromSeconds(5);
///
/// Gets or sets how long to wait for a log response before failing.
///
public TimeSpan ResponseTimeout { get; set; } = TimeSpan.FromSeconds(5);
///
/// Gets or sets a callback used to customize each ephemeral fetch client.
///
public Action? ConfigureClient { get; set; }
///
/// Gets or sets the schema-v0 client factory. Defaults to an endpoint-aware adapter:
/// rudp:// endpoints use the C# RUDP schema client and cultnet:// endpoints use LiteNetLib.
///
public Func? CreateClient { get; set; }
}
///
/// Fetches shard-log batches over the CultNet schema-v0 client transport.
///
public sealed class CultNetSchemaShardLogFetcher : ICultNetShardLogFetcher
{
private readonly CultNetSchemaShardLogFetcherOptions _options;
///
/// Creates a schema-v0 shard-log fetcher.
///
public CultNetSchemaShardLogFetcher(CultNetSchemaShardLogFetcherOptions? options = null)
{
_options = options ?? new CultNetSchemaShardLogFetcherOptions();
}
///
public async Task FetchAsync(
CultNetShardDescriptor shard,
long afterSequence,
int? limit = null)
{
if (shard == null) throw new ArgumentNullException(nameof(shard));
var endpoint = ResolvePrimaryEndpoint(shard);
var (host, port) = CultNetSchemaWriteForwarder.ParseEndpoint(endpoint);
var messageId = Guid.NewGuid().ToString("N");
var completion = new TaskCompletionSource(
TaskCreationOptions.RunContinuationsAsynchronously);
using var client = CreateClient(endpoint);
client.OnCultNet(response =>
{
if (string.Equals(response.MessageId, messageId, StringComparison.Ordinal))
{
completion.TrySetResult(response);
}
});
client.OnCultNet(error =>
completion.TrySetException(new InvalidOperationException(error.Error)));
client.Connect(host, port);
await WaitForConnectionAsync(client, endpoint).ConfigureAwait(false);
client.SendCultNet(new CultNetShardLogRequestMessage
{
MessageId = messageId,
ShardId = shard.ShardId,
ShardEpoch = shard.Epoch,
AfterSequence = afterSequence,
Limit = limit
});
return await WaitForResponseAsync(completion.Task, endpoint).ConfigureAwait(false);
}
private ICultNetSchemaClient CreateClient(string endpoint)
{
return _options.CreateClient?.Invoke()
?? CultNetSchemaClients.CreateForEndpoint(endpoint, _options.Security, _options.ConfigureClient);
}
private async Task WaitForConnectionAsync(ICultNetSchemaClient client, string endpoint)
{
var deadline = DateTimeOffset.UtcNow + _options.ConnectTimeout;
while (!client.Connected)
{
if (DateTimeOffset.UtcNow >= deadline)
{
throw new TimeoutException($"Timed out connecting to shard primary endpoint {endpoint}.");
}
await Task.Delay(TimeSpan.FromMilliseconds(25)).ConfigureAwait(false);
}
}
private async Task WaitForResponseAsync(
Task responseTask,
string endpoint)
{
var timeoutTask = Task.Delay(_options.ResponseTimeout);
var completed = await Task.WhenAny(responseTask, timeoutTask).ConfigureAwait(false);
if (completed != responseTask)
{
throw new TimeoutException($"Timed out waiting for shard log response from {endpoint}.");
}
return await responseTask.ConfigureAwait(false);
}
private static string ResolvePrimaryEndpoint(CultNetShardDescriptor shard)
{
return shard.PrimaryEndpoints.FirstOrDefault()
?? throw new InvalidOperationException(
$"Shard '{shard.ShardId}' does not advertise a primary endpoint.");
}
}
///
/// Options for the schema-v0 client-based shard snapshot fetcher.
///
public sealed class CultNetSchemaShardSnapshotFetcherOptions
{
///
/// Gets or sets client security options used to connect to primary endpoints.
///
public ClientSecurityOptions? Security { get; set; }
///
/// Gets or sets how long to wait for a connection before failing.
///
public TimeSpan ConnectTimeout { get; set; } = TimeSpan.FromSeconds(5);
///
/// Gets or sets how long to wait for a snapshot response before failing.
///
public TimeSpan ResponseTimeout { get; set; } = TimeSpan.FromSeconds(5);
///
/// Gets or sets schema ids to request. Null or empty requests every schema allowed by the shard.
///
public IReadOnlyList? SchemaIds { get; set; }
///
/// Gets or sets record keys to request. Null or empty requests every record allowed by the shard/schema filter.
///
public IReadOnlyList? RecordKeys { get; set; }
///
/// Gets or sets a callback used to customize each ephemeral fetch client.
///
public Action? ConfigureClient { get; set; }
///
/// Gets or sets the schema-v0 client factory. Defaults to an endpoint-aware adapter:
/// rudp:// endpoints use the C# RUDP schema client and cultnet:// endpoints use LiteNetLib.
///
public Func? CreateClient { get; set; }
}
///
/// Fetches shard snapshots over the CultNet schema-v0 client transport.
///
public sealed class CultNetSchemaShardSnapshotFetcher : ICultNetShardSnapshotFetcher
{
private readonly CultNetSchemaShardSnapshotFetcherOptions _options;
///
/// Creates a schema-v0 shard snapshot fetcher.
///
public CultNetSchemaShardSnapshotFetcher(CultNetSchemaShardSnapshotFetcherOptions? options = null)
{
_options = options ?? new CultNetSchemaShardSnapshotFetcherOptions();
}
///
public async Task FetchAsync(CultNetShardDescriptor shard)
{
if (shard == null) throw new ArgumentNullException(nameof(shard));
var endpoint = ResolvePrimaryEndpoint(shard);
var (host, port) = CultNetSchemaWriteForwarder.ParseEndpoint(endpoint);
var messageId = Guid.NewGuid().ToString("N");
var completion = new TaskCompletionSource(
TaskCreationOptions.RunContinuationsAsynchronously);
using var client = CreateClient(endpoint);
client.OnCultNet(response =>
{
if (string.Equals(response.MessageId, messageId, StringComparison.Ordinal))
{
completion.TrySetResult(response);
}
});
client.OnCultNet(error =>
completion.TrySetException(new InvalidOperationException(error.Error)));
client.Connect(host, port);
await WaitForConnectionAsync(client, endpoint).ConfigureAwait(false);
client.SendCultNet(new CultNetSnapshotRequestMessage
{
MessageId = messageId,
SchemaIds = ResolveRequestedSchemaIds(shard),
RecordKeys = ResolveRequestedRecordKeys(),
ShardId = shard.ShardId,
ShardEpoch = shard.Epoch
});
return await WaitForResponseAsync(completion.Task, endpoint).ConfigureAwait(false);
}
private ICultNetSchemaClient CreateClient(string endpoint)
{
return _options.CreateClient?.Invoke()
?? CultNetSchemaClients.CreateForEndpoint(endpoint, _options.Security, _options.ConfigureClient);
}
private string[]? ResolveRequestedSchemaIds(CultNetShardDescriptor shard)
{
if (_options.SchemaIds is { Count: > 0 })
{
return _options.SchemaIds
.Where(value => !string.IsNullOrWhiteSpace(value))
.Distinct(StringComparer.Ordinal)
.ToArray();
}
return shard.SchemaIds.Count == 0 ? null : shard.SchemaIds.ToArray();
}
private string[]? ResolveRequestedRecordKeys()
{
if (_options.RecordKeys is not { Count: > 0 })
{
return null;
}
var recordKeys = _options.RecordKeys
.Where(value => !string.IsNullOrWhiteSpace(value))
.Distinct(StringComparer.Ordinal)
.ToArray();
return recordKeys.Length == 0 ? null : recordKeys;
}
private async Task WaitForConnectionAsync(ICultNetSchemaClient client, string endpoint)
{
var deadline = DateTimeOffset.UtcNow + _options.ConnectTimeout;
while (!client.Connected)
{
if (DateTimeOffset.UtcNow >= deadline)
{
throw new TimeoutException($"Timed out connecting to shard primary endpoint {endpoint}.");
}
await Task.Delay(TimeSpan.FromMilliseconds(25)).ConfigureAwait(false);
}
}
private async Task WaitForResponseAsync(
Task responseTask,
string endpoint)
{
var timeoutTask = Task.Delay(_options.ResponseTimeout);
var completed = await Task.WhenAny(responseTask, timeoutTask).ConfigureAwait(false);
if (completed != responseTask)
{
throw new TimeoutException($"Timed out waiting for shard snapshot response from {endpoint}.");
}
return await responseTask.ConfigureAwait(false);
}
private static string ResolvePrimaryEndpoint(CultNetShardDescriptor shard)
{
return shard.PrimaryEndpoints.FirstOrDefault()
?? throw new InvalidOperationException(
$"Shard '{shard.ShardId}' does not advertise a primary endpoint.");
}
}
}