GameCult/CultLib
Watch
9
0
Fork
You've already forked CultLib
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add durable shard mutation log storage

Browse source
This commit is contained in:
Meta Vens 2026-05-19 19:55:29 +01:00
commit 2ff82365cb
8 changed files with 367 additions and 52 deletions
Download patch file Download diff file Expand all files Collapse all files

16
src/GameCult.Networking/Contracts/cultnet-distributed-database.md
View file

@ -59,6 +59,9 @@ Implemented:
- schema-v0 shard log fetcher for `cultnet://host:port` endpoints
- restart-safe replica cursor store, including a local MessagePack file-backed
implementation
- durable authoritative shard-log storage behind
`ICultNetShardMutationLogStore`, including a per-shard MessagePack file
implementation
- client-authority scopes for locally predicted input documents
- predicted and reconciled database change kinds for client-side prediction
- simulation witness observations for mesh-side opinions about frame facts
@ -73,7 +76,7 @@ Not implemented yet:
- membership/failure detection
- leader election or automatic shard failover
- durable log persistence and compaction
- log compaction
- snapshot fallback for compacted log history
- peer-to-peer fanout for observation gossip and candidate propagation
- rollback/resimulation helpers for simulation frames
@ -89,6 +92,8 @@ Not implemented yet:
applied.
- Raw snapshot/put/delete messages must pass through `CultNetDatabase`, not
bypass shard policy.
- The accepted shard log is stored in replica catch-up wire form. Object logs
are for local inspection; wire logs are the durable replication authority.
- Realtime subscriptions publish domain changes, not storage envelopes.
- Client-owned input documents may be predicted locally only inside explicit
`CultNetClientAuthorityScope` declarations.
@ -103,11 +108,12 @@ Not implemented yet:
## Next Coherent Slice
Add durable mutation-log storage and snapshot fallback:
Add compaction-aware snapshot fallback and discovery fanout:
- persist per-shard mutation logs instead of keeping them only in memory
- define the compaction boundary that turns old log reads into snapshot reads
- return `ResyncRequired` when a replica asks for compacted history
- expose operator-facing defaults for durable shard-log paths in CultMesh nodes
- add Verse discovery over the schema-v0 mesh lane
- after that, add simulation-frame rollback/resimulation helpers around
predicted input streams
- add peer-to-peer fanout for observation gossip and candidate propagation
@ -120,4 +126,6 @@ observations into deterministic consensus candidates for simulation facts like
"who shot first," and those reports now have schema-v0 wire contracts. The next
useful layer is making authoritative mutation logs durable and compaction-aware.
Membership and leader election should wait until basic replica catch-up is
boring.
boring. Authoritative mutation logs can now survive process restart through
`ICultNetShardMutationLogStore`; compaction and snapshot fallback are the next
places where the machine needs an honest boundary.

178
src/GameCult.Networking/CultNetDatabase.cs
View file

@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Threading.Tasks;
using GameCult.Caching;
using R3;
@ -225,6 +226,10 @@ namespace GameCult.Networking
/// Gets or sets the document registry used to create and apply raw CultNet document messages.
/// </summary>
public CultNetDocumentRegistry? DocumentRegistry { get; set; }
/// <summary>
/// Gets or sets the durable store for accepted shard mutation logs.
/// </summary>
public ICultNetShardMutationLogStore? MutationLogStore { get; set; }
}
/// <summary>
@ -385,6 +390,7 @@ namespace GameCult.Networking
private readonly string _runtimeId;
private readonly List<CultNetShardDescriptor> _shards;
private readonly List<CultNetClientAuthorityScope> _clientAuthorityScopes;
private readonly ICultNetShardMutationLogStore? _mutationLogStore;
private readonly Dictionary<string, object> _predictedDocuments = new(StringComparer.Ordinal);
private readonly Dictionary<string, List<CultNetShardMutationLogEntry>> _mutationLogs =
new(StringComparer.Ordinal);
@ -402,11 +408,13 @@ namespace GameCult.Networking
options ??= new CultNetDatabaseOptions();
_runtimeId = options.RuntimeId;
_documents = options.DocumentRegistry ?? new CultNetDocumentRegistry(cache.Registry);
_mutationLogStore = options.MutationLogStore;
_shards = (options.Shards == null || options.Shards.Count == 0
? new[] { CultNetShardDescriptor.PrimaryAll(options.RuntimeId) }
: options.Shards)
.ToList();
_clientAuthorityScopes = (options.ClientAuthorityScopes ?? Array.Empty<CultNetClientAuthorityScope>()).ToList();
InitializeLogSequencesFromStore();
_cache.OnUpdate += PublishCacheUpdate;
}
@ -452,6 +460,26 @@ namespace GameCult.Networking
return query.ToArray();
}
/// <summary>
/// Gets replica catch-up log entries for a shard after the supplied sequence.
/// </summary>
public IReadOnlyList<CultNetShardLogEntryMessage> GetMutationLogMessages(
string shardId,
long afterSequence = 0,
int? limit = null)
{
ThrowIfDisposed();
if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId));
if (_mutationLogStore != null)
{
return _mutationLogStore.Read(shardId, afterSequence, limit);
}
return GetMutationLog(shardId, afterSequence, limit)
.Select(ToLogEntryMessage)
.ToArray();
}
/// <summary>
/// Gets the last replicated sequence applied for a shard.
/// </summary>
@ -639,7 +667,12 @@ namespace GameCult.Networking
key,
descriptor.SchemaId,
document,
previous);
previous,
new CultNetShardLogEntryMessage
{
ChangeKind = previous == null ? "added" : "updated",
Put = message
});
PublishUntyped(
descriptor.DocumentType,
kind,
@ -680,7 +713,12 @@ namespace GameCult.Networking
key,
descriptor.SchemaId,
document: null,
previousDocument: previous);
previousDocument: previous,
wireEntry: new CultNetShardLogEntryMessage
{
ChangeKind = "removed",
Delete = message
});
PublishUntyped(
descriptor.DocumentType,
CultNetDatabaseChangeKind.Removed,
@ -877,22 +915,30 @@ namespace GameCult.Networking
CultRecordKey key,
string schemaId,
object? document,
object? previousDocument)
object? previousDocument,
CultNetShardLogEntryMessage? wireEntry = null)
{
var sequence = NextMutationLogSequence(shard.ShardId);
RecordMutationLogEntry(new CultNetShardMutationLogEntry(
var committedAt = DateTimeOffset.UtcNow.ToString("O");
var entry = new CultNetShardMutationLogEntry(
shard.ShardId,
shard.Epoch,
sequence,
DateTimeOffset.UtcNow.ToString("O"),
committedAt,
kind,
schemaId,
key,
document,
previousDocument));
previousDocument);
var storedWireEntry = _mutationLogStore == null
? null
: wireEntry ?? ToLogEntryMessage(entry);
RecordMutationLogEntry(entry, storedWireEntry);
}
private void RecordMutationLogEntry(CultNetShardMutationLogEntry entry)
private void RecordMutationLogEntry(
CultNetShardMutationLogEntry entry,
CultNetShardLogEntryMessage? wireEntry = null)
{
if (!_mutationLogs.TryGetValue(entry.ShardId, out var entries))
{
@ -906,6 +952,11 @@ namespace GameCult.Networking
{
_nextLogSequences[entry.ShardId] = entry.Sequence + 1;
}
if (_mutationLogStore != null && wireEntry != null)
{
_mutationLogStore.Append(entry.ShardId, NormalizeWireLogEntry(entry, wireEntry));
}
}
private long NextMutationLogSequence(string shardId)
@ -917,6 +968,26 @@ namespace GameCult.Networking
return sequence;
}
private void InitializeLogSequencesFromStore()
{
if (_mutationLogStore == null)
{
return;
}
foreach (var shard in _shards)
{
var highest = _mutationLogStore.Read(shard.ShardId)
.Select(entry => entry.Sequence)
.DefaultIfEmpty(0)
.Max();
if (highest > 0)
{
_nextLogSequences[shard.ShardId] = highest + 1;
}
}
}
private async Task ApplyCommittedShardLogEntryAsync(
CultNetShardDescriptor shard,
CultNetShardLogEntryMessage entry)
@ -983,7 +1054,8 @@ namespace GameCult.Networking
descriptor.SchemaId,
key,
document,
previous));
previous),
entry);
PublishUntyped(
descriptor.DocumentType,
kind,
@ -1029,7 +1101,8 @@ namespace GameCult.Networking
descriptor.SchemaId,
key,
document: null,
previousDocument: previous));
previousDocument: previous),
entry);
PublishUntyped(
descriptor.DocumentType,
CultNetDatabaseChangeKind.Removed,
@ -1040,6 +1113,93 @@ namespace GameCult.Networking
previousDocument: previous);
}
private CultNetShardLogEntryMessage ToLogEntryMessage(CultNetShardMutationLogEntry entry)
{
if (entry.Kind == CultNetDatabaseChangeKind.Removed || entry.Document == null)
{
return new CultNetShardLogEntryMessage
{
Sequence = entry.Sequence,
CommittedAt = entry.CommittedAt,
ChangeKind = "removed",
Delete = new CultNetDocumentDeleteMessage
{
MessageId = Guid.NewGuid().ToString("N"),
SchemaId = entry.SchemaId,
RecordKey = entry.Key.Value,
ShardId = entry.ShardId,
ShardEpoch = entry.ShardEpoch
}
};
}
return new CultNetShardLogEntryMessage
{
Sequence = entry.Sequence,
CommittedAt = entry.CommittedAt,
ChangeKind = entry.Kind == CultNetDatabaseChangeKind.Added ? "added" : "updated",
Put = new CultNetDocumentPutRawMessage
{
MessageId = Guid.NewGuid().ToString("N"),
Document = CreateRawDocumentRecord(entry.Key, entry.Document),
ShardId = entry.ShardId,
ShardEpoch = entry.ShardEpoch
}
};
}
private CultNetShardLogEntryMessage NormalizeWireLogEntry(
CultNetShardMutationLogEntry entry,
CultNetShardLogEntryMessage wireEntry)
{
return new CultNetShardLogEntryMessage
{
Sequence = entry.Sequence,
CommittedAt = string.IsNullOrWhiteSpace(entry.CommittedAt) ? wireEntry.CommittedAt : entry.CommittedAt,
ChangeKind = entry.Kind == CultNetDatabaseChangeKind.Removed
? "removed"
: entry.Kind == CultNetDatabaseChangeKind.Added ? "added" : "updated",
Put = wireEntry.Put == null
? null
: new CultNetDocumentPutRawMessage
{
MessageId = string.IsNullOrWhiteSpace(wireEntry.Put.MessageId)
? Guid.NewGuid().ToString("N")
: wireEntry.Put.MessageId,
Document = wireEntry.Put.Document,
ShardId = entry.ShardId,
ShardEpoch = entry.ShardEpoch
},
Delete = wireEntry.Delete == null
? null
: new CultNetDocumentDeleteMessage
{
MessageId = string.IsNullOrWhiteSpace(wireEntry.Delete.MessageId)
? Guid.NewGuid().ToString("N")
: wireEntry.Delete.MessageId,
SchemaId = wireEntry.Delete.SchemaId,
RecordKey = wireEntry.Delete.RecordKey,
ShardId = entry.ShardId,
ShardEpoch = entry.ShardEpoch
}
};
}
private CultNetRawDocumentRecord CreateRawDocumentRecord(CultRecordKey key, object document)
{
var method = typeof(CultNetDocumentRegistry)
.GetMethods(BindingFlags.Public | BindingFlags.Instance)
.Single(candidate => candidate.Name == nameof(CultNetDocumentRegistry.CreateRawDocumentPutMessage) &&
candidate.IsGenericMethodDefinition);
var documentType = document.GetType();
var handleType = typeof(CultRecordHandle<>).MakeGenericType(documentType);
var handle = Activator.CreateInstance(handleType, new object[] { key });
var message = method
.MakeGenericMethod(documentType)
.Invoke(_documents, new[] { Guid.NewGuid().ToString("N"), handle, document, null });
return ((CultNetDocumentPutRawMessage)message!).Document;
}
private static CultNetDatabaseChangeKind ChangeKindFromWire(string changeKind)
{
return changeKind switch

38
src/GameCult.Networking/CultNetDatabaseServer.cs
View file

@ -132,8 +132,7 @@ namespace GameCult.Networking
};
}
var entries = _database.GetMutationLog(request.ShardId, request.AfterSequence, request.Limit)
.Select(ToLogEntryMessage)
var entries = _database.GetMutationLogMessages(request.ShardId, request.AfterSequence, request.Limit)
.ToArray();
return new CultNetShardLogResponseMessage
@ -334,41 +333,6 @@ namespace GameCult.Networking
};
}
private CultNetShardLogEntryMessage ToLogEntryMessage(CultNetShardMutationLogEntry entry)
{
if (entry.Kind == CultNetDatabaseChangeKind.Removed || entry.Document == null)
{
return new CultNetShardLogEntryMessage
{
Sequence = entry.Sequence,
CommittedAt = entry.CommittedAt,
ChangeKind = "removed",
Delete = new CultNetDocumentDeleteMessage
{
MessageId = Guid.NewGuid().ToString("N"),
SchemaId = entry.SchemaId,
RecordKey = entry.Key.Value,
ShardId = entry.ShardId,
ShardEpoch = entry.ShardEpoch
}
};
}
return new CultNetShardLogEntryMessage
{
Sequence = entry.Sequence,
CommittedAt = entry.CommittedAt,
ChangeKind = entry.Kind == CultNetDatabaseChangeKind.Added ? "added" : "updated",
Put = new CultNetDocumentPutRawMessage
{
MessageId = Guid.NewGuid().ToString("N"),
Document = CreateRawDocumentRecord(entry.Key, entry.Document),
ShardId = entry.ShardId,
ShardEpoch = entry.ShardEpoch
}
};
}
private CultNetRawDocumentRecord CreateRawDocumentRecord(CultRecordKey key, object document)
{
var method = typeof(CultNetDocumentRegistry)

109
src/GameCult.Networking/CultNetShardMutationLogStorage.cs Normal file
View file

@ -0,0 +1,109 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using MessagePack;
namespace GameCult.Networking
{
/// <summary>
/// Stores accepted shard-log entries in their replica catch-up wire shape.
/// </summary>
public interface ICultNetShardMutationLogStore
{
/// <summary>
/// Reads accepted log entries for a shard after the supplied sequence.
/// </summary>
IReadOnlyList<CultNetShardLogEntryMessage> Read(string shardId, long afterSequence = 0, int? limit = null);
/// <summary>
/// Appends or replaces one accepted log entry for a shard.
/// </summary>
void Append(string shardId, CultNetShardLogEntryMessage entry);
}
/// <summary>
/// Persists shard-log entries as one MessagePack file per shard.
/// </summary>
public sealed class CultNetFileShardMutationLogStore : ICultNetShardMutationLogStore
{
private readonly string _rootPath;
private readonly object _gate = new();
/// <summary>
/// Creates a file-backed shard mutation log store.
/// </summary>
public CultNetFileShardMutationLogStore(string rootPath)
{
if (string.IsNullOrWhiteSpace(rootPath)) throw new ArgumentException("Value must be non-empty.", nameof(rootPath));
_rootPath = rootPath;
Directory.CreateDirectory(_rootPath);
}
/// <inheritdoc />
public IReadOnlyList<CultNetShardLogEntryMessage> Read(string shardId, long afterSequence = 0, int? limit = null)
{
if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId));
if (afterSequence < 0) throw new ArgumentOutOfRangeException(nameof(afterSequence), "Sequence must be non-negative.");
lock (_gate)
{
IEnumerable<CultNetShardLogEntryMessage> entries = ReadAll(shardId)
.Where(entry => entry.Sequence > afterSequence)
.OrderBy(entry => entry.Sequence);
if (limit.HasValue)
{
entries = entries.Take(limit.Value);
}
return entries.ToArray();
}
}
/// <inheritdoc />
public void Append(string shardId, CultNetShardLogEntryMessage entry)
{
if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId));
if (entry == null) throw new ArgumentNullException(nameof(entry));
if (entry.Sequence <= 0) throw new ArgumentOutOfRangeException(nameof(entry), "Log entry sequence must be positive.");
lock (_gate)
{
var entries = ReadAll(shardId)
.Where(existing => existing.Sequence != entry.Sequence)
.Append(entry)
.OrderBy(existing => existing.Sequence)
.ToArray();
var payload = MessagePackSerializer.Serialize(entries, CultNetSchemaMessageSerialization.Options);
File.WriteAllBytes(GetShardPath(shardId), payload);
}
}
private CultNetShardLogEntryMessage[] ReadAll(string shardId)
{
var path = GetShardPath(shardId);
if (!File.Exists(path))
{
return Array.Empty<CultNetShardLogEntryMessage>();
}
var payload = File.ReadAllBytes(path);
return MessagePackSerializer.Deserialize<CultNetShardLogEntryMessage[]>(
payload,
CultNetSchemaMessageSerialization.Options) ?? Array.Empty<CultNetShardLogEntryMessage>();
}
private string GetShardPath(string shardId)
{
return Path.Combine(_rootPath, HashShardId(shardId) + ".mpack");
}
private static string HashShardId(string shardId)
{
using var sha = SHA256.Create();
var hash = sha.ComputeHash(System.Text.Encoding.UTF8.GetBytes(shardId));
return string.Concat(hash.Select(value => value.ToString("x2")));
}
}
}

5
src/GameCult.Networking/README.md
View file

@ -156,7 +156,10 @@ primary endpoints, using `CultNetSchemaShardLogFetcher` for the schema-v0
transport path. Replica cursors can be stored through
`ICultNetShardReplicaCursorStore`; `CultNetFileShardReplicaCursorStore` keeps
those cursors in one local MessagePack file so replicas can resume after a
restart.
restart. Authoritative shard logs can be stored through
`ICultNetShardMutationLogStore`; `CultNetFileShardMutationLogStore` keeps one
MessagePack log file per shard and lets a restarted primary continue answering
`cultnet.shard_log_request.v0` from durable wire entries.
For client-side prediction, `CultNetClientAuthorityScope` declares which input
documents a runtime may author optimistically. `PutPredictedAsync` updates the