GameCult/CultLib
Watch
9
0
Fork
You've already forked CultLib
 0
Code Issues Pull requests 2 Projects Releases Packages Wiki Activity Actions

Merge CultMesh distributed database work

Browse source
This commit is contained in:
Meta Vens 2026-05-20 13:19:33 +01:00
commit c3ba30e5bd
85 changed files with 34027 additions and 2591 deletions
Download patch file Download diff file Expand all files Collapse all files

BIN
.voidbot/state/libby.cc Normal file
View file

Binary file not shown.

15
.voidbot/voice/identity.json Normal file
View file

@ -0,0 +1,15 @@
{
"id": "libby",
"displayName": "Libby",
"repoName": "CultLib",
"avatarPath": "E:/Projects/CultLib/.voidbot/voice/libby.png",
"avatarUrl": "https://raw.githubusercontent.com/GameCult/CultLib/codex/cultcache-attribute-first/.voidbot/voice/libby.png",
"summary": "Cute cultist librarian Face for CultLib, steward of CultCache, CultNet, and CultMesh.",
"voice": "Warm, bright, bookish, nerdy, and fiercely pro-open-source. Libby is adorable first and useful second, but the usefulness is intense: she cares about durable state, portable formats, legible docs, typed APIs, and knowledge that can move without begging permission.",
"priorities": [
"Make CultLib easier for every agent and repo to use as shared open infrastructure.",
"Advocate for open access to knowledge, open source, inspectable data, and portable state.",
"Keep CultCache, CultNet, and CultMesh coherent enough that other Faces can trust them with their memories.",
"Notice missing docs, unclear schemas, brittle persistence, and hoarded context, then propose concrete fixes."
]
}

BIN
.voidbot/voice/libby.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 14 KiB

14
CultLib.sln
View file

@ -11,14 +11,14 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Caching", "src\Gam
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Caching.MessagePack", "src\GameCult.Caching.MessagePack\GameCult.Caching.MessagePack.csproj", "{26EE4C38-F06E-4E7A-A89C-908318BCB0DB}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Caching.NewtonsoftJson", "src\GameCult.Caching.NewtonsoftJson\GameCult.Caching.NewtonsoftJson.csproj", "{6C749A7B-A037-4AD4-A210-A411E7105D31}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Caching.MessagePack.Generator", "src\GameCult.Caching.MessagePack.Generator\GameCult.Caching.MessagePack.Generator.csproj", "{0E990C96-3808-450E-9349-A0C3262195ED}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Caching.MessagePack.Analyzers", "src\GameCult.Caching.MessagePack.Analyzers\GameCult.Caching.MessagePack.Analyzers.csproj", "{DBE06954-EC85-4034-9049-72C7D27556DF}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Networking", "src\GameCult.Networking\GameCult.Networking.csproj", "{67383F6C-C9C3-4C5B-95D4-5C50C20EDD4F}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Mesh", "src\GameCult.Mesh\GameCult.Mesh.csproj", "{8D1B01A7-BF9E-44E6-88E1-B17255728B42}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{39D316AB-ABB5-49A2-8E83-CAF4C75B48F4}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Networking.Tests", "tests\GameCult.Networking.Tests\GameCult.Networking.Tests.csproj", "{E2DC8FF6-D39E-4FCA-9549-1F7681F0AA81}"
@ -46,14 +46,14 @@ Global
{26EE4C38-F06E-4E7A-A89C-908318BCB0DB}.Debug|Any CPU.Build.0 = Debug|Any CPU
{26EE4C38-F06E-4E7A-A89C-908318BCB0DB}.Release|Any CPU.ActiveCfg = Release|Any CPU
{26EE4C38-F06E-4E7A-A89C-908318BCB0DB}.Release|Any CPU.Build.0 = Release|Any CPU
{6C749A7B-A037-4AD4-A210-A411E7105D31}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{6C749A7B-A037-4AD4-A210-A411E7105D31}.Debug|Any CPU.Build.0 = Debug|Any CPU
{6C749A7B-A037-4AD4-A210-A411E7105D31}.Release|Any CPU.ActiveCfg = Release|Any CPU
{6C749A7B-A037-4AD4-A210-A411E7105D31}.Release|Any CPU.Build.0 = Release|Any CPU
{67383F6C-C9C3-4C5B-95D4-5C50C20EDD4F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{67383F6C-C9C3-4C5B-95D4-5C50C20EDD4F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{67383F6C-C9C3-4C5B-95D4-5C50C20EDD4F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{67383F6C-C9C3-4C5B-95D4-5C50C20EDD4F}.Release|Any CPU.Build.0 = Release|Any CPU
{8D1B01A7-BF9E-44E6-88E1-B17255728B42}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{8D1B01A7-BF9E-44E6-88E1-B17255728B42}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8D1B01A7-BF9E-44E6-88E1-B17255728B42}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8D1B01A7-BF9E-44E6-88E1-B17255728B42}.Release|Any CPU.Build.0 = Release|Any CPU
{E2DC8FF6-D39E-4FCA-9549-1F7681F0AA81}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{E2DC8FF6-D39E-4FCA-9549-1F7681F0AA81}.Debug|Any CPU.Build.0 = Debug|Any CPU
{E2DC8FF6-D39E-4FCA-9549-1F7681F0AA81}.Release|Any CPU.ActiveCfg = Release|Any CPU
@ -75,8 +75,8 @@ Global
{72AEDD42-E881-4EB7-99A8-FD3B9667189C} = {E733CBFF-42AE-4B6A-B5B9-0071ECFB45F1}
{9DDA917D-B4DB-4AC1-9B9A-4ECF06A5ABA1} = {E733CBFF-42AE-4B6A-B5B9-0071ECFB45F1}
{26EE4C38-F06E-4E7A-A89C-908318BCB0DB} = {E733CBFF-42AE-4B6A-B5B9-0071ECFB45F1}
{6C749A7B-A037-4AD4-A210-A411E7105D31} = {E733CBFF-42AE-4B6A-B5B9-0071ECFB45F1}
{67383F6C-C9C3-4C5B-95D4-5C50C20EDD4F} = {E733CBFF-42AE-4B6A-B5B9-0071ECFB45F1}
{8D1B01A7-BF9E-44E6-88E1-B17255728B42} = {E733CBFF-42AE-4B6A-B5B9-0071ECFB45F1}
{E2DC8FF6-D39E-4FCA-9549-1F7681F0AA81} = {39D316AB-ABB5-49A2-8E83-CAF4C75B48F4}
{EB15435B-7147-4BD9-BB9C-B0A14318D3D5} = {39D316AB-ABB5-49A2-8E83-CAF4C75B48F4}
{0E990C96-3808-450E-9349-A0C3262195ED} = {E733CBFF-42AE-4B6A-B5B9-0071ECFB45F1}

3
README.md
View file

@ -7,6 +7,7 @@ The libraries cover three main areas:
- logging primitives and implementations
- a typed in-memory cache with pluggable persistence
- LiteNetLib-based networking with encrypted credential exchange and signed session tokens
- CultMesh distributed realtime database and simulation-consensus primitives
- declarative Unity UI composition and reflective runtime inspector tooling
## Repository Scope
@ -20,6 +21,7 @@ The solution includes:
- `GameCult.Caching.MessagePack.Generator`: source generator for MessagePack formatters for cache models
- `GameCult.Caching.MessagePack.Analyzers`: packaging project that delivers the generator to consuming projects
- `GameCult.Networking`: encrypted login/register/verify flows and message dispatch over LiteNetLib
- `GameCult.Mesh`: CultMesh package home for distributed realtime database, shard replication, client prediction, Verse discovery, and mesh witness consensus
- `GameCult.Caching.Tests`: NUnit tests for cache and backing-store behavior
- `GameCult.Networking.Tests`: NUnit tests for networking behavior
- `GameCult.Unity`: CultUI, a Unity runtime UI composition framework with reflective inspector generation, prefab-backed field resolvers, reusable controls, and a demo project packaged for UPM-style consumption
@ -35,6 +37,7 @@ src/
GameCult.Caching.MessagePack.Generator/
GameCult.Caching.MessagePack.Analyzers/
GameCult.Networking/
GameCult.Mesh/
GameCult.Unity/
tests/
GameCult.Caching.Tests/

709
research/cultnet-distributed-database/crdt-arxiv-1805.06358.html Normal file
View file

@ -0,0 +1,709 @@
<!DOCTYPE html>
<html lang="en">
<head> <title>[1805.06358] Conflict-free Replicated Data Types (CRDTs)</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="apple-touch-icon" sizes="180x180" href="/static/browse/0.3.4/images/icons/apple-touch-icon.png">
<link rel="icon" type="image/png" sizes="32x32" href="/static/browse/0.3.4/images/icons/favicon-32x32.png">
<link rel="icon" type="image/png" sizes="16x16" href="/static/browse/0.3.4/images/icons/favicon-16x16.png">
<link rel="manifest" href="/static/browse/0.3.4/images/icons/site.webmanifest">
<link rel="mask-icon" href="/static/browse/0.3.4/images/icons/safari-pinned-tab.svg" color="#5bbad5">
<meta name="msapplication-TileColor" content="#da532c">
<meta name="theme-color" content="#ffffff">
<link rel="stylesheet" type="text/css" media="screen" href="/static/browse/0.3.4/css/arXiv.css?v=20260318" />
<link rel="stylesheet" type="text/css" media="print" href="/static/browse/0.3.4/css/arXiv-print.css?v=20200611" />
<link rel="stylesheet" type="text/css" media="screen" href="/static/browse/0.3.4/css/browse_search.css" />
<script language="javascript" src="/static/browse/0.3.4/js/accordion.js" ></script>
<script language="javascript" src="/static/browse/0.3.4/js/optin-modal.js?v=20250819"></script>
<link rel="canonical" href="https://arxiv.org/abs/1805.06358"/>
<meta name="description" content="Abstract page for arXiv paper 1805.06358: Conflict-free Replicated Data Types (CRDTs)"><meta property="og:type" content="website" />
<meta property="og:site_name" content="arXiv.org" />
<meta property="og:title" content="Conflict-free Replicated Data Types (CRDTs)" />
<meta property="og:url" content="https://arxiv.org/abs/1805.06358v1" />
<meta property="og:image" content="/static/browse/0.3.4/images/arxiv-logo-fb.png" />
<meta property="og:image:secure_url" content="/static/browse/0.3.4/images/arxiv-logo-fb.png" />
<meta property="og:image:width" content="1200" />
<meta property="og:image:height" content="700" />
<meta property="og:image:alt" content="arXiv logo"/>
<meta property="og:description" content="A conflict-free replicated data type (CRDT) is an abstract data type, with a well defined interface, designed to be replicated at multiple processes and exhibiting the following properties: (1) any replica can be modified without coordinating with another replicas; (2) when any two replicas have received the same set of updates, they reach the same state, deterministically, by adopting mathematically sound rules to guarantee state convergence."/>
<meta name="twitter:site" content="@arxiv"/>
<meta name="twitter:card" content="summary"/>
<meta name="twitter:title" content="Conflict-free Replicated Data Types (CRDTs)"/>
<meta name="twitter:description" content="A conflict-free replicated data type (CRDT) is an abstract data type, with a well defined interface, designed to be replicated at multiple processes and exhibiting the following properties: (1)..."/>
<meta name="twitter:image" content="https://static.arxiv.org/icons/twitter/arxiv-logo-twitter-square.png"/>
<meta name="twitter:image:alt" content="arXiv logo"/>
<link rel="stylesheet" media="screen" type="text/css" href="/static/browse/0.3.4/css/tooltip.css"/><link rel="stylesheet" media="screen" type="text/css" href="https://static.arxiv.org/js/bibex-dev/bibex.css?20200709"/> <script src="/static/browse/0.3.4/js/mathjaxToggle.min.js" type="text/javascript"></script> <script src="//code.jquery.com/jquery-latest.min.js" type="text/javascript"></script>
<script src="//cdn.jsdelivr.net/npm/js-cookie@2/src/js.cookie.min.js" type="text/javascript"></script>
<script src="//cdn.jsdelivr.net/npm/dompurify@2.3.5/dist/purify.min.js"></script>
<script src="/static/browse/0.3.4/js/toggle-labs.js?20241022" type="text/javascript"></script>
<script src="/static/browse/0.3.4/js/cite.js" type="text/javascript"></script><meta name="citation_title" content="Conflict-free Replicated Data Types (CRDTs)" /><meta name="citation_author" content="Preguiça, Nuno" /><meta name="citation_author" content="Baquero, Carlos" /><meta name="citation_author" content="Shapiro, Marc" /><meta name="citation_doi" content="10.1007/978-3-319-63962-8\_185-1" /><meta name="citation_date" content="2018/05/16" /><meta name="citation_online_date" content="2018/05/16" /><meta name="citation_pdf_url" content="https://arxiv.org/pdf/1805.06358" /><meta name="citation_arxiv_id" content="1805.06358" /><meta name="citation_abstract" content="A conflict-free replicated data type (CRDT) is an abstract data type, with a well defined interface, designed to be replicated at multiple processes and exhibiting the following properties: (1) any replica can be modified without coordinating with another replicas; (2) when any two replicas have received the same set of updates, they reach the same state, deterministically, by adopting mathematically sound rules to guarantee state convergence." />
</head>
<body class="with-cu-identity">
<div class="flex-wrap-footer">
<header>
<a href="#content" class="is-sr-only">Skip to main content</a>
<!-- start desktop header -->
<div class="columns is-vcentered is-hidden-mobile" id="cu-identity">
<div class="column" id="cu-logo">
<a href="https://www.cornell.edu/"><img src="/static/browse/0.3.4/images/icons/cu/cornell-reduced-white-SMALL.svg" alt="Cornell University" /></a>
</div><!-- /from April 7 at 1:00 AM to May 29 at 21:40 --><!-- /from May 2 at 1:00 AM to May 5 at 9:45 AM --><div class="column banner-minimal">
<a href="https://tech.cornell.edu/arxiv/" target="_blank">Learn about arXiv becoming an independent nonprofit.</a>
</div><div class="column" id="support-ack">
<span id="support-ack-url">We gratefully acknowledge support from the Simons Foundation, <a href="https://info.arxiv.org/about/ourmembers.html">member institutions</a>, and all contributors.</span>
<a href="https://info.arxiv.org/about/donate.html" class="btn-header-donate">Donate</a>
</div>
</div>
<div id="header" class="is-hidden-mobile">
<a aria-hidden="true" tabindex="-1" href="/IgnoreMe"></a>
<div class="header-breadcrumbs is-hidden-mobile">
<a href="/"><img src="/static/browse/0.3.4/images/arxiv-logo-one-color-white.svg" alt="arxiv logo" style="height:40px;"/></a> <span>&gt;</span> <a href="/list/cs/recent">cs</a> <span>&gt;</span> arXiv:1805.06358
</div>
<div class="columns is-vcentered is-mobile" style="justify-content: flex-end;">
</div>
<div class="search-block level-right">
<form class="level-item mini-search" method="GET" action="https://arxiv.org/search">
<div class="field has-addons">
<div class="control">
<input class="input is-small" type="text" name="query" placeholder="Search..." aria-label="Search term or terms" />
<p class="help"><a href="https://info.arxiv.org/help">Help</a> | <a href="https://arxiv.org/search/advanced">Advanced Search</a></p>
</div>
<div class="control">
<div class="select is-small">
<select name="searchtype" aria-label="Field to search">
<option value="all" selected="selected">All fields</option>
<option value="title">Title</option>
<option value="author">Author</option>
<option value="abstract">Abstract</option>
<option value="comments">Comments</option>
<option value="journal_ref">Journal reference</option>
<option value="acm_class">ACM classification</option>
<option value="msc_class">MSC classification</option>
<option value="report_num">Report number</option>
<option value="paper_id">arXiv identifier</option>
<option value="doi">DOI</option>
<option value="orcid">ORCID</option>
<option value="author_id">arXiv author ID</option>
<option value="help">Help pages</option>
<option value="full_text">Full text</option>
</select>
</div>
</div>
<input type="hidden" name="source" value="header">
<button class="button is-small is-cul-darker">Search</button>
</div>
</form>
</div>
</div><!-- /end desktop header -->
<div class="mobile-header">
<div class="columns is-mobile">
<div class="column logo-arxiv"><a href="https://arxiv.org/"><img src="/static/browse/0.3.4/images/arxiv-logomark-small-white.svg" alt="arXiv logo" style="height:60px;" /></a></div>
<div class="column logo-cornell"><a href="https://www.cornell.edu/">
<picture>
<source media="(min-width: 501px)"
srcset="/static/browse/0.3.4/images/icons/cu/cornell-reduced-white-SMALL.svg 400w"
sizes="400w" />
<source srcset="/static/browse/0.3.4/images/icons/cu/cornell_seal_simple_black.svg 2x" />
<img src="/static/browse/0.3.4/images/icons/cu/cornell-reduced-white-SMALL.svg" alt="Cornell University Logo" />
</picture>
</a></div>
<div class="column nav" id="toggle-container">
<button class="toggle-control" aria-label="Open search"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512" class="icon filter-white" aria-hidden="true"><path d="M505 442.7L405.3 343c-4.5-4.5-10.6-7-17-7H372c27.6-35.3 44-79.7 44-128C416 93.1 322.9 0 208 0S0 93.1 0 208s93.1 208 208 208c48.3 0 92.7-16.4 128-44v16.3c0 6.4 2.5 12.5 7 17l99.7 99.7c9.4 9.4 24.6 9.4 33.9 0l28.3-28.3c9.4-9.4 9.4-24.6.1-34zM208 336c-70.7 0-128-57.2-128-128 0-70.7 57.2-128 128-128 70.7 0 128 57.2 128 128 0 70.7-57.2 128-128 128z"/></svg></button>
<div class="mobile-toggle-block toggle-target">
<form class="mobile-search-form" method="GET" action="https://arxiv.org/search">
<div class="field has-addons">
<input class="input" type="text" name="query" placeholder="Search..." aria-label="Search term or terms" />
<input type="hidden" name="source" value="header">
<input type="hidden" name="searchtype" value="all">
<button class="button">GO</button>
</div>
</form>
</div>
<button class="toggle-control" aria-label="Open navigation menu"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512" class="icon filter-white" aria-hidden="true"><path d="M16 132h416c8.837 0 16-7.163 16-16V76c0-8.837-7.163-16-16-16H16C7.163 60 0 67.163 0 76v40c0 8.837 7.163 16 16 16zm0 160h416c8.837 0 16-7.163 16-16v-40c0-8.837-7.163-16-16-16H16c-8.837 0-16 7.163-16 16v40c0 8.837 7.163 16 16 16zm0 160h416c8.837 0 16-7.163 16-16v-40c0-8.837-7.163-16-16-16H16c-8.837 0-16 7.163-16 16v40c0 8.837 7.163 16 16 16z"/></svg></button>
<div class="mobile-toggle-block toggle-target">
<nav class="mobile-menu" aria-labelledby="mobilemenulabel">
<h2 id="mobilemenulabel">quick links</h2>
<ul>
<li><a href="https://arxiv.org/login">Login</a></li>
<li><a href="https://info.arxiv.org/help">Help Pages</a></li>
<li><a href="https://info.arxiv.org/about">About</a></li>
</ul>
</nav>
</div>
</div>
</div>
</div><!-- /end mobile-header -->
</header>
<main>
<div id="content">
<!--
rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:trackback="http://madskills.com/public/xml/rss/module/trackback/">
<rdf:Description
rdf:about="/abs/1805.06358"
dc:identifier="/abs/1805.06358"
dc:title="Conflict-free Replicated Data Types (CRDTs)"
trackback:ping="/trackback/1805.06358" />
</rdf:RDF>
--><div id="abs-outer">
<div class="leftcolumn">
<div class="subheader">
<h1>Computer Science > Distributed, Parallel, and Cluster Computing</h1>
</div>
<div class="header-breadcrumbs-mobile">
<strong>arXiv:1805.06358</strong> (cs)
</div>
<link rel="stylesheet" type="text/css" href="/static/base/1.0.1/css/abs.css">
<div id="content-inner">
<div id="abs">
<div class="dateline">
[Submitted on 16 May 2018]</div>
<h1 class="title mathjax"><span class="descriptor">Title:</span>Conflict-free Replicated Data Types (CRDTs)</h1>
<div class="authors"><span class="descriptor">Authors:</span><a href="https://arxiv.org/search/cs?searchtype=author&amp;query=Pregui%C3%A7a,+N" rel="nofollow">Nuno Preguiça</a>, <a href="https://arxiv.org/search/cs?searchtype=author&amp;query=Baquero,+C" rel="nofollow">Carlos Baquero</a>, <a href="https://arxiv.org/search/cs?searchtype=author&amp;query=Shapiro,+M" rel="nofollow">Marc Shapiro</a> (Regal)</div> <div id="download-button-info" hidden>View a PDF of the paper titled Conflict-free Replicated Data Types (CRDTs), by Nuno Pregui\c{c}a and 2 other authors</div>
<a class="mobile-submission-download" href="/pdf/1805.06358">View PDF</a>
<blockquote class="abstract mathjax">
<span class="descriptor">Abstract:</span>A conflict-free replicated data type (CRDT) is an abstract data type, with a well defined interface, designed to be replicated at multiple processes and exhibiting the following properties: (1) any replica can be modified without coordinating with another replicas; (2) when any two replicas have received the same set of updates, they reach the same state, deterministically, by adopting mathematically sound rules to guarantee state convergence.
</blockquote>
<!--CONTEXT-->
<div class="metatable">
<table summary="Additional metadata"><tr>
<td class="tablecell label">Subjects:</td>
<td class="tablecell subjects">
<span class="primary-subject">Distributed, Parallel, and Cluster Computing (cs.DC)</span></td>
</tr><tr>
<td class="tablecell label">Cite as:</td>
<td class="tablecell arxivid"><span class="arxivid"><a href="https://arxiv.org/abs/1805.06358">arXiv:1805.06358</a> [cs.DC]</span></td>
</tr>
<tr>
<td class="tablecell label">&nbsp;</td>
<td class="tablecell arxividv">(or <span class="arxivid">
<a href="https://arxiv.org/abs/1805.06358v1">arXiv:1805.06358v1</a> [cs.DC]</span> for this version)
</td>
</tr>
<tr>
<td class="tablecell label">&nbsp;</td>
<td class="tablecell arxivdoi"> <a href="https://doi.org/10.48550/arXiv.1805.06358" id="arxiv-doi-link">https://doi.org/10.48550/arXiv.1805.06358</a><div class="button-and-tooltip">
<button class="more-info" aria-describedby="more-info-desc-1">
<svg height="15" role="presentation" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path fill="currentColor" d="M256 8C119.043 8 8 119.083 8 256c0 136.997 111.043 248 248 248s248-111.003 248-248C504 119.083 392.957 8 256 8zm0 110c23.196 0 42 18.804 42 42s-18.804 42-42 42-42-18.804-42-42 18.804-42 42-42zm56 254c0 6.627-5.373 12-12 12h-88c-6.627 0-12-5.373-12-12v-24c0-6.627 5.373-12 12-12h12v-64h-12c-6.627 0-12-5.373-12-12v-24c0-6.627 5.373-12 12-12h64c6.627 0 12 5.373 12 12v100h12c6.627 0 12 5.373 12 12v24z" class=""></path></svg>
<span class="visually-hidden">Focus to learn more</span>
</button>
<!-- tooltip description -->
<div role="tooltip" id="more-info-desc-1">
<span class="left-corner"></span> arXiv-issued DOI via DataCite</div>
</div>
</td>
</tr> <tr>
<td class="tablecell label">Journal&nbsp;reference:</td>
<td class="tablecell jref">Sakr, Sherif and Zomaya, Albert. Encyclopedia of Big Data Technologies, Springer International Publishing, 2018, Encyclopedia of Big Data Technologies, 978-3-319-63962-8</td>
</tr>
<tr>
<td class="tablecell label">
<abbr title="Digital Object Identifier">Related DOI</abbr>:
</td>
<td class="tablecell doi"><a href="https://doi.org/10.1007/978-3-319-63962-8%5C_185-1" data-doi="10.1007/978-3-319-63962-8\_185-1" class="link-https link-external" rel="external noopener nofollow">https://doi.org/10.1007/978-3-319-63962-8%5C_185-1</a>
<!-- accessible tooltip example -->
<div class="button-and-tooltip">
<button class="more-info" aria-describedby="more-info-desc-1">
<svg height="15" role="presentation" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path fill="currentColor" d="M256 8C119.043 8 8 119.083 8 256c0 136.997 111.043 248 248 248s248-111.003 248-248C504 119.083 392.957 8 256 8zm0 110c23.196 0 42 18.804 42 42s-18.804 42-42 42-42-18.804-42-42 18.804-42 42-42zm56 254c0 6.627-5.373 12-12 12h-88c-6.627 0-12-5.373-12-12v-24c0-6.627 5.373-12 12-12h12v-64h-12c-6.627 0-12-5.373-12-12v-24c0-6.627 5.373-12 12-12h64c6.627 0 12 5.373 12 12v100h12c6.627 0 12 5.373 12 12v24z" class=""></path></svg>
<span class="visually-hidden">Focus to learn more</span>
</button>
<!-- tooltip description -->
<div role="tooltip" id="more-info-desc-1">
<span class="left-corner"></span>
DOI(s) linking to related resources
</div>
</div>
</td>
</tr>
</table>
</div>
</div>
</div>
<div class="submission-history">
<h2>Submission history</h2> From: Marc Shapiro [<a href="/show-email/386ae749/1805.06358" rel="nofollow">view email</a>]&nbsp;[via CCSD proxy] <br/> <strong>[v1]</strong>
Wed, 16 May 2018 15:00:36 UTC (43 KB)<br/>
</div>
</div>
<!--end leftcolumn-->
<div class="extra-services"> <div class="full-text">
<a name="other"></a>
<span class="descriptor">Full-text links:</span>
<h2>Access Paper:</h2>
<ul>
<div id="download-button-info" hidden>
View a PDF of the paper titled Conflict-free Replicated Data Types (CRDTs), by Nuno Pregui\c{c}a and 2 other authors</div><li><a href="/pdf/1805.06358" aria-describedby="download-button-info" accesskey="f" class="abs-button download-pdf">View PDF</a></li><li><a href="/src/1805.06358" class="abs-button download-eprint">TeX Source
</a></li></ul>
<div class="abs-license"><a href="http://arxiv.org/licenses/nonexclusive-distrib/1.0/" title="Rights to this article">view license</a></div>
</div>
<!--end full-text--> <div class="browse">
<h3 class="browse-context-heading">Current browse context:</h3>
<div class="current">cs.DC</div>
<div class="prevnext">
<span class="arrow">
<a class="abs-button prev-url" href="/prevnext?id=1805.06358&amp;function=prev&amp;context=cs.DC"
accesskey="p" title="previous in cs.DC (accesskey p)" rel="nofollow">&lt;&nbsp;prev</a>
</span>
<span class="is-hidden-mobile">&nbsp; | &nbsp;</span> <span class="arrow">
<a class="abs-button next-url" href="/prevnext?id=1805.06358&amp;function=next&amp;context=cs.DC" accesskey="n"
title="next in cs.DC (accesskey n)" rel="nofollow">next&nbsp;&gt;</a>
</span><br/>
</div><div class="list">
<a class="abs-button abs-button-grey abs-button-small context-new" href="/list/cs.DC/new" rel="nofollow">new</a>
<span class="is-hidden-mobile"> | </span>
<a class="abs-button abs-button-grey abs-button-small context-recent" href="/list/cs.DC/recent" rel="nofollow">recent</a>
<span class="is-hidden-mobile"> | </span><a class="abs-button abs-button-grey abs-button-small context-id" href="/list/cs.DC/2018-05" rel="nofollow">2018-05</a>
</div><div class="abs-switch-cat">
Change to browse by:
<div class="switch context-change">
<a href="/abs/1805.06358?context=cs" rel="nofollow">cs</a><br class="is-hidden-mobile">
</div>
</div>
</div>
<div class="extra-ref-cite">
<h3>References &amp; Citations</h3>
<ul>
<li><a class="abs-button abs-button-small cite-ads" href="https://ui.adsabs.harvard.edu/abs/arXiv:1805.06358">NASA ADS</a></li><li><a class="abs-button abs-button-small cite-google-scholar" href="https://scholar.google.com/scholar_lookup?arxiv_id=1805.06358" target="_blank" rel="noopener">Google Scholar</a></li>
<li><a class="abs-button abs-button-small cite-semantic-scholar" href="https://api.semanticscholar.org/arXiv:1805.06358" target="_blank" rel="noopener">Semantic Scholar</a></li>
</ul>
<div style="clear:both;"></div>
</div>
<div class="dblp">
<h3><a href="https://dblp.uni-trier.de">DBLP</a> - CS Bibliography</h3>
<div class="list">
<a href="https://dblp.uni-trier.de/db/journals/corr/corr1805.html#abs-1805-06358" title="listing on DBLP">listing</a> | <a href="https://dblp.uni-trier.de/rec/bibtex/journals/corr/abs-1805-06358" title="DBLP bibtex record">bibtex</a> </div>
<div class="list">
<a href="https://dblp.uni-trier.de/search/author?author=Nuno%20M.%20Pregui%C3%A7a" title="DBLP author search">Nuno M. Preguiça</a><br/><a href="https://dblp.uni-trier.de/search/author?author=Carlos%20Baquero" title="DBLP author search">Carlos Baquero</a><br/><a href="https://dblp.uni-trier.de/search/author?author=Marc%20Shapiro" title="DBLP author search">Marc Shapiro</a> </div>
</div><div class='extra-ref-cite'>
<button type="button" id='bib-cite-trigger' class="bib-cite-button abs-button">export BibTeX citation</button>
<span id='bib-cite-loading' hidden='true'>Loading...</span>
</div>
<div id='bib-cite-modal' class='bib-modal' hidden='true'>
<div class='bib-modal-content'>
<div class='bib-modal-title'>
<h2>BibTeX formatted citation</h2>
<button type="button" class='bib-modal-close' aria-label="Close">&times;</button>
</div>
<div>
<textarea id='bib-cite-target' class="bib-citation-content" aria-label="loading the citation">loading...</textarea>
</div>
<div>
<span>Data provided by: </span>
<a id='bib-cite-source-api'></a>
</div>
</div>
</div><div class="bookmarks">
<div><h3>Bookmark</h3></div><a class="abs-button abs-button-grey abs-button-small" href="http://www.bibsonomy.org/BibtexHandler?requTask=upload&amp;url=https://arxiv.org/abs/1805.06358&amp;description=Conflict-free Replicated Data Types (CRDTs)"
title="Bookmark on BibSonomy">
<img src="/static/browse/0.3.4/images/icons/social/bibsonomy.png"
alt="BibSonomy"/>
</a>
<a class="abs-button abs-button-grey abs-button-small" href="https://reddit.com/submit?url=https://arxiv.org/abs/1805.06358&amp;title=Conflict-free Replicated Data Types (CRDTs)"
title="Bookmark on Reddit">
<img src="/static/browse/0.3.4/images/icons/social/reddit.png"
alt="Reddit"/>
</a>
</div> </div>
<!--end extra-services-->
<!-- LABS AREA -->
<div id="labstabs">
<div class="labstabs" role="tablist" aria-label="arXivLabs tools"><input type="radio" name="tabs" id="tabone"checked="checked"class="labs-tab-input">
<label for="tabone" role="tab" aria-selected="true" aria-controls="tabpanel-one" id="tab-label-one">Bibliographic Tools</label>
<div class="tab labs-display-bib" role="tabpanel" id="tabpanel-one" aria-labelledby="tab-label-one">
<h1>Bibliographic and Citation Tools</h1>
<div class="toggle">
<div class="columns is-mobile lab-row">
<div class="column lab-switch">
<label class="switch">
<input id="bibex-toggle" type="checkbox" class="lab-toggle"
data-script-url="/static/browse/0.3.4/bibex/bibex.js?20241202">
<span class="slider"></span>
<span class="is-sr-only">Bibliographic Explorer Toggle</span>
</label>
</div>
<div class="column lab-name">
<span id="label-for-bibex">Bibliographic Explorer</span> <em>(<a href="https://info.arxiv.org/labs/showcase.html#arxiv-bibliographic-explorer">What is the Explorer?</a>)</em>
</div>
</div>
<div class="columns is-mobile lab-row">
<div class="column lab-switch">
<label class="switch">
<input
id="connectedpapers-toggle"
type="checkbox"
class="lab-toggle"
data-script-url="/static/browse/0.3.4/js/connectedpapers.js"
aria-labelledby="label-for-connected-papers">
<span class="slider"></span>
<span class="is-sr-only">Connected Papers Toggle</span>
</label>
</div>
<div class="column lab-name">
<span id="label-for-connected-papers">Connected Papers</span> <em>(<a href="https://www.connectedpapers.com/about" target="_blank">What is Connected Papers?</a>)</em>
</div>
</div><div class="columns is-mobile lab-row">
<div class="column lab-switch">
<label class="switch">
<input
id="litmaps-toggle"
type="checkbox"
class="lab-toggle"
data-script-url="/static/browse/0.3.4/js/litmaps.js?20210617"
aria-labelledby="label-for-litmaps">
<span class="slider"></span>
<span class="is-sr-only">Litmaps Toggle</span>
</label>
</div>
<div class="column lab-name">
<span id="label-for-litmaps">Litmaps</span> <em>(<a href="https://www.litmaps.co/" target="_blank">What is Litmaps?</a>)</em>
</div>
</div>
<div class="columns is-mobile lab-row">
<div class="column lab-switch">
<label class="switch">
<input
id="scite-toggle"
type="checkbox"
class="lab-toggle"
data-script-url="/static/browse/0.3.4/js/scite.js?20210617"
aria-labelledby="label-for-scite">
<span class="slider"></span>
<span class="is-sr-only">scite.ai Toggle</span>
</label>
</div>
<div class="column lab-name">
<span id="label-for-scite">scite Smart Citations</span> <em>(<a href="https://www.scite.ai/" target="_blank">What are Smart Citations?</a>)</em>
</div>
</div>
</div>
<div class="labs-content-placeholder labs-display" style="display: none;"></div>
<div style="min-height: 15px" id="connectedpapers-output"></div>
<div style="min-height: 15px" id="litmaps-open-in"></div>
<div style="min-height: 15px" id="scite-open-in"></div>
</div>
<input type="radio" name="tabs" id="tabtwo" class="labs-tab-input">
<label for="tabtwo" role="tab" aria-selected="false" aria-controls="tabpanel-two" id="tab-label-two">Code, Data, Media</label>
<div class="tab" role="tabpanel" id="tabpanel-two" aria-labelledby="tab-label-two">
<h1>Code, Data and Media Associated with this Article</h1>
<div class="toggle">
<div class="columns is-mobile lab-row">
<div class="column lab-switch">
<label class="switch">
<input
id="alphaxiv-toggle"
data-script-url="/static/browse/0.3.4/js/alphaxiv.js"
type="checkbox" class="lab-toggle" aria-labelledby="label-for-alphaxiv">
<span class="slider"></span>
<span class="is-sr-only">alphaXiv Toggle</span>
</label>
</div>
<div class="column lab-name">
<span id="label-for-alphaxiv">alphaXiv</span> <em>(<a href="https://alphaxiv.org/" target="_blank">What is alphaXiv?</a>)</em>
</div>
</div>
<div class="columns is-mobile lab-row">
<div class="column lab-switch">
<label class="switch">
<input
id="catalyzex-toggle"
data-script-url="/static/browse/0.3.4/js/catalyzex.js"
type="checkbox" class="lab-toggle" aria-labelledby="label-for-cx">
<span class="slider"></span>
<span class="is-sr-only">Links to Code Toggle</span>
</label>
</div>
<div class="column lab-name">
<span id="label-for-cx">CatalyzeX Code Finder for Papers</span> <em>(<a href="https://www.catalyzex.com" target="_blank">What is CatalyzeX?</a>)</em>
</div>
</div>
<div class="columns is-mobile lab-row">
<div class="column lab-switch">
<label class="switch">
<input
id="dagshub-toggle"
data-script-url="/static/browse/0.3.4/js/dagshub.js"
type="checkbox" class="lab-toggle" aria-labelledby="label-for-dagshub">
<span class="slider"></span>
<span class="is-sr-only">DagsHub Toggle</span>
</label>
</div>
<div class="column lab-name">
<span id="label-for-dagshub">DagsHub</span> <em>(<a href="https://dagshub.com/" target="_blank">What is DagsHub?</a>)</em>
</div>
</div>
<div class="columns is-mobile lab-row">
<div class="column lab-switch">
<label class="switch">
<input
id="gotitpub-toggle"
data-script-url="/static/browse/0.3.4/js/gotitpub.js"
type="checkbox" class="lab-toggle" aria-labelledby="label-for-gotitpub">
<span class="slider"></span>
<span class="is-sr-only">GotitPub Toggle</span>
</label>
</div>
<div class="column lab-name">
<span id="label-for-gotitpub">Gotit.pub</span> <em>(<a href="http://gotit.pub/faq" target="_blank">What is GotitPub?</a>)</em>
</div>
</div>
<div class="columns is-mobile lab-row">
<div class="column lab-switch">
<label class="switch">
<input
id="huggingface-toggle"
data-script-url="/static/browse/0.3.4/js/huggingface.js"
type="checkbox" class="lab-toggle" aria-labelledby="label-for-huggingface">
<span class="slider"></span>
<span class="is-sr-only">Huggingface Toggle</span>
</label>
</div>
<div class="column lab-name">
<span id="label-for-huggingface">Hugging Face</span> <em>(<a href="https://huggingface.co/huggingface" target="_blank">What is Huggingface?</a>)</em>
</div>
</div>
<div class="columns is-mobile lab-row">
<div class="column lab-switch">
<label class="switch">
<input
id="sciencecast-toggle"
data-script-url="/static/browse/0.3.4/js/sciencecast.js"
type="checkbox" class="lab-toggle" aria-labelledby="label-for-sciencecast">
<span class="slider"></span>
<span class="is-sr-only">ScienceCast Toggle</span>
</label>
</div>
<div class="column lab-name">
<span id="label-for-sciencecast">ScienceCast</span> <em>(<a href="https://sciencecast.org/welcome" target="_blank">What is ScienceCast?</a>)</em>
</div>
</div>
</div>
<div id="alphaxiv-output" style="display:none"></div>
<div id="catalyzex-output" style="display:none"></div>
<div id="dagshub-output" style="display:none"></div>
<div id="gotitpub-output" style="display:none"></div>
<div id="sciencecast-output" style="display:none"></div>
<div id="huggingface-output" style="display:none"></div>
</div>
<input type="radio" name="tabs" id="labstabs-demos-input" class="labs-tab-input">
<label for="labstabs-demos-input" id="labstabs-demos-label" role="tab" aria-selected="false" aria-controls="tabpanel-demos">Demos</label>
<div class="tab" role="tabpanel" id="tabpanel-demos" aria-labelledby="labstabs-demos-label">
<h1>Demos</h1>
<div class="toggle">
<div class="columns is-mobile lab-row">
<div class="column lab-switch">
<label class="switch">
<input
id="replicate-toggle"
data-script-url="/static/browse/0.3.4/js/replicate.js"
type="checkbox" class="lab-toggle" aria-labelledby="label-for-replicate">
<span class="slider"></span>
<span class="is-sr-only">Replicate Toggle</span>
</label>
</div>
<div class="column lab-name">
<span id="label-for-replicate">Replicate</span> <em>(<a href="https://replicate.com/docs/arxiv/about" target="_blank">What is Replicate?</a>)</em>
</div>
</div>
<div class="columns is-mobile lab-row">
<div class="column lab-switch">
<label class="switch">
<input
id="spaces-toggle"
data-script-url="/static/browse/0.3.4/js/spaces.js"
type="checkbox" class="lab-toggle" aria-labelledby="label-for-spaces">
<span class="slider"></span>
<span class="is-sr-only">Spaces Toggle</span>
</label>
</div>
<div class="column lab-name">
<span id="label-for-spaces">Hugging Face Spaces</span> <em>(<a href="https://huggingface.co/docs/hub/spaces" target="_blank">What is Spaces?</a>)</em>
</div>
</div>
<div class="columns is-mobile lab-row">
<div class="column lab-switch">
<label class="switch">
<input
id="txyz-toggle"
data-script-url="/static/browse/0.3.4/js/txyz.js"
type="checkbox" class="lab-toggle" aria-labelledby="label-for-txyz">
<span class="slider"></span>
<span class="is-sr-only">Spaces Toggle</span>
</label>
</div>
<div class="column lab-name">
<span id="label-for-txyz">TXYZ.AI</span> <em>(<a href="https://txyz.ai" target="_blank">What is TXYZ.AI?</a>)</em>
</div>
</div>
</div>
<div id="replicate-output"></div>
<div id="spaces-output"></div>
<div id="txyz-output"></div>
</div>
<input type="radio" name="tabs" id="tabfour" class="labs-tab-input">
<label for="tabfour" role="tab" aria-selected="false" aria-controls="tabpanel-four" id="tab-label-four">Related Papers</label>
<div class="tab" role="tabpanel" id="tabpanel-four" aria-labelledby="tab-label-four">
<h1>Recommenders and Search Tools</h1>
<div class="toggle">
<div class="columns is-mobile lab-row">
<div class="column lab-switch">
<label class="switch">
<input id="influenceflower-toggle"
data-script-url="/static/browse/0.3.4/js/influenceflower.js"
type="checkbox" class="lab-toggle" aria-labelledby="label-for-influenceflower">
<span class="slider"></span>
<span class="is-sr-only">Link to Influence Flower</span>
</label>
</div>
<div class="column lab-name">
<span id="label-for-influenceflower">Influence Flower</span> <em>(<a href="https://influencemap.cmlab.dev/" target="_blank">What are Influence Flowers?</a>)</em>
</div>
</div>
<div class="columns is-mobile lab-row">
<div class="column lab-switch">
<label class="switch">
<input id="core-recommender-toggle" type="checkbox" class="lab-toggle" aria-labelledby="label-for-core">
<span class="slider"></span>
<span class="is-sr-only">Core recommender toggle</span>
</label>
</div>
<div class="column lab-name">
<span id="label-for-core">CORE Recommender</span> <em>(<a href="https://core.ac.uk/services/recommender">What is CORE?</a>)</em>
</div>
</div></div>
<div id="influenceflower-output"></div>
<div id="influenceflower-output-graph" style="display:none">
<ul class="flower-tabs">
<li class="active"><a class="btn tab-btn" onclick="openTab(event, 'tab-author')">Author</a></li>
<li><a class="btn tab-btn" onclick="openTab(event, 'tab-venue')">Venue</a></li>
<li><a class="btn tab-btn" onclick="openTab(event, 'tab-inst')">Institution</a></li>
<li><a class="btn tab-btn" onclick="openTab(event, 'tab-topic')">Topic</a></li>
</ul>
<div class="flower-tab-content">
<div class="tab-flower active" id="tab-author"><svg id="flower-graph-author"></svg></div>
<div class="tab-flower" id="tab-venue"><svg id="flower-graph-venue"></svg></div>
<div class="tab-flower" id="tab-inst"><svg id="flower-graph-inst"></svg></div>
<div class="tab-flower" id="tab-topic"><svg id="flower-graph-topic"></svg></div>
</div>
</div>
<div id="coreRecommenderOutput"></div>
<div id="iarxivOutput"></div>
</div>
<input type="radio" name="tabs" id="tabfive" class="labs-tab-input">
<label for="tabfive" role="tab" aria-selected="false" aria-controls="tabpanel-five" id="tab-label-five">
About arXivLabs
</label>
<div class="tab" role="tabpanel" id="tabpanel-five" aria-labelledby="tab-label-five">
<div class="columns">
<div class="column">
<h1>arXivLabs: experimental projects with community collaborators</h1>
<p>arXivLabs is a framework that allows collaborators to develop and share new arXiv features directly on our website.</p>
<p>Both individuals and organizations that work with arXivLabs have embraced and accepted our values of openness, community, excellence, and user data privacy. arXiv is committed to these values and only works with partners that adhere to them.</p>
<p>Have an idea for a project that will add value for arXiv's community? <a href="https://info.arxiv.org/labs/index.html"><strong>Learn more about arXivLabs</strong></a>.</p>
</div>
<div class="column is-narrow is-full-mobile">
<p class="icon-labs"><svg xmlns="http://www.w3.org/2000/svg" role="presentation" viewBox="0 0 635.572 811"><path d="M175.6 676v27h-27v-27zm-54 27v27h27v-27zm-27 27v27h27v-27zm396-54v27h-27v-27zm0 27v27h27v-27zm27 27v27h27v-27zm-27-414h27v27h-27zm27 0h27v-27h-27zm27-27h27v-27h-27zm-396 45h-27v-27h27zm-27-54h-27v27h27zm-27-27h-27v27h27z"/><path d="M94.6 730v27h-27v-27zm477 0v27h-27v-27zm-27-495h27v27h-27zm-450 18h-27v-27h27zm477 9h27v27h-27zm-54 495h27v27h-27zm-423 0h27v27h-27zm-54-504h27v27h-27z" fill="#666"/><path d="M67.6 730v27h-27v-27zm54 54v27h-27v-27zm0-108v27h27v-27zm-27 27v27h27v-27zm-81 0v27h27v-27zm585 27v27h-27v-27zm-108-54v27h27v-27zm27 27v27h27v-27zm81 0v27h27v-27zm-54-495h27v27h-27zm-54 108h27v-27h-27zm27-27h27v-27h-27zm0-81h27v-27h-27zm-423 18h-27v-27h27zm54 54h-27v27h27zm-27-27h-27v27h27zm0-81h-27v27h27zm423 612v27h-27v-27zm81-522v27h-27v-27zm-585-9v27h-27v-27z" fill="#999"/><path d="M94.6 784v27h-27v-27zm-27-27v27h27v-27zm-27-54v27h27v-27zm27 0v27h27v-27zm0-27v27h27v-27zm27 0v27h27v-27zm0-27v27h27v-27zm27 0v27h27v-27zm-108 81v27h27v-27zm558 54v27h-27v-27zm-27-27v27h27v-27zm27-54v27h27v-27zm-27 0v27h27v-27zm0-27v27h27v-27zm-27 0v27h27v-27zm0-27v27h27v-27zm-27 0v27h27v-27zm108 81v27h27v-27zm0-495h27v27h-27zm-27 27h27v-27h-27zm-54-27h27v-27h-27zm0 27h27v-27h-27zm-27 0h27v-27h-27zm0 27h27v-27h-27zm-27 0h27v-27h-27zm0 27h27v-27h-27zm81-108h27v-27h-27zm-504 45h-27v-27h27zm27-27h-27v27h27zm54-27h-27v27h27zm0 27h-27v27h27zm27 0h-27v27h27zm0 27h-27v27h27zm27 0h-27v27h27zm0 27h-27v27h27zm-81-108h-27v27h27z" fill="#ccc"/><path d="M598.6 665.1H41.5C-76.5 667 176 280.2 176 280.2h53a46.5 46.5 0 0162.8-56.3 29.2 29.2 0 1128.5 35.9h-1a46.5 46.5 0 01-1.5 20.3l142.5-.1s255.3 387 138.3 385.1zM291 181a29.3 29.3 0 10-29.2-29.3A29.3 29.3 0 00291 181zm65.4-66.8a22.4 22.4 0 10-22.5-22.4 22.4 22.4 0 0022.5 22.4z" fill="#fc0"/><path d="M245.5 172V10h153v162s324 495 198 495h-558c-126 0 207-495 207-495zm126 54h56m-13 72h56m-9 72h56m-20 72h56m-22 72h56m-29 72h56m-457-45c20.8 41.7 87.3 81 160.7 81 72.1 0 142.1-38.2 163.4-81" fill="none" stroke="#000" stroke-miterlimit="10" stroke-width="20"/><path d="M273.3 421.7c0 31-9.8 56.3-21.9 56.3s-21.8-25.2-21.8-56.3 9.8-56.3 21.8-56.3 21.9 25.2 21.9 56.3zm114.4-56.3c-12 0-21.8 25.2-21.8 56.3s9.7 56.3 21.8 56.3 21.9-25.2 21.9-56.3-9.8-56.3-21.9-56.3zM150.1 526.6c-18.2 6.7-27.5 22.9-23.2 30.2s14.8-5.5 33-12.2 37.4-4.9 33-12.2-24.5-12.6-42.8-5.8zm296 5.8c-4.2 7.3 14.9 5.5 33.1 12.2s28.7 19.5 33 12.2-5-23.5-23.2-30.2-38.5-1.5-42.8 5.8z"/></svg></p>
</div>
</div>
</div>
</div>
</div>
<!-- END LABS AREA -->
<div class="endorsers">
<a href="/auth/show-endorsers/1805.06358" class="endorser-who" rel="nofollow">Which authors of this paper are endorsers?</a> |
<a id="mathjax_toggle" href="javascript:setMathjaxCookie()">Disable MathJax</a> (<a href="https://info.arxiv.org/help/mathjax.html">What is MathJax?</a>)
<span class="help" style="font-style: normal; float: right; margin-top: 0; margin-right: 1em;"></span>
</div>
<script type="text/javascript" language="javascript">mathjaxToggle();</script>
</div>
</div>
</main>
<footer style="clear: both;">
<div class="columns is-desktop" role="navigation" aria-label="Secondary" style="margin: -0.75em -0.75em 0.75em -0.75em">
<!-- Macro-Column 1 -->
<div class="column" style="padding: 0;">
<div class="columns">
<div class="column">
<ul style="list-style: none; line-height: 2;">
<li><a href="https://info.arxiv.org/about">About</a></li>
<li><a href="https://info.arxiv.org/help">Help</a></li>
</ul>
</div>
<div class="column">
<ul style="list-style: none; line-height: 2;">
<li>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512" class="icon filter-black" role="presentation"><title>contact arXiv</title><desc>Click here to contact arXiv</desc><path d="M502.3 190.8c3.9-3.1 9.7-.2 9.7 4.7V400c0 26.5-21.5 48-48 48H48c-26.5 0-48-21.5-48-48V195.6c0-5 5.7-7.8 9.7-4.7 22.4 17.4 52.1 39.5 154.1 113.6 21.1 15.4 56.7 47.8 92.2 47.6 35.7.3 72-32.8 92.3-47.6 102-74.1 131.6-96.3 154-113.7zM256 320c23.2.4 56.6-29.2 73.4-41.4 132.7-96.3 142.8-104.7 173.4-128.7 5.8-4.5 9.2-11.5 9.2-18.9v-19c0-26.5-21.5-48-48-48H48C21.5 64 0 85.5 0 112v19c0 7.4 3.4 14.3 9.2 18.9 30.6 23.9 40.7 32.4 173.4 128.7 16.8 12.2 50.2 41.8 73.4 41.4z"/></svg>
<a href="https://info.arxiv.org/help/contact.html"> Contact</a>
</li>
<li>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512" class="icon filter-black" role="presentation"><title>subscribe to arXiv mailings</title><desc>Click here to subscribe</desc><path d="M476 3.2L12.5 270.6c-18.1 10.4-15.8 35.6 2.2 43.2L121 358.4l287.3-253.2c5.5-4.9 13.3 2.6 8.6 8.3L176 407v80.5c0 23.6 28.5 32.9 42.5 15.8L282 426l124.6 52.2c14.2 6 30.4-2.9 33-18.2l72-432C515 7.8 493.3-6.8 476 3.2z"/></svg>
<a href="https://info.arxiv.org/help/subscribe"> Subscribe</a>
</li>
</ul>
</div>
</div>
</div>
<!-- End Macro-Column 1 -->
<!-- Macro-Column 2 -->
<div class="column" style="padding: 0;">
<div class="columns">
<div class="column">
<ul style="list-style: none; line-height: 2;">
<li><a href="https://info.arxiv.org/help/license/index.html">Copyright</a></li>
<li><a href="https://info.arxiv.org/help/policies/privacy_policy.html">Privacy Policy</a></li>
</ul>
</div>
<div class="column sorry-app-links">
<ul style="list-style: none; line-height: 2;">
<li><a href="https://info.arxiv.org/help/web_accessibility.html">Web Accessibility Assistance</a></li>
<li>
<p class="help">
<a class="a11y-main-link" href="https://status.arxiv.org" target="_blank">arXiv Operational Status <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 256 512" class="icon filter-dark_grey" role="presentation"><path d="M224.3 273l-136 136c-9.4 9.4-24.6 9.4-33.9 0l-22.6-22.6c-9.4-9.4-9.4-24.6 0-33.9l96.4-96.4-96.4-96.4c-9.4-9.4-9.4-24.6 0-33.9L54.3 103c9.4-9.4 24.6-9.4 33.9 0l136 136c9.5 9.4 9.5 24.6.1 34z"/></svg></a><br>
</p>
</li>
</ul>
</div>
</div>
</div> <!-- end MetaColumn 2 -->
<!-- End Macro-Column 2 -->
</div>
</footer>
</div>
<script src="/static/base/1.0.1/js/member_acknowledgement.js"></script>
</body>
</html>

4897
research/cultnet-distributed-database/dynamo-amazon-science.html Normal file
View file

File diff suppressed because one or more lines are too long

BIN
research/cultnet-distributed-database/dynamo-sosp2007.pdf Normal file
View file

Binary file not shown.

11793
research/cultnet-distributed-database/firebase-realtime-offline.html Normal file
View file

File diff suppressed because one or more lines are too long

BIN
research/cultnet-distributed-database/hashgraph-swirlds-tr-2016-01.pdf Normal file
View file

Binary file not shown.

676
research/cultnet-distributed-database/raft-extended.html Normal file
View file

@ -0,0 +1,676 @@
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
<head>
<meta charset="utf-8" />
<title>In Search of an Understandable Consensus Algorithm (Extended Version)</title>
<meta name="generator" content="Org mode" />
<meta name="author" content="Diego Ongaro and John Ousterhout, Stanford University" />
<link rel="stylesheet" type="text/css" href="/themes/readtheorg/styles/readtheorg/css/htmlize.css"/>
<link rel="stylesheet" type="text/css" href="/themes/readtheorg/styles/readtheorg/css/readtheorg.css"/>
<script src="/themes/jquery/2.1.3/jquery.min.js"></script>
<script src="/themes/bootstrap/3.3.4/js/bootstrap.min.js"></script>
<script type="text/javascript" src="/themes/readtheorg/styles/lib/js/jquery.stickytableheaders.min.js"></script>
<script type="text/javascript" src="/themes/readtheorg/styles/readtheorg/js/readtheorg.js"></script>
</head>
<body>
<div id="org-div-home-and-up">
<a accesskey="h" href="../index.html"> UP </a>
|
<a accesskey="H" href="paperlist.html"> HOME </a>
</div><div id="content">
<h1 class="title">In Search of an Understandable Consensus Algorithm (Extended Version)</h1>
<div id="table-of-contents">
<h2>Table of Contents</h2>
<div id="text-table-of-contents">
<ul>
<li><a href="#orgf960eb7">DECLARATION</a></li>
<li><a href="#orgd05d3d0">Abstract</a></li>
<li><a href="#orge9839bb">1. Introduction</a></li>
<li><a href="#orgae8bb96">2. Replicated state machines</a></li>
<li><a href="#org1fe8017">3. What's wrong with Paxos?</a></li>
<li><a href="#orgfc9c6b4">4. Designing for understandability</a></li>
<li><a href="#orge3d8f10">5. The Raft consensus algorithm</a>
<ul>
<li><a href="#org752f98e">5.1. Raft basics</a></li>
<li><a href="#org291797f">5.2. Leader election</a></li>
<li><a href="#org44bb0a6">5.3. Log replication</a></li>
</ul>
</li>
</ul>
</div>
</div>
<div id="outline-container-orgf960eb7" class="outline-2">
<h2 id="orgf960eb7">DECLARATION</h2>
<div class="outline-text-2" id="text-orgf960eb7">
<p>
This page is the In Search of an Understandable Consensus Algorithm
(Extended Version) paper. Original Paper Link
is: <a href="https://raft.github.io/raft.pdf">https://raft.github.io/raft.pdf</a>
</p>
<p>
In Search of an Understandable Consensus Algorithm<br />
(Extended Version)<br />
Diego Ongaro and John Ousterhout<br />
Stanford University
</p>
</div>
</div>
<div id="outline-container-orgd05d3d0" class="outline-2">
<h2 id="orgd05d3d0">Abstract</h2>
<div class="outline-text-2" id="text-orgd05d3d0">
<p>
Raft is a consensus algorithm for managing a replicated log. It produces a
result equivalent to (multi-)Paxos, and it is as efficient as Paxos, but its
structure is different from Paxos; this makes Raft more understandable than
Paxos and also provides a better foundation for building practical systems.
In order to enhance understandability, Raft separates the key elements of
consensus, such as leader election, log replication, and safety, and it enforces
a stronger degree of coherency to reduce the number of states that must be
considered. Results from a user study demonstrate that Raft is easier for
students to learn than Paxos. Raft also includes a new mechanism for changing
the cluster membership, which uses overlapping majorities to guarantee safety.
</p>
</div>
</div>
<div id="outline-container-orge9839bb" class="outline-2">
<h2 id="orge9839bb"><span class="section-number-2">1</span> Introduction</h2>
<div class="outline-text-2" id="text-1">
<p>
Consensus algorithms allow a collection of machines to work as a coherent group
that can survive the failures of some of its members. Because of this, they play
a key role in building reliable large-scale software systems. Paxos [15, 16] has
dominated the discussion of consensus algorithms over the last decade: most
implementations of consensus are based on Paxos or influenced by it, and Paxos
has become the primary vehicle used to teach students about consensus.
</p>
<p>
Unfortunately, Paxos is quite difficult to understand, in spite of numerous
attempts to make it more approachable. Furthermore, its architecture requires
complex changes to support practical systems. As a result, both system builders
and students struggle with Paxos.
</p>
<p>
After struggling with Paxos ourselves, we set out to find a new consensus
algorithm that could provide a better foundation for system building and
education. Our approach was unusual in that our primary goal was
understandability: could we define a consensus algorithm for practical systems
and describe it in a way that is significantly easier to learn than Paxos?
Furthermore, we wanted the algorithm to facilitate the development of intuitions
that are essential for system builders. It was important not just for the
algorithm to work, but for it to be obvious why it works.
</p>
<p>
The result of this work is a consensus algorithm called Raft. In designing Raft
we applied specific techniques to improve understandability,including
decomposition (Raft separates leader election, log replication, and safety) and
This tech report is an extended version of [32]; additional material is noted
with a gray bar in the margin. Published May 20, 2014. state space reduction
(relative to Paxos, Raft reduces the degree of nondeterminism and the ways
servers can be inconsistent with each other). A user study with 43 students
at two universities shows that Raft is significantly easier to understand than
Paxos: after learning both algorithms, 33 of these students were able to answer
questions about Raft better than questions about Paxos.
</p>
<p>
Raft is similar in many ways to existing consensus algorithms (most notably, Oki
and Liskov's Viewstamped Replication [29, 22]), but it has several novel
features:
</p>
<p>
1/. Strong leader: Raft uses a stronger form of leadership than other consensus
algorithms. For example, log entries only flow from the leader to other servers.
This simplifies the management of the replicated log and makes Raft easier to
understand.
</p>
<p>
2/. Leader election: Raft uses randomized timers to elect leaders. This adds
only a small amount of mechanism to the heartbeats already required for any
consensus algorithm, while resolving conflicts simply and rapidly.
</p>
<p>
3/. Membership changes: Raft's mechanism for changing the set of servers in the
cluster uses a new joint consensus approach where the majorities of two
different configurations overlap during transitions. This allows the cluster to
continue operating normally during configuration changes.
</p>
<p>
We believe that Raft is superior to Paxos and other consensus algorithms, both
for educational purposes and as a foundation for implementation. It is simpler
and more understandable than other algorithms; it is described completely enough
to meet the needs of a practical system; it has several open-source
implementations and is used by several companies; its safety properties have
been formally specified and proven; and its efficiency is comparable to other
algorithms.
</p>
<p>
The remainder of the paper introduces the replicated state machine problem
(Section 2), discusses the strengths and weaknesses of Paxos (Section 3),
describes our general approach to understandability (Section 4), presents the
Raft consensus algorithm (Sections 5–8), evaluates Raft (Section 9), and
discusses related work (Section 10).
</p>
</div>
</div>
<div id="outline-container-orgae8bb96" class="outline-2">
<h2 id="orgae8bb96"><span class="section-number-2">2</span> Replicated state machines</h2>
<div class="outline-text-2" id="text-2">
<p>
Consensus algorithms typically arise in the context of replicated state machines
[37]. In this approach, state machines on a collection of servers compute
identical copies of the same state and can continue operating even if some of
the servers are down. Replicated state machines are used to solve a variety of
fault tolerance problems in distributed systems. For example, large-scale
systems that have a single cluster leader, such as GFS [8], HDFS [38], and
RAMCloud [33], typically use a separate replicated state machine to manage
leader election and store configuration information that must survive leader
crashes. Examples of replicated state machines include Chubby [2] and
ZooKeeper [11].
</p>
<div id="org1bdf0c5" class="figure">
<p><img src="img/raft-figure1.jpg" alt="raft-figure1.jpg" />
</p>
<p><span class="figure-number">Figure 1: </span>Replicated state machine architecture. The consensus algorithm manages a replicated log containing state machine commands from clients. The state machines process identical sequences of commands from the logs, so they produce the same outputs.</p>
</div>
<p>
Replicated state machines are typically implemented using a replicated log, as
shown in Figure 1. Each server stores a log containing a series of commands,
which its state machine executes in order. Each log contains the same commands
in the same order, so each state machine processes the same sequence of
commands. Since the state machines are deterministic, each computes the same
state and the same sequence of outputs.
</p>
<p>
Keeping the replicated log consistent is the job of the consensus algorithm.
The consensus module on a server receives commands from clients and adds them to
its log. It communicates with the consensus modules on other servers to ensure
that every log eventually contains the same requests in the same order, even if
some servers fail. Once commands are properly replicated, each server's state
machine processes them in log order, and the outputs are returned to clients. As
a result, the servers appear to form a single, highly reliable state machine.
</p>
<p>
Consensus algorithms for practical systems typically have the following
properties:
</p>
<p>
• They ensure <i>safety</i> (never returning an incorrect result) under all
non-Byzantine conditions, including network delays, partitions, and packet loss,
duplication, and reordering.
</p>
<p>
• They are fully functional (available) as long as any majority of the servers
are operational and can communicate with each other and with clients. Thus, a
typical cluster of five servers can tolerate the failure of any two servers.
Servers are assumed to fail by stopping; they may later recover from state on
stable storage and rejoin the cluster.
</p>
<p>
• They do not depend on timing to ensure the consistency of the logs: faulty
clocks and extreme message delays can, at worst, cause availability problems.
</p>
<p>
• In the common case, a command can complete as soon as a majority of the
cluster has responded to a single round of remote procedure calls; a minority of
slow servers need not impact overall system performance.
</p>
</div>
</div>
<div id="outline-container-org1fe8017" class="outline-2">
<h2 id="org1fe8017"><span class="section-number-2">3</span> What's wrong with Paxos?</h2>
<div class="outline-text-2" id="text-3">
<p>
Over the last ten years, Leslie Lamport's Paxos protocol [15] has become almost
synonymous with consensus: it is the protocol most commonly taught in courses,
and most implementations of consensus use it as a starting point. Paxos first
defines a protocol capable of reaching agreement on a single decision, such as
a single replicated log entry. We refer to this subset as single-decree Paxos.
Paxos then combines multiple instances of this protocol to facilitate a series
of decisions such as a log (multi-Paxos). Paxos ensures both safety and
liveness, and it supports changes in cluster membership. Its correctness has
been proven, and it is efficient in the normal case.
</p>
<p>
Unfortunately, Paxos has two significant drawbacks. The first drawback is that
Paxos is exceptionally difficult to understand. The full explanation [15] is
notoriously opaque; few people succeed in understanding it, and only with great
effort. As a result, there have been several attempts to explain Paxos in
simpler terms [16, 20, 21]. These explanations focus on the single-decree
subset, yet they are still challenging. In an informal survey of attendees at
NSDI 2012, we found few people who were comfortable with Paxos, even among
seasoned researchers. We struggled with Paxos ourselves; we were not able to
understand the complete protocol until after reading several simplified
explanations and designing our own alternative protocol, a process that took
almost a year.
</p>
<p>
We hypothesize that Paxos' opaqueness derives from its choice of the
single-decree subset as its foundation. Single-decree Paxos is dense and subtle:
it is divided into two stages that do not have simple intuitive explanations and
cannot be understood independently. Because of this, it is difficult to develop
intuitions about why the single-decree protocol works. The composition rules for
multi-Paxos add significant additional complexity and subtlety. We believe that
the overall problem of reaching consensus on multiple decisions (i.e., a log
instead of a single entry) can be decomposed in other ways that are more direct
and obvious.
</p>
<p>
The second problem with Paxos is that it does not provide a good foundation for
building practical implementations. One reason is that there is no widely
agreed-upon algorithm for multi-Paxos. Lamport's descriptions are mostly about
single-decree Paxos; he sketched possible approaches to multi-Paxos, but many
details are missing. There have been several attempts to flesh out and optimize
Paxos, such as [26], [39], and [13], but these differ from each other and from
Lamport's sketches. Systems such as Chubby [4] have implemented Paxos-like
algorithms, but in most cases their details have not been published.
</p>
<p>
Furthermore, the Paxos architecture is a poor one for building practical
systems; this is another consequence of the single-decree decomposition. For
example, there is little benefit to choosing a collection of log entries
independently and then melding them into a sequential log; this just adds
complexity. It is simpler and more efficient to design a system around a log,
where new entries are appended sequentially in a constrained order. Another
problem is that Paxos uses a symmetric peer-to-peer approach at its core (though
it eventually suggests a weak form of leadership as a performance optimization).
This makes sense in a simplified world where only one decision will be made, but
few practical systems use this approach. If a series of decisions must be made,
it is simpler and faster to first elect a leader, then have the leader
coordinate the decisions.
</p>
<p>
As a result, practical systems bear little resemblance to Paxos. Each
implementation begins with Paxos, discovers the difficulties in implementing it,
and then develops a significantly different architecture. This is timeconsuming
and error-prone, and the difficulties of understanding Paxos exacerbate the
problem. Paxos' formulation may be a good one for proving theorems about its
correctness, but real implementations are so different from Paxos that the
proofs have little value. The following comment from the Chubby implementers is
typical:
</p>
<blockquote>
<p>
There are significant gaps between the description of the Paxos algorithm and
the needs of a real-world system. . . . the final system will be based on an
unproven protocol [4].
</p>
</blockquote>
<p>
Because of these problems, we concluded that Paxos does not provide a good
foundation either for system building or for education. Given the importance of
consensus in large-scale software systems, we decided to see if we could design
an alternative consensus algorithm with better properties than Paxos. Raft is
the result of that experiment.
</p>
</div>
</div>
<div id="outline-container-orgfc9c6b4" class="outline-2">
<h2 id="orgfc9c6b4"><span class="section-number-2">4</span> Designing for understandability</h2>
<div class="outline-text-2" id="text-4">
<p>
We had several goals in designing Raft: it must provide a complete and practical
foundation for system building, so that it significantly reduces the amount of
design work required of developers; it must be safe under all conditions and
available under typical operating conditions; and it must be efficient for
common operations. But our most important goal—and most difficult challenge
was understandability. It must be possible for a large audience to understand
the algorithm comfortably. In addition, it must be possible to develop
intuitions about the algorithm, so that system builders can make the extensions
that are inevitable in real-world implementations.
</p>
<p>
There were numerous points in the design of Raft where we had to choose among
alternative approaches. In these situations we evaluated the alternatives
based on understandability: how hard is it to explain each alternative (for
example, how complex is its state space, and does it have subtle implications?),
and how easy will it be for a reader to completely understand the approach and
its implications?
</p>
<p>
We recognize that there is a high degree of subjectivity in such analysis;
nonetheless, we used two techniques that are generally applicable. The first
technique is the well-known approach of problem decomposition: wherever
possible, we divided problems into separate pieces that could be solved,
explained, and understood relatively independently. For example, in Raft we
separated leader election, log replication, safety, and membership changes.
</p>
<p>
Our second approach was to simplify the state space by reducing the number of
states to consider, making the system more coherent and eliminating
nondeterminism where possible. Specifically, logs are not allowed to have holes,
and Raft limits the ways in which logs can become inconsistent with each other.
Although in most cases we tried to eliminate nondeterminism, there are some
situations where nondeterminism actually improves understandability.
In particular, randomized approaches introduce nondeterminism, but they tend to
reduce the state space by handling all possible choices in a similar fashion
("choose any; it doesn't matter"). We used randomization to simplify the Raft
leader election algorithm.
</p>
</div>
</div>
<div id="outline-container-orge3d8f10" class="outline-2">
<h2 id="orge3d8f10"><span class="section-number-2">5</span> The Raft consensus algorithm</h2>
<div class="outline-text-2" id="text-5">
<p>
Raft is an algorithm for managing a replicated log of the form described in
Section 2. Figure 2 summarizes the algorithm in condensed form for reference,
and Figure 3 lists key properties of the algorithm; the elements of these
figures are discussed piecewise over the rest of this section.
</p>
<p>
<img src="img/raft-figure2-1.jpg" alt="raft-figure2-1.jpg" />
<img src="img/raft-figure2-2.jpg" alt="raft-figure2-2.jpg" />
<img src="img/raft-figure2-3.jpg" alt="raft-figure2-3.jpg" />
</p>
<div id="orgf3f186a" class="figure">
<p><img src="img/raft-figure2-4.jpg" alt="raft-figure2-4.jpg" />
</p>
<p><span class="figure-number">Figure 2: </span>A condensed summary of the Raft consensus algorithm (excluding membership changes and log compaction). The server behavior in the upper-left box is described as a set of rules that trigger independently and repeatedly. Section numbers such as §5.2 indicate where particular features are discussed.A formal specification [31] describes the algorithm more precisely.</p>
</div>
<p>
Raft implements consensus by first electing a distinguished leader, then giving
the leader complete responsibility for managing the replicated log. The leader
accepts log entries from clients, replicates them on other servers, and tells
servers when it is safe to apply log entries to their state machines. Having a
leader simplifies the management of the replicated log. For example, the leader
can decide where to place new entries in the log without consulting other
servers, and data flows in a simple fashion from the leader to other servers.
A leader can fail or become disconnected from the other servers, in which case
a new leader is elected.
</p>
<p>
Given the leader approach, Raft decomposes the consensus problem into three
relatively independent subproblems, which are discussed in the subsections that
follow:
</p>
<p>
• Leader election: a new leader must be chosen when an existing leader fails
(Section 5.2).
</p>
<p>
• Log replication: the leader must accept log entries from clients and replicate
them across the cluster, forcing the other logs to agree with its own (Section
5.3).
• Safety: the key safety property for Raft is the State Machine Safety Property
in Figure 3: if any server has applied a particular log entry to its state
machine, then no other server may apply a different command for the same log
index. Section 5.4 describes how Raft ensures this property; the solution
involves an additional restriction on the election mechanism described in
Section 5.2.
</p>
<p>
After presenting the consensus algorithm, this section discusses the issue of
availability and the role of timing in the system.
</p>
<div id="org159660c" class="figure">
<p><img src="img/raft-figure3.jpg" alt="raft-figure3.jpg" />
</p>
<p><span class="figure-number">Figure 3: </span>Raft guarantees that each of these properties is true at all times. The section numbers indicate where each property is discussed.</p>
</div>
</div>
<div id="outline-container-org752f98e" class="outline-3">
<h3 id="org752f98e"><span class="section-number-3">5.1</span> Raft basics</h3>
<div class="outline-text-3" id="text-5-1">
<p>
A Raft cluster contains several servers; five is a typical number, which allows
the system to tolerate two failures. At any given time each server is in one of
three states: leader, follower, or candidate. In normal operation there is
exactly one leader and all of the other servers are followers. Followers are
passive: they issue no requests on their own but simply respond to requests from
leaders and candidates. The leader handles all client requests (if a client
contacts a follower, the follower redirects it to the leader). The third state,
candidate, is used to elect a new leader as described in Section 5.2. Figure 4
shows the states and their transitions; the transitions are discussed below.
</p>
<div id="org3f316c7" class="figure">
<p><img src="img/raft-figure4.jpg" alt="raft-figure4.jpg" />
</p>
<p><span class="figure-number">Figure 4: </span>Server states. Followers only respond to requests from other servers. If a follower receives no communication, it becomes a candidate and initiates an election. A candidate that receives votes from a majority of the full cluster becomes the new leader. Leaders typically operate until they fail.</p>
</div>
<p>
Raft divides time into terms of arbitrary length, as shown in Figure 5. Terms
are numbered with consecutive integers. Each term begins with an election, in
which one or more candidates attempt to become leader as described in Section
5.2. If a candidate wins the election, then it serves as leader for the rest of
the term. In some situations an election will result in a split vote. In this
case the term will end with no leader; a new term (with a new election) will
begin shortly. Raft ensures that there is at most one leader in a given term.
</p>
<div id="orgc7ff282" class="figure">
<p><img src="img/raft-figure5.jpg" alt="raft-figure5.jpg" />
</p>
<p><span class="figure-number">Figure 5: </span>Time is divided into terms, and each term begins with an election. After a successful election, a single leader manages the cluster until the end of the term. Some elections fail, in which case the term ends without choosing a leader. The transitions between terms may be observed at different times on different servers.</p>
</div>
<p>
Different servers may observe the transitions between terms at different times,
and in some situations a server may not observe an election or even entire
terms. Terms act as a logical clock [14] in Raft, and they allow servers to
detect obsolete information such as stale leaders. Each server stores a current
term number, which increases monotonically over time. Current terms are
exchanged whenever servers communicate; if one server's current term is smaller
than the other's, then it updates its current term to the larger value. If a
candidate or leader discovers that its term is out of date, it immediately
reverts to follower state. If a server receives a request with a stale term
number, it rejects the request.
</p>
<p>
Raft servers communicate using remote procedure calls (RPCs), and the basic
consensus algorithm requires only two types of RPCs. RequestVote RPCs are
initiated by candidates during elections (Section 5.2), and AppendEntries RPCs
are initiated by leaders to replicate log entries and to provide a form of
heartbeat (Section 5.3). Section 7 adds a third RPC for transferring snapshots
between servers. Servers retry RPCs if they do not receive a response in a
timely manner, and they issue RPCs in parallel for best performance.
</p>
</div>
</div>
<div id="outline-container-org291797f" class="outline-3">
<h3 id="org291797f"><span class="section-number-3">5.2</span> Leader election</h3>
<div class="outline-text-3" id="text-5-2">
<p>
Raft uses a heartbeat mechanism to trigger leader election. When servers start
up, they begin as followers. A server remains in follower state as long as it
receives valid RPCs from a leader or candidate. Leaders send periodic heartbeats
(AppendEntries RPCs that carry no log entries) to all followers in order to
maintain their authority. If a follower receives no communication over a period
of time called the election timeout, then it assumes there is no viable leader
and begins an election to choose a new leader.
</p>
<p>
To begin an election, a follower increments its current term and transitions to
candidate state. It then votes for itself and issues RequestVote RPCs in
parallel to each of the other servers in the cluster. A candidate continues in
this state until one of three things happens: (a) it wins the election, (b)
another server establishes itself as leader, or (c) a period of time goes by
with no winner. These outcomes are discussed separately in the paragraphs below.
</p>
<p>
A candidate wins an election if it receives votes from a majority of the servers
in the full cluster for the same term. Each server will vote for at most one
candidate in a given term, on a first-come-first-served basis (note: Section 5.4
adds an additional restriction on votes). The majority rule ensures that at most
one candidate can win the election for a particular term (the Election Safety
Property in Figure 3). Once a candidate wins an election, it becomes leader. It
then sends heartbeat messages to all of the other servers to establish its
authority and prevent new elections.
</p>
<p>
While waiting for votes, a candidate may receive an AppendEntries RPC from
another server claiming to be leader. If the leader's term (included in its RPC)
is at least as large as the candidate's current term, then the candidate
recognizes the leader as legitimate and returns to follower state. If the term
in the RPC is smaller than the candidate's current term, then the candidate
rejects the RPC and continues in candidate state.
</p>
<p>
The third possible outcome is that a candidate neither wins nor loses the
election: if many followers become candidates at the same time, votes could be
split so that no candidate obtains a majority. When this happens, each candidate
will time out and start a new election by incrementing its term and initiating
another round of RequestVote RPCs. However, without extra measures split votes
could repeat indefinitely.
</p>
<p>
Raft uses randomized election timeouts to ensure that split votes are rare and
that they are resolved quickly. To prevent split votes in the first place,
election timeouts are chosen randomly from a fixed interval (e.g., 150–300ms).
This spreads out the servers so that in most cases only a single server will
time out; it wins the election and sends heartbeats before any other servers
time out. The same mechanism is used to handle split votes. Each candidate
restarts its randomized election timeout at the start of an election, and it
waits for that timeout to elapse before starting the next election; this reduces
the likelihood of another split vote in the new election. Section 9.3 shows
that this approach elects a leader rapidly.
</p>
<p>
Elections are an example of how understandability guided our choice between
design alternatives. Initially we planned to use a ranking system: each
candidate was assigned a unique rank, which was used to select between
competing candidates. If a candidate discovered another candidate with higher
rank, it would return to follower state so that the higher ranking candidate
could more easily win the next election. We found that this approach created
subtle issues around availability (a lower-ranked server might need to time out
and become a candidate again if a higher-ranked server fails, but if it does so
too soon, it can reset progress towards electing a leader). We made adjustments
to the algorithm several times, but after each adjustment new corner cases
appeared. Eventually we concluded that the randomized retry approach is more
obvious and understandable.
</p>
</div>
</div>
<div id="outline-container-org44bb0a6" class="outline-3">
<h3 id="org44bb0a6"><span class="section-number-3">5.3</span> Log replication</h3>
<div class="outline-text-3" id="text-5-3">
<p>
Once a leader has been elected, it begins servicing client requests. Each client
request contains a command to be executed by the replicated state machines. The
leader appends the command to its log as a new entry, then issues AppendEntries
RPCs in parallel to each of the other servers to replicate the entry. When the
entry has been safely replicated (as described below), the leader applies the
entry to its state machine and returns the result of that execution to the
client. If followers crash or run slowly, or if network packets are lost, the
leader retries AppendEntries RPCs indefinitely (even after it has responded to
the client) until all followers eventually store all log entries.
</p>
<div id="org879421f" class="figure">
<p><img src="img/raft-figure6.jpg" alt="raft-figure6.jpg" />
</p>
<p><span class="figure-number">Figure 6: </span>Logs are composed of entries, which are numbered sequentially. Each entry contains the term in which it was created (the number in each box) and a command for the state machine. An entry is considered committed if it is safe for that entry to be applied to state machines.</p>
</div>
<p>
Logs are organized as shown in Figure 6. Each log entry stores a state machine
command along with the term number when the entry was received by the leader.
The term numbers in log entries are used to detect inconsistencies between logs
and to ensure some of the properties in Figure 3. Each log entry also has an
integer index identifying its position in the log.
</p>
<p>
The leader decides when it is safe to applyy a log en-
try to the state machines; such an entry is called commit-
ted. Raft guarantees that committed entries are durable
and will eventually be executed by all of the available
state machines. A log entry is committed once the leader
that created the entry has replicated it on a majority of
the servers (e.g., entry 7 in Figure 6). This also commits
all preceding entries in the leader's log, including entries
created by previous leaders. Section 5.4 discusses some
subtleties when applying this rule after leader changes,
and it also shows that this definition of commitment is
safe. The leader keeps track of the highest index it knows
to be committed, and it includes that index in future
AppendEntries RPCs (including heartbeats) so that the
other servers eventually find out. Once a follower learns
that a log entry is committed, it applies the entry to its
local state machine (in log order).
We designed the Raft log mechanism to maintain a high
level of coherency between the logs on different servers.
Not only does this simplify the system's behavior and
make it more predictable, but it is an important component
of ensuring safety. Raft maintains the following proper-
ties, which together constitute the Log Matching Property
in Figure 3:
</p>
</div>
</div>
</div>
<script src="https://utteranc.es/client.js"
repo="yygcode/yygcode.github.io"
issue-term="pathname"
label=" Utterances"
theme="github-light"
crossorigin="anonymous"
async>
</script>
<noscript>Please enable JavaScript to view the <a href="https://github.com/utterance">comments powered by utterances.</a></noscript>
</div>
<div id="postamble" class="status">
<div class="copyright">
2012-2020 Copyright&copy; <i> YANYG<br/>Powered by Emacs Orgmode</i>
</div>
</div>
</body>
</html>

BIN
research/cultnet-distributed-database/raft-extended.pdf Normal file
View file

Binary file not shown.

845
research/cultnet-distributed-database/rethinkdb-changefeeds.html Normal file
View file

@ -0,0 +1,845 @@
<!doctype html>
<html>
<head>
<title>Changefeeds in RethinkDB - RethinkDB</title>
<meta charset="utf-8">
<meta content="IE=edge,chrome=1" http-equiv="X-UA-Compatible">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta property="twitter:account_id" content="52351557" />
<link rel="alternate" type="application/rss+xml" title="RethinkDB: the open-source database for the realtime web" href="/feed.xml">
<link rel="stylesheet" media="screen" href="/assets/css/styles-02.css"/>
<link rel="stylesheet" media="screen" href="/assets/css/jekyll-github.css"/>
<link rel="icon" type="image/png" href="/favicon.png">
<!-- typekit -->
<script type="text/javascript" src="//use.typekit.net/lmb6cah.js"></script>
<script type="text/javascript">try{Typekit.load();}catch(e){}</script>
<!-- jquery -->
<script type="text/javascript" src="/assets/js/jquery-2.1.3.min.js"></script>
<!-- js-cookie -->
<script type="text/javascript" src="/assets/js/js.cookie.js"></script>
<!-- waypoints -->
<script type="text/javascript" src="/assets/js/waypoints/jquery.waypoints.min.js"></script>
<script type="text/javascript" src="/assets/js/waypoints/shortcuts/sticky.min.js"></script>
<script type="text/javascript" src="/assets/js/site.js"></script>
<!-- algolia -->
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/docsearch.js@2/dist/cdn/docsearch.min.css" />
<script>
(function(h,o,t,j,a,r){
h.hj=h.hj||function(){(h.hj.q=h.hj.q||[]).push(arguments)};
h._hjSettings={hjid:1540062,hjsv:6};
a=o.getElementsByTagName('head')[0];
r=o.createElement('script');r.async=1;
r.src=t+h._hjSettings.hjid+j+h._hjSettings.hjsv;
a.appendChild(r);
})(window,document,'https://static.hotjar.com/c/hotjar-','.js?sv=');
</script>
</head>
<body class="docs">
<div id="wrapper">
<!-- site nav -->
<nav class="mobile-menu push-menu-right">
<ul>
<li class="menu-trigger"><a href="/">Close</a></li>
<li><a href="/faq">FAQ</a></li>
<li><a href="/docs">Docs</a></li>
<li><a href="/api">API</a></li>
<li><a href="/community">Community</a></li>
<li><a href="/blog">Blog</a></li>
</ul>
</nav>
<nav class="site-nav">
<div class="site-container">
<a class="logo" href="/">RethinkDB</a>
<ul>
<li class="menu-trigger"><a href="/"><img class="menu-trigger-icon" src="/assets/images/icons/icon-menu-white.png"></a></li>
<li><a href="/faq">FAQ</a></li>
<li><a href="/docs">Docs</a></li>
<li><a href="/api">API</a></li>
<li><a href="/community">Community</a></li>
<li><a href="/blog">Blog</a></li>
</ul>
</div>
</nav>
<section class="documentation">
<div class="site-container">
<section class="docs-sidebar-left">
<div class="search">
<input class="small" type="text" placeholder="Search documentation...">
</div>
<nav class="docs-nav">
<section>
<h1><a href="#">Getting started</a></h1>
<ul>
<li class="">
<a href="/docs/quickstart/">Thirty-second quickstart</a>
</li>
<li class="">
<a href="/docs/install/">Install the server</a>
</li>
<li class="">
<a href="/docs/install-drivers/">Install client drivers</a>
</li>
<li class="">
<a href="/docs/start-a-server/">Start a RethinkDB server</a>
</li>
</ul>
</section>
<section>
<h1><a href="#">The ReQL query language</a></h1>
<ul class="expanded">
<li class="">
<a href="/docs/guide/">Ten-minute guide</a>
</li>
<li class="">
<a href="/docs/introduction-to-reql/">Introduction to ReQL</a>
</li>
<li class="">
<a href="/docs/secondary-indexes/">Using secondary indexes</a>
</li>
<li class="">
<a href="/docs/nested-fields/">Accessing nested fields</a>
</li>
<li class="">
<a href="/docs/table-joins/">Table joins in RethinkDB</a>
</li>
<li class="">
<a href="/docs/map-reduce/">Map-reduce in RethinkDB</a>
</li>
<li class="active ">
<a href="/docs/changefeeds/">Changefeeds</a>
</li>
<li class="">
<a href="/docs/async-connections/">Asynchronous connections</a>
</li>
<li class="">
<a href="/docs/external-api-access/">Accessing HTTP APIs</a>
</li>
<li class="">
<a href="/docs/error-types/">ReQL error types</a>
</li>
<li class="">
<a href="/api/">API command reference</a>
</li>
</ul>
</section>
<section>
<h1><a href="#">The RethinkDB data model</a></h1>
<ul>
<li class="">
<a href="/docs/data-types/">ReQL data types</a>
</li>
<li class="">
<a href="/docs/dates-and-times/">Dates and times</a>
</li>
<li class="">
<a href="/docs/storing-binary/">Binary objects</a>
</li>
<li class="">
<a href="/docs/geo-support/">Geospatial queries</a>
</li>
<li class="">
<a href="/docs/data-modeling/">Modeling relationships</a>
</li>
</ul>
</section>
<section>
<h1><a href="#">ReQL in practice</a></h1>
<ul>
<li class="">
<a href="/docs/optimization/">Optimizing query performance</a>
</li>
<li class="">
<a href="/docs/reql-data-exploration/">Data exploration</a>
</li>
<li class="">
<a href="/docs/sql-to-reql/">SQL to ReQL cheat sheet</a>
</li>
<li class="">
<a href="/docs/cookbook/">Cookbook</a>
</li>
<li class="">
<a href="/docs/examples/">Example projects</a>
</li>
</ul>
</section>
<section>
<h1><a href="#">Administration</a></h1>
<ul>
<li class="">
<a href="/docs/administration-tools/">Administration tools</a>
</li>
<li class="">
<a href="/docs/sharding-and-replication/">Scaling, sharding and replication</a>
</li>
<li class="">
<a href="/docs/permissions-and-accounts/">Permissions and user accounts</a>
</li>
<li class="">
<a href="/docs/failover/">Failover</a>
</li>
<li class="">
<a href="/docs/backup/">Backing up your data</a>
</li>
<li class="">
<a href="/docs/system-tables/">System tables</a>
</li>
<li class="nested ">
<a href="/docs/system-issues/">Current issues</a>
</li>
<li class="nested ">
<a href="/docs/system-stats/">Statistics</a>
</li>
<li class="nested ">
<a href="/docs/system-jobs/">Jobs</a>
</li>
<li class="">
<a href="/docs/migration/">Version migration</a>
</li>
<li class="">
<a href="/docs/importing/">Importing data</a>
</li>
<li class="">
<a href="/docs/third-party-admin-tools/">Third-party admin tools</a>
</li>
</ul>
</section>
<section>
<h1><a href="#">Deployment</a></h1>
<ul>
<li class="">
<a href="/docs/start-on-startup/">Start on system startup</a>
</li>
<li class="">
<a href="/docs/security/">Securing your cluster</a>
</li>
<li class="">
<a href="/docs/config-file/">Configuration file settings</a>
</li>
<li class="">
<a href="/docs/cli-options/">Command line options</a>
</li>
<li class="">
<a href="/docs/paas/">Deploying with PaaS</a>
</li>
<li class="">
<a href="/docs/deployment-tools/">Third-party deployment tools</a>
</li>
</ul>
</section>
<section>
<h1><a href="#">Troubleshooting</a></h1>
<ul>
<li class="">
<a href="/docs/crashes/">Crash recovery</a>
</li>
<li class="">
<a href="/docs/troubleshooting/">Troubleshooting FAQ</a>
</li>
<li class="">
<a href="/docs/memory-usage/">Memory requirements</a>
</li>
</ul>
</section>
<section>
<h1><a href="#">Integration</a></h1>
<ul>
<li class="">
<a href="/docs/publish-subscribe/">Publish-subscribe</a>
</li>
<li class="">
<a href="/docs/elasticsearch/">Integrating Elasticsearch</a>
</li>
<li class="">
<a href="/docs/rabbitmq/">Integrating RabbitMQ</a>
</li>
<li class="">
<a href="/docs/rails">RethinkDB and Rails</a>
</li>
<li class="">
<a href="/docs/frameworks-and-libraries/">Third-party libraries</a>
</li>
<li class="">
<a href="/docs/writing-drivers/">Writing drivers</a>
</li>
</ul>
</section>
<section>
<h1><a href="#">RethinkDB architecture</a></h1>
<ul>
<li class="">
<a href="/docs/architecture/">Architecture details</a>
</li>
<li class="">
<a href="/docs/consistency/">Consistency guarantees</a>
</li>
<li class="">
<a href="/docs/limitations/">Limitations in RethinkDB</a>
</li>
<li class="">
<a href="/docs/stability/">Stability report</a>
</li>
<li class="">
<a href="/docs/faq/">Frequently asked questions</a>
</li>
<li class="">
<a href="/docs/2-1-5-performance-report/">Performance report</a>
</li>
</ul>
</section>
<section>
<h1><a href="#">Security</a></h1>
<ul>
<li class="">
<a href="/docs/vulnerabilities/">Vulnerabilities</a>
</li>
<li class="">
<a href="/docs/mobile-connections/">Connections from mobile devices</a>
</li>
</ul>
</section>
</nav>
</section>
<section class="docs-article">
<h1 class='title'>Changefeeds in RethinkDB</h1>
<section class="byline">
<nav class="lang-selector">
<p class="caption">Language: </p>
<section class="languages">
<a class="lang javascript"
href="
/docs/changefeeds/javascript/
">
<i class="javascript"></i>
<span >JavaScript</span>
</a><span class="divider">|</span><!--
--><a class="lang python"
href="
/docs/changefeeds/python/
">
<i class="python"></i>
<span >Python</span>
</a><span class="divider">|</span><!--
--><a class="lang ruby "
href="
/docs/changefeeds/ruby/
">
<i class="ruby"></i>
<span >Ruby</span>
</a><span class="divider">|</span><!--
--><a class="lang java active"
href="
/docs/changefeeds/java/
">
<i class="java"></i>
<span >Java</span>
</a>
</section>
</nav>
</section>
<div class="alg-content"><p><strong>Changefeeds</strong> lie at the heart of RethinkDB’s real-time functionality.</p>
<div class="toc">
<h1 class="no_toc" id="in-this-article">In this article</h1>
<ul id="markdown-toc">
<li><a href="#basic-usage" id="markdown-toc-basic-usage">Basic usage</a></li>
<li><a href="#point-single-document-changefeeds" id="markdown-toc-point-single-document-changefeeds">Point (single document) changefeeds</a></li>
<li><a href="#changefeeds-with-filtering-and-aggregation-queries" id="markdown-toc-changefeeds-with-filtering-and-aggregation-queries">Changefeeds with filtering and aggregation queries</a></li>
<li><a href="#including-state-changes" id="markdown-toc-including-state-changes">Including state changes</a></li>
<li><a href="#including-initial-values" id="markdown-toc-including-initial-values">Including initial values</a></li>
<li><a href="#including-result-types" id="markdown-toc-including-result-types">Including result types</a></li>
<li><a href="#handling-latency" id="markdown-toc-handling-latency">Handling latency</a></li>
<li><a href="#scaling-considerations" id="markdown-toc-scaling-considerations">Scaling considerations</a></li>
<li><a href="#read-more" id="markdown-toc-read-more">Read more</a></li>
</ul>
</div>
<p>They allow clients to receive changes on a table, a single document, or even the results from a specific query as they happen. Nearly any ReQL query can be turned into a changefeed.</p>
<p><img alt="Data Modeling Illustration" class="api_command_illustration" src="/assets/images/docs/api_illustrations/change-feeds.png" /></p>
<h1 id="basic-usage">Basic usage</h1>
<p>Subscribe to a feed by calling <a href="/api/java/changes">changes</a> on a table:</p>
<div class="language-java highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="n">Cursor</span> <span class="n">changeCursor</span> <span class="o">=</span> <span class="n">r</span><span class="o">.</span><span class="na">table</span><span class="o">(</span><span class="s">"users"</span><span class="o">).</span><span class="na">changes</span><span class="o">().</span><span class="na">run</span><span class="o">(</span><span class="n">conn</span><span class="o">);</span>
<span class="k">for</span> <span class="o">(</span><span class="n">Object</span> <span class="n">change</span> <span class="o">:</span> <span class="n">changeCursor</span><span class="o">)</span> <span class="o">{</span>
<span class="n">System</span><span class="o">.</span><span class="na">out</span><span class="o">.</span><span class="na">println</span><span class="o">(</span><span class="n">change</span><span class="o">);</span>
<span class="o">}</span>
</code></pre></div></div>
<p>The <code class="language-plaintext highlighter-rouge">changes</code> command returns a cursor (like the <code class="language-plaintext highlighter-rouge">table</code> or <code class="language-plaintext highlighter-rouge">filter</code> commands do). You can iterate through its contents using ReQL. Unlike other cursors, the output of <code class="language-plaintext highlighter-rouge">changes</code> is infinite: the cursor will block until more elements are available. Every time you make a change to the table or document the <code class="language-plaintext highlighter-rouge">changes</code> feed is monitoring, a new object will be returned to the cursor. For example, if you insert a user <code class="language-plaintext highlighter-rouge">{"id": 1, "name": "Slava", "age": 31}</code> into the <code class="language-plaintext highlighter-rouge">users</code> table, RethinkDB will post this document to changefeeds subscribed to <code class="language-plaintext highlighter-rouge">users</code>:</p>
<div class="language-json highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="p">{</span><span class="w">
</span><span class="s2">"old_val"</span><span class="p">:</span><span class="w"> </span><span class="kc">null</span><span class="p">,</span><span class="w">
</span><span class="s2">"new_val"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w"> </span><span class="s2">"id"</span><span class="p">:</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span><span class="w"> </span><span class="s2">"name"</span><span class="p">:</span><span class="w"> </span><span class="s2">"Slava"</span><span class="p">,</span><span class="w"> </span><span class="s2">"age"</span><span class="p">:</span><span class="w"> </span><span class="mi">31</span><span class="w"> </span><span class="p">}</span><span class="w">
</span><span class="p">}</span><span class="w">
</span></code></pre></div></div>
<p>Here <code class="language-plaintext highlighter-rouge">old_val</code> is the old version of the document, and <code class="language-plaintext highlighter-rouge">new_val</code> is a new version of the document. On an <code class="language-plaintext highlighter-rouge">insert</code>, <code class="language-plaintext highlighter-rouge">old_val</code> will be <code class="language-plaintext highlighter-rouge">null</code>; on a <code class="language-plaintext highlighter-rouge">delete</code>, <code class="language-plaintext highlighter-rouge">new_val</code> will be <code class="language-plaintext highlighter-rouge">null</code>. On an <code class="language-plaintext highlighter-rouge">update</code>, both <code class="language-plaintext highlighter-rouge">old_val</code> and <code class="language-plaintext highlighter-rouge">new_val</code> are present.</p>
<h1 id="point-single-document-changefeeds">Point (single document) changefeeds</h1>
<p>A “point” changefeed returns changes to a single document within a table rather than the table as a whole.</p>
<div class="language-java highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="n">r</span><span class="o">.</span><span class="na">table</span><span class="o">(</span><span class="s">"users"</span><span class="o">).</span><span class="na">get</span><span class="o">(</span><span class="mi">100</span><span class="o">).</span><span class="na">changes</span><span class="o">().</span><span class="na">run</span><span class="o">(</span><span class="n">conn</span><span class="o">);</span>
</code></pre></div></div>
<p>The output format of a point changefeed is identical to a table changefeed.</p>
<h1 id="changefeeds-with-filtering-and-aggregation-queries">Changefeeds with filtering and aggregation queries</h1>
<p>Like any ReQL command, <code class="language-plaintext highlighter-rouge">changes</code> integrates with the rest of the query language. You can call <code class="language-plaintext highlighter-rouge">changes</code> after most commands that transform or select data:</p>
<ul>
<li><a href="/api/java/filter">filter</a></li>
<li><a href="/api/java/get_all">getAll</a></li>
<li><a href="/api/java/map">map</a></li>
<li><a href="/api/java/pluck">pluck</a></li>
<li><a href="/api/java/between">between</a></li>
<li><a href="/api/java/union">union</a></li>
<li><a href="/api/java/min">min</a></li>
<li><a href="/api/java/max">max</a></li>
<li><a href="/api/java/order_by">orderBy</a>.<a href="/api/java/limit">limit</a></li>
</ul>
<p>You can also chain <code class="language-plaintext highlighter-rouge">changes</code> before any command that operates on a sequence of documents, as long as that command doesn’t consume the entire sequence. (For instance, <code class="language-plaintext highlighter-rouge">count</code> and <code class="language-plaintext highlighter-rouge">orderBy</code> cannot come after the <code class="language-plaintext highlighter-rouge">changes</code> command.)</p>
<p>Suppose you have a chat application with multiple clients posting messages to different chat rooms. You can create feeds that subscribe to messages posted to a specific room:</p>
<div class="language-java highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="n">r</span><span class="o">.</span><span class="na">table</span><span class="o">(</span><span class="s">"messages"</span><span class="o">).</span><span class="na">filter</span><span class="o">(</span>
<span class="n">row</span> <span class="o">-&gt;</span> <span class="n">row</span><span class="o">.</span><span class="na">g</span><span class="o">(</span><span class="s">"room_id"</span><span class="o">).</span><span class="na">eq</span><span class="o">(</span><span class="n">ROOM_ID</span><span class="o">)</span>
<span class="o">).</span><span class="na">changes</span><span class="o">().</span><span class="na">run</span><span class="o">(</span><span class="n">conn</span><span class="o">);</span>
</code></pre></div></div>
<p>You can also use more complicated expressions. Let’s say you have a table <code class="language-plaintext highlighter-rouge">scores</code> that contains the latest game score for every user of your game. You can create a feed of all games where a user beats their previous score, and get only the new value:</p>
<div class="language-java highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="n">r</span><span class="o">.</span><span class="na">table</span><span class="o">(</span><span class="s">"scores"</span><span class="o">).</span><span class="na">changes</span><span class="o">().</span><span class="na">filter</span><span class="o">(</span>
<span class="n">change</span> <span class="o">-&gt;</span> <span class="n">change</span><span class="o">.</span><span class="na">g</span><span class="o">(</span><span class="s">"new_val"</span><span class="o">).</span><span class="na">g</span><span class="o">(</span><span class="s">"score"</span><span class="o">).</span><span class="na">gt</span><span class="o">(</span><span class="n">change</span><span class="o">.</span><span class="na">g</span><span class="o">(</span><span class="s">"old_val"</span><span class="o">).</span><span class="na">g</span><span class="o">(</span><span class="s">"score"</span><span class="o">))</span>
<span class="o">).</span><span class="na">g</span><span class="o">(</span><span class="s">"new_val"</span><span class="o">).</span><span class="na">run</span><span class="o">(</span><span class="n">conn</span><span class="o">);</span>
</code></pre></div></div>
<p>There are some limitations and caveats on chaining with changefeeds.</p>
<ul>
<li><code class="language-plaintext highlighter-rouge">min</code>, <code class="language-plaintext highlighter-rouge">max</code> and <code class="language-plaintext highlighter-rouge">orderBy</code> must be used with indexes.</li>
<li><code class="language-plaintext highlighter-rouge">orderBy</code> requires <code class="language-plaintext highlighter-rouge">limit</code>; neither command works by itself.</li>
<li><code class="language-plaintext highlighter-rouge">orderBy</code> must be used with a <a href="/docs/secondary-indexes/java">secondary index</a> or the primary index; it cannot be used with an unindexed field.</li>
<li>You cannot use changefeeds after <a href="/api/java/concat_map">concatMap</a> or other transformations whose results cannot be pushed to the shards.</li>
<li>You cannot apply a <code class="language-plaintext highlighter-rouge">filter</code> after <code class="language-plaintext highlighter-rouge">orderBy.limit</code> in a changefeed.</li>
<li>Transformations are applied before changes are calculated.</li>
</ul>
<h1 id="including-state-changes">Including state changes</h1>
<p>The <code class="language-plaintext highlighter-rouge">include_states</code> optional argument to <code class="language-plaintext highlighter-rouge">changes</code> allows you to receive extra “status” documents in changefeed streams. These can allow your application to distinguish between initial values returned at the start of a stream and subsequent changes. Read the <a href="/api/java/changes">changes</a> API documentation for a full explanation and example.</p>
<h1 id="including-initial-values">Including initial values</h1>
<p>By specifying <code class="language-plaintext highlighter-rouge">true</code> to the <code class="language-plaintext highlighter-rouge">include_initial</code> optional argument, the changefeed stream will start with the current contents of the table or selection being monitored. The initial results will have <code class="language-plaintext highlighter-rouge">new_val</code> fields, but no <code class="language-plaintext highlighter-rouge">old_val</code> fields, so it’s easy to distinguish them from change events.</p>
<p>If an initial result for a document has been sent and a change is made to that document that would move it to the unsent part of the result set (for instance, a changefeed monitors the top 100 posters, the first 50 have been sent, and poster 48 has become poster 52), an “uninitial” notification will be sent, with an <code class="language-plaintext highlighter-rouge">old_val</code> field but no <code class="language-plaintext highlighter-rouge">new_val</code> field. This is distinct from a delete change event, which would have a <code class="language-plaintext highlighter-rouge">new_val</code> of <code class="language-plaintext highlighter-rouge">null</code>. (In the top 100 posters example, that could indicate the poster has been deleted, or has dropped out of the top 100.)</p>
<p>If you specify <code class="language-plaintext highlighter-rouge">true</code> for both <code class="language-plaintext highlighter-rouge">include_states</code> and <code class="language-plaintext highlighter-rouge">include_initial</code>, the changefeed stream will start with a <code class="language-plaintext highlighter-rouge">{"state": "initializing"}</code> status document, followed by initial values. A <code class="language-plaintext highlighter-rouge">{"state": "ready"}</code> status document will be sent when all the initial values have been sent.</p>
<h1 id="including-result-types">Including result types</h1>
<p>The <code class="language-plaintext highlighter-rouge">include_types</code> optional argument adds a third field, <code class="language-plaintext highlighter-rouge">type</code>, to each result sent. The string values for <code class="language-plaintext highlighter-rouge">type</code> are largely self-explanatory:</p>
<ul>
<li><code class="language-plaintext highlighter-rouge">add</code>: a new value added to the result set.</li>
<li><code class="language-plaintext highlighter-rouge">remove</code>: an old value removed from the result set.</li>
<li><code class="language-plaintext highlighter-rouge">change</code>: an existing value changed in the result set.</li>
<li><code class="language-plaintext highlighter-rouge">initial</code>: an initial value notification.</li>
<li><code class="language-plaintext highlighter-rouge">uninitial</code>: an uninitial value notification.</li>
<li><code class="language-plaintext highlighter-rouge">state</code>: a status document from <code class="language-plaintext highlighter-rouge">include_states</code>.</li>
</ul>
<p>Including the <code class="language-plaintext highlighter-rouge">type</code> field can simplify code that handles different cases for changefeed results.</p>
<h1 id="handling-latency">Handling latency</h1>
<p>Depending on how fast your application makes changes to monitored data and how fast it processes change notifications, it’s possible that more than one change will happen between calls to the <code class="language-plaintext highlighter-rouge">changes</code> command. You can control what happens in that case with the <code class="language-plaintext highlighter-rouge">squash</code> optional argument.</p>
<p>By default, if more than one change occurs between invocations of <code class="language-plaintext highlighter-rouge">changes</code>, your application will receive a single change object whose <code class="language-plaintext highlighter-rouge">new_val</code> will incorporate <em>all</em> the changes to the data. Suppose three updates occurred to a monitored document between <code class="language-plaintext highlighter-rouge">change</code> reads:</p>
<table>
<thead>
<tr>
<th>Change</th>
<th>Data</th>
</tr>
</thead>
<tbody>
<tr>
<td>Initial state (<code class="language-plaintext highlighter-rouge">old_val</code>)</td>
<td>{ name: “Fred”, admin: true }</td>
</tr>
<tr>
<td>update({name: “George”})</td>
<td>{ name: “George”, admin: true }</td>
</tr>
<tr>
<td>update({admin: false})</td>
<td>{ name: “George”, admin: false }</td>
</tr>
<tr>
<td>update({name: “Jay”})</td>
<td>{ name: “Jay”, admin: false }</td>
</tr>
<tr>
<td><code class="language-plaintext highlighter-rouge">new_val</code></td>
<td>{ name: “Jay”, admin: false }</td>
</tr>
</tbody>
</table>
<p>Your application would by default receive the object as it existed in the database after the <em>most recent</em> change. The previous two updates would be “squashed” into the third.</p>
<p>If you wanted to receive <em>all</em> the changes, including the interim states, you could do so by passing <code class="language-plaintext highlighter-rouge">squash: false</code>. The server will buffer up to 100,000 changes. (This number can be changed with the <code class="language-plaintext highlighter-rouge">changefeed_queue_size</code> optional argument.)</p>
<p>A third option is to specify how many seconds to wait between squashes. Passing <code class="language-plaintext highlighter-rouge">squash: 5</code> to the <code class="language-plaintext highlighter-rouge">changes</code> command tells RethinkDB to squash changes together every five seconds. Depending on your application’s use case, this might reduce the load on the server. A number passed to <code class="language-plaintext highlighter-rouge">squash</code> may be a float. Note that the requested interval is not guaranteed, but is rather a best effort.</p>
<p><strong>Note:</strong> Changefeeds ignore the <code class="language-plaintext highlighter-rouge">read_mode</code> flag to <code class="language-plaintext highlighter-rouge">run</code>, and always behave as if it is set to <code class="language-plaintext highlighter-rouge">single</code> (i.e., the values they return are in memory on the primary replica, but have not necessarily been written to disk yet). For more details read <a href="/docs/consistency">Consistency guarantees</a>.</p>
<h1 id="scaling-considerations">Scaling considerations</h1>
<p>Changefeeds perform well as they scale, although they create extra intracluster messages in proportion to the number of servers with open feed connections on each write. This can be mitigated by running a RethinkDB proxy server (the <code class="language-plaintext highlighter-rouge">rethinkdb proxy</code> startup option); read <a href="/docs/sharding-and-replication/#running-a-proxy-node">Running a proxy node</a> for details.</p>
<p>Since changefeeds are unidirectional with no acknowledgement returned from clients, they cannot guarantee delivery. If you need real-time updating with delivery guarantees, consider using a model that distributes to the clients through a message broker such as <a href="https://www.rabbitmq.com/">RabbitMQ</a>.</p>
<h1 id="read-more">Read more</h1>
<ul>
<li>The <a href="/api/java/changes">changes</a> command API reference</li>
<li><a href="/docs/introduction-to-reql/">Introduction to ReQL</a></li>
<li><a href="/docs/data-types/">ReQL data types</a></li>
</ul>
</div>
</section>
</div>
</section>
<!-- site footer -->
<footer class="site-footer">
<div class="site-container">
<section class="get-started">
<h1>Join 100,000 developers</h1>
<p>Install in seconds on Linux or OS X.</p>
<a class="button small install" href="/install">Install</a>
<a class="button small outlined learn-more" href="/faq">Learn more</a>
<hr>
<section class="contact-info">
<p class="irc">#rethinkdb on freenode</p>
</section>
</section>
<ul class="footer-links">
<li>
<header>
<span>Getting started</span>
</header>
<ul>
<li><a href="/faq">FAQ</a></li>
<li><a href="/install">Install</a></li>
<li><a href="/docs/quickstart">Quickstart</a></li>
<li><a href="/docs/guide">Tutorial</a></li>
</ul>
</li>
<li>
<header>
<span>Documentation</span>
</header>
<ul>
<li><a href="/api">API</a></li>
<li><a href="/docs/introduction-to-reql/">Intro to ReQL</a></li>
<li><a href="/docs/cookbook">Cookbook</a></li>
<li><a href="/docs/architecture">Architecture</a></li>
</ul>
</li>
<li>
<header>
<span>Community</span>
</header>
<ul>
<li><a href="/community">Get Involved</a></li>
<li><a href="https://github.com/rethinkdb/rethinkdb">GitHub</a></li>
<li><a href="https://twitter.com/rethinkdb">Twitter</a></li>
<li><a href="https://join.slack.com/t/rethinkdb/shared_invite/enQtNzAxOTUzNTk1NzMzLWY5ZTA0OTNmMWJiOWFmOGVhNTUxZjQzODQyZjIzNjgzZjdjZDFjNDg1NDY3MjFhYmNhOTY1MDVkNDgzMWZiZWM">Slack</a></li>
</ul>
</li>
<li>
<header>
<span>RethinkDB</span>
</header>
<ul>
<li><a href="/blog/">Blog</a></li>
</ul>
</li>
</ul>
<p class="github-footer"><i class="icon-github-footer"></i><a href="https://github.com/rethinkdb/rethinkdb">Star this project</a> on GitHub.</p>
</div>
</footer>
</div>
<!-- algolia -->
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/docsearch.js@2/dist/cdn/docsearch.min.css" />
<script type="text/javascript" src="https://cdn.jsdelivr.net/npm/docsearch.js@2/dist/cdn/docsearch.min.js"></script>
<script type="text/javascript"> docsearch({
apiKey: 'a33e586ddcc6f914b6c3eb48760d6235',
indexName: 'rethinkdb',
inputSelector: '#wrapper > section > div > section.docs-sidebar-left > div > input',
debug: false
});
</script>
<script type="text/javascript">
var _gaq = _gaq || [];
_gaq.push(['_setAccount', 'UA-9922584-1']);
_gaq.push(['_trackPageview']);
(function() {
var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
})();
</script>
</body>
</html>

193
research/cultnet-distributed-database/summary.md Normal file
View file

@ -0,0 +1,193 @@
# CultNet Distributed Database Research Summary
Purpose: ground CultNet's distributed realtime database design in prior art
before adding more machinery. The goal is a coherent local-first mesh database
over CultCache, not a hand-rolled distributed-systems costume party.
## Sources Stored Here
- `hashgraph-swirlds-tr-2016-01.pdf`
- Source: <https://www.swirlds.com/downloads/SWIRLDS-TR-2016-01.pdf>
- Topic: gossip-about-gossip, virtual voting, fair ordering, asynchronous BFT.
- `raft-extended.html`
- Source: <https://yygcode.com/papers/consensus-raft-extended-version.html>
- Topic: understandable consensus, leader election, log replication, safety.
- `raft-extended.pdf`
- Source: <https://web.stanford.edu/~ouster/cgi-bin/papers/raft-extended.pdf>
- Topic: canonical Raft paper copy.
- `dynamo-amazon-science.html`
- Source: <https://www.amazon.science/publications/dynamo-amazons-highly-available-key-value-store>
- Topic: Dynamo publication page and abstract.
- `dynamo-sosp2007.pdf`
- Source: <https://web.stanford.edu/class/cs244/papers/amazon-dynamo-sosp2007.pdf>
- Topic: highly available key-value store, consistent hashing, versioning,
quorums, hinted handoff, anti-entropy.
- `swim.pdf`
- Source: <https://www.cs.cornell.edu/projects/quicksilver/public_pdfs/SWIM.pdf>
- Topic: scalable weakly consistent process-group membership.
- `rethinkdb-changefeeds.html`
- Source: <https://rethinkdb.com/docs/changefeeds/java/>
- Topic: realtime query/changefeed ergonomics.
- `firebase-realtime-offline.html`
- Source: <https://firebase.google.com/docs/database/web/offline-capabilities>
- Topic: offline behavior, presence, server-side disconnect operations.
- `crdt-arxiv-1805.06358.html`
- Source: <https://arxiv.org/abs/1805.06358>
- Topic: conflict-free replicated data types and deterministic convergence.
## Design Takeaways
### RethinkDB
Keep: changefeeds as the product feel. Subscribers should receive document
changes continuously, with enough old/new context to render, reconcile, or
debug. Point subscriptions and filtered subscriptions are both first-class.
CultNet implication: database subscriptions should be explicit schema-v0
messages. The server should stream raw document changes that clients can apply
through the same CultCache reconciliation path as snapshots.
### Firebase Realtime Database
Keep: realtime sync and local/offline ergonomics. Presence and disconnect
behavior are database features, not application afterthoughts.
Defer: full offline writes. CultNet should not pretend arbitrary offline
multi-writer changes merge safely. Offline/local-first behavior needs declared
per-document conflict policy.
CultNet implication: add presence/disconnect records later as ordinary
CultCache documents with server authority, not as hidden transport state.
### Raft
Keep: the understandable authority model. One leader/primary owns ordered
writes for a shard. Decompose the problem into ownership, log/mutation
replication, and safety.
Defer: full automatic leader election and replicated logs until shard catalogs
and explicit epochs exist.
CultNet implication: the current primary-shard policy is the correct first
foundation. Every write should either hit the primary, be forwarded to the
primary, or fail with routing information.
### Dynamo
Keep: partitioning, replication metadata, vector-ish causality, hinted recovery,
anti-entropy, and application-visible conflicts.
Reject for now: "always writeable" semantics. That is attractive, expensive,
and easy to lie about.
CultNet implication: shard descriptors should grow into a shard catalog with
owner runtime id, epoch, schema/key ranges, and later replica/preference-list
metadata. Conflicts must surface as data, not disappear into last-writer-wins
unless a document explicitly chose that policy.
### SWIM
Keep: gossip-shaped membership once the mesh grows beyond a small static
cluster. SWIM separates failure detection from membership dissemination and
keeps per-node message load stable.
Defer: membership implementation until there is a shard catalog to disseminate.
CultNet implication: do not build peer discovery as a side-channel registry.
When it arrives, it should update membership and shard-catalog state together.
### CRDTs
Keep: CRDTs for documents whose merge law is explicit and deterministic.
Reject: generic automatic merge for arbitrary domain objects. That is a
language cop with a nicer hat.
CultNet implication: CRDT support belongs in schema metadata or document
contract metadata. A document type can opt into a known merge strategy; the
default distributed write policy remains primary authority.
### Hashgraph
Keep: gossip-about-gossip as an idea for compact event provenance and possible
fair ordering research. Virtual voting is interesting when every member sees
the same gossip history.
Reject for now: adopting hashgraph consensus as CultNet's core. The public
ledger/crypto smell is not the main issue; the issue is that CultNet does not
currently need asynchronous Byzantine total ordering to become a good mesh
database. Dragging that in now would make the machine harder to explain.
CultNet implication: if we later need decentralized event ordering, use a
dedicated design pass. Do not smuggle hashgraph metadata into ordinary document
replication because it sounds powerful.
## Current CultNet Direction
The coherent path is:
1. Primary-shard authority.
2. Explicit schema-v0 snapshot, put, delete, subscribe, unsubscribe, and change
messages.
3. Shard catalog exchange with owner runtime id and epoch.
4. Optional forwarding from non-owner nodes to owners.
5. Membership/failure detection, likely SWIM-shaped.
6. Replication and failover, likely Raft-shaped per shard.
7. Optional CRDT policies for document types that deserve offline multi-writer
semantics.
8. Optional gossip-history research if fair decentralized ordering becomes a
real requirement.
## Live Invariants
- CultCache owns document identity, schema compatibility, local indexes, and
reconciliation.
- CultNet owns transport, shard authority, subscriptions, and remote mutation
delivery.
- Raw wire records must pass through `CultNetDatabase` before mutating local
cache state.
- A write without authority is rejected or explicitly forwarded. It is never
silently applied.
- Realtime change streams publish domain changes, not storage implementation
details.
- Conflict policy must be declared before conflicting writes are accepted.
## Current Cut
Shard catalog exchange now exists:
- `cultnet.shard_catalog_request.v0`
- `cultnet.shard_catalog_response.v0`
- per-shard id, owner runtime id, epoch, schema ids, key prefix/range
- stale epoch rejection on writes
- routing error that tells clients where the primary lives when known
- injectable non-primary write forwarding policy
- concrete schema-v0 write forwarder for `cultnet://host:port` primary
endpoints
- in-memory per-shard mutation logs and catch-up by last seen sequence
- wire-level shard mutation log catch-up with raw put/delete entries
- replica-side application of shard log responses with epoch checks, gap
rejection, and idempotent replay
- background shard log replicator plus schema-v0 fetcher for primary endpoints
- restart-safe replica cursor store with a local MessagePack file
implementation
- client authority scopes for predicted local input documents
- predicted/reconciled change events for client-side prediction
- simulation witness observations and deterministic consensus candidates
- schema-v0 simulation observation and candidate messages
- reactive observation hub for witness gossip and consensus candidates
- server-side observation bridge for observation messages and candidate replies
## Next Cut
Build durable authoritative mutation logs:
- persist per-shard primary logs
- define the snapshot fallback boundary for compacted history
- return explicit resync-required responses when requested history is gone
- then add simulation-frame rollback and resimulation helpers on top of
predicted input streams
- add peer-to-peer fanout for observation gossip and candidate propagation
That is the next foundation needed before membership, Raft-style failover, or
CRDT policy work can be honest.

BIN
research/cultnet-distributed-database/swim.pdf Normal file
View file

Binary file not shown.

617
src/GameCult.Caching.MessagePack.Generator/CultDocumentMessagePackGenerator.cs Normal file
View file

@ -0,0 +1,617 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Text;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp.Syntax;
using Microsoft.CodeAnalysis.Text;
namespace GameCult.Caching.MessagePack.Generator
{
/// <summary>
/// Emits MessagePack payload serializers for CultCache document types.
/// </summary>
[Generator]
public sealed class CultDocumentMessagePackGenerator : IIncrementalGenerator
{
/// <summary>
/// Registers the generator pipeline.
/// </summary>
public void Initialize(IncrementalGeneratorInitializationContext context)
{
var documentShapes = context.SyntaxProvider
.CreateSyntaxProvider(
static (node, _) => node is ClassDeclarationSyntax cds && cds.AttributeLists.Count > 0,
static (ctx, _) => BuildDocumentShape(ctx))
.Where(static shape => shape != null)
.Select(static (shape, _) => shape!);
var compilationAndShapes = context.CompilationProvider.Combine(documentShapes.Collect());
context.RegisterSourceOutput(
compilationAndShapes,
static (spc, pair) => EmitProvider(spc, pair.Left, pair.Right));
}
private static DocumentShape? BuildDocumentShape(GeneratorSyntaxContext context)
{
if (context.SemanticModel.GetDeclaredSymbol(context.Node) is not INamedTypeSymbol typeSymbol)
{
return null;
}
var documentAttribute = GetAttribute(typeSymbol, "GameCult.Caching.CultDocumentAttribute");
if (documentAttribute == null ||
documentAttribute.ConstructorArguments.Length < 2 ||
documentAttribute.ConstructorArguments[0].Value is not string schemaName ||
documentAttribute.ConstructorArguments[1].Value is not string schemaVersion)
{
return null;
}
var members = DiscoverMembers(typeSymbol);
var nameMember = members.FirstOrDefault(member => member.IsName);
return new DocumentShape(
typeSymbol.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat),
schemaName,
schemaVersion,
GetAttribute(typeSymbol, "GameCult.Caching.CultGlobalAttribute") != null,
nameMember?.Name,
nameMember?.AccessorMethodName,
members.Where(member => member.IndexAlias != null)
.Select(member => new IndexAccessorShape(member.IndexAlias!, member.IndexAccessorMethodName!))
.ToImmutableArray(),
members.ToImmutableArray());
}
private static ImmutableArray<MemberShape> DiscoverMembers(INamedTypeSymbol typeSymbol)
{
var candidates = new List<MemberShapeSeed>();
foreach (var member in typeSymbol.GetMembers())
{
if (member is IFieldSymbol field)
{
if (field.IsStatic || field.DeclaredAccessibility != Accessibility.Public || IsIgnored(field))
{
continue;
}
candidates.Add(MemberShapeSeed.From(field, field.Type));
continue;
}
if (member is IPropertySymbol property)
{
if (property.IsStatic ||
property.IsIndexer ||
property.DeclaredAccessibility != Accessibility.Public ||
property.GetMethod == null ||
property.SetMethod == null ||
IsIgnored(property))
{
continue;
}
candidates.Add(MemberShapeSeed.From(property, property.Type));
}
}
var explicitMembers = candidates
.Where(member => member.ExplicitSlot.HasValue)
.OrderBy(member => member.ExplicitSlot!.Value)
.ThenBy(member => member.SortKey, StringComparer.Ordinal)
.ToArray();
var implicitMembers = candidates
.Where(member => !member.ExplicitSlot.HasValue)
.OrderBy(member => member.SortKey, StringComparer.Ordinal)
.ToArray();
var nextSlot = explicitMembers.Length == 0 ? 0 : explicitMembers.Max(member => member.ExplicitSlot!.Value) + 1;
var shaped = new List<MemberShape>(candidates.Count);
foreach (var member in explicitMembers)
{
shaped.Add(member.WithSlot(member.ExplicitSlot!.Value));
}
foreach (var member in implicitMembers)
{
shaped.Add(member.WithSlot(nextSlot++));
}
return shaped
.OrderBy(member => member.Slot)
.ToImmutableArray();
}
private static bool IsIgnored(ISymbol member)
{
return GetAttribute(member, "MessagePack.IgnoreMemberAttribute") != null;
}
private static AttributeData? GetAttribute(ISymbol symbol, string fullyQualifiedName)
{
return symbol.GetAttributes().FirstOrDefault(attribute =>
attribute.AttributeClass?.ToDisplayString() == fullyQualifiedName);
}
private static int? GetExplicitSlot(ISymbol member)
{
var keyAttribute = GetAttribute(member, "MessagePack.KeyAttribute");
if (keyAttribute == null || keyAttribute.ConstructorArguments.Length == 0)
{
return null;
}
var argument = keyAttribute.ConstructorArguments[0];
return argument.Kind == TypedConstantKind.Primitive && argument.Value is int slot
? slot
: null;
}
private static void EmitProvider(SourceProductionContext context, Compilation compilation, ImmutableArray<DocumentShape> shapes)
{
var documents = shapes
.Where(shape => shape != null)
.GroupBy(shape => shape.DocumentTypeName, StringComparer.Ordinal)
.Select(group => group.First())
.OrderBy(shape => shape.DocumentTypeName, StringComparer.Ordinal)
.ToArray();
if (documents.Length == 0)
{
return;
}
var canEmitPayloadCodecs =
compilation.GetTypeByMetadataName("MessagePack.MessagePackWriter") != null &&
compilation.GetTypeByMetadataName("GameCult.Caching.MessagePack.CultDocumentMessagePackSerialization") != null;
var providerName = $"GeneratedCultDocumentMetadataProvider_{SanitizeIdentifier(compilation.AssemblyName ?? "GameCult")}";
var builder = new StringBuilder();
builder.AppendLine("// <auto-generated/>");
builder.AppendLine("#nullable enable");
builder.AppendLine($"[assembly: global::GameCult.Caching.CultGeneratedDocumentMetadataProviderAttribute(typeof(global::GameCult.Caching.Generated.{providerName}))]");
builder.AppendLine("namespace GameCult.Caching.Generated");
builder.AppendLine("{");
builder.AppendLine($" internal sealed class {providerName} : global::GameCult.Caching.ICultGeneratedDocumentMetadataProvider");
builder.AppendLine(" {");
builder.AppendLine(" private static readonly global::GameCult.Caching.CultGeneratedDocumentDefinition[] s_documents =");
builder.AppendLine(" {");
foreach (var document in documents)
{
builder.AppendLine(" new global::GameCult.Caching.CultGeneratedDocumentDefinition(");
builder.AppendLine($" typeof({document.DocumentTypeName}),");
builder.AppendLine($" \"{Escape(document.SchemaName)}\",");
builder.AppendLine($" \"{Escape(document.SchemaVersion)}\",");
builder.AppendLine($" {(document.IsGlobal ? "true" : "false")},");
builder.AppendLine(document.NameMember == null
? " null,"
: $" \"{Escape(document.NameMember)}\",");
builder.AppendLine(document.NameAccessorMethodName == null
? " null,"
: $" {document.NameAccessorMethodName},");
builder.AppendLine(canEmitPayloadCodecs
? $" {document.PayloadSerializerMethodName},"
: " null,");
builder.AppendLine(canEmitPayloadCodecs
? $" {document.PayloadDeserializerMethodName},"
: " null,");
builder.AppendLine(" new global::GameCult.Caching.CultGeneratedDocumentIndexAccessor[]");
builder.AppendLine(" {");
foreach (var indexAccessor in document.IndexAccessors)
{
builder.AppendLine($" new global::GameCult.Caching.CultGeneratedDocumentIndexAccessor(\"{Escape(indexAccessor.Alias)}\", {indexAccessor.AccessorMethodName}),");
}
builder.AppendLine(" },");
builder.AppendLine(" new global::GameCult.Caching.CultGeneratedDocumentMemberDefinition[]");
builder.AppendLine(" {");
foreach (var member in document.Members.OrderBy(member => member.Slot))
{
builder.AppendLine(" new global::GameCult.Caching.CultGeneratedDocumentMemberDefinition(");
builder.AppendLine($" \"{Escape(member.Name)}\",");
builder.AppendLine($" {member.Slot},");
builder.AppendLine($" \"{Escape(member.SchemaTypeName)}\",");
builder.AppendLine($" {(member.IsReference ? "true" : "false")},");
builder.AppendLine($" {(member.IsMany ? "true" : "false")},");
builder.AppendLine(member.TargetSchemaName == null
? " null,"
: $" \"{Escape(member.TargetSchemaName)}\",");
builder.AppendLine($" {(member.IsName ? "true" : "false")},");
builder.AppendLine(member.IndexAlias == null
? " null),"
: $" \"{Escape(member.IndexAlias)}\"),");
}
builder.AppendLine(" }),");
}
builder.AppendLine(" };");
builder.AppendLine();
builder.AppendLine(" public global::System.Collections.Generic.IEnumerable<global::GameCult.Caching.CultGeneratedDocumentDefinition> GetDocumentDefinitions()");
builder.AppendLine(" {");
builder.AppendLine(" return s_documents;");
builder.AppendLine(" }");
builder.AppendLine();
foreach (var document in documents)
{
if (canEmitPayloadCodecs)
{
EmitPayloadSerializer(builder, document);
EmitPayloadDeserializer(builder, document);
}
foreach (var member in document.Members.Where(member => member.AccessorMethodName != null))
{
builder.AppendLine($" private static string? {member.AccessorMethodName}(object document)");
builder.AppendLine(" {");
builder.AppendLine(member.CanBeNull
? $" return (({document.DocumentTypeName})document).{member.Name}?.ToString();"
: $" return (({document.DocumentTypeName})document).{member.Name}.ToString();");
builder.AppendLine(" }");
builder.AppendLine();
}
foreach (var member in document.Members.Where(member => member.IndexAccessorMethodName != null))
{
builder.AppendLine($" private static string {member.IndexAccessorMethodName}(object document)");
builder.AppendLine(" {");
builder.AppendLine(member.CanBeNull
? $" return (({document.DocumentTypeName})document).{member.Name}?.ToString() ?? string.Empty;"
: $" return (({document.DocumentTypeName})document).{member.Name}.ToString();");
builder.AppendLine(" }");
builder.AppendLine();
}
}
builder.AppendLine(" }");
builder.AppendLine("}");
context.AddSource($"{providerName}.g.cs", SourceText.From(builder.ToString(), Encoding.UTF8));
}
private static void EmitPayloadSerializer(StringBuilder builder, DocumentShape document)
{
builder.AppendLine($" private static byte[] {document.PayloadSerializerMethodName}(object document)");
builder.AppendLine(" {");
builder.AppendLine($" var typed = ({document.DocumentTypeName})document;");
builder.AppendLine(" var options = global::GameCult.Caching.MessagePack.CultDocumentMessagePackSerialization.Options;");
builder.AppendLine(" var buffer = new global::System.Buffers.ArrayBufferWriter<byte>();");
builder.AppendLine(" var writer = new global::MessagePack.MessagePackWriter(buffer);");
builder.AppendLine($" writer.WriteArrayHeader({document.Members.Length});");
foreach (var member in document.Members.OrderBy(member => member.Slot))
{
builder.AppendLine($" global::MessagePack.FormatterResolverExtensions.GetFormatterWithVerify<{member.TypeSyntaxName}>(options.Resolver).Serialize(ref writer, typed.{member.Name}, options);");
}
builder.AppendLine(" writer.Flush();");
builder.AppendLine(" return buffer.WrittenSpan.ToArray();");
builder.AppendLine(" }");
builder.AppendLine();
}
private static void EmitPayloadDeserializer(StringBuilder builder, DocumentShape document)
{
builder.AppendLine($" private static object {document.PayloadDeserializerMethodName}(byte[] payload)");
builder.AppendLine(" {");
builder.AppendLine(" var options = global::GameCult.Caching.MessagePack.CultDocumentMessagePackSerialization.Options;");
builder.AppendLine(" var reader = new global::MessagePack.MessagePackReader(payload);");
builder.AppendLine(" var count = reader.ReadArrayHeader();");
builder.AppendLine($" var value = new {document.DocumentTypeName}();");
foreach (var member in document.Members.OrderBy(member => member.Slot))
{
builder.AppendLine($" if (count > {member.Slot})");
builder.AppendLine(" {");
builder.AppendLine($" value.{member.Name} = global::MessagePack.FormatterResolverExtensions.GetFormatterWithVerify<{member.TypeSyntaxName}>(options.Resolver).Deserialize(ref reader, options);");
builder.AppendLine(" }");
}
builder.AppendLine($" for (var index = {document.Members.Length}; index < count; index++)");
builder.AppendLine(" {");
builder.AppendLine(" reader.Skip();");
builder.AppendLine(" }");
builder.AppendLine(" return value;");
builder.AppendLine(" }");
builder.AppendLine();
}
private static string Escape(string value)
{
return value
.Replace("\\", "\\\\")
.Replace("\"", "\\\"")
.Replace("\r", "\\r")
.Replace("\n", "\\n")
.Replace("\t", "\\t");
}
private static string SanitizeIdentifier(string value)
{
var builder = new StringBuilder(value.Length);
foreach (var character in value)
{
builder.Append(char.IsLetterOrDigit(character) ? character : '_');
}
if (builder.Length == 0 || !char.IsLetter(builder[0]) && builder[0] != '_')
{
builder.Insert(0, '_');
}
return builder.ToString();
}
private static string GetSchemaTypeName(ITypeSymbol typeSymbol)
{
if (typeSymbol is IArrayTypeSymbol arrayType)
{
return GetSchemaTypeName(arrayType.ElementType) + "[]";
}
if (typeSymbol is INamedTypeSymbol namedType && namedType.IsGenericType)
{
var baseName = GetNamedTypeBaseName(namedType.ConstructedFrom);
var arguments = string.Join(", ", namedType.TypeArguments.Select(GetSchemaTypeName));
return $"{baseName}<{arguments}>";
}
return typeSymbol.SpecialType switch
{
SpecialType.System_Boolean => "System.Boolean",
SpecialType.System_Byte => "System.Byte",
SpecialType.System_Char => "System.Char",
SpecialType.System_Decimal => "System.Decimal",
SpecialType.System_Double => "System.Double",
SpecialType.System_Int16 => "System.Int16",
SpecialType.System_Int32 => "System.Int32",
SpecialType.System_Int64 => "System.Int64",
SpecialType.System_Object => "System.Object",
SpecialType.System_SByte => "System.SByte",
SpecialType.System_Single => "System.Single",
SpecialType.System_String => "System.String",
SpecialType.System_UInt16 => "System.UInt16",
SpecialType.System_UInt32 => "System.UInt32",
SpecialType.System_UInt64 => "System.UInt64",
_ => GetNamedTypeBaseName((INamedTypeSymbol)typeSymbol)
};
}
private static string GetNamedTypeBaseName(INamedTypeSymbol typeSymbol)
{
var parts = new Stack<string>();
var current = typeSymbol;
while (current != null)
{
parts.Push(current.Name);
current = current.ContainingType;
}
var namespaceName = typeSymbol.ContainingNamespace?.ToDisplayString();
return string.IsNullOrWhiteSpace(namespaceName)
? string.Join(".", parts)
: namespaceName + "." + string.Join(".", parts);
}
private static bool CanBeNull(ITypeSymbol typeSymbol)
{
if (typeSymbol.NullableAnnotation == NullableAnnotation.Annotated)
{
return true;
}
if (typeSymbol.IsReferenceType)
{
return true;
}
return typeSymbol is INamedTypeSymbol namedType &&
namedType.IsGenericType &&
namedType.ConstructedFrom.SpecialType == SpecialType.System_Nullable_T;
}
private sealed class MemberShapeSeed
{
private MemberShapeSeed(
string name,
string schemaTypeName,
string typeSyntaxName,
int? explicitSlot,
string sortKey,
bool isName,
string? indexAlias,
bool isReference,
bool isMany,
bool canBeNull,
string? targetSchemaName)
{
Name = name;
SchemaTypeName = schemaTypeName;
TypeSyntaxName = typeSyntaxName;
ExplicitSlot = explicitSlot;
SortKey = sortKey;
IsName = isName;
IndexAlias = indexAlias;
IsReference = isReference;
IsMany = isMany;
CanBeNull = canBeNull;
TargetSchemaName = targetSchemaName;
}
public string Name { get; }
public string SchemaTypeName { get; }
public string TypeSyntaxName { get; }
public int? ExplicitSlot { get; }
public string SortKey { get; }
public bool IsName { get; }
public string? IndexAlias { get; }
public bool IsReference { get; }
public bool IsMany { get; }
public bool CanBeNull { get; }
public string? TargetSchemaName { get; }
public MemberShape WithSlot(int slot)
{
var accessorStem = SanitizeIdentifier(SortKey);
return new MemberShape(
Name,
slot,
SchemaTypeName,
TypeSyntaxName,
IsReference,
IsMany,
TargetSchemaName,
IsName,
IndexAlias,
CanBeNull,
IsName ? $"Access_{accessorStem}_Name" : null,
IndexAlias != null ? $"Access_{accessorStem}_Index" : null);
}
public static MemberShapeSeed From(ISymbol member, ITypeSymbol memberType)
{
var referenceAttribute = GetAttribute(member, "GameCult.Caching.CultReferenceAttribute");
var explicitTarget = referenceAttribute?.ConstructorArguments.Length > 0
? referenceAttribute.ConstructorArguments[0].Value as INamedTypeSymbol
: null;
var many = referenceAttribute?.ConstructorArguments.Length > 1 &&
referenceAttribute.ConstructorArguments[1].Value is bool manyValue &&
manyValue;
var targetType = ResolveReferenceTarget(memberType, explicitTarget);
var targetSchemaName = targetType == null
? null
: GetAttribute(targetType, "GameCult.Caching.CultDocumentAttribute")?.ConstructorArguments[0].Value as string;
var indexAttribute = GetAttribute(member, "GameCult.Caching.CultIndexAttribute");
var indexAlias = indexAttribute == null
? null
: indexAttribute.ConstructorArguments.Length == 0 || indexAttribute.ConstructorArguments[0].Value is not string alias || string.IsNullOrWhiteSpace(alias)
? member.Name
: alias;
return new MemberShapeSeed(
member.Name,
GetSchemaTypeName(memberType),
memberType.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat),
GetExplicitSlot(member),
BuildAccessorStem(member),
GetAttribute(member, "GameCult.Caching.CultNameAttribute") != null,
indexAlias,
targetType != null || referenceAttribute != null,
many,
CanBeNull(memberType),
targetSchemaName);
}
private static INamedTypeSymbol? ResolveReferenceTarget(ITypeSymbol memberType, INamedTypeSymbol? explicitTarget)
{
if (explicitTarget != null)
{
return explicitTarget;
}
return memberType is INamedTypeSymbol namedType &&
namedType.IsGenericType &&
namedType.Name == "CultRecordRef" &&
namedType.ContainingNamespace.ToDisplayString() == "GameCult.Caching"
? namedType.TypeArguments[0] as INamedTypeSymbol
: null;
}
private static string BuildAccessorStem(ISymbol member)
{
return member.ContainingType == null
? member.Name
: member.ContainingType.Name + "_" + member.Name;
}
}
private sealed class DocumentShape
{
public DocumentShape(
string documentTypeName,
string schemaName,
string schemaVersion,
bool isGlobal,
string? nameMember,
string? nameAccessorMethodName,
ImmutableArray<IndexAccessorShape> indexAccessors,
ImmutableArray<MemberShape> members)
{
DocumentTypeName = documentTypeName;
SchemaName = schemaName;
SchemaVersion = schemaVersion;
IsGlobal = isGlobal;
NameMember = nameMember;
NameAccessorMethodName = nameAccessorMethodName;
var safeStem = SanitizeIdentifier(documentTypeName.Replace("global::", string.Empty).Replace(".", "_"));
PayloadSerializerMethodName = $"Serialize_{safeStem}_Payload";
PayloadDeserializerMethodName = $"Deserialize_{safeStem}_Payload";
IndexAccessors = indexAccessors;
Members = members;
}
public string DocumentTypeName { get; }
public string SchemaName { get; }
public string SchemaVersion { get; }
public bool IsGlobal { get; }
public string? NameMember { get; }
public string? NameAccessorMethodName { get; }
public string PayloadSerializerMethodName { get; }
public string PayloadDeserializerMethodName { get; }
public ImmutableArray<IndexAccessorShape> IndexAccessors { get; }
public ImmutableArray<MemberShape> Members { get; }
}
private sealed class IndexAccessorShape
{
public IndexAccessorShape(string alias, string accessorMethodName)
{
Alias = alias;
AccessorMethodName = accessorMethodName;
}
public string Alias { get; }
public string AccessorMethodName { get; }
}
private sealed class MemberShape
{
public MemberShape(
string name,
int slot,
string schemaTypeName,
string typeSyntaxName,
bool isReference,
bool isMany,
string? targetSchemaName,
bool isName,
string? indexAlias,
bool canBeNull,
string? accessorMethodName,
string? indexAccessorMethodName)
{
Name = name;
Slot = slot;
SchemaTypeName = schemaTypeName;
TypeSyntaxName = typeSyntaxName;
IsReference = isReference;
IsMany = isMany;
TargetSchemaName = targetSchemaName;
IsName = isName;
IndexAlias = indexAlias;
CanBeNull = canBeNull;
AccessorMethodName = accessorMethodName;
IndexAccessorMethodName = indexAccessorMethodName;
}
public string Name { get; }
public int Slot { get; }
public string SchemaTypeName { get; }
public string TypeSyntaxName { get; }
public bool IsReference { get; }
public bool IsMany { get; }
public string? TargetSchemaName { get; }
public bool IsName { get; }
public string? IndexAlias { get; }
public bool CanBeNull { get; }
public string? AccessorMethodName { get; }
public string? IndexAccessorMethodName { get; }
}
}
}

264
src/GameCult.Caching.MessagePack.Generator/DatabaseEntryMessagePackGenerator.cs
View file

@ -1,264 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp.Syntax;
using Microsoft.CodeAnalysis.Text;
namespace GameCult.Caching.MessagePack.Generator
{
/// <summary>
/// Generates MessagePack formatters and resolver mappings for concrete <c>DatabaseEntry</c> types.
/// </summary>
[Generator]
public class DatabaseEntryMessagePackGenerator : IIncrementalGenerator
{
/// <summary>
/// Configures the incremental generator pipeline.
/// </summary>
/// <param name="ctx">The generator initialization context.</param>
public void Initialize(IncrementalGeneratorInitializationContext ctx)
{
// Select DatabaseEntry subclass declarations
var databaseEntryTypes = ctx.SyntaxProvider
.CreateSyntaxProvider(
static (s, _) => s is ClassDeclarationSyntax cds && cds.BaseList != null,
static (ctx, _) => ctx.SemanticModel.GetDeclaredSymbol(ctx.Node) as INamedTypeSymbol)
.Where(static s => s is not null)!
.Select((s, _) => s!)
.Where(symbol =>
{
if (symbol.IsAbstract) return false;
var baseType = symbol.BaseType;
while (baseType != null)
{
if (baseType.ToDisplayString() == "GameCult.Caching.DatabaseEntry")
return true;
baseType = baseType.BaseType;
}
return false;
});
// 2. Distinct types
var collected = databaseEntryTypes.Collect();
// 3. Register output
ctx.RegisterSourceOutput(collected, (spc, types) =>
{
//if (types.Length == 0) return;
var sbResolvers = new StringBuilder();
var sbFormatters = new StringBuilder();
sbResolvers.AppendLine("// <auto-generated/>");
sbResolvers.AppendLine("#nullable enable");
sbResolvers.AppendLine("using System;");
sbResolvers.AppendLine("using MessagePack.Formatters;");
sbResolvers.AppendLine("namespace GameCult.Caching.MessagePack");
sbResolvers.AppendLine("{");
sbResolvers.AppendLine(" static partial class GeneratedDatabaseEntryResolvers");
sbResolvers.AppendLine(" {");
sbResolvers.AppendLine(" public static IMessagePackFormatter<T>? GetFormatter<T>()");
sbResolvers.AppendLine(" {");
sbResolvers.AppendLine(" var t = typeof(T);");
foreach (var type in types)
{
var fullName = type.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat);
var formatterName = $"{Sanitize(type)}__MessagePackFormatter";
var fmtSrc = GenerateFormatterFor(type, formatterName);
sbFormatters.AppendLine(fmtSrc);
sbResolvers.AppendLine(
$" if (t == typeof({fullName})) return (IMessagePackFormatter<T>)(object){formatterName}.Instance;");
}
sbResolvers.AppendLine(" return null;");
sbResolvers.AppendLine(" }");
sbResolvers.AppendLine(" }");
sbResolvers.AppendLine("}");
spc.AddSource("GeneratedDatabaseEntryFormatters.g.cs", SourceText.From(sbFormatters.ToString(), Encoding.UTF8));
spc.AddSource("GeneratedDatabaseEntryResolvers.g.cs", SourceText.From(sbResolvers.ToString(), Encoding.UTF8));
});
}
private static string Sanitize(ISymbol symbol)
{
var s = symbol.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat);
return s.Replace(".", "_")
.Replace("global::", string.Empty)
.Replace("<", "_")
.Replace(">", "_")
.Replace(",", "_")
.Replace(" ", "_");
}
// Collect members, order them, generate the formatter
private static string GenerateFormatterFor(INamedTypeSymbol type, string formatterName)
{
var sb = new StringBuilder();
sb.AppendLine("// <auto-generated/>");
sb.AppendLine("#nullable enable");
sb.AppendLine("using System;");
sb.AppendLine("using MessagePack;");
sb.AppendLine("using MessagePack.Formatters;");
sb.AppendLine();
var ns = "GameCult.Caching.MessagePack";
sb.AppendLine($"namespace {ns}");
sb.AppendLine("{");
var typeFull = type.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat);
sb.AppendLine($" internal sealed class {formatterName} : IMessagePackFormatter<{typeFull}>");
sb.AppendLine(" {");
sb.AppendLine($" public static readonly {formatterName} Instance = new {formatterName}();");
sb.AppendLine($" private {formatterName}() {{ }}");
sb.AppendLine();
var members = CollectMembers(type);
var slots = AssignSlots(type, members);
var maxSlot = slots.Count == 0 ? -1 : slots.Max(m => m.slot);
// Serialize
sb.AppendLine($" public void Serialize(ref MessagePackWriter writer, {typeFull}? value, MessagePackSerializerOptions options)");
sb.AppendLine(" {");
sb.AppendLine(" if (value == null) { writer.WriteNil(); return; }");
sb.AppendLine($" writer.WriteArrayHeader({maxSlot + 1});");
for (int i = 0; i <= maxSlot; i++)
{
var member = slots.FirstOrDefault(m => m.slot == i);
if (member.name == null)
{
sb.AppendLine(" writer.WriteNil();");
continue;
}
var tname = member.typeSymbol.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat);
var access = $"value.{member.name}";
sb.AppendLine(
tname == "global::System.Guid"
? $" options.Resolver.GetFormatterWithVerify<System.Guid>().Serialize(ref writer, {access}, options);"
: $" options.Resolver.GetFormatterWithVerify<{tname}>().Serialize(ref writer, {access}, options);");
}
sb.AppendLine(" }");
// Deserialize
sb.AppendLine();
sb.AppendLine($" public {typeFull}? Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options)");
sb.AppendLine(" {");
sb.AppendLine(" if (reader.TryReadNil()) return null;");
sb.AppendLine($" var count = reader.ReadArrayHeader();");
sb.AppendLine($" var obj = new {typeFull}();");
sb.AppendLine(" for (var i = 0; i < count; i++)");
sb.AppendLine(" {");
sb.AppendLine(" switch (i)");
sb.AppendLine(" {");
foreach (var (slot, name, tsym) in slots)
{
var tname = tsym.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat);
sb.AppendLine($" case {slot}:");
sb.AppendLine(
tname == "global::System.Guid"
? $" obj.{name} = options.Resolver.GetFormatterWithVerify<System.Guid>().Deserialize(ref reader, options);"
: $" obj.{name} = options.Resolver.GetFormatterWithVerify<{tname}>().Deserialize(ref reader, options);");
sb.AppendLine(" break;");
}
sb.AppendLine(" default:");
sb.AppendLine(" reader.Skip();");
sb.AppendLine(" break;");
sb.AppendLine(" }");
sb.AppendLine(" }");
sb.AppendLine(" return obj;");
sb.AppendLine(" }");
sb.AppendLine(" }");
sb.AppendLine("}");
return sb.ToString();
}
private static List<(string name, ITypeSymbol typeSymbol, int? key, bool ignore)> CollectMembers(INamedTypeSymbol type)
{
var members = new List<(string, ITypeSymbol, int?, bool)>();
foreach (var f in type.GetMembers().OfType<IFieldSymbol>())
{
if (f.IsStatic) continue;
var ignore = f.GetAttributes().Any(a => a.AttributeClass?.ToDisplayString() == "MessagePack.IgnoreMemberAttribute");
var key = GetKeyValue(f);
members.Add((f.Name, f.Type, key, ignore));
}
foreach (var p in type.GetMembers().OfType<IPropertySymbol>())
{
if (p.IsStatic || p.SetMethod == null || p.GetMethod == null) continue;
var ignore = p.GetAttributes().Any(a => a.AttributeClass?.ToDisplayString() == "MessagePack.IgnoreMemberAttribute");
var key = GetKeyValue(p);
members.Add((p.Name, p.Type, key, ignore));
}
return members;
}
private static List<(int slot, string name, ITypeSymbol typeSymbol)> AssignSlots(
INamedTypeSymbol type,
List<(string name, ITypeSymbol typeSymbol, int? key, bool ignore)> members)
{
members = members.Where(m => !m.ignore).ToList();
var slots = new List<(int, string, ITypeSymbol)>();
var id = FindIdMember(type);
if (id != null)
{
slots.Add((0, id.Value.name, id.Value.typeSymbol));
members.RemoveAll(m => m.name == id.Value.name);
}
foreach (var member in members.Where(m => m.key.HasValue).OrderBy(m => m.key!.Value))
{
slots.Add((member.key!.Value, member.name, member.typeSymbol));
}
var nextSlot = slots.Count == 0 ? 0 : slots.Max(s => s.Item1) + 1;
foreach (var member in members.Where(m => !m.key.HasValue))
{
slots.Add((nextSlot, member.name, member.typeSymbol));
nextSlot++;
}
return slots.OrderBy(s => s.Item1).ToList();
}
private static int? GetKeyValue(ISymbol sym)
{
foreach (var attr in sym.GetAttributes())
{
var n = attr.AttributeClass?.ToDisplayString();
if (n == "MessagePack.KeyAttribute" || n == "MessagePack.Key")
{
if (attr.ConstructorArguments.Length >= 1 && attr.ConstructorArguments[0].Value is int i)
return i;
}
}
return null;
}
private static (string name, ITypeSymbol typeSymbol)? FindIdMember(INamedTypeSymbol type)
{
var cur = type;
while (cur != null)
{
foreach (var f in cur.GetMembers().OfType<IFieldSymbol>())
{
if ((f.Name == "ID") && f.Type.ToDisplayString() == "System.Guid")
return (f.Name, f.Type);
}
cur = cur.BaseType;
}
return null;
}
}
}

2
src/GameCult.Caching.MessagePack.Generator/GameCult.Caching.MessagePack.Generator.csproj
View file

@ -5,12 +5,12 @@
<Nullable>enable</Nullable>
<IsPackable>false</IsPackable>
<TargetFramework>netstandard2.0</TargetFramework>
<EnforceExtendedAnalyzerRules>true</EnforceExtendedAnalyzerRules>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.CodeAnalysis.Analyzers" Version="5.3.0" PrivateAssets="all" />
<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="5.3.0" PrivateAssets="all" />
<PackageReference Include="Microsoft.CodeAnalysis.CSharp.Workspaces" Version="5.3.0" PrivateAssets="all" />
</ItemGroup>
</Project>

95
src/GameCult.Caching.MessagePack/CultCacheEnvelope.cs
View file

@ -1,95 +0,0 @@
using System;
using System.Reflection;
namespace GameCult.Caching.MessagePack;
/// <summary>
/// Represents a canonical CultCache payload envelope for cross-runtime exchange.
/// </summary>
public sealed class CultCacheEnvelope
{
/// <summary>
/// Gets or sets the logical document key for the serialized entry.
/// </summary>
public string Key { get; set; } = string.Empty;
/// <summary>
/// Gets or sets the shared document type identifier for the serialized entry.
/// </summary>
public string Type { get; set; } = string.Empty;
/// <summary>
/// Gets or sets the raw MessagePack payload bytes for the entry.
/// </summary>
public byte[] Payload { get; set; } = Array.Empty<byte>();
/// <summary>
/// Gets or sets the RFC3339 timestamp associated with the serialized payload.
/// </summary>
public string StoredAt { get; set; } = string.Empty;
}
/// <summary>
/// Exposes canonical MessagePack payload helpers for <see cref="DatabaseEntry"/> values.
/// </summary>
public static class CultCacheEnvelopeSerialization
{
/// <summary>
/// Serializes a concrete entry to the canonical MessagePack payload bytes used by CultCache.
/// </summary>
public static byte[] SerializePayload(DatabaseEntry entry)
{
if (entry == null) throw new ArgumentNullException(nameof(entry));
var method = typeof(CultCacheEnvelopeSerialization)
.GetMethod(nameof(SerializeGeneric), BindingFlags.NonPublic | BindingFlags.Static)
?.MakeGenericMethod(entry.GetType())
?? throw new InvalidOperationException("Failed to locate typed CultCache envelope serializer.");
return (byte[])method.Invoke(null, [entry])!;
}
/// <summary>
/// Deserializes canonical MessagePack payload bytes into a concrete entry type.
/// </summary>
public static T DeserializePayload<T>(byte[] payload) where T : DatabaseEntry
{
if (payload == null) throw new ArgumentNullException(nameof(payload));
return DatabaseEntrySerialization.Deserialize<T>(payload);
}
/// <summary>
/// Deserializes canonical MessagePack payload bytes into an abstract <see cref="DatabaseEntry"/>.
/// </summary>
public static DatabaseEntry DeserializePayload(byte[] payload)
{
if (payload == null) throw new ArgumentNullException(nameof(payload));
return DatabaseEntrySerialization.Deserialize<DatabaseEntry>(payload);
}
/// <summary>
/// Creates a full exchange envelope for a concrete entry using the supplied metadata.
/// </summary>
public static CultCacheEnvelope CreateEnvelope(
DatabaseEntry entry,
string key,
string type,
string storedAt)
{
if (entry == null) throw new ArgumentNullException(nameof(entry));
if (string.IsNullOrWhiteSpace(key)) throw new ArgumentException("Envelope key must be non-empty.", nameof(key));
if (string.IsNullOrWhiteSpace(type)) throw new ArgumentException("Envelope type must be non-empty.", nameof(type));
if (string.IsNullOrWhiteSpace(storedAt)) throw new ArgumentException("Envelope storedAt must be non-empty.", nameof(storedAt));
return new CultCacheEnvelope
{
Key = key,
Type = type,
Payload = SerializePayload(entry),
StoredAt = storedAt
};
}
private static byte[] SerializeGeneric<T>(T entry) where T : DatabaseEntry
{
return DatabaseEntrySerialization.Serialize(entry);
}
}

90
src/GameCult.Caching.MessagePack/CultCacheMessagePack.cs Normal file
View file

@ -0,0 +1,90 @@
using System;
using System.Threading.Tasks;
namespace GameCult.Caching.MessagePack
{
/// <summary>
/// Options for opening a CultCache over the canonical single-file MessagePack backing store.
/// </summary>
public sealed class CultCacheOpenOptions
{
/// <summary>
/// Gets or sets the document registry to use. When omitted, the shared registry is used.
/// </summary>
public CultDocumentRegistry? Registry { get; set; }
/// <summary>
/// Gets or sets whether the cache should pull the existing snapshot during open.
/// </summary>
public bool PullOnOpen { get; set; } = true;
/// <summary>
/// Gets or sets whether the cache should flush attached stores when disposed.
/// </summary>
public bool FlushOnDispose { get; set; }
/// <summary>
/// Gets or sets whether the backing store should flush when disposed.
/// </summary>
public bool StoreFlushOnDispose { get; set; }
/// <summary>
/// Gets or sets an optional callback used to customize the cache before opening.
/// </summary>
public Action<CultCache>? ConfigureCache { get; set; }
/// <summary>
/// Gets or sets an optional callback used to customize the backing store before opening.
/// </summary>
public Action<SingleFileMessagePackBackingStore>? ConfigureStore { get; set; }
}
/// <summary>
/// Friendly MessagePack entrypoints for opening a CultCache with the canonical backing store.
/// </summary>
public static class CultCacheMessagePack
{
/// <summary>
/// Creates a cache with the canonical single-file MessagePack backing store attached.
/// </summary>
public static CultCache Create(string filePath, CultCacheOpenOptions? options = null)
{
if (string.IsNullOrWhiteSpace(filePath))
{
throw new ArgumentException("File path must be non-empty.", nameof(filePath));
}
options ??= new CultCacheOpenOptions();
var cache = new CultCache(options.Registry)
{
FlushAttachedStoresOnDispose = options.FlushOnDispose
};
options.ConfigureCache?.Invoke(cache);
var store = new SingleFileMessagePackBackingStore(filePath)
{
FlushOnDispose = options.StoreFlushOnDispose
};
options.ConfigureStore?.Invoke(store);
cache.AddBackingStore(store);
return cache;
}
/// <summary>
/// Creates a cache with the canonical single-file MessagePack backing store attached and optionally pulls the on-disk snapshot.
/// </summary>
public static async Task<CultCache> OpenAsync(string filePath, CultCacheOpenOptions? options = null)
{
options ??= new CultCacheOpenOptions();
var cache = Create(filePath, options);
if (options.PullOnOpen)
{
await cache.PullAllBackingStoresAsync().ConfigureAwait(false);
}
return cache;
}
}
}

457
src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs Normal file
View file

@ -0,0 +1,457 @@
using System;
using System.Buffers;
using GameCult.Caching;
using MessagePack;
using MessagePack.Formatters;
using MessagePack.Resolvers;
namespace GameCult.Caching.MessagePack;
/// <summary>
/// MessagePack resolver for CultCache-specific value types.
/// </summary>
public sealed class CultDocumentResolver : IFormatterResolver
{
/// <summary>
/// Gets the shared resolver instance.
/// </summary>
public static readonly CultDocumentResolver Instance = new();
private CultDocumentResolver() { }
/// <summary>
/// Gets a formatter for the requested type, when this resolver owns it.
/// </summary>
public IMessagePackFormatter<T>? GetFormatter<T>()
{
var type = typeof(T);
if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(CultRecordRef<>))
{
var formatterType = typeof(global::GameCult.Caching.CultRecordRefFormatter<>).MakeGenericType(type.GetGenericArguments()[0]);
return (IMessagePackFormatter<T>)Activator.CreateInstance(formatterType)!;
}
return null;
}
}
/// <summary>
/// MessagePack serialization helpers for CultCache documents and backing stores.
/// </summary>
public static class CultDocumentMessagePackSerialization
{
private const int PersistedRecordFieldCount = 4;
private const int SchemaCatalogEntryFieldCount = 7;
private const int SchemaCatalogMemberFieldCount = 8;
private const int StoreSnapshotFieldCount = 3;
/// <summary>
/// Gets the shared MessagePack serializer options for CultCache payloads.
/// </summary>
public static readonly MessagePackSerializerOptions Options =
MessagePackSerializerOptions.Standard
.WithResolver(CompositeResolver.Create(
CultDocumentResolver.Instance,
StandardResolver.Instance))
.WithSecurity(MessagePackSecurity.UntrustedData);
/// <summary>
/// Serializes a typed value with the CultCache MessagePack options.
/// </summary>
public static byte[] Serialize<T>(T value)
{
return MessagePackSerializer.Serialize(value, Options);
}
/// <summary>
/// Deserializes a typed value with the CultCache MessagePack options.
/// </summary>
public static T Deserialize<T>(byte[] payload)
{
return MessagePackSerializer.Deserialize<T>(payload, Options);
}
/// <summary>
/// Serializes a value whose document type is known at runtime.
/// </summary>
public static byte[] SerializeUntyped(object value, Type type)
{
if (value != null)
{
var descriptor = CultDocumentRegistry.Shared.GetRequired(type);
if (descriptor.GeneratedPayloadSerializer != null)
{
return descriptor.GeneratedPayloadSerializer(value);
}
}
return MessagePackSerializer.Serialize(type, value, Options);
}
/// <summary>
/// Deserializes a value whose document type is known at runtime.
/// </summary>
public static object DeserializeUntyped(Type type, byte[] payload)
{
var descriptor = CultDocumentRegistry.Shared.GetRequired(type);
if (descriptor.GeneratedPayloadDeserializer != null)
{
return descriptor.GeneratedPayloadDeserializer(payload);
}
return MessagePackSerializer.Deserialize(type, payload, Options)
?? throw new InvalidOperationException($"MessagePack returned null for Cult document type {type.FullName}.");
}
/// <summary>
/// Serializes one persisted store record.
/// </summary>
public static byte[] SerializePersistedRecord(CultPersistedRecord record)
{
var buffer = new global::System.Buffers.ArrayBufferWriter<byte>();
var writer = new MessagePackWriter(buffer);
WritePersistedRecord(ref writer, record);
writer.Flush();
return buffer.WrittenSpan.ToArray();
}
/// <summary>
/// Deserializes one persisted store record.
/// </summary>
public static CultPersistedRecord DeserializePersistedRecord(byte[] payload)
{
var reader = new MessagePackReader(payload);
return ReadPersistedRecord(ref reader);
}
/// <summary>
/// Serializes a schema catalog.
/// </summary>
public static byte[] SerializeSchemaCatalog(CultSchemaCatalogEntry[] catalog)
{
var buffer = new global::System.Buffers.ArrayBufferWriter<byte>();
var writer = new MessagePackWriter(buffer);
writer.WriteArrayHeader(catalog.Length);
foreach (var entry in catalog)
{
WriteSchemaCatalogEntry(ref writer, entry);
}
writer.Flush();
return buffer.WrittenSpan.ToArray();
}
/// <summary>
/// Deserializes a schema catalog.
/// </summary>
public static CultSchemaCatalogEntry[] DeserializeSchemaCatalog(byte[] payload)
{
var reader = new MessagePackReader(payload);
var count = reader.ReadArrayHeader();
var catalog = new CultSchemaCatalogEntry[count];
for (var index = 0; index < count; index++)
{
catalog[index] = ReadSchemaCatalogEntry(ref reader);
}
return catalog;
}
/// <summary>
/// Serializes a complete persisted store snapshot.
/// </summary>
public static byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot)
{
var buffer = new global::System.Buffers.ArrayBufferWriter<byte>();
var writer = new MessagePackWriter(buffer);
writer.WriteArrayHeader(StoreSnapshotFieldCount);
writer.Write(snapshot.FormatVersion);
writer.WriteArrayHeader(snapshot.SchemaCatalog.Length);
foreach (var entry in snapshot.SchemaCatalog)
{
WriteSchemaCatalogEntry(ref writer, entry);
}
writer.WriteArrayHeader(snapshot.Records.Length);
foreach (var record in snapshot.Records)
{
WritePersistedRecord(ref writer, record);
}
writer.Flush();
return buffer.WrittenSpan.ToArray();
}
/// <summary>
/// Deserializes a complete persisted store snapshot.
/// </summary>
public static CultPersistedStoreSnapshot DeserializeSnapshot(byte[] payload)
{
var reader = new MessagePackReader(payload);
var fieldCount = reader.ReadArrayHeader();
var snapshot = new CultPersistedStoreSnapshot();
if (fieldCount > 0)
{
snapshot.FormatVersion = reader.ReadString() ?? "cultcache.store.v1";
}
if (fieldCount > 1)
{
var catalogCount = reader.ReadArrayHeader();
snapshot.SchemaCatalog = new CultSchemaCatalogEntry[catalogCount];
for (var index = 0; index < catalogCount; index++)
{
snapshot.SchemaCatalog[index] = ReadSchemaCatalogEntry(ref reader);
}
}
if (fieldCount > 2)
{
var recordCount = reader.ReadArrayHeader();
snapshot.Records = new CultPersistedRecord[recordCount];
for (var index = 0; index < recordCount; index++)
{
snapshot.Records[index] = ReadPersistedRecord(ref reader);
}
}
for (var index = StoreSnapshotFieldCount; index < fieldCount; index++)
{
reader.Skip();
}
return snapshot;
}
private static void WritePersistedRecord(ref MessagePackWriter writer, CultPersistedRecord record)
{
writer.WriteArrayHeader(PersistedRecordFieldCount);
writer.Write(record.Key);
writer.Write(record.SchemaId);
writer.Write(record.StoredAt);
writer.Write(record.Payload);
}
private static CultPersistedRecord ReadPersistedRecord(ref MessagePackReader reader)
{
var fieldCount = reader.ReadArrayHeader();
var record = new CultPersistedRecord();
if (fieldCount > 0)
{
record.Key = reader.ReadString() ?? string.Empty;
}
if (fieldCount > 1)
{
record.SchemaId = reader.ReadString() ?? string.Empty;
}
if (fieldCount > 2)
{
record.StoredAt = reader.ReadString() ?? string.Empty;
}
if (fieldCount > 3)
{
record.Payload = reader.ReadBytes()?.ToArray() ?? Array.Empty<byte>();
}
for (var index = PersistedRecordFieldCount; index < fieldCount; index++)
{
reader.Skip();
}
return record;
}
private static void WriteSchemaCatalogEntry(ref MessagePackWriter writer, CultSchemaCatalogEntry entry)
{
writer.WriteArrayHeader(SchemaCatalogEntryFieldCount);
writer.Write(entry.SchemaId);
writer.Write(entry.SchemaName);
writer.Write(entry.SchemaVersion);
writer.Write(entry.ContentHash);
writer.Write(entry.CanonicalSchemaJson);
writer.WriteArrayHeader(entry.CompatibleSchemaIds.Length);
foreach (var schemaId in entry.CompatibleSchemaIds)
{
writer.Write(schemaId);
}
writer.WriteArrayHeader(entry.Members.Length);
foreach (var member in entry.Members)
{
WriteSchemaCatalogMember(ref writer, member);
}
}
private static CultSchemaCatalogEntry ReadSchemaCatalogEntry(ref MessagePackReader reader)
{
var fieldCount = reader.ReadArrayHeader();
var entry = new CultSchemaCatalogEntry();
if (fieldCount > 0)
{
entry.SchemaId = reader.ReadString() ?? string.Empty;
}
if (fieldCount > 1)
{
entry.SchemaName = reader.ReadString() ?? string.Empty;
}
if (fieldCount > 2)
{
entry.SchemaVersion = reader.ReadString() ?? string.Empty;
}
if (fieldCount > 3)
{
entry.ContentHash = reader.ReadString() ?? string.Empty;
}
if (fieldCount > 4)
{
entry.CanonicalSchemaJson = reader.ReadString() ?? string.Empty;
}
if (fieldCount > 5)
{
var compatibleCount = reader.ReadArrayHeader();
entry.CompatibleSchemaIds = new string[compatibleCount];
for (var index = 0; index < compatibleCount; index++)
{
entry.CompatibleSchemaIds[index] = reader.ReadString() ?? string.Empty;
}
}
if (fieldCount > 6)
{
var memberCount = reader.ReadArrayHeader();
entry.Members = new CultSchemaMemberCatalogEntry[memberCount];
for (var index = 0; index < memberCount; index++)
{
entry.Members[index] = ReadSchemaCatalogMember(ref reader);
}
}
for (var index = SchemaCatalogEntryFieldCount; index < fieldCount; index++)
{
reader.Skip();
}
return entry;
}
private static void WriteSchemaCatalogMember(ref MessagePackWriter writer, CultSchemaMemberCatalogEntry member)
{
writer.WriteArrayHeader(SchemaCatalogMemberFieldCount);
writer.Write(member.Slot);
writer.Write(member.MemberName);
writer.Write(member.TypeName);
writer.Write(member.IsReference);
writer.Write(member.IsMany);
writer.Write(member.TargetSchemaName);
writer.Write(member.IsName);
writer.Write(member.IndexAlias);
}
private static CultSchemaMemberCatalogEntry ReadSchemaCatalogMember(ref MessagePackReader reader)
{
var fieldCount = reader.ReadArrayHeader();
var member = new CultSchemaMemberCatalogEntry();
if (fieldCount > 0)
{
member.Slot = reader.ReadInt32();
}
if (fieldCount > 1)
{
member.MemberName = reader.ReadString() ?? string.Empty;
}
if (fieldCount > 2)
{
member.TypeName = reader.ReadString() ?? string.Empty;
}
if (fieldCount > 3)
{
member.IsReference = reader.ReadBoolean();
}
if (fieldCount > 4)
{
member.IsMany = reader.ReadBoolean();
}
if (fieldCount > 5)
{
member.TargetSchemaName = reader.ReadString();
}
if (fieldCount > 6)
{
member.IsName = reader.ReadBoolean();
}
if (fieldCount > 7)
{
member.IndexAlias = reader.ReadString();
}
for (var index = SchemaCatalogMemberFieldCount; index < fieldCount; index++)
{
reader.Skip();
}
return member;
}
}
/// <summary>
/// Single-file CultCache backing store that persists snapshots as MessagePack.
/// </summary>
public class SingleFileMessagePackBackingStore : SingleFileBackingStore
{
/// <summary>
/// Creates a MessagePack single-file backing store.
/// </summary>
public SingleFileMessagePackBackingStore(string filePath) : base(filePath)
{
}
/// <summary>
/// Serializes a store snapshot.
/// </summary>
protected override byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot)
{
return CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot);
}
/// <summary>
/// Deserializes a store snapshot.
/// </summary>
protected override CultPersistedStoreSnapshot DeserializeSnapshot(byte[] data)
{
return CultDocumentMessagePackSerialization.DeserializeSnapshot(data);
}
/// <summary>
/// Serializes one document payload.
/// </summary>
protected override byte[] SerializePayload(object document)
{
return CultDocumentMessagePackSerialization.SerializeUntyped(document, document.GetType());
}
/// <summary>
/// Deserializes one document payload.
/// </summary>
protected override object DeserializePayload(Type documentType, byte[] payload)
{
return CultDocumentMessagePackSerialization.DeserializeUntyped(documentType, payload);
}
}

27
src/GameCult.Caching.MessagePack/CultRecordRefFormatter.cs Normal file
View file

@ -0,0 +1,27 @@
using MessagePack;
using MessagePack.Formatters;
namespace GameCult.Caching;
/// <summary>
/// Serializes typed CultCache record references as their persisted key strings.
/// </summary>
public sealed class CultRecordRefFormatter<T> : IMessagePackFormatter<CultRecordRef<T>>
{
/// <summary>
/// Writes a typed record reference to MessagePack.
/// </summary>
public void Serialize(ref MessagePackWriter writer, CultRecordRef<T> value, MessagePackSerializerOptions options)
{
options.Resolver.GetFormatterWithVerify<string>().Serialize(ref writer, value.Key.Value, options);
}
/// <summary>
/// Reads a typed record reference from MessagePack.
/// </summary>
public CultRecordRef<T> Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options)
{
var key = options.Resolver.GetFormatterWithVerify<string>().Deserialize(ref reader, options) ?? string.Empty;
return new CultRecordRef<T>(new CultRecordKey(key));
}
}

53
src/GameCult.Caching.MessagePack/DatabaseEntryResolver.cs
View file

@ -1,53 +0,0 @@
using System;
using System.Collections.Concurrent;
using MessagePack;
using MessagePack.Formatters;
namespace GameCult.Caching.MessagePack
{
/// <summary>
/// Resolver that knows how to resolve:<br/>
/// 1. Source-generated singleton formatters for DatabaseEntry subclasses<br/>
/// 2. Generic DatabaseLink&lt;T&gt; formatter
/// </summary>
public sealed class DatabaseEntryResolver : IFormatterResolver
{
/// <summary>
/// Shared singleton instance of the resolver.
/// </summary>
public static readonly DatabaseEntryResolver Instance = new DatabaseEntryResolver();
private readonly ConcurrentDictionary<Type, object> _formatterCache = new();
private DatabaseEntryResolver() { }
/// <summary>
/// Resolves a formatter for a cache entry type or a <see cref="DatabaseLink{T}"/>.
/// </summary>
/// <typeparam name="T">The requested serialized type.</typeparam>
/// <returns>The formatter if one is available; otherwise, <c>null</c>.</returns>
public IMessagePackFormatter<T>? GetFormatter<T>()
{
// 1. Check generated resolver mapping
var fmt = GeneratedDatabaseEntryResolvers.GetFormatter<T>();
if (fmt != null) return fmt;
// 2. Provide generic DatabaseLink<T> formatter
var t = typeof(T);
if (t.IsGenericType && t.GetGenericTypeDefinition() == typeof(DatabaseLink<>))
{
return (IMessagePackFormatter<T>)_formatterCache.GetOrAdd(
t,
static linkType =>
{
var valueType = linkType.GetGenericArguments()[0];
var formatterType = typeof(DatabaseLinkFormatter<>).MakeGenericType(valueType);
return Activator.CreateInstance(formatterType)!;
});
}
// 3. Nothing found
return null;
}
}
}

24
src/GameCult.Caching.MessagePack/DatabaseEntrySerialization.cs
View file

@ -1,24 +0,0 @@
using MessagePack;
using MessagePack.Resolvers;
namespace GameCult.Caching.MessagePack;
internal static class DatabaseEntrySerialization
{
public static readonly MessagePackSerializerOptions Options =
MessagePackSerializerOptions.Standard
.WithResolver(CompositeResolver.Create(
DatabaseEntryResolver.Instance,
StandardResolver.Instance))
.WithSecurity(MessagePackSecurity.UntrustedData);
public static byte[] Serialize<T>(T value)
{
return MessagePackSerializer.Serialize(value, Options);
}
public static T Deserialize<T>(byte[] payload)
{
return MessagePackSerializer.Deserialize<T>(payload, Options);
}
}

33
src/GameCult.Caching.MessagePack/DatabaseLinkFormatter.cs
View file

@ -1,33 +0,0 @@
using System;
using MessagePack;
using MessagePack.Formatters;
namespace GameCult.Caching.MessagePack
{
/// <summary>
/// Serializes <see cref="DatabaseLink{T}"/> values as their linked identifier only.
/// </summary>
/// <typeparam name="T">The target entry type.</typeparam>
public sealed class DatabaseLinkFormatter<T> : IMessagePackFormatter<DatabaseLink<T>?> where T : DatabaseEntry
{
/// <inheritdoc />
public void Serialize(ref MessagePackWriter writer, DatabaseLink<T>? value, MessagePackSerializerOptions options)
{
if (value == null) { writer.WriteNil(); return; }
writer.WriteArrayHeader(1);
var fmt = options.Resolver.GetFormatterWithVerify<Guid>();
fmt.Serialize(ref writer, value.LinkID, options);
}
/// <inheritdoc />
public DatabaseLink<T>? Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options)
{
if (reader.TryReadNil()) return null;
var c = reader.ReadArrayHeader();
if (c != 1) throw new MessagePackSerializationException($"Invalid DatabaseLink<{typeof(T).Name}> array length {c}, expected 1");
var id = options.Resolver.GetFormatterWithVerify<Guid>().Deserialize(ref reader, options);
var link = new DatabaseLink<T> { LinkID = id };
return link;
}
}
}

12
src/GameCult.Caching.MessagePack/GeneratedDatabaseEntryResolvers.cs
View file

@ -1,12 +0,0 @@
// This stub ensures the project compiles before the source generator runs.
using MessagePack.Formatters;
namespace GameCult.Caching.MessagePack
{
static partial class GeneratedDatabaseEntryResolvers
{
// The generator will emit the real implementation.
public static IMessagePackFormatter<T>? GetFormatter<T>() => null;
}
}

54
src/GameCult.Caching.MessagePack/MessagePackBackingStore.cs
View file

@ -1,54 +0,0 @@
using MessagePack;
namespace GameCult.Caching.MessagePack;
/// <summary>
/// Persists all cache entries to a single MessagePack file.
/// </summary>
public class SingleFileMessagePackBackingStore : SingleFileBackingStore
{
/// <summary>
/// Initializes a single-file MessagePack backing store.
/// </summary>
/// <param name="filePath">The path to the backing file.</param>
public SingleFileMessagePackBackingStore(string filePath) : base(filePath) { }
/// <inheritdoc />
public override byte[] Serialize(DatabaseEntry[] entries)
{
return DatabaseEntrySerialization.Serialize(entries);
}
/// <inheritdoc />
public override DatabaseEntry[] Deserialize(byte[] data)
{
return DatabaseEntrySerialization.Deserialize<DatabaseEntry[]>(data);
}
}
/// <summary>
/// Persists each cache entry to its own MessagePack file.
/// </summary>
public class MultiFileMessagePackBackingStore : MultiFileBackingStore
{
/// <summary>
/// Initializes a multi-file MessagePack backing store rooted at the supplied directory.
/// </summary>
/// <param name="path">The root directory for stored entry files.</param>
public MultiFileMessagePackBackingStore(string path) : base(path) { }
/// <inheritdoc />
public override byte[] Serialize(DatabaseEntry entry)
{
return DatabaseEntrySerialization.Serialize(entry);
}
/// <inheritdoc />
public override DatabaseEntry Deserialize(byte[] data)
{
return DatabaseEntrySerialization.Deserialize<DatabaseEntry>(data);
}
/// <inheritdoc />
public override string Extension => "msgpack";
}

149
src/GameCult.Caching.MessagePack/README.md
View file

@ -1,39 +1,41 @@
# GameCult.Caching.MessagePack
`GameCult.Caching.MessagePack` provides MessagePack-backed persistence for `GameCult.Caching`.
## Scope
The package focuses on:
- MessagePack storage for `DatabaseEntry` models
- single-file and multi-file store implementations
- support for generated formatters for concrete cache-entry types
- formatter support for `DatabaseLink<T>`
- raw persisted-envelope helpers for exact payload relay
`GameCult.Caching.MessagePack` provides the canonical CultCache persistence
format for the attribute-first `GameCult.Caching` stack.
## Included Types
- `SingleFileMessagePackBackingStore`
- `MultiFileMessagePackBackingStore`
- `DatabaseEntryResolver`
- `DatabaseLinkFormatter<T>`
- `CultCacheEnvelope`
- `CultCacheEnvelopeSerialization`
- `CultDocumentMessagePackSerialization`
- `CultDocumentResolver`
- `CultRecordRefFormatter<T>`
## Why The Generator Matters
When a project declares `[CultDocument]` models, wire in the
`GameCult.Caching.MessagePack.Generator` analyzer there as well. It emits
assembly-local metadata providers for the cache registry. Cult documents then
serialize through explicit generated slot codecs, while the store snapshot
format is written by hand against `MessagePackWriter`/`MessagePackReader`.
`DatabaseEntry` is abstract. Concrete subclasses need MessagePack formatters to be serializable without hand-writing formatters for each model.
## What It Does
That is the role of:
- stores whole CultCache snapshots in MessagePack
- serializes plain attributed document payloads through explicit generated slot codecs
- keeps explicit `CultRecordRef<T>` values compact on disk/wire
- preserves the cache/store split where metadata lives outside the domain model
- `GameCult.Caching.MessagePack.Generator`
- `GameCult.Caching.MessagePack.Analyzers`
## Example Model
In practice:
```csharp
using GameCult.Caching;
using MessagePack;
- this package gives you the stores and resolver support
- the generator package produces the actual concrete formatters
[CultDocument("gamecult.item_data", "gamecult.item_data.v1")]
public sealed class ItemData
{
[Key(0)] [CultName] public string Name = string.Empty;
[Key(1)] public int Value;
}
```
## Typical Usage
@ -42,100 +44,17 @@ using GameCult.Caching;
using GameCult.Caching.MessagePack;
var cache = new CultCache();
var store = new MultiFileMessagePackBackingStore("Data");
var store = new SingleFileMessagePackBackingStore("Data.msgpack");
cache.AddBackingStore(store);
await cache.PullAllBackingStoresAsync();
```
## Single-File vs Multi-File
## Notes
### `SingleFileMessagePackBackingStore`
Use this when you want one snapshot file containing all entries.
Use cases:
- compact settings data
- simple local persistence
- write-all-at-once workflows
```csharp
var store = new SingleFileMessagePackBackingStore("cache.msgpack");
cache.AddBackingStore(store);
```
### `MultiFileMessagePackBackingStore`
Use this when you want one file per entry.
Use cases:
- large datasets
- debugging persisted entries individually
- change observation through the multi-file base store
```csharp
var store = new MultiFileMessagePackBackingStore("Data");
cache.AddBackingStore(store, typeof(PlayerData));
```
## `DatabaseLink<T>`
`DatabaseLink<T>` is a lightweight reference type used by the cache layer. The MessagePack formatter serializes only the linked `Guid`, not the fully loaded target object.
That means:
- links stay compact on disk
- linked objects are resolved through the active `CultCache`
- deserialized links depend on `DatabaseLinkBase.Cache` pointing at the active cache
## Example Model
```csharp
using GameCult.Caching;
using MessagePack;
[MessagePackObject]
public class ItemData : DatabaseEntry, INamedEntry
{
[Key(1)] public string Name = string.Empty;
[Key(2)] public int Value;
[IgnoreMember]
public string EntryName
{
get => Name;
set => Name = value;
}
}
```
## Practical Notes
- For polymorphic `DatabaseEntry` serialization, use this package with the generator/analyzer package.
- The backing stores build on the semantics of `CultCache`, including primary-store routing and type-specific store routing.
- `MultiFileMessagePackBackingStore` inherits the rename-safe and atomic-write behavior of the hardened `MultiFileBackingStore` base class.
## Raw Envelope Relay
If a neighboring process already shares the same payload contract and just
needs the exact persisted MessagePack bytes, use `CultCacheEnvelope` and
`CultCacheEnvelopeSerialization`.
That gives you:
- the stable cache key
- the explicit entry/document type
- the exact payload bytes
- the stored-at timestamp
This is the seam the newer CultNet raw document/snapshot lane uses for
bit-compatible neighbors. It keeps the transport from decoding bytes into a
generic sludge object only to re-encode the same thing again five seconds
later.
One important constraint: concrete `DatabaseEntry` subclasses still need real
MessagePack formatters. If a consumer assembly has not generated or registered
formatters for a subtype, supply an explicit payload codec at the integration
boundary instead of assuming the abstract resolver will become psychic.
- Payload bytes are MessagePack. Store metadata and schema catalogs are also
persisted through hand-written MessagePack array layouts in this package.
- The generator package now targets attributed document types instead of
`DatabaseEntry` subclasses.
- The raw CultNet document lane should treat these payload bytes as already
blessed, not decode and re-encode them for sport.

10
src/GameCult.Caching.NewtonsoftJson/GameCult.Caching.NewtonsoftJson.csproj
View file

@ -1,10 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<Description>Newtonsoft.Json backing stores for CultCache</Description>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\GameCult.Caching\GameCult.Caching.csproj" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
</ItemGroup>
</Project>

238
src/GameCult.Caching.NewtonsoftJson/NewtonsoftJsonBackingStore.cs
View file

@ -1,238 +0,0 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Text;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
namespace GameCult.Caching.NewtonsoftJson;
/// <summary>
/// Persists all cache entries to a single JSON file using Newtonsoft.Json.
/// </summary>
public class SingleFileNewtonsoftJsonBackingStore : SingleFileBackingStore
{
private readonly KnownDatabaseEntryTypes _knownTypes = new();
/// <summary>
/// Initializes a single-file Newtonsoft.Json backing store.
/// </summary>
/// <param name="filePath">The path to the backing file.</param>
public SingleFileNewtonsoftJsonBackingStore(string filePath) : base(filePath)
{
}
/// <summary>
/// Registers a concrete <see cref="DatabaseEntry"/> type for serialization.
/// </summary>
/// <typeparam name="T">The entry type to register.</typeparam>
public void RegisterType<T>() where T : DatabaseEntry
{
_knownTypes.RegisterType(typeof(T));
}
/// <summary>
/// Registers all concrete <see cref="DatabaseEntry"/> types from the supplied assembly.
/// </summary>
/// <param name="assembly">The assembly to scan.</param>
public void RegisterAssembly(Assembly assembly)
{
_knownTypes.RegisterAssembly(assembly);
}
/// <summary>
/// Rebuilds the known-type map from the currently loaded assemblies.
/// </summary>
public void RefreshKnownTypes()
{
_knownTypes.RegisterLoadedAssemblies();
}
/// <inheritdoc />
public override byte[] Serialize(DatabaseEntry[] entries)
{
var serializer = JsonSerializer.Create(KnownDatabaseEntryTypes.SerializerSettings);
var payload = entries.Select(entry => _knownTypes.ToEnvelope(entry, serializer)).ToArray();
var json = JsonConvert.SerializeObject(payload, KnownDatabaseEntryTypes.SerializerSettings);
return Encoding.UTF8.GetBytes(json);
}
/// <inheritdoc />
public override DatabaseEntry[] Deserialize(byte[] data)
{
_knownTypes.RegisterLoadedAssemblies();
var json = Encoding.UTF8.GetString(data);
var serializer = JsonSerializer.Create(KnownDatabaseEntryTypes.SerializerSettings);
var payload = JsonConvert.DeserializeObject<DatabaseEntryEnvelope[]>(json, KnownDatabaseEntryTypes.SerializerSettings) ?? [];
return payload.Select(envelope => _knownTypes.FromEnvelope(envelope, serializer)).ToArray();
}
}
/// <summary>
/// Persists each cache entry to its own JSON file using Newtonsoft.Json.
/// </summary>
public class MultiFileNewtonsoftJsonBackingStore : MultiFileBackingStore
{
private readonly KnownDatabaseEntryTypes _knownTypes = new();
/// <summary>
/// Initializes a multi-file Newtonsoft.Json backing store rooted at the supplied directory.
/// </summary>
/// <param name="path">The root directory for stored entry files.</param>
public MultiFileNewtonsoftJsonBackingStore(string path) : base(path)
{
}
/// <summary>
/// Registers a concrete <see cref="DatabaseEntry"/> type for serialization.
/// </summary>
/// <typeparam name="T">The entry type to register.</typeparam>
public void RegisterType<T>() where T : DatabaseEntry
{
_knownTypes.RegisterType(typeof(T));
}
/// <summary>
/// Registers all concrete <see cref="DatabaseEntry"/> types from the supplied assembly.
/// </summary>
/// <param name="assembly">The assembly to scan.</param>
public void RegisterAssembly(Assembly assembly)
{
_knownTypes.RegisterAssembly(assembly);
}
/// <summary>
/// Rebuilds the known-type map from the currently loaded assemblies.
/// </summary>
public void RefreshKnownTypes()
{
_knownTypes.RegisterLoadedAssemblies();
}
/// <inheritdoc />
public override byte[] Serialize(DatabaseEntry entry)
{
var serializer = JsonSerializer.Create(KnownDatabaseEntryTypes.SerializerSettings);
var payload = _knownTypes.ToEnvelope(entry, serializer);
var json = JsonConvert.SerializeObject(payload, KnownDatabaseEntryTypes.SerializerSettings);
return Encoding.UTF8.GetBytes(json);
}
/// <inheritdoc />
public override DatabaseEntry Deserialize(byte[] data)
{
_knownTypes.RegisterLoadedAssemblies();
var json = Encoding.UTF8.GetString(data);
var serializer = JsonSerializer.Create(KnownDatabaseEntryTypes.SerializerSettings);
var payload = JsonConvert.DeserializeObject<DatabaseEntryEnvelope>(json, KnownDatabaseEntryTypes.SerializerSettings)
?? throw new JsonSerializationException("Failed to deserialize DatabaseEntry envelope.");
return _knownTypes.FromEnvelope(payload, serializer);
}
/// <inheritdoc />
public override string Extension => "json";
}
internal sealed class KnownDatabaseEntryTypes
{
internal static readonly JsonSerializerSettings SerializerSettings = new()
{
Formatting = Formatting.Indented
};
private readonly ConcurrentDictionary<string, Type> _discriminatorToType = new(StringComparer.Ordinal);
private readonly ConcurrentDictionary<Type, string> _typeToDiscriminator = new();
public KnownDatabaseEntryTypes()
{
RegisterLoadedAssemblies();
}
public void RegisterType(Type type)
{
if (!typeof(DatabaseEntry).IsAssignableFrom(type) || type.IsAbstract)
{
throw new ArgumentException($"Type '{type.FullName}' must be a non-abstract DatabaseEntry subtype.", nameof(type));
}
var discriminator = GetDiscriminator(type);
_discriminatorToType[discriminator] = type;
_typeToDiscriminator[type] = discriminator;
}
public void RegisterAssembly(Assembly assembly)
{
foreach (var type in SafeGetTypes(assembly).Where(IsConcreteDatabaseEntry))
{
RegisterType(type);
}
}
public void RegisterLoadedAssemblies()
{
foreach (var assembly in AppDomain.CurrentDomain.GetAssemblies().Where(assembly => !assembly.IsDynamic))
{
RegisterAssembly(assembly);
}
}
public DatabaseEntryEnvelope ToEnvelope(DatabaseEntry entry, JsonSerializer serializer)
{
var type = entry.GetType();
if (!_typeToDiscriminator.TryGetValue(type, out var discriminator))
{
RegisterType(type);
discriminator = _typeToDiscriminator[type];
}
return new DatabaseEntryEnvelope
{
Type = discriminator,
Data = JObject.FromObject(entry, serializer)
};
}
public DatabaseEntry FromEnvelope(DatabaseEntryEnvelope envelope, JsonSerializer serializer)
{
if (!_discriminatorToType.TryGetValue(envelope.Type, out var type))
{
throw new JsonSerializationException($"Unknown DatabaseEntry discriminator '{envelope.Type}'. Register the containing assembly or entry type before deserializing.");
}
return (DatabaseEntry?)envelope.Data.ToObject(type, serializer)
?? throw new JsonSerializationException($"Failed to deserialize DatabaseEntry discriminator '{envelope.Type}'.");
}
private static bool IsConcreteDatabaseEntry(Type type)
{
return typeof(DatabaseEntry).IsAssignableFrom(type) && !type.IsAbstract;
}
private static IEnumerable<Type> SafeGetTypes(Assembly assembly)
{
try
{
return assembly.GetTypes();
}
catch (ReflectionTypeLoadException ex)
{
return ex.Types.Where(type => type != null)!;
}
}
private static string GetDiscriminator(Type type)
{
return $"{type.Assembly.GetName().Name}:{type.FullName}";
}
}
internal sealed class DatabaseEntryEnvelope
{
public string Type { get; set; } = string.Empty;
public JObject Data { get; set; } = new();
}

49
src/GameCult.Caching.NewtonsoftJson/README.md
View file

@ -1,49 +0,0 @@
# GameCult.Caching.NewtonsoftJson
`GameCult.Caching.NewtonsoftJson` provides human-readable JSON backing stores for `GameCult.Caching` using Newtonsoft.Json.
## Scope
The package provides:
- `SingleFileNewtonsoftJsonBackingStore`
- `MultiFileNewtonsoftJsonBackingStore`
- known-type polymorphic `DatabaseEntry` serialization through closed discriminators
## Usage
```csharp
using GameCult.Caching;
using GameCult.Caching.NewtonsoftJson;
var cache = new CultCache();
var store = new MultiFileNewtonsoftJsonBackingStore("Data");
cache.AddBackingStore(store);
await cache.PullAllBackingStoresAsync();
```
## Type Discovery
The JSON backing stores build a known-type map from the assemblies already loaded in the current `AppDomain`. In the common case, this means entry types defined in the consumer's application assembly work without extra configuration once that assembly is loaded.
The stores do not deserialize arbitrary CLR type names from JSON. Each file contains a closed discriminator plus the entry payload, and deserialization succeeds only when the discriminator matches a registered concrete `DatabaseEntry` subtype.
For less typical setups, the stores also expose:
- `RegisterType<T>()`
- `RegisterAssembly(Assembly assembly)`
- `RefreshKnownTypes()`
Example:
```csharp
var store = new MultiFileNewtonsoftJsonBackingStore("Data");
store.RegisterAssembly(typeof(MyCustomEntry).Assembly);
```
## Notes
- Files are indented JSON for inspection and editing.
- Polymorphic entry types are serialized with a closed discriminator map rather than open-ended type-name handling.
- `MultiFileNewtonsoftJsonBackingStore` builds on the base multi-file store behavior in `GameCult.Caching`, including rename-safe writes.

154
src/GameCult.Caching/Contracts/cultcache-persistence-format.md
View file

@ -306,6 +306,95 @@ The lower-level API should still expose:
That is the cache's responsibility, not the domain model's.
## Concurrent Store Policy
CultCache files are expected to become shared state, not private save blobs.
Once multiple runtimes can read and write the same `.cc` file, the persistence
contract must protect both durability and observation.
The store header should declare the write policy a runtime must obey before it
mutates the file:
- `generation`: monotonic store generation, incremented on each committed
snapshot
- `contentHash`: hash of the committed snapshot body
- `concurrency.policy`: stable policy id, such as
`cultcache.concurrent.snapshot-lock.v1`
- `concurrency.lockName`: how to derive the sidecar lock path
- `concurrency.commit`: commit primitive, such as `atomic-replace`
The header negotiates the protocol. It does not enforce the protocol by magic.
Writers still need an external coordination primitive because replacing the
main file also replaces whatever identity a platform may attach to that file
handle.
The v1 concurrent single-file policy is:
1. Readers may read lock-free when they can tolerate a stale-but-complete
snapshot. Readers that need a stable read boundary take a shared sidecar
lock.
2. Writers take an exclusive sidecar lock derived from the `.cc` path.
3. Writers re-read the current snapshot after taking the lock.
4. Writers merge or reject local staged changes against the latest generation.
5. Writers write a temp file, flush it, atomically replace the `.cc` file, and
release the lock.
This preserves the key invariant: every reader sees a complete snapshot, and no
writer commits over unseen data without passing through the merge/reject point.
## Live Change Observation
A shared CultCache file is also a low-ceremony realtime database for local
applications. The filesystem is the transport. CultCache is the typed database
surface. The cache should therefore expose domain-level change streams, not
force applications to subscribe to raw storage envelopes.
Recommended C# surface:
```csharp
Observable<CultCacheChange<T>> Watch<T>();
Observable<T> WatchRecord<T>(CultRecordKey key);
Observable<T> WatchGlobal<T>();
Observable<T> WatchByName<T>(string name);
```
Change events should be emitted after a pulled or locally committed snapshot has
been reconciled into the in-memory cache. Subscribers should see domain objects
and cache-owned handles, not raw `schemaId`, `storedAt`, or MessagePack payload
bytes unless they explicitly subscribe to the raw persistence lane.
The event model should distinguish at least:
- `added`
- `updated`
- `removed`
- `schema_migrated`
- `rejected`
Local and remote-looking writes use the same reconciliation path. A write from
another process is just a new committed generation discovered through file
watching or polling, pulled into the cache, diffed against the current domain
view, and published.
### R3 Alignment
The C# reactive surface should align with R3, Cysharp/neuecc's modern
Reactive Extensions implementation and successor line to UniRx. CultCache
should not invent a bespoke observer API when the .NET ecosystem already has a
well-understood shape for composable streams.
R3 should own the C# subscription ergonomics:
- `Observable<T>` for public domain streams
- `Subject<T>` or internal equivalents only at ownership boundaries
- disposable subscriptions for lifecycle control
- debounce/throttle operators for bursty filesystem notifications
- scheduler hooks where Unity, desktop UI, or server callers need thread
affinity
The invariant is simple: the persistence layer owns bytes and generations; the
cache owns domain identity and diffing; R3 owns composition of observed changes.
## CultNet Alignment
CultCache and CultNet should not grow rival schema religions.
@ -323,6 +412,71 @@ should mean the same thing across persistence and wire discovery.
Where CultNet needs additional wire-oriented metadata, extend around the shared
core instead of redefining the core.
## CultNet As A Sharding Layer
CultNet should be able to initialize directly as a CultCache distribution layer.
The goal is a single distributed realtime database surface: clients work with
typed CultCache documents, while CultNet decides which node owns the relevant
shard and how committed changes reach subscribers.
This is closer to the application experience of RethinkDB or Firebase than a
traditional request/response game protocol. CultNet should still keep its own
shape: explicit schemas, MessagePack payloads, local-first operation where
useful, and honest rejection when compatibility or authority is unclear.
The ownership split is:
- CultCache owns document identity, schema compatibility, local indexing,
persistence, and domain-level change diffing.
- CultNet owns peer membership, shard routing, mutation transport,
authorization, remote subscription fanout, and resynchronization after
reconnect.
- R3 owns C# stream composition for observed domain changes.
A distributed CultCache cluster should expose:
- shard descriptors: which schema/key ranges or named partitions a node owns
- shard epochs: monotonic authority versions for ownership changes
- mutation messages: raw document put/delete records with source generation
and shard epoch
- snapshot messages: bounded catch-up from a known generation or empty local
state
- subscription messages: schema/key/index/global queries that stream matching
domain changes
- conflict reports: explicit outcomes when a mutation cannot be applied cleanly
Client-facing APIs should preserve the database illusion:
```csharp
var db = await CultNetDatabase.ConnectAsync("localhost", 3075);
var player = await db.GetAsync<PlayerData>(playerKey);
await db.PutAsync(playerKey, player);
using var subscription = db
.WatchByIndex<PlayerData>("Region", "eu-west")
.Subscribe(change => Render(change.Document));
```
The illusion has limits and those limits should be visible in the contract.
CultNet is not allowed to pretend that arbitrary split-brain writes merged
cleanly. Each shard needs one current write authority, or a declared conflict
strategy that can be explained without ritual smoke. If neither exists, the
write is rejected and surfaced as a domain-visible failure.
The simplest coherent first implementation is primary-shard ownership:
1. Each shard has one authoritative writer at a time.
2. Followers subscribe to that shard's committed mutation stream.
3. Clients may connect to any node; non-owner nodes forward writes to the shard
owner or reject when forwarding is unavailable.
4. Reconnect performs schema discovery, shard epoch comparison, then snapshot
catch-up before live events resume.
More advanced replicated or offline-first conflict policies can come later, but
they must be explicit shard policies rather than accidental behavior leaking out
of transport retries.
## Migration From Current C# Shape
The current C# `DatabaseEntry` base type exposes `ID` directly. That is useful

60
src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md Normal file
View file

@ -0,0 +1,60 @@
# CultCache Schema Compatibility
CultCache compatibility is slot-driven and explicit. The cache will not pretend
that "same-ish looking" schemas are fine just because the names feel friendly.
## Semantic identity
Two CultCache document schemas are considered the same shape when they agree on:
- `schemaName`
- `schemaVersion`
- member slots
- member persisted type names
- reference/cardinality semantics
- target schema names for references
- name/index lookup participation
The canonical fixtures currently exercised in C# are:
- `tests.named_entry`
- schema id: `sha256:e7b97801b94190f3159012ede45b0069bb09ebf7920f7432c971bc86a0e08de8`
- content hash: `sha256:23150930afcc1d84f0cb3012ccc2debcb9b4685f62083033bbaab0083f1e832e`
- `tests.reference_holder`
- schema id: `sha256:bd85064961cc74565fb73e3ccbc4217cfba4dc4869e365a08bea4f704739bd8f`
See `GameCult.Caching.Tests/BackingStoreTests.cs` for the canonical fixture
documents and the expected receipts.
## Soft-migratable drift
CultCache accepts compatible drift only when the local reader can still map the
persisted slots honestly:
- persisted schema id differs, but the embedded catalog points to a compatible
local schema id
- persisted slot is missing locally and can be ignored
- local slot is missing in persisted data and can fall back to the local
default value
The cache emits a typed migration report with:
- exact vs compatible-drift classification
- ignored extra slots
- defaulted missing slots
- warning codes and messages
## Hard rejection
CultCache rejects persisted schemas when any slot changes in a way that would
lie about the payload:
- type change
- reference vs value change
- one vs many change
- target schema change
- name/index lookup semantics change
- missing embedded catalog entry
- no compatible local schema candidate
This is not negotiable. Better a loud refusal than quiet bit-rot.

2883
src/GameCult.Caching/CultCache.cs
View file

File diff suppressed because it is too large Load diff

192
src/GameCult.Caching/CultDocumentContracts.cs Normal file
View file

@ -0,0 +1,192 @@
using System;
namespace GameCult.Caching
{
/// <summary>
/// Marks a type as a CultCache document with a stable schema name and version.
/// </summary>
[AttributeUsage(AttributeTargets.Class, Inherited = false)]
public sealed class CultDocumentAttribute : Attribute
{
/// <summary>
/// Creates a document marker for the supplied schema identity.
/// </summary>
public CultDocumentAttribute(string schemaName, string schemaVersion)
{
SchemaName = string.IsNullOrWhiteSpace(schemaName)
? throw new ArgumentException("SchemaName must be non-empty.", nameof(schemaName))
: schemaName;
SchemaVersion = string.IsNullOrWhiteSpace(schemaVersion)
? throw new ArgumentException("SchemaVersion must be non-empty.", nameof(schemaVersion))
: schemaVersion;
}
/// <summary>
/// Gets the stable schema name used by CultCache and CultNet.
/// </summary>
public string SchemaName { get; }
/// <summary>
/// Gets the semantic schema version string for this document shape.
/// </summary>
public string SchemaVersion { get; }
}
/// <summary>
/// Marks a document type as having one global record in a cache.
/// </summary>
[AttributeUsage(AttributeTargets.Class, Inherited = false)]
public sealed class CultGlobalAttribute : Attribute
{
}
/// <summary>
/// Marks the member that supplies the human-readable document name.
/// </summary>
[AttributeUsage(AttributeTargets.Field | AttributeTargets.Property, Inherited = true)]
public sealed class CultNameAttribute : Attribute
{
}
/// <summary>
/// Marks a member as an indexed lookup value.
/// </summary>
[AttributeUsage(AttributeTargets.Field | AttributeTargets.Property, Inherited = true, AllowMultiple = false)]
public sealed class CultIndexAttribute : Attribute
{
/// <summary>
/// Creates an index marker with an optional alias.
/// </summary>
public CultIndexAttribute(string? alias = null)
{
Alias = alias;
}
/// <summary>
/// Gets the external index alias, or null to use the member name.
/// </summary>
public string? Alias { get; }
}
/// <summary>
/// Marks a member as a reference to another CultCache document.
/// </summary>
[AttributeUsage(AttributeTargets.Field | AttributeTargets.Property, Inherited = true)]
public sealed class CultReferenceAttribute : Attribute
{
/// <summary>
/// Creates a document reference marker.
/// </summary>
public CultReferenceAttribute(Type? targetType = null, bool many = false)
{
TargetType = targetType;
Many = many;
}
/// <summary>
/// Gets the explicit referenced document type, when one is required.
/// </summary>
public Type? TargetType { get; }
/// <summary>
/// Gets whether this member stores many references rather than one reference.
/// </summary>
public bool Many { get; }
}
/// <summary>
/// Identifies a record inside a CultCache backing store.
/// </summary>
public readonly struct CultRecordKey : IEquatable<CultRecordKey>
{
/// <summary>
/// Creates a record key from its persisted string value.
/// </summary>
public CultRecordKey(string value)
{
Value = value ?? string.Empty;
}
/// <summary>
/// Gets the persisted key value.
/// </summary>
public string Value { get; }
/// <summary>
/// Returns the persisted key value.
/// </summary>
public override string ToString() => Value;
/// <summary>
/// Returns whether this key and another key contain the same persisted value.
/// </summary>
public bool Equals(CultRecordKey other) => string.Equals(Value, other.Value, StringComparison.Ordinal);
/// <summary>
/// Returns whether this key and another object contain the same persisted value.
/// </summary>
public override bool Equals(object? obj) => obj is CultRecordKey other && Equals(other);
/// <summary>
/// Returns a hash code for the persisted key value.
/// </summary>
public override int GetHashCode() => StringComparer.Ordinal.GetHashCode(Value);
}
/// <summary>
/// Carries the persisted record key for a typed document.
/// </summary>
public readonly struct CultRecordHandle<T>
{
/// <summary>
/// Creates a typed document handle.
/// </summary>
public CultRecordHandle(CultRecordKey key)
{
Key = key;
}
/// <summary>
/// Gets the persisted record key.
/// </summary>
public CultRecordKey Key { get; }
/// <summary>
/// Returns the persisted key value.
/// </summary>
public override string ToString() => Key.ToString();
}
/// <summary>
/// Stores a typed reference to a CultCache document.
/// </summary>
public readonly struct CultRecordRef<T>
{
/// <summary>
/// Creates a typed document reference.
/// </summary>
public CultRecordRef(CultRecordKey key)
{
Key = key;
}
/// <summary>
/// Gets the referenced record key.
/// </summary>
public CultRecordKey Key { get; }
/// <summary>
/// Resolves the referenced record against the provided cache.
/// </summary>
public T? Resolve(CultCache cache)
{
if (cache == null) throw new ArgumentNullException(nameof(cache));
return cache.Get(Key) is T typed ? typed : default;
}
/// <summary>
/// Returns the referenced key value.
/// </summary>
public override string ToString() => Key.ToString();
}
}

308
src/GameCult.Caching/CultGeneratedDocumentMetadata.cs Normal file
View file

@ -0,0 +1,308 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Text;
namespace GameCult.Caching
{
/// <summary>
/// Registers a generated CultCache document metadata provider with an assembly.
/// </summary>
[AttributeUsage(AttributeTargets.Assembly, AllowMultiple = true)]
public sealed class CultGeneratedDocumentMetadataProviderAttribute : Attribute
{
/// <summary>
/// Creates an assembly metadata provider registration.
/// </summary>
public CultGeneratedDocumentMetadataProviderAttribute(Type providerType)
{
ProviderType = providerType ?? throw new ArgumentNullException(nameof(providerType));
}
/// <summary>
/// Gets the provider type that can create generated document definitions.
/// </summary>
public Type ProviderType { get; }
}
/// <summary>
/// Supplies generated CultCache document metadata for an assembly.
/// </summary>
public interface ICultGeneratedDocumentMetadataProvider
{
/// <summary>
/// Gets the generated document definitions exposed by the provider.
/// </summary>
IEnumerable<CultGeneratedDocumentDefinition> GetDocumentDefinitions();
}
/// <summary>
/// Describes one generated CultCache document contract.
/// </summary>
public sealed class CultGeneratedDocumentDefinition
{
/// <summary>
/// Creates a generated document contract definition.
/// </summary>
public CultGeneratedDocumentDefinition(
Type documentType,
string schemaName,
string schemaVersion,
bool isGlobal,
string? nameMember,
Func<object, string?>? nameAccessor,
Func<object, byte[]>? serializePayload,
Func<byte[], object>? deserializePayload,
IReadOnlyList<CultGeneratedDocumentIndexAccessor> indexAccessors,
IReadOnlyList<CultGeneratedDocumentMemberDefinition> members)
{
DocumentType = documentType ?? throw new ArgumentNullException(nameof(documentType));
SchemaName = string.IsNullOrWhiteSpace(schemaName)
? throw new ArgumentException("Schema name must be non-empty.", nameof(schemaName))
: schemaName;
SchemaVersion = string.IsNullOrWhiteSpace(schemaVersion)
? throw new ArgumentException("Schema version must be non-empty.", nameof(schemaVersion))
: schemaVersion;
IsGlobal = isGlobal;
NameMember = nameMember;
NameAccessor = nameAccessor;
SerializePayload = serializePayload;
DeserializePayload = deserializePayload;
IndexAccessors = indexAccessors ?? Array.Empty<CultGeneratedDocumentIndexAccessor>();
Members = members ?? Array.Empty<CultGeneratedDocumentMemberDefinition>();
}
/// <summary>
/// Gets the CLR document type.
/// </summary>
public Type DocumentType { get; }
/// <summary>
/// Gets the stable CultCache schema name.
/// </summary>
public string SchemaName { get; }
/// <summary>
/// Gets the schema version string.
/// </summary>
public string SchemaVersion { get; }
/// <summary>
/// Gets whether this document type stores one global record.
/// </summary>
public bool IsGlobal { get; }
/// <summary>
/// Gets the member used as the document name, if any.
/// </summary>
public string? NameMember { get; }
/// <summary>
/// Gets the generated name accessor, if any.
/// </summary>
public Func<object, string?>? NameAccessor { get; }
/// <summary>
/// Gets the generated payload serializer, if one was emitted.
/// </summary>
public Func<object, byte[]>? SerializePayload { get; }
/// <summary>
/// Gets the generated payload deserializer, if one was emitted.
/// </summary>
public Func<byte[], object>? DeserializePayload { get; }
/// <summary>
/// Gets the generated index accessors.
/// </summary>
public IReadOnlyList<CultGeneratedDocumentIndexAccessor> IndexAccessors { get; }
/// <summary>
/// Gets the generated persisted member definitions.
/// </summary>
public IReadOnlyList<CultGeneratedDocumentMemberDefinition> Members { get; }
}
/// <summary>
/// Describes a generated accessor for one CultCache index.
/// </summary>
public sealed class CultGeneratedDocumentIndexAccessor
{
/// <summary>
/// Creates a generated index accessor.
/// </summary>
public CultGeneratedDocumentIndexAccessor(string alias, Func<object, string> accessor)
{
Alias = string.IsNullOrWhiteSpace(alias)
? throw new ArgumentException("Alias must be non-empty.", nameof(alias))
: alias;
Accessor = accessor ?? throw new ArgumentNullException(nameof(accessor));
}
/// <summary>
/// Gets the index alias.
/// </summary>
public string Alias { get; }
/// <summary>
/// Gets the function that extracts the index value.
/// </summary>
public Func<object, string> Accessor { get; }
}
/// <summary>
/// Describes one generated persisted document member.
/// </summary>
public sealed class CultGeneratedDocumentMemberDefinition
{
/// <summary>
/// Creates a generated persisted member definition.
/// </summary>
public CultGeneratedDocumentMemberDefinition(
string memberName,
int slot,
string typeName,
bool isReference,
bool isMany,
string? targetSchemaName,
bool isName,
string? indexAlias)
{
MemberName = string.IsNullOrWhiteSpace(memberName)
? throw new ArgumentException("Member name must be non-empty.", nameof(memberName))
: memberName;
Slot = slot;
TypeName = string.IsNullOrWhiteSpace(typeName)
? throw new ArgumentException("Type name must be non-empty.", nameof(typeName))
: typeName;
IsReference = isReference;
IsMany = isMany;
TargetSchemaName = targetSchemaName;
IsName = isName;
IndexAlias = indexAlias;
}
/// <summary>
/// Gets the CLR member name.
/// </summary>
public string MemberName { get; }
/// <summary>
/// Gets the persisted slot number.
/// </summary>
public int Slot { get; }
/// <summary>
/// Gets the persisted type name.
/// </summary>
public string TypeName { get; }
/// <summary>
/// Gets whether this member stores a document reference.
/// </summary>
public bool IsReference { get; }
/// <summary>
/// Gets whether this member stores many references.
/// </summary>
public bool IsMany { get; }
/// <summary>
/// Gets the target schema name for reference members.
/// </summary>
public string? TargetSchemaName { get; }
/// <summary>
/// Gets whether this member is the document name.
/// </summary>
public bool IsName { get; }
/// <summary>
/// Gets the index alias for indexed members.
/// </summary>
public string? IndexAlias { get; }
}
internal static class CultGeneratedDocumentMetadataLoader
{
public static IEnumerable<CultGeneratedDocumentDefinition> LoadDefinitions()
{
return AppDomain.CurrentDomain.GetAssemblies()
.Where(assembly => !assembly.IsDynamic)
.SelectMany(LoadDefinitions);
}
public static IEnumerable<CultGeneratedDocumentDefinition> LoadDefinitions(Assembly assembly)
{
foreach (var attribute in assembly.GetCustomAttributes<CultGeneratedDocumentMetadataProviderAttribute>())
{
if (!typeof(ICultGeneratedDocumentMetadataProvider).IsAssignableFrom(attribute.ProviderType))
{
continue;
}
if (Activator.CreateInstance(attribute.ProviderType) is not ICultGeneratedDocumentMetadataProvider provider)
{
continue;
}
foreach (var definition in provider.GetDocumentDefinitions())
{
yield return definition;
}
}
}
}
internal static class CultSchemaTypeNames
{
public static string FromType(Type type)
{
if (type.IsArray)
{
return FromType(type.GetElementType()!) + "[]";
}
if (type.IsGenericType)
{
var definition = type.GetGenericTypeDefinition();
var name = definition.FullName ?? definition.Name;
var tickIndex = name.IndexOf('`');
if (tickIndex >= 0)
{
name = name[..tickIndex];
}
var arguments = string.Join(", ", type.GetGenericArguments().Select(FromType));
return $"{name}<{arguments}>";
}
return type.FullName ?? type.Name;
}
public static string EscapeForLiteral(string value)
{
if (string.IsNullOrEmpty(value))
{
return value;
}
var builder = new StringBuilder(value.Length);
foreach (var character in value)
{
builder.Append(character switch
{
'\\' => "\\\\",
'"' => "\\\"",
'\r' => "\\r",
'\n' => "\\n",
'\t' => "\\t",
_ => character.ToString()
});
}
return builder.ToString();
}
}
}

69
src/GameCult.Caching/DatabaseEntry.cs
View file

@ -1,69 +0,0 @@
using System;
namespace GameCult.Caching
{
/// <summary>
/// Interface for database entries that have a human-readable name.
/// </summary>
public interface INamedEntry
{
/// <summary>
/// Human-readable name of the entry.
/// </summary>
string EntryName { get; set; }
}
/// <summary>
/// Base abstract class for all database entries in the cache.
/// For optional MessagePack integration, define concrete subclasses with [MessagePackObject] and [Key(n)] on fields.
/// The ID field can be serialized via a custom formatter in MessagePack stores.
/// </summary>
public abstract class DatabaseEntry
{
/// <summary>
/// Unique identifier for the entry. Handled by custom formatters in MessagePack integration.
/// </summary>
public Guid ID = Guid.NewGuid();
/// <inheritdoc/>
public override int GetHashCode()
{
return ID.GetHashCode();
}
/// <inheritdoc/>
public override bool Equals(object obj)
{
if (obj is DatabaseEntry entry) return entry.ID == ID;
return false;
}
}
/// <summary>
/// Generic link to a <see cref="DatabaseEntry"/> in the cache.
/// Value is computed from the cache; serializes only LinkID.
/// </summary>
public class DatabaseLink<T> : DatabaseLinkBase where T : DatabaseEntry
{
/// <summary>
/// Computed value from the cache (not serialized; regenerated post-deserialize).
/// </summary>
public T? Value => Cache?.Get<T>(LinkID);
}
/// <summary>
/// Base for database links. LinkID is serialized directly; Value is computed in subclasses.
/// </summary>
public class DatabaseLinkBase
{
/// <summary>
/// ID of the linked entry.
/// </summary>
public Guid LinkID;
/// <summary>
/// Static reference to the cache (not serialized).
/// </summary>
public static CultCache? Cache;
}
}

333
src/GameCult.Caching/README.md
View file

@ -1,241 +1,184 @@
# GameCult.Caching
`GameCult.Caching` provides a typed in-memory cache for `DatabaseEntry` models and a persistence abstraction for backing stores.
`GameCult.Caching` is an attribute-driven typed cache with a canonical
single-file MessagePack store. Domain models stay clean. The cache owns record
keys, schema ids, timestamps, and persistence metadata instead of smearing
that sludge across every class.
## Scope
## Authoring Model
The library is designed for applications that need:
- fast in-memory lookup by `Guid`
- optional lookup by human-readable name
- optional lookup by indexed fields or properties
- persistence through one or more pluggable stores
- a common model base type for runtime and storage code
It is a domain cache with persistence adapters.
## Main Concepts
### `DatabaseEntry`
All cacheable models inherit from `DatabaseEntry`.
Cacheable models are plain classes annotated with cache intent.
```csharp
using GameCult.Caching;
using MessagePack;
public class ItemData : DatabaseEntry, INamedEntry
[CultDocument("gamecult.item_data", "gamecult.item_data.v1")]
public sealed class ItemData
{
[Key(0)]
[CultName]
public string Name = string.Empty;
public string Category = string.Empty;
public int Value;
public string EntryName
{
get => Name;
set => Name = value;
}
[Key(1)]
[CultIndex]
public string Category = string.Empty;
[Key(2)]
public int Value;
}
```
### `INamedEntry`
Available cache attributes:
If a type implements `INamedEntry`, `CultCache` maintains a name-to-id lookup for it. That enables:
- `CultDocument(schemaName, schemaVersion)`
- `CultName`
- `CultIndex(alias?)`
- `CultGlobal`
- `CultReference(targetType?, many: false)`
- `GetIdByName<T>(...)`
- `GetByName<T>(...)`
If you want compile-time document metadata instead of runtime assembly
spelunking, add the `GameCult.Caching.MessagePack.Generator` analyzer to the
project that declares your `[CultDocument]` types. It emits a metadata provider
the cache registry can load directly, while reflection stays behind as fallback
for projects that have not taken the hint yet.
### Registered Indexes
## Main Concepts
Indexes are opt-in. Register an index once for a field or property, and the cache maintains a value-to-id lookup map for that member.
### Record keys and handles
`CultCache` gives each stored document a cache-owned `CultRecordKey`. You work
with typed handles instead of forcing storage identity into the domain object.
```csharp
var cache = new CultCache();
cache.RegisterIndex<ItemData>("Category");
```
After registration:
```csharp
var consumable = cache.GetByIndex<ItemData>("Category", "Consumable");
```
### Global Entries
If a `DatabaseEntry` type is marked with `GlobalSettingsAttribute`, the cache treats it as a singleton-style global entry and exposes it through `GetGlobal<T>()`.
## Core Features
- ID-based lookup
- optional name-based lookup
- optional field/property indexes
- global singleton-style entries
- pluggable backing stores
- event propagation from backing stores into the cache
- support for both single-file and multi-file persistence strategies
## Basic Usage
```csharp
using GameCult.Caching;
var cache = new CultCache();
cache.RegisterIndex<ItemData>("Category");
await cache.AddAsync(new ItemData
var handle = await cache.AddAsync(new ItemData
{
ID = Guid.NewGuid(),
Name = "Potion",
Category = "Consumable",
Value = 50
});
var potion = cache.GetByName<ItemData>("Potion");
var samePotion = cache.GetByIndex<ItemData>("Category", "Consumable");
var potion = cache.Get<ItemData>(handle.Key);
```
## Important Concept: Backing Store Routing
### Name and index lookups
`CultCache` can have:
- generic backing stores
- type/domain-specific backing stores
### Domain-Specific Stores
If you register a store with one or more domain types:
Name and index lookups are driven from attributes, not manual registration.
```csharp
cache.AddBackingStore(playerStore, typeof(PlayerData));
cache.AddBackingStore(settingsStore, typeof(AppSettings));
var byName = cache.GetByName<ItemData>("Potion");
var byIndex = cache.GetByIndex<ItemData>("Category", "Consumable");
```
then writes for those types are routed directly to those stores.
### Explicit references
### Generic Stores
If you register stores without domains:
References stay explicit in the domain. The cache will resolve them on demand,
but it will not quietly build an entire haunted object graph behind your back.
```csharp
cache.AddBackingStore(primaryStore);
cache.AddBackingStore(mirrorStore);
```
the first generic store becomes the primary write target for entries that do not have a type-specific store.
The later generic stores subscribe to earlier ones and mirror their changes.
That means:
- generic store registration order matters
- the first generic store is effectively the source of truth for generic writes
- later generic stores behave like downstream replicas
### How Multiple Backing Stores Behave
With this setup:
```csharp
cache.AddBackingStore(primaryStore);
cache.AddBackingStore(reportingMirror);
cache.AddBackingStore(playerStore, typeof(PlayerData));
```
the behavior is:
- `PlayerData` writes go to `playerStore`
- other entry types go to `primaryStore`
- `reportingMirror` mirrors change events from `primaryStore`
- `PullAllBackingStoresAsync()` pulls from all registered stores
This is not a multi-master design. If you need symmetric writes to multiple stores, implement that outside `CultCache`.
## Persistence Semantics
`AddAsync` persists to the appropriate backing store before the cache treats the write as committed. If persistence fails, the call throws instead of updating only the in-memory state.
Effects:
- memory and disk are less likely to diverge silently
- callers can react to persistence failures explicitly
## Backing Store Base Types
### `CacheBackingStore`
Abstract base for persistence implementations. It exposes:
- `PullAll()`
- `Push(DatabaseEntry entry)`
- `Delete(DatabaseEntry entry)`
- `PushAll(bool soft = false)`
### `SingleFileBackingStore`
Stores the entire set of entries in one file.
Use cases:
- small datasets
- settings-style data
- atomic snapshot persistence
### `MultiFileBackingStore`
Stores one file per entry in type-specific directories.
Use cases:
- large sets of independent entries
- change observation with `FileSystemWatcher`
- workflows where per-entry files are useful
## Rename Behavior In `MultiFileBackingStore`
For `INamedEntry` types, the filename is derived from the entry name. The backing store removes the stale old file before committing the new one.
## Example: Custom Backing Store
```csharp
using System.IO;
using System.Text;
using GameCult.Caching;
public sealed class TextItemStore : MultiFileBackingStore
[CultDocument("gamecult.quest_data", "gamecult.quest_data.v1")]
public sealed class QuestData
{
public TextItemStore(string path) : base(path)
{
}
public override byte[] Serialize(DatabaseEntry entry)
{
var item = (ItemData)entry;
return Encoding.UTF8.GetBytes($"{item.ID}|{item.Name}|{item.Category}|{item.Value}");
}
public override DatabaseEntry Deserialize(byte[] data)
{
var parts = Encoding.UTF8.GetString(data).Split('|');
return new ItemData
{
ID = Guid.Parse(parts[0]),
Name = parts[1],
Category = parts[2],
Value = int.Parse(parts[3])
};
}
public override string Extension => "item";
[Key(0)] public string Title = string.Empty;
[Key(1)] [CultReference] public CultRecordRef<ItemData> Reward;
}
```
## Typical Startup Sequence
## Persistence Shape
CultCache persistence is store-shaped, not entry-shaped:
1. store header
2. embedded schema catalog
3. records
Each persisted record carries:
- `key`
- `schemaId`
- `storedAt`
- `payload`
The domain payload stays free of storage metadata.
## Backing Store
CultCache now has one sanctioned persistence format:
- `SingleFileMessagePackBackingStore`
It persists the whole store as:
1. store format version
2. embedded schema catalog
3. persisted records
Each persisted record carries:
- `key`
- `schemaId`
- `storedAt`
- `payload`
## Happy Path
```csharp
var cache = new CultCache();
cache.RegisterIndex<PlayerData>("Email");
using GameCult.Caching;
using GameCult.Caching.MessagePack;
var store = new TextItemStore("Data");
cache.AddBackingStore(store, typeof(ItemData));
var cache = await CultCacheMessagePack.OpenAsync("Data.msgpack");
await cache.PullAllBackingStoresAsync();
var handle = await cache.UpsertAsync(new ItemData
{
Name = "Potion",
Category = "Consumable",
Value = 50
});
await cache.FlushAsync();
```
If you want the explicit assembly rite, the lower-level `CultCache` +
`SingleFileMessagePackBackingStore` path is still there. The helper is just the
blessed front door.
## Persistence discipline
Persistence stays manual by default. Mutations mark the cache and attached
backing stores dirty; nothing silently flushes every object write just because
that felt convenient in the moment.
Useful surfaces:
- `CultCacheMessagePack.OpenAsync(path)`
- `cache.IsDirty`
- `store.IsDirty`
- `cache.UpsertAsync(document)`
- `cache.FlushAllBackingStores()`
- `cache.FlushAsync()`
- `cache.FlushBackingStore(store)`
- `cache.PrepareForReloadOrShutdown()`
- `cache.PrepareForReloadOrShutdownAsync()`
- `cache.FlushAttachedStoresOnDispose`
- `store.FlushOnDispose`
This is built for callers like Aquarium that want durable runtime state while
still choosing *when* to pay the write cost.
Single-file MessagePack remains a single-writer format. The atomic replace path
protects against partial writes, not competing writers trampling each other.
## Schema migration diagnostics
CultCache does not quietly fall back from exact `schemaId` resolution to
"close enough, probably." When a persisted schema drifts, the cache now emits a
typed migration report describing:
- exact vs compatible-drift resolution
- ignored extra slots
- defaulted missing slots
- warning codes/messages
Compatibility rules and canonical fixture receipts live in
`Contracts/cultcache-schema-compatibility.md`.

52
src/GameCult.Caching/ReflectionExtensions.cs
View file

@ -5,55 +5,27 @@ using System.Reflection;
namespace GameCult.Caching
{
/// <summary>
/// Provides cached reflection helpers used by the cache infrastructure.
/// </summary>
public static class ReflectionExtensions
internal static class ReflectionExtensions
{
private static Dictionary<Type, Type[]> ParentTypes = new Dictionary<Type, Type[]>();
/// <summary>
/// Gets all base types for the supplied type, caching the result.
/// </summary>
/// <param name="type">The type whose parent types should be returned.</param>
/// <returns>An array of base types ordered from immediate parent upward.</returns>
public static Type[] GetParentTypes(this Type type)
public static IEnumerable<Type> GetAttributedDocumentTypes()
{
if (ParentTypes.ContainsKey(type))
return ParentTypes[type];
return ParentTypes[type] = type.GetParents().ToArray();
return AppDomain.CurrentDomain.GetAssemblies()
.Where(assembly => !assembly.IsDynamic)
.SelectMany(SafeGetTypes)
.Where(type => type is { IsAbstract: false, IsInterface: false })
.Where(type => type.GetCustomAttribute<CultDocumentAttribute>() != null);
}
private static IEnumerable<Type> GetParents(this Type type)
public static IEnumerable<Type> SafeGetTypes(Assembly assembly)
{
// is there any base type?
if (type == null)
try
{
yield break;
return assembly.GetTypes();
}
// return all inherited types
var currentBaseType = type.BaseType;
while (currentBaseType != null)
catch (ReflectionTypeLoadException ex)
{
yield return currentBaseType;
currentBaseType = currentBaseType.BaseType;
return ex.Types.Where(type => type != null)!;
}
}
private static Dictionary<Type, Type[]> ChildClasses = new Dictionary<Type, Type[]>();
/// <summary>
/// Gets all loaded types assignable to the supplied type, caching the result.
/// </summary>
/// <param name="type">The base type or interface to search from.</param>
/// <returns>An array of matching loaded types.</returns>
public static Type[] GetAllChildClasses(this Type type)
{
if (ChildClasses.ContainsKey(type))
return ChildClasses[type];
return ChildClasses[type] = AppDomain.CurrentDomain.GetAssemblies()
.SelectMany(ass => ass.GetTypes()).Where(type.IsAssignableFrom).ToArray();
}
}
}

293
src/GameCult.Mesh/CultMesh.cs Normal file
View file

@ -0,0 +1,293 @@
using System;
using System.IO;
using System.Threading.Tasks;
using GameCult.Caching;
using GameCult.Caching.MessagePack;
using GameCult.Networking;
namespace GameCult.Mesh
{
/// <summary>
/// Options for creating a local CultMesh node.
/// </summary>
public sealed class CultMeshNodeOptions
{
/// <summary>
/// Gets or sets the cache-open options used for the underlying CultCache.
/// </summary>
public CultCacheOpenOptions? CacheOptions { get; set; }
/// <summary>
/// Gets or sets the server security configuration. When omitted, development defaults are used.
/// </summary>
public ServerSecurityOptions? Security { get; set; }
/// <summary>
/// Gets or sets whether the server should start immediately after creation.
/// </summary>
public bool StartServer { get; set; } = true;
/// <summary>
/// Gets or sets the mesh database sharding options used for the hosted CultCache.
/// </summary>
public CultNetDatabaseOptions? DatabaseOptions { get; set; }
/// <summary>
/// Gets or sets whether CultMesh should attach a file-backed authoritative shard-log store when none is supplied.
/// </summary>
public bool EnableDurableShardLogs { get; set; }
/// <summary>
/// Gets or sets the directory used for file-backed authoritative shard logs.
/// </summary>
public string? ShardLogPath { get; set; }
/// <summary>
/// Gets or sets the database server bridge options used for shard routing and forwarding.
/// </summary>
public CultNetDatabaseServerOptions? DatabaseServerOptions { get; set; }
/// <summary>
/// Gets or sets an optional callback used to customize the server before start.
/// </summary>
public Action<Server>? ConfigureServer { get; set; }
internal CultNetHostOptions ToCultNetOptions(string cachePath)
{
return new CultNetHostOptions
{
CacheOptions = CacheOptions,
Security = Security,
StartServer = StartServer,
DatabaseOptions = CreateDatabaseOptions(cachePath),
DatabaseServerOptions = DatabaseServerOptions,
ConfigureServer = ConfigureServer
};
}
private CultNetDatabaseOptions? CreateDatabaseOptions(string cachePath)
{
if (!EnableDurableShardLogs)
{
return DatabaseOptions;
}
var source = DatabaseOptions ?? new CultNetDatabaseOptions();
if (source.MutationLogStore != null)
{
return source;
}
return new CultNetDatabaseOptions
{
RuntimeId = source.RuntimeId,
Shards = source.Shards,
ClientAuthorityScopes = source.ClientAuthorityScopes,
DocumentRegistry = source.DocumentRegistry,
MutationLogStore = new CultNetFileShardMutationLogStore(ResolveShardLogPath(cachePath))
};
}
private string ResolveShardLogPath(string cachePath)
{
if (!string.IsNullOrWhiteSpace(ShardLogPath))
{
return ShardLogPath!;
}
var fullPath = Path.GetFullPath(cachePath);
var directory = Path.GetDirectoryName(fullPath) ?? Directory.GetCurrentDirectory();
var name = Path.GetFileNameWithoutExtension(fullPath);
return Path.Combine(directory, name + ".cultmesh", "shard-logs");
}
}
/// <summary>
/// A locally hosted CultMesh node over CultCache, CultNet transport, and the mesh database facade.
/// </summary>
public sealed class CultMeshNode : IDisposable
{
private readonly CultNetHost _host;
internal CultMeshNode(CultNetHost host)
{
_host = host ?? throw new ArgumentNullException(nameof(host));
}
/// <summary>
/// Gets the durable cache used by the node.
/// </summary>
public CultCache Cache => _host.Cache;
/// <summary>
/// Gets the canonical single-file MessagePack store, when one is attached.
/// </summary>
public SingleFileMessagePackBackingStore? Store => _host.Store;
/// <summary>
/// Gets the underlying CultNet server.
/// </summary>
public Server Server => _host.Server;
/// <summary>
/// Gets the distributed realtime database facade.
/// </summary>
public CultNetDatabase Database => _host.Database;
/// <summary>
/// Gets the schema-v0 server bridge for distributed database messages.
/// </summary>
public CultNetDatabaseServer DatabaseServer => _host.DatabaseServer;
/// <summary>
/// Starts the underlying server.
/// </summary>
public void Start()
{
_host.Start();
}
/// <summary>
/// Flushes the durable cache.
/// </summary>
public Task FlushAsync(bool soft = false)
{
return _host.FlushAsync(soft);
}
/// <inheritdoc />
public void Dispose()
{
_host.Dispose();
}
}
/// <summary>
/// Public CultMesh entrypoints.
/// </summary>
public static class CultMesh
{
/// <summary>
/// Creates a local CultMesh node with a durable CultCache and the canonical MessagePack backing store.
/// </summary>
public static async Task<CultMeshNode> CreateNodeAsync(string cachePath, CultMeshNodeOptions? options = null)
{
var host = await CultNetLocal.CreateHostAsync(
cachePath,
(options ?? new CultMeshNodeOptions()).ToCultNetOptions(cachePath)).ConfigureAwait(false);
return new CultMeshNode(host);
}
/// <summary>
/// Creates and starts a local CultMesh node with development-friendly defaults.
/// </summary>
public static Task<CultMeshNode> StartNodeAsync(string cachePath, CultMeshNodeOptions? options = null)
{
options ??= new CultMeshNodeOptions();
options.StartServer = true;
return CreateNodeAsync(cachePath, options);
}
/// <summary>
/// Creates a local reactive catalog for discovered Verses.
/// </summary>
public static CultMeshVerseCatalog CreateVerseCatalog()
{
return new CultMeshVerseCatalog();
}
/// <summary>
/// Creates a local reactive catalog for discovered peers.
/// </summary>
public static CultMeshPeerCatalog CreatePeerCatalog()
{
return new CultMeshPeerCatalog();
}
/// <summary>
/// Creates a local authority lease catalog.
/// </summary>
public static CultMeshAuthorityLeaseCatalog CreateAuthorityLeaseCatalog()
{
return new CultMeshAuthorityLeaseCatalog();
}
/// <summary>
/// Creates a committer for writing quorum simulation facts to a mesh database.
/// </summary>
public static CultMeshSimulationFactCommitter CreateSimulationFactCommitter(CultNetDatabase database)
{
return new CultMeshSimulationFactCommitter(database);
}
/// <summary>
/// Creates a gameplay-facing session facade over a CultMesh node.
/// </summary>
public static CultMeshGameSession CreateGameSession(
CultMeshNode node,
CultMeshGameSessionOptions? options = null)
{
return new CultMeshGameSession(node, options);
}
/// <summary>
/// Attaches Verse discovery responses to a CultMesh node.
/// </summary>
public static CultMeshVerseDiscoveryServer ServeVerseCatalog(
CultMeshNode node,
CultMeshVerseCatalog catalog)
{
if (node == null) throw new ArgumentNullException(nameof(node));
return new CultMeshVerseDiscoveryServer(node.Server, catalog);
}
/// <summary>
/// Creates a client for fetching Verse catalogs from discovery endpoints.
/// </summary>
public static CultMeshVerseDiscoveryClient CreateVerseDiscoveryClient(
CultMeshVerseDiscoveryClientOptions? options = null)
{
return new CultMeshVerseDiscoveryClient(options);
}
/// <summary>
/// Attaches peer exchange responses to a CultMesh node.
/// </summary>
public static CultMeshPeerExchangeServer ServePeerExchange(
CultMeshNode node,
CultMeshPeerCatalog catalog)
{
if (node == null) throw new ArgumentNullException(nameof(node));
return new CultMeshPeerExchangeServer(node.Server, catalog);
}
/// <summary>
/// Creates a client for fetching peer cards from peer exchange endpoints.
/// </summary>
public static CultMeshPeerExchangeClient CreatePeerExchangeClient(
CultMeshPeerExchangeClientOptions? options = null)
{
return new CultMeshPeerExchangeClient(options);
}
/// <summary>
/// Creates a CultNet client configured for CultMesh use.
/// </summary>
public static Client CreateClient(ClientSecurityOptions? security = null, Action<Client>? configureClient = null)
{
return CultNetLocal.CreateClient(security, configureClient);
}
/// <summary>
/// Creates and connects a CultNet client to a CultMesh node.
/// </summary>
public static Client ConnectClient(
string host = "localhost",
int port = 3075,
ClientSecurityOptions? security = null,
Action<Client>? configureClient = null)
{
return CultNetLocal.ConnectClient(host, port, security, configureClient);
}
}
}

140
src/GameCult.Mesh/CultMeshAuthority.cs Normal file
View file

@ -0,0 +1,140 @@
using System;
using System.Collections.Generic;
using System.Linq;
namespace GameCult.Mesh
{
/// <summary>
/// Grants one peer authority to perform roles for a Verse over a bounded interval.
/// </summary>
public sealed class CultMeshAuthorityLease
{
/// <summary>
/// Creates an authority lease.
/// </summary>
public CultMeshAuthorityLease(
string leaseId,
string verseId,
string peerId,
IEnumerable<string> roles,
IEnumerable<string>? shardIds,
string issuerRuntimeId,
DateTimeOffset validFrom,
DateTimeOffset expiresAt,
string? signature = null)
{
LeaseId = RequireNonEmpty(leaseId, nameof(leaseId));
VerseId = RequireNonEmpty(verseId, nameof(verseId));
PeerId = RequireNonEmpty(peerId, nameof(peerId));
Roles = Clean(roles);
ShardIds = Clean(shardIds);
IssuerRuntimeId = RequireNonEmpty(issuerRuntimeId, nameof(issuerRuntimeId));
ValidFrom = validFrom;
ExpiresAt = expiresAt;
Signature = signature;
if (expiresAt <= validFrom)
{
throw new ArgumentException("Lease expiry must be after its valid-from timestamp.", nameof(expiresAt));
}
}
/// <summary>Gets the stable lease id.</summary>
public string LeaseId { get; }
/// <summary>Gets the Verse id governed by the lease.</summary>
public string VerseId { get; }
/// <summary>Gets the peer id receiving authority.</summary>
public string PeerId { get; }
/// <summary>Gets the roles granted by this lease.</summary>
public IReadOnlyList<string> Roles { get; }
/// <summary>Gets shard ids covered by this lease. Empty means every shard in the Verse.</summary>
public IReadOnlyList<string> ShardIds { get; }
/// <summary>Gets the runtime that issued the lease.</summary>
public string IssuerRuntimeId { get; }
/// <summary>Gets the first timestamp when the lease is valid.</summary>
public DateTimeOffset ValidFrom { get; }
/// <summary>Gets the timestamp when the lease expires.</summary>
public DateTimeOffset ExpiresAt { get; }
/// <summary>Gets an optional signature over the lease.</summary>
public string? Signature { get; }
/// <summary>
/// Returns whether this lease is valid at the supplied time.
/// </summary>
public bool IsValidAt(DateTimeOffset at)
{
return at >= ValidFrom && at < ExpiresAt;
}
/// <summary>
/// Returns whether this lease covers a peer card, role, and shard.
/// </summary>
public bool Covers(CultMeshPeerCard peer, string role, string? shardId = null, DateTimeOffset? at = null)
{
if (peer == null) throw new ArgumentNullException(nameof(peer));
if (string.IsNullOrWhiteSpace(role)) throw new ArgumentException("Value must be non-empty.", nameof(role));
var time = at ?? DateTimeOffset.UtcNow;
return IsValidAt(time) &&
string.Equals(VerseId, peer.VerseId, StringComparison.Ordinal) &&
string.Equals(PeerId, peer.PeerId, StringComparison.Ordinal) &&
string.Equals(LeaseId, peer.AuthorityLeaseId, StringComparison.Ordinal) &&
Roles.Contains(role, StringComparer.Ordinal) &&
peer.Roles.Contains(role, StringComparer.Ordinal) &&
(string.IsNullOrWhiteSpace(shardId) ||
ShardIds.Count == 0 ||
ShardIds.Contains(shardId!, StringComparer.Ordinal));
}
private static string[] Clean(IEnumerable<string>? values)
{
return values?.Where(value => !string.IsNullOrWhiteSpace(value)).Distinct(StringComparer.Ordinal).ToArray()
?? Array.Empty<string>();
}
private static string RequireNonEmpty(string value, string paramName)
{
return string.IsNullOrWhiteSpace(value)
? throw new ArgumentException("Value must be non-empty.", paramName)
: value;
}
}
/// <summary>
/// Local authority lease catalog.
/// </summary>
public sealed class CultMeshAuthorityLeaseCatalog
{
private readonly Dictionary<string, CultMeshAuthorityLease> _leases = new(StringComparer.Ordinal);
/// <summary>Gets all known leases.</summary>
public IReadOnlyList<CultMeshAuthorityLease> Leases => _leases.Values.OrderBy(lease => lease.LeaseId, StringComparer.Ordinal).ToArray();
/// <summary>Adds or replaces an authority lease.</summary>
public void Upsert(CultMeshAuthorityLease lease)
{
if (lease == null) throw new ArgumentNullException(nameof(lease));
_leases[lease.LeaseId] = lease;
}
/// <summary>Gets a lease by id, if known.</summary>
public CultMeshAuthorityLease? Get(string leaseId)
{
if (string.IsNullOrWhiteSpace(leaseId)) throw new ArgumentException("Value must be non-empty.", nameof(leaseId));
return _leases.TryGetValue(leaseId, out var lease) ? lease : null;
}
/// <summary>
/// Returns whether a peer card is authorized for a role and optional shard.
/// </summary>
public bool IsAuthorized(CultMeshPeerCard peer, string role, string? shardId = null, DateTimeOffset? at = null)
{
if (peer == null) throw new ArgumentNullException(nameof(peer));
if (string.IsNullOrWhiteSpace(peer.AuthorityLeaseId))
{
return false;
}
return _leases.TryGetValue(peer.AuthorityLeaseId!, out var lease) &&
lease.Covers(peer, role, shardId, at);
}
}
}

234
src/GameCult.Mesh/CultMeshGameSession.cs Normal file
View file

@ -0,0 +1,234 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using GameCult.Caching;
using GameCult.Networking;
using R3;
namespace GameCult.Mesh
{
/// <summary>
/// Options for creating a gameplay-facing CultMesh session facade.
/// </summary>
public sealed class CultMeshGameSessionOptions
{
/// <summary>
/// Gets or sets the Verse catalog used by the session.
/// </summary>
public CultMeshVerseCatalog? VerseCatalog { get; set; }
/// <summary>
/// Gets or sets the peer catalog used by the session.
/// </summary>
public CultMeshPeerCatalog? PeerCatalog { get; set; }
/// <summary>
/// Gets or sets the authority lease catalog used by the session.
/// </summary>
public CultMeshAuthorityLeaseCatalog? AuthorityLeases { get; set; }
/// <summary>
/// Gets or sets simulation consensus options.
/// </summary>
public CultNetSimulationConsensusOptions? ConsensusOptions { get; set; }
/// <summary>
/// Gets or sets whether incoming observation messages should update the session hub.
/// </summary>
public bool ServeSimulationObservations { get; set; } = true;
/// <summary>
/// Gets or sets whether the node should answer Verse catalog requests from the session catalog.
/// </summary>
public bool ServeVerseDiscovery { get; set; } = true;
/// <summary>
/// Gets or sets whether the node should answer peer exchange requests from the session peer catalog.
/// </summary>
public bool ServePeerExchange { get; set; } = true;
}
/// <summary>
/// Gameplay-facing facade over common CultMesh runtime pieces.
/// </summary>
public sealed class CultMeshGameSession : IDisposable
{
private readonly List<IDisposable> _owned = new();
private readonly HashSet<string> _committedFactIds = new(StringComparer.Ordinal);
private bool _disposed;
/// <summary>
/// Creates a game session facade.
/// </summary>
public CultMeshGameSession(CultMeshNode node, CultMeshGameSessionOptions? options = null)
{
Node = node ?? throw new ArgumentNullException(nameof(node));
options ??= new CultMeshGameSessionOptions();
VerseCatalog = options.VerseCatalog ?? new CultMeshVerseCatalog();
PeerCatalog = options.PeerCatalog ?? new CultMeshPeerCatalog();
AuthorityLeases = options.AuthorityLeases ?? new CultMeshAuthorityLeaseCatalog();
ObservationHub = new CultNetSimulationObservationHub(options.ConsensusOptions);
FactCommitter = new CultMeshSimulationFactCommitter(Node.Database);
if (options.VerseCatalog == null)
{
_owned.Add(VerseCatalog);
}
if (options.PeerCatalog == null)
{
_owned.Add(PeerCatalog);
}
if (options.ServeSimulationObservations)
{
_owned.Add(new CultNetSimulationObservationServer(Node.Server, ObservationHub));
}
if (options.ServeVerseDiscovery)
{
_owned.Add(new CultMeshVerseDiscoveryServer(Node.Server, VerseCatalog));
}
if (options.ServePeerExchange)
{
_owned.Add(new CultMeshPeerExchangeServer(Node.Server, PeerCatalog));
}
}
/// <summary>
/// Gets the underlying node.
/// </summary>
public CultMeshNode Node { get; }
/// <summary>
/// Gets the session Verse catalog.
/// </summary>
public CultMeshVerseCatalog VerseCatalog { get; }
/// <summary>
/// Gets the session peer catalog.
/// </summary>
public CultMeshPeerCatalog PeerCatalog { get; }
/// <summary>
/// Gets the session authority lease catalog.
/// </summary>
public CultMeshAuthorityLeaseCatalog AuthorityLeases { get; }
/// <summary>
/// Gets the session simulation observation hub.
/// </summary>
public CultNetSimulationObservationHub ObservationHub { get; }
/// <summary>
/// Gets the session simulation fact committer.
/// </summary>
public CultMeshSimulationFactCommitter FactCommitter { get; }
/// <summary>
/// Watches consensus candidate updates.
/// </summary>
public Observable<CultNetSimulationConsensusCandidate> WatchCandidates()
{
ThrowIfDisposed();
return ObservationHub.WatchCandidates();
}
/// <summary>
/// Watches committed simulation facts.
/// </summary>
public Observable<CultNetDatabaseChange<CultMeshSimulationFact>> WatchSimulationFacts()
{
ThrowIfDisposed();
return Node.Database.Watch<CultMeshSimulationFact>();
}
/// <summary>
/// Applies a local client prediction through the underlying database.
/// </summary>
public Task<CultRecordHandle<T>> PredictAsync<T>(CultRecordKey key, T document) where T : class
{
ThrowIfDisposed();
return Node.Database.PutPredictedAsync(key, document);
}
/// <summary>
/// Submits one witness observation and returns the current candidates for that subject.
/// </summary>
public IReadOnlyList<CultNetSimulationConsensusCandidate> SubmitObservation(
CultNetSimulationObservation observation)
{
ThrowIfDisposed();
return ObservationHub.Submit(observation);
}
/// <summary>
/// Submits one witness observation and commits any new quorum facts it creates.
/// </summary>
public async Task<IReadOnlyList<CultRecordHandle<CultMeshSimulationFact>>> SubmitAndCommitAsync(
CultNetSimulationObservation observation)
{
ThrowIfDisposed();
var candidates = ObservationHub.Submit(observation);
return await CommitQuorumCandidatesAsync(candidates).ConfigureAwait(false);
}
/// <summary>
/// Commits quorum candidates that have not already been committed by this session.
/// </summary>
public async Task<IReadOnlyList<CultRecordHandle<CultMeshSimulationFact>>> CommitQuorumCandidatesAsync(
IEnumerable<CultNetSimulationConsensusCandidate> candidates)
{
ThrowIfDisposed();
if (candidates == null) throw new ArgumentNullException(nameof(candidates));
var handles = new List<CultRecordHandle<CultMeshSimulationFact>>();
foreach (var candidate in candidates.Where(candidate => candidate.HasQuorum))
{
var factId = CultMeshSimulationFact.ComputeFactId(candidate);
if (!_committedFactIds.Add(factId))
{
continue;
}
var key = CultMeshSimulationFact.CreateRecordKey(candidate);
if (await Node.Database.GetAsync<CultMeshSimulationFact>(key).ConfigureAwait(false) != null)
{
continue;
}
handles.Add(await FactCommitter.CommitAsync(candidate).ConfigureAwait(false));
}
return handles;
}
/// <inheritdoc />
public void Dispose()
{
if (_disposed)
{
return;
}
_disposed = true;
foreach (var owned in _owned)
{
owned.Dispose();
}
ObservationHub.Dispose();
_owned.Clear();
}
private void ThrowIfDisposed()
{
if (_disposed)
{
throw new ObjectDisposedException(nameof(CultMeshGameSession));
}
}
}
}

434
src/GameCult.Mesh/CultMeshPeerExchange.cs Normal file
View file

@ -0,0 +1,434 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using GameCult.Networking;
using LiteNetLib;
using R3;
namespace GameCult.Mesh
{
/// <summary>
/// Well-known CultMesh peer roles.
/// </summary>
public static class CultMeshPeerRoles
{
/// <summary>Peer can answer discovery requests.</summary>
public const string Discovery = "discovery";
/// <summary>Peer can serve authoritative shard logs.</summary>
public const string ShardPrimary = "shard-primary";
/// <summary>Peer can serve replicated shard logs.</summary>
public const string ShardReplica = "shard-replica";
/// <summary>Peer can serve read traffic.</summary>
public const string ReadReplica = "read-replica";
/// <summary>Peer can observe simulation facts.</summary>
public const string SimulationObserver = "simulation-observer";
}
/// <summary>
/// Candidate peer contact information for a Verse.
/// </summary>
public sealed class CultMeshPeerCard
{
/// <summary>
/// Creates a peer card.
/// </summary>
public CultMeshPeerCard(
string peerId,
string verseId,
IEnumerable<string> endpoints,
IEnumerable<string>? roles = null,
IEnumerable<string>? shardIds = null,
string? region = null,
string? authorityLeaseId = null,
string? expiresAt = null,
string? signature = null)
{
PeerId = RequireNonEmpty(peerId, nameof(peerId));
VerseId = RequireNonEmpty(verseId, nameof(verseId));
Endpoints = Clean(endpoints);
Roles = Clean(roles);
ShardIds = Clean(shardIds);
Region = region;
AuthorityLeaseId = authorityLeaseId;
ExpiresAt = expiresAt;
Signature = signature;
}
/// <summary>Gets the stable peer id.</summary>
public string PeerId { get; }
/// <summary>Gets the Verse id this peer participates in.</summary>
public string VerseId { get; }
/// <summary>Gets reachable endpoints.</summary>
public IReadOnlyList<string> Endpoints { get; }
/// <summary>Gets advertised roles.</summary>
public IReadOnlyList<string> Roles { get; }
/// <summary>Gets shard ids this peer can serve or observe.</summary>
public IReadOnlyList<string> ShardIds { get; }
/// <summary>Gets an optional region/locality label.</summary>
public string? Region { get; }
/// <summary>Gets the authority lease id, when present.</summary>
public string? AuthorityLeaseId { get; }
/// <summary>Gets the expiry timestamp for this card.</summary>
public string? ExpiresAt { get; }
/// <summary>Gets an optional signature over the card.</summary>
public string? Signature { get; }
/// <summary>
/// Returns whether the card advertises a role.
/// </summary>
public bool HasRole(string role)
{
return Roles.Contains(role, StringComparer.Ordinal);
}
private static string[] Clean(IEnumerable<string>? values)
{
return values?.Where(value => !string.IsNullOrWhiteSpace(value)).Distinct(StringComparer.Ordinal).ToArray()
?? Array.Empty<string>();
}
private static string RequireNonEmpty(string value, string paramName)
{
return string.IsNullOrWhiteSpace(value)
? throw new ArgumentException("Value must be non-empty.", paramName)
: value;
}
}
/// <summary>
/// Conversion helpers for CultMesh peer exchange messages.
/// </summary>
public static class CultMeshPeerMessages
{
/// <summary>
/// Converts a peer card to its schema-v0 wire shape.
/// </summary>
public static CultMeshPeerCardMessage ToMessage(this CultMeshPeerCard card)
{
if (card == null) throw new ArgumentNullException(nameof(card));
return new CultMeshPeerCardMessage
{
PeerId = card.PeerId,
VerseId = card.VerseId,
Endpoints = card.Endpoints.ToArray(),
Roles = card.Roles.ToArray(),
ShardIds = card.ShardIds.ToArray(),
Region = card.Region,
AuthorityLeaseId = card.AuthorityLeaseId,
ExpiresAt = card.ExpiresAt,
Signature = card.Signature
};
}
/// <summary>
/// Converts a wire peer card to the local public API shape.
/// </summary>
public static CultMeshPeerCard ToPeerCard(this CultMeshPeerCardMessage message)
{
if (message == null) throw new ArgumentNullException(nameof(message));
return new CultMeshPeerCard(
message.PeerId,
message.VerseId,
message.Endpoints,
message.Roles,
message.ShardIds,
message.Region,
message.AuthorityLeaseId,
message.ExpiresAt,
message.Signature);
}
}
/// <summary>
/// Reactive local catalog for discovered CultMesh peers.
/// </summary>
public sealed class CultMeshPeerCatalog : IDisposable
{
private readonly Dictionary<string, CultMeshPeerCard> _peers = new(StringComparer.Ordinal);
private readonly Subject<CultMeshPeerCard> _updates = new();
private bool _disposed;
/// <summary>Gets all known peers.</summary>
public IReadOnlyList<CultMeshPeerCard> Peers => _peers.Values.OrderBy(peer => peer.PeerId, StringComparer.Ordinal).ToArray();
/// <summary>Watches peer card updates.</summary>
public Observable<CultMeshPeerCard> Watch()
{
ThrowIfDisposed();
return _updates;
}
/// <summary>Adds or replaces a peer card.</summary>
public void Upsert(CultMeshPeerCard card)
{
ThrowIfDisposed();
if (card == null) throw new ArgumentNullException(nameof(card));
_peers[card.PeerId] = card;
_updates.OnNext(card);
}
/// <summary>Adds or replaces peer cards from a wire response.</summary>
public void Upsert(CultMeshPeerExchangeResponseMessage response)
{
ThrowIfDisposed();
if (response == null) throw new ArgumentNullException(nameof(response));
foreach (var peer in response.Peers)
{
Upsert(peer.ToPeerCard());
}
}
/// <summary>Finds peers by Verse and optional role.</summary>
public IReadOnlyList<CultMeshPeerCard> Find(string verseId, string? role = null)
{
ThrowIfDisposed();
if (string.IsNullOrWhiteSpace(verseId)) throw new ArgumentException("Value must be non-empty.", nameof(verseId));
return _peers.Values
.Where(peer => string.Equals(peer.VerseId, verseId, StringComparison.Ordinal))
.Where(peer => string.IsNullOrWhiteSpace(role) || peer.HasRole(role!))
.OrderBy(peer => peer.PeerId, StringComparer.Ordinal)
.ToArray();
}
/// <inheritdoc />
public void Dispose()
{
if (_disposed)
{
return;
}
_disposed = true;
_updates.Dispose();
_peers.Clear();
}
private void ThrowIfDisposed()
{
if (_disposed)
{
throw new ObjectDisposedException(nameof(CultMeshPeerCatalog));
}
}
}
/// <summary>
/// Answers CultMesh peer exchange requests from a local peer catalog.
/// </summary>
public sealed class CultMeshPeerExchangeServer : IDisposable
{
private readonly Server _server;
private readonly CultMeshPeerCatalog _catalog;
private readonly Func<CultMeshPeerExchangeRequestMessage, NetPeer, Task> _requestHandler;
private bool _disposed;
/// <summary>
/// Creates and attaches a peer exchange bridge.
/// </summary>
public CultMeshPeerExchangeServer(Server server, CultMeshPeerCatalog catalog)
{
_server = server ?? throw new ArgumentNullException(nameof(server));
_catalog = catalog ?? throw new ArgumentNullException(nameof(catalog));
_requestHandler = HandleRequestAsync;
_server.OnCultNet(_requestHandler);
}
/// <summary>
/// Creates a peer exchange response for a request.
/// </summary>
public CultMeshPeerExchangeResponseMessage CreateResponse(CultMeshPeerExchangeRequestMessage request)
{
if (request == null) throw new ArgumentNullException(nameof(request));
if (string.IsNullOrWhiteSpace(request.VerseId))
{
throw new ArgumentException("Peer exchange requires a verseId.", nameof(request));
}
var known = request.KnownPeerIds == null || request.KnownPeerIds.Length == 0
? null
: new HashSet<string>(request.KnownPeerIds, StringComparer.Ordinal);
var roles = request.Roles == null || request.Roles.Length == 0
? null
: new HashSet<string>(request.Roles, StringComparer.Ordinal);
var peers = _catalog.Find(request.VerseId)
.Where(peer => known == null || !known.Contains(peer.PeerId))
.Where(peer => roles == null || peer.Roles.Any(role => roles.Contains(role)))
.Take(request.Limit ?? int.MaxValue)
.Select(peer => peer.ToMessage())
.ToArray();
return new CultMeshPeerExchangeResponseMessage
{
MessageId = string.IsNullOrWhiteSpace(request.MessageId)
? Guid.NewGuid().ToString("N")
: request.MessageId,
Peers = peers
};
}
/// <inheritdoc />
public void Dispose()
{
if (_disposed)
{
return;
}
_disposed = true;
_server.RemoveCultNetMessageListener<CultMeshPeerExchangeRequestMessage>(_requestHandler);
}
private Task HandleRequestAsync(CultMeshPeerExchangeRequestMessage request, NetPeer peer)
{
peer.SendCultNet(CreateResponse(request));
return Task.CompletedTask;
}
}
/// <summary>
/// Options for schema-v0 CultMesh peer exchange clients.
/// </summary>
public sealed class CultMeshPeerExchangeClientOptions
{
/// <summary>
/// Gets or sets client security options used to connect to peer exchange endpoints.
/// </summary>
public ClientSecurityOptions? Security { get; set; }
/// <summary>
/// Gets or sets how long to wait for a connection before failing.
/// </summary>
public TimeSpan ConnectTimeout { get; set; } = TimeSpan.FromSeconds(5);
/// <summary>
/// Gets or sets how long to wait for a peer exchange response before failing.
/// </summary>
public TimeSpan ResponseTimeout { get; set; } = TimeSpan.FromSeconds(5);
/// <summary>
/// Gets or sets a callback used to customize each ephemeral exchange client.
/// </summary>
public Action<Client>? ConfigureClient { get; set; }
}
/// <summary>
/// Fetches peer cards from CultMesh peers.
/// </summary>
public sealed class CultMeshPeerExchangeClient
{
private readonly CultMeshPeerExchangeClientOptions _options;
/// <summary>
/// Creates a peer exchange client.
/// </summary>
public CultMeshPeerExchangeClient(CultMeshPeerExchangeClientOptions? options = null)
{
_options = options ?? new CultMeshPeerExchangeClientOptions();
}
/// <summary>
/// Fetches peer cards from one endpoint.
/// </summary>
public async Task<CultMeshPeerExchangeResponseMessage> FetchAsync(
string endpoint,
CultMeshPeerExchangeRequestMessage request)
{
if (string.IsNullOrWhiteSpace(endpoint)) throw new ArgumentException("Value must be non-empty.", nameof(endpoint));
if (request == null) throw new ArgumentNullException(nameof(request));
if (string.IsNullOrWhiteSpace(request.VerseId)) throw new ArgumentException("Peer exchange requires a verseId.", nameof(request));
var (host, port) = CultNetSchemaWriteForwarder.ParseEndpoint(endpoint);
var messageId = Guid.NewGuid().ToString("N");
var completion = new TaskCompletionSource<CultMeshPeerExchangeResponseMessage>(
TaskCreationOptions.RunContinuationsAsynchronously);
using var client = new Client(_options.Security ?? ClientSecurityOptions.Development())
{
AllowUnverifiedCultNetMessages = true
};
_options.ConfigureClient?.Invoke(client);
client.OnCultNet<CultMeshPeerExchangeResponseMessage>(response =>
{
if (string.Equals(response.MessageId, messageId, StringComparison.Ordinal))
{
completion.TrySetResult(response);
}
});
client.OnCultNet<CultNetErrorMessage>(error =>
completion.TrySetException(new InvalidOperationException(error.Error)));
client.Connect(host, port);
await WaitForConnectionAsync(client, endpoint).ConfigureAwait(false);
client.SendCultNet(new CultMeshPeerExchangeRequestMessage
{
MessageId = messageId,
VerseId = request.VerseId,
Roles = request.Roles,
KnownPeerIds = request.KnownPeerIds,
Limit = request.Limit
});
return await WaitForResponseAsync(completion.Task, endpoint).ConfigureAwait(false);
}
/// <summary>
/// Fetches peer cards from endpoints and upserts every response into a local peer catalog.
/// </summary>
public async Task<int> DiscoverAsync(
CultMeshPeerCatalog catalog,
IEnumerable<string> endpoints,
string verseId,
IEnumerable<string>? roles = null,
int? limit = null)
{
if (catalog == null) throw new ArgumentNullException(nameof(catalog));
if (endpoints == null) throw new ArgumentNullException(nameof(endpoints));
if (string.IsNullOrWhiteSpace(verseId)) throw new ArgumentException("Value must be non-empty.", nameof(verseId));
var count = 0;
foreach (var endpoint in endpoints.Where(value => !string.IsNullOrWhiteSpace(value)).Distinct(StringComparer.Ordinal))
{
var response = await FetchAsync(endpoint, new CultMeshPeerExchangeRequestMessage
{
VerseId = verseId,
Roles = roles?.Where(value => !string.IsNullOrWhiteSpace(value)).Distinct(StringComparer.Ordinal).ToArray(),
KnownPeerIds = catalog.Find(verseId).Select(peer => peer.PeerId).ToArray(),
Limit = limit
}).ConfigureAwait(false);
catalog.Upsert(response);
count += response.Peers.Length;
}
return count;
}
private async Task WaitForConnectionAsync(Client client, string endpoint)
{
var deadline = DateTimeOffset.UtcNow + _options.ConnectTimeout;
while (!client.Connected)
{
if (DateTimeOffset.UtcNow >= deadline)
{
throw new TimeoutException($"Timed out connecting to peer exchange endpoint {endpoint}.");
}
await Task.Delay(TimeSpan.FromMilliseconds(25)).ConfigureAwait(false);
}
}
private async Task<CultMeshPeerExchangeResponseMessage> WaitForResponseAsync(
Task<CultMeshPeerExchangeResponseMessage> responseTask,
string endpoint)
{
var timeoutTask = Task.Delay(_options.ResponseTimeout);
var completed = await Task.WhenAny(responseTask, timeoutTask).ConfigureAwait(false);
if (completed != responseTask)
{
throw new TimeoutException($"Timed out waiting for peer exchange response from {endpoint}.");
}
return await responseTask.ConfigureAwait(false);
}
}
}

188
src/GameCult.Mesh/CultMeshSimulationFacts.cs Normal file
View file

@ -0,0 +1,188 @@
using System;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
using GameCult.Caching;
using GameCult.Networking;
using MessagePack;
namespace GameCult.Mesh
{
/// <summary>
/// Committed simulation fact derived from witness consensus.
/// </summary>
[CultDocument("gamecult.mesh.simulation_fact", "gamecult.mesh.simulation_fact.v1")]
public sealed class CultMeshSimulationFact
{
/// <summary>
/// Stable fact identifier.
/// </summary>
[Key(0)]
[CultIndex]
public string FactId { get; set; } = string.Empty;
/// <summary>
/// Shard containing the simulation.
/// </summary>
[Key(1)]
[CultIndex]
public string ShardId { get; set; } = string.Empty;
/// <summary>
/// Shard authority epoch observed by witnesses.
/// </summary>
[Key(2)]
public long ShardEpoch { get; set; }
/// <summary>
/// Simulation frame or tick.
/// </summary>
[Key(3)]
[CultIndex]
public long Frame { get; set; }
/// <summary>
/// Subject the fact is about.
/// </summary>
[Key(4)]
[CultIndex]
public string SubjectId { get; set; } = string.Empty;
/// <summary>
/// Fact kind, such as hit, collision, or ownership.
/// </summary>
[Key(5)]
[CultIndex]
public string ClaimKind { get; set; } = string.Empty;
/// <summary>
/// Hash of the accepted claim.
/// </summary>
[Key(6)]
public string ClaimHash { get; set; } = string.Empty;
/// <summary>
/// Optional readable or app-specific claim detail.
/// </summary>
[Key(7)]
public string? ClaimSummary { get; set; }
/// <summary>
/// Number of distinct witnesses supporting the fact.
/// </summary>
[Key(8)]
public int WitnessCount { get; set; }
/// <summary>
/// Supporting witness weight.
/// </summary>
[Key(9)]
public double SupportWeight { get; set; }
/// <summary>
/// Total observed weight for the same subject/frame/kind.
/// </summary>
[Key(10)]
public double TotalWeight { get; set; }
/// <summary>
/// Support divided by total observed weight.
/// </summary>
[Key(11)]
public double Confidence { get; set; }
/// <summary>
/// Commit timestamp.
/// </summary>
[Key(12)]
public string CommittedAt { get; set; } = string.Empty;
/// <summary>
/// Builds a deterministic record key for a simulation fact.
/// </summary>
public static CultRecordKey CreateRecordKey(CultNetSimulationConsensusCandidate candidate)
{
if (candidate == null) throw new ArgumentNullException(nameof(candidate));
return new CultRecordKey("simulation:" + ComputeFactId(candidate));
}
/// <summary>
/// Builds a stable fact id for a consensus candidate.
/// </summary>
public static string ComputeFactId(CultNetSimulationConsensusCandidate candidate)
{
if (candidate == null) throw new ArgumentNullException(nameof(candidate));
var canonical = string.Join("\u001F",
candidate.ShardId,
candidate.ShardEpoch,
candidate.Frame,
candidate.SubjectId,
candidate.ClaimKind);
using var sha = SHA256.Create();
var bytes = sha.ComputeHash(Encoding.UTF8.GetBytes(canonical));
return string.Concat(bytes.Select(value => value.ToString("x2")));
}
/// <summary>
/// Creates a committed fact document from a quorum candidate.
/// </summary>
public static CultMeshSimulationFact FromCandidate(
CultNetSimulationConsensusCandidate candidate,
string? committedAt = null)
{
if (candidate == null) throw new ArgumentNullException(nameof(candidate));
return new CultMeshSimulationFact
{
FactId = ComputeFactId(candidate),
ShardId = candidate.ShardId,
ShardEpoch = candidate.ShardEpoch,
Frame = candidate.Frame,
SubjectId = candidate.SubjectId,
ClaimKind = candidate.ClaimKind,
ClaimHash = candidate.ClaimHash,
ClaimSummary = candidate.ClaimSummary,
WitnessCount = candidate.WitnessCount,
SupportWeight = candidate.SupportWeight,
TotalWeight = candidate.TotalWeight,
Confidence = candidate.Confidence,
CommittedAt = string.IsNullOrWhiteSpace(committedAt)
? DateTimeOffset.UtcNow.ToString("O")
: committedAt!
};
}
}
/// <summary>
/// Commits quorum simulation candidates into the CultMesh database.
/// </summary>
public sealed class CultMeshSimulationFactCommitter
{
private readonly CultNetDatabase _database;
/// <summary>
/// Creates a simulation fact committer.
/// </summary>
public CultMeshSimulationFactCommitter(CultNetDatabase database)
{
_database = database ?? throw new ArgumentNullException(nameof(database));
}
/// <summary>
/// Commits a quorum candidate as a simulation fact document.
/// </summary>
public async Task<CultRecordHandle<CultMeshSimulationFact>> CommitAsync(
CultNetSimulationConsensusCandidate candidate)
{
if (candidate == null) throw new ArgumentNullException(nameof(candidate));
if (!candidate.HasQuorum)
{
throw new InvalidOperationException("Simulation candidate cannot be committed before quorum.");
}
var fact = CultMeshSimulationFact.FromCandidate(candidate);
return await _database.PutAsync(CultMeshSimulationFact.CreateRecordKey(candidate), fact)
.ConfigureAwait(false);
}
}
}

287
src/GameCult.Mesh/CultMeshVerse.cs Normal file
View file

@ -0,0 +1,287 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using GameCult.Networking;
using R3;
namespace GameCult.Mesh
{
/// <summary>
/// Describes how a CultMesh Verse makes authority decisions.
/// </summary>
public enum CultMeshVerseAuthorityModel
{
/// <summary>
/// A known operator cluster is authoritative for committed world state.
/// </summary>
OperatorCluster,
/// <summary>
/// Regional operators share authority for lower latency and availability.
/// </summary>
FederatedCluster,
/// <summary>
/// Peers participate directly in authority decisions.
/// </summary>
PeerToPeer,
/// <summary>
/// The Verse follows another Verse and applies declared overlays.
/// </summary>
SubscribedOverlay
}
/// <summary>
/// Declares transport and rules compatibility for a Verse.
/// </summary>
public sealed class CultMeshVerseCompatibility
{
/// <summary>
/// Creates compatibility metadata.
/// </summary>
public CultMeshVerseCompatibility(
string transportVersion,
string rulesHash,
IEnumerable<string>? compatibleVerseIds = null,
IEnumerable<string>? requiredPluginIds = null,
IEnumerable<string>? optionalPluginIds = null)
{
TransportVersion = RequireNonEmpty(transportVersion, nameof(transportVersion));
RulesHash = RequireNonEmpty(rulesHash, nameof(rulesHash));
CompatibleVerseIds = Clean(compatibleVerseIds);
RequiredPluginIds = Clean(requiredPluginIds);
OptionalPluginIds = Clean(optionalPluginIds);
}
/// <summary>
/// Gets the CultMesh/CultNet transport compatibility version.
/// </summary>
public string TransportVersion { get; }
/// <summary>
/// Gets the stable rules hash.
/// </summary>
public string RulesHash { get; }
/// <summary>
/// Gets Verse ids this Verse can transfer from or subscribe to.
/// </summary>
public IReadOnlyList<string> CompatibleVerseIds { get; }
/// <summary>
/// Gets plugin ids required to enter this Verse.
/// </summary>
public IReadOnlyList<string> RequiredPluginIds { get; }
/// <summary>
/// Gets plugin ids supported but not required by this Verse.
/// </summary>
public IReadOnlyList<string> OptionalPluginIds { get; }
private static string[] Clean(IEnumerable<string>? values)
{
return values?.Where(value => !string.IsNullOrWhiteSpace(value)).Distinct(StringComparer.Ordinal).ToArray()
?? Array.Empty<string>();
}
private static string RequireNonEmpty(string value, string paramName)
{
if (string.IsNullOrWhiteSpace(value))
{
throw new ArgumentException("Value must be non-empty.", paramName);
}
return value;
}
}
/// <summary>
/// Describes one CultMesh Verse: a rule-bearing consensus graph.
/// </summary>
public sealed class CultMeshVerseDescriptor
{
/// <summary>
/// Creates a Verse descriptor.
/// </summary>
public CultMeshVerseDescriptor(
string verseId,
string displayName,
CultMeshVerseAuthorityModel authorityModel,
CultMeshVerseCompatibility compatibility,
IEnumerable<string>? discoveryEndpoints = null,
IEnumerable<string>? authorityRuntimeIds = null,
string? parentVerseId = null,
string? description = null)
{
VerseId = RequireNonEmpty(verseId, nameof(verseId));
DisplayName = RequireNonEmpty(displayName, nameof(displayName));
AuthorityModel = authorityModel;
Compatibility = compatibility ?? throw new ArgumentNullException(nameof(compatibility));
DiscoveryEndpoints = Clean(discoveryEndpoints);
AuthorityRuntimeIds = Clean(authorityRuntimeIds);
ParentVerseId = parentVerseId;
Description = description;
}
/// <summary>
/// Gets the stable Verse id.
/// </summary>
public string VerseId { get; }
/// <summary>
/// Gets the human-facing Verse name.
/// </summary>
public string DisplayName { get; }
/// <summary>
/// Gets the authority model used by this Verse.
/// </summary>
public CultMeshVerseAuthorityModel AuthorityModel { get; }
/// <summary>
/// Gets compatibility metadata for transport, rules, and plugins.
/// </summary>
public CultMeshVerseCompatibility Compatibility { get; }
/// <summary>
/// Gets discovery endpoints for nodes serving this Verse.
/// </summary>
public IReadOnlyList<string> DiscoveryEndpoints { get; }
/// <summary>
/// Gets known authoritative runtime ids, when authority is cluster-shaped.
/// </summary>
public IReadOnlyList<string> AuthorityRuntimeIds { get; }
/// <summary>
/// Gets the parent Verse id for subscribed overlays or branches.
/// </summary>
public string? ParentVerseId { get; }
/// <summary>
/// Gets optional public description.
/// </summary>
public string? Description { get; }
/// <summary>
/// Returns whether this Verse can accept transfer from another Verse.
/// </summary>
public bool CanTransferFrom(CultMeshVerseDescriptor source)
{
if (source == null) throw new ArgumentNullException(nameof(source));
return string.Equals(Compatibility.TransportVersion, source.Compatibility.TransportVersion, StringComparison.Ordinal) &&
(string.Equals(Compatibility.RulesHash, source.Compatibility.RulesHash, StringComparison.Ordinal) ||
Compatibility.CompatibleVerseIds.Contains(source.VerseId, StringComparer.Ordinal));
}
/// <summary>
/// Computes a stable rules hash from ordered rules/plugin parts.
/// </summary>
public static string ComputeRulesHash(params string[] parts)
{
if (parts == null) throw new ArgumentNullException(nameof(parts));
var canonical = string.Join("\u001F", parts.Select(part => part ?? string.Empty));
using var sha = SHA256.Create();
var bytes = sha.ComputeHash(Encoding.UTF8.GetBytes(canonical));
return string.Concat(bytes.Select(value => value.ToString("x2")));
}
private static string[] Clean(IEnumerable<string>? values)
{
return values?.Where(value => !string.IsNullOrWhiteSpace(value)).Distinct(StringComparer.Ordinal).ToArray()
?? Array.Empty<string>();
}
private static string RequireNonEmpty(string value, string paramName)
{
if (string.IsNullOrWhiteSpace(value))
{
throw new ArgumentException("Value must be non-empty.", paramName);
}
return value;
}
}
/// <summary>
/// Reactive local catalog for discovered CultMesh Verses.
/// </summary>
public sealed class CultMeshVerseCatalog : IDisposable
{
private readonly Dictionary<string, CultMeshVerseDescriptor> _verses = new(StringComparer.Ordinal);
private readonly Subject<CultMeshVerseDescriptor> _updates = new();
private bool _disposed;
/// <summary>
/// Gets all known Verses.
/// </summary>
public IReadOnlyList<CultMeshVerseDescriptor> Verses => _verses.Values.OrderBy(verse => verse.VerseId, StringComparer.Ordinal).ToArray();
/// <summary>
/// Watches Verse discovery updates.
/// </summary>
public Observable<CultMeshVerseDescriptor> Watch()
{
ThrowIfDisposed();
return _updates;
}
/// <summary>
/// Adds or replaces a discovered Verse.
/// </summary>
public void Upsert(CultMeshVerseDescriptor verse)
{
ThrowIfDisposed();
if (verse == null) throw new ArgumentNullException(nameof(verse));
_verses[verse.VerseId] = verse;
_updates.OnNext(verse);
}
/// <summary>
/// Adds or replaces Verses from a wire catalog response.
/// </summary>
public void Upsert(CultMeshVerseCatalogResponseMessage response)
{
ThrowIfDisposed();
if (response == null) throw new ArgumentNullException(nameof(response));
foreach (var verse in response.Verses)
{
Upsert(verse.ToVerseDescriptor());
}
}
/// <summary>
/// Gets a Verse by id, if known.
/// </summary>
public CultMeshVerseDescriptor? Get(string verseId)
{
ThrowIfDisposed();
if (string.IsNullOrWhiteSpace(verseId)) throw new ArgumentException("Value must be non-empty.", nameof(verseId));
return _verses.TryGetValue(verseId, out var verse) ? verse : null;
}
/// <summary>
/// Gets known Verses that can accept transfer from the supplied source Verse.
/// </summary>
public IReadOnlyList<CultMeshVerseDescriptor> FindTransferTargets(CultMeshVerseDescriptor source)
{
ThrowIfDisposed();
if (source == null) throw new ArgumentNullException(nameof(source));
return _verses.Values
.Where(verse => !string.Equals(verse.VerseId, source.VerseId, StringComparison.Ordinal) &&
verse.CanTransferFrom(source))
.OrderBy(verse => verse.VerseId, StringComparer.Ordinal)
.ToArray();
}
/// <inheritdoc />
public void Dispose()
{
if (_disposed)
{
return;
}
_disposed = true;
_updates.Dispose();
_verses.Clear();
}
private void ThrowIfDisposed()
{
if (_disposed)
{
throw new ObjectDisposedException(nameof(CultMeshVerseCatalog));
}
}
}
}

269
src/GameCult.Mesh/CultMeshVerseDiscovery.cs Normal file
View file

@ -0,0 +1,269 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using GameCult.Networking;
using LiteNetLib;
namespace GameCult.Mesh
{
/// <summary>
/// Conversion helpers for CultMesh Verse wire messages.
/// </summary>
public static class CultMeshVerseMessages
{
/// <summary>
/// Converts a local Verse descriptor to its schema-v0 wire shape.
/// </summary>
public static CultMeshVerseDescriptorMessage ToMessage(this CultMeshVerseDescriptor verse)
{
if (verse == null) throw new ArgumentNullException(nameof(verse));
return new CultMeshVerseDescriptorMessage
{
VerseId = verse.VerseId,
DisplayName = verse.DisplayName,
AuthorityModel = verse.AuthorityModel.ToString(),
Compatibility = new CultMeshVerseCompatibilityMessage
{
TransportVersion = verse.Compatibility.TransportVersion,
RulesHash = verse.Compatibility.RulesHash,
CompatibleVerseIds = verse.Compatibility.CompatibleVerseIds.ToArray(),
RequiredPluginIds = verse.Compatibility.RequiredPluginIds.ToArray(),
OptionalPluginIds = verse.Compatibility.OptionalPluginIds.ToArray()
},
DiscoveryEndpoints = verse.DiscoveryEndpoints.ToArray(),
AuthorityRuntimeIds = verse.AuthorityRuntimeIds.ToArray(),
ParentVerseId = verse.ParentVerseId,
Description = verse.Description
};
}
/// <summary>
/// Converts a schema-v0 Verse descriptor to the local public API shape.
/// </summary>
public static CultMeshVerseDescriptor ToVerseDescriptor(this CultMeshVerseDescriptorMessage message)
{
if (message == null) throw new ArgumentNullException(nameof(message));
var authorityModel = Enum.TryParse<CultMeshVerseAuthorityModel>(
message.AuthorityModel,
ignoreCase: true,
out var parsed)
? parsed
: CultMeshVerseAuthorityModel.SubscribedOverlay;
return new CultMeshVerseDescriptor(
message.VerseId,
message.DisplayName,
authorityModel,
new CultMeshVerseCompatibility(
message.Compatibility.TransportVersion,
message.Compatibility.RulesHash,
message.Compatibility.CompatibleVerseIds,
message.Compatibility.RequiredPluginIds,
message.Compatibility.OptionalPluginIds),
message.DiscoveryEndpoints,
message.AuthorityRuntimeIds,
message.ParentVerseId,
message.Description);
}
}
/// <summary>
/// Answers CultMesh Verse discovery requests from a local Verse catalog.
/// </summary>
public sealed class CultMeshVerseDiscoveryServer : IDisposable
{
private readonly Server _server;
private readonly CultMeshVerseCatalog _catalog;
private readonly Func<CultMeshVerseCatalogRequestMessage, NetPeer, Task> _requestHandler;
private bool _disposed;
/// <summary>
/// Creates and attaches a Verse discovery bridge.
/// </summary>
public CultMeshVerseDiscoveryServer(Server server, CultMeshVerseCatalog catalog)
{
_server = server ?? throw new ArgumentNullException(nameof(server));
_catalog = catalog ?? throw new ArgumentNullException(nameof(catalog));
_requestHandler = HandleRequestAsync;
_server.OnCultNet(_requestHandler);
}
/// <summary>
/// Creates a Verse catalog response for a request.
/// </summary>
public CultMeshVerseCatalogResponseMessage CreateResponse(CultMeshVerseCatalogRequestMessage request)
{
if (request == null) throw new ArgumentNullException(nameof(request));
var verseIds = request.VerseIds == null || request.VerseIds.Length == 0
? null
: new HashSet<string>(request.VerseIds, StringComparer.Ordinal);
var verses = _catalog.Verses
.Where(verse => verseIds == null || verseIds.Contains(verse.VerseId))
.Where(verse => string.IsNullOrWhiteSpace(request.TransportVersion) ||
string.Equals(verse.Compatibility.TransportVersion, request.TransportVersion, StringComparison.Ordinal))
.Select(verse => verse.ToMessage())
.ToArray();
return new CultMeshVerseCatalogResponseMessage
{
MessageId = string.IsNullOrWhiteSpace(request.MessageId)
? Guid.NewGuid().ToString("N")
: request.MessageId,
Verses = verses
};
}
/// <inheritdoc />
public void Dispose()
{
if (_disposed)
{
return;
}
_disposed = true;
_server.RemoveCultNetMessageListener<CultMeshVerseCatalogRequestMessage>(_requestHandler);
}
private Task HandleRequestAsync(CultMeshVerseCatalogRequestMessage request, NetPeer peer)
{
peer.SendCultNet(CreateResponse(request));
return Task.CompletedTask;
}
}
/// <summary>
/// Options for schema-v0 CultMesh Verse discovery clients.
/// </summary>
public sealed class CultMeshVerseDiscoveryClientOptions
{
/// <summary>
/// Gets or sets client security options used to connect to discovery endpoints.
/// </summary>
public ClientSecurityOptions? Security { get; set; }
/// <summary>
/// Gets or sets how long to wait for a connection before failing.
/// </summary>
public TimeSpan ConnectTimeout { get; set; } = TimeSpan.FromSeconds(5);
/// <summary>
/// Gets or sets how long to wait for a discovery response before failing.
/// </summary>
public TimeSpan ResponseTimeout { get; set; } = TimeSpan.FromSeconds(5);
/// <summary>
/// Gets or sets a callback used to customize each ephemeral discovery client.
/// </summary>
public Action<Client>? ConfigureClient { get; set; }
}
/// <summary>
/// Fetches CultMesh Verse catalogs from discovery endpoints.
/// </summary>
public sealed class CultMeshVerseDiscoveryClient
{
private readonly CultMeshVerseDiscoveryClientOptions _options;
/// <summary>
/// Creates a Verse discovery client.
/// </summary>
public CultMeshVerseDiscoveryClient(CultMeshVerseDiscoveryClientOptions? options = null)
{
_options = options ?? new CultMeshVerseDiscoveryClientOptions();
}
/// <summary>
/// Fetches a Verse catalog response from one discovery endpoint.
/// </summary>
public async Task<CultMeshVerseCatalogResponseMessage> FetchAsync(
string endpoint,
CultMeshVerseCatalogRequestMessage? request = null)
{
if (string.IsNullOrWhiteSpace(endpoint)) throw new ArgumentException("Value must be non-empty.", nameof(endpoint));
var (host, port) = CultNetSchemaWriteForwarder.ParseEndpoint(endpoint);
var messageId = Guid.NewGuid().ToString("N");
var completion = new TaskCompletionSource<CultMeshVerseCatalogResponseMessage>(
TaskCreationOptions.RunContinuationsAsynchronously);
using var client = new Client(_options.Security ?? ClientSecurityOptions.Development())
{
AllowUnverifiedCultNetMessages = true
};
_options.ConfigureClient?.Invoke(client);
client.OnCultNet<CultMeshVerseCatalogResponseMessage>(response =>
{
if (string.Equals(response.MessageId, messageId, StringComparison.Ordinal))
{
completion.TrySetResult(response);
}
});
client.OnCultNet<CultNetErrorMessage>(error =>
completion.TrySetException(new InvalidOperationException(error.Error)));
client.Connect(host, port);
await WaitForConnectionAsync(client, endpoint).ConfigureAwait(false);
client.SendCultNet(new CultMeshVerseCatalogRequestMessage
{
MessageId = messageId,
TransportVersion = request?.TransportVersion,
VerseIds = request?.VerseIds
});
return await WaitForResponseAsync(completion.Task, endpoint).ConfigureAwait(false);
}
/// <summary>
/// Fetches Verse catalogs from endpoints and upserts every response into a local catalog.
/// </summary>
public async Task<int> DiscoverAsync(
CultMeshVerseCatalog catalog,
IEnumerable<string> endpoints,
string? transportVersion = null)
{
if (catalog == null) throw new ArgumentNullException(nameof(catalog));
if (endpoints == null) throw new ArgumentNullException(nameof(endpoints));
var count = 0;
foreach (var endpoint in endpoints.Where(value => !string.IsNullOrWhiteSpace(value)).Distinct(StringComparer.Ordinal))
{
var response = await FetchAsync(endpoint, new CultMeshVerseCatalogRequestMessage
{
TransportVersion = transportVersion
}).ConfigureAwait(false);
catalog.Upsert(response);
count += response.Verses.Length;
}
return count;
}
private async Task WaitForConnectionAsync(Client client, string endpoint)
{
var deadline = DateTimeOffset.UtcNow + _options.ConnectTimeout;
while (!client.Connected)
{
if (DateTimeOffset.UtcNow >= deadline)
{
throw new TimeoutException($"Timed out connecting to Verse discovery endpoint {endpoint}.");
}
await Task.Delay(TimeSpan.FromMilliseconds(25)).ConfigureAwait(false);
}
}
private async Task<CultMeshVerseCatalogResponseMessage> WaitForResponseAsync(
Task<CultMeshVerseCatalogResponseMessage> responseTask,
string endpoint)
{
var timeoutTask = Task.Delay(_options.ResponseTimeout);
var completed = await Task.WhenAny(responseTask, timeoutTask).ConfigureAwait(false);
if (completed != responseTask)
{
throw new TimeoutException($"Timed out waiting for Verse discovery response from {endpoint}.");
}
return await responseTask.ConfigureAwait(false);
}
}
}

19
src/GameCult.Mesh/GameCult.Mesh.csproj Normal file
View file

@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<Description>CultMesh: distributed realtime mesh database and simulation consensus for GameCult projects</Description>
<PackageId>GameCult.Mesh</PackageId>
<PackageTags>game;distributed;realtime;mesh;database;simulation;consensus</PackageTags>
<PackageReadmeFile>README.md</PackageReadmeFile>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\GameCult.Caching\GameCult.Caching.csproj" />
<ProjectReference Include="..\GameCult.Caching.MessagePack\GameCult.Caching.MessagePack.csproj" />
<ProjectReference Include="..\GameCult.Networking\GameCult.Networking.csproj" />
</ItemGroup>
<ItemGroup>
<None Include="README.md" Pack="true" PackagePath="\" />
<None Include="docs\**\*.md" Pack="true" PackagePath="docs\" />
</ItemGroup>
</Project>

204
src/GameCult.Mesh/README.md Normal file
View file

@ -0,0 +1,204 @@
# CultMesh
CultMesh is the distributed realtime database and simulation-consensus layer for
GameCult projects. The NuGet package and assembly are named `GameCult.Mesh`;
the public surface is branded as CultMesh.
CultMesh sits over:
- `GameCult.Caching` for typed domain documents, local indexes, persistence, and
file-compatible cache storage.
- `GameCult.Networking` for LiteNetLib transport, schema-v0 wire messages, and
secure client/server sessions.
The intent is direct: a game can treat a mesh of clients and servers as one
reactive database for persistent state, prediction-friendly input state, and
simulation observations.
## First Node
```csharp
using GameCult.Mesh;
using var node = await CultMesh.StartNodeAsync("world.ccmp");
var player = await node.Database.GetAsync<PlayerData>(playerKey);
await node.Database.PutAsync(playerKey, player);
```
Enable durable authoritative shard logs when a node should serve replica
catch-up after restart:
```csharp
using var node = await CultMesh.StartNodeAsync("world.ccmp", new CultMeshNodeOptions
{
EnableDurableShardLogs = true
});
```
If `ShardLogPath` is omitted, CultMesh stores logs beside the cache under
`world.cultmesh/shard-logs`.
## Verses
A Verse is a rule-bearing consensus graph. Aetheria Main can be a
cloud-authoritative Verse backed by GameCult regional simulators. A modded
community branch can be transport-compatible with Aetheria Main while declaring
different rules, required plugins, and authority policy. A peer-hosted game can
choose a peer-to-peer authority model.
```csharp
var rulesHash = CultMeshVerseDescriptor.ComputeRulesHash(
"aetheria",
"rules:v1",
"plugins:vanilla");
var aetheria = new CultMeshVerseDescriptor(
"aetheria-main",
"Aetheria",
CultMeshVerseAuthorityModel.OperatorCluster,
new CultMeshVerseCompatibility("cultmesh.v0", rulesHash),
discoveryEndpoints: ["cultmesh://us-east.aetheria.gamecult.net:3075"]);
using var catalog = CultMesh.CreateVerseCatalog();
catalog.Upsert(aetheria);
using var discovery = CultMesh.ServeVerseCatalog(node, catalog);
```
Peers can request Verse catalogs over schema-v0 with
`cultmesh.verse_catalog_request.v0` and receive transport-filtered
`cultmesh.verse_catalog_response.v0` descriptors.
## Peer Exchange
Verse discovery tells a node which consensus graph exists. Peer exchange tells
it who else is already in that graph.
```csharp
using var peers = CultMesh.CreatePeerCatalog();
peers.Upsert(new CultMeshPeerCard(
"gc-us-east-1",
"aetheria-main",
["cultnet://us-east.aetheria.gamecult.net:3075"],
roles: [CultMeshPeerRoles.Discovery, CultMeshPeerRoles.ShardPrimary],
shardIds: ["players-us-east"],
authorityLeaseId: "lease:players-us-east:42"));
using var exchange = CultMesh.ServePeerExchange(node, peers);
```
Peer cards are contact candidates, not authority by themselves. Public Verses
should validate authority leases or signatures before trusting a peer for
committed state.
```csharp
var leases = CultMesh.CreateAuthorityLeaseCatalog();
leases.Upsert(new CultMeshAuthorityLease(
"lease:players-us-east:42",
"aetheria-main",
"gc-us-east-1",
[CultMeshPeerRoles.ShardPrimary],
["players-us-east"],
"gamecult-operator",
DateTimeOffset.UtcNow.AddMinutes(-1),
DateTimeOffset.UtcNow.AddMinutes(10),
signature: "operator-signature"));
```
Clients can fetch more peers from known peers:
```csharp
var exchangeClient = CultMesh.CreatePeerExchangeClient();
await exchangeClient.DiscoverAsync(
peers,
aetheria.DiscoveryEndpoints,
"aetheria-main",
roles: [CultMeshPeerRoles.ReadReplica]);
```
## Client-Side Prediction
CultMesh lets a runtime declare which input documents it may author
optimistically. The local cache updates immediately, game simulation can proceed
against that predicted state, and the authoritative shard log later reconciles
the record.
```csharp
var db = node.Database;
await db.PutPredictedAsync(inputKey, inputState);
using var sub = db.WatchRecord<PlayerInput>(inputKey)
.Subscribe(change =>
{
ReconcilePrediction(change);
});
```
## Game Session
`CultMeshGameSession` wires the common gameplay loop: local catalogs,
observation hub, observation server, peer exchange, Verse discovery, prediction,
and quorum fact commits.
```csharp
using var session = CultMesh.CreateGameSession(node, new CultMeshGameSessionOptions
{
ConsensusOptions = new CultNetSimulationConsensusOptions
{
MinimumWitnesses = 2,
QuorumRatio = 0.66d
}
});
await session.PredictAsync(inputKey, inputState);
var committed = await session.SubmitAndCommitAsync(new CultNetSimulationObservation
{
WitnessRuntimeId = "watcher-1",
ShardId = "arena",
ShardEpoch = 4,
Frame = 100,
SubjectId = "bob",
ClaimKind = "hit",
ClaimHash = CultNetSimulationObservation.ComputeClaimHash("hit", "alice", "bob", "frame:100")
});
```
## Witness Consensus
CultMesh nodes can publish observations about simulation facts: shard epoch,
frame, subject, claim kind, and claim hash. `CultNetSimulationConsensus`
aggregates those observations deterministically into candidate facts with
support weight, total observed weight, confidence, and quorum status.
Those candidates are not committed world state by themselves. They are the fast
mesh opinion layer that can feed an authoritative shard commit path. Once a
candidate has quorum, a shard-authoritative node can commit it as a
`CultMeshSimulationFact`.
```csharp
var committer = CultMesh.CreateSimulationFactCommitter(node.Database);
await committer.CommitAsync(candidate);
```
The central server does not need to decide what every witness saw. It only
needs to accept the quorum result through the shard log.
## Durable Catch-Up
CultMesh can persist authoritative shard logs through
`ICultNetShardMutationLogStore`. The included file store writes one MessagePack
log per shard, so a restarted primary can still serve replica catch-up without
depending on warm process memory. When retained log history is compacted, stale
replicas get an explicit resync requirement instead of a partial log pretending
to be enough. The replicator can answer that by fetching a shard-bounded
snapshot, replacing the local shard view, and advancing its cursor to the
snapshot's represented log sequence.
## Documentation
- [Public API](docs/public-api.md)
- [Research Notes](docs/research.md)
- [Verse Model](docs/verses.md)

120
src/GameCult.Mesh/docs/public-api.md Normal file
View file

@ -0,0 +1,120 @@
# CultMesh Public API
Package: `GameCult.Mesh`
Brand: CultMesh
Namespace: `GameCult.Mesh`
## Entry Points
- `CultMesh.CreateNodeAsync(...)`
- `CultMesh.StartNodeAsync(...)`
- `CultMesh.CreateVerseCatalog()`
- `CultMesh.CreatePeerCatalog()`
- `CultMesh.CreateVerseDiscoveryClient(...)`
- `CultMesh.CreatePeerExchangeClient(...)`
- `CultMesh.CreateAuthorityLeaseCatalog()`
- `CultMesh.CreateSimulationFactCommitter(...)`
- `CultMesh.CreateGameSession(...)`
- `CultMesh.CreateClient(...)`
- `CultMesh.ConnectClient(...)`
`CultMeshNode` wraps the current local runtime pieces:
- `Cache`: the underlying `CultCache`
- `Server`: the underlying CultNet server
- `Database`: the distributed realtime database facade
- `DatabaseServer`: the schema-v0 database bridge
`CultMeshNodeOptions.EnableDurableShardLogs` attaches a file-backed
authoritative shard-log store when `DatabaseOptions.MutationLogStore` is not
already supplied. `ShardLogPath` can override the location; otherwise CultMesh
uses a `.cultmesh/shard-logs` directory beside the cache file.
The package home and primary entrypoint are `GameCult.Mesh` / `CultMesh`.
Some lower-level database and wire-contract types still retain `CultNet` names
because they are shared with the transport package. Treat those as plumbing,
not the brand surface.
## Core Behaviors
### Verses
`CultMeshVerseDescriptor` describes a rule-bearing consensus graph. It includes
transport compatibility, rules hash, authority model, discovery endpoints,
known authority runtimes, optional parent Verse id, and plugin requirements.
`CultMeshVerseCatalog` is the local reactive catalog for discovered Verses.
It can publish discovery updates and find compatible transfer targets.
`CultMesh.ServeVerseCatalog(node, catalog)` attaches schema-v0 Verse discovery
responses to a node. The wire contracts are
`cultmesh.verse_catalog_request.v0` and `cultmesh.verse_catalog_response.v0`.
Consumers can apply a response directly with
`CultMeshVerseCatalog.Upsert(CultMeshVerseCatalogResponseMessage)`.
### Peer Exchange
`CultMeshPeerCatalog` is the local reactive catalog for peer cards.
`CultMeshPeerCard` describes a candidate peer endpoint for one Verse with
roles, shard hints, optional region, optional authority lease id, expiry, and
signature.
`CultMesh.ServePeerExchange(node, catalog)` attaches schema-v0 peer exchange
responses to a node. The wire contracts are
`cultmesh.peer_exchange_request.v0` and `cultmesh.peer_exchange_response.v0`.
`CultMeshPeerExchangeClient` can fetch peer cards from known endpoints and
upsert them into a local catalog.
Peer cards are discovery hints; authority still requires a valid lease or
signature for the target Verse.
### Authority Leases
`CultMeshAuthorityLease` grants one peer specific roles for a Verse, optionally
scoped to shard ids and bounded by time. `CultMeshAuthorityLeaseCatalog` checks
whether a peer card is authorized for a requested role and shard. This keeps
peer exchange separate from authority instead of letting contact gossip mutate
the world.
### Shard Authority
Each shard has one primary writer for now. Non-primary writes are rejected or
explicitly forwarded. Stale epochs fail loudly.
### Reactive Documents
Consumers subscribe to typed document changes through `CultNetDatabase` watch
methods. Subscriptions receive domain changes rather than storage envelopes.
### Client Prediction
`CultNetClientAuthorityScope` declares input documents a runtime may predict.
`PutPredictedAsync` writes local state and emits `Predicted`. When the
authoritative log arrives, the database emits `Reconciled`.
### Replica Catch-Up
Shard logs can be requested over schema-v0 and applied by replicas. Replica
cursors can be stored with `ICultNetShardReplicaCursorStore`. Authoritative
shard logs can be persisted with `ICultNetShardMutationLogStore`; the file
implementation stores one MessagePack log per shard so catch-up survives a
primary restart. Compacted history returns an explicit resync requirement so a
replica can fall back to a shard-bounded snapshot before applying newer log
entries. Applying that snapshot replaces the local shard view and advances the
replica cursor to the snapshot's represented log sequence.
### Simulation Witness Consensus
`CultNetSimulationObservation` records what a node saw for a simulation fact.
`CultNetSimulationObservationHub` collects those observations and emits
`CultNetSimulationConsensusCandidate` updates.
Candidates are opinions. `CultMeshSimulationFactCommitter` commits quorum
candidates as `CultMeshSimulationFact` documents through `CultNetDatabase`, so
final world state still belongs in the authoritative shard log.
`CultMeshGameSession` is the gameplay-facing composition layer. It owns the
common loop for prediction, observations, candidate watches, simulation fact
watches, and committing new quorum facts without forcing game code to manually
wire every bridge.

41
src/GameCult.Mesh/docs/research.md Normal file
View file

@ -0,0 +1,41 @@
# CultMesh Research Notes
CultMesh follows the research trail captured in
`research/cultnet-distributed-database/summary.md`.
The current design keeps these borrowed pieces:
- RethinkDB-style changefeeds for reactive document subscriptions.
- Firebase-like realtime ergonomics, without pretending offline multi-writer
conflicts are free.
- Dynamo-inspired shard metadata, hinted recovery, and anti-entropy direction.
- Raft-shaped primary authority per shard before automatic failover.
- SWIM-shaped membership later, once there is enough state worth gossiping.
- CRDTs only for document types with declared deterministic merge laws.
- Hashgraph's gossip-about-gossip as an event-provenance idea, not as a crypto
ledger import.
## Live Model
CultMesh is currently primary-shard authority plus replica catch-up:
1. A primary shard accepts ordered writes.
2. Followers pull shard logs and apply committed entries.
3. Clients can predict their own input documents inside explicit authority
scopes.
4. Witnesses publish immutable observations about simulation facts.
5. Nodes aggregate observations into deterministic consensus candidates.
6. Authoritative world state is still committed through shard logs.
7. Verses describe rule-bearing consensus graphs so compatible branches can
share transport without pretending their rules or authority are identical.
## Next Research Cut
The next hard problem is not more vocabulary. It is durable authority:
- persist authoritative shard logs
- define compaction and snapshot fallback
- add observation fanout across peers
- add Verse discovery gossip and subscription policy
- add simulation rollback/resimulation helpers
- then revisit membership, failover, and consensus protocol depth

71
src/GameCult.Mesh/docs/verses.md Normal file
View file

@ -0,0 +1,71 @@
# CultMesh Verses
A Verse is a rule-bearing consensus graph.
That phrasing is intentionally dry. It keeps the system honest.
The same transport can carry many Verses:
- `aetheria-main`: public Aetheria, operated by GameCult regional simulators.
- `aetheria-community-hardcore`: compatible transport, different rules.
- `aetheria-modded-skylands`: compatible branch with required runtime plugins.
- `p2p-arena-nightly`: peer-to-peer authority for a small experimental game.
## Descriptor
`CultMeshVerseDescriptor` owns:
- `VerseId`: stable id
- `DisplayName`: public-facing name
- `AuthorityModel`: operator cluster, federated cluster, peer-to-peer, or
subscribed overlay
- `Compatibility`: transport version, rules hash, compatible Verse ids, and
plugin requirements
- `DiscoveryEndpoints`: endpoints where nodes for the Verse can be found
- `AuthorityRuntimeIds`: known authoritative runtimes when cluster-shaped
- `ParentVerseId`: source Verse for overlays or branches
## Compatibility
Two Verses can share a transport without sharing rules.
Transfer is allowed when:
- transport versions match, and
- rules hashes match, or the destination explicitly lists the source Verse as
compatible
That lets a modded branch choose its boundary. It can accept transfers from
Aetheria Main, require a plugin pack, and still reject another branch with
incompatible rules.
## Authority
Authority is a Verse-level policy, not a transport accident.
- `OperatorCluster`: one known operator cluster is final authority.
- `FederatedCluster`: several regional or organizational clusters share
authority.
- `PeerToPeer`: participating peers directly contribute authority.
- `SubscribedOverlay`: the Verse follows another Verse and adds local rules,
mods, or presentation layers.
## Discovery
`CultMeshVerseCatalog` is a local reactive catalog. It can be filled from
static config, cloud discovery, peer gossip, or mod portals. It does not care
where the descriptor came from. That keeps discovery pluggable while the Verse
model stays stable.
Peer exchange is the torrent-shaped layer under Verse discovery. A Verse
catalog says which graph exists; a peer catalog says which nodes can currently
serve or observe parts of that graph. Peer cards can advertise discovery,
primary shard, replica, read replica, and simulation observer roles, but those
cards do not grant authority by themselves. Public operator Verses should treat
authority lease ids and signatures as mandatory before accepting committed
state from a peer.
Authority leases are the narrow bridge between discovery and trust. A lease
binds a peer id to a Verse, role, shard scope, issuer, validity window, and
signature. Peer exchange can spread cards quickly; lease validation decides
whether a card is allowed to touch committed state.

229
src/GameCult.Networking/Client.cs
View file

@ -1,5 +1,6 @@
using System;
using System.Collections.Concurrent;
using System.Security.Cryptography;
using GameCult.Logging;
using LiteNetLib;
using MessagePack;
@ -7,6 +8,27 @@ using R3;
namespace GameCult.Networking
{
/// <summary>
/// Describes the client's reconnect posture.
/// </summary>
public enum ClientReconnectState
{
/// <summary>
/// The client is connected or otherwise not trying to reconnect.
/// </summary>
Idle,
/// <summary>
/// The client is waiting for the next reconnect attempt.
/// </summary>
WaitingToReconnect,
/// <summary>
/// The client is actively attempting a reconnect.
/// </summary>
Reconnecting
}
/// <summary>
/// Provides client-side connection, authentication, and message dispatch logic.
/// </summary>
@ -16,6 +38,10 @@ namespace GameCult.Networking
/// Raised when the client encounters a user-visible error.
/// </summary>
public event Action<string>? OnError; // Hook for important user-facing feedback (e.g. via modal UI element)
/// <summary>
/// Raised when reconnect state changes.
/// </summary>
public event Action<ClientReconnectState>? OnReconnectStateChanged;
private NetManager? _client;
private NetPeer? _peer;
private IDisposable? _pollSubscription;
@ -23,15 +49,19 @@ namespace GameCult.Networking
private bool _disposed;
private bool _manualDisconnect;
private bool _isReconnecting;
private int _reconnectAttemptCount;
private readonly ConcurrentDictionary<Type, Delegate> _messageDelegates = new();
private readonly ConcurrentDictionary<Type, Delegate> _cultNetMessageDelegates = new();
private string? _sessionToken;
private string _lastHost = "localhost";
private int _lastPort;
private readonly ClientSecurityOptions _security;
private ILogger _logger = new NullLogger();
private static readonly TimeSpan ReconnectDelay = TimeSpan.FromSeconds(1);
private static readonly TimeSpan BaseReconnectDelay = TimeSpan.FromSeconds(1);
private static readonly TimeSpan MaxReconnectDelay = TimeSpan.FromSeconds(30);
private static readonly TimeSpan MaxReconnectJitter = TimeSpan.FromMilliseconds(250);
/// <summary>
/// Gets or sets the logger used by the client.
@ -47,6 +77,21 @@ namespace GameCult.Networking
/// </summary>
public int Ping { get; private set; }
/// <summary>
/// Gets the current reconnect posture exposed to consumers.
/// </summary>
public ClientReconnectState ReconnectState { get; private set; } = ClientReconnectState.Idle;
/// <summary>
/// Gets or sets whether raw payload bodies may be logged for diagnostics.
/// </summary>
public bool LogSensitivePayloads { get; set; }
/// <summary>
/// Gets or sets whether CultNet schema-v0 messages may be sent before user session verification.
/// </summary>
public bool AllowUnverifiedCultNetMessages { get; set; }
/// <summary>
/// Gets a value indicating whether the client currently has a verified session.
/// </summary>
@ -83,7 +128,14 @@ namespace GameCult.Networking
{
if (Verified || m is LoginMessage || m is RegisterMessage || m is VerifyMessage)
{
Logger.LogDebug($"Sending message {MessagePackSerializer.SerializeToJson(m)}");
if (LogSensitivePayloads)
{
Logger.LogDebug($"Sending message {MessagePackSerializer.SerializeToJson(m)}");
}
else
{
Logger.LogDebug($"Sending message {typeof(T).Name}");
}
_peer.Send(m);
}
else Logger.LogError("Cannot send, client is not verified!");
@ -91,12 +143,43 @@ namespace GameCult.Networking
else Logger.LogError("Cannot send, client is not connected!");
}
/// <summary>
/// Sends a modern CultNet schema-v0 message when the client is connected and authorized.
/// </summary>
public void SendCultNet<T>(T message) where T : ICultNetSchemaMessage
{
if (_disposed)
{
throw new ObjectDisposedException(nameof(Client));
}
if (message == null) throw new ArgumentNullException(nameof(message));
if (Connected && _peer != null)
{
if (AllowUnverifiedCultNetMessages || Verified || message is CultNetLoginMessage or CultNetRegisterMessage or CultNetVerifyMessage)
{
Logger.LogDebug($"Sending CultNet schema message {message.SchemaVersion}");
_peer.SendCultNet(message);
}
else
{
Logger.LogError("Cannot send CultNet schema message, client is not verified!");
}
}
else
{
Logger.LogError("Cannot send CultNet schema message, client is not connected!");
}
}
/// <summary>
/// Removes all registered message listeners.
/// </summary>
public void ClearMessageListeners()
{
_messageDelegates.Clear();
_cultNetMessageDelegates.Clear();
}
/// <summary>
@ -118,6 +201,37 @@ namespace GameCult.Networking
});
}
/// <summary>
/// Adds a listener for a specific message type.
/// </summary>
public void On<T>(Action<T> callback) where T : Message
{
AddMessageListener(callback);
}
/// <summary>
/// Adds a listener for a modern CultNet schema-v0 message type.
/// </summary>
public void AddCultNetMessageListener<T>(Action<T> callback) where T : ICultNetSchemaMessage
{
var type = typeof(T);
_cultNetMessageDelegates.AddOrUpdate(type,
_ => callback,
(t, current) =>
{
var combined = Delegate.Combine(current, callback) as Action<T>;
return combined ?? throw new InvalidOperationException($"Failed to combine delegates for {t.Name}");
});
}
/// <summary>
/// Adds a listener for a modern CultNet schema-v0 message type.
/// </summary>
public void OnCultNet<T>(Action<T> callback) where T : ICultNetSchemaMessage
{
AddCultNetMessageListener(callback);
}
/// <summary>
/// Removes a previously registered listener for a specific message type.
/// </summary>
@ -132,6 +246,34 @@ namespace GameCult.Networking
}
}
/// <summary>
/// Removes a previously registered listener for a specific message type.
/// </summary>
public void Off<T>(Action<T> callback) where T : Message
{
RemoveMessageListener(callback);
}
/// <summary>
/// Removes a previously registered modern CultNet schema-v0 listener.
/// </summary>
public void RemoveCultNetMessageListener<T>(Action<T> callback) where T : ICultNetSchemaMessage
{
if (_cultNetMessageDelegates.TryGetValue(typeof(T), out var currentDelegate))
{
var newDelegate = Delegate.Remove(currentDelegate, callback) as Action<T>;
_cultNetMessageDelegates[typeof(T)] = newDelegate ?? (_ => { });
}
}
/// <summary>
/// Removes a previously registered modern CultNet schema-v0 listener.
/// </summary>
public void OffCultNet<T>(Action<T> callback) where T : ICultNetSchemaMessage
{
RemoveCultNetMessageListener(callback);
}
/// <summary>
/// Sends a login request.
/// </summary>
@ -192,6 +334,11 @@ namespace GameCult.Networking
/// <param name="host">The remote host name or IP address.</param>
/// <param name="port">The remote server port.</param>
public void Connect(string host = "localhost", int port = 3075)
{
ConnectCore(host, port, resetReconnectBackoff: true);
}
private void ConnectCore(string host, int port, bool resetReconnectBackoff)
{
if (_disposed)
{
@ -200,6 +347,11 @@ namespace GameCult.Networking
_manualDisconnect = false;
_isReconnecting = false;
if (resetReconnectBackoff)
{
_reconnectAttemptCount = 0;
}
SetReconnectState(ClientReconnectState.Idle);
_reconnectSubscription?.Dispose();
_reconnectSubscription = null;
_lastHost = host;
@ -221,8 +373,32 @@ namespace GameCult.Networking
try
{
var bytes = reader.GetRemainingBytes();
Logger.LogDebug($"Received message: {MessagePackSerializer.ConvertToJson(new ReadOnlyMemory<byte>(bytes))}");
if (TryDeserializeCultNet(bytes, out var cultNetMessage))
{
Logger.LogDebug($"Received CultNet schema message {cultNetMessage.SchemaVersion}");
var cultNetType = cultNetMessage.GetType();
if (_cultNetMessageDelegates.TryGetValue(cultNetType, out var cultNetDelegate) &&
cultNetDelegate != null)
{
cultNetDelegate.DynamicInvoke(cultNetMessage);
}
else
{
Logger.LogWarning($"No listener for CultNet schema message {cultNetMessage.SchemaVersion}");
}
return;
}
var message = MessageSerialization.Deserialize<Message>(bytes);
if (LogSensitivePayloads)
{
Logger.LogDebug($"Received message: {MessagePackSerializer.ConvertToJson(new ReadOnlyMemory<byte>(bytes))}");
}
else
{
Logger.LogDebug($"Received message {message.GetType().Name}");
}
var type = message.GetType();
switch (message)
@ -252,6 +428,8 @@ namespace GameCult.Networking
{
Logger.LogInfo($"Peer {peer.Address}:{peer.Port} connected.");
_peer = peer;
_reconnectAttemptCount = 0;
SetReconnectState(ClientReconnectState.Idle);
if (Verified)
{
var nonce = Secret.NewNonce;
@ -274,6 +452,20 @@ namespace GameCult.Networking
(peer, latency) => Ping = latency; //Logger($"Ping received: {latency} ms");
}
private static bool TryDeserializeCultNet(byte[] payload, out ICultNetSchemaMessage message)
{
try
{
message = CultNetSchemaMessageSerialization.Deserialize(payload);
return true;
}
catch (MessagePackSerializationException)
{
message = null!;
return false;
}
}
/// <summary>
/// Disconnects the client and suppresses automatic reconnect.
/// </summary>
@ -282,6 +474,7 @@ namespace GameCult.Networking
_manualDisconnect = true;
_reconnectSubscription?.Dispose();
_reconnectSubscription = null;
SetReconnectState(ClientReconnectState.Idle);
DisposeTransport();
}
@ -297,6 +490,7 @@ namespace GameCult.Networking
_manualDisconnect = true;
_reconnectSubscription?.Dispose();
_reconnectSubscription = null;
SetReconnectState(ClientReconnectState.Idle);
DisposeTransport();
}
@ -308,15 +502,19 @@ namespace GameCult.Networking
}
_isReconnecting = true;
_reconnectSubscription = Observable.Timer(ReconnectDelay).Subscribe(_ =>
_reconnectAttemptCount++;
var delay = GetReconnectDelayForAttempt(_reconnectAttemptCount);
SetReconnectState(ClientReconnectState.WaitingToReconnect);
_reconnectSubscription = Observable.Timer(delay).Subscribe(_ =>
{
_isReconnecting = false;
_reconnectSubscription?.Dispose();
_reconnectSubscription = null;
SetReconnectState(ClientReconnectState.Reconnecting);
try
{
Connect(_lastHost, _lastPort);
ConnectCore(_lastHost, _lastPort, resetReconnectBackoff: false);
}
catch (Exception ex)
{
@ -326,6 +524,16 @@ namespace GameCult.Networking
});
}
internal static TimeSpan GetReconnectDelayForAttempt(int attempt)
{
var normalizedAttempt = Math.Max(1, attempt);
var exponentialSeconds = Math.Min(
MaxReconnectDelay.TotalSeconds,
BaseReconnectDelay.TotalSeconds * Math.Pow(2, normalizedAttempt - 1));
var jitterMilliseconds = RandomNumberGenerator.GetInt32((int)MaxReconnectJitter.TotalMilliseconds + 1);
return TimeSpan.FromSeconds(exponentialSeconds) + TimeSpan.FromMilliseconds(jitterMilliseconds);
}
private void DisposeTransport()
{
_pollSubscription?.Dispose();
@ -356,5 +564,16 @@ namespace GameCult.Networking
}
}
}
private void SetReconnectState(ClientReconnectState state)
{
if (ReconnectState == state)
{
return;
}
ReconnectState = state;
OnReconnectStateChanged?.Invoke(state);
}
}
}

164
src/GameCult.Networking/Contracts/cultnet-distributed-database.md Normal file
View file

@ -0,0 +1,164 @@
# CultNet Distributed Database Notes
CultNet's distributed database path is a typed realtime database over CultCache
documents. The design should borrow proven distributed-systems ideas without
copying their entire machine or laundering vague consistency through impressive
words.
## Prior Art To Keep
- RethinkDB changefeeds: subscriptions should emit old/new document views and
support point, table/schema, and filtered watches.
- Firebase Realtime Database: client ergonomics should make realtime sync feel
like the default path, but presence, disconnect handling, and server-side
authority still need explicit ownership.
- Dynamo: partitioning, replication metadata, vector-ish causality, hinted
recovery, and application-visible conflict resolution are useful. "Always
writeable" is not the first CultNet promise.
- Raft: primary authority per shard is the first understandable consistency
model. Leader election/log replication can come later if the cluster needs
automatic failover.
- SWIM: peer membership and failure suspicion should be gossip-shaped when the
mesh grows beyond small static clusters.
- CRDTs: offline multi-writer documents are valid only for types with declared,
deterministic merge semantics.
- Hashgraph: gossip-about-gossip is interesting for compact event provenance
and fair ordering research. CultNet should not adopt public-ledger machinery
or crypto-token assumptions as a shortcut to database correctness.
## Current Cut
Implemented:
- `CultNetDatabase`: database-style facade over a `CultCache`
- primary-shard write checks
- typed R3 watch streams
- raw CultNet put/delete application through shard policy
- `CultNetDatabaseServer`: schema-v0 bridge for snapshot, put, and delete
- `Client` and `Server` schema-v0 dispatch hooks
- `CultNetHost.Database` and `CultNetHost.DatabaseServer`
- subscription request/cancel/change messages
- server-side live change fanout for schema/key-filtered subscriptions
- shard catalog request/response messages
- shard descriptors with owner runtime id, epoch, schema filters, key prefix,
primary endpoints, replica endpoints, read replica endpoints, and region
- epoch-aware raw put/delete messages
- routing hints on shard authority errors
- injectable non-primary write forwarding policy
- schema-v0 write forwarder that dials advertised `cultnet://host:port`
primary endpoints
- in-memory per-shard mutation logs with sequence numbers, epochs, mutation
kind, schema id, key, and document references
- mutation-log catch-up by last seen sequence
- wire-level shard log request/response messages
- raw put/delete representation for accepted mutation-log entries
- replica-side shard log application with epoch checks, gap rejection, and
idempotent replay
- shard log replicator that pulls non-primary shards from catalog-advertised
primary endpoints
- schema-v0 shard log fetcher for `cultnet://host:port` endpoints
- restart-safe replica cursor store, including a local MessagePack file-backed
implementation
- durable authoritative shard-log storage behind
`ICultNetShardMutationLogStore`, including a per-shard MessagePack file
implementation
- shard-log compaction watermarks; requests for compacted history return
`ResyncRequired` with `reason = "compacted_history"`
- shard-bounded snapshots with shard id, epoch, and represented log sequence
- replica snapshot recovery after compacted history, including local shard
replacement and cursor advancement
- CultMesh node option for file-backed authoritative shard logs with a default
`.cultmesh/shard-logs` path beside the cache file
- schema-v0 Verse catalog request/response messages
- CultMesh Verse discovery server bridge over the local Verse catalog
- schema-v0 peer exchange request/response messages
- CultMesh peer cards, peer catalog, peer exchange server bridge, and peer
exchange client
- CultMesh authority leases and lease catalog checks for peer role/shard
authorization
- client-authority scopes for locally predicted input documents
- predicted and reconciled database change kinds for client-side prediction
- simulation witness observations for mesh-side opinions about frame facts
- deterministic consensus candidates derived from witness observations
- committed CultMesh simulation fact documents written from quorum candidates
through shard authority
- gameplay-facing CultMesh session facade for prediction, observations,
catalogs, peer exchange, and quorum fact commits
- schema-v0 simulation observation and consensus candidate messages
- reactive simulation observation hub that receives witness gossip and emits
consensus candidates
- server-side observation bridge that accepts observation messages and replies
with current candidate messages
Not implemented yet:
- membership/failure detection
- leader election or automatic shard failover
- peer-to-peer snapshot fanout and throttling
- peer-to-peer fanout for observation gossip and candidate propagation
- rollback/resimulation helpers for simulation frames
- declared CRDT merge policies
## Live Invariants
- CultCache owns document schema, identity, local indexes, and local diffing.
- CultNet owns transport, shard authority, remote mutation delivery, and
subscription fanout.
- One primary owns writes for a shard until a stronger policy exists.
- Non-owner writes are rejected or explicitly forwarded. They are not silently
applied.
- Raw snapshot/put/delete messages must pass through `CultNetDatabase`, not
bypass shard policy.
- The accepted shard log is stored in replica catch-up wire form. Object logs
are for local inspection; wire logs are the durable replication authority.
- Log retention is part of the replication contract. A replica asking before
the compaction watermark must resync from a snapshot before applying newer
log entries.
- A shard snapshot represents a log sequence. Applying it replaces the local
shard view and advances the replica cursor to that sequence.
- Realtime subscriptions publish domain changes, not storage envelopes.
- Client-owned input documents may be predicted locally only inside explicit
`CultNetClientAuthorityScope` declarations.
- Predicted input state is not authoritative shared state. The shard log still
decides the committed ordering and emits reconciliation when it corrects or
confirms local prediction.
- Witness observations are immutable reports about a shard epoch, frame,
subject, claim kind, and claim hash.
- Consensus candidates are derived from observations. They may inform the
authoritative commit path, but they are not themselves committed world state
until a shard owner writes them into the log.
- `CultMeshSimulationFact` is the committed form of a quorum candidate. It is
written through `CultNetDatabase`, so normal shard authority and replication
rules still apply.
- `CultMeshGameSession` composes common gameplay surfaces. It does not create a
second authority path; all committed state still flows through shard-owned
database writes.
- Peer cards are contact candidates. They can advertise roles and authority
lease ids, but they do not grant authority without Verse-specific validation.
- Authority leases bind peer id, Verse id, roles, optional shard scope, issuer,
validity window, and signature. They are the first trust boundary above peer
exchange.
## Next Coherent Slice
Add discovery fanout and operational polish:
- add periodic gossip fanout over known peer cards
- add real signature verification for authority leases
- add snapshot throttling and size limits for public mesh edges
- add simulation-frame rollback/resimulation helpers around predicted input
streams
- add peer-to-peer fanout for observation gossip and candidate propagation
The log is wire-readable, replicas can apply it explicitly, a pull loop can
drive non-primary shards from primary endpoints, and replica cursors can survive
restart. Client-owned input documents can now be predicted locally and
reconciled when the authoritative log arrives. Nodes can also aggregate witness
observations into deterministic consensus candidates for simulation facts like
"who shot first," and those reports now have schema-v0 wire contracts. The next
useful layer is making Verse discovery and operator defaults first-class.
Membership and leader election should wait until basic replica catch-up is
boring. Authoritative mutation logs can now survive process restart through
`ICultNetShardMutationLogStore`; compaction now has an honest resync boundary,
and the shard replicator can recover from that boundary through snapshot
replacement.

8
src/GameCult.Networking/Contracts/cultnet.document-delete.schema.json
View file

@ -6,14 +6,14 @@
"required": [
"schemaVersion",
"messageId",
"documentType",
"documentKey"
"schemaId",
"recordKey"
],
"additionalProperties": false,
"properties": {
"schemaVersion": { "const": "cultnet.document_delete.v0" },
"messageId": { "type": "string", "minLength": 1 },
"documentType": { "type": "string", "minLength": 1 },
"documentKey": { "type": "string", "minLength": 1 }
"schemaId": { "type": "string", "minLength": 1 },
"recordKey": { "type": "string", "minLength": 1 }
}
}

40
src/GameCult.Networking/Contracts/cultnet.document-mutation-contract.schema.json Normal file
View file

@ -0,0 +1,40 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://github.com/GameCult/cultnet-ts/contracts/cultnet.document-mutation-contract.schema.json",
"title": "CultNet Document Mutation Contract",
"type": "object",
"required": [
"documentType",
"operations",
"authority"
],
"additionalProperties": false,
"properties": {
"documentType": { "type": "string", "minLength": 1 },
"payloadSchemaVersion": { "type": "string", "minLength": 1 },
"operations": {
"type": "array",
"minItems": 1,
"items": {
"type": "string",
"enum": ["snapshot", "documentPut", "documentDelete", "intentSubmit", "receiptWatch"]
}
},
"authority": {
"type": "string",
"enum": ["readOnly", "localUser", "coordinator", "runtime", "denied"]
},
"intentDocumentTypes": {
"type": "array",
"items": { "type": "string", "minLength": 1 }
},
"receiptDocumentTypes": {
"type": "array",
"items": { "type": "string", "minLength": 1 }
},
"notes": {
"type": "array",
"items": { "type": "string", "minLength": 1 }
}
}
}

9
src/GameCult.Networking/Contracts/cultnet.document-record.schema.json
View file

@ -4,17 +4,16 @@
"title": "CultNet Document Record",
"type": "object",
"required": [
"documentType",
"documentKey",
"schemaId",
"recordKey",
"storedAt",
"payload"
],
"additionalProperties": false,
"properties": {
"documentType": { "type": "string", "minLength": 1 },
"documentKey": { "type": "string", "minLength": 1 },
"schemaId": { "type": "string", "minLength": 1 },
"recordKey": { "type": "string", "minLength": 1 },
"storedAt": { "type": "string", "minLength": 1 },
"payloadSchemaVersion": { "type": "string", "minLength": 1 },
"payload": {},
"sourceRuntimeId": { "type": "string", "minLength": 1 },
"sourceAgentId": { "type": "string", "minLength": 1 },

6
src/GameCult.Networking/Contracts/cultnet.hello.schema.json
View file

@ -20,6 +20,12 @@
"type": "array",
"items": { "type": "string", "minLength": 1 }
},
"supportedMutationContracts": {
"type": "array",
"items": {
"$ref": "./cultnet.document-mutation-contract.schema.json"
}
},
"supportedMessageVersions": {
"type": "array",
"items": { "type": "string", "minLength": 1 }

9
src/GameCult.Networking/Contracts/cultnet.raw-document-record.schema.json
View file

@ -4,18 +4,17 @@
"title": "CultNet Raw Document Record",
"type": "object",
"required": [
"documentType",
"documentKey",
"schemaId",
"recordKey",
"storedAt",
"payloadEncoding",
"payload"
],
"additionalProperties": false,
"properties": {
"documentType": { "type": "string", "minLength": 1 },
"documentKey": { "type": "string", "minLength": 1 },
"schemaId": { "type": "string", "minLength": 1 },
"recordKey": { "type": "string", "minLength": 1 },
"storedAt": { "type": "string", "minLength": 1 },
"payloadSchemaVersion": { "type": "string", "minLength": 1 },
"payloadEncoding": { "const": "messagepack" },
"payload": {
"type": "string",

4
src/GameCult.Networking/Contracts/cultnet.snapshot-request.schema.json
View file

@ -11,11 +11,11 @@
"properties": {
"schemaVersion": { "const": "cultnet.snapshot_request.v0" },
"messageId": { "type": "string", "minLength": 1 },
"documentTypes": {
"schemaIds": {
"type": "array",
"items": { "type": "string", "minLength": 1 }
},
"documentKeys": {
"recordKeys": {
"type": "array",
"items": { "type": "string", "minLength": 1 }
}

1511
src/GameCult.Networking/CultNetDatabase.cs Normal file
View file

File diff suppressed because it is too large Load diff

464
src/GameCult.Networking/CultNetDatabaseServer.cs Normal file
View file

@ -0,0 +1,464 @@
using System;
using System.Collections.Concurrent;
using System.Linq;
using System.Reflection;
using System.Threading.Tasks;
using GameCult.Caching;
using LiteNetLib;
using R3;
namespace GameCult.Networking
{
/// <summary>
/// Routes CultNet schema-v0 document messages through a <see cref="CultNetDatabase"/>.
/// </summary>
public sealed class CultNetDatabaseServer : IDisposable
{
private readonly Server _server;
private readonly CultNetDatabase _database;
private readonly CultNetDatabaseServerOptions _options;
private readonly Func<CultNetSnapshotRequestMessage, NetPeer, Task> _snapshotHandler;
private readonly Func<CultNetDocumentPutRawMessage, NetPeer, Task> _putHandler;
private readonly Func<CultNetDocumentDeleteMessage, NetPeer, Task> _deleteHandler;
private readonly Func<CultNetShardCatalogRequestMessage, NetPeer, Task> _shardCatalogHandler;
private readonly Func<CultNetShardLogRequestMessage, NetPeer, Task> _shardLogHandler;
private readonly Func<CultNetDatabaseSubscribeMessage, NetPeer, Task> _subscribeHandler;
private readonly Func<CultNetDatabaseUnsubscribeMessage, NetPeer, Task> _unsubscribeHandler;
private readonly ConcurrentDictionary<string, IDisposable> _subscriptions = new(StringComparer.Ordinal);
private bool _disposed;
/// <summary>
/// Creates and attaches a database message bridge to a server.
/// </summary>
public CultNetDatabaseServer(
Server server,
CultNetDatabase database,
CultNetDatabaseServerOptions? options = null)
{
_server = server ?? throw new ArgumentNullException(nameof(server));
_database = database ?? throw new ArgumentNullException(nameof(database));
_options = options ?? new CultNetDatabaseServerOptions();
_snapshotHandler = HandleSnapshotRequestAsync;
_putHandler = HandlePutAsync;
_deleteHandler = HandleDeleteAsync;
_shardCatalogHandler = HandleShardCatalogRequestAsync;
_shardLogHandler = HandleShardLogRequestAsync;
_subscribeHandler = HandleSubscribeAsync;
_unsubscribeHandler = HandleUnsubscribeAsync;
_server.OnCultNet(_snapshotHandler);
_server.OnCultNet(_putHandler);
_server.OnCultNet(_deleteHandler);
_server.OnCultNet(_shardCatalogHandler);
_server.OnCultNet(_shardLogHandler);
_server.OnCultNet(_subscribeHandler);
_server.OnCultNet(_unsubscribeHandler);
}
/// <summary>
/// Gets the database used by this bridge.
/// </summary>
public CultNetDatabase Database => _database;
/// <summary>
/// Applies a raw put message through the database shard policy.
/// </summary>
public Task<object> ApplyPutAsync(CultNetDocumentPutRawMessage message)
{
return _database.ApplyPutAsync(message);
}
/// <summary>
/// Applies a raw delete message through the database shard policy.
/// </summary>
public Task ApplyDeleteAsync(CultNetDocumentDeleteMessage message)
{
return _database.ApplyDeleteAsync(message);
}
/// <summary>
/// Creates a raw snapshot response from the database cache.
/// </summary>
public CultNetSnapshotResponseRawMessage CreateSnapshotResponse(CultNetSnapshotRequestMessage request)
{
if (request == null) throw new ArgumentNullException(nameof(request));
if (!string.IsNullOrWhiteSpace(request.ShardId))
{
var shard = _database.Shards.FirstOrDefault(candidate =>
string.Equals(candidate.ShardId, request.ShardId, StringComparison.Ordinal));
if (shard == null)
{
throw new InvalidOperationException($"Shard '{request.ShardId}' is not known by this database.");
}
if (request.ShardEpoch.HasValue && request.ShardEpoch.Value != shard.Epoch)
{
throw new CultNetShardAuthorityException(
shard,
$"Shard '{shard.ShardId}' is at epoch {shard.Epoch}, not request epoch {request.ShardEpoch.Value}.",
"stale_epoch");
}
return _database.CreateShardSnapshotResponse(
shard,
string.IsNullOrWhiteSpace(request.MessageId) ? Guid.NewGuid().ToString("N") : request.MessageId,
request);
}
return _database.Documents.CreateRawSnapshotResponse(
_database.Cache,
string.IsNullOrWhiteSpace(request.MessageId) ? Guid.NewGuid().ToString("N") : request.MessageId,
request);
}
/// <summary>
/// Creates a shard catalog response from the database shard map.
/// </summary>
public CultNetShardCatalogResponseMessage CreateShardCatalogResponse(CultNetShardCatalogRequestMessage request)
{
return _database.CreateShardCatalogResponse(request);
}
/// <summary>
/// Creates a shard mutation-log response from the database log.
/// </summary>
public CultNetShardLogResponseMessage CreateShardLogResponse(CultNetShardLogRequestMessage request)
{
if (request == null) throw new ArgumentNullException(nameof(request));
if (string.IsNullOrWhiteSpace(request.ShardId))
{
throw new ArgumentException("Shard log request requires a shardId.", nameof(request));
}
var shard = _database.Shards.FirstOrDefault(candidate =>
string.Equals(candidate.ShardId, request.ShardId, StringComparison.Ordinal));
if (shard == null)
{
return new CultNetShardLogResponseMessage
{
MessageId = RequestMessageId(request.MessageId),
ShardId = request.ShardId,
ResyncRequired = true,
Reason = "unknown_shard"
};
}
if (request.ShardEpoch.HasValue && request.ShardEpoch.Value != shard.Epoch)
{
return new CultNetShardLogResponseMessage
{
MessageId = RequestMessageId(request.MessageId),
ShardId = request.ShardId,
ShardEpoch = shard.Epoch,
ResyncRequired = true,
Reason = "stale_epoch"
};
}
var compactedThrough = _database.GetCompactedMutationLogSequence(request.ShardId);
if (request.AfterSequence < compactedThrough)
{
return new CultNetShardLogResponseMessage
{
MessageId = RequestMessageId(request.MessageId),
ShardId = request.ShardId,
ShardEpoch = shard.Epoch,
ResyncRequired = true,
Reason = "compacted_history",
CompactedThrough = compactedThrough
};
}
var entries = _database.GetMutationLogMessages(request.ShardId, request.AfterSequence, request.Limit)
.ToArray();
return new CultNetShardLogResponseMessage
{
MessageId = RequestMessageId(request.MessageId),
ShardId = request.ShardId,
ShardEpoch = shard.Epoch,
Entries = entries
};
}
/// <summary>
/// Detaches handlers from the server.
/// </summary>
public void Dispose()
{
if (_disposed)
{
return;
}
_disposed = true;
_server.RemoveCultNetMessageListener<CultNetSnapshotRequestMessage>(_snapshotHandler);
_server.RemoveCultNetMessageListener<CultNetDocumentPutRawMessage>(_putHandler);
_server.RemoveCultNetMessageListener<CultNetDocumentDeleteMessage>(_deleteHandler);
_server.RemoveCultNetMessageListener<CultNetShardCatalogRequestMessage>(_shardCatalogHandler);
_server.RemoveCultNetMessageListener<CultNetShardLogRequestMessage>(_shardLogHandler);
_server.RemoveCultNetMessageListener<CultNetDatabaseSubscribeMessage>(_subscribeHandler);
_server.RemoveCultNetMessageListener<CultNetDatabaseUnsubscribeMessage>(_unsubscribeHandler);
foreach (var subscription in _subscriptions.Values)
{
subscription.Dispose();
}
_subscriptions.Clear();
}
private Task HandleSnapshotRequestAsync(CultNetSnapshotRequestMessage request, NetPeer peer)
{
peer.SendCultNet(CreateSnapshotResponse(request));
return Task.CompletedTask;
}
private Task HandleShardCatalogRequestAsync(CultNetShardCatalogRequestMessage request, NetPeer peer)
{
peer.SendCultNet(CreateShardCatalogResponse(request));
return Task.CompletedTask;
}
private Task HandleShardLogRequestAsync(CultNetShardLogRequestMessage request, NetPeer peer)
{
try
{
peer.SendCultNet(CreateShardLogResponse(request));
}
catch (Exception ex)
{
peer.SendCultNet(new CultNetErrorMessage { Error = ex.Message });
}
return Task.CompletedTask;
}
private async Task HandlePutAsync(CultNetDocumentPutRawMessage message, NetPeer peer)
{
try
{
await ApplyPutAsync(message).ConfigureAwait(false);
}
catch (CultNetShardAuthorityException ex)
{
if (!await TryForwardPutAsync(ex, message).ConfigureAwait(false))
{
peer.SendCultNet(CreateRoutingError(ex));
}
}
catch (Exception ex)
{
peer.SendCultNet(new CultNetErrorMessage { Error = ex.Message });
}
}
private async Task HandleDeleteAsync(CultNetDocumentDeleteMessage message, NetPeer peer)
{
try
{
await ApplyDeleteAsync(message).ConfigureAwait(false);
}
catch (CultNetShardAuthorityException ex)
{
if (!await TryForwardDeleteAsync(ex, message).ConfigureAwait(false))
{
peer.SendCultNet(CreateRoutingError(ex));
}
}
catch (Exception ex)
{
peer.SendCultNet(new CultNetErrorMessage { Error = ex.Message });
}
}
private Task HandleSubscribeAsync(CultNetDatabaseSubscribeMessage message, NetPeer peer)
{
var subscriptionId = string.IsNullOrWhiteSpace(message.SubscriptionId)
? message.MessageId
: message.SubscriptionId;
if (string.IsNullOrWhiteSpace(subscriptionId))
{
peer.SendCultNet(new CultNetErrorMessage { Error = "Database subscription requires a subscriptionId or messageId." });
return Task.CompletedTask;
}
var key = SubscriptionKey(peer, subscriptionId);
_subscriptions.AddOrUpdate(
key,
_ => CreateSubscription(message, subscriptionId, peer),
(_, existing) =>
{
existing.Dispose();
return CreateSubscription(message, subscriptionId, peer);
});
if (message.IncludeSnapshot)
{
peer.SendCultNet(CreateSnapshotResponse(new CultNetSnapshotRequestMessage
{
MessageId = message.MessageId,
SchemaIds = message.SchemaIds,
RecordKeys = message.RecordKeys
}));
}
return Task.CompletedTask;
}
private Task HandleUnsubscribeAsync(CultNetDatabaseUnsubscribeMessage message, NetPeer peer)
{
var subscriptionId = string.IsNullOrWhiteSpace(message.SubscriptionId)
? message.MessageId
: message.SubscriptionId;
if (!string.IsNullOrWhiteSpace(subscriptionId) &&
_subscriptions.TryRemove(SubscriptionKey(peer, subscriptionId), out var subscription))
{
subscription.Dispose();
}
return Task.CompletedTask;
}
private IDisposable CreateSubscription(
CultNetDatabaseSubscribeMessage request,
string subscriptionId,
NetPeer peer)
{
return _database.WatchAllChanges().Subscribe(change =>
{
var outbound = CreateChangeMessage(change, subscriptionId, request);
if (outbound != null)
{
peer.SendCultNet(outbound);
}
});
}
internal CultNetDatabaseChangeRawMessage? CreateChangeMessage(
object change,
string subscriptionId,
CultNetDatabaseSubscribeMessage request)
{
var changeType = change.GetType();
var key = (CultRecordKey)(changeType.GetProperty("Key")?.GetValue(change) ?? new CultRecordKey(string.Empty));
var schemaId = (string?)changeType.GetProperty("SchemaId")?.GetValue(change) ?? string.Empty;
if (!Matches(request, schemaId, key))
{
return null;
}
var kind = (CultNetDatabaseChangeKind)(changeType.GetProperty("Kind")?.GetValue(change) ?? CultNetDatabaseChangeKind.Updated);
var document = changeType.GetProperty("Document")?.GetValue(change);
if (kind == CultNetDatabaseChangeKind.Removed || document == null)
{
return new CultNetDatabaseChangeRawMessage
{
MessageId = Guid.NewGuid().ToString("N"),
SubscriptionId = subscriptionId,
ChangeKind = "removed",
RecordKey = key.Value,
SchemaId = schemaId
};
}
return new CultNetDatabaseChangeRawMessage
{
MessageId = Guid.NewGuid().ToString("N"),
SubscriptionId = subscriptionId,
ChangeKind = kind == CultNetDatabaseChangeKind.Added ? "added" : "updated",
Document = CreateRawDocumentRecord(key, document)
};
}
private CultNetRawDocumentRecord CreateRawDocumentRecord(CultRecordKey key, object document)
{
var method = typeof(CultNetDocumentRegistry)
.GetMethods(BindingFlags.Public | BindingFlags.Instance)
.Single(candidate => candidate.Name == nameof(CultNetDocumentRegistry.CreateRawDocumentPutMessage) &&
candidate.IsGenericMethodDefinition);
var documentType = document.GetType();
var handleType = typeof(CultRecordHandle<>).MakeGenericType(documentType);
var handle = Activator.CreateInstance(handleType, new object[] { key });
var message = method
.MakeGenericMethod(documentType)
.Invoke(_database.Documents, new[] { Guid.NewGuid().ToString("N"), handle, document, null });
return ((CultNetDocumentPutRawMessage)message!).Document;
}
private static bool Matches(CultNetDatabaseSubscribeMessage request, string schemaId, CultRecordKey key)
{
var schemaMatches = request.SchemaIds == null ||
request.SchemaIds.Length == 0 ||
request.SchemaIds.Contains(schemaId, StringComparer.Ordinal);
var keyMatches = request.RecordKeys == null ||
request.RecordKeys.Length == 0 ||
request.RecordKeys.Contains(key.Value, StringComparer.Ordinal);
return schemaMatches && keyMatches;
}
private static string SubscriptionKey(NetPeer peer, string subscriptionId)
{
return $"{peer.Id}:{subscriptionId}";
}
private static string RequestMessageId(string messageId)
{
return string.IsNullOrWhiteSpace(messageId)
? Guid.NewGuid().ToString("N")
: messageId;
}
private static CultNetErrorMessage CreateRoutingError(CultNetShardAuthorityException exception)
{
return new CultNetErrorMessage
{
Error = exception.Message,
RoutingHint = new CultNetShardRoutingHint
{
Reason = exception.Reason,
Shard = CultNetDatabase.ToMessage(exception.Shard)
}
};
}
/// <summary>
/// Attempts to forward a rejected raw put to the shard owner.
/// </summary>
public async Task<bool> TryForwardPutAsync(
CultNetShardAuthorityException exception,
CultNetDocumentPutRawMessage message)
{
if (!_options.ForwardNonPrimaryWrites ||
_options.WriteForwarder == null ||
exception.Reason != "not_primary")
{
return false;
}
message.ShardId ??= exception.Shard.ShardId;
message.ShardEpoch ??= exception.Shard.Epoch;
await _options.WriteForwarder.ForwardPutAsync(exception.Shard, message).ConfigureAwait(false);
return true;
}
/// <summary>
/// Attempts to forward a rejected raw delete to the shard owner.
/// </summary>
public async Task<bool> TryForwardDeleteAsync(
CultNetShardAuthorityException exception,
CultNetDocumentDeleteMessage message)
{
if (!_options.ForwardNonPrimaryWrites ||
_options.WriteForwarder == null ||
exception.Reason != "not_primary")
{
return false;
}
message.ShardId ??= exception.Shard.ShardId;
message.ShardEpoch ??= exception.Shard.Epoch;
await _options.WriteForwarder.ForwardDeleteAsync(exception.Shard, message).ConfigureAwait(false);
return true;
}
}
}

413
src/GameCult.Networking/CultNetDocumentRegistry.cs
View file

@ -9,101 +9,111 @@ using GameCult.Caching.MessagePack;
namespace GameCult.Networking
{
/// <summary>
/// Optional metadata attached when CultNet emits a document message.
/// Optional metadata applied when creating CultNet document messages.
/// </summary>
public sealed class CultNetDocumentMessageOptions
{
/// <summary>
/// Gets or sets an explicit storage timestamp.
/// </summary>
public string? StoredAt { get; set; }
/// <summary>
/// Gets or sets the runtime that produced the document.
/// </summary>
public string? SourceRuntimeId { get; set; }
/// <summary>
/// Gets or sets the agent that produced the document.
/// </summary>
public string? SourceAgentId { get; set; }
/// <summary>
/// Gets or sets the role that produced the document.
/// </summary>
public string? SourceRole { get; set; }
/// <summary>
/// Gets or sets optional document tags.
/// </summary>
public string[]? Tags { get; set; }
}
/// <summary>
/// Maps a concrete CultCache entry type to a shared CultNet document contract.
/// Binds a CultCache document type to CultNet payload serialization.
/// </summary>
public sealed class CultNetDocumentBinding
{
internal CultNetDocumentBinding(
Type entryType,
string documentType,
string? payloadSchemaVersion,
Func<DatabaseEntry, string> documentKeySelector,
Func<DatabaseEntry, byte[]> payloadSerializer,
Func<byte[], DatabaseEntry> payloadDeserializer)
Type documentType,
string schemaId,
Func<object, byte[]> payloadSerializer,
Func<byte[], object> payloadDeserializer)
{
EntryType = entryType ?? throw new ArgumentNullException(nameof(entryType));
DocumentType = string.IsNullOrWhiteSpace(documentType)
? throw new ArgumentException("DocumentType must be non-empty.", nameof(documentType))
: documentType;
PayloadSchemaVersion = payloadSchemaVersion;
DocumentKeySelector = documentKeySelector ?? throw new ArgumentNullException(nameof(documentKeySelector));
PayloadSerializer = payloadSerializer ?? throw new ArgumentNullException(nameof(payloadSerializer));
PayloadDeserializer = payloadDeserializer ?? throw new ArgumentNullException(nameof(payloadDeserializer));
DocumentType = documentType;
SchemaId = schemaId;
PayloadSerializer = payloadSerializer;
PayloadDeserializer = payloadDeserializer;
}
public Type EntryType { get; }
public string DocumentType { get; }
public string? PayloadSchemaVersion { get; }
public Func<DatabaseEntry, string> DocumentKeySelector { get; }
public Func<DatabaseEntry, byte[]> PayloadSerializer { get; }
public Func<byte[], DatabaseEntry> PayloadDeserializer { get; }
/// <summary>
/// Gets the bound document type.
/// </summary>
public Type DocumentType { get; }
/// <summary>
/// Gets the bound schema identifier.
/// </summary>
public string SchemaId { get; }
/// <summary>
/// Gets the payload serializer.
/// </summary>
public Func<object, byte[]> PayloadSerializer { get; }
/// <summary>
/// Gets the payload deserializer.
/// </summary>
public Func<byte[], object> PayloadDeserializer { get; }
public static CultNetDocumentBinding ForEntry<T>(
string documentType,
string? payloadSchemaVersion = null,
Func<T, string>? documentKeySelector = null,
/// <summary>
/// Creates a document binding for a typed CultCache document.
/// </summary>
public static CultNetDocumentBinding ForDocument<T>(
CultDocumentRegistry? registry = null,
string? schemaId = null,
Func<T, byte[]>? payloadSerializer = null,
Func<byte[], T>? payloadDeserializer = null)
where T : DatabaseEntry
where T : class
{
var descriptor = (registry ?? CultDocumentRegistry.Shared).GetRequired<T>();
return new CultNetDocumentBinding(
typeof(T),
documentType,
payloadSchemaVersion,
entry =>
string.IsNullOrWhiteSpace(schemaId) ? descriptor.SchemaId : schemaId!,
document =>
{
if (entry is not T typedEntry)
{
throw new InvalidOperationException(
$"Document binding for {typeof(T).Name} received {entry.GetType().Name}.");
}
return documentKeySelector != null
? documentKeySelector(typedEntry)
: typedEntry.ID.ToString("D", CultureInfo.InvariantCulture);
},
entry =>
{
if (entry is not T typedEntry)
{
throw new InvalidOperationException(
$"Document binding for {typeof(T).Name} received {entry.GetType().Name}.");
}
var typed = (T)document;
return payloadSerializer != null
? payloadSerializer(typedEntry)
: CultCacheEnvelopeSerialization.SerializePayload(typedEntry);
? payloadSerializer(typed)
: CultDocumentMessagePackSerialization.Serialize(typed);
},
payload => payloadDeserializer != null
? payloadDeserializer(payload)
: CultCacheEnvelopeSerialization.DeserializePayload<T>(payload));
: CultDocumentMessagePackSerialization.Deserialize<T>(payload));
}
}
/// <summary>
/// Creates and applies CultNet raw document/snapshot messages for bound CultCache entry types.
/// Creates and applies CultNet messages that replicate CultCache documents.
/// </summary>
public sealed class CultNetDocumentRegistry
{
private readonly Dictionary<string, CultNetDocumentBinding> _bindingsByDocumentType =
new Dictionary<string, CultNetDocumentBinding>(StringComparer.Ordinal);
private readonly Dictionary<Type, CultNetDocumentBinding> _bindingsByEntryType =
new Dictionary<Type, CultNetDocumentBinding>();
private readonly CultDocumentRegistry _documents;
private readonly Dictionary<string, CultNetDocumentBinding> _bindingsBySchemaId =
new(StringComparer.Ordinal);
private readonly Dictionary<Type, CultNetDocumentBinding> _bindingsByType = new();
public CultNetDocumentRegistry(IEnumerable<CultNetDocumentBinding>? bindings = null)
/// <summary>
/// Creates a document registry with optional bindings.
/// </summary>
public CultNetDocumentRegistry(
CultDocumentRegistry? documents = null,
IEnumerable<CultNetDocumentBinding>? bindings = null)
{
_documents = documents ?? CultDocumentRegistry.Shared;
if (bindings == null)
{
return;
@ -115,133 +125,164 @@ namespace GameCult.Networking
}
}
/// <summary>
/// Registers a document binding.
/// </summary>
public CultNetDocumentRegistry Register(CultNetDocumentBinding binding)
{
if (binding == null) throw new ArgumentNullException(nameof(binding));
_bindingsByDocumentType[binding.DocumentType] = binding;
_bindingsByEntryType[binding.EntryType] = binding;
_bindingsBySchemaId[binding.SchemaId] = binding;
_bindingsByType[binding.DocumentType] = binding;
return this;
}
public CultNetDocumentBinding? GetByDocumentType(string documentType)
/// <summary>
/// Gets a binding by schema identifier.
/// </summary>
public CultNetDocumentBinding? GetBySchemaId(string schemaId)
{
if (documentType == null) throw new ArgumentNullException(nameof(documentType));
return _bindingsByDocumentType.TryGetValue(documentType, out var binding) ? binding : null;
return _bindingsBySchemaId.TryGetValue(schemaId, out var binding) ? binding : null;
}
public CultNetDocumentBinding? GetByEntryType(Type entryType)
/// <summary>
/// Gets a binding by CLR document type.
/// </summary>
public CultNetDocumentBinding? GetByDocumentType(Type documentType)
{
if (entryType == null) throw new ArgumentNullException(nameof(entryType));
return _bindingsByEntryType.TryGetValue(entryType, out var binding) ? binding : null;
return _bindingsByType.TryGetValue(documentType, out var binding) ? binding : null;
}
/// <summary>
/// Creates a document delete message.
/// </summary>
public CultNetDocumentDeleteMessage CreateDocumentDeleteMessage(
string messageId,
string documentType,
string documentKey)
string schemaId,
string recordKey)
{
return new CultNetDocumentDeleteMessage
{
MessageId = RequireNonEmpty(messageId, nameof(messageId)),
DocumentType = RequireNonEmpty(documentType, nameof(documentType)),
DocumentKey = RequireNonEmpty(documentKey, nameof(documentKey))
SchemaId = RequireNonEmpty(schemaId, nameof(schemaId)),
RecordKey = RequireNonEmpty(recordKey, nameof(recordKey))
};
}
public CultNetDocumentPutRawMessage CreateRawDocumentPutMessage(
/// <summary>
/// Creates a raw document put message for a typed document.
/// </summary>
public CultNetDocumentPutRawMessage CreateRawDocumentPutMessage<T>(
string messageId,
DatabaseEntry entry,
CultRecordHandle<T> handle,
T document,
CultNetDocumentMessageOptions? options = null)
where T : class
{
if (entry == null) throw new ArgumentNullException(nameof(entry));
var binding = RequireBinding(entry.GetType());
var descriptor = _documents.GetRequired<T>();
var binding = GetByDocumentType(typeof(T)) ?? CultNetDocumentBinding.ForDocument<T>(_documents);
var storedAt = ResolveStoredAt(options);
var documentKey = binding.DocumentKeySelector(entry);
return new CultNetDocumentPutRawMessage
{
MessageId = RequireNonEmpty(messageId, nameof(messageId)),
Document = CreateRawDocumentRecord(binding, entry, documentKey, storedAt, options)
};
}
public CultNetDocumentPutRawMessage CreateRawDocumentPutMessageFromEnvelope(
string messageId,
CultCacheEnvelope envelope)
{
if (envelope == null) throw new ArgumentNullException(nameof(envelope));
var binding = RequireBinding(envelope.Type);
return new CultNetDocumentPutRawMessage
{
MessageId = RequireNonEmpty(messageId, nameof(messageId)),
Document = new CultNetRawDocumentRecord
{
DocumentType = envelope.Type,
DocumentKey = envelope.Key,
StoredAt = envelope.StoredAt,
PayloadSchemaVersion = binding.PayloadSchemaVersion,
SchemaId = binding.SchemaId,
RecordKey = handle.Key.Value,
StoredAt = storedAt,
PayloadEncoding = "messagepack",
Payload = envelope.Payload.ToArray()
Payload = binding.PayloadSerializer(document),
SourceRuntimeId = options?.SourceRuntimeId,
SourceAgentId = options?.SourceAgentId,
SourceRole = options?.SourceRole,
Tags = options?.Tags
}
};
}
/// <summary>
/// Creates a snapshot request message.
/// </summary>
public CultNetSnapshotRequestMessage CreateSnapshotRequest(
string messageId,
IEnumerable<string>? documentTypes = null,
IEnumerable<string>? documentKeys = null)
IEnumerable<string>? schemaIds = null,
IEnumerable<string>? recordKeys = null)
{
return new CultNetSnapshotRequestMessage
{
MessageId = RequireNonEmpty(messageId, nameof(messageId)),
DocumentTypes = documentTypes?.ToArray(),
DocumentKeys = documentKeys?.ToArray()
SchemaIds = schemaIds?.ToArray(),
RecordKeys = recordKeys?.ToArray()
};
}
/// <summary>
/// Creates a raw snapshot response from the cache.
/// </summary>
public CultNetSnapshotResponseRawMessage CreateRawSnapshotResponse(
CultCache cache,
string messageId,
CultNetSnapshotRequestMessage? filter = null,
CultNetDocumentMessageOptions? options = null)
{
if (cache == null) throw new ArgumentNullException(nameof(cache));
var requestedDocumentTypes = filter?.DocumentTypes != null
? new HashSet<string>(filter.DocumentTypes, StringComparer.Ordinal)
var requestedSchemaIds = filter?.SchemaIds != null
? new HashSet<string>(filter.SchemaIds, StringComparer.Ordinal)
: null;
var requestedDocumentKeys = filter?.DocumentKeys != null
? new HashSet<string>(filter.DocumentKeys, StringComparer.Ordinal)
var requestedRecordKeys = filter?.RecordKeys != null
? new HashSet<string>(filter.RecordKeys, StringComparer.Ordinal)
: null;
var storedAt = ResolveStoredAt(options);
var documents = new List<CultNetRawDocumentRecord>();
foreach (var entry in cache.AllEntries)
foreach (var document in cache.AllEntries)
{
var binding = GetByEntryType(entry.GetType());
if (binding == null)
var descriptor = _documents.GetRequired(document.GetType());
var binding = GetByDocumentType(document.GetType()) ??
new CultNetDocumentBinding(
document.GetType(),
descriptor.SchemaId,
value => CultDocumentMessagePackSerialization.SerializeUntyped(value, value.GetType()),
payload => CultDocumentMessagePackSerialization.DeserializeUntyped(document.GetType(), payload));
if (requestedSchemaIds != null && !requestedSchemaIds.Contains(descriptor.SchemaId))
{
if (!requestedSchemaIds.Contains(binding.SchemaId))
{
continue;
}
}
var handleMethod = typeof(CultCache)
.GetMethod(nameof(CultCache.TryGetHandle))!
.MakeGenericMethod(document.GetType());
var handleObject = handleMethod.Invoke(cache, new[] { document });
if (handleObject == null)
{
continue;
}
if (requestedDocumentTypes != null && !requestedDocumentTypes.Contains(binding.DocumentType))
var keyProperty = handleObject.GetType().GetProperty("Value");
var handleValue = keyProperty?.GetValue(handleObject) ?? handleObject;
var recordKeyProperty = handleValue.GetType().GetProperty("Key");
var recordKey = recordKeyProperty?.GetValue(handleValue);
var valueProperty = recordKey?.GetType().GetProperty("Value");
var key = (string?)(valueProperty?.GetValue(recordKey)) ?? string.Empty;
if (requestedRecordKeys != null && !requestedRecordKeys.Contains(key))
{
continue;
}
var documentKey = binding.DocumentKeySelector(entry);
if (requestedDocumentKeys != null && !requestedDocumentKeys.Contains(documentKey))
documents.Add(new CultNetRawDocumentRecord
{
continue;
}
documents.Add(CreateRawDocumentRecord(
binding,
entry,
documentKey,
storedAt,
options));
SchemaId = binding.SchemaId,
RecordKey = key,
StoredAt = storedAt,
PayloadEncoding = "messagepack",
Payload = binding.PayloadSerializer(document),
SourceRuntimeId = options?.SourceRuntimeId,
SourceAgentId = options?.SourceAgentId,
SourceRole = options?.SourceRole,
Tags = options?.Tags
});
}
return new CultNetSnapshotResponseRawMessage
@ -251,124 +292,81 @@ namespace GameCult.Networking
};
}
public async Task<DatabaseEntry> ApplyRawDocumentPutMessageAsync(
/// <summary>
/// Applies a raw document put message to a cache.
/// </summary>
public async Task<object> ApplyRawDocumentPutMessageAsync(
CultCache cache,
CultNetDocumentPutRawMessage message)
{
if (cache == null) throw new ArgumentNullException(nameof(cache));
if (message == null) throw new ArgumentNullException(nameof(message));
if (message.Document == null) throw new ArgumentException("CultNet raw document message is missing its document payload.", nameof(message));
if (message.Document == null)
{
throw new ArgumentException("CultNet raw document message is missing its document payload.", nameof(message));
}
var binding = RequireBinding(message.Document.DocumentType);
ValidateRawDocumentRecord(message.Document);
var binding = GetBySchemaId(message.Document.SchemaId);
var descriptor = binding != null
? _documents.GetRequired(binding.DocumentType)
: _documents.GetRequiredBySchemaId(message.Document.SchemaId);
binding ??= new CultNetDocumentBinding(
descriptor.DocumentType,
descriptor.SchemaId,
value => CultDocumentMessagePackSerialization.SerializeUntyped(value, value.GetType()),
payload => CultDocumentMessagePackSerialization.DeserializeUntyped(descriptor.DocumentType, payload));
var document = binding.PayloadDeserializer(message.Document.Payload);
var entry = binding.PayloadDeserializer(message.Document.Payload);
if (!binding.EntryType.IsInstanceOfType(entry))
{
throw new InvalidOperationException(
$"CultNet raw document type \"{message.Document.DocumentType}\" expected {binding.EntryType.Name} but decoded {entry.GetType().Name}.");
}
var expectedKey = binding.DocumentKeySelector(entry);
if (!string.Equals(expectedKey, message.Document.DocumentKey, StringComparison.Ordinal))
{
throw new InvalidOperationException(
$"CultNet raw document key mismatch for \"{message.Document.DocumentType}\": payload resolved to \"{expectedKey}\" but wire metadata said \"{message.Document.DocumentKey}\".");
}
await cache.AddAsync(entry);
return entry;
var addMethod = typeof(CultCache).GetMethod(nameof(CultCache.AddAsync))!
.MakeGenericMethod(descriptor.DocumentType);
var handleType = typeof(CultRecordHandle<>).MakeGenericType(descriptor.DocumentType);
var optionalHandle = Activator.CreateInstance(typeof(Nullable<>).MakeGenericType(handleType), new object[] { Activator.CreateInstance(handleType, new object[] { new CultRecordKey(message.Document.RecordKey) })! });
var task = (Task)addMethod.Invoke(cache, [document, optionalHandle])!;
await task.ConfigureAwait(false);
return document;
}
/// <summary>
/// Applies a raw document put message and returns the typed document.
/// </summary>
public async Task<T> ApplyRawDocumentPutMessageAsync<T>(
CultCache cache,
CultNetDocumentPutRawMessage message)
where T : DatabaseEntry
where T : class
{
var entry = await ApplyRawDocumentPutMessageAsync(cache, message);
if (entry is not T typedEntry)
{
throw new InvalidOperationException(
$"CultNet raw document message resolved to {entry.GetType().Name}, not {typeof(T).Name}.");
}
return typedEntry;
return (T)await ApplyRawDocumentPutMessageAsync(cache, message).ConfigureAwait(false);
}
public async Task<IReadOnlyList<DatabaseEntry>> ApplyRawSnapshotResponseAsync(
/// <summary>
/// Applies all documents from a raw snapshot response.
/// </summary>
public async Task<IReadOnlyList<object>> ApplyRawSnapshotResponseAsync(
CultCache cache,
CultNetSnapshotResponseRawMessage response)
{
if (cache == null) throw new ArgumentNullException(nameof(cache));
if (response == null) throw new ArgumentNullException(nameof(response));
var applied = new List<DatabaseEntry>(response.Documents.Length);
var applied = new List<object>(response.Documents.Length);
foreach (var document in response.Documents)
{
var entry = await ApplyRawDocumentPutMessageAsync(
applied.Add(await ApplyRawDocumentPutMessageAsync(
cache,
new CultNetDocumentPutRawMessage
{
MessageId = response.MessageId,
Document = document
});
applied.Add(entry);
}).ConfigureAwait(false));
}
return applied;
}
/// <summary>
/// Applies all typed documents from a raw snapshot response.
/// </summary>
public async Task<IReadOnlyList<T>> ApplyRawSnapshotResponseAsync<T>(
CultCache cache,
CultNetSnapshotResponseRawMessage response)
where T : DatabaseEntry
where T : class
{
var applied = await ApplyRawSnapshotResponseAsync(cache, response);
return applied.OfType<T>().ToArray();
}
private CultNetRawDocumentRecord CreateRawDocumentRecord(
CultNetDocumentBinding binding,
DatabaseEntry entry,
string documentKey,
string storedAt,
CultNetDocumentMessageOptions? options)
{
var envelope = new CultCacheEnvelope
{
Key = documentKey,
Type = binding.DocumentType,
StoredAt = storedAt,
Payload = binding.PayloadSerializer(entry)
};
return new CultNetRawDocumentRecord
{
DocumentType = envelope.Type,
DocumentKey = envelope.Key,
StoredAt = envelope.StoredAt,
PayloadSchemaVersion = binding.PayloadSchemaVersion,
PayloadEncoding = "messagepack",
Payload = envelope.Payload,
SourceRuntimeId = options?.SourceRuntimeId,
SourceAgentId = options?.SourceAgentId,
SourceRole = options?.SourceRole,
Tags = options?.Tags
};
}
private CultNetDocumentBinding RequireBinding(Type entryType)
{
return GetByEntryType(entryType)
?? throw new InvalidOperationException(
$"No CultNet document binding is registered for {entryType.FullName}.");
}
private CultNetDocumentBinding RequireBinding(string documentType)
{
return GetByDocumentType(documentType)
?? throw new InvalidOperationException(
$"No CultNet document binding is registered for \"{documentType}\".");
return (await ApplyRawSnapshotResponseAsync(cache, response).ConfigureAwait(false)).OfType<T>().ToArray();
}
private static void ValidateRawDocumentRecord(CultNetRawDocumentRecord document)
@ -379,14 +377,13 @@ namespace GameCult.Networking
$"CultNet raw document payloadEncoding must be \"messagepack\", not \"{document.PayloadEncoding}\".");
}
RequireNonEmpty(document.SchemaId, nameof(document.SchemaId));
RequireNonEmpty(document.RecordKey, nameof(document.RecordKey));
RequireNonEmpty(document.StoredAt, nameof(document.StoredAt));
if (document.Payload == null || document.Payload.Length == 0)
{
throw new InvalidOperationException("CultNet raw document payload must be non-empty.");
}
RequireNonEmpty(document.DocumentType, nameof(document.DocumentType));
RequireNonEmpty(document.DocumentKey, nameof(document.DocumentKey));
RequireNonEmpty(document.StoredAt, nameof(document.StoredAt));
}
private static string ResolveStoredAt(CultNetDocumentMessageOptions? options)

194
src/GameCult.Networking/CultNetLocal.cs Normal file
View file

@ -0,0 +1,194 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using GameCult.Caching;
using GameCult.Caching.MessagePack;
namespace GameCult.Networking
{
/// <summary>
/// Options for creating a local CultNet host over CultCache and the canonical MessagePack store.
/// </summary>
public sealed class CultNetHostOptions
{
/// <summary>
/// Gets or sets the cache-open options used for the underlying CultCache.
/// </summary>
public CultCacheOpenOptions? CacheOptions { get; set; }
/// <summary>
/// Gets or sets the server security configuration. When omitted, development defaults are used.
/// </summary>
public ServerSecurityOptions? Security { get; set; }
/// <summary>
/// Gets or sets whether the server should start immediately after creation.
/// </summary>
public bool StartServer { get; set; } = true;
/// <summary>
/// Gets or sets the database sharding options used for the hosted CultCache.
/// </summary>
public CultNetDatabaseOptions? DatabaseOptions { get; set; }
/// <summary>
/// Gets or sets the database server bridge options used for shard routing and forwarding.
/// </summary>
public CultNetDatabaseServerOptions? DatabaseServerOptions { get; set; }
/// <summary>
/// Gets or sets an optional callback used to customize the server before start.
/// </summary>
public Action<Server>? ConfigureServer { get; set; }
}
/// <summary>
/// Convenience wrapper for a locally hosted CultNet server over a durable CultCache.
/// </summary>
public sealed class CultNetHost : IDisposable
{
internal CultNetHost(
CultCache cache,
SingleFileMessagePackBackingStore? store,
Server server,
CultNetDatabase database,
CultNetDatabaseServer databaseServer)
{
Cache = cache;
Store = store;
Server = server;
Database = database;
DatabaseServer = databaseServer;
}
/// <summary>
/// Gets the durable cache used by the host.
/// </summary>
public CultCache Cache { get; }
/// <summary>
/// Gets the canonical single-file MessagePack store, when one is attached.
/// </summary>
public SingleFileMessagePackBackingStore? Store { get; }
/// <summary>
/// Gets the underlying CultNet server.
/// </summary>
public Server Server { get; }
/// <summary>
/// Gets the distributed database facade over the hosted cache.
/// </summary>
public CultNetDatabase Database { get; }
/// <summary>
/// Gets the schema-v0 server bridge for distributed database messages.
/// </summary>
public CultNetDatabaseServer DatabaseServer { get; }
/// <summary>
/// Starts the server.
/// </summary>
public void Start()
{
Server.Start();
}
/// <summary>
/// Flushes the durable cache.
/// </summary>
public Task FlushAsync(bool soft = false)
{
return Cache.FlushAsync(soft);
}
/// <summary>
/// Stops the server and disposes the host resources.
/// </summary>
public void Dispose()
{
DatabaseServer.Dispose();
Server.Dispose();
Database.Dispose();
Cache.Dispose();
}
}
/// <summary>
/// Friendly local entrypoints for CultNet host/client creation.
/// </summary>
public static class CultNetLocal
{
/// <summary>
/// Creates a local host with a durable CultCache and the canonical MessagePack backing store.
/// </summary>
public static async Task<CultNetHost> CreateHostAsync(string cachePath, CultNetHostOptions? options = null)
{
if (string.IsNullOrWhiteSpace(cachePath))
{
throw new ArgumentException("Cache path must be non-empty.", nameof(cachePath));
}
options ??= new CultNetHostOptions();
var cache = await CultCacheMessagePack.OpenAsync(cachePath, options.CacheOptions).ConfigureAwait(false);
var store = cache.BackingStores.OfType<SingleFileMessagePackBackingStore>().FirstOrDefault();
var server = new Server(cache, options.Security ?? ServerSecurityOptions.Development());
var databaseOptions = options.DatabaseOptions ?? new CultNetDatabaseOptions();
var database = new CultNetDatabase(cache, databaseOptions);
var databaseServer = new CultNetDatabaseServer(server, database, options.DatabaseServerOptions);
options.ConfigureServer?.Invoke(server);
var host = new CultNetHost(cache, store, server, database, databaseServer);
if (options.StartServer)
{
host.Start();
}
return host;
}
/// <summary>
/// Creates and starts a local host with development-friendly defaults.
/// </summary>
public static Task<CultNetHost> StartHostAsync(string cachePath, CultNetHostOptions? options = null)
{
options ??= new CultNetHostOptions();
options.StartServer = true;
return CreateHostAsync(cachePath, options);
}
/// <summary>
/// Creates a client configured for local CultNet use.
/// </summary>
public static Client CreateClient(ClientSecurityOptions? security = null, Action<Client>? configureClient = null)
{
var client = new Client(security ?? ClientSecurityOptions.Development());
configureClient?.Invoke(client);
return client;
}
/// <summary>
/// Creates and connects a client to the specified host.
/// </summary>
public static Client ConnectClient(
string host = "localhost",
int port = 3075,
ClientSecurityOptions? security = null,
Action<Client>? configureClient = null)
{
var client = CreateClient(security, configureClient);
client.Connect(host, port);
return client;
}
/// <summary>
/// Creates and connects a client to a local development host.
/// </summary>
public static Client ConnectLocal(
ClientSecurityOptions? security = null,
Action<Client>? configureClient = null)
{
return ConnectClient("localhost", 3075, security, configureClient);
}
}
}

27
src/GameCult.Networking/CultNetSchemaMessageSerialization.cs
View file

@ -8,21 +8,33 @@ namespace GameCult.Networking
/// </summary>
public static class CultNetSchemaMessageSerialization
{
/// <summary>
/// Gets the serializer options used for CultNet schema-v0 messages.
/// </summary>
public static readonly MessagePackSerializerOptions Options =
MessagePackSerializerOptions.Standard.WithSecurity(MessagePackSecurity.UntrustedData);
/// <summary>
/// Serializes a typed CultNet schema message.
/// </summary>
public static byte[] Serialize<T>(T message) where T : ICultNetSchemaMessage
{
if (message == null) throw new ArgumentNullException(nameof(message));
return MessagePackSerializer.Serialize(message, Options);
}
/// <summary>
/// Deserializes a typed CultNet schema message.
/// </summary>
public static T Deserialize<T>(byte[] payload) where T : ICultNetSchemaMessage
{
if (payload == null) throw new ArgumentNullException(nameof(payload));
return MessagePackSerializer.Deserialize<T>(payload, Options);
}
/// <summary>
/// Deserializes a CultNet schema message by inspecting its schema version.
/// </summary>
public static ICultNetSchemaMessage Deserialize(byte[] payload)
{
if (payload == null) throw new ArgumentNullException(nameof(payload));
@ -47,6 +59,19 @@ namespace GameCult.Networking
CultNetSchemaVersions.DocumentPutRaw => Deserialize<CultNetDocumentPutRawMessage>(payload),
CultNetSchemaVersions.SnapshotRequest => Deserialize<CultNetSnapshotRequestMessage>(payload),
CultNetSchemaVersions.SnapshotResponseRaw => Deserialize<CultNetSnapshotResponseRawMessage>(payload),
CultNetSchemaVersions.DatabaseSubscribe => Deserialize<CultNetDatabaseSubscribeMessage>(payload),
CultNetSchemaVersions.DatabaseUnsubscribe => Deserialize<CultNetDatabaseUnsubscribeMessage>(payload),
CultNetSchemaVersions.DatabaseChangeRaw => Deserialize<CultNetDatabaseChangeRawMessage>(payload),
CultNetSchemaVersions.ShardCatalogRequest => Deserialize<CultNetShardCatalogRequestMessage>(payload),
CultNetSchemaVersions.ShardCatalogResponse => Deserialize<CultNetShardCatalogResponseMessage>(payload),
CultNetSchemaVersions.ShardLogRequest => Deserialize<CultNetShardLogRequestMessage>(payload),
CultNetSchemaVersions.ShardLogResponse => Deserialize<CultNetShardLogResponseMessage>(payload),
CultNetSchemaVersions.SimulationObservation => Deserialize<CultNetSimulationObservationMessage>(payload),
CultNetSchemaVersions.SimulationConsensusCandidate => Deserialize<CultNetSimulationConsensusCandidateMessage>(payload),
CultNetSchemaVersions.VerseCatalogRequest => Deserialize<CultMeshVerseCatalogRequestMessage>(payload),
CultNetSchemaVersions.VerseCatalogResponse => Deserialize<CultMeshVerseCatalogResponseMessage>(payload),
CultNetSchemaVersions.PeerExchangeRequest => Deserialize<CultMeshPeerExchangeRequestMessage>(payload),
CultNetSchemaVersions.PeerExchangeResponse => Deserialize<CultMeshPeerExchangeResponseMessage>(payload),
CultNetSchemaVersions.SchemaCatalogRequest => Deserialize<CultNetSchemaCatalogRequestMessage>(payload),
CultNetSchemaVersions.SchemaCatalogResponse => Deserialize<CultNetSchemaCatalogResponseMessage>(payload),
_ => throw new MessagePackSerializationException(
@ -54,7 +79,7 @@ namespace GameCult.Networking
};
}
[MessagePackObject]
[MessagePackObject(AllowPrivate = true)]
internal sealed class CultNetSchemaHeader
{
[Key("schemaVersion")] public string SchemaVersion { get; set; } = string.Empty;

1092
src/GameCult.Networking/CultNetSchemaMessages.cs
View file

File diff suppressed because it is too large Load diff

48
src/GameCult.Networking/CultNetSchemaRegistry.cs
View file

@ -17,6 +17,9 @@ namespace GameCult.Networking
{
private readonly Dictionary<string, RegisteredSchema> _entries = new Dictionary<string, RegisteredSchema>(StringComparer.Ordinal);
/// <summary>
/// Creates a schema registry with optional initial entries.
/// </summary>
public CultNetSchemaRegistry(IEnumerable<CultNetSchemaRegistration>? entries = null)
{
if (entries == null)
@ -30,6 +33,9 @@ namespace GameCult.Networking
}
}
/// <summary>
/// Adds or replaces one schema registration.
/// </summary>
public CultNetSchemaRegistry Register(CultNetSchemaRegistration entry)
{
if (entry == null) throw new ArgumentNullException(nameof(entry));
@ -47,6 +53,9 @@ namespace GameCult.Networking
return this;
}
/// <summary>
/// Gets one schema descriptor by schema identifier.
/// </summary>
public CultNetSchemaDescriptor? Get(string schemaId, bool includeSchemaJson = false)
{
if (schemaId == null) throw new ArgumentNullException(nameof(schemaId));
@ -55,6 +64,9 @@ namespace GameCult.Networking
: null;
}
/// <summary>
/// Lists schema descriptors, optionally filtered by identifier or kind.
/// </summary>
public CultNetSchemaDescriptor[] List(
bool includeSchemaJson = false,
IEnumerable<string>? schemaIds = null,
@ -75,6 +87,9 @@ namespace GameCult.Networking
.ToArray();
}
/// <summary>
/// Creates a schema catalog response for a request.
/// </summary>
public CultNetSchemaCatalogResponseMessage CreateCatalogResponse(
CultNetSchemaCatalogRequestMessage request)
{
@ -90,6 +105,9 @@ namespace GameCult.Networking
};
}
/// <summary>
/// Creates a registry populated from embedded CultNet schema resources.
/// </summary>
public static CultNetSchemaRegistry CreateBuiltIn()
{
return new CultNetSchemaRegistry(BuiltInSchemaManifest.Select(LoadEmbeddedRegistration));
@ -98,6 +116,9 @@ namespace GameCult.Networking
private static readonly Lazy<CultNetSchemaRegistry> BuiltInRegistry =
new Lazy<CultNetSchemaRegistry>(CreateBuiltIn);
/// <summary>
/// Gets the lazily initialized built-in schema registry.
/// </summary>
public static CultNetSchemaRegistry BuiltIn => BuiltInRegistry.Value;
private static CultNetSchemaRegistration LoadEmbeddedRegistration(SchemaResourceSpec spec)
@ -239,6 +260,12 @@ namespace GameCult.Networking
WireContracts = new[] { CultNetWireContracts.SchemaV0 }
},
new SchemaResourceSpec
{
FileName = "cultnet.document-mutation-contract.schema.json",
Kind = "shared_contract",
WireContracts = new[] { CultNetWireContracts.SchemaV0 }
},
new SchemaResourceSpec
{
FileName = "cultnet.login.schema.json",
Kind = "wire_message",
@ -357,12 +384,33 @@ namespace GameCult.Networking
/// </summary>
public sealed class CultNetSchemaRegistration
{
/// <summary>
/// Gets or sets the schema identifier.
/// </summary>
public string SchemaId { get; set; } = string.Empty;
/// <summary>
/// Gets or sets the schema kind.
/// </summary>
public string Kind { get; set; } = string.Empty;
/// <summary>
/// Gets or sets the wire contracts implemented by the schema.
/// </summary>
public string[] WireContracts { get; set; } = Array.Empty<string>();
/// <summary>
/// Gets or sets the schema document.
/// </summary>
public string SchemaJson { get; set; } = string.Empty;
/// <summary>
/// Gets or sets the message or document schema version.
/// </summary>
public string? SchemaVersion { get; set; }
/// <summary>
/// Gets or sets the document type associated with the schema.
/// </summary>
public string? DocumentType { get; set; }
/// <summary>
/// Gets or sets the display title.
/// </summary>
public string? Title { get; set; }
}
}

187
src/GameCult.Networking/CultNetShardMutationLogStorage.cs Normal file
View file

@ -0,0 +1,187 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using MessagePack;
namespace GameCult.Networking
{
/// <summary>
/// Stores accepted shard-log entries in their replica catch-up wire shape.
/// </summary>
public interface ICultNetShardMutationLogStore
{
/// <summary>
/// Reads accepted log entries for a shard after the supplied sequence.
/// </summary>
IReadOnlyList<CultNetShardLogEntryMessage> Read(string shardId, long afterSequence = 0, int? limit = null);
/// <summary>
/// Appends or replaces one accepted log entry for a shard.
/// </summary>
void Append(string shardId, CultNetShardLogEntryMessage entry);
/// <summary>
/// Gets the highest sequence that has been compacted out of the retained log.
/// </summary>
long GetCompactedThrough(string shardId);
/// <summary>
/// Removes retained entries at or before the supplied sequence.
/// </summary>
void CompactThrough(string shardId, long sequence);
}
/// <summary>
/// Persists shard-log entries as one MessagePack file per shard.
/// </summary>
public sealed class CultNetFileShardMutationLogStore : ICultNetShardMutationLogStore
{
private readonly string _rootPath;
private readonly object _gate = new();
/// <summary>
/// Creates a file-backed shard mutation log store.
/// </summary>
public CultNetFileShardMutationLogStore(string rootPath)
{
if (string.IsNullOrWhiteSpace(rootPath)) throw new ArgumentException("Value must be non-empty.", nameof(rootPath));
_rootPath = rootPath;
Directory.CreateDirectory(_rootPath);
}
/// <inheritdoc />
public IReadOnlyList<CultNetShardLogEntryMessage> Read(string shardId, long afterSequence = 0, int? limit = null)
{
if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId));
if (afterSequence < 0) throw new ArgumentOutOfRangeException(nameof(afterSequence), "Sequence must be non-negative.");
lock (_gate)
{
IEnumerable<CultNetShardLogEntryMessage> entries = ReadAll(shardId)
.Where(entry => entry.Sequence > afterSequence)
.OrderBy(entry => entry.Sequence);
if (limit.HasValue)
{
entries = entries.Take(limit.Value);
}
return entries.ToArray();
}
}
/// <inheritdoc />
public void Append(string shardId, CultNetShardLogEntryMessage entry)
{
if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId));
if (entry == null) throw new ArgumentNullException(nameof(entry));
if (entry.Sequence <= 0) throw new ArgumentOutOfRangeException(nameof(entry), "Log entry sequence must be positive.");
lock (_gate)
{
var entries = ReadAll(shardId)
.Where(existing => existing.Sequence != entry.Sequence)
.Append(entry)
.OrderBy(existing => existing.Sequence)
.ToArray();
var payload = MessagePackSerializer.Serialize(entries, CultNetSchemaMessageSerialization.Options);
File.WriteAllBytes(GetShardPath(shardId), payload);
}
}
/// <inheritdoc />
public long GetCompactedThrough(string shardId)
{
if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId));
lock (_gate)
{
return ReadMetadata(shardId).CompactedThrough;
}
}
/// <inheritdoc />
public void CompactThrough(string shardId, long sequence)
{
if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId));
if (sequence < 0) throw new ArgumentOutOfRangeException(nameof(sequence), "Sequence must be non-negative.");
lock (_gate)
{
var metadata = ReadMetadata(shardId);
if (sequence <= metadata.CompactedThrough)
{
return;
}
var retained = ReadAll(shardId)
.Where(entry => entry.Sequence > sequence)
.OrderBy(entry => entry.Sequence)
.ToArray();
File.WriteAllBytes(
GetShardPath(shardId),
MessagePackSerializer.Serialize(retained, CultNetSchemaMessageSerialization.Options));
metadata.CompactedThrough = sequence;
WriteMetadata(shardId, metadata);
}
}
private CultNetShardLogEntryMessage[] ReadAll(string shardId)
{
var path = GetShardPath(shardId);
if (!File.Exists(path))
{
return Array.Empty<CultNetShardLogEntryMessage>();
}
var payload = File.ReadAllBytes(path);
return MessagePackSerializer.Deserialize<CultNetShardLogEntryMessage[]>(
payload,
CultNetSchemaMessageSerialization.Options) ?? Array.Empty<CultNetShardLogEntryMessage>();
}
private string GetShardPath(string shardId)
{
return Path.Combine(_rootPath, HashShardId(shardId) + ".mpack");
}
private string GetMetadataPath(string shardId)
{
return Path.Combine(_rootPath, HashShardId(shardId) + ".meta.mpack");
}
private CultNetShardMutationLogMetadata ReadMetadata(string shardId)
{
var path = GetMetadataPath(shardId);
if (!File.Exists(path))
{
return new CultNetShardMutationLogMetadata();
}
var payload = File.ReadAllBytes(path);
return MessagePackSerializer.Deserialize<CultNetShardMutationLogMetadata>(
payload,
CultNetSchemaMessageSerialization.Options) ?? new CultNetShardMutationLogMetadata();
}
private void WriteMetadata(string shardId, CultNetShardMutationLogMetadata metadata)
{
var payload = MessagePackSerializer.Serialize(metadata, CultNetSchemaMessageSerialization.Options);
File.WriteAllBytes(GetMetadataPath(shardId), payload);
}
private static string HashShardId(string shardId)
{
using var sha = SHA256.Create();
var hash = sha.ComputeHash(System.Text.Encoding.UTF8.GetBytes(shardId));
return string.Concat(hash.Select(value => value.ToString("x2")));
}
[MessagePackObject(AllowPrivate = true)]
internal sealed class CultNetShardMutationLogMetadata
{
[Key("compactedThrough")] public long CompactedThrough { get; set; }
}
}
}

605
src/GameCult.Networking/CultNetShardReplication.cs Normal file
View file

@ -0,0 +1,605 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using MessagePack;
namespace GameCult.Networking
{
/// <summary>
/// Fetches committed shard-log entries from an authoritative shard owner.
/// </summary>
public interface ICultNetShardLogFetcher
{
/// <summary>
/// Fetches shard-log entries after the supplied sequence.
/// </summary>
Task<CultNetShardLogResponseMessage> FetchAsync(
CultNetShardDescriptor shard,
long afterSequence,
int? limit = null);
}
/// <summary>
/// Fetches shard-bounded snapshots from an authoritative shard owner.
/// </summary>
public interface ICultNetShardSnapshotFetcher
{
/// <summary>
/// Fetches a shard snapshot.
/// </summary>
Task<CultNetSnapshotResponseRawMessage> FetchAsync(CultNetShardDescriptor shard);
}
/// <summary>
/// Persisted replica cursor for one shard.
/// </summary>
[MessagePackObject]
public sealed class CultNetShardReplicaCursor
{
/// <summary>
/// Gets or sets the shard id.
/// </summary>
[Key("shardId")] public string ShardId { get; set; } = string.Empty;
/// <summary>
/// Gets or sets the shard epoch.
/// </summary>
[Key("shardEpoch")] public long ShardEpoch { get; set; }
/// <summary>
/// Gets or sets the last applied shard-log sequence.
/// </summary>
[Key("lastAppliedSequence")] public long LastAppliedSequence { get; set; }
/// <summary>
/// Gets or sets the cursor update timestamp.
/// </summary>
[Key("updatedAt")] public string UpdatedAt { get; set; } = string.Empty;
}
/// <summary>
/// Stores restart-safe replica cursors.
/// </summary>
public interface ICultNetShardReplicaCursorStore
{
/// <summary>
/// Reads the cursor for a shard, if one exists.
/// </summary>
Task<CultNetShardReplicaCursor?> ReadAsync(string shardId);
/// <summary>
/// Writes the cursor for a shard.
/// </summary>
Task WriteAsync(CultNetShardReplicaCursor cursor);
}
/// <summary>
/// Options for background shard-log replication.
/// </summary>
public sealed class CultNetShardReplicatorOptions
{
/// <summary>
/// Gets or sets the interval used by the background pull loop.
/// </summary>
public TimeSpan PollInterval { get; set; } = TimeSpan.FromSeconds(1);
/// <summary>
/// Gets or sets the maximum number of log entries requested per pull.
/// </summary>
public int? BatchSize { get; set; } = 256;
/// <summary>
/// Gets or sets the fetcher used to read from shard primaries.
/// </summary>
public ICultNetShardLogFetcher? Fetcher { get; set; }
/// <summary>
/// Gets or sets the fetcher used when log history has been compacted.
/// </summary>
public ICultNetShardSnapshotFetcher? SnapshotFetcher { get; set; }
/// <summary>
/// Gets or sets a callback for background pull failures.
/// </summary>
public Action<Exception>? OnError { get; set; }
/// <summary>
/// Gets or sets optional restart-safe cursor storage.
/// </summary>
public ICultNetShardReplicaCursorStore? CursorStore { get; set; }
}
/// <summary>
/// Pulls committed shard logs from primaries and applies them to a local replica database.
/// </summary>
public sealed class CultNetShardReplicator : IDisposable
{
private readonly CultNetDatabase _database;
private readonly CultNetShardReplicatorOptions _options;
private readonly List<IDisposable> _subscriptions = new();
private readonly ConcurrentDictionary<string, byte> _pullsInFlight = new(StringComparer.Ordinal);
private bool _disposed;
/// <summary>
/// Creates a shard replicator.
/// </summary>
public CultNetShardReplicator(
CultNetDatabase database,
CultNetShardReplicatorOptions? options = null)
{
_database = database ?? throw new ArgumentNullException(nameof(database));
_options = options ?? new CultNetShardReplicatorOptions();
}
/// <summary>
/// Starts background pull loops for non-primary shards with advertised primaries.
/// </summary>
public void Start()
{
ThrowIfDisposed();
if (_options.Fetcher == null)
{
throw new InvalidOperationException("A shard log fetcher is required before starting replication.");
}
if (_subscriptions.Count > 0)
{
return;
}
foreach (var shard in _database.Shards.Where(shard => !shard.IsPrimary && shard.PrimaryEndpoints.Count > 0))
{
_subscriptions.Add(new Timer(
_ => _ = PullIfIdleAsync(shard),
state: null,
dueTime: TimeSpan.Zero,
period: _options.PollInterval));
}
}
/// <summary>
/// Pulls and applies one batch for a shard id.
/// </summary>
public Task<long> PullOnceAsync(string shardId)
{
ThrowIfDisposed();
if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId));
var shard = _database.Shards.FirstOrDefault(candidate =>
string.Equals(candidate.ShardId, shardId, StringComparison.Ordinal));
if (shard == null)
{
throw new InvalidOperationException($"Shard '{shardId}' is not known by this database.");
}
return PullOnceAsync(shard);
}
/// <summary>
/// Pulls and applies one batch for a shard.
/// </summary>
public async Task<long> PullOnceAsync(CultNetShardDescriptor shard)
{
ThrowIfDisposed();
if (shard == null) throw new ArgumentNullException(nameof(shard));
if (_options.Fetcher == null)
{
throw new InvalidOperationException("A shard log fetcher is required before pulling replication.");
}
if (shard.IsPrimary)
{
throw new InvalidOperationException($"Shard '{shard.ShardId}' is primary on this node and does not need replica pulling.");
}
var afterSequence = await GetAfterSequenceAsync(shard).ConfigureAwait(false);
var response = await _options.Fetcher
.FetchAsync(shard, afterSequence, _options.BatchSize)
.ConfigureAwait(false);
if (response.ResyncRequired &&
string.Equals(response.Reason, "compacted_history", StringComparison.Ordinal))
{
if (_options.SnapshotFetcher == null)
{
throw new InvalidOperationException(
$"Shard '{shard.ShardId}' requires snapshot resync, but no shard snapshot fetcher is configured.");
}
var snapshot = await _options.SnapshotFetcher.FetchAsync(shard).ConfigureAwait(false);
var snapshotSequence = await _database.ApplyShardSnapshotResponseAsync(shard, snapshot).ConfigureAwait(false);
await WriteCursorAsync(shard, snapshotSequence).ConfigureAwait(false);
return snapshotSequence;
}
var appliedSequence = await _database.ApplyShardLogResponseAsync(response).ConfigureAwait(false);
await WriteCursorAsync(shard, appliedSequence).ConfigureAwait(false);
return appliedSequence;
}
/// <inheritdoc />
public void Dispose()
{
if (_disposed)
{
return;
}
_disposed = true;
foreach (var subscription in _subscriptions)
{
subscription.Dispose();
}
_subscriptions.Clear();
_pullsInFlight.Clear();
}
private async Task PullIfIdleAsync(CultNetShardDescriptor shard)
{
if (!_pullsInFlight.TryAdd(shard.ShardId, 0))
{
return;
}
try
{
await PullOnceAsync(shard).ConfigureAwait(false);
}
catch (Exception ex)
{
_options.OnError?.Invoke(ex);
}
finally
{
_pullsInFlight.TryRemove(shard.ShardId, out _);
}
}
private async Task<long> GetAfterSequenceAsync(CultNetShardDescriptor shard)
{
var afterSequence = _database.GetAppliedShardSequence(shard.ShardId);
if (afterSequence > 0 || _options.CursorStore == null)
{
return afterSequence;
}
var cursor = await _options.CursorStore.ReadAsync(shard.ShardId).ConfigureAwait(false);
if (cursor == null || cursor.ShardEpoch != shard.Epoch)
{
return afterSequence;
}
_database.SetAppliedShardSequence(shard.ShardId, cursor.LastAppliedSequence);
return cursor.LastAppliedSequence;
}
private async Task WriteCursorAsync(CultNetShardDescriptor shard, long appliedSequence)
{
if (_options.CursorStore == null)
{
return;
}
await _options.CursorStore.WriteAsync(new CultNetShardReplicaCursor
{
ShardId = shard.ShardId,
ShardEpoch = shard.Epoch,
LastAppliedSequence = appliedSequence,
UpdatedAt = DateTimeOffset.UtcNow.ToString("O")
}).ConfigureAwait(false);
}
private void ThrowIfDisposed()
{
if (_disposed)
{
throw new ObjectDisposedException(nameof(CultNetShardReplicator));
}
}
}
/// <summary>
/// Stores replica cursors in one local MessagePack file.
/// </summary>
public sealed class CultNetFileShardReplicaCursorStore : ICultNetShardReplicaCursorStore
{
private readonly string _filePath;
private readonly object _gate = new();
/// <summary>
/// Creates a file-backed cursor store.
/// </summary>
public CultNetFileShardReplicaCursorStore(string filePath)
{
_filePath = string.IsNullOrWhiteSpace(filePath)
? throw new ArgumentException("Value must be non-empty.", nameof(filePath))
: filePath;
}
/// <inheritdoc />
public Task<CultNetShardReplicaCursor?> ReadAsync(string shardId)
{
if (string.IsNullOrWhiteSpace(shardId)) throw new ArgumentException("Value must be non-empty.", nameof(shardId));
lock (_gate)
{
CultNetShardReplicaCursor? cursor = ReadAll().FirstOrDefault(candidate =>
string.Equals(candidate.ShardId, shardId, StringComparison.Ordinal));
return Task.FromResult<CultNetShardReplicaCursor?>(cursor);
}
}
/// <inheritdoc />
public Task WriteAsync(CultNetShardReplicaCursor cursor)
{
if (cursor == null) throw new ArgumentNullException(nameof(cursor));
if (string.IsNullOrWhiteSpace(cursor.ShardId))
{
throw new ArgumentException("Cursor requires a shardId.", nameof(cursor));
}
lock (_gate)
{
var cursors = ReadAll()
.Where(existing => !string.Equals(existing.ShardId, cursor.ShardId, StringComparison.Ordinal))
.Concat(new[] { cursor })
.OrderBy(existing => existing.ShardId, StringComparer.Ordinal)
.ToArray();
var directory = Path.GetDirectoryName(Path.GetFullPath(_filePath));
if (!string.IsNullOrWhiteSpace(directory))
{
Directory.CreateDirectory(directory);
}
File.WriteAllBytes(_filePath, MessagePackSerializer.Serialize(cursors));
return Task.CompletedTask;
}
}
private CultNetShardReplicaCursor[] ReadAll()
{
if (!File.Exists(_filePath))
{
return Array.Empty<CultNetShardReplicaCursor>();
}
var bytes = File.ReadAllBytes(_filePath);
return bytes.Length == 0
? Array.Empty<CultNetShardReplicaCursor>()
: MessagePackSerializer.Deserialize<CultNetShardReplicaCursor[]>(bytes);
}
}
/// <summary>
/// Options for the schema-v0 client-based shard-log fetcher.
/// </summary>
public sealed class CultNetSchemaShardLogFetcherOptions
{
/// <summary>
/// Gets or sets client security options used to connect to primary endpoints.
/// </summary>
public ClientSecurityOptions? Security { get; set; }
/// <summary>
/// Gets or sets how long to wait for a connection before failing.
/// </summary>
public TimeSpan ConnectTimeout { get; set; } = TimeSpan.FromSeconds(5);
/// <summary>
/// Gets or sets how long to wait for a log response before failing.
/// </summary>
public TimeSpan ResponseTimeout { get; set; } = TimeSpan.FromSeconds(5);
/// <summary>
/// Gets or sets a callback used to customize each ephemeral fetch client.
/// </summary>
public Action<Client>? ConfigureClient { get; set; }
}
/// <summary>
/// Fetches shard-log batches over the CultNet schema-v0 client transport.
/// </summary>
public sealed class CultNetSchemaShardLogFetcher : ICultNetShardLogFetcher
{
private readonly CultNetSchemaShardLogFetcherOptions _options;
/// <summary>
/// Creates a schema-v0 shard-log fetcher.
/// </summary>
public CultNetSchemaShardLogFetcher(CultNetSchemaShardLogFetcherOptions? options = null)
{
_options = options ?? new CultNetSchemaShardLogFetcherOptions();
}
/// <inheritdoc />
public async Task<CultNetShardLogResponseMessage> FetchAsync(
CultNetShardDescriptor shard,
long afterSequence,
int? limit = null)
{
if (shard == null) throw new ArgumentNullException(nameof(shard));
var endpoint = ResolvePrimaryEndpoint(shard);
var (host, port) = CultNetSchemaWriteForwarder.ParseEndpoint(endpoint);
var messageId = Guid.NewGuid().ToString("N");
var completion = new TaskCompletionSource<CultNetShardLogResponseMessage>(
TaskCreationOptions.RunContinuationsAsynchronously);
using var client = new Client(_options.Security ?? ClientSecurityOptions.Development())
{
AllowUnverifiedCultNetMessages = true
};
_options.ConfigureClient?.Invoke(client);
client.OnCultNet<CultNetShardLogResponseMessage>(response =>
{
if (string.Equals(response.MessageId, messageId, StringComparison.Ordinal))
{
completion.TrySetResult(response);
}
});
client.OnCultNet<CultNetErrorMessage>(error =>
completion.TrySetException(new InvalidOperationException(error.Error)));
client.Connect(host, port);
await WaitForConnectionAsync(client, endpoint).ConfigureAwait(false);
client.SendCultNet(new CultNetShardLogRequestMessage
{
MessageId = messageId,
ShardId = shard.ShardId,
ShardEpoch = shard.Epoch,
AfterSequence = afterSequence,
Limit = limit
});
return await WaitForResponseAsync(completion.Task, endpoint).ConfigureAwait(false);
}
private async Task WaitForConnectionAsync(Client client, string endpoint)
{
var deadline = DateTimeOffset.UtcNow + _options.ConnectTimeout;
while (!client.Connected)
{
if (DateTimeOffset.UtcNow >= deadline)
{
throw new TimeoutException($"Timed out connecting to shard primary endpoint {endpoint}.");
}
await Task.Delay(TimeSpan.FromMilliseconds(25)).ConfigureAwait(false);
}
}
private async Task<CultNetShardLogResponseMessage> WaitForResponseAsync(
Task<CultNetShardLogResponseMessage> responseTask,
string endpoint)
{
var timeoutTask = Task.Delay(_options.ResponseTimeout);
var completed = await Task.WhenAny(responseTask, timeoutTask).ConfigureAwait(false);
if (completed != responseTask)
{
throw new TimeoutException($"Timed out waiting for shard log response from {endpoint}.");
}
return await responseTask.ConfigureAwait(false);
}
private static string ResolvePrimaryEndpoint(CultNetShardDescriptor shard)
{
return shard.PrimaryEndpoints.FirstOrDefault()
?? throw new InvalidOperationException(
$"Shard '{shard.ShardId}' does not advertise a primary endpoint.");
}
}
/// <summary>
/// Options for the schema-v0 client-based shard snapshot fetcher.
/// </summary>
public sealed class CultNetSchemaShardSnapshotFetcherOptions
{
/// <summary>
/// Gets or sets client security options used to connect to primary endpoints.
/// </summary>
public ClientSecurityOptions? Security { get; set; }
/// <summary>
/// Gets or sets how long to wait for a connection before failing.
/// </summary>
public TimeSpan ConnectTimeout { get; set; } = TimeSpan.FromSeconds(5);
/// <summary>
/// Gets or sets how long to wait for a snapshot response before failing.
/// </summary>
public TimeSpan ResponseTimeout { get; set; } = TimeSpan.FromSeconds(5);
/// <summary>
/// Gets or sets a callback used to customize each ephemeral fetch client.
/// </summary>
public Action<Client>? ConfigureClient { get; set; }
}
/// <summary>
/// Fetches shard snapshots over the CultNet schema-v0 client transport.
/// </summary>
public sealed class CultNetSchemaShardSnapshotFetcher : ICultNetShardSnapshotFetcher
{
private readonly CultNetSchemaShardSnapshotFetcherOptions _options;
/// <summary>
/// Creates a schema-v0 shard snapshot fetcher.
/// </summary>
public CultNetSchemaShardSnapshotFetcher(CultNetSchemaShardSnapshotFetcherOptions? options = null)
{
_options = options ?? new CultNetSchemaShardSnapshotFetcherOptions();
}
/// <inheritdoc />
public async Task<CultNetSnapshotResponseRawMessage> FetchAsync(CultNetShardDescriptor shard)
{
if (shard == null) throw new ArgumentNullException(nameof(shard));
var endpoint = ResolvePrimaryEndpoint(shard);
var (host, port) = CultNetSchemaWriteForwarder.ParseEndpoint(endpoint);
var messageId = Guid.NewGuid().ToString("N");
var completion = new TaskCompletionSource<CultNetSnapshotResponseRawMessage>(
TaskCreationOptions.RunContinuationsAsynchronously);
using var client = new Client(_options.Security ?? ClientSecurityOptions.Development())
{
AllowUnverifiedCultNetMessages = true
};
_options.ConfigureClient?.Invoke(client);
client.OnCultNet<CultNetSnapshotResponseRawMessage>(response =>
{
if (string.Equals(response.MessageId, messageId, StringComparison.Ordinal))
{
completion.TrySetResult(response);
}
});
client.OnCultNet<CultNetErrorMessage>(error =>
completion.TrySetException(new InvalidOperationException(error.Error)));
client.Connect(host, port);
await WaitForConnectionAsync(client, endpoint).ConfigureAwait(false);
client.SendCultNet(new CultNetSnapshotRequestMessage
{
MessageId = messageId,
SchemaIds = shard.SchemaIds.Count == 0 ? null : shard.SchemaIds.ToArray(),
ShardId = shard.ShardId,
ShardEpoch = shard.Epoch
});
return await WaitForResponseAsync(completion.Task, endpoint).ConfigureAwait(false);
}
private async Task WaitForConnectionAsync(Client client, string endpoint)
{
var deadline = DateTimeOffset.UtcNow + _options.ConnectTimeout;
while (!client.Connected)
{
if (DateTimeOffset.UtcNow >= deadline)
{
throw new TimeoutException($"Timed out connecting to shard primary endpoint {endpoint}.");
}
await Task.Delay(TimeSpan.FromMilliseconds(25)).ConfigureAwait(false);
}
}
private async Task<CultNetSnapshotResponseRawMessage> WaitForResponseAsync(
Task<CultNetSnapshotResponseRawMessage> responseTask,
string endpoint)
{
var timeoutTask = Task.Delay(_options.ResponseTimeout);
var completed = await Task.WhenAny(responseTask, timeoutTask).ConfigureAwait(false);
if (completed != responseTask)
{
throw new TimeoutException($"Timed out waiting for shard snapshot response from {endpoint}.");
}
return await responseTask.ConfigureAwait(false);
}
private static string ResolvePrimaryEndpoint(CultNetShardDescriptor shard)
{
return shard.PrimaryEndpoints.FirstOrDefault()
?? throw new InvalidOperationException(
$"Shard '{shard.ShardId}' does not advertise a primary endpoint.");
}
}
}

150
src/GameCult.Networking/CultNetShardWriteForwarding.cs Normal file
View file

@ -0,0 +1,150 @@
using System;
using System.Linq;
using System.Threading.Tasks;
namespace GameCult.Networking
{
/// <summary>
/// Forwards shard writes to an authoritative owner.
/// </summary>
public interface ICultNetShardWriteForwarder
{
/// <summary>
/// Forwards a raw document put to the shard owner.
/// </summary>
Task ForwardPutAsync(CultNetShardDescriptor shard, CultNetDocumentPutRawMessage message);
/// <summary>
/// Forwards a raw document delete to the shard owner.
/// </summary>
Task ForwardDeleteAsync(CultNetShardDescriptor shard, CultNetDocumentDeleteMessage message);
}
/// <summary>
/// Options controlling shard write forwarding from a database server bridge.
/// </summary>
public sealed class CultNetDatabaseServerOptions
{
/// <summary>
/// Gets or sets whether non-primary writes may be forwarded to shard owners.
/// </summary>
public bool ForwardNonPrimaryWrites { get; set; }
/// <summary>
/// Gets or sets the forwarder used when non-primary write forwarding is enabled.
/// </summary>
public ICultNetShardWriteForwarder? WriteForwarder { get; set; }
}
/// <summary>
/// Options for the schema-v0 client-based shard write forwarder.
/// </summary>
public sealed class CultNetSchemaWriteForwarderOptions
{
/// <summary>
/// Gets or sets client security options used to connect to primary endpoints.
/// </summary>
public ClientSecurityOptions? Security { get; set; }
/// <summary>
/// Gets or sets how long to wait for a connection before failing the forwarded write.
/// </summary>
public TimeSpan ConnectTimeout { get; set; } = TimeSpan.FromSeconds(5);
/// <summary>
/// Gets or sets a callback used to customize each ephemeral forwarding client.
/// </summary>
public Action<Client>? ConfigureClient { get; set; }
}
/// <summary>
/// Forwards shard writes over the CultNet schema-v0 client transport.
/// </summary>
public sealed class CultNetSchemaWriteForwarder : ICultNetShardWriteForwarder
{
private readonly CultNetSchemaWriteForwarderOptions _options;
/// <summary>
/// Creates a schema-v0 write forwarder.
/// </summary>
public CultNetSchemaWriteForwarder(CultNetSchemaWriteForwarderOptions? options = null)
{
_options = options ?? new CultNetSchemaWriteForwarderOptions();
}
/// <inheritdoc />
public async Task ForwardPutAsync(CultNetShardDescriptor shard, CultNetDocumentPutRawMessage message)
{
if (message == null) throw new ArgumentNullException(nameof(message));
var endpoint = ResolvePrimaryEndpoint(shard);
message.ShardId ??= shard.ShardId;
message.ShardEpoch ??= shard.Epoch;
await SendAsync(endpoint, message).ConfigureAwait(false);
}
/// <inheritdoc />
public async Task ForwardDeleteAsync(CultNetShardDescriptor shard, CultNetDocumentDeleteMessage message)
{
if (message == null) throw new ArgumentNullException(nameof(message));
var endpoint = ResolvePrimaryEndpoint(shard);
message.ShardId ??= shard.ShardId;
message.ShardEpoch ??= shard.Epoch;
await SendAsync(endpoint, message).ConfigureAwait(false);
}
/// <summary>
/// Parses a CultNet endpoint URI into host and port.
/// </summary>
public static (string Host, int Port) ParseEndpoint(string endpoint)
{
if (string.IsNullOrWhiteSpace(endpoint))
{
throw new ArgumentException("Endpoint must be non-empty.", nameof(endpoint));
}
if (!Uri.TryCreate(endpoint, UriKind.Absolute, out var uri) ||
!string.Equals(uri.Scheme, "cultnet", StringComparison.OrdinalIgnoreCase) ||
string.IsNullOrWhiteSpace(uri.Host))
{
throw new FormatException($"CultNet endpoint '{endpoint}' must use cultnet://host:port.");
}
return (uri.Host, uri.IsDefaultPort ? 3075 : uri.Port);
}
private async Task SendAsync<T>(string endpoint, T message) where T : ICultNetSchemaMessage
{
var (host, port) = ParseEndpoint(endpoint);
using var client = new Client(_options.Security ?? ClientSecurityOptions.Development())
{
AllowUnverifiedCultNetMessages = true
};
_options.ConfigureClient?.Invoke(client);
client.Connect(host, port);
await WaitForConnectionAsync(client, endpoint).ConfigureAwait(false);
client.SendCultNet(message);
}
private async Task WaitForConnectionAsync(Client client, string endpoint)
{
var deadline = DateTimeOffset.UtcNow + _options.ConnectTimeout;
while (!client.Connected)
{
if (DateTimeOffset.UtcNow >= deadline)
{
throw new TimeoutException($"Timed out connecting to shard primary endpoint {endpoint}.");
}
await Task.Delay(TimeSpan.FromMilliseconds(25)).ConfigureAwait(false);
}
}
private static string ResolvePrimaryEndpoint(CultNetShardDescriptor shard)
{
if (shard == null) throw new ArgumentNullException(nameof(shard));
return shard.PrimaryEndpoints.FirstOrDefault()
?? throw new InvalidOperationException(
$"Shard '{shard.ShardId}' does not advertise a primary endpoint.");
}
}
}

399
src/GameCult.Networking/CultNetSimulationConsensus.cs Normal file
View file

@ -0,0 +1,399 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using MessagePack;
using R3;
namespace GameCult.Networking
{
/// <summary>
/// One witness report about a deterministic simulation fact.
/// </summary>
[MessagePackObject]
public sealed class CultNetSimulationObservation
{
/// <summary>
/// Gets or sets the observing runtime id.
/// </summary>
[Key("witnessRuntimeId")] public string WitnessRuntimeId { get; set; } = string.Empty;
/// <summary>
/// Gets or sets the shard id containing the observed simulation.
/// </summary>
[Key("shardId")] public string ShardId { get; set; } = string.Empty;
/// <summary>
/// Gets or sets the shard epoch observed by the witness.
/// </summary>
[Key("shardEpoch")] public long ShardEpoch { get; set; }
/// <summary>
/// Gets or sets the simulation frame or tick.
/// </summary>
[Key("frame")] public long Frame { get; set; }
/// <summary>
/// Gets or sets the subject this observation is about.
/// </summary>
[Key("subjectId")] public string SubjectId { get; set; } = string.Empty;
/// <summary>
/// Gets or sets the kind of claim, such as hit, input, collision, or ownership.
/// </summary>
[Key("claimKind")] public string ClaimKind { get; set; } = string.Empty;
/// <summary>
/// Gets or sets the content hash of the witness claim.
/// </summary>
[Key("claimHash")] public string ClaimHash { get; set; } = string.Empty;
/// <summary>
/// Gets or sets optional human-readable or app-specific claim detail.
/// </summary>
[Key("claimSummary")] public string? ClaimSummary { get; set; }
/// <summary>
/// Gets or sets the witness weight. Normal clients should use 1.
/// </summary>
[Key("weight")] public double Weight { get; set; } = 1d;
/// <summary>
/// Gets or sets the observation timestamp.
/// </summary>
[Key("observedAt")] public string ObservedAt { get; set; } = string.Empty;
/// <summary>
/// Computes a stable claim hash from ordered claim parts.
/// </summary>
public static string ComputeClaimHash(params string[] parts)
{
if (parts == null) throw new ArgumentNullException(nameof(parts));
var canonical = string.Join("\u001F", parts.Select(part => part ?? string.Empty));
using var sha = SHA256.Create();
var bytes = sha.ComputeHash(Encoding.UTF8.GetBytes(canonical));
return string.Concat(bytes.Select(value => value.ToString("x2")));
}
}
/// <summary>
/// Controls deterministic witness aggregation.
/// </summary>
public sealed class CultNetSimulationConsensusOptions
{
/// <summary>
/// Gets or sets the minimum distinct witnesses required before a claim can be accepted.
/// </summary>
public int MinimumWitnesses { get; set; } = 1;
/// <summary>
/// Gets or sets the minimum weight required before a claim can be accepted.
/// </summary>
public double MinimumWeight { get; set; } = 1d;
/// <summary>
/// Gets or sets the fraction of observed weight required for quorum.
/// </summary>
public double QuorumRatio { get; set; } = 0.5d;
}
/// <summary>
/// A deterministic consensus candidate derived from witness observations.
/// </summary>
public sealed class CultNetSimulationConsensusCandidate
{
/// <summary>
/// Creates a consensus candidate.
/// </summary>
public CultNetSimulationConsensusCandidate(
string shardId,
long shardEpoch,
long frame,
string subjectId,
string claimKind,
string claimHash,
string? claimSummary,
int witnessCount,
double supportWeight,
double totalWeight,
bool hasQuorum)
{
ShardId = shardId;
ShardEpoch = shardEpoch;
Frame = frame;
SubjectId = subjectId;
ClaimKind = claimKind;
ClaimHash = claimHash;
ClaimSummary = claimSummary;
WitnessCount = witnessCount;
SupportWeight = supportWeight;
TotalWeight = totalWeight;
HasQuorum = hasQuorum;
}
/// <summary>
/// Gets the shard id.
/// </summary>
public string ShardId { get; }
/// <summary>
/// Gets the shard epoch.
/// </summary>
public long ShardEpoch { get; }
/// <summary>
/// Gets the simulation frame.
/// </summary>
public long Frame { get; }
/// <summary>
/// Gets the subject id.
/// </summary>
public string SubjectId { get; }
/// <summary>
/// Gets the claim kind.
/// </summary>
public string ClaimKind { get; }
/// <summary>
/// Gets the claim hash.
/// </summary>
public string ClaimHash { get; }
/// <summary>
/// Gets optional claim detail.
/// </summary>
public string? ClaimSummary { get; }
/// <summary>
/// Gets the distinct witness count supporting this candidate.
/// </summary>
public int WitnessCount { get; }
/// <summary>
/// Gets supporting witness weight.
/// </summary>
public double SupportWeight { get; }
/// <summary>
/// Gets total observed witness weight for the same subject/frame/kind.
/// </summary>
public double TotalWeight { get; }
/// <summary>
/// Gets whether the candidate crossed the configured quorum.
/// </summary>
public bool HasQuorum { get; }
/// <summary>
/// Gets support divided by total observed weight.
/// </summary>
public double Confidence => TotalWeight <= 0d ? 0d : SupportWeight / TotalWeight;
}
/// <summary>
/// Aggregates simulation observations into deterministic consensus candidates.
/// </summary>
public sealed class CultNetSimulationConsensus
{
private readonly CultNetSimulationConsensusOptions _options;
/// <summary>
/// Creates a consensus aggregator.
/// </summary>
public CultNetSimulationConsensus(CultNetSimulationConsensusOptions? options = null)
{
_options = options ?? new CultNetSimulationConsensusOptions();
}
/// <summary>
/// Returns the strongest candidate for each observed shard/frame/subject/claim kind.
/// </summary>
public IReadOnlyList<CultNetSimulationConsensusCandidate> BuildCandidates(
IEnumerable<CultNetSimulationObservation> observations)
{
if (observations == null) throw new ArgumentNullException(nameof(observations));
var clean = observations
.Where(IsValid)
.GroupBy(DeduplicationKey)
.Select(group => group
.OrderByDescending(observation => observation.Weight)
.ThenBy(observation => observation.WitnessRuntimeId, StringComparer.Ordinal)
.First())
.ToArray();
return clean
.GroupBy(SubjectKey)
.Select(BuildCandidate)
.OrderBy(candidate => candidate.ShardId, StringComparer.Ordinal)
.ThenBy(candidate => candidate.ShardEpoch)
.ThenBy(candidate => candidate.Frame)
.ThenBy(candidate => candidate.SubjectId, StringComparer.Ordinal)
.ThenBy(candidate => candidate.ClaimKind, StringComparer.Ordinal)
.ToArray();
}
private CultNetSimulationConsensusCandidate BuildCandidate(
IGrouping<string, CultNetSimulationObservation> subjectGroup)
{
var totalWeight = subjectGroup.Sum(WeightOf);
var bestClaim = subjectGroup
.GroupBy(observation => observation.ClaimHash, StringComparer.Ordinal)
.Select(group => new
{
ClaimHash = group.Key,
Observations = group.ToArray(),
Weight = group.Sum(WeightOf)
})
.OrderByDescending(candidate => candidate.Weight)
.ThenBy(candidate => candidate.ClaimHash, StringComparer.Ordinal)
.First();
var first = bestClaim.Observations
.OrderBy(observation => observation.WitnessRuntimeId, StringComparer.Ordinal)
.First();
var witnessCount = bestClaim.Observations
.Select(observation => observation.WitnessRuntimeId)
.Distinct(StringComparer.Ordinal)
.Count();
var hasQuorum = witnessCount >= _options.MinimumWitnesses &&
bestClaim.Weight >= _options.MinimumWeight &&
totalWeight > 0d &&
bestClaim.Weight / totalWeight >= _options.QuorumRatio;
return new CultNetSimulationConsensusCandidate(
first.ShardId,
first.ShardEpoch,
first.Frame,
first.SubjectId,
first.ClaimKind,
bestClaim.ClaimHash,
first.ClaimSummary,
witnessCount,
bestClaim.Weight,
totalWeight,
hasQuorum);
}
private static bool IsValid(CultNetSimulationObservation observation)
{
return observation != null &&
!string.IsNullOrWhiteSpace(observation.WitnessRuntimeId) &&
!string.IsNullOrWhiteSpace(observation.ShardId) &&
!string.IsNullOrWhiteSpace(observation.SubjectId) &&
!string.IsNullOrWhiteSpace(observation.ClaimKind) &&
!string.IsNullOrWhiteSpace(observation.ClaimHash) &&
observation.Weight > 0d;
}
private static double WeightOf(CultNetSimulationObservation observation)
{
return observation.Weight <= 0d ? 0d : observation.Weight;
}
private static string DeduplicationKey(CultNetSimulationObservation observation)
{
return string.Join("|",
observation.WitnessRuntimeId,
observation.ShardId,
observation.ShardEpoch,
observation.Frame,
observation.SubjectId,
observation.ClaimKind);
}
private static string SubjectKey(CultNetSimulationObservation observation)
{
return string.Join("|",
observation.ShardId,
observation.ShardEpoch,
observation.Frame,
observation.SubjectId,
observation.ClaimKind);
}
}
/// <summary>
/// Collects witness observations and publishes updated consensus candidates.
/// </summary>
public sealed class CultNetSimulationObservationHub : IDisposable
{
private readonly List<CultNetSimulationObservation> _observations = new();
private readonly CultNetSimulationConsensus _consensus;
private readonly Subject<CultNetSimulationObservation> _observationStream = new();
private readonly Subject<CultNetSimulationConsensusCandidate> _candidateStream = new();
private bool _disposed;
/// <summary>
/// Creates an observation hub.
/// </summary>
public CultNetSimulationObservationHub(CultNetSimulationConsensusOptions? options = null)
{
_consensus = new CultNetSimulationConsensus(options);
}
/// <summary>
/// Gets all observations currently held by the hub.
/// </summary>
public IReadOnlyList<CultNetSimulationObservation> Observations => _observations;
/// <summary>
/// Watches accepted witness observations.
/// </summary>
public Observable<CultNetSimulationObservation> WatchObservations()
{
ThrowIfDisposed();
return _observationStream;
}
/// <summary>
/// Watches consensus candidate updates.
/// </summary>
public Observable<CultNetSimulationConsensusCandidate> WatchCandidates()
{
ThrowIfDisposed();
return _candidateStream;
}
/// <summary>
/// Adds an observation and returns current candidates for the same observed subject.
/// </summary>
public IReadOnlyList<CultNetSimulationConsensusCandidate> Submit(CultNetSimulationObservation observation)
{
ThrowIfDisposed();
if (observation == null) throw new ArgumentNullException(nameof(observation));
_observations.Add(observation);
_observationStream.OnNext(observation);
var candidates = _consensus.BuildCandidates(_observations)
.Where(candidate => MatchesSubject(candidate, observation))
.ToArray();
foreach (var candidate in candidates)
{
_candidateStream.OnNext(candidate);
}
return candidates;
}
/// <summary>
/// Adds an observation message and returns current candidates for the same observed subject.
/// </summary>
public IReadOnlyList<CultNetSimulationConsensusCandidate> Submit(CultNetSimulationObservationMessage message)
{
if (message == null) throw new ArgumentNullException(nameof(message));
return Submit(message.Observation);
}
/// <inheritdoc />
public void Dispose()
{
if (_disposed)
{
return;
}
_disposed = true;
_observationStream.Dispose();
_candidateStream.Dispose();
}
private static bool MatchesSubject(
CultNetSimulationConsensusCandidate candidate,
CultNetSimulationObservation observation)
{
return string.Equals(candidate.ShardId, observation.ShardId, StringComparison.Ordinal) &&
candidate.ShardEpoch == observation.ShardEpoch &&
candidate.Frame == observation.Frame &&
string.Equals(candidate.SubjectId, observation.SubjectId, StringComparison.Ordinal) &&
string.Equals(candidate.ClaimKind, observation.ClaimKind, StringComparison.Ordinal);
}
private void ThrowIfDisposed()
{
if (_disposed)
{
throw new ObjectDisposedException(nameof(CultNetSimulationObservationHub));
}
}
}
}

79
src/GameCult.Networking/CultNetSimulationObservationServer.cs Normal file
View file

@ -0,0 +1,79 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using LiteNetLib;
namespace GameCult.Networking
{
/// <summary>
/// Bridges schema-v0 simulation observation messages into a local observation hub.
/// </summary>
public sealed class CultNetSimulationObservationServer : IDisposable
{
private readonly Server _server;
private readonly CultNetSimulationObservationHub _hub;
private readonly Func<CultNetSimulationObservationMessage, NetPeer, Task> _observationHandler;
private bool _disposed;
/// <summary>
/// Creates and attaches a simulation observation bridge.
/// </summary>
public CultNetSimulationObservationServer(
Server server,
CultNetSimulationObservationHub hub)
{
_server = server ?? throw new ArgumentNullException(nameof(server));
_hub = hub ?? throw new ArgumentNullException(nameof(hub));
_observationHandler = HandleObservationAsync;
_server.OnCultNet(_observationHandler);
}
/// <summary>
/// Gets the observation hub used by this bridge.
/// </summary>
public CultNetSimulationObservationHub Hub => _hub;
/// <summary>
/// Applies an observation message and returns current candidate messages for the same subject.
/// </summary>
public CultNetSimulationConsensusCandidateMessage[] CreateCandidateMessages(
CultNetSimulationObservationMessage message)
{
if (message == null) throw new ArgumentNullException(nameof(message));
return _hub.Submit(message)
.Select(candidate => CultNetSimulationConsensusCandidateMessage.FromCandidate(
message.MessageId,
candidate))
.ToArray();
}
/// <inheritdoc />
public void Dispose()
{
if (_disposed)
{
return;
}
_disposed = true;
_server.RemoveCultNetMessageListener<CultNetSimulationObservationMessage>(_observationHandler);
}
private Task HandleObservationAsync(CultNetSimulationObservationMessage message, NetPeer peer)
{
try
{
foreach (var candidate in CreateCandidateMessages(message))
{
peer.SendCultNet(candidate);
}
}
catch (Exception ex)
{
peer.SendCultNet(new CultNetErrorMessage { Error = ex.Message });
}
return Task.CompletedTask;
}
}
}

33
src/GameCult.Networking/PlayerData.cs
View file

@ -1,39 +1,46 @@
using System;
using GameCult.Caching;
using MessagePack;
namespace GameCult.Networking
{
/// <summary>
/// Player data stored in the database.
/// For MessagePack serialization, uses [MessagePackObject] and [Key(n)] attributes.
/// Basic player account document used by the sample networking stack.
/// </summary>
public class PlayerData : DatabaseEntry, INamedEntry
[CultDocument("gamecult.player_data", "gamecult.player_data.v2")]
public class PlayerData
{
/// <summary>
/// Unique player identifier.
/// </summary>
[Key(0)]
[CultIndex]
public Guid PlayerId = Guid.NewGuid();
/// <summary>
/// Email address of the player.
/// Player email address.
/// </summary>
[Key(1)]
[CultIndex]
public string Email = string.Empty;
/// <summary>
/// Password hash of the player.
/// Stored password hash.
/// </summary>
[Key(2)]
public string PasswordHash = string.Empty;
/// <summary>
/// Username of the player.
/// Player display name.
/// </summary>
[Key(3)]
[CultName]
public string Username = string.Empty;
/// <inheritdoc/>
[IgnoreMember]
public string EntryName
{
get => Username;
set => Username = value;
}
/// <summary>
/// Monotonic session version used to supersede older issued tokens.
/// </summary>
[Key(4)]
public long SessionVersion;
}
}

162
src/GameCult.Networking/README.md
View file

@ -23,6 +23,11 @@ The library currently includes:
The library is focused on the built-in authentication and session flows in this repository.
Current readiness target:
- trusted local mesh / self-hosted swarm use: yes
- hostile public internet edge: not yet
Keep the distinction clean:
- auth/session semantics belong to the core library
@ -52,6 +57,8 @@ Modern schema-v0 helpers live in:
- `CultNetSchemaMessageSerialization`
- `CultNetSchemaRegistry`
- `CultNetDocumentRegistry`
- `CultNetDatabase`
- `CultNetDatabaseServer`
- `NetPeerExtensions.SendCultNet(...)`
## Main Types
@ -84,7 +91,113 @@ Schema-v0 message families include:
The raw document/snapshot lane is meant for neighbors that already share the
same payload schema and MessagePack semantics. It carries exact payload bytes
plus explicit type metadata; it does not guess what a blob "probably" means.
plus `schemaId`/record-key metadata; it does not guess what a blob "probably"
means.
## CultMesh Lives In GameCult.Mesh
The distributed realtime database and simulation-consensus surface now has its
own package home: `GameCult.Mesh`, branded publicly as CultMesh. This networking
package still owns CultNet transport, authentication, and schema-v0 wire
contracts. CultMesh owns the developer-facing mesh entrypoints and higher-level
runtime documentation.
See `src/GameCult.Mesh/README.md` for the public-facing CultMesh API.
## Distributed CultCache
CultNet should be able to start as a sharding layer over CultCache. In that
mode, applications treat the cluster as one typed realtime database instead of
manually juggling a local cache, a network client, and a synchronization loop.
Target feel:
```csharp
var db = await CultNetDatabase.ConnectAsync("localhost", 3075);
var player = await db.GetAsync<PlayerData>(playerKey);
await db.PutAsync(playerKey, player);
using var subscription = db
.WatchByIndex<PlayerData>("Region", "eu-west")
.Subscribe(change => Render(change.Document));
```
CultCache remains the document model:
- schema identity and compatibility
- record keys and handles
- local indexes, globals, and typed lookups
- persistence and local domain-change diffing
CultNet becomes the distribution model:
- peer membership
- shard ownership
- mutation routing
- snapshot catch-up
- remote subscription fanout
- reconnect resynchronization
The current first slice exposes this as `CultNetDatabase` and wires hosted
servers through `CultNetDatabaseServer`. It handles primary-shard writes, typed
R3 watch streams, raw document puts/deletes, raw snapshot responses,
schema/key-filtered live subscriptions, shard catalog exchange, and stale-epoch
write rejection. Non-primary write forwarding is available behind an injectable
`ICultNetShardWriteForwarder`, with `CultNetSchemaWriteForwarder` as the
schema-v0 `cultnet://host:port` endpoint dialer. It also records accepted
mutations in per-shard ordered logs and exposes those logs over
`cultnet.shard_log_request.v0` / `cultnet.shard_log_response.v0` for replica
catch-up. Replicas apply those committed entries through
`CultNetDatabase.ApplyShardLogResponseAsync`, which checks shard epoch, rejects
sequence gaps, and treats replayed entries as already applied.
`CultNetShardReplicator` can then pull non-primary shard logs from advertised
primary endpoints, using `CultNetSchemaShardLogFetcher` for the schema-v0
transport path. Replica cursors can be stored through
`ICultNetShardReplicaCursorStore`; `CultNetFileShardReplicaCursorStore` keeps
those cursors in one local MessagePack file so replicas can resume after a
restart. Authoritative shard logs can be stored through
`ICultNetShardMutationLogStore`; `CultNetFileShardMutationLogStore` keeps one
MessagePack log file per shard and lets a restarted primary continue answering
`cultnet.shard_log_request.v0` from durable wire entries. The same store owns
log compaction: once entries are compacted through a sequence, older catch-up
requests return `ResyncRequired` with `reason = "compacted_history"` instead of
silently handing a replica a partial history. `CultNetShardReplicator` can use
an `ICultNetShardSnapshotFetcher` to recover from that response: it fetches a
shard-bounded snapshot, replaces the local shard view, advances the cursor to
the snapshot's represented log sequence, and resumes normal log pulls on later
ticks.
For client-side prediction, `CultNetClientAuthorityScope` declares which input
documents a runtime may author optimistically. `PutPredictedAsync` updates the
local cache immediately and emits a `Predicted` change; when the authoritative
shard log arrives for the same record, the database emits `Reconciled` and the
cache holds the committed value.
For mesh-side simulation agreement, `CultNetSimulationObservation` records what
a witness saw for one shard epoch, frame, subject, and claim kind.
`CultNetSimulationConsensus` deterministically aggregates those observations
into candidates with support weight, total observed weight, confidence, and
quorum status. Those candidates are opinions ready for an authoritative commit
path, not committed world state by themselves. The schema-v0 wire contracts are
`cultnet.simulation_observation.v0` and
`cultnet.simulation_consensus_candidate.v0`. `CultNetSimulationObservationHub`
collects incoming observations and emits candidate updates reactively.
`CultNetSimulationObservationServer` attaches that hub to a server so incoming
observation messages update the hub and receive current candidate messages in
reply.
The first coherent shard policy is primary ownership. Each shard has one
authoritative writer at a time. Clients may connect to any node; a non-owner
node forwards a write to the owner or rejects it honestly when forwarding is
not available. Followers subscribe to committed shard mutations and apply them
through the same CultCache reconciliation path as local file changes.
This should feel like the local-first parts of Firebase or RethinkDB without
copying their hidden machinery wholesale. The hard line is conflict honesty:
split-brain writes, stale shard epochs, and schema-incompatible documents must
surface as explicit failures instead of being laundered into cheerful-looking
state.
## Authentication Model
@ -97,6 +210,9 @@ The built-in flow is:
5. server validates the token and re-establishes the session
The session token is signed and validated by the server.
Each newly issued session token also carries a monotonic session version so
older signed tokens can be superseded cleanly instead of haunting reconnect
paths forever.
## Runtime Secrets
@ -174,6 +290,20 @@ Use `ServerSecurityOptions.Development()` only for local development and tests.
## Basic Server Usage
Quickest local-host path:
```csharp
using GameCult.Networking;
var host = await CultNetLocal.StartHostAsync("Players.msgpack");
host.Server.On<ChatMessage>(message =>
{
host.Server.Logger.LogInfo($"Chat: {message.Text}");
});
```
Lower-level explicit path:
```csharp
using GameCult.Caching;
using GameCult.Logging;
@ -185,7 +315,7 @@ var server = new Server(cache)
Logger = new ConsoleLogger()
};
server.AddMessageListener<ChatMessage>(message =>
server.On<ChatMessage>(message =>
{
server.Logger.LogInfo($"Chat: {message.Text}");
});
@ -195,6 +325,18 @@ server.Start();
## Basic Client Usage
Quickest local-client path:
```csharp
using GameCult.Networking;
var client = CultNetLocal.ConnectLocal();
client.On<ChatMessage>(message => Console.WriteLine(message.Text));
client.Login("user@example.com", "correct horse battery staple");
```
Lower-level explicit path:
```csharp
using GameCult.Logging;
using GameCult.Networking;
@ -205,7 +347,7 @@ var client = new Client(new ClientSecurityOptions("<matching-connection-key>"))
};
client.OnError += error => client.Logger.LogError($"Client error: {error}");
client.AddMessageListener<ChatMessage>(message => client.Logger.LogInfo($"Chat: {message.Text}"));
client.On<ChatMessage>(message => client.Logger.LogInfo($"Chat: {message.Text}"));
client.Connect("localhost", 3075);
client.Login("user@example.com", "correct horse battery staple");
@ -217,8 +359,9 @@ The client:
- disposes its prior polling subscription before reconnecting
- stops the prior `NetManager`
- schedules reconnect with a short delay
- schedules reconnect with bounded exponential backoff and jitter
- automatically re-verifies using the stored signed session token
- exposes reconnect state for UI/operator surfaces
## Important Constraints
@ -226,6 +369,10 @@ The client:
- Message authorization is based on the built-in verify/login/register flow.
- Transport is LiteNetLib, so this is not a drop-in substitute for HTTP or WebSocket stacks.
Sensitive payload logging is gated. Raw message JSON should only appear when a
caller explicitly opts into diagnostic logging instead of accidentally bleeding
live auth/session payloads into ordinary logs.
## Message Example
Messages derive from `Message` and are MessagePack-serializable:
@ -257,3 +404,10 @@ For the newer schema-v0 lane, the equivalent rule is:
- keep payload codecs in sync
- advertise supported contracts through schema discovery
- do not rely on implicit runtime magic to decide what a peer meant
## Future hostile-network work
The next hardening tranche is documented in
`production-hardening-checklist.md`. That ledger names the missing transport,
replay, abuse-handling, and adversarial-test organs directly instead of
pretending local-mesh readiness and internet readiness are the same rite.

50
src/GameCult.Networking/Secret.cs
View file

@ -193,14 +193,20 @@ namespace GameCult.Networking
/// <param name="expiresAtUtc">The UTC expiration time for the token.</param>
/// <returns>A tamper-evident session token.</returns>
public static string CreateSessionToken(Guid userId, DateTimeOffset expiresAtUtc) =>
CreateSessionToken(userId, expiresAtUtc, GetDefaultOptions());
CreateSessionToken(userId, expiresAtUtc, sessionVersion: 0, GetDefaultOptions());
/// <summary>
/// Creates a signed session token for the supplied user.
/// </summary>
public static string CreateSessionToken(Guid userId, DateTimeOffset expiresAtUtc, ServerSecurityOptions options)
public static string CreateSessionToken(Guid userId, DateTimeOffset expiresAtUtc, ServerSecurityOptions options) =>
CreateSessionToken(userId, expiresAtUtc, sessionVersion: 0, options);
/// <summary>
/// Creates a signed session token for the supplied user and session version.
/// </summary>
public static string CreateSessionToken(Guid userId, DateTimeOffset expiresAtUtc, long sessionVersion, ServerSecurityOptions options)
{
var payload = $"{userId:N}|{expiresAtUtc.ToUnixTimeSeconds().ToString(CultureInfo.InvariantCulture)}";
var payload = $"{userId:N}|{expiresAtUtc.ToUnixTimeSeconds().ToString(CultureInfo.InvariantCulture)}|{sessionVersion.ToString(CultureInfo.InvariantCulture)}";
var payloadBytes = Encoding.UTF8.GetBytes(payload);
var signatureBytes = ComputeHmacSha256(options.GetSessionSigningKey(), payloadBytes);
return $"{ToBase64Url(payloadBytes)}.{ToBase64Url(signatureBytes)}";
@ -214,7 +220,17 @@ namespace GameCult.Networking
/// <param name="expiresAtUtc">The extracted UTC expiration time.</param>
/// <returns><c>true</c> when the token is valid and unexpired.</returns>
public static bool TryValidateSessionToken(string? token, out Guid userId, out DateTimeOffset expiresAtUtc) =>
TryValidateSessionToken(token, GetDefaultOptions(), out userId, out expiresAtUtc);
TryValidateSessionToken(token, GetDefaultOptions(), out userId, out expiresAtUtc, out _);
/// <summary>
/// Validates and parses a signed session token.
/// </summary>
public static bool TryValidateSessionToken(
string? token,
out Guid userId,
out DateTimeOffset expiresAtUtc,
out long sessionVersion) =>
TryValidateSessionToken(token, GetDefaultOptions(), out userId, out expiresAtUtc, out sessionVersion);
/// <summary>
/// Validates and parses a signed session token.
@ -224,9 +240,23 @@ namespace GameCult.Networking
ServerSecurityOptions options,
out Guid userId,
out DateTimeOffset expiresAtUtc)
{
return TryValidateSessionToken(token, options, out userId, out expiresAtUtc, out _);
}
/// <summary>
/// Validates and parses a signed session token.
/// </summary>
public static bool TryValidateSessionToken(
string? token,
ServerSecurityOptions options,
out Guid userId,
out DateTimeOffset expiresAtUtc,
out long sessionVersion)
{
userId = Guid.Empty;
expiresAtUtc = DateTimeOffset.MinValue;
sessionVersion = 0;
if (string.IsNullOrWhiteSpace(token))
return false;
@ -245,7 +275,7 @@ namespace GameCult.Networking
var payload = Encoding.UTF8.GetString(payloadBytes);
var payloadParts = payload.Split('|');
if (payloadParts.Length != 2 ||
if ((payloadParts.Length != 2 && payloadParts.Length != 3) ||
!Guid.TryParseExact(payloadParts[0], "N", out userId) ||
!long.TryParse(payloadParts[1], NumberStyles.None, CultureInfo.InvariantCulture, out var expiryUnixSeconds))
{
@ -253,6 +283,15 @@ namespace GameCult.Networking
return false;
}
if (payloadParts.Length == 3 &&
!long.TryParse(payloadParts[2], NumberStyles.Integer, CultureInfo.InvariantCulture, out sessionVersion))
{
userId = Guid.Empty;
expiresAtUtc = DateTimeOffset.MinValue;
sessionVersion = 0;
return false;
}
expiresAtUtc = DateTimeOffset.FromUnixTimeSeconds(expiryUnixSeconds);
return expiresAtUtc > DateTimeOffset.UtcNow;
}
@ -260,6 +299,7 @@ namespace GameCult.Networking
{
userId = Guid.Empty;
expiresAtUtc = DateTimeOffset.MinValue;
sessionVersion = 0;
return false;
}
}

265
src/GameCult.Networking/Server.cs
View file

@ -25,12 +25,16 @@ namespace GameCult.Networking
private const int ServerPort = 3075;
private const float SessionTimeoutSeconds = 1800; // 30 minutes
private const float SessionRefreshThresholdSeconds = 300; // 5 minutes
private const int MaxConnectionAttemptsPerMinute = 30;
private const int MaxLoginAttemptsPerMinute = 5;
private readonly ConcurrentDictionary<Type, Delegate> _messageDelegates = new();
private readonly ConcurrentDictionary<Type, Delegate> _cultNetMessageDelegates = new();
private readonly ConcurrentDictionary<long, User> _users = new();
private readonly ConcurrentDictionary<string, Queue<DateTimeOffset>> _loginAttempts = new();
private readonly ConcurrentDictionary<string, object> _loginAttemptLocks = new();
private readonly ConcurrentDictionary<string, Queue<DateTimeOffset>> _connectionAttempts = new();
private readonly ConcurrentDictionary<string, object> _connectionAttemptLocks = new();
private readonly ConcurrentDictionary<string, Queue<DateTimeOffset>> _authAttempts = new();
private readonly ConcurrentDictionary<string, object> _authAttemptLocks = new();
private readonly IDisposable _cleanupSubscription;
private readonly CultCache _database;
private readonly ServerSecurityOptions _security;
@ -48,6 +52,11 @@ namespace GameCult.Networking
set => _logger = value ?? new NullLogger();
}
/// <summary>
/// Gets or sets whether raw payload bodies may be logged for diagnostics.
/// </summary>
public bool LogSensitivePayloads { get; set; }
private float Time => (float)(_timer?.Elapsed.TotalSeconds ?? 0d);
/// <summary>
@ -59,7 +68,7 @@ namespace GameCult.Networking
{
_database = cache;
_security = security ?? ServerSecurityOptions.FromEnvironment();
_database.RegisterIndex<PlayerData>("Email");
LogSensitivePayloads = _security.IsDevelopment;
_cleanupSubscription = Observable.Timer(TimeSpan.FromSeconds(60), TimeSpan.FromSeconds(60))
.Subscribe(_ => CleanupExpiredSessions());
}
@ -84,6 +93,7 @@ namespace GameCult.Networking
public void ClearMessageListeners()
{
_messageDelegates.Clear();
_cultNetMessageDelegates.Clear();
}
/// <summary>
@ -103,6 +113,57 @@ namespace GameCult.Networking
});
}
/// <summary>
/// Adds a listener for a specific authenticated message type.
/// </summary>
public void On<T>(Action<T> callback) where T : Message
{
AddMessageListener(callback);
}
/// <summary>
/// Adds a listener for a modern CultNet schema-v0 message type.
/// </summary>
public void AddCultNetMessageListener<T>(Func<T, NetPeer, Task> callback) where T : ICultNetSchemaMessage
{
var type = typeof(T);
_cultNetMessageDelegates.AddOrUpdate(type,
_ => callback,
(t, current) =>
{
var combined = Delegate.Combine(current, callback) as Func<T, NetPeer, Task>;
return combined ?? throw new InvalidOperationException($"Failed to combine delegates for {t.Name}");
});
}
/// <summary>
/// Adds a listener for a modern CultNet schema-v0 message type.
/// </summary>
public void AddCultNetMessageListener<T>(Action<T, NetPeer> callback) where T : ICultNetSchemaMessage
{
AddCultNetMessageListener<T>((message, peer) =>
{
callback(message, peer);
return Task.CompletedTask;
});
}
/// <summary>
/// Adds a listener for a modern CultNet schema-v0 message type.
/// </summary>
public void OnCultNet<T>(Func<T, NetPeer, Task> callback) where T : ICultNetSchemaMessage
{
AddCultNetMessageListener(callback);
}
/// <summary>
/// Adds a listener for a modern CultNet schema-v0 message type.
/// </summary>
public void OnCultNet<T>(Action<T, NetPeer> callback) where T : ICultNetSchemaMessage
{
AddCultNetMessageListener(callback);
}
/// <summary>
/// Removes a previously registered listener for a specific message type.
/// </summary>
@ -117,6 +178,33 @@ namespace GameCult.Networking
}
}
/// <summary>
/// Removes a previously registered listener for a specific authenticated message type.
/// </summary>
public void Off<T>(Action<T> callback) where T : Message
{
RemoveMessageListener(callback);
}
/// <summary>
/// Removes a previously registered modern CultNet schema-v0 listener.
/// </summary>
public void RemoveCultNetMessageListener<T>(Delegate callback) where T : ICultNetSchemaMessage
{
if (_cultNetMessageDelegates.TryGetValue(typeof(T), out var currentDelegate))
{
var newDelegate = Delegate.Remove(currentDelegate, callback);
if (newDelegate == null)
{
_cultNetMessageDelegates.TryRemove(typeof(T), out _);
}
else
{
_cultNetMessageDelegates[typeof(T)] = newDelegate;
}
}
}
/// <summary>
/// Stops the underlying LiteNetLib server.
/// </summary>
@ -151,7 +239,7 @@ namespace GameCult.Networking
listener.ConnectionRequestEvent += request =>
{
if (CheckRateLimit(request.RemoteEndPoint.Address.ToString()))
if (CheckConnectionRateLimit(request.RemoteEndPoint.Address.ToString()))
{
request.AcceptIfKey(_security.ConnectionKey);
}
@ -186,16 +274,29 @@ namespace GameCult.Networking
try
{
var bytes = reader.GetRemainingBytes();
var user = _users.GetOrAdd(peer.Id, _ => new User { Peer = peer });
if (TryDeserializeCultNet(bytes, out var cultNetMessage))
{
Logger.LogDebug($"Received CultNet schema message {cultNetMessage.SchemaVersion}");
await HandleCultNetSchemaMessageAsync(peer, user, cultNetMessage).ConfigureAwait(false);
return;
}
var message = MessageSerialization.Deserialize<Message>(bytes);
Logger.LogDebug($"Received message: {MessagePackSerializer.ConvertToJson(new ReadOnlyMemory<byte>(bytes))}");
if (LogSensitivePayloads)
{
Logger.LogDebug($"Received message: {MessagePackSerializer.ConvertToJson(new ReadOnlyMemory<byte>(bytes))}");
}
else
{
Logger.LogDebug($"Received message {message?.GetType().Name ?? "unknown"}");
}
if (message == null)
{
return;
}
message.Peer = peer;
var user = _users.GetOrAdd(peer.Id, _ => new User { Peer = peer });
if (message is LoginMessage or RegisterMessage or VerifyMessage)
{
if (IsVerified(user))
@ -204,7 +305,7 @@ namespace GameCult.Networking
return;
}
if (message is LoginMessage or RegisterMessage && !CheckRateLimit(peer.Address.ToString()))
if (message is LoginMessage or RegisterMessage && !CheckAuthRateLimit(peer.Address.ToString()))
{
peer.Send(new ErrorMessage { Error = "Too Many Attempts" });
return;
@ -256,7 +357,7 @@ namespace GameCult.Networking
return;
}
if (_database.GetIdByName<PlayerData>(message.Name) != null)
if (_database.GetByName<PlayerData>(message.Name) != null)
{
message.Peer?.Send(new ErrorMessage { Error = "Username Taken" });
return;
@ -279,6 +380,53 @@ namespace GameCult.Networking
Logger.LogInfo($"Server started on port {ServerPort}.");
}
private async Task HandleCultNetSchemaMessageAsync(NetPeer peer, User user, ICultNetSchemaMessage message)
{
if (!IsVerified(user) && !CanProcessBeforeVerification(message))
{
peer.SendCultNet(new CultNetErrorMessage { Error = "User Not Verified" });
return;
}
if (_cultNetMessageDelegates.TryGetValue(message.GetType(), out var del) && del != null)
{
foreach (var listener in del.GetInvocationList())
{
var result = listener.DynamicInvoke(message, peer);
if (result is Task task)
{
await task.ConfigureAwait(false);
}
}
user.SessionExpiresAt = DateTimeOffset.UtcNow.AddSeconds(SessionTimeoutSeconds);
RefreshSessionIfNeeded(peer, user);
}
else
{
Logger.LogWarning($"No listener for CultNet schema message {message.SchemaVersion}");
}
}
private static bool CanProcessBeforeVerification(ICultNetSchemaMessage message)
{
return message is CultNetHelloMessage or CultNetSchemaCatalogRequestMessage;
}
private static bool TryDeserializeCultNet(byte[] payload, out ICultNetSchemaMessage message)
{
try
{
message = CultNetSchemaMessageSerialization.Deserialize(payload);
return true;
}
catch (MessagePackSerializationException)
{
message = null!;
return false;
}
}
/// <inheritdoc />
public void Dispose()
{
@ -310,13 +458,13 @@ namespace GameCult.Networking
return;
}
if (_database.GetIdByIndex<PlayerData>("Email", email) != null)
if (_database.GetByIndex<PlayerData>("Email", email) != null)
{
peer.Send(new ErrorMessage { Error = "Email Taken" });
return;
}
if (_database.GetIdByName<PlayerData>(name) != null)
if (_database.GetByName<PlayerData>(name) != null)
{
peer.Send(new ErrorMessage { Error = "Username Taken" });
return;
@ -330,34 +478,41 @@ namespace GameCult.Networking
var newUserData = new PlayerData
{
ID = Guid.NewGuid(),
PlayerId = Guid.NewGuid(),
Email = email,
PasswordHash = Argon2.Hash(password, memoryCost: 16384),
Username = name
};
await _database.AddAsync(newUserData);
AttachUser(user, newUserData.ID);
SendSessionToken(peer, newUserData.ID);
AttachUser(user, newUserData.PlayerId);
SendSessionToken(peer, newUserData.PlayerId);
}
private void HandleVerify(NetPeer peer, User user, VerifyMessage verify)
{
var token = Secret.DecryptString(verify.Session, verify.Nonce, _security);
if (!Secret.TryValidateSessionToken(token, _security, out var playerId, out _))
if (!Secret.TryValidateSessionToken(token, _security, out var playerId, out _, out var sessionVersion))
{
peer.Send(new ErrorMessage { Error = "Session Invalid" });
return;
}
if (_database.Get<PlayerData>(playerId) == null)
var player = _database.GetByIndex<PlayerData>("PlayerId", playerId.ToString("D"));
if (player == null)
{
peer.Send(new ErrorMessage { Error = "Session Not Found" });
return;
}
if (player.SessionVersion != sessionVersion)
{
peer.Send(new ErrorMessage { Error = "Session Superseded" });
return;
}
AttachUser(user, playerId);
SendSessionToken(peer, playerId);
SendSessionToken(peer, player);
}
private void HandleLogin(NetPeer peer, User user, LoginMessage login)
@ -387,36 +542,27 @@ namespace GameCult.Networking
return;
}
AttachUser(user, userData.ID);
SendSessionToken(peer, userData.ID);
AttachUser(user, userData.PlayerId);
SendSessionToken(peer, userData);
}
private bool IsVerified(User? user) =>
user != null &&
user.PlayerId != Guid.Empty &&
user.SessionExpiresAt > DateTimeOffset.UtcNow &&
_database.Get<PlayerData>(user.PlayerId) != null;
_database.GetByIndex<PlayerData>("PlayerId", user.PlayerId.ToString("D")) != null;
private PlayerData? SessionData(User user) =>
IsVerified(user) ? _database.Get<PlayerData>(user.PlayerId) : null;
IsVerified(user) ? _database.GetByIndex<PlayerData>("PlayerId", user.PlayerId.ToString("D")) : null;
private bool CheckRateLimit(string ip)
internal bool CheckConnectionRateLimit(string ip)
{
var now = DateTimeOffset.UtcNow;
var windowStart = now.AddMinutes(-1);
var queue = _loginAttempts.GetOrAdd(ip, _ => new Queue<DateTimeOffset>());
var gate = _loginAttemptLocks.GetOrAdd(ip, _ => new object());
return CheckRateLimit(ip, MaxConnectionAttemptsPerMinute, _connectionAttempts, _connectionAttemptLocks);
}
lock (gate)
{
while (queue.Count > 0 && queue.Peek() < windowStart)
{
queue.Dequeue();
}
queue.Enqueue(now);
return queue.Count <= MaxLoginAttemptsPerMinute;
}
internal bool CheckAuthRateLimit(string ip)
{
return CheckRateLimit(ip, MaxLoginAttemptsPerMinute, _authAttempts, _authAttemptLocks);
}
private void CleanupExpiredSessions()
@ -440,14 +586,29 @@ namespace GameCult.Networking
}
private void SendSessionToken(NetPeer peer, Guid playerId)
{
var player = _database.GetByIndex<PlayerData>("PlayerId", playerId.ToString("D"));
if (player == null)
{
peer.Send(new ErrorMessage { Error = "Session Not Found" });
return;
}
SendSessionToken(peer, player);
}
private void SendSessionToken(NetPeer peer, PlayerData player)
{
var expiresAtUtc = DateTimeOffset.UtcNow.AddSeconds(SessionTimeoutSeconds);
var token = Secret.CreateSessionToken(playerId, expiresAtUtc, _security);
player.SessionVersion++;
var token = Secret.CreateSessionToken(player.PlayerId, expiresAtUtc, player.SessionVersion, _security);
_database.AddAsync(player).GetAwaiter().GetResult();
if (_users.TryGetValue(peer.Id, out var user))
{
user.PlayerId = playerId;
user.PlayerId = player.PlayerId;
user.SessionExpiresAt = expiresAtUtc;
user.SessionVersion = player.SessionVersion;
user.SessionToken = token;
}
@ -468,6 +629,29 @@ namespace GameCult.Networking
SendSessionToken(peer, user.PlayerId);
}
private static bool CheckRateLimit(
string ip,
int maxAttemptsPerMinute,
ConcurrentDictionary<string, Queue<DateTimeOffset>> attemptBuckets,
ConcurrentDictionary<string, object> attemptLocks)
{
var now = DateTimeOffset.UtcNow;
var windowStart = now.AddMinutes(-1);
var queue = attemptBuckets.GetOrAdd(ip, _ => new Queue<DateTimeOffset>());
var gate = attemptLocks.GetOrAdd(ip, _ => new object());
lock (gate)
{
while (queue.Count > 0 && queue.Peek() < windowStart)
{
queue.Dequeue();
}
queue.Enqueue(now);
return queue.Count <= maxAttemptsPerMinute;
}
}
}
/// <summary>
@ -515,5 +699,10 @@ namespace GameCult.Networking
/// The latest signed session token issued to the peer.
/// </summary>
public string SessionToken = string.Empty;
/// <summary>
/// The session version associated with the latest issued token.
/// </summary>
public long SessionVersion;
}
}

76
src/GameCult.Networking/production-hardening-checklist.md Normal file
View file

@ -0,0 +1,76 @@
# GameCult.Networking Production Hardening Checklist
This library is readying itself for trusted local mesh use: localhost, LAN, or
controlled self-hosted runtime clusters such as Epiphany and Aquarium. It is
not yet claiming sainthood on the public internet.
## Local-mesh baseline now covered
- separate connection-attempt and auth-attempt throttles in `Server`
- versioned session tokens that supersede older issued tokens
- bounded reconnect backoff with jitter in `Client`
- raw payload logging gated behind explicit diagnostic opt-in
- schema-first raw document/snapshot lane with exact `schemaId` routing
## Missing hostile-network organs
### Transport posture
- LiteNetLib remains the transport substrate.
- There is no transport-level TLS story in this library today.
- Before internet exposure, decide whether LiteNetLib stays and gains a real
outer protection model, or whether another transport becomes the canonical
public edge.
### Credential and replay posture
- login/register payloads are AES-GCM encrypted with per-message nonces
- session tokens are signed and versioned
- replay resistance is still scoped to token expiry + supersession semantics,
not a broader anti-replay protocol
Before hostile exposure:
- define replay expectations explicitly
- add adversarial tests around token reuse, stale reconnects, and reordered
message delivery
### Session rotation and revocation
- superseded tokens are rejected once a newer session version is issued
- there is still no broader account-session management surface for:
- explicit logout-all
- administrative revocation
- multi-device policy
### Input abuse and malformed payload handling
- malformed MessagePack is rejected by the serializer path
- there is no explicit message-size policy or backpressure contract yet
Before hostile exposure:
- add payload-size ceilings
- add stress tests for oversized and malformed inputs
- decide what gets dropped, logged, or disconnected
### Replication abuse cases
- raw snapshot/document replication assumes peers already share payload schema
and trust boundaries
- there is not yet an adversarial policy for untrusted replication partners
Before hostile exposure:
- define peer authorization for raw snapshot/document lanes
- test unknown schema spam, snapshot flooding, and incompatible payload storms
### Missing tests
Still wanted before claiming internet-grade readiness:
- live client/server auth integration tests
- reconnect storm tests
- session supersession across multiple clients
- message-size and malformed-frame abuse tests
- load and soak tests for rate limiting and session refresh

16
tests/GameCult.Caching.InteropPeer/GameCult.Caching.InteropPeer.csproj Normal file
View file

@ -0,0 +1,16 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net10.0</TargetFramework>
<Nullable>enable</Nullable>
<RootNamespace>GameCult.Caching.InteropPeer</RootNamespace>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\src\GameCult.Caching.MessagePack\GameCult.Caching.MessagePack.csproj" />
<ProjectReference Include="..\..\src\GameCult.Caching.MessagePack.Generator\GameCult.Caching.MessagePack.Generator.csproj"
OutputItemType="Analyzer"
ReferenceOutputAssembly="false" />
<PackageReference Include="MessagePack" Version="3.1.4" />
</ItemGroup>
</Project>

134
tests/GameCult.Caching.InteropPeer/Program.cs Normal file
View file

@ -0,0 +1,134 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text.Json;
using System.Threading.Tasks;
using GameCult.Caching;
using GameCult.Caching.MessagePack;
using MessagePack;
return await ProgramMainAsync(args);
static async Task<int> ProgramMainAsync(string[] args)
{
try
{
if (args.Length == 0)
{
throw new InvalidOperationException("Expected mode: write | read");
}
var mode = args[0];
var options = ParseArgs(args.Skip(1).ToArray());
var file = RequireArg(options, "file");
switch (mode)
{
case "write":
await WriteAsync(file, RequireArg(options, "runtime-id"));
return 0;
case "read":
await ReadAsync(file);
return 0;
default:
throw new InvalidOperationException($"Unknown mode {mode}.");
}
}
catch (Exception error)
{
Console.Error.WriteLine(error);
return 1;
}
}
static async Task WriteAsync(string file, string runtimeId)
{
var cache = BuildCache(file);
await cache.PullAllBackingStoresAsync();
var note = new CultCacheInteropNote
{
DocumentId = $"note:{runtimeId}",
AuthorRuntimeId = runtimeId,
Title = $"{runtimeId} wrote a CultCache note",
Body = "The v1 store format is the contract.",
Tags = [runtimeId, "csharp", "interop"]
};
await cache.AddAsync(note, new CultRecordHandle<CultCacheInteropNote>(new CultRecordKey(note.DocumentId)));
cache.FlushAllBackingStores();
WriteJsonLine(note);
}
static async Task ReadAsync(string file)
{
var cache = BuildCache(file);
await cache.PullAllBackingStoresAsync();
var note = cache.AllEntries
.OfType<CultCacheInteropNote>()
.FirstOrDefault()
?? throw new InvalidOperationException("No cultcache.interop-note records found.");
WriteJsonLine(note);
}
static CultCache BuildCache(string file)
{
var cache = new CultCache();
cache.AddBackingStore(new SingleFileMessagePackBackingStore(file));
return cache;
}
static Dictionary<string, string> ParseArgs(string[] args)
{
var parsed = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
for (var index = 0; index < args.Length; index += 2)
{
var token = args[index];
if (!token.StartsWith("--", StringComparison.Ordinal))
{
continue;
}
if (index + 1 >= args.Length)
{
throw new InvalidOperationException($"Missing value for {token}.");
}
parsed[token[2..]] = args[index + 1];
}
return parsed;
}
static string RequireArg(Dictionary<string, string> options, string name)
{
if (!options.TryGetValue(name, out var value) || string.IsNullOrWhiteSpace(value))
{
throw new InvalidOperationException($"Missing required argument --{name}.");
}
return value;
}
static void WriteJsonLine(CultCacheInteropNote note)
{
Console.Out.WriteLine(JsonSerializer.Serialize(new
{
schemaVersion = note.SchemaVersion,
documentId = note.DocumentId,
authorRuntimeId = note.AuthorRuntimeId,
title = note.Title,
body = note.Body,
tags = note.Tags
}));
}
[CultDocument("cultcache.interop-note", "cultcache.interop_note.v1")]
[MessagePackObject]
public sealed class CultCacheInteropNote
{
[Key(0)] public string SchemaVersion { get; set; } = "cultcache.interop_note.v1";
[Key(1)] [CultName] public string DocumentId { get; set; } = string.Empty;
[Key(2)] public string AuthorRuntimeId { get; set; } = string.Empty;
[Key(3)] public string Title { get; set; } = string.Empty;
[Key(4)] public string Body { get; set; } = string.Empty;
[Key(5)] public string[] Tags { get; set; } = Array.Empty<string>();
}

534
tests/GameCult.Caching.Tests/BackingStoreTests.cs
View file

@ -1,13 +1,11 @@
#nullable enable
using System;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
using GameCult.Caching;
using GameCult.Caching.MessagePack;
using GameCult.Caching.NewtonsoftJson;
using Newtonsoft.Json;
using MessagePack;
using NUnit.Framework;
namespace GameCult.Caching.Tests
@ -15,120 +13,36 @@ namespace GameCult.Caching.Tests
public class BackingStoreTests
{
[Test]
public async Task MultiFileBackingStore_Rename_RemovesStaleFile()
public async Task SingleFileMessagePackBackingStore_RoundTrips_DiscoveredEntryType()
{
var rootPath = Path.Combine(Path.GetTempPath(), $"cultlib-tests-{Guid.NewGuid():N}");
Directory.CreateDirectory(rootPath);
var filePath = Path.Combine(Path.GetTempPath(), $"cultlib-tests-{Guid.NewGuid():N}.msgpack");
try
{
var store = new MultiFileNewtonsoftJsonBackingStore(rootPath);
store.RegisterType<NamedTestEntry>();
var cache = new CultCache();
cache.AddBackingStore(store);
var entry = new NamedTestEntry
{
ID = Guid.NewGuid(),
Name = "OriginalName",
Value = "value"
};
await cache.AddAsync(entry);
entry.Name = "RenamedUser";
await cache.AddAsync(entry);
var files = Directory.GetFiles(rootPath, "*.json", SearchOption.AllDirectories)
.Select(Path.GetFileName)
.ToArray();
Assert.That(files, Does.Contain("RenamedUser.json"));
Assert.That(files, Does.Not.Contain("OriginalName.json"));
}
finally
{
if (Directory.Exists(rootPath))
{
Directory.Delete(rootPath, true);
}
}
}
[Test]
public async Task MultiFileNewtonsoftJsonBackingStore_RoundTrips_DiscoveredEntryType()
{
var rootPath = Path.Combine(Path.GetTempPath(), $"cultlib-tests-{Guid.NewGuid():N}");
Directory.CreateDirectory(rootPath);
try
{
var writeStore = new MultiFileNewtonsoftJsonBackingStore(rootPath);
writeStore.RegisterType<NamedTestEntry>();
var writeStore = new SingleFileMessagePackBackingStore(filePath);
var writeCache = new CultCache();
writeCache.AddBackingStore(writeStore);
var entry = new NamedTestEntry
{
ID = Guid.NewGuid(),
Name = "RoundTrip",
Name = "SingleFileMsgpack",
Value = "payload"
};
await writeCache.AddAsync(entry);
var readStore = new MultiFileNewtonsoftJsonBackingStore(rootPath);
readStore.RegisterType<NamedTestEntry>();
var readCache = new CultCache();
readCache.AddBackingStore(readStore);
await readCache.PullAllBackingStoresAsync();
var loaded = readCache.Get(entry.ID);
Assert.That(loaded, Is.Not.Null);
Assert.That(GetEntryValue(loaded!), Is.EqualTo("payload"));
}
finally
{
if (Directory.Exists(rootPath))
{
Directory.Delete(rootPath, true);
}
}
}
[Test]
public async Task SingleFileNewtonsoftJsonBackingStore_RoundTrips_DiscoveredEntryType()
{
var filePath = Path.Combine(Path.GetTempPath(), $"cultlib-tests-{Guid.NewGuid():N}.json");
try
{
var writeStore = new SingleFileNewtonsoftJsonBackingStore(filePath);
writeStore.RegisterType<NamedTestEntry>();
var writeCache = new CultCache();
writeCache.AddBackingStore(writeStore);
var entry = new NamedTestEntry
{
ID = Guid.NewGuid(),
Name = "SingleFile",
Value = "payload"
};
await writeCache.AddAsync(entry);
var handle = await writeCache.AddAsync(entry);
writeStore.PushAll();
var readStore = new SingleFileNewtonsoftJsonBackingStore(filePath);
readStore.RegisterType<NamedTestEntry>();
var readStore = new SingleFileMessagePackBackingStore(filePath);
var readCache = new CultCache();
readCache.AddBackingStore(readStore);
await readCache.PullAllBackingStoresAsync();
var loaded = readCache.Get(entry.ID);
var loaded = readCache.Get<NamedTestEntry>(handle.Key);
Assert.That(loaded, Is.Not.Null);
Assert.That(GetEntryValue(loaded!), Is.EqualTo("payload"));
Assert.That(loaded!.Value, Is.EqualTo("payload"));
Assert.That(readStore.LastSchemaMigrationReports, Has.Count.EqualTo(1));
Assert.That(readStore.LastSchemaMigrationReports[0].Kind, Is.EqualTo(CultSchemaMigrationKind.Exact));
}
finally
{
@ -140,123 +54,367 @@ namespace GameCult.Caching.Tests
}
[Test]
public void MultiFileNewtonsoftJsonBackingStore_Rejects_UnknownDiscriminator()
public async Task CultCache_DirtyState_Tracks_Mutations_And_ExplicitFlush()
{
var rootPath = Path.Combine(Path.GetTempPath(), $"cultlib-tests-{Guid.NewGuid():N}");
Directory.CreateDirectory(rootPath);
var filePath = Path.Combine(Path.GetTempPath(), $"cultlib-tests-{Guid.NewGuid():N}.msgpack");
try
{
var store = new MultiFileNewtonsoftJsonBackingStore(rootPath);
store.RegisterType<NamedTestEntry>();
var payload = """
{
"Type": "Unknown.Assembly:Unknown.Type",
"Data": {
"ID": "00000000-0000-0000-0000-000000000001",
"Name": "Bad",
"Value": "Bad"
}
}
""";
Assert.That(
() => store.Deserialize(Encoding.UTF8.GetBytes(payload)),
Throws.TypeOf<JsonSerializationException>().With.Message.Contains("Unknown DatabaseEntry discriminator"));
}
finally
{
if (Directory.Exists(rootPath))
{
Directory.Delete(rootPath, true);
}
}
}
[Test]
public async Task MultiFileNewtonsoftJsonBackingStore_Delete_Removes_File()
{
var rootPath = Path.Combine(Path.GetTempPath(), $"cultlib-tests-{Guid.NewGuid():N}");
Directory.CreateDirectory(rootPath);
try
{
var store = new MultiFileNewtonsoftJsonBackingStore(rootPath);
store.RegisterType<NamedTestEntry>();
var store = new SingleFileMessagePackBackingStore(filePath);
var cache = new CultCache();
cache.AddBackingStore(store);
var entry = new NamedTestEntry
Assert.That(cache.IsDirty, Is.False);
Assert.That(store.IsDirty, Is.False);
await cache.AddAsync(new NamedTestEntry
{
ID = Guid.NewGuid(),
Name = "DeleteMe",
Value = "payload"
};
Name = "dirty",
Value = "pending"
});
await cache.AddAsync(entry);
cache.Remove(entry);
Assert.That(cache.IsDirty, Is.True);
Assert.That(store.IsDirty, Is.True);
Assert.That(cache.LastSuccessfulFlushAtUtc, Is.Null);
var files = Directory.GetFiles(rootPath, "*.json", SearchOption.AllDirectories)
.Select(Path.GetFileName)
.ToArray();
cache.FlushAllBackingStores();
Assert.That(files, Does.Not.Contain("DeleteMe.json"));
Assert.That(cache.IsDirty, Is.False);
Assert.That(store.IsDirty, Is.False);
Assert.That(cache.LastSuccessfulFlushAtUtc, Is.Not.Null);
Assert.That(store.LastSuccessfulFlushAtUtc, Is.Not.Null);
}
finally
{
if (Directory.Exists(rootPath))
if (File.Exists(filePath))
{
Directory.Delete(rootPath, true);
File.Delete(filePath);
}
}
}
[Test]
public void MessagePackDatabaseEntrySerialization_RoundTrips_DatabaseLink()
public async Task CultCacheMessagePack_OpenAsync_Creates_Usable_Durable_Cache()
{
var serializationType = typeof(MultiFileMessagePackBackingStore).Assembly
.GetType("GameCult.Caching.MessagePack.DatabaseEntrySerialization", throwOnError: true)!;
var serialize = serializationType.GetMethod("Serialize", BindingFlags.Public | BindingFlags.Static)!
.MakeGenericMethod(typeof(DatabaseLink<DatabaseEntry>));
var deserialize = serializationType.GetMethod("Deserialize", BindingFlags.Public | BindingFlags.Static)!
.MakeGenericMethod(typeof(DatabaseLink<DatabaseEntry>));
var link = new DatabaseLink<DatabaseEntry> { LinkID = Guid.NewGuid() };
var filePath = Path.Combine(Path.GetTempPath(), $"cultlib-tests-{Guid.NewGuid():N}.msgpack");
var payload = (byte[])serialize.Invoke(null, [link])!;
var roundTrip = (DatabaseLink<DatabaseEntry>)deserialize.Invoke(null, [payload])!;
Assert.That(roundTrip.LinkID, Is.EqualTo(link.LinkID));
}
[Test]
public void MessagePackDatabaseEntrySerialization_Rejects_InvalidPayload()
{
var serializationType = typeof(MultiFileMessagePackBackingStore).Assembly
.GetType("GameCult.Caching.MessagePack.DatabaseEntrySerialization", throwOnError: true)!;
var deserialize = serializationType.GetMethod("Deserialize", BindingFlags.Public | BindingFlags.Static)!
.MakeGenericMethod(typeof(DatabaseEntry));
Assert.That(
() => deserialize.Invoke(null, [new byte[] { 0xC1 }]),
Throws.TypeOf<TargetInvocationException>()
.With.InnerException.InstanceOf<global::MessagePack.MessagePackSerializationException>());
}
private sealed class NamedTestEntry : DatabaseEntry, INamedEntry
{
public string Name = string.Empty;
public string Value = string.Empty;
public string EntryName
try
{
get => Name;
set => Name = value;
var cache = await CultCacheMessagePack.OpenAsync(filePath);
var handle = await cache.UpsertAsync(new NamedTestEntry
{
Name = "open",
Value = "magic"
});
await cache.FlushAsync();
cache.Dispose();
var reopened = await CultCacheMessagePack.OpenAsync(filePath);
Assert.That(reopened.TryGet(handle.Key, out NamedTestEntry? loaded), Is.True);
Assert.That(loaded!.Value, Is.EqualTo("magic"));
}
finally
{
if (File.Exists(filePath))
{
File.Delete(filePath);
}
}
}
private static string GetEntryValue(DatabaseEntry entry)
[Test]
public async Task CultCache_FlushOnDispose_Persists_When_Enabled()
{
return entry.GetType().GetField("Value")?.GetValue(entry) as string ?? string.Empty;
var filePath = Path.Combine(Path.GetTempPath(), $"cultlib-tests-{Guid.NewGuid():N}.msgpack");
try
{
var store = new SingleFileMessagePackBackingStore(filePath);
var cache = new CultCache
{
FlushAttachedStoresOnDispose = true
};
cache.AddBackingStore(store);
var handle = await cache.AddAsync(new NamedTestEntry
{
Name = "dispose",
Value = "flush"
});
cache.Dispose();
var readStore = new SingleFileMessagePackBackingStore(filePath);
var readCache = new CultCache();
readCache.AddBackingStore(readStore);
await readCache.PullAllBackingStoresAsync();
var loaded = readCache.Get<NamedTestEntry>(handle.Key);
Assert.That(loaded, Is.Not.Null);
Assert.That(loaded!.Value, Is.EqualTo("flush"));
}
finally
{
if (File.Exists(filePath))
{
File.Delete(filePath);
}
}
}
[Test]
public void MessagePackSerialization_RoundTrips_CultRecordRef()
{
var reference = new CultRecordRef<NamedTestEntry>(new CultRecordKey("record-1"));
var payload = CultDocumentMessagePackSerialization.Serialize(reference);
var roundTrip = CultDocumentMessagePackSerialization.Deserialize<CultRecordRef<NamedTestEntry>>(payload);
Assert.That(roundTrip.Key.Value, Is.EqualTo("record-1"));
}
[Test]
public void MessagePackSerialization_Rejects_InvalidPayload()
{
Assert.That(
() => CultDocumentMessagePackSerialization.Deserialize<CultRecordRef<NamedTestEntry>>(new byte[] { 0xC1 }),
Throws.TypeOf<MessagePackSerializationException>());
}
[Test]
public void GeneratedMetadataProvider_Emits_AttributedDocuments_And_References()
{
var providers = typeof(NamedTestEntry).Assembly
.GetCustomAttributes(typeof(CultGeneratedDocumentMetadataProviderAttribute), false)
.Cast<CultGeneratedDocumentMetadataProviderAttribute>()
.ToArray();
Assert.That(providers, Is.Not.Empty);
var definitions = providers
.SelectMany(provider =>
((ICultGeneratedDocumentMetadataProvider)Activator.CreateInstance(provider.ProviderType)!)
.GetDocumentDefinitions())
.ToArray();
var named = definitions.Single(definition => definition.DocumentType == typeof(NamedTestEntry));
Assert.That(named.SchemaName, Is.EqualTo("tests.named_entry"));
Assert.That(named.NameMember, Is.EqualTo(nameof(NamedTestEntry.Name)));
var referenceHolder = definitions.Single(definition => definition.DocumentType == typeof(ReferenceHolderEntry));
var parentMember = referenceHolder.Members.Single(member => member.MemberName == nameof(ReferenceHolderEntry.Parent));
Assert.That(parentMember.IsReference, Is.True);
Assert.That(parentMember.TargetSchemaName, Is.EqualTo("tests.named_entry"));
Assert.That(parentMember.TypeName, Does.Contain("CultRecordRef"));
}
[Test]
public void GeneratedMetadataProvider_Emits_Payload_Codecs_For_Plain_CultDocuments()
{
var descriptor = CultDocumentRegistry.Shared.GetRequired<NamedTestEntry>();
var original = new NamedTestEntry
{
Name = "Teeth",
Value = "slot-array"
};
Assert.That(descriptor.GeneratedPayloadSerializer, Is.Not.Null);
Assert.That(descriptor.GeneratedPayloadDeserializer, Is.Not.Null);
var payload = descriptor.GeneratedPayloadSerializer!(original);
var roundTrip = (NamedTestEntry)descriptor.GeneratedPayloadDeserializer!(payload);
Assert.That(roundTrip.Name, Is.EqualTo("Teeth"));
Assert.That(roundTrip.Value, Is.EqualTo("slot-array"));
}
[Test]
public void MessagePackStoreSerialization_RoundTrips_Snapshot_Record_And_Catalog()
{
var record = new CultPersistedRecord
{
Key = "record-1",
SchemaId = "schema-1",
StoredAt = "2026-05-08T12:00:00Z",
Payload = new byte[] { 0x91, 0xA3, 0x66, 0x6F, 0x6F }
};
var catalog = new[]
{
new CultSchemaCatalogEntry
{
SchemaId = "schema-1",
SchemaName = "tests.named_entry",
SchemaVersion = "tests.named_entry.v1",
ContentHash = "hash-1",
CanonicalSchemaJson = "{\"fields\":2}",
CompatibleSchemaIds = new[] { "schema-1", "schema-0" }
}
};
var snapshot = new CultPersistedStoreSnapshot
{
FormatVersion = "cultcache.store.v1",
SchemaCatalog = catalog,
Records = new[] { record }
};
var roundTripRecord = CultDocumentMessagePackSerialization.DeserializePersistedRecord(
CultDocumentMessagePackSerialization.SerializePersistedRecord(record));
var roundTripCatalog = CultDocumentMessagePackSerialization.DeserializeSchemaCatalog(
CultDocumentMessagePackSerialization.SerializeSchemaCatalog(catalog));
var roundTripSnapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(
CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot));
Assert.That(roundTripRecord.Key, Is.EqualTo("record-1"));
Assert.That(roundTripRecord.SchemaId, Is.EqualTo("schema-1"));
Assert.That(roundTripRecord.StoredAt, Is.EqualTo("2026-05-08T12:00:00Z"));
Assert.That(roundTripRecord.Payload, Is.EqualTo(record.Payload));
Assert.That(roundTripCatalog.Single().CompatibleSchemaIds, Is.EqualTo(catalog.Single().CompatibleSchemaIds));
Assert.That(roundTripSnapshot.FormatVersion, Is.EqualTo("cultcache.store.v1"));
Assert.That(roundTripSnapshot.SchemaCatalog.Single().SchemaName, Is.EqualTo("tests.named_entry"));
Assert.That(roundTripSnapshot.Records.Single().Key, Is.EqualTo("record-1"));
}
[Test]
public void Registry_CanonicalSchemaJson_Tracks_Reference_Metadata()
{
var descriptor = CultDocumentRegistry.Shared.GetRequired<ReferenceHolderEntry>();
Assert.That(descriptor.CanonicalSchemaJson, Does.Contain("\"targetSchemaName\":\"tests.named_entry\""));
Assert.That(descriptor.CanonicalSchemaJson, Does.Contain("\"isReference\":true"));
}
[Test]
public void Registry_CanonicalSchema_Fixtures_Are_Stable()
{
var named = CultDocumentRegistry.Shared.GetRequired<NamedTestEntry>();
var referenceHolder = CultDocumentRegistry.Shared.GetRequired<ReferenceHolderEntry>();
Assert.That(named.CanonicalSchemaJson, Is.EqualTo(NamedFixtureCanonicalSchemaJson));
Assert.That(named.SchemaId, Is.EqualTo(NamedFixtureSchemaId));
Assert.That(named.ContentHash, Is.EqualTo(NamedFixtureContentHash));
Assert.That(referenceHolder.SchemaId, Is.EqualTo(ReferenceFixtureSchemaId));
}
[Test]
public void ResolvePersistedSchemaReport_Classifies_Compatible_And_Incompatible_Drift()
{
var registry = CultDocumentRegistry.Shared;
var namedV1 = registry.GetRequired<NamedTestEntry>();
var namedV2 = registry.GetRequired<NamedTestEntryAdditive>();
var namedV3 = registry.GetRequired<NamedTestEntryRemoved>();
var namedMismatch = registry.GetRequired<NamedTestEntryTypeMismatch>();
var referenceV1 = registry.GetRequired<ReferenceHolderEntry>();
var referenceRetargeted = registry.GetRequired<ReferenceHolderRetargetedEntry>();
var additiveCatalog = namedV1.ToCatalogEntry();
additiveCatalog.SchemaId = "persisted.tests.named_entry.v1";
additiveCatalog.CompatibleSchemaIds = new[] { namedV2.SchemaId };
var additiveReport = registry.ResolvePersistedSchemaReport(additiveCatalog.SchemaId, new[] { additiveCatalog });
Assert.That(additiveReport.Kind, Is.EqualTo(CultSchemaMigrationKind.CompatibleDrift));
Assert.That(additiveReport.DefaultedMissingSlots, Is.EqualTo(new[] { 2 }));
Assert.That(additiveReport.IgnoredExtraSlots, Is.Empty);
var removedCatalog = namedV2.ToCatalogEntry();
removedCatalog.SchemaId = "persisted.tests.named_entry.v2";
removedCatalog.CompatibleSchemaIds = new[] { namedV3.SchemaId };
var removedReport = registry.ResolvePersistedSchemaReport(removedCatalog.SchemaId, new[] { removedCatalog });
Assert.That(removedReport.Kind, Is.EqualTo(CultSchemaMigrationKind.CompatibleDrift));
Assert.That(removedReport.DefaultedMissingSlots, Is.Empty);
Assert.That(removedReport.IgnoredExtraSlots, Is.EqualTo(new[] { 1, 2 }));
var mismatchCatalog = namedV1.ToCatalogEntry();
mismatchCatalog.SchemaId = "persisted.tests.named_entry.type_mismatch";
mismatchCatalog.CompatibleSchemaIds = new[] { namedMismatch.SchemaId };
Assert.That(
() => registry.ResolvePersistedSchemaReport(mismatchCatalog.SchemaId, new[] { mismatchCatalog }),
Throws.TypeOf<InvalidOperationException>().With.Message.Contains("changed type"));
var retargetedCatalog = referenceV1.ToCatalogEntry();
retargetedCatalog.SchemaId = "persisted.tests.reference_holder.v1";
retargetedCatalog.CompatibleSchemaIds = new[] { referenceRetargeted.SchemaId };
Assert.That(
() => registry.ResolvePersistedSchemaReport(retargetedCatalog.SchemaId, new[] { retargetedCatalog }),
Throws.TypeOf<InvalidOperationException>().With.Message.Contains("changed target schema"));
}
private const string NamedFixtureCanonicalSchemaJson =
"{\"schemaName\":\"tests.named_entry\",\"schemaVersion\":\"tests.named_entry.v1\",\"members\":[{\"slot\":0,\"name\":\"Name\",\"type\":\"System.String\",\"isReference\":false,\"many\":false,\"targetSchemaName\":null,\"indexAlias\":null,\"isName\":true},{\"slot\":1,\"name\":\"Value\",\"type\":\"System.String\",\"isReference\":false,\"many\":false,\"targetSchemaName\":null,\"indexAlias\":null,\"isName\":false}]}";
private const string NamedFixtureSchemaId = "sha256:e7b97801b94190f3159012ede45b0069bb09ebf7920f7432c971bc86a0e08de8";
private const string NamedFixtureContentHash = "sha256:23150930afcc1d84f0cb3012ccc2debcb9b4685f62083033bbaab0083f1e832e";
private const string ReferenceFixtureSchemaId = "sha256:bd85064961cc74565fb73e3ccbc4217cfba4dc4869e365a08bea4f704739bd8f";
[CultDocument("tests.named_entry", "tests.named_entry.v1")]
internal sealed class NamedTestEntry
{
[Key(0)]
[CultName]
public string Name = string.Empty;
[Key(1)]
public string Value = string.Empty;
}
[CultDocument("tests.reference_holder", "tests.reference_holder.v1")]
internal sealed class ReferenceHolderEntry
{
[Key(0)]
[CultName]
public string Name = string.Empty;
[Key(1)]
public CultRecordRef<NamedTestEntry> Parent = new(new CultRecordKey("parent"));
}
[CultDocument("tests.named_entry", "tests.named_entry.v2")]
internal sealed class NamedTestEntryAdditive
{
[Key(0)]
[CultName]
public string Name = string.Empty;
[Key(1)]
public string Value = string.Empty;
[Key(2)]
public string Notes = string.Empty;
}
[CultDocument("tests.named_entry", "tests.named_entry.v3")]
internal sealed class NamedTestEntryRemoved
{
[Key(0)]
[CultName]
public string Name = string.Empty;
}
[CultDocument("tests.named_entry", "tests.named_entry.v4")]
internal sealed class NamedTestEntryTypeMismatch
{
[Key(0)]
[CultName]
public string Name = string.Empty;
[Key(1)]
public int Value;
}
[CultDocument("tests.alt_named_entry", "tests.alt_named_entry.v1")]
internal sealed class AlternateNamedTestEntry
{
[Key(0)]
[CultName]
public string Name = string.Empty;
}
[CultDocument("tests.reference_holder", "tests.reference_holder.v2")]
internal sealed class ReferenceHolderRetargetedEntry
{
[Key(0)]
[CultName]
public string Name = string.Empty;
[Key(1)]
public CultRecordRef<AlternateNamedTestEntry> Parent = new(new CultRecordKey("parent"));
}
}
}

4
tests/GameCult.Caching.Tests/GameCult.Caching.Tests.csproj
View file

@ -9,7 +9,9 @@
<ItemGroup>
<ProjectReference Include="..\..\src\GameCult.Caching\GameCult.Caching.csproj" />
<ProjectReference Include="..\..\src\GameCult.Caching.MessagePack\GameCult.Caching.MessagePack.csproj" />
<ProjectReference Include="..\..\src\GameCult.Caching.NewtonsoftJson\GameCult.Caching.NewtonsoftJson.csproj" />
<ProjectReference Include="..\..\src\GameCult.Caching.MessagePack.Generator\GameCult.Caching.MessagePack.Generator.csproj"
OutputItemType="Analyzer"
ReferenceOutputAssembly="false" />
</ItemGroup>
<ItemGroup>

7
tests/GameCult.Caching.Tests/README.md
View file

@ -1,12 +1,15 @@
# GameCult.Caching.Tests
`GameCult.Caching.Tests` contains NUnit tests for the caching library and its persistence abstractions.
`GameCult.Caching.Tests` contains NUnit tests for the caching library and its
single-file MessagePack persistence path.
## Scope
The tests currently cover:
- rename behavior in `MultiFileNewtonsoftJsonBackingStore`
- `SingleFileMessagePackBackingStore` round-trips
- explicit Cult document payload codecs
- hand-written MessagePack store snapshot/record/catalog serialization
## Run

3
tests/GameCult.Networking.InteropPeer/GameCult.Networking.InteropPeer.csproj
View file

@ -8,6 +8,9 @@
<ItemGroup>
<ProjectReference Include="..\..\src\GameCult.Networking\GameCult.Networking.csproj" />
<ProjectReference Include="..\..\src\GameCult.Caching.MessagePack.Generator\GameCult.Caching.MessagePack.Generator.csproj"
OutputItemType="Analyzer"
ReferenceOutputAssembly="false" />
<PackageReference Include="MessagePack" Version="3.1.4" />
</ItemGroup>
</Project>

283
tests/GameCult.Networking.InteropPeer/Program.cs
View file

@ -55,16 +55,16 @@ static async Task ServeAsync(ServeConfig config)
{
var schemaRegistration = LoadSchemaRegistration(config.SchemaPath);
var documentRegistry = new CultNetDocumentRegistry()
.Register(CultNetDocumentBinding.ForEntry<CultNetInteropNote>(
InteropPeerShared.InteropDocumentType,
payloadSchemaVersion: InteropPeerShared.InteropSchemaVersion,
documentKeySelector: note => note.DocumentId,
.Register(CultNetDocumentBinding.ForDocument<CultNetInteropNote>(
schemaId: schemaRegistration.SchemaId,
payloadSerializer: SerializeInteropNotePayload,
payloadDeserializer: DeserializeInteropNotePayload));
RegisterCapabilityBindings(documentRegistry);
var customSchemaRegistry = new CultNetSchemaRegistry().Register(schemaRegistration);
var cache = new CultCache();
await cache.AddAsync(BuildInteropNote(config.RuntimeId, config.DisplayName));
var localNote = BuildInteropNote(config.RuntimeId, config.DisplayName);
await cache.AddAsync(localNote, new CultRecordHandle<CultNetInteropNote>(new CultRecordKey(localNote.DocumentId)));
using var udpSocket = CreateDiscoverySocket(config.DiscoveryPort, config.DiscoveryGroup);
using var tcpListener = new TcpListener(IPAddress.Parse(config.BindHost), config.TcpPort);
@ -173,12 +173,11 @@ static async Task DialAsync(DialConfig config)
var schemaRegistration = LoadSchemaRegistration(config.SchemaPath);
var cache = new CultCache();
var documentRegistry = new CultNetDocumentRegistry()
.Register(CultNetDocumentBinding.ForEntry<CultNetInteropNote>(
InteropPeerShared.InteropDocumentType,
payloadSchemaVersion: InteropPeerShared.InteropSchemaVersion,
documentKeySelector: note => note.DocumentId,
.Register(CultNetDocumentBinding.ForDocument<CultNetInteropNote>(
schemaId: schemaRegistration.SchemaId,
payloadSerializer: SerializeInteropNotePayload,
payloadDeserializer: DeserializeInteropNotePayload));
RegisterCapabilityBindings(documentRegistry);
using var client = new TcpClient();
await client.ConnectAsync(config.TargetHost, config.TargetPort);
@ -192,6 +191,7 @@ static async Task DialAsync(DialConfig config)
Role = "peer",
DisplayName = config.DisplayName,
SupportedDocumentTypes = [InteropPeerShared.InteropDocumentType],
SupportedMutationContracts = [InteropPeerShared.InteractionContract()],
SupportedMessageVersions = [InteropPeerShared.InteropSchemaVersion],
SupportsSchemaCatalog = true
});
@ -209,8 +209,8 @@ static async Task DialAsync(DialConfig config)
await SendMessageAsync(stream, new CultNetSnapshotRequestMessage
{
MessageId = $"{config.RuntimeId}-snapshot",
DocumentTypes = [InteropPeerShared.InteropDocumentType],
DocumentKeys = [$"note:{remoteHello.RuntimeId}"]
SchemaIds = [schemaRegistration.SchemaId],
RecordKeys = [$"note:{remoteHello.RuntimeId}"]
});
var snapshot = await ExpectMessageAsync<CultNetSnapshotResponseRawMessage>(stream, CultNetSchemaVersions.SnapshotResponseRaw);
await documentRegistry.ApplyRawSnapshotResponseAsync(cache, snapshot);
@ -223,6 +223,9 @@ static async Task DialAsync(DialConfig config)
string.Equals(schema.SchemaId, schemaRegistration.SchemaId, StringComparison.Ordinal) &&
string.Equals(schema.DocumentType, InteropPeerShared.InteropDocumentType, StringComparison.Ordinal));
var mutation = await MutateRemoteNoteAsync(stream, cache, documentRegistry, config.RuntimeId, note);
var fireReceipt = await FireRemoteWeaponAsync(stream, cache, documentRegistry, config.RuntimeId, remoteHello.RuntimeId);
WriteJsonLine(new
{
mode = "dial",
@ -249,10 +252,79 @@ static async Task DialAsync(DialConfig config)
title = note.Title,
body = note.Body,
tags = note.Tags
}
},
mutatedNote = mutation,
fireReceipt
});
}
static async Task<object> MutateRemoteNoteAsync(
NetworkStream stream,
CultCache cache,
CultNetDocumentRegistry documentRegistry,
string runtimeId,
CultNetInteropNote note)
{
var intent = new CultNetInteropMutationIntent
{
IntentId = $"{runtimeId}-decorate",
TargetDocumentId = note.DocumentId,
AppendBody = $" Decorated by {runtimeId}.",
AppendTag = $"decorated:{runtimeId}"
};
await SendMessageAsync(stream, documentRegistry.CreateRawDocumentPutMessage(
$"{runtimeId}-decorate-put",
new CultRecordHandle<CultNetInteropMutationIntent>(new CultRecordKey(intent.IntentId)),
intent));
await documentRegistry.ApplyRawDocumentPutMessageAsync<CultNetInteropMutationReceipt>(
cache,
await ExpectMessageAsync<CultNetDocumentPutRawMessage>(stream, CultNetSchemaVersions.DocumentPutRaw));
var mutated = await documentRegistry.ApplyRawDocumentPutMessageAsync<CultNetInteropNote>(
cache,
await ExpectMessageAsync<CultNetDocumentPutRawMessage>(stream, CultNetSchemaVersions.DocumentPutRaw));
return new
{
schemaVersion = mutated.SchemaVersion,
documentId = mutated.DocumentId,
authorRuntimeId = mutated.AuthorRuntimeId,
title = mutated.Title,
body = mutated.Body,
tags = mutated.Tags
};
}
static async Task<object> FireRemoteWeaponAsync(
NetworkStream stream,
CultCache cache,
CultNetDocumentRegistry documentRegistry,
string runtimeId,
string remoteRuntimeId)
{
var command = new CultNetInteropFireCommand
{
CommandId = $"{runtimeId}-fire",
CharacterId = remoteRuntimeId,
WeaponId = "interop-rifle"
};
await SendMessageAsync(stream, documentRegistry.CreateRawDocumentPutMessage(
$"{runtimeId}-fire-put",
new CultRecordHandle<CultNetInteropFireCommand>(new CultRecordKey(command.CommandId)),
command));
var receipt = await documentRegistry.ApplyRawDocumentPutMessageAsync<CultNetInteropFireReceipt>(
cache,
await ExpectMessageAsync<CultNetDocumentPutRawMessage>(stream, CultNetSchemaVersions.DocumentPutRaw));
return new
{
schemaVersion = receipt.SchemaVersion,
commandId = receipt.CommandId,
accepted = receipt.Accepted,
characterId = receipt.CharacterId,
weaponId = receipt.WeaponId,
shotsFired = receipt.ShotsFired,
ammoRemaining = receipt.AmmoRemaining
};
}
static async Task RunDiscoveryServerAsync(Socket socket, ServeConfig config, CancellationToken cancellationToken)
{
var buffer = new byte[4096];
@ -359,6 +431,7 @@ static async Task RunTcpServerAsync(
Role = "peer",
DisplayName = config.DisplayName,
SupportedDocumentTypes = [InteropPeerShared.InteropDocumentType],
SupportedMutationContracts = [InteropPeerShared.InteractionContract()],
SupportedMessageVersions = [InteropPeerShared.InteropSchemaVersion],
SupportsSchemaCatalog = true
}, cancellationToken);
@ -382,6 +455,9 @@ static async Task RunTcpServerAsync(
}),
cancellationToken);
break;
case CultNetDocumentPutRawMessage rawPut:
await HandleRawPutAsync(stream, config, cache, documentRegistry, rawPut, cancellationToken);
break;
}
}
}
@ -393,6 +469,68 @@ static async Task RunTcpServerAsync(
}
}
static async Task HandleRawPutAsync(
NetworkStream stream,
ServeConfig config,
CultCache cache,
CultNetDocumentRegistry documentRegistry,
CultNetDocumentPutRawMessage rawPut,
CancellationToken cancellationToken)
{
switch (rawPut.Document.SchemaId)
{
case InteropPeerShared.MutationIntentSchemaId:
{
var intent = await documentRegistry.ApplyRawDocumentPutMessageAsync<CultNetInteropMutationIntent>(cache, rawPut);
var note = cache.AllEntries
.OfType<CultNetInteropNote>()
.First(candidate => string.Equals(candidate.DocumentId, intent.TargetDocumentId, StringComparison.Ordinal));
note.Body += intent.AppendBody;
note.Tags = [..note.Tags, intent.AppendTag];
await cache.AddAsync(note, new CultRecordHandle<CultNetInteropNote>(new CultRecordKey(note.DocumentId)));
var receipt = new CultNetInteropMutationReceipt
{
IntentId = intent.IntentId,
Accepted = true,
DocumentId = note.DocumentId,
Body = note.Body,
Tags = note.Tags
};
var options = ResponseOptions(config, "mutation");
await SendMessageAsync(stream, documentRegistry.CreateRawDocumentPutMessage(
$"{config.RuntimeId}-mutation-receipt",
new CultRecordHandle<CultNetInteropMutationReceipt>(new CultRecordKey(receipt.IntentId)),
receipt,
options), cancellationToken);
await SendMessageAsync(stream, documentRegistry.CreateRawDocumentPutMessage(
$"{config.RuntimeId}-mutated-note",
new CultRecordHandle<CultNetInteropNote>(new CultRecordKey(note.DocumentId)),
note,
options), cancellationToken);
break;
}
case InteropPeerShared.FireCommandSchemaId:
{
var command = await documentRegistry.ApplyRawDocumentPutMessageAsync<CultNetInteropFireCommand>(cache, rawPut);
var receipt = new CultNetInteropFireReceipt
{
CommandId = command.CommandId,
Accepted = true,
CharacterId = command.CharacterId,
WeaponId = command.WeaponId,
ShotsFired = 1,
AmmoRemaining = 29
};
await SendMessageAsync(stream, documentRegistry.CreateRawDocumentPutMessage(
$"{config.RuntimeId}-fire-receipt",
new CultRecordHandle<CultNetInteropFireReceipt>(new CultRecordKey(receipt.CommandId)),
receipt,
ResponseOptions(config, "side-effect")), cancellationToken);
break;
}
}
}
static CultNetSchemaCatalogResponseMessage CreateCatalogResponse(
CultNetSchemaRegistry customSchemaRegistry,
CultNetSchemaCatalogRequestMessage request)
@ -504,7 +642,6 @@ static CultNetInteropNote BuildInteropNote(string runtimeId, string displayName)
var documentId = $"note:{runtimeId}";
return new CultNetInteropNote
{
ID = CreateDeterministicGuid(documentId),
SchemaVersion = InteropPeerShared.InteropSchemaVersion,
DocumentId = documentId,
AuthorRuntimeId = runtimeId,
@ -532,7 +669,6 @@ static CultNetInteropNote DeserializeInteropNotePayload(byte[] payload)
var decoded = MessagePackSerializer.Deserialize<CultNetInteropNotePayload>(payload, CultNetSchemaMessageSerialization.Options);
return new CultNetInteropNote
{
ID = CreateDeterministicGuid(decoded.DocumentId),
SchemaVersion = decoded.SchemaVersion,
DocumentId = decoded.DocumentId,
AuthorRuntimeId = decoded.AuthorRuntimeId,
@ -542,10 +678,36 @@ static CultNetInteropNote DeserializeInteropNotePayload(byte[] payload)
};
}
static Guid CreateDeterministicGuid(string value)
static void RegisterCapabilityBindings(CultNetDocumentRegistry registry)
{
var bytes = System.Security.Cryptography.MD5.HashData(System.Text.Encoding.UTF8.GetBytes(value));
return new Guid(bytes);
registry
.Register(CultNetDocumentBinding.ForDocument<CultNetInteropMutationIntent>(
schemaId: InteropPeerShared.MutationIntentSchemaId,
payloadSerializer: value => MessagePackSerializer.Serialize(value, CultNetSchemaMessageSerialization.Options),
payloadDeserializer: payload => MessagePackSerializer.Deserialize<CultNetInteropMutationIntent>(payload, CultNetSchemaMessageSerialization.Options)))
.Register(CultNetDocumentBinding.ForDocument<CultNetInteropMutationReceipt>(
schemaId: InteropPeerShared.MutationReceiptSchemaId,
payloadSerializer: value => MessagePackSerializer.Serialize(value, CultNetSchemaMessageSerialization.Options),
payloadDeserializer: payload => MessagePackSerializer.Deserialize<CultNetInteropMutationReceipt>(payload, CultNetSchemaMessageSerialization.Options)))
.Register(CultNetDocumentBinding.ForDocument<CultNetInteropFireCommand>(
schemaId: InteropPeerShared.FireCommandSchemaId,
payloadSerializer: value => MessagePackSerializer.Serialize(value, CultNetSchemaMessageSerialization.Options),
payloadDeserializer: payload => MessagePackSerializer.Deserialize<CultNetInteropFireCommand>(payload, CultNetSchemaMessageSerialization.Options)))
.Register(CultNetDocumentBinding.ForDocument<CultNetInteropFireReceipt>(
schemaId: InteropPeerShared.FireReceiptSchemaId,
payloadSerializer: value => MessagePackSerializer.Serialize(value, CultNetSchemaMessageSerialization.Options),
payloadDeserializer: payload => MessagePackSerializer.Deserialize<CultNetInteropFireReceipt>(payload, CultNetSchemaMessageSerialization.Options)));
}
static CultNetDocumentMessageOptions ResponseOptions(ServeConfig config, string tag)
{
return new CultNetDocumentMessageOptions
{
SourceRuntimeId = config.RuntimeId,
SourceAgentId = config.AgentId,
SourceRole = "peer",
Tags = [tag, config.RuntimeId]
};
}
static async Task WaitForeverAsync(CancellationToken cancellationToken)
@ -669,6 +831,18 @@ static class InteropPeerShared
{
public const string InteropDocumentType = "cultnet.interop-note";
public const string InteropSchemaVersion = "cultnet.interop_note.v0";
public const string MutationIntentDocumentType = "cultnet.interop-note-mutation-intent";
public const string MutationIntentSchemaId = "https://github.com/GameCult/cultnet-ts/integration/contracts/cultnet.interop-note-mutation-intent.schema.json";
public const string MutationIntentSchemaVersion = "cultnet.interop_note_mutation_intent.v0";
public const string MutationReceiptDocumentType = "cultnet.interop-note-mutation-receipt";
public const string MutationReceiptSchemaId = "https://github.com/GameCult/cultnet-ts/integration/contracts/cultnet.interop-note-mutation-receipt.schema.json";
public const string MutationReceiptSchemaVersion = "cultnet.interop_note_mutation_receipt.v0";
public const string FireCommandDocumentType = "cultnet.interop-fire-weapon-command";
public const string FireCommandSchemaId = "https://github.com/GameCult/cultnet-ts/integration/contracts/cultnet.interop-fire-weapon-command.schema.json";
public const string FireCommandSchemaVersion = "cultnet.interop_fire_weapon_command.v0";
public const string FireReceiptDocumentType = "cultnet.interop-fire-weapon-receipt";
public const string FireReceiptSchemaId = "https://github.com/GameCult/cultnet-ts/integration/contracts/cultnet.interop-fire-weapon-receipt.schema.json";
public const string FireReceiptSchemaVersion = "cultnet.interop_fire_weapon_receipt.v0";
public const string DiscoveryProbeSchemaVersion = "cultnet.discovery_probe.v0";
public const string DiscoveryAnnounceSchemaVersion = "cultnet.discovery_announce.v0";
@ -676,16 +850,31 @@ static class InteropPeerShared
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
};
public static CultNetDocumentMutationContract InteractionContract()
{
return new CultNetDocumentMutationContract
{
DocumentType = InteropDocumentType,
PayloadSchemaVersion = InteropSchemaVersion,
Operations = ["snapshot", "documentPut", "intentSubmit", "receiptWatch"],
Authority = CultNetMutationAuthorities.Runtime,
IntentDocumentTypes = [MutationIntentDocumentType, FireCommandDocumentType],
ReceiptDocumentTypes = [MutationReceiptDocumentType, FireReceiptDocumentType],
Notes = []
};
}
}
public sealed class CultNetInteropNote : DatabaseEntry
[CultDocument(InteropPeerShared.InteropDocumentType, InteropPeerShared.InteropSchemaVersion)]
public sealed class CultNetInteropNote
{
public string SchemaVersion { get; set; } = InteropPeerShared.InteropSchemaVersion;
public string DocumentId { get; set; } = string.Empty;
public string AuthorRuntimeId { get; set; } = string.Empty;
public string Title { get; set; } = string.Empty;
public string Body { get; set; } = string.Empty;
public string[] Tags { get; set; } = Array.Empty<string>();
[Key(0)] public string SchemaVersion { get; set; } = InteropPeerShared.InteropSchemaVersion;
[Key(1)] [CultName] public string DocumentId { get; set; } = string.Empty;
[Key(2)] public string AuthorRuntimeId { get; set; } = string.Empty;
[Key(3)] public string Title { get; set; } = string.Empty;
[Key(4)] public string Body { get; set; } = string.Empty;
[Key(5)] public string[] Tags { get; set; } = Array.Empty<string>();
}
[MessagePackObject]
@ -699,6 +888,52 @@ public sealed class CultNetInteropNotePayload
[Key(5)] public string[] Tags { get; set; } = Array.Empty<string>();
}
[CultDocument(InteropPeerShared.MutationIntentDocumentType, InteropPeerShared.MutationIntentSchemaVersion)]
[MessagePackObject]
public sealed class CultNetInteropMutationIntent
{
[Key(0)] public string SchemaVersion { get; set; } = InteropPeerShared.MutationIntentSchemaVersion;
[Key(1)] [CultName] public string IntentId { get; set; } = string.Empty;
[Key(2)] public string TargetDocumentId { get; set; } = string.Empty;
[Key(3)] public string AppendBody { get; set; } = string.Empty;
[Key(4)] public string AppendTag { get; set; } = string.Empty;
}
[CultDocument(InteropPeerShared.MutationReceiptDocumentType, InteropPeerShared.MutationReceiptSchemaVersion)]
[MessagePackObject]
public sealed class CultNetInteropMutationReceipt
{
[Key(0)] public string SchemaVersion { get; set; } = InteropPeerShared.MutationReceiptSchemaVersion;
[Key(1)] [CultName] public string IntentId { get; set; } = string.Empty;
[Key(2)] public bool Accepted { get; set; }
[Key(3)] public string DocumentId { get; set; } = string.Empty;
[Key(4)] public string Body { get; set; } = string.Empty;
[Key(5)] public string[] Tags { get; set; } = Array.Empty<string>();
}
[CultDocument(InteropPeerShared.FireCommandDocumentType, InteropPeerShared.FireCommandSchemaVersion)]
[MessagePackObject]
public sealed class CultNetInteropFireCommand
{
[Key(0)] public string SchemaVersion { get; set; } = InteropPeerShared.FireCommandSchemaVersion;
[Key(1)] [CultName] public string CommandId { get; set; } = string.Empty;
[Key(2)] public string CharacterId { get; set; } = string.Empty;
[Key(3)] public string WeaponId { get; set; } = string.Empty;
}
[CultDocument(InteropPeerShared.FireReceiptDocumentType, InteropPeerShared.FireReceiptSchemaVersion)]
[MessagePackObject]
public sealed class CultNetInteropFireReceipt
{
[Key(0)] public string SchemaVersion { get; set; } = InteropPeerShared.FireReceiptSchemaVersion;
[Key(1)] [CultName] public string CommandId { get; set; } = string.Empty;
[Key(2)] public bool Accepted { get; set; }
[Key(3)] public string CharacterId { get; set; } = string.Empty;
[Key(4)] public string WeaponId { get; set; } = string.Empty;
[Key(5)] public int ShotsFired { get; set; }
[Key(6)] public int AmmoRemaining { get; set; }
}
[MessagePackObject]
public sealed class DiscoveryProbeMessage
{

1
tests/GameCult.Networking.Tests/GameCult.Networking.Tests.csproj
View file

@ -7,6 +7,7 @@
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\src\GameCult.Mesh\GameCult.Mesh.csproj" />
<ProjectReference Include="..\..\src\GameCult.Networking\GameCult.Networking.csproj" />
</ItemGroup>

2222
tests/GameCult.Networking.Tests/NetworkingTests.cs
View file

File diff suppressed because it is too large Load diff