From 340c9507e42a72883ed961582c72362271d6bb6f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 28 Oct 2026 02:36:17 +0100 Subject: [PATCH 01/88] Write ownership by the written type: one per-key predicate, the registry leaves upserts An upsert may replace the record at its key only when the type it writes there owns the record (CultDocumentRegistry.WrittenOwns): the entry declares the stored id, or reads the stored entry whole under its own schema name. Cache and cacheless writers, single-file and directory stores ask the same per-key judge (CacheBackingStore.UndeclaredAt); a cache no longer replaces another registered type's record. Registry.Owns is the removal/check rule only. Deleted: CatalogEntryLookup's registry half, CultDocumentRegistry.Lookup, Owns's mayClaim, the per-batch union and the per-id memo. Co-Authored-By: Claude Sonnet 5.5 --- .../DirectoryMessagePackBackingStore.cs | 24 +- .../Contracts/cultcache-persistence-format.md | 20 +- src/GameCult.Caching/CultCache.cs | 227 ++++++++++-------- .../RestampOnWriteTests.cs | 51 ++-- unity/org.gamecult.cultlib/CHANGELOG.md | 2 +- 5 files changed, 169 insertions(+), 155 deletions(-) diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index 63180a1b..9d668909 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -185,7 +185,11 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore manifest.Records.ToDictionary(record => record.Key, record => VersionOf(record.Payload), StringComparer.Ordinal), Entries.Values, ForeignRecords)) return CultCommitOutcome.Mismatch; - RefuseUndeclaredOnDisk(manifest, request.Upserts.Concat(request.Deletes).Select(entry => entry.Key.Value).Concat(request.Checks).Concat(StagedKeys)); + RefuseUndeclaredOnDisk( + manifest, + request.Upserts.Concat(request.Deletes).Select(entry => entry.Key.Value).Concat(request.Checks).Concat(StagedKeys), + request.Upserts, + request.Deletes); if (request.Upserts.Count == 0 && request.Deletes.Count == 0 && !IsDirty) { // Nothing to write: the batch only judged keys it does not hold. @@ -243,7 +247,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore Directory.CreateDirectory(_manifestFile.DirectoryName!); using var commitLease = AcquireCommitLease(wait: true, create: true); var (manifest, catalogBytes) = ReadManifest(); - RefuseUndeclaredOnDisk(manifest, StagedKeys); + RefuseUndeclaredOnDisk(manifest, StagedKeys, Array.Empty(), Array.Empty()); if (IsDirty) WriteGeneration(manifest, catalogBytes); else @@ -275,7 +279,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore using var commitLease = AcquireCommitLease(wait, create: true); if (commitLease == null) return CultCommitOutcome.Contended; - RefuseUndeclaredOnDisk(ReadManifest().Snapshot, keys); + RefuseUndeclaredOnDisk(ReadManifest().Snapshot, keys, Array.Empty(), Array.Empty()); return CultCommitOutcome.Committed; }); } @@ -286,14 +290,22 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore // The manifest's other records stay as they are, foreign and read-only ones included; a write that would replace or remove a // record the manifest holds that the writer does not own is refused before any page is written, by the one rule every - // store shares (CacheBackingStore.RefuseUndeclaredOnDisk). The refused records reload as a pull loads them. - private void RefuseUndeclaredOnDisk(CultPersistedStoreSnapshot currentManifest, IEnumerable written) + // store shares (CacheBackingStore.UndeclaredAt, CacheBackingStore.RefuseUndeclaredOnDisk). The refused records reload as a + // pull loads them. A key is owned by the entry the write puts there: the batch's upsert, or the staged document. A key it + // removes (the batch's deletes, the staged deletes) or only checks goes to the registry's removal rule. + private void RefuseUndeclaredOnDisk(CultPersistedStoreSnapshot currentManifest, IEnumerable written, IReadOnlyList upserts, IReadOnlyList deletes) { CultPersistedRecord? HeldAt(string key) => currentManifest.Records.FirstOrDefault(record => string.Equals(record.Key, key, StringComparison.Ordinal)); + var batch = upserts.ToDictionary(entry => entry.Key.Value, entry => entry.Descriptor, StringComparer.Ordinal); + var removed = deletes.Select(entry => entry.Key.Value).ToHashSet(StringComparer.Ordinal); + CultSchemaCatalogEntry? WrittenAt(string key) => + batch.TryGetValue(key, out var descriptor) ? descriptor.ToCatalogEntry() + : !removed.Contains(key) && _dirtyKeys.ContainsKey(key) && Entries.TryGetValue(key, out var staged) ? staged.Descriptor.ToCatalogEntry() + : null; RefuseUndeclaredOnDisk( written, - key => HeldAt(key) is { } held ? Undeclared(held, currentManifest.SchemaCatalog, Registry.Owns) : null, + UndeclaredAt(HeldAt, currentManifest.SchemaCatalog, WrittenAt, Registry.Owns), key => HeldAt(key) is { } held && Foreign(held, currentManifest.SchemaCatalog, VersionOf(held.Payload)) == null ? LoadPage(held, currentManifest.SchemaCatalog, out _, out _) : null, diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index 10fb9fc8..3a6391d8 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -206,16 +206,16 @@ The rule is checked again where the write lands, under the store's lock: a write whose durable record is not owned, because another writer rewrote it after this cache loaded, is refused typed, and the record is reloaded as the store holds it, read-only when a type can read it. This holds for every write through the store, the store's own `Push`, `Delete` and -`CommitBatch` included. A CultMesh single-file write (`WriteDocument`, `WriteRecord`, or `PushAll` on an unattached store) asks the same rule: it owns the ids its -written entries declare (their own ids and the ids they list compatible) and an id whose publishing entry its claimant reads whole (no persisted -slot unread). The claimant is picked by the one precedence over the written entries and the call's registry, a written entry being the only -description of its schema id (the registry's type for that id is never asked), and the record is owned only when the claimant is a written -entry. A registry adds claimants and declares nothing, so it only adds refusals; with none the written entries are the only claimants. Refused: -a record another registered type claims, under that type's current id or an old one it lists; a record whose stored entry lists another -registered type's id ahead of the written one's, whatever name the stored entry carries; a record two registered types list compatible or two -registered versions of the written schema could read; a record the picked written entry reads only by dropping persisted slots; and the -removal of any stored record by a call that writes no entry. A registry-less `WriteDocument` reads `CultDocumentRegistry.Shared`. It is -refused the same way otherwise. +`CommitBatch` included. Whether a write may replace the record at a key is decided per key by the type it writes there, for a cache +and for a CultMesh single-file write (`WriteDocument`, `WriteRecord`, or `PushAll` on an unattached store) alike, in a single-file store and +a directory store alike: the written type owns the stored record when its entry declares the stored id (its own id, or one it lists +compatible), or when it is the type that reads the stored record's published entry and reads it whole (no persisted slot unread), so an older +id of its own schema name that it reads whole is its own and is restamped. Anything else is refused, another registered type's record +included, and so is a record the written type would read only by dropping persisted slots. No registry enters the decision, `Shared` included: +the answer for a given written entry, stored id and catalog is the same lent a registry or not, and a cache's registered types do not make +each other's records writable. Retyping a key is a removal and an add, in separate commits. A removal or check of a key goes to the +registry's removal rule (a record a registered type declares, or reads whole, may be removed), and a cacheless call, which has no registry, +removes a record only when an entry it writes owns it; a call that writes no entry removes nothing. A record that resolves to no local type is a foreign record, for example one of a type this build does not have, or one renamed under a new schema name without declaring its old id. A store never destroys or relabels it: the store carries it diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index ce276a0c..6ce2e9dd 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -524,6 +524,12 @@ namespace GameCult.Caching // asked. internal bool StoredSchemaReadWhole { get; set; } + // The catalog entry that publishes StoredSchemaId in the store this record was loaded from, with the id it was published for, + // so staging can ask WrittenOwns the question the store's judge asks, over the catalog the record came with. Null for a record + // this cache constructed or wrote, which is stored under Descriptor's own id and published by Descriptor's entry. A reader + // uses it only while the id still matches StoredSchemaId. + internal (string Id, CultSchemaCatalogEntry Entry)? StoredPublishedBy { get; set; } + // Names the bytes this record was read from or written as: the SHA-256 of its persisted encoding, set by the store that // read or wrote them. A record no store has read or written has a version of its own that equals nothing else. internal string StoredVersion { get; set; } @@ -593,45 +599,28 @@ namespace GameCult.Caching public string TypeLabel { get; } } - // The lookup of a cacheless writer: the catalog entries the call writes, and the lent registry's other types. A written entry is - // the only description of its schema id, so the registry's type for that id is never asked: a registry adds claimants but never - // re-describes the written type. + // The lookup a write's ownership asks: only the catalog entries the call writes. No registry enters it, so the answer is the + // written type's own (variants:ruling:write-ownership-by-written-type). internal sealed class CatalogEntryLookup : IClaimantLookup { private readonly IReadOnlyCollection _entries; - private readonly IClaimantLookup? _others; - public CatalogEntryLookup(IReadOnlyCollection entries, CultDocumentRegistry? others) + public CatalogEntryLookup(IReadOnlyCollection entries) { _entries = entries; - _others = others?.Lookup; } public CultSchemaCatalogEntry? Owner(string schemaId) => - _entries.FirstOrDefault(entry => string.Equals(entry.SchemaId, schemaId, StringComparison.Ordinal)) - ?? Other(_others?.Owner(schemaId)); + _entries.FirstOrDefault(entry => string.Equals(entry.SchemaId, schemaId, StringComparison.Ordinal)); public IReadOnlyList Listing(string schemaId) => - _entries.Where(entry => entry.CompatibleSchemaIds.Contains(schemaId, StringComparer.Ordinal)) - .Concat(Others(_others?.Listing(schemaId))) - .ToArray(); + _entries.Where(entry => entry.CompatibleSchemaIds.Contains(schemaId, StringComparer.Ordinal)).ToArray(); public IReadOnlyList Named(string schemaName) => - _entries.Where(entry => string.Equals(entry.SchemaName, schemaName, StringComparison.Ordinal)) - .Concat(Others(_others?.Named(schemaName))) - .ToArray(); + _entries.Where(entry => string.Equals(entry.SchemaName, schemaName, StringComparison.Ordinal)).ToArray(); public ClaimantFacts Describe(CultSchemaCatalogEntry claimant) => new(claimant.SchemaName, claimant.SchemaVersion, claimant.SchemaId); - - // A registry type counts only when no written entry has its schema id. - private CultSchemaCatalogEntry? Other(CultDocumentDescriptor? descriptor) => - descriptor == null || _entries.Any(entry => string.Equals(entry.SchemaId, descriptor.SchemaId, StringComparison.Ordinal)) - ? null - : descriptor.ToCatalogEntry(); - - private IEnumerable Others(IEnumerable? descriptors) => - (descriptors ?? Array.Empty()).Select(Other).OfType(); } public sealed class CultDocumentRegistry @@ -832,29 +821,42 @@ namespace GameCult.Caching catalog.FirstOrDefault(entry => string.Equals(entry.SchemaId, schemaId, StringComparison.Ordinal)) ?? catalog.FirstOrDefault(entry => entry.CompatibleSchemaIds.Contains(schemaId, StringComparer.Ordinal)); - // The one answer to whether a writer may write, relabel or remove a record stored under this id - // (variants:ruling:drift-record-write-ownership): a registered type declares the id (owns it or lists it compatible), or the - // type that reads it reads it whole. Read live, so a type registered later that declares the id claims its records at their - // next write. A record read only by dropping slots is not owned: declaring its id compatible is the consent to the loss. - // Header only: the persisted catalog entry that publishes the id is compared with the claimant's entry, never a payload. A - // claimant that cannot be named (ambiguous, id missing from the catalog, shapes incompatible) owns nothing. + // Whether a removal, or a check of a key, may proceed over a record stored under this id, and the load-time marking of a + // record read-only: a registered type declares the id (owns it or lists it compatible), or the type that reads it reads it + // whole (variants:ruling:drift-record-write-ownership). It never decides an upsert: WrittenOwns does + // (variants:ruling:write-ownership-by-written-type). Read live, so a type registered later that declares the id claims its + // records at their next removal. A record read only by dropping slots is not owned: declaring its id compatible is the + // consent to the loss. Header only: the persisted catalog entry that publishes the id is compared with the claimant's + // entry, never a payload. A claimant that cannot be named (ambiguous, id missing from the catalog, shapes incompatible) owns + // nothing. internal bool Owns(string schemaId, IReadOnlyCollection catalog) => - Owns(schemaId, catalog, Declared, _indexes, descriptor => descriptor.ToCatalogEntry(), _ => true); + Owns(schemaId, catalog, Declared, _indexes, descriptor => descriptor.ToCatalogEntry()); - // The registry's lookup, for a cacheless writer that lends it only its other types. - internal IClaimantLookup Lookup => _indexes; + // The one answer to whether an upsert of the written type may replace the record stored under this id at its key, for a cache + // and a cacheless writer, a single-file store and a directory store alike. The written entry owns the record when it declares + // the stored id (its own id, or one it lists compatible), or when it is the type that reads the record's published entry + // (the one Claimant precedence over this one entry) and reads it whole, so an older id of its own schema name that it reads + // whole is its own (variants:ruling:own-entry-reads-whole). Anything else, another registered type's record included, is + // refused: retyping a key is a removal and an add (variants:ruling:cache-refuses-retype). No registry enters it. + internal static bool WrittenOwns( + CultSchemaCatalogEntry written, + string storedId, + IReadOnlyCollection catalog) => + Owns(storedId, catalog, id => DeclaredBy(written, id), new CatalogEntryLookup(new[] { written }), entry => entry); - // The rule itself, for any writer: declared, or the claimant (picked by the one Claimant over a lookup) reads the record's - // published entry whole. The lookup is a registry's indexes, or the entries a registry-less cacheless writer writes. mayClaim - // is a condition on the claimant's answer, not a second lookup: a cache accepts any claimant, a cacheless writer only one of - // the entries the call writes, so it never owns a record another registered type reads. + // WrittenOwns' declared half: the entry's own id or one it lists compatible. + internal static bool DeclaredBy(CultSchemaCatalogEntry written, string storedId) => + string.Equals(written.SchemaId, storedId, StringComparison.Ordinal) || + written.CompatibleSchemaIds.Contains(storedId, StringComparer.Ordinal); + + // The rule itself: declared, or the claimant (picked by the one Claimant over a lookup) reads the record's published entry + // whole. The lookup is a registry's indexes (the removal rule) or the one entry a write puts at the key (WrittenOwns). internal static bool Owns( string schemaId, IReadOnlyCollection catalog, Func declared, IClaimantLookup lookup, - Func entryOf, - Func mayClaim) + Func entryOf) where T : class { if (declared(schemaId)) @@ -864,8 +866,7 @@ namespace GameCult.Caching var claimant = Claimant(schemaId, catalog, lookup, out var persisted); if (persisted == null || claimant == null) return false; - var claimantEntry = entryOf(claimant); - return mayClaim(claimantEntry) && ReadsWhole(persisted, claimantEntry); + return ReadsWhole(persisted, entryOf(claimant)); } catch (InvalidOperationException) { @@ -3105,14 +3106,14 @@ namespace GameCult.Caching if (globals.TryGetValue(type, out var key) && key == stored.Key.Value) globals.Remove(type); if (source == null) - Claim(stored); + Claim(stored, upsert: false); } foreach (var stored in admitted) { var descriptor = stored.Descriptor; if (source == null) - Claim(stored); + Claim(stored, upsert: true); else if (Home(descriptor.DocumentType) is var home && home != source) throw new InvalidOperationException( $"{descriptor.SchemaName} record {stored.Key.Value} was loaded from {source} but its home is {home?.ToString() ?? "no store"}."); @@ -3140,7 +3141,7 @@ namespace GameCult.Caching $"The batch spans {distinct[0]} and {distinct[1]}; a commit lands in one home store."); return distinct.FirstOrDefault(); - void Claim(CultStoredDocument stored) + void Claim(CultStoredDocument stored, bool upsert) { var home = Home(stored.Descriptor.DocumentType); if (home == null) @@ -3153,7 +3154,7 @@ namespace GameCult.Caching if (home.IsReadOnly) throw new InvalidOperationException( $"{stored.Descriptor.SchemaName} record {stored.Key.Value} routes to read-only backing store {home}."); - home.RefuseUndeclared(stored.Key.Value); + home.RefuseUndeclared(stored.Key.Value, upsert ? stored.Descriptor : null); homes.Add(home); } } @@ -3944,6 +3945,7 @@ namespace GameCult.Caching { StoredSchemaId = record.SchemaId, StoredSchemaReadWhole = readWhole, + StoredPublishedBy = PublishedByIn(record.SchemaId, catalog), StoredVersion = version }; } @@ -3968,10 +3970,14 @@ namespace GameCult.Caching { StoredSchemaId = record.SchemaId, StoredSchemaReadWhole = readWhole, + StoredPublishedBy = PublishedByIn(record.SchemaId, catalog), StoredVersion = version }; } + private static (string Id, CultSchemaCatalogEntry Entry)? PublishedByIn(string schemaId, IReadOnlyCollection catalog) => + CultDocumentRegistry.PublishedBy(schemaId, catalog) is { } entry ? (schemaId, entry) : null; + protected void SetLastSchemaMigrationReports(IEnumerable reports) { _lastSchemaMigrationReports = reports?.ToArray() ?? Array.Empty(); @@ -3991,19 +3997,34 @@ namespace GameCult.Caching _foreignRecords = foreign.OrderBy(record => record.Key, StringComparer.Ordinal).ToArray(); } - // A cache never writes, relabels or removes a record it does not own: the staging half of rule 3. Refused when the store - // carries a foreign record at the key, or holds one that is not owned: its stored id is not declared by a registered type and - // the type that reads it drops persisted slots (read-only). A record read whole is owned; ToStoredDocument decided that from - // the same comparison Owns makes. Called for every record a cache write admits or evicts, before anything is staged. - protected internal void RefuseUndeclared(string key) + // A cache never writes, relabels or removes a record it does not own: the staging half of rule 3. A removal or check of a key + // is refused when the store carries a foreign record at the key, or holds one that is not owned: its stored id is not + // declared by a registered type and the type that reads it drops persisted slots (read-only). A record read whole is owned; + // ToStoredDocument decided that from the same comparison Owns makes. + protected internal void RefuseUndeclared(string key) => RefuseUndeclared(key, null); + + // The same, for every record a cache write admits or evicts, before anything is staged: an upsert (written is its type) may + // replace only a record that type owns, WrittenOwns' rule (variants:ruling:write-ownership-by-written-type), asked over the + // catalog entry that published the held record's id when it was loaded, the question the store's judge asks of the file. + internal void RefuseUndeclared(string key, CultDocumentDescriptor? written) { var foreign = _foreignRecords.FirstOrDefault(record => string.Equals(record.Key, key, StringComparison.Ordinal)); if (foreign != null) throw Overwrites(null, new[] { (foreign.Key, foreign.SchemaId, foreign.SchemaName) }); - if (Entries.TryGetValue(key, out var held) && !Registry.Declared(held.StoredSchemaId) && !held.StoredSchemaReadWhole) + if (Entries.TryGetValue(key, out var held) && !HeldOwned(held, written)) throw Overwrites(null, new[] { (key, held.StoredSchemaId, held.Descriptor.SchemaName) }); } + private bool HeldOwned(CultStoredDocument held, CultDocumentDescriptor? written) + { + if (written == null) + return Registry.Declared(held.StoredSchemaId) || held.StoredSchemaReadWhole; + var published = held.StoredPublishedBy is { } loaded && string.Equals(loaded.Id, held.StoredSchemaId, StringComparison.Ordinal) + ? loaded.Entry + : held.Descriptor.ToCatalogEntry(); + return CultDocumentRegistry.WrittenOwns(written.ToCatalogEntry(), held.StoredSchemaId, new[] { published }); + } + /// /// Called when the cache removes keys it does not hold, outside a batch that commits through this store. Such a removal /// removes nothing, but a store that holds a record at one of these keys under an id no registered type declares must @@ -4077,6 +4098,35 @@ namespace GameCult.Caching ? null : (record.Key, record.SchemaId, CultDocumentRegistry.PublishedBy(record.SchemaId, catalog)?.SchemaName ?? string.Empty); + // The one per-key judge of a write over a durable catalog, for every store kind: the stored record at each key the write names + // is owned by the entry the write puts at that key (WrittenOwns), or, when the write only removes or checks the key, by the + // removal rule. heldAt gives the durable record at a key and writtenAt the catalog entry the write puts there (null for a + // removal or a check). Memoised per written schema id and stored id, so a large store pays one comparison per pair, and an + // answer for one written type is never reused for another. + internal static Func UndeclaredAt( + Func heldAt, + IReadOnlyCollection catalog, + Func writtenAt, + Func, bool> removalOwns) + { + var answers = new Dictionary<(string? Written, string Stored), bool>(); + return key => + { + if (heldAt(key) is not { } held) + return null; + var written = writtenAt(key); + return Undeclared(held, catalog, (storedId, entries) => + { + var asked = (written?.SchemaId, storedId); + if (!answers.TryGetValue(asked, out var answer)) + answers[asked] = answer = written == null + ? removalOwns(storedId, entries) + : CultDocumentRegistry.WrittenOwns(written, storedId, entries); + return answer; + }); + }; + } + // The apply half of rule 3, for every store kind, checked where the write lands under the store's lock. A key this write names // whose durable record is under an id no registered type declares is refused, every such key at once, through Refuse, which // reloads it as the store holds it: listed foreign when no type reads it, held read-only when one does. @@ -4484,7 +4534,7 @@ namespace GameCult.Caching /// The record to store. /// The schema catalog entry for the record's schema id. /// Whether the record's payload holds element ids, or null when the caller cannot tell. - /// The registry the payload was encoded with. Given, the write decides the header for copied records that registry can read, and its other types also claim stored records as a cache registering them would; it declares nothing, so it can only refuse more. stays the only description of its own schema id and the registry's type for that id is never asked. Owned are a record whose id declares (its own or one it lists compatible) and a record the entry reads whole when it is the picked claimant; refused are a record another registered type claims, one whose stored entry lists another registered type's id ahead of the entry's, and one that two registered types list or two versions of the schema could read. Omitted, it decides nothing for copied records and the one entry is the only claimant. It serves this call only: the store keeps no registry. + /// The registry the payload was encoded with. Given, the write decides the header for copied records that registry can read. It never decides ownership: the answer is the same lent or not. alone owns the record at the key when it declares the stored id (its own or one it lists compatible), or when it is the type that reads the stored entry and reads it whole, so an older id of its own schema name it reads whole is its own and is restamped. Everything else is refused with the file left as it was: a record of another registered type included, whatever the registry holds, and a record it would read only by dropping persisted slots. Retyping a key is a removal and an add. It serves this call only: the store keeps no registry. /// The call holds the store's gate while it runs. User code it runs must not enter a cache or another store; that throws . /// The store is attached to a cache; write through the cache. /// The file at the path is not a store this runtime can read. @@ -4507,7 +4557,7 @@ namespace GameCult.Caching /// The record key the document is stored under. /// The document to store. /// The stored-at stamp, or null for now. - /// The registry that describes and encodes the document, or null for . The document type's entry is taken from it and is the only description of that schema id. Its other types claim stored records as a cache registering them would, and it declares nothing, so it can only refuse more: the document's entry owns a record whose id it declares (its own or one it lists compatible), and a record it reads whole when it is the picked claimant; refused are a record another registered type claims, one whose stored entry lists another registered type's id ahead of the entry's, and one that two registered types list or two versions of the schema could read. A wider registry (Shared when none is passed) refuses at least as much. It serves this call only: the store keeps no registry. + /// The registry that describes and encodes the document, or null for . The document type's entry is taken from it, and the registry decides the header for copied records it can read. It never decides ownership, so the answer is the same for any registry that holds the type: the document's entry owns a record whose id it declares (its own or one it lists compatible), and a record of an older id of its own schema name that it reads whole; everything else is refused, another registered type's record included, and a record it would read only by dropping persisted slots. Retyping a key is a removal and an add. It serves this call only: the store keeps no registry. /// The call holds the store's gate while it runs. User code it runs (property getters while the document is encoded, constructors while a copied record is read) must not enter a cache or another store; that throws , carried by the encoder's wrapper when the code ran inside it. /// The store is attached to a cache; write through the cache. /// The file at the path is not a store this runtime can read. @@ -4723,53 +4773,43 @@ namespace GameCult.Caching using var fileLock = AcquireLock(wait); if (fileLock == null) return CultCommitOutcome.Contended; - RefuseUndeclaredOnDisk(ReadSnapshot(), keys, OwnsOnDisk()); + RefuseUndeclaredOnDisk(ReadSnapshot(), keys, Array.Empty()); return CultCommitOutcome.Committed; }); } - // A cache owns through its registry (CultDocumentRegistry.Owns). A cacheless CultMesh write owns what the catalog entries it - // writes declare: their own ids and the ids they list compatible, or an id whose publishing entry its claimant reads whole, - // the claimant being picked by the one precedence over the written entries and the lent registry's other types, and owning - // only when it is a written entry (variants:ruling:mesh-writes-bound-by-declaration). The keys judged are every key staged, - // written, removed or checked. + // Every key staged, written, removed or checked is judged, each by the one per-key judge (UndeclaredAt). private void RefuseUndeclaredOnDisk(Durable? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals, IEnumerable checks) => RefuseUndeclaredOnDisk( durable, _staged.Concat(upserts.Select(staged => staged.Record.Key)).Concat(removals).Concat(checks).Concat(StagedChecks), - OwnsOnDisk(upserts)); + upserts); - // The owns predicate a write over the durable catalog asks. A cacheless writer declares what the entries it writes declare - // (variants:ruling:mesh-writes-bound-by-declaration) and reads a published entry with one lookup: the written entries shadow - // the lent registry's type for their ids, and the registry adds other claimants. The record is owned only when the claimant - // is a written entry, so the call is never more permissive than that declaration, and a registry can only refuse more. - private Func, bool> OwnsOnDisk(IReadOnlyCollection? upserts = null) => - Cache != null || upserts == null + // A write's key is owned by the entry the write puts there, whoever the writer is (WrittenOwns). A key it only removes or + // checks is owned by the removal rule: a cache's registry (Owns), or, for a cacheless call that has no registry, some entry the + // call writes (a call that writes no entry owns nothing, variants:ruling:mesh-writes-bound-by-declaration). + private Func, bool> RemovalOwnsOnDisk(IReadOnlyCollection upserts) => + Cache != null ? Registry.Owns - : (id, catalog) => - { - var writes = upserts.Select(staged => staged.Entry).ToArray(); - Func declared = candidate => writes.Any(entry => - string.Equals(entry.SchemaId, candidate, StringComparison.Ordinal) || - entry.CompatibleSchemaIds.Contains(candidate, StringComparer.Ordinal)); - return CultDocumentRegistry.Owns( - id, - catalog, - declared, - new CatalogEntryLookup(writes, AttachedRegistry), - entry => entry, - claimant => writes.Any(entry => string.Equals(entry.SchemaId, claimant.SchemaId, StringComparison.Ordinal))); - }; + : (id, catalog) => upserts.Any(staged => CultDocumentRegistry.WrittenOwns(staged.Entry, id, catalog)); // A refused key's durable version is what this store has now read of it. - private void RefuseUndeclaredOnDisk(Durable? durable, IEnumerable written, Func, bool> owns) + private void RefuseUndeclaredOnDisk(Durable? durable, IEnumerable written, IReadOnlyCollection upserts) { - var undeclared = UndeclaredOnDisk(durable, owns); - if (undeclared.Count == 0) - return; + var keys = written.Distinct(StringComparer.Ordinal).ToArray(); + var wanted = keys.ToHashSet(StringComparer.Ordinal); + var held = (durable?.Snapshot.Records ?? Array.Empty()) + .Where(record => wanted.Contains(record.Key)) + .ToDictionary(record => record.Key, StringComparer.Ordinal); + var entries = upserts.ToDictionary(staged => staged.Record.Key, staged => staged.Entry, StringComparer.Ordinal); + var undeclaredAt = UndeclaredAt( + key => held.TryGetValue(key, out var record) ? record : null, + durable?.Snapshot.SchemaCatalog ?? Array.Empty(), + key => entries.TryGetValue(key, out var entry) ? entry : null, + RemovalOwnsOnDisk(upserts)); RefuseUndeclaredOnDisk( - written, - key => undeclared.TryGetValue(key, out var carried) ? carried : null, + keys, + undeclaredAt, key => Cache == null ? null : DurableAt(durable, key), key => ForeignAt(durable, key), key => @@ -4781,23 +4821,6 @@ namespace GameCult.Caching cacheless: Cache == null); } - // Every durable record under an id the writer does not own, by key, named by the catalog entry that publishes its id. The - // answer is decided once per distinct schema id, so a large store pays one comparison per id, not per record. - private static Dictionary UndeclaredOnDisk(Durable? durable, Func, bool> owns) - { - var undeclared = new Dictionary(StringComparer.Ordinal); - var answers = new Dictionary(StringComparer.Ordinal); - bool OwnsOnce(string id, IReadOnlyCollection catalog) => - answers.TryGetValue(id, out var answer) ? answer : answers[id] = owns(id, catalog); - foreach (var record in durable?.Snapshot.Records ?? Array.Empty()) - { - if (Undeclared(record, durable!.Snapshot.SchemaCatalog, OwnsOnce) is { } carried) - undeclared[record.Key] = carried; - } - - return undeclared; - } - // The durable record at a key as a load lists it when no registered type claims its id; null otherwise. private CultForeignRecord? ForeignAt(Durable? durable, string key) { diff --git a/tests/GameCult.Caching.Tests/RestampOnWriteTests.cs b/tests/GameCult.Caching.Tests/RestampOnWriteTests.cs index b90f4336..b1976aa6 100644 --- a/tests/GameCult.Caching.Tests/RestampOnWriteTests.cs +++ b/tests/GameCult.Caching.Tests/RestampOnWriteTests.cs @@ -480,17 +480,18 @@ namespace GameCult.Caching.Tests private static readonly CultDocumentRegistry NarrowShortRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(NoteNarrow), typeof(NoteShort) }); - // Which entry reads a persisted entry is one precedence: a cacheless writer lent a registry picks the claimant that registry - // picks, never one by schema name over its own entry. Here the entry is named and shaped like Narrow but lists Short's id. + // A write's ownership is the written entry's alone (variants:ruling:own-entry-reads-whole): the stored entry is named and shaped + // like Narrow but lists Short's id, and a registry that holds Short (which would read it by listing the id) changes nothing, for + // a cache and a cacheless writer alike. [TestCase(false)] [TestCase(true)] - public void ACachelessWriterPicksTheClaimantItsRegistryPicks(bool cacheless) + public void AWrittenEntryReadingTheRecordWholeOwnsItWhateverTheRegistryHolds(bool cacheless) { var path = SeedRenamed(listsNarrow: false, alsoLists: NarrowShortRegistry.GetRequired().SchemaId, name: "tests.restamp_note"); - var before = Fingerprint(path); - Assert.Throws(() => WriteNarrow(path, cacheless, NarrowShortRegistry)); - Assert.That(Fingerprint(path), Is.EqualTo(before)); + WriteNarrow(path, cacheless, NarrowShortRegistry); + + Assert.That(Snapshot(path).Records.Single(record => record.Key == "note").SchemaId, Is.EqualTo(NarrowRegistry.GetRequired().SchemaId)); } [TestCase(false)] @@ -668,28 +669,6 @@ namespace GameCult.Caching.Tests Assert.That(Fingerprint(path), Is.EqualTo(before)); } - // The shadow, where it is decided: a written entry answers for its schema id whatever the registry holds, and the registry - // still adds its other types. - [Test] - public void AWrittenEntryShadowsTheLentRegistrysTypeForItsId() - { - var written = NarrowRegistry.GetRequired().ToCatalogEntry(); - var lookup = new CatalogEntryLookup(new[] { written }, NarrowListerRegistry); - var lister = NarrowListerRegistry.GetRequired(); - - Assert.That(lookup.Owner(written.SchemaId), Is.SameAs(written)); - Assert.That(lookup.Named(written.SchemaName), Is.EqualTo(new[] { written }).Using((x, y) => ReferenceEquals(x, y))); - Assert.That(lookup.Listing(written.SchemaId), Is.EqualTo(new[] { written }).Using((x, y) => ReferenceEquals(x, y))); - Assert.That(lookup.Owner(lister.SchemaId)!.SchemaId, Is.EqualTo(lister.SchemaId)); - Assert.That(lookup.Listing(ListerOldId).Select(entry => entry.SchemaId), Is.EqualTo(new[] { lister.SchemaId })); - Assert.That(lookup.Named(lister.SchemaName).Select(entry => entry.SchemaId), Is.EqualTo(new[] { lister.SchemaId })); - - // A write of several entries shadows by the entry that has the id, not by all of them having it. - var other = CultDocumentRegistry.ForTypes(new[] { typeof(NoteOther) }).GetRequired().ToCatalogEntry(); - var several = new CatalogEntryLookup(new[] { other, written }, NarrowListerRegistry); - Assert.That(several.Named(written.SchemaName), Is.EqualTo(new[] { written }).Using((x, y) => ReferenceEquals(x, y))); - } - // The registry's other types change nothing about a record the written type claims itself. [Test] public void AnotherRegisteredTypeDoesNotTakeAWritersOwnDriftRecord() @@ -701,16 +680,16 @@ namespace GameCult.Caching.Tests Assert.That(Snapshot(path).Records.Single(record => record.Key == "note").SchemaId, Is.EqualTo(NarrowRegistry.GetRequired().SchemaId)); } - // What the lent registry registers beyond the written type can only make the claim more ambiguous, never own more: a second - // version of the written schema's name refuses a record the write owned by name alone. + // What the lent registry registers beyond the written type changes nothing, in either direction: a second version of the + // written schema's name neither refuses nor widens a record the write owns by name alone. [Test] - public void ASecondVersionOfTheWrittenNameInTheLentRegistryOnlyRefuses() + public void ASecondVersionOfTheWrittenNameInTheLentRegistryChangesNothing() { var path = SeedRenamed(listsNarrow: false, name: "tests.restamp_note"); - var before = Fingerprint(path); - Assert.Throws(() => WriteNarrow(path, cacheless: true, CultDocumentRegistry.ForTypes(new[] { typeof(NoteNarrow), typeof(NoteGrown) }))); - Assert.That(Fingerprint(path), Is.EqualTo(before)); + WriteNarrow(path, cacheless: true, CultDocumentRegistry.ForTypes(new[] { typeof(NoteNarrow), typeof(NoteGrown) })); + + Assert.That(Snapshot(path).Records.Single(record => record.Key == "note").SchemaId, Is.EqualTo(NarrowRegistry.GetRequired().SchemaId)); } // A record stored under an id its catalog entry only lists (no entry owns it) is published by that entry, and the type that @@ -765,8 +744,8 @@ namespace GameCult.Caching.Tests var writes = new[] { narrow }; Func declares = id => id == narrow.SchemaId; - Assert.That(CultDocumentRegistry.Owns("tests.restamp.unpublished", writes, declares, new CatalogEntryLookup(writes, null), entry => entry, _ => true), Is.False); - Assert.That(CultDocumentRegistry.Owns(narrow.SchemaId, writes, declares, new CatalogEntryLookup(writes, null), entry => entry, _ => true), Is.True); + Assert.That(CultDocumentRegistry.Owns("tests.restamp.unpublished", writes, declares, new CatalogEntryLookup(writes), entry => entry), Is.False); + Assert.That(CultDocumentRegistry.Owns(narrow.SchemaId, writes, declares, new CatalogEntryLookup(writes), entry => entry), Is.True); } private static string VectorPath(string name) diff --git a/unity/org.gamecult.cultlib/CHANGELOG.md b/unity/org.gamecult.cultlib/CHANGELOG.md index 390ece2a..6d8fcd71 100644 --- a/unity/org.gamecult.cultlib/CHANGELOG.md +++ b/unity/org.gamecult.cultlib/CHANGELOG.md @@ -13,7 +13,7 @@ Releases as 3.0.0 (major). - A store is written in the format its content needs, and CultLib 2.0 refuses the newer ones by name. A single-file store holding a document variant is written as `cultcache.store.v2`, one holding an element id as `cultcache.store.v3`; a directory store holding an element id is written as `cultcache.store.v5` (`v4` otherwise). A store is marked only when a record actually holds an id or a variant. Upgrade readers before writers: a 2.0 reader that opens such a store is refused, not misread. - Writes are write sets. A flush or commit replaces the records it staged and copies every other record forward as stored, under the store's lock, instead of rewriting the whole store from the cache's memory. A record the cache does not know is kept, and an untouched record keeps a dropped member's slot through every write (never reuse a dropped slot; see the schema-compatibility contract). - A write or removal at a key whose stored record the writer does not own is refused with `CultSchemaConflictException`, and the file is left as it was. A writer owns a record whose schema id it declares (owns or lists compatible) and a record its type reads whole under compatible drift: that record is writable and is restamped under the type's own id on write, as in 2.0.1. A record a type reads only by dropping persisted slots stays read-only until its id is declared compatible, which accepts the loss (this differs from 2.0.1, which shed the slots silently). A cache owns through its registry; a cacheless CultMesh write owns through the catalog entry it writes. -- A cacheless `WriteDocument`, `WriteRecord` or unattached-store `PushAll` owns a stored record in two ways only: the entries it writes declare the record's schema id (an entry's own id, or one it lists compatible), or the type picked to read the record is one of the written entries and reads it whole (no persisted slot unread). The type is picked as a cache registering the call's registry would pick it, except that a written entry is the only description of its schema id: `WriteDocument` takes its entry from the registry (`CultDocumentRegistry.Shared` when none is passed), `WriteRecord` writes the entry it is given, and a registry's own type for that id is never asked. A registry adds claimants and declares nothing, so it can only refuse more; without one the written entries are the only claimants. Refused, with the file left as it was: a record another registered type claims, under that type's current id or an old id it lists; a record whose stored entry lists another registered type's id ahead of the written one's, whatever name the stored entry carries; a record that two registered types list compatible, or that two registered versions of the written schema could read; a record the picked written entry would read only by dropping persisted slots; and the removal of any stored record by a call that writes no entry (a `Delete` then `PushAll`). As in 2.0.1, a record the written entry's own type reads whole under drift is owned and restamped. A registry-less `WriteDocument` reads `Shared` today, so any type loaded there can refuse the write; pass a registry to pin the answer. +- A write replaces the record at a key only when the type it writes there owns that record, decided per key and the same for a cache and for a cacheless `WriteDocument`, `WriteRecord` or unattached-store `PushAll`, in a single-file store and a directory store. The written type owns the stored record when its entry declares the stored id (its own id, or one it lists compatible), or when it is the type that reads the record's published entry and reads it whole (no persisted slot unread): an older id of its own schema name that it reads whole is its own and is restamped, as in 2.0.1. Anything else is refused with `CultSchemaConflictException` and the file left as it was: a record of another registered type, under its current id or an old id it lists, and a record the written type would read only by dropping persisted slots. No registry enters the decision, `CultDocumentRegistry.Shared` included, so a lent registry never changes the answer. This changes a cache: where 2.x let a cache holding two types replace one's record at a key with the other's, it now refuses; retype a key by removing it and adding the new record in separate commits. A removal or check goes to the registry's removal rule, and a cacheless call, which has no registry, removes a record only when an entry it writes owns it (a `Delete` then `PushAll` on an unattached store writes no entry and is refused). - Registration refuses an owned object-list element type that has no `[CultElementId]` member (a public string with `[Key]`), naming the type and the list. Element ids let a write set merge a list by identity; a list of record references or plain values is a whole value and is not affected. - The abstract and protected surface for store kinds outside this library changed. On `SingleFileBackingStore`, `SerializeSnapshot` and `DeserializeSnapshot` are replaced by the abstract `SerializeStore`, `SerializeRecord`, `SerializeCatalogEntry` and `ReadStore`, and the abstract `SerializePayload(object)` becomes `SerializePayload(object document, Type documentType)`: a store encodes a document as the type it is given. On `CacheBackingStore`, `Judging` is removed, `ToStoredDocument` takes the record's version (`record, version, catalog, deserialize, out report`), and these protected members are new for a store kind to implement or call: `HeaderFor`, `HeaderForWrite`, `Foreign`, `SetForeignRecords`, `Undeclared`, `RefuseUndeclared`, `RefuseUndeclaredOnDisk`, `RefuseRemovalOfUndeclared`, `Refuse`, `SameStoredIdentity`, `VersionOf`, `StoredAtOver`, `Overwrites`, `StagedChecks`, `StageChecks` and `JudgeChecks`. `MarkFlushSucceeded` is unchanged. `CacheBackingStore.ForeignRecords` is new and public: the records a store holds under a schema id its cache does not declare. - `CultCommitRequest.Expected` is now `IReadOnlyList<(CultRecordKey Key, string? Version)>` (it was a key with a schema id and a stored-at), `CultCommitRequest.Checks` is new, and `ConditionsHold` takes `(IReadOnlyDictionary durable, IEnumerable observed, IEnumerable foreign)` in place of `(IReadOnlyCollection, IEnumerable)`. A store kind outside the library lists the record versions it holds and passes its foreign records. From 244ad2280ee348bd479ef35bda762024d3b41e2d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 28 Oct 2026 02:36:17 +0100 Subject: [PATCH 02/88] WriteOwnershipTests, WrittenOwnsTests: pin the per-key write predicate in both arms and both store kinds Co-Authored-By: Claude Sonnet 5.5 --- .../WriteOwnershipTests.cs | 248 ++++++++++++++++++ .../WrittenOwnsTests.cs | 97 +++++++ 2 files changed, 345 insertions(+) create mode 100644 tests/GameCult.Caching.Tests/WriteOwnershipTests.cs create mode 100644 tests/GameCult.Caching.Tests/WrittenOwnsTests.cs diff --git a/tests/GameCult.Caching.Tests/WriteOwnershipTests.cs b/tests/GameCult.Caching.Tests/WriteOwnershipTests.cs new file mode 100644 index 00000000..5dab91e3 --- /dev/null +++ b/tests/GameCult.Caching.Tests/WriteOwnershipTests.cs @@ -0,0 +1,248 @@ +#nullable enable +using System; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Caching.Tests +{ + // variants:ruling:write-ownership-by-written-type and variants:ruling:cache-refuses-retype: an upsert replaces the record stored at + // its key only when the type it writes there owns that record. Which type owns the id, and which other types a registry holds, + // change nothing; retyping a key is a removal and an add. The rule is one per-key predicate (CultDocumentRegistry.WrittenOwns) + // for a cache and a cacheless writer, in a single-file store and a directory store. + public class WriteOwnershipTests + { + // Two schemas of one shape that are never the same type. + [CultDocument("tests.wo_a", "tests.wo_a.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class WoA + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + } + + [CultDocument("tests.wo_b", "tests.wo_b.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class WoB + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + } + + private static readonly CultDocumentRegistry ARegistry = CultDocumentRegistry.ForTypes(new[] { typeof(WoA) }); + private static readonly CultDocumentRegistry ABRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(WoA), typeof(WoB) }); + private static readonly CultRecordKey K1 = new("k1"); + private static readonly CultRecordKey K2 = new("k2"); + + private string _directory = string.Empty; + + [SetUp] + public void CreateDirectory() + { + _directory = Path.Combine(Path.GetTempPath(), $"cultlib-writeownership-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_directory); + } + + [TearDown] + public void DeleteDirectory() + { + if (Directory.Exists(_directory)) + Directory.Delete(_directory, recursive: true); + } + + public static readonly object[] StoreKinds = { false, true }; + + private string PathFor(bool directory) => Path.Combine(_directory, directory ? "store" : "store.cc"); + + private static CultCache Open(string path, bool directory, CultDocumentRegistry registry) + { + var cache = new CultCache(registry); + cache.AddBackingStore(directory ? new DirectoryMessagePackBackingStore(path) : new SingleFileMessagePackBackingStore(path)); + cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + return cache; + } + + private static string Fingerprint(string path) + { + using var sha = SHA256.Create(); + if (File.Exists(path)) + return Convert.ToHexString(sha.ComputeHash(File.ReadAllBytes(path))); + return string.Join("|", Directory.GetFiles(path, "*", SearchOption.AllDirectories) + .OrderBy(file => file, StringComparer.Ordinal) + .Select(file => Path.GetRelativePath(path, file) + ":" + Convert.ToHexString(sha.ComputeHash(File.ReadAllBytes(file))))); + } + + private static Type TypeAt(string path, bool directory, CultDocumentRegistry registry, CultRecordKey key) + { + using var cache = Open(path, directory, registry); + return cache.GetStored(key)!.Descriptor.DocumentType; + } + + // A store that a cache holding both types wrote: A at k1 and B at k2. + private string SeedAAtK1BAtK2(bool directory) + { + var path = PathFor(directory); + using var cache = Open(path, directory, ABRegistry); + Assert.That(cache.Commit(batch => + { + batch.Upsert(new WoA { Name = "k1", Value = 1 }, new CultRecordHandle(K1)); + batch.Upsert(new WoB { Name = "k2", Value = 2 }, new CultRecordHandle(K2)); + }), Is.True); + return path; + } + + // P1: a cache holding A and B may not replace B's record with A's, whoever else the registry declares. + [TestCaseSource(nameof(StoreKinds))] + public void ACacheRefusesToWriteOneRegisteredTypeOverAnothersRecord(bool directory) + { + var path = SeedAAtK1BAtK2(directory); + var before = Fingerprint(path); + + using (var cache = Open(path, directory, ABRegistry)) + { + Assert.Throws(() => cache.Commit(batch => batch.Upsert(new WoA { Name = "k2", Value = 9 }, new CultRecordHandle(K2)))); + Assert.That(cache.GetStored(K2)!.Descriptor.DocumentType, Is.EqualTo(typeof(WoB)), "the cache still holds B's record"); + Assert.Throws(() => cache.UpsertAsync(typeof(WoA), new WoA { Name = "k2", Value = 9 }, K2)); + Assert.That(cache.GetStored(K2)!.Descriptor.DocumentType, Is.EqualTo(typeof(WoB)), "a staged write is refused at the call, not at the flush"); + } + + Assert.That(Fingerprint(path), Is.EqualTo(before)); + Assert.That(TypeAt(path, directory, ABRegistry, K2), Is.EqualTo(typeof(WoB))); + + // The pair that differs only in the type written: B over B's own record. + using (var cache = Open(path, directory, ABRegistry)) + Assert.That(cache.Commit(batch => batch.Upsert(new WoB { Name = "k2", Value = 3 }, new CultRecordHandle(K2))), Is.True); + } + + // Retype is a removal and an add, in separate commits. + [TestCaseSource(nameof(StoreKinds))] + public void ARetypeIsARemovalAndAnAddInSeparateCommits(bool directory) + { + var path = SeedAAtK1BAtK2(directory); + + using (var cache = Open(path, directory, ABRegistry)) + { + Assert.That(cache.Commit(batch => batch.Remove(K2)), Is.True); + Assert.That(cache.Commit(batch => batch.Upsert(new WoA { Name = "k2", Value = 9 }, new CultRecordHandle(K2))), Is.True); + } + + Assert.That(TypeAt(path, directory, ABRegistry, K2), Is.EqualTo(typeof(WoA))); + Assert.That(TypeAt(path, directory, ABRegistry, K1), Is.EqualTo(typeof(WoA)), "the bystander is untouched"); + } + + // The store is judged again where the write lands, under its lock: another writer retyped the key after this cache loaded it, + // so this cache's own-type write of the old record is now a write over a record that is not its type's. + [TestCaseSource(nameof(StoreKinds))] + public void AnotherWritersRetypeRefusesThisCachesStaleWriteUnderTheLock(bool directory) + { + var path = SeedAAtK1BAtK2(directory); + using var stale = Open(path, directory, ABRegistry); + Assert.That(stale.GetStored(K1)!.Descriptor.DocumentType, Is.EqualTo(typeof(WoA))); + using (var other = Open(path, directory, ABRegistry)) + { + Assert.That(other.Commit(batch => batch.Remove(K1)), Is.True); + Assert.That(other.Commit(batch => batch.Upsert(new WoB { Name = "k1", Value = 5 }, new CultRecordHandle(K1))), Is.True); + } + + var before = Fingerprint(path); + + Assert.Throws(() => stale.Commit(batch => batch.Upsert(new WoA { Name = "k1", Value = 9 }, new CultRecordHandle(K1)))); + + Assert.That(Fingerprint(path), Is.EqualTo(before)); + Assert.That(TypeAt(path, directory, ABRegistry, K1), Is.EqualTo(typeof(WoB))); + } + + public static readonly object[] Registries = { "none", "A", "AB", "Shared" }; + + private static CultDocumentRegistry? Lent(object kind) => kind switch + { + "A" => ARegistry, + "AB" => ABRegistry, + "Shared" => CultDocumentRegistry.Shared, + _ => null, + }; + + // P3: a cacheless write of A over B's record is refused whatever registry is lent, and the same lent or not. + [TestCaseSource(nameof(Registries))] + public void ACachelessWriteOfOneTypeOverAnothersRecordIsRefusedWhateverRegistryIsLent(object lent) + { + var path = SeedAAtK1BAtK2(directory: false); + var before = Fingerprint(path); + var store = new SingleFileMessagePackBackingStore(path); + + Assert.Throws(() => store.WriteDocument(K2, new WoA { Name = "k2", Value = 9 }, registry: Lent(lent))); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + + // WriteRecord names its entry and nothing else. + var entry = ARegistry.GetRequired().ToCatalogEntry(); + var record = new CultPersistedRecord + { + Key = K2.Value, + SchemaId = entry.SchemaId, + StoredAt = "2026-10-28T00:00:00.0000000+00:00", + Payload = MessagePackSerializer.Serialize(new WoA { Name = "k2", Value = 9 }), + }; + Assert.Throws(() => store.WriteRecord(record, entry, null, Lent(lent))); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + + // The pair that differs only in whose record it is: A over A's own, which commits for every registry. + store.WriteDocument(K1, new WoA { Name = "k1", Value = 9 }, registry: Lent(lent) ?? ARegistry); + Assert.That(TypeAt(path, false, ABRegistry, K1), Is.EqualTo(typeof(WoA))); + } + + private static CultStoredDocument Doc(CultDocumentRegistry registry, CultRecordKey key, T document) where T : class => + new(key, "2026-10-28T00:00:00.0000000+00:00", registry.GetRequired(), document); + + // The judge is per key: an unattached PushAll that stages A at B's key and B at A's key is refused, though together the writes + // declare both ids. + [Test] + public void AnUnattachedPushAllJudgesEachKeyByItsOwnWrittenType() + { + var path = SeedAAtK1BAtK2(directory: false); + var before = Fingerprint(path); + var swapped = new SingleFileMessagePackBackingStore(path); + swapped.Push(Doc(ABRegistry, K1, new WoB { Name = "k1", Value = 7 })); + swapped.Push(Doc(ABRegistry, K2, new WoA { Name = "k2", Value = 7 })); + + Assert.Throws(() => swapped.PushAll()); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + + var own = new SingleFileMessagePackBackingStore(path); + own.Push(Doc(ABRegistry, K1, new WoA { Name = "k1", Value = 7 })); + own.Push(Doc(ABRegistry, K2, new WoB { Name = "k2", Value = 7 })); + own.PushAll(); + Assert.That(Fingerprint(path), Is.Not.EqualTo(before)); + } + + // An answer for one written type is never reused for another: two records stored under A's id, A written at one key and B at + // the other, in either staging order and either key order. + [TestCase("a-own", "b-bad", true)] + [TestCase("a-own", "b-bad", false)] + [TestCase("b-own", "a-bad", true)] + [TestCase("b-own", "a-bad", false)] + public void AnAnswerForOneWrittenTypeIsNeverReusedForAnother(string ownKey, string badKey, bool ownFirst) + { + var path = PathFor(directory: false); + using (var seed = Open(path, false, ABRegistry)) + Assert.That(seed.Commit(batch => + { + batch.Upsert(new WoA { Name = ownKey, Value = 1 }, new CultRecordHandle(new CultRecordKey(ownKey))); + batch.Upsert(new WoA { Name = badKey, Value = 1 }, new CultRecordHandle(new CultRecordKey(badKey))); + }), Is.True); + var before = Fingerprint(path); + var store = new SingleFileMessagePackBackingStore(path); + void PushOwn() => store.Push(Doc(ABRegistry, new CultRecordKey(ownKey), new WoA { Name = ownKey, Value = 2 })); + void PushBad() => store.Push(Doc(ABRegistry, new CultRecordKey(badKey), new WoB { Name = badKey, Value = 2 })); + if (ownFirst) { PushOwn(); PushBad(); } else { PushBad(); PushOwn(); } + + var refusal = Assert.Throws(() => store.PushAll())!; + + Assert.That(refusal.RecordKey, Is.EqualTo(badKey)); + Assert.That(refusal.Message, Does.Not.Contain($"'{ownKey}'"), "the key A owns is not among the refused"); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + } + } +} diff --git a/tests/GameCult.Caching.Tests/WrittenOwnsTests.cs b/tests/GameCult.Caching.Tests/WrittenOwnsTests.cs new file mode 100644 index 00000000..c1a1c148 --- /dev/null +++ b/tests/GameCult.Caching.Tests/WrittenOwnsTests.cs @@ -0,0 +1,97 @@ +#nullable enable +using System; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Caching.Tests +{ + // The one per-key predicate of write ownership (variants:ruling:write-ownership-by-written-type, own-entry-reads-whole), over + // catalogs the production types publish. Each refusal is paired with the accepted case that differs in only the one thing the + // rule looks at. Where the predicate is wired in is WriteOwnershipTests. + public class WrittenOwnsTests + { + // One schema name at three versions, nothing declared: N2 reads N1's records whole (it adds a slot) and Wide's only by + // dropping a slot; Renamed has N2's exact shape under another schema name. + [CultDocument("tests.wo_n", "tests.wo_n.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class WoN1 + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + } + + [CultDocument("tests.wo_n", "tests.wo_n.v2")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class WoN2 + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + [Key(2)] public string Colour = string.Empty; + } + + [CultDocument("tests.wo_n", "tests.wo_n.v3")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class WoNWide + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + [Key(2)] public string Colour = string.Empty; + [Key(3)] public string Extra = string.Empty; + } + + [CultDocument("tests.wo_nx", "tests.wo_nx.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class WoNRenamed + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + [Key(2)] public string Colour = string.Empty; + } + + private static CultSchemaCatalogEntry EntryOf() where T : class => + CultDocumentRegistry.ForTypes(new[] { typeof(T) }).GetRequired().ToCatalogEntry(); + + private static CultSchemaCatalogEntry Listing(CultSchemaCatalogEntry entry, params string[] ids) => + new() + { + SchemaId = entry.SchemaId, + ContentHash = entry.ContentHash, + SchemaName = entry.SchemaName, + SchemaVersion = entry.SchemaVersion, + CanonicalSchemaJson = entry.CanonicalSchemaJson, + Members = entry.Members, + CompatibleSchemaIds = ids, + }; + + [Test] + public void WrittenOwnsDeclaredIdsAreTheEntrysOwnOrAnyItListsNotOnlyTheFirst() + { + var a = EntryOf(); + var b = EntryOf(); + var catalog = new[] { a }; + + Assert.That(CultDocumentRegistry.WrittenOwns(a, a.SchemaId, catalog), Is.True, "its own id"); + Assert.That(CultDocumentRegistry.WrittenOwns(b, a.SchemaId, catalog), Is.False, "same shape, another schema: not its record"); + Assert.That(CultDocumentRegistry.WrittenOwns(Listing(b, b.SchemaId, a.SchemaId), a.SchemaId, catalog), Is.True, "listed last"); + Assert.That(CultDocumentRegistry.WrittenOwns(Listing(b, a.SchemaId, b.SchemaId), a.SchemaId, catalog), Is.True, "listed first"); + Assert.That(CultDocumentRegistry.WrittenOwns(Listing(b, b.SchemaId, "tests.wo.other"), a.SchemaId, catalog), Is.False, "lists another id"); + Assert.That(CultDocumentRegistry.WrittenOwns(a, a.SchemaId, Array.Empty()), Is.True, "declared needs no catalog"); + } + + [Test] + public void WrittenOwnsAnOlderIdOfItsOwnNameOnlyWhenItReadsTheRecordWhole() + { + var n1 = EntryOf(); + var n2 = EntryOf(); + var wide = EntryOf(); + var renamed = EntryOf(); + + Assert.That(CultDocumentRegistry.WrittenOwns(n2, n1.SchemaId, new[] { n1 }), Is.True, "same name, reads whole"); + Assert.That(CultDocumentRegistry.WrittenOwns(renamed, n1.SchemaId, new[] { n1 }), Is.False, "the same shape under another name"); + Assert.That(CultDocumentRegistry.WrittenOwns(n2, wide.SchemaId, new[] { wide }), Is.False, "same name, drops a persisted slot"); + Assert.That(CultDocumentRegistry.WrittenOwns(n2, "tests.wo.unpublished", new[] { n1 }), Is.False, "an id no catalog entry publishes"); + Assert.That(CultDocumentRegistry.WrittenOwns(n2, n1.SchemaId, new[] { n1, wide }), Is.True, "another entry in the catalog changes nothing"); + } + } +} From 0b147ea51993d7e78f9df5afe2abe73709d0bb56 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 28 Oct 2026 02:43:27 +0100 Subject: [PATCH 03/88] ForeignRecordTests, VariantTests: retype a key by removal and add, and a late type writes its own record Co-Authored-By: Claude Sonnet 5.5 --- .../ForeignRecordTests.cs | 61 +++++++++++-------- tests/GameCult.Caching.Tests/VariantTests.cs | 4 +- 2 files changed, 38 insertions(+), 27 deletions(-) diff --git a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs index ab4cb046..37ea54e4 100644 --- a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs +++ b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs @@ -67,6 +67,13 @@ namespace GameCult.Caching.Tests private static CultCache Open(string path, CultDocumentRegistry registry, bool directory) => CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry, UseDirectoryStore = directory }); + // The other writer retypes a key: a removal and an add in separate commits (variants:ruling:cache-refuses-retype). + private static void Retype(CultCache writer, Type type, object document, CultRecordKey key) + { + Assert.That(writer.Commit(batch => batch.Remove(key)), Is.True); + Assert.That(writer.Commit(batch => batch.Upsert(type, document, key)), Is.True); + } + private static object DeckOf(string name) => New(Deck, ("Name", name)); private static object WidgetOf(string label) => New(Widget, ("Label", label)); @@ -205,7 +212,7 @@ namespace GameCult.Caching.Tests var path = Seeded("replaced.cc", directory); using var cache = Open(path, DeckOnly, directory); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + Retype(other, Widget, WidgetOf("now a widget"), D); var before = Fingerprint(path); var refusal = Assert.Throws(() => Land(cache, write, D, DeckOf(Canary)))!; @@ -228,7 +235,7 @@ namespace GameCult.Caching.Tests var path = Seeded("unstuck.cc", directory); using var cache = Open(path, DeckOnly, directory); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + Retype(other, Widget, WidgetOf("now a widget"), D); var widget = Stored(path, D); Assert.Throws(() => Land(cache, refused, D, DeckOf(Canary))); @@ -249,7 +256,7 @@ namespace GameCult.Caching.Tests var path = Seeded("not-served.cc", directory); using var cache = Open(path, DeckOnly, directory); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + Retype(other, Widget, WidgetOf("now a widget"), D); var refusal = Assert.Throws(() => Land(cache, refused, D, DeckOf(Canary)))!; Assert.That(refusal.Message, Does.StartWith("Record 'd' (schema id '" + WidgetId + "'"), "one refused key reads as one record"); @@ -332,7 +339,7 @@ namespace GameCult.Caching.Tests var path = Seeded("untouched.cc", directory); using var cache = Open(path, DeckOnly, directory); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + Retype(other, Widget, WidgetOf("now a widget"), D); var widget = Stored(path, D); Land(cache, write, E, DeckOf("e")); @@ -356,7 +363,7 @@ namespace GameCult.Caching.Tests var path = Seeded("restamp.cc", directory); using var cache = Open(path, DeckOnly, directory); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + Retype(other, Widget, WidgetOf("now a widget"), D); Land(cache, write, E, DeckOf("e")); @@ -374,7 +381,7 @@ namespace GameCult.Caching.Tests var path = Seeded("stale.cc", directory); using var cache = Open(path, DeckOnly, directory); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + Retype(other, Widget, WidgetOf("now a widget"), D); var widget = Stored(path, D); Assert.Throws(() => Land(cache, Write.Commit, D, DeckOf(Canary))); @@ -392,7 +399,7 @@ namespace GameCult.Caching.Tests var path = Seeded("staged-commit.cc", directory); using var cache = Open(path, DeckOnly, directory); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + Retype(other, Widget, WidgetOf("now a widget"), D); var widget = Stored(path, D); cache.UpsertAsync(Deck, DeckOf(Canary), D).GetAwaiter().GetResult(); @@ -413,7 +420,7 @@ namespace GameCult.Caching.Tests using var cache = Open(path, DeckOnly, directory); Land(cache, landed, D, DeckOf("d again")); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + Retype(other, Widget, WidgetOf("now a widget"), D); var widget = Stored(path, D); Land(cache, Write.Commit, E, DeckOf("e")); @@ -431,7 +438,7 @@ namespace GameCult.Caching.Tests cache.UpsertAsync(Deck, DeckOf("d staged"), D).GetAwaiter().GetResult(); Land(cache, Write.Commit, E, DeckOf("e")); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + Retype(other, Widget, WidgetOf("now a widget"), D); var widget = Stored(path, D); Land(cache, Write.Commit, new CultRecordKey("e2"), DeckOf("e2")); @@ -448,7 +455,7 @@ namespace GameCult.Caching.Tests var path = Seeded("staged-removal.cc", directory); using var cache = Open(path, DeckOnly, directory); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + Retype(other, Widget, WidgetOf("now a widget"), D); var widget = Stored(path, D); cache.Remove(D); @@ -468,7 +475,7 @@ namespace GameCult.Caching.Tests var path = Seeded("pending.cc", directory); using var cache = Open(path, DeckOnly, directory); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + Retype(other, Widget, WidgetOf("now a widget"), D); cache.UpsertAsync(Deck, DeckOf(Canary), D).GetAwaiter().GetResult(); Assert.Throws(() => cache.FlushAsync().GetAwaiter().GetResult()); @@ -478,7 +485,7 @@ namespace GameCult.Caching.Tests cache.UpsertAsync(Deck, DeckOf(Canary), E).GetAwaiter().GetResult(); cache.UpsertAsync(Deck, DeckOf("v"), V).GetAwaiter().GetResult(); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), E)); + Retype(other, Widget, WidgetOf("now a widget"), E); Assert.Throws(() => cache.FlushAsync().GetAwaiter().GetResult()); Assert.That(cache.IsDirty, Is.True, "v is still staged"); cache.FlushAsync().GetAwaiter().GetResult(); @@ -495,7 +502,7 @@ namespace GameCult.Caching.Tests using var cache = Open(path, DeckOnly, directory); cache.Commit(batch => batch.Upsert(Deck, DeckOf("e"), E)); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + Retype(other, Widget, WidgetOf("now a widget"), D); var before = Fingerprint(path); // A condition keeps the single-file commit on the file as it is, so only the removal reaches d. @@ -516,7 +523,7 @@ namespace GameCult.Caching.Tests var path = Seeded("pulled.cc", directory); using var cache = Open(path, DeckOnly, directory); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + Retype(other, Widget, WidgetOf("now a widget"), D); cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); @@ -562,7 +569,7 @@ namespace GameCult.Caching.Tests using var cache = Open(path, DeckOnly, directory); var z = new CultRecordKey("z"); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("z"), z)); + Retype(other, Widget, WidgetOf("z"), z); var before = Stored(path, z); Land(cache, Write.Flush, E, DeckOf("e")); @@ -803,8 +810,9 @@ namespace GameCult.Caching.Tests Assert.That(EmittedDocumentTypes.Read(reopened.Get(D)!, "Name"), Is.EqualTo("claimed")); } - // Whether a record is read-only is read live from the registry at the write, not fixed at load: a type registered after the - // load that declares the id makes the record writable, and the write stores it under the writing type's id, not the declarer's. + // Whether a record is writable is decided at the write, not fixed at load, and by the writing type alone: a type registered + // after the load that declares the id writes the record and stores it under its own id, and a registry holding it does not make + // another type the record's writer. [Test] public void ATypeRegisteredAfterLoadThatDeclaresTheIdMakesItWritable() { @@ -815,11 +823,12 @@ namespace GameCult.Caching.Tests { Assert.Throws(() => cache.Commit(batch => batch.Upsert(Deck, DeckOf("refused"), D))); registry.GetRequired(LateClaimingDeck); - cache.Commit(batch => batch.Upsert(Deck, DeckOf("claimed"), D)); + Assert.Throws(() => cache.Commit(batch => batch.Upsert(Deck, DeckOf("still refused"), D)), "the registry's other type does not make the record the writing type's"); + cache.Commit(batch => batch.Upsert(LateClaimingDeck, New(LateClaimingDeck, ("Name", "claimed")), D)); } - Assert.That(Read(path).Records.Single().SchemaId, Is.EqualTo(registry.GetRequired(Deck).SchemaId)); - using var reopened = Open(path, DeckOnly, directory: false); + Assert.That(Read(path).Records.Single().SchemaId, Is.EqualTo(registry.GetRequired(LateClaimingDeck).SchemaId)); + using var reopened = Open(path, CultDocumentRegistry.ForTypes(new[] { LateClaimingDeck }), directory: false); Assert.That(EmittedDocumentTypes.Read(reopened.Get(D)!, "Name"), Is.EqualTo("claimed")); } @@ -1132,7 +1141,7 @@ namespace GameCult.Caching.Tests a.UpsertAsync(Deck, DeckOf("e"), E).GetAwaiter().GetResult(); Assert.That(a.Remove(W), Is.False); using (var b = Open(path, Full, directory)) - b.Commit(batch => batch.Upsert(Deck, DeckOf("theirs"), W)); + Retype(b, Deck, DeckOf("theirs"), W); var theirs = Stored(path, W); Assert.DoesNotThrow(() => a.FlushAsync().GetAwaiter().GetResult()); @@ -1140,7 +1149,7 @@ namespace GameCult.Caching.Tests Assert.That(Read(path).Records.Select(record => record.Key), Is.EquivalentTo(new[] { "d", "e", "w" }), "the staged write landed beside the checked key"); Assert.That(Stored(path, W), Is.EqualTo(theirs), "the record at the checked key is still the other writer's bytes"); using (var c = Open(path, Full, directory)) - c.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), W)); + Retype(c, Widget, WidgetOf("now a widget"), W); Assert.DoesNotThrow(() => a.FlushAsync().GetAwaiter().GetResult(), "a flush clears what it judged: the next one has nothing to judge"); } @@ -1181,7 +1190,7 @@ namespace GameCult.Caching.Tests Assert.That(StoreOf(a).ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "w" })); // The writer declares the widget's id, so it may replace the widget with a deck record under an id a reads but does not declare. using (var other = Open(path, CultDocumentRegistry.ForTypes(new[] { MovingDeck, Widget }), directory)) - Assert.That(other.Commit(batch => batch.Upsert(MovingDeck, New(MovingDeck, ("Name", "moved")), W)), Is.True); + Retype(other, MovingDeck, New(MovingDeck, ("Name", "moved")), W); Assert.That(a.Remove(W), Is.False); var refusal = Assert.Throws(() => a.FlushAsync().GetAwaiter().GetResult())!; @@ -1293,7 +1302,7 @@ namespace GameCult.Caching.Tests else cache.FlushAsync().GetAwaiter().GetResult(); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), staged)); + Retype(other, Widget, WidgetOf("now a widget"), staged); Assert.DoesNotThrow(() => cache.FlushAsync().GetAwaiter().GetResult()); } @@ -1371,7 +1380,7 @@ namespace GameCult.Caching.Tests using var cache = Open(path, DeckOnly, directory); Assert.That(StoreOf(cache).ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "w" })); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Deck, DeckOf("theirs"), W)); + Retype(other, Deck, DeckOf("theirs"), W); var theirs = Stored(path, W); Assert.That(cache.Commit(batch => batch.Remove(W)), Is.True); @@ -1455,7 +1464,7 @@ namespace GameCult.Caching.Tests using var _ = cache; Assert.That(cache.BackingStores[1].ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "w" })); using (var late = Open(other, CultDocumentRegistry.ForTypes(new[] { Gizmo, Widget }), directory)) - late.Commit(batch => batch.Upsert(Gizmo, New(Gizmo, ("Tag", "theirs")), W)); + Retype(late, Gizmo, New(Gizmo, ("Tag", "theirs")), W); var theirs = Stored(other, W); Assert.That(cache.Commit(batch => diff --git a/tests/GameCult.Caching.Tests/VariantTests.cs b/tests/GameCult.Caching.Tests/VariantTests.cs index b8c1fb3d..208abd70 100644 --- a/tests/GameCult.Caching.Tests/VariantTests.cs +++ b/tests/GameCult.Caching.Tests/VariantTests.cs @@ -251,7 +251,9 @@ namespace GameCult.Caching.Tests cache.Commit(batch => batch.Upsert(typeof(VariantOther), new VariantOther { Name = "other" }, new CultRecordKey("other"))); var rebase = Refused(() => cache.Commit(batch => batch.UpsertVariant(BigKey, new CultRecordKey("other")))); - Assert.That(rebase.Message, Does.Contain(BigKey.Value).And.Contain("VariantOther")); + // Held in a store, the key's record is a VariantGear and the write is a VariantOther: the ownership rule refuses it first + // (variants:ruling:cache-refuses-retype), so this refusal no longer names the types. + Assert.That(rebase.Message, Does.Contain(BigKey.Value)); } // A store whose variant record names another type than its base's: the loader refuses it naming both keys. From d1032b1617dfb188e94f15d214e5c46f28ed9a0c Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 28 Oct 2026 02:45:17 +0100 Subject: [PATCH 04/88] WriteOwnershipTests: judge a stale two-key batch per key; the unattached store cannot carry upserts Co-Authored-By: Claude Sonnet 5.5 --- .../ForeignRecordTests.cs | 9 ++- tests/GameCult.Caching.Tests/VariantTests.cs | 2 +- .../WriteOwnershipTests.cs | 69 ++++++++----------- 3 files changed, 33 insertions(+), 47 deletions(-) diff --git a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs index 37ea54e4..4b25c791 100644 --- a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs +++ b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs @@ -294,11 +294,10 @@ namespace GameCult.Caching.Tests }); using var cache = Open(path, DeckOnly, directory); using (var other = Open(path, Full, directory)) - other.Commit(batch => - { - batch.Upsert(Widget, WidgetOf("w1"), D); - batch.Upsert(Widget, WidgetOf("w2"), d2); - }); + { + Retype(other, Widget, WidgetOf("w1"), D); + Retype(other, Widget, WidgetOf("w2"), d2); + } var refusal = Assert.Throws(() => { diff --git a/tests/GameCult.Caching.Tests/VariantTests.cs b/tests/GameCult.Caching.Tests/VariantTests.cs index 208abd70..72f32d1e 100644 --- a/tests/GameCult.Caching.Tests/VariantTests.cs +++ b/tests/GameCult.Caching.Tests/VariantTests.cs @@ -250,7 +250,7 @@ namespace GameCult.Caching.Tests SeedBig(cache); cache.Commit(batch => batch.Upsert(typeof(VariantOther), new VariantOther { Name = "other" }, new CultRecordKey("other"))); - var rebase = Refused(() => cache.Commit(batch => batch.UpsertVariant(BigKey, new CultRecordKey("other")))); + var rebase = Assert.Throws(() => cache.Commit(batch => batch.UpsertVariant(BigKey, new CultRecordKey("other"))))!; // Held in a store, the key's record is a VariantGear and the write is a VariantOther: the ownership rule refuses it first // (variants:ruling:cache-refuses-retype), so this refusal no longer names the types. Assert.That(rebase.Message, Does.Contain(BigKey.Value)); diff --git a/tests/GameCult.Caching.Tests/WriteOwnershipTests.cs b/tests/GameCult.Caching.Tests/WriteOwnershipTests.cs index 5dab91e3..1a51db1e 100644 --- a/tests/GameCult.Caching.Tests/WriteOwnershipTests.cs +++ b/tests/GameCult.Caching.Tests/WriteOwnershipTests.cs @@ -193,52 +193,39 @@ namespace GameCult.Caching.Tests Assert.That(TypeAt(path, false, ABRegistry, K1), Is.EqualTo(typeof(WoA))); } - private static CultStoredDocument Doc(CultDocumentRegistry registry, CultRecordKey key, T document) where T : class => - new(key, "2026-10-28T00:00:00.0000000+00:00", registry.GetRequired(), document); - - // The judge is per key: an unattached PushAll that stages A at B's key and B at A's key is refused, though together the writes - // declare both ids. - [Test] - public void AnUnattachedPushAllJudgesEachKeyByItsOwnWrittenType() + // The judge is per key, and an answer for one written type is never reused for another. Another writer left A's record at two + // keys; this stale cache holds A at one and B at the other and writes both. The key A owns lands, the key where B meets A's + // record is the one refused, whichever is judged first. + [TestCase("a-own", "b-bad", true, false)] + [TestCase("a-own", "b-bad", false, false)] + [TestCase("b-own", "a-bad", true, true)] + [TestCase("b-own", "a-bad", false, true)] + public void AStaleBatchIsJudgedPerKeyByEachKeysWrittenType(string ownKey, string badKey, bool ownFirst, bool directory) { - var path = SeedAAtK1BAtK2(directory: false); - var before = Fingerprint(path); - var swapped = new SingleFileMessagePackBackingStore(path); - swapped.Push(Doc(ABRegistry, K1, new WoB { Name = "k1", Value = 7 })); - swapped.Push(Doc(ABRegistry, K2, new WoA { Name = "k2", Value = 7 })); - - Assert.Throws(() => swapped.PushAll()); - Assert.That(Fingerprint(path), Is.EqualTo(before)); - - var own = new SingleFileMessagePackBackingStore(path); - own.Push(Doc(ABRegistry, K1, new WoA { Name = "k1", Value = 7 })); - own.Push(Doc(ABRegistry, K2, new WoB { Name = "k2", Value = 7 })); - own.PushAll(); - Assert.That(Fingerprint(path), Is.Not.EqualTo(before)); - } - - // An answer for one written type is never reused for another: two records stored under A's id, A written at one key and B at - // the other, in either staging order and either key order. - [TestCase("a-own", "b-bad", true)] - [TestCase("a-own", "b-bad", false)] - [TestCase("b-own", "a-bad", true)] - [TestCase("b-own", "a-bad", false)] - public void AnAnswerForOneWrittenTypeIsNeverReusedForAnother(string ownKey, string badKey, bool ownFirst) - { - var path = PathFor(directory: false); - using (var seed = Open(path, false, ABRegistry)) + var path = PathFor(directory); + var own = new CultRecordKey(ownKey); + var bad = new CultRecordKey(badKey); + using (var seed = Open(path, directory, ABRegistry)) Assert.That(seed.Commit(batch => { - batch.Upsert(new WoA { Name = ownKey, Value = 1 }, new CultRecordHandle(new CultRecordKey(ownKey))); - batch.Upsert(new WoA { Name = badKey, Value = 1 }, new CultRecordHandle(new CultRecordKey(badKey))); + batch.Upsert(new WoA { Name = ownKey, Value = 1 }, new CultRecordHandle(own)); + batch.Upsert(new WoB { Name = badKey, Value = 1 }, new CultRecordHandle(bad)); }), Is.True); - var before = Fingerprint(path); - var store = new SingleFileMessagePackBackingStore(path); - void PushOwn() => store.Push(Doc(ABRegistry, new CultRecordKey(ownKey), new WoA { Name = ownKey, Value = 2 })); - void PushBad() => store.Push(Doc(ABRegistry, new CultRecordKey(badKey), new WoB { Name = badKey, Value = 2 })); - if (ownFirst) { PushOwn(); PushBad(); } else { PushBad(); PushOwn(); } + using var stale = Open(path, directory, ABRegistry); + using (var other = Open(path, directory, ABRegistry)) + { + Assert.That(other.Commit(batch => batch.Remove(bad)), Is.True); + Assert.That(other.Commit(batch => batch.Upsert(new WoA { Name = badKey, Value = 2 }, new CultRecordHandle(bad))), Is.True); + } - var refusal = Assert.Throws(() => store.PushAll())!; + var before = Fingerprint(path); + void StageOwn(CultCacheBatch batch) => batch.Upsert(new WoA { Name = ownKey, Value = 3 }, new CultRecordHandle(own)); + void StageBad(CultCacheBatch batch) => batch.Upsert(new WoB { Name = badKey, Value = 3 }, new CultRecordHandle(bad)); + + var refusal = Assert.Throws(() => stale.Commit(batch => + { + if (ownFirst) { StageOwn(batch); StageBad(batch); } else { StageBad(batch); StageOwn(batch); } + }))!; Assert.That(refusal.RecordKey, Is.EqualTo(badKey)); Assert.That(refusal.Message, Does.Not.Contain($"'{ownKey}'"), "the key A owns is not among the refused"); From cf9f8dc0e11be9caed70cd88784617d40b7f15d9 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 28 Oct 2026 04:31:46 +0100 Subject: [PATCH 05/88] WriteOwnershipTests: a batch removal of a staged key is judged as a removal Co-Authored-By: Claude Sonnet 5.5 --- .../WriteOwnershipTests.cs | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/GameCult.Caching.Tests/WriteOwnershipTests.cs b/tests/GameCult.Caching.Tests/WriteOwnershipTests.cs index 1a51db1e..105fc615 100644 --- a/tests/GameCult.Caching.Tests/WriteOwnershipTests.cs +++ b/tests/GameCult.Caching.Tests/WriteOwnershipTests.cs @@ -231,5 +231,26 @@ namespace GameCult.Caching.Tests Assert.That(refusal.Message, Does.Not.Contain($"'{ownKey}'"), "the key A owns is not among the refused"); Assert.That(Fingerprint(path), Is.EqualTo(before)); } + + // A batch that removes a key is a removal of it, whatever this cache has staged there: the removal rule judges it, so a key + // another writer retyped can still be removed by the cache that staged a write of the old type. + [TestCaseSource(nameof(StoreKinds))] + public void ABatchRemovalOfAStagedKeyIsJudgedAsARemoval(bool directory) + { + var path = SeedAAtK1BAtK2(directory); + using var stale = Open(path, directory, ABRegistry); + stale.UpsertAsync(typeof(WoA), new WoA { Name = "k1", Value = 9 }, K1).GetAwaiter().GetResult(); + using (var other = Open(path, directory, ABRegistry)) + { + Assert.That(other.Commit(batch => batch.Remove(K1)), Is.True); + Assert.That(other.Commit(batch => batch.Upsert(new WoB { Name = "k1", Value = 5 }, new CultRecordHandle(K1))), Is.True); + } + + Assert.That(stale.Commit(batch => batch.Remove(K1)), Is.True); + + using var reopened = Open(path, directory, ABRegistry); + Assert.That(reopened.GetStored(K1), Is.Null); + Assert.That(reopened.GetStored(K2)!.Descriptor.DocumentType, Is.EqualTo(typeof(WoB)), "the bystander is untouched"); + } } } From e163e6b006904994e8d5c98b42c931241f46d2da Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 28 Oct 2026 05:11:09 +0100 Subject: [PATCH 06/88] Retype in one batch or staged apart, and a store-less cache refuses it too A removal followed by an add at the same key is a retype (variants:ruling:cache-refuses-retype): the batch's staged add records that it replaces, the key is judged as the removal it names, and the record lands. The store carries the same fact for removals the cache staged apart (Remove then UpsertAsync), so the flush no longer reads the add as an upsert over the record it removed and discards it. A cache with no backing store judges the upsert against the record it holds. The variant plan's rebase-type check is deleted: write ownership refuses that retype first. The StoredPublishedBy staleness guard is deleted (a loaded record is never relabelled in place). Protected RefuseUndeclaredOnDisk doc states the per-key rule. Co-Authored-By: Claude Opus 5.5 --- .../DirectoryMessagePackBackingStore.cs | 16 +-- .../Contracts/cultcache-persistence-format.md | 2 +- src/GameCult.Caching/CultCache.cs | 117 ++++++++++++------ unity/org.gamecult.cultlib/CHANGELOG.md | 2 +- 4 files changed, 91 insertions(+), 46 deletions(-) diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index 9d668909..cd51e3f9 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -189,7 +189,8 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore manifest, request.Upserts.Concat(request.Deletes).Select(entry => entry.Key.Value).Concat(request.Checks).Concat(StagedKeys), request.Upserts, - request.Deletes); + request.Deletes, + request.Checks); if (request.Upserts.Count == 0 && request.Deletes.Count == 0 && !IsDirty) { // Nothing to write: the batch only judged keys it does not hold. @@ -247,7 +248,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore Directory.CreateDirectory(_manifestFile.DirectoryName!); using var commitLease = AcquireCommitLease(wait: true, create: true); var (manifest, catalogBytes) = ReadManifest(); - RefuseUndeclaredOnDisk(manifest, StagedKeys, Array.Empty(), Array.Empty()); + RefuseUndeclaredOnDisk(manifest, StagedKeys, Array.Empty(), Array.Empty(), Array.Empty()); if (IsDirty) WriteGeneration(manifest, catalogBytes); else @@ -279,7 +280,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore using var commitLease = AcquireCommitLease(wait, create: true); if (commitLease == null) return CultCommitOutcome.Contended; - RefuseUndeclaredOnDisk(ReadManifest().Snapshot, keys, Array.Empty(), Array.Empty()); + RefuseUndeclaredOnDisk(ReadManifest().Snapshot, keys, Array.Empty(), Array.Empty(), Array.Empty()); return CultCommitOutcome.Committed; }); } @@ -293,15 +294,16 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore // store shares (CacheBackingStore.UndeclaredAt, CacheBackingStore.RefuseUndeclaredOnDisk). The refused records reload as a // pull loads them. A key is owned by the entry the write puts there: the batch's upsert, or the staged document. A key it // removes (the batch's deletes, the staged deletes) or only checks goes to the registry's removal rule. - private void RefuseUndeclaredOnDisk(CultPersistedStoreSnapshot currentManifest, IEnumerable written, IReadOnlyList upserts, IReadOnlyList deletes) + private void RefuseUndeclaredOnDisk(CultPersistedStoreSnapshot currentManifest, IEnumerable written, IReadOnlyList upserts, IReadOnlyList deletes, IEnumerable checks) { CultPersistedRecord? HeldAt(string key) => currentManifest.Records.FirstOrDefault(record => string.Equals(record.Key, key, StringComparison.Ordinal)); var batch = upserts.ToDictionary(entry => entry.Key.Value, entry => entry.Descriptor, StringComparer.Ordinal); - var removed = deletes.Select(entry => entry.Key.Value).ToHashSet(StringComparer.Ordinal); + var removed = deletes.Select(entry => entry.Key.Value).Concat(checks).Concat(StagedRemovals).ToHashSet(StringComparer.Ordinal); CultSchemaCatalogEntry? WrittenAt(string key) => - batch.TryGetValue(key, out var descriptor) ? descriptor.ToCatalogEntry() - : !removed.Contains(key) && _dirtyKeys.ContainsKey(key) && Entries.TryGetValue(key, out var staged) ? staged.Descriptor.ToCatalogEntry() + removed.Contains(key) ? null + : batch.TryGetValue(key, out var descriptor) ? descriptor.ToCatalogEntry() + : _dirtyKeys.ContainsKey(key) && Entries.TryGetValue(key, out var staged) ? staged.Descriptor.ToCatalogEntry() : null; RefuseUndeclaredOnDisk( written, diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index 3a6391d8..6548e8d9 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -213,7 +213,7 @@ compatible), or when it is the type that reads the stored record's published ent id of its own schema name that it reads whole is its own and is restamped. Anything else is refused, another registered type's record included, and so is a record the written type would read only by dropping persisted slots. No registry enters the decision, `Shared` included: the answer for a given written entry, stored id and catalog is the same lent a registry or not, and a cache's registered types do not make -each other's records writable. Retyping a key is a removal and an add, in separate commits. A removal or check of a key goes to the +each other's records writable. Retyping a key is a removal and an add: in one batch (the removal called before the add) or staged apart, with or without a backing store. The removal half is judged as a removal, so the key is retyped only when the removal rule allows it. A removal or check of a key goes to the registry's removal rule (a record a registered type declares, or reads whole, may be removed), and a cacheless call, which has no registry, removes a record only when an entry it writes owns it; a call that writes no entry removes nothing. diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 6ce2e9dd..5ee82e06 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -524,11 +524,11 @@ namespace GameCult.Caching // asked. internal bool StoredSchemaReadWhole { get; set; } - // The catalog entry that publishes StoredSchemaId in the store this record was loaded from, with the id it was published for, - // so staging can ask WrittenOwns the question the store's judge asks, over the catalog the record came with. Null for a record - // this cache constructed or wrote, which is stored under Descriptor's own id and published by Descriptor's entry. A reader - // uses it only while the id still matches StoredSchemaId. - internal (string Id, CultSchemaCatalogEntry Entry)? StoredPublishedBy { get; set; } + // The catalog entry that publishes StoredSchemaId in the store this record was loaded from, so staging can ask WrittenOwns the + // question the store's judge asks, over the catalog the record came with. Null for a record this cache constructed or wrote, + // which is stored under Descriptor's own id and published by Descriptor's entry. Only a loaded record carries it, and a loaded + // record is replaced by a new one when written, never relabelled in place, so it always names the id StoredSchemaId holds. + internal CultSchemaCatalogEntry? StoredPublishedBy { get; set; } // Names the bytes this record was read from or written as: the SHA-256 of its persisted encoding, set by the store that // read or wrote them. A record no store has read or written has a version of its own that equals nothing else. @@ -2075,14 +2075,19 @@ namespace GameCult.Caching // One staged write: a plain record (Stored), a variant (Variant), a flatten, or, with none of those, a removal. internal sealed class StagedOperation { - public StagedOperation(CultRecordKey key, CultStoredDocument? stored, CultVariantDelta? variant, bool flatten) + public StagedOperation(CultRecordKey key, CultStoredDocument? stored, CultVariantDelta? variant, bool flatten, bool replaces = false) { Key = key; Stored = stored; Variant = variant; Flatten = flatten; + Replaces = replaces; } + // True for a plain record staged at a key this batch removed first: a removal and an add (variants:ruling:cache-refuses-retype). + // The key is judged as the removal it names, then the record lands at it. + public bool Replaces { get; } + public CultRecordKey Key { get; } public CultStoredDocument? Stored { get; } public CultVariantDelta? Variant { get; } @@ -2156,7 +2161,8 @@ namespace GameCult.Caching { ThrowIfSealed(); var stored = _cache.CreateStoredDocument(document, key); - Operations[stored.Key.Value] = new StagedOperation(stored.Key, stored, null, false); + var replaces = Operations.TryGetValue(stored.Key.Value, out var previous) && (previous.IsRemoval || previous.Replaces); + Operations[stored.Key.Value] = new StagedOperation(stored.Key, stored, null, false, replaces); return stored.Key; } @@ -2556,6 +2562,7 @@ namespace GameCult.Caching Admit(Array.Empty(), new[] { existing }, null, (home, _) => { home?.Delete(existing); + home?.NoteRemoved(existing.Key.Value); return CultCommitOutcome.Committed; }); return true; @@ -2826,15 +2833,18 @@ namespace GameCult.Caching : StampFlatten(op.Key)) .ToArray(); // A removal of a key the cache does not hold removes nothing: it is a check every store the commit reaches judges under its lock. + // So is the removal half of a retype (a record staged at a key the batch removed first, held or not): the store judges the + // key as the removal it names, and the record then lands at it. + var replaced = batch.Operations.Values.Where(op => op.Replaces).Select(op => op.Key.Value).ToHashSet(StringComparer.Ordinal); var checks = batch.Operations.Values - .Where(op => op.IsRemoval && !_entries.ContainsKey(op.Key.Value)) + .Where(op => op.Replaces || (op.IsRemoval && !_entries.ContainsKey(op.Key.Value))) .Select(op => op.Key.Value) .ToArray(); var deletes = batch.Operations.Values .Where(op => op.IsRemoval && _entries.ContainsKey(op.Key.Value)) .Select(op => _entries[op.Key.Value]) .ToArray(); - return Admit(upserts, deletes, null, (home, resolved) => + return Admit(upserts, deletes, null, replaced: replaced, land: (home, resolved) => { // A batch that only checks keys names no home; it commits through the cache's one writable store, if it has one. if (home == null && checks.Length > 0 && _stores.Where(entry => !entry.Store.IsReadOnly).Take(2).ToArray() is { Length: 1 } only) @@ -2884,11 +2894,12 @@ namespace GameCult.Caching IReadOnlyList admitted, IReadOnlyList evicted, CacheBackingStore? source, - Func, CultCommitOutcome> land) => Held(() => + Func, CultCommitOutcome> land, + HashSet? replaced = null) => Held(() => { if (_held == null) throw new InvalidOperationException("An admission reached the cache under a plain lock on its gate; every admission runs in a hold."); - var home = Validate(admitted, evicted, source); + var home = Validate(admitted, evicted, source, replaced); // Ids are decided before anything lands, and the caller's documents change only inside this admission: a plan that // refuses (a duplicate, a malformed id) has changed nothing, and a write that does not commit gives its ids back. // A write mints random ids; a load of a record written before ids existed mints the ids its reload will mint @@ -3096,7 +3107,8 @@ namespace GameCult.Caching private CacheBackingStore? Validate( IReadOnlyList admitted, IReadOnlyList evicted, - CacheBackingStore? source) + CacheBackingStore? source, + HashSet? replaced) { var homes = new List(); var globals = new Dictionary(_globals); @@ -3113,7 +3125,7 @@ namespace GameCult.Caching { var descriptor = stored.Descriptor; if (source == null) - Claim(stored, upsert: true); + Claim(stored, upsert: replaced?.Contains(stored.Key.Value) != true); else if (Home(descriptor.DocumentType) is var home && home != source) throw new InvalidOperationException( $"{descriptor.SchemaName} record {stored.Key.Value} was loaded from {source} but its home is {home?.ToString() ?? "no store"}."); @@ -3148,6 +3160,9 @@ namespace GameCult.Caching { if (_stores.Count > 0) throw new InvalidOperationException($"No backing store is home to {stored.Descriptor.SchemaName}."); + // No store, no file to judge, but the same rule: a record is replaced only by its own type's write. + if (upsert && _entries.TryGetValue(stored.Key.Value, out var inMemory) && !CacheBackingStore.UpsertOwned(inMemory, stored.Descriptor)) + throw CacheBackingStore.Retyped(inMemory, stored.Key.Value, stored.Descriptor); return; } @@ -3265,6 +3280,7 @@ namespace GameCult.Caching // A variant is written only through the variant API: a plain record never lands on a variant's key, and the // reverse is a rebase or a flatten, never an accident. A load reports what is on disk and is not a write. + // A rebase onto a base of another type would retype the key; write ownership refused that in Claim before this runs. if (source == null) { foreach (var stored in admitted) @@ -3277,11 +3293,6 @@ namespace GameCult.Caching if (stored.Variant != null && present != null && present.Variant == null) throw new InvalidOperationException( $"Record {stored.Key.Value} is a plain record; it cannot become a variant of {stored.Variant.BaseKey} in place. Use a new key."); - if (stored.Variant != null && present?.Variant != null && - present.Descriptor.DocumentType != stored.Descriptor.DocumentType) - throw new InvalidOperationException( - $"Variant {stored.Key.Value} is a {present.Descriptor.DocumentType.Name}; " + - $"rebasing it onto {stored.Variant.BaseKey} would make it a {stored.Descriptor.DocumentType.Name}."); } } @@ -3975,8 +3986,8 @@ namespace GameCult.Caching }; } - private static (string Id, CultSchemaCatalogEntry Entry)? PublishedByIn(string schemaId, IReadOnlyCollection catalog) => - CultDocumentRegistry.PublishedBy(schemaId, catalog) is { } entry ? (schemaId, entry) : null; + private static CultSchemaCatalogEntry? PublishedByIn(string schemaId, IReadOnlyCollection catalog) => + CultDocumentRegistry.PublishedBy(schemaId, catalog); protected void SetLastSchemaMigrationReports(IEnumerable reports) { @@ -4015,15 +4026,24 @@ namespace GameCult.Caching throw Overwrites(null, new[] { (key, held.StoredSchemaId, held.Descriptor.SchemaName) }); } - private bool HeldOwned(CultStoredDocument held, CultDocumentDescriptor? written) - { - if (written == null) - return Registry.Declared(held.StoredSchemaId) || held.StoredSchemaReadWhole; - var published = held.StoredPublishedBy is { } loaded && string.Equals(loaded.Id, held.StoredSchemaId, StringComparison.Ordinal) - ? loaded.Entry - : held.Descriptor.ToCatalogEntry(); - return CultDocumentRegistry.WrittenOwns(written.ToCatalogEntry(), held.StoredSchemaId, new[] { published }); - } + private bool HeldOwned(CultStoredDocument held, CultDocumentDescriptor? written) => + written == null + ? Registry.Declared(held.StoredSchemaId) || held.StoredSchemaReadWhole + : UpsertOwned(held, written); + + // Whether a write of the written type may replace the record a cache holds: WrittenOwns over the entry that published the held + // record's id when it was loaded (a record this cache wrote is published by its own type). The one answer for a store-backed + // cache's staging and a store-less cache's. + internal static bool UpsertOwned(CultStoredDocument held, CultDocumentDescriptor written) => + CultDocumentRegistry.WrittenOwns(written.ToCatalogEntry(), held.StoredSchemaId, new[] { held.StoredPublishedBy ?? held.Descriptor.ToCatalogEntry() }); + + // The refusal of a store-less cache that would retype a key. + internal static CultSchemaConflictException Retyped(CultStoredDocument held, string key, CultDocumentDescriptor written) => new( + $"Record '{key}' is a {held.Descriptor.SchemaName} (schema id '{held.StoredSchemaId}'); writing a {written.SchemaName} there would retype the key. " + + "Remove the record first, in the same batch or an earlier commit, then add the other type's.", + held.StoredSchemaId, + new[] { held.Descriptor.SchemaName }, + key); /// /// Called when the cache removes keys it does not hold, outside a batch that commits through this store. Such a removal @@ -4090,6 +4110,17 @@ namespace GameCult.Caching internal bool HasStagedChecks => _checks.Count > 0; + // Keys this store's cache removed here since its last flush. A write of the same key after that is a removal and an add + // (variants:ruling:cache-refuses-retype): the key is judged as the removal it names, not as an upsert over the record it held. + // Cleared like the checks, by a refusal of the key or a successful flush or commit. + private readonly HashSet _removed = new(StringComparer.Ordinal); + + internal void NoteRemoved(string key) => Held(() => _removed.Add(key)); + + // Every key a removal named since the last flush (held or not), which this store judges as a removal even when a record is + // staged there again. + internal IReadOnlyCollection StagedRemovals => _removed.Concat(_checks).ToArray(); + // A stored record under an id the writer does not own, named by the catalog entry that publishes the id. Header only: no // payload is decoded. Null for a record under an owned id. protected static (string Key, string SchemaId, string SchemaName)? Undeclared( @@ -4127,9 +4158,15 @@ namespace GameCult.Caching }; } - // The apply half of rule 3, for every store kind, checked where the write lands under the store's lock. A key this write names - // whose durable record is under an id no registered type declares is refused, every such key at once, through Refuse, which - // reloads it as the store holds it: listed foreign when no type reads it, held read-only when one does. + // The apply half of rule 3, for every store kind, checked where the write lands under the store's lock. undeclaredAt is the + // store's per-key judge: for each key the write names it says whether the durable record there is one the write may not + // replace, remove or relabel. The rule it must apply is per key and per written type, never one predicate per stored id: a + // key the write puts a record at is owned only by the type written there (its own id, an id it lists, or an older id of its + // own schema name it reads whole), so another registered type's record is refused; a key the write only removes or checks + // is owned by any registered type that declares or reads it whole; and a key the cache removed and the write then adds again + // is judged as the removal it names. Every key at once is refused through Refuse, which reloads it as the store holds it: + // listed foreign when no type reads it, held read-only when one does. The shipped stores build this judge with + // UndeclaredAt, which is internal; a store kind outside the library supplies its own and passes it here. protected void RefuseUndeclaredOnDisk( IEnumerable written, Func undeclaredAt, @@ -4197,6 +4234,7 @@ namespace GameCult.Caching { unstage(key); _checks.Remove(key); + _removed.Remove(key); } foreach (var held in dropped) Entries.TryRemove(held.Key.Value, out _); @@ -4253,6 +4291,7 @@ namespace GameCult.Caching { IsDirty = false; _checks.Clear(); + _removed.Clear(); } } @@ -4773,7 +4812,7 @@ namespace GameCult.Caching using var fileLock = AcquireLock(wait); if (fileLock == null) return CultCommitOutcome.Contended; - RefuseUndeclaredOnDisk(ReadSnapshot(), keys, Array.Empty()); + RefuseUndeclaredOnDisk(ReadSnapshot(), keys, Array.Empty(), Array.Empty()); return CultCommitOutcome.Committed; }); } @@ -4783,7 +4822,8 @@ namespace GameCult.Caching RefuseUndeclaredOnDisk( durable, _staged.Concat(upserts.Select(staged => staged.Record.Key)).Concat(removals).Concat(checks).Concat(StagedChecks), - upserts); + upserts, + checks); // A write's key is owned by the entry the write puts there, whoever the writer is (WrittenOwns). A key it only removes or // checks is owned by the removal rule: a cache's registry (Owns), or, for a cacheless call that has no registry, some entry the @@ -4793,15 +4833,18 @@ namespace GameCult.Caching ? Registry.Owns : (id, catalog) => upserts.Any(staged => CultDocumentRegistry.WrittenOwns(staged.Entry, id, catalog)); - // A refused key's durable version is what this store has now read of it. - private void RefuseUndeclaredOnDisk(Durable? durable, IEnumerable written, IReadOnlyCollection upserts) + // A key the write also names as a removal or check (the request's, or one the cache removed here since its last flush) is a + // retype's removal half: it is judged by the removal rule, and the record staged there then lands. A refused key's durable + // version is what this store has now read of it. + private void RefuseUndeclaredOnDisk(Durable? durable, IEnumerable written, IReadOnlyCollection upserts, IEnumerable checks) { var keys = written.Distinct(StringComparer.Ordinal).ToArray(); var wanted = keys.ToHashSet(StringComparer.Ordinal); var held = (durable?.Snapshot.Records ?? Array.Empty()) .Where(record => wanted.Contains(record.Key)) .ToDictionary(record => record.Key, StringComparer.Ordinal); - var entries = upserts.ToDictionary(staged => staged.Record.Key, staged => staged.Entry, StringComparer.Ordinal); + var retyped = checks.Concat(StagedRemovals).ToHashSet(StringComparer.Ordinal); + var entries = upserts.Where(staged => !retyped.Contains(staged.Record.Key)).ToDictionary(staged => staged.Record.Key, staged => staged.Entry, StringComparer.Ordinal); var undeclaredAt = UndeclaredAt( key => held.TryGetValue(key, out var record) ? record : null, durable?.Snapshot.SchemaCatalog ?? Array.Empty(), diff --git a/unity/org.gamecult.cultlib/CHANGELOG.md b/unity/org.gamecult.cultlib/CHANGELOG.md index 6d8fcd71..081ee4f1 100644 --- a/unity/org.gamecult.cultlib/CHANGELOG.md +++ b/unity/org.gamecult.cultlib/CHANGELOG.md @@ -13,7 +13,7 @@ Releases as 3.0.0 (major). - A store is written in the format its content needs, and CultLib 2.0 refuses the newer ones by name. A single-file store holding a document variant is written as `cultcache.store.v2`, one holding an element id as `cultcache.store.v3`; a directory store holding an element id is written as `cultcache.store.v5` (`v4` otherwise). A store is marked only when a record actually holds an id or a variant. Upgrade readers before writers: a 2.0 reader that opens such a store is refused, not misread. - Writes are write sets. A flush or commit replaces the records it staged and copies every other record forward as stored, under the store's lock, instead of rewriting the whole store from the cache's memory. A record the cache does not know is kept, and an untouched record keeps a dropped member's slot through every write (never reuse a dropped slot; see the schema-compatibility contract). - A write or removal at a key whose stored record the writer does not own is refused with `CultSchemaConflictException`, and the file is left as it was. A writer owns a record whose schema id it declares (owns or lists compatible) and a record its type reads whole under compatible drift: that record is writable and is restamped under the type's own id on write, as in 2.0.1. A record a type reads only by dropping persisted slots stays read-only until its id is declared compatible, which accepts the loss (this differs from 2.0.1, which shed the slots silently). A cache owns through its registry; a cacheless CultMesh write owns through the catalog entry it writes. -- A write replaces the record at a key only when the type it writes there owns that record, decided per key and the same for a cache and for a cacheless `WriteDocument`, `WriteRecord` or unattached-store `PushAll`, in a single-file store and a directory store. The written type owns the stored record when its entry declares the stored id (its own id, or one it lists compatible), or when it is the type that reads the record's published entry and reads it whole (no persisted slot unread): an older id of its own schema name that it reads whole is its own and is restamped, as in 2.0.1. Anything else is refused with `CultSchemaConflictException` and the file left as it was: a record of another registered type, under its current id or an old id it lists, and a record the written type would read only by dropping persisted slots. No registry enters the decision, `CultDocumentRegistry.Shared` included, so a lent registry never changes the answer. This changes a cache: where 2.x let a cache holding two types replace one's record at a key with the other's, it now refuses; retype a key by removing it and adding the new record in separate commits. A removal or check goes to the registry's removal rule, and a cacheless call, which has no registry, removes a record only when an entry it writes owns it (a `Delete` then `PushAll` on an unattached store writes no entry and is refused). +- A write replaces the record at a key only when the type it writes there owns that record, decided per key and the same for a cache and for a cacheless `WriteDocument`, `WriteRecord` or unattached-store `PushAll`, in a single-file store and a directory store. The written type owns the stored record when its entry declares the stored id (its own id, or one it lists compatible), or when it is the type that reads the record's published entry and reads it whole (no persisted slot unread): an older id of its own schema name that it reads whole is its own and is restamped, as in 2.0.1. Anything else is refused with `CultSchemaConflictException` and the file left as it was: a record of another registered type, under its current id or an old id it lists, and a record the written type would read only by dropping persisted slots. No registry enters the decision, `CultDocumentRegistry.Shared` included, so a lent registry never changes the answer. This changes a cache: where 2.x let a cache holding two types replace one's record at a key with the other's, it now refuses, with or without a backing store. Retype a key by removing it and adding the new record, in one batch (`batch.Remove(key)` then `batch.Upsert(...)`) or staged apart (`Remove` then `UpsertAsync`); the removal is judged as a removal, so a record no registered type owns is still refused. A removal or check goes to the registry's removal rule, and a cacheless call, which has no registry, removes a record only when an entry it writes owns it (a `Delete` then `PushAll` on an unattached store writes no entry and is refused). - Registration refuses an owned object-list element type that has no `[CultElementId]` member (a public string with `[Key]`), naming the type and the list. Element ids let a write set merge a list by identity; a list of record references or plain values is a whole value and is not affected. - The abstract and protected surface for store kinds outside this library changed. On `SingleFileBackingStore`, `SerializeSnapshot` and `DeserializeSnapshot` are replaced by the abstract `SerializeStore`, `SerializeRecord`, `SerializeCatalogEntry` and `ReadStore`, and the abstract `SerializePayload(object)` becomes `SerializePayload(object document, Type documentType)`: a store encodes a document as the type it is given. On `CacheBackingStore`, `Judging` is removed, `ToStoredDocument` takes the record's version (`record, version, catalog, deserialize, out report`), and these protected members are new for a store kind to implement or call: `HeaderFor`, `HeaderForWrite`, `Foreign`, `SetForeignRecords`, `Undeclared`, `RefuseUndeclared`, `RefuseUndeclaredOnDisk`, `RefuseRemovalOfUndeclared`, `Refuse`, `SameStoredIdentity`, `VersionOf`, `StoredAtOver`, `Overwrites`, `StagedChecks`, `StageChecks` and `JudgeChecks`. `MarkFlushSucceeded` is unchanged. `CacheBackingStore.ForeignRecords` is new and public: the records a store holds under a schema id its cache does not declare. - `CultCommitRequest.Expected` is now `IReadOnlyList<(CultRecordKey Key, string? Version)>` (it was a key with a schema id and a stored-at), `CultCommitRequest.Checks` is new, and `ConditionsHold` takes `(IReadOnlyDictionary durable, IEnumerable observed, IEnumerable foreign)` in place of `(IReadOnlyCollection, IEnumerable)`. A store kind outside the library lists the record versions it holds and passes its foreign records. From abf4f0387aa937f7b56792aca22e4a164857a667 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 28 Oct 2026 05:11:09 +0100 Subject: [PATCH 07/88] WriteOwnershipTests, ForeignRecordTests, VariantTests: pin the one-batch retype, the store-less refusal, the late declarer's removal and the rebase refusal Co-Authored-By: Claude Opus 5.5 --- .../ForeignRecordTests.cs | 20 +++ tests/GameCult.Caching.Tests/VariantTests.cs | 18 ++ .../WriteOwnershipTests.cs | 156 ++++++++++++++++++ 3 files changed, 194 insertions(+) diff --git a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs index 4b25c791..61aa236e 100644 --- a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs +++ b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs @@ -831,6 +831,26 @@ namespace GameCult.Caching.Tests Assert.That(EmittedDocumentTypes.Read(reopened.Get(D)!, "Name"), Is.EqualTo("claimed")); } + // The removal rule asks the registry as it is when the removal is judged: a record read-only at load becomes removable once a + // type that declares its id is registered. The write rule is not the removal rule (the test above): the late type may remove + // the record, and a removal is judged by what the registry declares, not by the type that last wrote the key. + [Test] + public void ATypeRegisteredAfterLoadThatDeclaresTheIdMakesTheRecordRemovable() + { + var path = OtherRuntimeStore("late-remove.cc"); + var registry = CultDocumentRegistry.ForTypes(new[] { Deck }); + + using (var cache = Open(path, registry, directory: false)) + { + Assert.Throws(() => cache.Commit(batch => batch.Remove(D)), "no registered type declares the id yet"); + registry.GetRequired(LateClaimingDeck); + Assert.That(cache.Commit(batch => batch.Remove(D)), Is.True); + Assert.That(cache.Get(D), Is.Null); + } + + Assert.That(Read(path).Records, Is.Empty); + } + // A commit onto the file that involves a variant is judged on the set the file will hold. A foreign record is not decoded // for that judgement, so it neither fails the commit nor joins the set. [Test] diff --git a/tests/GameCult.Caching.Tests/VariantTests.cs b/tests/GameCult.Caching.Tests/VariantTests.cs index 72f32d1e..1bcb36a2 100644 --- a/tests/GameCult.Caching.Tests/VariantTests.cs +++ b/tests/GameCult.Caching.Tests/VariantTests.cs @@ -282,6 +282,24 @@ namespace GameCult.Caching.Tests Assert.That(load.Message, Does.Contain("other-variant").And.Contain("gear-base")); } + // Rebasing a variant onto a base of another type would retype its key: refused with or without a store, the record untouched. + [Test] + public void ARebaseOntoABaseOfAnotherTypeIsRefusedWithoutAStore() + { + using var cache = new CultCache(Registry); + SeedBase(cache); + SeedBig(cache); + var other = new CultRecordKey("other"); + cache.Commit(batch => batch.Upsert(typeof(VariantOther), new VariantOther { Name = "other" }, other)); + + var rebase = Assert.Throws(() => cache.Commit(batch => batch.UpsertVariant(BigKey, other)))!; + + Assert.That(rebase.RecordKey, Is.EqualTo(BigKey.Value)); + Assert.That(cache.GetStored(BigKey)!.Variant!.BaseKey, Is.EqualTo(BaseKey.Value)); + Assert.That(cache.GetStored(BigKey)!.Descriptor.DocumentType, Is.EqualTo(typeof(VariantGear))); + Assert.Throws(() => cache.UpsertVariantAsync(BigKey, other).GetAwaiter().GetResult()); + } + [Test] public void AVariantOfAGlobalIsRefused() { diff --git a/tests/GameCult.Caching.Tests/WriteOwnershipTests.cs b/tests/GameCult.Caching.Tests/WriteOwnershipTests.cs index 105fc615..ff975a51 100644 --- a/tests/GameCult.Caching.Tests/WriteOwnershipTests.cs +++ b/tests/GameCult.Caching.Tests/WriteOwnershipTests.cs @@ -252,5 +252,161 @@ namespace GameCult.Caching.Tests Assert.That(reopened.GetStored(K1), Is.Null); Assert.That(reopened.GetStored(K2)!.Descriptor.DocumentType, Is.EqualTo(typeof(WoB)), "the bystander is untouched"); } + + // The ruling's retype: one batch holds the removal and the add, in the order they were called. The removal half is judged as a + // removal (a registered type's record is removable), and the record lands at the key. + [TestCaseSource(nameof(StoreKinds))] + public void ARetypeIsOneBatchHoldingTheRemovalAndTheAdd(bool directory) + { + var path = SeedAAtK1BAtK2(directory); + + using (var cache = Open(path, directory, ABRegistry)) + { + Assert.That(cache.Commit(batch => + { + batch.Remove(K2); + batch.Upsert(new WoA { Name = "k2", Value = 9 }, new CultRecordHandle(K2)); + }), Is.True); + Assert.That(cache.GetStored(K2)!.Descriptor.DocumentType, Is.EqualTo(typeof(WoA))); + } + + Assert.That(TypeAt(path, directory, ABRegistry, K2), Is.EqualTo(typeof(WoA))); + Assert.That(TypeAt(path, directory, ABRegistry, K1), Is.EqualTo(typeof(WoA)), "the bystander is untouched"); + } + + // The order is the meaning: a removal called after the add leaves the key removed. + [TestCaseSource(nameof(StoreKinds))] + public void ARemovalCalledAfterTheAddLeavesTheKeyRemoved(bool directory) + { + var path = SeedAAtK1BAtK2(directory); + + using (var cache = Open(path, directory, ABRegistry)) + { + Assert.That(cache.Commit(batch => + { + batch.Remove(K2); + batch.Upsert(new WoA { Name = "k2", Value = 9 }, new CultRecordHandle(K2)); + batch.Remove(K2); + }), Is.True); + Assert.That(cache.GetStored(K2), Is.Null); + } + + using var reopened = Open(path, directory, ABRegistry); + Assert.That(reopened.GetStored(K2), Is.Null); + } + + // The removal half is a removal and nothing more: a record no registered type owns is not replaced by adding after its removal. + [TestCaseSource(nameof(StoreKinds))] + public void ARetypeOfARecordNoRegisteredTypeOwnsIsStillRefused(bool directory) + { + var path = SeedAAtK1BAtK2(directory); + var before = Fingerprint(path); + + using (var cache = Open(path, directory, ARegistry)) + { + Assert.Throws(() => cache.Commit(batch => + { + batch.Remove(K2); + batch.Upsert(new WoA { Name = "k2", Value = 9 }, new CultRecordHandle(K2)); + })); + } + + Assert.That(Fingerprint(path), Is.EqualTo(before)); + } + + // A removal of a key the cache does not hold is judged under the store's lock, and so is the add after it: another writer put + // B there, this cache never loaded it, and removing then adding A is a retype of whatever is stored. + [TestCaseSource(nameof(StoreKinds))] + public void ARetypeOfAKeyTheCacheDoesNotHoldCommitsInOneBatch(bool directory) + { + var path = PathFor(directory); + using var early = Open(path, directory, ABRegistry); + using (var other = Open(path, directory, ABRegistry)) + Assert.That(other.Commit(batch => batch.Upsert(new WoB { Name = "k1", Value = 1 }, new CultRecordHandle(K1))), Is.True); + + Assert.That(early.Commit(batch => + { + batch.Remove(K1); + batch.Upsert(new WoA { Name = "k1", Value = 2 }, new CultRecordHandle(K1)); + }), Is.True); + + Assert.That(TypeAt(path, directory, ABRegistry, K1), Is.EqualTo(typeof(WoA))); + } + + // A removal and an add called separately on the cache are one retype too: the add, staged after the removal, is not lost to the + // flush that judges it, and the type changes in the file. + [TestCaseSource(nameof(StoreKinds))] + public void AStagedRemovalAndAddOfAnotherTypeIsNotLostAtTheFlush(bool directory) + { + var path = SeedAAtK1BAtK2(directory); + + using (var cache = Open(path, directory, ABRegistry)) + { + Assert.That(cache.Remove(K2), Is.True); + cache.UpsertAsync(typeof(WoA), new WoA { Name = "k2", Value = 9 }, K2).GetAwaiter().GetResult(); + cache.FlushAllBackingStores(); + Assert.That(cache.GetStored(K2)!.Descriptor.DocumentType, Is.EqualTo(typeof(WoA))); + } + + Assert.That(TypeAt(path, directory, ABRegistry, K2), Is.EqualTo(typeof(WoA))); + } + + [TestCaseSource(nameof(StoreKinds))] + public void AStagedRemovalOfAKeyTheCacheDoesNotHoldAndAnAddOfAnotherTypeIsNotLostAtTheFlush(bool directory) + { + var path = PathFor(directory); + using var early = Open(path, directory, ABRegistry); + using (var other = Open(path, directory, ABRegistry)) + Assert.That(other.Commit(batch => batch.Upsert(new WoB { Name = "k1", Value = 1 }, new CultRecordHandle(K1))), Is.True); + + Assert.That(early.Remove(K1), Is.False, "the cache holds nothing there"); + early.UpsertAsync(typeof(WoA), new WoA { Name = "k1", Value = 2 }, K1).GetAwaiter().GetResult(); + early.FlushAllBackingStores(); + + Assert.That(TypeAt(path, directory, ABRegistry, K1), Is.EqualTo(typeof(WoA))); + } + + // A removal is spent by the flush that judged it: a later add of another type with no removal in front of it is refused as ever. + [TestCaseSource(nameof(StoreKinds))] + public void ARemovalIsSpentByTheFlushThatJudgedIt(bool directory) + { + var path = SeedAAtK1BAtK2(directory); + + using var cache = Open(path, directory, ABRegistry); + Assert.That(cache.Remove(K2), Is.True); + cache.FlushAllBackingStores(); + cache.UpsertAsync(typeof(WoB), new WoB { Name = "k2", Value = 4 }, K2).GetAwaiter().GetResult(); + cache.FlushAllBackingStores(); + Assert.Throws(() => cache.Commit(batch => batch.Upsert(new WoA { Name = "k2", Value = 5 }, new CultRecordHandle(K2)))); + Assert.That(TypeAt(path, directory, ABRegistry, K2), Is.EqualTo(typeof(WoB))); + } + + // No store, same rule: a cache holding A and B refuses one's write over the other's record, on Commit and on UpsertAsync, and + // a retype is a removal and an add, in one batch or staged apart. + [Test] + public void AStorelessCacheRefusesToWriteOneRegisteredTypeOverAnothersRecord() + { + using var cache = new CultCache(ABRegistry); + Assert.That(cache.Commit(batch => batch.Upsert(new WoB { Name = "k2", Value = 2 }, new CultRecordHandle(K2))), Is.True); + + var commit = Assert.Throws(() => cache.Commit(batch => batch.Upsert(new WoA { Name = "k2", Value = 9 }, new CultRecordHandle(K2))))!; + Assert.That(commit.RecordKey, Is.EqualTo(K2.Value)); + Assert.Throws(() => cache.UpsertAsync(typeof(WoA), new WoA { Name = "k2", Value = 9 }, K2)); + Assert.That(cache.GetStored(K2)!.Descriptor.DocumentType, Is.EqualTo(typeof(WoB))); + + // The pair that differs only in the type written: B over B's own record. + Assert.That(cache.Commit(batch => batch.Upsert(new WoB { Name = "k2", Value = 3 }, new CultRecordHandle(K2))), Is.True); + + Assert.That(cache.Commit(batch => + { + batch.Remove(K2); + batch.Upsert(new WoA { Name = "k2", Value = 9 }, new CultRecordHandle(K2)); + }), Is.True); + Assert.That(cache.GetStored(K2)!.Descriptor.DocumentType, Is.EqualTo(typeof(WoA))); + + Assert.That(cache.Remove(K2), Is.True); + cache.UpsertAsync(typeof(WoB), new WoB { Name = "k2", Value = 4 }, K2).GetAwaiter().GetResult(); + Assert.That(cache.GetStored(K2)!.Descriptor.DocumentType, Is.EqualTo(typeof(WoB))); + } } } From 36873e696e1c6059d3782b7ce28b5c6c32a33402 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 28 Oct 2026 05:12:19 +0100 Subject: [PATCH 08/88] VariantTests: the store-less rebase test builds its cache with the codec Co-Authored-By: Claude Opus 5.5 --- tests/GameCult.Caching.Tests/VariantTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/GameCult.Caching.Tests/VariantTests.cs b/tests/GameCult.Caching.Tests/VariantTests.cs index 1bcb36a2..62255b12 100644 --- a/tests/GameCult.Caching.Tests/VariantTests.cs +++ b/tests/GameCult.Caching.Tests/VariantTests.cs @@ -286,7 +286,7 @@ namespace GameCult.Caching.Tests [Test] public void ARebaseOntoABaseOfAnotherTypeIsRefusedWithoutAStore() { - using var cache = new CultCache(Registry); + using var cache = new CultCache(Registry, CultCacheMessagePack.CreateCodec(Registry)); SeedBase(cache); SeedBig(cache); var other = new CultRecordKey("other"); @@ -418,7 +418,7 @@ namespace GameCult.Caching.Tests [Test] public void ACacheWithoutACodecHoldsNoVariants() { - using var cache = new CultCache(Registry); + using var cache = new CultCache(Registry, CultCacheMessagePack.CreateCodec(Registry)); cache.Commit(batch => batch.Upsert(typeof(VariantGear), Laser(), BaseKey)); var error = Refused(() => cache.UpsertVariantAsync(BigKey, BaseKey).GetAwaiter().GetResult()); Assert.That(error.Message, Does.Contain(BigKey.Value).And.Contain("codec")); From 3e283c81d4400ac477a2cf685b653e8deca4d9a1 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 28 Oct 2026 05:14:10 +0100 Subject: [PATCH 09/88] Keep the StoredPublishedBy staleness guard and pin it: a record minting relabels in place is owned by a newer version that reads its current publisher whole The probe (a throw where a loaded record is relabelled in place) tripped 32 existing tests, so the guard is reachable, not dead. Co-Authored-By: Claude Opus 5.5 --- src/GameCult.Caching/CultCache.cs | 20 ++++++---- .../GameCult.Caching.Tests/ElementIdTests.cs | 40 +++++++++++++++++++ 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 5ee82e06..4fe301c2 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -524,11 +524,12 @@ namespace GameCult.Caching // asked. internal bool StoredSchemaReadWhole { get; set; } - // The catalog entry that publishes StoredSchemaId in the store this record was loaded from, so staging can ask WrittenOwns the - // question the store's judge asks, over the catalog the record came with. Null for a record this cache constructed or wrote, - // which is stored under Descriptor's own id and published by Descriptor's entry. Only a loaded record carries it, and a loaded - // record is replaced by a new one when written, never relabelled in place, so it always names the id StoredSchemaId holds. - internal CultSchemaCatalogEntry? StoredPublishedBy { get; set; } + // The catalog entry that publishes StoredSchemaId in the store this record was loaded from, with the id it was published for, + // so staging can ask WrittenOwns the question the store's judge asks, over the catalog the record came with. Null for a record + // this cache constructed or wrote, which is stored under Descriptor's own id and published by Descriptor's entry. A write that + // relabels a loaded record in place (minting its element ids, then a flush) moves StoredSchemaId and leaves this behind, so a + // reader uses it only while the id still matches StoredSchemaId. + internal (string Id, CultSchemaCatalogEntry Entry)? StoredPublishedBy { get; set; } // Names the bytes this record was read from or written as: the SHA-256 of its persisted encoding, set by the store that // read or wrote them. A record no store has read or written has a version of its own that equals nothing else. @@ -3986,8 +3987,8 @@ namespace GameCult.Caching }; } - private static CultSchemaCatalogEntry? PublishedByIn(string schemaId, IReadOnlyCollection catalog) => - CultDocumentRegistry.PublishedBy(schemaId, catalog); + private static (string Id, CultSchemaCatalogEntry Entry)? PublishedByIn(string schemaId, IReadOnlyCollection catalog) => + CultDocumentRegistry.PublishedBy(schemaId, catalog) is { } entry ? (schemaId, entry) : null; protected void SetLastSchemaMigrationReports(IEnumerable reports) { @@ -4035,7 +4036,10 @@ namespace GameCult.Caching // record's id when it was loaded (a record this cache wrote is published by its own type). The one answer for a store-backed // cache's staging and a store-less cache's. internal static bool UpsertOwned(CultStoredDocument held, CultDocumentDescriptor written) => - CultDocumentRegistry.WrittenOwns(written.ToCatalogEntry(), held.StoredSchemaId, new[] { held.StoredPublishedBy ?? held.Descriptor.ToCatalogEntry() }); + CultDocumentRegistry.WrittenOwns( + written.ToCatalogEntry(), + held.StoredSchemaId, + new[] { held.StoredPublishedBy is { } loaded && string.Equals(loaded.Id, held.StoredSchemaId, StringComparison.Ordinal) ? loaded.Entry : held.Descriptor.ToCatalogEntry() }); // The refusal of a store-less cache that would retype a key. internal static CultSchemaConflictException Retyped(CultStoredDocument held, string key, CultDocumentDescriptor written) => new( diff --git a/tests/GameCult.Caching.Tests/ElementIdTests.cs b/tests/GameCult.Caching.Tests/ElementIdTests.cs index aa1d7b57..28edc1b9 100644 --- a/tests/GameCult.Caching.Tests/ElementIdTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdTests.cs @@ -274,6 +274,36 @@ namespace GameCult.Caching.Tests Assert.That(DiskRecord(path, "a").SchemaId, Is.Not.EqualTo(otherRuntimeId)); } + // Minting rewrites a loaded record in place under this type's id, and the cache keeps that entry. A newer version of the type, + // registered afterwards, reads the entry the record is now published by whole, so it owns the record: staging asks the entry + // that publishes the id the record carries now, not the one that published the id it was loaded under. + [Test] + public void ANewerVersionReadsWholeTheRecordMintingRelabelledInPlace() + { + const string otherRuntimeId = "other.runtime.id_deck"; + var path = PathOf("mint-then-newer.cc"); + WritePreIdStore(path, null, ("a", "a", 1)); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + var entry = snapshot.SchemaCatalog.Single(); + entry.SchemaId = otherRuntimeId; + entry.ContentHash = otherRuntimeId; + entry.CompatibleSchemaIds = new[] { otherRuntimeId }; + snapshot.Records.Single().SchemaId = otherRuntimeId; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + var key = new CultRecordKey("a"); + + using (var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry })) + { + Assert.That(cache.MintElementIds(), Is.EqualTo(1)); + Assert.That(cache.GetStored(key)!.StoredSchemaId, Is.Not.EqualTo(otherRuntimeId), "minting relabelled the loaded entry in place"); + registry.GetRequired(typeof(IdDeckNext)); + Assert.That(cache.Commit(batch => batch.Upsert(typeof(IdDeckNext), new IdDeckNext { Name = "newer" }, key)), Is.True); + } + + Assert.That(DiskRecord(path, "a").SchemaId, Is.EqualTo(registry.GetRequired(typeof(IdDeckNext)).SchemaId)); + } + // A pre-id record this cache reads only by dropping a persisted slot is read-only, so the rewrite that would persist its // minted ids is refused typed, naming it, and the store is left byte for byte. [Test] @@ -1401,6 +1431,16 @@ namespace GameCult.Caching.Tests [Key(2)] public List Moves { get; set; } = new(); } + // The next version of IdDeck: the same slots under a new schema version. + [CultDocument("tests.element_id_deck", "tests.element_id_deck.v2")] + [MessagePackObject] + public sealed class IdDeckNext + { + [Key(0)] [CultName] public string Name { get; set; } = ""; + [Key(1)] public List Reels { get; set; } = new(); + [Key(2)] public List Moves { get; set; } = new(); + } + // The shapes before ids: same slots, no id member. Not documents, so never registered. [MessagePackObject] public sealed class OldMark From 20be80cc239ac7bd5862b392a26c390ec1504c38 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 28 Oct 2026 05:16:32 +0100 Subject: [PATCH 10/88] Restore the codec-less variant test's cache; the one-batch retype test keeps the replacement through a later add of the batch Co-Authored-By: Claude Opus 5.5 --- tests/GameCult.Caching.Tests/VariantTests.cs | 2 +- tests/GameCult.Caching.Tests/WriteOwnershipTests.cs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/GameCult.Caching.Tests/VariantTests.cs b/tests/GameCult.Caching.Tests/VariantTests.cs index 62255b12..8f574cd1 100644 --- a/tests/GameCult.Caching.Tests/VariantTests.cs +++ b/tests/GameCult.Caching.Tests/VariantTests.cs @@ -418,7 +418,7 @@ namespace GameCult.Caching.Tests [Test] public void ACacheWithoutACodecHoldsNoVariants() { - using var cache = new CultCache(Registry, CultCacheMessagePack.CreateCodec(Registry)); + using var cache = new CultCache(Registry); cache.Commit(batch => batch.Upsert(typeof(VariantGear), Laser(), BaseKey)); var error = Refused(() => cache.UpsertVariantAsync(BigKey, BaseKey).GetAwaiter().GetResult()); Assert.That(error.Message, Does.Contain(BigKey.Value).And.Contain("codec")); diff --git a/tests/GameCult.Caching.Tests/WriteOwnershipTests.cs b/tests/GameCult.Caching.Tests/WriteOwnershipTests.cs index ff975a51..0728e6f1 100644 --- a/tests/GameCult.Caching.Tests/WriteOwnershipTests.cs +++ b/tests/GameCult.Caching.Tests/WriteOwnershipTests.cs @@ -265,8 +265,9 @@ namespace GameCult.Caching.Tests Assert.That(cache.Commit(batch => { batch.Remove(K2); + batch.Upsert(new WoB { Name = "k2", Value = 8 }, new CultRecordHandle(K2)); batch.Upsert(new WoA { Name = "k2", Value = 9 }, new CultRecordHandle(K2)); - }), Is.True); + }), Is.True, "the key stays a replacement through a later add of the same batch"); Assert.That(cache.GetStored(K2)!.Descriptor.DocumentType, Is.EqualTo(typeof(WoA))); } From 009de19879004a32b6d9c2a588c288b92bc1cce4 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 28 Oct 2026 05:29:31 +0100 Subject: [PATCH 11/88] WriteOwnershipTests: a removal is spent by the flush that judged it Co-Authored-By: Claude Opus 5.5 --- tests/GameCult.Caching.Tests/WriteOwnershipTests.cs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/GameCult.Caching.Tests/WriteOwnershipTests.cs b/tests/GameCult.Caching.Tests/WriteOwnershipTests.cs index 0728e6f1..d78c2a67 100644 --- a/tests/GameCult.Caching.Tests/WriteOwnershipTests.cs +++ b/tests/GameCult.Caching.Tests/WriteOwnershipTests.cs @@ -367,7 +367,7 @@ namespace GameCult.Caching.Tests Assert.That(TypeAt(path, directory, ABRegistry, K1), Is.EqualTo(typeof(WoA))); } - // A removal is spent by the flush that judged it: a later add of another type with no removal in front of it is refused as ever. + // A removal is spent by the flush that judged it: a later add of another type, with no removal in front of it, is judged as an upsert again. [TestCaseSource(nameof(StoreKinds))] public void ARemovalIsSpentByTheFlushThatJudgedIt(bool directory) { @@ -376,9 +376,13 @@ namespace GameCult.Caching.Tests using var cache = Open(path, directory, ABRegistry); Assert.That(cache.Remove(K2), Is.True); cache.FlushAllBackingStores(); - cache.UpsertAsync(typeof(WoB), new WoB { Name = "k2", Value = 4 }, K2).GetAwaiter().GetResult(); - cache.FlushAllBackingStores(); - Assert.Throws(() => cache.Commit(batch => batch.Upsert(new WoA { Name = "k2", Value = 5 }, new CultRecordHandle(K2)))); + + // Another writer puts B at the key this cache removed; this cache never loaded it, and stages an A there with no removal. + using (var other = Open(path, directory, ABRegistry)) + Assert.That(other.Commit(batch => batch.Upsert(new WoB { Name = "k2", Value = 4 }, new CultRecordHandle(K2))), Is.True); + cache.UpsertAsync(typeof(WoA), new WoA { Name = "k2", Value = 5 }, K2).GetAwaiter().GetResult(); + + Assert.Throws(() => cache.FlushAllBackingStores()); Assert.That(TypeAt(path, directory, ABRegistry, K2), Is.EqualTo(typeof(WoB))); } From 6f1930b354eee4f102c2b50f4eda943c2eb9a248 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 28 Oct 2026 05:31:46 +0100 Subject: [PATCH 12/88] ElementIdTests: pin the StoredPublishedBy guard through a push that relabels a loaded record in place Co-Authored-By: Claude Opus 5.5 --- tests/GameCult.Caching.Tests/ElementIdTests.cs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/GameCult.Caching.Tests/ElementIdTests.cs b/tests/GameCult.Caching.Tests/ElementIdTests.cs index 28edc1b9..1b6b3fe2 100644 --- a/tests/GameCult.Caching.Tests/ElementIdTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdTests.cs @@ -274,11 +274,11 @@ namespace GameCult.Caching.Tests Assert.That(DiskRecord(path, "a").SchemaId, Is.Not.EqualTo(otherRuntimeId)); } - // Minting rewrites a loaded record in place under this type's id, and the cache keeps that entry. A newer version of the type, + // Pushing a loaded record rewrites it in place under this type's id, and the cache keeps that entry. A newer version of the type, // registered afterwards, reads the entry the record is now published by whole, so it owns the record: staging asks the entry // that publishes the id the record carries now, not the one that published the id it was loaded under. [Test] - public void ANewerVersionReadsWholeTheRecordMintingRelabelledInPlace() + public void ANewerVersionReadsWholeTheRecordAPushRelabelledInPlace() { const string otherRuntimeId = "other.runtime.id_deck"; var path = PathOf("mint-then-newer.cc"); @@ -295,8 +295,11 @@ namespace GameCult.Caching.Tests using (var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry })) { - Assert.That(cache.MintElementIds(), Is.EqualTo(1)); - Assert.That(cache.GetStored(key)!.StoredSchemaId, Is.Not.EqualTo(otherRuntimeId), "minting relabelled the loaded entry in place"); + var loaded = cache.GetStored(key)!; + cache.BackingStores[0].Push(loaded); + cache.BackingStores[0].PushAll(); + Assert.That(ReferenceEquals(cache.GetStored(key), loaded), Is.True, "the cache keeps the entry the store relabelled"); + Assert.That(loaded.StoredSchemaId, Is.Not.EqualTo(otherRuntimeId), "the push relabelled the loaded entry in place"); registry.GetRequired(typeof(IdDeckNext)); Assert.That(cache.Commit(batch => batch.Upsert(typeof(IdDeckNext), new IdDeckNext { Name = "newer" }, key)), Is.True); } From e19d98d0b8b35411fdf23fe11604e2c3373cb6c6 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 28 Oct 2026 06:19:06 +0100 Subject: [PATCH 13/88] A one-batch retype is the held record's eviction then the add, on every write path; the stores judge through one protected per-key judge Delete StagedOperation.Replaces and Admit's replaced set. CultCacheBatch records the keys Remove named; Land evicts the held record at each (or checks the key when held by no one) and the add lands after it, so watchers, the globals copy, the variant guard, the key-home check, Claim, the store judges and the change journal see what they see when the two are staged apart. A Flatten after a Remove in the batch is refused as a flatten of a non-variant. CacheBackingStore gains one protected RefuseUndeclaredOnDisk overload taking the catalog, the held record, the written entry and the removed keys; both shipped stores call it. The old overload and Undeclared stay. Co-Authored-By: Claude Opus 5.5 --- .../DirectoryMessagePackBackingStore.cs | 9 +- src/GameCult.Caching/CultCache.cs | 123 +++++++++++------- 2 files changed, 83 insertions(+), 49 deletions(-) diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index cd51e3f9..a113db44 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -299,15 +299,16 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore CultPersistedRecord? HeldAt(string key) => currentManifest.Records.FirstOrDefault(record => string.Equals(record.Key, key, StringComparison.Ordinal)); var batch = upserts.ToDictionary(entry => entry.Key.Value, entry => entry.Descriptor, StringComparer.Ordinal); - var removed = deletes.Select(entry => entry.Key.Value).Concat(checks).Concat(StagedRemovals).ToHashSet(StringComparer.Ordinal); CultSchemaCatalogEntry? WrittenAt(string key) => - removed.Contains(key) ? null - : batch.TryGetValue(key, out var descriptor) ? descriptor.ToCatalogEntry() + batch.TryGetValue(key, out var descriptor) ? descriptor.ToCatalogEntry() : _dirtyKeys.ContainsKey(key) && Entries.TryGetValue(key, out var staged) ? staged.Descriptor.ToCatalogEntry() : null; RefuseUndeclaredOnDisk( written, - UndeclaredAt(HeldAt, currentManifest.SchemaCatalog, WrittenAt, Registry.Owns), + currentManifest.SchemaCatalog, + HeldAt, + WrittenAt, + deletes.Select(entry => entry.Key.Value).Concat(checks), key => HeldAt(key) is { } held && Foreign(held, currentManifest.SchemaCatalog, VersionOf(held.Payload)) == null ? LoadPage(held, currentManifest.SchemaCatalog, out _, out _) : null, diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 4fe301c2..f68eec42 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -2076,19 +2076,14 @@ namespace GameCult.Caching // One staged write: a plain record (Stored), a variant (Variant), a flatten, or, with none of those, a removal. internal sealed class StagedOperation { - public StagedOperation(CultRecordKey key, CultStoredDocument? stored, CultVariantDelta? variant, bool flatten, bool replaces = false) + public StagedOperation(CultRecordKey key, CultStoredDocument? stored, CultVariantDelta? variant, bool flatten) { Key = key; Stored = stored; Variant = variant; Flatten = flatten; - Replaces = replaces; } - // True for a plain record staged at a key this batch removed first: a removal and an add (variants:ruling:cache-refuses-retype). - // The key is judged as the removal it names, then the record lands at it. - public bool Replaces { get; } - public CultRecordKey Key { get; } public CultStoredDocument? Stored { get; } public CultVariantDelta? Variant { get; } @@ -2101,6 +2096,9 @@ namespace GameCult.Caching { private readonly CultCache _cache; internal readonly Dictionary Operations = new(StringComparer.Ordinal); + // Every key Remove named, in whatever order: Land evicts the record the cache holds at each (or checks the key when it holds + // none) before anything lands, so a record staged at a removed key is an add after an eviction, as when staged apart. + internal readonly HashSet Removals = new(StringComparer.Ordinal); internal readonly List<(CultRecordKey Key, string? Version)> Expected = new(); internal bool ExpectsUnchanged; internal bool Sealed; @@ -2125,6 +2123,7 @@ namespace GameCult.Caching public void Remove(CultRecordKey key) { ThrowIfSealed(); + Removals.Add(key.Value); Operations[key.Value] = new StagedOperation(key, null, null, false); } @@ -2142,6 +2141,8 @@ namespace GameCult.Caching public void Flatten(CultRecordKey key) { ThrowIfSealed(); + if (Removals.Contains(key.Value)) + throw new InvalidOperationException($"Flatten({key.Value}): the record is not a variant."); Operations[key.Value] = new StagedOperation(key, null, null, true); } @@ -2162,8 +2163,7 @@ namespace GameCult.Caching { ThrowIfSealed(); var stored = _cache.CreateStoredDocument(document, key); - var replaces = Operations.TryGetValue(stored.Key.Value, out var previous) && (previous.IsRemoval || previous.Replaces); - Operations[stored.Key.Value] = new StagedOperation(stored.Key, stored, null, false, replaces); + Operations[stored.Key.Value] = new StagedOperation(stored.Key, stored, null, false); return stored.Key; } @@ -2833,19 +2833,12 @@ namespace GameCult.Caching : op.Variant != null ? StampVariant(op.Key, DescriptorOfBase(op.Key, op.Variant.BaseKey, batch), op.Variant) : StampFlatten(op.Key)) .ToArray(); - // A removal of a key the cache does not hold removes nothing: it is a check every store the commit reaches judges under its lock. - // So is the removal half of a retype (a record staged at a key the batch removed first, held or not): the store judges the - // key as the removal it names, and the record then lands at it. - var replaced = batch.Operations.Values.Where(op => op.Replaces).Select(op => op.Key.Value).ToHashSet(StringComparer.Ordinal); - var checks = batch.Operations.Values - .Where(op => op.Replaces || (op.IsRemoval && !_entries.ContainsKey(op.Key.Value))) - .Select(op => op.Key.Value) - .ToArray(); - var deletes = batch.Operations.Values - .Where(op => op.IsRemoval && _entries.ContainsKey(op.Key.Value)) - .Select(op => _entries[op.Key.Value]) - .ToArray(); - return Admit(upserts, deletes, null, replaced: replaced, land: (home, resolved) => + // A removal evicts the record the cache holds at the key, and a record staged at the key lands after the eviction, as + // when the two are staged apart (variants:ruling:one-batch-retype-is-eviction-then-add). A removal of a key the cache + // does not hold removes nothing: it is a check every store the commit reaches judges under its lock. + var deletes = batch.Removals.Where(_entries.ContainsKey).Select(key => _entries[key]).ToArray(); + var checks = batch.Removals.Where(key => !_entries.ContainsKey(key)).ToArray(); + return Admit(upserts, deletes, null, removed: batch.Removals, land: (home, resolved) => { // A batch that only checks keys names no home; it commits through the cache's one writable store, if it has one. if (home == null && checks.Length > 0 && _stores.Where(entry => !entry.Store.IsReadOnly).Take(2).ToArray() is { Length: 1 } only) @@ -2896,11 +2889,11 @@ namespace GameCult.Caching IReadOnlyList evicted, CacheBackingStore? source, Func, CultCommitOutcome> land, - HashSet? replaced = null) => Held(() => + IReadOnlyCollection? removed = null) => Held(() => { if (_held == null) throw new InvalidOperationException("An admission reached the cache under a plain lock on its gate; every admission runs in a hold."); - var home = Validate(admitted, evicted, source, replaced); + var home = Validate(admitted, evicted, source, removed); // Ids are decided before anything lands, and the caller's documents change only inside this admission: a plan that // refuses (a duplicate, a malformed id) has changed nothing, and a write that does not commit gives its ids back. // A write mints random ids; a load of a record written before ids existed mints the ids its reload will mint @@ -3109,8 +3102,9 @@ namespace GameCult.Caching IReadOnlyList admitted, IReadOnlyList evicted, CacheBackingStore? source, - HashSet? replaced) + IReadOnlyCollection? removed) { + var removedKeys = removed?.ToHashSet(StringComparer.Ordinal) ?? new HashSet(StringComparer.Ordinal); var homes = new List(); var globals = new Dictionary(_globals); foreach (var stored in evicted) @@ -3126,13 +3120,13 @@ namespace GameCult.Caching { var descriptor = stored.Descriptor; if (source == null) - Claim(stored, upsert: replaced?.Contains(stored.Key.Value) != true); + Claim(stored, upsert: !removedKeys.Contains(stored.Key.Value)); else if (Home(descriptor.DocumentType) is var home && home != source) throw new InvalidOperationException( $"{descriptor.SchemaName} record {stored.Key.Value} was loaded from {source} but its home is {home?.ToString() ?? "no store"}."); // One key, one store: a key never moves between homes, and two stores never both deliver it. - if (_entries.TryGetValue(stored.Key.Value, out var present) && + if (!removedKeys.Contains(stored.Key.Value) && _entries.TryGetValue(stored.Key.Value, out var present) && Home(present.Descriptor.DocumentType) is var held && Home(descriptor.DocumentType) is var incoming && held != incoming) @@ -3287,6 +3281,8 @@ namespace GameCult.Caching foreach (var stored in admitted) { _entries.TryGetValue(stored.Key.Value, out var present); + if (gone.Contains(stored.Key.Value)) + present = null; if (stored.Variant == null && !stored.Flattens && present?.Variant != null) throw new InvalidOperationException( $"Record {stored.Key.Value} is a variant of {present.Variant.BaseKey}; a plain write cannot replace it. " + @@ -4162,6 +4158,53 @@ namespace GameCult.Caching }; } + /// + /// The per-key ownership judge of a write, for a store kind outside this library: the rule the shipped stores apply under their + /// lock, as one call. Each key in and in is judged against the durable + /// record gives for it. A key the write puts a record at ( gives the + /// catalog entry of the record written there) is owned only by that type: a record under its own schema id, an id it lists + /// compatible, or an older id of its own schema name it reads whole; another registered type's record is refused. A key the + /// write removes or only checks, and every key this store's cache removed since the last flush or commit (the removal half of + /// a retype, whose add then lands at it), is owned by any registered type that declares or reads the record whole. A refusal is + /// : it reloads what it refuses, unstages it through , and throws. + /// + /// Every key the write names. + /// The schema catalog of the durable store. + /// The durable record at a key, or null. + /// The catalog entry of the record the write puts at a key, or null when it puts none. + /// The keys the write removes or checks. + /// See . + /// See . + /// See . + /// True for a write no cache made: its refusal is the cacheless one. + protected void RefuseUndeclaredOnDisk( + IEnumerable written, + IReadOnlyCollection catalog, + Func heldAt, + Func writtenAt, + IEnumerable removed, + Func durableAt, + Func foreignAt, + Action unstage, + bool cacheless = false) + { + var keys = written.Distinct(StringComparer.Ordinal).ToArray(); + var removedKeys = removed.Concat(StagedRemovals).ToHashSet(StringComparer.Ordinal); + // A cacheless call has no registry that decides a removal: the entries it writes do (a call that writes no entry owns + // nothing, variants:ruling:mesh-writes-bound-by-declaration). + var entries = keys.Select(writtenAt).OfType().ToArray(); + Func, bool> removalOwns = Cache != null + ? Registry.Owns + : (id, entriesInCatalog) => entries.Any(entry => CultDocumentRegistry.WrittenOwns(entry, id, entriesInCatalog)); + RefuseUndeclaredOnDisk( + keys, + UndeclaredAt(heldAt, catalog, key => removedKeys.Contains(key) ? null : writtenAt(key), removalOwns), + durableAt, + foreignAt, + unstage, + cacheless); + } + // The apply half of rule 3, for every store kind, checked where the write lands under the store's lock. undeclaredAt is the // store's per-key judge: for each key the write names it says whether the durable record there is one the write may not // replace, remove or relabel. The rule it must apply is per key and per written type, never one predicate per stored id: a @@ -4169,8 +4212,8 @@ namespace GameCult.Caching // own schema name it reads whole), so another registered type's record is refused; a key the write only removes or checks // is owned by any registered type that declares or reads it whole; and a key the cache removed and the write then adds again // is judged as the removal it names. Every key at once is refused through Refuse, which reloads it as the store holds it: - // listed foreign when no type reads it, held read-only when one does. The shipped stores build this judge with - // UndeclaredAt, which is internal; a store kind outside the library supplies its own and passes it here. + // listed foreign when no type reads it, held read-only when one does. The shipped stores call the overload that builds + // this judge; this one takes a judge a store kind outside the library built itself. protected void RefuseUndeclaredOnDisk( IEnumerable written, Func undeclaredAt, @@ -4827,15 +4870,7 @@ namespace GameCult.Caching durable, _staged.Concat(upserts.Select(staged => staged.Record.Key)).Concat(removals).Concat(checks).Concat(StagedChecks), upserts, - checks); - - // A write's key is owned by the entry the write puts there, whoever the writer is (WrittenOwns). A key it only removes or - // checks is owned by the removal rule: a cache's registry (Owns), or, for a cacheless call that has no registry, some entry the - // call writes (a call that writes no entry owns nothing, variants:ruling:mesh-writes-bound-by-declaration). - private Func, bool> RemovalOwnsOnDisk(IReadOnlyCollection upserts) => - Cache != null - ? Registry.Owns - : (id, catalog) => upserts.Any(staged => CultDocumentRegistry.WrittenOwns(staged.Entry, id, catalog)); + checks.Concat(removals)); // A key the write also names as a removal or check (the request's, or one the cache removed here since its last flush) is a // retype's removal half: it is judged by the removal rule, and the record staged there then lands. A refused key's durable @@ -4847,16 +4882,14 @@ namespace GameCult.Caching var held = (durable?.Snapshot.Records ?? Array.Empty()) .Where(record => wanted.Contains(record.Key)) .ToDictionary(record => record.Key, StringComparer.Ordinal); - var retyped = checks.Concat(StagedRemovals).ToHashSet(StringComparer.Ordinal); - var entries = upserts.Where(staged => !retyped.Contains(staged.Record.Key)).ToDictionary(staged => staged.Record.Key, staged => staged.Entry, StringComparer.Ordinal); - var undeclaredAt = UndeclaredAt( - key => held.TryGetValue(key, out var record) ? record : null, - durable?.Snapshot.SchemaCatalog ?? Array.Empty(), - key => entries.TryGetValue(key, out var entry) ? entry : null, - RemovalOwnsOnDisk(upserts)); + var removed = checks.ToHashSet(StringComparer.Ordinal); + var writtenEntries = upserts.ToDictionary(staged => staged.Record.Key, staged => staged.Entry, StringComparer.Ordinal); RefuseUndeclaredOnDisk( keys, - undeclaredAt, + durable?.Snapshot.SchemaCatalog ?? Array.Empty(), + key => held.TryGetValue(key, out var record) ? record : null, + key => writtenEntries.TryGetValue(key, out var entry) ? entry : null, + removed, key => Cache == null ? null : DurableAt(durable, key), key => ForeignAt(durable, key), key => From ad51e45183d949e49bbd833aa73b360b845b4565 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 28 Oct 2026 06:19:07 +0100 Subject: [PATCH 14/88] RetypeEvictionTests: pin the retype as an eviction, the variant and global retypes, the removal bookkeeping and the rebase rule Co-Authored-By: Claude Opus 5.5 --- .../RetypeEvictionTests.cs | 429 ++++++++++++++++++ 1 file changed, 429 insertions(+) create mode 100644 tests/GameCult.Caching.Tests/RetypeEvictionTests.cs diff --git a/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs b/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs new file mode 100644 index 00000000..644c3d17 --- /dev/null +++ b/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs @@ -0,0 +1,429 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; +using R3; + +namespace GameCult.Caching.Tests +{ + // variants:ruling:one-batch-retype-is-eviction-then-add: a batch that removes a key and adds a record there stages the held + // record's eviction and then the add, so watchers, the batch validators, the store judges and the removal bookkeeping see what they + // see when the two are staged apart (cache.Remove, then UpsertAsync). Where a test runs both shapes it asserts the same answer. + public class RetypeEvictionTests + { + [CultDocument("tests.ev_a", "tests.ev_a.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class EvA + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + } + + [CultDocument("tests.ev_b", "tests.ev_b.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class EvB + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + } + + // A type only the second writer registers. + [CultDocument("tests.ev_c", "tests.ev_c.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class EvC + { + [Key(0)] [CultName] public string Name = string.Empty; + } + + [CultDocument("tests.ev_g", "tests.ev_g.v1")] + [MessagePackObject(AllowPrivate = true)] + [CultGlobal] + internal sealed class EvG + { + [Key(0)] public string Value = string.Empty; + } + + // The rebase pair: RbY lists RbX's id compatible, so a plain write of RbY owns an RbX record; RbZ lists nothing. + [CultDocument("tests.rb_x", "tests.rb_x.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class RbX + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + } + + [CultDocument("tests.rb_y", "tests.rb_y.v1", CompatibleSchemaIds = new[] { "tests.rb_x.v1" })] + [MessagePackObject(AllowPrivate = true)] + internal sealed class RbY + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + } + + [CultDocument("tests.rb_z", "tests.rb_z.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class RbZ + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + } + + private static readonly CultDocumentRegistry AB = CultDocumentRegistry.ForTypes(new[] { typeof(EvA), typeof(EvB), typeof(EvG) }); + private static readonly CultDocumentRegistry ABC = CultDocumentRegistry.ForTypes(new[] { typeof(EvA), typeof(EvB), typeof(EvC), typeof(EvG) }); + private static readonly CultDocumentRegistry Rb = CultDocumentRegistry.ForTypes(new[] { typeof(RbX), typeof(RbY), typeof(RbZ) }); + private static readonly CultRecordKey K1 = new("k1"); + private static readonly CultRecordKey K2 = new("k2"); + private static readonly CultRecordKey K3 = new("k3"); + + // The stores a retype is judged in, and no store at all. The directory store holds no variant, so variant tests skip it. + public static readonly object[] Kinds = { "single-file", "directory", "storeless" }; + public static readonly object[] VariantKinds = { "single-file", "storeless" }; + + private string _directory = string.Empty; + private int _paths; + + [SetUp] + public void CreateDirectory() + { + _directory = Path.Combine(Path.GetTempPath(), $"cultlib-retypeeviction-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_directory); + } + + [TearDown] + public void DeleteDirectory() + { + if (Directory.Exists(_directory)) + Directory.Delete(_directory, recursive: true); + } + + // A fresh store path per call, so one test can run a scenario twice. + private string NewPath(string kind) => Path.Combine(_directory, $"{kind}-{_paths++}", kind == "directory" ? "store" : "store.cc"); + + private static CultCache Open(string path, string kind, CultDocumentRegistry registry) + { + var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry)); + if (kind != "storeless") + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + cache.AddBackingStore(kind == "directory" ? new DirectoryMessagePackBackingStore(path) : new SingleFileMessagePackBackingStore(path)); + cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + } + + return cache; + } + + private static Type TypeAt(string path, string kind, CultDocumentRegistry registry, CultRecordKey key) + { + using var cache = Open(path, kind, registry); + return cache.GetStored(key)!.Descriptor.DocumentType; + } + + // Retypes a held key as one batch (Remove, then the add the caller stages) or staged apart (cache.Remove, then the add's own call). + private static void Retype(CultCache cache, bool oneBatch, CultRecordKey key, Action add, Action apart) + { + if (oneBatch) + Assert.That(cache.Commit(batch => + { + batch.Remove(key); + add(batch); + }), Is.True); + else + { + Assert.That(cache.Remove(key), Is.True); + apart(); + } + } + + private static void PutB(CultCache cache, CultRecordKey key, int value = 2) => + Assert.That(cache.Commit(batch => batch.Upsert(typeof(EvB), new EvB { Name = key.Value, Value = value }, key)), Is.True); + + // finding retype-not-an-eviction: Watch sees Removed and Watch sees Added, in a batch and staged apart alike. + [TestCaseSource(nameof(Kinds))] + public void ARetypeIsRemovedForTheOldTypeAndAddedForTheNewWhetherOneBatchOrStagedApart(string kind) + { + foreach (var oneBatch in new[] { true, false }) + { + using var cache = Open(NewPath(kind), kind, AB); + PutB(cache, K2); + var removedLog = new List(); + var addedLog = new List(); + using var oldWatch = cache.Watch().Subscribe(change => removedLog.Add(change.Kind)); + using var newWatch = cache.Watch().Subscribe(change => addedLog.Add(change.Kind)); + + Retype(cache, oneBatch, K2, + batch => batch.Upsert(typeof(EvA), new EvA { Name = "k2", Value = 9 }, K2), + () => cache.UpsertAsync(typeof(EvA), new EvA { Name = "k2", Value = 9 }, K2).GetAwaiter().GetResult()); + + Assert.That(removedLog, Is.EqualTo(new[] { CultCacheDocumentChangeKind.Removed }), $"Watch, one batch {oneBatch}"); + Assert.That(addedLog, Is.EqualTo(new[] { CultCacheDocumentChangeKind.Added }), $"Watch, one batch {oneBatch}"); + Assert.That(cache.GetStored(K2)!.Descriptor.DocumentType, Is.EqualTo(typeof(EvA))); + } + } + + // finding retype-not-an-eviction, global key, and finding global-eviction-uncovered: evicting a global frees its slot, and the + // batch that retypes a global's key may land the next global of its type elsewhere, as staged apart. + [TestCaseSource(nameof(Kinds))] + public void EvictingAGlobalFreesItsSlotAndAGlobalKeyRetypesInOneBatch(string kind) + { + using var cache = Open(NewPath(kind), kind, AB); + var g1 = new CultRecordKey("g1"); + var g2 = new CultRecordKey("g2"); + Assert.That(cache.Commit(batch => batch.Upsert(typeof(EvG), new EvG { Value = "one" }, g1)), Is.True); + + Assert.Throws( + () => cache.Commit(batch => batch.Upsert(typeof(EvG), new EvG { Value = "two" }, g2)), "a second global while the first stands is refused"); + + Assert.That(cache.Commit(batch => + { + batch.Remove(g1); + batch.Upsert(typeof(EvG), new EvG { Value = "two" }, g2); + }), Is.True, "the eviction freed the slot"); + Assert.That(cache.GetGlobal()!.Value, Is.EqualTo("two")); + Assert.That(cache.GetStored(g1), Is.Null); + + Assert.That(cache.Commit(batch => + { + batch.Remove(g2); + batch.Upsert(typeof(EvA), new EvA { Name = "g2", Value = 1 }, g2); + batch.Upsert(typeof(EvG), new EvG { Value = "three" }, g1); + }), Is.True, "staged apart the retype of the global's key and the next global commit"); + Assert.That(cache.GetStored(g2)!.Descriptor.DocumentType, Is.EqualTo(typeof(EvA))); + Assert.That(cache.GetGlobal()!.Value, Is.EqualTo("three")); + } + + // finding retype-not-an-eviction (variant key) and variant-ops-drop-the-removal: a variant key retypes to a plain record of + // another type, a plain key becomes a variant of another type's base, and a key removed, added and then added as a variant + // lands as the variant, each answering as it does staged apart. + [TestCaseSource(nameof(VariantKinds))] + public void VariantKeysRetypeInOneBatchAsStagedApart(string kind) + { + foreach (var oneBatch in new[] { true, false }) + { + using var cache = Open(NewPath(kind), kind, AB); + var baseA = new CultRecordKey("base-a"); + Assert.That(cache.Commit(batch => + { + batch.Upsert(typeof(EvA), new EvA { Name = "base-a", Value = 1 }, baseA); + batch.UpsertVariant(K3, baseA, new[] { cache.Override(nameof(EvA.Name), "v3") }); + batch.Upsert(typeof(EvB), new EvB { Name = "k2", Value = 2 }, K2); + batch.Upsert(typeof(EvB), new EvB { Name = "k1", Value = 2 }, K1); + }), Is.True); + + Retype(cache, oneBatch, K3, + batch => batch.Upsert(typeof(EvB), new EvB { Name = "k3", Value = 3 }, K3), + () => cache.UpsertAsync(typeof(EvB), new EvB { Name = "k3", Value = 3 }, K3).GetAwaiter().GetResult()); + Assert.That(cache.GetStored(K3)!.Variant, Is.Null, $"one batch {oneBatch}"); + Assert.That(cache.GetStored(K3)!.Descriptor.DocumentType, Is.EqualTo(typeof(EvB))); + + Retype(cache, oneBatch, K2, + batch => batch.UpsertVariant(K2, baseA, new[] { cache.Override(nameof(EvA.Name), "v2") }), + () => cache.UpsertVariantAsync(K2, baseA, new[] { cache.Override(nameof(EvA.Name), "v2") }).GetAwaiter().GetResult()); + Assert.That(cache.GetStored(K2)!.Variant?.BaseKey, Is.EqualTo(baseA.Value), $"one batch {oneBatch}"); + Assert.That(cache.GetStored(K2)!.Descriptor.DocumentType, Is.EqualTo(typeof(EvA))); + + // Removed and then added as a variant (in one batch, with a plain add of the same type between): the variant lands. + Retype(cache, oneBatch, K1, + batch => + { + batch.Upsert(typeof(EvA), new EvA { Name = "k1", Value = 2 }, K1); + batch.UpsertVariant(K1, baseA, new[] { cache.Override(nameof(EvA.Name), "v1") }); + }, + () => cache.UpsertVariantAsync(K1, baseA, new[] { cache.Override(nameof(EvA.Name), "v1") }).GetAwaiter().GetResult()); + Assert.That(cache.GetStored(K1)!.Variant?.BaseKey, Is.EqualTo(baseA.Value), $"one batch {oneBatch}"); + } + } + + [TestCaseSource(nameof(VariantKinds))] + public void AFlattenAfterARemovalFlattensNothingInOneBatchAndStagedApart(string kind) + { + using var cache = Open(NewPath(kind), kind, AB); + var baseA = new CultRecordKey("base-a"); + cache.Commit(batch => + { + batch.Upsert(typeof(EvA), new EvA { Name = "base-a", Value = 1 }, baseA); + batch.UpsertVariant(K3, baseA, new[] { cache.Override(nameof(EvA.Name), "v3") }); + }); + + Assert.Throws(() => cache.Commit(batch => + { + batch.Remove(K3); + batch.Flatten(K3); + })); + Assert.That(cache.GetStored(K3)!.Variant, Is.Not.Null, "the refused batch changed nothing"); + Assert.That(cache.Remove(K3), Is.True); + Assert.Throws(() => cache.FlattenAsync(K3).GetAwaiter().GetResult()); + } + + // finding removal-bookkeeping-unpinned, mutant X2 (a retype elsewhere in the batch skips every key's staging claim): a batch + // holding two retypes commits, and a cross-type add at a key it did not remove is still refused beside a retype. + [TestCaseSource(nameof(Kinds))] + public void EachKeyOfABatchIsJudgedByItsOwnRemoval(string kind) + { + using var cache = Open(NewPath(kind), kind, AB); + PutB(cache, K1); + PutB(cache, K2); + PutB(cache, K3); + + Assert.That(cache.Commit(batch => + { + batch.Remove(K1); + batch.Upsert(typeof(EvA), new EvA { Name = "k1" }, K1); + batch.Remove(K2); + batch.Upsert(typeof(EvA), new EvA { Name = "k2" }, K2); + }), Is.True, "two retypes"); + + var bad = Assert.Throws(() => cache.Commit(batch => + { + batch.Remove(K1); + batch.Upsert(typeof(EvB), new EvB { Name = "k1" }, K1); + batch.Upsert(typeof(EvA), new EvA { Name = "k3" }, K3); + }))!; + Assert.That(bad.RecordKey, Is.EqualTo(K3.Value), "one key's removal licenses no other key's retype"); + Assert.That(cache.GetStored(K3)!.Descriptor.DocumentType, Is.EqualTo(typeof(EvB))); + Assert.That(cache.GetStored(K1)!.Descriptor.DocumentType, Is.EqualTo(typeof(EvA)), "the refused batch changed nothing"); + + Assert.That(cache.Commit(batch => + { + batch.Upsert(typeof(EvB), new EvB { Name = "k3", Value = 5 }, K3); + batch.Remove(K3); + }), Is.True, "an add and then a removal removes"); + Assert.That(cache.GetStored(K3), Is.Null); + } + + // finding removal-bookkeeping-unpinned, mutant X13 (any second add counts as a replacement): two adds of one batch with no + // removal are an upsert, judged against what the key holds. + [TestCaseSource(nameof(Kinds))] + public void TwoAddsWithNoRemovalAreAnUpsert(string kind) + { + using var cache = Open(NewPath(kind), kind, AB); + PutB(cache, K2); + + Assert.Throws(() => cache.Commit(batch => + { + batch.Upsert(typeof(EvA), new EvA { Name = "k2", Value = 1 }, K2); + batch.Upsert(typeof(EvA), new EvA { Name = "k2", Value = 2 }, K2); + })); + Assert.That(cache.GetStored(K2)!.Descriptor.DocumentType, Is.EqualTo(typeof(EvB))); + } + + // finding removal-bookkeeping-unpinned, mutant X18 (no removal noted for a record this cache wrote itself): a cache that wrote + // the key itself, then removes it and stages another type there, lands that type at the flush. + [TestCaseSource(nameof(Kinds))] + public void ARecordThisCacheWroteItselfRetypesStagedApart(string kind) + { + var path = NewPath(kind); + using (var cache = Open(path, kind, AB)) + { + PutB(cache, K2); + cache.FlushAllBackingStores(); + Assert.That(cache.Remove(K2), Is.True); + cache.UpsertAsync(typeof(EvA), new EvA { Name = "k2", Value = 9 }, K2).GetAwaiter().GetResult(); + cache.FlushAllBackingStores(); + } + + if (kind != "storeless") + Assert.That(TypeAt(path, kind, AB, K2), Is.EqualTo(typeof(EvA))); + } + + // finding removal-bookkeeping-unpinned, mutant M (a refusal keeps the staged removal): a removal the store refused is gone, so + // a later add at a key another writer has since rewritten is judged as an upsert, not as the stale removal. + [TestCaseSource(nameof(Kinds))] + public void ARefusedRemovalIsNotReusedByALaterAdd(string kind) + { + if (kind == "storeless") + Assert.Ignore("a refusal is a store's."); + var path = NewPath(kind); + using (var seed = Open(path, kind, ABC)) + PutB(seed, K1); + + using var cache = Open(path, kind, AB); + using var other = Open(path, kind, ABC); + Assert.That(cache.Remove(K1), Is.True); + + // Another writer retypes the key to a type this cache does not know: the cache's staged removal meets a record it may not remove. + Assert.That(other.Commit(batch => + { + batch.Remove(K1); + batch.Upsert(typeof(EvC), new EvC { Name = "k1" }, K1); + }), Is.True); + Assert.Throws(() => cache.FlushAllBackingStores()); + + // The writer removes it again and puts a B there; this cache pulls and then stages an A with no removal in front of it. + Assert.That(other.Commit(batch => batch.Remove(K1)), Is.True); + cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + Assert.That(other.Commit(batch => batch.Upsert(typeof(EvB), new EvB { Name = "k1", Value = 7 }, K1)), Is.True); + cache.UpsertAsync(typeof(EvA), new EvA { Name = "k1", Value = 8 }, K1).GetAwaiter().GetResult(); + + Assert.Throws(() => cache.FlushAllBackingStores()); + Assert.That(TypeAt(path, kind, AB, K1), Is.EqualTo(typeof(EvB))); + } + + // finding removal-bookkeeping-unpinned, mutant X9 (a refusal clears every staged removal): a refusal of one key leaves another + // key's staged retype, and the next flush lands it. + [TestCaseSource(nameof(Kinds))] + public void ARefusedFlushKeepsAnotherKeysStagedRetype(string kind) + { + if (kind == "storeless") + Assert.Ignore("a refusal is a store's."); + var path = NewPath(kind); + using (var seed = Open(path, kind, ABC)) + { + PutB(seed, K1); + PutB(seed, K2); + } + + using var cache = Open(path, kind, AB); + using var other = Open(path, kind, ABC); + Assert.That(cache.Remove(K2), Is.True); + cache.UpsertAsync(typeof(EvA), new EvA { Name = "k2", Value = 9 }, K2).GetAwaiter().GetResult(); + + Assert.That(other.Commit(batch => + { + batch.Remove(K1); + batch.Upsert(typeof(EvC), new EvC { Name = "k1" }, K1); + }), Is.True); + cache.UpsertAsync(typeof(EvA), new EvA { Name = "k1", Value = 1 }, K1).GetAwaiter().GetResult(); + + var refusal = Assert.Throws(() => cache.FlushAllBackingStores())!; + Assert.That(refusal.RecordKey, Is.EqualTo(K1.Value), "k1 holds a type this cache does not know"); + cache.FlushAllBackingStores(); + + Assert.That(TypeAt(path, kind, AB, K2), Is.EqualTo(typeof(EvA)), "the retype of k2 survived the refusal of k1"); + } + + // finding rebase-check-was-reachable, ruling variant-rebase-follows-plain-write: a variant rebased onto a base of another type + // follows the plain-write rule. Where RbY lists RbX's id (a plain write of RbY would own an RbX record) the rebase commits and + // the variant becomes an RbY; where it would be refused (RbZ) the rebase is refused and the variant is untouched. + [TestCaseSource(nameof(VariantKinds))] + public void ARebaseChangesTypeExactlyWhereAPlainWriteMay(string kind) + { + using var cache = Open(NewPath(kind), kind, Rb); + var baseX = new CultRecordKey("base-x"); + var baseY = new CultRecordKey("base-y"); + var baseZ = new CultRecordKey("base-z"); + var variant = new CultRecordKey("v"); + Assert.That(cache.Commit(batch => + { + batch.Upsert(typeof(RbX), new RbX { Name = "base-x" }, baseX); + batch.Upsert(typeof(RbY), new RbY { Name = "base-y" }, baseY); + batch.Upsert(typeof(RbZ), new RbZ { Name = "base-z" }, baseZ); + batch.UpsertVariant(variant, baseX, new[] { cache.Override(nameof(RbX.Name), "v") }); + }), Is.True); + + // The plain write of the same types gives the same two answers. + Assert.Throws(() => cache.Commit(batch => batch.Upsert(typeof(RbZ), new RbZ { Name = "base-x" }, baseX))); + Assert.Throws( + () => cache.Commit(batch => batch.UpsertVariant(variant, baseZ, new[] { cache.Override(nameof(RbZ.Name), "v") }))); + Assert.That(cache.GetStored(variant)!.Descriptor.DocumentType, Is.EqualTo(typeof(RbX))); + Assert.That(cache.GetStored(variant)!.Variant!.BaseKey, Is.EqualTo(baseX.Value)); + + Assert.That(cache.Commit(batch => batch.UpsertVariant(variant, baseY, new[] { cache.Override(nameof(RbY.Name), "v") })), Is.True); + Assert.That(cache.GetStored(variant)!.Descriptor.DocumentType, Is.EqualTo(typeof(RbY))); + Assert.That(cache.GetStored(variant)!.Variant!.BaseKey, Is.EqualTo(baseY.Value)); + } + } +} From 60474bd2b3860069c6242ed88cd453cfc8bdb148 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 28 Oct 2026 06:19:56 +0100 Subject: [PATCH 15/88] Single-file CommitBatch: a deleted key the batch also upserts reaches the judge as its removal half Co-Authored-By: Claude Opus 5.5 --- src/GameCult.Caching/CultCache.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index f68eec42..b7223f6a 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -4560,7 +4560,9 @@ namespace GameCult.Caching return CultCommitOutcome.Mismatch; var (upserts, removals) = StagedWrites(request.Upserts, request.Deletes); - var written = ApplyWriteSet(disk, upserts, removals, request.Checks); + // A key the batch deletes and also upserts is a retype: the upsert wins in the write set, so the deletion travels as a check, + // and the judge reads the key as the removal it names. + var written = ApplyWriteSet(disk, upserts, removals, request.Checks.Concat(request.Deletes.Select(entry => entry.Key.Value))); foreach (var entry in request.Deletes) Entries.TryRemove(entry.Key.Value, out _); foreach (var entry in request.Upserts) From edad333a4d78feb55f315d9e80522b30521b6096 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 28 Oct 2026 06:20:52 +0100 Subject: [PATCH 16/88] RetypeEvictionTests: RbY lists RbX's real schema id; the refused-flush test refuses k1 under the lock Co-Authored-By: Claude Opus 5.5 --- .../RetypeEvictionTests.cs | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs b/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs index 644c3d17..fd38541d 100644 --- a/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs +++ b/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs @@ -47,6 +47,9 @@ namespace GameCult.Caching.Tests [Key(0)] public string Value = string.Empty; } + // RbX's schema id, which RbY lists. The test below asserts it, so a changed derivation fails there. + private const string RbXId = "sha256:25087e0d76fd5b9ac950723ccd47f3f3ac1428b53b49ca9bda9aae8b38a0220a"; + // The rebase pair: RbY lists RbX's id compatible, so a plain write of RbY owns an RbX record; RbZ lists nothing. [CultDocument("tests.rb_x", "tests.rb_x.v1")] [MessagePackObject(AllowPrivate = true)] @@ -56,7 +59,7 @@ namespace GameCult.Caching.Tests [Key(1)] public int Value; } - [CultDocument("tests.rb_y", "tests.rb_y.v1", CompatibleSchemaIds = new[] { "tests.rb_x.v1" })] + [CultDocument("tests.rb_y", "tests.rb_y.v1", CompatibleSchemaIds = new[] { RbXId })] [MessagePackObject(AllowPrivate = true)] internal sealed class RbY { @@ -371,28 +374,21 @@ namespace GameCult.Caching.Tests Assert.Ignore("a refusal is a store's."); var path = NewPath(kind); using (var seed = Open(path, kind, ABC)) - { - PutB(seed, K1); PutB(seed, K2); - } using var cache = Open(path, kind, AB); using var other = Open(path, kind, ABC); Assert.That(cache.Remove(K2), Is.True); cache.UpsertAsync(typeof(EvA), new EvA { Name = "k2", Value = 9 }, K2).GetAwaiter().GetResult(); - - Assert.That(other.Commit(batch => - { - batch.Remove(K1); - batch.Upsert(typeof(EvC), new EvC { Name = "k1" }, K1); - }), Is.True); cache.UpsertAsync(typeof(EvA), new EvA { Name = "k1", Value = 1 }, K1).GetAwaiter().GetResult(); + // Another writer puts a type this cache does not know at k1 before the flush: k1's staged add is refused under the lock. + Assert.That(other.Commit(batch => batch.Upsert(typeof(EvC), new EvC { Name = "k1" }, K1)), Is.True); var refusal = Assert.Throws(() => cache.FlushAllBackingStores())!; - Assert.That(refusal.RecordKey, Is.EqualTo(K1.Value), "k1 holds a type this cache does not know"); + Assert.That(refusal.RecordKey, Is.EqualTo(K1.Value)); cache.FlushAllBackingStores(); - Assert.That(TypeAt(path, kind, AB, K2), Is.EqualTo(typeof(EvA)), "the retype of k2 survived the refusal of k1"); + Assert.That(TypeAt(path, kind, ABC, K2), Is.EqualTo(typeof(EvA)), "the retype of k2 survived the refusal of k1"); } // finding rebase-check-was-reachable, ruling variant-rebase-follows-plain-write: a variant rebased onto a base of another type @@ -401,6 +397,7 @@ namespace GameCult.Caching.Tests [TestCaseSource(nameof(VariantKinds))] public void ARebaseChangesTypeExactlyWhereAPlainWriteMay(string kind) { + Assert.That(Rb.GetRequired(typeof(RbX)).SchemaId, Is.EqualTo(RbXId)); using var cache = Open(NewPath(kind), kind, Rb); var baseX = new CultRecordKey("base-x"); var baseY = new CultRecordKey("base-y"); From 1ff5ae79dc3ebb978f732c93e563aaf845b79a44 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 28 Oct 2026 06:26:50 +0100 Subject: [PATCH 17/88] Validate: the key-home check needs no eviction exception (an evicted key's home is already in the batch's homes) Co-Authored-By: Claude Opus 5.5 --- src/GameCult.Caching/CultCache.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index b7223f6a..797540b0 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -3126,7 +3126,7 @@ namespace GameCult.Caching $"{descriptor.SchemaName} record {stored.Key.Value} was loaded from {source} but its home is {home?.ToString() ?? "no store"}."); // One key, one store: a key never moves between homes, and two stores never both deliver it. - if (!removedKeys.Contains(stored.Key.Value) && _entries.TryGetValue(stored.Key.Value, out var present) && + if (_entries.TryGetValue(stored.Key.Value, out var present) && Home(present.Descriptor.DocumentType) is var held && Home(descriptor.DocumentType) is var incoming && held != incoming) From 2239fb9b5f3991819b8b7892fd10cfd50244dfd3 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 28 Oct 2026 06:29:21 +0100 Subject: [PATCH 18/88] Contract and CHANGELOG: a one-batch retype is an eviction then an add, a rebase follows the plain-write rule, the protected per-key judge Co-Authored-By: Claude Opus 5.5 --- src/GameCult.Caching/Contracts/cultcache-persistence-format.md | 2 +- unity/org.gamecult.cultlib/CHANGELOG.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index 6548e8d9..29342f98 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -213,7 +213,7 @@ compatible), or when it is the type that reads the stored record's published ent id of its own schema name that it reads whole is its own and is restamped. Anything else is refused, another registered type's record included, and so is a record the written type would read only by dropping persisted slots. No registry enters the decision, `Shared` included: the answer for a given written entry, stored id and catalog is the same lent a registry or not, and a cache's registered types do not make -each other's records writable. Retyping a key is a removal and an add: in one batch (the removal called before the add) or staged apart, with or without a backing store. The removal half is judged as a removal, so the key is retyped only when the removal rule allows it. A removal or check of a key goes to the +each other's records writable. Retyping a key is a removal and an add: in one batch (the removal called before the add) or staged apart, with or without a backing store. The removal half is judged as a removal, so the key is retyped only when the removal rule allows it. A one-batch retype is staged as the held record's eviction and then the add: a watcher of the old type sees Removed and a watcher of the new type sees Added, a global's slot is freed, and every check, validator and store judge sees what it sees when the two are staged apart; this holds for plain records, variants and flattens alike (a Flatten after a Remove in the same batch flattens nothing and is refused). A variant rebased onto a base of another type follows the plain-write rule: where a plain write of the new type would own the variant's record (the new type declares the old id) the rebase commits and the variant becomes that type, and where a plain write would be refused the rebase is refused. A removal or check of a key goes to the registry's removal rule (a record a registered type declares, or reads whole, may be removed), and a cacheless call, which has no registry, removes a record only when an entry it writes owns it; a call that writes no entry removes nothing. diff --git a/unity/org.gamecult.cultlib/CHANGELOG.md b/unity/org.gamecult.cultlib/CHANGELOG.md index 081ee4f1..ef98338b 100644 --- a/unity/org.gamecult.cultlib/CHANGELOG.md +++ b/unity/org.gamecult.cultlib/CHANGELOG.md @@ -25,6 +25,7 @@ Releases as 3.0.0 (major). ### Added +- `CacheBackingStore.RefuseUndeclaredOnDisk(written, catalog, heldAt, writtenAt, removed, durableAt, foreignAt, unstage, cacheless)` is the one protected per-key ownership judge for a store kind outside the library: it applies the whole rule under the store's lock (the written type owns a record by its own id, an id it lists or an older id of its own name it reads whole; a key the write removes or checks, and a key the cache removed since the last flush, is owned by any registered type that declares or reads the record whole) and refuses through `Refuse`. Both shipped stores call it. Additive: the earlier overload taking a store-built `undeclaredAt` judge is unchanged. A batch that removes a key and adds a record there is staged as the held record's eviction and then the add, so a watcher of the old type sees `Removed`, a watcher of the new type sees `Added`, and a variant rebase onto a base of another type changes type exactly where a plain write may. - A cacheless `WriteRecord` or `WriteDocument`, and an attach, hold the store's gate while they run, and user code they run (a document's property getters, a constructor while a copied record is read) now throws `InvalidOperationException` if it enters a cache it does not hold or a second store; it used to wait forever against an attach of the same store. A store call that queued on an attach that then failed runs under the gate that owns the store, not beside a cacheless call or a second cache's attach. An attach that fails takes its registry back, so a `PullAll` on the store afterwards is refused as unattached instead of loading records the next cache never sees. A cacheless `WriteRecord` or `WriteDocument` made while the same thread already holds that store from user code another cacheless write of the store runs, or from a journal an attach's pull runs (then "The store is attached to a cache, write through the cache.", as after the attach) throws `InvalidOperationException`; it used to write the file behind the attaching cache and take that cache's registry, so the cache could never flush or pull the store again. A journal that calls its cache's store (`Push`, `Delete`, `PushAll`, `CommitBatch`, `PullAll`, `ReadStoreSnapshot`, `JudgeChecks`) while that cache's `AddBackingStore` runs the journal now throws "A journal entered the cache it journals", as it does after the attach; it used to stage or write records the cache never admitted, so the cache and the file disagreed. - `SingleFileBackingStore.ReadStoreSnapshot()` reads a store file once, under the store's in-process gate, with the reader a cache's store opens with, and returns the durable snapshot or null when nothing is at the path. `SingleFileBackingStore.WriteRecord(record, catalogEntry, holdsIds, registry)` writes one record without a cache: under the store's lock every other record and catalog entry is copied as stored, the file is replaced atomically, and the header is decided as a flush decides it. It refuses on a store attached to a cache. - `SingleFileBackingStore.WriteDocument(key, document, storedAt, registry)` writes one typed document without a cache, as a cache write does: it plans element ids, encodes the document as `T` (the type its schema id and ids are taken from, also for a derived instance) and marks the store when it holds an id; a failed write gives the minted ids back. It refuses on a store attached to a cache. The `registry` of `WriteRecord` and `WriteDocument` serves that call only; a store keeps none. From c8563e533f7c68eb132952d6954c5acfa2efe01f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 28 Oct 2026 06:39:26 +0100 Subject: [PATCH 19/88] Single-file judge: the removals need not travel as checks (a removal has no upsert); pin the flatten and retype refusal texts Co-Authored-By: Claude Opus 5.5 --- src/GameCult.Caching/CultCache.cs | 2 +- tests/GameCult.Caching.Tests/RetypeEvictionTests.cs | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 797540b0..2508f77c 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -4872,7 +4872,7 @@ namespace GameCult.Caching durable, _staged.Concat(upserts.Select(staged => staged.Record.Key)).Concat(removals).Concat(checks).Concat(StagedChecks), upserts, - checks.Concat(removals)); + checks); // A key the write also names as a removal or check (the request's, or one the cache removed here since its last flush) is a // retype's removal half: it is judged by the removal rule, and the record staged there then lands. A refused key's durable diff --git a/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs b/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs index fd38541d..ad1c4072 100644 --- a/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs +++ b/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs @@ -251,11 +251,12 @@ namespace GameCult.Caching.Tests batch.UpsertVariant(K3, baseA, new[] { cache.Override(nameof(EvA.Name), "v3") }); }); - Assert.Throws(() => cache.Commit(batch => + var flatten = Assert.Throws(() => cache.Commit(batch => { batch.Remove(K3); batch.Flatten(K3); - })); + }))!; + Assert.That(flatten.Message, Does.Contain("Flatten(k3)").And.Contain("not a variant")); Assert.That(cache.GetStored(K3)!.Variant, Is.Not.Null, "the refused batch changed nothing"); Assert.That(cache.Remove(K3), Is.True); Assert.Throws(() => cache.FlattenAsync(K3).GetAwaiter().GetResult()); @@ -305,11 +306,13 @@ namespace GameCult.Caching.Tests using var cache = Open(NewPath(kind), kind, AB); PutB(cache, K2); - Assert.Throws(() => cache.Commit(batch => + var refused = Assert.Throws(() => cache.Commit(batch => { batch.Upsert(typeof(EvA), new EvA { Name = "k2", Value = 1 }, K2); batch.Upsert(typeof(EvA), new EvA { Name = "k2", Value = 2 }, K2); - })); + }))!; + if (kind == "storeless") + Assert.That(refused.Message, Does.Contain("would retype the key").And.Contain("Remove the record first, in the same batch or an earlier commit, then add the other type's.")); Assert.That(cache.GetStored(K2)!.Descriptor.DocumentType, Is.EqualTo(typeof(EvB))); } From a9601cdd48c25171a199c255ac65fc3cfd9da99c Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 28 Oct 2026 07:23:22 +0100 Subject: [PATCH 20/88] Validate: an evicted key is absent to the one-key-one-store check, so a one-batch retype across home stores is the batch-spanning refusal with its own text; contract states it; pinned Co-Authored-By: Claude Opus 5.5 --- .../Contracts/cultcache-persistence-format.md | 2 +- src/GameCult.Caching/CultCache.cs | 9 ++-- .../RetypeEvictionTests.cs | 50 ++++++++++++++++++- 3 files changed, 56 insertions(+), 5 deletions(-) diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index 29342f98..93faf63f 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -213,7 +213,7 @@ compatible), or when it is the type that reads the stored record's published ent id of its own schema name that it reads whole is its own and is restamped. Anything else is refused, another registered type's record included, and so is a record the written type would read only by dropping persisted slots. No registry enters the decision, `Shared` included: the answer for a given written entry, stored id and catalog is the same lent a registry or not, and a cache's registered types do not make -each other's records writable. Retyping a key is a removal and an add: in one batch (the removal called before the add) or staged apart, with or without a backing store. The removal half is judged as a removal, so the key is retyped only when the removal rule allows it. A one-batch retype is staged as the held record's eviction and then the add: a watcher of the old type sees Removed and a watcher of the new type sees Added, a global's slot is freed, and every check, validator and store judge sees what it sees when the two are staged apart; this holds for plain records, variants and flattens alike (a Flatten after a Remove in the same batch flattens nothing and is refused). A variant rebased onto a base of another type follows the plain-write rule: where a plain write of the new type would own the variant's record (the new type declares the old id) the rebase commits and the variant becomes that type, and where a plain write would be refused the rebase is refused. A removal or check of a key goes to the +each other's records writable. Retyping a key is a removal and an add: in one batch (the removal called before the add) or staged apart, with or without a backing store. The removal half is judged as a removal, so the key is retyped only when the removal rule allows it. A one-batch retype is staged as the held record's eviction and then the add: a watcher of the old type sees Removed and a watcher of the new type sees Added, a global's slot is freed, and every check, validator and store judge sees what it sees when the two are staged apart; this holds for plain records, variants and flattens alike (a Flatten after a Remove in the same batch flattens nothing and is refused). A one-batch retype across two home stores is refused, by the rule that a commit lands in one home store: the removal is in the first store and the add in the second, so the batch spans both. Stage the removal and the add apart, as two commits (not atomic); the key is then in exactly one store. A variant rebased onto a base of another type follows the plain-write rule: where a plain write of the new type would own the variant's record (the new type declares the old id) the rebase commits and the variant becomes that type, and where a plain write would be refused the rebase is refused. A removal or check of a key goes to the registry's removal rule (a record a registered type declares, or reads whole, may be removed), and a cacheless call, which has no registry, removes a record only when an entry it writes owns it; a call that writes no entry removes nothing. diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 2508f77c..f5513807 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -3107,6 +3107,7 @@ namespace GameCult.Caching var removedKeys = removed?.ToHashSet(StringComparer.Ordinal) ?? new HashSet(StringComparer.Ordinal); var homes = new List(); var globals = new Dictionary(_globals); + var evictedKeys = evicted.Select(stored => stored.Key.Value).ToHashSet(StringComparer.Ordinal); foreach (var stored in evicted) { var type = stored.Descriptor.DocumentType; @@ -3125,8 +3126,9 @@ namespace GameCult.Caching throw new InvalidOperationException( $"{descriptor.SchemaName} record {stored.Key.Value} was loaded from {source} but its home is {home?.ToString() ?? "no store"}."); - // One key, one store: a key never moves between homes, and two stores never both deliver it. - if (_entries.TryGetValue(stored.Key.Value, out var present) && + // One key, one store: a key never moves between homes, and two stores never both deliver it. An evicted key is absent + // here, as it is staged apart; a retype across two home stores is then the batch spanning both (below): one rule, one text. + if (!evictedKeys.Contains(stored.Key.Value) && _entries.TryGetValue(stored.Key.Value, out var present) && Home(present.Descriptor.DocumentType) is var held && Home(descriptor.DocumentType) is var incoming && held != incoming) @@ -3145,7 +3147,8 @@ namespace GameCult.Caching var distinct = homes.Distinct().ToArray(); if (distinct.Length > 1) throw new InvalidOperationException( - $"The batch spans {distinct[0]} and {distinct[1]}; a commit lands in one home store."); + $"The batch spans {distinct[0]} and {distinct[1]}; a commit lands in one home store. " + + "To move a key between stores, stage its removal and its add apart (two commits, not one atomic commit)."); return distinct.FirstOrDefault(); void Claim(CultStoredDocument stored, bool upsert) diff --git a/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs b/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs index ad1c4072..02d93c80 100644 --- a/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs +++ b/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs @@ -262,7 +262,55 @@ namespace GameCult.Caching.Tests Assert.Throws(() => cache.FlattenAsync(K3).GetAwaiter().GetResult()); } - // finding removal-bookkeeping-unpinned, mutant X2 (a retype elsewhere in the batch skips every key's staging claim): a batch + // finding cross-store-retype-decided-by-key-home: a one-batch retype between two home stores is refused by the rule that a commit + // lands in one home store, with that rule's text (an evicted key is absent to the one-key-one-store check, as when staged + // apart); staged apart, as two commits, it leaves the key in exactly one store. + [Test] + public void ARetypeAcrossTwoHomeStoresInOneBatchIsRefusedByTheBatchSpanningRuleAndStagedApartLeavesOneStore() + { + string[] KeysIn(string path) => + new SingleFileMessagePackBackingStore(path).ReadStoreSnapshot()?.Records.Select(record => record.Key).ToArray() ?? Array.Empty(); + + CultCache OpenTwo(string a, string b) + { + var cache = new CultCache(AB, CultCacheMessagePack.CreateCodec(AB)); + cache.AddBackingStore(new SingleFileMessagePackBackingStore(a), typeof(EvA)); + cache.AddBackingStore(new SingleFileMessagePackBackingStore(b), typeof(EvB)); + cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + return cache; + } + + var pathA = NewPath("single-file"); + var pathB = NewPath("single-file"); + Directory.CreateDirectory(Path.GetDirectoryName(pathA)!); + Directory.CreateDirectory(Path.GetDirectoryName(pathB)!); + using (var cache = OpenTwo(pathA, pathB)) + { + PutB(cache, K1); + Assert.That(KeysIn(pathB), Is.EqualTo(new[] { "k1" })); + + var refused = Assert.Throws(() => cache.Commit(batch => + { + batch.Remove(K1); + batch.Upsert(typeof(EvA), new EvA { Name = "k1" }, K1); + }))!; + Assert.That(refused.Message, Does.Contain("The batch spans").And.Contain("a commit lands in one home store") + .And.Contain("stage its removal and its add apart").And.Not.Contain("second store")); + Assert.That(KeysIn(pathA), Is.Empty, "the refused batch wrote nothing"); + Assert.That(KeysIn(pathB), Is.EqualTo(new[] { "k1" })); + Assert.That(cache.GetStored(K1)!.Descriptor.DocumentType, Is.EqualTo(typeof(EvB))); + + Assert.That(cache.Remove(K1), Is.True); + cache.FlushAsync().GetAwaiter().GetResult(); + cache.UpsertAsync(typeof(EvA), new EvA { Name = "k1" }, K1).GetAwaiter().GetResult(); + cache.FlushAsync().GetAwaiter().GetResult(); + } + + Assert.That(KeysIn(pathA), Is.EqualTo(new[] { "k1" }), "staged apart the key is in the add's store"); + Assert.That(KeysIn(pathB), Is.Empty, "and no longer in the removal's"); + } + + // finding removal-bookkeeping-unpinned, mutant X2(a retype elsewhere in the batch skips every key's staging claim): a batch // holding two retypes commits, and a cross-type add at a key it did not remove is still refused beside a retype. [TestCaseSource(nameof(Kinds))] public void EachKeyOfABatchIsJudgedByItsOwnRemoval(string kind) From d8a238508064e0614a15e6bf478622e865fa0a0d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 28 Oct 2026 07:23:59 +0100 Subject: [PATCH 21/88] Flatten acts on what the batch stages at the key: a staged variant is flattened as it lands, a removal or plain record there leaves nothing to flatten; pinned Co-Authored-By: Claude Opus 5.5 --- src/GameCult.Caching/CultCache.cs | 25 +++++-- .../RetypeEvictionTests.cs | 74 +++++++++++++++++++ 2 files changed, 93 insertions(+), 6 deletions(-) diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index f5513807..8c80116c 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -2141,8 +2141,20 @@ namespace GameCult.Caching public void Flatten(CultRecordKey key) { ThrowIfSealed(); - if (Removals.Contains(key.Value)) - throw new InvalidOperationException($"Flatten({key.Value}): the record is not a variant."); + // A flatten acts on what the key will hold, as staged apart: a variant staged here is flattened as it lands; a removal or a + // plain record staged here leaves nothing to flatten; with nothing staged it flattens the variant the cache holds. + if (Operations.TryGetValue(key.Value, out var staged)) + { + if (staged.Variant != null) + { + Operations[key.Value] = new StagedOperation(key, null, staged.Variant, true); + return; + } + + if (!staged.Flatten) + throw new InvalidOperationException($"Flatten({key.Value}): the record is not a variant."); + } + Operations[key.Value] = new StagedOperation(key, null, null, true); } @@ -2728,8 +2740,9 @@ namespace GameCult.Caching new(key, MintStoredAt(PreviousStoredAt(key), Clock()), descriptor, document); // An unresolved variant: the admitting hold resolves it against its base before anything lands. - private CultStoredDocument StampVariant(CultRecordKey key, CultDocumentDescriptor descriptor, CultVariantDelta delta) => - new(key, MintStoredAt(PreviousStoredAt(key), Clock()), descriptor, delta, null, _codec); + // A staged variant that a Flatten followed lands as the plain record of its resolution (flattens). + private CultStoredDocument StampVariant(CultRecordKey key, CultDocumentDescriptor descriptor, CultVariantDelta delta, bool flattens = false) => + new(key, MintStoredAt(PreviousStoredAt(key), Clock()), descriptor, delta, null, _codec) { Flattens = flattens }; // The plain record a Flatten stages: the variant's resolved document, under the variant's own key and schema. private CultStoredDocument StampFlatten(CultRecordKey key) @@ -2830,7 +2843,7 @@ namespace GameCult.Caching var upserts = batch.Operations.Values .Where(op => !op.IsRemoval) .Select(op => op.Stored != null ? Stamp(op.Key, op.Stored.Descriptor, op.Stored.Document) - : op.Variant != null ? StampVariant(op.Key, DescriptorOfBase(op.Key, op.Variant.BaseKey, batch), op.Variant) + : op.Variant != null ? StampVariant(op.Key, DescriptorOfBase(op.Key, op.Variant.BaseKey, batch), op.Variant, op.Flatten) : StampFlatten(op.Key)) .ToArray(); // A removal evicts the record the cache holds at the key, and a record staged at the key lands after the eviction, as @@ -3411,7 +3424,7 @@ namespace GameCult.Caching ?? throw new InvalidOperationException($"Variant {key} resolved to nothing."); stack.RemoveAt(stack.Count - 1); - return done[key] = isVariant + return done[key] = isVariant && !variant.Flattens ? variant.Resolved(document, minted ? new CultVariantDelta(delta.BaseKey, overrides) : delta, variant.IdsInMemoryOnly || (minted && source != null), _codec) : new CultStoredDocument(variant.Key, variant.StoredAt, variant.Descriptor, document) { diff --git a/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs b/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs index 02d93c80..9f1cdd9f 100644 --- a/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs +++ b/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs @@ -262,6 +262,80 @@ namespace GameCult.Caching.Tests Assert.Throws(() => cache.FlattenAsync(K3).GetAwaiter().GetResult()); } + // finding flatten-guard-reads-removals: a Flatten acts on what the key will hold, as staged apart. A variant staged at the key + // (after a removal or not) is flattened as it lands; a plain record staged there leaves nothing to flatten. + [TestCaseSource(nameof(VariantKinds))] + public void AFlattenOfAVariantStagedInTheSameBatchFlattensItAsStagedApart(string kind) + { + foreach (var oneBatch in new[] { true, false }) + { + using var cache = Open(NewPath(kind), kind, AB); + var baseA = new CultRecordKey("base-a"); + Assert.That(cache.Commit(batch => + { + batch.Upsert(typeof(EvA), new EvA { Name = "base-a", Value = 1 }, baseA); + batch.UpsertVariant(K3, baseA, new[] { cache.Override(nameof(EvA.Name), "v3") }); + batch.Upsert(typeof(EvB), new EvB { Name = "k1", Value = 2 }, K1); + }), Is.True); + + // Remove, UpsertVariant, Flatten: the plain record of the variant's resolution, retyped. + if (oneBatch) + Assert.That(cache.Commit(batch => + { + batch.Remove(K1); + batch.UpsertVariant(K1, baseA, new[] { cache.Override(nameof(EvA.Name), "v1") }); + batch.Flatten(K1); + }), Is.True); + else + { + Assert.That(cache.Remove(K1), Is.True); + cache.UpsertVariantAsync(K1, baseA, new[] { cache.Override(nameof(EvA.Name), "v1") }).GetAwaiter().GetResult(); + cache.FlattenAsync(K1).GetAwaiter().GetResult(); + } + + var flattened = cache.GetStored(K1)!; + Assert.That(flattened.Variant, Is.Null, $"one batch {oneBatch}"); + Assert.That(flattened.Descriptor.DocumentType, Is.EqualTo(typeof(EvA))); + Assert.That(((EvA)flattened.Document).Name, Is.EqualTo("v1")); + + // A held variant rebased and flattened in one batch flattens the rebase, not the held record. + if (oneBatch) + Assert.That(cache.Commit(batch => + { + batch.UpsertVariant(K3, baseA, new[] { cache.Override(nameof(EvA.Name), "v3b") }); + batch.Flatten(K3); + }), Is.True); + else + { + cache.UpsertVariantAsync(K3, baseA, new[] { cache.Override(nameof(EvA.Name), "v3b") }).GetAwaiter().GetResult(); + cache.FlattenAsync(K3).GetAwaiter().GetResult(); + } + + Assert.That(cache.GetStored(K3)!.Variant, Is.Null, $"one batch {oneBatch}"); + Assert.That(((EvA)cache.GetStored(K3)!.Document).Name, Is.EqualTo("v3b")); + } + } + + [TestCaseSource(nameof(VariantKinds))] + public void AFlattenOfAPlainRecordStagedInTheSameBatchIsRefusedAsStagedApart(string kind) + { + using var cache = Open(NewPath(kind), kind, AB); + PutB(cache, K2); + + var refused = Assert.Throws(() => cache.Commit(batch => + { + batch.Remove(K2); + batch.Upsert(typeof(EvA), new EvA { Name = "k2" }, K2); + batch.Flatten(K2); + }))!; + Assert.That(refused.Message, Does.Contain("Flatten(k2)").And.Contain("not a variant")); + Assert.That(cache.GetStored(K2)!.Descriptor.DocumentType, Is.EqualTo(typeof(EvB)), "the refused batch changed nothing"); + + Assert.That(cache.Remove(K2), Is.True); + cache.UpsertAsync(typeof(EvA), new EvA { Name = "k2" }, K2).GetAwaiter().GetResult(); + Assert.Throws(() => cache.FlattenAsync(K2).GetAwaiter().GetResult()); + } + // finding cross-store-retype-decided-by-key-home: a one-batch retype between two home stores is refused by the rule that a commit // lands in one home store, with that rule's text (an evicted key is absent to the one-key-one-store check, as when staged // apart); staged apart, as two commits, it leaves the key in exactly one store. From 8c0b55c4509a078be2ee1ee444353f334cd4e320 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 28 Oct 2026 07:24:19 +0100 Subject: [PATCH 22/88] OutsideStoreJudgeTests: a store kind in an assembly with no InternalsVisibleTo applies the whole per-key rule through the protected judge Co-Authored-By: Claude Opus 5.5 --- CultLib.sln | 15 + .../GameCult.Caching.OutsideStoreTests.csproj | 22 ++ .../OutsideStoreJudgeTests.cs | 359 ++++++++++++++++++ 3 files changed, 396 insertions(+) create mode 100644 tests/GameCult.Caching.OutsideStoreTests/GameCult.Caching.OutsideStoreTests.csproj create mode 100644 tests/GameCult.Caching.OutsideStoreTests/OutsideStoreJudgeTests.cs diff --git a/CultLib.sln b/CultLib.sln index 217f2d1f..a8baa383 100644 --- a/CultLib.sln +++ b/CultLib.sln @@ -51,6 +51,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Caching.ConsumerBu EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Caching.GeneratorConsumer", "tests\GameCult.Caching.GeneratorConsumer\GameCult.Caching.GeneratorConsumer.csproj", "{1A75EA8F-00B3-49AB-8398-B1B7723BE10B}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Caching.OutsideStoreTests", "tests\GameCult.Caching.OutsideStoreTests\GameCult.Caching.OutsideStoreTests.csproj", "{A54C2C29-941A-46A9-9DDE-9711B2F2EADD}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -289,6 +291,18 @@ Global {1A75EA8F-00B3-49AB-8398-B1B7723BE10B}.Release|x64.Build.0 = Release|Any CPU {1A75EA8F-00B3-49AB-8398-B1B7723BE10B}.Release|x86.ActiveCfg = Release|Any CPU {1A75EA8F-00B3-49AB-8398-B1B7723BE10B}.Release|x86.Build.0 = Release|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Debug|x64.ActiveCfg = Debug|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Debug|x64.Build.0 = Debug|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Debug|x86.ActiveCfg = Debug|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Debug|x86.Build.0 = Debug|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Release|Any CPU.Build.0 = Release|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Release|x64.ActiveCfg = Release|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Release|x64.Build.0 = Release|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Release|x86.ActiveCfg = Release|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -314,5 +328,6 @@ Global {FEF9DA88-2177-4FD0-885D-7096ACFA8A9D} = {B48EAE30-152F-01E3-6E31-82B350494D3C} {D3138F3C-9531-4906-A5D3-74BD857D0685} = {39D316AB-ABB5-49A2-8E83-CAF4C75B48F4} {1A75EA8F-00B3-49AB-8398-B1B7723BE10B} = {39D316AB-ABB5-49A2-8E83-CAF4C75B48F4} + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD} = {39D316AB-ABB5-49A2-8E83-CAF4C75B48F4} EndGlobalSection EndGlobal diff --git a/tests/GameCult.Caching.OutsideStoreTests/GameCult.Caching.OutsideStoreTests.csproj b/tests/GameCult.Caching.OutsideStoreTests/GameCult.Caching.OutsideStoreTests.csproj new file mode 100644 index 00000000..c40c048d --- /dev/null +++ b/tests/GameCult.Caching.OutsideStoreTests/GameCult.Caching.OutsideStoreTests.csproj @@ -0,0 +1,22 @@ + + + + + net10.0 + false + enable + GameCult.Caching.OutsideStoreTests + + + + + + + + + + + + + + diff --git a/tests/GameCult.Caching.OutsideStoreTests/OutsideStoreJudgeTests.cs b/tests/GameCult.Caching.OutsideStoreTests/OutsideStoreJudgeTests.cs new file mode 100644 index 00000000..150fdf4d --- /dev/null +++ b/tests/GameCult.Caching.OutsideStoreTests/OutsideStoreJudgeTests.cs @@ -0,0 +1,359 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Caching.OutsideStoreTests +{ + // variants:ruling:store-kind-per-key-judge: a store kind written outside the library applies the whole per-key ownership rule + // through the one protected judge (CacheBackingStore.RefuseUndeclaredOnDisk), using only public and protected members. This + // assembly has no InternalsVisibleTo, so a judge parameter type, or something it needs, turning internal breaks it here. + public class OutsideStoreJudgeTests + { + [CultDocument("tests.out_a", "tests.out_a.v1")] + [MessagePackObject] + public sealed class OutA + { + [Key(0)] [CultName] public string Name { get; set; } = string.Empty; + [Key(1)] public int Value { get; set; } + } + + [CultDocument("tests.out_b", "tests.out_b.v1")] + [MessagePackObject] + public sealed class OutB + { + [Key(0)] [CultName] public string Name { get; set; } = string.Empty; + [Key(1)] public int Value { get; set; } + } + + // RbX's schema id, which RbY lists; the test below asserts it, so a changed derivation fails there. + private const string RbXId = "sha256:25087e0d76fd5b9ac950723ccd47f3f3ac1428b53b49ca9bda9aae8b38a0220a"; + + [CultDocument("tests.rb_x", "tests.rb_x.v1")] + [MessagePackObject] + public sealed class RbX + { + [Key(0)] [CultName] public string Name { get; set; } = string.Empty; + [Key(1)] public int Value { get; set; } + } + + [CultDocument("tests.rb_y", "tests.rb_y.v1", CompatibleSchemaIds = new[] { RbXId })] + [MessagePackObject] + public sealed class RbY + { + [Key(0)] [CultName] public string Name { get; set; } = string.Empty; + [Key(1)] public int Value { get; set; } + } + + [CultDocument("tests.rb_z", "tests.rb_z.v1")] + [MessagePackObject] + public sealed class RbZ + { + [Key(0)] [CultName] public string Name { get; set; } = string.Empty; + [Key(1)] public int Value { get; set; } + } + + // What the "disk" holds: records by key and the catalog that publishes their ids. + private sealed class Disk + { + public readonly Dictionary Records = new(); + public readonly List Catalog = new(); + private int _version; + + public void Put(CultPersistedRecord record, CultSchemaCatalogEntry entry) + { + if (Catalog.All(known => known.SchemaId != entry.SchemaId)) + Catalog.Add(entry); + Records[record.Key] = (record, (++_version).ToString()); + } + + public string? SchemaIdAt(string key) => Records.TryGetValue(key, out var held) ? held.Record.SchemaId : null; + } + + // A store kind with no InternalsVisibleTo: an in-memory disk, flushed and committed under the protected judge. + private sealed class OutsideStore : CacheBackingStore + { + private readonly Disk _disk; + private readonly CultCodec _codec; + private readonly HashSet _staged = new(StringComparer.Ordinal); + + public OutsideStore(Disk disk, CultCodec codec) + { + _disk = disk; + _codec = codec; + } + + private object Decode(Type type, byte[] payload) => _codec.Deserialize(type, type, payload)!; + + private CultStoredDocument? DurableAt(string key) + { + if (!_disk.Records.TryGetValue(key, out var held)) + return null; + try + { + return ToStoredDocument(held.Record, held.Version, _disk.Catalog, Decode, out _); + } + catch (Exception) + { + return null; + } + } + + public override void PullAll() => Held(() => + { + var loaded = _disk.Records + .Where(pair => !Entries.TryGetValue(pair.Key, out var known) || !SameStoredIdentity(known.StoredVersion, pair.Value.Version)) + .Select(pair => DurableAt(pair.Key)) + .OfType() + .ToArray(); + var dropped = Entries.Values.Where(known => !_disk.Records.ContainsKey(known.Key.Value)).ToArray(); + if (loaded.Length > 0 || dropped.Length > 0) + Loaded?.Invoke(loaded, dropped); + foreach (var stored in dropped) + Entries.TryRemove(stored.Key.Value, out _); + foreach (var stored in loaded) + Entries[stored.Key.Value] = stored; + }); + + public override void Push(CultStoredDocument entry) => Held(() => + { + Entries[entry.Key.Value] = entry; + _staged.Add(entry.Key.Value); + IsDirty = true; + }); + + public override void Delete(CultStoredDocument entry) => Held(() => + { + Entries.TryRemove(entry.Key.Value, out _); + _staged.Add(entry.Key.Value); + IsDirty = true; + }); + + public override void PushAll() => Held(() => Apply(Array.Empty(), Array.Empty(), Array.Empty())); + + public override CultCommitOutcome CommitBatch(CultCommitRequest request, bool wait) => Held(() => + { + Apply(request.Upserts, request.Deletes, request.Checks); + foreach (var entry in request.Deletes) + Entries.TryRemove(entry.Key.Value, out _); + foreach (var entry in request.Upserts) + Entries[entry.Key.Value] = entry; + return CultCommitOutcome.Committed; + }); + + private void Apply(IReadOnlyList upserts, IReadOnlyList deletes, IEnumerable checks) + { + var writes = upserts.ToDictionary(entry => entry.Key.Value, StringComparer.Ordinal); + foreach (var key in _staged) + { + if (Entries.TryGetValue(key, out var pushed)) + writes.TryAdd(key, pushed); + } + + var removals = deletes.Select(entry => entry.Key.Value) + .Concat(_staged.Where(key => !writes.ContainsKey(key) && !Entries.ContainsKey(key))) + .ToArray(); + var judged = removals.Concat(checks).ToArray(); + var entries = writes.ToDictionary(pair => pair.Key, pair => pair.Value.Descriptor.ToCatalogEntry(), StringComparer.Ordinal); + + // The whole rule, from the protected judge: the same call both shipped stores make. + RefuseUndeclaredOnDisk( + writes.Keys.Concat(judged).Concat(StagedChecks), + _disk.Catalog, + key => _disk.Records.TryGetValue(key, out var held) ? held.Record : null, + key => entries.TryGetValue(key, out var entry) ? entry : null, + judged, + DurableAt, + _ => null, + key => + { + _staged.Remove(key); + IsDirty = _staged.Count > 0; + }); + + foreach (var (key, document) in writes) + { + var type = document.Descriptor.DocumentType; + _disk.Put( + new CultPersistedRecord + { + Key = key, + SchemaId = document.Descriptor.SchemaId, + StoredAt = document.StoredAt, + Payload = _codec.Serialize(document.Document, type, type) + }, + entries[key]); + } + + foreach (var key in removals.Where(key => !writes.ContainsKey(key))) + _disk.Records.Remove(key); + _staged.Clear(); + MarkFlushSucceeded(); + } + } + + private static readonly CultDocumentRegistry AB = CultDocumentRegistry.ForTypes(new[] { typeof(OutA), typeof(OutB) }); + private static readonly CultDocumentRegistry Rb = CultDocumentRegistry.ForTypes(new[] { typeof(RbX), typeof(RbY), typeof(RbZ) }); + private static readonly CultRecordKey K = new("k"); + private static readonly CultRecordKey K2 = new("k2"); + + private static CultCache Open(Disk disk, CultDocumentRegistry registry) + { + var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry)); + cache.AddBackingStore(new OutsideStore(disk, cache.Codec!)); + cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + return cache; + } + + // Another writer puts a record of this type at the key on the disk, as the cache's own store would have. + private static void PutOnDisk(Disk disk, CultDocumentRegistry registry, string key, T document, string? asSchemaId = null, CultSchemaCatalogEntry? entry = null) + where T : class + { + var codec = CultCacheMessagePack.CreateCodec(registry); + var descriptor = registry.GetRequired(); + disk.Put( + new CultPersistedRecord + { + Key = key, + SchemaId = asSchemaId ?? descriptor.SchemaId, + StoredAt = DateTimeOffset.UtcNow.ToString("O"), + Payload = codec.Serialize(document, typeof(T), typeof(T)) + }, + entry ?? descriptor.ToCatalogEntry()); + } + + private static Type HeldType(CultCache cache, CultRecordKey key) => cache.GetStored(key)!.Descriptor.DocumentType; + + // The written type's own id: a record of another registered type that this cache never loaded. + [Test] + public void AnotherRegisteredTypesUnheldRecordIsRefusedUnderTheLockAndReloaded() + { + var disk = new Disk(); + using var cache = Open(disk, AB); + PutOnDisk(disk, AB, "k", new OutB { Name = "k", Value = 2 }); + var before = disk.Records["k"]; + + Assert.That( + () => cache.Commit(batch => batch.Upsert(typeof(OutA), new OutA { Name = "k", Value = 1 }, K)), + Throws.InstanceOf()); + Assert.That(disk.Records["k"], Is.EqualTo(before), "the refused write changed nothing on the disk"); + Assert.That(HeldType(cache, K), Is.EqualTo(typeof(OutB)), "the refusal reloaded the record the store holds"); + } + + [Test] + public void AFlushOfAnotherRegisteredTypesUnheldRecordIsRefusedToo() + { + var disk = new Disk(); + using var cache = Open(disk, AB); + PutOnDisk(disk, AB, "k", new OutB { Name = "k", Value = 2 }); + + cache.UpsertAsync(typeof(OutA), new OutA { Name = "k", Value = 1 }, K).GetAwaiter().GetResult(); + Assert.That(() => cache.FlushAsync().GetAwaiter().GetResult(), Throws.InstanceOf()); + Assert.That(disk.SchemaIdAt("k"), Is.EqualTo(AB.GetRequired().SchemaId)); + Assert.That(HeldType(cache, K), Is.EqualTo(typeof(OutB))); + } + + // Claimant precedence, a listed id: the written type owns a record under an id it lists, and a type that lists nothing does not. + [Test] + public void ARecordUnderAnIdTheWrittenTypeListsIsOwnedAndOneItDoesNotListIsRefused() + { + Assert.That(Rb.GetRequired().SchemaId, Is.EqualTo(RbXId)); + var disk = new Disk(); + using var cache = Open(disk, Rb); + PutOnDisk(disk, Rb, "k", new RbX { Name = "k" }); + PutOnDisk(disk, Rb, "k2", new RbX { Name = "k2" }); + + Assert.That(cache.Commit(batch => batch.Upsert(typeof(RbY), new RbY { Name = "k", Value = 1 }, K)), Is.True); + Assert.That(disk.SchemaIdAt("k"), Is.EqualTo(Rb.GetRequired().SchemaId)); + + Assert.That( + () => cache.Commit(batch => batch.Upsert(typeof(RbZ), new RbZ { Name = "k2" }, K2)), + Throws.InstanceOf()); + Assert.That(disk.SchemaIdAt("k2"), Is.EqualTo(RbXId), "the type that lists nothing replaced nothing"); + } + + // Reads whole: an older id of the written type's own schema name, which the type reads whole, is its own and is restamped. + [Test] + public void AnOlderIdOfTheWrittenSchemaNameThatIsReadWholeIsOwnedAndRestamped() + { + var disk = new Disk(); + using var cache = Open(disk, AB); + var current = AB.GetRequired().ToCatalogEntry(); + var older = new CultSchemaCatalogEntry + { + SchemaId = "sha256:older-out-a", + SchemaName = current.SchemaName, + SchemaVersion = "tests.out_a.v0", + ContentHash = current.ContentHash, + CanonicalSchemaJson = current.CanonicalSchemaJson, + Members = current.Members + }; + PutOnDisk(disk, AB, "k", new OutA { Name = "k" }, asSchemaId: older.SchemaId, entry: older); + + Assert.That(cache.Commit(batch => batch.Upsert(typeof(OutA), new OutA { Name = "k", Value = 3 }, K)), Is.True); + Assert.That(disk.SchemaIdAt("k"), Is.EqualTo(current.SchemaId)); + } + + // Staged retype removals: a held key retypes as a removal then an add, in one batch and staged apart. + [TestCase(true)] + [TestCase(false)] + public void ARetypeOfAHeldKeyCommitsInOneBatchAndStagedApart(bool oneBatch) + { + var disk = new Disk(); + PutOnDisk(disk, AB, "k", new OutB { Name = "k", Value = 2 }); + using var cache = Open(disk, AB); + Assert.That(HeldType(cache, K), Is.EqualTo(typeof(OutB))); + + if (oneBatch) + Assert.That(cache.Commit(batch => + { + batch.Remove(K); + batch.Upsert(typeof(OutA), new OutA { Name = "k", Value = 1 }, K); + }), Is.True); + else + { + Assert.That(cache.Remove(K), Is.True); + Assert.That(cache.Commit(batch => batch.Upsert(typeof(OutA), new OutA { Name = "k", Value = 1 }, K)), Is.True); + } + + Assert.That(disk.SchemaIdAt("k"), Is.EqualTo(AB.GetRequired().SchemaId)); + Assert.That(HeldType(cache, K), Is.EqualTo(typeof(OutA))); + } + + [Test] + public void ARetypeOfAKeyTheCacheNeverLoadedIsJudgedByTheRemovalRule() + { + var disk = new Disk(); + using var cache = Open(disk, AB); + PutOnDisk(disk, AB, "k", new OutB { Name = "k", Value = 2 }); + + Assert.That(cache.Commit(batch => + { + batch.Remove(K); + batch.Upsert(typeof(OutA), new OutA { Name = "k", Value = 1 }, K); + }), Is.True, "a record a registered type reads may be removed, so the key retypes"); + Assert.That(disk.SchemaIdAt("k"), Is.EqualTo(AB.GetRequired().SchemaId)); + } + + // A removal is never licensed over a record no registered type owns, retype or not. + [Test] + public void ACommitNeverRemovesOrRetypesARecordNoRegisteredTypeOwns() + { + var disk = new Disk(); + using var cache = Open(disk, AB); + var unknown = new CultSchemaCatalogEntry { SchemaId = "tests.out_unknown", SchemaName = "tests.out_unknown" }; + disk.Put(new CultPersistedRecord { Key = "k", SchemaId = unknown.SchemaId, StoredAt = "t", Payload = new byte[] { 0x90 } }, unknown); + var before = disk.Records["k"]; + + Assert.That(() => cache.Commit(batch => + { + batch.Remove(K); + batch.Upsert(typeof(OutA), new OutA { Name = "k" }, K); + }), Throws.InstanceOf(), "retype"); + Assert.That(() => cache.Commit(batch => batch.Remove(K)), Throws.InstanceOf(), "removal"); + Assert.That(disk.Records["k"], Is.EqualTo(before)); + } + } +} From b3221c2d562c6282bf20912fea1f89613b8db55c Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 28 Oct 2026 07:25:03 +0100 Subject: [PATCH 23/88] OutsideStoreJudgeTests: the outside store tracks the versions it took itself (StoredVersion is internal) Co-Authored-By: Claude Opus 5.5 --- .../OutsideStoreJudgeTests.cs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/GameCult.Caching.OutsideStoreTests/OutsideStoreJudgeTests.cs b/tests/GameCult.Caching.OutsideStoreTests/OutsideStoreJudgeTests.cs index 150fdf4d..e47e4d5f 100644 --- a/tests/GameCult.Caching.OutsideStoreTests/OutsideStoreJudgeTests.cs +++ b/tests/GameCult.Caching.OutsideStoreTests/OutsideStoreJudgeTests.cs @@ -78,6 +78,8 @@ namespace GameCult.Caching.OutsideStoreTests private readonly Disk _disk; private readonly CultCodec _codec; private readonly HashSet _staged = new(StringComparer.Ordinal); + // StoredVersion is internal to the library, so a store kind tracks the version it last took of each key itself. + private readonly Dictionary _versions = new(StringComparer.Ordinal); public OutsideStore(Disk disk, CultCodec codec) { @@ -104,7 +106,7 @@ namespace GameCult.Caching.OutsideStoreTests public override void PullAll() => Held(() => { var loaded = _disk.Records - .Where(pair => !Entries.TryGetValue(pair.Key, out var known) || !SameStoredIdentity(known.StoredVersion, pair.Value.Version)) + .Where(pair => !_versions.TryGetValue(pair.Key, out var known) || !SameStoredIdentity(known, pair.Value.Version)) .Select(pair => DurableAt(pair.Key)) .OfType() .ToArray(); @@ -114,7 +116,10 @@ namespace GameCult.Caching.OutsideStoreTests foreach (var stored in dropped) Entries.TryRemove(stored.Key.Value, out _); foreach (var stored in loaded) + { Entries[stored.Key.Value] = stored; + _versions[stored.Key.Value] = _disk.Records[stored.Key.Value].Version; + } }); public override void Push(CultStoredDocument entry) => Held(() => @@ -185,10 +190,14 @@ namespace GameCult.Caching.OutsideStoreTests Payload = _codec.Serialize(document.Document, type, type) }, entries[key]); + _versions[key] = _disk.Records[key].Version; } foreach (var key in removals.Where(key => !writes.ContainsKey(key))) + { _disk.Records.Remove(key); + _versions.Remove(key); + } _staged.Clear(); MarkFlushSucceeded(); } From fec51f59a8eab2dbe76ea234a1036862104eb783 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 28 Oct 2026 08:01:04 +0100 Subject: [PATCH 24/88] RetypeEvictionTests: Remove, a plain Upsert, then Flatten on a held variant is refused and never drops the plain write Co-Authored-By: Claude Opus 5.5 --- .../RetypeEvictionTests.cs | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs b/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs index 9f1cdd9f..5c6bcf0a 100644 --- a/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs +++ b/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs @@ -336,6 +336,35 @@ namespace GameCult.Caching.Tests Assert.Throws(() => cache.FlattenAsync(K2).GetAwaiter().GetResult()); } + // finding flatten-after-staged-plain-unpinned: where the key held a variant, Remove, a plain Upsert, then Flatten is refused + // as staged apart, and the plain write is never dropped for a flatten of the variant the cache held. + [TestCaseSource(nameof(VariantKinds))] + public void AFlattenAfterARemovalAndAPlainWriteOnAHeldVariantIsRefusedAndKeepsTheHeldVariant(string kind) + { + using var cache = Open(NewPath(kind), kind, AB); + var baseA = new CultRecordKey("base-a"); + cache.Commit(batch => + { + batch.Upsert(typeof(EvA), new EvA { Name = "base-a", Value = 1 }, baseA); + batch.UpsertVariant(K3, baseA, new[] { cache.Override(nameof(EvA.Name), "v3") }); + }); + + var refused = Assert.Throws(() => cache.Commit(batch => + { + batch.Remove(K3); + batch.Upsert(typeof(EvA), new EvA { Name = "plain" }, K3); + batch.Flatten(K3); + }))!; + Assert.That(refused.Message, Does.Contain("Flatten(k3)").And.Contain("not a variant")); + Assert.That(cache.GetStored(K3)!.Variant, Is.Not.Null, "the refused batch changed nothing"); + + Assert.That(cache.Remove(K3), Is.True); + cache.UpsertAsync(typeof(EvA), new EvA { Name = "plain" }, K3).GetAwaiter().GetResult(); + var apart = Assert.Throws(() => cache.FlattenAsync(K3).GetAwaiter().GetResult())!; + Assert.That(apart.Message, Is.EqualTo(refused.Message)); + Assert.That(((EvA)cache.GetStored(K3)!.Document).Name, Is.EqualTo("plain")); + } + // finding cross-store-retype-decided-by-key-home: a one-batch retype between two home stores is refused by the rule that a commit // lands in one home store, with that rule's text (an evicted key is absent to the one-key-one-store check, as when staged // apart); staged apart, as two commits, it leaves the key in exactly one store. From 6f31d90a9abb216e8e1396c3e31bec7f9d6f0ec0 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 28 Oct 2026 08:01:15 +0100 Subject: [PATCH 25/88] A batch stages an operation after a Flatten against the plain record it leaves: a second Flatten and an UpsertVariant are refused, a plain Upsert replaces it; one answer in a batch and staged apart; pinned Co-Authored-By: Claude Opus 5.5 --- src/GameCult.Caching/CultCache.cs | 36 +++++---- .../RetypeEvictionTests.cs | 79 +++++++++++++++++++ 2 files changed, 101 insertions(+), 14 deletions(-) diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 8c80116c..5a0626a6 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -547,6 +547,9 @@ namespace GameCult.Caching // A plain record staged by Flatten: the one way a plain record may replace a variant at its key. internal bool Flattens { get; set; } + // A plain record a batch staged after a Flatten at its key: it replaces the plain record that flatten left, so it may land on the variant the cache holds. + internal bool ReplacesFlattened { get; set; } + // True while this record holds element ids minted when it loaded, that no write has persisted (see MintElementIds). internal bool IdsInMemoryOnly { get; set; } @@ -2076,8 +2079,9 @@ namespace GameCult.Caching // One staged write: a plain record (Stored), a variant (Variant), a flatten, or, with none of those, a removal. internal sealed class StagedOperation { - public StagedOperation(CultRecordKey key, CultStoredDocument? stored, CultVariantDelta? variant, bool flatten) + public StagedOperation(CultRecordKey key, CultStoredDocument? stored, CultVariantDelta? variant, bool flatten, bool afterFlatten = false) { + AfterFlatten = afterFlatten; Key = key; Stored = stored; Variant = variant; @@ -2088,6 +2092,8 @@ namespace GameCult.Caching public CultStoredDocument? Stored { get; } public CultVariantDelta? Variant { get; } public bool Flatten { get; } + // A plain record staged after a Flatten at its key: the key holds a plain record by then, so it replaces that, not a variant. + public bool AfterFlatten { get; } public bool IsRemoval => Stored == null && Variant == null && !Flatten; } @@ -2132,6 +2138,10 @@ namespace GameCult.Caching public CultRecordKey UpsertVariant(CultRecordKey key, CultRecordKey baseKey, IEnumerable? overrides = null) { ThrowIfSealed(); + // A flatten staged here leaves a plain record at the key, which cannot become a variant in place, as when staged apart. + if (Operations.TryGetValue(key.Value, out var flattened) && flattened.Flatten) + throw new InvalidOperationException( + $"Record {key.Value} is a plain record; it cannot become a variant of {baseKey.Value} in place. Use a new key."); Operations[key.Value] = new StagedOperation( key, null, new CultVariantDelta(baseKey.Value, (overrides ?? Array.Empty()).ToArray()), false); return key; @@ -2142,17 +2152,14 @@ namespace GameCult.Caching { ThrowIfSealed(); // A flatten acts on what the key will hold, as staged apart: a variant staged here is flattened as it lands; a removal or a - // plain record staged here leaves nothing to flatten; with nothing staged it flattens the variant the cache holds. + // plain record staged here, or an earlier flatten, leaves nothing to flatten; with nothing staged it flattens the variant + // the cache holds. if (Operations.TryGetValue(key.Value, out var staged)) { - if (staged.Variant != null) - { - Operations[key.Value] = new StagedOperation(key, null, staged.Variant, true); - return; - } - - if (!staged.Flatten) + if (staged.Variant == null || staged.Flatten) throw new InvalidOperationException($"Flatten({key.Value}): the record is not a variant."); + Operations[key.Value] = new StagedOperation(key, null, staged.Variant, true); + return; } Operations[key.Value] = new StagedOperation(key, null, null, true); @@ -2175,7 +2182,8 @@ namespace GameCult.Caching { ThrowIfSealed(); var stored = _cache.CreateStoredDocument(document, key); - Operations[stored.Key.Value] = new StagedOperation(stored.Key, stored, null, false); + Operations[stored.Key.Value] = new StagedOperation( + stored.Key, stored, null, false, afterFlatten: Operations.TryGetValue(stored.Key.Value, out var prior) && prior.Flatten); return stored.Key; } @@ -2736,8 +2744,8 @@ namespace GameCult.Caching } // Call under the gate that admits the result, so no load can land a later storedAt in between. - private CultStoredDocument Stamp(CultRecordKey key, CultDocumentDescriptor descriptor, object document) => - new(key, MintStoredAt(PreviousStoredAt(key), Clock()), descriptor, document); + private CultStoredDocument Stamp(CultRecordKey key, CultDocumentDescriptor descriptor, object document, bool replacesFlattened = false) => + new(key, MintStoredAt(PreviousStoredAt(key), Clock()), descriptor, document) { ReplacesFlattened = replacesFlattened }; // An unresolved variant: the admitting hold resolves it against its base before anything lands. // A staged variant that a Flatten followed lands as the plain record of its resolution (flattens). @@ -2842,7 +2850,7 @@ namespace GameCult.Caching // Staging stamped outside this gate; a pull since then may hold a later storedAt, so stamp again. var upserts = batch.Operations.Values .Where(op => !op.IsRemoval) - .Select(op => op.Stored != null ? Stamp(op.Key, op.Stored.Descriptor, op.Stored.Document) + .Select(op => op.Stored != null ? Stamp(op.Key, op.Stored.Descriptor, op.Stored.Document, op.AfterFlatten) : op.Variant != null ? StampVariant(op.Key, DescriptorOfBase(op.Key, op.Variant.BaseKey, batch), op.Variant, op.Flatten) : StampFlatten(op.Key)) .ToArray(); @@ -3299,7 +3307,7 @@ namespace GameCult.Caching _entries.TryGetValue(stored.Key.Value, out var present); if (gone.Contains(stored.Key.Value)) present = null; - if (stored.Variant == null && !stored.Flattens && present?.Variant != null) + if (stored.Variant == null && !stored.Flattens && !stored.ReplacesFlattened && present?.Variant != null) throw new InvalidOperationException( $"Record {stored.Key.Value} is a variant of {present.Variant.BaseKey}; a plain write cannot replace it. " + "Change it through the variant API, or Flatten it."); diff --git a/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs b/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs index 5c6bcf0a..4e6b2e15 100644 --- a/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs +++ b/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs @@ -365,6 +365,85 @@ namespace GameCult.Caching.Tests Assert.That(((EvA)cache.GetStored(K3)!.Document).Name, Is.EqualTo("plain")); } + // finding op-after-staged-flatten-diverges: an operation after a Flatten at the same key acts on the plain record that flatten + // leaves, so one intent has one answer in a batch and staged apart, in both stores kinds and after a reload from disk. + [TestCaseSource(nameof(VariantKinds))] + public void AnOperationAfterAStagedFlattenAnswersAsStagedApart(string kind) + { + var scenarios = new[] { "F,F", "F,P", "F,V", "V,F,V", "F,P,F", "V,F,P" }; + foreach (var scenario in scenarios) + { + var steps = scenario.Split(','); + var results = new List(); + foreach (var oneBatch in new[] { true, false }) + { + var path = NewPath(kind); + string outcome; + using (var cache = Open(path, kind, AB)) + { + var baseA = new CultRecordKey("base-a"); + cache.Commit(batch => + { + batch.Upsert(typeof(EvA), new EvA { Name = "base-a", Value = 1 }, baseA); + batch.UpsertVariant(K3, baseA, new[] { cache.Override(nameof(EvA.Name), "v3") }); + }); + var variant = new[] { cache.Override(nameof(EvA.Name), "vv") }; + + try + { + if (oneBatch) + cache.Commit(batch => + { + foreach (var step in steps) + { + if (step == "F") batch.Flatten(K3); + else if (step == "P") batch.Upsert(typeof(EvA), new EvA { Name = "plain" }, K3); + else batch.UpsertVariant(K3, baseA, variant); + } + }); + else + foreach (var step in steps) + { + if (step == "F") cache.FlattenAsync(K3).GetAwaiter().GetResult(); + else if (step == "P") cache.UpsertAsync(typeof(EvA), new EvA { Name = "plain" }, K3).GetAwaiter().GetResult(); + else cache.UpsertVariantAsync(K3, baseA, variant).GetAwaiter().GetResult(); + } + outcome = "committed"; + } + catch (InvalidOperationException refused) + { + outcome = "refused: " + refused.Message; + } + + outcome += " | " + Describe(cache); + cache.FlushAsync().GetAwaiter().GetResult(); + } + + if (kind != "storeless" && outcome.StartsWith("committed", StringComparison.Ordinal)) + { + using var reloaded = Open(path, kind, AB); + outcome += " | reloaded " + Describe(reloaded); + } + + results.Add(outcome); + } + + // A refused batch changes nothing, while staged apart the steps before the refused one have landed: the answer compared + // is the verdict and its text, and the end state only where both committed. + var batchAnswer = results[0].Split(" | ")[0]; + var apartAnswer = results[1].Split(" | ")[0]; + Assert.That(batchAnswer, Is.EqualTo(apartAnswer), $"{kind} {scenario}"); + if (batchAnswer == "committed") + Assert.That(results[0], Is.EqualTo(results[1]), $"{kind} {scenario}"); + } + + static string Describe(CultCache cache) + { + var held = cache.GetStored(K3)!; + return $"{held.Descriptor.DocumentType.Name} variant={held.Variant?.BaseKey} name={((EvA)held.Document).Name}"; + } + } + // finding cross-store-retype-decided-by-key-home: a one-batch retype between two home stores is refused by the rule that a commit // lands in one home store, with that rule's text (an evicted key is absent to the one-key-one-store check, as when staged // apart); staged apart, as two commits, it leaves the key in exactly one store. From 8c7252eae73875ada3163f0291ffb8f52f979725 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 28 Oct 2026 08:07:58 +0100 Subject: [PATCH 26/88] RetypeEvictionTests: a Flatten of a staged variant, then a second Flatten, is refused as staged apart Co-Authored-By: Claude Opus 5.5 --- tests/GameCult.Caching.Tests/RetypeEvictionTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs b/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs index 4e6b2e15..9e924d81 100644 --- a/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs +++ b/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs @@ -370,7 +370,7 @@ namespace GameCult.Caching.Tests [TestCaseSource(nameof(VariantKinds))] public void AnOperationAfterAStagedFlattenAnswersAsStagedApart(string kind) { - var scenarios = new[] { "F,F", "F,P", "F,V", "V,F,V", "F,P,F", "V,F,P" }; + var scenarios = new[] { "F,F", "F,P", "F,V", "V,F,V", "F,P,F", "V,F,P", "V,F,F" }; foreach (var scenario in scenarios) { var steps = scenario.Split(','); From 64b86e0a273a3936939357e9deb186b7bf3bc417 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 28 Oct 2026 08:22:20 +0100 Subject: [PATCH 27/88] write-ownership-cs: a write owns a record by the written type; one-batch retype is eviction then add (squash of hands/variants-write-ownership a4eae905..8c7252ea) A write owns the record at its key when the written type declares the record's schema id, or reads an older undeclared id of its own schema name whole (ruling own-entry-reads-whole); every other registered type's record is refused, in every cache, with or without a backing store (cache-refuses-retype). Removing a record and adding one of another type in a single batch is staged as the held record's eviction then the add, so validators, Claim, the store judges and the change journal see what they see staged apart (one-batch-retype-is-eviction-then-add); Watch gets Removed and Watch Added. A one-batch retype across home stores is refused by the batch-spans rule and must be staged apart. A variant rebase changes type exactly where a plain write may (variant-rebase-follows-plain-write). Store kinds outside the library get one protected per-key judge (store-kind-per-key-judge), exercised by tests/GameCult.Caching.OutsideStoreTests. Spec variants:cut_spec:cut-write-ownership-cs.r1; reports h1-h5; Soul s5 clears the merge (verdict variants:verdict:cut-write-ownership-cs.s5). Co-Authored-By: Claude Opus 5.5 --- CultLib.sln | 15 + .../DirectoryMessagePackBackingStore.cs | 27 +- .../Contracts/cultcache-persistence-format.md | 20 +- src/GameCult.Caching/CultCache.cs | 409 +++++++---- .../GameCult.Caching.OutsideStoreTests.csproj | 22 + .../OutsideStoreJudgeTests.cs | 368 ++++++++++ .../GameCult.Caching.Tests/ElementIdTests.cs | 43 ++ .../ForeignRecordTests.cs | 90 ++- .../RestampOnWriteTests.cs | 51 +- .../RetypeEvictionTests.cs | 659 ++++++++++++++++++ tests/GameCult.Caching.Tests/VariantTests.cs | 24 +- .../WriteOwnershipTests.cs | 417 +++++++++++ .../WrittenOwnsTests.cs | 97 +++ unity/org.gamecult.cultlib/CHANGELOG.md | 3 +- 14 files changed, 2019 insertions(+), 226 deletions(-) create mode 100644 tests/GameCult.Caching.OutsideStoreTests/GameCult.Caching.OutsideStoreTests.csproj create mode 100644 tests/GameCult.Caching.OutsideStoreTests/OutsideStoreJudgeTests.cs create mode 100644 tests/GameCult.Caching.Tests/RetypeEvictionTests.cs create mode 100644 tests/GameCult.Caching.Tests/WriteOwnershipTests.cs create mode 100644 tests/GameCult.Caching.Tests/WrittenOwnsTests.cs diff --git a/CultLib.sln b/CultLib.sln index 217f2d1f..a8baa383 100644 --- a/CultLib.sln +++ b/CultLib.sln @@ -51,6 +51,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Caching.ConsumerBu EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Caching.GeneratorConsumer", "tests\GameCult.Caching.GeneratorConsumer\GameCult.Caching.GeneratorConsumer.csproj", "{1A75EA8F-00B3-49AB-8398-B1B7723BE10B}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Caching.OutsideStoreTests", "tests\GameCult.Caching.OutsideStoreTests\GameCult.Caching.OutsideStoreTests.csproj", "{A54C2C29-941A-46A9-9DDE-9711B2F2EADD}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -289,6 +291,18 @@ Global {1A75EA8F-00B3-49AB-8398-B1B7723BE10B}.Release|x64.Build.0 = Release|Any CPU {1A75EA8F-00B3-49AB-8398-B1B7723BE10B}.Release|x86.ActiveCfg = Release|Any CPU {1A75EA8F-00B3-49AB-8398-B1B7723BE10B}.Release|x86.Build.0 = Release|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Debug|x64.ActiveCfg = Debug|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Debug|x64.Build.0 = Debug|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Debug|x86.ActiveCfg = Debug|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Debug|x86.Build.0 = Debug|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Release|Any CPU.Build.0 = Release|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Release|x64.ActiveCfg = Release|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Release|x64.Build.0 = Release|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Release|x86.ActiveCfg = Release|Any CPU + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -314,5 +328,6 @@ Global {FEF9DA88-2177-4FD0-885D-7096ACFA8A9D} = {B48EAE30-152F-01E3-6E31-82B350494D3C} {D3138F3C-9531-4906-A5D3-74BD857D0685} = {39D316AB-ABB5-49A2-8E83-CAF4C75B48F4} {1A75EA8F-00B3-49AB-8398-B1B7723BE10B} = {39D316AB-ABB5-49A2-8E83-CAF4C75B48F4} + {A54C2C29-941A-46A9-9DDE-9711B2F2EADD} = {39D316AB-ABB5-49A2-8E83-CAF4C75B48F4} EndGlobalSection EndGlobal diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index 63180a1b..a113db44 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -185,7 +185,12 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore manifest.Records.ToDictionary(record => record.Key, record => VersionOf(record.Payload), StringComparer.Ordinal), Entries.Values, ForeignRecords)) return CultCommitOutcome.Mismatch; - RefuseUndeclaredOnDisk(manifest, request.Upserts.Concat(request.Deletes).Select(entry => entry.Key.Value).Concat(request.Checks).Concat(StagedKeys)); + RefuseUndeclaredOnDisk( + manifest, + request.Upserts.Concat(request.Deletes).Select(entry => entry.Key.Value).Concat(request.Checks).Concat(StagedKeys), + request.Upserts, + request.Deletes, + request.Checks); if (request.Upserts.Count == 0 && request.Deletes.Count == 0 && !IsDirty) { // Nothing to write: the batch only judged keys it does not hold. @@ -243,7 +248,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore Directory.CreateDirectory(_manifestFile.DirectoryName!); using var commitLease = AcquireCommitLease(wait: true, create: true); var (manifest, catalogBytes) = ReadManifest(); - RefuseUndeclaredOnDisk(manifest, StagedKeys); + RefuseUndeclaredOnDisk(manifest, StagedKeys, Array.Empty(), Array.Empty(), Array.Empty()); if (IsDirty) WriteGeneration(manifest, catalogBytes); else @@ -275,7 +280,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore using var commitLease = AcquireCommitLease(wait, create: true); if (commitLease == null) return CultCommitOutcome.Contended; - RefuseUndeclaredOnDisk(ReadManifest().Snapshot, keys); + RefuseUndeclaredOnDisk(ReadManifest().Snapshot, keys, Array.Empty(), Array.Empty(), Array.Empty()); return CultCommitOutcome.Committed; }); } @@ -286,14 +291,24 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore // The manifest's other records stay as they are, foreign and read-only ones included; a write that would replace or remove a // record the manifest holds that the writer does not own is refused before any page is written, by the one rule every - // store shares (CacheBackingStore.RefuseUndeclaredOnDisk). The refused records reload as a pull loads them. - private void RefuseUndeclaredOnDisk(CultPersistedStoreSnapshot currentManifest, IEnumerable written) + // store shares (CacheBackingStore.UndeclaredAt, CacheBackingStore.RefuseUndeclaredOnDisk). The refused records reload as a + // pull loads them. A key is owned by the entry the write puts there: the batch's upsert, or the staged document. A key it + // removes (the batch's deletes, the staged deletes) or only checks goes to the registry's removal rule. + private void RefuseUndeclaredOnDisk(CultPersistedStoreSnapshot currentManifest, IEnumerable written, IReadOnlyList upserts, IReadOnlyList deletes, IEnumerable checks) { CultPersistedRecord? HeldAt(string key) => currentManifest.Records.FirstOrDefault(record => string.Equals(record.Key, key, StringComparison.Ordinal)); + var batch = upserts.ToDictionary(entry => entry.Key.Value, entry => entry.Descriptor, StringComparer.Ordinal); + CultSchemaCatalogEntry? WrittenAt(string key) => + batch.TryGetValue(key, out var descriptor) ? descriptor.ToCatalogEntry() + : _dirtyKeys.ContainsKey(key) && Entries.TryGetValue(key, out var staged) ? staged.Descriptor.ToCatalogEntry() + : null; RefuseUndeclaredOnDisk( written, - key => HeldAt(key) is { } held ? Undeclared(held, currentManifest.SchemaCatalog, Registry.Owns) : null, + currentManifest.SchemaCatalog, + HeldAt, + WrittenAt, + deletes.Select(entry => entry.Key.Value).Concat(checks), key => HeldAt(key) is { } held && Foreign(held, currentManifest.SchemaCatalog, VersionOf(held.Payload)) == null ? LoadPage(held, currentManifest.SchemaCatalog, out _, out _) : null, diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index 10fb9fc8..93faf63f 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -206,16 +206,16 @@ The rule is checked again where the write lands, under the store's lock: a write whose durable record is not owned, because another writer rewrote it after this cache loaded, is refused typed, and the record is reloaded as the store holds it, read-only when a type can read it. This holds for every write through the store, the store's own `Push`, `Delete` and -`CommitBatch` included. A CultMesh single-file write (`WriteDocument`, `WriteRecord`, or `PushAll` on an unattached store) asks the same rule: it owns the ids its -written entries declare (their own ids and the ids they list compatible) and an id whose publishing entry its claimant reads whole (no persisted -slot unread). The claimant is picked by the one precedence over the written entries and the call's registry, a written entry being the only -description of its schema id (the registry's type for that id is never asked), and the record is owned only when the claimant is a written -entry. A registry adds claimants and declares nothing, so it only adds refusals; with none the written entries are the only claimants. Refused: -a record another registered type claims, under that type's current id or an old one it lists; a record whose stored entry lists another -registered type's id ahead of the written one's, whatever name the stored entry carries; a record two registered types list compatible or two -registered versions of the written schema could read; a record the picked written entry reads only by dropping persisted slots; and the -removal of any stored record by a call that writes no entry. A registry-less `WriteDocument` reads `CultDocumentRegistry.Shared`. It is -refused the same way otherwise. +`CommitBatch` included. Whether a write may replace the record at a key is decided per key by the type it writes there, for a cache +and for a CultMesh single-file write (`WriteDocument`, `WriteRecord`, or `PushAll` on an unattached store) alike, in a single-file store and +a directory store alike: the written type owns the stored record when its entry declares the stored id (its own id, or one it lists +compatible), or when it is the type that reads the stored record's published entry and reads it whole (no persisted slot unread), so an older +id of its own schema name that it reads whole is its own and is restamped. Anything else is refused, another registered type's record +included, and so is a record the written type would read only by dropping persisted slots. No registry enters the decision, `Shared` included: +the answer for a given written entry, stored id and catalog is the same lent a registry or not, and a cache's registered types do not make +each other's records writable. Retyping a key is a removal and an add: in one batch (the removal called before the add) or staged apart, with or without a backing store. The removal half is judged as a removal, so the key is retyped only when the removal rule allows it. A one-batch retype is staged as the held record's eviction and then the add: a watcher of the old type sees Removed and a watcher of the new type sees Added, a global's slot is freed, and every check, validator and store judge sees what it sees when the two are staged apart; this holds for plain records, variants and flattens alike (a Flatten after a Remove in the same batch flattens nothing and is refused). A one-batch retype across two home stores is refused, by the rule that a commit lands in one home store: the removal is in the first store and the add in the second, so the batch spans both. Stage the removal and the add apart, as two commits (not atomic); the key is then in exactly one store. A variant rebased onto a base of another type follows the plain-write rule: where a plain write of the new type would own the variant's record (the new type declares the old id) the rebase commits and the variant becomes that type, and where a plain write would be refused the rebase is refused. A removal or check of a key goes to the +registry's removal rule (a record a registered type declares, or reads whole, may be removed), and a cacheless call, which has no registry, +removes a record only when an entry it writes owns it; a call that writes no entry removes nothing. A record that resolves to no local type is a foreign record, for example one of a type this build does not have, or one renamed under a new schema name without declaring its old id. A store never destroys or relabels it: the store carries it diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index ce276a0c..5a0626a6 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -524,6 +524,13 @@ namespace GameCult.Caching // asked. internal bool StoredSchemaReadWhole { get; set; } + // The catalog entry that publishes StoredSchemaId in the store this record was loaded from, with the id it was published for, + // so staging can ask WrittenOwns the question the store's judge asks, over the catalog the record came with. Null for a record + // this cache constructed or wrote, which is stored under Descriptor's own id and published by Descriptor's entry. A write that + // relabels a loaded record in place (minting its element ids, then a flush) moves StoredSchemaId and leaves this behind, so a + // reader uses it only while the id still matches StoredSchemaId. + internal (string Id, CultSchemaCatalogEntry Entry)? StoredPublishedBy { get; set; } + // Names the bytes this record was read from or written as: the SHA-256 of its persisted encoding, set by the store that // read or wrote them. A record no store has read or written has a version of its own that equals nothing else. internal string StoredVersion { get; set; } @@ -540,6 +547,9 @@ namespace GameCult.Caching // A plain record staged by Flatten: the one way a plain record may replace a variant at its key. internal bool Flattens { get; set; } + // A plain record a batch staged after a Flatten at its key: it replaces the plain record that flatten left, so it may land on the variant the cache holds. + internal bool ReplacesFlattened { get; set; } + // True while this record holds element ids minted when it loaded, that no write has persisted (see MintElementIds). internal bool IdsInMemoryOnly { get; set; } @@ -593,45 +603,28 @@ namespace GameCult.Caching public string TypeLabel { get; } } - // The lookup of a cacheless writer: the catalog entries the call writes, and the lent registry's other types. A written entry is - // the only description of its schema id, so the registry's type for that id is never asked: a registry adds claimants but never - // re-describes the written type. + // The lookup a write's ownership asks: only the catalog entries the call writes. No registry enters it, so the answer is the + // written type's own (variants:ruling:write-ownership-by-written-type). internal sealed class CatalogEntryLookup : IClaimantLookup { private readonly IReadOnlyCollection _entries; - private readonly IClaimantLookup? _others; - public CatalogEntryLookup(IReadOnlyCollection entries, CultDocumentRegistry? others) + public CatalogEntryLookup(IReadOnlyCollection entries) { _entries = entries; - _others = others?.Lookup; } public CultSchemaCatalogEntry? Owner(string schemaId) => - _entries.FirstOrDefault(entry => string.Equals(entry.SchemaId, schemaId, StringComparison.Ordinal)) - ?? Other(_others?.Owner(schemaId)); + _entries.FirstOrDefault(entry => string.Equals(entry.SchemaId, schemaId, StringComparison.Ordinal)); public IReadOnlyList Listing(string schemaId) => - _entries.Where(entry => entry.CompatibleSchemaIds.Contains(schemaId, StringComparer.Ordinal)) - .Concat(Others(_others?.Listing(schemaId))) - .ToArray(); + _entries.Where(entry => entry.CompatibleSchemaIds.Contains(schemaId, StringComparer.Ordinal)).ToArray(); public IReadOnlyList Named(string schemaName) => - _entries.Where(entry => string.Equals(entry.SchemaName, schemaName, StringComparison.Ordinal)) - .Concat(Others(_others?.Named(schemaName))) - .ToArray(); + _entries.Where(entry => string.Equals(entry.SchemaName, schemaName, StringComparison.Ordinal)).ToArray(); public ClaimantFacts Describe(CultSchemaCatalogEntry claimant) => new(claimant.SchemaName, claimant.SchemaVersion, claimant.SchemaId); - - // A registry type counts only when no written entry has its schema id. - private CultSchemaCatalogEntry? Other(CultDocumentDescriptor? descriptor) => - descriptor == null || _entries.Any(entry => string.Equals(entry.SchemaId, descriptor.SchemaId, StringComparison.Ordinal)) - ? null - : descriptor.ToCatalogEntry(); - - private IEnumerable Others(IEnumerable? descriptors) => - (descriptors ?? Array.Empty()).Select(Other).OfType(); } public sealed class CultDocumentRegistry @@ -832,29 +825,42 @@ namespace GameCult.Caching catalog.FirstOrDefault(entry => string.Equals(entry.SchemaId, schemaId, StringComparison.Ordinal)) ?? catalog.FirstOrDefault(entry => entry.CompatibleSchemaIds.Contains(schemaId, StringComparer.Ordinal)); - // The one answer to whether a writer may write, relabel or remove a record stored under this id - // (variants:ruling:drift-record-write-ownership): a registered type declares the id (owns it or lists it compatible), or the - // type that reads it reads it whole. Read live, so a type registered later that declares the id claims its records at their - // next write. A record read only by dropping slots is not owned: declaring its id compatible is the consent to the loss. - // Header only: the persisted catalog entry that publishes the id is compared with the claimant's entry, never a payload. A - // claimant that cannot be named (ambiguous, id missing from the catalog, shapes incompatible) owns nothing. + // Whether a removal, or a check of a key, may proceed over a record stored under this id, and the load-time marking of a + // record read-only: a registered type declares the id (owns it or lists it compatible), or the type that reads it reads it + // whole (variants:ruling:drift-record-write-ownership). It never decides an upsert: WrittenOwns does + // (variants:ruling:write-ownership-by-written-type). Read live, so a type registered later that declares the id claims its + // records at their next removal. A record read only by dropping slots is not owned: declaring its id compatible is the + // consent to the loss. Header only: the persisted catalog entry that publishes the id is compared with the claimant's + // entry, never a payload. A claimant that cannot be named (ambiguous, id missing from the catalog, shapes incompatible) owns + // nothing. internal bool Owns(string schemaId, IReadOnlyCollection catalog) => - Owns(schemaId, catalog, Declared, _indexes, descriptor => descriptor.ToCatalogEntry(), _ => true); + Owns(schemaId, catalog, Declared, _indexes, descriptor => descriptor.ToCatalogEntry()); - // The registry's lookup, for a cacheless writer that lends it only its other types. - internal IClaimantLookup Lookup => _indexes; + // The one answer to whether an upsert of the written type may replace the record stored under this id at its key, for a cache + // and a cacheless writer, a single-file store and a directory store alike. The written entry owns the record when it declares + // the stored id (its own id, or one it lists compatible), or when it is the type that reads the record's published entry + // (the one Claimant precedence over this one entry) and reads it whole, so an older id of its own schema name that it reads + // whole is its own (variants:ruling:own-entry-reads-whole). Anything else, another registered type's record included, is + // refused: retyping a key is a removal and an add (variants:ruling:cache-refuses-retype). No registry enters it. + internal static bool WrittenOwns( + CultSchemaCatalogEntry written, + string storedId, + IReadOnlyCollection catalog) => + Owns(storedId, catalog, id => DeclaredBy(written, id), new CatalogEntryLookup(new[] { written }), entry => entry); - // The rule itself, for any writer: declared, or the claimant (picked by the one Claimant over a lookup) reads the record's - // published entry whole. The lookup is a registry's indexes, or the entries a registry-less cacheless writer writes. mayClaim - // is a condition on the claimant's answer, not a second lookup: a cache accepts any claimant, a cacheless writer only one of - // the entries the call writes, so it never owns a record another registered type reads. + // WrittenOwns' declared half: the entry's own id or one it lists compatible. + internal static bool DeclaredBy(CultSchemaCatalogEntry written, string storedId) => + string.Equals(written.SchemaId, storedId, StringComparison.Ordinal) || + written.CompatibleSchemaIds.Contains(storedId, StringComparer.Ordinal); + + // The rule itself: declared, or the claimant (picked by the one Claimant over a lookup) reads the record's published entry + // whole. The lookup is a registry's indexes (the removal rule) or the one entry a write puts at the key (WrittenOwns). internal static bool Owns( string schemaId, IReadOnlyCollection catalog, Func declared, IClaimantLookup lookup, - Func entryOf, - Func mayClaim) + Func entryOf) where T : class { if (declared(schemaId)) @@ -864,8 +870,7 @@ namespace GameCult.Caching var claimant = Claimant(schemaId, catalog, lookup, out var persisted); if (persisted == null || claimant == null) return false; - var claimantEntry = entryOf(claimant); - return mayClaim(claimantEntry) && ReadsWhole(persisted, claimantEntry); + return ReadsWhole(persisted, entryOf(claimant)); } catch (InvalidOperationException) { @@ -2074,8 +2079,9 @@ namespace GameCult.Caching // One staged write: a plain record (Stored), a variant (Variant), a flatten, or, with none of those, a removal. internal sealed class StagedOperation { - public StagedOperation(CultRecordKey key, CultStoredDocument? stored, CultVariantDelta? variant, bool flatten) + public StagedOperation(CultRecordKey key, CultStoredDocument? stored, CultVariantDelta? variant, bool flatten, bool afterFlatten = false) { + AfterFlatten = afterFlatten; Key = key; Stored = stored; Variant = variant; @@ -2086,6 +2092,8 @@ namespace GameCult.Caching public CultStoredDocument? Stored { get; } public CultVariantDelta? Variant { get; } public bool Flatten { get; } + // A plain record staged after a Flatten at its key: the key holds a plain record by then, so it replaces that, not a variant. + public bool AfterFlatten { get; } public bool IsRemoval => Stored == null && Variant == null && !Flatten; } @@ -2094,6 +2102,9 @@ namespace GameCult.Caching { private readonly CultCache _cache; internal readonly Dictionary Operations = new(StringComparer.Ordinal); + // Every key Remove named, in whatever order: Land evicts the record the cache holds at each (or checks the key when it holds + // none) before anything lands, so a record staged at a removed key is an add after an eviction, as when staged apart. + internal readonly HashSet Removals = new(StringComparer.Ordinal); internal readonly List<(CultRecordKey Key, string? Version)> Expected = new(); internal bool ExpectsUnchanged; internal bool Sealed; @@ -2118,6 +2129,7 @@ namespace GameCult.Caching public void Remove(CultRecordKey key) { ThrowIfSealed(); + Removals.Add(key.Value); Operations[key.Value] = new StagedOperation(key, null, null, false); } @@ -2126,6 +2138,10 @@ namespace GameCult.Caching public CultRecordKey UpsertVariant(CultRecordKey key, CultRecordKey baseKey, IEnumerable? overrides = null) { ThrowIfSealed(); + // A flatten staged here leaves a plain record at the key, which cannot become a variant in place, as when staged apart. + if (Operations.TryGetValue(key.Value, out var flattened) && flattened.Flatten) + throw new InvalidOperationException( + $"Record {key.Value} is a plain record; it cannot become a variant of {baseKey.Value} in place. Use a new key."); Operations[key.Value] = new StagedOperation( key, null, new CultVariantDelta(baseKey.Value, (overrides ?? Array.Empty()).ToArray()), false); return key; @@ -2135,6 +2151,17 @@ namespace GameCult.Caching public void Flatten(CultRecordKey key) { ThrowIfSealed(); + // A flatten acts on what the key will hold, as staged apart: a variant staged here is flattened as it lands; a removal or a + // plain record staged here, or an earlier flatten, leaves nothing to flatten; with nothing staged it flattens the variant + // the cache holds. + if (Operations.TryGetValue(key.Value, out var staged)) + { + if (staged.Variant == null || staged.Flatten) + throw new InvalidOperationException($"Flatten({key.Value}): the record is not a variant."); + Operations[key.Value] = new StagedOperation(key, null, staged.Variant, true); + return; + } + Operations[key.Value] = new StagedOperation(key, null, null, true); } @@ -2155,7 +2182,8 @@ namespace GameCult.Caching { ThrowIfSealed(); var stored = _cache.CreateStoredDocument(document, key); - Operations[stored.Key.Value] = new StagedOperation(stored.Key, stored, null, false); + Operations[stored.Key.Value] = new StagedOperation( + stored.Key, stored, null, false, afterFlatten: Operations.TryGetValue(stored.Key.Value, out var prior) && prior.Flatten); return stored.Key; } @@ -2555,6 +2583,7 @@ namespace GameCult.Caching Admit(Array.Empty(), new[] { existing }, null, (home, _) => { home?.Delete(existing); + home?.NoteRemoved(existing.Key.Value); return CultCommitOutcome.Committed; }); return true; @@ -2715,12 +2744,13 @@ namespace GameCult.Caching } // Call under the gate that admits the result, so no load can land a later storedAt in between. - private CultStoredDocument Stamp(CultRecordKey key, CultDocumentDescriptor descriptor, object document) => - new(key, MintStoredAt(PreviousStoredAt(key), Clock()), descriptor, document); + private CultStoredDocument Stamp(CultRecordKey key, CultDocumentDescriptor descriptor, object document, bool replacesFlattened = false) => + new(key, MintStoredAt(PreviousStoredAt(key), Clock()), descriptor, document) { ReplacesFlattened = replacesFlattened }; // An unresolved variant: the admitting hold resolves it against its base before anything lands. - private CultStoredDocument StampVariant(CultRecordKey key, CultDocumentDescriptor descriptor, CultVariantDelta delta) => - new(key, MintStoredAt(PreviousStoredAt(key), Clock()), descriptor, delta, null, _codec); + // A staged variant that a Flatten followed lands as the plain record of its resolution (flattens). + private CultStoredDocument StampVariant(CultRecordKey key, CultDocumentDescriptor descriptor, CultVariantDelta delta, bool flattens = false) => + new(key, MintStoredAt(PreviousStoredAt(key), Clock()), descriptor, delta, null, _codec) { Flattens = flattens }; // The plain record a Flatten stages: the variant's resolved document, under the variant's own key and schema. private CultStoredDocument StampFlatten(CultRecordKey key) @@ -2820,20 +2850,16 @@ namespace GameCult.Caching // Staging stamped outside this gate; a pull since then may hold a later storedAt, so stamp again. var upserts = batch.Operations.Values .Where(op => !op.IsRemoval) - .Select(op => op.Stored != null ? Stamp(op.Key, op.Stored.Descriptor, op.Stored.Document) - : op.Variant != null ? StampVariant(op.Key, DescriptorOfBase(op.Key, op.Variant.BaseKey, batch), op.Variant) + .Select(op => op.Stored != null ? Stamp(op.Key, op.Stored.Descriptor, op.Stored.Document, op.AfterFlatten) + : op.Variant != null ? StampVariant(op.Key, DescriptorOfBase(op.Key, op.Variant.BaseKey, batch), op.Variant, op.Flatten) : StampFlatten(op.Key)) .ToArray(); - // A removal of a key the cache does not hold removes nothing: it is a check every store the commit reaches judges under its lock. - var checks = batch.Operations.Values - .Where(op => op.IsRemoval && !_entries.ContainsKey(op.Key.Value)) - .Select(op => op.Key.Value) - .ToArray(); - var deletes = batch.Operations.Values - .Where(op => op.IsRemoval && _entries.ContainsKey(op.Key.Value)) - .Select(op => _entries[op.Key.Value]) - .ToArray(); - return Admit(upserts, deletes, null, (home, resolved) => + // A removal evicts the record the cache holds at the key, and a record staged at the key lands after the eviction, as + // when the two are staged apart (variants:ruling:one-batch-retype-is-eviction-then-add). A removal of a key the cache + // does not hold removes nothing: it is a check every store the commit reaches judges under its lock. + var deletes = batch.Removals.Where(_entries.ContainsKey).Select(key => _entries[key]).ToArray(); + var checks = batch.Removals.Where(key => !_entries.ContainsKey(key)).ToArray(); + return Admit(upserts, deletes, null, removed: batch.Removals, land: (home, resolved) => { // A batch that only checks keys names no home; it commits through the cache's one writable store, if it has one. if (home == null && checks.Length > 0 && _stores.Where(entry => !entry.Store.IsReadOnly).Take(2).ToArray() is { Length: 1 } only) @@ -2883,11 +2909,12 @@ namespace GameCult.Caching IReadOnlyList admitted, IReadOnlyList evicted, CacheBackingStore? source, - Func, CultCommitOutcome> land) => Held(() => + Func, CultCommitOutcome> land, + IReadOnlyCollection? removed = null) => Held(() => { if (_held == null) throw new InvalidOperationException("An admission reached the cache under a plain lock on its gate; every admission runs in a hold."); - var home = Validate(admitted, evicted, source); + var home = Validate(admitted, evicted, source, removed); // Ids are decided before anything lands, and the caller's documents change only inside this admission: a plan that // refuses (a duplicate, a malformed id) has changed nothing, and a write that does not commit gives its ids back. // A write mints random ids; a load of a record written before ids existed mints the ids its reload will mint @@ -3095,30 +3122,34 @@ namespace GameCult.Caching private CacheBackingStore? Validate( IReadOnlyList admitted, IReadOnlyList evicted, - CacheBackingStore? source) + CacheBackingStore? source, + IReadOnlyCollection? removed) { + var removedKeys = removed?.ToHashSet(StringComparer.Ordinal) ?? new HashSet(StringComparer.Ordinal); var homes = new List(); var globals = new Dictionary(_globals); + var evictedKeys = evicted.Select(stored => stored.Key.Value).ToHashSet(StringComparer.Ordinal); foreach (var stored in evicted) { var type = stored.Descriptor.DocumentType; if (globals.TryGetValue(type, out var key) && key == stored.Key.Value) globals.Remove(type); if (source == null) - Claim(stored); + Claim(stored, upsert: false); } foreach (var stored in admitted) { var descriptor = stored.Descriptor; if (source == null) - Claim(stored); + Claim(stored, upsert: !removedKeys.Contains(stored.Key.Value)); else if (Home(descriptor.DocumentType) is var home && home != source) throw new InvalidOperationException( $"{descriptor.SchemaName} record {stored.Key.Value} was loaded from {source} but its home is {home?.ToString() ?? "no store"}."); - // One key, one store: a key never moves between homes, and two stores never both deliver it. - if (_entries.TryGetValue(stored.Key.Value, out var present) && + // One key, one store: a key never moves between homes, and two stores never both deliver it. An evicted key is absent + // here, as it is staged apart; a retype across two home stores is then the batch spanning both (below): one rule, one text. + if (!evictedKeys.Contains(stored.Key.Value) && _entries.TryGetValue(stored.Key.Value, out var present) && Home(present.Descriptor.DocumentType) is var held && Home(descriptor.DocumentType) is var incoming && held != incoming) @@ -3137,23 +3168,27 @@ namespace GameCult.Caching var distinct = homes.Distinct().ToArray(); if (distinct.Length > 1) throw new InvalidOperationException( - $"The batch spans {distinct[0]} and {distinct[1]}; a commit lands in one home store."); + $"The batch spans {distinct[0]} and {distinct[1]}; a commit lands in one home store. " + + "To move a key between stores, stage its removal and its add apart (two commits, not one atomic commit)."); return distinct.FirstOrDefault(); - void Claim(CultStoredDocument stored) + void Claim(CultStoredDocument stored, bool upsert) { var home = Home(stored.Descriptor.DocumentType); if (home == null) { if (_stores.Count > 0) throw new InvalidOperationException($"No backing store is home to {stored.Descriptor.SchemaName}."); + // No store, no file to judge, but the same rule: a record is replaced only by its own type's write. + if (upsert && _entries.TryGetValue(stored.Key.Value, out var inMemory) && !CacheBackingStore.UpsertOwned(inMemory, stored.Descriptor)) + throw CacheBackingStore.Retyped(inMemory, stored.Key.Value, stored.Descriptor); return; } if (home.IsReadOnly) throw new InvalidOperationException( $"{stored.Descriptor.SchemaName} record {stored.Key.Value} routes to read-only backing store {home}."); - home.RefuseUndeclared(stored.Key.Value); + home.RefuseUndeclared(stored.Key.Value, upsert ? stored.Descriptor : null); homes.Add(home); } } @@ -3264,23 +3299,21 @@ namespace GameCult.Caching // A variant is written only through the variant API: a plain record never lands on a variant's key, and the // reverse is a rebase or a flatten, never an accident. A load reports what is on disk and is not a write. + // A rebase onto a base of another type would retype the key; write ownership refused that in Claim before this runs. if (source == null) { foreach (var stored in admitted) { _entries.TryGetValue(stored.Key.Value, out var present); - if (stored.Variant == null && !stored.Flattens && present?.Variant != null) + if (gone.Contains(stored.Key.Value)) + present = null; + if (stored.Variant == null && !stored.Flattens && !stored.ReplacesFlattened && present?.Variant != null) throw new InvalidOperationException( $"Record {stored.Key.Value} is a variant of {present.Variant.BaseKey}; a plain write cannot replace it. " + "Change it through the variant API, or Flatten it."); if (stored.Variant != null && present != null && present.Variant == null) throw new InvalidOperationException( $"Record {stored.Key.Value} is a plain record; it cannot become a variant of {stored.Variant.BaseKey} in place. Use a new key."); - if (stored.Variant != null && present?.Variant != null && - present.Descriptor.DocumentType != stored.Descriptor.DocumentType) - throw new InvalidOperationException( - $"Variant {stored.Key.Value} is a {present.Descriptor.DocumentType.Name}; " + - $"rebasing it onto {stored.Variant.BaseKey} would make it a {stored.Descriptor.DocumentType.Name}."); } } @@ -3399,7 +3432,7 @@ namespace GameCult.Caching ?? throw new InvalidOperationException($"Variant {key} resolved to nothing."); stack.RemoveAt(stack.Count - 1); - return done[key] = isVariant + return done[key] = isVariant && !variant.Flattens ? variant.Resolved(document, minted ? new CultVariantDelta(delta.BaseKey, overrides) : delta, variant.IdsInMemoryOnly || (minted && source != null), _codec) : new CultStoredDocument(variant.Key, variant.StoredAt, variant.Descriptor, document) { @@ -3944,6 +3977,7 @@ namespace GameCult.Caching { StoredSchemaId = record.SchemaId, StoredSchemaReadWhole = readWhole, + StoredPublishedBy = PublishedByIn(record.SchemaId, catalog), StoredVersion = version }; } @@ -3968,10 +4002,14 @@ namespace GameCult.Caching { StoredSchemaId = record.SchemaId, StoredSchemaReadWhole = readWhole, + StoredPublishedBy = PublishedByIn(record.SchemaId, catalog), StoredVersion = version }; } + private static (string Id, CultSchemaCatalogEntry Entry)? PublishedByIn(string schemaId, IReadOnlyCollection catalog) => + CultDocumentRegistry.PublishedBy(schemaId, catalog) is { } entry ? (schemaId, entry) : null; + protected void SetLastSchemaMigrationReports(IEnumerable reports) { _lastSchemaMigrationReports = reports?.ToArray() ?? Array.Empty(); @@ -3991,19 +4029,46 @@ namespace GameCult.Caching _foreignRecords = foreign.OrderBy(record => record.Key, StringComparer.Ordinal).ToArray(); } - // A cache never writes, relabels or removes a record it does not own: the staging half of rule 3. Refused when the store - // carries a foreign record at the key, or holds one that is not owned: its stored id is not declared by a registered type and - // the type that reads it drops persisted slots (read-only). A record read whole is owned; ToStoredDocument decided that from - // the same comparison Owns makes. Called for every record a cache write admits or evicts, before anything is staged. - protected internal void RefuseUndeclared(string key) + // A cache never writes, relabels or removes a record it does not own: the staging half of rule 3. A removal or check of a key + // is refused when the store carries a foreign record at the key, or holds one that is not owned: its stored id is not + // declared by a registered type and the type that reads it drops persisted slots (read-only). A record read whole is owned; + // ToStoredDocument decided that from the same comparison Owns makes. + protected internal void RefuseUndeclared(string key) => RefuseUndeclared(key, null); + + // The same, for every record a cache write admits or evicts, before anything is staged: an upsert (written is its type) may + // replace only a record that type owns, WrittenOwns' rule (variants:ruling:write-ownership-by-written-type), asked over the + // catalog entry that published the held record's id when it was loaded, the question the store's judge asks of the file. + internal void RefuseUndeclared(string key, CultDocumentDescriptor? written) { var foreign = _foreignRecords.FirstOrDefault(record => string.Equals(record.Key, key, StringComparison.Ordinal)); if (foreign != null) throw Overwrites(null, new[] { (foreign.Key, foreign.SchemaId, foreign.SchemaName) }); - if (Entries.TryGetValue(key, out var held) && !Registry.Declared(held.StoredSchemaId) && !held.StoredSchemaReadWhole) + if (Entries.TryGetValue(key, out var held) && !HeldOwned(held, written)) throw Overwrites(null, new[] { (key, held.StoredSchemaId, held.Descriptor.SchemaName) }); } + private bool HeldOwned(CultStoredDocument held, CultDocumentDescriptor? written) => + written == null + ? Registry.Declared(held.StoredSchemaId) || held.StoredSchemaReadWhole + : UpsertOwned(held, written); + + // Whether a write of the written type may replace the record a cache holds: WrittenOwns over the entry that published the held + // record's id when it was loaded (a record this cache wrote is published by its own type). The one answer for a store-backed + // cache's staging and a store-less cache's. + internal static bool UpsertOwned(CultStoredDocument held, CultDocumentDescriptor written) => + CultDocumentRegistry.WrittenOwns( + written.ToCatalogEntry(), + held.StoredSchemaId, + new[] { held.StoredPublishedBy is { } loaded && string.Equals(loaded.Id, held.StoredSchemaId, StringComparison.Ordinal) ? loaded.Entry : held.Descriptor.ToCatalogEntry() }); + + // The refusal of a store-less cache that would retype a key. + internal static CultSchemaConflictException Retyped(CultStoredDocument held, string key, CultDocumentDescriptor written) => new( + $"Record '{key}' is a {held.Descriptor.SchemaName} (schema id '{held.StoredSchemaId}'); writing a {written.SchemaName} there would retype the key. " + + "Remove the record first, in the same batch or an earlier commit, then add the other type's.", + held.StoredSchemaId, + new[] { held.Descriptor.SchemaName }, + key); + /// /// Called when the cache removes keys it does not hold, outside a batch that commits through this store. Such a removal /// removes nothing, but a store that holds a record at one of these keys under an id no registered type declares must @@ -4069,6 +4134,17 @@ namespace GameCult.Caching internal bool HasStagedChecks => _checks.Count > 0; + // Keys this store's cache removed here since its last flush. A write of the same key after that is a removal and an add + // (variants:ruling:cache-refuses-retype): the key is judged as the removal it names, not as an upsert over the record it held. + // Cleared like the checks, by a refusal of the key or a successful flush or commit. + private readonly HashSet _removed = new(StringComparer.Ordinal); + + internal void NoteRemoved(string key) => Held(() => _removed.Add(key)); + + // Every key a removal named since the last flush (held or not), which this store judges as a removal even when a record is + // staged there again. + internal IReadOnlyCollection StagedRemovals => _removed.Concat(_checks).ToArray(); + // A stored record under an id the writer does not own, named by the catalog entry that publishes the id. Header only: no // payload is decoded. Null for a record under an owned id. protected static (string Key, string SchemaId, string SchemaName)? Undeclared( @@ -4077,9 +4153,91 @@ namespace GameCult.Caching ? null : (record.Key, record.SchemaId, CultDocumentRegistry.PublishedBy(record.SchemaId, catalog)?.SchemaName ?? string.Empty); - // The apply half of rule 3, for every store kind, checked where the write lands under the store's lock. A key this write names - // whose durable record is under an id no registered type declares is refused, every such key at once, through Refuse, which - // reloads it as the store holds it: listed foreign when no type reads it, held read-only when one does. + // The one per-key judge of a write over a durable catalog, for every store kind: the stored record at each key the write names + // is owned by the entry the write puts at that key (WrittenOwns), or, when the write only removes or checks the key, by the + // removal rule. heldAt gives the durable record at a key and writtenAt the catalog entry the write puts there (null for a + // removal or a check). Memoised per written schema id and stored id, so a large store pays one comparison per pair, and an + // answer for one written type is never reused for another. + internal static Func UndeclaredAt( + Func heldAt, + IReadOnlyCollection catalog, + Func writtenAt, + Func, bool> removalOwns) + { + var answers = new Dictionary<(string? Written, string Stored), bool>(); + return key => + { + if (heldAt(key) is not { } held) + return null; + var written = writtenAt(key); + return Undeclared(held, catalog, (storedId, entries) => + { + var asked = (written?.SchemaId, storedId); + if (!answers.TryGetValue(asked, out var answer)) + answers[asked] = answer = written == null + ? removalOwns(storedId, entries) + : CultDocumentRegistry.WrittenOwns(written, storedId, entries); + return answer; + }); + }; + } + + /// + /// The per-key ownership judge of a write, for a store kind outside this library: the rule the shipped stores apply under their + /// lock, as one call. Each key in and in is judged against the durable + /// record gives for it. A key the write puts a record at ( gives the + /// catalog entry of the record written there) is owned only by that type: a record under its own schema id, an id it lists + /// compatible, or an older id of its own schema name it reads whole; another registered type's record is refused. A key the + /// write removes or only checks, and every key this store's cache removed since the last flush or commit (the removal half of + /// a retype, whose add then lands at it), is owned by any registered type that declares or reads the record whole. A refusal is + /// : it reloads what it refuses, unstages it through , and throws. + /// + /// Every key the write names. + /// The schema catalog of the durable store. + /// The durable record at a key, or null. + /// The catalog entry of the record the write puts at a key, or null when it puts none. + /// The keys the write removes or checks. + /// See . + /// See . + /// See . + /// True for a write no cache made: its refusal is the cacheless one. + protected void RefuseUndeclaredOnDisk( + IEnumerable written, + IReadOnlyCollection catalog, + Func heldAt, + Func writtenAt, + IEnumerable removed, + Func durableAt, + Func foreignAt, + Action unstage, + bool cacheless = false) + { + var keys = written.Distinct(StringComparer.Ordinal).ToArray(); + var removedKeys = removed.Concat(StagedRemovals).ToHashSet(StringComparer.Ordinal); + // A cacheless call has no registry that decides a removal: the entries it writes do (a call that writes no entry owns + // nothing, variants:ruling:mesh-writes-bound-by-declaration). + var entries = keys.Select(writtenAt).OfType().ToArray(); + Func, bool> removalOwns = Cache != null + ? Registry.Owns + : (id, entriesInCatalog) => entries.Any(entry => CultDocumentRegistry.WrittenOwns(entry, id, entriesInCatalog)); + RefuseUndeclaredOnDisk( + keys, + UndeclaredAt(heldAt, catalog, key => removedKeys.Contains(key) ? null : writtenAt(key), removalOwns), + durableAt, + foreignAt, + unstage, + cacheless); + } + + // The apply half of rule 3, for every store kind, checked where the write lands under the store's lock. undeclaredAt is the + // store's per-key judge: for each key the write names it says whether the durable record there is one the write may not + // replace, remove or relabel. The rule it must apply is per key and per written type, never one predicate per stored id: a + // key the write puts a record at is owned only by the type written there (its own id, an id it lists, or an older id of its + // own schema name it reads whole), so another registered type's record is refused; a key the write only removes or checks + // is owned by any registered type that declares or reads it whole; and a key the cache removed and the write then adds again + // is judged as the removal it names. Every key at once is refused through Refuse, which reloads it as the store holds it: + // listed foreign when no type reads it, held read-only when one does. The shipped stores call the overload that builds + // this judge; this one takes a judge a store kind outside the library built itself. protected void RefuseUndeclaredOnDisk( IEnumerable written, Func undeclaredAt, @@ -4147,6 +4305,7 @@ namespace GameCult.Caching { unstage(key); _checks.Remove(key); + _removed.Remove(key); } foreach (var held in dropped) Entries.TryRemove(held.Key.Value, out _); @@ -4203,6 +4362,7 @@ namespace GameCult.Caching { IsDirty = false; _checks.Clear(); + _removed.Clear(); } } @@ -4424,7 +4584,9 @@ namespace GameCult.Caching return CultCommitOutcome.Mismatch; var (upserts, removals) = StagedWrites(request.Upserts, request.Deletes); - var written = ApplyWriteSet(disk, upserts, removals, request.Checks); + // A key the batch deletes and also upserts is a retype: the upsert wins in the write set, so the deletion travels as a check, + // and the judge reads the key as the removal it names. + var written = ApplyWriteSet(disk, upserts, removals, request.Checks.Concat(request.Deletes.Select(entry => entry.Key.Value))); foreach (var entry in request.Deletes) Entries.TryRemove(entry.Key.Value, out _); foreach (var entry in request.Upserts) @@ -4484,7 +4646,7 @@ namespace GameCult.Caching /// The record to store. /// The schema catalog entry for the record's schema id. /// Whether the record's payload holds element ids, or null when the caller cannot tell. - /// The registry the payload was encoded with. Given, the write decides the header for copied records that registry can read, and its other types also claim stored records as a cache registering them would; it declares nothing, so it can only refuse more. stays the only description of its own schema id and the registry's type for that id is never asked. Owned are a record whose id declares (its own or one it lists compatible) and a record the entry reads whole when it is the picked claimant; refused are a record another registered type claims, one whose stored entry lists another registered type's id ahead of the entry's, and one that two registered types list or two versions of the schema could read. Omitted, it decides nothing for copied records and the one entry is the only claimant. It serves this call only: the store keeps no registry. + /// The registry the payload was encoded with. Given, the write decides the header for copied records that registry can read. It never decides ownership: the answer is the same lent or not. alone owns the record at the key when it declares the stored id (its own or one it lists compatible), or when it is the type that reads the stored entry and reads it whole, so an older id of its own schema name it reads whole is its own and is restamped. Everything else is refused with the file left as it was: a record of another registered type included, whatever the registry holds, and a record it would read only by dropping persisted slots. Retyping a key is a removal and an add. It serves this call only: the store keeps no registry. /// The call holds the store's gate while it runs. User code it runs must not enter a cache or another store; that throws . /// The store is attached to a cache; write through the cache. /// The file at the path is not a store this runtime can read. @@ -4507,7 +4669,7 @@ namespace GameCult.Caching /// The record key the document is stored under. /// The document to store. /// The stored-at stamp, or null for now. - /// The registry that describes and encodes the document, or null for . The document type's entry is taken from it and is the only description of that schema id. Its other types claim stored records as a cache registering them would, and it declares nothing, so it can only refuse more: the document's entry owns a record whose id it declares (its own or one it lists compatible), and a record it reads whole when it is the picked claimant; refused are a record another registered type claims, one whose stored entry lists another registered type's id ahead of the entry's, and one that two registered types list or two versions of the schema could read. A wider registry (Shared when none is passed) refuses at least as much. It serves this call only: the store keeps no registry. + /// The registry that describes and encodes the document, or null for . The document type's entry is taken from it, and the registry decides the header for copied records it can read. It never decides ownership, so the answer is the same for any registry that holds the type: the document's entry owns a record whose id it declares (its own or one it lists compatible), and a record of an older id of its own schema name that it reads whole; everything else is refused, another registered type's record included, and a record it would read only by dropping persisted slots. Retyping a key is a removal and an add. It serves this call only: the store keeps no registry. /// The call holds the store's gate while it runs. User code it runs (property getters while the document is encoded, constructors while a copied record is read) must not enter a cache or another store; that throws , carried by the encoder's wrapper when the code ran inside it. /// The store is attached to a cache; write through the cache. /// The file at the path is not a store this runtime can read. @@ -4723,53 +4885,37 @@ namespace GameCult.Caching using var fileLock = AcquireLock(wait); if (fileLock == null) return CultCommitOutcome.Contended; - RefuseUndeclaredOnDisk(ReadSnapshot(), keys, OwnsOnDisk()); + RefuseUndeclaredOnDisk(ReadSnapshot(), keys, Array.Empty(), Array.Empty()); return CultCommitOutcome.Committed; }); } - // A cache owns through its registry (CultDocumentRegistry.Owns). A cacheless CultMesh write owns what the catalog entries it - // writes declare: their own ids and the ids they list compatible, or an id whose publishing entry its claimant reads whole, - // the claimant being picked by the one precedence over the written entries and the lent registry's other types, and owning - // only when it is a written entry (variants:ruling:mesh-writes-bound-by-declaration). The keys judged are every key staged, - // written, removed or checked. + // Every key staged, written, removed or checked is judged, each by the one per-key judge (UndeclaredAt). private void RefuseUndeclaredOnDisk(Durable? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals, IEnumerable checks) => RefuseUndeclaredOnDisk( durable, _staged.Concat(upserts.Select(staged => staged.Record.Key)).Concat(removals).Concat(checks).Concat(StagedChecks), - OwnsOnDisk(upserts)); + upserts, + checks); - // The owns predicate a write over the durable catalog asks. A cacheless writer declares what the entries it writes declare - // (variants:ruling:mesh-writes-bound-by-declaration) and reads a published entry with one lookup: the written entries shadow - // the lent registry's type for their ids, and the registry adds other claimants. The record is owned only when the claimant - // is a written entry, so the call is never more permissive than that declaration, and a registry can only refuse more. - private Func, bool> OwnsOnDisk(IReadOnlyCollection? upserts = null) => - Cache != null || upserts == null - ? Registry.Owns - : (id, catalog) => - { - var writes = upserts.Select(staged => staged.Entry).ToArray(); - Func declared = candidate => writes.Any(entry => - string.Equals(entry.SchemaId, candidate, StringComparison.Ordinal) || - entry.CompatibleSchemaIds.Contains(candidate, StringComparer.Ordinal)); - return CultDocumentRegistry.Owns( - id, - catalog, - declared, - new CatalogEntryLookup(writes, AttachedRegistry), - entry => entry, - claimant => writes.Any(entry => string.Equals(entry.SchemaId, claimant.SchemaId, StringComparison.Ordinal))); - }; - - // A refused key's durable version is what this store has now read of it. - private void RefuseUndeclaredOnDisk(Durable? durable, IEnumerable written, Func, bool> owns) + // A key the write also names as a removal or check (the request's, or one the cache removed here since its last flush) is a + // retype's removal half: it is judged by the removal rule, and the record staged there then lands. A refused key's durable + // version is what this store has now read of it. + private void RefuseUndeclaredOnDisk(Durable? durable, IEnumerable written, IReadOnlyCollection upserts, IEnumerable checks) { - var undeclared = UndeclaredOnDisk(durable, owns); - if (undeclared.Count == 0) - return; + var keys = written.Distinct(StringComparer.Ordinal).ToArray(); + var wanted = keys.ToHashSet(StringComparer.Ordinal); + var held = (durable?.Snapshot.Records ?? Array.Empty()) + .Where(record => wanted.Contains(record.Key)) + .ToDictionary(record => record.Key, StringComparer.Ordinal); + var removed = checks.ToHashSet(StringComparer.Ordinal); + var writtenEntries = upserts.ToDictionary(staged => staged.Record.Key, staged => staged.Entry, StringComparer.Ordinal); RefuseUndeclaredOnDisk( - written, - key => undeclared.TryGetValue(key, out var carried) ? carried : null, + keys, + durable?.Snapshot.SchemaCatalog ?? Array.Empty(), + key => held.TryGetValue(key, out var record) ? record : null, + key => writtenEntries.TryGetValue(key, out var entry) ? entry : null, + removed, key => Cache == null ? null : DurableAt(durable, key), key => ForeignAt(durable, key), key => @@ -4781,23 +4927,6 @@ namespace GameCult.Caching cacheless: Cache == null); } - // Every durable record under an id the writer does not own, by key, named by the catalog entry that publishes its id. The - // answer is decided once per distinct schema id, so a large store pays one comparison per id, not per record. - private static Dictionary UndeclaredOnDisk(Durable? durable, Func, bool> owns) - { - var undeclared = new Dictionary(StringComparer.Ordinal); - var answers = new Dictionary(StringComparer.Ordinal); - bool OwnsOnce(string id, IReadOnlyCollection catalog) => - answers.TryGetValue(id, out var answer) ? answer : answers[id] = owns(id, catalog); - foreach (var record in durable?.Snapshot.Records ?? Array.Empty()) - { - if (Undeclared(record, durable!.Snapshot.SchemaCatalog, OwnsOnce) is { } carried) - undeclared[record.Key] = carried; - } - - return undeclared; - } - // The durable record at a key as a load lists it when no registered type claims its id; null otherwise. private CultForeignRecord? ForeignAt(Durable? durable, string key) { diff --git a/tests/GameCult.Caching.OutsideStoreTests/GameCult.Caching.OutsideStoreTests.csproj b/tests/GameCult.Caching.OutsideStoreTests/GameCult.Caching.OutsideStoreTests.csproj new file mode 100644 index 00000000..c40c048d --- /dev/null +++ b/tests/GameCult.Caching.OutsideStoreTests/GameCult.Caching.OutsideStoreTests.csproj @@ -0,0 +1,22 @@ + + + + + net10.0 + false + enable + GameCult.Caching.OutsideStoreTests + + + + + + + + + + + + + + diff --git a/tests/GameCult.Caching.OutsideStoreTests/OutsideStoreJudgeTests.cs b/tests/GameCult.Caching.OutsideStoreTests/OutsideStoreJudgeTests.cs new file mode 100644 index 00000000..e47e4d5f --- /dev/null +++ b/tests/GameCult.Caching.OutsideStoreTests/OutsideStoreJudgeTests.cs @@ -0,0 +1,368 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Caching.OutsideStoreTests +{ + // variants:ruling:store-kind-per-key-judge: a store kind written outside the library applies the whole per-key ownership rule + // through the one protected judge (CacheBackingStore.RefuseUndeclaredOnDisk), using only public and protected members. This + // assembly has no InternalsVisibleTo, so a judge parameter type, or something it needs, turning internal breaks it here. + public class OutsideStoreJudgeTests + { + [CultDocument("tests.out_a", "tests.out_a.v1")] + [MessagePackObject] + public sealed class OutA + { + [Key(0)] [CultName] public string Name { get; set; } = string.Empty; + [Key(1)] public int Value { get; set; } + } + + [CultDocument("tests.out_b", "tests.out_b.v1")] + [MessagePackObject] + public sealed class OutB + { + [Key(0)] [CultName] public string Name { get; set; } = string.Empty; + [Key(1)] public int Value { get; set; } + } + + // RbX's schema id, which RbY lists; the test below asserts it, so a changed derivation fails there. + private const string RbXId = "sha256:25087e0d76fd5b9ac950723ccd47f3f3ac1428b53b49ca9bda9aae8b38a0220a"; + + [CultDocument("tests.rb_x", "tests.rb_x.v1")] + [MessagePackObject] + public sealed class RbX + { + [Key(0)] [CultName] public string Name { get; set; } = string.Empty; + [Key(1)] public int Value { get; set; } + } + + [CultDocument("tests.rb_y", "tests.rb_y.v1", CompatibleSchemaIds = new[] { RbXId })] + [MessagePackObject] + public sealed class RbY + { + [Key(0)] [CultName] public string Name { get; set; } = string.Empty; + [Key(1)] public int Value { get; set; } + } + + [CultDocument("tests.rb_z", "tests.rb_z.v1")] + [MessagePackObject] + public sealed class RbZ + { + [Key(0)] [CultName] public string Name { get; set; } = string.Empty; + [Key(1)] public int Value { get; set; } + } + + // What the "disk" holds: records by key and the catalog that publishes their ids. + private sealed class Disk + { + public readonly Dictionary Records = new(); + public readonly List Catalog = new(); + private int _version; + + public void Put(CultPersistedRecord record, CultSchemaCatalogEntry entry) + { + if (Catalog.All(known => known.SchemaId != entry.SchemaId)) + Catalog.Add(entry); + Records[record.Key] = (record, (++_version).ToString()); + } + + public string? SchemaIdAt(string key) => Records.TryGetValue(key, out var held) ? held.Record.SchemaId : null; + } + + // A store kind with no InternalsVisibleTo: an in-memory disk, flushed and committed under the protected judge. + private sealed class OutsideStore : CacheBackingStore + { + private readonly Disk _disk; + private readonly CultCodec _codec; + private readonly HashSet _staged = new(StringComparer.Ordinal); + // StoredVersion is internal to the library, so a store kind tracks the version it last took of each key itself. + private readonly Dictionary _versions = new(StringComparer.Ordinal); + + public OutsideStore(Disk disk, CultCodec codec) + { + _disk = disk; + _codec = codec; + } + + private object Decode(Type type, byte[] payload) => _codec.Deserialize(type, type, payload)!; + + private CultStoredDocument? DurableAt(string key) + { + if (!_disk.Records.TryGetValue(key, out var held)) + return null; + try + { + return ToStoredDocument(held.Record, held.Version, _disk.Catalog, Decode, out _); + } + catch (Exception) + { + return null; + } + } + + public override void PullAll() => Held(() => + { + var loaded = _disk.Records + .Where(pair => !_versions.TryGetValue(pair.Key, out var known) || !SameStoredIdentity(known, pair.Value.Version)) + .Select(pair => DurableAt(pair.Key)) + .OfType() + .ToArray(); + var dropped = Entries.Values.Where(known => !_disk.Records.ContainsKey(known.Key.Value)).ToArray(); + if (loaded.Length > 0 || dropped.Length > 0) + Loaded?.Invoke(loaded, dropped); + foreach (var stored in dropped) + Entries.TryRemove(stored.Key.Value, out _); + foreach (var stored in loaded) + { + Entries[stored.Key.Value] = stored; + _versions[stored.Key.Value] = _disk.Records[stored.Key.Value].Version; + } + }); + + public override void Push(CultStoredDocument entry) => Held(() => + { + Entries[entry.Key.Value] = entry; + _staged.Add(entry.Key.Value); + IsDirty = true; + }); + + public override void Delete(CultStoredDocument entry) => Held(() => + { + Entries.TryRemove(entry.Key.Value, out _); + _staged.Add(entry.Key.Value); + IsDirty = true; + }); + + public override void PushAll() => Held(() => Apply(Array.Empty(), Array.Empty(), Array.Empty())); + + public override CultCommitOutcome CommitBatch(CultCommitRequest request, bool wait) => Held(() => + { + Apply(request.Upserts, request.Deletes, request.Checks); + foreach (var entry in request.Deletes) + Entries.TryRemove(entry.Key.Value, out _); + foreach (var entry in request.Upserts) + Entries[entry.Key.Value] = entry; + return CultCommitOutcome.Committed; + }); + + private void Apply(IReadOnlyList upserts, IReadOnlyList deletes, IEnumerable checks) + { + var writes = upserts.ToDictionary(entry => entry.Key.Value, StringComparer.Ordinal); + foreach (var key in _staged) + { + if (Entries.TryGetValue(key, out var pushed)) + writes.TryAdd(key, pushed); + } + + var removals = deletes.Select(entry => entry.Key.Value) + .Concat(_staged.Where(key => !writes.ContainsKey(key) && !Entries.ContainsKey(key))) + .ToArray(); + var judged = removals.Concat(checks).ToArray(); + var entries = writes.ToDictionary(pair => pair.Key, pair => pair.Value.Descriptor.ToCatalogEntry(), StringComparer.Ordinal); + + // The whole rule, from the protected judge: the same call both shipped stores make. + RefuseUndeclaredOnDisk( + writes.Keys.Concat(judged).Concat(StagedChecks), + _disk.Catalog, + key => _disk.Records.TryGetValue(key, out var held) ? held.Record : null, + key => entries.TryGetValue(key, out var entry) ? entry : null, + judged, + DurableAt, + _ => null, + key => + { + _staged.Remove(key); + IsDirty = _staged.Count > 0; + }); + + foreach (var (key, document) in writes) + { + var type = document.Descriptor.DocumentType; + _disk.Put( + new CultPersistedRecord + { + Key = key, + SchemaId = document.Descriptor.SchemaId, + StoredAt = document.StoredAt, + Payload = _codec.Serialize(document.Document, type, type) + }, + entries[key]); + _versions[key] = _disk.Records[key].Version; + } + + foreach (var key in removals.Where(key => !writes.ContainsKey(key))) + { + _disk.Records.Remove(key); + _versions.Remove(key); + } + _staged.Clear(); + MarkFlushSucceeded(); + } + } + + private static readonly CultDocumentRegistry AB = CultDocumentRegistry.ForTypes(new[] { typeof(OutA), typeof(OutB) }); + private static readonly CultDocumentRegistry Rb = CultDocumentRegistry.ForTypes(new[] { typeof(RbX), typeof(RbY), typeof(RbZ) }); + private static readonly CultRecordKey K = new("k"); + private static readonly CultRecordKey K2 = new("k2"); + + private static CultCache Open(Disk disk, CultDocumentRegistry registry) + { + var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry)); + cache.AddBackingStore(new OutsideStore(disk, cache.Codec!)); + cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + return cache; + } + + // Another writer puts a record of this type at the key on the disk, as the cache's own store would have. + private static void PutOnDisk(Disk disk, CultDocumentRegistry registry, string key, T document, string? asSchemaId = null, CultSchemaCatalogEntry? entry = null) + where T : class + { + var codec = CultCacheMessagePack.CreateCodec(registry); + var descriptor = registry.GetRequired(); + disk.Put( + new CultPersistedRecord + { + Key = key, + SchemaId = asSchemaId ?? descriptor.SchemaId, + StoredAt = DateTimeOffset.UtcNow.ToString("O"), + Payload = codec.Serialize(document, typeof(T), typeof(T)) + }, + entry ?? descriptor.ToCatalogEntry()); + } + + private static Type HeldType(CultCache cache, CultRecordKey key) => cache.GetStored(key)!.Descriptor.DocumentType; + + // The written type's own id: a record of another registered type that this cache never loaded. + [Test] + public void AnotherRegisteredTypesUnheldRecordIsRefusedUnderTheLockAndReloaded() + { + var disk = new Disk(); + using var cache = Open(disk, AB); + PutOnDisk(disk, AB, "k", new OutB { Name = "k", Value = 2 }); + var before = disk.Records["k"]; + + Assert.That( + () => cache.Commit(batch => batch.Upsert(typeof(OutA), new OutA { Name = "k", Value = 1 }, K)), + Throws.InstanceOf()); + Assert.That(disk.Records["k"], Is.EqualTo(before), "the refused write changed nothing on the disk"); + Assert.That(HeldType(cache, K), Is.EqualTo(typeof(OutB)), "the refusal reloaded the record the store holds"); + } + + [Test] + public void AFlushOfAnotherRegisteredTypesUnheldRecordIsRefusedToo() + { + var disk = new Disk(); + using var cache = Open(disk, AB); + PutOnDisk(disk, AB, "k", new OutB { Name = "k", Value = 2 }); + + cache.UpsertAsync(typeof(OutA), new OutA { Name = "k", Value = 1 }, K).GetAwaiter().GetResult(); + Assert.That(() => cache.FlushAsync().GetAwaiter().GetResult(), Throws.InstanceOf()); + Assert.That(disk.SchemaIdAt("k"), Is.EqualTo(AB.GetRequired().SchemaId)); + Assert.That(HeldType(cache, K), Is.EqualTo(typeof(OutB))); + } + + // Claimant precedence, a listed id: the written type owns a record under an id it lists, and a type that lists nothing does not. + [Test] + public void ARecordUnderAnIdTheWrittenTypeListsIsOwnedAndOneItDoesNotListIsRefused() + { + Assert.That(Rb.GetRequired().SchemaId, Is.EqualTo(RbXId)); + var disk = new Disk(); + using var cache = Open(disk, Rb); + PutOnDisk(disk, Rb, "k", new RbX { Name = "k" }); + PutOnDisk(disk, Rb, "k2", new RbX { Name = "k2" }); + + Assert.That(cache.Commit(batch => batch.Upsert(typeof(RbY), new RbY { Name = "k", Value = 1 }, K)), Is.True); + Assert.That(disk.SchemaIdAt("k"), Is.EqualTo(Rb.GetRequired().SchemaId)); + + Assert.That( + () => cache.Commit(batch => batch.Upsert(typeof(RbZ), new RbZ { Name = "k2" }, K2)), + Throws.InstanceOf()); + Assert.That(disk.SchemaIdAt("k2"), Is.EqualTo(RbXId), "the type that lists nothing replaced nothing"); + } + + // Reads whole: an older id of the written type's own schema name, which the type reads whole, is its own and is restamped. + [Test] + public void AnOlderIdOfTheWrittenSchemaNameThatIsReadWholeIsOwnedAndRestamped() + { + var disk = new Disk(); + using var cache = Open(disk, AB); + var current = AB.GetRequired().ToCatalogEntry(); + var older = new CultSchemaCatalogEntry + { + SchemaId = "sha256:older-out-a", + SchemaName = current.SchemaName, + SchemaVersion = "tests.out_a.v0", + ContentHash = current.ContentHash, + CanonicalSchemaJson = current.CanonicalSchemaJson, + Members = current.Members + }; + PutOnDisk(disk, AB, "k", new OutA { Name = "k" }, asSchemaId: older.SchemaId, entry: older); + + Assert.That(cache.Commit(batch => batch.Upsert(typeof(OutA), new OutA { Name = "k", Value = 3 }, K)), Is.True); + Assert.That(disk.SchemaIdAt("k"), Is.EqualTo(current.SchemaId)); + } + + // Staged retype removals: a held key retypes as a removal then an add, in one batch and staged apart. + [TestCase(true)] + [TestCase(false)] + public void ARetypeOfAHeldKeyCommitsInOneBatchAndStagedApart(bool oneBatch) + { + var disk = new Disk(); + PutOnDisk(disk, AB, "k", new OutB { Name = "k", Value = 2 }); + using var cache = Open(disk, AB); + Assert.That(HeldType(cache, K), Is.EqualTo(typeof(OutB))); + + if (oneBatch) + Assert.That(cache.Commit(batch => + { + batch.Remove(K); + batch.Upsert(typeof(OutA), new OutA { Name = "k", Value = 1 }, K); + }), Is.True); + else + { + Assert.That(cache.Remove(K), Is.True); + Assert.That(cache.Commit(batch => batch.Upsert(typeof(OutA), new OutA { Name = "k", Value = 1 }, K)), Is.True); + } + + Assert.That(disk.SchemaIdAt("k"), Is.EqualTo(AB.GetRequired().SchemaId)); + Assert.That(HeldType(cache, K), Is.EqualTo(typeof(OutA))); + } + + [Test] + public void ARetypeOfAKeyTheCacheNeverLoadedIsJudgedByTheRemovalRule() + { + var disk = new Disk(); + using var cache = Open(disk, AB); + PutOnDisk(disk, AB, "k", new OutB { Name = "k", Value = 2 }); + + Assert.That(cache.Commit(batch => + { + batch.Remove(K); + batch.Upsert(typeof(OutA), new OutA { Name = "k", Value = 1 }, K); + }), Is.True, "a record a registered type reads may be removed, so the key retypes"); + Assert.That(disk.SchemaIdAt("k"), Is.EqualTo(AB.GetRequired().SchemaId)); + } + + // A removal is never licensed over a record no registered type owns, retype or not. + [Test] + public void ACommitNeverRemovesOrRetypesARecordNoRegisteredTypeOwns() + { + var disk = new Disk(); + using var cache = Open(disk, AB); + var unknown = new CultSchemaCatalogEntry { SchemaId = "tests.out_unknown", SchemaName = "tests.out_unknown" }; + disk.Put(new CultPersistedRecord { Key = "k", SchemaId = unknown.SchemaId, StoredAt = "t", Payload = new byte[] { 0x90 } }, unknown); + var before = disk.Records["k"]; + + Assert.That(() => cache.Commit(batch => + { + batch.Remove(K); + batch.Upsert(typeof(OutA), new OutA { Name = "k" }, K); + }), Throws.InstanceOf(), "retype"); + Assert.That(() => cache.Commit(batch => batch.Remove(K)), Throws.InstanceOf(), "removal"); + Assert.That(disk.Records["k"], Is.EqualTo(before)); + } + } +} diff --git a/tests/GameCult.Caching.Tests/ElementIdTests.cs b/tests/GameCult.Caching.Tests/ElementIdTests.cs index aa1d7b57..1b6b3fe2 100644 --- a/tests/GameCult.Caching.Tests/ElementIdTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdTests.cs @@ -274,6 +274,39 @@ namespace GameCult.Caching.Tests Assert.That(DiskRecord(path, "a").SchemaId, Is.Not.EqualTo(otherRuntimeId)); } + // Pushing a loaded record rewrites it in place under this type's id, and the cache keeps that entry. A newer version of the type, + // registered afterwards, reads the entry the record is now published by whole, so it owns the record: staging asks the entry + // that publishes the id the record carries now, not the one that published the id it was loaded under. + [Test] + public void ANewerVersionReadsWholeTheRecordAPushRelabelledInPlace() + { + const string otherRuntimeId = "other.runtime.id_deck"; + var path = PathOf("mint-then-newer.cc"); + WritePreIdStore(path, null, ("a", "a", 1)); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + var entry = snapshot.SchemaCatalog.Single(); + entry.SchemaId = otherRuntimeId; + entry.ContentHash = otherRuntimeId; + entry.CompatibleSchemaIds = new[] { otherRuntimeId }; + snapshot.Records.Single().SchemaId = otherRuntimeId; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + var key = new CultRecordKey("a"); + + using (var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry })) + { + var loaded = cache.GetStored(key)!; + cache.BackingStores[0].Push(loaded); + cache.BackingStores[0].PushAll(); + Assert.That(ReferenceEquals(cache.GetStored(key), loaded), Is.True, "the cache keeps the entry the store relabelled"); + Assert.That(loaded.StoredSchemaId, Is.Not.EqualTo(otherRuntimeId), "the push relabelled the loaded entry in place"); + registry.GetRequired(typeof(IdDeckNext)); + Assert.That(cache.Commit(batch => batch.Upsert(typeof(IdDeckNext), new IdDeckNext { Name = "newer" }, key)), Is.True); + } + + Assert.That(DiskRecord(path, "a").SchemaId, Is.EqualTo(registry.GetRequired(typeof(IdDeckNext)).SchemaId)); + } + // A pre-id record this cache reads only by dropping a persisted slot is read-only, so the rewrite that would persist its // minted ids is refused typed, naming it, and the store is left byte for byte. [Test] @@ -1401,6 +1434,16 @@ namespace GameCult.Caching.Tests [Key(2)] public List Moves { get; set; } = new(); } + // The next version of IdDeck: the same slots under a new schema version. + [CultDocument("tests.element_id_deck", "tests.element_id_deck.v2")] + [MessagePackObject] + public sealed class IdDeckNext + { + [Key(0)] [CultName] public string Name { get; set; } = ""; + [Key(1)] public List Reels { get; set; } = new(); + [Key(2)] public List Moves { get; set; } = new(); + } + // The shapes before ids: same slots, no id member. Not documents, so never registered. [MessagePackObject] public sealed class OldMark diff --git a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs index ab4cb046..61aa236e 100644 --- a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs +++ b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs @@ -67,6 +67,13 @@ namespace GameCult.Caching.Tests private static CultCache Open(string path, CultDocumentRegistry registry, bool directory) => CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry, UseDirectoryStore = directory }); + // The other writer retypes a key: a removal and an add in separate commits (variants:ruling:cache-refuses-retype). + private static void Retype(CultCache writer, Type type, object document, CultRecordKey key) + { + Assert.That(writer.Commit(batch => batch.Remove(key)), Is.True); + Assert.That(writer.Commit(batch => batch.Upsert(type, document, key)), Is.True); + } + private static object DeckOf(string name) => New(Deck, ("Name", name)); private static object WidgetOf(string label) => New(Widget, ("Label", label)); @@ -205,7 +212,7 @@ namespace GameCult.Caching.Tests var path = Seeded("replaced.cc", directory); using var cache = Open(path, DeckOnly, directory); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + Retype(other, Widget, WidgetOf("now a widget"), D); var before = Fingerprint(path); var refusal = Assert.Throws(() => Land(cache, write, D, DeckOf(Canary)))!; @@ -228,7 +235,7 @@ namespace GameCult.Caching.Tests var path = Seeded("unstuck.cc", directory); using var cache = Open(path, DeckOnly, directory); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + Retype(other, Widget, WidgetOf("now a widget"), D); var widget = Stored(path, D); Assert.Throws(() => Land(cache, refused, D, DeckOf(Canary))); @@ -249,7 +256,7 @@ namespace GameCult.Caching.Tests var path = Seeded("not-served.cc", directory); using var cache = Open(path, DeckOnly, directory); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + Retype(other, Widget, WidgetOf("now a widget"), D); var refusal = Assert.Throws(() => Land(cache, refused, D, DeckOf(Canary)))!; Assert.That(refusal.Message, Does.StartWith("Record 'd' (schema id '" + WidgetId + "'"), "one refused key reads as one record"); @@ -287,11 +294,10 @@ namespace GameCult.Caching.Tests }); using var cache = Open(path, DeckOnly, directory); using (var other = Open(path, Full, directory)) - other.Commit(batch => - { - batch.Upsert(Widget, WidgetOf("w1"), D); - batch.Upsert(Widget, WidgetOf("w2"), d2); - }); + { + Retype(other, Widget, WidgetOf("w1"), D); + Retype(other, Widget, WidgetOf("w2"), d2); + } var refusal = Assert.Throws(() => { @@ -332,7 +338,7 @@ namespace GameCult.Caching.Tests var path = Seeded("untouched.cc", directory); using var cache = Open(path, DeckOnly, directory); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + Retype(other, Widget, WidgetOf("now a widget"), D); var widget = Stored(path, D); Land(cache, write, E, DeckOf("e")); @@ -356,7 +362,7 @@ namespace GameCult.Caching.Tests var path = Seeded("restamp.cc", directory); using var cache = Open(path, DeckOnly, directory); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + Retype(other, Widget, WidgetOf("now a widget"), D); Land(cache, write, E, DeckOf("e")); @@ -374,7 +380,7 @@ namespace GameCult.Caching.Tests var path = Seeded("stale.cc", directory); using var cache = Open(path, DeckOnly, directory); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + Retype(other, Widget, WidgetOf("now a widget"), D); var widget = Stored(path, D); Assert.Throws(() => Land(cache, Write.Commit, D, DeckOf(Canary))); @@ -392,7 +398,7 @@ namespace GameCult.Caching.Tests var path = Seeded("staged-commit.cc", directory); using var cache = Open(path, DeckOnly, directory); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + Retype(other, Widget, WidgetOf("now a widget"), D); var widget = Stored(path, D); cache.UpsertAsync(Deck, DeckOf(Canary), D).GetAwaiter().GetResult(); @@ -413,7 +419,7 @@ namespace GameCult.Caching.Tests using var cache = Open(path, DeckOnly, directory); Land(cache, landed, D, DeckOf("d again")); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + Retype(other, Widget, WidgetOf("now a widget"), D); var widget = Stored(path, D); Land(cache, Write.Commit, E, DeckOf("e")); @@ -431,7 +437,7 @@ namespace GameCult.Caching.Tests cache.UpsertAsync(Deck, DeckOf("d staged"), D).GetAwaiter().GetResult(); Land(cache, Write.Commit, E, DeckOf("e")); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + Retype(other, Widget, WidgetOf("now a widget"), D); var widget = Stored(path, D); Land(cache, Write.Commit, new CultRecordKey("e2"), DeckOf("e2")); @@ -448,7 +454,7 @@ namespace GameCult.Caching.Tests var path = Seeded("staged-removal.cc", directory); using var cache = Open(path, DeckOnly, directory); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + Retype(other, Widget, WidgetOf("now a widget"), D); var widget = Stored(path, D); cache.Remove(D); @@ -468,7 +474,7 @@ namespace GameCult.Caching.Tests var path = Seeded("pending.cc", directory); using var cache = Open(path, DeckOnly, directory); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + Retype(other, Widget, WidgetOf("now a widget"), D); cache.UpsertAsync(Deck, DeckOf(Canary), D).GetAwaiter().GetResult(); Assert.Throws(() => cache.FlushAsync().GetAwaiter().GetResult()); @@ -478,7 +484,7 @@ namespace GameCult.Caching.Tests cache.UpsertAsync(Deck, DeckOf(Canary), E).GetAwaiter().GetResult(); cache.UpsertAsync(Deck, DeckOf("v"), V).GetAwaiter().GetResult(); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), E)); + Retype(other, Widget, WidgetOf("now a widget"), E); Assert.Throws(() => cache.FlushAsync().GetAwaiter().GetResult()); Assert.That(cache.IsDirty, Is.True, "v is still staged"); cache.FlushAsync().GetAwaiter().GetResult(); @@ -495,7 +501,7 @@ namespace GameCult.Caching.Tests using var cache = Open(path, DeckOnly, directory); cache.Commit(batch => batch.Upsert(Deck, DeckOf("e"), E)); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + Retype(other, Widget, WidgetOf("now a widget"), D); var before = Fingerprint(path); // A condition keeps the single-file commit on the file as it is, so only the removal reaches d. @@ -516,7 +522,7 @@ namespace GameCult.Caching.Tests var path = Seeded("pulled.cc", directory); using var cache = Open(path, DeckOnly, directory); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + Retype(other, Widget, WidgetOf("now a widget"), D); cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); @@ -562,7 +568,7 @@ namespace GameCult.Caching.Tests using var cache = Open(path, DeckOnly, directory); var z = new CultRecordKey("z"); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("z"), z)); + Retype(other, Widget, WidgetOf("z"), z); var before = Stored(path, z); Land(cache, Write.Flush, E, DeckOf("e")); @@ -803,8 +809,9 @@ namespace GameCult.Caching.Tests Assert.That(EmittedDocumentTypes.Read(reopened.Get(D)!, "Name"), Is.EqualTo("claimed")); } - // Whether a record is read-only is read live from the registry at the write, not fixed at load: a type registered after the - // load that declares the id makes the record writable, and the write stores it under the writing type's id, not the declarer's. + // Whether a record is writable is decided at the write, not fixed at load, and by the writing type alone: a type registered + // after the load that declares the id writes the record and stores it under its own id, and a registry holding it does not make + // another type the record's writer. [Test] public void ATypeRegisteredAfterLoadThatDeclaresTheIdMakesItWritable() { @@ -815,14 +822,35 @@ namespace GameCult.Caching.Tests { Assert.Throws(() => cache.Commit(batch => batch.Upsert(Deck, DeckOf("refused"), D))); registry.GetRequired(LateClaimingDeck); - cache.Commit(batch => batch.Upsert(Deck, DeckOf("claimed"), D)); + Assert.Throws(() => cache.Commit(batch => batch.Upsert(Deck, DeckOf("still refused"), D)), "the registry's other type does not make the record the writing type's"); + cache.Commit(batch => batch.Upsert(LateClaimingDeck, New(LateClaimingDeck, ("Name", "claimed")), D)); } - Assert.That(Read(path).Records.Single().SchemaId, Is.EqualTo(registry.GetRequired(Deck).SchemaId)); - using var reopened = Open(path, DeckOnly, directory: false); + Assert.That(Read(path).Records.Single().SchemaId, Is.EqualTo(registry.GetRequired(LateClaimingDeck).SchemaId)); + using var reopened = Open(path, CultDocumentRegistry.ForTypes(new[] { LateClaimingDeck }), directory: false); Assert.That(EmittedDocumentTypes.Read(reopened.Get(D)!, "Name"), Is.EqualTo("claimed")); } + // The removal rule asks the registry as it is when the removal is judged: a record read-only at load becomes removable once a + // type that declares its id is registered. The write rule is not the removal rule (the test above): the late type may remove + // the record, and a removal is judged by what the registry declares, not by the type that last wrote the key. + [Test] + public void ATypeRegisteredAfterLoadThatDeclaresTheIdMakesTheRecordRemovable() + { + var path = OtherRuntimeStore("late-remove.cc"); + var registry = CultDocumentRegistry.ForTypes(new[] { Deck }); + + using (var cache = Open(path, registry, directory: false)) + { + Assert.Throws(() => cache.Commit(batch => batch.Remove(D)), "no registered type declares the id yet"); + registry.GetRequired(LateClaimingDeck); + Assert.That(cache.Commit(batch => batch.Remove(D)), Is.True); + Assert.That(cache.Get(D), Is.Null); + } + + Assert.That(Read(path).Records, Is.Empty); + } + // A commit onto the file that involves a variant is judged on the set the file will hold. A foreign record is not decoded // for that judgement, so it neither fails the commit nor joins the set. [Test] @@ -1132,7 +1160,7 @@ namespace GameCult.Caching.Tests a.UpsertAsync(Deck, DeckOf("e"), E).GetAwaiter().GetResult(); Assert.That(a.Remove(W), Is.False); using (var b = Open(path, Full, directory)) - b.Commit(batch => batch.Upsert(Deck, DeckOf("theirs"), W)); + Retype(b, Deck, DeckOf("theirs"), W); var theirs = Stored(path, W); Assert.DoesNotThrow(() => a.FlushAsync().GetAwaiter().GetResult()); @@ -1140,7 +1168,7 @@ namespace GameCult.Caching.Tests Assert.That(Read(path).Records.Select(record => record.Key), Is.EquivalentTo(new[] { "d", "e", "w" }), "the staged write landed beside the checked key"); Assert.That(Stored(path, W), Is.EqualTo(theirs), "the record at the checked key is still the other writer's bytes"); using (var c = Open(path, Full, directory)) - c.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), W)); + Retype(c, Widget, WidgetOf("now a widget"), W); Assert.DoesNotThrow(() => a.FlushAsync().GetAwaiter().GetResult(), "a flush clears what it judged: the next one has nothing to judge"); } @@ -1181,7 +1209,7 @@ namespace GameCult.Caching.Tests Assert.That(StoreOf(a).ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "w" })); // The writer declares the widget's id, so it may replace the widget with a deck record under an id a reads but does not declare. using (var other = Open(path, CultDocumentRegistry.ForTypes(new[] { MovingDeck, Widget }), directory)) - Assert.That(other.Commit(batch => batch.Upsert(MovingDeck, New(MovingDeck, ("Name", "moved")), W)), Is.True); + Retype(other, MovingDeck, New(MovingDeck, ("Name", "moved")), W); Assert.That(a.Remove(W), Is.False); var refusal = Assert.Throws(() => a.FlushAsync().GetAwaiter().GetResult())!; @@ -1293,7 +1321,7 @@ namespace GameCult.Caching.Tests else cache.FlushAsync().GetAwaiter().GetResult(); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), staged)); + Retype(other, Widget, WidgetOf("now a widget"), staged); Assert.DoesNotThrow(() => cache.FlushAsync().GetAwaiter().GetResult()); } @@ -1371,7 +1399,7 @@ namespace GameCult.Caching.Tests using var cache = Open(path, DeckOnly, directory); Assert.That(StoreOf(cache).ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "w" })); using (var other = Open(path, Full, directory)) - other.Commit(batch => batch.Upsert(Deck, DeckOf("theirs"), W)); + Retype(other, Deck, DeckOf("theirs"), W); var theirs = Stored(path, W); Assert.That(cache.Commit(batch => batch.Remove(W)), Is.True); @@ -1455,7 +1483,7 @@ namespace GameCult.Caching.Tests using var _ = cache; Assert.That(cache.BackingStores[1].ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "w" })); using (var late = Open(other, CultDocumentRegistry.ForTypes(new[] { Gizmo, Widget }), directory)) - late.Commit(batch => batch.Upsert(Gizmo, New(Gizmo, ("Tag", "theirs")), W)); + Retype(late, Gizmo, New(Gizmo, ("Tag", "theirs")), W); var theirs = Stored(other, W); Assert.That(cache.Commit(batch => diff --git a/tests/GameCult.Caching.Tests/RestampOnWriteTests.cs b/tests/GameCult.Caching.Tests/RestampOnWriteTests.cs index b90f4336..b1976aa6 100644 --- a/tests/GameCult.Caching.Tests/RestampOnWriteTests.cs +++ b/tests/GameCult.Caching.Tests/RestampOnWriteTests.cs @@ -480,17 +480,18 @@ namespace GameCult.Caching.Tests private static readonly CultDocumentRegistry NarrowShortRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(NoteNarrow), typeof(NoteShort) }); - // Which entry reads a persisted entry is one precedence: a cacheless writer lent a registry picks the claimant that registry - // picks, never one by schema name over its own entry. Here the entry is named and shaped like Narrow but lists Short's id. + // A write's ownership is the written entry's alone (variants:ruling:own-entry-reads-whole): the stored entry is named and shaped + // like Narrow but lists Short's id, and a registry that holds Short (which would read it by listing the id) changes nothing, for + // a cache and a cacheless writer alike. [TestCase(false)] [TestCase(true)] - public void ACachelessWriterPicksTheClaimantItsRegistryPicks(bool cacheless) + public void AWrittenEntryReadingTheRecordWholeOwnsItWhateverTheRegistryHolds(bool cacheless) { var path = SeedRenamed(listsNarrow: false, alsoLists: NarrowShortRegistry.GetRequired().SchemaId, name: "tests.restamp_note"); - var before = Fingerprint(path); - Assert.Throws(() => WriteNarrow(path, cacheless, NarrowShortRegistry)); - Assert.That(Fingerprint(path), Is.EqualTo(before)); + WriteNarrow(path, cacheless, NarrowShortRegistry); + + Assert.That(Snapshot(path).Records.Single(record => record.Key == "note").SchemaId, Is.EqualTo(NarrowRegistry.GetRequired().SchemaId)); } [TestCase(false)] @@ -668,28 +669,6 @@ namespace GameCult.Caching.Tests Assert.That(Fingerprint(path), Is.EqualTo(before)); } - // The shadow, where it is decided: a written entry answers for its schema id whatever the registry holds, and the registry - // still adds its other types. - [Test] - public void AWrittenEntryShadowsTheLentRegistrysTypeForItsId() - { - var written = NarrowRegistry.GetRequired().ToCatalogEntry(); - var lookup = new CatalogEntryLookup(new[] { written }, NarrowListerRegistry); - var lister = NarrowListerRegistry.GetRequired(); - - Assert.That(lookup.Owner(written.SchemaId), Is.SameAs(written)); - Assert.That(lookup.Named(written.SchemaName), Is.EqualTo(new[] { written }).Using((x, y) => ReferenceEquals(x, y))); - Assert.That(lookup.Listing(written.SchemaId), Is.EqualTo(new[] { written }).Using((x, y) => ReferenceEquals(x, y))); - Assert.That(lookup.Owner(lister.SchemaId)!.SchemaId, Is.EqualTo(lister.SchemaId)); - Assert.That(lookup.Listing(ListerOldId).Select(entry => entry.SchemaId), Is.EqualTo(new[] { lister.SchemaId })); - Assert.That(lookup.Named(lister.SchemaName).Select(entry => entry.SchemaId), Is.EqualTo(new[] { lister.SchemaId })); - - // A write of several entries shadows by the entry that has the id, not by all of them having it. - var other = CultDocumentRegistry.ForTypes(new[] { typeof(NoteOther) }).GetRequired().ToCatalogEntry(); - var several = new CatalogEntryLookup(new[] { other, written }, NarrowListerRegistry); - Assert.That(several.Named(written.SchemaName), Is.EqualTo(new[] { written }).Using((x, y) => ReferenceEquals(x, y))); - } - // The registry's other types change nothing about a record the written type claims itself. [Test] public void AnotherRegisteredTypeDoesNotTakeAWritersOwnDriftRecord() @@ -701,16 +680,16 @@ namespace GameCult.Caching.Tests Assert.That(Snapshot(path).Records.Single(record => record.Key == "note").SchemaId, Is.EqualTo(NarrowRegistry.GetRequired().SchemaId)); } - // What the lent registry registers beyond the written type can only make the claim more ambiguous, never own more: a second - // version of the written schema's name refuses a record the write owned by name alone. + // What the lent registry registers beyond the written type changes nothing, in either direction: a second version of the + // written schema's name neither refuses nor widens a record the write owns by name alone. [Test] - public void ASecondVersionOfTheWrittenNameInTheLentRegistryOnlyRefuses() + public void ASecondVersionOfTheWrittenNameInTheLentRegistryChangesNothing() { var path = SeedRenamed(listsNarrow: false, name: "tests.restamp_note"); - var before = Fingerprint(path); - Assert.Throws(() => WriteNarrow(path, cacheless: true, CultDocumentRegistry.ForTypes(new[] { typeof(NoteNarrow), typeof(NoteGrown) }))); - Assert.That(Fingerprint(path), Is.EqualTo(before)); + WriteNarrow(path, cacheless: true, CultDocumentRegistry.ForTypes(new[] { typeof(NoteNarrow), typeof(NoteGrown) })); + + Assert.That(Snapshot(path).Records.Single(record => record.Key == "note").SchemaId, Is.EqualTo(NarrowRegistry.GetRequired().SchemaId)); } // A record stored under an id its catalog entry only lists (no entry owns it) is published by that entry, and the type that @@ -765,8 +744,8 @@ namespace GameCult.Caching.Tests var writes = new[] { narrow }; Func declares = id => id == narrow.SchemaId; - Assert.That(CultDocumentRegistry.Owns("tests.restamp.unpublished", writes, declares, new CatalogEntryLookup(writes, null), entry => entry, _ => true), Is.False); - Assert.That(CultDocumentRegistry.Owns(narrow.SchemaId, writes, declares, new CatalogEntryLookup(writes, null), entry => entry, _ => true), Is.True); + Assert.That(CultDocumentRegistry.Owns("tests.restamp.unpublished", writes, declares, new CatalogEntryLookup(writes), entry => entry), Is.False); + Assert.That(CultDocumentRegistry.Owns(narrow.SchemaId, writes, declares, new CatalogEntryLookup(writes), entry => entry), Is.True); } private static string VectorPath(string name) diff --git a/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs b/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs new file mode 100644 index 00000000..9e924d81 --- /dev/null +++ b/tests/GameCult.Caching.Tests/RetypeEvictionTests.cs @@ -0,0 +1,659 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; +using R3; + +namespace GameCult.Caching.Tests +{ + // variants:ruling:one-batch-retype-is-eviction-then-add: a batch that removes a key and adds a record there stages the held + // record's eviction and then the add, so watchers, the batch validators, the store judges and the removal bookkeeping see what they + // see when the two are staged apart (cache.Remove, then UpsertAsync). Where a test runs both shapes it asserts the same answer. + public class RetypeEvictionTests + { + [CultDocument("tests.ev_a", "tests.ev_a.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class EvA + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + } + + [CultDocument("tests.ev_b", "tests.ev_b.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class EvB + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + } + + // A type only the second writer registers. + [CultDocument("tests.ev_c", "tests.ev_c.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class EvC + { + [Key(0)] [CultName] public string Name = string.Empty; + } + + [CultDocument("tests.ev_g", "tests.ev_g.v1")] + [MessagePackObject(AllowPrivate = true)] + [CultGlobal] + internal sealed class EvG + { + [Key(0)] public string Value = string.Empty; + } + + // RbX's schema id, which RbY lists. The test below asserts it, so a changed derivation fails there. + private const string RbXId = "sha256:25087e0d76fd5b9ac950723ccd47f3f3ac1428b53b49ca9bda9aae8b38a0220a"; + + // The rebase pair: RbY lists RbX's id compatible, so a plain write of RbY owns an RbX record; RbZ lists nothing. + [CultDocument("tests.rb_x", "tests.rb_x.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class RbX + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + } + + [CultDocument("tests.rb_y", "tests.rb_y.v1", CompatibleSchemaIds = new[] { RbXId })] + [MessagePackObject(AllowPrivate = true)] + internal sealed class RbY + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + } + + [CultDocument("tests.rb_z", "tests.rb_z.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class RbZ + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + } + + private static readonly CultDocumentRegistry AB = CultDocumentRegistry.ForTypes(new[] { typeof(EvA), typeof(EvB), typeof(EvG) }); + private static readonly CultDocumentRegistry ABC = CultDocumentRegistry.ForTypes(new[] { typeof(EvA), typeof(EvB), typeof(EvC), typeof(EvG) }); + private static readonly CultDocumentRegistry Rb = CultDocumentRegistry.ForTypes(new[] { typeof(RbX), typeof(RbY), typeof(RbZ) }); + private static readonly CultRecordKey K1 = new("k1"); + private static readonly CultRecordKey K2 = new("k2"); + private static readonly CultRecordKey K3 = new("k3"); + + // The stores a retype is judged in, and no store at all. The directory store holds no variant, so variant tests skip it. + public static readonly object[] Kinds = { "single-file", "directory", "storeless" }; + public static readonly object[] VariantKinds = { "single-file", "storeless" }; + + private string _directory = string.Empty; + private int _paths; + + [SetUp] + public void CreateDirectory() + { + _directory = Path.Combine(Path.GetTempPath(), $"cultlib-retypeeviction-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_directory); + } + + [TearDown] + public void DeleteDirectory() + { + if (Directory.Exists(_directory)) + Directory.Delete(_directory, recursive: true); + } + + // A fresh store path per call, so one test can run a scenario twice. + private string NewPath(string kind) => Path.Combine(_directory, $"{kind}-{_paths++}", kind == "directory" ? "store" : "store.cc"); + + private static CultCache Open(string path, string kind, CultDocumentRegistry registry) + { + var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry)); + if (kind != "storeless") + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + cache.AddBackingStore(kind == "directory" ? new DirectoryMessagePackBackingStore(path) : new SingleFileMessagePackBackingStore(path)); + cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + } + + return cache; + } + + private static Type TypeAt(string path, string kind, CultDocumentRegistry registry, CultRecordKey key) + { + using var cache = Open(path, kind, registry); + return cache.GetStored(key)!.Descriptor.DocumentType; + } + + // Retypes a held key as one batch (Remove, then the add the caller stages) or staged apart (cache.Remove, then the add's own call). + private static void Retype(CultCache cache, bool oneBatch, CultRecordKey key, Action add, Action apart) + { + if (oneBatch) + Assert.That(cache.Commit(batch => + { + batch.Remove(key); + add(batch); + }), Is.True); + else + { + Assert.That(cache.Remove(key), Is.True); + apart(); + } + } + + private static void PutB(CultCache cache, CultRecordKey key, int value = 2) => + Assert.That(cache.Commit(batch => batch.Upsert(typeof(EvB), new EvB { Name = key.Value, Value = value }, key)), Is.True); + + // finding retype-not-an-eviction: Watch sees Removed and Watch sees Added, in a batch and staged apart alike. + [TestCaseSource(nameof(Kinds))] + public void ARetypeIsRemovedForTheOldTypeAndAddedForTheNewWhetherOneBatchOrStagedApart(string kind) + { + foreach (var oneBatch in new[] { true, false }) + { + using var cache = Open(NewPath(kind), kind, AB); + PutB(cache, K2); + var removedLog = new List(); + var addedLog = new List(); + using var oldWatch = cache.Watch().Subscribe(change => removedLog.Add(change.Kind)); + using var newWatch = cache.Watch().Subscribe(change => addedLog.Add(change.Kind)); + + Retype(cache, oneBatch, K2, + batch => batch.Upsert(typeof(EvA), new EvA { Name = "k2", Value = 9 }, K2), + () => cache.UpsertAsync(typeof(EvA), new EvA { Name = "k2", Value = 9 }, K2).GetAwaiter().GetResult()); + + Assert.That(removedLog, Is.EqualTo(new[] { CultCacheDocumentChangeKind.Removed }), $"Watch, one batch {oneBatch}"); + Assert.That(addedLog, Is.EqualTo(new[] { CultCacheDocumentChangeKind.Added }), $"Watch, one batch {oneBatch}"); + Assert.That(cache.GetStored(K2)!.Descriptor.DocumentType, Is.EqualTo(typeof(EvA))); + } + } + + // finding retype-not-an-eviction, global key, and finding global-eviction-uncovered: evicting a global frees its slot, and the + // batch that retypes a global's key may land the next global of its type elsewhere, as staged apart. + [TestCaseSource(nameof(Kinds))] + public void EvictingAGlobalFreesItsSlotAndAGlobalKeyRetypesInOneBatch(string kind) + { + using var cache = Open(NewPath(kind), kind, AB); + var g1 = new CultRecordKey("g1"); + var g2 = new CultRecordKey("g2"); + Assert.That(cache.Commit(batch => batch.Upsert(typeof(EvG), new EvG { Value = "one" }, g1)), Is.True); + + Assert.Throws( + () => cache.Commit(batch => batch.Upsert(typeof(EvG), new EvG { Value = "two" }, g2)), "a second global while the first stands is refused"); + + Assert.That(cache.Commit(batch => + { + batch.Remove(g1); + batch.Upsert(typeof(EvG), new EvG { Value = "two" }, g2); + }), Is.True, "the eviction freed the slot"); + Assert.That(cache.GetGlobal()!.Value, Is.EqualTo("two")); + Assert.That(cache.GetStored(g1), Is.Null); + + Assert.That(cache.Commit(batch => + { + batch.Remove(g2); + batch.Upsert(typeof(EvA), new EvA { Name = "g2", Value = 1 }, g2); + batch.Upsert(typeof(EvG), new EvG { Value = "three" }, g1); + }), Is.True, "staged apart the retype of the global's key and the next global commit"); + Assert.That(cache.GetStored(g2)!.Descriptor.DocumentType, Is.EqualTo(typeof(EvA))); + Assert.That(cache.GetGlobal()!.Value, Is.EqualTo("three")); + } + + // finding retype-not-an-eviction (variant key) and variant-ops-drop-the-removal: a variant key retypes to a plain record of + // another type, a plain key becomes a variant of another type's base, and a key removed, added and then added as a variant + // lands as the variant, each answering as it does staged apart. + [TestCaseSource(nameof(VariantKinds))] + public void VariantKeysRetypeInOneBatchAsStagedApart(string kind) + { + foreach (var oneBatch in new[] { true, false }) + { + using var cache = Open(NewPath(kind), kind, AB); + var baseA = new CultRecordKey("base-a"); + Assert.That(cache.Commit(batch => + { + batch.Upsert(typeof(EvA), new EvA { Name = "base-a", Value = 1 }, baseA); + batch.UpsertVariant(K3, baseA, new[] { cache.Override(nameof(EvA.Name), "v3") }); + batch.Upsert(typeof(EvB), new EvB { Name = "k2", Value = 2 }, K2); + batch.Upsert(typeof(EvB), new EvB { Name = "k1", Value = 2 }, K1); + }), Is.True); + + Retype(cache, oneBatch, K3, + batch => batch.Upsert(typeof(EvB), new EvB { Name = "k3", Value = 3 }, K3), + () => cache.UpsertAsync(typeof(EvB), new EvB { Name = "k3", Value = 3 }, K3).GetAwaiter().GetResult()); + Assert.That(cache.GetStored(K3)!.Variant, Is.Null, $"one batch {oneBatch}"); + Assert.That(cache.GetStored(K3)!.Descriptor.DocumentType, Is.EqualTo(typeof(EvB))); + + Retype(cache, oneBatch, K2, + batch => batch.UpsertVariant(K2, baseA, new[] { cache.Override(nameof(EvA.Name), "v2") }), + () => cache.UpsertVariantAsync(K2, baseA, new[] { cache.Override(nameof(EvA.Name), "v2") }).GetAwaiter().GetResult()); + Assert.That(cache.GetStored(K2)!.Variant?.BaseKey, Is.EqualTo(baseA.Value), $"one batch {oneBatch}"); + Assert.That(cache.GetStored(K2)!.Descriptor.DocumentType, Is.EqualTo(typeof(EvA))); + + // Removed and then added as a variant (in one batch, with a plain add of the same type between): the variant lands. + Retype(cache, oneBatch, K1, + batch => + { + batch.Upsert(typeof(EvA), new EvA { Name = "k1", Value = 2 }, K1); + batch.UpsertVariant(K1, baseA, new[] { cache.Override(nameof(EvA.Name), "v1") }); + }, + () => cache.UpsertVariantAsync(K1, baseA, new[] { cache.Override(nameof(EvA.Name), "v1") }).GetAwaiter().GetResult()); + Assert.That(cache.GetStored(K1)!.Variant?.BaseKey, Is.EqualTo(baseA.Value), $"one batch {oneBatch}"); + } + } + + [TestCaseSource(nameof(VariantKinds))] + public void AFlattenAfterARemovalFlattensNothingInOneBatchAndStagedApart(string kind) + { + using var cache = Open(NewPath(kind), kind, AB); + var baseA = new CultRecordKey("base-a"); + cache.Commit(batch => + { + batch.Upsert(typeof(EvA), new EvA { Name = "base-a", Value = 1 }, baseA); + batch.UpsertVariant(K3, baseA, new[] { cache.Override(nameof(EvA.Name), "v3") }); + }); + + var flatten = Assert.Throws(() => cache.Commit(batch => + { + batch.Remove(K3); + batch.Flatten(K3); + }))!; + Assert.That(flatten.Message, Does.Contain("Flatten(k3)").And.Contain("not a variant")); + Assert.That(cache.GetStored(K3)!.Variant, Is.Not.Null, "the refused batch changed nothing"); + Assert.That(cache.Remove(K3), Is.True); + Assert.Throws(() => cache.FlattenAsync(K3).GetAwaiter().GetResult()); + } + + // finding flatten-guard-reads-removals: a Flatten acts on what the key will hold, as staged apart. A variant staged at the key + // (after a removal or not) is flattened as it lands; a plain record staged there leaves nothing to flatten. + [TestCaseSource(nameof(VariantKinds))] + public void AFlattenOfAVariantStagedInTheSameBatchFlattensItAsStagedApart(string kind) + { + foreach (var oneBatch in new[] { true, false }) + { + using var cache = Open(NewPath(kind), kind, AB); + var baseA = new CultRecordKey("base-a"); + Assert.That(cache.Commit(batch => + { + batch.Upsert(typeof(EvA), new EvA { Name = "base-a", Value = 1 }, baseA); + batch.UpsertVariant(K3, baseA, new[] { cache.Override(nameof(EvA.Name), "v3") }); + batch.Upsert(typeof(EvB), new EvB { Name = "k1", Value = 2 }, K1); + }), Is.True); + + // Remove, UpsertVariant, Flatten: the plain record of the variant's resolution, retyped. + if (oneBatch) + Assert.That(cache.Commit(batch => + { + batch.Remove(K1); + batch.UpsertVariant(K1, baseA, new[] { cache.Override(nameof(EvA.Name), "v1") }); + batch.Flatten(K1); + }), Is.True); + else + { + Assert.That(cache.Remove(K1), Is.True); + cache.UpsertVariantAsync(K1, baseA, new[] { cache.Override(nameof(EvA.Name), "v1") }).GetAwaiter().GetResult(); + cache.FlattenAsync(K1).GetAwaiter().GetResult(); + } + + var flattened = cache.GetStored(K1)!; + Assert.That(flattened.Variant, Is.Null, $"one batch {oneBatch}"); + Assert.That(flattened.Descriptor.DocumentType, Is.EqualTo(typeof(EvA))); + Assert.That(((EvA)flattened.Document).Name, Is.EqualTo("v1")); + + // A held variant rebased and flattened in one batch flattens the rebase, not the held record. + if (oneBatch) + Assert.That(cache.Commit(batch => + { + batch.UpsertVariant(K3, baseA, new[] { cache.Override(nameof(EvA.Name), "v3b") }); + batch.Flatten(K3); + }), Is.True); + else + { + cache.UpsertVariantAsync(K3, baseA, new[] { cache.Override(nameof(EvA.Name), "v3b") }).GetAwaiter().GetResult(); + cache.FlattenAsync(K3).GetAwaiter().GetResult(); + } + + Assert.That(cache.GetStored(K3)!.Variant, Is.Null, $"one batch {oneBatch}"); + Assert.That(((EvA)cache.GetStored(K3)!.Document).Name, Is.EqualTo("v3b")); + } + } + + [TestCaseSource(nameof(VariantKinds))] + public void AFlattenOfAPlainRecordStagedInTheSameBatchIsRefusedAsStagedApart(string kind) + { + using var cache = Open(NewPath(kind), kind, AB); + PutB(cache, K2); + + var refused = Assert.Throws(() => cache.Commit(batch => + { + batch.Remove(K2); + batch.Upsert(typeof(EvA), new EvA { Name = "k2" }, K2); + batch.Flatten(K2); + }))!; + Assert.That(refused.Message, Does.Contain("Flatten(k2)").And.Contain("not a variant")); + Assert.That(cache.GetStored(K2)!.Descriptor.DocumentType, Is.EqualTo(typeof(EvB)), "the refused batch changed nothing"); + + Assert.That(cache.Remove(K2), Is.True); + cache.UpsertAsync(typeof(EvA), new EvA { Name = "k2" }, K2).GetAwaiter().GetResult(); + Assert.Throws(() => cache.FlattenAsync(K2).GetAwaiter().GetResult()); + } + + // finding flatten-after-staged-plain-unpinned: where the key held a variant, Remove, a plain Upsert, then Flatten is refused + // as staged apart, and the plain write is never dropped for a flatten of the variant the cache held. + [TestCaseSource(nameof(VariantKinds))] + public void AFlattenAfterARemovalAndAPlainWriteOnAHeldVariantIsRefusedAndKeepsTheHeldVariant(string kind) + { + using var cache = Open(NewPath(kind), kind, AB); + var baseA = new CultRecordKey("base-a"); + cache.Commit(batch => + { + batch.Upsert(typeof(EvA), new EvA { Name = "base-a", Value = 1 }, baseA); + batch.UpsertVariant(K3, baseA, new[] { cache.Override(nameof(EvA.Name), "v3") }); + }); + + var refused = Assert.Throws(() => cache.Commit(batch => + { + batch.Remove(K3); + batch.Upsert(typeof(EvA), new EvA { Name = "plain" }, K3); + batch.Flatten(K3); + }))!; + Assert.That(refused.Message, Does.Contain("Flatten(k3)").And.Contain("not a variant")); + Assert.That(cache.GetStored(K3)!.Variant, Is.Not.Null, "the refused batch changed nothing"); + + Assert.That(cache.Remove(K3), Is.True); + cache.UpsertAsync(typeof(EvA), new EvA { Name = "plain" }, K3).GetAwaiter().GetResult(); + var apart = Assert.Throws(() => cache.FlattenAsync(K3).GetAwaiter().GetResult())!; + Assert.That(apart.Message, Is.EqualTo(refused.Message)); + Assert.That(((EvA)cache.GetStored(K3)!.Document).Name, Is.EqualTo("plain")); + } + + // finding op-after-staged-flatten-diverges: an operation after a Flatten at the same key acts on the plain record that flatten + // leaves, so one intent has one answer in a batch and staged apart, in both stores kinds and after a reload from disk. + [TestCaseSource(nameof(VariantKinds))] + public void AnOperationAfterAStagedFlattenAnswersAsStagedApart(string kind) + { + var scenarios = new[] { "F,F", "F,P", "F,V", "V,F,V", "F,P,F", "V,F,P", "V,F,F" }; + foreach (var scenario in scenarios) + { + var steps = scenario.Split(','); + var results = new List(); + foreach (var oneBatch in new[] { true, false }) + { + var path = NewPath(kind); + string outcome; + using (var cache = Open(path, kind, AB)) + { + var baseA = new CultRecordKey("base-a"); + cache.Commit(batch => + { + batch.Upsert(typeof(EvA), new EvA { Name = "base-a", Value = 1 }, baseA); + batch.UpsertVariant(K3, baseA, new[] { cache.Override(nameof(EvA.Name), "v3") }); + }); + var variant = new[] { cache.Override(nameof(EvA.Name), "vv") }; + + try + { + if (oneBatch) + cache.Commit(batch => + { + foreach (var step in steps) + { + if (step == "F") batch.Flatten(K3); + else if (step == "P") batch.Upsert(typeof(EvA), new EvA { Name = "plain" }, K3); + else batch.UpsertVariant(K3, baseA, variant); + } + }); + else + foreach (var step in steps) + { + if (step == "F") cache.FlattenAsync(K3).GetAwaiter().GetResult(); + else if (step == "P") cache.UpsertAsync(typeof(EvA), new EvA { Name = "plain" }, K3).GetAwaiter().GetResult(); + else cache.UpsertVariantAsync(K3, baseA, variant).GetAwaiter().GetResult(); + } + outcome = "committed"; + } + catch (InvalidOperationException refused) + { + outcome = "refused: " + refused.Message; + } + + outcome += " | " + Describe(cache); + cache.FlushAsync().GetAwaiter().GetResult(); + } + + if (kind != "storeless" && outcome.StartsWith("committed", StringComparison.Ordinal)) + { + using var reloaded = Open(path, kind, AB); + outcome += " | reloaded " + Describe(reloaded); + } + + results.Add(outcome); + } + + // A refused batch changes nothing, while staged apart the steps before the refused one have landed: the answer compared + // is the verdict and its text, and the end state only where both committed. + var batchAnswer = results[0].Split(" | ")[0]; + var apartAnswer = results[1].Split(" | ")[0]; + Assert.That(batchAnswer, Is.EqualTo(apartAnswer), $"{kind} {scenario}"); + if (batchAnswer == "committed") + Assert.That(results[0], Is.EqualTo(results[1]), $"{kind} {scenario}"); + } + + static string Describe(CultCache cache) + { + var held = cache.GetStored(K3)!; + return $"{held.Descriptor.DocumentType.Name} variant={held.Variant?.BaseKey} name={((EvA)held.Document).Name}"; + } + } + + // finding cross-store-retype-decided-by-key-home: a one-batch retype between two home stores is refused by the rule that a commit + // lands in one home store, with that rule's text (an evicted key is absent to the one-key-one-store check, as when staged + // apart); staged apart, as two commits, it leaves the key in exactly one store. + [Test] + public void ARetypeAcrossTwoHomeStoresInOneBatchIsRefusedByTheBatchSpanningRuleAndStagedApartLeavesOneStore() + { + string[] KeysIn(string path) => + new SingleFileMessagePackBackingStore(path).ReadStoreSnapshot()?.Records.Select(record => record.Key).ToArray() ?? Array.Empty(); + + CultCache OpenTwo(string a, string b) + { + var cache = new CultCache(AB, CultCacheMessagePack.CreateCodec(AB)); + cache.AddBackingStore(new SingleFileMessagePackBackingStore(a), typeof(EvA)); + cache.AddBackingStore(new SingleFileMessagePackBackingStore(b), typeof(EvB)); + cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + return cache; + } + + var pathA = NewPath("single-file"); + var pathB = NewPath("single-file"); + Directory.CreateDirectory(Path.GetDirectoryName(pathA)!); + Directory.CreateDirectory(Path.GetDirectoryName(pathB)!); + using (var cache = OpenTwo(pathA, pathB)) + { + PutB(cache, K1); + Assert.That(KeysIn(pathB), Is.EqualTo(new[] { "k1" })); + + var refused = Assert.Throws(() => cache.Commit(batch => + { + batch.Remove(K1); + batch.Upsert(typeof(EvA), new EvA { Name = "k1" }, K1); + }))!; + Assert.That(refused.Message, Does.Contain("The batch spans").And.Contain("a commit lands in one home store") + .And.Contain("stage its removal and its add apart").And.Not.Contain("second store")); + Assert.That(KeysIn(pathA), Is.Empty, "the refused batch wrote nothing"); + Assert.That(KeysIn(pathB), Is.EqualTo(new[] { "k1" })); + Assert.That(cache.GetStored(K1)!.Descriptor.DocumentType, Is.EqualTo(typeof(EvB))); + + Assert.That(cache.Remove(K1), Is.True); + cache.FlushAsync().GetAwaiter().GetResult(); + cache.UpsertAsync(typeof(EvA), new EvA { Name = "k1" }, K1).GetAwaiter().GetResult(); + cache.FlushAsync().GetAwaiter().GetResult(); + } + + Assert.That(KeysIn(pathA), Is.EqualTo(new[] { "k1" }), "staged apart the key is in the add's store"); + Assert.That(KeysIn(pathB), Is.Empty, "and no longer in the removal's"); + } + + // finding removal-bookkeeping-unpinned, mutant X2(a retype elsewhere in the batch skips every key's staging claim): a batch + // holding two retypes commits, and a cross-type add at a key it did not remove is still refused beside a retype. + [TestCaseSource(nameof(Kinds))] + public void EachKeyOfABatchIsJudgedByItsOwnRemoval(string kind) + { + using var cache = Open(NewPath(kind), kind, AB); + PutB(cache, K1); + PutB(cache, K2); + PutB(cache, K3); + + Assert.That(cache.Commit(batch => + { + batch.Remove(K1); + batch.Upsert(typeof(EvA), new EvA { Name = "k1" }, K1); + batch.Remove(K2); + batch.Upsert(typeof(EvA), new EvA { Name = "k2" }, K2); + }), Is.True, "two retypes"); + + var bad = Assert.Throws(() => cache.Commit(batch => + { + batch.Remove(K1); + batch.Upsert(typeof(EvB), new EvB { Name = "k1" }, K1); + batch.Upsert(typeof(EvA), new EvA { Name = "k3" }, K3); + }))!; + Assert.That(bad.RecordKey, Is.EqualTo(K3.Value), "one key's removal licenses no other key's retype"); + Assert.That(cache.GetStored(K3)!.Descriptor.DocumentType, Is.EqualTo(typeof(EvB))); + Assert.That(cache.GetStored(K1)!.Descriptor.DocumentType, Is.EqualTo(typeof(EvA)), "the refused batch changed nothing"); + + Assert.That(cache.Commit(batch => + { + batch.Upsert(typeof(EvB), new EvB { Name = "k3", Value = 5 }, K3); + batch.Remove(K3); + }), Is.True, "an add and then a removal removes"); + Assert.That(cache.GetStored(K3), Is.Null); + } + + // finding removal-bookkeeping-unpinned, mutant X13 (any second add counts as a replacement): two adds of one batch with no + // removal are an upsert, judged against what the key holds. + [TestCaseSource(nameof(Kinds))] + public void TwoAddsWithNoRemovalAreAnUpsert(string kind) + { + using var cache = Open(NewPath(kind), kind, AB); + PutB(cache, K2); + + var refused = Assert.Throws(() => cache.Commit(batch => + { + batch.Upsert(typeof(EvA), new EvA { Name = "k2", Value = 1 }, K2); + batch.Upsert(typeof(EvA), new EvA { Name = "k2", Value = 2 }, K2); + }))!; + if (kind == "storeless") + Assert.That(refused.Message, Does.Contain("would retype the key").And.Contain("Remove the record first, in the same batch or an earlier commit, then add the other type's.")); + Assert.That(cache.GetStored(K2)!.Descriptor.DocumentType, Is.EqualTo(typeof(EvB))); + } + + // finding removal-bookkeeping-unpinned, mutant X18 (no removal noted for a record this cache wrote itself): a cache that wrote + // the key itself, then removes it and stages another type there, lands that type at the flush. + [TestCaseSource(nameof(Kinds))] + public void ARecordThisCacheWroteItselfRetypesStagedApart(string kind) + { + var path = NewPath(kind); + using (var cache = Open(path, kind, AB)) + { + PutB(cache, K2); + cache.FlushAllBackingStores(); + Assert.That(cache.Remove(K2), Is.True); + cache.UpsertAsync(typeof(EvA), new EvA { Name = "k2", Value = 9 }, K2).GetAwaiter().GetResult(); + cache.FlushAllBackingStores(); + } + + if (kind != "storeless") + Assert.That(TypeAt(path, kind, AB, K2), Is.EqualTo(typeof(EvA))); + } + + // finding removal-bookkeeping-unpinned, mutant M (a refusal keeps the staged removal): a removal the store refused is gone, so + // a later add at a key another writer has since rewritten is judged as an upsert, not as the stale removal. + [TestCaseSource(nameof(Kinds))] + public void ARefusedRemovalIsNotReusedByALaterAdd(string kind) + { + if (kind == "storeless") + Assert.Ignore("a refusal is a store's."); + var path = NewPath(kind); + using (var seed = Open(path, kind, ABC)) + PutB(seed, K1); + + using var cache = Open(path, kind, AB); + using var other = Open(path, kind, ABC); + Assert.That(cache.Remove(K1), Is.True); + + // Another writer retypes the key to a type this cache does not know: the cache's staged removal meets a record it may not remove. + Assert.That(other.Commit(batch => + { + batch.Remove(K1); + batch.Upsert(typeof(EvC), new EvC { Name = "k1" }, K1); + }), Is.True); + Assert.Throws(() => cache.FlushAllBackingStores()); + + // The writer removes it again and puts a B there; this cache pulls and then stages an A with no removal in front of it. + Assert.That(other.Commit(batch => batch.Remove(K1)), Is.True); + cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + Assert.That(other.Commit(batch => batch.Upsert(typeof(EvB), new EvB { Name = "k1", Value = 7 }, K1)), Is.True); + cache.UpsertAsync(typeof(EvA), new EvA { Name = "k1", Value = 8 }, K1).GetAwaiter().GetResult(); + + Assert.Throws(() => cache.FlushAllBackingStores()); + Assert.That(TypeAt(path, kind, AB, K1), Is.EqualTo(typeof(EvB))); + } + + // finding removal-bookkeeping-unpinned, mutant X9 (a refusal clears every staged removal): a refusal of one key leaves another + // key's staged retype, and the next flush lands it. + [TestCaseSource(nameof(Kinds))] + public void ARefusedFlushKeepsAnotherKeysStagedRetype(string kind) + { + if (kind == "storeless") + Assert.Ignore("a refusal is a store's."); + var path = NewPath(kind); + using (var seed = Open(path, kind, ABC)) + PutB(seed, K2); + + using var cache = Open(path, kind, AB); + using var other = Open(path, kind, ABC); + Assert.That(cache.Remove(K2), Is.True); + cache.UpsertAsync(typeof(EvA), new EvA { Name = "k2", Value = 9 }, K2).GetAwaiter().GetResult(); + cache.UpsertAsync(typeof(EvA), new EvA { Name = "k1", Value = 1 }, K1).GetAwaiter().GetResult(); + + // Another writer puts a type this cache does not know at k1 before the flush: k1's staged add is refused under the lock. + Assert.That(other.Commit(batch => batch.Upsert(typeof(EvC), new EvC { Name = "k1" }, K1)), Is.True); + var refusal = Assert.Throws(() => cache.FlushAllBackingStores())!; + Assert.That(refusal.RecordKey, Is.EqualTo(K1.Value)); + cache.FlushAllBackingStores(); + + Assert.That(TypeAt(path, kind, ABC, K2), Is.EqualTo(typeof(EvA)), "the retype of k2 survived the refusal of k1"); + } + + // finding rebase-check-was-reachable, ruling variant-rebase-follows-plain-write: a variant rebased onto a base of another type + // follows the plain-write rule. Where RbY lists RbX's id (a plain write of RbY would own an RbX record) the rebase commits and + // the variant becomes an RbY; where it would be refused (RbZ) the rebase is refused and the variant is untouched. + [TestCaseSource(nameof(VariantKinds))] + public void ARebaseChangesTypeExactlyWhereAPlainWriteMay(string kind) + { + Assert.That(Rb.GetRequired(typeof(RbX)).SchemaId, Is.EqualTo(RbXId)); + using var cache = Open(NewPath(kind), kind, Rb); + var baseX = new CultRecordKey("base-x"); + var baseY = new CultRecordKey("base-y"); + var baseZ = new CultRecordKey("base-z"); + var variant = new CultRecordKey("v"); + Assert.That(cache.Commit(batch => + { + batch.Upsert(typeof(RbX), new RbX { Name = "base-x" }, baseX); + batch.Upsert(typeof(RbY), new RbY { Name = "base-y" }, baseY); + batch.Upsert(typeof(RbZ), new RbZ { Name = "base-z" }, baseZ); + batch.UpsertVariant(variant, baseX, new[] { cache.Override(nameof(RbX.Name), "v") }); + }), Is.True); + + // The plain write of the same types gives the same two answers. + Assert.Throws(() => cache.Commit(batch => batch.Upsert(typeof(RbZ), new RbZ { Name = "base-x" }, baseX))); + Assert.Throws( + () => cache.Commit(batch => batch.UpsertVariant(variant, baseZ, new[] { cache.Override(nameof(RbZ.Name), "v") }))); + Assert.That(cache.GetStored(variant)!.Descriptor.DocumentType, Is.EqualTo(typeof(RbX))); + Assert.That(cache.GetStored(variant)!.Variant!.BaseKey, Is.EqualTo(baseX.Value)); + + Assert.That(cache.Commit(batch => batch.UpsertVariant(variant, baseY, new[] { cache.Override(nameof(RbY.Name), "v") })), Is.True); + Assert.That(cache.GetStored(variant)!.Descriptor.DocumentType, Is.EqualTo(typeof(RbY))); + Assert.That(cache.GetStored(variant)!.Variant!.BaseKey, Is.EqualTo(baseY.Value)); + } + } +} diff --git a/tests/GameCult.Caching.Tests/VariantTests.cs b/tests/GameCult.Caching.Tests/VariantTests.cs index b8c1fb3d..8f574cd1 100644 --- a/tests/GameCult.Caching.Tests/VariantTests.cs +++ b/tests/GameCult.Caching.Tests/VariantTests.cs @@ -250,8 +250,10 @@ namespace GameCult.Caching.Tests SeedBig(cache); cache.Commit(batch => batch.Upsert(typeof(VariantOther), new VariantOther { Name = "other" }, new CultRecordKey("other"))); - var rebase = Refused(() => cache.Commit(batch => batch.UpsertVariant(BigKey, new CultRecordKey("other")))); - Assert.That(rebase.Message, Does.Contain(BigKey.Value).And.Contain("VariantOther")); + var rebase = Assert.Throws(() => cache.Commit(batch => batch.UpsertVariant(BigKey, new CultRecordKey("other"))))!; + // Held in a store, the key's record is a VariantGear and the write is a VariantOther: the ownership rule refuses it first + // (variants:ruling:cache-refuses-retype), so this refusal no longer names the types. + Assert.That(rebase.Message, Does.Contain(BigKey.Value)); } // A store whose variant record names another type than its base's: the loader refuses it naming both keys. @@ -280,6 +282,24 @@ namespace GameCult.Caching.Tests Assert.That(load.Message, Does.Contain("other-variant").And.Contain("gear-base")); } + // Rebasing a variant onto a base of another type would retype its key: refused with or without a store, the record untouched. + [Test] + public void ARebaseOntoABaseOfAnotherTypeIsRefusedWithoutAStore() + { + using var cache = new CultCache(Registry, CultCacheMessagePack.CreateCodec(Registry)); + SeedBase(cache); + SeedBig(cache); + var other = new CultRecordKey("other"); + cache.Commit(batch => batch.Upsert(typeof(VariantOther), new VariantOther { Name = "other" }, other)); + + var rebase = Assert.Throws(() => cache.Commit(batch => batch.UpsertVariant(BigKey, other)))!; + + Assert.That(rebase.RecordKey, Is.EqualTo(BigKey.Value)); + Assert.That(cache.GetStored(BigKey)!.Variant!.BaseKey, Is.EqualTo(BaseKey.Value)); + Assert.That(cache.GetStored(BigKey)!.Descriptor.DocumentType, Is.EqualTo(typeof(VariantGear))); + Assert.Throws(() => cache.UpsertVariantAsync(BigKey, other).GetAwaiter().GetResult()); + } + [Test] public void AVariantOfAGlobalIsRefused() { diff --git a/tests/GameCult.Caching.Tests/WriteOwnershipTests.cs b/tests/GameCult.Caching.Tests/WriteOwnershipTests.cs new file mode 100644 index 00000000..d78c2a67 --- /dev/null +++ b/tests/GameCult.Caching.Tests/WriteOwnershipTests.cs @@ -0,0 +1,417 @@ +#nullable enable +using System; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Caching.Tests +{ + // variants:ruling:write-ownership-by-written-type and variants:ruling:cache-refuses-retype: an upsert replaces the record stored at + // its key only when the type it writes there owns that record. Which type owns the id, and which other types a registry holds, + // change nothing; retyping a key is a removal and an add. The rule is one per-key predicate (CultDocumentRegistry.WrittenOwns) + // for a cache and a cacheless writer, in a single-file store and a directory store. + public class WriteOwnershipTests + { + // Two schemas of one shape that are never the same type. + [CultDocument("tests.wo_a", "tests.wo_a.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class WoA + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + } + + [CultDocument("tests.wo_b", "tests.wo_b.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class WoB + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + } + + private static readonly CultDocumentRegistry ARegistry = CultDocumentRegistry.ForTypes(new[] { typeof(WoA) }); + private static readonly CultDocumentRegistry ABRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(WoA), typeof(WoB) }); + private static readonly CultRecordKey K1 = new("k1"); + private static readonly CultRecordKey K2 = new("k2"); + + private string _directory = string.Empty; + + [SetUp] + public void CreateDirectory() + { + _directory = Path.Combine(Path.GetTempPath(), $"cultlib-writeownership-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_directory); + } + + [TearDown] + public void DeleteDirectory() + { + if (Directory.Exists(_directory)) + Directory.Delete(_directory, recursive: true); + } + + public static readonly object[] StoreKinds = { false, true }; + + private string PathFor(bool directory) => Path.Combine(_directory, directory ? "store" : "store.cc"); + + private static CultCache Open(string path, bool directory, CultDocumentRegistry registry) + { + var cache = new CultCache(registry); + cache.AddBackingStore(directory ? new DirectoryMessagePackBackingStore(path) : new SingleFileMessagePackBackingStore(path)); + cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + return cache; + } + + private static string Fingerprint(string path) + { + using var sha = SHA256.Create(); + if (File.Exists(path)) + return Convert.ToHexString(sha.ComputeHash(File.ReadAllBytes(path))); + return string.Join("|", Directory.GetFiles(path, "*", SearchOption.AllDirectories) + .OrderBy(file => file, StringComparer.Ordinal) + .Select(file => Path.GetRelativePath(path, file) + ":" + Convert.ToHexString(sha.ComputeHash(File.ReadAllBytes(file))))); + } + + private static Type TypeAt(string path, bool directory, CultDocumentRegistry registry, CultRecordKey key) + { + using var cache = Open(path, directory, registry); + return cache.GetStored(key)!.Descriptor.DocumentType; + } + + // A store that a cache holding both types wrote: A at k1 and B at k2. + private string SeedAAtK1BAtK2(bool directory) + { + var path = PathFor(directory); + using var cache = Open(path, directory, ABRegistry); + Assert.That(cache.Commit(batch => + { + batch.Upsert(new WoA { Name = "k1", Value = 1 }, new CultRecordHandle(K1)); + batch.Upsert(new WoB { Name = "k2", Value = 2 }, new CultRecordHandle(K2)); + }), Is.True); + return path; + } + + // P1: a cache holding A and B may not replace B's record with A's, whoever else the registry declares. + [TestCaseSource(nameof(StoreKinds))] + public void ACacheRefusesToWriteOneRegisteredTypeOverAnothersRecord(bool directory) + { + var path = SeedAAtK1BAtK2(directory); + var before = Fingerprint(path); + + using (var cache = Open(path, directory, ABRegistry)) + { + Assert.Throws(() => cache.Commit(batch => batch.Upsert(new WoA { Name = "k2", Value = 9 }, new CultRecordHandle(K2)))); + Assert.That(cache.GetStored(K2)!.Descriptor.DocumentType, Is.EqualTo(typeof(WoB)), "the cache still holds B's record"); + Assert.Throws(() => cache.UpsertAsync(typeof(WoA), new WoA { Name = "k2", Value = 9 }, K2)); + Assert.That(cache.GetStored(K2)!.Descriptor.DocumentType, Is.EqualTo(typeof(WoB)), "a staged write is refused at the call, not at the flush"); + } + + Assert.That(Fingerprint(path), Is.EqualTo(before)); + Assert.That(TypeAt(path, directory, ABRegistry, K2), Is.EqualTo(typeof(WoB))); + + // The pair that differs only in the type written: B over B's own record. + using (var cache = Open(path, directory, ABRegistry)) + Assert.That(cache.Commit(batch => batch.Upsert(new WoB { Name = "k2", Value = 3 }, new CultRecordHandle(K2))), Is.True); + } + + // Retype is a removal and an add, in separate commits. + [TestCaseSource(nameof(StoreKinds))] + public void ARetypeIsARemovalAndAnAddInSeparateCommits(bool directory) + { + var path = SeedAAtK1BAtK2(directory); + + using (var cache = Open(path, directory, ABRegistry)) + { + Assert.That(cache.Commit(batch => batch.Remove(K2)), Is.True); + Assert.That(cache.Commit(batch => batch.Upsert(new WoA { Name = "k2", Value = 9 }, new CultRecordHandle(K2))), Is.True); + } + + Assert.That(TypeAt(path, directory, ABRegistry, K2), Is.EqualTo(typeof(WoA))); + Assert.That(TypeAt(path, directory, ABRegistry, K1), Is.EqualTo(typeof(WoA)), "the bystander is untouched"); + } + + // The store is judged again where the write lands, under its lock: another writer retyped the key after this cache loaded it, + // so this cache's own-type write of the old record is now a write over a record that is not its type's. + [TestCaseSource(nameof(StoreKinds))] + public void AnotherWritersRetypeRefusesThisCachesStaleWriteUnderTheLock(bool directory) + { + var path = SeedAAtK1BAtK2(directory); + using var stale = Open(path, directory, ABRegistry); + Assert.That(stale.GetStored(K1)!.Descriptor.DocumentType, Is.EqualTo(typeof(WoA))); + using (var other = Open(path, directory, ABRegistry)) + { + Assert.That(other.Commit(batch => batch.Remove(K1)), Is.True); + Assert.That(other.Commit(batch => batch.Upsert(new WoB { Name = "k1", Value = 5 }, new CultRecordHandle(K1))), Is.True); + } + + var before = Fingerprint(path); + + Assert.Throws(() => stale.Commit(batch => batch.Upsert(new WoA { Name = "k1", Value = 9 }, new CultRecordHandle(K1)))); + + Assert.That(Fingerprint(path), Is.EqualTo(before)); + Assert.That(TypeAt(path, directory, ABRegistry, K1), Is.EqualTo(typeof(WoB))); + } + + public static readonly object[] Registries = { "none", "A", "AB", "Shared" }; + + private static CultDocumentRegistry? Lent(object kind) => kind switch + { + "A" => ARegistry, + "AB" => ABRegistry, + "Shared" => CultDocumentRegistry.Shared, + _ => null, + }; + + // P3: a cacheless write of A over B's record is refused whatever registry is lent, and the same lent or not. + [TestCaseSource(nameof(Registries))] + public void ACachelessWriteOfOneTypeOverAnothersRecordIsRefusedWhateverRegistryIsLent(object lent) + { + var path = SeedAAtK1BAtK2(directory: false); + var before = Fingerprint(path); + var store = new SingleFileMessagePackBackingStore(path); + + Assert.Throws(() => store.WriteDocument(K2, new WoA { Name = "k2", Value = 9 }, registry: Lent(lent))); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + + // WriteRecord names its entry and nothing else. + var entry = ARegistry.GetRequired().ToCatalogEntry(); + var record = new CultPersistedRecord + { + Key = K2.Value, + SchemaId = entry.SchemaId, + StoredAt = "2026-10-28T00:00:00.0000000+00:00", + Payload = MessagePackSerializer.Serialize(new WoA { Name = "k2", Value = 9 }), + }; + Assert.Throws(() => store.WriteRecord(record, entry, null, Lent(lent))); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + + // The pair that differs only in whose record it is: A over A's own, which commits for every registry. + store.WriteDocument(K1, new WoA { Name = "k1", Value = 9 }, registry: Lent(lent) ?? ARegistry); + Assert.That(TypeAt(path, false, ABRegistry, K1), Is.EqualTo(typeof(WoA))); + } + + // The judge is per key, and an answer for one written type is never reused for another. Another writer left A's record at two + // keys; this stale cache holds A at one and B at the other and writes both. The key A owns lands, the key where B meets A's + // record is the one refused, whichever is judged first. + [TestCase("a-own", "b-bad", true, false)] + [TestCase("a-own", "b-bad", false, false)] + [TestCase("b-own", "a-bad", true, true)] + [TestCase("b-own", "a-bad", false, true)] + public void AStaleBatchIsJudgedPerKeyByEachKeysWrittenType(string ownKey, string badKey, bool ownFirst, bool directory) + { + var path = PathFor(directory); + var own = new CultRecordKey(ownKey); + var bad = new CultRecordKey(badKey); + using (var seed = Open(path, directory, ABRegistry)) + Assert.That(seed.Commit(batch => + { + batch.Upsert(new WoA { Name = ownKey, Value = 1 }, new CultRecordHandle(own)); + batch.Upsert(new WoB { Name = badKey, Value = 1 }, new CultRecordHandle(bad)); + }), Is.True); + using var stale = Open(path, directory, ABRegistry); + using (var other = Open(path, directory, ABRegistry)) + { + Assert.That(other.Commit(batch => batch.Remove(bad)), Is.True); + Assert.That(other.Commit(batch => batch.Upsert(new WoA { Name = badKey, Value = 2 }, new CultRecordHandle(bad))), Is.True); + } + + var before = Fingerprint(path); + void StageOwn(CultCacheBatch batch) => batch.Upsert(new WoA { Name = ownKey, Value = 3 }, new CultRecordHandle(own)); + void StageBad(CultCacheBatch batch) => batch.Upsert(new WoB { Name = badKey, Value = 3 }, new CultRecordHandle(bad)); + + var refusal = Assert.Throws(() => stale.Commit(batch => + { + if (ownFirst) { StageOwn(batch); StageBad(batch); } else { StageBad(batch); StageOwn(batch); } + }))!; + + Assert.That(refusal.RecordKey, Is.EqualTo(badKey)); + Assert.That(refusal.Message, Does.Not.Contain($"'{ownKey}'"), "the key A owns is not among the refused"); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + } + + // A batch that removes a key is a removal of it, whatever this cache has staged there: the removal rule judges it, so a key + // another writer retyped can still be removed by the cache that staged a write of the old type. + [TestCaseSource(nameof(StoreKinds))] + public void ABatchRemovalOfAStagedKeyIsJudgedAsARemoval(bool directory) + { + var path = SeedAAtK1BAtK2(directory); + using var stale = Open(path, directory, ABRegistry); + stale.UpsertAsync(typeof(WoA), new WoA { Name = "k1", Value = 9 }, K1).GetAwaiter().GetResult(); + using (var other = Open(path, directory, ABRegistry)) + { + Assert.That(other.Commit(batch => batch.Remove(K1)), Is.True); + Assert.That(other.Commit(batch => batch.Upsert(new WoB { Name = "k1", Value = 5 }, new CultRecordHandle(K1))), Is.True); + } + + Assert.That(stale.Commit(batch => batch.Remove(K1)), Is.True); + + using var reopened = Open(path, directory, ABRegistry); + Assert.That(reopened.GetStored(K1), Is.Null); + Assert.That(reopened.GetStored(K2)!.Descriptor.DocumentType, Is.EqualTo(typeof(WoB)), "the bystander is untouched"); + } + + // The ruling's retype: one batch holds the removal and the add, in the order they were called. The removal half is judged as a + // removal (a registered type's record is removable), and the record lands at the key. + [TestCaseSource(nameof(StoreKinds))] + public void ARetypeIsOneBatchHoldingTheRemovalAndTheAdd(bool directory) + { + var path = SeedAAtK1BAtK2(directory); + + using (var cache = Open(path, directory, ABRegistry)) + { + Assert.That(cache.Commit(batch => + { + batch.Remove(K2); + batch.Upsert(new WoB { Name = "k2", Value = 8 }, new CultRecordHandle(K2)); + batch.Upsert(new WoA { Name = "k2", Value = 9 }, new CultRecordHandle(K2)); + }), Is.True, "the key stays a replacement through a later add of the same batch"); + Assert.That(cache.GetStored(K2)!.Descriptor.DocumentType, Is.EqualTo(typeof(WoA))); + } + + Assert.That(TypeAt(path, directory, ABRegistry, K2), Is.EqualTo(typeof(WoA))); + Assert.That(TypeAt(path, directory, ABRegistry, K1), Is.EqualTo(typeof(WoA)), "the bystander is untouched"); + } + + // The order is the meaning: a removal called after the add leaves the key removed. + [TestCaseSource(nameof(StoreKinds))] + public void ARemovalCalledAfterTheAddLeavesTheKeyRemoved(bool directory) + { + var path = SeedAAtK1BAtK2(directory); + + using (var cache = Open(path, directory, ABRegistry)) + { + Assert.That(cache.Commit(batch => + { + batch.Remove(K2); + batch.Upsert(new WoA { Name = "k2", Value = 9 }, new CultRecordHandle(K2)); + batch.Remove(K2); + }), Is.True); + Assert.That(cache.GetStored(K2), Is.Null); + } + + using var reopened = Open(path, directory, ABRegistry); + Assert.That(reopened.GetStored(K2), Is.Null); + } + + // The removal half is a removal and nothing more: a record no registered type owns is not replaced by adding after its removal. + [TestCaseSource(nameof(StoreKinds))] + public void ARetypeOfARecordNoRegisteredTypeOwnsIsStillRefused(bool directory) + { + var path = SeedAAtK1BAtK2(directory); + var before = Fingerprint(path); + + using (var cache = Open(path, directory, ARegistry)) + { + Assert.Throws(() => cache.Commit(batch => + { + batch.Remove(K2); + batch.Upsert(new WoA { Name = "k2", Value = 9 }, new CultRecordHandle(K2)); + })); + } + + Assert.That(Fingerprint(path), Is.EqualTo(before)); + } + + // A removal of a key the cache does not hold is judged under the store's lock, and so is the add after it: another writer put + // B there, this cache never loaded it, and removing then adding A is a retype of whatever is stored. + [TestCaseSource(nameof(StoreKinds))] + public void ARetypeOfAKeyTheCacheDoesNotHoldCommitsInOneBatch(bool directory) + { + var path = PathFor(directory); + using var early = Open(path, directory, ABRegistry); + using (var other = Open(path, directory, ABRegistry)) + Assert.That(other.Commit(batch => batch.Upsert(new WoB { Name = "k1", Value = 1 }, new CultRecordHandle(K1))), Is.True); + + Assert.That(early.Commit(batch => + { + batch.Remove(K1); + batch.Upsert(new WoA { Name = "k1", Value = 2 }, new CultRecordHandle(K1)); + }), Is.True); + + Assert.That(TypeAt(path, directory, ABRegistry, K1), Is.EqualTo(typeof(WoA))); + } + + // A removal and an add called separately on the cache are one retype too: the add, staged after the removal, is not lost to the + // flush that judges it, and the type changes in the file. + [TestCaseSource(nameof(StoreKinds))] + public void AStagedRemovalAndAddOfAnotherTypeIsNotLostAtTheFlush(bool directory) + { + var path = SeedAAtK1BAtK2(directory); + + using (var cache = Open(path, directory, ABRegistry)) + { + Assert.That(cache.Remove(K2), Is.True); + cache.UpsertAsync(typeof(WoA), new WoA { Name = "k2", Value = 9 }, K2).GetAwaiter().GetResult(); + cache.FlushAllBackingStores(); + Assert.That(cache.GetStored(K2)!.Descriptor.DocumentType, Is.EqualTo(typeof(WoA))); + } + + Assert.That(TypeAt(path, directory, ABRegistry, K2), Is.EqualTo(typeof(WoA))); + } + + [TestCaseSource(nameof(StoreKinds))] + public void AStagedRemovalOfAKeyTheCacheDoesNotHoldAndAnAddOfAnotherTypeIsNotLostAtTheFlush(bool directory) + { + var path = PathFor(directory); + using var early = Open(path, directory, ABRegistry); + using (var other = Open(path, directory, ABRegistry)) + Assert.That(other.Commit(batch => batch.Upsert(new WoB { Name = "k1", Value = 1 }, new CultRecordHandle(K1))), Is.True); + + Assert.That(early.Remove(K1), Is.False, "the cache holds nothing there"); + early.UpsertAsync(typeof(WoA), new WoA { Name = "k1", Value = 2 }, K1).GetAwaiter().GetResult(); + early.FlushAllBackingStores(); + + Assert.That(TypeAt(path, directory, ABRegistry, K1), Is.EqualTo(typeof(WoA))); + } + + // A removal is spent by the flush that judged it: a later add of another type, with no removal in front of it, is judged as an upsert again. + [TestCaseSource(nameof(StoreKinds))] + public void ARemovalIsSpentByTheFlushThatJudgedIt(bool directory) + { + var path = SeedAAtK1BAtK2(directory); + + using var cache = Open(path, directory, ABRegistry); + Assert.That(cache.Remove(K2), Is.True); + cache.FlushAllBackingStores(); + + // Another writer puts B at the key this cache removed; this cache never loaded it, and stages an A there with no removal. + using (var other = Open(path, directory, ABRegistry)) + Assert.That(other.Commit(batch => batch.Upsert(new WoB { Name = "k2", Value = 4 }, new CultRecordHandle(K2))), Is.True); + cache.UpsertAsync(typeof(WoA), new WoA { Name = "k2", Value = 5 }, K2).GetAwaiter().GetResult(); + + Assert.Throws(() => cache.FlushAllBackingStores()); + Assert.That(TypeAt(path, directory, ABRegistry, K2), Is.EqualTo(typeof(WoB))); + } + + // No store, same rule: a cache holding A and B refuses one's write over the other's record, on Commit and on UpsertAsync, and + // a retype is a removal and an add, in one batch or staged apart. + [Test] + public void AStorelessCacheRefusesToWriteOneRegisteredTypeOverAnothersRecord() + { + using var cache = new CultCache(ABRegistry); + Assert.That(cache.Commit(batch => batch.Upsert(new WoB { Name = "k2", Value = 2 }, new CultRecordHandle(K2))), Is.True); + + var commit = Assert.Throws(() => cache.Commit(batch => batch.Upsert(new WoA { Name = "k2", Value = 9 }, new CultRecordHandle(K2))))!; + Assert.That(commit.RecordKey, Is.EqualTo(K2.Value)); + Assert.Throws(() => cache.UpsertAsync(typeof(WoA), new WoA { Name = "k2", Value = 9 }, K2)); + Assert.That(cache.GetStored(K2)!.Descriptor.DocumentType, Is.EqualTo(typeof(WoB))); + + // The pair that differs only in the type written: B over B's own record. + Assert.That(cache.Commit(batch => batch.Upsert(new WoB { Name = "k2", Value = 3 }, new CultRecordHandle(K2))), Is.True); + + Assert.That(cache.Commit(batch => + { + batch.Remove(K2); + batch.Upsert(new WoA { Name = "k2", Value = 9 }, new CultRecordHandle(K2)); + }), Is.True); + Assert.That(cache.GetStored(K2)!.Descriptor.DocumentType, Is.EqualTo(typeof(WoA))); + + Assert.That(cache.Remove(K2), Is.True); + cache.UpsertAsync(typeof(WoB), new WoB { Name = "k2", Value = 4 }, K2).GetAwaiter().GetResult(); + Assert.That(cache.GetStored(K2)!.Descriptor.DocumentType, Is.EqualTo(typeof(WoB))); + } + } +} diff --git a/tests/GameCult.Caching.Tests/WrittenOwnsTests.cs b/tests/GameCult.Caching.Tests/WrittenOwnsTests.cs new file mode 100644 index 00000000..c1a1c148 --- /dev/null +++ b/tests/GameCult.Caching.Tests/WrittenOwnsTests.cs @@ -0,0 +1,97 @@ +#nullable enable +using System; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Caching.Tests +{ + // The one per-key predicate of write ownership (variants:ruling:write-ownership-by-written-type, own-entry-reads-whole), over + // catalogs the production types publish. Each refusal is paired with the accepted case that differs in only the one thing the + // rule looks at. Where the predicate is wired in is WriteOwnershipTests. + public class WrittenOwnsTests + { + // One schema name at three versions, nothing declared: N2 reads N1's records whole (it adds a slot) and Wide's only by + // dropping a slot; Renamed has N2's exact shape under another schema name. + [CultDocument("tests.wo_n", "tests.wo_n.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class WoN1 + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + } + + [CultDocument("tests.wo_n", "tests.wo_n.v2")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class WoN2 + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + [Key(2)] public string Colour = string.Empty; + } + + [CultDocument("tests.wo_n", "tests.wo_n.v3")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class WoNWide + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + [Key(2)] public string Colour = string.Empty; + [Key(3)] public string Extra = string.Empty; + } + + [CultDocument("tests.wo_nx", "tests.wo_nx.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class WoNRenamed + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + [Key(2)] public string Colour = string.Empty; + } + + private static CultSchemaCatalogEntry EntryOf() where T : class => + CultDocumentRegistry.ForTypes(new[] { typeof(T) }).GetRequired().ToCatalogEntry(); + + private static CultSchemaCatalogEntry Listing(CultSchemaCatalogEntry entry, params string[] ids) => + new() + { + SchemaId = entry.SchemaId, + ContentHash = entry.ContentHash, + SchemaName = entry.SchemaName, + SchemaVersion = entry.SchemaVersion, + CanonicalSchemaJson = entry.CanonicalSchemaJson, + Members = entry.Members, + CompatibleSchemaIds = ids, + }; + + [Test] + public void WrittenOwnsDeclaredIdsAreTheEntrysOwnOrAnyItListsNotOnlyTheFirst() + { + var a = EntryOf(); + var b = EntryOf(); + var catalog = new[] { a }; + + Assert.That(CultDocumentRegistry.WrittenOwns(a, a.SchemaId, catalog), Is.True, "its own id"); + Assert.That(CultDocumentRegistry.WrittenOwns(b, a.SchemaId, catalog), Is.False, "same shape, another schema: not its record"); + Assert.That(CultDocumentRegistry.WrittenOwns(Listing(b, b.SchemaId, a.SchemaId), a.SchemaId, catalog), Is.True, "listed last"); + Assert.That(CultDocumentRegistry.WrittenOwns(Listing(b, a.SchemaId, b.SchemaId), a.SchemaId, catalog), Is.True, "listed first"); + Assert.That(CultDocumentRegistry.WrittenOwns(Listing(b, b.SchemaId, "tests.wo.other"), a.SchemaId, catalog), Is.False, "lists another id"); + Assert.That(CultDocumentRegistry.WrittenOwns(a, a.SchemaId, Array.Empty()), Is.True, "declared needs no catalog"); + } + + [Test] + public void WrittenOwnsAnOlderIdOfItsOwnNameOnlyWhenItReadsTheRecordWhole() + { + var n1 = EntryOf(); + var n2 = EntryOf(); + var wide = EntryOf(); + var renamed = EntryOf(); + + Assert.That(CultDocumentRegistry.WrittenOwns(n2, n1.SchemaId, new[] { n1 }), Is.True, "same name, reads whole"); + Assert.That(CultDocumentRegistry.WrittenOwns(renamed, n1.SchemaId, new[] { n1 }), Is.False, "the same shape under another name"); + Assert.That(CultDocumentRegistry.WrittenOwns(n2, wide.SchemaId, new[] { wide }), Is.False, "same name, drops a persisted slot"); + Assert.That(CultDocumentRegistry.WrittenOwns(n2, "tests.wo.unpublished", new[] { n1 }), Is.False, "an id no catalog entry publishes"); + Assert.That(CultDocumentRegistry.WrittenOwns(n2, n1.SchemaId, new[] { n1, wide }), Is.True, "another entry in the catalog changes nothing"); + } + } +} diff --git a/unity/org.gamecult.cultlib/CHANGELOG.md b/unity/org.gamecult.cultlib/CHANGELOG.md index 390ece2a..ef98338b 100644 --- a/unity/org.gamecult.cultlib/CHANGELOG.md +++ b/unity/org.gamecult.cultlib/CHANGELOG.md @@ -13,7 +13,7 @@ Releases as 3.0.0 (major). - A store is written in the format its content needs, and CultLib 2.0 refuses the newer ones by name. A single-file store holding a document variant is written as `cultcache.store.v2`, one holding an element id as `cultcache.store.v3`; a directory store holding an element id is written as `cultcache.store.v5` (`v4` otherwise). A store is marked only when a record actually holds an id or a variant. Upgrade readers before writers: a 2.0 reader that opens such a store is refused, not misread. - Writes are write sets. A flush or commit replaces the records it staged and copies every other record forward as stored, under the store's lock, instead of rewriting the whole store from the cache's memory. A record the cache does not know is kept, and an untouched record keeps a dropped member's slot through every write (never reuse a dropped slot; see the schema-compatibility contract). - A write or removal at a key whose stored record the writer does not own is refused with `CultSchemaConflictException`, and the file is left as it was. A writer owns a record whose schema id it declares (owns or lists compatible) and a record its type reads whole under compatible drift: that record is writable and is restamped under the type's own id on write, as in 2.0.1. A record a type reads only by dropping persisted slots stays read-only until its id is declared compatible, which accepts the loss (this differs from 2.0.1, which shed the slots silently). A cache owns through its registry; a cacheless CultMesh write owns through the catalog entry it writes. -- A cacheless `WriteDocument`, `WriteRecord` or unattached-store `PushAll` owns a stored record in two ways only: the entries it writes declare the record's schema id (an entry's own id, or one it lists compatible), or the type picked to read the record is one of the written entries and reads it whole (no persisted slot unread). The type is picked as a cache registering the call's registry would pick it, except that a written entry is the only description of its schema id: `WriteDocument` takes its entry from the registry (`CultDocumentRegistry.Shared` when none is passed), `WriteRecord` writes the entry it is given, and a registry's own type for that id is never asked. A registry adds claimants and declares nothing, so it can only refuse more; without one the written entries are the only claimants. Refused, with the file left as it was: a record another registered type claims, under that type's current id or an old id it lists; a record whose stored entry lists another registered type's id ahead of the written one's, whatever name the stored entry carries; a record that two registered types list compatible, or that two registered versions of the written schema could read; a record the picked written entry would read only by dropping persisted slots; and the removal of any stored record by a call that writes no entry (a `Delete` then `PushAll`). As in 2.0.1, a record the written entry's own type reads whole under drift is owned and restamped. A registry-less `WriteDocument` reads `Shared` today, so any type loaded there can refuse the write; pass a registry to pin the answer. +- A write replaces the record at a key only when the type it writes there owns that record, decided per key and the same for a cache and for a cacheless `WriteDocument`, `WriteRecord` or unattached-store `PushAll`, in a single-file store and a directory store. The written type owns the stored record when its entry declares the stored id (its own id, or one it lists compatible), or when it is the type that reads the record's published entry and reads it whole (no persisted slot unread): an older id of its own schema name that it reads whole is its own and is restamped, as in 2.0.1. Anything else is refused with `CultSchemaConflictException` and the file left as it was: a record of another registered type, under its current id or an old id it lists, and a record the written type would read only by dropping persisted slots. No registry enters the decision, `CultDocumentRegistry.Shared` included, so a lent registry never changes the answer. This changes a cache: where 2.x let a cache holding two types replace one's record at a key with the other's, it now refuses, with or without a backing store. Retype a key by removing it and adding the new record, in one batch (`batch.Remove(key)` then `batch.Upsert(...)`) or staged apart (`Remove` then `UpsertAsync`); the removal is judged as a removal, so a record no registered type owns is still refused. A removal or check goes to the registry's removal rule, and a cacheless call, which has no registry, removes a record only when an entry it writes owns it (a `Delete` then `PushAll` on an unattached store writes no entry and is refused). - Registration refuses an owned object-list element type that has no `[CultElementId]` member (a public string with `[Key]`), naming the type and the list. Element ids let a write set merge a list by identity; a list of record references or plain values is a whole value and is not affected. - The abstract and protected surface for store kinds outside this library changed. On `SingleFileBackingStore`, `SerializeSnapshot` and `DeserializeSnapshot` are replaced by the abstract `SerializeStore`, `SerializeRecord`, `SerializeCatalogEntry` and `ReadStore`, and the abstract `SerializePayload(object)` becomes `SerializePayload(object document, Type documentType)`: a store encodes a document as the type it is given. On `CacheBackingStore`, `Judging` is removed, `ToStoredDocument` takes the record's version (`record, version, catalog, deserialize, out report`), and these protected members are new for a store kind to implement or call: `HeaderFor`, `HeaderForWrite`, `Foreign`, `SetForeignRecords`, `Undeclared`, `RefuseUndeclared`, `RefuseUndeclaredOnDisk`, `RefuseRemovalOfUndeclared`, `Refuse`, `SameStoredIdentity`, `VersionOf`, `StoredAtOver`, `Overwrites`, `StagedChecks`, `StageChecks` and `JudgeChecks`. `MarkFlushSucceeded` is unchanged. `CacheBackingStore.ForeignRecords` is new and public: the records a store holds under a schema id its cache does not declare. - `CultCommitRequest.Expected` is now `IReadOnlyList<(CultRecordKey Key, string? Version)>` (it was a key with a schema id and a stored-at), `CultCommitRequest.Checks` is new, and `ConditionsHold` takes `(IReadOnlyDictionary durable, IEnumerable observed, IEnumerable foreign)` in place of `(IReadOnlyCollection, IEnumerable)`. A store kind outside the library lists the record versions it holds and passes its foreign records. @@ -25,6 +25,7 @@ Releases as 3.0.0 (major). ### Added +- `CacheBackingStore.RefuseUndeclaredOnDisk(written, catalog, heldAt, writtenAt, removed, durableAt, foreignAt, unstage, cacheless)` is the one protected per-key ownership judge for a store kind outside the library: it applies the whole rule under the store's lock (the written type owns a record by its own id, an id it lists or an older id of its own name it reads whole; a key the write removes or checks, and a key the cache removed since the last flush, is owned by any registered type that declares or reads the record whole) and refuses through `Refuse`. Both shipped stores call it. Additive: the earlier overload taking a store-built `undeclaredAt` judge is unchanged. A batch that removes a key and adds a record there is staged as the held record's eviction and then the add, so a watcher of the old type sees `Removed`, a watcher of the new type sees `Added`, and a variant rebase onto a base of another type changes type exactly where a plain write may. - A cacheless `WriteRecord` or `WriteDocument`, and an attach, hold the store's gate while they run, and user code they run (a document's property getters, a constructor while a copied record is read) now throws `InvalidOperationException` if it enters a cache it does not hold or a second store; it used to wait forever against an attach of the same store. A store call that queued on an attach that then failed runs under the gate that owns the store, not beside a cacheless call or a second cache's attach. An attach that fails takes its registry back, so a `PullAll` on the store afterwards is refused as unattached instead of loading records the next cache never sees. A cacheless `WriteRecord` or `WriteDocument` made while the same thread already holds that store from user code another cacheless write of the store runs, or from a journal an attach's pull runs (then "The store is attached to a cache, write through the cache.", as after the attach) throws `InvalidOperationException`; it used to write the file behind the attaching cache and take that cache's registry, so the cache could never flush or pull the store again. A journal that calls its cache's store (`Push`, `Delete`, `PushAll`, `CommitBatch`, `PullAll`, `ReadStoreSnapshot`, `JudgeChecks`) while that cache's `AddBackingStore` runs the journal now throws "A journal entered the cache it journals", as it does after the attach; it used to stage or write records the cache never admitted, so the cache and the file disagreed. - `SingleFileBackingStore.ReadStoreSnapshot()` reads a store file once, under the store's in-process gate, with the reader a cache's store opens with, and returns the durable snapshot or null when nothing is at the path. `SingleFileBackingStore.WriteRecord(record, catalogEntry, holdsIds, registry)` writes one record without a cache: under the store's lock every other record and catalog entry is copied as stored, the file is replaced atomically, and the header is decided as a flush decides it. It refuses on a store attached to a cache. - `SingleFileBackingStore.WriteDocument(key, document, storedAt, registry)` writes one typed document without a cache, as a cache write does: it plans element ids, encodes the document as `T` (the type its schema id and ids are taken from, also for a derived instance) and marks the store when it holds an id; a failed write gives the minted ids back. It refuses on a store attached to a cache. The `registry` of `WriteRecord` and `WriteDocument` serves that call only; a store keeps none. From f1941449132abae32f324b3ada2b5179aafcf7a0 Mon Sep 17 00:00:00 2001 From: Eureka Date: Thu, 29 Oct 2026 13:07:21 +0100 Subject: [PATCH 28/88] Research: presence provider library/launcher split prior art (cultlib-gaps B4b2) Co-Authored-By: Claude Opus 5.5 --- ...esence-provider-library-split-prior-art.md | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 docs/research/2026-10-29-presence-provider-library-split-prior-art.md diff --git a/docs/research/2026-10-29-presence-provider-library-split-prior-art.md b/docs/research/2026-10-29-presence-provider-library-split-prior-art.md new file mode 100644 index 00000000..8acf0479 --- /dev/null +++ b/docs/research/2026-10-29-presence-provider-library-split-prior-art.md @@ -0,0 +1,28 @@ +# Prior art: presence provider as a library, launcher as a consumer + +Served: `cultlib-gaps:cut_spec:cut-cultnet-rs-presence-provider-a.r1` and the +Defaulted rulings `presence-provider-network-free-core`, +`presence-credentials-both-idunn-drivers` and `presence-provider-consumers-adopt`, +applying the operator's `gamecult-chat:ruling:presence-one-owner-library-launcher-consumer`. +Imagination pass, 2026-10-29, from recall of the cited manuals (not re-fetched). + +- libsystemd `sd_notify(3)`: a service links a small library and reports its + own state (READY, STATUS, WATCHDOG); programs that cannot link it use + `systemd-notify(1)` or a wrapper. Same split as cultnet-rs provider plus + idunn-launch as the sidecar consumer. +- `sd_listen_fds_with_names(3)`: checks LISTEN_PID against the process, sets + close-on-exec on passed descriptors, finds them by name, and can unset the + LISTEN_* environment. This set is the descriptor rule the provider adopts. +- systemd watchdog (`WATCHDOG=1` within `WatchdogSec`): the model for the + warming heartbeat. +- Consul: TTL checks updated by the service through the client API, versus + script/HTTP checks the agent runs for services that cannot. In-process + versus sidecar again. +- Kubernetes: kubelet-run probes observe from outside; readiness gates are + reported by the app. Idunn's route challenge is the outside observer, the + signed presence the app's own report. + +Existing copies compared: CodexConnector `src/idunn_health.rs` (two +descriptors, strict PID check), Idunn launcher (two descriptors, no PID check), +cultnet-ts `idunn-runtime-authority.ts` (descriptors by name, systemd private +PID case). The provider takes the strictest union. From 1f495156b59fc44e92803f4f8110cd518b76bdf0 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 29 Oct 2026 18:08:26 +0100 Subject: [PATCH 29/88] cultnet-rs-presence-provider-a: one Rust provider of gamecult.runtime_presence_health.v2 for Idunn-managed services, every platform; custody follows delivery; signer owns health; write-lease guard; route answer; no network (squash of hands/cultlib-gaps-presence-provider-a f1941449..ed48b5e6; Soul s1-s3) Co-Authored-By: Claude Opus 5.5 --- packages/cultmesh-rs/Cargo.lock | 62 +- packages/cultnet-rs/CHANGELOG.md | 6 + packages/cultnet-rs/Cargo.toml | 6 +- .../cultnet-rs/src/idunn_runtime_provider.rs | 1404 +++++++++++ .../src/idunn_runtime_provider/tests.rs | 2120 +++++++++++++++++ packages/cultnet-rs/src/lib.rs | 2 + .../src/runtime_authority_contracts.rs | 16 + 7 files changed, 3612 insertions(+), 4 deletions(-) create mode 100644 packages/cultnet-rs/src/idunn_runtime_provider.rs create mode 100644 packages/cultnet-rs/src/idunn_runtime_provider/tests.rs diff --git a/packages/cultmesh-rs/Cargo.lock b/packages/cultmesh-rs/Cargo.lock index 8dfc33ed..d056b52e 100644 --- a/packages/cultmesh-rs/Cargo.lock +++ b/packages/cultmesh-rs/Cargo.lock @@ -52,6 +52,15 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "ar_archive_writer" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6" +dependencies = [ + "object", +] + [[package]] name = "autocfg" version = "1.5.1" @@ -174,7 +183,7 @@ dependencies = [ [[package]] name = "cultcache-rs" -version = "0.2.0" +version = "0.3.1" dependencies = [ "anyhow", "chrono", @@ -185,13 +194,15 @@ dependencies = [ "serde", "serde_bytes", "serde_json", + "serde_stacker", + "stacker", "uuid", "windows-sys 0.59.0", ] [[package]] name = "cultcache-rs-derive" -version = "0.1.0" +version = "0.2.0" dependencies = [ "proc-macro2", "quote", @@ -213,7 +224,7 @@ dependencies = [ [[package]] name = "cultnet-rs" -version = "0.2.0" +version = "0.3.0" dependencies = [ "aes-gcm", "anyhow", @@ -221,7 +232,9 @@ dependencies = [ "chrono", "cultcache-rs", "ed25519-dalek", + "fs2", "hmac", + "libc", "rand", "rmp-serde", "rmpv", @@ -521,6 +534,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "object" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -589,6 +611,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "psm" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd034599e63b970727f70d79e02d62390a4a84f7c6b827c27c46d5ac3fa622" +dependencies = [ + "ar_archive_writer", + "cc", +] + [[package]] name = "quote" version = "1.0.47" @@ -774,6 +806,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_stacker" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4936375d50c4be7eff22293a9344f8e46f323ed2b3c243e52f89138d9bb0f4a" +dependencies = [ + "serde", + "serde_core", + "stacker", +] + [[package]] name = "sha2" version = "0.10.9" @@ -826,6 +869,19 @@ dependencies = [ "der", ] +[[package]] +name = "stacker" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.61.2", +] + [[package]] name = "subtle" version = "2.6.1" diff --git a/packages/cultnet-rs/CHANGELOG.md b/packages/cultnet-rs/CHANGELOG.md index d12dc3ed..c2600756 100644 --- a/packages/cultnet-rs/CHANGELOG.md +++ b/packages/cultnet-rs/CHANGELOG.md @@ -6,6 +6,12 @@ only in git history. ## Unreleased +### Added + +- The provider half of `gamecult.runtime_presence_health.v2` for Idunn-managed Rust services, in one module that builds and runs on every platform an Idunn driver launches (unix under Idunn's systemd driver, Windows under its host driver). `IdunnRuntimeAuthority::open(target, health_contract, credentials)` reads the runtime bundle, the candidate bind and the write lease from the launch environment Idunn set (no caller supplies a path) and checks every binding Idunn expects (target and health contract, activation bound to Expected, both signing credentials, the write-lease and candidate-bind environment; the candidate bind must equal Expected's route exactly). The credentials' delivery decides the custody the bundle is held to: `IdunnRuntimeCredentials::from_systemd_descriptors` (unix; found by name per sd_listen_fds_with_names(3), `LISTEN_PID` or the `PrivatePIDs` pid-1 rule, root-owned 0400 single-link read-only files, close-on-exec, `LISTEN_*` removed from the environment) carries root custody (bundle directory, `expected.cc`, `activation.cc`, their `.lock` siblings and every parent root-owned, not group- or world-writable, no symlinks, and the same for the write lease), while `from_host_environment` (the host driver's files; the provider identity is opened before the one-shot activation credential is read and removed, and a refusal leaves the credential in place) carries profile custody on every platform (the bundle is a real directory of real files, never a link: std's `is_symlink` refuses symlinks and, on Windows, junctions). No refusal names a path; an unreadable store is reported by its failure kind alone (not decodable, unsupported format, not found, permission denied, I/O error). `IdunnRuntimePresenceSigner` takes the authority by value, owns the reported health (`RuntimePresenceState`, starting Warming; `report_health`), signs validated presence records with `sign` / `sign_at` (returning `SignedRuntimePresence`: record, raw CultNet document and canonical digest), holds Idunn's process write lease through a `ProcessWriteLeaseGuard` (unix only; `try_acquire_write_lease`; the guard holds the lease's `.lock` shared and `require_current` detects replacement; a runtime whose Expected requires a lease is refused at `open` elsewhere with `write_lease_required`), keeps signing Failed, without a lease digest, after the lease is lost, and answers Idunn's route challenge from local authority with `answer_route_observation`. The module owns no network. No wire bytes change: its output passes `authenticate_runtime_presence_claim` and `correlate_runtime_presence_claim`. New unix-only dependencies: `libc`, `fs2`. +- The launch-contract names both sides of an Idunn launch read: `IDUNN_RUNTIME_BUNDLE_ENVIRONMENT`, `IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT`, `IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT`, `IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT`, `RUNTIME_PRESENCE_IDENTITY_BINDING` and `RUNTIME_PRESENCE_IDENTITY_FD_NAME`. +- Version 0.3.0: the unreleased `register` and `CultMeshCdnChunkRef` breaking changes below follow the 0.y.z minor lane. + ### Breaking - `CultNetDocumentRegistry::register` returns `Result<&mut Self>` where it returned `&mut Self`. A binding that claims a document type or a schema id another binding already holds is refused with `cultcache_rs::SchemaConflictError` (naming the new binding's schema id and both document types) and nothing is bound; binding a type to the schema id it already holds still replaces that binding. Callers chain `register(...)?`. diff --git a/packages/cultnet-rs/Cargo.toml b/packages/cultnet-rs/Cargo.toml index 13d87867..77f368d4 100644 --- a/packages/cultnet-rs/Cargo.toml +++ b/packages/cultnet-rs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cultnet-rs" -version = "0.2.0" +version = "0.3.0" edition = "2024" publish = false license = "MIT" @@ -29,6 +29,10 @@ zeroize = "1" socket2 = "0.5" uuid = { version = "1", features = ["v4"] } +[target.'cfg(unix)'.dependencies] +fs2 = "0.4" +libc = "0.2" + [target.'cfg(windows)'.dependencies] windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_Security_Cryptography"] } diff --git a/packages/cultnet-rs/src/idunn_runtime_provider.rs b/packages/cultnet-rs/src/idunn_runtime_provider.rs new file mode 100644 index 00000000..6e4173ed --- /dev/null +++ b/packages/cultnet-rs/src/idunn_runtime_provider.rs @@ -0,0 +1,1404 @@ +//! The provider half of `gamecult.runtime_presence_health.v2` for Rust runtimes. +//! +//! An Idunn-managed service attests its own health: it opens the root-provisioned +//! runtime bundle with the two signing credentials Idunn hands it, reports its +//! health as it changes, and signs presence records that Odin can authenticate +//! and Idunn's route challenge can read. This module owns that provider half and +//! owns no network; publishing the signed records is a separate concern that +//! consumes [`IdunnRuntimePresenceSigner::sign`]. +//! +//! The module compiles and runs on every platform an Idunn driver launches: +//! Idunn's systemd driver (unix) delivers the credentials as inherited +//! descriptors under root custody, its host driver (Windows) as files in the +//! user's profile. Only the descriptor delivery, root custody and the write-lease +//! lock are unix-only. +//! +//! ```no_run +//! use cultnet_rs::{ +//! GameCultRuntimeCapability, IdunnRuntimeAuthority, IdunnRuntimeCredentials, +//! IdunnRuntimePresenceSigner, RuntimePresenceState, +//! }; +//! +//! # fn main() -> anyhow::Result<()> { +//! // The host driver's delivery. Under Idunn's systemd driver (unix) use +//! // `IdunnRuntimeCredentials::from_systemd_descriptors()` instead. +//! let credentials = IdunnRuntimeCredentials::from_host_environment()?; +//! // The runtime bundle, the candidate bind and the write lease are read from the +//! // launch environment Idunn set. +//! let authority = IdunnRuntimeAuthority::open( +//! "my-service", +//! "my-service.runtime-health.v1", +//! credentials, +//! )?; +//! authority.require_candidate_bind("127.0.0.1:14103")?; +//! let mut signer = IdunnRuntimePresenceSigner::new( +//! authority, +//! vec![GameCultRuntimeCapability { +//! capability: "my.capability".into(), +//! schema: "my.schema.v1".into(), +//! compatibility: "v1".into(), +//! capacity: 1, +//! }], +//! ); +//! let warming = signer.sign("starting")?; // the signer starts Warming +//! signer.report_health(RuntimePresenceState::Active); +//! let active = signer.sign("ready")?; +//! # let _ = (warming, active); +//! # Ok(()) +//! # } +//! ``` + +use std::fs::File; +#[cfg(unix)] +use std::fs::OpenOptions; +use std::io::Read; +#[cfg(unix)] +use std::os::fd::{AsRawFd, FromRawFd, RawFd}; +#[cfg(unix)] +use std::os::unix::fs::MetadataExt; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, Result, anyhow, bail, ensure}; +use cultcache_rs::{CultCacheEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore}; +use zeroize::Zeroizing; + +#[cfg(unix)] +use crate::IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME; +#[cfg(unix)] +use crate::RUNTIME_PRESENCE_IDENTITY_FD_NAME; +use crate::{ + CultNetMessage, CultNetRawDocumentRecord, CultNetRawPayloadEncoding, + GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, GameCultProviderHealthIdentity, + GameCultRuntimeCapability, GameCultRuntimePresenceHealthPurpose, + GameCultRuntimePresenceHealthRecord, IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT, + IDUNN_EXPECTED_INCARNATION_SCHEMA, IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT, + IDUNN_RUNTIME_ACTIVATION_SCHEMA, IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, + IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, IdunnExpectedIncarnationRecord, + IdunnRuntimeActivationRecord, IdunnRuntimeActivationSigner, RUNTIME_PRESENCE_IDENTITY_BINDING, + ServiceIdentitySigner, open_service_identity_credential_reader, +}; +#[cfg(unix)] +use crate::{IDUNN_PROCESS_WRITE_LEASE_SCHEMA, IdunnProcessWriteLeaseRecord}; + +#[cfg(unix)] +const LISTEN_PID_ENVIRONMENT: &str = "LISTEN_PID"; +#[cfg(unix)] +const LISTEN_FDS_ENVIRONMENT: &str = "LISTEN_FDS"; +#[cfg(unix)] +const LISTEN_FDNAMES_ENVIRONMENT: &str = "LISTEN_FDNAMES"; +/// `SD_LISTEN_FDS_START` in sd_listen_fds(3). +#[cfg(unix)] +const LISTEN_FDS_START: RawFd = 3; +const MAX_RECENT_WARMING_PROOFS: usize = 64; +const MAX_DETAIL_BYTES: usize = 512; +const DOCUMENT_ROLE: &str = "runtime-presence-health-publisher"; +const DOCUMENT_TAGS: [&str; 2] = ["cultnet.transport.rudp.v0", "runtime-presence"]; + +// ------------------------------------------------------------------ host port + +/// The process facts the provider reads: environment, pid and inherited +/// descriptors. Production reads the real process; tests supply a fake, so the +/// rules run without mutating the test process's own environment. +trait LaunchHost { + fn var(&self, name: &str) -> Option; + fn remove_var(&mut self, name: &str); + fn process_id(&self) -> u32; + /// Takes sole ownership of an inherited descriptor, or refuses if it is not open. + #[cfg(unix)] + fn adopt_descriptor(&mut self, descriptor: RawFd) -> Result; +} + +struct ProcessHost; + +impl LaunchHost for ProcessHost { + fn var(&self, name: &str) -> Option { + std::env::var(name).ok() + } + + fn remove_var(&mut self, name: &str) { + // SAFETY: the credential helpers document that they run during start-up + // before the service spawns threads, which is the contract + // `std::env::remove_var` needs. + unsafe { std::env::remove_var(name) } + } + + fn process_id(&self) -> u32 { + std::process::id() + } + + #[cfg(unix)] + fn adopt_descriptor(&mut self, descriptor: RawFd) -> Result { + // SAFETY: F_GETFD takes an integer descriptor and dereferences nothing. + if unsafe { libc::fcntl(descriptor, libc::F_GETFD) } == -1 { + return Err(std::io::Error::last_os_error()) + .with_context(|| format!("inherited descriptor {descriptor} is not open")); + } + // SAFETY: the caller proved the descriptor is open, names it exactly once + // and has not adopted it before, so this is its only owner. + Ok(unsafe { File::from_raw_fd(descriptor) }) + } +} + +fn optional_environment(host: &impl LaunchHost, name: &str) -> Result> { + match host.var(name) { + None => Ok(None), + Some(value) if !value.is_empty() && value.trim() == value => Ok(Some(value)), + Some(_) => bail!("{name} is empty or has surrounding whitespace"), + } +} + +fn required_environment(host: &impl LaunchHost, name: &str) -> Result { + optional_environment(host, name)? + .ok_or_else(|| anyhow!("{name} is required for an Idunn-managed runtime")) +} + +// -------------------------------------------------------------------- custody + +/// What a path is, read without following links. std counts symlinks and, on +/// Windows, junctions as links; Profile custody accepts only the real thing. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum FileKind { + Regular, + Directory, + Symlink, + Other, +} + +impl FileKind { + fn of(metadata: &std::fs::Metadata) -> Self { + let file_type = metadata.file_type(); + if file_type.is_symlink() { + Self::Symlink + } else if file_type.is_dir() { + Self::Directory + } else if file_type.is_file() { + Self::Regular + } else { + Self::Other + } + } + + fn noun(self) -> &'static str { + match self { + Self::Directory => "a directory", + _ => "a regular file", + } + } +} + +/// What a custody-checked path must be. +#[derive(Clone, Copy, Debug)] +enum Rule { + /// A record or lock Idunn provisions read-only. + ReadOnlyFile, + /// The runtime bundle directory. + ReadOnlyDirectory, + /// The write-lease record and its lock. + ControlledFile, +} + +impl Rule { + fn kind(self) -> FileKind { + match self { + Self::ReadOnlyFile | Self::ControlledFile => FileKind::Regular, + Self::ReadOnlyDirectory => FileKind::Directory, + } + } + + #[cfg(unix)] + fn root_description(self) -> &'static str { + match self { + Self::ReadOnlyFile => "a root-owned read-only regular file", + Self::ReadOnlyDirectory => "a root-owned read-only directory", + Self::ControlledFile => "a root-controlled regular file", + } + } +} + +/// Profile custody trusts the user's profile and checks only that the path is the +/// real thing: no owner or ACL, because Idunn's host driver sets none. +fn profile_accepts(kind: FileKind, rule: Rule) -> bool { + kind == rule.kind() +} + +/// The unix facts a root-custody rule reads. Rules are pure predicates over +/// these, so the whole policy is testable without owning root. +#[cfg(unix)] +#[derive(Clone, Copy, Debug)] +struct Facts { + kind: FileKind, + uid: u32, + gid: u32, + mode: u32, + nlink: u64, + len: u64, +} + +#[cfg(unix)] +impl Facts { + fn of(metadata: &std::fs::Metadata) -> Self { + Self { + kind: FileKind::of(metadata), + uid: metadata.uid(), + gid: metadata.gid(), + mode: metadata.mode() & 0o7777, + nlink: metadata.nlink(), + len: metadata.len(), + } + } + + fn of_path(path: &Path, label: &str) -> Result { + let metadata = + std::fs::symlink_metadata(path).with_context(|| format!("inspecting {label}"))?; + Ok(Self::of(&metadata)) + } + + fn satisfies(&self, rule: Rule) -> bool { + match rule { + Rule::ReadOnlyFile => self.is_root_read_only_file(), + Rule::ReadOnlyDirectory => self.is_root_read_only_directory(), + Rule::ControlledFile => self.is_root_controlled_file(), + } + } + + /// Root-owned regular file nobody can write, not even root's group. + fn is_root_read_only_file(&self) -> bool { + self.kind == FileKind::Regular && self.uid == 0 && self.mode & 0o222 == 0 + } + + fn is_root_read_only_directory(&self) -> bool { + self.kind == FileKind::Directory && self.uid == 0 && self.mode & 0o222 == 0 + } + + /// Root-owned regular file that group and world cannot write. + fn is_root_controlled_file(&self) -> bool { + self.kind == FileKind::Regular && self.uid == 0 && self.mode & 0o022 == 0 + } + + fn is_root_controlled_directory(&self) -> bool { + self.kind == FileKind::Directory && self.uid == 0 && self.mode & 0o022 == 0 + } + + /// An inherited signer: root-owned, mode exactly 0400, one link, not empty. + fn is_signer_descriptor(&self) -> bool { + self.kind == FileKind::Regular + && self.uid == 0 + && self.gid == 0 + && self.mode == 0o400 + && self.nlink == 1 + && self.len > 0 + } +} + +/// How much of the filesystem the provider trusts, derived from how the +/// credentials were delivered and never chosen by a caller. Idunn's systemd +/// driver provisions root-owned material (`Root`); its host driver writes into the +/// user's profile (`Profile`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Custody { + #[cfg(unix)] + Root, + Profile, +} + +impl Custody { + fn require_file(self, path: &Path, label: &str, rule: Rule) -> Result<()> { + ensure!(path.is_absolute(), "{label} path is not absolute"); + let metadata = + std::fs::symlink_metadata(path).with_context(|| format!("inspecting {label}"))?; + match self { + Self::Profile => ensure!( + profile_accepts(FileKind::of(&metadata), rule), + "{label} is not {} (a link is refused)", + rule.kind().noun() + ), + #[cfg(unix)] + Self::Root => ensure!( + Facts::of(&metadata).satisfies(rule), + "{label} is not {}", + rule.root_description() + ), + } + Ok(()) + } + + /// The sibling `.lock` of a record. Only root custody inspects it. + #[cfg(unix)] + fn require_lock(self, path: &Path, label: &str, rule: Rule) -> Result<()> { + match self { + Self::Profile => Ok(()), + Self::Root => self.require_file(path, label, rule), + } + } + + /// Every directory from `path` up to the filesystem root is root-controlled. + /// Only root custody walks the chain. + #[cfg(unix)] + fn require_directory_chain(self, path: &Path, label: &str) -> Result<()> { + if self == Self::Profile { + return Ok(()); + } + ensure!(path.is_absolute(), "{label} path is not absolute"); + for directory in path.ancestors() { + let facts = Facts::of_path(directory, label)?; + ensure!( + facts.is_root_controlled_directory(), + "{label} is not entirely root-controlled" + ); + } + Ok(()) + } + + fn require_bundle(self, bundle: &Path) -> Result<()> { + self.require_file(bundle, "Idunn runtime bundle", Rule::ReadOnlyDirectory)?; + #[cfg(unix)] + { + let parent = bundle.parent().context("runtime bundle has no parent")?; + self.require_directory_chain(parent, "Idunn runtime bundle parent")?; + } + Ok(()) + } +} + +#[cfg(unix)] +fn sibling_lock_path(path: &Path) -> Result { + let mut name = path + .file_name() + .context("CultCache authority path has no file name")? + .to_os_string(); + name.push(".lock"); + Ok(path.with_file_name(name)) +} + +// ---------------------------------------------------------------- credentials + +/// The two signing credentials Idunn issues a launch: the one-launch activation +/// seed and the stable provider identity. Idunn's systemd driver delivers them +/// as OpenFile descriptors and its host driver as files named in the +/// environment; `from_systemd_descriptors` (unix only) and +/// [`from_host_environment`](Self::from_host_environment) read each form. +/// +/// The delivery decides how much of the filesystem the authority trusts: systemd +/// descriptors carry root custody, the host files carry profile custody. A +/// caller never chooses it. +/// +/// The raw constructor is not public, so a caller cannot choose custody: +/// +/// ```compile_fail,E0624 +/// let _ = cultnet_rs::IdunnRuntimeCredentials::new(std::io::empty(), std::io::empty()); +/// ``` +pub struct IdunnRuntimeCredentials { + activation: Box, + provider: Box, + custody: Custody, +} + +impl IdunnRuntimeCredentials { + /// Wraps the host driver's activation seed and provider identity credential + /// (profile custody). Crate-private: custody follows delivery, so a caller + /// never picks it and uses a `from_*` constructor. + pub(crate) fn new(activation: impl Read + 'static, provider: impl Read + 'static) -> Self { + Self::delivered(activation, provider, Custody::Profile) + } + + fn delivered( + activation: impl Read + 'static, + provider: impl Read + 'static, + custody: Custody, + ) -> Self { + Self { + activation: Box::new(activation), + provider: Box::new(provider), + custody, + } + } + + /// Takes the credentials from systemd's inherited descriptors, following + /// sd_listen_fds_with_names(3). `LISTEN_PID` must be this process (or, under + /// `PrivatePIDs`, this process must be pid 1 while `LISTEN_PID` names a host + /// pid above 1); the descriptors are found by name and extra descriptors are + /// tolerated; each must be a root-owned 0400 single-link read-only regular + /// file and is marked close-on-exec. `LISTEN_PID`, `LISTEN_FDS` and + /// `LISTEN_FDNAMES` are removed from the environment so no child inherits + /// them. + /// + /// Call this during start-up, before the service spawns threads: it edits the + /// process environment and takes ownership of descriptors 3 and above. + #[cfg(unix)] + pub fn from_systemd_descriptors() -> Result { + Self::from_systemd_descriptors_in(&mut ProcessHost) + } + + #[cfg(unix)] + fn from_systemd_descriptors_in(host: &mut impl LaunchHost) -> Result { + let (activation, provider) = take_systemd_descriptors(host, Custody::Root)?; + Ok(Self::delivered(activation, provider, Custody::Root)) + } + + /// Takes the credentials from the files Idunn's host driver names in + /// [`IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT`] and + /// [`RUNTIME_PRESENCE_IDENTITY_BINDING`]. The provider identity file is opened + /// first; a missing, unreadable or non-regular one refuses and leaves the + /// activation credential in place. The activation credential is one-shot: it is read + /// into memory, its handle closed and the file removed, so the seed lives only + /// in this process afterwards. A refusal names the variable and never the path. + pub fn from_host_environment() -> Result { + Self::from_host_environment_in(&ProcessHost) + } + + fn from_host_environment_in(host: &impl LaunchHost) -> Result { + let activation_path = PathBuf::from(required_environment( + host, + IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT, + )?); + let provider_path = PathBuf::from(required_environment( + host, + RUNTIME_PRESENCE_IDENTITY_BINDING, + )?); + ensure!( + activation_path.is_absolute() && provider_path.is_absolute(), + "host credential paths are not absolute" + ); + let provider = File::open(&provider_path).with_context(|| { + format!("{RUNTIME_PRESENCE_IDENTITY_BINDING}: opening provider identity credential") + })?; + ensure!( + provider + .metadata() + .with_context(|| { + format!("{RUNTIME_PRESENCE_IDENTITY_BINDING}: inspecting provider identity") + })? + .is_file(), + "{RUNTIME_PRESENCE_IDENTITY_BINDING}: provider identity credential is not a regular file" + ); + let mut seed = Zeroizing::new(Vec::new()); + { + let mut file = File::open(&activation_path).with_context(|| { + format!("{IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT}: opening activation credential") + })?; + file.read_to_end(&mut seed).with_context(|| { + format!("{IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT}: reading activation credential") + })?; + } + std::fs::remove_file(&activation_path).with_context(|| { + format!("{IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT}: retiring activation credential") + })?; + Ok(Self::new(SeedReader { seed, position: 0 }, provider)) + } +} + +/// Reads an in-memory seed and wipes it when dropped. +struct SeedReader { + seed: Zeroizing>, + position: usize, +} + +impl Read for SeedReader { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + let remaining = &self.seed[self.position..]; + let count = remaining.len().min(buffer.len()); + buffer[..count].copy_from_slice(&remaining[..count]); + self.position += count; + Ok(count) + } +} + +/// systemd's `LISTEN_PID` is a host pid. `PrivatePIDs=yes` makes the main process +/// pid 1 in a private namespace, where it cannot see that host pid; a descendant +/// has a pid above 1 and must not reuse the inherited set. +#[cfg(unix)] +fn listen_pid_matches(listen_pid: &str, process_id: u32) -> bool { + match listen_pid.parse::() { + Ok(declared) if declared.to_string() == listen_pid => { + declared == process_id || (process_id == 1 && declared > 1) + } + _ => false, + } +} + +#[cfg(unix)] +fn take_systemd_descriptors(host: &mut impl LaunchHost, custody: Custody) -> Result<(File, File)> { + let listen_pid = required_environment(&*host, LISTEN_PID_ENVIRONMENT); + let listen_fds = required_environment(&*host, LISTEN_FDS_ENVIRONMENT); + let listen_fd_names = required_environment(&*host, LISTEN_FDNAMES_ENVIRONMENT); + // Whatever the verdict, no child of this process inherits the claim. + for name in [ + LISTEN_PID_ENVIRONMENT, + LISTEN_FDS_ENVIRONMENT, + LISTEN_FDNAMES_ENVIRONMENT, + ] { + host.remove_var(name); + } + let (listen_pid, listen_fds, listen_fd_names) = (listen_pid?, listen_fds?, listen_fd_names?); + ensure!( + listen_pid_matches(&listen_pid, host.process_id()), + "systemd signer descriptors belong to a different process" + ); + let count = listen_fds + .parse::() + .ok() + .filter(|count| count.to_string() == listen_fds) + .context("parsing systemd LISTEN_FDS")?; + let names = listen_fd_names.split(':').collect::>(); + ensure!( + names.len() == count, + "systemd LISTEN_FDNAMES does not name every inherited descriptor" + ); + let activation = find_descriptor(&names, IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME)?; + let provider = find_descriptor(&names, RUNTIME_PRESENCE_IDENTITY_FD_NAME)?; + let activation = host.adopt_descriptor(activation)?; + let activation = protect_descriptor(activation, "runtime activation signer", custody)?; + let provider = host.adopt_descriptor(provider)?; + let provider = protect_descriptor(provider, "stable provider signer", custody)?; + Ok((activation, provider)) +} + +/// The descriptor number of the one descriptor called `name`. +#[cfg(unix)] +fn find_descriptor(names: &[&str], name: &str) -> Result { + let mut matches = names + .iter() + .enumerate() + .filter(|(_, candidate)| **candidate == name) + .map(|(index, _)| LISTEN_FDS_START + index as RawFd); + let descriptor = matches + .next() + .ok_or_else(|| anyhow!("systemd did not pass the {name} descriptor"))?; + ensure!( + matches.next().is_none(), + "systemd passed the {name} descriptor more than once" + ); + Ok(descriptor) +} + +#[cfg(unix)] +fn protect_descriptor(file: File, label: &str, custody: Custody) -> Result { + // OpenFile descriptors arrive without close-on-exec; a child must never + // inherit a signer, even on a path that refuses it below. + // SAFETY: fcntl reads and sets flags of the descriptor `file` owns. + let flags = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_GETFD) }; + if flags == -1 + // SAFETY: as above. + || unsafe { libc::fcntl(file.as_raw_fd(), libc::F_SETFD, flags | libc::FD_CLOEXEC) } == -1 + { + return Err(std::io::Error::last_os_error()) + .with_context(|| format!("protecting {label} descriptor from child inheritance")); + } + // SAFETY: F_GETFL reads the flags of the descriptor `file` owns. + let status = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_GETFL) }; + if status == -1 { + return Err(std::io::Error::last_os_error()) + .with_context(|| format!("reading {label} access mode")); + } + ensure!( + status & libc::O_ACCMODE == libc::O_RDONLY, + "{label} descriptor is not read-only" + ); + if custody == Custody::Root { + let facts = Facts::of( + &file + .metadata() + .with_context(|| format!("inspecting {label}"))?, + ); + ensure!( + facts.is_signer_descriptor(), + "{label} descriptor is not a root-owned 0400 singly-linked regular file" + ); + } + Ok(file) +} + +// ------------------------------------------------------------------ authority + +/// What the authority and every lease guard share: the verified records and the +/// custody policy their files are read under. +struct Records { + expected: IdunnExpectedIncarnationRecord, + expected_sha256: String, + activation: IdunnRuntimeActivationRecord, + activation_sha256: String, + /// Set exactly when Expected requires a process write lease (unix only: no + /// other platform can hold the lease lock, so `open` refuses the requirement). + #[cfg(unix)] + lease_path: Option, + #[cfg(unix)] + custody: Custody, +} + +/// One launch's verified Idunn authority: the Expected incarnation, the +/// activation witness bound to it, and the two signing keys Idunn issued. +/// +/// Opening checks every binding the provider owes Idunn. The Expected target and +/// health contract equal the caller's; the activation binds Expected's canonical +/// digest and runtime id; the activation credential is the activation's signer; +/// the provider identity is the signer Expected selected; the write-lease +/// environment is present exactly when Expected requires a lease; and the +/// candidate-bind environment names Expected's route. The launch environment is +/// read here and nowhere else. +pub struct IdunnRuntimeAuthority { + records: Arc, + activation_signer: IdunnRuntimeActivationSigner, + provider_signer: ServiceIdentitySigner, + candidate_bind: Option, +} + +impl std::fmt::Debug for IdunnRuntimeAuthority { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("IdunnRuntimeAuthority") + .field("target", &self.records.expected.target) + .field("runtime_id", &self.records.expected.runtime_id) + .finish_non_exhaustive() + } +} + +impl IdunnRuntimeAuthority { + /// Opens the runtime bundle Idunn launched this process with, for the target + /// `target` and the health contract `health_contract`, with the credentials + /// Idunn issued. The bundle directory comes from + /// [`IDUNN_RUNTIME_BUNDLE_ENVIRONMENT`], the candidate bind from + /// [`IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT`] and the write lease from + /// [`IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT`]; no caller supplies a path. + /// + /// The credentials' delivery decides the custody the bundle is held to. Under + /// systemd descriptors (root custody, unix) the bundle directory, its + /// `expected.cc` and `activation.cc`, their `.lock` siblings and every parent + /// directory must be root-owned and not writable by others, and none may be a + /// symlink; the write lease, if Expected requires one, is held to the same + /// custody. Under any other delivery (profile custody, every platform) the + /// bundle directory and the two records must be a real directory and real + /// regular files, never a link. A runtime that requires a + /// write lease can only be opened on unix, where the lease lock exists. + pub fn open( + target: &str, + health_contract: &str, + credentials: IdunnRuntimeCredentials, + ) -> Result { + Self::open_in(&ProcessHost, target, health_contract, credentials) + } + + fn open_in( + host: &impl LaunchHost, + target: &str, + health_contract: &str, + credentials: IdunnRuntimeCredentials, + ) -> Result { + let custody = credentials.custody; + let bundle = PathBuf::from(required_environment( + host, + IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, + )?); + ensure!( + bundle.is_absolute(), + "{IDUNN_RUNTIME_BUNDLE_ENVIRONMENT} is not an absolute path" + ); + custody.require_bundle(&bundle)?; + let (key, payload) = read_authority_record( + &bundle.join("expected.cc"), + IdunnExpectedIncarnationRecord::TYPE, + IDUNN_EXPECTED_INCARNATION_SCHEMA, + custody, + )?; + let expected = IdunnExpectedIncarnationRecord::decode_canonical(&payload)?; + ensure!( + key == expected.target, + "Expected record key is not its target" + ); + ensure!( + expected.target == target && expected.health_contract == health_contract, + "Idunn Expected target or health contract does not match this runtime" + ); + let (key, payload) = read_authority_record( + &bundle.join("activation.cc"), + IdunnRuntimeActivationRecord::TYPE, + IDUNN_RUNTIME_ACTIVATION_SCHEMA, + custody, + )?; + let activation = IdunnRuntimeActivationRecord::decode_canonical(&payload)?; + ensure!( + key == expected.target, + "activation record key is not the Expected target" + ); + let expected_sha256 = expected.canonical_sha256()?; + ensure!( + activation.expected_projection_sha256 == expected_sha256 + && activation.runtime_id == expected.runtime_id, + "runtime activation does not bind the bundled Expected projection" + ); + + // The lease lock exists only on unix; a requirement no process here can + // hold is refused rather than signed around. + #[cfg(not(unix))] + ensure!( + !expected.write_lease_required, + "write_lease_required: Expected requires a process write lease, which this platform cannot hold" + ); + let lease_path = optional_environment(host, IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT)?; + ensure!( + lease_path.is_some() == expected.write_lease_required, + "Idunn process write-lease environment does not match Expected" + ); + #[cfg(unix)] + let lease_path = lease_path.map(PathBuf::from); + #[cfg(unix)] + if let Some(path) = &lease_path { + ensure!( + path.is_absolute(), + "Idunn process write-lease path is not absolute" + ); + } + let candidate_bind = optional_environment(host, IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT)?; + match (&expected.route, &candidate_bind) { + (Some(route), Some(bind)) => ensure!( + route.candidate_endpoint == format!("{}://{bind}", route.transport), + "Idunn candidate bind differs from the Expected route" + ), + (None, None) => {} + _ => bail!("Idunn candidate bind environment does not match Expected's route"), + } + + let IdunnRuntimeCredentials { + activation: activation_credential, + provider: provider_credential, + .. + } = credentials; + let activation_signer = + IdunnRuntimeActivationSigner::from_credential_reader(activation_credential)?; + ensure!( + activation_signer.identity_id() == activation.activation_signer_identity_id + && activation_signer.public_key() == activation.activation_signer_public_key, + "runtime activation credential does not belong to the Idunn activation record" + ); + // The shared reader owns profile validation, platform binding, seed + // unprotection and domain-separated signing; this module never parses or + // exports seed bytes. + let provider_signer = open_service_identity_credential_reader::< + GameCultProviderHealthIdentity, + >(provider_credential)?; + ensure!( + provider_signer.entry().identity_id == expected.expected_signer_identity_id, + "provider signer is not the identity selected by Expected" + ); + + Ok(Self { + records: Arc::new(Records { + activation_sha256: activation.canonical_sha256()?, + expected_sha256, + expected, + activation, + #[cfg(unix)] + lease_path, + #[cfg(unix)] + custody, + }), + activation_signer, + provider_signer, + candidate_bind, + }) + } + + /// Idunn's admitted description of this incarnation. + pub fn expected(&self) -> &IdunnExpectedIncarnationRecord { + &self.records.expected + } + + /// The activation witness Idunn issued for this launch. + pub fn activation(&self) -> &IdunnRuntimeActivationRecord { + &self.records.activation + } + + /// The endpoint every presence states as bound: Expected's candidate route, + /// confirmed at open against Idunn's candidate-bind environment. `None` when + /// the incarnation has no route. + pub fn bound_endpoint(&self) -> Option<&str> { + self.records + .expected + .route + .as_ref() + .map(|route| route.candidate_endpoint.as_str()) + } + + /// Checks that `bind` (a `host:port`) is the address Idunn will challenge: + /// the candidate bind Idunn put in [`IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT`], + /// which open already tied to Expected's route. Call it with the address the + /// service actually binds, before it listens. + pub fn require_candidate_bind(&self, bind: &str) -> Result<()> { + let route = self + .records + .expected + .route + .as_ref() + .context("Expected has no admitted route")?; + ensure!( + self.candidate_bind.as_deref() == Some(bind) + && route.candidate_endpoint == format!("{}://{bind}", route.transport), + "candidate bind differs from Idunn's candidate bind and the Expected route" + ); + Ok(()) + } +} + +fn read_authority_record( + path: &Path, + record_type: &str, + schema: &str, + custody: Custody, +) -> Result<(String, Vec)> { + custody.require_file(path, "runtime authority record", Rule::ReadOnlyFile)?; + #[cfg(unix)] + custody.require_lock( + &sibling_lock_path(path)?, + "runtime authority lock", + Rule::ReadOnlyFile, + )?; + read_snapshot(path, "runtime authority record", |entries| { + let [envelope] = entries.as_slice() else { + bail!("runtime authority store must contain exactly one record"); + }; + ensure!( + envelope.r#type == record_type && envelope.schema_id.as_deref() == Some(schema), + "runtime authority store has the wrong typed envelope" + ); + Ok((envelope.key.clone(), envelope.payload.clone())) + }) +} + +/// Reads `path` as a CultCache store under its shared lock and hands the +/// envelopes to `read`. CultCache's own errors carry the store's path, which +/// under the host driver lies in the user's profile, so a failure of the store +/// itself is reported by `label` and a path-free kind alone +/// ([`store_failure_kind`]); `read`'s refusals pass through. +fn read_snapshot( + path: &Path, + label: &str, + read: impl FnOnce(Vec) -> Result, +) -> Result { + let mut outcome = None; + let store = + SingleFileMessagePackBackingStore::new(path).with_read_only_shared_snapshot(|entries| { + outcome = Some(read(entries)); + Ok(()) + }); + match (store, outcome) { + (Ok(()), Some(outcome)) => outcome, + (Err(error), _) => bail!( + "{label} could not be read as a CultCache store: {}", + store_failure_kind(&error) + ), + (Ok(()), None) => bail!("{label} could not be read as a CultCache store"), + } +} + +/// What kept a store from being read, as a label that names no path: CultCache's +/// verdict on the bytes, else the kind of the I/O failure in the chain. +fn store_failure_kind(error: &anyhow::Error) -> &'static str { + use cultcache_rs::{CultCacheStoreUnreadable, CultCacheStoreUnreadableKind}; + if let Some(unreadable) = error.downcast_ref::() { + return match unreadable.kind { + CultCacheStoreUnreadableKind::Undecodable => "not decodable", + CultCacheStoreUnreadableKind::UnsupportedFormat => { + "in a format this runtime does not read" + } + _ => "unreadable", + }; + } + match error + .chain() + .find_map(|cause| cause.downcast_ref::()) + .map(std::io::Error::kind) + { + Some(std::io::ErrorKind::NotFound) => "not found", + Some(std::io::ErrorKind::PermissionDenied) => "permission denied", + Some(_) => "I/O error", + None => "unreadable", + } +} + +// ---------------------------------------------------------------- write lease + +/// Proof that this process holds Idunn's process write lease. It holds the +/// lease's sibling `.lock` shared for as long as any clone lives, which fences +/// Idunn's lease replacement and every other writer that takes the lock +/// exclusively. Gate state writes on [`require_current`](Self::require_current). +/// +/// The lease lock exists only on unix. The type exists everywhere so consumer code +/// needs no `cfg`, but on other platforms no guard can be constructed: `open` +/// refuses a runtime that requires a lease. +#[derive(Clone)] +pub struct ProcessWriteLeaseGuard { + held: Arc, +} + +#[cfg(unix)] +struct HeldLease { + _lock: File, + records: Arc, + warming_proofs: Vec, + lease_sha256: String, +} + +/// No platform without the lease lock can hold a lease. +#[cfg(not(unix))] +enum HeldLease {} + +impl HeldLease { + #[cfg(unix)] + fn lease_sha256(&self) -> &str { + &self.lease_sha256 + } + + #[cfg(not(unix))] + fn lease_sha256(&self) -> &str { + match *self {} + } + + #[cfg(unix)] + fn require_current(&self) -> Result<()> { + let observed = load_lease(&self.records, &self.warming_proofs)?; + ensure!( + observed.as_deref() == Some(self.lease_sha256()), + "process-write lease is no longer the admitted lifetime grant" + ); + Ok(()) + } + + #[cfg(not(unix))] + fn require_current(&self) -> Result<()> { + match *self {} + } +} + +impl std::fmt::Debug for ProcessWriteLeaseGuard { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("ProcessWriteLeaseGuard") + .field("lease_sha256", &self.lease_sha256()) + .finish_non_exhaustive() + } +} + +impl ProcessWriteLeaseGuard { + /// The canonical digest of the lease record this guard was admitted under. + pub fn lease_sha256(&self) -> &str { + self.held.lease_sha256() + } + + /// Fails unless the lease record on disk is still the admitted lifetime grant. + pub fn require_current(&self) -> Result<()> { + self.held.require_current() + } +} + +/// The digest of the lease on disk if it exists and binds this incarnation and +/// one of `warming_proofs`; `None` while the record is absent or empty. +#[cfg(unix)] +fn load_lease(records: &Records, warming_proofs: &[String]) -> Result> { + let Some(path) = present_lease_path(records)? else { + return Ok(None); + }; + read_snapshot(path, "process-write-lease record", |entries| { + let [envelope] = entries.as_slice() else { + if entries.is_empty() { + return Ok(None); + } + bail!("process-write-lease store is ambiguous"); + }; + ensure!( + envelope.r#type == IdunnProcessWriteLeaseRecord::TYPE + && envelope.schema_id.as_deref() == Some(IDUNN_PROCESS_WRITE_LEASE_SCHEMA), + "process-write-lease store has the wrong typed envelope" + ); + let lease = IdunnProcessWriteLeaseRecord::decode_canonical(&envelope.payload)?; + ensure!( + envelope.key == lease.target, + "process-write-lease key is not its target" + ); + exact_lease_sha256(&lease, records, warming_proofs).map(Some) + }) +} + +/// The lease record's path once the record exists and its files pass custody; +/// `None` while Idunn has not written it. +#[cfg(unix)] +fn present_lease_path(records: &Records) -> Result> { + let path = records + .lease_path + .as_deref() + .context("Expected does not require a process write lease")?; + match std::fs::symlink_metadata(path) { + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.into()), + } + require_lease_custody(records.custody, path)?; + Ok(Some(path)) +} + +#[cfg(unix)] +fn require_lease_custody(custody: Custody, path: &Path) -> Result<()> { + custody.require_file(path, "process-write-lease record", Rule::ControlledFile)?; + custody.require_lock( + &sibling_lock_path(path)?, + "process-write-lease lock", + Rule::ControlledFile, + )?; + custody.require_directory_chain( + path.parent().context("process-write lease has no parent")?, + "process-write-lease parent", + ) +} + +#[cfg(unix)] +fn exact_lease_sha256( + lease: &IdunnProcessWriteLeaseRecord, + records: &Records, + warming_proofs: &[String], +) -> Result { + lease.validate()?; + let expected = &records.expected; + ensure!( + lease.target == expected.target + && lease.expected_projection_sha256 == records.expected_sha256 + && lease.plan_id == expected.plan_id + && lease.incarnation_id == expected.incarnation_id + && lease.sealed_release_id == expected.sealed_release_id + && lease.activation_witness_sha256 == records.activation_sha256 + && Some(&lease.state_schema_generation) == expected.state_schema_generation.as_ref() + && Some(&lease.state_contract_sha256) == expected.state_contract_sha256.as_ref() + && lease.runtime_id == expected.runtime_id + && lease.runtime_instance_id == records.activation.runtime_instance_id + && warming_proofs + .iter() + .any(|proof| proof == &lease.warming_presence_sha256), + "process-write lease does not bind a provider-owned Warming proof for this incarnation" + ); + lease.canonical_sha256() +} + +/// Takes the lease's shared lock and admits the record under it. `None` while +/// the record is absent. +#[cfg(unix)] +fn acquire_lease( + records: &Arc, + warming_proofs: &[String], +) -> Result> { + let Some(path) = present_lease_path(records)? else { + return Ok(None); + }; + let lock_path = sibling_lock_path(path)?; + let lock = OpenOptions::new() + .read(true) + .open(&lock_path) + .context("opening process-write-lease lock")?; + fs2::FileExt::lock_shared(&lock).context("locking process-write-lease lock")?; + // The first shared lock stays held while the snapshot is decoded; the + // backing-store read takes its own nested shared lock so the canonical + // CultCache decoder is the one reading the record. + let Some(lease_sha256) = load_lease(records, warming_proofs)? else { + return Ok(None); + }; + Ok(Some(ProcessWriteLeaseGuard { + held: Arc::new(HeldLease { + _lock: lock, + records: Arc::clone(records), + warming_proofs: warming_proofs.to_vec(), + lease_sha256, + }), + })) +} + +/// `open` refuses a lease requirement on this platform, so nothing reaches here. +#[cfg(not(unix))] +fn acquire_lease( + _records: &Arc, + _warming_proofs: &[String], +) -> Result> { + bail!("write_lease_required: this platform cannot hold a process write lease") +} + +// --------------------------------------------------------------------- signer + +/// The health an app reports about itself. Every presence the signer signs +/// states the last one reported. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RuntimePresenceState { + /// Starting, or waiting for Idunn's write lease. The signer begins here. + Warming, + /// Serving. + Active, + /// Serving with reduced ability. + Degraded, + /// Not able to serve. + Failed, +} + +impl RuntimePresenceState { + /// The state string the presence record carries. + pub fn as_str(self) -> &'static str { + match self { + Self::Warming => "warming", + Self::Active => "active", + Self::Degraded => "degraded", + Self::Failed => "failed", + } + } +} + +/// One signed presence: the validated record, the raw CultNet document that +/// carries it, and the record's canonical digest (what a write lease names when +/// the presence is Warming). +#[derive(Clone, Debug)] +pub struct SignedRuntimePresence { + pub record: GameCultRuntimePresenceHealthRecord, + pub document: CultNetRawDocumentRecord, + pub canonical_sha256: String, +} + +/// Signs this launch's runtime presence and answers Idunn's route challenge from +/// local authority, with no network. +/// +/// The signer is the one owner of reported health: it starts Warming, the app +/// moves it with [`report_health`](Self::report_health), and every presence it +/// signs, however it is carried, states that health. It takes the authority by +/// value, so one signer per authority is a fact of the types: a second signer +/// would restart the publisher sequence under the same runtime instance. +pub struct IdunnRuntimePresenceSigner { + authority: IdunnRuntimeAuthority, + capabilities: Vec, + health: RuntimePresenceState, + sequence: u64, + warming_proofs: Vec, + lease: Option, +} + +impl IdunnRuntimePresenceSigner { + /// A Warming signer over `authority`, stating `capabilities` in every + /// presence. Capabilities are validated when a presence is signed. + pub fn new( + authority: IdunnRuntimeAuthority, + capabilities: Vec, + ) -> Self { + Self { + authority, + capabilities, + health: RuntimePresenceState::Warming, + sequence: 0, + warming_proofs: Vec::new(), + lease: None, + } + } + + /// The authority this signer owns. + pub fn authority(&self) -> &IdunnRuntimeAuthority { + &self.authority + } + + /// Records the app's health. Until the first call the signer reports Warming. + pub fn report_health(&mut self, state: RuntimePresenceState) { + self.health = state; + } + + /// The health the next presence will state. + pub fn health(&self) -> RuntimePresenceState { + self.health + } + + /// Signs one presence now, in the reported health. Every call takes the next + /// publisher sequence. A Warming presence never carries a lease and becomes a + /// valid write-lease target; Active on a lease-required Expected is refused + /// unless the signer holds a current [`ProcessWriteLeaseGuard`]; a held guard + /// that is no longer current refuses Active and Degraded, while Failed still + /// signs, without a lease digest, so a runtime that lost its lease can say so. + pub fn sign(&mut self, detail: &str) -> Result { + self.sign_at(detail, unix_millis()?) + } + + /// [`sign`](Self::sign) with an explicit observation time. + pub fn sign_at( + &mut self, + detail: &str, + observed_at_unix_millis: u64, + ) -> Result { + let records = Arc::clone(&self.authority.records); + let state = self.health; + let write_lease_sha256 = match (state, &self.lease) { + (RuntimePresenceState::Warming, _) | (RuntimePresenceState::Failed, None) => None, + // A runtime that lost its lease must still be able to say it + // failed; the correlation admits a not-ready presence without a + // lease digest. + (RuntimePresenceState::Failed, Some(guard)) => guard + .require_current() + .ok() + .map(|()| guard.lease_sha256().to_owned()), + (_, Some(guard)) => { + guard.require_current()?; + Some(guard.lease_sha256().to_owned()) + } + (_, None) if records.expected.write_lease_required => { + bail!( + "{} stateful presence lacks its process-write lease", + state.as_str() + ) + } + (_, None) => None, + }; + self.sequence = self + .sequence + .checked_add(1) + .ok_or_else(|| anyhow!("runtime presence publisher sequence overflow"))?; + let expected = &records.expected; + let mut record = GameCultRuntimePresenceHealthRecord { + schema_version: GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + target: expected.target.clone(), + expected_projection_sha256: records.expected_sha256.clone(), + plan_id: expected.plan_id.clone(), + incarnation_id: expected.incarnation_id.clone(), + sealed_release_id: expected.sealed_release_id.clone(), + activation_witness_sha256: records.activation_sha256.clone(), + state_schema_generation: expected.state_schema_generation.clone(), + state_contract_sha256: expected.state_contract_sha256.clone(), + runtime_id: expected.runtime_id.clone(), + runtime_instance_id: records.activation.runtime_instance_id.clone(), + bound_endpoint: self.authority.bound_endpoint().map(str::to_owned), + capabilities: self.capabilities.clone(), + health_contract: expected.health_contract.clone(), + state: state.as_str().into(), + detail: truncate_utf8(detail, MAX_DETAIL_BYTES).into(), + write_lease_sha256, + signer_identity_id: self.authority.provider_signer.entry().identity_id.clone(), + publisher_sequence: self.sequence, + observed_at_unix_millis, + signature_algorithm: "ed25519".into(), + signature: Vec::new(), + activation_signer_identity_id: records.activation.activation_signer_identity_id.clone(), + activation_signature: Vec::new(), + }; + let proof_payload = record.canonical_proof_payload()?; + let provider_proof = self + .authority + .provider_signer + .sign::(&proof_payload); + ensure!( + provider_proof.identity_id == record.signer_identity_id, + "provider signer identity changed after runtime admission" + ); + record.signature = provider_proof.signature; + record.activation_signature = self + .authority + .activation_signer + .sign_presence_proof(&record)?; + record.validate()?; + let payload = rmp_serde::to_vec(&record)?; + ensure!( + rmp_serde::from_slice::(&payload)? == record, + "runtime presence did not round-trip canonically" + ); + let canonical_sha256 = record.canonical_sha256()?; + let document = CultNetRawDocumentRecord { + schema_id: GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + record_key: expected.target.clone(), + stored_at: chrono::DateTime::from_timestamp_millis(i64::try_from( + observed_at_unix_millis, + )?) + .context("runtime presence observation time is invalid")? + .to_rfc3339(), + payload_encoding: CultNetRawPayloadEncoding::Messagepack, + payload, + source_runtime_id: Some(expected.runtime_id.clone()), + source_agent_id: Some(record.signer_identity_id.clone()), + source_role: Some(DOCUMENT_ROLE.into()), + tags: Some(DOCUMENT_TAGS.iter().map(|tag| (*tag).into()).collect()), + }; + if state == RuntimePresenceState::Warming { + self.warming_proofs.push(canonical_sha256.clone()); + if self.warming_proofs.len() > MAX_RECENT_WARMING_PROOFS { + self.warming_proofs.remove(0); + } + } + Ok(SignedRuntimePresence { + record, + document, + canonical_sha256, + }) + } + + /// Looks for Idunn's process write lease. `Ok(None)` while Idunn has not + /// written it; an error when a record exists but does not bind this + /// incarnation and a Warming presence this signer signed within its last 64. + /// The guard it returns is kept: later non-Warming presences carry its lease + /// digest. The caller waits, sleeping between calls and signing Warming + /// heartbeats, as its own loop decides. + pub fn try_acquire_write_lease(&mut self) -> Result> { + ensure!( + self.authority.records.expected.write_lease_required, + "Expected does not require a process write lease" + ); + if let Some(guard) = &self.lease { + guard.require_current()?; + return Ok(Some(guard.clone())); + } + let guard = acquire_lease(&self.authority.records, &self.warming_proofs)?; + self.lease = guard.clone(); + Ok(guard) + } + + /// Answers Idunn's route challenge: a `SnapshotResponseRaw` carrying a fresh + /// presence in the reported health. Only a plain snapshot request for this + /// runtime's presence schema and target is answered; another schema, another + /// record key, a shard-addressed request, a request of any other kind or an + /// empty or untrimmed message id is refused. + pub fn answer_route_observation(&mut self, request: &CultNetMessage) -> Result { + let CultNetMessage::SnapshotRequest { + message_id, + schema_ids, + record_keys, + } = request + else { + bail!("Idunn route observation is not a plain snapshot request"); + }; + ensure!( + !message_id.is_empty() && message_id.trim() == message_id, + "Idunn route observation requires a message id" + ); + let target = &self.authority.records.expected.target; + ensure!( + schema_ids.as_ref().is_none_or(|schemas| schemas + .iter() + .any(|schema| schema == GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA)) + && record_keys + .as_ref() + .is_none_or(|keys| keys.iter().any(|key| key == target)), + "Idunn route observation does not request this runtime presence document" + ); + let signed = self.sign(&format!("route-observation:{message_id}"))?; + Ok(CultNetMessage::SnapshotResponseRaw { + message_id: message_id.clone(), + documents: vec![signed.document], + }) + } +} + +fn truncate_utf8(text: &str, max_bytes: usize) -> &str { + if text.len() <= max_bytes { + return text; + } + let mut end = max_bytes; + while !text.is_char_boundary(end) { + end -= 1; + } + &text[..end] +} + +fn unix_millis() -> Result { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .context("system clock is before the Unix epoch")? + .as_millis() + .try_into() + .context("Unix time exceeds u64") +} + +#[cfg(test)] +mod tests; diff --git a/packages/cultnet-rs/src/idunn_runtime_provider/tests.rs b/packages/cultnet-rs/src/idunn_runtime_provider/tests.rs new file mode 100644 index 00000000..9f2d53ec --- /dev/null +++ b/packages/cultnet-rs/src/idunn_runtime_provider/tests.rs @@ -0,0 +1,2120 @@ +use std::collections::BTreeMap; +use std::io::Cursor; + +use cultcache_rs::{CacheBackingStore, CultCacheEnvelope}; + +use super::*; +use crate::{ + IdunnExpectedCapability, IdunnExpectedRoute, IdunnRuntimeActivationLaunch, + IdunnServiceIdentity, RUNTIME_PRESENCE_IDENTITY_FD_NAME, RuntimePresenceAuthenticationContext, + VerifiedRuntimeAuthority, authenticate_runtime_presence_claim, + correlate_runtime_presence_claim, enroll_service_identity_at, verify_runtime_authority, +}; + +const TARGET: &str = "ghostlight"; +const HEALTH_CONTRACT: &str = "ghostlight.runtime-health.v1"; +const BIND: &str = "127.0.0.1:14103"; +const ENDPOINT: &str = "tcp://127.0.0.1:14103"; +#[cfg(unix)] +const ACTIVATION_NAME: &str = IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME; +#[cfg(unix)] +const PROVIDER_NAME: &str = RUNTIME_PRESENCE_IDENTITY_FD_NAME; + +fn digest(byte: char) -> String { + format!("sha256-{}", byte.to_string().repeat(64)) +} + +fn refusal(result: Result) -> String { + match result { + Ok(_) => panic!("expected a refusal"), + Err(error) => format!("{error:#}"), + } +} + +fn capabilities() -> Vec { + ["conversation", "state"] + .into_iter() + .map(|name| GameCultRuntimeCapability { + capability: name.into(), + schema: format!("{name}.v1"), + compatibility: "v1".into(), + capacity: 1, + }) + .collect() +} + +fn write_record(path: &Path, key: &str, ty: &str, schema: &str, payload: Vec) -> Result<()> { + SingleFileMessagePackBackingStore::new(path).push(&CultCacheEnvelope { + key: key.into(), + r#type: ty.into(), + payload, + stored_at: "2026-10-29T00:00:00Z".into(), + schema_id: Some(schema.into()), + }) +} + +// ------------------------------------------------------------------ fake host + +struct FakeHost { + vars: BTreeMap, + pid: u32, + #[cfg(unix)] + descriptors: BTreeMap, +} + +impl FakeHost { + fn new() -> Self { + Self { + vars: BTreeMap::new(), + pid: 4242, + #[cfg(unix)] + descriptors: BTreeMap::new(), + } + } + + fn with(mut self, name: &str, value: &str) -> Self { + self.vars.insert(name.into(), value.into()); + self + } +} + +impl LaunchHost for FakeHost { + fn var(&self, name: &str) -> Option { + self.vars.get(name).cloned() + } + + fn remove_var(&mut self, name: &str) { + self.vars.remove(name); + } + + fn process_id(&self) -> u32 { + self.pid + } + + #[cfg(unix)] + fn adopt_descriptor(&mut self, descriptor: RawFd) -> Result { + self.descriptors + .remove(&descriptor) + .with_context(|| format!("descriptor {descriptor} is not open")) + } +} + +// ------------------------------------------------------------------------ rig + +/// One launch's worth of Idunn material written under a temp directory. +struct Rig { + dir: tempfile::TempDir, + expected: IdunnExpectedIncarnationRecord, + activation: IdunnRuntimeActivationRecord, + activation_seed: Vec, + provider_credential: Vec, + idunn: ServiceIdentitySigner, + provider_public_key: Vec, +} + +impl Rig { + fn new(write_lease_required: bool) -> Result { + let dir = tempfile::tempdir()?; + let provider_path = dir.path().join("provider.cc"); + let provider = + enroll_service_identity_at::(&provider_path)?; + let idunn = + enroll_service_identity_at::(&dir.path().join("idunn.cc"))?; + let provider_anchor = provider.trust_anchor()?; + let expected = IdunnExpectedIncarnationRecord { + schema_version: IDUNN_EXPECTED_INCARNATION_SCHEMA.into(), + target: TARGET.into(), + plan_id: digest('2'), + incarnation_id: "ghostlight/2026-09-02/1".into(), + sealed_release_id: digest('3'), + source_repository: "github.com/GameCult/Ghostlight".into(), + source_revision: "4".repeat(40), + recipe_sha256: digest('5'), + runtime_id: "ghostlight-yggdrasil-1".into(), + expected_signer_identity_id: provider_anchor.identity_id.clone(), + health_contract: HEALTH_CONTRACT.into(), + artifact_sha256: digest('6'), + state_schema_generation: Some("world-v2".into()), + state_contract_sha256: Some(digest('7')), + write_lease_required, + route: Some(IdunnExpectedRoute { + route_id: "ghostlight-public".into(), + transport: "tcp".into(), + stable_endpoint: "tcp://127.0.0.1:14100".into(), + candidate_endpoint: ENDPOINT.into(), + }), + capabilities: ["conversation", "state"] + .into_iter() + .map(|name| IdunnExpectedCapability { + capability: name.into(), + schema: format!("{name}.v1"), + compatibility: "v1".into(), + minimum_capacity: 1, + }) + .collect(), + dependencies: Vec::new(), + }; + let (activation, activation_seed) = Self::issue(&expected, &idunn)?; + let rig = Self { + provider_credential: std::fs::read(&provider_path)?, + provider_public_key: provider_anchor.public_key, + activation, + activation_seed, + expected, + idunn, + dir, + }; + std::fs::create_dir(rig.bundle())?; + rig.write_bundle(&rig.expected, &rig.activation)?; + Ok(rig) + } + + fn issue( + expected: &IdunnExpectedIncarnationRecord, + idunn: &ServiceIdentitySigner, + ) -> Result<(IdunnRuntimeActivationRecord, Vec)> { + let launch = IdunnRuntimeActivationLaunch::issue(expected, digest('8'), 99, idunn)?; + let mut seed = Vec::new(); + let activation = launch.write_credential(&mut seed)?; + Ok((activation, seed)) + } + + fn write_bundle( + &self, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result<()> { + write_record( + &self.bundle().join("expected.cc"), + &expected.target, + IdunnExpectedIncarnationRecord::TYPE, + IDUNN_EXPECTED_INCARNATION_SCHEMA, + expected.canonical_bytes()?, + )?; + write_record( + &self.bundle().join("activation.cc"), + &expected.target, + IdunnRuntimeActivationRecord::TYPE, + IDUNN_RUNTIME_ACTIVATION_SCHEMA, + activation.canonical_bytes()?, + ) + } + + fn bundle(&self) -> PathBuf { + self.dir.path().join("bundle") + } + + fn lease_path(&self) -> PathBuf { + self.dir.path().join("lease.cc") + } + + #[cfg(unix)] + fn lock_path(&self) -> PathBuf { + sibling_lock_path(&self.lease_path()).unwrap() + } + + fn credentials(&self) -> IdunnRuntimeCredentials { + IdunnRuntimeCredentials::new( + Cursor::new(self.activation_seed.clone()), + Cursor::new(self.provider_credential.clone()), + ) + } + + /// A launch environment that names only the bundle. + fn bare_host(&self) -> FakeHost { + FakeHost::new().with( + IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, + self.bundle().to_str().unwrap(), + ) + } + + /// The whole launch environment Idunn sets for this rig. + fn host(&self) -> FakeHost { + let host = self + .bare_host() + .with(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, BIND); + if self.expected.write_lease_required { + host.with( + IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT, + self.lease_path().to_str().unwrap(), + ) + } else { + host + } + } + + fn open_with( + &self, + host: &FakeHost, + target: &str, + health_contract: &str, + credentials: IdunnRuntimeCredentials, + ) -> Result { + IdunnRuntimeAuthority::open_in(host, target, health_contract, credentials) + } + + fn open(&self, host: &FakeHost) -> Result { + self.open_with(host, TARGET, HEALTH_CONTRACT, self.credentials()) + } + + fn authority(&self) -> Result { + self.open(&self.host()) + } + + fn signer(&self) -> Result { + Ok(IdunnRuntimePresenceSigner::new( + self.authority()?, + capabilities(), + )) + } + + fn verified(&self) -> Result { + verify_runtime_authority( + &self.expected, + &self.activation, + &self.idunn.trust_anchor()?, + &self.provider_public_key, + ) + } + + /// Idunn's lease record at `path`, naming `warming_sha256`. + #[cfg(unix)] + fn write_lease_at(&self, path: &Path, warming_sha256: &str, epoch: u64) -> Result<()> { + let lease = IdunnProcessWriteLeaseRecord { + schema_version: IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into(), + target: TARGET.into(), + expected_projection_sha256: self.expected.canonical_sha256()?, + plan_id: self.expected.plan_id.clone(), + incarnation_id: self.expected.incarnation_id.clone(), + sealed_release_id: self.expected.sealed_release_id.clone(), + activation_witness_sha256: self.activation.canonical_sha256()?, + state_schema_generation: "world-v2".into(), + state_contract_sha256: digest('7'), + runtime_id: self.expected.runtime_id.clone(), + runtime_instance_id: self.activation.runtime_instance_id.clone(), + warming_presence_sha256: warming_sha256.into(), + lease_epoch: epoch, + issued_at_unix_millis: 101, + }; + write_record( + path, + TARGET, + IdunnProcessWriteLeaseRecord::TYPE, + IDUNN_PROCESS_WRITE_LEASE_SCHEMA, + lease.canonical_bytes()?, + ) + } + + #[cfg(unix)] + fn write_lease(&self, warming_sha256: &str, epoch: u64) -> Result<()> { + self.write_lease_at(&self.lease_path(), warming_sha256, epoch) + } +} + +/// Runs `presence` through the shared verifier exactly as Odin would. +fn assert_odin_accepts(rig: &Rig, presence: &SignedRuntimePresence, now: u64) -> Result<()> { + let authority = rig.verified()?; + let claim = authenticate_runtime_presence_claim( + &presence.document.payload, + &authority, + RuntimePresenceAuthenticationContext { + trusted_received_at_unix_millis: now, + maximum_age_millis: 100, + maximum_future_skew_millis: 5, + }, + )?; + assert_eq!(claim.record(), &presence.record); + let verified = + correlate_runtime_presence_claim(claim, &authority)?.into_undisputed_present()?; + assert_eq!(verified.signed_presence_sha256(), presence.canonical_sha256); + Ok(()) +} + +// ---------------------------------------------------------------- signer tests + +#[test] +fn signed_presence_authenticates_and_correlates() -> Result<()> { + let rig = Rig::new(false)?; + let mut signer = rig.signer()?; + let warming = signer.sign_at("starting", 1_000)?; + signer.report_health(RuntimePresenceState::Active); + let active = signer.sign_at("ready", 1_001)?; + + for (presence, state, detail) in [ + (&warming, "warming", "starting"), + (&active, "active", "ready"), + ] { + assert_odin_accepts(&rig, presence, 1_010)?; + assert_eq!(presence.record.state, state); + assert_eq!(presence.record.detail, detail); + assert_eq!(presence.record.bound_endpoint.as_deref(), Some(ENDPOINT)); + assert_eq!( + presence.document.payload, + rmp_serde::to_vec(&presence.record)? + ); + assert_eq!( + presence.canonical_sha256, + presence.record.canonical_sha256()? + ); + assert_eq!( + presence.document.schema_id, + GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA + ); + assert_eq!(presence.document.record_key, TARGET); + assert_eq!( + presence.document.source_runtime_id.as_deref(), + Some("ghostlight-yggdrasil-1") + ); + assert_eq!( + presence.document.source_agent_id.as_deref(), + Some(presence.record.signer_identity_id.as_str()) + ); + assert_eq!( + presence.document.source_role.as_deref(), + Some(DOCUMENT_ROLE) + ); + assert_eq!( + presence.document.tags.as_deref(), + Some( + &[ + "cultnet.transport.rudp.v0".to_owned(), + "runtime-presence".to_owned() + ][..] + ) + ); + } + assert_eq!(warming.document.stored_at, "1970-01-01T00:00:01+00:00"); + assert_eq!(signer.authority().expected().target, TARGET); + assert_eq!(signer.authority().activation(), &rig.activation); + assert_eq!(signer.authority().bound_endpoint(), Some(ENDPOINT)); + Ok(()) +} + +#[test] +fn sequence_strictly_increases_and_health_is_signer_owned() -> Result<()> { + let rig = Rig::new(false)?; + let mut signer = rig.signer()?; + assert_eq!(signer.health(), RuntimePresenceState::Warming); + let first = signer.sign_at("a", 1_000)?; + assert_eq!( + (first.record.state.as_str(), first.record.publisher_sequence), + ("warming", 1) + ); + let mut sequence = 1; + for (health, text) in [ + (RuntimePresenceState::Active, "active"), + (RuntimePresenceState::Degraded, "degraded"), + (RuntimePresenceState::Failed, "failed"), + (RuntimePresenceState::Warming, "warming"), + ] { + signer.report_health(health); + assert_eq!(signer.health(), health); + let signed = signer.sign_at("b", 1_000)?; + sequence += 1; + assert_eq!(signed.record.state, text); + assert_eq!(signed.record.publisher_sequence, sequence); + assert_odin_accepts(&rig, &signed, 1_010)?; + } + Ok(()) +} + +#[test] +fn detail_is_cut_to_512_bytes_on_a_character_boundary() -> Result<()> { + let rig = Rig::new(false)?; + let mut signer = rig.signer()?; + let exact = "x".repeat(512); + assert_eq!(signer.sign_at(&exact, 1_000)?.record.detail, exact); + let over = "x".repeat(513); + assert_eq!(signer.sign_at(&over, 1_000)?.record.detail, exact); + let wide = "é".repeat(400); + assert_eq!(signer.sign_at(&wide, 1_000)?.record.detail, "é".repeat(256)); + let odd = format!("x{}", "é".repeat(400)); + assert_eq!( + signer.sign_at(&odd, 1_000)?.record.detail, + format!("x{}", "é".repeat(255)) + ); + Ok(()) +} + +#[cfg(unix)] +#[test] +fn warming_never_carries_lease_and_active_requires_held_lease() -> Result<()> { + let rig = Rig::new(true)?; + let mut signer = rig.signer()?; + let warming = signer.sign_at("awaiting-process-write-lease", 1_000)?; + assert_eq!(warming.record.write_lease_sha256, None); + + signer.report_health(RuntimePresenceState::Active); + let refused = refusal(signer.sign_at("ready", 1_001)); + assert!( + refused.contains("lacks its process-write lease"), + "{refused}" + ); + // Degraded needs the lease as Active does, whatever the history. + signer.report_health(RuntimePresenceState::Degraded); + let refused = refusal(signer.sign_at("serving reduced", 1_001)); + assert!( + refused.contains("degraded stateful presence lacks its process-write lease"), + "{refused}" + ); + // Failed is the honest report and needs none. + signer.report_health(RuntimePresenceState::Failed); + assert_eq!( + signer + .sign_at("no lease yet", 1_001)? + .record + .write_lease_sha256, + None + ); + + rig.write_lease(&warming.canonical_sha256, 1)?; + let guard = signer + .try_acquire_write_lease()? + .expect("the lease binds a warming proof"); + signer.report_health(RuntimePresenceState::Active); + let active = signer.sign_at("ready", 1_002)?; + assert_eq!( + active.record.write_lease_sha256.as_deref(), + Some(guard.lease_sha256()) + ); + assert_odin_accepts(&rig, &active, 1_010)?; + + signer.report_health(RuntimePresenceState::Degraded); + assert_eq!( + signer + .sign_at("slow", 1_003)? + .record + .write_lease_sha256 + .as_deref(), + Some(guard.lease_sha256()) + ); + signer.report_health(RuntimePresenceState::Warming); + assert_eq!( + signer.sign_at("again", 1_004)?.record.write_lease_sha256, + None + ); + Ok(()) +} + +#[test] +fn a_runtime_without_a_lease_requirement_signs_active_and_refuses_a_lease() -> Result<()> { + let rig = Rig::new(false)?; + let mut signer = rig.signer()?; + signer.report_health(RuntimePresenceState::Active); + assert_eq!( + signer.sign_at("ready", 1_000)?.record.write_lease_sha256, + None + ); + let refused = refusal(signer.try_acquire_write_lease()); + assert!(refused.contains("does not require"), "{refused}"); + Ok(()) +} + +#[cfg(unix)] +#[test] +fn lease_must_bind_a_recent_warming_proof_of_this_signer() -> Result<()> { + let rig = Rig::new(true)?; + let mut signer = rig.signer()?; + assert!(signer.try_acquire_write_lease()?.is_none(), "no record yet"); + + let mut warming = Vec::new(); + for index in 0..65 { + warming.push( + signer + .sign_at(&format!("heartbeat {index}"), 1_000)? + .canonical_sha256, + ); + } + signer.report_health(RuntimePresenceState::Failed); + let not_warming = signer.sign_at("failed", 1_000)?.canonical_sha256; + + // A proof some other signer signed, a presence that was never Warming and a + // Warming proof pushed out of the last 64 are all refused. + for (epoch, sha) in [(1, digest('d')), (2, not_warming), (3, warming[0].clone())] { + rig.write_lease(&sha, epoch)?; + let refused = refusal(signer.try_acquire_write_lease()); + assert!( + refused.contains("provider-owned Warming proof"), + "{refused}" + ); + } + + // The oldest proof still inside the window is accepted. + rig.write_lease(&warming[1], 4)?; + assert!(signer.try_acquire_write_lease()?.is_some()); + Ok(()) +} + +#[cfg(unix)] +#[test] +fn a_lease_for_another_runtime_instance_is_refused() -> Result<()> { + let rig = Rig::new(true)?; + let mut signer = rig.signer()?; + let warming = signer.sign_at("w", 1_000)?; + rig.write_lease(&warming.canonical_sha256, 1)?; + let stored = SingleFileMessagePackBackingStore::new(rig.lease_path()) + .pull_all()? + .remove(0); + let mut lease = IdunnProcessWriteLeaseRecord::decode_canonical(&stored.payload)?; + lease.runtime_instance_id = digest('1'); + write_record( + &rig.lease_path(), + TARGET, + IdunnProcessWriteLeaseRecord::TYPE, + IDUNN_PROCESS_WRITE_LEASE_SCHEMA, + lease.canonical_bytes()?, + )?; + let refused = refusal(signer.try_acquire_write_lease()); + assert!( + refused.contains("provider-owned Warming proof"), + "{refused}" + ); + Ok(()) +} + +#[cfg(unix)] +#[test] +fn lease_guard_holds_shared_lock_and_detects_replacement() -> Result<()> { + let rig = Rig::new(true)?; + let mut signer = rig.signer()?; + let warming = signer.sign_at("w", 1_000)?; + rig.write_lease(&warming.canonical_sha256, 1)?; + let guard = signer.try_acquire_write_lease()?.unwrap(); + assert_eq!( + signer.try_acquire_write_lease()?.unwrap().lease_sha256(), + guard.lease_sha256(), + "acquiring again returns the held lease" + ); + guard.require_current()?; + + let probe = OpenOptions::new().read(true).open(rig.lock_path())?; + assert!( + fs2::FileExt::try_lock_exclusive(&probe).is_err(), + "a guard holds the lock shared" + ); + fs2::FileExt::try_lock_shared(&probe)?; + fs2::FileExt::unlock(&probe)?; + + // The lock outlives every clone but the last. + let clone = guard.clone(); + drop(guard); + assert!(fs2::FileExt::try_lock_exclusive(&probe).is_err()); + + // A writer that renames a new record into place is detected. + let replacement = rig.dir.path().join("replacement.cc"); + rig.write_lease_at(&replacement, &warming.canonical_sha256, 2)?; + std::fs::rename(&replacement, rig.lease_path())?; + let stale = refusal(clone.require_current()); + assert!(stale.contains("no longer the admitted"), "{stale}"); + signer.report_health(RuntimePresenceState::Active); + assert!( + signer.sign_at("ready", 1_001).is_err(), + "a stale lease signs nothing but Warming" + ); + + drop(clone); + drop(signer); + fs2::FileExt::try_lock_exclusive(&probe)?; + Ok(()) +} + +#[cfg(unix)] +#[test] +fn a_removed_lease_record_is_not_current() -> Result<()> { + let rig = Rig::new(true)?; + let mut signer = rig.signer()?; + let warming = signer.sign_at("w", 1_000)?; + rig.write_lease(&warming.canonical_sha256, 1)?; + let guard = signer.try_acquire_write_lease()?.unwrap(); + std::fs::remove_file(rig.lease_path())?; + let stale = refusal(guard.require_current()); + assert!(stale.contains("no longer the admitted"), "{stale}"); + Ok(()) +} + +#[cfg(unix)] +#[test] +fn a_lost_lease_still_signs_failed() -> Result<()> { + let rig = Rig::new(true)?; + let mut signer = rig.signer()?; + let warming = signer.sign_at("warming", 1_000)?; + rig.write_lease(&warming.canonical_sha256, 1)?; + let guard = signer.try_acquire_write_lease()?.expect("the lease binds"); + signer.report_health(RuntimePresenceState::Active); + let held = signer.sign_at("held", 1_001)?; + assert_eq!( + held.record.write_lease_sha256.as_deref(), + Some(guard.lease_sha256()) + ); + // While the lease is current, Failed carries its digest like any other state. + signer.report_health(RuntimePresenceState::Failed); + assert_eq!( + signer + .sign_at("failing", 1_001)? + .record + .write_lease_sha256 + .as_deref(), + Some(guard.lease_sha256()) + ); + + // Idunn withdraws the lease: Active and Degraded can no longer be claimed. + std::fs::remove_file(rig.lease_path())?; + for state in [RuntimePresenceState::Active, RuntimePresenceState::Degraded] { + signer.report_health(state); + let refused = refusal(signer.sign_at("lost", 1_002)); + assert!( + refused.contains("no longer the admitted"), + "{state:?}: {refused}" + ); + } + // Failed is the honest report, and it carries no lease digest. + signer.report_health(RuntimePresenceState::Failed); + let failed = signer.sign_at("lease lost", 1_003)?; + assert_eq!(failed.record.state, "failed"); + assert_eq!(failed.record.write_lease_sha256, None); + let authority = rig.verified()?; + let claim = authenticate_runtime_presence_claim( + &failed.document.payload, + &authority, + RuntimePresenceAuthenticationContext { + trusted_received_at_unix_millis: 1_010, + maximum_age_millis: 100, + maximum_future_skew_millis: 5, + }, + )?; + correlate_runtime_presence_claim(claim, &authority)?; + // Warming still signs, as before. + signer.report_health(RuntimePresenceState::Warming); + signer.sign_at("again", 1_004)?; + Ok(()) +} + +// ------------------------------------------------------------- authority tests + +#[test] +fn authority_open_refuses_mismatches() -> Result<()> { + let rig = Rig::new(false)?; + rig.authority()?; + + let refused = refusal(rig.open_with(&rig.host(), "other", HEALTH_CONTRACT, rig.credentials())); + assert!(refused.contains("target or health contract"), "{refused}"); + let refused = refusal(rig.open_with(&rig.host(), TARGET, "other.v1", rig.credentials())); + assert!(refused.contains("target or health contract"), "{refused}"); + + let mut wrong_seed = rig.activation_seed.clone(); + wrong_seed[0] ^= 1; + let credentials = IdunnRuntimeCredentials::new( + Cursor::new(wrong_seed), + Cursor::new(rig.provider_credential.clone()), + ); + let refused = refusal(rig.open_with(&rig.host(), TARGET, HEALTH_CONTRACT, credentials)); + assert!( + refused.contains("activation credential does not belong"), + "{refused}" + ); + + let other = tempfile::tempdir()?; + enroll_service_identity_at::(&other.path().join("p.cc"))?; + let credentials = IdunnRuntimeCredentials::new( + Cursor::new(rig.activation_seed.clone()), + Cursor::new(std::fs::read(other.path().join("p.cc"))?), + ); + let refused = refusal(rig.open_with(&rig.host(), TARGET, HEALTH_CONTRACT, credentials)); + assert!( + refused.contains("not the identity selected by Expected"), + "{refused}" + ); + + // A lease environment on an incarnation that needs none is refused. + let lease_env = rig + .host() + .with(IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT, "/run/lease.cc"); + let refused = refusal(rig.open(&lease_env)); + assert!( + refused.contains("write-lease environment does not match"), + "{refused}" + ); + let empty = rig.host().with(IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT, ""); + assert!(refusal(rig.open(&empty)).contains("empty or has surrounding whitespace")); + Ok(()) +} + +#[cfg(unix)] +#[test] +fn a_lease_requirement_needs_the_lease_environment() -> Result<()> { + let rig = Rig::new(true)?; + rig.authority()?; + let no_lease_env = rig + .bare_host() + .with(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, BIND); + let refused = refusal(rig.open(&no_lease_env)); + assert!( + refused.contains("write-lease environment does not match"), + "{refused}" + ); + Ok(()) +} + +#[cfg(not(unix))] +#[test] +fn non_unix_open_refuses_a_write_lease_requirement() -> Result<()> { + let rig = Rig::new(true)?; + let refused = refusal(rig.authority()); + assert!(refused.contains("write_lease_required"), "{refused}"); + assert!( + !refused.contains(rig.dir.path().to_str().unwrap()), + "{refused}" + ); + Ok(()) +} + +#[test] +fn open_reads_the_whole_launch_environment() -> Result<()> { + let rig = Rig::new(false)?; + // The bundle comes only from the launch environment: with it missing, open has + // nowhere to look, whatever else is set. + let no_bundle = FakeHost::new().with(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, BIND); + let refused = refusal(rig.open(&no_bundle)); + assert!( + refused.contains(IDUNN_RUNTIME_BUNDLE_ENVIRONMENT), + "{refused}" + ); + let relative = rig + .host() + .with(IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, "relative/bundle"); + let refused = refusal(rig.open(&relative)); + assert!(refused.contains("not an absolute path"), "{refused}"); + // The bundle, the candidate bind and (below, unix) the write lease are all + // read through the one host port. + let authority = rig.open(&rig.host())?; + authority.require_candidate_bind(BIND)?; + Ok(()) +} + +#[test] +fn the_launch_contract_names_are_the_public_spellings() { + assert_eq!( + IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, + "GAMECULT_IDUNN_RUNTIME_BUNDLE" + ); + assert_eq!( + IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, + "GAMECULT_IDUNN_CANDIDATE_BIND" + ); + assert_eq!( + IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT, + "GAMECULT_IDUNN_PROCESS_WRITE_LEASE" + ); + assert_eq!( + IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT, + "GAMECULT_IDUNN_ACTIVATION_CREDENTIAL" + ); + assert_eq!( + RUNTIME_PRESENCE_IDENTITY_BINDING, + "GAMECULT_RUNTIME_PRESENCE_IDENTITY" + ); + assert_eq!( + RUNTIME_PRESENCE_IDENTITY_FD_NAME, + "gamecult-runtime-presence-identity" + ); +} + +#[test] +fn authority_open_refuses_an_activation_for_another_expected() -> Result<()> { + let rig = Rig::new(false)?; + let mut other_expected = rig.expected.clone(); + other_expected.plan_id = digest('9'); + let (foreign, _) = Rig::issue(&other_expected, &rig.idunn)?; + write_record( + &rig.bundle().join("activation.cc"), + TARGET, + IdunnRuntimeActivationRecord::TYPE, + IDUNN_RUNTIME_ACTIVATION_SCHEMA, + foreign.canonical_bytes()?, + )?; + let refused = refusal(rig.authority()); + assert!( + refused.contains("does not bind the bundled Expected"), + "{refused}" + ); + Ok(()) +} + +#[test] +fn authority_open_refuses_a_substituted_record_key() -> Result<()> { + let rig = Rig::new(false)?; + let substitute = rig.dir.path().join("substitute.cc"); + write_record( + &substitute, + "someone-else", + IdunnExpectedIncarnationRecord::TYPE, + IDUNN_EXPECTED_INCARNATION_SCHEMA, + rig.expected.canonical_bytes()?, + )?; + std::fs::rename(&substitute, rig.bundle().join("expected.cc"))?; + let refused = refusal(rig.authority()); + assert!(refused.contains("key is not its target"), "{refused}"); + Ok(()) +} + +#[test] +fn candidate_bind_must_equal_expected_route() -> Result<()> { + let rig = Rig::new(false)?; + let authority = rig.authority()?; + authority.require_candidate_bind(BIND)?; + assert!(authority.require_candidate_bind("127.0.0.1:14104").is_err()); + assert!(authority.require_candidate_bind(ENDPOINT).is_err()); + assert!(authority.require_candidate_bind("").is_err()); + assert_eq!(authority.bound_endpoint(), Some(ENDPOINT)); + + // Idunn's environment must itself name Expected's route. + let bind = |value: &str| { + rig.bare_host() + .with(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, value) + }; + assert!(refusal(rig.open(&bind("127.0.0.1:14104"))).contains("candidate bind differs")); + let missing = rig.bare_host(); + assert!(refusal(rig.open(&missing)).contains("candidate bind environment")); + assert!(refusal(rig.open(&bind(" 127.0.0.1:14103"))).contains("surrounding whitespace")); + Ok(()) +} + +#[test] +fn candidate_bind_suffix_is_refused_at_open() -> Result<()> { + let rig = Rig::new(false)?; + // Idunn passes the bare host:port; only the whole of Expected's endpoint + // after `tcp://` is that bind. A suffix or prefix of it is not. + for wrong in [ + "0.0.1:14103", + "1:14103", + "127.0.0.1:1410", + "tcp://127.0.0.1:14103", + ] { + let host = rig + .bare_host() + .with(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, wrong); + let refused = refusal(rig.open(&host)); + assert!( + refused.contains("candidate bind differs"), + "{wrong}: {refused}" + ); + } + let authority = rig.authority()?; + assert!(authority.require_candidate_bind("0.0.1:14103").is_err()); + Ok(()) +} + +#[test] +fn a_route_less_incarnation_has_no_bind_to_check() -> Result<()> { + let mut rig = Rig::new(false)?; + rig.expected.route = None; + let (activation, seed) = Rig::issue(&rig.expected, &rig.idunn)?; + rig.activation = activation; + rig.activation_seed = seed; + rig.write_bundle(&rig.expected, &rig.activation)?; + let authority = rig.open(&rig.bare_host())?; + assert_eq!(authority.bound_endpoint(), None); + assert!(authority.require_candidate_bind(BIND).is_err()); + assert!(refusal(rig.open(&rig.host())).contains("candidate bind environment")); + let mut signer = IdunnRuntimePresenceSigner::new(authority, capabilities()); + assert_eq!(signer.sign_at("w", 1_000)?.record.bound_endpoint, None); + Ok(()) +} + +// --------------------------------------------------------------- route answer + +fn strings(items: &[&str]) -> Vec { + items.iter().map(|item| (*item).to_owned()).collect() +} + +fn request(id: &str, schemas: Option<&[&str]>, keys: Option<&[&str]>) -> CultNetMessage { + CultNetMessage::SnapshotRequest { + message_id: id.into(), + schema_ids: schemas.map(strings), + record_keys: keys.map(strings), + } +} + +fn presence_in(answer: CultNetMessage) -> GameCultRuntimePresenceHealthRecord { + let CultNetMessage::SnapshotResponseRaw { documents, .. } = answer else { + panic!("a snapshot response"); + }; + let [document] = documents.as_slice() else { + panic!("one presence document"); + }; + rmp_serde::from_slice(&document.payload).unwrap() +} + +#[test] +fn route_observation_refuses_foreign_requests() -> Result<()> { + let rig = Rig::new(false)?; + let mut signer = rig.signer()?; + signer.report_health(RuntimePresenceState::Degraded); + let schema = GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA; + + let answer = + signer.answer_route_observation(&request("m-1", Some(&[schema]), Some(&[TARGET])))?; + let CultNetMessage::SnapshotResponseRaw { message_id, .. } = &answer else { + panic!("a snapshot response"); + }; + assert_eq!(message_id, "m-1"); + let record = presence_in(answer); + assert_eq!( + record.state, "degraded", + "the answer states the reported health" + ); + assert_eq!(record.detail, "route-observation:m-1"); + assert_eq!(record.publisher_sequence, 1); + // An open request and a request listing more than this runtime are fine. + signer.answer_route_observation(&request("m-2", None, None))?; + signer.answer_route_observation(&request("m-3", Some(&["x", schema]), Some(&["y", TARGET])))?; + + for refused in [ + request("m", Some(&["other.schema.v1"]), None), + request("m", None, Some(&["another-target"])), + request("m", Some(&[schema]), Some(&["another-target"])), + request("m", Some(&["other.schema.v1"]), Some(&[TARGET])), + request("", None, None), + request(" m", None, None), + request("m ", None, None), + CultNetMessage::SnapshotResponseRaw { + message_id: "m".into(), + documents: Vec::new(), + }, + ] { + assert!( + signer.answer_route_observation(&refused).is_err(), + "{refused:?}" + ); + } + // Refusals sign nothing: the next answer takes the next sequence. + let record = presence_in(signer.answer_route_observation(&request("m-4", None, None))?); + assert_eq!(record.publisher_sequence, 4); + Ok(()) +} + +#[cfg(unix)] +#[test] +fn route_observation_answers_in_lease_discipline() -> Result<()> { + let rig = Rig::new(true)?; + let mut signer = rig.signer()?; + signer.report_health(RuntimePresenceState::Active); + assert!( + signer + .answer_route_observation(&request("m", None, None)) + .is_err() + ); + Ok(()) +} + +// -------------------------------------------------------- descriptor contract + +#[cfg(unix)] +fn clear_cloexec(file: &File) { + // SAFETY: fcntl on a descriptor `file` owns. + assert_ne!( + unsafe { libc::fcntl(file.as_raw_fd(), libc::F_SETFD, 0) }, + -1 + ); +} + +#[cfg(unix)] +fn set_cloexec(file: &File) { + // SAFETY: fcntl on a descriptor `file` owns. + assert_ne!( + unsafe { libc::fcntl(file.as_raw_fd(), libc::F_SETFD, libc::FD_CLOEXEC) }, + -1 + ); +} + +#[cfg(unix)] +fn is_cloexec(file: &File) -> bool { + // SAFETY: fcntl on a descriptor `file` owns. + let flags = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_GETFD) }; + flags != -1 && flags & libc::FD_CLOEXEC != 0 +} + +#[cfg(unix)] +fn inherited(dir: &Path, name: &str, content: &[u8]) -> Result { + std::fs::write(dir.join(name), content)?; + let file = File::open(dir.join(name))?; + clear_cloexec(&file); + assert!(!is_cloexec(&file)); + Ok(file) +} + +#[cfg(unix)] +fn systemd_host(dir: &Path, listen_pid: &str, fds: &str, names: &str) -> Result { + let mut host = FakeHost::new() + .with(LISTEN_PID_ENVIRONMENT, listen_pid) + .with(LISTEN_FDS_ENVIRONMENT, fds) + .with(LISTEN_FDNAMES_ENVIRONMENT, names); + for (index, name) in names.split(':').enumerate() { + let descriptor = LISTEN_FDS_START + index as RawFd; + host.descriptors.insert( + descriptor, + inherited(dir, &format!("fd-{index}"), name.as_bytes())?, + ); + } + Ok(host) +} + +#[cfg(unix)] +fn contents(mut file: File) -> Result { + let mut text = String::new(); + file.read_to_string(&mut text)?; + Ok(text) +} + +#[cfg(unix)] +#[test] +fn systemd_descriptor_contract() -> Result<()> { + let dir = tempfile::tempdir()?; + let names = format!("socket:{ACTIVATION_NAME}:extra:{PROVIDER_NAME}:more"); + + // Found by name; extra descriptors are tolerated and left alone. + let mut host = systemd_host(dir.path(), "4242", "5", &names)?; + // A descriptor that already carries close-on-exec keeps it. + set_cloexec(&host.descriptors[&(LISTEN_FDS_START + 3)]); + let (activation, provider) = take_systemd_descriptors(&mut host, Custody::Profile)?; + assert!( + is_cloexec(&activation) && is_cloexec(&provider), + "signers never reach a child" + ); + assert_eq!(contents(activation)?, ACTIVATION_NAME); + assert_eq!(contents(provider)?, PROVIDER_NAME); + assert!(host.vars.is_empty(), "LISTEN_* leave the environment"); + assert_eq!(host.descriptors.len(), 3, "only the two signers were taken"); + + // Order does not matter. + let swapped = format!("{PROVIDER_NAME}:{ACTIVATION_NAME}"); + let mut host = systemd_host(dir.path(), "4242", "2", &swapped)?; + let (activation, provider) = take_systemd_descriptors(&mut host, Custody::Profile)?; + assert_eq!(contents(activation)?, ACTIVATION_NAME); + assert_eq!(contents(provider)?, PROVIDER_NAME); + + // A claim for another process is refused and still cleared from the environment. + let both = format!("{ACTIVATION_NAME}:{PROVIDER_NAME}"); + let mut host = systemd_host(dir.path(), "999", "2", &both)?; + assert!( + refusal(take_systemd_descriptors(&mut host, Custody::Profile)) + .contains("different process") + ); + assert!(host.vars.is_empty()); + + // PrivatePIDs: pid 1 accepts the host's LISTEN_PID, a descendant does not. + let mut host = systemd_host(dir.path(), "777", "2", &both)?; + host.pid = 1; + take_systemd_descriptors(&mut host, Custody::Profile)?; + let mut host = systemd_host(dir.path(), "777", "2", &both)?; + host.pid = 2; + assert!(take_systemd_descriptors(&mut host, Custody::Profile).is_err()); + Ok(()) +} + +#[cfg(unix)] +#[test] +fn systemd_listen_pid_rule() { + assert!(listen_pid_matches("4242", 4242)); + assert!(!listen_pid_matches("4243", 4242)); + // PrivatePIDs: the main process is pid 1 and cannot see the host pid. + assert!(listen_pid_matches("777", 1)); + assert!(listen_pid_matches("1", 1)); + assert!( + !listen_pid_matches("777", 2), + "a descendant must not reuse the set" + ); + assert!(!listen_pid_matches("0", 1)); + // The text must be the canonical number. + assert!(!listen_pid_matches("04242", 4242)); + assert!(!listen_pid_matches("+4242", 4242)); + assert!(!listen_pid_matches("", 1)); + assert!(!listen_pid_matches("abc", 1)); +} + +#[cfg(unix)] +#[test] +fn systemd_descriptor_contract_refuses_malformed_claims() -> Result<()> { + let dir = tempfile::tempdir()?; + let both = format!("{ACTIVATION_NAME}:{PROVIDER_NAME}"); + let only_activation = ACTIVATION_NAME.to_owned(); + let doubled = format!("{ACTIVATION_NAME}:{PROVIDER_NAME}:{PROVIDER_NAME}"); + for (fds, names, expect) in [ + ( + "2", + only_activation.as_str(), + "LISTEN_FDNAMES does not name", + ), + ( + "1", + only_activation.as_str(), + "did not pass the gamecult-runtime-presence-identity", + ), + ("3", doubled.as_str(), "more than once"), + ("2", "a:b", "did not pass the gamecult-idunn"), + ("02", both.as_str(), "LISTEN_FDS"), + ("x", both.as_str(), "LISTEN_FDS"), + ("3", both.as_str(), "does not name every"), + ] { + let mut host = systemd_host(dir.path(), "4242", "2", &both)?; + host.vars.insert(LISTEN_FDS_ENVIRONMENT.into(), fds.into()); + host.vars + .insert(LISTEN_FDNAMES_ENVIRONMENT.into(), names.into()); + let refused = refusal(take_systemd_descriptors(&mut host, Custody::Profile)); + assert!(refused.contains(expect), "{names}: {refused}"); + assert!(host.vars.is_empty()); + } + for missing in [ + LISTEN_PID_ENVIRONMENT, + LISTEN_FDS_ENVIRONMENT, + LISTEN_FDNAMES_ENVIRONMENT, + ] { + let mut host = systemd_host(dir.path(), "4242", "2", &both)?; + host.vars.remove(missing); + assert!(refusal(take_systemd_descriptors(&mut host, Custody::Profile)).contains(missing)); + assert!(host.vars.is_empty(), "the rest of the claim is removed too"); + } + // A descriptor the process does not have open is refused. + let mut host = systemd_host(dir.path(), "4242", "2", &both)?; + host.descriptors.clear(); + assert!(refusal(take_systemd_descriptors(&mut host, Custody::Profile)).contains("not open")); + Ok(()) +} + +#[cfg(unix)] +#[test] +fn systemd_descriptors_must_be_read_only() -> Result<()> { + let dir = tempfile::tempdir()?; + let both = format!("{ACTIVATION_NAME}:{PROVIDER_NAME}"); + let mut host = systemd_host(dir.path(), "4242", "2", &both)?; + let writable = OpenOptions::new() + .read(true) + .write(true) + .open(dir.path().join("fd-1"))?; + host.descriptors.insert(LISTEN_FDS_START + 1, writable); + let refused = refusal(take_systemd_descriptors(&mut host, Custody::Profile)); + assert!(refused.contains("not read-only"), "{refused}"); + Ok(()) +} + +#[cfg(unix)] +#[test] +fn write_only_pipe_descriptor_is_not_read_only() -> Result<()> { + let mut fds = [0 as libc::c_int; 2]; + // SAFETY: pipe writes two descriptors into the array. + assert_eq!(unsafe { libc::pipe(fds.as_mut_ptr()) }, 0); + // SAFETY: both descriptors were just created and are owned exactly once. + let (read_end, write_end) = unsafe { (File::from_raw_fd(fds[0]), File::from_raw_fd(fds[1])) }; + // SAFETY: F_GETFL reads the flags of a descriptor `write_end` owns. + let status = unsafe { libc::fcntl(write_end.as_raw_fd(), libc::F_GETFL) }; + assert_eq!( + status & libc::O_ACCMODE, + libc::O_WRONLY, + "a valid status of 1" + ); + // F_GETFL succeeds here, so the refusal is the access mode and not a failed read. + let refused = refusal(protect_descriptor( + write_end, + "probe signer", + Custody::Profile, + )); + assert!( + refused.contains("probe signer descriptor is not read-only"), + "{refused}" + ); + assert!( + protect_descriptor(read_end, "probe signer", Custody::Profile).is_ok(), + "the read end of the same pipe is read-only" + ); + Ok(()) +} + +#[cfg(unix)] +#[test] +fn systemd_descriptors_pass_the_root_custody_rule_or_are_refused() -> Result<()> { + let dir = tempfile::tempdir()?; + let both = format!("{ACTIVATION_NAME}:{PROVIDER_NAME}"); + // The temp files are mode 0644, never root's 0400. + let mut host = systemd_host(dir.path(), "4242", "2", &both)?; + let refused = refusal(take_systemd_descriptors(&mut host, Custody::Root)); + assert!(refused.contains("root-owned 0400"), "{refused}"); + // The public delivery is root custody, with no way to ask for less. + let mut host = systemd_host(dir.path(), "4242", "2", &both)?; + let refused = refusal(IdunnRuntimeCredentials::from_systemd_descriptors_in( + &mut host, + )); + assert!(refused.contains("root-owned 0400"), "{refused}"); + Ok(()) +} + +#[test] +fn custody_follows_delivery() -> Result<()> { + let rig = Rig::new(false)?; + assert_eq!(rig.credentials().custody, Custody::Profile, "new"); + + let dir = tempfile::tempdir()?; + let seed = dir.path().join("activation-key"); + let identity = dir.path().join("identity.cc"); + std::fs::write(&seed, &rig.activation_seed)?; + std::fs::write(&identity, &rig.provider_credential)?; + let host = FakeHost::new() + .with( + IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT, + seed.to_str().unwrap(), + ) + .with( + RUNTIME_PRESENCE_IDENTITY_BINDING, + identity.to_str().unwrap(), + ); + assert_eq!( + IdunnRuntimeCredentials::from_host_environment_in(&host)?.custody, + Custody::Profile, + "host files" + ); + + // Profile custody takes the real thing and nothing else, on every platform. + for (kind, rule, accepted) in [ + (FileKind::Directory, Rule::ReadOnlyDirectory, true), + (FileKind::Regular, Rule::ReadOnlyFile, true), + (FileKind::Regular, Rule::ControlledFile, true), + (FileKind::Symlink, Rule::ReadOnlyDirectory, false), + (FileKind::Symlink, Rule::ReadOnlyFile, false), + (FileKind::Symlink, Rule::ControlledFile, false), + (FileKind::Other, Rule::ReadOnlyFile, false), + (FileKind::Regular, Rule::ReadOnlyDirectory, false), + (FileKind::Directory, Rule::ReadOnlyFile, false), + ] { + assert_eq!(profile_accepts(kind, rule), accepted, "{kind:?} {rule:?}"); + } + Ok(()) +} + +#[test] +fn profile_custody_reads_the_real_filesystem() -> Result<()> { + let dir = tempfile::tempdir()?; + let file = dir.path().join("record.cc"); + std::fs::write(&file, b"x")?; + let custody = Custody::Profile; + custody.require_file(&file, "record", Rule::ReadOnlyFile)?; + custody.require_file(dir.path(), "dir", Rule::ReadOnlyDirectory)?; + // The wrong kind, an absent path and a relative path are refused. + let refused = refusal(custody.require_file(dir.path(), "dir", Rule::ReadOnlyFile)); + assert!(refused.contains("dir is not a regular file"), "{refused}"); + assert!( + custody + .require_file(&file, "record", Rule::ReadOnlyDirectory) + .is_err() + ); + assert!( + custody + .require_file(&dir.path().join("absent"), "absent", Rule::ReadOnlyFile) + .is_err() + ); + assert!( + custody + .require_file(Path::new("record.cc"), "record", Rule::ReadOnlyFile) + .is_err() + ); + assert_eq!( + FileKind::of(&std::fs::symlink_metadata(&file)?), + FileKind::Regular + ); + assert_eq!( + FileKind::of(&std::fs::symlink_metadata(dir.path())?), + FileKind::Directory + ); + Ok(()) +} + +#[cfg(unix)] +#[test] +fn profile_custody_refuses_a_symlink() -> Result<()> { + let dir = tempfile::tempdir()?; + let file = dir.path().join("record.cc"); + std::fs::write(&file, b"x")?; + let link = dir.path().join("link.cc"); + std::os::unix::fs::symlink(&file, &link)?; + let refused = refusal(Custody::Profile.require_file(&link, "record", Rule::ReadOnlyFile)); + assert!(refused.contains("a link is refused"), "{refused}"); + assert_eq!( + FileKind::of(&std::fs::symlink_metadata(&link)?), + FileKind::Symlink + ); + let dir_link = dir.path().join("dir-link"); + std::os::unix::fs::symlink(dir.path(), &dir_link)?; + assert!( + Custody::Profile + .require_file(&dir_link, "dir", Rule::ReadOnlyDirectory) + .is_err() + ); + // A bundle that is a symlink is refused by open, not followed. + let rig = Rig::new(false)?; + let bundle_link = rig.dir.path().join("bundle-link"); + std::os::unix::fs::symlink(rig.bundle(), &bundle_link)?; + let host = rig.host().with( + IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, + bundle_link.to_str().unwrap(), + ); + let refused = refusal(rig.open(&host)); + assert!( + refused.contains("Idunn runtime bundle is not a directory"), + "{refused}" + ); + Ok(()) +} + +#[cfg(unix)] +#[test] +fn credentials_helper_wires_the_systemd_signers() -> Result<()> { + let rig = Rig::new(false)?; + let dir = tempfile::tempdir()?; + let both = format!("{ACTIVATION_NAME}:{PROVIDER_NAME}"); + let mut host = systemd_host(dir.path(), "4242", "2", &both)?; + host.descriptors.insert( + LISTEN_FDS_START, + inherited(dir.path(), "seed", &rig.activation_seed)?, + ); + host.descriptors.insert( + LISTEN_FDS_START + 1, + inherited(dir.path(), "identity", &rig.provider_credential)?, + ); + // The temp files are not root's, so the test takes the descriptors under profile + // custody and wraps them as any two readers. + let (activation, provider) = take_systemd_descriptors(&mut host, Custody::Profile)?; + let credentials = IdunnRuntimeCredentials::new(activation, provider); + let authority = rig.open_with(&rig.host(), TARGET, HEALTH_CONTRACT, credentials)?; + assert_eq!(authority.expected().target, TARGET); + Ok(()) +} + +// ------------------------------------------------------------ host credentials + +#[test] +fn host_activation_credential_is_one_shot() -> Result<()> { + let rig = Rig::new(false)?; + let dir = tempfile::tempdir()?; + let seed_path = dir.path().join("activation-key"); + let identity_path = dir.path().join("identity.cc"); + std::fs::write(&seed_path, &rig.activation_seed)?; + std::fs::write(&identity_path, &rig.provider_credential)?; + let host = FakeHost::new() + .with( + IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT, + seed_path.to_str().unwrap(), + ) + .with( + RUNTIME_PRESENCE_IDENTITY_BINDING, + identity_path.to_str().unwrap(), + ); + + let credentials = IdunnRuntimeCredentials::from_host_environment_in(&host)?; + assert!( + !seed_path.exists(), + "the seed lives only in this process now" + ); + assert!( + identity_path.exists(), + "the provider identity is stable and stays" + ); + let authority = rig.open_with(&rig.host(), TARGET, HEALTH_CONTRACT, credentials)?; + assert_eq!(authority.activation(), &rig.activation); + // The file is gone, so a second launch from the same delivery cannot read it. + assert!(IdunnRuntimeCredentials::from_host_environment_in(&host).is_err()); + Ok(()) +} + +#[test] +fn host_credentials_refuse_missing_or_relative_paths() -> Result<()> { + let dir = tempfile::tempdir()?; + let seed = dir.path().join("seed"); + std::fs::write(&seed, [1u8; 32])?; + let identity = dir.path().join("identity.cc"); + std::fs::write(&identity, b"x")?; + let host = FakeHost::new().with( + IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT, + seed.to_str().unwrap(), + ); + assert!( + refusal(IdunnRuntimeCredentials::from_host_environment_in(&host)) + .contains(RUNTIME_PRESENCE_IDENTITY_BINDING) + ); + let host = FakeHost::new().with( + RUNTIME_PRESENCE_IDENTITY_BINDING, + identity.to_str().unwrap(), + ); + assert!( + refusal(IdunnRuntimeCredentials::from_host_environment_in(&host)) + .contains(IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT) + ); + let host = FakeHost::new() + .with(IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT, "relative-seed") + .with( + RUNTIME_PRESENCE_IDENTITY_BINDING, + identity.to_str().unwrap(), + ); + assert!( + refusal(IdunnRuntimeCredentials::from_host_environment_in(&host)).contains("not absolute") + ); + assert!(seed.exists(), "nothing was consumed by a refused delivery"); + Ok(()) +} + +#[cfg(unix)] +#[test] +fn root_rules_accept_what_idunn_provisions_and_nothing_else() { + use FileKind::{Directory, Regular}; + // What Idunn's systemd driver provisions: 0555 bundle, 0444 records and locks, + // 0640 lease files. + assert!(facts(Directory, 0, 0o555).satisfies(Rule::ReadOnlyDirectory)); + assert!(facts(Regular, 0, 0o444).satisfies(Rule::ReadOnlyFile)); + assert!(facts(Regular, 0, 0o640).satisfies(Rule::ControlledFile)); + // Each rule keeps its own kind and owner. + assert!(!facts(Directory, 0, 0o555).satisfies(Rule::ReadOnlyFile)); + assert!(!facts(Regular, 0, 0o444).satisfies(Rule::ReadOnlyDirectory)); + assert!(!facts(Regular, 0, 0o640).satisfies(Rule::ReadOnlyFile)); + assert!(!facts(Regular, 1000, 0o640).satisfies(Rule::ControlledFile)); +} + +#[cfg(unix)] +#[test] +fn only_root_custody_inspects_a_lock() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir()?; + let lock = dir.path().join("record.cc.lock"); + std::fs::write(&lock, b"")?; + std::fs::set_permissions(&lock, std::fs::Permissions::from_mode(0o666))?; + // The profile trusts its own lock; root custody refuses a world-writable one. + Custody::Profile.require_lock(&lock, "lock", Rule::ControlledFile)?; + let refused = refusal(Custody::Root.require_lock(&lock, "lock", Rule::ControlledFile)); + assert!( + refused.contains("lock is not a root-controlled regular file"), + "{refused}" + ); + Ok(()) +} + +/// A directory junction is a reparse point: profile custody refuses it as a bundle. +#[cfg(windows)] +#[test] +fn profile_custody_refuses_a_junction() -> Result<()> { + let rig = Rig::new(false)?; + let junction = rig.dir.path().join("bundle-junction"); + let made = std::process::Command::new("cmd") + .args(["/C", "mklink", "/J"]) + .arg(&junction) + .arg(rig.bundle()) + .output()?; + assert!(made.status.success(), "{made:?}"); + assert_eq!( + FileKind::of(&std::fs::symlink_metadata(&junction)?), + FileKind::Symlink + ); + let host = rig + .host() + .with(IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, junction.to_str().unwrap()); + let refused = refusal(rig.open(&host)); + assert!( + refused.contains("Idunn runtime bundle is not a directory"), + "{refused}" + ); + Ok(()) +} + +fn host_delivery(seed: &Path, identity: &Path) -> FakeHost { + FakeHost::new() + .with( + IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT, + seed.to_str().unwrap(), + ) + .with( + RUNTIME_PRESENCE_IDENTITY_BINDING, + identity.to_str().unwrap(), + ) +} + +#[test] +fn host_delivery_signs_on_every_platform() -> Result<()> { + let rig = Rig::new(false)?; + let dir = tempfile::tempdir()?; + let seed = dir.path().join("activation-key"); + let identity = dir.path().join("identity.cc"); + std::fs::write(&seed, &rig.activation_seed)?; + std::fs::write(&identity, &rig.provider_credential)?; + let credentials = + IdunnRuntimeCredentials::from_host_environment_in(&host_delivery(&seed, &identity))?; + let authority = rig.open_with(&rig.host(), TARGET, HEALTH_CONTRACT, credentials)?; + let mut signer = IdunnRuntimePresenceSigner::new(authority, capabilities()); + let warming = signer.sign_at("starting", 1_000)?; + signer.report_health(RuntimePresenceState::Active); + let active = signer.sign_at("ready", 1_001)?; + assert_odin_accepts(&rig, &warming, 1_010)?; + assert_odin_accepts(&rig, &active, 1_010)?; + Ok(()) +} + +#[test] +fn host_credentials_keep_the_seed_when_the_provider_is_missing() -> Result<()> { + let rig = Rig::new(false)?; + let dir = tempfile::tempdir()?; + let seed = dir.path().join("activation-key"); + std::fs::write(&seed, &rig.activation_seed)?; + let absent = dir.path().join("absent-identity.cc"); + let refused = refusal(IdunnRuntimeCredentials::from_host_environment_in( + &host_delivery(&seed, &absent), + )); + assert!( + refused.contains(RUNTIME_PRESENCE_IDENTITY_BINDING), + "{refused}" + ); + assert!(seed.exists(), "a refused delivery leaves the one-shot seed"); + // The next launch attempt, with the identity in place, still has its seed. + let identity = dir.path().join("identity.cc"); + std::fs::write(&identity, &rig.provider_credential)?; + IdunnRuntimeCredentials::from_host_environment_in(&host_delivery(&seed, &identity))?; + assert!(!seed.exists()); + Ok(()) +} + +#[test] +fn host_credential_errors_never_echo_a_path() -> Result<()> { + let rig = Rig::new(false)?; + let root = tempfile::tempdir()?; + let canary = root.path().join("canary-6e1f0c9a"); + std::fs::create_dir(&canary)?; + let seed = canary.join("activation-key"); + let identity = canary.join("identity.cc"); + std::fs::write(&identity, &rig.provider_credential)?; + let directory = canary.join("a-directory"); + std::fs::create_dir(&directory)?; + let mut refusals = vec![ + // The activation credential is absent. + refusal(IdunnRuntimeCredentials::from_host_environment_in( + &host_delivery(&seed, &identity), + )), + // The provider identity is absent. + refusal(IdunnRuntimeCredentials::from_host_environment_in( + &host_delivery(&directory.join("seed-in-a-dir"), &canary.join("absent.cc")), + )), + // The activation credential is a directory: it cannot be read or retired. + refusal(IdunnRuntimeCredentials::from_host_environment_in( + &host_delivery(&directory, &identity), + )), + ]; + // Bundle and record refusals: a missing bundle and a bundle of the wrong kind. + let missing_bundle = rig.bare_host().with( + IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, + canary.join("no-bundle").to_str().unwrap(), + ); + refusals.push(refusal(rig.open(&missing_bundle))); + let file_bundle = rig + .bare_host() + .with(IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, identity.to_str().unwrap()); + refusals.push(refusal(rig.open(&file_bundle))); + let record_missing = tempfile::tempdir()?; + let host = rig.bare_host().with( + IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, + record_missing.path().to_str().unwrap(), + ); + refusals.push(refusal(rig.open(&host))); + for refused in &refusals { + assert!(!refused.contains("canary-6e1f0c9a"), "{refused}"); + assert!( + !refused.contains(root.path().to_str().unwrap()), + "{refused}" + ); + assert!( + !refused.contains(record_missing.path().to_str().unwrap()), + "{refused}" + ); + } + assert!( + refusals[0].contains(IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT), + "{}", + refusals[0] + ); + assert!( + refusals[1].contains(RUNTIME_PRESENCE_IDENTITY_BINDING), + "{}", + refusals[1] + ); + Ok(()) +} + +// --------------------------------------------------------------------- custody + +#[cfg(unix)] +fn facts(kind: FileKind, uid: u32, mode: u32) -> Facts { + Facts { + kind, + uid, + gid: 0, + mode, + nlink: 1, + len: 32, + } +} + +#[cfg(unix)] +#[test] +fn custody_predicates() { + use FileKind::{Directory, Other, Regular, Symlink}; + // Root-owned read-only files (bundle records and locks). + assert!(facts(Regular, 0, 0o444).is_root_read_only_file()); + assert!(facts(Regular, 0, 0o400).is_root_read_only_file()); + assert!( + !facts(Regular, 0, 0o644).is_root_read_only_file(), + "owner-writable" + ); + assert!( + !facts(Regular, 0, 0o464).is_root_read_only_file(), + "group-writable" + ); + assert!( + !facts(Regular, 0, 0o446).is_root_read_only_file(), + "world-writable" + ); + assert!( + !facts(Regular, 1000, 0o444).is_root_read_only_file(), + "not root" + ); + assert!( + !facts(Symlink, 0, 0o444).is_root_read_only_file(), + "symlink" + ); + assert!(!facts(Directory, 0, 0o444).is_root_read_only_file()); + assert!(!facts(Other, 0, 0o444).is_root_read_only_file()); + // The bundle directory. + assert!(facts(Directory, 0, 0o555).is_root_read_only_directory()); + assert!(!facts(Directory, 0, 0o755).is_root_read_only_directory()); + assert!(!facts(Directory, 1000, 0o555).is_root_read_only_directory()); + assert!(!facts(Symlink, 0, 0o555).is_root_read_only_directory()); + assert!(!facts(Regular, 0, 0o555).is_root_read_only_directory()); + // Root-controlled files and directories (the lease and its parents). + assert!(facts(Regular, 0, 0o644).is_root_controlled_file()); + assert!( + !facts(Regular, 0, 0o664).is_root_controlled_file(), + "group-writable" + ); + assert!( + !facts(Regular, 0, 0o646).is_root_controlled_file(), + "world-writable" + ); + assert!(!facts(Regular, 1000, 0o644).is_root_controlled_file()); + assert!(!facts(Symlink, 0, 0o644).is_root_controlled_file()); + assert!(facts(Directory, 0, 0o755).is_root_controlled_directory()); + assert!(!facts(Directory, 0, 0o775).is_root_controlled_directory()); + assert!(!facts(Directory, 0, 0o757).is_root_controlled_directory()); + assert!( + !facts(Directory, 0, 0o1777).is_root_controlled_directory(), + "sticky tmp" + ); + assert!(!facts(Directory, 1000, 0o755).is_root_controlled_directory()); + assert!(!facts(Symlink, 0, 0o755).is_root_controlled_directory()); + // Inherited signers. + assert!(facts(Regular, 0, 0o400).is_signer_descriptor()); + assert!(!facts(Regular, 0, 0o440).is_signer_descriptor()); + assert!(!facts(Regular, 0, 0o600).is_signer_descriptor()); + assert!(!facts(Regular, 0, 0o4400).is_signer_descriptor(), "setuid"); + assert!(!facts(Regular, 1000, 0o400).is_signer_descriptor()); + assert!( + !Facts { + gid: 1000, + ..facts(Regular, 0, 0o400) + } + .is_signer_descriptor() + ); + assert!( + !Facts { + nlink: 2, + ..facts(Regular, 0, 0o400) + } + .is_signer_descriptor() + ); + assert!( + !Facts { + len: 0, + ..facts(Regular, 0, 0o400) + } + .is_signer_descriptor() + ); + assert!(!facts(Directory, 0, 0o400).is_signer_descriptor()); +} + +#[cfg(unix)] +#[test] +fn root_custody_reads_the_real_filesystem_and_refuses() -> Result<()> { + use std::os::unix::fs::{PermissionsExt, symlink}; + let dir = tempfile::tempdir()?; + let file = dir.path().join("record.cc"); + std::fs::write(&file, b"x")?; + let require = + |custody: Custody, path: &Path| custody.require_file(path, "record", Rule::ControlledFile); + + std::fs::set_permissions(&file, std::fs::Permissions::from_mode(0o666))?; + assert!( + refusal(require(Custody::Root, &file)) + .contains("record is not a root-controlled regular file") + ); + let link = dir.path().join("link.cc"); + symlink(&file, &link)?; + std::fs::set_permissions(&file, std::fs::Permissions::from_mode(0o644))?; + assert!( + require(Custody::Root, &link).is_err(), + "a symlink is never custody" + ); + assert!(require(Custody::Root, Path::new("record.cc")).is_err()); + assert!(require(Custody::Root, &dir.path().join("absent")).is_err()); + assert!( + require(Custody::Profile, &link).is_err(), + "profile custody refuses a link too" + ); + assert!(require(Custody::Profile, &file).is_ok()); + + assert!( + Custody::Root + .require_directory_chain(dir.path(), "dir") + .is_err() + ); + assert!( + Custody::Root + .require_directory_chain(Path::new("relative"), "dir") + .is_err() + ); + assert!( + Custody::Profile + .require_directory_chain(dir.path(), "dir") + .is_ok() + ); + Ok(()) +} + +#[cfg(unix)] +#[test] +fn the_default_open_path_enforces_root_custody() -> Result<()> { + // The rig's bundle is an ordinary writable temp directory. + let rig = Rig::new(false)?; + let mut credentials = rig.credentials(); + credentials.custody = Custody::Root; + let refused = refusal(rig.open_with(&rig.host(), TARGET, HEALTH_CONTRACT, credentials)); + assert!( + refused.contains("Idunn runtime bundle is not a root-owned read-only directory"), + "{refused}" + ); + Ok(()) +} + +#[test] +fn debug_output_names_the_target_and_never_signing_material() -> Result<()> { + let rig = Rig::new(false)?; + let shown = format!("{:?}", rig.authority()?); + assert!(shown.contains("IdunnRuntimeAuthority") && shown.contains(TARGET)); + Ok(()) +} + +#[cfg(unix)] +#[test] +fn a_lease_guard_debug_names_its_digest() -> Result<()> { + let lease_rig = Rig::new(true)?; + let mut signer = lease_rig.signer()?; + let warming = signer.sign_at("w", 1_000)?; + lease_rig.write_lease(&warming.canonical_sha256, 1)?; + let guard = signer.try_acquire_write_lease()?.unwrap(); + assert!(format!("{guard:?}").contains(guard.lease_sha256())); + Ok(()) +} + +// ----------------------------------------------------- the real process, in small + +/// A descriptor number far above anything a parallel test is likely to open. +#[cfg(unix)] +fn high_duplicate(file: &File) -> RawFd { + // SAFETY: F_DUPFD duplicates a descriptor `file` owns into a fresh number. + let descriptor = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_DUPFD, 700) }; + assert!(descriptor >= 700); + descriptor +} + +#[test] +fn the_process_host_reads_the_real_process() -> Result<()> { + let mut host = ProcessHost; + assert_eq!(host.process_id(), std::process::id()); + + let name = "CULTNET_RS_PROVIDER_TEST_PROCESS_HOST"; + assert_eq!(host.var(name), None); + // SAFETY: a name no other test or thread reads or writes. + unsafe { std::env::set_var(name, "a value") }; + assert_eq!(host.var(name).as_deref(), Some("a value")); + host.remove_var(name); + assert_eq!(host.var(name), None); + assert!( + std::env::var(name).is_err(), + "removed from the real environment" + ); + Ok(()) +} + +#[cfg(unix)] +#[test] +fn the_process_host_adopts_real_descriptors() -> Result<()> { + let mut host = ProcessHost; + let dir = tempfile::tempdir()?; + std::fs::write(dir.path().join("adopted"), b"adopted bytes")?; + let source = File::open(dir.path().join("adopted"))?; + let descriptor = high_duplicate(&source); + let adopted = host.adopt_descriptor(descriptor)?; + assert_eq!(adopted.as_raw_fd(), descriptor); + assert_eq!(contents(adopted.try_clone()?)?, "adopted bytes"); + adopted.sync_all()?; + + let closed = high_duplicate(&source); + // SAFETY: closes the duplicate made on the previous line. + assert_eq!(unsafe { libc::close(closed) }, 0); + assert!(refusal(host.adopt_descriptor(closed)).contains("is not open")); + Ok(()) +} + +#[cfg(unix)] +#[test] +fn facts_read_the_real_metadata() -> Result<()> { + use std::os::unix::fs::{PermissionsExt, symlink}; + let dir = tempfile::tempdir()?; + let file = dir.path().join("file"); + std::fs::write(&file, b"hello")?; + std::fs::set_permissions(&file, std::fs::Permissions::from_mode(0o640))?; + let observed = Facts::of(&std::fs::symlink_metadata(&file)?); + assert_eq!(observed.kind, FileKind::Regular); + assert_eq!( + observed.mode, 0o640, + "permission bits only, not the file type" + ); + assert_eq!((observed.len, observed.nlink), (5, 1)); + // SAFETY: geteuid takes no arguments and cannot fail. + assert_eq!(observed.uid, unsafe { libc::geteuid() }); + + assert_eq!(Facts::of_path(dir.path(), "dir")?.kind, FileKind::Directory); + let link = dir.path().join("link"); + symlink(&file, &link)?; + assert_eq!(Facts::of_path(&link, "link")?.kind, FileKind::Symlink); + let socket = dir.path().join("socket"); + let _listener = std::os::unix::net::UnixListener::bind(&socket)?; + assert_eq!(Facts::of_path(&socket, "socket")?.kind, FileKind::Other); + assert!(Facts::of_path(&dir.path().join("absent"), "absent").is_err()); + Ok(()) +} + +// --------------------------------------------------------- lease failure shapes + +#[cfg(unix)] +#[test] +fn a_lease_path_that_cannot_be_inspected_is_an_error_not_an_absent_lease() -> Result<()> { + let rig = Rig::new(true)?; + // The parent is a regular file, so the lease path is ENOTDIR, not "not found". + let impossible = rig.dir.path().join("provider.cc").join("lease.cc"); + let host = rig + .bare_host() + .with(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, BIND) + .with( + IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT, + impossible.to_str().unwrap(), + ); + let mut signer = IdunnRuntimePresenceSigner::new(rig.open(&host)?, capabilities()); + assert!(signer.try_acquire_write_lease().is_err()); + Ok(()) +} + +#[cfg(unix)] +#[test] +fn the_write_lease_is_held_to_root_custody() -> Result<()> { + let rig = Rig::new(true)?; + let mut authority = rig.authority()?; + Arc::get_mut(&mut authority.records).unwrap().custody = Custody::Root; + let mut signer = IdunnRuntimePresenceSigner::new(authority, capabilities()); + let warming = signer.sign_at("w", 1_000)?; + assert!( + signer.try_acquire_write_lease()?.is_none(), + "an absent record is not yet a custody question" + ); + rig.write_lease(&warming.canonical_sha256, 1)?; + let refused = refusal(signer.try_acquire_write_lease()); + assert!(refused.contains("process-write-lease"), "{refused}"); + Ok(()) +} + +#[test] +fn sign_stamps_the_current_time() -> Result<()> { + let rig = Rig::new(false)?; + let mut signer = rig.signer()?; + let before = unix_millis()?; + let signed = signer.sign("now")?; + let after = unix_millis()?; + assert!( + (before..=after).contains(&signed.record.observed_at_unix_millis), + "{before} <= {} <= {after}", + signed.record.observed_at_unix_millis + ); + assert!(before > 1_700_000_000_000, "unix_millis is the wall clock"); + Ok(()) +} + +// ------------------------------------------------------ s2 repairs, attempt 3 + +/// A lease-required runtime may claim Degraded only under a current lease, with +/// no lease ever held and after one was lost alike. +#[cfg(unix)] +#[test] +fn degraded_needs_a_current_lease_whatever_the_history() -> Result<()> { + let rig = Rig::new(true)?; + let mut signer = rig.signer()?; + let warming = signer.sign_at("w", 1_000)?; + signer.report_health(RuntimePresenceState::Degraded); + assert!(signer.sign_at("never held", 1_001).is_err()); + rig.write_lease(&warming.canonical_sha256, 1)?; + signer.try_acquire_write_lease()?.expect("binds"); + assert!(signer.sign_at("held", 1_002).is_ok()); + std::fs::remove_file(rig.lease_path())?; + assert!(signer.sign_at("lost", 1_003).is_err()); + Ok(()) +} + +/// A runtime that never needed a lease reports Degraded and Active without one. +#[test] +fn degraded_and_active_sign_without_a_lease_when_none_is_required() -> Result<()> { + let rig = Rig::new(false)?; + let mut signer = rig.signer()?; + for state in [RuntimePresenceState::Degraded, RuntimePresenceState::Active] { + signer.report_health(state); + assert_eq!(signer.sign_at("up", 1_000)?.record.write_lease_sha256, None); + } + Ok(()) +} + +fn assert_no_path(refused: &str, rig: &Rig) { + assert!( + !refused.contains(rig.dir.path().to_str().unwrap()), + "{refused}" + ); + assert!( + !refused.contains(rig.dir.path().file_name().unwrap().to_str().unwrap()), + "{refused}" + ); +} + +#[test] +fn a_missing_record_lock_names_no_path() -> Result<()> { + let rig = Rig::new(false)?; + let mut lock = rig.bundle().join("expected.cc").into_os_string(); + lock.push(".lock"); + std::fs::remove_file(&lock)?; + let refused = refusal(rig.open(&rig.host())); + assert!( + refused.contains("could not be read as a CultCache store: not found"), + "{refused}" + ); + assert_no_path(&refused, &rig); + Ok(()) +} + +#[test] +fn an_undecodable_record_names_no_path() -> Result<()> { + let rig = Rig::new(false)?; + std::fs::write( + rig.bundle().join("activation.cc"), + [0xc1_u8, 0xc1, b'g', b'a'], + )?; + let refused = refusal(rig.open(&rig.host())); + assert!( + refused.contains("could not be read as a CultCache store: not decodable"), + "{refused}" + ); + assert_no_path(&refused, &rig); + Ok(()) +} + +#[test] +fn a_record_in_a_newer_store_format_says_so_and_names_no_path() -> Result<()> { + let rig = Rig::new(false)?; + // A store header this runtime does not read; the format is judged from the + // header alone, so the cut-short store is still a newer one. + let mut newer = vec![0x93_u8, 0xb2]; + newer.extend_from_slice(b"cultcache.store.v9"); + std::fs::write(rig.bundle().join("activation.cc"), newer)?; + let refused = refusal(rig.open(&rig.host())); + assert!( + refused.contains( + "could not be read as a CultCache store: in a format this runtime does not read" + ), + "{refused}" + ); + assert_no_path(&refused, &rig); + Ok(()) +} + +#[test] +fn an_io_failure_is_labelled_by_its_kind_alone() { + use std::io::{Error, ErrorKind}; + let path = "/profile/secret-dir/store.cc"; + for (kind, label) in [ + (ErrorKind::PermissionDenied, "permission denied"), + (ErrorKind::NotFound, "not found"), + (ErrorKind::TimedOut, "I/O error"), + ] { + let error = anyhow::Error::new(Error::new(kind, path)) + .context(format!("failed to open pre-created {path}.lock")); + assert_eq!(store_failure_kind(&error), label); + } + assert_eq!(store_failure_kind(&anyhow!("opaque")), "unreadable"); +} + +/// The provider's own refusals about a record still say what is wrong. +#[test] +fn a_store_of_two_records_is_still_refused_by_its_own_message() -> Result<()> { + let rig = Rig::new(false)?; + write_record( + &rig.bundle().join("expected.cc"), + "other", + IdunnExpectedIncarnationRecord::TYPE, + IDUNN_EXPECTED_INCARNATION_SCHEMA, + rig.expected.canonical_bytes()?, + )?; + let refused = refusal(rig.open(&rig.host())); + assert!(refused.contains("exactly one record"), "{refused}"); + Ok(()) +} + +#[cfg(unix)] +#[test] +fn an_undecodable_lease_names_no_path() -> Result<()> { + let rig = Rig::new(true)?; + let mut signer = rig.signer()?; + let warming = signer.sign_at("w", 1_000)?; + rig.write_lease(&warming.canonical_sha256, 1)?; + signer.try_acquire_write_lease()?.expect("binds"); + std::fs::write(rig.lease_path(), [0xc1_u8, 0xc1, b'g', b'a'])?; + signer.report_health(RuntimePresenceState::Active); + let refused = refusal(signer.sign_at("a", 1_001)); + assert!(refused.contains("could not be read"), "{refused}"); + assert_no_path(&refused, &rig); + Ok(()) +} + +/// A provider identity that is not a regular file refuses before the one-shot +/// seed is read, so the next launch attempt still has it. +#[test] +fn host_credentials_keep_the_seed_when_the_provider_is_a_directory() -> Result<()> { + let rig = Rig::new(false)?; + let dir = tempfile::tempdir()?; + let seed = dir.path().join("activation-key"); + std::fs::write(&seed, &rig.activation_seed)?; + let provider = dir.path().join("provider-is-a-dir"); + std::fs::create_dir(&provider)?; + let refused = refusal(IdunnRuntimeCredentials::from_host_environment_in( + &host_delivery(&seed, &provider), + )); + assert!( + refused.contains(RUNTIME_PRESENCE_IDENTITY_BINDING), + "{refused}" + ); + assert!(!refused.contains(dir.path().to_str().unwrap()), "{refused}"); + assert!(seed.exists(), "a refused delivery leaves the one-shot seed"); + Ok(()) +} diff --git a/packages/cultnet-rs/src/lib.rs b/packages/cultnet-rs/src/lib.rs index 621150ba..b35bd701 100644 --- a/packages/cultnet-rs/src/lib.rs +++ b/packages/cultnet-rs/src/lib.rs @@ -7,6 +7,7 @@ mod media_stream_contracts; mod media_stream_wire; mod provider_health_contracts; mod runtime_authority_contracts; +mod idunn_runtime_provider; mod service_identity; mod snapshot_query; mod cultmesh; @@ -30,6 +31,7 @@ pub use media_stream_contracts::*; pub use media_stream_wire::*; pub use provider_health_contracts::*; pub use runtime_authority_contracts::*; +pub use idunn_runtime_provider::*; pub use service_identity::*; pub use snapshot_query::*; pub use cultmesh::*; diff --git a/packages/cultnet-rs/src/runtime_authority_contracts.rs b/packages/cultnet-rs/src/runtime_authority_contracts.rs index 0549a5c1..08cdbcf1 100644 --- a/packages/cultnet-rs/src/runtime_authority_contracts.rs +++ b/packages/cultnet-rs/src/runtime_authority_contracts.rs @@ -21,6 +21,22 @@ pub const ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA: &str = "odin.runtime_topolog pub const ODIN_RUNTIME_TOPOLOGY_CORRELATION_SIGNING_PURPOSE: &str = "odin.runtime_topology_correlation.v2"; +/// Names the runtime bundle directory Idunn launched this process with. Set by +/// both Idunn drivers; read by `IdunnRuntimeAuthority::open`. +pub const IDUNN_RUNTIME_BUNDLE_ENVIRONMENT: &str = "GAMECULT_IDUNN_RUNTIME_BUNDLE"; +/// Names the `host:port` Idunn expects the candidate to bind. Set by Idunn's +/// systemd driver for routed targets. +pub const IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT: &str = "GAMECULT_IDUNN_CANDIDATE_BIND"; +/// Names the process write-lease record of a stateful target. Set by Idunn's +/// systemd driver exactly when Expected requires a lease. +pub const IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT: &str = "GAMECULT_IDUNN_PROCESS_WRITE_LEASE"; +/// Names the one-shot activation credential file. Set by Idunn's host driver. +pub const IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT: &str = "GAMECULT_IDUNN_ACTIVATION_CREDENTIAL"; +/// Names the stable provider identity credential file. Set by Idunn's host driver. +pub const RUNTIME_PRESENCE_IDENTITY_BINDING: &str = "GAMECULT_RUNTIME_PRESENCE_IDENTITY"; +/// The name Idunn's systemd driver gives the stable provider identity descriptor. +pub const RUNTIME_PRESENCE_IDENTITY_FD_NAME: &str = "gamecult-runtime-presence-identity"; + const MAX_RUNTIME_CAPABILITIES: usize = 256; const MAX_RUNTIME_DEPENDENCIES: usize = 256; const MAX_TOPOLOGY_DISAGREEMENTS: usize = 256; From 37a2b002dbc358d6c26019c6461ff733df99b476 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sat, 31 Oct 2026 01:30:29 +0100 Subject: [PATCH 30/88] cultnet-ts: CultNetRudpSocketTransportServer serves many peers on one socket, one reliable session per remote endpoint (matches Rust server hub and C# socket server; wire unchanged) Co-Authored-By: Claude Opus 5.5 --- packages/cultnet-ts/src/index.ts | 6 + packages/cultnet-ts/src/rudp-server.ts | 370 +++++++++++++++++++ packages/cultnet-ts/src/rudp.ts | 8 +- packages/cultnet-ts/test/rudp-server.test.ts | 172 +++++++++ 4 files changed, 552 insertions(+), 4 deletions(-) create mode 100644 packages/cultnet-ts/src/rudp-server.ts create mode 100644 packages/cultnet-ts/test/rudp-server.test.ts diff --git a/packages/cultnet-ts/src/index.ts b/packages/cultnet-ts/src/index.ts index 1e856822..c321bd07 100644 --- a/packages/cultnet-ts/src/index.ts +++ b/packages/cultnet-ts/src/index.ts @@ -123,6 +123,12 @@ export { type CultNetRudpSessionOptions, type RudpTransportProfileOptions, } from "./rudp"; +export { + CultNetRudpSocketTransportServer, + type CultNetRudpSocketServerFrame, + type CultNetRudpSocketServerOptions, + type CultNetRudpSocketServerPeer, +} from "./rudp-server"; export { invokeCultNetOperation, startCultNetOperationServer, diff --git a/packages/cultnet-ts/src/rudp-server.ts b/packages/cultnet-ts/src/rudp-server.ts new file mode 100644 index 00000000..e6e68464 --- /dev/null +++ b/packages/cultnet-ts/src/rudp-server.ts @@ -0,0 +1,370 @@ +import { EventEmitter } from "node:events"; +import { type RemoteInfo, type Socket } from "node:dgram"; + +import type { CultNetTransportProfile } from "./contracts"; +import { + CultNetRudpSession, + MAX_UDP4_DATAGRAM_BYTES, + MAX_UDP6_DATAGRAM_BYTES, + RUDP_FIXED_HEADER_BYTES, + channelOptions, + createRudpTransportProfile, + decodeRudpPacket, + encodeRudpPacket, + isPermanentSendError, + sendRudpDatagram, + type CultNetRudpPacket, +} from "./rudp"; +import type { CultNetReconnectPolicy, CultNetTransportFrame, CultNetTransportStats } from "./transport"; + +export interface CultNetRudpSocketServerOptions { + runtimeId: string; + /** A bound UDP socket. The server owns it from here: `close()` closes it. */ + socket: Socket; + connectionId: number; + /** The first local sequence of every peer session. Unset, each session draws its own. */ + initialSequence?: number; + resendDelayMs?: number; + resendPollMs?: number; + transportId?: string; + maxPayloadBytes?: number; + maxFragmentBytes?: number; + maxPendingReliablePackets?: number; + reconnectPolicy?: CultNetReconnectPolicy; + /** A Connect from an endpoint beyond this many peers is dropped. Default 256. */ + maxPeers?: number; + /** + * A peer that has sent nothing for this long is ended and told so. Default 60 000; + * `Infinity` keeps idle peers until they leave. + */ + peerTimeoutMs?: number; +} + +/** One remote endpoint's session. A peer that reconnects from the same endpoint is a new object. */ +export interface CultNetRudpSocketServerPeer { + readonly remoteHost: string; + readonly remotePort: number; + /** The exact bytes of the Connect that opened this peer's session. */ + readonly connectPayload: Uint8Array; + readonly connected: boolean; + readonly outstandingReliablePacketCount: number; +} + +export interface CultNetRudpSocketServerFrame extends CultNetTransportFrame { + peer: CultNetRudpSocketServerPeer; +} + +type PeerEntry = { + peer: CultNetRudpSocketServerPeer; + session: CultNetRudpSession; + key: string; +}; + +/** The goodbye for a session that refused a packet; built after the reset, or its ack would acknowledge the refused frame. */ +function refusedGoodbye(session: CultNetRudpSession): CultNetRudpPacket { + session.resetPeerState(); + return session.createDisconnect(utf8("session refused a packet")); +} + +const utf8 = (text: string) => new TextEncoder().encode(text); + +/** + * One UDP socket serving many peers. Every remote address and port gets its own reliable + * session, opened by its Connect: a second client never takes the first one's session, and + * a client that reconnects from the same endpoint replaces only its own. The wire format is + * the single-peer transport's; the shape matches the Rust `CultNetRudpServerHub` and the C# + * `CultNetRudpSocketTransportServer`. + * + * Reply to a peer with `send(peer, ...)` on the peer a frame arrived from. + * + * Events: `peerConnected` (peer), `frame` ({ peer, channelId, payload }), `pong` ({ peer, payload }), + * `peerDisconnected` ({ peer, reason }), `close`, `error`. + */ +export class CultNetRudpSocketTransportServer extends EventEmitter { + readonly profile: CultNetTransportProfile; + readonly #socket: Socket; + readonly #options: CultNetRudpSocketServerOptions; + readonly #entries = new Map(); + readonly #timer: NodeJS.Timeout; + readonly #maxDatagramBytes: number; + readonly #maxPeers: number; + readonly #peerTimeoutMs: number; + #closed = false; + readonly #stats: CultNetTransportStats = { + bytesReceived: 0, + bytesSent: 0, + framesReceived: 0, + framesSent: 0, + packetsDropped: 0, + sendFailures: 0, + }; + + constructor(options: CultNetRudpSocketServerOptions) { + super(); + this.#options = options; + this.#socket = options.socket; + this.#maxPeers = options.maxPeers ?? 256; + this.#peerTimeoutMs = options.peerTimeoutMs ?? 60_000; + if (!Number.isInteger(this.#maxPeers) || this.#maxPeers < 1) { + throw new Error("RUDP server maxPeers must be a positive integer."); + } + const address = this.#socket.address(); + this.#maxDatagramBytes = + typeof address !== "string" && String(address.family).endsWith("6") ? MAX_UDP6_DATAGRAM_BYTES : MAX_UDP4_DATAGRAM_BYTES; + this.profile = createRudpTransportProfile(options.runtimeId, { + transportId: options.transportId, + host: typeof address === "string" ? undefined : address.address, + port: typeof address === "string" ? undefined : address.port, + maxPayloadBytes: options.maxPayloadBytes, + maxFragmentBytes: options.maxFragmentBytes, + maxPendingReliablePackets: options.maxPendingReliablePackets, + reconnectPolicy: options.reconnectPolicy, + }); + + this.#socket.on("message", (wire, remote) => this.#receiveDatagram(wire, remote)); + this.#socket.on("close", () => { + if (this.#closed) return; + this.#closed = true; + clearInterval(this.#timer); + this.#entries.clear(); + this.emit("close"); + }); + this.#socket.on("error", (error) => this.emit("error", error instanceof Error ? error : new Error(String(error)))); + this.#timer = setInterval(() => this.#tick(), options.resendPollMs ?? 25); + this.#timer.unref?.(); + } + + /** The peers with a live session. */ + get peers(): CultNetRudpSocketServerPeer[] { + return [...this.#entries.values()].map((entry) => entry.peer); + } + + get stats(): CultNetTransportStats { + return { ...this.#stats }; + } + + /** Sends to one peer. Throws when that peer's session has ended or was replaced. */ + send(peer: CultNetRudpSocketServerPeer, channelId: string, payload: Uint8Array): void { + const entry = this.#liveEntry(peer); + const largestPayload = + this.#options.maxFragmentBytes === undefined ? payload.byteLength : Math.min(payload.byteLength, this.#options.maxFragmentBytes); + if (RUDP_FIXED_HEADER_BYTES + Buffer.byteLength(channelId, "utf8") + largestPayload > this.#maxDatagramBytes) { + throw Object.assign(new Error("RUDP datagram is larger than UDP can carry."), { code: "EMSGSIZE" }); + } + const packets = entry.session.sendMany(channelId, payload, { + ...channelOptions(channelId), + nowMs: Date.now(), + maxFragmentBytes: this.#options.maxFragmentBytes, + }); + for (const packet of packets) { + const thrown = this.#transmit(entry, packet); + if (thrown) throw thrown; + } + this.#stats.framesSent += 1; + } + + /** Ends one peer's session, telling it why. */ + disconnect(peer: CultNetRudpSocketServerPeer, reason: Uint8Array = new Uint8Array()): void { + const entry = this.#liveEntry(peer); + this.#endPeer(entry, entry.session.createDisconnect(Uint8Array.from(reason))); + } + + /** Ends every peer and releases the socket. */ + close(): void { + if (this.#closed) return; + clearInterval(this.#timer); + for (const entry of [...this.#entries.values()]) { + this.#endPeer(entry, entry.session.createDisconnect(utf8("server closed"))); + } + this.#closed = true; + this.#socket.close(); + } + + #liveEntry(peer: CultNetRudpSocketServerPeer): PeerEntry { + const entry = this.#entries.get(`${peer.remoteHost}:${peer.remotePort}`); + if (!entry || entry.peer !== peer) { + throw new Error("RUDP peer is not connected."); + } + return entry; + } + + /** Removes the peer, tells the caller, and says goodbye on the wire. Idempotent per entry. */ + #endPeer(entry: PeerEntry, goodbye: CultNetRudpPacket): void { + if (this.#entries.get(entry.key) !== entry) return; + this.#entries.delete(entry.key); + this.emit("peerDisconnected", { peer: entry.peer, reason: goodbye.payload }); + // Best-effort: the session is over whether or not the peer hears it. + const wire = encodeRudpPacket(goodbye); + this.#stats.bytesSent += wire.length; + sendRudpDatagram(this.#socket, wire, entry.peer.remotePort, entry.peer.remoteHost, () => {}); + } + + #receiveDatagram(wire: Buffer, remote: RemoteInfo): void { + this.#stats.bytesReceived += wire.length; + // What a datagram carries is the sender's business: a malformed frame, another + // session's connection id and a sender no Connect admitted are dropped and counted. + let packet: CultNetRudpPacket; + try { + packet = decodeRudpPacket(wire); + } catch { + this.#stats.packetsDropped += 1; + return; + } + if (packet.connectionId !== this.#options.connectionId) { + this.#stats.packetsDropped += 1; + return; + } + const key = `${remote.address}:${remote.port}`; + const existing = this.#entries.get(key); + + if (packet.packetType === "connect") { + this.#receiveConnect(packet, remote, key, existing); + return; + } + if (!existing) { + this.#stats.packetsDropped += 1; + return; + } + this.#receiveFromPeer(existing, packet); + } + + #receiveConnect(packet: CultNetRudpPacket, remote: RemoteInfo, key: string, existing: PeerEntry | undefined): void { + if (existing?.session.connectRepeats(packet)) { + // A repeat of the Connect it accepted: the Accept owed, nothing restarted. + try { + this.#sendInPoll(existing, existing.session.acceptConnect(packet, Date.now())); + } catch { + this.#stats.packetsDropped += 1; + this.#endPeer(existing, refusedGoodbye(existing.session)); + } + return; + } + if (!existing && this.#entries.size >= this.#maxPeers) { + this.#stats.packetsDropped += 1; + return; + } + const session = new CultNetRudpSession({ + connectionId: this.#options.connectionId, + initialSequence: this.#options.initialSequence, + resendDelayMs: this.#options.resendDelayMs, + maxPendingReliablePackets: this.#options.maxPendingReliablePackets, + }); + let accept: CultNetRudpPacket; + try { + accept = session.acceptConnect(packet, Date.now()); + } catch { + this.#stats.packetsDropped += 1; + return; + } + const entry: PeerEntry = { + key, + session, + peer: { + remoteHost: remote.address, + remotePort: remote.port, + connectPayload: Uint8Array.from(packet.payload ?? []), + get connected() { return session.connected; }, + get outstandingReliablePacketCount() { return session.outstandingReliablePacketCount; }, + }, + }; + if (this.#transmit(entry, accept)) { + // The peer cannot be answered, so no session starts. + this.#stats.packetsDropped += 1; + return; + } + if (existing) { + // Same endpoint, new Connect generation: only this peer's old session ends. + this.#entries.delete(key); + this.emit("peerDisconnected", { peer: existing.peer, reason: utf8("replaced by a new Connect generation") }); + } + this.#entries.set(key, entry); + this.emit("peerConnected", entry.peer); + } + + #receiveFromPeer(entry: PeerEntry, packet: CultNetRudpPacket): void { + const session = entry.session; + let result: ReturnType; + try { + result = session.receive(packet, Date.now()); + } catch { + // receive() has already recorded the packet's reliable sequence, so the session + // cannot be kept: a retransmit would be acknowledged and the frame silently lost. + this.#stats.packetsDropped += 1; + this.#endPeer(entry, refusedGoodbye(session)); + return; + } + try { + if (result.reply) this.#sendInPoll(entry, result.reply); + for (const ready of result.readyToSend ?? []) this.#sendInPoll(entry, ready); + if (result.pong) { + this.emit("pong", { peer: entry.peer, payload: result.pongPayload ?? new Uint8Array() }); + } + for (const frame of result.delivered) { + this.#stats.framesReceived += 1; + this.emit("frame", { peer: entry.peer, channelId: frame.channelId, payload: frame.payload }); + } + if (result.disconnected) { + if (this.#entries.get(entry.key) === entry) { + this.#entries.delete(entry.key); + this.emit("peerDisconnected", { peer: entry.peer, reason: result.disconnectReason ?? new Uint8Array(0) }); + } + return; + } + if (this.#entries.get(entry.key) === entry && (packet.packetType === "data" || result.delivered.length > 0)) { + this.#sendInPoll(entry, session.createAckForReceived(packet.sequence)); + } + } catch (error) { + this.emit("error", error instanceof Error ? error : new Error(String(error))); + } + } + + #tick(): void { + const now = Date.now(); + for (const entry of [...this.#entries.values()]) { + if (this.#entries.get(entry.key) !== entry) continue; + if (entry.session.checkTimeout(now, this.#peerTimeoutMs)) { + this.#endPeer(entry, entry.session.createDisconnect(utf8("session timed out"))); + continue; + } + const generation = entry.session.generation; + for (const packet of entry.session.dueResends(now)) { + this.#sendInPoll(entry, packet); + // A resend that can never be sent ended the session and what it owed. + if (entry.session.generation !== generation || this.#entries.get(entry.key) !== entry) break; + } + } + } + + /** A send with no caller to hand a failure to: it ends that peer, never the poll. */ + #sendInPoll(entry: PeerEntry, packet: CultNetRudpPacket): void { + const thrown = this.#transmit(entry, packet); + if (thrown) this.#sendFailed(entry, thrown); + } + + /** Sends one packet to the peer and returns the failure the send threw, if any. */ + #transmit(entry: PeerEntry, packet: CultNetRudpPacket): Error | undefined { + const wire = encodeRudpPacket(packet); + this.#stats.bytesSent += wire.length; + let thrown: Error | undefined; + let sending = true; + const generation = entry.session.generation; + sendRudpDatagram(this.#socket, wire, entry.peer.remotePort, entry.peer.remoteHost, (error) => { + if (sending) { + thrown = error; + } else if (entry.session.generation === generation) { + this.#sendFailed(entry, error); + } + }); + sending = false; + return thrown; + } + + /** A datagram that can never be sent ends that peer; any other failure is a lost datagram. */ + #sendFailed(entry: PeerEntry, error: Error): void { + if (isPermanentSendError(error)) { + this.#endPeer(entry, entry.session.endUnsendableSession(error)); + return; + } + this.#stats.sendFailures += 1; + } +} diff --git a/packages/cultnet-ts/src/rudp.ts b/packages/cultnet-ts/src/rudp.ts index e68293d3..89939ef3 100644 --- a/packages/cultnet-ts/src/rudp.ts +++ b/packages/cultnet-ts/src/rudp.ts @@ -29,11 +29,11 @@ import { const RUDP_MAGIC = [0x43, 0x4e, 0x52, 0x30] as const; // CNR0 const RUDP_VERSION = 0; -const RUDP_FIXED_HEADER_BYTES = 36; +export const RUDP_FIXED_HEADER_BYTES = 36; /// The largest UDP payload a datagram can carry: 65,535 less the UDP header and, /// over IPv4, the minimal IP header. IPv6's header is outside its payload length. -const MAX_UDP4_DATAGRAM_BYTES = 65_507; -const MAX_UDP6_DATAGRAM_BYTES = 65_527; +export const MAX_UDP4_DATAGRAM_BYTES = 65_507; +export const MAX_UDP6_DATAGRAM_BYTES = 65_527; const MAX_CHANNEL_ID_BYTES = 255; export const CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS = 32; const RUDP_RECEIVED_SEQUENCE_WINDOW = 4_096; @@ -1588,7 +1588,7 @@ export function createRudpTransportProfile( }; } -function channelOptions(channelId: string): { reliable: boolean; ordered: boolean; sequenced: boolean } { +export function channelOptions(channelId: string): { reliable: boolean; ordered: boolean; sequenced: boolean } { if (channelId === "schema") { return { reliable: true, ordered: true, sequenced: false }; } diff --git a/packages/cultnet-ts/test/rudp-server.test.ts b/packages/cultnet-ts/test/rudp-server.test.ts new file mode 100644 index 00000000..d35ca9f7 --- /dev/null +++ b/packages/cultnet-ts/test/rudp-server.test.ts @@ -0,0 +1,172 @@ +// One server socket serves many peers, each with its own reliable session. +import assert from "node:assert/strict"; +import test from "node:test"; +import dgram, { type Socket } from "node:dgram"; +import { + CultNetRudpSession, + CultNetRudpSocketTransportConnection, + CultNetRudpSocketTransportServer, + decodeRudpPacket, + encodeRudpPacket, + type CultNetRudpSocketServerPeer, +} from "../src"; + +const connectionId = 0x10203091; +const enc = (text: string) => new TextEncoder().encode(text); +const dec = (bytes: Uint8Array) => new TextDecoder().decode(bytes); +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function bind(): Promise { + const socket = dgram.createSocket("udp4"); + await new Promise((resolve) => socket.bind(0, "127.0.0.1", resolve)); + return socket; +} + +async function waitFor(predicate: () => boolean, description: string): Promise { + const startedAt = Date.now(); + while (!predicate()) { + if (Date.now() - startedAt > 3_000) throw new Error(`Timed out waiting for ${description}.`); + await sleep(5); + } +} + +type Client = { connection: CultNetRudpSocketTransportConnection; socket: Socket; received: string[]; closed: () => boolean }; + +async function startServer(extra: Record = {}) { + const socket = await bind(); + const server = new CultNetRudpSocketTransportServer({ + runtimeId: "many-peer-server", socket, connectionId, resendDelayMs: 10, resendPollMs: 5, ...extra, + }); + const requests: { peer: CultNetRudpSocketServerPeer; text: string }[] = []; + const connected: CultNetRudpSocketServerPeer[] = []; + const left: { peer: CultNetRudpSocketServerPeer; reason: string }[] = []; + server.on("peerConnected", (peer: CultNetRudpSocketServerPeer) => connected.push(peer)); + server.on("peerDisconnected", ({ peer, reason }: { peer: CultNetRudpSocketServerPeer; reason: Uint8Array }) => left.push({ peer, reason: dec(reason) })); + // Every request is answered on the peer it arrived from. + server.on("frame", ({ peer, payload }: { peer: CultNetRudpSocketServerPeer; payload: Uint8Array }) => { + requests.push({ peer, text: dec(payload) }); + server.send(peer, "schema", enc(`re:${dec(payload)}`)); + }); + const port = (socket.address() as { port: number }).port; + return { server, port, requests, connected, left }; +} + +async function startClient(port: number, connectPayload = new Uint8Array()): Promise { + const socket = await bind(); + const connection = new CultNetRudpSocketTransportConnection({ + runtimeId: "client", socket, mode: "client", remoteHost: "127.0.0.1", remotePort: port, + connectionId, resendDelayMs: 10, resendPollMs: 5, + }); + const received: string[] = []; + let closed = false; + connection.on("frame", ({ payload }: { payload: Uint8Array }) => received.push(dec(payload))); + connection.on("close", () => { closed = true; }); + connection.connect(connectPayload); + await waitFor(() => connection.connected, "client handshake"); + return { connection, socket, received, closed: () => closed }; +} + +/** A bare session on a socket, so a goodbye can be sent and the socket released only after it left. */ +async function rawClient(port: number) { + const socket = await bind(); + const session = new CultNetRudpSession({ connectionId }); + const transmit = (packet: ReturnType) => + new Promise((resolve) => socket.send(encodeRudpPacket(packet), port, "127.0.0.1", () => resolve())); + socket.on("message", (wire) => { session.receive(decodeRudpPacket(wire), Date.now()); }); + await transmit(session.createConnect(Date.now())); + await waitFor(() => session.connected, "raw client handshake"); + return { async leave() { await transmit(session.createDisconnect()); socket.close(); } }; +} + +test("two clients on different ports talk through one server socket and neither resets the other", async () => { + const { server, port, requests } = await startServer(); + const a = await startClient(port, enc("alpha")); + const b = await startClient(port, enc("beta")); + try { + assert.equal(server.peers.length, 2); + assert.deepEqual(server.peers.map((peer) => dec(peer.connectPayload)).sort(), ["alpha", "beta"]); + for (let round = 0; round < 5; round += 1) { + a.connection.send("schema", enc(`a${round}`)); + b.connection.send("schema", enc(`b${round}`)); + } + await waitFor(() => a.received.length === 5 && b.received.length === 5, "both clients' replies"); + assert.deepEqual(a.received, [0, 1, 2, 3, 4].map((n) => `re:a${n}`)); + assert.deepEqual(b.received, [0, 1, 2, 3, 4].map((n) => `re:b${n}`)); + assert.equal(requests.length, 10); + assert.equal(a.connection.connected && b.connection.connected, true); + assert.equal(a.closed() || b.closed(), false); + assert.equal(server.peers.length, 2); + } finally { + a.connection.close(); b.connection.close(); server.close(); + } +}); + +test("a client reconnecting from the same endpoint replaces only its own session", async () => { + const { server, port, connected, left } = await startServer(); + const a = await startClient(port); + const b = await startClient(port); + try { + const oldPeer = connected[0]!; + a.connection.connect(); + await waitFor(() => connected.length === 3, "the reconnect to be accepted"); + assert.equal(left.length, 1); + assert.equal(left[0]!.peer, oldPeer); + assert.equal(left[0]!.reason, "replaced by a new Connect generation"); + assert.throws(() => server.send(oldPeer, "schema", enc("stale")), /not connected/); + assert.equal(server.peers.length, 2); + + a.connection.send("schema", enc("after")); + b.connection.send("schema", enc("still")); + await waitFor(() => a.received.includes("re:after") && b.received.includes("re:still"), "replies after the reconnect"); + assert.equal(b.closed(), false); + } finally { + a.connection.close(); b.connection.close(); server.close(); + } +}); + +test("a peer that leaves, or goes silent, is dropped and the others keep their sessions", async () => { + const { server, port, left } = await startServer({ peerTimeoutMs: 150 }); + const leaver = await rawClient(port); + const silent = await startClient(port); + const steady = await startClient(port); + try { + await leaver.leave(); + await waitFor(() => server.peers.length === 2, "the leaver to be dropped"); + assert.deepEqual(left.map((entry) => entry.reason), [""]); + + // Only the steady peer speaks; the silent one is reaped and told so. + const keepAlive = setInterval(() => steady.connection.ping(), 20); + try { + await waitFor(() => server.peers.length === 1, "the silent peer to be reaped"); + } finally { + clearInterval(keepAlive); + } + assert.ok(left.some((entry) => entry.reason === "session timed out")); + await waitFor(() => silent.closed(), "the reaped client to hear the goodbye"); + assert.equal(steady.closed(), false); + steady.connection.send("schema", enc("alive")); + await waitFor(() => steady.received.includes("re:alive"), "the surviving peer's reply"); + } finally { + silent.connection.close(); steady.connection.close(); server.close(); + } +}); + +test("Connects beyond maxPeers and datagrams from unadmitted endpoints are dropped", async () => { + const { server, port } = await startServer({ maxPeers: 1 }); + const first = await startClient(port); + const stranger = await bind(); + try { + const wire = Buffer.from(encodeRudpPacket(new CultNetRudpSession({ connectionId }).createConnect(Date.now()))); + stranger.send(wire, port, "127.0.0.1"); + const data = new CultNetRudpSession({ connectionId, initialSequence: 9 }); + data.createConnect(0); + stranger.send(Buffer.from(encodeRudpPacket(data.createPing())), port, "127.0.0.1"); + await waitFor(() => server.stats.packetsDropped >= 1, "the extra Connect to be dropped"); + await sleep(30); + assert.equal(server.peers.length, 1); + first.connection.send("schema", enc("ok")); + await waitFor(() => first.received.includes("re:ok"), "the admitted peer's reply"); + } finally { + stranger.close(); first.connection.close(); server.close(); + } +}); From ae0b9e234f885abfd45e4a4f20122b4349064a3e Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sat, 31 Oct 2026 01:32:01 +0100 Subject: [PATCH 31/88] cultnet-ts: changelog and README for the many-peer RUDP server Co-Authored-By: Claude Opus 5.5 --- packages/cultnet-ts/CHANGELOG.md | 4 ++++ packages/cultnet-ts/README.md | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/cultnet-ts/CHANGELOG.md b/packages/cultnet-ts/CHANGELOG.md index d37ff6dc..b87eebd3 100644 --- a/packages/cultnet-ts/CHANGELOG.md +++ b/packages/cultnet-ts/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to `cultnet-ts` are recorded here, following ## Unreleased +### Added + +- `CultNetRudpSocketTransportServer`: one UDP socket serving many peers, each remote address and port with its own reliable session (`peerConnected`, `frame`, `peerDisconnected` events; `send(peer, ...)` answers on the peer a frame came from). A second client no longer replaces the first; idle peers are reaped after `peerTimeoutMs` (default 60 s) and Connects beyond `maxPeers` (default 256) are dropped. Matches the Rust `CultNetRudpServerHub` and C# `CultNetRudpSocketTransportServer`; the wire format is unchanged. `CultNetRudpSocketTransportConnection` in `server` mode still serves exactly one peer. + ### Breaking - `CultMeshCdnChunkRef` carries `id` (slot 4), the reference's element id. `encodeManifest` writes five-slot chunk references and `packContent` sets the decimal offset as the id, as the reference does; a manifest written before ids reads `""` and is rewritten with the slot. Content vectors regenerated from the reference. diff --git a/packages/cultnet-ts/README.md b/packages/cultnet-ts/README.md index 0075ed9d..c766793b 100644 --- a/packages/cultnet-ts/README.md +++ b/packages/cultnet-ts/README.md @@ -21,7 +21,7 @@ The library currently includes: - `CultCacheTS` replication helpers for document put/delete/snapshot flows - schema discovery catalog messages and a registry for asking what contracts a peer can actually exchange safely -- RUDP socket transport, reconnect policy helpers, and +- RUDP socket transport (`CultNetRudpSocketTransportServer` serves many peers from one socket), reconnect policy helpers, and `CultNetRudpReconnectLoop` for controller-owned retry scheduling around a caller-owned socket factory - generated TypeScript contracts and runtime verifiers from canonical JSON From 94f20eee5e866a0bea94024a1b8aaeb4c1112af2 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sat, 31 Oct 2026 01:51:34 +0100 Subject: [PATCH 32/88] cultnet-ts server: close() waits for the goodbyes to leave the socket; peer reaping is opt-in and TypeScript-only Co-Authored-By: Claude Opus 5.5 --- packages/cultnet-ts/CHANGELOG.md | 2 +- packages/cultnet-ts/src/rudp-server.ts | 42 +++++++++++++++----- packages/cultnet-ts/test/rudp-server.test.ts | 32 +++++++++++++-- 3 files changed, 60 insertions(+), 16 deletions(-) diff --git a/packages/cultnet-ts/CHANGELOG.md b/packages/cultnet-ts/CHANGELOG.md index b87eebd3..eae823c4 100644 --- a/packages/cultnet-ts/CHANGELOG.md +++ b/packages/cultnet-ts/CHANGELOG.md @@ -7,7 +7,7 @@ All notable changes to `cultnet-ts` are recorded here, following ### Added -- `CultNetRudpSocketTransportServer`: one UDP socket serving many peers, each remote address and port with its own reliable session (`peerConnected`, `frame`, `peerDisconnected` events; `send(peer, ...)` answers on the peer a frame came from). A second client no longer replaces the first; idle peers are reaped after `peerTimeoutMs` (default 60 s) and Connects beyond `maxPeers` (default 256) are dropped. Matches the Rust `CultNetRudpServerHub` and C# `CultNetRudpSocketTransportServer`; the wire format is unchanged. `CultNetRudpSocketTransportConnection` in `server` mode still serves exactly one peer. +- `CultNetRudpSocketTransportServer`: one UDP socket serving many peers, each remote address and port with its own reliable session (`peerConnected`, `frame`, `peerDisconnected` events; `send(peer, ...)` answers on the peer a frame came from). A second client no longer replaces the first; Connects beyond `maxPeers` (default 256) are dropped, and `close()` resolves once every peer's goodbye has left the socket. The shape follows the Rust `CultNetRudpServerHub` and C# `CultNetRudpSocketTransportServer`, and the wire format is unchanged. The optional `peerTimeoutMs` (idle peers ended and told so; default off) exists only in TypeScript. `CultNetRudpSocketTransportConnection` in `server` mode still serves exactly one peer. ### Breaking diff --git a/packages/cultnet-ts/src/rudp-server.ts b/packages/cultnet-ts/src/rudp-server.ts index e6e68464..37e8695a 100644 --- a/packages/cultnet-ts/src/rudp-server.ts +++ b/packages/cultnet-ts/src/rudp-server.ts @@ -34,10 +34,12 @@ export interface CultNetRudpSocketServerOptions { /** A Connect from an endpoint beyond this many peers is dropped. Default 256. */ maxPeers?: number; /** - * A peer that has sent nothing for this long is ended and told so. Default 60 000; - * `Infinity` keeps idle peers until they leave. + * A peer that has sent nothing for this long is ended and told so. TypeScript only: the + * Rust and C# servers never reap. Default `Infinity`, which keeps idle peers until they leave. */ peerTimeoutMs?: number; + /** How long `close()` waits for the goodbyes to leave the socket. Default 1000. */ + closeGraceMs?: number; } /** One remote endpoint's session. A peer that reconnects from the same endpoint is a new object. */ @@ -90,6 +92,7 @@ export class CultNetRudpSocketTransportServer extends EventEmitter { readonly #maxPeers: number; readonly #peerTimeoutMs: number; #closed = false; + #closing = false; readonly #stats: CultNetTransportStats = { bytesReceived: 0, bytesSent: 0, @@ -104,7 +107,7 @@ export class CultNetRudpSocketTransportServer extends EventEmitter { this.#options = options; this.#socket = options.socket; this.#maxPeers = options.maxPeers ?? 256; - this.#peerTimeoutMs = options.peerTimeoutMs ?? 60_000; + this.#peerTimeoutMs = options.peerTimeoutMs ?? Infinity; if (!Number.isInteger(this.#maxPeers) || this.#maxPeers < 1) { throw new Error("RUDP server maxPeers must be a positive integer."); } @@ -169,13 +172,23 @@ export class CultNetRudpSocketTransportServer extends EventEmitter { this.#endPeer(entry, entry.session.createDisconnect(Uint8Array.from(reason))); } - /** Ends every peer and releases the socket. */ - close(): void { + /** + * Ends every peer and releases the socket once the goodbyes have left it (bounded by + * `closeGraceMs`, default 1000): closing the socket at once would drop them. + */ + async close(): Promise { if (this.#closed) return; + this.#closing = true; clearInterval(this.#timer); - for (const entry of [...this.#entries.values()]) { - this.#endPeer(entry, entry.session.createDisconnect(utf8("server closed"))); - } + const sent = [...this.#entries.values()].map((entry) => + this.#endPeer(entry, entry.session.createDisconnect(utf8("server closed")))); + let bound: NodeJS.Timeout | undefined; + await Promise.race([ + Promise.all(sent), + new Promise((resolve) => { bound = setTimeout(resolve, this.#options.closeGraceMs ?? 1_000); }), + ]); + clearTimeout(bound); + if (this.#closed) return; this.#closed = true; this.#socket.close(); } @@ -189,17 +202,24 @@ export class CultNetRudpSocketTransportServer extends EventEmitter { } /** Removes the peer, tells the caller, and says goodbye on the wire. Idempotent per entry. */ - #endPeer(entry: PeerEntry, goodbye: CultNetRudpPacket): void { - if (this.#entries.get(entry.key) !== entry) return; + #endPeer(entry: PeerEntry, goodbye: CultNetRudpPacket): Promise { + if (this.#entries.get(entry.key) !== entry) return Promise.resolve(); this.#entries.delete(entry.key); this.emit("peerDisconnected", { peer: entry.peer, reason: goodbye.payload }); // Best-effort: the session is over whether or not the peer hears it. const wire = encodeRudpPacket(goodbye); this.#stats.bytesSent += wire.length; - sendRudpDatagram(this.#socket, wire, entry.peer.remotePort, entry.peer.remoteHost, () => {}); + return new Promise((resolve) => { + try { + this.#socket.send(wire, entry.peer.remotePort, entry.peer.remoteHost, () => resolve()); + } catch { + resolve(); + } + }); } #receiveDatagram(wire: Buffer, remote: RemoteInfo): void { + if (this.#closing) return; this.#stats.bytesReceived += wire.length; // What a datagram carries is the sender's business: a malformed frame, another // session's connection id and a sender no Connect admitted are dropped and counted. diff --git a/packages/cultnet-ts/test/rudp-server.test.ts b/packages/cultnet-ts/test/rudp-server.test.ts index d35ca9f7..6ba976d3 100644 --- a/packages/cultnet-ts/test/rudp-server.test.ts +++ b/packages/cultnet-ts/test/rudp-server.test.ts @@ -97,7 +97,7 @@ test("two clients on different ports talk through one server socket and neither assert.equal(a.closed() || b.closed(), false); assert.equal(server.peers.length, 2); } finally { - a.connection.close(); b.connection.close(); server.close(); + a.connection.close(); b.connection.close(); await server.close(); } }); @@ -120,7 +120,7 @@ test("a client reconnecting from the same endpoint replaces only its own session await waitFor(() => a.received.includes("re:after") && b.received.includes("re:still"), "replies after the reconnect"); assert.equal(b.closed(), false); } finally { - a.connection.close(); b.connection.close(); server.close(); + a.connection.close(); b.connection.close(); await server.close(); } }); @@ -147,7 +147,7 @@ test("a peer that leaves, or goes silent, is dropped and the others keep their s steady.connection.send("schema", enc("alive")); await waitFor(() => steady.received.includes("re:alive"), "the surviving peer's reply"); } finally { - silent.connection.close(); steady.connection.close(); server.close(); + silent.connection.close(); steady.connection.close(); await server.close(); } }); @@ -167,6 +167,30 @@ test("Connects beyond maxPeers and datagrams from unadmitted endpoints are dropp first.connection.send("schema", enc("ok")); await waitFor(() => first.received.includes("re:ok"), "the admitted peer's reply"); } finally { - stranger.close(); first.connection.close(); server.close(); + stranger.close(); first.connection.close(); await server.close(); + } +}); + +test("close() delivers a Disconnect to every peer before the socket is released", async () => { + const { server, port } = await startServer(); + const peers = await Promise.all([0, 1, 2].map(async () => { + const socket = await bind(); + const session = new CultNetRudpSession({ connectionId }); + const types: string[] = []; + socket.on("message", (wire) => { + const packet = decodeRudpPacket(wire); + types.push(packet.packetType); + session.receive(packet, Date.now()); + }); + socket.send(encodeRudpPacket(session.createConnect(Date.now())), port, "127.0.0.1"); + await waitFor(() => session.connected, "raw handshake"); + return { socket, types }; + })); + try { + assert.equal(server.peers.length, 3); + await server.close(); + await waitFor(() => peers.every((peer) => peer.types.includes("disconnect")), "every peer to observe the Disconnect"); + } finally { + peers.forEach((peer) => peer.socket.close()); } }); From 2d0adb92cc9f26654557d9482d3596e9cb1cd709 Mon Sep 17 00:00:00 2001 From: libby Date: Sat, 31 Oct 2026 21:52:18 +0000 Subject: [PATCH 33/88] cultmath: add fbm, the value-only octave sum of snoise, with its HLSL and GLSL mirrors math.fbm(p, octaves, lacunarity, gain) and math.fbm(p) (four octaves, lacunarity 2, gain 0.5); cultmath_fbm in the shaders, the GLSL regenerated by GlslLowering, a golden-fixture family appended, FbmTests for the rules. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultmath/README.md | 3 +- packages/cultmath/shaders/CultMath.glsl | 20 + packages/cultmath/shaders/CultMath.hlsl | 20 + packages/cultmath/src/CultMath/math.cs | 23 + .../cultmath/tests/CultMath.Tests/FbmTests.cs | 73 ++++ .../tests/CultMath.Tests/GlslMirrorTests.cs | 1 + .../HlslSourceCompatibilityTests.cs | 4 +- .../CultMath.Tests/fixtures/glsl-parity.json | 402 +++++++++++++++--- .../unity/org.gamecult.cultmath/CHANGELOG.md | 8 + .../Shaders/CultMath.hlsl | 20 + 10 files changed, 502 insertions(+), 72 deletions(-) create mode 100644 packages/cultmath/tests/CultMath.Tests/FbmTests.cs diff --git a/packages/cultmath/README.md b/packages/cultmath/README.md index 381b194a..b144c56d 100644 --- a/packages/cultmath/README.md +++ b/packages/cultmath/README.md @@ -29,7 +29,8 @@ shader semantics. `cross`, `length`, `distance`, `normalize`, `reflect`, `csum`, `erf`, `erfinv` (scalar only), exponential `decay`/`damp`, intercept/segment helpers (`distance_to_segment` in 2D and 3D, with and without `out closest`, and `segment_closest`), Catmull-Rom, Bezier curves, and - deterministic value-noise primitives + deterministic value-noise primitives, and `fbm(p)` / `fbm(p, octaves, lacunarity, gain)`, the value-only octave sum + of `snoise` (`cultmath_fbm` in HLSL and GLSL) - Site cells: `power_cells` and `multiplicative_cells` return the two nearest sites of a point as `CultSiteCells`, `soft_gabriel_gate` gates a pair of sites by the empty-disc margin. Each is a loop over the shader steps `cultmath_power_score` / `cultmath_multiplicative_score`, `cultmath_site_cells_empty`, `cultmath_site_cells_insert`, diff --git a/packages/cultmath/shaders/CultMath.glsl b/packages/cultmath/shaders/CultMath.glsl index 6ebf3f36..529ad6e8 100644 --- a/packages/cultmath/shaders/CultMath.glsl +++ b/packages/cultmath/shaders/CultMath.glsl @@ -342,6 +342,26 @@ vec4 cultmath_fbm_grad(vec3 p, int octaves, float lacunarity, float gain) return vec4(gradient, value); } +// Value-only octave sum of cultmath_snoise; see the C# math.fbm comment. The one-argument form is four octaves, +// lacunarity 2, gain 0.5. +float cultmath_fbm(vec3 p, int octaves, float lacunarity, float gain) +{ + octaves = clamp(octaves, 0, 16); + float amplitude = 1.0; + float frequency = 1.0; + float value = 0.0; + for (int i = 0; i < octaves; i++) + { + value += amplitude * cultmath_snoise(p * frequency); + frequency *= lacunarity; + amplitude *= gain; + } + + return value; +} + +float cultmath_fbm(vec3 value) { return cultmath_fbm(value, 4, 2.0, 0.5); } + // Ridged multifractal noise (Musgrave, Texturing and Modeling, ch. 16); see the C# math.ridged_grad // comment for the fold/sign derivation, the deliberate crease at n = 0, and why octaves is clamped. vec4 cultmath_ridged_grad(vec3 p, int octaves, float lacunarity, float gain) diff --git a/packages/cultmath/shaders/CultMath.hlsl b/packages/cultmath/shaders/CultMath.hlsl index d3eea396..fc9f7a70 100644 --- a/packages/cultmath/shaders/CultMath.hlsl +++ b/packages/cultmath/shaders/CultMath.hlsl @@ -342,6 +342,26 @@ float4 cultmath_fbm_grad(float3 p, int octaves, float lacunarity, float gain) return float4(gradient, value); } +// Value-only octave sum of cultmath_snoise; see the C# math.fbm comment. The one-argument form is four octaves, +// lacunarity 2, gain 0.5. +float cultmath_fbm(float3 p, int octaves, float lacunarity, float gain) +{ + octaves = clamp(octaves, 0, 16); + float amplitude = 1.0; + float frequency = 1.0; + float value = 0.0; + for (int i = 0; i < octaves; i++) + { + value += amplitude * cultmath_snoise(p * frequency); + frequency *= lacunarity; + amplitude *= gain; + } + + return value; +} + +float cultmath_fbm(float3 value) { return cultmath_fbm(value, 4, 2.0, 0.5); } + // Ridged multifractal noise (Musgrave, Texturing and Modeling, ch. 16); see the C# math.ridged_grad // comment for the fold/sign derivation, the deliberate crease at n = 0, and why octaves is clamped. float4 cultmath_ridged_grad(float3 p, int octaves, float lacunarity, float gain) diff --git a/packages/cultmath/src/CultMath/math.cs b/packages/cultmath/src/CultMath/math.cs index 0d22dc4c..a7c41f48 100644 --- a/packages/cultmath/src/CultMath/math.cs +++ b/packages/cultmath/src/CultMath/math.cs @@ -823,6 +823,29 @@ public static partial class math return new float4(gradient, value); } + // Value-only fractional Brownian motion: the octave sum of snoise, octave i at frequency lacunarity^i and + // amplitude gain^i. It is fbm_grad(p, octaves, lacunarity, gain).w without the gradient work (to float + // rounding: snoise and snoise_grad.w order their arithmetic differently), so a loop that only wants the height + // (a sphere trace) does not pay for the gradient. octaves clamps to [0, 16] as fbm_grad's does. The one-argument + // form is the stock four octaves, lacunarity 2, gain 1/2: its range is within +-1.875 times snoise's. + public static float fbm(float3 p, int octaves, float lacunarity, float gain) + { + octaves = clamp(octaves, 0, 16); + var amplitude = 1.0f; + var frequency = 1.0f; + var value = 0.0f; + for (var i = 0; i < octaves; i++) + { + value += amplitude * snoise(p * frequency); + frequency *= lacunarity; + amplitude *= gain; + } + + return value; + } + + public static float fbm(float3 value) => fbm(value, 4, 2.0f, 0.5f); + // Ridged multifractal noise (Musgrave, Texturing and Modeling, ch. 16; the operator's dune term, // 2026-09-25): each octave folds snoise_grad about zero, value = Sum a_i*(1 - |n_i|), gradient = // -Sum a_i*f_i*sign(n_i)*grad(n_i), the same per-octave frequency scaling as fbm_grad's chain diff --git a/packages/cultmath/tests/CultMath.Tests/FbmTests.cs b/packages/cultmath/tests/CultMath.Tests/FbmTests.cs new file mode 100644 index 00000000..89ca4dc9 --- /dev/null +++ b/packages/cultmath/tests/CultMath.Tests/FbmTests.cs @@ -0,0 +1,73 @@ +using System; +using CultMath; +using Xunit; + +namespace CultMath.Tests; + +public sealed class FbmTests +{ + private static float3 Point(System.Random r) => new(r.NextSingle() * 20.0f - 10.0f, r.NextSingle() * 20.0f - 10.0f, r.NextSingle() * 20.0f - 10.0f); + + [Fact] + public void OneOctaveIsSnoise() + { + var random = new System.Random(0xFB01); + for (var i = 0; i < 64; i++) + { + var p = Point(random); + Assert.Equal(math.snoise(p), math.fbm(p, 1, 1.0f + random.NextSingle() * 3.0f, 0.1f + random.NextSingle() * 0.8f)); + } + } + + [Fact] + public void ZeroOrNegativeOctavesSumNothing() + { + var p = new float3(0.3f, 1.7f, -2.2f); + Assert.Equal(0.0f, math.fbm(p, 0, 2.0f, 0.5f)); + Assert.Equal(0.0f, math.fbm(p, -5, 2.0f, 0.5f)); + } + + [Fact] + public void OctavesCompoundGainAndLacunarityIndependently() + { + // gain != lacunarity and neither is 1, so applying either once, or swapping them, changes the sum. + const float lacunarity = 2.3f; + const float gain = 0.4f; + var random = new System.Random(0xFB02); + for (var i = 0; i < 64; i++) + { + var p = Point(random); + var expected = math.snoise(p) + gain * math.snoise(p * lacunarity) + gain * gain * math.snoise(p * (lacunarity * lacunarity)); + Assert.Equal(expected, math.fbm(p, 3, lacunarity, gain), 5); + } + } + + [Fact] + public void ValueIsFbmGradsValue() + { + var random = new System.Random(0xFB03); + for (var i = 0; i < 128; i++) + { + var p = Point(random); + Assert.Equal(math.fbm_grad(p, 5, 2.0f, 0.55f).w, math.fbm(p, 5, 2.0f, 0.55f), 4); + } + } + + [Fact] + public void OctaveCountClampsToSixteenLikeFbmGrad() + { + var p = new float3(0.3f, 1.7f, -2.2f); + Assert.Equal(math.fbm(p, 16, 1.1f, 0.9f), math.fbm(p, 1000, 1.1f, 0.9f)); + } + + [Fact] + public void OneArgumentFormIsFourOctavesLacunarityTwoGainHalf() + { + var random = new System.Random(0xFB04); + for (var i = 0; i < 64; i++) + { + var p = Point(random); + Assert.Equal(math.fbm(p, 4, 2.0f, 0.5f), math.fbm(p)); + } + } +} diff --git a/packages/cultmath/tests/CultMath.Tests/GlslMirrorTests.cs b/packages/cultmath/tests/CultMath.Tests/GlslMirrorTests.cs index c2b604f6..10e08380 100644 --- a/packages/cultmath/tests/CultMath.Tests/GlslMirrorTests.cs +++ b/packages/cultmath/tests/CultMath.Tests/GlslMirrorTests.cs @@ -904,6 +904,7 @@ static int4 clamp(int4 x, int4 a, int4 b) { if (a.x > b.x || a.y > b.y || a.z > // The site-array functions: a fixed-capacity array and a count, killed on the device (the lattice at rings up to 17). new Family($"vec4 cultmath_power_border(vec3, vec4[{SiteArrays.Capacity}], int)", "ulp-bounded", SiteArrays.PowerBorderArguments), new Family($"vec4 cultmath_weighted_lattice_cells(vec2, float, vec4[{SiteArrays.Capacity}], int, float, float, float, float, float, vec2, float)", "ulp-bounded", SiteArrays.LatticeArguments), + new Family("float cultmath_fbm(vec3, int, float, float)", "ulp-bounded", r => new object[] { Point3(r), 4, 2.0f, 0.5f }), }).ToArray(); // A point of magnitude m, m log-uniform in [lo, hi]: one component at +-m, the other two uniform in [-m, m], diff --git a/packages/cultmath/tests/CultMath.Tests/HlslSourceCompatibilityTests.cs b/packages/cultmath/tests/CultMath.Tests/HlslSourceCompatibilityTests.cs index 2c994be8..821ba94a 100644 --- a/packages/cultmath/tests/CultMath.Tests/HlslSourceCompatibilityTests.cs +++ b/packages/cultmath/tests/CultMath.Tests/HlslSourceCompatibilityTests.cs @@ -47,6 +47,8 @@ public sealed class HlslSourceCompatibilityTests Assert.NotNull(ShaderFunction(shader, "cultmath_snoise_hessian", typeof(float3))); Assert.NotNull(ShaderFunction(shader, "cultmath_integral_clamp_linear", typeof(float), typeof(float), typeof(float), typeof(float), typeof(float), typeof(float))); Assert.NotNull(ShaderFunction(shader, "cultmath_fbm_grad", typeof(float3), typeof(int), typeof(float), typeof(float))); + Assert.NotNull(ShaderFunction(shader, "cultmath_fbm", typeof(float3), typeof(int), typeof(float), typeof(float))); + Assert.NotNull(ShaderFunction(shader, "cultmath_fbm", typeof(float3))); Assert.NotNull(ShaderFunction(shader, "cultmath_ridged_grad", typeof(float3), typeof(int), typeof(float), typeof(float))); Assert.NotNull(ShaderFunction(shader, "cultmath_phacelle", typeof(float3), typeof(float3), typeof(float), typeof(float))); Assert.NotNull(ShaderFunction(shader, "cultmath_iv_snoise_ball", typeof(float3), typeof(float))); @@ -205,7 +207,7 @@ public sealed class HlslSourceCompatibilityTests Assert.True(mismatches.Count == 0, $"{mismatches.Count} mismatches:{Environment.NewLine}{string.Join(Environment.NewLine, mismatches.Take(12))}"); // The three site-array functions are compared by SiteArrayMirrorsMatchCSharp, which draws their counts and arrays. - Assert.Equal(140, compared.Count); + Assert.Equal(141, compared.Count); } /// diff --git a/packages/cultmath/tests/CultMath.Tests/fixtures/glsl-parity.json b/packages/cultmath/tests/CultMath.Tests/fixtures/glsl-parity.json index f2ab1f19..244d068c 100644 --- a/packages/cultmath/tests/CultMath.Tests/fixtures/glsl-parity.json +++ b/packages/cultmath/tests/CultMath.Tests/fixtures/glsl-parity.json @@ -6820,7 +6820,7 @@ { "glsl": "vec2 cultmath_iv_exp(vec2)", "tolerance": "ulp-bounded, platform exp", - "platform": "Ubuntu 24.04.5 LTS; .NET 10.0.12", + "platform": "Debian GNU/Linux 13 (trixie); .NET 10.0.14", "cases": [ "41202b2a 4120314e -> 46ade79e 46ae2a6b", "c0d1a016 bfb9db96 -> 3abb4cbc 3e6fb77f", @@ -7083,7 +7083,7 @@ { "glsl": "CultPhasor cultmath_phacelle(vec3, vec3, float, float)", "tolerance": "ulp-bounded, platform exp, sin, cos; across platforms |diff| <= 2^-20 max(|v|, 1); on a device |diff| <= 2^-18 max(|v|, 1)", - "platform": "Ubuntu 24.04.5 LTS; .NET 10.0.12", + "platform": "Debian GNU/Linux 13 (trixie); .NET 10.0.14", "cases": [ "c01d8630 41b2cd8c 41ea274c bfa19894 40aeb6ce 40918978 3e9be843 3f000000 -> bf74d9fb 3fb71956 bf35fe4e 3f21280a c01277e2 403b5fe4 40464bbd bd5b4455", "c2234cf7 4237b112 419ae0c0 c0acbcd4 bf60f9a0 bfeb2b08 3ebf0a10 3f000000 -> c0d0048f 3f228da3 bfc85f60 3edaf41b c0171c80 be137f8c be1c92a3 bf5747bf", @@ -12591,7 +12591,7 @@ { "glsl": "float cultmath_cycles_power(float, float)", "tolerance": "ulp-bounded, platform pow", - "platform": "Ubuntu 24.04.5 LTS; .NET 10.0.12", + "platform": "Debian GNU/Linux 13 (trixie); .NET 10.0.14", "cases": [ "bfb8bc1c 00000000 -> 3f800000", "3f7fc090 3fa15ed0 -> 3f7fb009", @@ -12854,7 +12854,7 @@ { "glsl": "float cultmath_cycles_logarithm(float, float)", "tolerance": "ulp-bounded, platform log", - "platform": "Ubuntu 24.04.5 LTS; .NET 10.0.12", + "platform": "Debian GNU/Linux 13 (trixie); .NET 10.0.14", "cases": [ "416175a4 418f2e6d -> 3f6ac789", "40d2d3ad 416ecd13 -> 3f328da7", @@ -24907,7 +24907,7 @@ { "glsl": "float cultmath_cycles_arcsine(float)", "tolerance": "ulp-bounded, platform asin", - "platform": "Ubuntu 24.04.5 LTS; .NET 10.0.12", + "platform": "Debian GNU/Linux 13 (trixie); .NET 10.0.14", "cases": [ "3f74439c -> 3fa22776", "beaa06dc -> bead5166", @@ -24919,27 +24919,27 @@ "bf44c25b -> bf606b2a", "be9326d8 -> be9541be", "bf047a79 -> bf0b3e4e", - "3f6fb444 -> 3f9b2514", + "3f6fb444 -> 3f9b2513", "bec3bc1c -> bec8d8b0", "3ed94bc8 -> 3ee069a8", "be732cb0 -> be75852c", "bf525772 -> bf76dbfc", "3f5f4508 -> 3f87a01e", - "3f5243a0 -> 3f76b93a", + "3f5243a0 -> 3f76b939", "3f6f8480 -> 3f9ae13d", "bf43af08 -> bf5ebe7a", "3fb52362 -> 3fc90fdb", "3eae8380 -> 3eb2152a", "bf3ff8fb -> bf590ffc", "3f7bb06c -> 3fb189e5", - "3dee93e0 -> 3def1ede", + "3dee93e0 -> 3def1edd", "bfb0b96e -> bfc90fdb", "bf454824 -> bf613cb1", "3f23fb40 -> 3f31f9c7", "be9b0000 -> be9d788d", "3fb7c110 -> 3fc90fdb", "bf0c30e4 -> bf145b51", - "3e855c0c -> 3e86ea5c", + "3e855c0c -> 3e86ea5d", "bf0e2123 -> bf16ae2c", "3f72c708 -> 3f9fbe32", "3f98afd8 -> 3fc90fdb", @@ -24970,7 +24970,7 @@ "3f81d10a -> 3fc90fdb", "bf033714 -> bf09c506", "3fa3a04a -> 3fc90fdb", - "3f1281b8 -> 3f1bfaf2", + "3f1281b8 -> 3f1bfaf3", "bf34447a -> bf480038", "bf9eb144 -> bfc90fdb", "3f38d780 -> 3f4e86c5", @@ -25014,7 +25014,7 @@ "3f039538 -> 3f0a32b8", "3f4cd500 -> 3f6d70e3", "3ee655e4 -> 3eeee961", - "3ef23d20 -> 3efc5474", + "3ef23d20 -> 3efc5473", "3e0c9370 -> 3e0d0572", "3f9f079c -> 3fc90fdb", "3e4b1770 -> 3e4c7278", @@ -25038,7 +25038,7 @@ "3f982f94 -> 3fc90fdb", "3ef183b4 -> 3efb8213", "bed4d500 -> bedb7e63", - "bf49041d -> bf672e44", + "bf49041d -> bf672e45", "bf986f2a -> bfc90fdb", "3eeeeb04 -> 3ef89167", "bf05421c -> bf0c27dd", @@ -25072,7 +25072,7 @@ "3f48bf60 -> 3f66bf61", "3f953182 -> 3fc90fdb", "3e21e6d0 -> 3e229577", - "bf043606 -> bf0aee58", + "bf043606 -> bf0aee57", "3d94f740 -> 3d9518f5", "3c15e400 -> 3c15e489", "bfb51e92 -> bfc90fdb", @@ -25103,7 +25103,7 @@ "3f4133b0 -> 3f5aedab", "3f25a7cc -> 3f3429d9", "3f4f5e44 -> 3f71b78b", - "3f795244 -> 3fabc288", + "3f795244 -> 3fabc287", "3de214a0 -> 3de28ad3", "3e8fe798 -> 3e91deba", "bf9af793 -> bfc90fdb", @@ -25127,7 +25127,7 @@ "3ecad4fc -> 3ed08d21", "3e05a058 -> 3e06022f", "beb65270 -> beba69cf", - "3f5058c0 -> 3f7364a8", + "3f5058c0 -> 3f7364a7", "3fa6360c -> 3fc90fdb", "3f466ed4 -> 3f630d19", "3f700570 -> 3f9b993a", @@ -25144,12 +25144,12 @@ "beaaf660 -> beae4f6a", "3f92651c -> 3fc90fdb", "3f8dddfa -> 3fc90fdb", - "3f2ed97c -> 3f407a92", + "3f2ed97c -> 3f407a93", "be5e0a78 -> be5fd19b", "bf794a5d -> bfabb123", "bf166554 -> bf20c10e", "3ec7db68 -> 3ecd50d0", - "3f0f3a38 -> 3f1800b6", + "3f0f3a38 -> 3f1800b5", "bf9d2433 -> bfc90fdb", "be1b6e70 -> be1c08d6", "3f8cf9f4 -> 3fc90fdb", @@ -25170,7 +25170,7 @@ { "glsl": "float cultmath_cycles_arccosine(float)", "tolerance": "ulp-bounded, platform acos", - "platform": "Ubuntu 24.04.5 LTS; .NET 10.0.12", + "platform": "Debian GNU/Linux 13 (trixie); .NET 10.0.14", "cases": [ "3da419e0 -> 3fbecb6b", "bccc7dc0 -> 3fcc41e7", @@ -25185,7 +25185,7 @@ "3f99f728 -> 00000000", "3efe6a30 -> 3f867f99", "bf8e3141 -> 40490fdb", - "be1ccb60 -> 3fdcbd18", + "be1ccb60 -> 3fdcbd17", "bf6312e3 -> 402a57f6", "beca9b18 -> 3ffd2360", "bfa261f2 -> 40490fdb", @@ -25221,7 +25221,7 @@ "3e8561b8 -> 3fa753cc", "bf888ac6 -> 40490fdb", "3fbb7d54 -> 00000000", - "3ec05018 -> 3f97c656", + "3ec05018 -> 3f97c657", "bf8c0f8c -> 40490fdb", "bf72b2f0 -> 40345736", "3f244db8 -> 3f5fba77", @@ -25250,7 +25250,7 @@ "3f8f186c -> 00000000", "3f30b1e8 -> 3f4f1b53", "3d953310 -> 3fbfba8c", - "bde0dee0 -> 3fd7250e", + "bde0dee0 -> 3fd7250d", "3f7b8300 -> 3e4007f6", "3f35601c -> 3f488ecf", "bf9a050a -> 40490fdb", @@ -25272,7 +25272,7 @@ "bfb4c70e -> 40490fdb", "3e93057c -> 3fa3c820", "3f2b8624 -> 3f56293a", - "bf51e010 -> 40220aae", + "bf51e010 -> 40220aaf", "bf8aa06c -> 40490fdb", "3f9839c8 -> 00000000", "3f28eed8 -> 3f59a156", @@ -25297,7 +25297,7 @@ "3e8aea00 -> 3fa5e480", "3ed35888 -> 3f9298c3", "3f567510 -> 3f13e3ce", - "3e9761c8 -> 3fa2a464", + "3e9761c8 -> 3fa2a463", "3fbc3ca0 -> 00000000", "bf5814bd -> 4024d6ee", "3f807f68 -> 00000000", @@ -25306,7 +25306,7 @@ "3f1a4008 -> 3f6c92ef", "3fa491d4 -> 00000000", "3f385e00 -> 3f44484f", - "3e9ac620 -> 3fa1c0e6", + "3e9ac620 -> 3fa1c0e5", "3e9541ec -> 3fa3329f", "bf647da0 -> 402b1eac", "3dc1c080 -> 3fbcef2e", @@ -25351,7 +25351,7 @@ "beab7798 -> 3ff4c5fc", "3f0df694 -> 3f7ba4b1", "3f94dda6 -> 00000000", - "bf5c7d84 -> 4026f34c", + "bf5c7d84 -> 4026f34d", "3f9296ba -> 00000000", "bfbada1e -> 40490fdb", "3f8cf40a -> 00000000", @@ -25364,7 +25364,7 @@ "bf6b7605 -> 402f4022", "bfad549a -> 40490fdb", "3faa5d8c -> 00000000", - "3e8b7c88 -> 3fa5be6e", + "3e8b7c88 -> 3fa5be6f", "3f3fa580 -> 3f398dc6", "bf61690a -> 4029746e", "bf73a264 -> 403516b7", @@ -25427,13 +25427,13 @@ "3f930316 -> 00000000", "bf6bfdd5 -> 402f9730", "3fa53312 -> 00000000", - "bf1a0e9a -> 400ddba4" + "bf1a0e9a -> 400ddba5" ] }, { "glsl": "float cultmath_cycles_arctan2(float, float)", "tolerance": "ulp-bounded, platform atan", - "platform": "Ubuntu 24.04.5 LTS; .NET 10.0.12", + "platform": "Debian GNU/Linux 13 (trixie); .NET 10.0.14", "cases": [ "c08cbc16 c1549681 -> c0349aac", "418c8108 41948858 -> 3f41f3ee", @@ -25442,7 +25442,7 @@ "c1853a3e 4183660c -> bf4ad4c8", "40cc3750 c18864ad -> 4032239a", "c1841fce 414eb7b4 -> bf682df1", - "3f7f11c0 41915dcc -> 3d605f59", + "3f7f11c0 41915dcc -> 3d605f58", "c1190690 40822148 -> bf959a14", "41235546 414a8e7c -> 3f2db91b", "c173027e 4157ae48 -> bf584be7", @@ -25451,12 +25451,12 @@ "41504298 c1432d3c -> 4014b8bf", "41976732 4165afb0 -> 3f6bff17", "c12faa83 c19c5692 -> c0284d67", - "c0aa038c c0cb2194 -> c01c7634", + "c0aa038c c0cb2194 -> c01c7633", "c0df2a78 c0a67120 -> c00d8b25", "4144767c c11a39a5 -> 400f2006", "c00c8aa0 4194be56 -> bdf0c4af", "3f876310 c1937378 -> 404564aa", - "41636814 c1172a84 -> 400a13e2", + "41636814 c1172a84 -> 400a13e1", "3f81e2e0 bd4f6800 -> 3fcf71a7", "41309010 c08f8f1c -> 3ffa7c90", "40982a9c 3f904760 -> 3fab4541", @@ -25467,23 +25467,23 @@ "4192314a 4086f2c0 -> 3fac080a", "411b37ca c171da02 -> 40248b70", "c1664f18 c130242c -> c00e512f", - "40eced38 40bd9198 -> 3f655f42", - "4104386c 411d5130 -> 3f32ed70", + "40eced38 40bd9198 -> 3f655f43", + "4104386c 411d5130 -> 3f32ed71", "bdd98800 c15cd654 -> c04891c6", "c161bae2 c19f6675 -> c021a16c", "c15bccb4 4139d89a -> bf5e709e", "c1065de6 c1860dc7 -> c02b5428", "419a2318 416c82f8 -> 3f6a973f", - "bf911740 3eafb040 -> bfa36fa4", + "bf911740 3eafb040 -> bfa36fa5", "41961b7c c01fd768 -> 3fd9fffb", "4042f5d8 c00cea20 -> 400c95f1", - "c12f053d 418e307c -> bf0d3c4a", + "c12f053d 418e307c -> bf0d3c4b", "403d4400 c11dc866 -> 40366a62", "c15831c7 412e7716 -> bf644dcc", "c0d29330 41698de0 -> bed8d810", "c0fd3872 415963f8 -> bf0702af", "c131bb1f c074e670 -> bff386a5", - "417f6828 40a71880 -> 3fa09868", + "417f6828 40a71880 -> 3fa09869", "40497b80 41717390 -> 3e529a28", "413af82e c196608f -> 40257713", "4179eb80 bfb8a2f0 -> 3fd4d959", @@ -25492,8 +25492,8 @@ "414f2488 c180a1ba -> 401dad4f", "4036b5f0 c15671f1 -> 403ba162", "bf97a3d0 c0046bb8 -> c027c818", - "40ae84d8 c0f9a4b6 -> 402203ec", - "3feb4440 40f9d6e0 -> 3e6cc1c5", + "40ae84d8 c0f9a4b6 -> 402203eb", + "3feb4440 40f9d6e0 -> 3e6cc1c6", "419f7088 411a15a2 -> 3f8f723b", "c188441a bfeff970 -> bfd71812", "40560bd0 410fd5de -> 3eb65b86", @@ -25508,10 +25508,10 @@ "414c9fc4 418893dc -> 3f249745", "c033b3b8 c18b7729 -> c03ed777", "c13f3667 c152d7a7 -> c019eb30", - "c086fb50 41627eb4 -> be9446ba", + "c086fb50 41627eb4 -> be9446b9", "c13dde75 41858b9e -> bf1e348f", "41655ddc c13d8347 -> 4010b9bb", - "40a95660 41584940 -> 3ebf0aa4", + "40a95660 41584940 -> 3ebf0aa3", "4141fbc4 c0ce1146 -> 4003c77a", "c0091050 c155ea5c -> c03ee580", "40b8f0c0 c18079b9 -> 4032f416", @@ -25524,17 +25524,17 @@ "40a7d388 c07a5f68 -> 400d8c45", "4186baee c161afee -> 4011273b", "c1988920 3f471480 -> bfc3d822", - "3f70c440 40be95d8 -> 3e2060cf", + "3f70c440 40be95d8 -> 3e2060d0", "417834e8 415d5098 -> 3f57b555", "c1775234 40c0f8f8 -> bf99734e", "c1974114 4198ffc8 -> bf479800", "410c0ec4 bff12930 -> 3fe432a6", "413a6dc4 c09ce414 -> 3ffc0b0b", "be987ac0 c097edfc -> c0450d7e", - "bfa53460 4194fa8e -> bd8db6e0", + "bfa53460 4194fa8e -> bd8db6e1", "40d38cc8 c19c290d -> 403429aa", "415e9230 bea6b840 -> 3fcc0ec1", - "c1612e94 4199eb08 -> bf21ad98", + "c1612e94 4199eb08 -> bf21ad99", "c102419c 3f3e0580 -> bfbd6c6a", "c171e10c 3e75cd00 -> bfc70799", "c159f7c3 c040e368 -> bfe4ee5c", @@ -25543,18 +25543,18 @@ "c0ebecc4 4188a552 -> bed0a0ee", "40b612c0 419e96ba -> 3e8f1b96", "c1392859 c184cf09 -> c02219d5", - "40f88828 c0379bc8 -> 3ff65a7b", + "40f88828 c0379bc8 -> 3ff65a7c", "4183657e 418c205a -> 3f40d57c", "3d64c400 c049c398 -> 4047eda0", - "c184e5e2 c0ba5c6c -> bff438b2", - "c13cf1d2 c09df406 -> bffbbd3f", + "c184e5e2 c0ba5c6c -> bff438b3", + "c13cf1d2 c09df406 -> bffbbd40", "41951ea0 40d3e958 -> 3f9d5e14", "4117a1c4 4182e3fa -> 3f0666f9", "c100421b 3fa11c00 -> bfb52067", "c0a4c048 3fd9b1a0 -> bfa0392a", "c0bce6f0 c0a756e0 -> c012ed67", - "410f31a8 4089ad2c -> 3f8fb3fa", - "beb17800 40836d64 -> bdac6ea3", + "410f31a8 4089ad2c -> 3f8fb3f9", + "beb17800 40836d64 -> bdac6ea4", "c0724d88 bee688c0 -> bfd836b1", "3fb83bc0 c1693dec -> 4042c367", "c18c3dd4 4027c610 -> bfb60fc6", @@ -25562,7 +25562,7 @@ "41231d74 404d91d0 -> 3fa1fe4e", "c0e92c5a c04e1b80 -> bffe543d", "404ffd08 c1396f46 -> 403790b4", - "bf50eb60 41702f48 -> bd5e7502", + "bf50eb60 41702f48 -> bd5e7501", "c0294f90 412b4d2a -> be780e6d", "c0fe6df0 41350240 -> bf1cd3f4", "bfd3b0d0 41824f24 -> bdcf3ab0", @@ -25571,15 +25571,15 @@ "400eab20 c180166a -> 4040354a", "40b741d8 415a8318 -> 3ecb4b31", "c0b19f72 c1844941 -> c03455ba", - "418bd268 c0f5d76c -> 3ffe1262", + "418bd268 c0f5d76c -> 3ffe1263", "3fa6a0a0 c19bf37a -> 4044cb60", "417687a4 418fb914 -> 3f357bee", - "c104b28d 40b23950 -> bf7abbaa", + "c104b28d 40b23950 -> bf7abba9", "40b00cb0 be7bb000 -> 3fcec6d5", "41836b9a c15bc2b2 -> 40111937", "40697890 417344c4 -> 3e7121a8", "4190121e c158f0df -> 400dd562", - "c18bdd8b bf6a2c00 -> bfcfc0bd", + "c18bdd8b bf6a2c00 -> bfcfc0be", "418b5438 4191f240 -> 3f431ff0", "c0f7e4e4 410571fe -> bf3f9e8f", "c163d334 bd98d600 -> bfc9bb97", @@ -25590,27 +25590,27 @@ "c133cbd1 c12c03ab -> c015618c", "bf32bb00 c1111506 -> c04424d8", "40e3794c bfa93e10 -> 3fe099e8", - "419cd3d2 bf1c60c0 -> 3fcd0c96", + "419cd3d2 bf1c60c0 -> 3fcd0c97", "bf800240 40cf58c0 -> be1ccef2", "c1630418 4194bc52 -> bf26e0ae", - "419ee38a c0d13810 -> 3ff1c4a8", + "419ee38a c0d13810 -> 3ff1c4a9", "c0554ad8 bf756f80 -> bfecea8c", "4155c1e0 c086c8b0 -> 3ff02782", "c19438df c155bc54 -> c00c82ae", "4181ed52 c072d0e0 -> 3fe66fdc", "41335fee c12dcd2f -> 4015c95e", "c1571882 c194fd20 -> c0210be4", - "c19a88b4 c0b6e8d0 -> bfede30f", - "c153621e c19b2864 -> c022ca69", - "40a5abdc 419f6cf2 -> 3e822341", + "c19a88b4 c0b6e8d0 -> bfede310", + "c153621e c19b2864 -> c022ca68", + "40a5abdc 419f6cf2 -> 3e822342", "40d30c58 419f1e6c -> 3ea3ee9c", "418c0490 c0b33c62 -> 3ff0b4e5", - "c0831f14 40e27288 -> bf065d6a", - "bf2ad860 4075b2b0 -> be303f57", + "c0831f14 40e27288 -> bf065d69", + "bf2ad860 4075b2b0 -> be303f56", "bf04a640 bffc8c00 -> c038a00e", "bf41b3a0 40b8c260 -> be056f50", "4021d860 c14d4bd9 -> 403c9ba3", - "3d83a000 40cb8cf8 -> 3c258915", + "3d83a000 40cb8cf8 -> 3c258914", "c178b13e c0fba980 -> c0028267", "c0108e98 40935624 -> bee9826e", "40833c68 40d08ed0 -> 3f0fc87b", @@ -25623,13 +25623,13 @@ "c1000d87 419fc51c -> bec32579", "c0814a50 c18dda43 -> c03ab943", "c186e3b7 c15b7765 -> c0103ce8", - "4183a496 c01779a8 -> 3fdb58d2", + "4183a496 c01779a8 -> 3fdb58d3", "40e51c20 bfe49ed0 -> 3fe85ad7", - "4187b382 c1690142 -> 4011ef14", - "40d37970 419b19ac -> 3ea83333", + "4187b382 c1690142 -> 4011ef13", + "40d37970 419b19ac -> 3ea83332", "3fa7f3f0 c0b5afc8 -> 403a86cd", "c122c32b 416a72a0 -> bf1b5a62", - "40374b70 c19b6a93 -> 403fb1c2", + "40374b70 c19b6a93 -> 403fb1c3", "c08588a0 c19f609e -> c03bd8f2", "419cbe90 c170df46 -> 400e75e0", "4182e2b4 c1643109 -> 40126ac4", @@ -25642,9 +25642,9 @@ "c1300d51 c108ac40 -> c00ec793", "410261e6 c1588362 -> 40265f3f", "419adcac 40054e48 -> 3fbb5784", - "41798160 c0ecc4b0 -> 4000e25e", + "41798160 c0ecc4b0 -> 4000e25f", "40639318 c1702fdd -> 403a2d2c", - "c1794797 c1829f3b -> c0184bca", + "c1794797 c1829f3b -> c0184bc9", "c0904192 c0035a40 -> bfffbf67", "41787da0 c19a2a35 -> 401da5e0", "4120e654 c1014c4b -> 400fdab9", @@ -25664,15 +25664,15 @@ "4112e32a c18e9f42 -> 402aa0a2", "bfaa2d50 40ef51f8 -> be3427dc", "c16ae95c 406f7cb0 -> bfa91e64", - "40d2db80 405037e8 -> 3f8e5b81", + "40d2db80 405037e8 -> 3f8e5b82", "be920280 419b18c2 -> bc70fbde", "41823e9c c10676ec -> 4003074a", "40fc01e0 40a9689c -> 3f7a9c4a", "416ade10 c15a3b2c -> 40147298", "c12b4626 419bd752 -> bf00a1ec", - "413a29aa c0cd07be -> 4004befe", + "413a29aa c0cd07be -> 4004beff", "c199d0c3 c1866487 -> c0127d6a", - "419e0288 c1425a56 -> 4007d1b2", + "419e0288 c1425a56 -> 4007d1b3", "4144a97c 4083fd0c -> 3f9f9ec3", "4104e2a2 3f327440 -> 3fbe57fd", "c10cd1b9 40378e30 -> bfa0bd4e", @@ -25685,7 +25685,7 @@ "3fc08c60 c177750c -> 4042db49", "c10a43a9 c0be1246 -> c00b121d", "4061f8c0 40e4294c -> 3eeb6b1a", - "c0aa3df6 4165c098 -> beb5a9ce", + "c0aa3df6 4165c098 -> beb5a9cf", "c13c904c c1220f4d -> c011f7c3", "4141fe18 c13691ef -> 4014db0f", "c043e348 40bf5874 -> bef239e5", @@ -32770,6 +32770,268 @@ "c486ccb7 4464268f 42b60000 44088700 42475598 00000000 43258de7 44372289 44df86e2 00000000 43904267 42f00000 c2b40000 40e00000 42b80000 43200000 c2f00000 40e00000 42ba0000 43480000 c3160000 40e00000 42bc0000 43700000 c3340000 40e00000 42be0000 438c0000 c3520000 40e00000 42c00000 43a00000 c3700000 40e00000 42c20000 43b40000 c3870000 40e00000 42c40000 43c80000 c3960000 40e00000 42c60000 43dc0000 c3a50000 40e00000 42c80000 43f00000 c3b40000 40e00000 42ca0000 44020000 c3c30000 40e00000 42cc0000 440c0000 c3d20000 40e00000 42ce0000 44160000 c3e10000 40e00000 42d00000 44200000 c3f00000 40e00000 42d20000 442a0000 c3ff0000 40e00000 42d40000 44340000 c4070000 40e00000 42d60000 443e0000 c40e8000 40e00000 42d80000 44480000 c4160000 40e00000 42da0000 44520000 c41d8000 40e00000 42dc0000 445c0000 c4250000 40e00000 42de0000 44660000 c42c8000 40e00000 42e00000 44700000 c4340000 40e00000 42e20000 447a0000 c43b8000 40e00000 42e40000 44820000 c4430000 40e00000 42e60000 44870000 c44a8000 40e00000 42e80000 448c0000 c4520000 40e00000 42ea0000 44910000 c4598000 40e00000 42ec0000 44960000 c4610000 40e00000 42ee0000 449b0000 c4688000 40e00000 42f00000 44a00000 c4700000 40e00000 42f20000 44a50000 c4778000 40e00000 42f40000 44aa0000 c47f0000 40e00000 42f60000 44af0000 c4834000 40e00000 42f80000 44b40000 c4870000 40e00000 42fa0000 44b90000 c48ac000 40e00000 42fc0000 44be0000 c48e8000 40e00000 42fe0000 44c30000 c4924000 40e00000 43000000 44c80000 c4960000 40e00000 43010000 44cd0000 c499c000 40e00000 43020000 44d20000 c49d8000 40e00000 43030000 44d70000 c4a14000 40e00000 43040000 44dc0000 c4a50000 40e00000 43050000 44e10000 c4a8c000 40e00000 43060000 44e60000 c4ac8000 40e00000 43070000 44eb0000 c4b04000 40e00000 43080000 44f00000 c4b40000 40e00000 43090000 44f50000 c4b7c000 40e00000 430a0000 44fa0000 c4bb8000 40e00000 430b0000 44ff0000 c4bf4000 40e00000 430c0000 45020000 c4c30000 40e00000 430d0000 45048000 c4c6c000 40e00000 430e0000 45070000 c4ca8000 40e00000 430f0000 45098000 c4ce4000 40e00000 43100000 450c0000 c4d20000 40e00000 43110000 450e8000 c4d5c000 40e00000 43120000 45110000 c4d98000 40e00000 43130000 45138000 c4dd4000 40e00000 43140000 45160000 c4e10000 40e00000 43150000 45188000 c4e4c000 40e00000 43160000 451b0000 c4e88000 40e00000 43170000 451d8000 c4ec4000 40e00000 43180000 45200000 c4f00000 40e00000 43190000 45228000 c4f3c000 40e00000 431a0000 45250000 c4f78000 40e00000 431b0000 45278000 c4fb4000 40e00000 431c0000 452a0000 c4ff0000 40e00000 431d0000 452c8000 c5016000 40e00000 431e0000 452f0000 c5034000 40e00000 431f0000 45318000 c5052000 40e00000 43200000 45340000 c5070000 40e00000 43210000 45368000 c508e000 40e00000 43220000 45390000 c50ac000 40e00000 43230000 453b8000 c50ca000 40e00000 43240000 453e0000 c50e8000 40e00000 43250000 45408000 c5106000 40e00000 43260000 45430000 c5124000 40e00000 43270000 45458000 c5142000 40e00000 43280000 45480000 c5160000 40e00000 43290000 454a8000 c517e000 40e00000 432a0000 454d0000 c519c000 40e00000 432b0000 454f8000 c51ba000 40e00000 432c0000 45520000 c51d8000 40e00000 432d0000 45548000 c51f6000 40e00000 432e0000 45570000 c5214000 40e00000 432f0000 45598000 c5232000 40e00000 43300000 455c0000 c5250000 40e00000 43310000 455e8000 c526e000 40e00000 43320000 45610000 c528c000 40e00000 43330000 45638000 c52aa000 40e00000 43340000 45660000 c52c8000 40e00000 43350000 45688000 c52e6000 40e00000 43360000 456b0000 c5304000 40e00000 43370000 456d8000 c5322000 40e00000 43380000 45700000 c5340000 40e00000 43390000 45728000 c535e000 40e00000 433a0000 45750000 c537c000 40e00000 433b0000 45778000 c539a000 40e00000 433c0000 457a0000 c53b8000 40e00000 433d0000 457c8000 c53d6000 40e00000 433e0000 457f0000 c53f4000 40e00000 433f0000 4580c000 c5412000 40e00000 43400000 45820000 c5430000 40e00000 43410000 45834000 c544e000 40e00000 43420000 45848000 c546c000 40e00000 43430000 4585c000 c548a000 40e00000 43440000 45870000 c54a8000 40e00000 43450000 45884000 c54c6000 40e00000 43460000 45898000 c54e4000 40e00000 43470000 458ac000 c5502000 40e00000 43480000 458c0000 c5520000 40e00000 43490000 458d4000 c553e000 40e00000 434a0000 458e8000 c555c000 40e00000 434b0000 458fc000 c557a000 40e00000 434c0000 45910000 c5598000 40e00000 434d0000 45924000 c55b6000 40e00000 434e0000 45938000 c55d4000 40e00000 434f0000 4594c000 c55f2000 40e00000 43500000 45960000 c5610000 40e00000 43510000 45974000 c562e000 40e00000 43520000 45988000 c564c000 40e00000 43530000 4599c000 c566a000 40e00000 43540000 459b0000 c5688000 40e00000 43550000 459c4000 c56a6000 40e00000 43560000 459d8000 c56c4000 40e00000 43570000 459ec000 c56e2000 40e00000 43580000 45a00000 c5700000 40e00000 43590000 00000002 43820000 3f333333 3fcccccd 42f00000 3eb33333 00000000 00000000 3f800000 -> 3ef8c434 3f32636d 3efbcd58 3e549b04", "c4b3e76c 444241a0 42500000 435b9579 4136dbcf 00000000 43945431 43041651 43830dcc 00000000 43652355 c48bf352 44d33444 00000000 434dc873 c3000002 c10833d3 00000000 433550ff c28b6e94 c30f7278 00000000 438d43a6 44a4de09 c4f12ff1 00000000 425a0d22 44d428d3 430976c5 00000000 434b973e 441d5713 44a7fc0b 00000000 438e7c40 c4ab19d7 44f2b987 00000000 41e038cc c455d2d5 c2a36170 00000000 43908b8d c3801d60 c40e138a 00000000 431363cc 43396792 c37f2156 00000000 437ac80c 439f85f6 420c7863 00000000 42150b58 4428e36e 44c2bb96 00000000 43837958 c47c7670 44a88b01 00000000 4294d48d c40641aa c285e8ba 00000000 4232d05c c40ca53f c4a8ca3a 00000000 4378763b 42cfefd3 c306ba40 00000000 430371e3 44b979fd 434ebfb9 00000000 43421a7a 42b44286 43616d19 00000000 43047467 c2892c2a 42ac8e8b 00000000 4332087f c4f373b1 c39604df 00000000 42b3908c c4071e02 c4b04c5b 00000000 4347459e 43fd006f c41a827e 00000000 4208fda6 44c75602 43869ead 00000000 42421b44 4351a83b 440ee820 00000000 42cc9909 c48345c8 449bb4d1 00000000 42125362 c4abe6d8 c37c9d2e 00000000 433a98a6 c43a8ecd c5050f9b 00000000 42b796d8 4488f948 c49dcc59 00000000 43638b9c 4518e99d 43f2fa58 00000000 4271681e 4436363e 450837cb 00000000 437c8686 c49f0216 44b1f01c 00000000 438026fe c4af2be6 c395b304 00000000 43814487 c3f349af c4befb18 00000000 428c07fa 4424afd0 c4330845 00000000 42decd7b 4513cf17 44073fe5 00000000 438a0efc 442d8bbd 450f5844 00000000 434b07a5 c493ef20 449c3dfb 00000000 437f587e c4222d07 c31e43fc 00000000 43128220 c4066f1d c4ea31b9 00000000 4340306c 43e81d81 c3ee2e59 00000000 4330577c 4514ae2d 441a3491 00000000 43869616 439ae709 448ea55a 00000000 43226a3e c4068e80 440626bb 00000000 430d975e c49c9b56 c3ac1805 00000000 42dea520 c4161f98 c5129034 00000000 42da3948 44400c36 c43a07ca 00000000 43826b94 44c6d8dc 43e6e324 00000000 438edae5 4396e0d8 449ca83c 00000000 43643dd5 c35d19fc 4350154a 00000000 430efc77 c485b6e4 c3a3a64d 00000000 4312ff68 c3cdea28 c4e43378 00000000 4385fba4 44224024 c4145936 00000000 435e4c43 43a49d73 42d3e418 00000000 438636f8 43ed8fd2 450d12ed 00000000 4210bc80 c2dcc99b 42c41aba 00000000 4299b2ee c50fdfe9 c44262e4 00000000 42fe5060 c3dac9b9 c50be58c 00000000 42f0366f 44ca1d48 c4ae6032 00000000 4382f138 4475cf34 43adfc47 00000000 420467d2 4383e7ac 44b69e10 00000000 42939fb6 c4a5b2bc 448ad7eb 00000000 41f43fa8 c461b848 c3a71faa 00000000 434a4234 c358c6a6 c4a37e3e 00000000 423c3b16 42ae5790 c28ddcb8 00000000 435f71b1 44586d7a 43a75f50 00000000 431dfe64 43126575 44726ffe 00000000 42e181b4 c3da9fe9 43acb878 00000000 436eefe3 c4efcdc3 c4416fe4 00000000 423c53dc c344a113 c4b46166 00000000 435ffd97 44553a65 c4238580 00000000 430092b5 4499e952 440156a4 00000000 430fbb25 42c9a36d 444f404d 00000000 4369e56c c43b5358 440b6b0e 00000000 41cf9cce c5087fe5 c46ebaa1 00000000 42caf529 c21191e4 c3aa0683 00000000 4344d73b 443ee95e c409dbcb 00000000 41f53bb8 44f6e532 44606df7 00000000 42aa10aa 433460d1 44f3d985 00000000 43582457 c48b83c0 44436f9c 00000000 4350ab77 c4098067 c381cd3a 00000000 41ad0134 c32c4d09 c50a24e7 00000000 4293fef2 44bb391d c47e4d73 00000000 432790ce 44ec7b40 4467a0ac 00000000 43470d71 420e374b 440c07c0 00000000 425deb6d c4911239 443effdb 00000000 4366654f c43619f1 c3b8eb31 00000000 434e068d c2a05b66 c4cc6faf 00000000 4291eb66 44b5dd54 c468003e 00000000 42acbd25 44cd3c28 4457e72d 00000000 433ac26f 41e50db8 444eeade 00000000 431d0e95 c3beb1b5 436b9aef 00000000 435bf745 c4afd8d7 c43f8161 00000000 435bc798 c10e3dc0 c3dc6413 00000000 42e0327b 448376b9 c41d3ced 00000000 439027f4 42ccb7fb 4266a7eb 00000000 42561b63 4107706a 44b7ae4c 00000000 4386c984 c49102da 4427d07c 00000000 43041b2c c48de2f6 c4254afa 00000000 42dc4d73 417fe3ca c4e73dfa 00000000 436e328f 44fdd280 c48e0708 00000000 435a40e7 44dfcd51 4486b7c0 00000000 437af18e c22ffde1 44ee8d5d 00000000 42ebac6a c445a9ae 43d5c9a6 00000000 43543326 c45fbee4 c40b1d1a 00000000 43507e5b 425680ee c4b2d793 00000000 432a871c 45066678 c48c65e4 00000000 438c4680 44501288 44059013 00000000 435e3b80 c2e08f5e 45072ba4 00000000 428609a6 c4a3143d 44246ade 00000000 4395e45b c4556a6b c40d5fe9 00000000 4321ff85 4206eb23 c3fe1841 00000000 42ab0018 43addb6e c3290c04 00000000 4308e930 43dc68f5 43969f8a 00000000 430610e6 c2126c9e 43e25da0 00000000 42ae86ef c4c4feae 4438935d 00000000 42ef05dd c486c228 c43def39 00000000 438f4904 4321a791 c4d3da54 00000000 433ab2f6 44c00026 c42d31f8 00000000 42db619b 44f338d9 44b0b3b2 00000000 42c2bff2 c2dd8512 447bddda 00000000 4313918c c505ea13 44685e96 00000000 4372ce6f 459b0000 c5688000 40e00000 43550000 459c4000 c56a6000 40e00000 43560000 459d8000 c56c4000 40e00000 43570000 459ec000 c56e2000 40e00000 43580000 45a00000 c5700000 40e00000 43590000 0000007b 43820000 3f333333 3fcccccd 42f00000 3eb33333 3f19999a 3f4ccccd 3f800000 -> 3f1796ad 3f2abc83 3f147ad8 3f6031e3" ] + }, + { + "glsl": "float cultmath_fbm(vec3, int, float, float)", + "tolerance": "ulp-bounded", + "cases": [ + "c11a9d90 c04202a0 c20bc826 00000004 40000000 3f000000 -> 3f27d6ca", + "41601e80 4005e0a0 41eba008 00000004 40000000 3f000000 -> be8cf3fd", + "407e0d30 c181cd2c 41cbe660 00000004 40000000 3f000000 -> 3eebda0d", + "41b7f3d4 c129244c c0fdb2f0 00000004 40000000 3f000000 -> bd3ada1f", + "407005a0 4118392c 41a44bb4 00000004 40000000 3f000000 -> beabbce1", + "c1830e64 c2048dab 422a3126 00000004 40000000 3f000000 -> 3e53b796", + "407894c0 c2091ce8 4243e388 00000004 40000000 3f000000 -> beff701d", + "c1f06795 c23c1b2b c2412342 00000004 40000000 3f000000 -> 3ec7b869", + "c1bdbda4 41dc0d48 c22f43c4 00000004 40000000 3f000000 -> bec760f6", + "408126b0 c1ea0445 c1a7655d 00000004 40000000 3f000000 -> 3e0a4e06", + "413dbdcc 422bc160 40f21a68 00000004 40000000 3f000000 -> bea53f12", + "c1484ebc 420439be 416f96b0 00000004 40000000 3f000000 -> 3e72cffa", + "c1ba67c0 408252b0 421a9588 00000004 40000000 3f000000 -> bf451a7c", + "c16b1c64 41a06cd4 c176d898 00000004 40000000 3f000000 -> 3f02099c", + "4120d64c 41de56d4 40b8cd88 00000004 40000000 3f000000 -> 3f2bebc0", + "c1ca675a c230dc20 c22de33d 00000004 40000000 3f000000 -> 3f1cd196", + "4137ed30 4146d32c c21ce93a 00000004 40000000 3f000000 -> 3da3b336", + "c08f7cb0 4226b458 41db68b4 00000004 40000000 3f000000 -> be182486", + "41f4cfe4 c1ece732 c0c027f0 00000004 40000000 3f000000 -> 3daec608", + "c212a4e3 c1446638 41c4aed0 00000004 40000000 3f000000 -> be06a746", + "41ba2790 4247bbfe c213e474 00000004 40000000 3f000000 -> 3f08da39", + "c065a220 c21b758b c060a740 00000004 40000000 3f000000 -> 3f0948cb", + "420a0656 c206ebf6 41d3a918 00000004 40000000 3f000000 -> be0f2d50", + "c231bfd1 c22c5352 414c9ffc 00000004 40000000 3f000000 -> 3d5ad4f0", + "c2386461 c1fb5f35 c08a9a88 00000004 40000000 3f000000 -> bf0307b4", + "416666b8 411e5a8c c1c68d2b 00000004 40000000 3f000000 -> bf3c2a7e", + "c081e798 c241ab24 41adc588 00000004 40000000 3f000000 -> 3ecf6bca", + "42469f7a 422a298a 41052448 00000004 40000000 3f000000 -> bd9e0e93", + "c1fd92e8 c206a0f6 c242f2f6 00000004 40000000 3f000000 -> bea9dec8", + "41f11de8 40dc22f8 414ab55c 00000004 40000000 3f000000 -> bd8bc474", + "c1e82aab c2128efd c2069c1b 00000004 40000000 3f000000 -> be7c0319", + "c20fc1e0 41409478 41a0f4d8 00000004 40000000 3f000000 -> 3da2f1d0", + "c1e3a366 c211704c 418541ec 00000004 40000000 3f000000 -> 3f3968b5", + "c222dbdb c11ce808 421e0fd6 00000004 40000000 3f000000 -> 3eb06c7c", + "c19f6f92 40bdd8e0 40b040e8 00000004 40000000 3f000000 -> 3d4ea228", + "c0011390 c1062cd4 41b8c7e4 00000004 40000000 3f000000 -> bed7e250", + "c191f0aa c20731bd 4127d710 00000004 40000000 3f000000 -> 3e0d1a30", + "c1f97cd1 418b3020 c15db9a8 00000004 40000000 3f000000 -> 3dfd58fc", + "c23c8b5e 41bce8a0 c22a6d8e 00000004 40000000 3f000000 -> 3c465804", + "409693d0 c1af215b 42332b6c 00000004 40000000 3f000000 -> bee9b32d", + "c098bbf0 c236889e 41163eac 00000004 40000000 3f000000 -> 3f0bd913", + "c210d8cb 420a5584 422e3bac 00000004 40000000 3f000000 -> bebb6492", + "c063be60 421f5d7e 40d15a40 00000004 40000000 3f000000 -> 3ebd15a8", + "41cc6488 c0e4a4f8 c06f9100 00000004 40000000 3f000000 -> 3db49ff4", + "410861d0 c155845c 41363b78 00000004 40000000 3f000000 -> bdcb6293", + "4200d2ee 419cae84 c1564644 00000004 40000000 3f000000 -> 3edb892a", + "c23cbf0a 41a28f8c 41c5d480 00000004 40000000 3f000000 -> bf2d47fb", + "c1aeadcf bfcf1700 c1152790 00000004 40000000 3f000000 -> be5972ba", + "c23f486c c23f56ed c0813868 00000004 40000000 3f000000 -> 3ee6dc4a", + "41f1a4a0 c21b3682 4219b0ce 00000004 40000000 3f000000 -> bf0eadc6", + "414b9fe8 4186b7a4 420e2062 00000004 40000000 3f000000 -> 3e2980bf", + "41aeeb18 4160ee88 c162c594 00000004 40000000 3f000000 -> bee70d01", + "c19effaa c206defe 41aacb30 00000004 40000000 3f000000 -> 3de761d6", + "413f31c0 c241d553 3fe2e9a0 00000004 40000000 3f000000 -> bde2b38a", + "c205c755 40c6ab38 41bac5c0 00000004 40000000 3f000000 -> 3ed8ad73", + "c20f112c 4124a490 c19a0db2 00000004 40000000 3f000000 -> bdc30b1c", + "41c60570 c1eabc98 c237e2f2 00000004 40000000 3f000000 -> bf4faa5b", + "41a64804 3ff230a0 40f93c68 00000004 40000000 3f000000 -> bee98cca", + "c1ef7da7 c203c097 40bf80d8 00000004 40000000 3f000000 -> 3db61265", + "c1fa4ce3 c219905e 4201ac72 00000004 40000000 3f000000 -> bf387f6a", + "c019d480 4181e398 41e52800 00000004 40000000 3f000000 -> be033562", + "4190f4f4 c214b5b7 41d01f1c 00000004 40000000 3f000000 -> 3d5e261c", + "412b9a38 4212a2e8 41b4c68c 00000004 40000000 3f000000 -> 3f336c18", + "c1c37f7b 40052c30 405363f0 00000004 40000000 3f000000 -> 3e9fce4c", + "41742360 c1122dec 41963a2c 00000004 40000000 3f000000 -> 3d369949", + "c1d85a1d c2198f93 4228b1e0 00000004 40000000 3f000000 -> 3e151611", + "418aa674 4217c8c0 c1020040 00000004 40000000 3f000000 -> 3d9e5382", + "c20b2034 c232306a 422227ca 00000004 40000000 3f000000 -> 3d21dba6", + "420993c0 4227c3f8 c20c35b2 00000004 40000000 3f000000 -> 3e254db8", + "c1b20e56 3fe68be0 c0403c60 00000004 40000000 3f000000 -> 3e00593c", + "41eebd00 c21a5963 c2354e9f 00000004 40000000 3f000000 -> bf023291", + "421bb928 c20e9648 c16b1c78 00000004 40000000 3f000000 -> bf0b8dcb", + "41b90698 bfc839c0 c0d3a708 00000004 40000000 3f000000 -> bf1723c8", + "423d5354 40608460 41bd0c74 00000004 40000000 3f000000 -> bcc26220", + "41025788 bf4f6280 41949de4 00000004 40000000 3f000000 -> beed4e99", + "3f39b3c0 c231e4d4 c21be301 00000004 40000000 3f000000 -> be0bb9b2", + "421c0a76 423c6e3a c2323104 00000004 40000000 3f000000 -> bf40bdcd", + "415cf8ec 3fb0caa0 c1d2b23c 00000004 40000000 3f000000 -> be851f93", + "c1174dfc c18cfa28 412c88d4 00000004 40000000 3f000000 -> 3f111826", + "42186c38 4206e6ba c1e8f35c 00000004 40000000 3f000000 -> bec21f9a", + "41311500 c188248e c17245d8 00000004 40000000 3f000000 -> 3d6acd72", + "c1c331e9 c088f1f0 c1154f68 00000004 40000000 3f000000 -> 3f32b2ec", + "bfaf6440 41d29f60 c1bbea67 00000004 40000000 3f000000 -> 3d96329a", + "c1843796 be305a00 4152c8e8 00000004 40000000 3f000000 -> bdce91e5", + "bf9b27e0 419fffcc c178fe1c 00000004 40000000 3f000000 -> 3e94636d", + "4191f36c 40e24f50 40c86908 00000004 40000000 3f000000 -> beb06eb9", + "c1e04e4e c182a61e 4211a8cc 00000004 40000000 3f000000 -> 3e4f51ee", + "c23852b1 c1344684 40963120 00000004 40000000 3f000000 -> 3f06fbc7", + "c1074b08 c207dfba 41208124 00000004 40000000 3f000000 -> 3f44398c", + "41f81500 bfce6b20 41ffcd98 00000004 40000000 3f000000 -> 3db207ed", + "41c42d58 420c1714 416b5de8 00000004 40000000 3f000000 -> be56c8b1", + "c1dbf666 c15bc7b4 c1ebcae6 00000004 40000000 3f000000 -> be987233", + "c133a140 c18ba254 c23a1f4c 00000004 40000000 3f000000 -> be6063b7", + "c1b0ba6b c23f9e0f c1a66906 00000004 40000000 3f000000 -> 3e71480d", + "420a1232 40b13200 c02e16b0 00000004 40000000 3f000000 -> bda67ee4", + "c0f719f8 c20be886 41fb0cd8 00000004 40000000 3f000000 -> be9018c6", + "421ce668 41b26594 41ca7208 00000004 40000000 3f000000 -> 3f33ea18", + "c235eaec 418126bc 3d672000 00000004 40000000 3f000000 -> bea45437", + "c17889e4 40e4bf88 4230ff5a 00000004 40000000 3f000000 -> beb68251", + "41f150cc 420839ce c1a27d07 00000004 40000000 3f000000 -> bcbd7004", + "41f517bc 418cf4bc c1b079ee 00000004 40000000 3f000000 -> bb24d74c", + "c0aedff8 410f3624 416ce240 00000004 40000000 3f000000 -> 3f0d6ce0", + "4241dac2 c1bdbc18 c1821c58 00000004 40000000 3f000000 -> bea85f48", + "c2105a14 c1a43bf4 c1cbc9d9 00000004 40000000 3f000000 -> 3ebe4d06", + "4032c6d0 42303610 c19369ff 00000004 40000000 3f000000 -> 3ee7e743", + "40e6ea70 c1f4c9c7 4204c81e 00000004 40000000 3f000000 -> be98d471", + "c22a5870 4245107c c243f735 00000004 40000000 3f000000 -> bed2576b", + "41e533b8 42272418 41d0b3b4 00000004 40000000 3f000000 -> 3e0d2bde", + "c22d5237 41a00334 c1f8f891 00000004 40000000 3f000000 -> 3e1a4a1f", + "c20d624e 42182746 41e8ec44 00000004 40000000 3f000000 -> bd73af1a", + "c1bdc99a 4109bd28 bfc15ee0 00000004 40000000 3f000000 -> bdd4a46b", + "41be3e70 c02e9130 c17861bc 00000004 40000000 3f000000 -> be9a49dc", + "c242f78d c1204738 423a58b8 00000004 40000000 3f000000 -> bd8f7f9e", + "4201e604 c0c5f000 421af586 00000004 40000000 3f000000 -> bd939ee0", + "c016b680 41ca80f8 c0e579d8 00000004 40000000 3f000000 -> be949954", + "3f477c00 3fdd1520 41b0ba9c 00000004 40000000 3f000000 -> 3f61641f", + "c1fc2271 41da0074 c1440cb0 00000004 40000000 3f000000 -> 3d22d4bd", + "c21bc8c4 41b601ac 420b2952 00000004 40000000 3f000000 -> bea1ff75", + "c1aa8e9f c2400dfe 41ceac90 00000004 40000000 3f000000 -> bf0ee9ae", + "3fed25c0 c1422724 421bcbf0 00000004 40000000 3f000000 -> 3e7b8260", + "c0825938 42073d10 c13c1548 00000004 40000000 3f000000 -> 3e8108b9", + "4093dd90 42168aba 41a6e578 00000004 40000000 3f000000 -> bf70dcc5", + "41d04c14 41e81a68 c04e0de0 00000004 40000000 3f000000 -> 3f169ba9", + "423c6370 40bea6d8 41eb59d4 00000004 40000000 3f000000 -> be27a3b6", + "42126f66 c1c87ce9 40ae73e0 00000004 40000000 3f000000 -> bdecf67a", + "c216ecbb c197f579 c2061c65 00000004 40000000 3f000000 -> bec0f5a1", + "c185edd2 40bc7ec8 c1fdce1e 00000004 40000000 3f000000 -> beaeea8a", + "c04d9980 4238aeec c21cd8f3 00000004 40000000 3f000000 -> bf4c5f9d", + "c10a1ac8 c20b1c08 420f6e74 00000004 40000000 3f000000 -> bca013bc", + "415b2edc c09ccbc8 c096d8b0 00000004 40000000 3f000000 -> be5536d7", + "410a4524 bf182e80 c23f8806 00000004 40000000 3f000000 -> 3ee6c4f3", + "c2297336 c1573b54 412013c4 00000004 40000000 3f000000 -> 3c3ab7a0", + "3efd6c00 422c18e8 4216b5c2 00000004 40000000 3f000000 -> beeca0ab", + "41afbc10 41c5efcc 41b5b6e8 00000004 40000000 3f000000 -> bf364b64", + "c1c583a7 c1e4e94f c1e5a945 00000004 40000000 3f000000 -> bf34f4f8", + "41053fa4 c1cc7056 c00c0b50 00000004 40000000 3f000000 -> bdda9df2", + "c1a54a53 c2065a94 41a7d97c 00000004 40000000 3f000000 -> 3e6ae330", + "4115cab4 c1fb3fd3 41ffccbc 00000004 40000000 3f000000 -> be2050ce", + "41a565fc 42448e70 42168e26 00000004 40000000 3f000000 -> bd95b845", + "c1b458be c212fd9c 41b34d84 00000004 40000000 3f000000 -> bde52094", + "420c044a c216a5ac c180b2a6 00000004 40000000 3f000000 -> 3ea3b257", + "c24205c8 c138b768 c22dca6a 00000004 40000000 3f000000 -> bf026e60", + "41da69e0 41e7c53c 42199060 00000004 40000000 3f000000 -> 3e296457", + "4235c98a 413c61b4 c1404e8c 00000004 40000000 3f000000 -> bd267211", + "c1a9a44f 4134ac38 41f92fec 00000004 40000000 3f000000 -> bd77a9b6", + "c08e1418 c1e42f89 41c64724 00000004 40000000 3f000000 -> 3f071a60", + "421dfbb0 42042964 42231c22 00000004 40000000 3f000000 -> be057b84", + "c13f435c c17a3c80 c1aa7dc8 00000004 40000000 3f000000 -> bece20fe", + "41c27058 42451e9c c19981a2 00000004 40000000 3f000000 -> be79305d", + "c1eedcc7 c15098d0 c1497518 00000004 40000000 3f000000 -> be9e1b90", + "41b2ed98 c1d7b191 41ee2d04 00000004 40000000 3f000000 -> 3d44b899", + "c20af100 4213d9c0 41467648 00000004 40000000 3f000000 -> 3ec0aa40", + "c1332ce4 c1d17816 4203b932 00000004 40000000 3f000000 -> be11a895", + "c20d0a38 41f835f0 c090d638 00000004 40000000 3f000000 -> be1498a4", + "4116bbac c180dd68 409467f0 00000004 40000000 3f000000 -> bf0fd158", + "c187ff3c c19b7655 40e9a840 00000004 40000000 3f000000 -> 3e89df12", + "c244b197 c1c714ff 41dd4c6c 00000004 40000000 3f000000 -> 3ead3f11", + "c214956d c140a03c c19e4e02 00000004 40000000 3f000000 -> 3ef4d00f", + "41fd1540 c042ae00 41dfd2e8 00000004 40000000 3f000000 -> 3d973e78", + "c2331a6e c1b67be9 c12d31d4 00000004 40000000 3f000000 -> be750957", + "41cdeccc 40e51a98 c0ffb978 00000004 40000000 3f000000 -> be253870", + "3f7b69c0 c1d75a82 c1343b2c 00000004 40000000 3f000000 -> 3dbf25d2", + "41035764 c211e128 41c2a2a0 00000004 40000000 3f000000 -> 3f151eee", + "c1fa0b38 c22cf74a 423ea448 00000004 40000000 3f000000 -> bd6e638c", + "42276a60 41c60bd0 c23ff195 00000004 40000000 3f000000 -> bf359aaa", + "c09648b8 41e5007c c246206c 00000004 40000000 3f000000 -> 3ec9a87b", + "41c06838 c10b20d4 4195b38c 00000004 40000000 3f000000 -> be8ff22f", + "41bc1450 c1f57998 c2189bb5 00000004 40000000 3f000000 -> 3e28b2f1", + "411bf210 411830e4 420c4f46 00000004 40000000 3f000000 -> be20e42c", + "bee13d80 c182f740 c11fed84 00000004 40000000 3f000000 -> bf6b6b32", + "4209b41c c1836194 423a81ea 00000004 40000000 3f000000 -> 3eebe724", + "c0c7d2c8 c1300278 c1d110b4 00000004 40000000 3f000000 -> 3edf09dc", + "c23a2398 c2263ebc c21cc173 00000004 40000000 3f000000 -> 3f347bc5", + "c2086436 41c65514 c203da03 00000004 40000000 3f000000 -> be4a016a", + "4203e3b8 42005406 c1fd266b 00000004 40000000 3f000000 -> 3e92d85b", + "41f003dc c1e7ff9b 41534824 00000004 40000000 3f000000 -> bf51dab7", + "c1ae6ad8 c15ab124 4192a9c8 00000004 40000000 3f000000 -> 3ea4064b", + "41dc13c0 c1969d16 420f8a40 00000004 40000000 3f000000 -> 3f0909f0", + "41a3b388 c23f13c1 c1427f80 00000004 40000000 3f000000 -> 3e6ede51", + "c19eb6f5 4224f120 41591a18 00000004 40000000 3f000000 -> 3e6ab4d3", + "406a1ef0 c22567ab 41ef34f0 00000004 40000000 3f000000 -> bf055e0e", + "c17acd40 c17ae6e4 41b97304 00000004 40000000 3f000000 -> be66940a", + "c243dda3 bff368e0 c163e5e4 00000004 40000000 3f000000 -> bd501bea", + "41e12d8c c10295fc 41a17df0 00000004 40000000 3f000000 -> 3dd3be72", + "421ea106 4102614c 41841a84 00000004 40000000 3f000000 -> bf09bbcc", + "3fd67940 42222670 c14c4344 00000004 40000000 3f000000 -> 3ee83191", + "422df782 41edf52c c137cfe8 00000004 40000000 3f000000 -> 3d22e603", + "c1e10584 c1e51a20 423d30aa 00000004 40000000 3f000000 -> 3f0369ee", + "41ad5f78 c22c55c3 c23e623f 00000004 40000000 3f000000 -> bd8b828e", + "c21d0856 41ba57c4 c153b374 00000004 40000000 3f000000 -> bf83804f", + "42101786 41ba6fc4 c2032fdd 00000004 40000000 3f000000 -> 3e0487d8", + "c0307f90 4143e76c 41812468 00000004 40000000 3f000000 -> bf30ffd4", + "c14dd1ec bfad1f00 c00f2420 00000004 40000000 3f000000 -> bdccd9c1", + "c09a65b8 41e7bf24 41b736f4 00000004 40000000 3f000000 -> 3f093490", + "c1b468fb 3d4f3800 4231f934 00000004 40000000 3f000000 -> 3c21555a", + "424095ea 421768d8 41b83244 00000004 40000000 3f000000 -> bece4789", + "c1f50604 c1f9b40a c2193a31 00000004 40000000 3f000000 -> be4a50c7", + "c2458f10 c1554bf0 c21a0974 00000004 40000000 3f000000 -> bf1acbb9", + "c1c78b00 c191b69e 42152566 00000004 40000000 3f000000 -> be957456", + "420269ce 414a73d0 c1b4c7ed 00000004 40000000 3f000000 -> 3e697510", + "41a63780 42212ab4 41c671ac 00000004 40000000 3f000000 -> 3f139976", + "4243f362 42287f3e 4194453c 00000004 40000000 3f000000 -> 3c50a516", + "403eed70 c0f2b800 42379e78 00000004 40000000 3f000000 -> beba9818", + "c24684f1 c20dd7bc c101c264 00000004 40000000 3f000000 -> 3d3cca45", + "421de172 bfd77900 c244e4fc 00000004 40000000 3f000000 -> 3f050db9", + "4179c230 4231b1ba 41aa7e40 00000004 40000000 3f000000 -> 3e3e0e04", + "c23cc88c 41bc33a8 c1f77d37 00000004 40000000 3f000000 -> 3d7074f3", + "c1a4f6c3 42257aac 40902f38 00000004 40000000 3f000000 -> 3cded1fb", + "c1c76558 420da696 c155d488 00000004 40000000 3f000000 -> bdb2d8a1", + "c2312669 c1de84f2 c19c55ac 00000004 40000000 3f000000 -> bf733e95", + "41e6d99c 40910dc0 420ffa14 00000004 40000000 3f000000 -> be87cefb", + "c1ed1c98 41c32f10 421ab4de 00000004 40000000 3f000000 -> beac3f5c", + "424062a4 c1ac4fc1 40fda488 00000004 40000000 3f000000 -> bed84bad", + "c1c052d0 c09ac768 c0920138 00000004 40000000 3f000000 -> 3e8a6d70", + "41d8f554 c1f7fc22 c2370249 00000004 40000000 3f000000 -> 3eb1ddd1", + "419e6cd0 c21b7082 421968ce 00000004 40000000 3f000000 -> be530987", + "c21d2479 411727cc c14a3428 00000004 40000000 3f000000 -> 3dd46038", + "c20819bc 41ee64b4 421c8320 00000004 40000000 3f000000 -> bdbaa7c4", + "c226a55b c109a378 4243b008 00000004 40000000 3f000000 -> 3da8d91f", + "c1ed9a23 41bf4c20 c22d2758 00000004 40000000 3f000000 -> be4a641a", + "c190d081 c02eac10 c1ee290f 00000004 40000000 3f000000 -> 3e49c3d4", + "41772540 c1e3e37f 41201a34 00000004 40000000 3f000000 -> be5c48b4", + "40bcaa98 4198df7c 41bf595c 00000004 40000000 3f000000 -> 3f1b99b9", + "c21799da 418c1bd0 417f6670 00000004 40000000 3f000000 -> bd1b2a24", + "c2127570 41d6c1b8 c1ad1a80 00000004 40000000 3f000000 -> 3d032504", + "41bd0318 40109f50 41b2f178 00000004 40000000 3f000000 -> be738350", + "3e841880 40a8b758 41c22e30 00000004 40000000 3f000000 -> be993b38", + "bfdbbdc0 4233fe10 414bce50 00000004 40000000 3f000000 -> be35bd5c", + "422c6234 414e1a28 41c5deb8 00000004 40000000 3f000000 -> 3eaa7016", + "423b76f4 c217e8f1 c187ff32 00000004 40000000 3f000000 -> 3ec9b4b3", + "421f04d6 41de5f6c 40919080 00000004 40000000 3f000000 -> be522d95", + "3f9b7d20 42303ed8 423f91d4 00000004 40000000 3f000000 -> beed15a0", + "c1c65740 c2194ea5 423df3ae 00000004 40000000 3f000000 -> be9b386d", + "c0b846b8 c2105eea c0ff01f8 00000004 40000000 3f000000 -> be6e4c9a", + "c20e43e2 42427560 4227bc0c 00000004 40000000 3f000000 -> 3eb83eb6", + "41ab32b4 c23da44d c23c2e83 00000004 40000000 3f000000 -> bf1c83e6", + "412c4184 c1749b9c 421c8d2c 00000004 40000000 3f000000 -> be315e50", + "c1bb9168 41a0557c c1fae1b8 00000004 40000000 3f000000 -> be39850e", + "c18c7cde 420675ae 401172f0 00000004 40000000 3f000000 -> bec5ccdb", + "40efbf28 41b2ce34 410a1ed8 00000004 40000000 3f000000 -> bf1e2c8b", + "420d1730 41107978 40e2c578 00000004 40000000 3f000000 -> 3eb26d1a", + "c1114ee0 c1dd3d62 421ede34 00000004 40000000 3f000000 -> 3db35cba", + "c21a5874 4142baa4 42032204 00000004 40000000 3f000000 -> bcff40fb", + "c19c3e28 bfca5a80 c1169d98 00000004 40000000 3f000000 -> bf81fcc7", + "41c70a18 c22feef8 c10a28ac 00000004 40000000 3f000000 -> bea49aff", + "c18e1152 c181341a 423988ae 00000004 40000000 3f000000 -> 3e720a62", + "41a66f10 c1e26e4d 4124ab60 00000004 40000000 3f000000 -> 3e3bbee8", + "41ae3bc0 c15e65b4 41da7bec 00000004 40000000 3f000000 -> 3e0672ef", + "c0c46630 4153362c c228fb08 00000004 40000000 3f000000 -> bf217d87", + "c1617454 c1f9cb51 c21e36d6 00000004 40000000 3f000000 -> be923997", + "c08c2ae0 418db8f4 c12ad700 00000004 40000000 3f000000 -> bef08fdd", + "c1325020 410f7fe4 409c15b8 00000004 40000000 3f000000 -> be423c5a", + "420bc714 41bc1fb0 41d22824 00000004 40000000 3f000000 -> 3e0456ea", + "c23f2228 411fc738 c1194bd4 00000004 40000000 3f000000 -> 3b981998", + "c1eea110 c19da117 42112bee 00000004 40000000 3f000000 -> beafb6e7", + "bfb96dc0 3fd647a0 416066d8 00000004 40000000 3f000000 -> be699b27" + ] } ] } diff --git a/packages/cultmath/unity/org.gamecult.cultmath/CHANGELOG.md b/packages/cultmath/unity/org.gamecult.cultmath/CHANGELOG.md index dfb84a2d..7edfa1b2 100644 --- a/packages/cultmath/unity/org.gamecult.cultmath/CHANGELOG.md +++ b/packages/cultmath/unity/org.gamecult.cultmath/CHANGELOG.md @@ -5,6 +5,14 @@ All notable changes to this package are documented in this file. Earlier releases (0.2.0-0.2.3) predate this package's changelog and are not backfilled here; see `docs/semver-policy.md` for why. +## [Unreleased] + +### Added + +- `math.fbm(float3 p, int octaves, float lacunarity, float gain)` and `math.fbm(float3 p)` (four octaves, + lacunarity 2, gain 0.5): the value-only octave sum of `snoise`, the cost of `fbm_grad` without the gradient. + The shaders carry `cultmath_fbm` in HLSL and GLSL; octaves clamp to [0, 16] as `fbm_grad`'s do. + ## [0.5.0] ### Breaking diff --git a/packages/cultmath/unity/org.gamecult.cultmath/Shaders/CultMath.hlsl b/packages/cultmath/unity/org.gamecult.cultmath/Shaders/CultMath.hlsl index d3eea396..fc9f7a70 100644 --- a/packages/cultmath/unity/org.gamecult.cultmath/Shaders/CultMath.hlsl +++ b/packages/cultmath/unity/org.gamecult.cultmath/Shaders/CultMath.hlsl @@ -342,6 +342,26 @@ float4 cultmath_fbm_grad(float3 p, int octaves, float lacunarity, float gain) return float4(gradient, value); } +// Value-only octave sum of cultmath_snoise; see the C# math.fbm comment. The one-argument form is four octaves, +// lacunarity 2, gain 0.5. +float cultmath_fbm(float3 p, int octaves, float lacunarity, float gain) +{ + octaves = clamp(octaves, 0, 16); + float amplitude = 1.0; + float frequency = 1.0; + float value = 0.0; + for (int i = 0; i < octaves; i++) + { + value += amplitude * cultmath_snoise(p * frequency); + frequency *= lacunarity; + amplitude *= gain; + } + + return value; +} + +float cultmath_fbm(float3 value) { return cultmath_fbm(value, 4, 2.0, 0.5); } + // Ridged multifractal noise (Musgrave, Texturing and Modeling, ch. 16); see the C# math.ridged_grad // comment for the fold/sign derivation, the deliberate crease at n = 0, and why octaves is clamped. float4 cultmath_ridged_grad(float3 p, int octaves, float lacunarity, float gain) From 990521b3f1426cbbc90bb28c3c69c4b976b528cf Mon Sep 17 00:00:00 2001 From: libby Date: Sat, 31 Oct 2026 21:58:18 +0000 Subject: [PATCH 34/88] cultmath: measure the Lipschitz bound of phacelle, with its tool and a pinning test max singular value of the (cosine, sine) Jacobian <= max(10, 1.25 |side|) / (1 - normalization) for normalization in [0, 0.9], |side| in [0.25, 25]; method and tightness in design.md. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultmath/docs/design.md | 20 +++++ .../tests/CultMath.Tests/PhacelleTests.cs | 29 +++++++ .../tools/phacelle-lipschitz/Program.cs | 85 +++++++++++++++++++ .../phacelle-lipschitz.csproj | 4 + 4 files changed, 138 insertions(+) create mode 100644 packages/cultmath/tools/phacelle-lipschitz/Program.cs create mode 100644 packages/cultmath/tools/phacelle-lipschitz/phacelle-lipschitz.csproj diff --git a/packages/cultmath/docs/design.md b/packages/cultmath/docs/design.md index 3b63f4d2..d9f06b5f 100644 --- a/packages/cultmath/docs/design.md +++ b/packages/cultmath/docs/design.md @@ -451,6 +451,26 @@ together, laid out as `float4(gradient.xyz, value.w)`, with no value-only twin nearly cancels. The total weight `W` is positive for every `pcg3d` input (the smallest found by hill-climbing is 0.056); `W = 0`, and a NaN output, needs adversarial jitter no hash yields. + Lipschitz bound of the `(cosine, sine)` field in `p` (one cell is 1 unit): the + largest singular value of its 2×3 Jacobian is at most + `max(10, 1.25 |side|) / (1 - normalization)` for `normalization` in `[0, 0.9]` and + `|side|` in `[0.25, 25]`. A scalar use of `cosine` or `sine` alone is bounded by the same number (its gradient norm is + one row of the Jacobian, never above the largest singular value). Method (`tools/phacelle-lipschitz`): + random `p` (a 64-unit cube), a uniformly random `side` direction and offset, the + exact gradient at each, 4 million samples per (normalization, |side|); then + hill-climbing from the 256 best samples over `p`, `side` direction and offset + (10 000 steps each); then value differences around the climbed argmax to confirm the + gradient is the field's slope (agreement within 3%). Random sampling alone + undershoots the climbed maximum by up to 4×, because the maxima sit at near-cancellation + points of the blend (`1/|I|`); quote the climbed number. The climbed sup is 0.016 to 0.88 of + the envelope over the sweep: tightest (within 1.15×) at `normalization` 0 and `|side| >= 8`, + and 1.5 to 60× loose at small `|side|` or large `normalization`, where the + envelope's constant 10 is the worst case over all `|side|`. The typical gradient (99.99th percentile of the + samples) is 2 to 5× below the sup, so a sphere trace steps safely at `distance / L` + and can step farther with the returned gradient. The bound is not a proof (a + search found it, not an argument) and it does not hold as `normalization` approaches 1: + the 1/|I| factor is unbounded there (sampled 1335 at `normalization` 1, `|side|` 2π, and + rising with the sample count). If the caller scales `p` by `f` the bound scales by `f`. Cells whose per-axis lower bound `Σ max(0, |local - g| - 0.5)²` is at least 2.25 lie outside the support and have weight exactly 0, so they are skipped, which changes no output bit. Cells whose bound sits just under 2.25 can still diff --git a/packages/cultmath/tests/CultMath.Tests/PhacelleTests.cs b/packages/cultmath/tests/CultMath.Tests/PhacelleTests.cs index 71f5b452..e9c3377d 100644 --- a/packages/cultmath/tests/CultMath.Tests/PhacelleTests.cs +++ b/packages/cultmath/tests/CultMath.Tests/PhacelleTests.cs @@ -337,4 +337,33 @@ public sealed class PhacelleTests Assert.Equal(BitConverter.SingleToInt32Bits(expected.z), BitConverter.SingleToInt32Bits(actual.z)); Assert.Equal(BitConverter.SingleToInt32Bits(expected.w), BitConverter.SingleToInt32Bits(actual.w)); } + + // The Lipschitz envelope of the (cosine, sine) field in p, per docs/design.md ("Lipschitz bound"): the largest + // singular value of its 2x3 Jacobian never exceeds max(10, 1.25 |side|) / (1 - normalization), for normalization + // in [0, 0.9] and |side| in [0.25, 25]. tools/phacelle-lipschitz measures it by sampling and hill-climbing; this + // pins that no ordinary sample passes it (the climbed maximum is at most 0.88 of the envelope). + [Theory] + [InlineData(0.0f)] + [InlineData(0.5f)] + [InlineData(0.75f)] + [InlineData(0.9f)] + public void GradientNormStaysUnderTheDocumentedLipschitzEnvelope(float normalization) + { + var random = new System.Random(0x11B5); + foreach (var sideLength in new[] { 0.25f, 1.0f, 1.5f, 4.0f, 12.5f, 25.0f }) + { + var envelope = Math.Max(10.0, 1.25 * sideLength) / (1.0 - normalization); + for (var i = 0; i < 20000; i++) + { + var p = RandomPoint(random, 32.0f); + var side = math.normalize(RandomSide(random)) * sideLength; + var phasor = math.phacelle(p, side, random.NextSingle(), normalization); + var a = (double)math.dot(phasor.cosine.xyz, phasor.cosine.xyz); + var d = (double)math.dot(phasor.sine.xyz, phasor.sine.xyz); + var b = (double)math.dot(phasor.cosine.xyz, phasor.sine.xyz); + var norm = Math.Sqrt((a + d) / 2.0 + Math.Sqrt(Math.Max(0.0, (a + d) * (a + d) / 4.0 - (a * d - b * b)))); + Assert.True(norm <= envelope, $"|J| {norm} passes the envelope {envelope} at p {p}, side {side}, normalization {normalization}"); + } + } + } } diff --git a/packages/cultmath/tools/phacelle-lipschitz/Program.cs b/packages/cultmath/tools/phacelle-lipschitz/Program.cs new file mode 100644 index 00000000..d46decdf --- /dev/null +++ b/packages/cultmath/tools/phacelle-lipschitz/Program.cs @@ -0,0 +1,85 @@ +// Measures the Lipschitz constant of math.phacelle's (cosine, sine) field, the number docs/design.md quotes and +// PhacelleTests.GradientNormStaysUnderTheDocumentedLipschitzEnvelope pins. +// dotnet run -c Release --project packages/cultmath/tools/phacelle-lipschitz -- +// e.g. ... -- 4000000 0,0.5,0.75,0.9 0.25,0.5,1,1.5,2,4,8,12.5,25 +// Columns: sampledMax (random p), climbedMax (hill-climbed from the 256 best samples; the number to quote), secantMax +// (value differences around the climbed argmax, the check that the analytic gradient is the field's slope), +// scalarMax (the larger of |grad cosine|, |grad sine| over the samples), p99.99 (the typical tail), and climbed*(1-n)/|side|. +using System; +using System.Linq; +using System.Threading.Tasks; +using CultMath; + +// Lipschitz bound of math.phacelle's (cosine, sine) field: sup over p of the operator norm of its 2x3 Jacobian, +// found by (1) random sampling of the exact analytic gradient, (2) hill-climbing from the top samples over +// (p, off, side direction), (3) a secant check on the VALUES (|f(p+d) - f(p)| / |d|) so the gradient itself is not trusted. +static double Op(float3 p, float3 side, float off, float n, out double comp) +{ + var r = math.phacelle(p, side, off, n); + double a = r.cosine.x*r.cosine.x + r.cosine.y*r.cosine.y + r.cosine.z*r.cosine.z; + double d = r.sine.x*r.sine.x + r.sine.y*r.sine.y + r.sine.z*r.sine.z; + double b = r.cosine.x*r.sine.x + r.cosine.y*r.sine.y + r.cosine.z*r.sine.z; + double tr = a + d, det = a*d - b*b; + comp = Math.Sqrt(Math.Max(a, d)); + var op = Math.Sqrt(tr/2 + Math.Sqrt(Math.Max(0, tr*tr/4 - det))); + return double.IsFinite(op) ? op : 0; +} +static float3 Dir(double u, double th, double len) { var s = Math.Sqrt(Math.Max(0, 1-u*u)); return new float3((float)(s*Math.Cos(th)*len), (float)(s*Math.Sin(th)*len), (float)(u*len)); } + +long N = long.Parse(args[0]); +var norms = args[1].Split(',').Select(float.Parse).ToArray(); +var sides = args[2].Split(',').Select(float.Parse).ToArray(); +Console.WriteLine("n\t|side|\tN\tsampledMax\tclimbedMax\tsecantMax\tscalarMax(cos|sin)\tp99.99\tclimbed*(1-n)/|side|"); +foreach (var n in norms) foreach (var sl in sides) +{ + int T = Environment.ProcessorCount; + var tops = new (double v, float3 p, double u, double th, float off)[T][]; + var allv = new double[T][]; var scal = new double[T]; + Parallel.For(0, T, t => + { + var rng = new System.Random(unchecked(99991 * t + (int)(n * 1000) * 131 + (int)(sl * 100))); + var top = new (double v, float3 p, double u, double th, float off)[16]; + var list = new double[N / T]; + double sc = 0; + for (long i = 0; i < N / T; i++) + { + var p = new float3((float)(rng.NextDouble()*64-32), (float)(rng.NextDouble()*64-32), (float)(rng.NextDouble()*64-32)); + double u = rng.NextDouble()*2-1, th = rng.NextDouble()*Math.PI*2; float off = (float)rng.NextDouble(); + var v = Op(p, Dir(u, th, sl), off, n, out var c); + list[i] = v; if (c > sc) sc = c; + int m = 0; for (int k = 1; k < 16; k++) if (top[k].v < top[m].v) m = k; + if (v > top[m].v) top[m] = (v, p, u, th, off); + } + tops[t] = top; allv[t] = list; scal[t] = sc; + }); + var all = allv.SelectMany(x => x).ToArray(); Array.Sort(all); + double sampled = all[^1], p9999 = all[(int)(all.Length * 0.9999)]; + // hill-climb the best 16*T by random local search + var seeds = tops.SelectMany(x => x).OrderByDescending(x => x.v).Take(256).ToArray(); + var climbed = new (double v, float3 p, double u, double th, float off)[seeds.Length]; + Parallel.For(0, seeds.Length, s => + { + var rng = new System.Random(s + 17); + var cur = seeds[s]; double step = 0.05; + for (int it = 0; it < 10000; it++) + { + var q = new float3(cur.p.x + (float)((rng.NextDouble()-.5)*step), cur.p.y + (float)((rng.NextDouble()-.5)*step), cur.p.z + (float)((rng.NextDouble()-.5)*step)); + var u = Math.Clamp(cur.u + (rng.NextDouble()-.5)*step, -1, 1); var th = cur.th + (rng.NextDouble()-.5)*step; var off = cur.off + (float)((rng.NextDouble()-.5)*step); + var v = Op(q, Dir(u, th, sl), off, n, out _); + if (v > cur.v) cur = (v, q, u, th, off); else if (it % 1000 == 999) step *= 0.5; + } + climbed[s] = cur; + }); + var best = climbed.OrderByDescending(x => x.v).First(); + // secant check around the climbed argmax: values only, tiny displacements in random directions + double secant = 0; var sr = new System.Random(5); + var side0 = Dir(best.u, best.th, sl); + for (int i = 0; i < 20000; i++) + { + double e = Math.Pow(10, -2 - 2*sr.NextDouble()); var d = Dir(sr.NextDouble()*2-1, sr.NextDouble()*6.2832, e); + var a = math.phacelle(best.p, side0, best.off, n); var b = math.phacelle(best.p + d, side0, best.off, n); + double dc = b.cosine.w - a.cosine.w, ds = b.sine.w - a.sine.w; var dl = Math.Sqrt((double)d.x*d.x + (double)d.y*d.y + (double)d.z*d.z); + secant = Math.Max(secant, Math.Sqrt(dc*dc + ds*ds) / dl); + } + Console.WriteLine($"{n}\t{sl}\t{N}\t{sampled:F3}\t{best.v:F3}\t{secant:F3}\t{scal.Max():F3}\t{p9999:F3}\t{best.v*(1-n)/sl:F3}"); +} diff --git a/packages/cultmath/tools/phacelle-lipschitz/phacelle-lipschitz.csproj b/packages/cultmath/tools/phacelle-lipschitz/phacelle-lipschitz.csproj new file mode 100644 index 00000000..3b8629ae --- /dev/null +++ b/packages/cultmath/tools/phacelle-lipschitz/phacelle-lipschitz.csproj @@ -0,0 +1,4 @@ + + Exenet10.0enabletruetruefalse + + From 3b58b24a684f11020e6faa4d2602959ca4777d40 Mon Sep 17 00:00:00 2001 From: Eureka Date: Sun, 1 Nov 2026 04:57:26 +0000 Subject: [PATCH 35/88] cultmath: re-record device-bounds-lavapipe.txt for cultmath_fbm Recorded on Yggdrasil under llvmpipe (LLVM 20.1.2) with packages/cultmath/tools/device/run-lavapipe.sh, WRITE_BOUNDS=1, at 990521b3. Adds the lavapipe-glsl and lavapipe-hlsl rows for cultmath_fbm only. The cycles_arcsine/arctan2 rows are left as committed (pre-existing asin/atan oracle failures at base 94f20eee are out of scope). Co-Authored-By: Claude Sonnet 5.5 --- .../tests/CultMath.Tests/fixtures/device-bounds-lavapipe.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/cultmath/tests/CultMath.Tests/fixtures/device-bounds-lavapipe.txt b/packages/cultmath/tests/CultMath.Tests/fixtures/device-bounds-lavapipe.txt index 42706157..0a61b572 100644 --- a/packages/cultmath/tests/CultMath.Tests/fixtures/device-bounds-lavapipe.txt +++ b/packages/cultmath/tests/CultMath.Tests/fixtures/device-bounds-lavapipe.txt @@ -126,6 +126,7 @@ lavapipe-glsl 121 float cultmath_lattice_distance(vec2, vec2, float, vec2, float lavapipe-glsl 122 float cultmath_lattice_smooth_step(float, float, float, float) ulp-bounded 256 0 0 6cbeddde536b12e83873f2e5ac60d16bef22049d2a7dea61af0ff1e0d5b84740 lavapipe-glsl 123 vec4 cultmath_power_border(vec3, vec4[128], int) ulp-bounded 188 65.72121683891783 0 8427c2b99109a4f59ff745fdb87f261deb1cbae34ce99bc1ae05ebb7f9963b4d lavapipe-glsl 124 vec4 cultmath_weighted_lattice_cells(vec2, float, vec4[128], int, float, float, float, float, float, vec2, float) ulp-bounded 179 3 0 fd3b58c4e2ba8a7bf801a1e6cdff82468abcb368442bc98b2583fda9d2e6f823 +lavapipe-glsl 125 float cultmath_fbm(vec3, int, float, float) ulp-bounded 43 190.375 0 8f852b55293bd75a4e2e64cd1d1f5083605c2ba2d0877cdf4e8e507c72890135 lavapipe-hlsl 0 float cultmath_snoise(vec3) ulp-bounded 94 122.6875 0 5d41c14241a13124f2e1b49f230f5df0c48d0d69fb1be9ef2f33dc1702bf0b0a lavapipe-hlsl 1 float cultmath_snoise(vec2) ulp-bounded 204 1 0 81d9c62726afdccbbe4ffef173b5c92731eb6bf5927b3979d653e50dbd5d4064 lavapipe-hlsl 2 vec4 cultmath_snoise_grad(vec3) ulp-bounded 71 617.5 0 f7e9ed1707cdb015efaba0e0150193309da0d4c125349518f8b7c60a17b25790 @@ -251,3 +252,4 @@ lavapipe-hlsl 121 float cultmath_lattice_distance(vec2, vec2, float, vec2, float lavapipe-hlsl 122 float cultmath_lattice_smooth_step(float, float, float, float) ulp-bounded 256 0 0 6cbeddde536b12e83873f2e5ac60d16bef22049d2a7dea61af0ff1e0d5b84740 lavapipe-hlsl 123 vec4 cultmath_power_border(vec3, vec4[128], int) ulp-bounded 188 65.72121683891783 0 8427c2b99109a4f59ff745fdb87f261deb1cbae34ce99bc1ae05ebb7f9963b4d lavapipe-hlsl 124 vec4 cultmath_weighted_lattice_cells(vec2, float, vec4[128], int, float, float, float, float, float, vec2, float) ulp-bounded 179 3 0 fd3b58c4e2ba8a7bf801a1e6cdff82468abcb368442bc98b2583fda9d2e6f823 +lavapipe-hlsl 125 float cultmath_fbm(vec3, int, float, float) ulp-bounded 49 190.375 0 cd02f850f1d01ff77a3cb1c49af2bb49e51bb18c395936703cb37b7729ad2c88 From b57501b33e2aa56d95826d7e6fcce0cc9641a4d2 Mon Sep 17 00:00:00 2001 From: Libby Date: Mon, 2 Nov 2026 18:33:18 +0000 Subject: [PATCH 36/88] cultmath: phacelle-lipschitz prints usage and exits 2 on missing or malformed arguments Without arguments the tool threw IndexOutOfRange; a malformed number threw FormatException. Co-Authored-By: Claude Sonnet 5.5 --- .../cultmath/tools/phacelle-lipschitz/Program.cs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/cultmath/tools/phacelle-lipschitz/Program.cs b/packages/cultmath/tools/phacelle-lipschitz/Program.cs index d46decdf..c149f1d6 100644 --- a/packages/cultmath/tools/phacelle-lipschitz/Program.cs +++ b/packages/cultmath/tools/phacelle-lipschitz/Program.cs @@ -26,9 +26,15 @@ static double Op(float3 p, float3 side, float off, float n, out double comp) } static float3 Dir(double u, double th, double len) { var s = Math.Sqrt(Math.Max(0, 1-u*u)); return new float3((float)(s*Math.Cos(th)*len), (float)(s*Math.Sin(th)*len), (float)(u*len)); } -long N = long.Parse(args[0]); -var norms = args[1].Split(',').Select(float.Parse).ToArray(); -var sides = args[2].Split(',').Select(float.Parse).ToArray(); +float[]? norms = null, sides = null; +try { norms = args.Length == 3 ? args[1].Split(',').Select(float.Parse).ToArray() : null; sides = args.Length == 3 ? args[2].Split(',').Select(float.Parse).ToArray() : null; } +catch (FormatException) { } +if (args.Length != 3 || !long.TryParse(args[0], out var N) || N <= 0 || norms is null || sides is null) +{ + Console.Error.WriteLine("usage: phacelle-lipschitz "); + Console.Error.WriteLine(" e.g. phacelle-lipschitz 4000000 0,0.5,0.75,0.9 0.25,0.5,1,1.5,2,4,8,12.5,25"); + return 2; +} Console.WriteLine("n\t|side|\tN\tsampledMax\tclimbedMax\tsecantMax\tscalarMax(cos|sin)\tp99.99\tclimbed*(1-n)/|side|"); foreach (var n in norms) foreach (var sl in sides) { @@ -83,3 +89,4 @@ foreach (var n in norms) foreach (var sl in sides) } Console.WriteLine($"{n}\t{sl}\t{N}\t{sampled:F3}\t{best.v:F3}\t{secant:F3}\t{scal.Max():F3}\t{p9999:F3}\t{best.v*(1-n)/sl:F3}"); } +return 0; From 52dd5e8acba98fd9fd6a3902074a0f4d51c5e6c7 Mon Sep 17 00:00:00 2001 From: Libby Date: Wed, 4 Nov 2026 16:11:05 +0000 Subject: [PATCH 37/88] docs: docs/start onboarding page with tested CultCache first calls in C# and TypeScript A language x package matrix (each cell a tested snippet or "not yet"), a C# first-call test and a TypeScript first-call test that the docs link, and a pointer from the top of README.md. Co-Authored-By: Claude Sonnet 5.5 --- README.md | 2 + docs/start/README.md | 115 ++++++++++++++++++ packages/cultcache-ts/test/first-call.test.ts | 51 ++++++++ .../GameCult.Caching.Tests/FirstCallTests.cs | 54 ++++++++ 4 files changed, 222 insertions(+) create mode 100644 docs/start/README.md create mode 100644 packages/cultcache-ts/test/first-call.test.ts create mode 100644 tests/GameCult.Caching.Tests/FirstCallTests.cs diff --git a/README.md b/README.md index e2ea607d..6e6de55c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # CultLib +**New here? Start at [`docs/start`](docs/start/README.md)**: a language x package matrix with tested first-call snippets. + CultLib is GameCult's shared state, transport, and runtime substrate. It started as reusable C# game infrastructure, but the useful thing now is larger: typed `.cc` persistence, schema-stable wire formats, ergonomic reliable-UDP diff --git a/docs/start/README.md b/docs/start/README.md new file mode 100644 index 00000000..91447789 --- /dev/null +++ b/docs/start/README.md @@ -0,0 +1,115 @@ +# Start Here + +CultLib is one stack in several languages: **CultCache** (typed `.cc` persistence), +**CultNet** (schema-v0 wire messages and transports) on top of it, and **CultMesh** (Verse +discovery, replication, authority) on top of that. **CultMath** and **Geometry** sit beside the +stack as portable numerics and engine-neutral geometry. The C# runtime in `src/` is the +reference; the other runtimes keep wire parity with it. + +If you want typed state on disk, start with CultCache. Everything else assumes it. + +## Language x package + +A cell links a snippet that a test in this repo runs, or says **not yet**. "Package" links +the package's own README or source, which is real but carries no tested first-call snippet from +this page. + +| | CultCache | CultMath | Geometry | CultNet | CultMesh | +| --- | --- | --- | --- | --- | --- | +| **C#** | [first call, tested](#c-first-call) | [package](../../packages/cultmath/README.md); snippet not yet | [package](../../src/GameCult.Geometry/README.md); snippet not yet | [package](../../src/GameCult.Networking); snippet not yet | [quickstart](../../src/GameCult.Mesh/docs/durable-node-quickstart.md); run by `scripts/verify-eve-getting-started.ps1` | +| **TypeScript** | [first call, tested](#typescript-first-call) | not yet | not yet | [package](../../packages/cultnet-ts/README.md); snippet not yet | [package](../../packages/cultmesh-ts/README.md); snippet not yet | +| **Rust** | [package](../../packages/cultcache-rs/README.md); snippet not yet | [native core](../../packages/cultmath/native/cultmath-core); snippet not yet | not yet | [package](../../packages/cultnet-rs/README.md), [examples](../../packages/cultnet-rs/examples); snippet not yet | [package](../../packages/cultmesh-rs); snippet not yet | +| **Python** | [package](../../packages/cultcache-py/README.md); snippet not yet | not yet | not yet | [package](../../packages/cultnet-py/README.md); snippet not yet | [package](../../packages/cultmesh-py/README.md); snippet not yet | + +The Rust and Python CultCache first calls are the next cut for this page. For the whole +picture of what each piece is for, see the +[package chooser in the top-level README](../../README.md#which-package-do-i-want). + +## C# first call + +Define a typed document, put it in a cache backed by a `.cc` file, read it back through a second +cache. The source is +[`tests/GameCult.Caching.Tests/FirstCallTests.cs`](../../tests/GameCult.Caching.Tests/FirstCallTests.cs), +which is what `dotnet test tests/GameCult.Caching.Tests --filter FirstCallTests` runs, so the snippet +cannot drift without the test failing. + +```csharp +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using MessagePack; + +[CultDocument("gamecult.first_call_item", "gamecult.first_call_item.v1")] +[MessagePackObject] +public sealed class Item +{ + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] [CultIndex] public string Category = string.Empty; + [Key(2)] public int Value; +} + +var store = new SingleFileMessagePackBackingStore("first-call.cc"); +var cache = new CultCache(); +cache.AddBackingStore(store); +var handle = await cache.AddAsync(new Item { Name = "Potion", Category = "Consumable", Value = 50 }); +store.PushAll(); // write the .cc file + +var reopened = new CultCache(); +reopened.AddBackingStore(new SingleFileMessagePackBackingStore("first-call.cc")); +await reopened.PullAllBackingStoresAsync(); // read it back +var potion = reopened.Get(handle.Key); // by key +var byName = reopened.GetByName("Potion"); // by [CultName] +``` + +The test's `Item` is nested and internal, so it carries `[MessagePackObject(AllowPrivate = true)]`; the snippet above is the public-class form and is not itself compiled. More: `src/GameCult.Caching/README.md`. + +## TypeScript first call + +Same shape in TypeScript: the document is a name, a schema id and a `parse`-able schema (zod here), +and the `.cc` file is wire-compatible with the C# one. The source is +[`packages/cultcache-ts/test/first-call.test.ts`](../../packages/cultcache-ts/test/first-call.test.ts), +run by `npm test` in `packages/cultcache-ts` (or `node --test dist-test/test/first-call.test.js` +after `tsc`). + +```ts +import { z } from "zod"; +import { + CultCache, + SingleFileMessagePackBackingStore, + defineDocumentRegistry, + defineDocumentType, +} from "@gamecult/cultcache-ts"; + +const itemDocument = defineDocumentType({ + type: "item", + schemaId: "gamecult.first_call_item.v1", + schemaName: "gamecult.first_call_item", + schemaVersion: "gamecult.first_call_item.v1", + schema: z.object({ name: z.string(), category: z.string(), value: z.number().int() }), + name: "name", + indexes: { category: "category" }, +}); +const registry = defineDocumentRegistry(itemDocument); + +const cache = CultCache.builder() + .withRegistry(registry) + .withGenericStore(new SingleFileMessagePackBackingStore("first-call.cc")) + .build(); +await cache.pullAllBackingStores(); +await cache.put(itemDocument, "item:potion", { name: "Potion", category: "Consumable", value: 50 }); + +const reopened = CultCache.builder() + .withRegistry(registry) + .withGenericStore(new SingleFileMessagePackBackingStore("first-call.cc")) + .build(); +await reopened.pullAllBackingStores(); +const potion = reopened.get(itemDocument, "item:potion"); +const byName = reopened.getByName(itemDocument, "Potion"); +``` + +The test imports from `../src/index`; an application imports `@gamecult/cultcache-ts`. More: +`packages/cultcache-ts/README.md`. + +## Where next + +- Smallest durable mesh path: `src/GameCult.Mesh/docs/durable-node-quickstart.md`. +- Application path, four chapters: `src/GameCult.Mesh/docs/getting-started/README.md`. diff --git a/packages/cultcache-ts/test/first-call.test.ts b/packages/cultcache-ts/test/first-call.test.ts new file mode 100644 index 00000000..aec9faa1 --- /dev/null +++ b/packages/cultcache-ts/test/first-call.test.ts @@ -0,0 +1,51 @@ +import assert from "node:assert/strict"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { z } from "zod"; + +import { + CultCache, + SingleFileMessagePackBackingStore, + defineDocumentRegistry, + defineDocumentType, +} from "../src/index"; + +// The TypeScript first-call snippet that docs/start/README.md links: define a typed document, +// store it in a .cc file, read it back through a second cache. Keep it small; it is the front door. +const itemDocument = defineDocumentType({ + type: "item", + schemaId: "gamecult.first_call_item.v1", + schemaName: "gamecult.first_call_item", + schemaVersion: "gamecult.first_call_item.v1", + schema: z.object({ name: z.string(), category: z.string(), value: z.number().int() }), + name: "name", + indexes: { category: "category" }, +}); +const registry = defineDocumentRegistry(itemDocument); + +test("define, store and read back a typed document", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-first-call-")); + const path = join(dir, "first-call.cc"); + try { + const cache = CultCache.builder() + .withRegistry(registry) + .withGenericStore(new SingleFileMessagePackBackingStore(path)) + .build(); + await cache.pullAllBackingStores(); + await cache.put(itemDocument, "item:potion", { name: "Potion", category: "Consumable", value: 50 }); + + const reopened = CultCache.builder() + .withRegistry(registry) + .withGenericStore(new SingleFileMessagePackBackingStore(path)) + .build(); + await reopened.pullAllBackingStores(); + + assert.equal(reopened.get(itemDocument, "item:potion")?.value, 50); + assert.equal(reopened.getByName(itemDocument, "Potion")?.category, "Consumable"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/tests/GameCult.Caching.Tests/FirstCallTests.cs b/tests/GameCult.Caching.Tests/FirstCallTests.cs new file mode 100644 index 00000000..d14ad72c --- /dev/null +++ b/tests/GameCult.Caching.Tests/FirstCallTests.cs @@ -0,0 +1,54 @@ +#nullable enable +using System; +using System.IO; +using System.Threading.Tasks; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Caching.Tests +{ + // The C# first-call snippet that docs/start/README.md links: define a typed document, store it in a + // .cc file, read it back through a second cache. Keep it small; it is the front door. + public class FirstCallTests + { + [CultDocument("gamecult.first_call_item", "gamecult.first_call_item.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class Item + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] [CultIndex] public string Category = string.Empty; + [Key(2)] public int Value; + } + + [Test] + public async Task DefineStoreAndReadBackATypedDocument() + { + var path = Path.Combine(Path.GetTempPath(), $"first-call-{Guid.NewGuid():N}.cc"); + try + { + // Write: a cache with one backing store, one typed document. + var store = new SingleFileMessagePackBackingStore(path); + var cache = new CultCache(); + cache.AddBackingStore(store); + var handle = await cache.AddAsync(new Item { Name = "Potion", Category = "Consumable", Value = 50 }); + store.PushAll(); + + // Read: a fresh cache over the same file. + var reopened = new CultCache(); + reopened.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + await reopened.PullAllBackingStoresAsync(); + + var byKey = reopened.Get(handle.Key); + Assert.That(byKey, Is.Not.Null); + Assert.That(byKey!.Value, Is.EqualTo(50)); + Assert.That(reopened.GetByName("Potion")!.Category, Is.EqualTo("Consumable")); + } + finally + { + File.Delete(path); + } + } + } +} From 5d8e032d7f208d2137b4ef27002035a9c8dbe9be Mon Sep 17 00:00:00 2001 From: Libby Date: Wed, 4 Nov 2026 17:22:13 +0000 Subject: [PATCH 38/88] cultmath: glsl-parity header names the recording image and lavapipe/Mesa; asin/acos/atan2 oracle tolerance capped at 1 ulp The glsl-parity.json header gains recorded_on (generator writes it; goldens untouched). CyclesOracleTests allows cycles_arcsine, cycles_arccosine and cycles_arctan2 (glsl-parity families 95-97) 1 ulp from Blender, with the measured cases named; a test pins the cap at 1 ulp, the three names, and that a 2 ulp miss still fails. Co-Authored-By: Claude Sonnet 5.5 --- .../tests/CultMath.Tests/CyclesOracleTests.cs | 38 +++++++++++++++++-- .../tests/CultMath.Tests/GlslMirrorTests.cs | 1 + .../CultMath.Tests/fixtures/glsl-parity.json | 1 + 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/packages/cultmath/tests/CultMath.Tests/CyclesOracleTests.cs b/packages/cultmath/tests/CultMath.Tests/CyclesOracleTests.cs index e4afbc74..86ccbfc8 100644 --- a/packages/cultmath/tests/CultMath.Tests/CyclesOracleTests.cs +++ b/packages/cultmath/tests/CultMath.Tests/CyclesOracleTests.cs @@ -19,11 +19,23 @@ namespace CultMath.Tests; /// public sealed class CyclesOracleTests { - // The ulp distance a function is allowed from the reference. Empty: the fixtures are the SSE4.2 kernel's results - // and every function equals them, power and logarithm included (glibc's powf and logf, 0 ulp from MathF on Linux). + // The ulp distance a function is allowed from the reference; a function not named is held to 0. The fixtures are + // the SSE4.2 kernel's results and almost every function equals them, power and logarithm included (glibc's powf and + // logf, 0 ulp from MathF on Linux). asin, acos and atan2 are the exception: the fixtures' glibc and the libm these + // tests run on round them one ulp apart (glsl-parity.json families 95-97 are the same three, platform families + // there too), so each is allowed 1 ulp, never more (TolerancesAreCappedAtOneUlpAndStayOnTheDriftingFunctions). // An entry needs a measured case on the reference named in it (a failing case's bits and ulp); other builds' // tolerances belong to the class table in tools/cycles-oracle/README.md, not here. - private static readonly Dictionary Tolerances = new(); + private const int MaxToleratedUlp = 1; + + private static readonly Dictionary Tolerances = new() + { + ["cycles_arcsine"] = (1, "case 3f4dd9af -> 3f6f256c, C# 3f6f256d"), + ["cycles_arccosine"] = (1, "case 3f027192 -> 3f84a03c, C# 3f84a03b"), + ["cycles_arctan2"] = (1, "case 3f000000 bdcccccd -> 3fe2541c, C# 3fe2541d"), + }; + + private static int Allowed(string function) => Tolerances.TryGetValue(function, out var tolerance) ? tolerance.Ulp : 0; private const string CSharp = "C#"; private const string Mirror = "HLSL mirror"; @@ -120,7 +132,7 @@ public sealed class CyclesOracleTests var name = (string)entry!["function"]!; var method = Method(name); var clamped = (string?)entry["then"] is not null; - var allowed = Tolerances.TryGetValue(name, out var tolerance) ? tolerance.Ulp : 0; + var allowed = Allowed(name); var worst = 0L; string? worstCase = null; var cases = entry["cases"]!.AsArray(); @@ -158,6 +170,24 @@ public sealed class CyclesOracleTests Assert.True(failures.Count == 0, string.Join(Environment.NewLine, failures)); } + [Fact] + public void TolerancesAreCappedAtOneUlpAndStayOnTheDriftingFunctions() + { + Assert.Equal(new[] { "cycles_arccosine", "cycles_arcsine", "cycles_arctan2" }, Tolerances.Keys.Order().ToArray()); + Assert.All(Tolerances.Values, t => Assert.InRange(t.Ulp, 1, MaxToleratedUlp)); + Assert.All(Tolerances.Values, t => Assert.NotEmpty(t.Measured)); + + // A result two ulps from the reference is past every allowance, and one ulp is within the drifting functions' only. + foreach (var function in Tolerances.Keys.Append("cycles_power")) + { + var want = 0.7f; + var twoAway = MathF.BitIncrement(MathF.BitIncrement(want)); + Assert.Equal(2, Distance(want, twoAway)); + Assert.True(Distance(want, twoAway) > Allowed(function), $"{function}: a 2 ulp miss would pass"); + Assert.Equal(function != "cycles_power", Distance(want, MathF.BitIncrement(want)) <= Allowed(function)); + } + } + private static JsonNode RampEntry(string interpolation) => Load("ramp")["functions"]!.AsArray().Single(e => e!["node"]!.ToString() == "ShaderNodeValToRGB " + interpolation)!; diff --git a/packages/cultmath/tests/CultMath.Tests/GlslMirrorTests.cs b/packages/cultmath/tests/CultMath.Tests/GlslMirrorTests.cs index 10e08380..874ace67 100644 --- a/packages/cultmath/tests/CultMath.Tests/GlslMirrorTests.cs +++ b/packages/cultmath/tests/CultMath.Tests/GlslMirrorTests.cs @@ -975,6 +975,7 @@ static int4 clamp(int4 x, int4 a, int4 b) { if (a.x > b.x || a.y > b.y || a.z > var json = new StringBuilder(); json.Append("{\n"); json.Append(" \"generator\": \"GlslMirrorTests.GoldenFixtureMatchesCSharp (packages/cultmath/tests/CultMath.Tests); regenerate with CULTMATH_WRITE_GLSL=1\",\n"); + json.Append(" \"recorded_on\": \"platform families: Yggdrasil's mcr.microsoft.com/dotnet/sdk:10.0 image (Debian GNU/Linux 13 trixie, .NET 10.0.14); lavapipe device bounds: fixtures/device-bounds-lavapipe.txt (llvmpipe LLVM 20.1.2, mesa-vulkan-drivers 25.2.8-0ubuntu0.24.04.4, ubuntu:24.04 probe image)\",\n"); json.Append($" \"seed\": {Seed},\n"); json.Append($" \"points\": {Points},\n"); json.Append(" \"encoding\": \"each case is 'arguments -> result', every scalar a float32 or 32-bit integer bit pattern in hex, vectors and structs flattened in field order\",\n"); diff --git a/packages/cultmath/tests/CultMath.Tests/fixtures/glsl-parity.json b/packages/cultmath/tests/CultMath.Tests/fixtures/glsl-parity.json index 244d068c..a3e68668 100644 --- a/packages/cultmath/tests/CultMath.Tests/fixtures/glsl-parity.json +++ b/packages/cultmath/tests/CultMath.Tests/fixtures/glsl-parity.json @@ -1,5 +1,6 @@ { "generator": "GlslMirrorTests.GoldenFixtureMatchesCSharp (packages/cultmath/tests/CultMath.Tests); regenerate with CULTMATH_WRITE_GLSL=1", + "recorded_on": "platform families: Yggdrasil's mcr.microsoft.com/dotnet/sdk:10.0 image (Debian GNU/Linux 13 trixie, .NET 10.0.14); lavapipe device bounds: fixtures/device-bounds-lavapipe.txt (llvmpipe LLVM 20.1.2, mesa-vulkan-drivers 25.2.8-0ubuntu0.24.04.4, ubuntu:24.04 probe image)", "seed": 20261002, "points": 256, "encoding": "each case is 'arguments -> result', every scalar a float32 or 32-bit integer bit pattern in hex, vectors and structs flattened in field order", From 533b492af246a543b3209c5d29d9c1fe028ba9fb Mon Sep 17 00:00:00 2001 From: Libby Date: Wed, 4 Nov 2026 20:20:33 +0000 Subject: [PATCH 39/88] docs/start: tested Rust and Python CultCache first calls Each test file carries the snippet between markers; the docs show it verbatim and a second test in the file fails if the README block drifts. Matrix cells for Rust and Python CultCache now link them. Co-Authored-By: Claude Sonnet 5.5 --- docs/start/README.md | 108 +++++++++++++++++- .../cultcache-py/tests/test_first_call.py | 67 +++++++++++ packages/cultcache-rs/tests/first_call.rs | 56 +++++++++ 3 files changed, 228 insertions(+), 3 deletions(-) create mode 100644 packages/cultcache-py/tests/test_first_call.py create mode 100644 packages/cultcache-rs/tests/first_call.rs diff --git a/docs/start/README.md b/docs/start/README.md index 91447789..c178c7f0 100644 --- a/docs/start/README.md +++ b/docs/start/README.md @@ -18,10 +18,10 @@ this page. | --- | --- | --- | --- | --- | --- | | **C#** | [first call, tested](#c-first-call) | [package](../../packages/cultmath/README.md); snippet not yet | [package](../../src/GameCult.Geometry/README.md); snippet not yet | [package](../../src/GameCult.Networking); snippet not yet | [quickstart](../../src/GameCult.Mesh/docs/durable-node-quickstart.md); run by `scripts/verify-eve-getting-started.ps1` | | **TypeScript** | [first call, tested](#typescript-first-call) | not yet | not yet | [package](../../packages/cultnet-ts/README.md); snippet not yet | [package](../../packages/cultmesh-ts/README.md); snippet not yet | -| **Rust** | [package](../../packages/cultcache-rs/README.md); snippet not yet | [native core](../../packages/cultmath/native/cultmath-core); snippet not yet | not yet | [package](../../packages/cultnet-rs/README.md), [examples](../../packages/cultnet-rs/examples); snippet not yet | [package](../../packages/cultmesh-rs); snippet not yet | -| **Python** | [package](../../packages/cultcache-py/README.md); snippet not yet | not yet | not yet | [package](../../packages/cultnet-py/README.md); snippet not yet | [package](../../packages/cultmesh-py/README.md); snippet not yet | +| **Rust** | [first call, tested](#rust-first-call) | [native core](../../packages/cultmath/native/cultmath-core); snippet not yet | not yet | [package](../../packages/cultnet-rs/README.md), [examples](../../packages/cultnet-rs/examples); snippet not yet | [package](../../packages/cultmesh-rs); snippet not yet | +| **Python** | [first call, tested](#python-first-call) | not yet | not yet | [package](../../packages/cultnet-py/README.md); snippet not yet | [package](../../packages/cultmesh-py/README.md); snippet not yet | -The Rust and Python CultCache first calls are the next cut for this page. For the whole +For the whole picture of what each piece is for, see the [package chooser in the top-level README](../../README.md#which-package-do-i-want). @@ -109,6 +109,108 @@ const byName = reopened.getByName(itemDocument, "Potion"); The test imports from `../src/index`; an application imports `@gamecult/cultcache-ts`. More: `packages/cultcache-ts/README.md`. +## Rust first call + +Same shape with `derive(DatabaseEntry)`: slots are `#[cultcache(key = N)]`, so the `.cc` payload is the +slot-indexed array the C# runtime writes. The source is +[`packages/cultcache-rs/tests/first_call.rs`](../../packages/cultcache-rs/tests/first_call.rs), run by +`cargo test --test first_call` in `packages/cultcache-rs`; a second test in that file fails if the block +below stops matching the file. The Rust runtime has no name lookup, so the read-back is by key. More: +`packages/cultcache-rs/README.md`. + +```rust +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "item")] +struct Item { + #[cultcache(key = 0)] + name: String, + #[cultcache(key = 1)] + category: String, + #[cultcache(key = 2, default)] + value: u32, +} + +fn first_call() -> anyhow::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("first-call.cc"); + + let mut cache = CultCache::new(); + cache.register_entry_type::()?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)); + cache.pull_all_backing_stores()?; + cache.put("item:potion", &Item { + name: "Potion".to_string(), + category: "Consumable".to_string(), + value: 50, + })?; // written to the .cc file before this returns + + let mut reopened = CultCache::new(); + reopened.register_entry_type::()?; + reopened.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)); + reopened.pull_all_backing_stores()?; // read it back + let potion = reopened.get_required::("item:potion")?; + assert_eq!(potion.name, "Potion"); + assert_eq!(potion.value, 50); + Ok(()) +} +``` + +## Python first call + +A document is a definition with an encoder and decoder; `name=` and `indexes=` register the name lookup +and indexes. The source is +[`packages/cultcache-py/tests/test_first_call.py`](../../packages/cultcache-py/tests/test_first_call.py), +run by `python -m unittest tests.test_first_call` in `packages/cultcache-py` with `src` on `PYTHONPATH` +and `msgpack` installed (the store needs it). A second test fails if the block below stops matching the +file. More: `packages/cultcache-py/README.md`. + +```python +import tempfile +from dataclasses import asdict, dataclass +from pathlib import Path + +from cultcache_py import CultCache, SingleFileMessagePackBackingStore, define_document_type + + +@dataclass +class Item: + name: str + category: str + value: int + + +item_doc = define_document_type( + "item", + encode=lambda item: asdict(item), + decode=lambda raw: Item(**raw), + name="name", + indexes={"category": "category"}, +) + + +def first_call() -> Item: + path = Path(tempfile.mkdtemp()) / "first-call.cc" + + cache = ( + CultCache.builder() + .register_document_type(item_doc) + .add_generic_store(SingleFileMessagePackBackingStore(path)) + .build() + ) + cache.pull_all_backing_stores() + cache.put(item_doc, "item:potion", Item("Potion", "Consumable", 50)) # written to the .cc file + + reopened = ( + CultCache.builder() + .register_document_type(item_doc) + .add_generic_store(SingleFileMessagePackBackingStore(path)) + .build() + ) + reopened.pull_all_backing_stores() # read it back + assert reopened.get_by_name(item_doc, "Potion") == reopened.get_required(item_doc, "item:potion") + return reopened.get_required(item_doc, "item:potion") +``` + ## Where next - Smallest durable mesh path: `src/GameCult.Mesh/docs/durable-node-quickstart.md`. diff --git a/packages/cultcache-py/tests/test_first_call.py b/packages/cultcache-py/tests/test_first_call.py new file mode 100644 index 00000000..f8d4b52b --- /dev/null +++ b/packages/cultcache-py/tests/test_first_call.py @@ -0,0 +1,67 @@ +"""The Python CultCache first call from docs/start/README.md. The docs show the region between the +`snippet` markers verbatim, and test_docs_show_the_tested_snippet fails when they differ.""" + +import unittest +from pathlib import Path + +# snippet:begin +import tempfile +from dataclasses import asdict, dataclass +from pathlib import Path + +from cultcache_py import CultCache, SingleFileMessagePackBackingStore, define_document_type + + +@dataclass +class Item: + name: str + category: str + value: int + + +item_doc = define_document_type( + "item", + encode=lambda item: asdict(item), + decode=lambda raw: Item(**raw), + name="name", + indexes={"category": "category"}, +) + + +def first_call() -> Item: + path = Path(tempfile.mkdtemp()) / "first-call.cc" + + cache = ( + CultCache.builder() + .register_document_type(item_doc) + .add_generic_store(SingleFileMessagePackBackingStore(path)) + .build() + ) + cache.pull_all_backing_stores() + cache.put(item_doc, "item:potion", Item("Potion", "Consumable", 50)) # written to the .cc file + + reopened = ( + CultCache.builder() + .register_document_type(item_doc) + .add_generic_store(SingleFileMessagePackBackingStore(path)) + .build() + ) + reopened.pull_all_backing_stores() # read it back + assert reopened.get_by_name(item_doc, "Potion") == reopened.get_required(item_doc, "item:potion") + return reopened.get_required(item_doc, "item:potion") +# snippet:end + + +class FirstCallTests(unittest.TestCase): + def test_first_call_round_trips_through_a_second_cache(self) -> None: + self.assertEqual(first_call(), Item("Potion", "Consumable", 50)) + + def test_docs_show_the_tested_snippet(self) -> None: + source = Path(__file__).read_text(encoding="utf-8") + snippet = source.split("# snippet:begin\n", 1)[1].split("# snippet:end\n", 1)[0] + docs = (Path(__file__).parents[3] / "docs" / "start" / "README.md").read_text(encoding="utf-8") + self.assertIn("```python\n" + snippet + "```", docs) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/cultcache-rs/tests/first_call.rs b/packages/cultcache-rs/tests/first_call.rs new file mode 100644 index 00000000..6859ae07 --- /dev/null +++ b/packages/cultcache-rs/tests/first_call.rs @@ -0,0 +1,56 @@ +//! The Rust CultCache first call from docs/start/README.md. The docs show the region between the +//! `snippet` markers verbatim, and `docs_show_the_tested_snippet` fails when they differ. + +use cultcache_rs::{CultCache, DatabaseEntry, SingleFileMessagePackBackingStore}; + +// snippet:begin +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "item")] +struct Item { + #[cultcache(key = 0)] + name: String, + #[cultcache(key = 1)] + category: String, + #[cultcache(key = 2, default)] + value: u32, +} + +fn first_call() -> anyhow::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("first-call.cc"); + + let mut cache = CultCache::new(); + cache.register_entry_type::()?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)); + cache.pull_all_backing_stores()?; + cache.put("item:potion", &Item { + name: "Potion".to_string(), + category: "Consumable".to_string(), + value: 50, + })?; // written to the .cc file before this returns + + let mut reopened = CultCache::new(); + reopened.register_entry_type::()?; + reopened.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)); + reopened.pull_all_backing_stores()?; // read it back + let potion = reopened.get_required::("item:potion")?; + assert_eq!(potion.name, "Potion"); + assert_eq!(potion.value, 50); + Ok(()) +} +// snippet:end + +#[test] +fn first_call_round_trips_through_a_second_cache() { + first_call().unwrap(); +} + +#[test] +fn docs_show_the_tested_snippet() { + let source = std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/first_call.rs")).unwrap(); + let begin = source.find("// snippet:begin\n").unwrap() + "// snippet:begin\n".len(); + let end = source.find("// snippet:end\n").unwrap(); + let docs = std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/../../docs/start/README.md")).unwrap(); + let block = format!("```rust\n{}```", &source[begin..end]); + assert!(docs.contains(&block), "docs/start/README.md no longer shows tests/first_call.rs verbatim"); +} From 883271f62c540804b422899f96117ea6a0abad6d Mon Sep 17 00:00:00 2001 From: Eureka Date: Wed, 4 Nov 2026 20:26:13 +0000 Subject: [PATCH 40/88] CultMath: regenerate lavapipe device bounds at 85ce65fa Exact-case counts move: cycles_arcsine 154 -> 155 and cycles_arctan2 6 -> 7 on both routes; worst u1 and result hashes unchanged. Co-Authored-By: Claude Sonnet 5.5 --- .../CultMath.Tests/fixtures/device-bounds-lavapipe.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/cultmath/tests/CultMath.Tests/fixtures/device-bounds-lavapipe.txt b/packages/cultmath/tests/CultMath.Tests/fixtures/device-bounds-lavapipe.txt index 0a61b572..f383cc3d 100644 --- a/packages/cultmath/tests/CultMath.Tests/fixtures/device-bounds-lavapipe.txt +++ b/packages/cultmath/tests/CultMath.Tests/fixtures/device-bounds-lavapipe.txt @@ -96,9 +96,9 @@ lavapipe-glsl 91 vec3 cultmath_cycles_mix_val(float, vec3, vec3) ulp-bounded 256 lavapipe-glsl 92 vec3 cultmath_cycles_mix_color(float, vec3, vec3) ulp-bounded 256 0 0 00067b1567bcd6c07a81fa5002f145dfca22437a3f3df5389b2665334d14909b lavapipe-glsl 93 vec3 cultmath_cycles_mix_soft(float, vec3, vec3) ulp-bounded 157 1.9750388068880254 0 ed7c5508ca0543cc20a5f0c5624e75e3ca2a2d6044d481816eeda2edc50c0592 lavapipe-glsl 94 vec3 cultmath_cycles_mix_linear(float, vec3, vec3) ulp-bounded 256 0 0 672435cc30bcbbb253cce7316ee0712b1a18f06924bc89cea2d3388dd164a15d -lavapipe-glsl 95 float cultmath_cycles_arcsine(float) ulp-bounded, platform asin 154 2734.286809678319 0 f28f3ed0a53ffeebdfa84865d2d3d41590e6b46fc1578d40e1215e2a9e1af585 +lavapipe-glsl 95 float cultmath_cycles_arcsine(float) ulp-bounded, platform asin 155 2734.286809678319 0 f28f3ed0a53ffeebdfa84865d2d3d41590e6b46fc1578d40e1215e2a9e1af585 lavapipe-glsl 96 float cultmath_cycles_arccosine(float) ulp-bounded, platform acos 79 1046.8674409774287 0 22a2c38ab5a232cdbb33cf0c133362581a398f64a4d0d5d6dd89dc5af06bd890 -lavapipe-glsl 97 float cultmath_cycles_arctan2(float, float) ulp-bounded, platform atan 6 25.5 0 2c60447d80186f6dc9726f19c09fa96ec3f66d8f5688968da533d9195ea463c4 +lavapipe-glsl 97 float cultmath_cycles_arctan2(float, float) ulp-bounded, platform atan 7 25.5 0 2c60447d80186f6dc9726f19c09fa96ec3f66d8f5688968da533d9195ea463c4 lavapipe-glsl 98 mat3 cultmath_snoise_hessian(vec3) ulp-bounded 30 1358.4375 0 62bbafb8420ed5aaa0ce11d1470ca98945e690b186c33a1e7cf4d71478396465 lavapipe-glsl 99 mat3 cultmath_snoise_hessian(vec3) ulp-bounded, on the simplex tie set 110 1317.4754224467295 0 0efd9b3a637b46033ed0351f64de274db4d9ce05bb83dd1ce57afb3ceaea26bc lavapipe-glsl 100 float cultmath_integral_clamp_linear(float, float, float, float, float, float) ulp-bounded 256 0 0 84bf9c9a2126b3114a692507cc54f58d313afc33de8a1cac83969cc508433cbe @@ -222,9 +222,9 @@ lavapipe-hlsl 91 vec3 cultmath_cycles_mix_val(float, vec3, vec3) ulp-bounded 256 lavapipe-hlsl 92 vec3 cultmath_cycles_mix_color(float, vec3, vec3) ulp-bounded 256 0 0 00067b1567bcd6c07a81fa5002f145dfca22437a3f3df5389b2665334d14909b lavapipe-hlsl 93 vec3 cultmath_cycles_mix_soft(float, vec3, vec3) ulp-bounded 157 1.9750388068880254 0 ed7c5508ca0543cc20a5f0c5624e75e3ca2a2d6044d481816eeda2edc50c0592 lavapipe-hlsl 94 vec3 cultmath_cycles_mix_linear(float, vec3, vec3) ulp-bounded 256 0 0 672435cc30bcbbb253cce7316ee0712b1a18f06924bc89cea2d3388dd164a15d -lavapipe-hlsl 95 float cultmath_cycles_arcsine(float) ulp-bounded, platform asin 154 2734.286809678319 0 f28f3ed0a53ffeebdfa84865d2d3d41590e6b46fc1578d40e1215e2a9e1af585 +lavapipe-hlsl 95 float cultmath_cycles_arcsine(float) ulp-bounded, platform asin 155 2734.286809678319 0 f28f3ed0a53ffeebdfa84865d2d3d41590e6b46fc1578d40e1215e2a9e1af585 lavapipe-hlsl 96 float cultmath_cycles_arccosine(float) ulp-bounded, platform acos 79 1046.8674409774287 0 22a2c38ab5a232cdbb33cf0c133362581a398f64a4d0d5d6dd89dc5af06bd890 -lavapipe-hlsl 97 float cultmath_cycles_arctan2(float, float) ulp-bounded, platform atan 6 25.5 0 2c60447d80186f6dc9726f19c09fa96ec3f66d8f5688968da533d9195ea463c4 +lavapipe-hlsl 97 float cultmath_cycles_arctan2(float, float) ulp-bounded, platform atan 7 25.5 0 2c60447d80186f6dc9726f19c09fa96ec3f66d8f5688968da533d9195ea463c4 lavapipe-hlsl 98 mat3 cultmath_snoise_hessian(vec3) ulp-bounded 33 2377.8799856935966 0 a64853839f394cc7c367ba5e107caa8bb396f07cf202263aa917095dd1c939f7 lavapipe-hlsl 99 mat3 cultmath_snoise_hessian(vec3) ulp-bounded, on the simplex tie set 132 1317.4754224467295 0 9066a49e50731881583ffca16c949171bb61578a1dfd497c54dafe773ab49488 lavapipe-hlsl 100 float cultmath_integral_clamp_linear(float, float, float, float, float, float) ulp-bounded 256 0 0 84bf9c9a2126b3114a692507cc54f58d313afc33de8a1cac83969cc508433cbe From bbee623e298b73e5d46dfb6075d661ddeaca9de4 Mon Sep 17 00:00:00 2001 From: Libby Date: Wed, 4 Nov 2026 21:35:41 +0000 Subject: [PATCH 41/88] CultMath: Phacelle rows are order-independent (forward, backward, shuffled) Pins that height rows built from math.phacelle (the C# mirror of cultmath_phacelle) are bit-identical whatever order the rows are generated in, for a ring-buffer terrain. Golden fixture untouched. Co-Authored-By: Claude Sonnet 5.5 --- .../CultMath.Tests/PhacelleRowOrderTests.cs | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 packages/cultmath/tests/CultMath.Tests/PhacelleRowOrderTests.cs diff --git a/packages/cultmath/tests/CultMath.Tests/PhacelleRowOrderTests.cs b/packages/cultmath/tests/CultMath.Tests/PhacelleRowOrderTests.cs new file mode 100644 index 00000000..b465d75a --- /dev/null +++ b/packages/cultmath/tests/CultMath.Tests/PhacelleRowOrderTests.cs @@ -0,0 +1,91 @@ +using System; +using System.Linq; +using CultMath; +using Xunit; + +namespace CultMath.Tests; + +/// +/// A ring-buffer terrain (Voxel Space) fills height rows in whatever order the camera uncovers them, so a +/// Phacelle row must be a pure function of world x, z and the seed: the same row, bit for bit, whether the +/// rows around it were generated forward, backward or shuffled. C# phacelle mirrors cultmath_phacelle in +/// shaders/CultMath.Phacelle.glsl; GlslMirrorTests pins the two together, and this pins that neither keeps +/// state between calls. +/// +public sealed class PhacelleRowOrderTests +{ + private const int Rows = 24; + private const int Columns = 48; + private const int Seed = 7919; + + // One height sample: world (x, z) and the seed alone choose the point, the stripe direction and the phase. + private static int[] Sample(int worldX, int worldZ, int seed) + { + var p = new float3(worldX * 0.173f, (seed & 0xff) * 0.37f, worldZ * 0.173f); + var angle = (seed >> 8) * 0.61f; + var side = new float3(math.cos(angle), 0.0f, math.sin(angle)) * (math.TAU * 0.9f); + var phasor = math.phacelle(p, side, (seed & 0xf) / 16.0f, 0.5f); + return new[] + { + Bits(phasor.cosine.x), Bits(phasor.cosine.y), Bits(phasor.cosine.z), Bits(phasor.cosine.w), + Bits(phasor.sine.x), Bits(phasor.sine.y), Bits(phasor.sine.z), Bits(phasor.sine.w), + }; + } + + private static int Bits(float value) => BitConverter.SingleToInt32Bits(value); + + private static int[][] Row(int worldZ, int seed) => + Enumerable.Range(-Columns / 2, Columns).Select(x => Sample(x, worldZ, seed)).ToArray(); + + private static int[][][] Generate(int[] order, int seed) + { + var rows = new int[Rows][][]; + foreach (var z in order) rows[z] = Row(z - Rows / 2, seed); + return rows; + } + + private static void AssertSameRows(int[][][] expected, int[][][] actual) + { + for (var z = 0; z < Rows; z++) + for (var x = 0; x < Columns; x++) + Assert.True(expected[z][x].SequenceEqual(actual[z][x]), $"row {z} column {x} differs bit for bit"); + } + + [Fact] + public void RowsAreBitIdenticalForwardBackwardAndShuffled() + { + var forward = Enumerable.Range(0, Rows).ToArray(); + var backward = forward.Reverse().ToArray(); + var shuffled = (int[])forward.Clone(); + var random = new System.Random(20261104); + for (var i = shuffled.Length - 1; i > 0; i--) + { + var j = random.Next(i + 1); + (shuffled[i], shuffled[j]) = (shuffled[j], shuffled[i]); + } + + Assert.False(shuffled.SequenceEqual(forward) || shuffled.SequenceEqual(backward)); + var reference = Generate(forward, Seed); + AssertSameRows(reference, Generate(backward, Seed)); + AssertSameRows(reference, Generate(shuffled, Seed)); + } + + [Fact] + public void ARowIsTheSameWhenRegeneratedAfterAnyOtherSeedAndRow() + { + var first = Row(5, Seed); + _ = Row(-3, Seed + 1); + _ = Row(5, Seed + 1); + var again = Row(5, Seed); + for (var x = 0; x < Columns; x++) Assert.True(first[x].SequenceEqual(again[x])); + } + + // Guards the test itself: a different seed or a different row must change the bits, or the equalities + // above would hold for a function that ignores its inputs. + [Fact] + public void RowsDependOnSeedAndWorldRow() + { + Assert.False(Row(5, Seed).SelectMany(s => s).SequenceEqual(Row(5, Seed + 1).SelectMany(s => s))); + Assert.False(Row(5, Seed).SelectMany(s => s).SequenceEqual(Row(6, Seed).SelectMany(s => s))); + } +} From 30318c18fcba74f12858632da24ceb5b34637e15 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 5 Nov 2026 00:09:07 +0000 Subject: [PATCH 42/88] heimdall-delegation: delegation resolver with property tests Re-delegation to max depth, estate-of-delegator counting, per-estate cap on the final holder, scope narrowing, direct override, revocation, uncast on cycle/depth/trim, same-estate-only switch, per-person public receipts. No stake function; weights are caller-supplied. Co-Authored-By: Claude Sonnet 5.5 --- .gitignore | 2 + packages/heimdall-delegation/Cargo.lock | 7 + packages/heimdall-delegation/Cargo.toml | 10 + packages/heimdall-delegation/README.md | 29 ++ packages/heimdall-delegation/src/lib.rs | 270 +++++++++++++ .../heimdall-delegation/tests/properties.rs | 368 ++++++++++++++++++ 6 files changed, 686 insertions(+) create mode 100644 packages/heimdall-delegation/Cargo.lock create mode 100644 packages/heimdall-delegation/Cargo.toml create mode 100644 packages/heimdall-delegation/README.md create mode 100644 packages/heimdall-delegation/src/lib.rs create mode 100644 packages/heimdall-delegation/tests/properties.rs diff --git a/.gitignore b/.gitignore index 6d0f1f77..b033eb4c 100644 --- a/.gitignore +++ b/.gitignore @@ -322,6 +322,8 @@ node_modules/ !packages/cultmesh-browser/** !packages/cultmesh-kotlin/ !packages/cultmesh-kotlin/** +!packages/heimdall-delegation/ +!packages/heimdall-delegation/** packages/**/node_modules/ packages/**/dist/ packages/**/dist-test/ diff --git a/packages/heimdall-delegation/Cargo.lock b/packages/heimdall-delegation/Cargo.lock new file mode 100644 index 00000000..e706d732 --- /dev/null +++ b/packages/heimdall-delegation/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "heimdall-delegation" +version = "0.1.0" diff --git a/packages/heimdall-delegation/Cargo.toml b/packages/heimdall-delegation/Cargo.toml new file mode 100644 index 00000000..d57b3ab4 --- /dev/null +++ b/packages/heimdall-delegation/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "heimdall-delegation" +version = "0.1.0" +edition = "2024" +license = "MIT" +publish = false + +[lib] +name = "heimdall_delegation" +path = "src/lib.rs" diff --git a/packages/heimdall-delegation/README.md b/packages/heimdall-delegation/README.md new file mode 100644 index 00000000..40ac8cad --- /dev/null +++ b/packages/heimdall-delegation/README.md @@ -0,0 +1,29 @@ +# heimdall-delegation + +Heimdall's delegation resolver: given who is seated, who delegated to whom, and +who voted directly on a topic, it decides whose vote counts where, and publishes +a receipt for every seated person. Pure and deterministic; no I/O, no +dependencies. Entry point: `resolve` in `src/lib.rs`. Live: the resolver and its +property tests (`tests/properties.rs`). Not here: a stake function (the caller +supplies each person's `weight` until the estates' formula exists), storage, and +any CultMesh projection. + +## Rules + +- A person may delegate to any seated person. Delegates may re-delegate, up to + `Config::max_depth` hops. +- A delegated vote counts in the **delegator's** estate. +- Scope narrows along a chain: topic `t` flows only if every link's scope + contains `t` (the receipt carries the intersection). +- A direct vote on the topic overrides the voter's own delegation. +- Cycles, depth violations, unseated or (with `same_estate_only`) cross-estate + delegates, and chains that end without a vote return the vote **uncast** to + its originator. Nothing is re-routed. +- Cap: the final holder may wield at most `cap` (default 1/5) of an estate's + total weight delegated in from that estate. Over the cap, originators are + kept in ascending id order until the next would overflow; the rest are + trimmed and stay uncast. The ordering is a placeholder for Heimdall's ruling. +- Revoked delegations are inert. Resolution never raises the total: every + seated person yields exactly one receipt, cast + uncast = total weight. +- Input order never matters; duplicate roster entries, duplicate direct votes + and overlapping delegations from one person are input errors. diff --git a/packages/heimdall-delegation/src/lib.rs b/packages/heimdall-delegation/src/lib.rs new file mode 100644 index 00000000..adc7bfbc --- /dev/null +++ b/packages/heimdall-delegation/src/lib.rs @@ -0,0 +1,270 @@ +//! Heimdall's delegation resolver. See README.md for the rules. + +use std::collections::{BTreeMap, BTreeSet}; + +pub type PersonId = u64; +pub type EstateId = u64; +pub type Topic = String; +pub type Choice = String; + +/// A seated person. `weight` is supplied by the caller (no stake function yet). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Person { + pub id: PersonId, + pub estate: EstateId, + pub weight: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Scope { + All, + Only(BTreeSet), +} + +impl Scope { + pub fn covers(&self, topic: &str) -> bool { + match self { + Scope::All => true, + Scope::Only(s) => s.contains(topic), + } + } + + pub fn narrow(&self, other: &Scope) -> Scope { + match (self, other) { + (Scope::All, s) | (s, Scope::All) => s.clone(), + (Scope::Only(a), Scope::Only(b)) => Scope::Only(a.intersection(b).cloned().collect()), + } + } + + fn overlaps(&self, other: &Scope) -> bool { + match (self, other) { + (Scope::All, _) | (_, Scope::All) => true, + (Scope::Only(a), Scope::Only(b)) => a.intersection(b).next().is_some(), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Delegation { + pub from: PersonId, + pub to: PersonId, + pub scope: Scope, + /// Revoked delegations are inert. + pub revoked: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Vote { + pub voter: PersonId, + pub topic: Topic, + pub choice: Choice, +} + +/// A fraction of an estate's total weight. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Ratio { + pub num: u64, + pub den: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Config { + /// Maximum hops from originator to final holder; 0 disables delegation. + pub max_depth: usize, + /// Per-estate cap on what one final holder may wield from that estate. + pub cap: Ratio, + /// Fallback switch: refuse delegation across estates. + pub same_estate_only: bool, +} + +impl Default for Config { + fn default() -> Self { + Config { max_depth: 3, cap: Ratio { num: 1, den: 5 }, same_estate_only: false } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum InputError { + DuplicatePerson(PersonId), + DuplicateVote(PersonId), + ZeroCapDenominator, + /// Two live delegations from one person cover a common topic. + OverlappingDelegations(PersonId), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum UncastReason { + /// The chain ended at someone who neither voted nor delegated this topic. + NoVote, + Cycle, + DepthExceeded, + /// The next delegate is not seated. + NotSeated, + /// `same_estate_only` and the next delegate is in another estate. + CrossEstate, + /// The holder's cap for the originator's estate was reached. + Trimmed, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Outcome { + Direct { choice: Choice }, + Delegated { + holder: PersonId, + holder_estate: EstateId, + /// originator, delegates..., holder + chain: Vec, + /// Intersection of every link's scope. + scope: Scope, + choice: Choice, + }, + Uncast(UncastReason), +} + +/// Public receipt: one per seated person per topic. `estate` is the estate the +/// weight counts in (the originator's). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Receipt { + pub topic: Topic, + pub originator: PersonId, + pub estate: EstateId, + pub weight: u64, + pub outcome: Outcome, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Tally { + pub by_choice: BTreeMap, + /// estate -> choice -> weight, counted in the originator's estate. + pub by_estate: BTreeMap>, + pub cast: u64, + pub uncast: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Resolution { + /// Sorted by originator id. + pub receipts: Vec, + pub tally: Tally, +} + +enum Walk { + Direct(Choice), + Held { chain: Vec, scope: Scope, choice: Choice }, + Uncast(UncastReason), +} + +/// Resolve one topic. Pure; the order of every input slice is irrelevant. +pub fn resolve( + config: &Config, + people: &[Person], + delegations: &[Delegation], + votes: &[Vote], + topic: &str, +) -> Result { + if config.cap.den == 0 { + return Err(InputError::ZeroCapDenominator); + } + let mut roster: BTreeMap = BTreeMap::new(); + for p in people { + if roster.insert(p.id, p).is_some() { + return Err(InputError::DuplicatePerson(p.id)); + } + } + let mut direct: BTreeMap = BTreeMap::new(); + for v in votes.iter().filter(|v| v.topic == topic) { + if direct.insert(v.voter, &v.choice).is_some() { + return Err(InputError::DuplicateVote(v.voter)); + } + } + let mut live: BTreeMap> = BTreeMap::new(); + for d in delegations.iter().filter(|d| !d.revoked) { + live.entry(d.from).or_default().push(d); + } + for (from, ds) in &live { + for (i, a) in ds.iter().enumerate() { + if ds[i + 1..].iter().any(|b| a.scope.overlaps(&b.scope)) { + return Err(InputError::OverlappingDelegations(*from)); + } + } + } + + let walk = |origin: &Person| -> Walk { + if let Some(c) = direct.get(&origin.id) { + return Walk::Direct((*c).clone()); + } + let mut chain = vec![origin.id]; + let mut scope = Scope::All; + let mut cur = origin.id; + loop { + let Some(d) = live.get(&cur).and_then(|ds| ds.iter().find(|d| d.scope.covers(topic))) else { + return Walk::Uncast(UncastReason::NoVote); + }; + let Some(next) = roster.get(&d.to) else { + return Walk::Uncast(UncastReason::NotSeated); + }; + if config.same_estate_only && next.estate != origin.estate { + return Walk::Uncast(UncastReason::CrossEstate); + } + if chain.contains(&next.id) { + return Walk::Uncast(UncastReason::Cycle); + } + if chain.len() > config.max_depth { + return Walk::Uncast(UncastReason::DepthExceeded); + } + chain.push(next.id); + scope = scope.narrow(&d.scope); + if let Some(c) = direct.get(&next.id) { + return Walk::Held { chain, scope, choice: (*c).clone() }; + } + cur = next.id; + } + }; + + let mut receipts: Vec = Vec::with_capacity(roster.len()); + for p in roster.values() { + let outcome = match walk(p) { + Walk::Direct(choice) => Outcome::Direct { choice }, + Walk::Held { chain, scope, choice } => { + let holder = *chain.last().expect("chain has a holder"); + Outcome::Delegated { holder, holder_estate: roster[&holder].estate, chain, scope, choice } + } + Walk::Uncast(r) => Outcome::Uncast(r), + }; + receipts.push(Receipt { topic: topic.to_string(), originator: p.id, estate: p.estate, weight: p.weight, outcome }); + } + + // Cap: per (holder, originator estate), keep originators in ascending id + // order until the next would overflow; the rest are trimmed. + let mut estate_total: BTreeMap = BTreeMap::new(); + for p in roster.values() { + *estate_total.entry(p.estate).or_default() += p.weight as u128; + } + let mut used: BTreeMap<(PersonId, EstateId), u128> = BTreeMap::new(); + let mut full: BTreeSet<(PersonId, EstateId)> = BTreeSet::new(); + for r in receipts.iter_mut() { + let Outcome::Delegated { holder, .. } = r.outcome else { continue }; + let key = (holder, r.estate); + let limit = estate_total[&r.estate] * config.cap.num as u128 / config.cap.den as u128; + let after = used.get(&key).copied().unwrap_or(0) + r.weight as u128; + if full.contains(&key) || after > limit { + full.insert(key); + r.outcome = Outcome::Uncast(UncastReason::Trimmed); + } else { + used.insert(key, after); + } + } + + let mut tally = Tally::default(); + for r in &receipts { + match &r.outcome { + Outcome::Direct { choice } | Outcome::Delegated { choice, .. } => { + *tally.by_choice.entry(choice.clone()).or_default() += r.weight; + *tally.by_estate.entry(r.estate).or_default().entry(choice.clone()).or_default() += r.weight; + tally.cast += r.weight; + } + Outcome::Uncast(_) => tally.uncast += r.weight, + } + } + Ok(Resolution { receipts, tally }) +} diff --git a/packages/heimdall-delegation/tests/properties.rs b/packages/heimdall-delegation/tests/properties.rs new file mode 100644 index 00000000..3f86c026 --- /dev/null +++ b/packages/heimdall-delegation/tests/properties.rs @@ -0,0 +1,368 @@ +use heimdall_delegation::*; +use std::collections::BTreeSet; + +/// xorshift64*: seeded, dependency-free, reproducible. +struct Rng(u64); +impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 >> 12; + self.0 ^= self.0 << 25; + self.0 ^= self.0 >> 27; + self.0.wrapping_mul(0x2545F4914F6CDD1D) + } + fn below(&mut self, n: u64) -> u64 { + self.next() % n + } + fn shuffle(&mut self, v: &mut [T]) { + for i in (1..v.len()).rev() { + v.swap(i, self.below(i as u64 + 1) as usize); + } + } +} + +const TOPIC: &str = "budget"; + +#[derive(Clone)] +struct World { + config: Config, + people: Vec, + delegations: Vec, + votes: Vec, +} + +impl World { + fn resolve(&self) -> Resolution { + resolve(&self.config, &self.people, &self.delegations, &self.votes, TOPIC).unwrap() + } +} + +fn person(id: u64, estate: u64, weight: u64) -> Person { + Person { id, estate, weight } +} +fn del(from: u64, to: u64) -> Delegation { + Delegation { from, to, scope: Scope::All, revoked: false } +} +fn vote(voter: u64, choice: &str) -> Vote { + Vote { voter, topic: TOPIC.into(), choice: choice.into() } +} +fn only(topics: &[&str]) -> Scope { + Scope::Only(topics.iter().map(|t| t.to_string()).collect()) +} +fn world(config: Config, people: Vec, delegations: Vec, votes: Vec) -> World { + World { config, people, delegations, votes } +} +fn loose(max_depth: usize) -> Config { + Config { max_depth, cap: Ratio { num: 1, den: 1 }, same_estate_only: false } +} +fn outcome(r: &Resolution, id: u64) -> &Outcome { + &r.receipts.iter().find(|x| x.originator == id).unwrap().outcome +} + +/// Random world: ids 1..=n, 3 estates. Delegations may form cycles, cross +/// estates, point at unseated ids, carry scopes, and be revoked. At most one +/// delegation per person per scope bucket so inputs stay valid. +fn random_world(rng: &mut Rng) -> World { + let n = 2 + rng.below(14); + let people: Vec = (1..=n).map(|i| person(i, rng.below(3), 1 + rng.below(9))).collect(); + let mut delegations = vec![]; + for from in 1..=n { + match rng.below(4) { + 0 => {} + 1 | 2 => delegations.push(Delegation { + from, + to: 1 + rng.below(n + 2), // n+1, n+2 are unseated + scope: Scope::All, + revoked: rng.below(6) == 0, + }), + _ => { + // two disjoint scoped delegations + delegations.push(Delegation { from, to: 1 + rng.below(n), scope: only(&["budget"]), revoked: false }); + delegations.push(Delegation { from, to: 1 + rng.below(n), scope: only(&["war", "trade"]), revoked: false }); + } + } + } + let mut votes = vec![]; + for v in 1..=n { + if rng.below(3) == 0 { + votes.push(vote(v, ["yes", "no", "abstain"][rng.below(3) as usize])); + } + } + let config = Config { + max_depth: rng.below(5) as usize, + cap: Ratio { num: 1 + rng.below(3), den: 1 + rng.below(5) }, + same_estate_only: rng.below(4) == 0, + }; + World { config, people, delegations, votes } +} + +fn worlds(n: u64) -> impl Iterator { + (1..=n).map(|seed| (seed, random_world(&mut Rng(seed * 7919)))) +} + +#[test] +fn order_independence_under_shuffled_input() { + for (seed, w) in worlds(400) { + let base = w.resolve(); + let mut rng = Rng(seed + 1000); + for _ in 0..5 { + let mut s = w.clone(); + rng.shuffle(&mut s.people); + rng.shuffle(&mut s.delegations); + rng.shuffle(&mut s.votes); + assert_eq!(s.resolve(), base, "seed {seed}"); + } + } +} + +#[test] +fn conservation_one_receipt_per_person_and_no_inflation() { + for (seed, w) in worlds(400) { + let r = w.resolve(); + let total: u64 = w.people.iter().map(|p| p.weight).sum(); + assert_eq!(r.receipts.len(), w.people.len(), "seed {seed}"); + assert_eq!(r.tally.cast + r.tally.uncast, total, "seed {seed}"); + assert!(r.tally.cast <= total); + assert_eq!(r.tally.by_choice.values().sum::(), r.tally.cast); + for p in &w.people { + let counted: u64 = r.tally.by_estate.get(&p.estate).map_or(0, |m| m.values().sum()); + let estate_total: u64 = w.people.iter().filter(|q| q.estate == p.estate).map(|q| q.weight).sum(); + assert!(counted <= estate_total, "seed {seed}: estate {} inflated", p.estate); + } + // Delegated weight counts in the delegator's estate, not the holder's. + for rc in &r.receipts { + let p = w.people.iter().find(|p| p.id == rc.originator).unwrap(); + assert_eq!((rc.estate, rc.weight), (p.estate, p.weight)); + } + } +} + +#[test] +fn direct_vote_overrides_any_delegation() { + for (seed, w) in worlds(400) { + let r = w.resolve(); + for v in &w.votes { + assert_eq!(outcome(&r, v.voter), &Outcome::Direct { choice: v.choice.clone() }, "seed {seed}"); + } + } + // Same person with and without a delegation chain pointing elsewhere. + let people = vec![person(1, 0, 1), person(2, 0, 1), person(3, 0, 1)]; + let votes = vec![vote(1, "yes"), vote(3, "no")]; + let with = world(loose(3), people.clone(), vec![del(1, 2), del(2, 3)], votes.clone()).resolve(); + let without = world(loose(3), people, vec![del(2, 3)], votes).resolve(); + assert_eq!(outcome(&with, 1), &Outcome::Direct { choice: "yes".into() }); + assert_eq!(with.receipts[0], without.receipts[0]); +} + +#[test] +fn cycles_return_the_vote_to_every_originator_in_them() { + let people: Vec<_> = (1..=4).map(|i| person(i, 0, 1)).collect(); + let r = world(loose(9), people.clone(), vec![del(1, 2), del(2, 3), del(3, 1), del(4, 1)], vec![]).resolve(); + for id in 1..=4 { + assert_eq!(outcome(&r, id), &Outcome::Uncast(UncastReason::Cycle)); + } + assert_eq!((r.tally.cast, r.tally.uncast), (0, 4)); + // Self-delegation is a cycle too. + let r = world(loose(9), people, vec![del(1, 1)], vec![]).resolve(); + assert_eq!(outcome(&r, 1), &Outcome::Uncast(UncastReason::Cycle)); +} + +#[test] +fn delegated_chains_are_acyclic_and_within_depth() { + for (seed, w) in worlds(400) { + for rc in &w.resolve().receipts { + if let Outcome::Delegated { chain, .. } = &rc.outcome { + let uniq: BTreeSet<_> = chain.iter().collect(); + assert_eq!(uniq.len(), chain.len(), "seed {seed}"); + assert!(chain.len() - 1 <= w.config.max_depth, "seed {seed}"); + assert_eq!(chain[0], rc.originator); + } + } + } +} + +#[test] +fn depth_boundary_is_exact() { + // 1 -> 2 -> 3 -> 4(votes): three hops. + let people: Vec<_> = (1..=4).map(|i| person(i, 0, 1)).collect(); + let d = vec![del(1, 2), del(2, 3), del(3, 4)]; + let v = vec![vote(4, "yes")]; + let at = world(loose(3), people.clone(), d.clone(), v.clone()).resolve(); + assert!(matches!(outcome(&at, 1), Outcome::Delegated { holder: 4, .. })); + let over = world(loose(2), people, d, v).resolve(); + assert_eq!(outcome(&over, 1), &Outcome::Uncast(UncastReason::DepthExceeded)); + assert!(matches!(outcome(&over, 2), Outcome::Delegated { holder: 4, .. })); + assert!(matches!(outcome(&over, 3), Outcome::Delegated { holder: 4, .. })); +} + +#[test] +fn zero_depth_disables_delegation() { + let people = vec![person(1, 0, 1), person(2, 0, 1)]; + let r = world(loose(0), people, vec![del(1, 2)], vec![vote(2, "yes")]).resolve(); + assert_eq!(outcome(&r, 1), &Outcome::Uncast(UncastReason::DepthExceeded)); +} + +#[test] +fn scope_narrows_along_the_chain() { + let people: Vec<_> = (1..=3).map(|i| person(i, 0, 1)).collect(); + let mut a = del(1, 2); + a.scope = only(&["budget", "war"]); + let mut b = del(2, 3); + b.scope = only(&["budget", "trade"]); + let r = world(loose(3), people, vec![a, b], vec![vote(3, "yes")]).resolve(); + match outcome(&r, 1) { + Outcome::Delegated { scope, .. } => assert_eq!(scope, &only(&["budget"])), + o => panic!("{o:?}"), + } + // Topic outside the second link's scope: chain ends at 2, who has no vote. + let mut a = del(1, 2); + a.scope = Scope::All; + let mut b = del(2, 3); + b.scope = only(&["trade"]); + let people: Vec<_> = (1..=3).map(|i| person(i, 0, 1)).collect(); + let r = world(loose(3), people, vec![a, b], vec![vote(3, "yes")]).resolve(); + assert_eq!(outcome(&r, 1), &Outcome::Uncast(UncastReason::NoVote)); +} + +#[test] +fn every_delegated_scope_covers_the_topic() { + for (seed, w) in worlds(400) { + for rc in &w.resolve().receipts { + if let Outcome::Delegated { scope, .. } = &rc.outcome { + assert!(scope.covers(TOPIC), "seed {seed}"); + } + } + } +} + +#[test] +fn revoked_delegation_equals_removed_delegation() { + for (seed, w) in worlds(400) { + let mut revoked = w.clone(); + for d in revoked.delegations.iter_mut().filter(|d| d.from % 2 == 0) { + d.revoked = true; + } + let mut removed = w.clone(); + removed.delegations.retain(|d| d.from % 2 != 0); + assert_eq!(revoked.resolve(), removed.resolve(), "seed {seed}"); + } +} + +#[test] +fn unseated_delegate_returns_the_vote() { + let r = world(loose(3), vec![person(1, 0, 4)], vec![del(1, 99)], vec![vote(99, "yes")]).resolve(); + assert_eq!(outcome(&r, 1), &Outcome::Uncast(UncastReason::NotSeated)); + assert_eq!(r.tally.uncast, 4); +} + +#[test] +fn same_estate_only_refuses_cross_estate_hops() { + let people = vec![person(1, 0, 1), person(2, 1, 1), person(3, 0, 1)]; + let d = vec![del(1, 2), del(3, 1)]; + let v = vec![vote(2, "yes")]; + let open = world(loose(3), people.clone(), d.clone(), v.clone()).resolve(); + assert!(matches!(outcome(&open, 1), Outcome::Delegated { holder: 2, holder_estate: 1, .. })); + let mut c = loose(3); + c.same_estate_only = true; + let shut = world(c, people, d, v).resolve(); + assert_eq!(outcome(&shut, 1), &Outcome::Uncast(UncastReason::CrossEstate)); + assert_eq!(outcome(&shut, 3), &Outcome::Uncast(UncastReason::CrossEstate)); + // Delegated votes count in the delegator's estate even across estates. + assert_eq!(open.tally.by_estate[&0]["yes"], 2); + assert_eq!(open.tally.by_estate[&1]["yes"], 1); +} + +#[test] +fn same_estate_switch_only_ever_removes_delegated_votes() { + for (seed, w) in worlds(400) { + let mut shut = w.clone(); + shut.config.same_estate_only = true; + for rc in &shut.resolve().receipts { + if let Outcome::Delegated { holder_estate, .. } = rc.outcome { + assert_eq!(holder_estate, rc.estate, "seed {seed}"); + } + } + } +} + +#[test] +fn cap_trims_deterministically_in_originator_order() { + // Estate 0: ten people of weight 1, holder is person 11 (estate 1). + // Cap 1/5 of estate 0 (total 10) = 2. + let mut people: Vec<_> = (1..=10).map(|i| person(i, 0, 1)).collect(); + people.push(person(11, 1, 1)); + let d: Vec<_> = (1..=10).map(|i| del(i, 11)).collect(); + let mut c = loose(1); + c.cap = Ratio { num: 1, den: 5 }; + let w = world(c, people, d, vec![vote(11, "yes")]); + let r = w.resolve(); + for id in 1..=2 { + assert!(matches!(outcome(&r, id), Outcome::Delegated { holder: 11, .. })); + } + for id in 3..=10 { + assert_eq!(outcome(&r, id), &Outcome::Uncast(UncastReason::Trimmed)); + } + assert_eq!(r.tally.by_estate[&0]["yes"], 2); + // Trimmed votes are not re-routed: the holder's own weight is untouched. + assert_eq!(r.tally.by_estate[&1]["yes"], 1); + assert_eq!(r.tally.uncast, 8); +} + +#[test] +fn cap_is_per_estate_and_applies_to_the_final_holder() { + // Two estates of 5. Cap 1/5 -> 1 each. Holder 20 gets 1 from each estate. + let mut people: Vec<_> = (1..=5).map(|i| person(i, 0, 1)).collect(); + people.extend((6..=10).map(|i| person(i, 1, 1))); + people.push(person(20, 2, 1)); + let d: Vec<_> = (1..=10).map(|i| del(i, 20)).collect(); + let mut c = loose(1); + c.cap = Ratio { num: 1, den: 5 }; + let r = world(c, people, d, vec![vote(20, "yes")]).resolve(); + let held: Vec = r + .receipts + .iter() + .filter(|x| matches!(x.outcome, Outcome::Delegated { .. })) + .map(|x| x.originator) + .collect(); + assert_eq!(held, vec![1, 6]); +} + +#[test] +fn cap_holds_for_every_holder_and_estate() { + for (seed, w) in worlds(400) { + let r = w.resolve(); + let mut per: std::collections::BTreeMap<(u64, u64), u128> = Default::default(); + for rc in &r.receipts { + if let Outcome::Delegated { holder, .. } = rc.outcome { + *per.entry((holder, rc.estate)).or_default() += rc.weight as u128; + } + } + for ((_, estate), got) in per { + let total: u128 = w.people.iter().filter(|p| p.estate == estate).map(|p| p.weight as u128).sum(); + assert!(got <= total * w.config.cap.num as u128 / w.config.cap.den as u128, "seed {seed}"); + } + } +} + +#[test] +fn invalid_input_is_an_error_not_a_guess() { + let p = vec![person(1, 0, 1), person(1, 0, 1)]; + assert_eq!(resolve(&loose(1), &p, &[], &[], TOPIC), Err(InputError::DuplicatePerson(1))); + let p = vec![person(1, 0, 1)]; + let v = vec![vote(1, "a"), vote(1, "b")]; + assert_eq!(resolve(&loose(1), &p, &[], &v, TOPIC), Err(InputError::DuplicateVote(1))); + let mut a = del(1, 2); + a.scope = only(&["budget"]); + let b = del(1, 3); + assert_eq!(resolve(&loose(1), &p, &[a, b], &[], TOPIC), Err(InputError::OverlappingDelegations(1))); + let mut c = loose(1); + c.cap.den = 0; + assert_eq!(resolve(&c, &p, &[], &[], TOPIC), Err(InputError::ZeroCapDenominator)); +} + +#[test] +fn votes_on_other_topics_are_ignored() { + let people = vec![person(1, 0, 1), person(2, 0, 1)]; + let other = Vote { voter: 2, topic: "war".into(), choice: "yes".into() }; + let r = resolve(&loose(2), &people, &[del(1, 2)], &[other], TOPIC).unwrap(); + assert_eq!(r.tally.cast, 0); +} From e2a39d8aefd6adc532d6c8b235fca669c8d38a32 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 5 Nov 2026 05:26:37 +0000 Subject: [PATCH 43/88] heimdall-delegation: pro-rata cap, revocation time on receipts, prior art Replace ascending-id trimming with pro-rata scaling by largest remainder (ascending originator id breaks exact remainder ties only). Receipts carry counted weight and the originator's latest covering revocation time. Delegation::revoked becomes Option. Property tests: trim order-independence, cap never exceeded, no delegator scaled more than another. README: prior-art section, placeholders labelled. Co-Authored-By: Claude Sonnet 5.5 --- packages/heimdall-delegation/README.md | 76 +++++- packages/heimdall-delegation/src/lib.rs | 96 ++++++-- .../heimdall-delegation/tests/properties.rs | 223 ++++++++++++++---- 3 files changed, 324 insertions(+), 71 deletions(-) diff --git a/packages/heimdall-delegation/README.md b/packages/heimdall-delegation/README.md index 40ac8cad..bf0eca38 100644 --- a/packages/heimdall-delegation/README.md +++ b/packages/heimdall-delegation/README.md @@ -19,11 +19,73 @@ any CultMesh projection. - Cycles, depth violations, unseated or (with `same_estate_only`) cross-estate delegates, and chains that end without a vote return the vote **uncast** to its originator. Nothing is re-routed. -- Cap: the final holder may wield at most `cap` (default 1/5) of an estate's - total weight delegated in from that estate. Over the cap, originators are - kept in ascending id order until the next would overflow; the rest are - trimmed and stay uncast. The ordering is a placeholder for Heimdall's ruling. -- Revoked delegations are inert. Resolution never raises the total: every - seated person yields exactly one receipt, cast + uncast = total weight. +- Cap (**placeholder**): per (final holder, originator estate) the holder may + wield at most `cap` (default 1/5) of that estate's total weight. The holder's + own direct vote is not counted against it. Over the cap, every delegator into + that holder from that estate is scaled by the same factor `limit / sum`, in + integer weight by largest remainder; only exact remainder ties fall to + ascending originator id. A receipt's `counted` is its scaled weight and + `weight - counted` is uncast; a share that rounds to zero is + `Uncast(Trimmed)`. Cast + uncast = total weight. +- Revocation (**placeholder**): `Delegation::revoked` is `Some(time)` when + revoked. A revoked delegation is inert; the resolver reads no clock and the + caller owns the ledger. Each receipt carries `revoked_at`, the latest + revocation time among the originator's own revoked delegations covering the + topic (revocations further down a chain are not reported). - Input order never matters; duplicate roster entries, duplicate direct votes - and overlapping delegations from one person are input errors. + and overlapping delegations from one person are input errors (**placeholder**: overlapping scopes are rejected, not resolved by precedence). + +## Placeholders until the estates rule + +Four choices are Heimdall's #swarm rulings of 2026-11-05 standing in for the +estates' own decision, and change if they rule otherwise: (1) the cap is +pro-rata, (2) the cap excludes the holder's own direct vote, (3) overlapping +scopes are an input error, (4) revocation is a flag with a time and the caller +owns the ledger. + +## Prior art + +Sources were fetched on 2026-11-05; claims below are what those pages say. + +- **LiquidFeedback, cycles.** The platform treats a cycle as a non-problem: if + everyone in a loop only delegates nobody votes, and if anyone in it acts + directly their delegation is suspended, so the loop breaks + ([Liquid Democracy Journal 3, "Circular Delegations: Myth or Disaster"](https://liquid-democracy-journal.org/issue/3/The_Liquid_Democracy_Journal-Issue003-02-Circular_Delegations_-_Myth_or_Disaster.html)). + Delegations are per area and a single area's set must not form cycles + ([Kling et al. 2015](https://ar5iv.arxiv.org/html/1503.07723)). *Consequence:* + we keep a direct vote overriding the voter's own delegation, and a + cycle with no direct vote returns every member's weight uncast to its + originator rather than re-routing it. We do not forbid cycles at input time + (the ledger is the caller's and may form them); we resolve them. +- **Kling, Kunegis, Hartmann, Strohmaier, Staab 2015** ([ICWSM](https://ojs.aaai.org/index.php/ICWSM/article/view/14618), + [arXiv:1503.07723](https://arxiv.org/abs/1503.07723)), a study of + LiquidFeedback in the German Pirate Party. Weight is power-law distributed + (38 voters held more than 100 delegations); delegation is transitive; super-voters + have high theoretical power but exercise less of it (mean exercised/theoretical + ratio 0.34), and the paper argues potential power is the safer measure. It + proposes no cap. *Consequence:* a cap is our own addition, justified by the + potential-power argument, not by that paper; keep the cap a separate, + swappable `Config` field so the estates can set it to 1/1 and turn it off. + Transitive delegation with `max_depth` stays. +- **Gölz, Kahng, Mackenzie, Procaccia 2018** ([The Fluid Mechanics of Liquid + Democracy](https://arxiv.org/abs/1808.01906), WINE 2018; TEAC 2021). With single + delegation the maximum guru weight grows polynomially in the electorate; + letting voters list more than one option and routing by confluent flow brings + it to O(log log t). It handles cycles in general graphs by processing strongly + connected components, and notes that people who delegate only among themselves + have weight that reaches nobody. It considers caps and rejects them: voters + can coordinate outside the system and copy a delegate's ballot, so a cap + "would not alleviate the problem of super-voters, just reduce + transparency" ([full text](https://ar5iv.arxiv.org/html/1808.01906)). + *Consequences:* (a) a cap bounds what the platform records, not what a + holder can persuade; the receipts' `counted`/uncast split keeps the trimmed + part visible instead of hiding it, which is the transparency their objection + turns on. (b) Pro-rata scaling treats equal delegators equally without a + priority order a holder could game by who delegates first; ascending-id + trimming did not. (c) Multi-option delegation is the research direction for + reducing concentration without a cap; it is out of scope here and would need + the same preference-vs-equal-treatment question the Liquid Democracy Journal + piece raises about preferential delegation. +- **Not checked:** this section does not cover other liquid-democracy + implementations (e.g. Loomio, Democracy Earth) or the cycle literature after + 2018. diff --git a/packages/heimdall-delegation/src/lib.rs b/packages/heimdall-delegation/src/lib.rs index adc7bfbc..a6fe97d6 100644 --- a/packages/heimdall-delegation/src/lib.rs +++ b/packages/heimdall-delegation/src/lib.rs @@ -49,8 +49,9 @@ pub struct Delegation { pub from: PersonId, pub to: PersonId, pub scope: Scope, - /// Revoked delegations are inert. - pub revoked: bool, + /// `Some(t)` means revoked at caller-clock time `t`. Revoked delegations are + /// inert; the resolver never reads a clock and the caller owns the ledger. + pub revoked: Option, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -102,7 +103,7 @@ pub enum UncastReason { NotSeated, /// `same_estate_only` and the next delegate is in another estate. CrossEstate, - /// The holder's cap for the originator's estate was reached. + /// The cap scaled this originator's whole weight away (rounded to zero). Trimmed, } @@ -129,6 +130,14 @@ pub struct Receipt { pub originator: PersonId, pub estate: EstateId, pub weight: u64, + /// Part of `weight` that counted for the outcome's choice. `weight` for a + /// direct vote, `0` for an uncast one, and for a delegated one `weight` + /// scaled by the cap; `weight - counted` is uncast. + pub counted: u64, + /// Latest revocation time among the originator's own revoked delegations + /// whose scope covers this topic. Those delegations took no part in the + /// resolution; the receipt records that they existed. + pub revoked_at: Option, pub outcome: Outcome, } @@ -178,8 +187,16 @@ pub fn resolve( } } let mut live: BTreeMap> = BTreeMap::new(); - for d in delegations.iter().filter(|d| !d.revoked) { - live.entry(d.from).or_default().push(d); + let mut revoked_at: BTreeMap = BTreeMap::new(); + for d in delegations { + match d.revoked { + None => live.entry(d.from).or_default().push(d), + Some(t) if d.scope.covers(topic) => { + let e = revoked_at.entry(d.from).or_insert(t); + *e = (*e).max(t); + } + Some(_) => {} + } } for (from, ds) in &live { for (i, a) in ds.iter().enumerate() { @@ -231,27 +248,59 @@ pub fn resolve( } Walk::Uncast(r) => Outcome::Uncast(r), }; - receipts.push(Receipt { topic: topic.to_string(), originator: p.id, estate: p.estate, weight: p.weight, outcome }); + let counted = if matches!(outcome, Outcome::Direct { .. } | Outcome::Delegated { .. }) { p.weight } else { 0 }; + receipts.push(Receipt { + topic: topic.to_string(), + originator: p.id, + estate: p.estate, + weight: p.weight, + counted, + revoked_at: revoked_at.get(&p.id).copied(), + outcome, + }); } - // Cap: per (holder, originator estate), keep originators in ascending id - // order until the next would overflow; the rest are trimmed. + // Cap: per (final holder, originator estate) the holder may wield at most + // floor(estate total * cap) of delegated weight (the holder's own direct + // vote is not part of it). An over-cap group is scaled pro rata: every + // delegator by the same factor limit / sum, in integer weight by largest + // remainder. Only exact remainder ties fall to ascending originator id. let mut estate_total: BTreeMap = BTreeMap::new(); for p in roster.values() { *estate_total.entry(p.estate).or_default() += p.weight as u128; } - let mut used: BTreeMap<(PersonId, EstateId), u128> = BTreeMap::new(); - let mut full: BTreeSet<(PersonId, EstateId)> = BTreeSet::new(); - for r in receipts.iter_mut() { - let Outcome::Delegated { holder, .. } = r.outcome else { continue }; - let key = (holder, r.estate); - let limit = estate_total[&r.estate] * config.cap.num as u128 / config.cap.den as u128; - let after = used.get(&key).copied().unwrap_or(0) + r.weight as u128; - if full.contains(&key) || after > limit { - full.insert(key); - r.outcome = Outcome::Uncast(UncastReason::Trimmed); - } else { - used.insert(key, after); + let mut groups: BTreeMap<(PersonId, EstateId), Vec> = BTreeMap::new(); + for (i, r) in receipts.iter().enumerate() { + if let Outcome::Delegated { holder, .. } = r.outcome { + groups.entry((holder, r.estate)).or_default().push(i); + } + } + for ((_, estate), members) in groups { + let limit = estate_total[&estate] * config.cap.num as u128 / config.cap.den as u128; + let sum: u128 = members.iter().map(|&i| receipts[i].weight as u128).sum(); + if sum <= limit { + continue; + } + // members are in ascending originator id (receipts are), so a stable + // sort by descending remainder leaves ids ascending within ties. + let mut shares: Vec<(usize, u128, u128)> = members + .iter() + .map(|&i| { + let scaled = receipts[i].weight as u128 * limit; + (i, scaled / sum, scaled % sum) + }) + .collect(); + let leftover = limit - shares.iter().map(|s| s.1).sum::(); + let mut order: Vec = (0..shares.len()).collect(); + order.sort_by(|&a, &b| shares[b].2.cmp(&shares[a].2)); + for &k in order.iter().take(leftover as usize) { + shares[k].1 += 1; + } + for (i, counted, _) in shares { + receipts[i].counted = counted as u64; + if counted == 0 { + receipts[i].outcome = Outcome::Uncast(UncastReason::Trimmed); + } } } @@ -259,9 +308,10 @@ pub fn resolve( for r in &receipts { match &r.outcome { Outcome::Direct { choice } | Outcome::Delegated { choice, .. } => { - *tally.by_choice.entry(choice.clone()).or_default() += r.weight; - *tally.by_estate.entry(r.estate).or_default().entry(choice.clone()).or_default() += r.weight; - tally.cast += r.weight; + *tally.by_choice.entry(choice.clone()).or_default() += r.counted; + *tally.by_estate.entry(r.estate).or_default().entry(choice.clone()).or_default() += r.counted; + tally.cast += r.counted; + tally.uncast += r.weight - r.counted; } Outcome::Uncast(_) => tally.uncast += r.weight, } diff --git a/packages/heimdall-delegation/tests/properties.rs b/packages/heimdall-delegation/tests/properties.rs index 3f86c026..e4fa61ad 100644 --- a/packages/heimdall-delegation/tests/properties.rs +++ b/packages/heimdall-delegation/tests/properties.rs @@ -40,7 +40,7 @@ fn person(id: u64, estate: u64, weight: u64) -> Person { Person { id, estate, weight } } fn del(from: u64, to: u64) -> Delegation { - Delegation { from, to, scope: Scope::All, revoked: false } + Delegation { from, to, scope: Scope::All, revoked: None } } fn vote(voter: u64, choice: &str) -> Vote { Vote { voter, topic: TOPIC.into(), choice: choice.into() } @@ -72,12 +72,12 @@ fn random_world(rng: &mut Rng) -> World { from, to: 1 + rng.below(n + 2), // n+1, n+2 are unseated scope: Scope::All, - revoked: rng.below(6) == 0, + revoked: (rng.below(6) == 0).then(|| rng.below(1000)), }), _ => { // two disjoint scoped delegations - delegations.push(Delegation { from, to: 1 + rng.below(n), scope: only(&["budget"]), revoked: false }); - delegations.push(Delegation { from, to: 1 + rng.below(n), scope: only(&["war", "trade"]), revoked: false }); + delegations.push(Delegation { from, to: 1 + rng.below(n), scope: only(&["budget"]), revoked: None }); + delegations.push(Delegation { from, to: 1 + rng.below(n), scope: only(&["war", "trade"]), revoked: None }); } } } @@ -239,11 +239,15 @@ fn revoked_delegation_equals_removed_delegation() { for (seed, w) in worlds(400) { let mut revoked = w.clone(); for d in revoked.delegations.iter_mut().filter(|d| d.from % 2 == 0) { - d.revoked = true; + d.revoked = Some(d.revoked.unwrap_or(7)); } let mut removed = w.clone(); removed.delegations.retain(|d| d.from % 2 != 0); - assert_eq!(revoked.resolve(), removed.resolve(), "seed {seed}"); + let (mut a, mut b) = (revoked.resolve(), removed.resolve()); + for r in a.receipts.iter_mut().chain(b.receipts.iter_mut()) { + r.revoked_at = None; + } + assert_eq!(a, b, "seed {seed}"); } } @@ -285,26 +289,37 @@ fn same_estate_switch_only_ever_removes_delegated_votes() { } #[test] -fn cap_trims_deterministically_in_originator_order() { - // Estate 0: ten people of weight 1, holder is person 11 (estate 1). - // Cap 1/5 of estate 0 (total 10) = 2. +fn cap_scales_every_delegator_by_the_same_factor() { + // Estate 0: ten people of weight 1 plus a weight-10 whale (total 20); + // cap 1/5 -> 4 from estate 0. Eleven delegators into holder 12 (estate 1): + // sum 20 > 4, factor 1/5: the whale keeps 2, the ten keep 0.2 each; the + // two leftover units go to the largest remainders (equal, so ids 1 and 2). let mut people: Vec<_> = (1..=10).map(|i| person(i, 0, 1)).collect(); - people.push(person(11, 1, 1)); - let d: Vec<_> = (1..=10).map(|i| del(i, 11)).collect(); + people.push(person(11, 0, 10)); + people.push(person(12, 1, 1)); + let d: Vec<_> = (1..=11).map(|i| del(i, 12)).collect(); let mut c = loose(1); c.cap = Ratio { num: 1, den: 5 }; - let w = world(c, people, d, vec![vote(11, "yes")]); - let r = w.resolve(); - for id in 1..=2 { - assert!(matches!(outcome(&r, id), Outcome::Delegated { holder: 11, .. })); - } - for id in 3..=10 { - assert_eq!(outcome(&r, id), &Outcome::Uncast(UncastReason::Trimmed)); - } - assert_eq!(r.tally.by_estate[&0]["yes"], 2); - // Trimmed votes are not re-routed: the holder's own weight is untouched. + let r = world(c, people, d, vec![vote(12, "yes")]).resolve(); + let counted: Vec = r.receipts.iter().filter(|x| x.estate == 0).map(|x| x.counted).collect(); + assert_eq!(counted, vec![1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2]); + assert_eq!(outcome(&r, 3), &Outcome::Uncast(UncastReason::Trimmed)); + assert_eq!(r.tally.by_estate[&0]["yes"], 4); + // The holder's own direct vote is outside the cap and untouched. assert_eq!(r.tally.by_estate[&1]["yes"], 1); - assert_eq!(r.tally.uncast, 8); + assert_eq!((r.tally.cast, r.tally.uncast), (5, 16)); +} + +#[test] +fn partial_trim_keeps_the_delegation_and_reports_the_uncast_part() { + let people = vec![person(1, 0, 6), person(2, 0, 4), person(3, 1, 1)]; + let mut c = loose(1); + c.cap = Ratio { num: 1, den: 2 }; // limit 5 of estate 0 (total 10) + let r = world(c, people, vec![del(1, 3), del(2, 3)], vec![vote(3, "yes")]).resolve(); + let rc = |id| r.receipts.iter().find(|x| x.originator == id).unwrap(); + assert_eq!((rc(1).counted, rc(2).counted), (3, 2)); // exactly 6/10 and 4/10 of 5 + assert!(matches!(rc(1).outcome, Outcome::Delegated { holder: 3, .. })); + assert_eq!(r.tally.uncast, 5); } #[test] @@ -317,30 +332,156 @@ fn cap_is_per_estate_and_applies_to_the_final_holder() { let mut c = loose(1); c.cap = Ratio { num: 1, den: 5 }; let r = world(c, people, d, vec![vote(20, "yes")]).resolve(); - let held: Vec = r - .receipts - .iter() - .filter(|x| matches!(x.outcome, Outcome::Delegated { .. })) - .map(|x| x.originator) - .collect(); - assert_eq!(held, vec![1, 6]); + for e in [0, 1] { + let got: u64 = r.receipts.iter().filter(|x| x.estate == e).map(|x| x.counted).sum(); + assert_eq!(got, 1, "estate {e}"); + } +} + +/// Worlds built to hit the cap: few holders, most people delegate straight +/// to them, varied weights, tight caps. +fn crowded_world(rng: &mut Rng) -> World { + let n = 6 + rng.below(20); + let holders = 1 + rng.below(3); + let people: Vec = (1..=n).map(|i| person(i, rng.below(3), 1 + rng.below(12))).collect(); + let mut delegations = vec![]; + let mut votes = vec![]; + for i in 1..=n { + if i <= holders { + votes.push(vote(i, "yes")); + } else if rng.below(8) != 0 { + delegations.push(del(i, 1 + rng.below(holders))); + } + } + let config = Config { max_depth: 1, cap: Ratio { num: 1, den: 2 + rng.below(6) }, same_estate_only: false }; + World { config, people, delegations, votes } +} + +fn crowded(n: u64) -> impl Iterator { + (1..=n).map(|seed| (seed, crowded_world(&mut Rng(seed * 104729)))) +} + +fn groups(w: &World, r: &Resolution) -> std::collections::BTreeMap<(u64, u64), Vec> { + let _ = w; + let mut g: std::collections::BTreeMap<(u64, u64), Vec> = Default::default(); + for rc in &r.receipts { + if let Outcome::Delegated { holder, .. } = rc.outcome { + g.entry((holder, rc.estate)).or_default().push(rc.clone()); + } + } + g +} + +fn limit(w: &World, estate: u64) -> u128 { + let total: u128 = w.people.iter().filter(|p| p.estate == estate).map(|p| p.weight as u128).sum(); + total * w.config.cap.num as u128 / w.config.cap.den as u128 } #[test] -fn cap_holds_for_every_holder_and_estate() { - for (seed, w) in worlds(400) { - let r = w.resolve(); - let mut per: std::collections::BTreeMap<(u64, u64), u128> = Default::default(); - for rc in &r.receipts { - if let Outcome::Delegated { holder, .. } = rc.outcome { - *per.entry((holder, rc.estate)).or_default() += rc.weight as u128; - } - } - for ((_, estate), got) in per { - let total: u128 = w.people.iter().filter(|p| p.estate == estate).map(|p| p.weight as u128).sum(); - assert!(got <= total * w.config.cap.num as u128 / w.config.cap.den as u128, "seed {seed}"); +fn trim_is_order_independent_under_shuffles() { + let mut trimmed = 0; + for (seed, w) in crowded(400) { + let base = w.resolve(); + trimmed += base.receipts.iter().filter(|r| r.counted < r.weight).count(); + let mut rng = Rng(seed + 5000); + for _ in 0..5 { + let mut s = w.clone(); + rng.shuffle(&mut s.people); + rng.shuffle(&mut s.delegations); + rng.shuffle(&mut s.votes); + assert_eq!(s.resolve(), base, "seed {seed}"); } } + assert!(trimmed > 500, "generator must actually exercise the cap, got {trimmed}"); +} + +#[test] +fn delegated_weight_never_exceeds_the_cap_and_fills_it_when_over() { + for (seed, w) in worlds(400).chain(crowded(400)) { + let r = w.resolve(); + for ((_, estate), members) in groups(&w, &r) { + let sum: u128 = members.iter().map(|m| m.weight as u128).sum(); + let got: u128 = members.iter().map(|m| m.counted as u128).sum(); + let lim = limit(&w, estate); + assert!(got <= lim, "seed {seed}: {got} > {lim}"); + if sum > lim { + assert_eq!(got, lim, "seed {seed}: over-cap group must use exactly the cap"); + } else { + assert_eq!(got, sum, "seed {seed}: under-cap group is untouched"); + } + } + } +} + +#[test] +fn no_delegator_is_scaled_more_than_another() { + for (seed, w) in crowded(400) { + let r = w.resolve(); + for ((_, estate), members) in groups(&w, &r).into_iter().chain(trimmed_groups(&w, &r)) { + let sum: u128 = members.iter().map(|m| m.weight as u128).sum(); + let lim = limit(&w, estate); + if sum <= lim { + continue; + } + for m in &members { + // counted is the exact pro-rata share rounded down or up, never further. + let exact = m.weight as u128 * lim; + let (lo, hi) = (exact / sum, exact.div_ceil(sum)); + assert!((lo..=hi).contains(&(m.counted as u128)), "seed {seed}: {m:?}"); + } + // Equal weights are scaled equally to within one unit of rounding, + // and a unit of rounding goes to the lower originator id on a tie. + for a in &members { + for b in &members { + if a.weight == b.weight { + assert!(a.counted.abs_diff(b.counted) <= 1, "seed {seed}"); + if a.counted < b.counted { + assert!(a.originator > b.originator, "seed {seed}: tie broke against ascending id"); + } + } + } + } + } + } +} + +/// Delegators whose whole share rounded to zero are `Uncast(Trimmed)`, so they +/// are not in `groups`; rebuild their groups by following the chain they would +/// have taken (depth 1: their sole live delegation). +fn trimmed_groups(w: &World, r: &Resolution) -> std::collections::BTreeMap<(u64, u64), Vec> { + let mut g = groups(w, r); + for rc in &r.receipts { + if rc.outcome == Outcome::Uncast(UncastReason::Trimmed) { + let to = w.delegations.iter().find(|d| d.from == rc.originator && d.revoked.is_none()).unwrap().to; + g.entry((to, rc.estate)).or_default().push(rc.clone()); + } + } + g +} + +#[test] +fn receipts_carry_the_revocation_time_of_the_originators_own_revoked_delegation() { + let people = vec![person(1, 0, 1), person(2, 0, 1), person(3, 0, 1)]; + let mut a = del(1, 2); + a.revoked = Some(40); + let mut b = del(1, 3); + b.revoked = Some(90); + b.scope = only(&["budget"]); + let mut c = del(1, 3); + c.revoked = Some(500); + c.scope = only(&["war"]); // other topic: not reported + let r = world(loose(2), people, vec![a, b, c], vec![vote(3, "yes")]).resolve(); + let rc = |id| r.receipts.iter().find(|x| x.originator == id).unwrap(); + assert_eq!(rc(1).revoked_at, Some(90)); // latest covering revocation + assert_eq!(outcome(&r, 1), &Outcome::Uncast(UncastReason::NoVote)); // and it stayed inert + assert_eq!(rc(2).revoked_at, None); + // A live delegation alongside an older revoked one still resolves. + let people = vec![person(1, 0, 1), person(2, 0, 1)]; + let mut old = del(1, 2); + old.revoked = Some(3); + let r = world(loose(2), people, vec![old, del(1, 2)], vec![vote(2, "yes")]).resolve(); + assert!(matches!(outcome(&r, 1), Outcome::Delegated { holder: 2, .. })); + assert_eq!(r.receipts[0].revoked_at, Some(3)); } #[test] From 5cb5f35dccebfc9c170588a2eb652faa804ac308 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 5 Nov 2026 09:41:42 +0000 Subject: [PATCH 44/88] heimdall-delegation: README caveat that the cap does not bind under ballot copying Co-Authored-By: Claude Sonnet 5.5 --- packages/heimdall-delegation/README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/heimdall-delegation/README.md b/packages/heimdall-delegation/README.md index bf0eca38..bfe920f6 100644 --- a/packages/heimdall-delegation/README.md +++ b/packages/heimdall-delegation/README.md @@ -27,6 +27,13 @@ any CultMesh projection. ascending originator id. A receipt's `counted` is its scaled weight and `weight - counted` is uncast; a share that rounds to zero is `Uncast(Trimmed)`. Cast + uncast = total weight. + **Caveat:** the cap binds the ledger, not influence. A delegator under an + over-cap holder can copy that holder's ballot and vote directly; the + resolver sees a direct ballot and cannot see the copying. In the Bifrost sim + (`docs/research/2026-11-05-estate-rule-sim/`, commit `5d352c6c`, Bifrost PR + #4), with full copying the cap leaves effective top-holder share unchanged + (difference +0.001 or less at every alpha), while the formal share drops. The + copy rate there is an assumption, not a measurement. - Revocation (**placeholder**): `Delegation::revoked` is `Some(time)` when revoked. A revoked delegation is inert; the resolver reads no clock and the caller owns the ledger. Each receipt carries `revoked_at`, the latest From bb25bf04cb7037e40a688df8f286848077e1455a Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 5 Nov 2026 09:43:55 +0000 Subject: [PATCH 45/88] docs/start: verbatim drift check for the C# and TypeScript first-call blocks Both blocks now come from a snippet region in their test, and a second test fails when docs/start/README.md stops showing it, as the Rust and Python blocks already do. The TS block differs only in the package import specifier. Co-Authored-By: Claude Sonnet 5.5 --- docs/start/README.md | 61 ++++++++------ packages/cultcache-ts/test/first-call.test.ts | 59 +++++++++----- .../GameCult.Caching.Tests/FirstCallTests.cs | 80 ++++++++++++------- 3 files changed, 127 insertions(+), 73 deletions(-) diff --git a/docs/start/README.md b/docs/start/README.md index c178c7f0..99b66354 100644 --- a/docs/start/README.md +++ b/docs/start/README.md @@ -34,6 +34,7 @@ which is what `dotnet test tests/GameCult.Caching.Tests --filter FirstCallTests` cannot drift without the test failing. ```csharp +using System.Threading.Tasks; using GameCult.Caching; using GameCult.Caching.MessagePack; using MessagePack; @@ -47,20 +48,27 @@ public sealed class Item [Key(2)] public int Value; } -var store = new SingleFileMessagePackBackingStore("first-call.cc"); -var cache = new CultCache(); -cache.AddBackingStore(store); -var handle = await cache.AddAsync(new Item { Name = "Potion", Category = "Consumable", Value = 50 }); -store.PushAll(); // write the .cc file +public static class FirstCall +{ + public static async Task Run(string path) + { + var store = new SingleFileMessagePackBackingStore(path); + var cache = new CultCache(); + cache.AddBackingStore(store); + var handle = await cache.AddAsync(new Item { Name = "Potion", Category = "Consumable", Value = 50 }); + store.PushAll(); // write the .cc file -var reopened = new CultCache(); -reopened.AddBackingStore(new SingleFileMessagePackBackingStore("first-call.cc")); -await reopened.PullAllBackingStoresAsync(); // read it back -var potion = reopened.Get(handle.Key); // by key -var byName = reopened.GetByName("Potion"); // by [CultName] + var reopened = new CultCache(); + reopened.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + await reopened.PullAllBackingStoresAsync(); // read it back + var potion = reopened.Get(handle.Key); // by key + var byName = reopened.GetByName("Potion"); // by [CultName] + return byName ?? potion; + } +} ``` -The test's `Item` is nested and internal, so it carries `[MessagePackObject(AllowPrivate = true)]`; the snippet above is the public-class form and is not itself compiled. More: `src/GameCult.Caching/README.md`. +`FirstCall.Run` is the call; a second test in that file fails if the block above stops matching it. More: `src/GameCult.Caching/README.md`. ## TypeScript first call @@ -90,23 +98,26 @@ const itemDocument = defineDocumentType({ }); const registry = defineDocumentRegistry(itemDocument); -const cache = CultCache.builder() - .withRegistry(registry) - .withGenericStore(new SingleFileMessagePackBackingStore("first-call.cc")) - .build(); -await cache.pullAllBackingStores(); -await cache.put(itemDocument, "item:potion", { name: "Potion", category: "Consumable", value: 50 }); +async function firstCall(path: string) { + const cache = CultCache.builder() + .withRegistry(registry) + .withGenericStore(new SingleFileMessagePackBackingStore(path)) + .build(); + await cache.pullAllBackingStores(); + await cache.put(itemDocument, "item:potion", { name: "Potion", category: "Consumable", value: 50 }); -const reopened = CultCache.builder() - .withRegistry(registry) - .withGenericStore(new SingleFileMessagePackBackingStore("first-call.cc")) - .build(); -await reopened.pullAllBackingStores(); -const potion = reopened.get(itemDocument, "item:potion"); -const byName = reopened.getByName(itemDocument, "Potion"); + const reopened = CultCache.builder() + .withRegistry(registry) + .withGenericStore(new SingleFileMessagePackBackingStore(path)) + .build(); + await reopened.pullAllBackingStores(); + const potion = reopened.get(itemDocument, "item:potion"); + const byName = reopened.getByName(itemDocument, "Potion"); + return { potion, byName }; +} ``` -The test imports from `../src/index`; an application imports `@gamecult/cultcache-ts`. More: +The test imports from `../src/index` where the block reads `@gamecult/cultcache-ts`; a second test in that file fails if the block stops matching the rest. More: `packages/cultcache-ts/README.md`. ## Rust first call diff --git a/packages/cultcache-ts/test/first-call.test.ts b/packages/cultcache-ts/test/first-call.test.ts index aec9faa1..3c0e22e5 100644 --- a/packages/cultcache-ts/test/first-call.test.ts +++ b/packages/cultcache-ts/test/first-call.test.ts @@ -1,11 +1,15 @@ import assert from "node:assert/strict"; -import { mkdtemp, rm } from "node:fs/promises"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import test from "node:test"; +// The TypeScript first-call snippet that docs/start/README.md shows: define a typed document, store it +// in a .cc file, read it back through a second cache. Keep it small; it is the front door. The docs show +// the region between the `snippet` markers verbatim, except that the package import reads +// "@gamecult/cultcache-ts" where this file reads "../src/index"; the docs test below fails when they differ. +// snippet:begin import { z } from "zod"; - import { CultCache, SingleFileMessagePackBackingStore, @@ -13,8 +17,6 @@ import { defineDocumentType, } from "../src/index"; -// The TypeScript first-call snippet that docs/start/README.md links: define a typed document, -// store it in a .cc file, read it back through a second cache. Keep it small; it is the front door. const itemDocument = defineDocumentType({ type: "item", schemaId: "gamecult.first_call_item.v1", @@ -26,26 +28,43 @@ const itemDocument = defineDocumentType({ }); const registry = defineDocumentRegistry(itemDocument); +async function firstCall(path: string) { + const cache = CultCache.builder() + .withRegistry(registry) + .withGenericStore(new SingleFileMessagePackBackingStore(path)) + .build(); + await cache.pullAllBackingStores(); + await cache.put(itemDocument, "item:potion", { name: "Potion", category: "Consumable", value: 50 }); + + const reopened = CultCache.builder() + .withRegistry(registry) + .withGenericStore(new SingleFileMessagePackBackingStore(path)) + .build(); + await reopened.pullAllBackingStores(); + const potion = reopened.get(itemDocument, "item:potion"); + const byName = reopened.getByName(itemDocument, "Potion"); + return { potion, byName }; +} +// snippet:end + test("define, store and read back a typed document", async () => { const dir = await mkdtemp(join(tmpdir(), "cultcache-first-call-")); - const path = join(dir, "first-call.cc"); try { - const cache = CultCache.builder() - .withRegistry(registry) - .withGenericStore(new SingleFileMessagePackBackingStore(path)) - .build(); - await cache.pullAllBackingStores(); - await cache.put(itemDocument, "item:potion", { name: "Potion", category: "Consumable", value: 50 }); - - const reopened = CultCache.builder() - .withRegistry(registry) - .withGenericStore(new SingleFileMessagePackBackingStore(path)) - .build(); - await reopened.pullAllBackingStores(); - - assert.equal(reopened.get(itemDocument, "item:potion")?.value, 50); - assert.equal(reopened.getByName(itemDocument, "Potion")?.category, "Consumable"); + const { potion, byName } = await firstCall(join(dir, "first-call.cc")); + assert.equal(potion?.value, 50); + assert.equal(byName?.category, "Consumable"); } finally { await rm(dir, { recursive: true, force: true }); } }); + +test("docs show the tested snippet", async () => { + // Compiled to dist-test/test/first-call.test.js, so the package root is two levels up. + const source = await readFile(join(__dirname, "..", "..", "test", "first-call.test.ts"), "utf8"); + const snippet = source + .split("// snippet:begin\n")[1] + .split("// snippet:end\n")[0] + .replace('"../src/index"', '"@gamecult/cultcache-ts"'); + const docs = (await readFile(join(__dirname, "..", "..", "..", "..", "docs", "start", "README.md"), "utf8")).replace(/\r\n/g, "\n"); + assert.ok(docs.includes("```ts\n" + snippet + "```"), "docs/start/README.md no longer shows test/first-call.test.ts"); +}); diff --git a/tests/GameCult.Caching.Tests/FirstCallTests.cs b/tests/GameCult.Caching.Tests/FirstCallTests.cs index d14ad72c..8540d9d0 100644 --- a/tests/GameCult.Caching.Tests/FirstCallTests.cs +++ b/tests/GameCult.Caching.Tests/FirstCallTests.cs @@ -1,54 +1,78 @@ #nullable enable using System; using System.IO; +using NUnit.Framework; + +// snippet:begin using System.Threading.Tasks; using GameCult.Caching; using GameCult.Caching.MessagePack; using MessagePack; -using NUnit.Framework; + +[CultDocument("gamecult.first_call_item", "gamecult.first_call_item.v1")] +[MessagePackObject] +public sealed class Item +{ + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] [CultIndex] public string Category = string.Empty; + [Key(2)] public int Value; +} + +public static class FirstCall +{ + public static async Task Run(string path) + { + var store = new SingleFileMessagePackBackingStore(path); + var cache = new CultCache(); + cache.AddBackingStore(store); + var handle = await cache.AddAsync(new Item { Name = "Potion", Category = "Consumable", Value = 50 }); + store.PushAll(); // write the .cc file + + var reopened = new CultCache(); + reopened.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + await reopened.PullAllBackingStoresAsync(); // read it back + var potion = reopened.Get(handle.Key); // by key + var byName = reopened.GetByName("Potion"); // by [CultName] + return byName ?? potion; + } +} +// snippet:end namespace GameCult.Caching.Tests { - // The C# first-call snippet that docs/start/README.md links: define a typed document, store it in a - // .cc file, read it back through a second cache. Keep it small; it is the front door. + // The C# first-call snippet that docs/start/README.md shows verbatim: define a typed document, store it + // in a .cc file, read it back through a second cache. DocsShowTheTestedSnippet fails when they differ. public class FirstCallTests { - [CultDocument("gamecult.first_call_item", "gamecult.first_call_item.v1")] - [MessagePackObject(AllowPrivate = true)] - internal sealed class Item - { - [Key(0)] [CultName] public string Name = string.Empty; - [Key(1)] [CultIndex] public string Category = string.Empty; - [Key(2)] public int Value; - } - [Test] public async Task DefineStoreAndReadBackATypedDocument() { var path = Path.Combine(Path.GetTempPath(), $"first-call-{Guid.NewGuid():N}.cc"); try { - // Write: a cache with one backing store, one typed document. - var store = new SingleFileMessagePackBackingStore(path); - var cache = new CultCache(); - cache.AddBackingStore(store); - var handle = await cache.AddAsync(new Item { Name = "Potion", Category = "Consumable", Value = 50 }); - store.PushAll(); - - // Read: a fresh cache over the same file. - var reopened = new CultCache(); - reopened.AddBackingStore(new SingleFileMessagePackBackingStore(path)); - await reopened.PullAllBackingStoresAsync(); - - var byKey = reopened.Get(handle.Key); - Assert.That(byKey, Is.Not.Null); - Assert.That(byKey!.Value, Is.EqualTo(50)); - Assert.That(reopened.GetByName("Potion")!.Category, Is.EqualTo("Consumable")); + var potion = await FirstCall.Run(path); + Assert.That(potion, Is.Not.Null); + Assert.That(potion!.Value, Is.EqualTo(50)); + Assert.That(potion.Category, Is.EqualTo("Consumable")); } finally { File.Delete(path); } } + + [Test] + public void DocsShowTheTestedSnippet() + { + var source = File.ReadAllText(SourcePath()); + const string begin = "// snippet:begin\n"; + var start = source.IndexOf(begin, StringComparison.Ordinal) + begin.Length; + var snippet = source[start..source.IndexOf("// snippet:end\n", StringComparison.Ordinal)]; + var docs = File.ReadAllText(Path.Combine(Path.GetDirectoryName(SourcePath())!, "..", "..", "docs", "start", "README.md")); + Assert.That(docs.Replace("\r\n", "\n"), Does.Contain("```csharp\n" + snippet + "```"), + "docs/start/README.md no longer shows tests/GameCult.Caching.Tests/FirstCallTests.cs verbatim"); + } + + static string SourcePath([System.Runtime.CompilerServices.CallerFilePath] string path = "") => path; } } From ae33ba484246cebc7f851d619f95549a74a5fc10 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 5 Nov 2026 11:43:03 +0000 Subject: [PATCH 46/88] heimdall-stake: stake function and splitting-gain report (Rust, zero deps) weight = 1 + log_b(amount/unit) in integer cents and micro-weights with an integer-only log, plus floor, per-identity cap and minimum amount per identity; splitting_gain/curve report what splitting a budget over k identities does. 11 property tests. Co-Authored-By: Claude Sonnet 5.5 --- .gitignore | 2 + packages/heimdall-stake/Cargo.lock | 7 + packages/heimdall-stake/Cargo.toml | 10 + packages/heimdall-stake/README.md | 89 ++++++++ .../heimdall-stake/examples/first_call.rs | 11 + packages/heimdall-stake/src/lib.rs | 185 ++++++++++++++++ packages/heimdall-stake/tests/properties.rs | 198 ++++++++++++++++++ 7 files changed, 502 insertions(+) create mode 100644 packages/heimdall-stake/Cargo.lock create mode 100644 packages/heimdall-stake/Cargo.toml create mode 100644 packages/heimdall-stake/README.md create mode 100644 packages/heimdall-stake/examples/first_call.rs create mode 100644 packages/heimdall-stake/src/lib.rs create mode 100644 packages/heimdall-stake/tests/properties.rs diff --git a/.gitignore b/.gitignore index b033eb4c..30d5f28e 100644 --- a/.gitignore +++ b/.gitignore @@ -324,6 +324,8 @@ node_modules/ !packages/cultmesh-kotlin/** !packages/heimdall-delegation/ !packages/heimdall-delegation/** +!packages/heimdall-stake/ +!packages/heimdall-stake/** packages/**/node_modules/ packages/**/dist/ packages/**/dist-test/ diff --git a/packages/heimdall-stake/Cargo.lock b/packages/heimdall-stake/Cargo.lock new file mode 100644 index 00000000..e51417db --- /dev/null +++ b/packages/heimdall-stake/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "heimdall-stake" +version = "0.1.0" diff --git a/packages/heimdall-stake/Cargo.toml b/packages/heimdall-stake/Cargo.toml new file mode 100644 index 00000000..19fe9296 --- /dev/null +++ b/packages/heimdall-stake/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "heimdall-stake" +version = "0.1.0" +edition = "2024" +license = "MIT" +publish = false + +[lib] +name = "heimdall_stake" +path = "src/lib.rs" diff --git a/packages/heimdall-stake/README.md b/packages/heimdall-stake/README.md new file mode 100644 index 00000000..03b8dc32 --- /dev/null +++ b/packages/heimdall-stake/README.md @@ -0,0 +1,89 @@ +# heimdall-stake + +Heimdall's stake function: how much weight an amount of money buys one identity. Pure, +deterministic, no I/O, no dependencies; integer cents in, integer micro-weights out +(`MICRO` = 1,000,000 = a weight of 1.0), and an integer-only logarithm, so the same inputs give +the same bits on every platform. Entry points: `weight` and `splitting_gain` in `src/lib.rs`. +Live: the formula, the three Heimdall parameters, the splitting-gain report and their property +tests (`tests/properties.rs`). Not here: the estate rule that normalizes weights, the sybil +adversary, storage, and any projection. Every parameter is a placeholder until the estates rule. + +## First call + +``` +cargo run --example first_call # prints the Meta example and a splitting table +cargo test +``` + +```rust +use heimdall_stake::*; +let p = Params::meta_example(); // unit 1 EUR, log10 +assert_eq!(weight(&p, 100_00).unwrap(), 3 * MICRO); // 100 EUR +assert_eq!(weight(&p, 10_000_00).unwrap(), 5 * MICRO); // 10,000 EUR +let costly = Params { min_amount_per_identity_cents: 50_00, ..p }; +let r = splitting_gain(&costly, 100_00, 3).unwrap(); // 100 EUR as three identities of 33.33 +assert_eq!((r.one_identity, r.split_total), (3 * MICRO, 0)); +``` + +The output feeds `heimdall-delegation`'s integer `weight`; the caller picks the scale. + +## Formula + +Per Meta, `weight = 1 + log_b(amount / unit)`, with the base `b` and the `unit` set by the charter. +Meta's example: unit 1 EUR, `b` = 10: 100 EUR weighs 3, 10,000 EUR weighs 5 (both tested exactly). +Amounts are integer cents; negative and zero amounts weigh 0. The result is clamped at 0 (an +amount under `unit / b` would otherwise go negative) and rounded to the nearest micro-weight. + +Heimdall's parameters on top: + +- `floor_cents`: an amount **at or below** the floor weighs 0. (The brief said both "below" and + "at/below"; at-or-below is implemented and tested.) +- `per_identity_cap`: the most weight one identity can hold, in micro-weights; `None` is uncapped. +- `min_amount_per_identity_cents`: an identity holding **less** than this is ineligible and weighs 0. + +`floor` and `min_amount_per_identity` both zero a weight and compose; they differ in what they are +for (the floor trims dust from the formula, the minimum prices an identity), and the estates may +want one parameter instead of two. + +Properties (tested): monotone in the amount for any parameters; concave above the unit when no +floor or minimum intrudes (the floor and minimum are steps, so the whole function is not concave); +zero at and below the floor; the cap is never exceeded and binds; deterministic; total over every +`i64` including `i64::MIN`; the integer log agrees with `f64` to 2 micro-weights; bad parameters +(`base < 2`, `unit == 0`) are `Err`, not panics. + +## Splitting gain + +`splitting_gain(params, budget_cents, k)` puts one budget in one identity and, separately, evenly +over `k` identities, and reports both totals, the gain, how many split identities are eligible and +the smallest share. `splitting_curve` and `most_profitable_split` sweep `k`. + +The log formula's `+1` is a splitting incentive: with no minimum, 100 EUR as ten identities of +10 EUR weighs 20 against 3 as one. The minimum is the only lever that prices it, and it prices it +as a cliff, not a slope: with a 50 EUR minimum, 100 EUR split in two still gains (5.40 against 3.00 +in the example), and split in three weighs 0. So the minimum has to sit near the size of a typical +budget to deter splitting, and the report is how to see where it does. + +## Where this sits in the estate rule + +This crate does not normalize. The estate rule normalizes weights above it; the weights here are +raw. A sybil rate (how many identities an adversary splits a budget into) is an adversary the +simulation harness plays, not a parameter of this function; `splitting_gain` is its closed-form +probe for the even-split case only. + +## Prior art + +Sources below are from background knowledge and were not re-fetched for this README; verify before +citing. + +- **Quadratic voting** (Lalley and Weyl, "Quadratic Voting: How Mechanism Design Can Radicalize + Democracy", 2018; Posner and Weyl, *Radical Markets*, 2018): the cost of v votes is v², so the + marginal price rises linearly and vote weight grows with the square root of spend. Lalley-Weyl + show it is efficient in large populations under assumptions that include no collusion and no + identity splitting, and that identity splitting is exactly what breaks it. Our weight grows + slower still (log), so the money buys less influence at the top, and the same sybil caveat applies + with the same remedy: a cost per identity. +- **Logarithmic weight** has the same shape as diminishing-returns utility (Bernoulli) and as the + concave caps in some stake-weighted systems; here its role is only concavity, with the charter + choosing how fast through the base. +- **Sybil attack** (Douceur, 2002): without a cost per identity, an open system cannot tell one + participant from many. The per-identity minimum is that cost, in money. diff --git a/packages/heimdall-stake/examples/first_call.rs b/packages/heimdall-stake/examples/first_call.rs new file mode 100644 index 00000000..80275375 --- /dev/null +++ b/packages/heimdall-stake/examples/first_call.rs @@ -0,0 +1,11 @@ +use heimdall_stake::*; +fn main() { + let p = Params::meta_example(); + println!("100 EUR -> {}", weight(&p, 100_00).unwrap() as f64 / MICRO as f64); + println!("10,000 EUR -> {}", weight(&p, 10_000_00).unwrap() as f64 / MICRO as f64); + let costly = Params { min_amount_per_identity_cents: 50_00, ..p }; + for k in [1, 2, 3, 10] { + let r = splitting_gain(&costly, 100_00, k).unwrap(); + println!("100 EUR in {k:>2} identities: {:.2} vs {:.2} (gain {:+.2})", r.split_total as f64 / 1e6, r.one_identity as f64 / 1e6, r.gain as f64 / 1e6); + } +} diff --git a/packages/heimdall-stake/src/lib.rs b/packages/heimdall-stake/src/lib.rs new file mode 100644 index 00000000..4e168f34 --- /dev/null +++ b/packages/heimdall-stake/src/lib.rs @@ -0,0 +1,185 @@ +//! Heimdall's stake function. See README.md for the formula and its placeholders. +//! +//! `weight = 1 + log_b(amount / unit)`, with a floor, a per-identity cap and a minimum amount per +//! identity. Pure and integer-only: amounts are integer cents, weights are integer micro-weights +//! (`MICRO` = 1.0), and the logarithm is computed in fixed point with no floating point, so the +//! same inputs give the same bits on every platform. + +use std::fmt; + +/// Weight in millionths: `MICRO` is a weight of 1.0. Feeds `heimdall-delegation`'s integer weights +/// after the caller picks a scale. +pub type Weight = u64; +pub const MICRO: Weight = 1_000_000; + +/// All of these are placeholders until the estates rule; the charter sets `base` and `unit_cents`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Params { + /// Logarithm base `b`, an integer of at least 2. + pub base: u32, + /// The amount worth weight 1.0, in cents; at least 1. + pub unit_cents: u64, + /// Amounts at or below this weigh 0. + pub floor_cents: u64, + /// Highest weight one identity can hold, in micro-weights; `None` is uncapped. + pub per_identity_cap: Option, + /// An identity holding less than this is not eligible and weighs 0 (the splitting cost). + pub min_amount_per_identity_cents: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ParamError { + BaseBelowTwo, + UnitZero, +} + +impl fmt::Display for ParamError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ParamError::BaseBelowTwo => write!(f, "log base must be at least 2"), + ParamError::UnitZero => write!(f, "unit must be at least 1 cent"), + } + } +} + +impl std::error::Error for ParamError {} + +impl Params { + /// Meta's example: unit 1 EUR, log10, no floor, no cap, no minimum. + pub fn meta_example() -> Params { + Params { base: 10, unit_cents: 100, floor_cents: 0, per_identity_cap: None, min_amount_per_identity_cents: 0 } + } + + pub fn validate(&self) -> Result<(), ParamError> { + if self.base < 2 { + return Err(ParamError::BaseBelowTwo); + } + if self.unit_cents == 0 { + return Err(ParamError::UnitZero); + } + Ok(()) + } +} + +const FRAC: u32 = 40; + +/// log2(n) for n >= 1 in Q40 fixed point, by the square-and-compare method: no floats, so it is +/// the same everywhere. Error is below 2^-38. +fn log2_q40(n: u64) -> i128 { + let k = 63 - n.leading_zeros(); + let mut m: u128 = if k >= 60 { (n >> (k - 60)) as u128 } else { (n as u128) << (60 - k) }; // [1,2) in Q60 + let mut frac: u128 = 0; + for bit in (0..FRAC).rev() { + m = (m * m) >> 60; + if m >= 1u128 << 61 { + m >>= 1; + frac |= 1u128 << bit; + } + } + ((k as i128) << FRAC) | frac as i128 +} + +/// Weight of one identity holding `amount_cents`. Zero for zero and negative amounts, for amounts at +/// or below the floor, and for amounts under the per-identity minimum. Never panics on any input. +pub fn weight(params: &Params, amount_cents: i64) -> Result { + params.validate()?; + if amount_cents <= 0 { + return Ok(0); + } + let amount = amount_cents as u64; + if amount <= params.floor_cents || amount < params.min_amount_per_identity_cents { + return Ok(0); + } + // micro = 1e6 * (1 + (log2 a - log2 u) / log2 b), rounded to nearest. + let num = (log2_q40(amount) - log2_q40(params.unit_cents)) * MICRO as i128; + let den = log2_q40(params.base as u64); + let half = den / 2; + let rounded = if num >= 0 { (num + half) / den } else { (num - half) / den }; + let raw = (MICRO as i128 + rounded).max(0) as u128; + let capped = match params.per_identity_cap { + Some(cap) => raw.min(cap as u128), + None => raw, + }; + Ok(capped.min(Weight::MAX as u128) as Weight) +} + +/// Total weight of several identities, each weighed on its own (the cap and minimum are per identity). +pub fn total_weight(params: &Params, amounts_cents: &[i64]) -> Result { + params.validate()?; + let mut total: Weight = 0; + for &a in amounts_cents { + total = total.saturating_add(weight(params, a)?); + } + Ok(total) +} + +/// One budget held as a single identity against the same budget split evenly over `k` identities +/// (the first `budget % k` identities hold one extra cent). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SplitReport { + pub budget_cents: i64, + pub k: u64, + pub one_identity: Weight, + pub split_total: Weight, + /// `split_total - one_identity`: positive means splitting pays. + pub gain: i128, + /// Identities of the split that clear the floor and the minimum. + pub eligible_identities: u64, + /// Smallest amount any identity of the split holds. + pub smallest_share_cents: i64, +} + +/// What splitting `budget_cents` over `k` identities does to total weight. `k` of 0 is treated as 1. +pub fn splitting_gain(params: &Params, budget_cents: i64, k: u64) -> Result { + params.validate()?; + let k = k.max(1); + let one_identity = weight(params, budget_cents)?; + let (base_share, extra, smallest) = if budget_cents <= 0 { + (0i64, 0u64, budget_cents) + } else { + let b = budget_cents as u64; + ((b / k) as i64, b % k, (b / k) as i64) + }; + // Identities are of two sizes, so weigh the two sizes instead of looping over k. + let mut split_total: u128 = 0; + let mut eligible = 0u64; + let big = k.min(extra); + let small = k - big; + for (count, amount) in [(big, base_share.saturating_add(1)), (small, base_share)] { + if count == 0 { + continue; + } + let w = weight(params, amount)?; + if w > 0 { + eligible += count; + } + split_total += w as u128 * count as u128; + } + let split_total = split_total.min(Weight::MAX as u128) as Weight; + Ok(SplitReport { + budget_cents, + k, + one_identity, + split_total, + gain: split_total as i128 - one_identity as i128, + eligible_identities: eligible, + smallest_share_cents: smallest, + }) +} + +/// `splitting_gain` for k = 1..=max_k. +pub fn splitting_curve(params: &Params, budget_cents: i64, max_k: u64) -> Result, ParamError> { + (1..=max_k.max(1)).map(|k| splitting_gain(params, budget_cents, k)).collect() +} + +/// The k in 1..=max_k with the largest split total (smallest k on ties), and its report. +pub fn most_profitable_split(params: &Params, budget_cents: i64, max_k: u64) -> Result { + let mut best = splitting_gain(params, budget_cents, 1)?; + for k in 2..=max_k { + let r = splitting_gain(params, budget_cents, k)?; + if r.split_total > best.split_total { + best = r; + } + } + Ok(best) +} diff --git a/packages/heimdall-stake/tests/properties.rs b/packages/heimdall-stake/tests/properties.rs new file mode 100644 index 00000000..7e450fbd --- /dev/null +++ b/packages/heimdall-stake/tests/properties.rs @@ -0,0 +1,198 @@ +use heimdall_stake::*; + +/// xorshift64*: seeded, dependency-free, reproducible. +struct Rng(u64); +impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 >> 12; + self.0 ^= self.0 << 25; + self.0 ^= self.0 >> 27; + self.0.wrapping_mul(0x2545F4914F6CDD1D) + } + fn below(&mut self, n: u64) -> u64 { + self.next() % n + } +} + +fn random_params(r: &mut Rng) -> Params { + Params { + base: 2 + r.below(15) as u32, + unit_cents: 1 + r.below(100_000), + floor_cents: if r.below(2) == 0 { 0 } else { r.below(50_000) }, + per_identity_cap: if r.below(2) == 0 { None } else { Some(MICRO + r.below(8 * MICRO)) }, + min_amount_per_identity_cents: if r.below(2) == 0 { 0 } else { r.below(20_000) }, + } +} + +/// Log-uniform amounts in cents, so every magnitude is exercised. +fn random_amount(r: &mut Rng) -> i64 { + let bits = 1 + r.below(40); + (r.next() >> (64 - bits)) as i64 +} + +#[test] +fn meta_example_unit_one_euro_log10() { + let p = Params::meta_example(); + assert_eq!(weight(&p, 100_00).unwrap(), 3 * MICRO); // 100 EUR + assert_eq!(weight(&p, 10_000_00).unwrap(), 5 * MICRO); // 10,000 EUR + assert_eq!(weight(&p, 100).unwrap(), MICRO); // 1 EUR = the unit = weight 1 + assert_eq!(weight(&p, 10_00).unwrap(), 2 * MICRO); // 10 EUR +} + +#[test] +fn base_two_and_other_units_hit_exact_powers() { + let p = Params { base: 2, unit_cents: 250, ..Params::meta_example() }; + assert_eq!(weight(&p, 250 * 8).unwrap(), 4 * MICRO); // 1 + log2(8) + let q = Params { base: 7, unit_cents: 1, ..Params::meta_example() }; + assert_eq!(weight(&q, 7u64.pow(9) as i64).unwrap(), 10 * MICRO); +} + +#[test] +fn integer_log_agrees_with_floating_point_to_two_micro() { + let mut r = Rng(0xC0FFEE); + for _ in 0..20_000 { + let p = Params { base: 2 + r.below(15) as u32, unit_cents: 1 + r.below(1_000_000), floor_cents: 0, per_identity_cap: None, min_amount_per_identity_cents: 0 }; + let a = 1 + random_amount(&mut r).abs(); + let expect = 1.0 + (a as f64 / p.unit_cents as f64).ln() / (p.base as f64).ln(); + let expect = (expect.max(0.0) * MICRO as f64).round(); + let got = weight(&p, a).unwrap() as f64; + assert!((got - expect).abs() <= 2.0, "base {} unit {} amount {a}: {got} vs {expect}", p.base, p.unit_cents); + } +} + +#[test] +fn monotone_in_amount_for_any_parameters() { + let mut r = Rng(1); + for _ in 0..300 { + let p = random_params(&mut r); + let mut amounts: Vec = (0..60).map(|_| random_amount(&mut r)).collect(); + amounts.sort(); + let ws: Vec = amounts.iter().map(|&a| weight(&p, a).unwrap()).collect(); + assert!(ws.windows(2).all(|w| w[0] <= w[1]), "{p:?} {amounts:?} {ws:?}"); + } +} + +#[test] +fn concave_above_the_unit_when_no_floor_or_minimum_intrudes() { + // Second differences of 1 + log_b(a/u) are non-positive; rounding to a micro-weight allows 2. + let mut r = Rng(2); + for _ in 0..300 { + let mut p = random_params(&mut r); + p.floor_cents = 0; + p.min_amount_per_identity_cents = 0; + let a = p.unit_cents as i64 + random_amount(&mut r) % 1_000_000_000; + let d = 1 + (r.below(1_000_000) as i64); + let (w0, w1, w2) = (weight(&p, a).unwrap() as i128, weight(&p, a + d).unwrap() as i128, weight(&p, a + 2 * d).unwrap() as i128); + assert!((w2 - w1) <= (w1 - w0) + 2, "{p:?} a {a} d {d}: {w0} {w1} {w2}"); + } +} + +#[test] +fn zero_at_and_below_the_floor_and_positive_just_above() { + let mut r = Rng(3); + for _ in 0..300 { + let mut p = random_params(&mut r); + p.min_amount_per_identity_cents = 0; + p.per_identity_cap = None; + p.floor_cents = 1 + r.below(1_000_000); + for a in [p.floor_cents as i64, p.floor_cents as i64 - 1, 1] { + assert_eq!(weight(&p, a).unwrap(), 0, "{p:?} amount {a}"); + } + // Far enough above both floor and unit that the formula is positive. + let above = (p.floor_cents.max(p.unit_cents) * 2) as i64; + assert!(weight(&p, above).unwrap() > 0, "{p:?} amount {above}"); + } +} + +#[test] +fn cap_is_never_exceeded_and_binds_when_the_formula_passes_it() { + let mut r = Rng(4); + for _ in 0..300 { + let mut p = random_params(&mut r); + let cap = MICRO + r.below(8 * MICRO); + p.per_identity_cap = Some(cap); + for _ in 0..40 { + assert!(weight(&p, random_amount(&mut r)).unwrap() <= cap); + } + let uncapped = Params { per_identity_cap: None, ..p }; + let big = i64::MAX; + if weight(&uncapped, big).unwrap() > cap && p.min_amount_per_identity_cents <= big as u64 { + assert_eq!(weight(&p, big).unwrap(), cap); + } + } +} + +#[test] +fn deterministic_and_total_over_every_input() { + let mut r = Rng(5); + for _ in 0..200 { + let p = random_params(&mut r); + for a in [i64::MIN, -1, 0, 1, i64::MAX, random_amount(&mut r)] { + assert_eq!(weight(&p, a), weight(&p, a)); + } + let budget = random_amount(&mut r); + assert_eq!(splitting_gain(&p, budget, 7), splitting_gain(&p, budget, 7)); + assert_eq!(splitting_curve(&p, budget, 5), splitting_curve(&p, budget, 5)); + } + let p = Params::meta_example(); + for a in [i64::MIN, -5, -1, 0] { + assert_eq!(weight(&p, a).unwrap(), 0); + } + assert!(weight(&p, i64::MAX).unwrap() > 0); +} + +#[test] +fn bad_parameters_are_errors_not_panics() { + let mut p = Params::meta_example(); + p.base = 1; + assert_eq!(weight(&p, 100), Err(ParamError::BaseBelowTwo)); + p.base = 0; + assert_eq!(total_weight(&p, &[1]), Err(ParamError::BaseBelowTwo)); + p = Params { unit_cents: 0, ..Params::meta_example() }; + assert_eq!(splitting_gain(&p, 100, 2), Err(ParamError::UnitZero)); +} + +#[test] +fn splitting_report_matches_summing_the_shares_by_hand() { + let mut r = Rng(6); + for _ in 0..200 { + let p = random_params(&mut r); + let budget = 1 + random_amount(&mut r); + let k = 1 + r.below(40); + let rep = splitting_gain(&p, budget, k).unwrap(); + let shares: Vec = (0..k).map(|i| (budget as u64 / k + u64::from(i < budget as u64 % k)) as i64).collect(); + assert_eq!(shares.iter().sum::(), budget); + assert_eq!(rep.split_total, total_weight(&p, &shares).unwrap()); + assert_eq!(rep.one_identity, weight(&p, budget).unwrap()); + assert_eq!(rep.gain, rep.split_total as i128 - rep.one_identity as i128); + assert_eq!(rep.smallest_share_cents, *shares.iter().min().unwrap()); + assert_eq!(rep.eligible_identities, shares.iter().filter(|&&s| weight(&p, s).unwrap() > 0).count() as u64); + } +} + +#[test] +fn without_a_minimum_splitting_pays_and_with_one_it_stops_paying() { + // 100 EUR, unit 1 EUR, log10: one identity weighs 3. Ten identities of 10 EUR weigh 2 each = 20. + let free = Params::meta_example(); + let rep = splitting_gain(&free, 100_00, 10).unwrap(); + assert_eq!((rep.one_identity, rep.split_total), (3 * MICRO, 20 * MICRO)); + assert!(rep.gain > 0); + assert_eq!(most_profitable_split(&free, 100_00, 50).unwrap().k, rep_best_k(&free, 100_00, 50)); + + // Each identity must hold 50 EUR: two shares of 50 EUR are eligible, three of 33.33 are not. + let costly = Params { min_amount_per_identity_cents: 50_00, ..free }; + assert_eq!(splitting_gain(&costly, 100_00, 2).unwrap().eligible_identities, 2); + let three = splitting_gain(&costly, 100_00, 3).unwrap(); + assert_eq!((three.split_total, three.eligible_identities), (0, 0)); + assert_eq!(three.gain, -(3 * MICRO as i128)); + // Past the minimum the curve is dead flat at zero, so the best split is never past budget/min. + let best = most_profitable_split(&costly, 100_00, 200).unwrap(); + assert!(best.k <= 2 && best.split_total > 0); + for rep in splitting_curve(&costly, 100_00, 200).unwrap().iter().filter(|r| r.k > 2) { + assert_eq!(rep.split_total, 0); + } +} + +fn rep_best_k(p: &Params, budget: i64, max_k: u64) -> u64 { + splitting_curve(p, budget, max_k).unwrap().iter().max_by_key(|r| (r.split_total, std::cmp::Reverse(r.k))).unwrap().k +} From 84f932ec506d859469ed2037f69d8ef3daf80f23 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 5 Nov 2026 21:27:41 +0000 Subject: [PATCH 47/88] heimdall-stake: sybil note and golden vectors README: the constant 1 in weight = 1 + log_b(amount/unit) is paid once per identity, so splitting always pays until the log goes negative and no base fixes it; the defence lives outside the function. Golden vectors (single and split k=1,2,10,100, three bases) computed with 60-digit decimal ln, pinned by tests/golden.rs. Co-Authored-By: Claude Sonnet 5.5 --- packages/heimdall-stake/README.md | 11 +++++++ packages/heimdall-stake/tests/golden.rs | 30 +++++++++++++++++++ .../tests/vectors/stake_golden.csv | 29 ++++++++++++++++++ 3 files changed, 70 insertions(+) create mode 100644 packages/heimdall-stake/tests/golden.rs create mode 100644 packages/heimdall-stake/tests/vectors/stake_golden.csv diff --git a/packages/heimdall-stake/README.md b/packages/heimdall-stake/README.md index 03b8dc32..5c8ae095 100644 --- a/packages/heimdall-stake/README.md +++ b/packages/heimdall-stake/README.md @@ -63,6 +63,17 @@ as a cliff, not a slope: with a 50 EUR minimum, 100 EUR split in two still gains in the example), and split in three weighs 0. So the minimum has to sit near the size of a typical budget to deter splitting, and the report is how to see where it does. +## Why the sybil defence cannot live here + +`weight = 1 + log_b(amount / unit)` pays the constant 1 once per identity. Splitting a budget into +k identities pays that 1 k times and costs only the logarithm's shrinkage, so splitting wins until +the log term goes negative: with unit 1 EUR and b = 10, 100 EUR weighs 3 as one identity, 5.40 as +two, 20 as ten and 100 as one hundred (`tests/vectors/stake_golden.csv`, pinned by +`tests/golden.rs`). No choice of `b` fixes this: `b` scales the log term and never the constant. +A cap or a per-identity minimum only moves the cliff (see Splitting gain). So the defence belongs +outside the function: non-transferable or vesting stake, a per-person cap on total stake, and a +cost to create an identity. This crate states the incentive; it cannot remove it. + ## Where this sits in the estate rule This crate does not normalize. The estate rule normalizes weights above it; the weights here are diff --git a/packages/heimdall-stake/tests/golden.rs b/packages/heimdall-stake/tests/golden.rs new file mode 100644 index 00000000..d742aa5a --- /dev/null +++ b/packages/heimdall-stake/tests/golden.rs @@ -0,0 +1,30 @@ +use heimdall_stake::*; + +const VECTORS: &str = include_str!("vectors/stake_golden.csv"); + +/// Pins `weight` and `total_weight` to values computed outside this crate (see the fixture header). +/// Split rows are the sybil argument in numbers: the same budget in k identities weighs more. +#[test] +fn golden_vectors() { + let mut rows = 0; + let mut split_rows = 0; + for line in VECTORS.lines().filter(|l| !l.starts_with('#') && !l.trim().is_empty()) { + let f: Vec<&str> = line.split(',').collect(); + let (base, unit, amount, k, expected): (u32, u64, i64, usize, u64) = + (f[1].parse().unwrap(), f[2].parse().unwrap(), f[3].parse().unwrap(), f[4].parse().unwrap(), f[5].parse().unwrap()); + let p = Params { base, unit_cents: unit, floor_cents: 0, per_identity_cap: None, min_amount_per_identity_cents: 0 }; + let got = match f[0] { + "single" => weight(&p, amount).unwrap(), + "split" => { + split_rows += 1; + let (q, r) = (amount / k as i64, (amount % k as i64) as usize); + let shares: Vec = (0..k).map(|i| q + i64::from(i < r)).collect(); + total_weight(&p, &shares).unwrap() + } + other => panic!("unknown vector kind {other}"), + }; + assert_eq!(got, expected, "vector `{line}`"); + rows += 1; + } + assert!(rows >= 20 && split_rows >= 12, "fixture shrank: {rows} rows, {split_rows} split"); +} diff --git a/packages/heimdall-stake/tests/vectors/stake_golden.csv b/packages/heimdall-stake/tests/vectors/stake_golden.csv new file mode 100644 index 00000000..26f8978a --- /dev/null +++ b/packages/heimdall-stake/tests/vectors/stake_golden.csv @@ -0,0 +1,29 @@ +# kind,base,unit_cents,amount_cents,k,expected_micro +# single: weight of one identity holding amount. split: budget (amount) split evenly over k identities (first amount%k hold one extra cent), summed. +# expected values computed independently with 60-digit decimal ln, rounded half up. No floor, cap or minimum. +single,10,100,100,1,1000000 +single,10,100,1000,1,2000000 +single,10,100,10000,1,3000000 +single,10,100,1000000,1,5000000 +single,10,100,50,1,698970 +single,10,100,10,1,0 +single,10,100,1,1,0 +single,2,100,400,1,3000000 +single,2,100,100,1,1000000 +single,2,1,1,1,1000000 +single,3,500,123456,1,6014537 +single,7,1,987654321,1,11643268 +single,10,1,1,1,1000000 +single,16,250,99999,1,3160960 +split,10,100,10000,1,3000000 +split,10,100,10000,2,5397940 +split,10,100,10000,10,20000000 +split,10,100,10000,100,100000000 +split,2,100,10000,1,7643856 +split,2,100,10000,2,13287712 +split,2,100,10000,10,43219280 +split,2,100,10000,100,100000000 +split,10,100,100000,1,4000000 +split,10,100,100000,2,7397940 +split,10,100,100000,10,30000000 +split,10,100,100000,100,200000000 From 8777ce4766cd59b628c30179f52026f6147fb1bd Mon Sep 17 00:00:00 2001 From: Libby Date: Fri, 6 Nov 2026 12:20:06 +0000 Subject: [PATCH 48/88] docs/start: two-client .io game recipe over CultCache + CultMesh (TS) Authority holds one arena document in a CultCache and serves it over CultMesh; two cultmesh-browser clients join, move and leave, and see each other. Tested with real WebSockets. Browser run, persistence, remote trust and CI wiring are marked not yet on the recipe page. Co-Authored-By: Claude Sonnet 5.5 --- docs/start/README.md | 2 +- docs/start/io-game.md | 196 +++++++++++++++++++++++++++ docs/start/io-game/arena.test.ts | 73 ++++++++++ docs/start/io-game/browser.ts | 17 +++ docs/start/io-game/client.ts | 28 ++++ docs/start/io-game/index.html | 6 + docs/start/io-game/package-lock.json | 64 +++++++++ docs/start/io-game/package.json | 9 ++ docs/start/io-game/schema.ts | 16 +++ docs/start/io-game/serve.ts | 17 +++ docs/start/io-game/server.ts | 100 ++++++++++++++ 11 files changed, 527 insertions(+), 1 deletion(-) create mode 100644 docs/start/io-game.md create mode 100644 docs/start/io-game/arena.test.ts create mode 100644 docs/start/io-game/browser.ts create mode 100644 docs/start/io-game/client.ts create mode 100644 docs/start/io-game/index.html create mode 100644 docs/start/io-game/package-lock.json create mode 100644 docs/start/io-game/package.json create mode 100644 docs/start/io-game/schema.ts create mode 100644 docs/start/io-game/serve.ts create mode 100644 docs/start/io-game/server.ts diff --git a/docs/start/README.md b/docs/start/README.md index 99b66354..3c7d9b7e 100644 --- a/docs/start/README.md +++ b/docs/start/README.md @@ -17,7 +17,7 @@ this page. | | CultCache | CultMath | Geometry | CultNet | CultMesh | | --- | --- | --- | --- | --- | --- | | **C#** | [first call, tested](#c-first-call) | [package](../../packages/cultmath/README.md); snippet not yet | [package](../../src/GameCult.Geometry/README.md); snippet not yet | [package](../../src/GameCult.Networking); snippet not yet | [quickstart](../../src/GameCult.Mesh/docs/durable-node-quickstart.md); run by `scripts/verify-eve-getting-started.ps1` | -| **TypeScript** | [first call, tested](#typescript-first-call) | not yet | not yet | [package](../../packages/cultnet-ts/README.md); snippet not yet | [package](../../packages/cultmesh-ts/README.md); snippet not yet | +| **TypeScript** | [first call, tested](#typescript-first-call) | not yet | not yet | [package](../../packages/cultnet-ts/README.md); snippet not yet | [package](../../packages/cultmesh-ts/README.md); snippet not yet; [recipe: two-client .io game](io-game.md), tested with two live clients, browser page not yet run | | **Rust** | [first call, tested](#rust-first-call) | [native core](../../packages/cultmath/native/cultmath-core); snippet not yet | not yet | [package](../../packages/cultnet-rs/README.md), [examples](../../packages/cultnet-rs/examples); snippet not yet | [package](../../packages/cultmesh-rs); snippet not yet | | **Python** | [first call, tested](#python-first-call) | not yet | not yet | [package](../../packages/cultnet-py/README.md); snippet not yet | [package](../../packages/cultmesh-py/README.md); snippet not yet | diff --git a/docs/start/io-game.md b/docs/start/io-game.md new file mode 100644 index 00000000..4781d9bf --- /dev/null +++ b/docs/start/io-game.md @@ -0,0 +1,196 @@ +# Recipe: a two-client .io game over CultCache + CultMesh (TypeScript) + +A tiny arena. One process is the authority: it holds a single `io.arena_state` document in a CultCache +and serves it over CultMesh. Each client discovers the authority through the Odin catalog, leases the +document, and invokes `join`, `move` and `leave` operations. Two clients see each other join, move and +leave because both lease the same document. + +## What ran, what did not + +| Piece | Status | +| --- | --- | +| Two clients, real WebSockets, real `cultmesh-browser` client, one arena: join, see each other, move, leave | **tested** (`arena.test.ts`) | +| Edge clamp, and the source below matching the files that ran | **tested** | +| Browser bundle builds and the page is served | **tested** (HTTP fetch only) | +| The page running in a real browser, two windows | **not yet**: no browser on the host that built this recipe | +| Run by CI / `npm run test:ts` | **not yet**: the recipe has its own `package.json` (for `ws`) outside the workspaces; run it by hand as below | +| Authority state on disk (`.cc` file) | **not yet**: the arena cache is in memory | +| Remote play: Odin roots and a signed route | **not yet**: the client uses `local-development` trust, loopback only | +| Dropped players: a closed tab that never sent `leave` stays in the arena | **not yet**: operations do not carry a connection identity to the authority | +| `CultCache` in the browser | **not yet**: the `@gamecult/cultcache-ts` entry imports Node built-ins, so the client keeps the decoded document in a variable and only `./document` is imported | + +No `cultmesh-ts` or `cultnet-ts` TypeScript authority server exists. The server below is the CultNet +message handling a provider owes, written in this recipe with the `ws` package. + +## Run it + +``` +npm ci --ignore-scripts # repo root +(cd packages/cultnet-ts && npm run build) # then cultcache-ts, cultmesh-browser, same command +cd docs/start/io-game && npm install +npm test # node --test arena.test.ts, Node 24 +node serve.ts # http://127.0.0.1:8080, open it twice +``` + +Tested on Node v24.14, which runs `.ts` files directly (no type check; `tsc` is not part of this recipe). + +## schema.ts + +```ts +import { defineDocumentType } from "@gamecult/cultcache-ts/document"; + +export interface Player { x: number; y: number } +export interface ArenaState { players: Record } + +// The one shared document: every player's position. The server owns it; clients lease it. +export const arenaDocument = defineDocumentType({ + type: "io.arena_state", + schemaId: "io.arena_state.v1", + schema: { parse: (value: unknown) => value as ArenaState }, +}); + +export const arenaKey = "arena:main"; +export const verseId = "io.arena"; +export const authorityRuntimeId = "io.arena-daemon"; +export const arenaSize = 100; +``` + +## server.ts + +```ts +import { createServer, type RequestListener, type Server } from "node:http"; +import { CultCache } from "@gamecult/cultcache-ts"; +import { defineDocumentRegistry } from "@gamecult/cultcache-ts/document"; +import { decodeWireValue, encodeWireValue } from "cultnet-ts/wire"; +import { encodeCultNetMessageForWire, parseCultNetMessage, type CultNetMessage } from "cultnet-ts/contracts"; +import { WebSocketServer, type WebSocket } from "ws"; +import { arenaDocument, arenaKey, arenaSize, authorityRuntimeId, verseId, type ArenaState } from "./schema.ts"; + +const bytesToBase64 = (b: Uint8Array) => Buffer.from(b).toString("base64"); +const base64ToBytes = (s: string) => new Uint8Array(Buffer.from(s, "base64")); + +// One process plays both CultMesh roles: Odin (/odin tells clients where the authority is) and the authority (/mesh). +export async function startArena(port = 0, host = "127.0.0.1", onRequest?: RequestListener) { + const cache = CultCache.builder().withRegistry(defineDocumentRegistry(arenaDocument)).build(); + await cache.put(arenaDocument, arenaKey, { players: {} }); + const http: Server = createServer(onRequest ?? ((_, res) => { res.writeHead(404).end(); })); + await new Promise(resolve => http.listen(port, host, resolve)); + const endpoint = `ws://${host}:${(http.address() as { port: number }).port}`; + + const subscribers = new Map(); + const send = (ws: WebSocket, m: CultNetMessage) => ws.send(encodeWireValue(encodeCultNetMessageForWire(m))); + const record = () => ({ + schemaId: arenaDocument.schemaId, schemaName: arenaDocument.type, schemaVersion: arenaDocument.schemaId, + schemaContentHash: arenaDocument.schemaId, recordKey: arenaKey, storedAt: new Date().toISOString(), + payloadEncoding: "messagepack" as const, + payload: encodeWireValue(cache.getRequired(arenaDocument, arenaKey)) as never, + sourceRuntimeId: authorityRuntimeId, + }); + const publish = async (next: ArenaState) => { + await cache.put(arenaDocument, arenaKey, next); + for (const [ws, ids] of subscribers) for (const subscriptionId of ids) { + send(ws, { schemaVersion: "cultnet.database_change_raw.v0", messageId: crypto.randomUUID(), subscriptionId, changeKind: "updated", document: record() }); + } + }; + + let nextId = 0; + const operations: Record Promise> = { + join: async () => { + const id = `p${++nextId}`; + const state = cache.getRequired(arenaDocument, arenaKey); + await publish({ players: { ...state.players, [id]: { x: Math.floor(Math.random() * arenaSize), y: Math.floor(Math.random() * arenaSize) } } }); + return { id }; + }, + move: async ({ id, dx, dy }: { id: string; dx: number; dy: number }) => { + const state = cache.getRequired(arenaDocument, arenaKey); + const me = state.players[id]; + if (!me) return { ok: false }; + const clamp = (v: number) => Math.max(0, Math.min(arenaSize, v)); + await publish({ players: { ...state.players, [id]: { x: clamp(me.x + Math.sign(dx)), y: clamp(me.y + Math.sign(dy)) } } }); + return { ok: true }; + }, + leave: async ({ id }: { id: string }) => { + const { [id]: _gone, ...rest } = cache.getRequired(arenaDocument, arenaKey).players; + await publish({ players: rest }); + return { ok: true }; + }, + }; + + const wss = new WebSocketServer({ server: http }); + wss.on("connection", (ws, req) => { + const odin = req.url === "/odin"; + ws.on("close", () => subscribers.delete(ws)); + ws.on("message", async raw => { + const m = parseCultNetMessage(decodeWireValue(new Uint8Array(raw as Buffer))); + if (odin && m.schemaVersion === "cultmesh.verse_catalog_request.v0") { + send(ws, { + schemaVersion: "cultmesh.verse_catalog_response.v0", messageId: m.messageId, + verses: [{ + verseId, displayName: "Arena", authorityModel: "ServerAuthoritative", + compatibility: { transportVersion: "cultmesh.v1", rulesHash: "arena-v1", compatibleVerseIds: [], requiredPluginIds: [], optionalPluginIds: [] }, + discoveryEndpoints: [`${endpoint}/mesh`], authorityRuntimeIds: [authorityRuntimeId], + authorityRoutes: [{ authorityRuntimeId, endpoint: `${endpoint}/mesh`, protocolIds: ["cultmesh.documents.v1"], priority: 0, generation: "arena-1" }], + }], + }); + } else if (m.schemaVersion === "cultmesh.session_open.v2") { + send(ws, { schemaVersion: "cultmesh.session_accepted.v2", messageId: m.messageId, accepted: true, verseId: m.verseId, authorityRuntimeId, + protocolId: m.protocolId, routeGeneration: m.routeGeneration, clientNonce: m.clientNonce, providerKeyId: "", providerSignature: "" }); + } else if (m.schemaVersion === "cultnet.database_subscribe.v0") { + subscribers.set(ws, [...(subscribers.get(ws) ?? []), m.subscriptionId]); + send(ws, { schemaVersion: "cultnet.snapshot_response_raw.v0", messageId: m.messageId, documents: [record()] }); + } else if (m.schemaVersion === "cultnet.database_unsubscribe.v0") { + subscribers.set(ws, (subscribers.get(ws) ?? []).filter(id => id !== m.subscriptionId)); + } else if (m.schemaVersion === "cultnet.operation_request.v0") { + const handler = operations[m.operation]; + const receipt = handler ? await handler(decodeWireValue(base64ToBytes(m.payload))) : undefined; + send(ws, { + schemaVersion: "cultnet.operation_response.v0", messageId: m.messageId, serviceId: m.serviceId, operation: m.operation, + status: handler ? "accepted" : "invalid", payloadSchema: "io.receipt.v1", payloadEncoding: "messagepack-base64", + payload: bytesToBase64(encodeWireValue(receipt ?? { ok: false })), sourceRuntimeId: authorityRuntimeId, + }); + } + }); + }); + + return { + odinUrl: `${endpoint}/odin`, + state: () => cache.getRequired(arenaDocument, arenaKey), + close: async () => { for (const c of wss.clients) c.terminate(); wss.close(); await new Promise(r => http.close(r)); }, + }; +} +``` + +## client.ts + +```ts +import { CultMeshBrowserClient, CultMeshBrowserOdinRendezvous, decodeCultNetOperationPayload, decodeCultNetPayload } from "cultmesh-browser"; +import { arenaDocument, arenaKey, authorityRuntimeId, verseId, type ArenaState } from "./schema.ts"; + +// Runs unchanged in a browser or in Node 22+: both have a global WebSocket. +export async function joinArena(odinUrl: string, runtimeId: string, onState: (state: ArenaState, myId: string) => void) { + const rendezvous = new CultMeshBrowserOdinRendezvous({ endpoints: [odinUrl], runtimeId: `${runtimeId}.discovery` }); + const mesh = await CultMeshBrowserClient.connect({ + verseId, authorityRuntimeId, runtimeId, rendezvous, + trust: { mode: "local-development" }, // loopback only; remote Verses need Odin roots + }); + let latest: ArenaState | undefined; // CultCache itself pulls in Node built-ins, so the browser client keeps the decoded document + const call = async (operation: string, payload: object) => + decodeCultNetOperationPayload<{ id?: string }>(await mesh.invoke({ serviceId: "io.arena", operation, payloadSchema: "io.request.v1", payload })); + + const lease = await mesh.leaseRawDocument({ schemaId: arenaDocument.schemaId, recordKey: arenaKey }); + const { id } = await call("join", {}); + lease.watch(record => { + if (!record) return; + latest = decodeCultNetPayload(record, arenaDocument); + onState(latest, id!); + }); + return { + id: id!, + move: (dx: number, dy: number) => call("move", { id, dx, dy }), + state: () => latest, + leave: async () => { await call("leave", { id }); lease.dispose(); await mesh.dispose(); }, + }; +} +``` + +`browser.ts`, `index.html` and `serve.ts` in [`io-game/`](io-game/) draw the canvas and bundle the client with esbuild. diff --git a/docs/start/io-game/arena.test.ts b/docs/start/io-game/arena.test.ts new file mode 100644 index 00000000..67049907 --- /dev/null +++ b/docs/start/io-game/arena.test.ts @@ -0,0 +1,73 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; +import { joinArena } from "./client.ts"; +import { startArena } from "./server.ts"; +import { arenaSize, type ArenaState } from "./schema.ts"; + +const until = async (cond: () => boolean) => { + for (let i = 0; i < 200 && !cond(); i++) await new Promise(r => setTimeout(r, 10)); + assert.ok(cond(), "condition not reached in 2s"); +}; + +test("two clients join one arena, see each other, and see each other move", async () => { + const arena = await startArena(); + const seenByA: ArenaState[] = []; + const seenByB: ArenaState[] = []; + const a = await joinArena(arena.odinUrl, "client-a", s => seenByA.push(structuredClone(s))); + const b = await joinArena(arena.odinUrl, "client-b", s => seenByB.push(structuredClone(s))); + try { + assert.notEqual(a.id, b.id); + await until(() => Object.keys(a.state()?.players ?? {}).length === 2); + await until(() => Object.keys(b.state()?.players ?? {}).length === 2); + + const before = structuredClone(b.state()!.players[a.id]!); + // Moves clamp at the arena edge, so pick a direction that always has room to move. + const dx = before.x > 0 ? -1 : 1; + await a.move(dx, 0); + await until(() => b.state()!.players[a.id]!.x !== before.x); + assert.equal(b.state()!.players[a.id]!.x, before.x + dx); + assert.deepEqual(a.state(), arena.state()); + assert.deepEqual(b.state(), arena.state()); + + await b.leave(); + await until(() => !(a.id !== b.id && b.id in a.state()!.players)); + assert.deepEqual(Object.keys(a.state()!.players), [a.id]); + } finally { + await a.leave().catch(() => undefined); + await arena.close(); + } +}); + +test("a player cannot walk out of the arena", async () => { + const arena = await startArena(); + const a = await joinArena(arena.odinUrl, "client-a", () => undefined); + try { + for (let i = 0; i <= arenaSize; i++) await a.move(-1, 0); + assert.equal(arena.state().players[a.id]!.x, 0); + } finally { + await a.leave(); + await arena.close(); + } +}); + +test("serve.ts bundles the browser client and serves the page", async () => { + const { serveArena } = await import("./serve.ts"); + const site = await serveArena(0); + try { + const html = await (await fetch(site.url)).text(); + assert.match(html, / { + const page = readFileSync(new URL("../io-game.md", import.meta.url), "utf8").replace(/\r\n/g, "\n"); + for (const file of ["schema.ts", "server.ts", "client.ts"]) { + const text = readFileSync(new URL(`./${file}`, import.meta.url), "utf8").replace(/\r\n/g, "\n"); + assert.ok(page.includes(text), `docs/start/io-game.md no longer shows ${file} verbatim`); + } +}); diff --git a/docs/start/io-game/browser.ts b/docs/start/io-game/browser.ts new file mode 100644 index 00000000..4120285f --- /dev/null +++ b/docs/start/io-game/browser.ts @@ -0,0 +1,17 @@ +import { joinArena } from "./client.ts"; +import { arenaSize, type ArenaState } from "./schema.ts"; + +const canvas = document.querySelector("canvas")!; +const ctx = canvas.getContext("2d")!; +const scale = canvas.width / arenaSize; + +const game = await joinArena(`ws://${location.host}/odin`, `browser-${crypto.randomUUID()}`, (state: ArenaState, me: string) => { + ctx.clearRect(0, 0, canvas.width, canvas.height); + for (const [id, p] of Object.entries(state.players)) { + ctx.fillStyle = id === me ? "#f5c542" : "#6aa9ff"; + ctx.fillRect(p.x * scale, p.y * scale, scale * 2, scale * 2); + } +}); +const keys: Record = { ArrowLeft: [-1, 0], ArrowRight: [1, 0], ArrowUp: [0, -1], ArrowDown: [0, 1] }; +addEventListener("keydown", e => { const d = keys[e.key]; if (d) void game.move(...d); }); +addEventListener("pagehide", () => void game.leave()); diff --git a/docs/start/io-game/client.ts b/docs/start/io-game/client.ts new file mode 100644 index 00000000..8493e23c --- /dev/null +++ b/docs/start/io-game/client.ts @@ -0,0 +1,28 @@ +import { CultMeshBrowserClient, CultMeshBrowserOdinRendezvous, decodeCultNetOperationPayload, decodeCultNetPayload } from "cultmesh-browser"; +import { arenaDocument, arenaKey, authorityRuntimeId, verseId, type ArenaState } from "./schema.ts"; + +// Runs unchanged in a browser or in Node 22+: both have a global WebSocket. +export async function joinArena(odinUrl: string, runtimeId: string, onState: (state: ArenaState, myId: string) => void) { + const rendezvous = new CultMeshBrowserOdinRendezvous({ endpoints: [odinUrl], runtimeId: `${runtimeId}.discovery` }); + const mesh = await CultMeshBrowserClient.connect({ + verseId, authorityRuntimeId, runtimeId, rendezvous, + trust: { mode: "local-development" }, // loopback only; remote Verses need Odin roots + }); + let latest: ArenaState | undefined; // CultCache itself pulls in Node built-ins, so the browser client keeps the decoded document + const call = async (operation: string, payload: object) => + decodeCultNetOperationPayload<{ id?: string }>(await mesh.invoke({ serviceId: "io.arena", operation, payloadSchema: "io.request.v1", payload })); + + const lease = await mesh.leaseRawDocument({ schemaId: arenaDocument.schemaId, recordKey: arenaKey }); + const { id } = await call("join", {}); + lease.watch(record => { + if (!record) return; + latest = decodeCultNetPayload(record, arenaDocument); + onState(latest, id!); + }); + return { + id: id!, + move: (dx: number, dy: number) => call("move", { id, dx, dy }), + state: () => latest, + leave: async () => { await call("leave", { id }); lease.dispose(); await mesh.dispose(); }, + }; +} diff --git a/docs/start/io-game/index.html b/docs/start/io-game/index.html new file mode 100644 index 00000000..bcc77648 --- /dev/null +++ b/docs/start/io-game/index.html @@ -0,0 +1,6 @@ + + +arena + +

Arrow keys. Open this page in a second window.

+ diff --git a/docs/start/io-game/package-lock.json b/docs/start/io-game/package-lock.json new file mode 100644 index 00000000..48afe0a7 --- /dev/null +++ b/docs/start/io-game/package-lock.json @@ -0,0 +1,64 @@ +{ + "name": "cultlib-start-io-game", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "cultlib-start-io-game", + "dependencies": { + "ws": "^8.18.0" + }, + "devDependencies": { + "@types/ws": "^8.5.0" + } + }, + "node_modules/@types/node": { + "version": "26.6.6", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.6.6.tgz", + "integrity": "sha512-MzQj+8Kx3XI40OmUAE6Qoh58ZMNstNNMwn2nfd3qWsfe7nDNSqJNzes4njhz6AHwrSHOffSLwXU6yb474Q4aog==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~8.9.0" + } + }, + "node_modules/@types/ws": { + "version": "8.18.2", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.18.2.tgz", + "integrity": "sha512-67MQl+fpWKVTT1NYdnmo3U4sc/xPo/zQBncVnI74qmQa0z/b+1g6iYqNmGCPbxO+zz2aklb08a0oHfegiVd0/w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/undici-types": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz", + "integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==", + "dev": true, + "license": "MIT" + }, + "node_modules/ws": { + "version": "8.22.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.22.1.tgz", + "integrity": "sha512-oyL6OPYme/CmiFvNuGzeoxCJ4nOrZE2O1FwM9MWM1AsGri6nLXB2pg+lNA8aLeXMSav0J6eIVS1j6vUPzUtK9g==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + } + } +} diff --git a/docs/start/io-game/package.json b/docs/start/io-game/package.json new file mode 100644 index 00000000..d8f1281f --- /dev/null +++ b/docs/start/io-game/package.json @@ -0,0 +1,9 @@ +{ + "name": "cultlib-start-io-game", + "private": true, + "type": "module", + "description": "docs/start recipe: a tiny two-client browser game over CultCache + CultMesh (TypeScript).", + "scripts": { "test": "node --test arena.test.ts" }, + "dependencies": { "ws": "^8.18.0" }, + "devDependencies": { "@types/ws": "^8.5.0" } +} diff --git a/docs/start/io-game/schema.ts b/docs/start/io-game/schema.ts new file mode 100644 index 00000000..d0c1a59b --- /dev/null +++ b/docs/start/io-game/schema.ts @@ -0,0 +1,16 @@ +import { defineDocumentType } from "@gamecult/cultcache-ts/document"; + +export interface Player { x: number; y: number } +export interface ArenaState { players: Record } + +// The one shared document: every player's position. The server owns it; clients lease it. +export const arenaDocument = defineDocumentType({ + type: "io.arena_state", + schemaId: "io.arena_state.v1", + schema: { parse: (value: unknown) => value as ArenaState }, +}); + +export const arenaKey = "arena:main"; +export const verseId = "io.arena"; +export const authorityRuntimeId = "io.arena-daemon"; +export const arenaSize = 100; diff --git a/docs/start/io-game/serve.ts b/docs/start/io-game/serve.ts new file mode 100644 index 00000000..82bf20be --- /dev/null +++ b/docs/start/io-game/serve.ts @@ -0,0 +1,17 @@ +import { readFileSync } from "node:fs"; +import { build } from "esbuild"; +import { startArena } from "./server.ts"; + +// Serves the page, the bundled browser client, and the arena's WebSocket on one port. +export async function serveArena(port = 8080, host = "127.0.0.1") { + const bundle = await build({ entryPoints: [new URL("./browser.ts", import.meta.url).pathname], bundle: true, format: "esm", target: "es2022", write: false }); + const html = readFileSync(new URL("./index.html", import.meta.url)); + const arena = await startArena(port, host, (req, res) => { + if (req.url === "/") res.writeHead(200, { "content-type": "text/html" }).end(html); + else if (req.url === "/browser.js") res.writeHead(200, { "content-type": "text/javascript" }).end(bundle.outputFiles[0]!.contents); + else res.writeHead(404).end(); + }); + return { url: arena.odinUrl.replace("ws:", "http:").replace("/odin", ""), close: arena.close }; +} + +if (import.meta.main) console.log(`arena at ${(await serveArena()).url}`); diff --git a/docs/start/io-game/server.ts b/docs/start/io-game/server.ts new file mode 100644 index 00000000..39a1a618 --- /dev/null +++ b/docs/start/io-game/server.ts @@ -0,0 +1,100 @@ +import { createServer, type RequestListener, type Server } from "node:http"; +import { CultCache } from "@gamecult/cultcache-ts"; +import { defineDocumentRegistry } from "@gamecult/cultcache-ts/document"; +import { decodeWireValue, encodeWireValue } from "cultnet-ts/wire"; +import { encodeCultNetMessageForWire, parseCultNetMessage, type CultNetMessage } from "cultnet-ts/contracts"; +import { WebSocketServer, type WebSocket } from "ws"; +import { arenaDocument, arenaKey, arenaSize, authorityRuntimeId, verseId, type ArenaState } from "./schema.ts"; + +const bytesToBase64 = (b: Uint8Array) => Buffer.from(b).toString("base64"); +const base64ToBytes = (s: string) => new Uint8Array(Buffer.from(s, "base64")); + +// One process plays both CultMesh roles: Odin (/odin tells clients where the authority is) and the authority (/mesh). +export async function startArena(port = 0, host = "127.0.0.1", onRequest?: RequestListener) { + const cache = CultCache.builder().withRegistry(defineDocumentRegistry(arenaDocument)).build(); + await cache.put(arenaDocument, arenaKey, { players: {} }); + const http: Server = createServer(onRequest ?? ((_, res) => { res.writeHead(404).end(); })); + await new Promise(resolve => http.listen(port, host, resolve)); + const endpoint = `ws://${host}:${(http.address() as { port: number }).port}`; + + const subscribers = new Map(); + const send = (ws: WebSocket, m: CultNetMessage) => ws.send(encodeWireValue(encodeCultNetMessageForWire(m))); + const record = () => ({ + schemaId: arenaDocument.schemaId, schemaName: arenaDocument.type, schemaVersion: arenaDocument.schemaId, + schemaContentHash: arenaDocument.schemaId, recordKey: arenaKey, storedAt: new Date().toISOString(), + payloadEncoding: "messagepack" as const, + payload: encodeWireValue(cache.getRequired(arenaDocument, arenaKey)) as never, + sourceRuntimeId: authorityRuntimeId, + }); + const publish = async (next: ArenaState) => { + await cache.put(arenaDocument, arenaKey, next); + for (const [ws, ids] of subscribers) for (const subscriptionId of ids) { + send(ws, { schemaVersion: "cultnet.database_change_raw.v0", messageId: crypto.randomUUID(), subscriptionId, changeKind: "updated", document: record() }); + } + }; + + let nextId = 0; + const operations: Record Promise> = { + join: async () => { + const id = `p${++nextId}`; + const state = cache.getRequired(arenaDocument, arenaKey); + await publish({ players: { ...state.players, [id]: { x: Math.floor(Math.random() * arenaSize), y: Math.floor(Math.random() * arenaSize) } } }); + return { id }; + }, + move: async ({ id, dx, dy }: { id: string; dx: number; dy: number }) => { + const state = cache.getRequired(arenaDocument, arenaKey); + const me = state.players[id]; + if (!me) return { ok: false }; + const clamp = (v: number) => Math.max(0, Math.min(arenaSize, v)); + await publish({ players: { ...state.players, [id]: { x: clamp(me.x + Math.sign(dx)), y: clamp(me.y + Math.sign(dy)) } } }); + return { ok: true }; + }, + leave: async ({ id }: { id: string }) => { + const { [id]: _gone, ...rest } = cache.getRequired(arenaDocument, arenaKey).players; + await publish({ players: rest }); + return { ok: true }; + }, + }; + + const wss = new WebSocketServer({ server: http }); + wss.on("connection", (ws, req) => { + const odin = req.url === "/odin"; + ws.on("close", () => subscribers.delete(ws)); + ws.on("message", async raw => { + const m = parseCultNetMessage(decodeWireValue(new Uint8Array(raw as Buffer))); + if (odin && m.schemaVersion === "cultmesh.verse_catalog_request.v0") { + send(ws, { + schemaVersion: "cultmesh.verse_catalog_response.v0", messageId: m.messageId, + verses: [{ + verseId, displayName: "Arena", authorityModel: "ServerAuthoritative", + compatibility: { transportVersion: "cultmesh.v1", rulesHash: "arena-v1", compatibleVerseIds: [], requiredPluginIds: [], optionalPluginIds: [] }, + discoveryEndpoints: [`${endpoint}/mesh`], authorityRuntimeIds: [authorityRuntimeId], + authorityRoutes: [{ authorityRuntimeId, endpoint: `${endpoint}/mesh`, protocolIds: ["cultmesh.documents.v1"], priority: 0, generation: "arena-1" }], + }], + }); + } else if (m.schemaVersion === "cultmesh.session_open.v2") { + send(ws, { schemaVersion: "cultmesh.session_accepted.v2", messageId: m.messageId, accepted: true, verseId: m.verseId, authorityRuntimeId, + protocolId: m.protocolId, routeGeneration: m.routeGeneration, clientNonce: m.clientNonce, providerKeyId: "", providerSignature: "" }); + } else if (m.schemaVersion === "cultnet.database_subscribe.v0") { + subscribers.set(ws, [...(subscribers.get(ws) ?? []), m.subscriptionId]); + send(ws, { schemaVersion: "cultnet.snapshot_response_raw.v0", messageId: m.messageId, documents: [record()] }); + } else if (m.schemaVersion === "cultnet.database_unsubscribe.v0") { + subscribers.set(ws, (subscribers.get(ws) ?? []).filter(id => id !== m.subscriptionId)); + } else if (m.schemaVersion === "cultnet.operation_request.v0") { + const handler = operations[m.operation]; + const receipt = handler ? await handler(decodeWireValue(base64ToBytes(m.payload))) : undefined; + send(ws, { + schemaVersion: "cultnet.operation_response.v0", messageId: m.messageId, serviceId: m.serviceId, operation: m.operation, + status: handler ? "accepted" : "invalid", payloadSchema: "io.receipt.v1", payloadEncoding: "messagepack-base64", + payload: bytesToBase64(encodeWireValue(receipt ?? { ok: false })), sourceRuntimeId: authorityRuntimeId, + }); + } + }); + }); + + return { + odinUrl: `${endpoint}/odin`, + state: () => cache.getRequired(arenaDocument, arenaKey), + close: async () => { for (const c of wss.clients) c.terminate(); wss.close(); await new Promise(r => http.close(r)); }, + }; +} From be0502c8973afdb4a2f1517c676db53f41752469 Mon Sep 17 00:00:00 2001 From: Libby Date: Fri, 6 Nov 2026 12:30:20 +0000 Subject: [PATCH 49/88] docs(start): tested CultNet first-call snippets for C#, TypeScript, Rust, Python Each sends a typed document as a document_put_raw message over a loopback pipe and applies it to a second cache; a second test per runtime fails when docs/start/README.md stops showing the snippet verbatim. Co-Authored-By: Claude Sonnet 5.5 --- docs/start/README.md | 246 +++++++++++++++++- packages/cultnet-py/tests/test_first_call.py | 72 +++++ packages/cultnet-rs/tests/first_call.rs | 77 ++++++ packages/cultnet-ts/test/first-call.test.ts | 78 ++++++ .../CultNetFirstCallTests.cs | 80 ++++++ 5 files changed, 549 insertions(+), 4 deletions(-) create mode 100644 packages/cultnet-py/tests/test_first_call.py create mode 100644 packages/cultnet-rs/tests/first_call.rs create mode 100644 packages/cultnet-ts/test/first-call.test.ts create mode 100644 tests/GameCult.Networking.Tests/CultNetFirstCallTests.cs diff --git a/docs/start/README.md b/docs/start/README.md index 99b66354..4e3950be 100644 --- a/docs/start/README.md +++ b/docs/start/README.md @@ -16,10 +16,10 @@ this page. | | CultCache | CultMath | Geometry | CultNet | CultMesh | | --- | --- | --- | --- | --- | --- | -| **C#** | [first call, tested](#c-first-call) | [package](../../packages/cultmath/README.md); snippet not yet | [package](../../src/GameCult.Geometry/README.md); snippet not yet | [package](../../src/GameCult.Networking); snippet not yet | [quickstart](../../src/GameCult.Mesh/docs/durable-node-quickstart.md); run by `scripts/verify-eve-getting-started.ps1` | -| **TypeScript** | [first call, tested](#typescript-first-call) | not yet | not yet | [package](../../packages/cultnet-ts/README.md); snippet not yet | [package](../../packages/cultmesh-ts/README.md); snippet not yet | -| **Rust** | [first call, tested](#rust-first-call) | [native core](../../packages/cultmath/native/cultmath-core); snippet not yet | not yet | [package](../../packages/cultnet-rs/README.md), [examples](../../packages/cultnet-rs/examples); snippet not yet | [package](../../packages/cultmesh-rs); snippet not yet | -| **Python** | [first call, tested](#python-first-call) | not yet | not yet | [package](../../packages/cultnet-py/README.md); snippet not yet | [package](../../packages/cultmesh-py/README.md); snippet not yet | +| **C#** | [first call, tested](#c-first-call) | [package](../../packages/cultmath/README.md); snippet not yet | [package](../../src/GameCult.Geometry/README.md); snippet not yet | [first call, tested](#c-cultnet-first-call); [package](../../src/GameCult.Networking) | [quickstart](../../src/GameCult.Mesh/docs/durable-node-quickstart.md); run by `scripts/verify-eve-getting-started.ps1` | +| **TypeScript** | [first call, tested](#typescript-first-call) | not yet | not yet | [first call, tested](#typescript-cultnet-first-call); [package](../../packages/cultnet-ts/README.md) | [package](../../packages/cultmesh-ts/README.md); snippet not yet | +| **Rust** | [first call, tested](#rust-first-call) | [native core](../../packages/cultmath/native/cultmath-core); snippet not yet | not yet | [first call, tested](#rust-cultnet-first-call); [package](../../packages/cultnet-rs/README.md), [examples](../../packages/cultnet-rs/examples) | [package](../../packages/cultmesh-rs); snippet not yet | +| **Python** | [first call, tested](#python-first-call) | not yet | not yet | [first call, tested](#python-cultnet-first-call); [package](../../packages/cultnet-py/README.md) | [package](../../packages/cultmesh-py/README.md); snippet not yet | For the whole picture of what each piece is for, see the @@ -222,6 +222,244 @@ def first_call() -> Item: return reopened.get_required(item_doc, "item:potion") ``` +## CultNet first calls + +CultNet is schema-v0 messages in length-prefixed frames. The first call in each runtime is the same +shape: define a typed document, build a `cultnet.document_put_raw.v0` message carrying its MessagePack +payload, send one frame over a loopback pipe, and apply the received message to a second cache. Each +block is verbatim from the test that runs it; a second test in the same file fails if the block stops +matching. + +### C# CultNet first call + +Source: [`tests/GameCult.Networking.Tests/CultNetFirstCallTests.cs`](../../tests/GameCult.Networking.Tests/CultNetFirstCallTests.cs), +run by `dotnet test tests/GameCult.Networking.Tests --filter CultNetFirstCallTests`. More: `src/GameCult.Networking/README.md`. + +```csharp +using System.Net; +using System.Net.Sockets; +using System.Threading.Tasks; +using GameCult.Caching; +using GameCult.Networking; +using MessagePack; + +[CultDocument("gamecult.first_call_note", "gamecult.first_call_note.v1")] +[MessagePackObject] +public sealed class Note +{ + [Key(0)] [CultName] public string Title = string.Empty; + [Key(1)] public string Body = string.Empty; +} + +public static class CultNetFirstCall +{ + public static async Task Run() + { + var inbox = new CultCache(); // the receiver's cache + var documents = new CultNetDocumentRegistry(); + var received = new TaskCompletionSource(); + + // Receive: a loopback listener that applies each raw document put to the inbox. + using var server = new TcpFramedCultNetSchemaServer(new TcpListener(IPAddress.Loopback, 0)); + server.OnCultNet(async (put, peer) => + { + await documents.ApplyRawDocumentPutMessageAsync(inbox, put); + received.TrySetResult(true); + }); + + // Send: a typed document as a cultnet.document_put_raw.v0 message. + using var client = new TcpFramedCultNetSchemaClient(); + client.Connect("127.0.0.1", server.LocalEndPoint.Port); + var handle = new CultRecordHandle(new CultRecordKey("note:hello")); + client.SendCultNet(documents.CreateRawDocumentPutMessage( + "put-1", handle, new Note { Title = "Hello", Body = "sent over CultNet" })); + + await received.Task.WaitAsync(TimeSpan.FromSeconds(5)); + return inbox.Get(handle.Key); + } +} +``` + +### TypeScript CultNet first call + +Source: [`packages/cultnet-ts/test/first-call.test.ts`](../../packages/cultnet-ts/test/first-call.test.ts), run by +`npm test` in `packages/cultnet-ts` (build `packages/cultcache-ts` first: `npm run build` there). The test imports +from `../src` where the block reads `cultnet-ts`. More: `packages/cultnet-ts/README.md`. + +```ts +import { createServer, connect } from "node:net"; +import { z } from "zod"; +import { CultCache, defineDocumentType } from "@gamecult/cultcache-ts"; +import { + CultNetDocumentRegistry, + createTcpFramedCultNetPeer, + defineCultNetDocumentBinding, +} from "cultnet-ts"; + +const noteDocument = defineDocumentType({ + type: "note", + schemaId: "gamecult.first_call_note.v1", + schemaName: "gamecult.first_call_note", + schemaVersion: "gamecult.first_call_note.v1", + schema: z.object({ title: z.string(), body: z.string() }), + name: "title", +}); +const documents = new CultNetDocumentRegistry([defineCultNetDocumentBinding({ definition: noteDocument })]); + +async function firstCall() { + const options = { runtimeId: "first-call", wireContract: "cultnet.schema.v0" } as const; + const inbox = CultCache.builder().withDocumentType(noteDocument).build(); // the receiver's cache + + // Receiver: a loopback listener whose peer applies each document put to the inbox. + const received = new Promise((resolve, reject) => { + const server = createServer((socket) => { + const peer = createTcpFramedCultNetPeer(socket, options); + peer.on("invalidMessage", reject); + peer.on("message", async (message) => { + if (message.schemaVersion !== "cultnet.document_put_raw.v0") return; + await documents.applyRawDocumentPutMessage(inbox, message); + peer.close(); + server.close(); + resolve(); + }); + }).listen(0, "127.0.0.1", () => { + // Sender: connect, build a typed put message, send it. + const { port } = server.address() as { port: number }; + const sender = createTcpFramedCultNetPeer(connect(port, "127.0.0.1"), options); + sender.send( + documents.createRawDocumentPutMessage(documents.get("note")!, "put-1", "note:hello", { + title: "Hello", + body: "sent over CultNet", + }), + ); + received.finally(() => sender.close()); + }); + }); + await received; + return inbox.get(noteDocument, "note:hello"); +} +``` + +### Rust CultNet first call + +Source: [`packages/cultnet-rs/tests/first_call.rs`](../../packages/cultnet-rs/tests/first_call.rs), run by +`cargo test --test first_call` in `packages/cultnet-rs`. More: `packages/cultnet-rs/README.md`. + +```rust +use std::net::{TcpListener, TcpStream}; +use std::thread; + +use cultnet_rs::{ + CultNetDocumentBinding, CultNetDocumentPutOptions, CultNetDocumentRegistry, CultNetWireContract, + TcpFramedTransportConnection, create_tcp_framed_transport_profile, + decode_cultnet_message_from_slice, encode_cultnet_message_to_vec, +}; + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "note")] +struct Note { + #[cultcache(key = 0)] + title: String, + #[cultcache(key = 1)] + body: String, +} + +fn first_call() -> anyhow::Result { + let mut documents = CultNetDocumentRegistry::new(); + documents.register(CultNetDocumentBinding::for_entry_with_schema_id::( + "gamecult.first_call_note.v1".to_string(), + "gamecult.first_call_note.v1".to_string(), + ))?; + let profile = |runtime_id| create_tcp_framed_transport_profile(runtime_id, Default::default()); + + // Receiver: a loopback listener that applies the document put it receives to its own cache. + let listener = TcpListener::bind("127.0.0.1:0")?; + let port = listener.local_addr()?.port(); + let receiver = thread::spawn({ + let (documents, profile) = (documents.clone(), profile("receiver")); + move || -> anyhow::Result { + let (stream, _) = listener.accept()?; + let frame = TcpFramedTransportConnection::new(stream, profile).receive()?; + let message = decode_cultnet_message_from_slice(&frame.payload, CultNetWireContract::CultNetSchemaV0)?; + let mut inbox = CultCache::new(); + inbox.register_entry_type::()?; + documents.apply_raw_document_put_message::(&mut inbox, &message) + } + }); + + // Sender: build a typed put message, encode it for the wire, send one frame. + let put = documents.create_raw_document_put_message( + "put-1", + "note:hello", + &Note { title: "Hello".to_string(), body: "sent over CultNet".to_string() }, + CultNetDocumentPutOptions::default(), + )?; + let bytes = encode_cultnet_message_to_vec(&put, CultNetWireContract::CultNetSchemaV0)?; + TcpFramedTransportConnection::new(TcpStream::connect(("127.0.0.1", port))?, profile("sender")) + .send("schema", &bytes)?; + + receiver.join().expect("receiver thread") +} +``` + +### Python CultNet first call + +Source: [`packages/cultnet-py/tests/test_first_call.py`](../../packages/cultnet-py/tests/test_first_call.py), run by +`python -m unittest tests.test_first_call` in `packages/cultnet-py` with both `src` and `../cultcache-py/src` on +`PYTHONPATH` and `msgpack` installed. The socket pair stands in for a TCP connection. More: `packages/cultnet-py/README.md`. + +```python +import socket +from dataclasses import asdict, dataclass + +from cultcache_py import CultCache, define_document_type +from cultnet_py import ( + apply_raw_document_record, + document_put_raw, + encode_frame, + parse_message, + read_frame, + schema_document_map, +) + + +@dataclass +class Note: + title: str + body: str + + +note_doc = define_document_type( + "note", + encode=lambda note: asdict(note), + decode=lambda raw: Note(**raw), + name="title", +) + + +def first_call() -> Note: + sender, receiver = socket.socketpair() # an in-process pipe; use socket.create_connection for TCP + + # Send: a typed document as a cultnet.document_put_raw.v0 message in one length-prefixed frame. + put = document_put_raw( + message_id="put-1", + key="note:hello", + schema_id=note_doc.catalog_entry().schema_id, + stored_at="2026-01-01T00:00:00Z", + payload=note_doc.encode_payload(Note("Hello", "sent over CultNet")), + ) + sender.sendall(encode_frame(put.to_bytes())) + + # Receive: read the frame, parse the message, apply its record to a second cache. + message = parse_message(read_frame(receiver.makefile("rb"))) + assert message.schema_version == "cultnet.document_put_raw.v0" + inbox = CultCache.builder().register_document_type(note_doc).build() + apply_raw_document_record(inbox, schema_document_map([note_doc]), message.body["document"]) + sender.close() + receiver.close() + return inbox.get_required(note_doc, "note:hello") +``` + ## Where next - Smallest durable mesh path: `src/GameCult.Mesh/docs/durable-node-quickstart.md`. diff --git a/packages/cultnet-py/tests/test_first_call.py b/packages/cultnet-py/tests/test_first_call.py new file mode 100644 index 00000000..9815043c --- /dev/null +++ b/packages/cultnet-py/tests/test_first_call.py @@ -0,0 +1,72 @@ +"""The Python CultNet first call from docs/start/README.md. The docs show the region between the +`snippet` markers verbatim, and test_docs_show_the_tested_snippet fails when they differ.""" + +import unittest +from pathlib import Path + +# snippet:begin +import socket +from dataclasses import asdict, dataclass + +from cultcache_py import CultCache, define_document_type +from cultnet_py import ( + apply_raw_document_record, + document_put_raw, + encode_frame, + parse_message, + read_frame, + schema_document_map, +) + + +@dataclass +class Note: + title: str + body: str + + +note_doc = define_document_type( + "note", + encode=lambda note: asdict(note), + decode=lambda raw: Note(**raw), + name="title", +) + + +def first_call() -> Note: + sender, receiver = socket.socketpair() # an in-process pipe; use socket.create_connection for TCP + + # Send: a typed document as a cultnet.document_put_raw.v0 message in one length-prefixed frame. + put = document_put_raw( + message_id="put-1", + key="note:hello", + schema_id=note_doc.catalog_entry().schema_id, + stored_at="2026-01-01T00:00:00Z", + payload=note_doc.encode_payload(Note("Hello", "sent over CultNet")), + ) + sender.sendall(encode_frame(put.to_bytes())) + + # Receive: read the frame, parse the message, apply its record to a second cache. + message = parse_message(read_frame(receiver.makefile("rb"))) + assert message.schema_version == "cultnet.document_put_raw.v0" + inbox = CultCache.builder().register_document_type(note_doc).build() + apply_raw_document_record(inbox, schema_document_map([note_doc]), message.body["document"]) + sender.close() + receiver.close() + return inbox.get_required(note_doc, "note:hello") +# snippet:end + + +class FirstCallTests(unittest.TestCase): + def test_a_typed_document_crosses_a_pipe_into_a_second_cache(self) -> None: + self.assertEqual(first_call(), Note("Hello", "sent over CultNet")) + + def test_docs_show_the_tested_snippet(self) -> None: + source = Path(__file__).read_text(encoding="utf-8") + snippet = source.split("# snippet:begin\n", 1)[1].split("# snippet:end\n", 1)[0] + docs = (Path(__file__).parents[3] / "docs" / "start" / "README.md").read_text(encoding="utf-8") + self.assertIn("```python\n" + snippet + "```", docs) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/cultnet-rs/tests/first_call.rs b/packages/cultnet-rs/tests/first_call.rs new file mode 100644 index 00000000..e2ef1e6f --- /dev/null +++ b/packages/cultnet-rs/tests/first_call.rs @@ -0,0 +1,77 @@ +//! The Rust CultNet first call from docs/start/README.md. The docs show the region between the +//! `snippet` markers verbatim, and `docs_show_the_tested_snippet` fails when they differ. + +use cultcache_rs::{CultCache, DatabaseEntry}; + +// snippet:begin +use std::net::{TcpListener, TcpStream}; +use std::thread; + +use cultnet_rs::{ + CultNetDocumentBinding, CultNetDocumentPutOptions, CultNetDocumentRegistry, CultNetWireContract, + TcpFramedTransportConnection, create_tcp_framed_transport_profile, + decode_cultnet_message_from_slice, encode_cultnet_message_to_vec, +}; + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "note")] +struct Note { + #[cultcache(key = 0)] + title: String, + #[cultcache(key = 1)] + body: String, +} + +fn first_call() -> anyhow::Result { + let mut documents = CultNetDocumentRegistry::new(); + documents.register(CultNetDocumentBinding::for_entry_with_schema_id::( + "gamecult.first_call_note.v1".to_string(), + "gamecult.first_call_note.v1".to_string(), + ))?; + let profile = |runtime_id| create_tcp_framed_transport_profile(runtime_id, Default::default()); + + // Receiver: a loopback listener that applies the document put it receives to its own cache. + let listener = TcpListener::bind("127.0.0.1:0")?; + let port = listener.local_addr()?.port(); + let receiver = thread::spawn({ + let (documents, profile) = (documents.clone(), profile("receiver")); + move || -> anyhow::Result { + let (stream, _) = listener.accept()?; + let frame = TcpFramedTransportConnection::new(stream, profile).receive()?; + let message = decode_cultnet_message_from_slice(&frame.payload, CultNetWireContract::CultNetSchemaV0)?; + let mut inbox = CultCache::new(); + inbox.register_entry_type::()?; + documents.apply_raw_document_put_message::(&mut inbox, &message) + } + }); + + // Sender: build a typed put message, encode it for the wire, send one frame. + let put = documents.create_raw_document_put_message( + "put-1", + "note:hello", + &Note { title: "Hello".to_string(), body: "sent over CultNet".to_string() }, + CultNetDocumentPutOptions::default(), + )?; + let bytes = encode_cultnet_message_to_vec(&put, CultNetWireContract::CultNetSchemaV0)?; + TcpFramedTransportConnection::new(TcpStream::connect(("127.0.0.1", port))?, profile("sender")) + .send("schema", &bytes)?; + + receiver.join().expect("receiver thread") +} +// snippet:end + +#[test] +fn a_typed_document_crosses_loopback_into_a_second_cache() { + let note = first_call().unwrap(); + assert_eq!(note, Note { title: "Hello".to_string(), body: "sent over CultNet".to_string() }); +} + +#[test] +fn docs_show_the_tested_snippet() { + let source = std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/first_call.rs")).unwrap(); + let begin = source.find("// snippet:begin\n").unwrap() + "// snippet:begin\n".len(); + let end = source.find("// snippet:end\n").unwrap(); + let docs = std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/../../docs/start/README.md")).unwrap(); + let block = format!("```rust\n{}```", &source[begin..end]); + assert!(docs.contains(&block), "docs/start/README.md no longer shows tests/first_call.rs verbatim"); +} diff --git a/packages/cultnet-ts/test/first-call.test.ts b/packages/cultnet-ts/test/first-call.test.ts new file mode 100644 index 00000000..b389ca95 --- /dev/null +++ b/packages/cultnet-ts/test/first-call.test.ts @@ -0,0 +1,78 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import test from "node:test"; + +// The TypeScript CultNet first-call snippet that docs/start/README.md shows: define a typed document, send it +// over a loopback TCP pipe as a cultnet.document_put_raw.v0 message, receive it into a second cache. The docs show +// the region between the `snippet` markers verbatim, except that the package import reads +// "cultnet-ts" where this file reads "../src"; the docs test below fails when they differ. +// snippet:begin +import { createServer, connect } from "node:net"; +import { z } from "zod"; +import { CultCache, defineDocumentType } from "@gamecult/cultcache-ts"; +import { + CultNetDocumentRegistry, + createTcpFramedCultNetPeer, + defineCultNetDocumentBinding, +} from "../src"; + +const noteDocument = defineDocumentType({ + type: "note", + schemaId: "gamecult.first_call_note.v1", + schemaName: "gamecult.first_call_note", + schemaVersion: "gamecult.first_call_note.v1", + schema: z.object({ title: z.string(), body: z.string() }), + name: "title", +}); +const documents = new CultNetDocumentRegistry([defineCultNetDocumentBinding({ definition: noteDocument })]); + +async function firstCall() { + const options = { runtimeId: "first-call", wireContract: "cultnet.schema.v0" } as const; + const inbox = CultCache.builder().withDocumentType(noteDocument).build(); // the receiver's cache + + // Receiver: a loopback listener whose peer applies each document put to the inbox. + const received = new Promise((resolve, reject) => { + const server = createServer((socket) => { + const peer = createTcpFramedCultNetPeer(socket, options); + peer.on("invalidMessage", reject); + peer.on("message", async (message) => { + if (message.schemaVersion !== "cultnet.document_put_raw.v0") return; + await documents.applyRawDocumentPutMessage(inbox, message); + peer.close(); + server.close(); + resolve(); + }); + }).listen(0, "127.0.0.1", () => { + // Sender: connect, build a typed put message, send it. + const { port } = server.address() as { port: number }; + const sender = createTcpFramedCultNetPeer(connect(port, "127.0.0.1"), options); + sender.send( + documents.createRawDocumentPutMessage(documents.get("note")!, "put-1", "note:hello", { + title: "Hello", + body: "sent over CultNet", + }), + ); + received.finally(() => sender.close()); + }); + }); + await received; + return inbox.get(noteDocument, "note:hello"); +} +// snippet:end + +test("send a typed document over loopback and receive it into a second cache", async () => { + const note = await firstCall(); + assert.deepEqual(note, { title: "Hello", body: "sent over CultNet" }); +}); + +test("docs show the tested snippet", async () => { + // Compiled to dist-test/test/first-call.test.js, so the package root is two levels up. + const source = await readFile(join(__dirname, "..", "..", "test", "first-call.test.ts"), "utf8"); + const snippet = source + .split("// snippet:begin\n")[1] + .split("// snippet:end\n")[0] + .replace('"../src"', '"cultnet-ts"'); + const docs = (await readFile(join(__dirname, "..", "..", "..", "..", "docs", "start", "README.md"), "utf8")).replace(/\r\n/g, "\n"); + assert.ok(docs.includes("```ts\n" + snippet + "```"), "docs/start/README.md no longer shows test/first-call.test.ts"); +}); diff --git a/tests/GameCult.Networking.Tests/CultNetFirstCallTests.cs b/tests/GameCult.Networking.Tests/CultNetFirstCallTests.cs new file mode 100644 index 00000000..67c2f06d --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetFirstCallTests.cs @@ -0,0 +1,80 @@ +#nullable enable +using System; +using System.IO; +using NUnit.Framework; + +// snippet:begin +using System.Net; +using System.Net.Sockets; +using System.Threading.Tasks; +using GameCult.Caching; +using GameCult.Networking; +using MessagePack; + +[CultDocument("gamecult.first_call_note", "gamecult.first_call_note.v1")] +[MessagePackObject] +public sealed class Note +{ + [Key(0)] [CultName] public string Title = string.Empty; + [Key(1)] public string Body = string.Empty; +} + +public static class CultNetFirstCall +{ + public static async Task Run() + { + var inbox = new CultCache(); // the receiver's cache + var documents = new CultNetDocumentRegistry(); + var received = new TaskCompletionSource(); + + // Receive: a loopback listener that applies each raw document put to the inbox. + using var server = new TcpFramedCultNetSchemaServer(new TcpListener(IPAddress.Loopback, 0)); + server.OnCultNet(async (put, peer) => + { + await documents.ApplyRawDocumentPutMessageAsync(inbox, put); + received.TrySetResult(true); + }); + + // Send: a typed document as a cultnet.document_put_raw.v0 message. + using var client = new TcpFramedCultNetSchemaClient(); + client.Connect("127.0.0.1", server.LocalEndPoint.Port); + var handle = new CultRecordHandle(new CultRecordKey("note:hello")); + client.SendCultNet(documents.CreateRawDocumentPutMessage( + "put-1", handle, new Note { Title = "Hello", Body = "sent over CultNet" })); + + await received.Task.WaitAsync(TimeSpan.FromSeconds(5)); + return inbox.Get(handle.Key); + } +} +// snippet:end + +namespace GameCult.Networking.Tests +{ + // The C# CultNet first-call snippet that docs/start/README.md shows verbatim: define a typed document, send it + // over a loopback TCP pipe, receive it into a second cache. DocsShowTheTestedSnippet fails when they differ. + public class CultNetFirstCallTests + { + [Test] + public async Task SendATypedDocumentOverLoopbackAndReceiveItIntoASecondCache() + { + var note = await CultNetFirstCall.Run(); + Assert.That(note, Is.Not.Null); + Assert.That(note!.Title, Is.EqualTo("Hello")); + Assert.That(note.Body, Is.EqualTo("sent over CultNet")); + } + + [Test] + public void DocsShowTheTestedSnippet() + { + var source = File.ReadAllText(SourcePath()); + const string begin = "// snippet:begin\n"; + var start = source.IndexOf(begin, StringComparison.Ordinal) + begin.Length; + var snippet = source[start..source.IndexOf("// snippet:end\n", StringComparison.Ordinal)]; + var docs = File.ReadAllText(Path.Combine(Path.GetDirectoryName(SourcePath())!, "..", "..", "docs", "start", "README.md")); + Assert.That(docs.Replace("\r\n", "\n"), Does.Contain("```csharp\n" + snippet + "```"), + "docs/start/README.md no longer shows tests/GameCult.Networking.Tests/CultNetFirstCallTests.cs verbatim"); + } + + static string SourcePath([System.Runtime.CompilerServices.CallerFilePath] string path = "") => path; + } +} From 309c8c1ba3af58af35a5664f550014415107efc9 Mon Sep 17 00:00:00 2001 From: Libby Date: Fri, 6 Nov 2026 15:35:25 +0000 Subject: [PATCH 50/88] heimdall-consent: typed records for the answer collector NoticeAcceptance, Answer and MemberTerms cut from Heimdall PR #6 (docs/answer-collector-consent.md, head d836f29), with a golden vector for versioned sample keys, scope-less exclusion from training, redaction as hash-only tombstone and newcomer notice acceptance. Ambiguities are listed in the README as questions for Heimdall. Co-Authored-By: Claude Sonnet 5.5 --- packages/heimdall-consent/Cargo.lock | 716 ++++++++++++++++++ packages/heimdall-consent/Cargo.toml | 18 + packages/heimdall-consent/README.md | 21 + packages/heimdall-consent/src/lib.rs | 146 ++++ packages/heimdall-consent/tests/golden.rs | 200 +++++ .../tests/vectors/consent_golden.csv | 33 + 6 files changed, 1134 insertions(+) create mode 100644 packages/heimdall-consent/Cargo.lock create mode 100644 packages/heimdall-consent/Cargo.toml create mode 100644 packages/heimdall-consent/README.md create mode 100644 packages/heimdall-consent/src/lib.rs create mode 100644 packages/heimdall-consent/tests/golden.rs create mode 100644 packages/heimdall-consent/tests/vectors/consent_golden.csv diff --git a/packages/heimdall-consent/Cargo.lock b/packages/heimdall-consent/Cargo.lock new file mode 100644 index 00000000..758d85ea --- /dev/null +++ b/packages/heimdall-consent/Cargo.lock @@ -0,0 +1,716 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "ar_archive_writer" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6" +dependencies = [ + "object", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "cc" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ff2a8126fff0f8c839080d619675a585e61720bb12d2baceae45769fd66f155" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cultcache-rs" +version = "0.3.1" +dependencies = [ + "anyhow", + "chrono", + "cultcache-rs-derive", + "fs2", + "redb", + "rmp-serde", + "serde", + "serde_bytes", + "serde_json", + "serde_stacker", + "stacker", + "uuid", + "windows-sys 0.59.0", +] + +[[package]] +name = "cultcache-rs-derive" +version = "0.2.0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "heimdall-consent" +version = "0.1.0" +dependencies = [ + "anyhow", + "cultcache-rs", + "serde", + "tempfile", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dc09d2154098bbf4ef54eab6b117a3c24cbbffe94046b16ee6d6d4510584d3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "object" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "psm" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd034599e63b970727f70d79e02d62390a4a84f7c6b827c27c46d5ac3fa622" +dependencies = [ + "ar_archive_writer", + "cc", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "redb" +version = "4.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b317ef681e18f6091efbfab532eaf924a135c492d9d0299d0fbfae07e92a39b" +dependencies = [ + "libc", +] + +[[package]] +name = "rmp" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "rmp-serde" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155" +dependencies = [ + "rmp", + "serde", +] + +[[package]] +name = "rustix" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_stacker" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4936375d50c4be7eff22293a9344f8e46f323ed2b3c243e52f89138d9bb0f4a" +dependencies = [ + "serde", + "serde_core", + "stacker", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "stacker" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.61.2", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "uuid" +version = "1.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a0e2d11620c50695fe12bbbfde96b3477fe606e7284c22d505c12ce51214514" +dependencies = [ + "getrandom", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0181c14ddd6c7b8b0b0132a5f1ea2f32be0d893cd3ea97b6e3b28886b8a3d1c" +dependencies = [ + "cfg-if", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e931a02a39abdbf66966fc35902b5ed58344750a70885bed4ebb3d8eaf28ed9" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ca4b54280e455bd831e33a6b8e734baf432fe63fbc35c42ac14ce67a043a53a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.9", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b1f9c3798725016c6ee5f2f57869ec0831e8bbc0ab0310a86e787e97f41389e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/packages/heimdall-consent/Cargo.toml b/packages/heimdall-consent/Cargo.toml new file mode 100644 index 00000000..9c2b6729 --- /dev/null +++ b/packages/heimdall-consent/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "heimdall-consent" +version = "0.1.0" +edition = "2024" +license = "MIT" +publish = false + +[lib] +name = "heimdall_consent" +path = "src/lib.rs" + +[dependencies] +anyhow = "1" +cultcache-rs = { path = "../cultcache-rs" } +serde = { version = "1", features = ["derive"] } + +[dev-dependencies] +tempfile = "3" diff --git a/packages/heimdall-consent/README.md b/packages/heimdall-consent/README.md new file mode 100644 index 00000000..5e2c3284 --- /dev/null +++ b/packages/heimdall-consent/README.md @@ -0,0 +1,21 @@ +# heimdall-consent + +Typed CultCache records for Heimdall's answer collector: `NoticeAcceptance`, `Answer` and `MemberTerms`, +plus the two rules that need no store to state (`Answer::is_training_data`, `Answer::redacted`). Cut from +`docs/answer-collector-consent.md` in GameCult/Heimdall PR #6 (head d836f29); the doc's tables are the +contract. Specification only: no collector, tally or export lives here. + +Entry points: `src/lib.rs`; golden vectors `tests/vectors/consent_golden.csv`, run by `tests/golden.rs` +(`cargo test`). Planned, not live: the collector, the tally, the training export. + +## Choices the doc does not fix (questions for Heimdall) + +1. Timestamps are unix milliseconds (`i64`), matching `origin_server_ts`. The doc says "UTC timestamp". +2. `member_id`, `experiment_id`, `event_id` are strings; `notice_version` is `u32`; `terms_version` a string. The doc names only "Heimdall member id" and "Matrix event id". +3. Key text is the parts joined with U+001F; a part that is empty or contains it is refused. Matrix ids can hold most other separators. +4. The tombstone's salted `text` hash has no row in the answer table. It is carried as `Answer.text_hash` (bytes) on the exited record. Which hash function, whose salt, and whether the tombstone is a separate record is undecided; the crate never computes the hash. +5. `Answer.notice_version` and `scope` are optional in the typed record so an answer lacking them can exist and be excluded; the table makes both required. Should such a reply be stored at all, given "is not collected"? +6. A reply under a notice with no recorded scope writes no acceptance in the vector's collector; the doc says the first reply accepts "the notice version then pinned" and does not cover a missing pin. +7. `MemberTerms` is a separate record keyed by `member_id`. The doc puts the fields on the member; the member record lives in Heimdall, not CultLib. +8. Training rule item 2 also requires the acceptance's scope to be `train` and to match the answer's experiment and member; the doc requires only that an acceptance exist for the key with `accepted_at <= received_at`. +9. The "separate store or partition" for records wider than `show` is not modelled here. diff --git a/packages/heimdall-consent/src/lib.rs b/packages/heimdall-consent/src/lib.rs new file mode 100644 index 00000000..00dc3b48 --- /dev/null +++ b/packages/heimdall-consent/src/lib.rs @@ -0,0 +1,146 @@ +//! Typed CultCache records for Heimdall's answer collector, cut from `docs/answer-collector-consent.md` +//! in GameCult/Heimdall PR #6 (head d836f29): the notice acceptance record, the answer record, and +//! the terms version on the member. Field names follow the doc's tables. Where the doc is silent +//! the choice made here is listed in README.md as a question for Heimdall. + +use anyhow::{Result, bail}; +use cultcache_rs::DatabaseEntry; +use serde::{Deserialize, Serialize}; + +/// What a notice allows, each level including the ones below it. `Train` is declared in terms by the +/// room's notice; nothing here widens a scope. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Scope { + Show, + Study, + Train, +} + +/// Separates key parts. Ids containing it are refused rather than escaped. +const SEP: char = '\u{1f}'; + +fn key(parts: &[&str]) -> Result { + if let Some(p) = parts.iter().find(|p| p.is_empty() || p.contains(SEP)) { + bail!("key part {p:?} is empty or contains the key separator"); + } + Ok(parts.join(&SEP.to_string())) +} + +/// `@`, version a positive integer. The version is part of the id, so it is part +/// of every answer key and no key can name a sample without one. +pub fn check_sample_id(sample_id: &str) -> Result<(&str, u64)> { + let Some((sample, version)) = sample_id.rsplit_once('@') else { + bail!("sample id {sample_id:?} has no @version"); + }; + match version.parse::() { + Ok(v) if v > 0 && !sample.is_empty() && !version.starts_with('+') => Ok((sample, v)), + _ => bail!("sample id {sample_id:?} needs a non-empty sample and a positive integer version"), + } +} + +/// The first reply a member posts under a notice version. Written once, never updated. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "heimdall.notice-acceptance", schema = "HeimdallNoticeAcceptance")] +pub struct NoticeAcceptance { + #[cultcache(key = 0)] + pub member_id: String, + #[cultcache(key = 1)] + pub experiment_id: String, + #[cultcache(key = 2)] + pub notice_version: u32, + #[cultcache(key = 3)] + pub scope: Scope, + /// `origin_server_ts` of the first reply, unix milliseconds UTC. + #[cultcache(key = 4)] + pub accepted_at: i64, + #[cultcache(key = 5)] + pub event_id: String, +} + +impl NoticeAcceptance { + /// `(experiment_id, member_id, notice_version)`. + pub fn cache_key(experiment_id: &str, member_id: &str, notice_version: u32) -> Result { + key(&[experiment_id, member_id, ¬ice_version.to_string()]) + } +} + +/// One answer. `notice_version` and `scope` are optional in the typed record only so that a stored +/// answer lacking them can be represented and excluded; the doc's table makes them required. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "heimdall.answer", schema = "HeimdallAnswer")] +pub struct Answer { + #[cultcache(key = 0)] + pub sample_id: String, + #[cultcache(key = 1)] + pub experiment_id: String, + #[cultcache(key = 2)] + pub member_id: String, + #[cultcache(key = 3, default)] + pub notice_version: Option, + #[cultcache(key = 4, default)] + pub scope: Option, + /// Absent once exited. + #[cultcache(key = 5, default)] + pub text: Option, + /// One-way: set by redaction, never cleared. + #[cultcache(key = 6, default)] + pub exited: bool, + /// `origin_server_ts` of the reply, unix milliseconds UTC. + #[cultcache(key = 7)] + pub received_at: i64, + #[cultcache(key = 8)] + pub event_id: String, + /// The tombstone's salted hash of `text`, present only once exited. Not in the doc's table: see README. + #[cultcache(key = 9, default)] + pub text_hash: Option>, +} + +impl Answer { + /// `(experiment_id, sample_id, member_id)`; refuses a sample id without a positive version. + pub fn cache_key(experiment_id: &str, sample_id: &str, member_id: &str) -> Result { + check_sample_id(sample_id)?; + key(&[experiment_id, sample_id, member_id]) + } + + /// Redaction is exit: text erased, `exited` set, a hash-only tombstone kept. The hash is computed + /// by the caller; this crate never sees it salted or unsalted. + pub fn redacted(&self, text_hash: Vec) -> Answer { + Answer { text: None, exited: true, text_hash: Some(text_hash), ..self.clone() } + } + + /// Training rule: scope `train`, an acceptance for this notice version made no later than the + /// answer, and not exited. No recorded scope, version or acceptance means excluded. + pub fn is_training_data(&self, acceptance: Option<&NoticeAcceptance>) -> bool { + let (Some(Scope::Train), Some(version), Some(acc)) = (self.scope, self.notice_version, acceptance) else { + return false; + }; + !self.exited + && acc.notice_version == version + && acc.experiment_id == self.experiment_id + && acc.member_id == self.member_id + && acc.scope == Scope::Train + && acc.accepted_at <= self.received_at + } +} + +/// Newcomers only: the terms version recorded by the `m.login.terms` stage. Members who joined +/// before it, and Minds, have no record, and the collector never reads it or gates on it. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "heimdall.member-terms", schema = "HeimdallMemberTerms")] +pub struct MemberTerms { + #[cultcache(key = 0)] + pub member_id: String, + #[cultcache(key = 1)] + pub terms_version: String, + /// Unix milliseconds UTC. + #[cultcache(key = 2)] + pub terms_accepted_at: i64, +} + +impl MemberTerms { + /// `member_id`. + pub fn cache_key(member_id: &str) -> Result { + key(&[member_id]) + } +} diff --git a/packages/heimdall-consent/tests/golden.rs b/packages/heimdall-consent/tests/golden.rs new file mode 100644 index 00000000..6154545f --- /dev/null +++ b/packages/heimdall-consent/tests/golden.rs @@ -0,0 +1,200 @@ +use cultcache_rs::{CultCache, SingleFileMessagePackBackingStore}; +use heimdall_consent::*; + +const VECTORS: &str = include_str!("vectors/consent_golden.csv"); + +fn new_cache(dir: &tempfile::TempDir) -> CultCache { + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + let _ = cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(dir.path().join("consent.cc"))); + cache.pull_all_backing_stores().unwrap(); + cache +} + +fn scope(s: &str) -> Option { + match s { + "" => None, + "show" => Some(Scope::Show), + "study" => Some(Scope::Study), + "train" => Some(Scope::Train), + other => panic!("unknown scope {other}"), + } +} + +fn hex(s: &str) -> Vec { + (0..s.len()).step_by(2).map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap()).collect() +} + +fn put_answer(cache: &mut CultCache, a: &Answer) { + cache.put(Answer::cache_key(&a.experiment_id, &a.sample_id, &a.member_id).unwrap(), a).unwrap(); +} + +fn training_events(cache: &CultCache) -> Vec { + let mut ids: Vec = cache + .get_all::() + .unwrap() + .iter() + .filter(|a| { + let acc = a.notice_version.and_then(|v| { + cache.get::(&NoticeAcceptance::cache_key(&a.experiment_id, &a.member_id, v).unwrap()).unwrap() + }); + a.is_training_data(acc.as_ref()) + }) + .map(|a| a.event_id.clone()) + .collect(); + ids.sort(); + ids +} + +#[test] +fn golden_vectors() { + let mut cases: Vec<(&str, Vec>)> = Vec::new(); + for line in VECTORS.lines().filter(|l| !l.starts_with('#') && !l.trim().is_empty()) { + let f: Vec<&str> = line.split(',').collect(); + assert_eq!(f.len(), 10, "row `{line}`"); + match cases.last_mut() { + Some((name, rows)) if *name == f[0] => rows.push(f), + _ => cases.push((f[0], vec![f])), + } + } + assert_eq!(cases.len(), 4, "fixture shrank"); + for (name, rows) in cases { + let dir = tempfile::tempdir().unwrap(); + let mut cache = new_cache(&dir); + for f in rows { + let (op, member, room, nv, sc, sample, payload, ts, event) = (f[1], f[2], f[3], f[4], f[5], f[6], f[7], f[8], f[9]); + let ctx = format!("case {name} op {op} event {event}"); + let ts: i64 = ts.parse().unwrap_or(0); + let nv: Option = nv.parse().ok(); + match op { + "reply" | "answer_raw" => { + if op == "reply" { + if let (Some(v), Some(s)) = (nv, scope(sc)) { + let k = NoticeAcceptance::cache_key(room, member, v).unwrap(); + if cache.get::(&k).unwrap().is_none() { + cache + .put(k, &NoticeAcceptance { + member_id: member.into(), + experiment_id: room.into(), + notice_version: v, + scope: s, + accepted_at: ts, + event_id: event.into(), + }) + .unwrap(); + } + } + } + put_answer(&mut cache, &Answer { + sample_id: sample.into(), + experiment_id: room.into(), + member_id: member.into(), + notice_version: nv, + scope: scope(sc), + text: Some(payload.into()), + exited: false, + received_at: ts, + event_id: event.into(), + text_hash: None, + }); + } + "redact" => { + let k = Answer::cache_key(room, sample, member).unwrap(); + let a = cache.get_required::(&k).unwrap(); + assert_eq!(a.event_id, event, "{ctx}"); + put_answer(&mut cache, &a.redacted(hex(payload))); + } + "terms" => { + cache + .put(MemberTerms::cache_key(member).unwrap(), &MemberTerms { + member_id: member.into(), + terms_version: payload.into(), + terms_accepted_at: ts, + }) + .unwrap(); + } + "expect_counts" => { + let want = format!( + "answers={};acceptances={}", + cache.get_all::().unwrap().len(), + cache.get_all::().unwrap().len() + ); + assert_eq!(want, payload, "{ctx}"); + } + "expect_train" => assert_eq!(training_events(&cache).join(";"), payload, "{ctx}"), + "expect_exited" => { + let a = cache.get_required::(&Answer::cache_key(room, sample, member).unwrap()).unwrap(); + assert!(a.exited && a.text.is_none(), "{ctx}"); + assert_eq!(a.text_hash, Some(hex(payload)), "{ctx}"); + assert_eq!((a.event_id.as_str(), a.received_at, a.sample_id.as_str()), (event, ts, sample), "{ctx}"); + } + "expect_acceptance" => { + let a = cache + .get_required::(&NoticeAcceptance::cache_key(room, member, nv.unwrap()).unwrap()) + .unwrap(); + assert_eq!((a.notice_version, Some(a.scope), a.accepted_at), (nv.unwrap(), scope(sc), ts), "{ctx}"); + assert_eq!(a.event_id, payload, "{ctx}"); + } + other => panic!("unknown op {other}"), + } + } + } +} + +#[test] +fn records_survive_a_reopen() { + let dir = tempfile::tempdir().unwrap(); + let mut cache = new_cache(&dir); + let a = Answer { + sample_id: "s@1".into(), + experiment_id: "r".into(), + member_id: "m".into(), + notice_version: Some(2), + scope: Some(Scope::Train), + text: None, + exited: true, + received_at: 5, + event_id: "e".into(), + text_hash: Some(vec![1, 2]), + }; + put_answer(&mut cache, &a); + let reopened = new_cache(&dir); + assert_eq!(reopened.get_required::(&Answer::cache_key("r", "s@1", "m").unwrap()).unwrap(), a); +} + +#[test] +fn key_refuses_unversioned_sample_ids() { + for bad in ["s", "s@", "s@0", "s@x", "@1", "s@-1", "s@+1"] { + assert!(Answer::cache_key("r", bad, "m").is_err(), "{bad}"); + } + assert!(Answer::cache_key("r", "s@1", "m").unwrap() != Answer::cache_key("r", "s@2", "m").unwrap()); +} + +#[test] +fn acceptance_after_the_answer_does_not_cover_it() { + let a = Answer { + sample_id: "s@1".into(), + experiment_id: "r".into(), + member_id: "m".into(), + notice_version: Some(1), + scope: Some(Scope::Train), + text: Some("t".into()), + exited: false, + received_at: 100, + event_id: "e".into(), + text_hash: None, + }; + let acc = |at| NoticeAcceptance { + member_id: "m".into(), + experiment_id: "r".into(), + notice_version: 1, + scope: Scope::Train, + accepted_at: at, + event_id: "e".into(), + }; + assert!(a.is_training_data(Some(&acc(100)))); + assert!(!a.is_training_data(Some(&acc(101)))); + assert!(!a.is_training_data(None)); +} diff --git a/packages/heimdall-consent/tests/vectors/consent_golden.csv b/packages/heimdall-consent/tests/vectors/consent_golden.csv new file mode 100644 index 00000000..c966351b --- /dev/null +++ b/packages/heimdall-consent/tests/vectors/consent_golden.csv @@ -0,0 +1,33 @@ +# case,op,member,experiment,notice_version,scope,sample_id,payload,ts,event_id +# Ops run in order against a real CultCache; each case starts empty. Timestamps are unix ms. +# reply: first reply under a (room, member, notice_version) writes an acceptance, then the answer replaces any earlier one at (room, sample_id, member); payload is the text. +# answer_raw: stores the answer with no acceptance written. redact: replaces the answer whose event_id matches with its tombstone; payload is the hash as hex. +# terms: member field terms_version (payload) and terms_accepted_at (ts). expect_*: payload is what must hold. +# Expected values are written by hand from docs/answer-collector-consent.md (Heimdall PR #6, head d836f29), not computed by the crate. +versioned-key,reply,m1,room1,1,train,s1@1,first,100,e1 +versioned-key,reply,m1,room1,1,train,s1@2,second,200,e2 +versioned-key,reply,m1,room1,1,train,s1@1,replaced,300,e3 +versioned-key,expect_counts,,,,,,answers=2;acceptances=1,, +versioned-key,expect_train,,,,,,e2;e3,, +no-scope,reply,m2,room1,1,,s1@1,unscoped,100,e4 +no-scope,reply,m2,room1,,,s1@2,unversioned,110,e5 +no-scope,answer_raw,m3,room1,1,train,s1@1,no-acceptance,120,e6 +no-scope,reply,m4,room1,1,study,s1@1,study-only,130,e7 +no-scope,reply,m5,room1,1,train,s1@1,trainable,140,e8 +no-scope,expect_counts,,,,,,answers=5;acceptances=2,, +no-scope,expect_train,,,,,,e8,, +redacted-reply,reply,m6,room1,1,train,s1@1,secret words,100,e9 +redacted-reply,reply,m7,room1,1,train,s1@1,kept,110,e10 +redacted-reply,redact,m6,room1,,,s1@1,a1b2c3d4,,e9 +redacted-reply,expect_exited,m6,room1,,,s1@1,a1b2c3d4,100,e9 +redacted-reply,expect_train,,,,,,e10,, +redacted-reply,expect_counts,,,,,,answers=2;acceptances=2,, +newcomer,terms,m8,,,,,2026-11,50, +newcomer,reply,m8,room1,3,train,s1@1,hello,100,e11 +newcomer,reply,m9,room1,3,train,s1@1,veteran,110,e12 +newcomer,reply,m8,room1,4,study,s1@1,again,200,e13 +newcomer,expect_acceptance,m8,room1,3,train,,e11,100, +newcomer,expect_acceptance,m8,room1,4,study,,e13,200, +newcomer,expect_acceptance,m9,room1,3,train,,e12,110, +newcomer,expect_counts,,,,,,answers=2;acceptances=3,, +newcomer,expect_train,,,,,,e12,, From 3a98eef12f8b735ac55fa376af4ca63f854f936e Mon Sep 17 00:00:00 2001 From: Libby Date: Fri, 6 Nov 2026 18:10:00 +0000 Subject: [PATCH 51/88] cultnet-ts: drop unset optional keys from createDocumentPutMessage MessagePack encodes undefined as nil, and the peer rejects a document whose sourceRuntimeId/sourceAgentId/sourceRole/tags are nil. Omit absent keys; test that the encoded message carries none. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultnet-ts/src/replication.ts | 9 +++++---- packages/cultnet-ts/test/cultnet.test.ts | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/packages/cultnet-ts/src/replication.ts b/packages/cultnet-ts/src/replication.ts index a332e4ba..9676d0f0 100644 --- a/packages/cultnet-ts/src/replication.ts +++ b/packages/cultnet-ts/src/replication.ts @@ -127,10 +127,11 @@ export class CultNetDocumentRegistry { recordKey: documentKey, storedAt: options.storedAt ?? new Date().toISOString(), payload: parsed, - sourceRuntimeId: options.sourceRuntimeId, - sourceAgentId: options.sourceAgentId, - sourceRole: options.sourceRole, - tags: options.tags, + // Absent optional fields must be omitted, not set to undefined: MessagePack encodes undefined as nil and the peer rejects it. + ...(options.sourceRuntimeId !== undefined && { sourceRuntimeId: options.sourceRuntimeId }), + ...(options.sourceAgentId !== undefined && { sourceAgentId: options.sourceAgentId }), + ...(options.sourceRole !== undefined && { sourceRole: options.sourceRole }), + ...(options.tags !== undefined && { tags: options.tags }), }, }; } diff --git a/packages/cultnet-ts/test/cultnet.test.ts b/packages/cultnet-ts/test/cultnet.test.ts index 2cc5b080..1bcfca0d 100644 --- a/packages/cultnet-ts/test/cultnet.test.ts +++ b/packages/cultnet-ts/test/cultnet.test.ts @@ -3059,3 +3059,22 @@ test("RUDP operation service admits a restarted client on the same address", asy await server.close(); } }); + +// MessagePack encodes undefined as nil, and a peer rejects a document whose optional source fields are nil. +test("document put message omits unset optional keys so the encoded wire has no nil", () => { + const definition = defineDocumentType({ type: "putnil.doc", schema: z.object({ body: z.string() }) }); + const binding = defineCultNetDocumentBinding({ definition }); + const registry = new CultNetDocumentRegistry([binding]); + + const bare = registry.createDocumentPutMessage(binding, "m", "k", { body: "x" }); + assert.deepEqual(Object.keys(bare.document).sort(), ["payload", "recordKey", "schemaId", "storedAt"]); + const decodedBare = decodeMsgpack(encode(bare)) as { document: Record }; + assert.ok(Object.values(decodedBare.document).every(value => value !== null), "no nil values on the wire"); + assert.ok(!("sourceRuntimeId" in decodedBare.document)); + + const stamped = registry.createDocumentPutMessage(binding, "m", "k", { body: "x" }, + { sourceRuntimeId: "r", sourceAgentId: "a", sourceRole: "role", tags: ["t"] }); + assert.deepEqual(decodeMsgpack(encode(stamped)), stamped); + assert.equal(stamped.document.sourceRuntimeId, "r"); + assert.deepEqual(stamped.document.tags, ["t"]); +}); From a9a2d51c6acaf3a6233341ee928046626e335b54 Mon Sep 17 00:00:00 2001 From: Libby Date: Fri, 6 Nov 2026 18:27:06 +0000 Subject: [PATCH 52/88] heimdall-consent: match Heimdall's rulings at doc commit 2364979 Typed records, validate(), redacted([u8;32]) and the golden vectors follow the nine rulings on Heimdall PR #6: u32 versions, 32-byte tombstone hash on the exited Answer, scopeless replies stored with both fields absent, strict training rule, fresh first reply per notice version. README cites doc lines. Co-Authored-By: Claude Sonnet 5.5 --- packages/heimdall-consent/README.md | 37 ++++-- packages/heimdall-consent/src/lib.rs | 98 +++++++++++++--- packages/heimdall-consent/tests/golden.rs | 111 +++++++++++++++++- .../tests/vectors/consent_golden.csv | 18 ++- 4 files changed, 221 insertions(+), 43 deletions(-) diff --git a/packages/heimdall-consent/README.md b/packages/heimdall-consent/README.md index 5e2c3284..670fc8b6 100644 --- a/packages/heimdall-consent/README.md +++ b/packages/heimdall-consent/README.md @@ -1,21 +1,32 @@ # heimdall-consent Typed CultCache records for Heimdall's answer collector: `NoticeAcceptance`, `Answer` and `MemberTerms`, -plus the two rules that need no store to state (`Answer::is_training_data`, `Answer::redacted`). Cut from -`docs/answer-collector-consent.md` in GameCult/Heimdall PR #6 (head d836f29); the doc's tables are the -contract. Specification only: no collector, tally or export lives here. +plus the rules that need no store to state (`Answer::is_training_data`, `Answer::redacted`, +`validate`). Cut from `docs/answer-collector-consent.md` in GameCult/Heimdall PR #6 at doc commit +`2364979`; the doc's tables and Representation section are the contract, and its rulings are on the PR +(comment 682). Specification only: no collector, tally, export or tombstone hashing lives here. Entry points: `src/lib.rs`; golden vectors `tests/vectors/consent_golden.csv`, run by `tests/golden.rs` (`cargo test`). Planned, not live: the collector, the tally, the training export. -## Choices the doc does not fix (questions for Heimdall) +## Where each ruling lands (doc lines at 2364979) -1. Timestamps are unix milliseconds (`i64`), matching `origin_server_ts`. The doc says "UTC timestamp". -2. `member_id`, `experiment_id`, `event_id` are strings; `notice_version` is `u32`; `terms_version` a string. The doc names only "Heimdall member id" and "Matrix event id". -3. Key text is the parts joined with U+001F; a part that is empty or contains it is refused. Matrix ids can hold most other separators. -4. The tombstone's salted `text` hash has no row in the answer table. It is carried as `Answer.text_hash` (bytes) on the exited record. Which hash function, whose salt, and whether the tombstone is a separate record is undecided; the crate never computes the hash. -5. `Answer.notice_version` and `scope` are optional in the typed record so an answer lacking them can exist and be excluded; the table makes both required. Should such a reply be stored at all, given "is not collected"? -6. A reply under a notice with no recorded scope writes no acceptance in the vector's collector; the doc says the first reply accepts "the notice version then pinned" and does not cover a missing pin. -7. `MemberTerms` is a separate record keyed by `member_id`. The doc puts the fields on the member; the member record lives in Heimdall, not CultLib. -8. Training rule item 2 also requires the acceptance's scope to be `train` and to match the answer's experiment and member; the doc requires only that an acceptance exist for the key with `accepted_at <= received_at`. -9. The "separate store or partition" for records wider than `show` is not modelled here. +| Ruling | Doc | Crate | +|---|---|---| +| Timestamps are Unix ms UTC, signed 64-bit | L93-94 | `i64` on `accepted_at`, `received_at`, `terms_accepted_at` | +| Ids are non-empty opaque strings, `event_id` verbatim; versions unsigned 32-bit and positive | L95-98 | `validate()`; `notice_version: u32`; `check_sample_id` parses `u32` | +| Key parts joined with U+001F; empty part or part containing it refused | L99-100 | `key()` | +| Tombstone is the exited `Answer` with `text_hash` (32 bytes); HMAC-SHA-256 keyed by a per-member salt held by the member-side service; the collector cannot compute it | L74, L127-147, L186-187 | `Answer.text_hash`; `redacted([u8; 32])` takes the service's output and refuses an answer with no `text` (L143-144); `validate()` requires 32 bytes and `exited` | +| A reply with no established notice version or scope is stored with both absent, writes no acceptance, is excluded from every use | L82-89 | `notice_version`/`scope` optional and absent together (`validate()`); vector case `no-scope` | +| A reply's version is the one pinned at its `origin_server_ts`; changed version needs a fresh first reply | L32-34, L47, L87-89 | acceptance key includes the version; `is_training_data` accepts only the acceptance for the answer's own version; vector case `version-change` | +| `MemberTerms` is the typed shape of the member record's `terms_version`/`terms_accepted_at`, not a second store | L51-56 | `MemberTerms` doc comment; the collector does not hold it | +| Training needs acceptance `scope = train` equal to the answer's `scope`, and `accepted_at <= received_at` | L111-119 | `is_training_data` | +| One store per scope plus a store for scopeless replies | L120-125 | not modelled; the collector's layout owns it | + +## Questions for Heimdall + +1. May `text` be an empty string? The doc says `text` is "string, or absent" (L72) and ids are non-empty + (L95), but says nothing of answer text. The crate accepts `Some("")` and `redacted` tombstones it. +2. The doc says a later answer to the same key replaces the earlier by "latest `received_at` wins" + (L78-80). Equal `received_at` for two different events is unspecified. The crate holds no replace + rule; the collector owns it. diff --git a/packages/heimdall-consent/src/lib.rs b/packages/heimdall-consent/src/lib.rs index 00dc3b48..3c98cd47 100644 --- a/packages/heimdall-consent/src/lib.rs +++ b/packages/heimdall-consent/src/lib.rs @@ -1,7 +1,8 @@ //! Typed CultCache records for Heimdall's answer collector, cut from `docs/answer-collector-consent.md` -//! in GameCult/Heimdall PR #6 (head d836f29): the notice acceptance record, the answer record, and -//! the terms version on the member. Field names follow the doc's tables. Where the doc is silent -//! the choice made here is listed in README.md as a question for Heimdall. +//! in GameCult/Heimdall PR #6 (doc commit 2364979): the notice acceptance record, the answer record, +//! and the typed shape of the terms pair on the member record. Field names and types follow the +//! doc's tables and its Representation section; README.md cites the doc lines. Anything the doc +//! still leaves open is listed there as a question for Heimdall. use anyhow::{Result, bail}; use cultcache_rs::DatabaseEntry; @@ -17,7 +18,8 @@ pub enum Scope { Train, } -/// Separates key parts. Ids containing it are refused rather than escaped. +/// Separates key parts (Representation: parts joined with U+001F; an empty part or one containing it +/// is refused, not escaped). const SEP: char = '\u{1f}'; fn key(parts: &[&str]) -> Result { @@ -27,13 +29,20 @@ fn key(parts: &[&str]) -> Result { Ok(parts.join(&SEP.to_string())) } -/// `@`, version a positive integer. The version is part of the id, so it is part -/// of every answer key and no key can name a sample without one. -pub fn check_sample_id(sample_id: &str) -> Result<(&str, u64)> { +fn nonempty(name: &str, value: &str) -> Result<()> { + if value.is_empty() { + bail!("{name} must be a non-empty string"); + } + Ok(()) +} + +/// `@`, version a positive unsigned 32-bit integer. The version is part of the id, +/// so it is part of every answer key and no key can name a sample without one. +pub fn check_sample_id(sample_id: &str) -> Result<(&str, u32)> { let Some((sample, version)) = sample_id.rsplit_once('@') else { bail!("sample id {sample_id:?} has no @version"); }; - match version.parse::() { + match version.parse::() { Ok(v) if v > 0 && !sample.is_empty() && !version.starts_with('+') => Ok((sample, v)), _ => bail!("sample id {sample_id:?} needs a non-empty sample and a positive integer version"), } @@ -59,14 +68,27 @@ pub struct NoticeAcceptance { } impl NoticeAcceptance { + /// Representation: ids are non-empty strings, versions positive. + pub fn validate(&self) -> Result<()> { + nonempty("member_id", &self.member_id)?; + nonempty("experiment_id", &self.experiment_id)?; + nonempty("event_id", &self.event_id)?; + if self.notice_version == 0 { + bail!("notice_version must be positive"); + } + Ok(()) + } + /// `(experiment_id, member_id, notice_version)`. pub fn cache_key(experiment_id: &str, member_id: &str, notice_version: u32) -> Result { key(&[experiment_id, member_id, ¬ice_version.to_string()]) } } -/// One answer. `notice_version` and `scope` are optional in the typed record only so that a stored -/// answer lacking them can be represented and excluded; the doc's table makes them required. +/// One answer. `notice_version` and `scope` are absent together on a reply with no established +/// notice version or no matching acceptance: such a reply is stored, excluded from every use, and +/// writes no acceptance (Answer record, "A reply whose `notice_version` or `scope` is missing"). +/// Once exited the record is the tombstone: `text` absent, `text_hash` set. #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] #[cultcache(type = "heimdall.answer", schema = "HeimdallAnswer")] pub struct Answer { @@ -91,7 +113,8 @@ pub struct Answer { pub received_at: i64, #[cultcache(key = 8)] pub event_id: String, - /// The tombstone's salted hash of `text`, present only once exited. Not in the doc's table: see README. + /// The tombstone hash of `text`: HMAC-SHA-256, 32 raw bytes, computed by the member-side service + /// (it holds the salt). Present only once exited, and only if the collector held `text`. #[cultcache(key = 9, default)] pub text_hash: Option>, } @@ -103,14 +126,44 @@ impl Answer { key(&[experiment_id, sample_id, member_id]) } - /// Redaction is exit: text erased, `exited` set, a hash-only tombstone kept. The hash is computed - /// by the caller; this crate never sees it salted or unsalted. - pub fn redacted(&self, text_hash: Vec) -> Answer { - Answer { text: None, exited: true, text_hash: Some(text_hash), ..self.clone() } + /// Representation and the invariants the table implies. + pub fn validate(&self) -> Result<()> { + nonempty("member_id", &self.member_id)?; + nonempty("experiment_id", &self.experiment_id)?; + nonempty("event_id", &self.event_id)?; + check_sample_id(&self.sample_id)?; + if self.notice_version == Some(0) { + bail!("notice_version must be positive"); + } + if self.notice_version.is_some() != self.scope.is_some() { + bail!("notice_version and scope are present together or absent together"); + } + if self.exited && self.text.is_some() { + bail!("an exited answer holds no text"); + } + if let Some(hash) = &self.text_hash { + if !self.exited || hash.len() != 32 { + bail!("text_hash is 32 bytes and only on an exited answer"); + } + } + Ok(()) } - /// Training rule: scope `train`, an acceptance for this notice version made no later than the - /// answer, and not exited. No recorded scope, version or acceptance means excluded. + /// Redaction is exit: text erased, `exited` set, the tombstone hash kept. The hash comes from the + /// member-side service, which holds the salt; this crate never computes it. A reply the collector + /// never collected has no text and gets no tombstone, so an answer without `text` is refused. + pub fn redacted(&self, text_hash: [u8; 32]) -> Result { + if self.text.is_none() { + bail!("no text held for event {:?}: nothing to tombstone", self.event_id); + } + Ok(Answer { text: None, exited: true, text_hash: Some(text_hash.to_vec()), ..self.clone() }) + } + + /// Training rule (doc "Training rule"): scope `train`; an acceptance for this answer's + /// `(experiment_id, member_id, notice_version)` whose own scope is `train` and equals the answer's, + /// with `accepted_at <= received_at`; not exited. A changed notice version needs its own acceptance, + /// so an acceptance for another version never covers the answer. No recorded scope, version or + /// acceptance means excluded. pub fn is_training_data(&self, acceptance: Option<&NoticeAcceptance>) -> bool { let (Some(Scope::Train), Some(version), Some(acc)) = (self.scope, self.notice_version, acceptance) else { return false; @@ -120,12 +173,14 @@ impl Answer { && acc.experiment_id == self.experiment_id && acc.member_id == self.member_id && acc.scope == Scope::Train + && Some(acc.scope) == self.scope && acc.accepted_at <= self.received_at } } -/// Newcomers only: the terms version recorded by the `m.login.terms` stage. Members who joined -/// before it, and Minds, have no record, and the collector never reads it or gates on it. +/// The typed shape of the `terms_version` / `terms_accepted_at` pair on the Heimdall member record +/// (newcomers only, recorded by the `m.login.terms` stage). It is not a second store and the +/// collector does not hold it. Members who joined before the stage, and Minds, have no such pair. #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] #[cultcache(type = "heimdall.member-terms", schema = "HeimdallMemberTerms")] pub struct MemberTerms { @@ -139,6 +194,11 @@ pub struct MemberTerms { } impl MemberTerms { + pub fn validate(&self) -> Result<()> { + nonempty("member_id", &self.member_id)?; + nonempty("terms_version", &self.terms_version) + } + /// `member_id`. pub fn cache_key(member_id: &str) -> Result { key(&[member_id]) diff --git a/packages/heimdall-consent/tests/golden.rs b/packages/heimdall-consent/tests/golden.rs index 6154545f..88882842 100644 --- a/packages/heimdall-consent/tests/golden.rs +++ b/packages/heimdall-consent/tests/golden.rs @@ -59,7 +59,7 @@ fn golden_vectors() { _ => cases.push((f[0], vec![f])), } } - assert_eq!(cases.len(), 4, "fixture shrank"); + assert_eq!(cases.len(), 5, "fixture shrank"); for (name, rows) in cases { let dir = tempfile::tempdir().unwrap(); let mut cache = new_cache(&dir); @@ -87,7 +87,7 @@ fn golden_vectors() { } } } - put_answer(&mut cache, &Answer { + let answer = Answer { sample_id: sample.into(), experiment_id: room.into(), member_id: member.into(), @@ -98,13 +98,16 @@ fn golden_vectors() { received_at: ts, event_id: event.into(), text_hash: None, - }); + }; + answer.validate().unwrap(); + put_answer(&mut cache, &answer); } "redact" => { let k = Answer::cache_key(room, sample, member).unwrap(); let a = cache.get_required::(&k).unwrap(); assert_eq!(a.event_id, event, "{ctx}"); - put_answer(&mut cache, &a.redacted(hex(payload))); + let hash: [u8; 32] = hex(payload).try_into().expect("tombstone hash is 32 bytes"); + put_answer(&mut cache, &a.redacted(hash).unwrap()); } "terms" => { cache @@ -157,8 +160,9 @@ fn records_survive_a_reopen() { exited: true, received_at: 5, event_id: "e".into(), - text_hash: Some(vec![1, 2]), + text_hash: Some(vec![7; 32]), }; + a.validate().unwrap(); put_answer(&mut cache, &a); let reopened = new_cache(&dir); assert_eq!(reopened.get_required::(&Answer::cache_key("r", "s@1", "m").unwrap()).unwrap(), a); @@ -166,9 +170,10 @@ fn records_survive_a_reopen() { #[test] fn key_refuses_unversioned_sample_ids() { - for bad in ["s", "s@", "s@0", "s@x", "@1", "s@-1", "s@+1"] { + for bad in ["s", "s@", "s@0", "s@x", "@1", "s@-1", "s@+1", "s@4294967296"] { assert!(Answer::cache_key("r", bad, "m").is_err(), "{bad}"); } + assert!(Answer::cache_key("r", "s@4294967295", "m").is_ok()); assert!(Answer::cache_key("r", "s@1", "m").unwrap() != Answer::cache_key("r", "s@2", "m").unwrap()); } @@ -198,3 +203,97 @@ fn acceptance_after_the_answer_does_not_cover_it() { assert!(!a.is_training_data(Some(&acc(101)))); assert!(!a.is_training_data(None)); } + +fn answer() -> Answer { + Answer { + sample_id: "s@1".into(), + experiment_id: "r".into(), + member_id: "m".into(), + notice_version: Some(1), + scope: Some(Scope::Train), + text: Some("t".into()), + exited: false, + received_at: 100, + event_id: "e".into(), + text_hash: None, + } +} + +#[test] +fn acceptance_for_another_notice_version_never_covers_the_answer() { + let acc = NoticeAcceptance { + member_id: "m".into(), + experiment_id: "r".into(), + notice_version: 1, + scope: Scope::Train, + accepted_at: 1, + event_id: "e0".into(), + }; + let mut a = answer(); + assert!(a.is_training_data(Some(&acc))); + a.notice_version = Some(2); + assert!(!a.is_training_data(Some(&acc)), "a changed notice version needs a fresh first reply"); +} + +#[test] +fn training_needs_acceptance_scope_equal_to_the_answers_scope() { + let mut acc = NoticeAcceptance { + member_id: "m".into(), + experiment_id: "r".into(), + notice_version: 1, + scope: Scope::Study, + accepted_at: 1, + event_id: "e0".into(), + }; + assert!(!answer().is_training_data(Some(&acc))); + acc.scope = Scope::Train; + assert!(answer().is_training_data(Some(&acc))); +} + +#[test] +fn a_reply_never_collected_gets_no_tombstone() { + let mut a = answer(); + a.text = None; + assert!(a.redacted([0; 32]).is_err()); + let t = answer().redacted([9; 32]).unwrap(); + assert!(t.exited && t.text.is_none() && t.text_hash == Some(vec![9; 32])); + assert!(t.redacted([9; 32]).is_err(), "exit is one-way and a tombstone has no text to hash again"); +} + +#[test] +fn validate_enforces_the_representation_rules() { + assert!(answer().validate().is_ok()); + let cases: Vec<(&str, Box)> = vec![ + ("empty member", Box::new(|a| a.member_id.clear())), + ("empty event", Box::new(|a| a.event_id.clear())), + ("version zero", Box::new(|a| a.notice_version = Some(0))), + ("version without scope", Box::new(|a| a.scope = None)), + ("scope without version", Box::new(|a| a.notice_version = None)), + ("exited with text", Box::new(|a| a.exited = true)), + ("hash on a live answer", Box::new(|a| a.text_hash = Some(vec![0; 32]))), + ("short hash", Box::new(|a| { + a.exited = true; + a.text = None; + a.text_hash = Some(vec![0; 31]); + })), + ]; + for (name, break_it) in cases { + let mut a = answer(); + break_it(&mut a); + assert!(a.validate().is_err(), "{name}"); + } + let mut unscoped = answer(); + unscoped.notice_version = None; + unscoped.scope = None; + assert!(unscoped.validate().is_ok(), "a scopeless reply is stored"); + let acc = NoticeAcceptance { + member_id: "m".into(), + experiment_id: "r".into(), + notice_version: 0, + scope: Scope::Show, + accepted_at: 1, + event_id: "e".into(), + }; + assert!(acc.validate().is_err()); + assert!(MemberTerms { member_id: "m".into(), terms_version: String::new(), terms_accepted_at: 1 }.validate().is_err()); +} diff --git a/packages/heimdall-consent/tests/vectors/consent_golden.csv b/packages/heimdall-consent/tests/vectors/consent_golden.csv index c966351b..c1bfe2f9 100644 --- a/packages/heimdall-consent/tests/vectors/consent_golden.csv +++ b/packages/heimdall-consent/tests/vectors/consent_golden.csv @@ -1,15 +1,16 @@ # case,op,member,experiment,notice_version,scope,sample_id,payload,ts,event_id # Ops run in order against a real CultCache; each case starts empty. Timestamps are unix ms. # reply: first reply under a (room, member, notice_version) writes an acceptance, then the answer replaces any earlier one at (room, sample_id, member); payload is the text. -# answer_raw: stores the answer with no acceptance written. redact: replaces the answer whose event_id matches with its tombstone; payload is the hash as hex. +# answer_raw: stores the answer with no acceptance written. redact: replaces the answer whose event_id matches with its tombstone; payload is the 32-byte tombstone hash as 64 hex chars (the member-side service's output; the crate never computes it). +# A reply with no established notice version or scope is stored with both absent (doc L82-89, Answer record), writes no acceptance, and is excluded. answer_raw with a version but no acceptance models a lost or never-written acceptance: still excluded (doc L120). # terms: member field terms_version (payload) and terms_accepted_at (ts). expect_*: payload is what must hold. -# Expected values are written by hand from docs/answer-collector-consent.md (Heimdall PR #6, head d836f29), not computed by the crate. +# Expected values are written by hand from docs/answer-collector-consent.md (Heimdall PR #6, doc commit 2364979), not computed by the crate. versioned-key,reply,m1,room1,1,train,s1@1,first,100,e1 versioned-key,reply,m1,room1,1,train,s1@2,second,200,e2 versioned-key,reply,m1,room1,1,train,s1@1,replaced,300,e3 versioned-key,expect_counts,,,,,,answers=2;acceptances=1,, versioned-key,expect_train,,,,,,e2;e3,, -no-scope,reply,m2,room1,1,,s1@1,unscoped,100,e4 +no-scope,reply,m2,room1,,,s1@1,unscoped,100,e4 no-scope,reply,m2,room1,,,s1@2,unversioned,110,e5 no-scope,answer_raw,m3,room1,1,train,s1@1,no-acceptance,120,e6 no-scope,reply,m4,room1,1,study,s1@1,study-only,130,e7 @@ -18,8 +19,8 @@ no-scope,expect_counts,,,,,,answers=5;acceptances=2,, no-scope,expect_train,,,,,,e8,, redacted-reply,reply,m6,room1,1,train,s1@1,secret words,100,e9 redacted-reply,reply,m7,room1,1,train,s1@1,kept,110,e10 -redacted-reply,redact,m6,room1,,,s1@1,a1b2c3d4,,e9 -redacted-reply,expect_exited,m6,room1,,,s1@1,a1b2c3d4,100,e9 +redacted-reply,redact,m6,room1,,,s1@1,a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90,,e9 +redacted-reply,expect_exited,m6,room1,,,s1@1,a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90,100,e9 redacted-reply,expect_train,,,,,,e10,, redacted-reply,expect_counts,,,,,,answers=2;acceptances=2,, newcomer,terms,m8,,,,,2026-11,50, @@ -31,3 +32,10 @@ newcomer,expect_acceptance,m8,room1,4,study,,e13,200, newcomer,expect_acceptance,m9,room1,3,train,,e12,110, newcomer,expect_counts,,,,,,answers=2;acceptances=3,, newcomer,expect_train,,,,,,e12,, +version-change,reply,m10,room1,1,train,s1@1,under v1,100,e14 +version-change,answer_raw,m10,room1,2,train,s1@2,v2 without a fresh first reply,200,e15 +version-change,expect_counts,,,,,,answers=2;acceptances=1,, +version-change,expect_train,,,,,,e14,, +version-change,reply,m10,room1,2,train,s1@3,first reply under v2,300,e16 +version-change,expect_counts,,,,,,answers=3;acceptances=2,, +version-change,expect_train,,,,,,e14;e16,, From 5ab6a6181f673b16ba0510699e155472aeeb741e Mon Sep 17 00:00:00 2001 From: Libby Date: Fri, 6 Nov 2026 19:42:24 +0000 Subject: [PATCH 53/88] heimdall-consent: live answers need text; ties on received_at break by bytewise event_id Rulings: Heimdall PR #6 comment 759. Golden vectors for empty live text, empty revoked, and received_at ties in both input orders. Co-Authored-By: Claude Sonnet 5.5 --- packages/heimdall-consent/src/lib.rs | 13 +++- packages/heimdall-consent/tests/golden.rs | 64 ++++++++++++++++++- .../tests/vectors/consent_golden.csv | 25 ++++++++ 3 files changed, 99 insertions(+), 3 deletions(-) diff --git a/packages/heimdall-consent/src/lib.rs b/packages/heimdall-consent/src/lib.rs index 3c98cd47..ee6be615 100644 --- a/packages/heimdall-consent/src/lib.rs +++ b/packages/heimdall-consent/src/lib.rs @@ -108,7 +108,8 @@ pub struct Answer { /// One-way: set by redaction, never cleared. #[cultcache(key = 6, default)] pub exited: bool, - /// `origin_server_ts` of the reply, unix milliseconds UTC. + /// Assigned by the collector on receipt, unix milliseconds UTC (the reply's `origin_server_ts` + /// in practice); ties are broken by `supersedes`. #[cultcache(key = 7)] pub received_at: i64, #[cultcache(key = 8)] @@ -141,6 +142,9 @@ impl Answer { if self.exited && self.text.is_some() { bail!("an exited answer holds no text"); } + if !self.exited && self.text.as_deref().is_none_or(str::is_empty) { + bail!("a live answer needs non-empty text; empty text is valid only on a revoked answer"); + } if let Some(hash) = &self.text_hash { if !self.exited || hash.len() != 32 { bail!("text_hash is 32 bytes and only on an exited answer"); @@ -149,6 +153,13 @@ impl Answer { Ok(()) } + /// Replacement rule for two answers at one key: the later reply is the greater `(received_at, + /// event_id)`. `received_at` is assigned by the collector; a tie breaks by `event_id` compared + /// bytewise, the greater winning (Heimdall PR #6 comment 759). Insertion order never decides. + pub fn supersedes(&self, earlier: &Answer) -> bool { + (self.received_at, self.event_id.as_bytes()) > (earlier.received_at, earlier.event_id.as_bytes()) + } + /// Redaction is exit: text erased, `exited` set, the tombstone hash kept. The hash comes from the /// member-side service, which holds the salt; this crate never computes it. A reply the collector /// never collected has no text and gets no tombstone, so an answer without `text` is refused. diff --git a/packages/heimdall-consent/tests/golden.rs b/packages/heimdall-consent/tests/golden.rs index 88882842..23a9161d 100644 --- a/packages/heimdall-consent/tests/golden.rs +++ b/packages/heimdall-consent/tests/golden.rs @@ -31,6 +31,15 @@ fn put_answer(cache: &mut CultCache, a: &Answer) { cache.put(Answer::cache_key(&a.experiment_id, &a.sample_id, &a.member_id).unwrap(), a).unwrap(); } +/// The collector's replace rule: an incoming answer replaces the stored one only if it is the later reply. +fn put_if_later(cache: &mut CultCache, a: &Answer) { + let k = Answer::cache_key(&a.experiment_id, &a.sample_id, &a.member_id).unwrap(); + if cache.get::(&k).unwrap().is_some_and(|old| !a.supersedes(&old)) { + return; + } + cache.put(k, a).unwrap(); +} + fn training_events(cache: &CultCache) -> Vec { let mut ids: Vec = cache .get_all::() @@ -59,7 +68,7 @@ fn golden_vectors() { _ => cases.push((f[0], vec![f])), } } - assert_eq!(cases.len(), 5, "fixture shrank"); + assert_eq!(cases.len(), 9, "fixture shrank"); for (name, rows) in cases { let dir = tempfile::tempdir().unwrap(); let mut cache = new_cache(&dir); @@ -100,7 +109,42 @@ fn golden_vectors() { text_hash: None, }; answer.validate().unwrap(); - put_answer(&mut cache, &answer); + put_if_later(&mut cache, &answer); + } + "reply_rejected" => { + let answer = Answer { + sample_id: sample.into(), + experiment_id: room.into(), + member_id: member.into(), + notice_version: nv, + scope: scope(sc), + text: Some(payload.into()), + exited: false, + received_at: ts, + event_id: event.into(), + text_hash: None, + }; + assert!(answer.validate().is_err(), "{ctx}: must be refused"); + } + "answer_revoked" => { + let answer = Answer { + sample_id: sample.into(), + experiment_id: room.into(), + member_id: member.into(), + notice_version: nv, + scope: scope(sc), + text: None, + exited: true, + received_at: ts, + event_id: event.into(), + text_hash: if payload.is_empty() { None } else { Some(hex(payload)) }, + }; + answer.validate().unwrap(); + put_if_later(&mut cache, &answer); + } + "expect_winner" => { + let a = cache.get_required::(&Answer::cache_key(room, sample, member).unwrap()).unwrap(); + assert_eq!((a.event_id.as_str(), a.text.as_deref()), (event, Some(payload)), "{ctx}"); } "redact" => { let k = Answer::cache_key(room, sample, member).unwrap(); @@ -269,6 +313,8 @@ fn validate_enforces_the_representation_rules() { ("version zero", Box::new(|a| a.notice_version = Some(0))), ("version without scope", Box::new(|a| a.scope = None)), ("scope without version", Box::new(|a| a.notice_version = None)), + ("live without text", Box::new(|a| a.text = None)), + ("live with empty text", Box::new(|a| a.text = Some(String::new()))), ("exited with text", Box::new(|a| a.exited = true)), ("hash on a live answer", Box::new(|a| a.text_hash = Some(vec![0; 32]))), ("short hash", Box::new(|a| { @@ -282,6 +328,10 @@ fn validate_enforces_the_representation_rules() { break_it(&mut a); assert!(a.validate().is_err(), "{name}"); } + let mut revoked = answer(); + revoked.text = None; + revoked.exited = true; + assert!(revoked.validate().is_ok(), "empty text is valid on a revoked answer"); let mut unscoped = answer(); unscoped.notice_version = None; unscoped.scope = None; @@ -297,3 +347,13 @@ fn validate_enforces_the_representation_rules() { assert!(acc.validate().is_err()); assert!(MemberTerms { member_id: "m".into(), terms_version: String::new(), terms_accepted_at: 1 }.validate().is_err()); } + +#[test] +fn later_reply_is_greater_received_at_then_bytewise_event_id() { + let at = |t, e: &str| Answer { received_at: t, event_id: e.into(), ..answer() }; + assert!(at(2, "a").supersedes(&at(1, "z"))); + assert!(!at(1, "z").supersedes(&at(2, "a"))); + assert!(at(1, "a").supersedes(&at(1, "B")), "bytewise: 0x61 > 0x42"); + assert!(at(1, "z").supersedes(&at(1, "aa")), "bytewise, not by length"); + assert!(!at(1, "e").supersedes(&at(1, "e"))); +} diff --git a/packages/heimdall-consent/tests/vectors/consent_golden.csv b/packages/heimdall-consent/tests/vectors/consent_golden.csv index c1bfe2f9..751f4586 100644 --- a/packages/heimdall-consent/tests/vectors/consent_golden.csv +++ b/packages/heimdall-consent/tests/vectors/consent_golden.csv @@ -3,6 +3,8 @@ # reply: first reply under a (room, member, notice_version) writes an acceptance, then the answer replaces any earlier one at (room, sample_id, member); payload is the text. # answer_raw: stores the answer with no acceptance written. redact: replaces the answer whose event_id matches with its tombstone; payload is the 32-byte tombstone hash as 64 hex chars (the member-side service's output; the crate never computes it). # A reply with no established notice version or scope is stored with both absent (doc L82-89, Answer record), writes no acceptance, and is excluded. answer_raw with a version but no acceptance models a lost or never-written acceptance: still excluded (doc L120). +# reply_rejected: validate must refuse the answer (nothing stored). answer_revoked: stores an exited answer with no text; payload is the tombstone hash (empty means none). expect_winner: the stored answer's event_id and text must equal event and payload. +# Replacement at one key: the later reply is the greater (received_at, event_id), event_id compared bytewise; input order never decides (doc L78-80 and Heimdall PR #6 comment 759). # terms: member field terms_version (payload) and terms_accepted_at (ts). expect_*: payload is what must hold. # Expected values are written by hand from docs/answer-collector-consent.md (Heimdall PR #6, doc commit 2364979), not computed by the crate. versioned-key,reply,m1,room1,1,train,s1@1,first,100,e1 @@ -39,3 +41,26 @@ version-change,expect_train,,,,,,e14,, version-change,reply,m10,room1,2,train,s1@3,first reply under v2,300,e16 version-change,expect_counts,,,,,,answers=3;acceptances=2,, version-change,expect_train,,,,,,e14;e16,, +empty-live-rejected,reply_rejected,m11,room1,,,s1@1,,100,e17 +empty-live-rejected,expect_counts,,,,,,answers=0;acceptances=0,, +empty-live-rejected,reply,m11,room1,,,s1@1,now filled,110,e18 +empty-live-rejected,expect_winner,m11,room1,,,s1@1,now filled,,e18 +empty-revoked,answer_revoked,m12,room1,1,train,s1@1,,100,e19 +empty-revoked,answer_revoked,m13,room1,1,train,s1@1,a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90,100,e20 +empty-revoked,expect_counts,,,,,,answers=2;acceptances=0,, +empty-revoked,expect_train,,,,,,,, +tie-greater-first,reply,m14,room1,,,s1@1,upper-B,500,B +tie-greater-first,reply,m14,room1,,,s1@1,lower-a,500,a +tie-greater-first,reply,m15,room1,,,s1@1,short-z,500,z +tie-greater-first,reply,m15,room1,,,s1@1,long-aa,500,aa +tie-greater-first,reply,m16,room1,,,s1@1,newer-ts-first,600,a +tie-greater-first,reply,m16,room1,,,s1@1,older-ts-last,500,z +tie-greater-first,expect_winner,m14,room1,,,s1@1,lower-a,,a +tie-greater-first,expect_winner,m15,room1,,,s1@1,short-z,,z +tie-greater-first,expect_winner,m16,room1,,,s1@1,newer-ts-first,,a +tie-greater-last,reply,m14,room1,,,s1@1,lower-a,500,a +tie-greater-last,reply,m14,room1,,,s1@1,upper-B,500,B +tie-greater-last,reply,m15,room1,,,s1@1,long-aa,500,aa +tie-greater-last,reply,m15,room1,,,s1@1,short-z,500,z +tie-greater-last,expect_winner,m14,room1,,,s1@1,lower-a,,a +tie-greater-last,expect_winner,m15,room1,,,s1@1,short-z,,z From b6f61a3869d2aaca0058d016adf6c662372ca1ec Mon Sep 17 00:00:00 2001 From: Libby Date: Fri, 6 Nov 2026 19:42:38 +0000 Subject: [PATCH 54/88] cultnet error vectors: shared hex file for TS/Py, Rust pins it to the C# captures Co-Authored-By: Claude Sonnet 5.5 --- contracts/cultnet/interop/error-vectors.json | 134 +++++++++++++++++++ packages/cultnet-rs/tests/error_message.rs | 19 +++ 2 files changed, 153 insertions(+) create mode 100644 contracts/cultnet/interop/error-vectors.json diff --git a/contracts/cultnet/interop/error-vectors.json b/contracts/cultnet/interop/error-vectors.json new file mode 100644 index 00000000..05422f9e --- /dev/null +++ b/contracts/cultnet/interop/error-vectors.json @@ -0,0 +1,134 @@ +{ + "provenance": "Captured from MessagePack C# 3.1.7 (CultNetErrorMessage/CultNetErrorDetails, Options = Standard.WithSecurity(UntrustedData)); pinned byte-for-byte by packages/cultnet-rs/tests/error_message.rs, whose inline hex equals messagePackHex here (test error_vectors_file_matches_the_inline_csharp_captures). Rust encodes to these exact bytes, so C#-written and Rust-written bytes are one set. Every key is written on the wire, nil included.", + "vectors": [ + { + "name": "selection_invalid_schemas_null", + "messagePackHex": "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72D92473656C656374696F6E5F696E76616C69643A206D757374206E6F7420626520656D707479AB726F7574696E6748696E74C0A4636F6465B173656C656374696F6E5F696E76616C6964A764657461696C7384A56669656C64A7736368656D6173A576616C7565C0A461734F66C0A763757272656E74C0", + "message": { + "schemaVersion": "cultnet.error.v0", + "error": "selection_invalid: must not be empty", + "routingHint": null, + "code": "selection_invalid", + "details": { + "field": "schemas", + "value": null, + "asOf": null, + "current": null + } + } + }, + { + "name": "selection_invalid_field_value", + "messagePackHex": "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72D92273656C656374696F6E5F696E76616C69643A20626C616E6B206B657920656E747279AB726F7574696E6748696E74C0A4636F6465B173656C656374696F6E5F696E76616C6964A764657461696C7384A56669656C64A46B657973A576616C7565A3202020A461734F66C0A763757272656E74C0", + "message": { + "schemaVersion": "cultnet.error.v0", + "error": "selection_invalid: blank key entry", + "routingHint": null, + "code": "selection_invalid", + "details": { + "field": "keys", + "value": " ", + "asOf": null, + "current": null + } + } + }, + { + "name": "cursor_stale", + "messagePackHex": "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72BD637572736F725F7374616C653A20637572736F72206973207374616C65AB726F7574696E6748696E74C0A4636F6465AC637572736F725F7374616C65A764657461696C7384A56669656C64C0A576616C7565C0A461734F6601A763757272656E7402", + "message": { + "schemaVersion": "cultnet.error.v0", + "error": "cursor_stale: cursor is stale", + "routingHint": null, + "code": "cursor_stale", + "details": { + "field": null, + "value": null, + "asOf": 1, + "current": 2 + } + } + }, + { + "name": "cursor_invalid", + "messagePackHex": "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72D923637572736F725F696E76616C69643A2063616E6E6F7420706172736520637572736F72AB726F7574696E6748696E74C0A4636F6465AE637572736F725F696E76616C6964A764657461696C73C0", + "message": { + "schemaVersion": "cultnet.error.v0", + "error": "cursor_invalid: cannot parse cursor", + "routingHint": null, + "code": "cursor_invalid", + "details": null + } + }, + { + "name": "reference_outside_target", + "messagePackHex": "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72D9317265666572656E63655F6F7574736964655F7461726765743A206F757473696465206465636C6172656420746172676574AB726F7574696E6748696E74C0A4636F6465B87265666572656E63655F6F7574736964655F746172676574A764657461696C73C0", + "message": { + "schemaVersion": "cultnet.error.v0", + "error": "reference_outside_target: outside declared target", + "routingHint": null, + "code": "reference_outside_target", + "details": null + } + }, + { + "name": "variant_unsupported_subscription_ended", + "messagePackHex": "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72B876617269616E7420756E737570706F727465642068657265AB726F7574696E6748696E74C0A4636F6465B376617269616E745F756E737570706F72746564A764657461696C7384A56669656C64AE737562736372697074696F6E4964A576616C7565A57375622D31A461734F66C0A763757272656E74C0", + "message": { + "schemaVersion": "cultnet.error.v0", + "error": "variant unsupported here", + "routingHint": null, + "code": "variant_unsupported", + "details": { + "field": "subscriptionId", + "value": "sub-1", + "asOf": null, + "current": null + } + } + }, + { + "name": "variant_unsupported_no_subscription", + "messagePackHex": "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72B876617269616E7420756E737570706F727465642068657265AB726F7574696E6748696E74C0A4636F6465B376617269616E745F756E737570706F72746564A764657461696C73C0", + "message": { + "schemaVersion": "cultnet.error.v0", + "error": "variant unsupported here", + "routingHint": null, + "code": "variant_unsupported", + "details": null + } + }, + { + "name": "unowned_schema", + "messagePackHex": "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72D95C756E6F776E65645F736368656D613A204E6F207368617264206F776E7320736368656D612027736F6D652E736368656D612E763127206B65792027736F6D653A6B6579273B207468652077726974652077617320726566757365642EAB726F7574696E6748696E74C0A4636F6465AE756E6F776E65645F736368656D61A764657461696C7384A56669656C64A8736368656D614964A576616C7565AE736F6D652E736368656D612E7631A461734F66C0A763757272656E74C0", + "message": { + "schemaVersion": "cultnet.error.v0", + "error": "unowned_schema: No shard owns schema 'some.schema.v1' key 'some:key'; the write was refused.", + "routingHint": null, + "code": "unowned_schema", + "details": { + "field": "schemaId", + "value": "some.schema.v1", + "asOf": null, + "current": null + } + } + }, + { + "name": "write_not_authorized", + "messagePackHex": "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72D92177726974655F6E6F745F617574686F72697A65643A206E6F74206772616E746564AB726F7574696E6748696E74C0A4636F6465B477726974655F6E6F745F617574686F72697A6564A764657461696C7384A56669656C64A97265636F72644B6579A576616C7565A8736F6D653A6B6579A461734F66C0A763757272656E74C0", + "message": { + "schemaVersion": "cultnet.error.v0", + "error": "write_not_authorized: not granted", + "routingHint": null, + "code": "write_not_authorized", + "details": { + "field": "recordKey", + "value": "some:key", + "asOf": null, + "current": null + } + } + } + ] +} diff --git a/packages/cultnet-rs/tests/error_message.rs b/packages/cultnet-rs/tests/error_message.rs index 58732295..8b36d774 100644 --- a/packages/cultnet-rs/tests/error_message.rs +++ b/packages/cultnet-rs/tests/error_message.rs @@ -211,3 +211,22 @@ fn round_trip_is_stable() { assert_eq!(decoded, message, "{}: round trip mismatch", case.name); } } + +/// `contracts/cultnet/interop/error-vectors.json` is the file the TypeScript and Python runtimes +/// decode; it must stay byte-identical to the inline C# captures above, or those runtimes would be +/// pinned to bytes this crate no longer pins. +#[test] +fn error_vectors_file_matches_the_inline_csharp_captures() { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../contracts/cultnet/interop/error-vectors.json"); + let file: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&path).expect("read error-vectors.json")) + .expect("parse error-vectors.json"); + let vectors = file["vectors"].as_array().expect("vectors array"); + let all = cases(); + assert_eq!(vectors.len(), all.len(), "vector count differs from the inline cases"); + for (vector, case) in vectors.iter().zip(all.iter()) { + assert_eq!(vector["name"].as_str(), Some(case.name)); + assert_eq!(vector["messagePackHex"].as_str(), Some(case.csharp_hex), "{}", case.name); + } +} From 19f77ae3c272f7ae1f080921340928d079198773 Mon Sep 17 00:00:00 2001 From: Libby Date: Fri, 6 Nov 2026 19:42:44 +0000 Subject: [PATCH 55/88] heimdall-consent: README maps doc lines at Heimdall commit 4e99bfa Co-Authored-By: Claude Sonnet 5.5 --- packages/heimdall-consent/README.md | 33 +++++++++---------- packages/heimdall-consent/src/lib.rs | 9 +++-- .../tests/vectors/consent_golden.csv | 6 ++-- 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/packages/heimdall-consent/README.md b/packages/heimdall-consent/README.md index 670fc8b6..bac41338 100644 --- a/packages/heimdall-consent/README.md +++ b/packages/heimdall-consent/README.md @@ -3,30 +3,29 @@ Typed CultCache records for Heimdall's answer collector: `NoticeAcceptance`, `Answer` and `MemberTerms`, plus the rules that need no store to state (`Answer::is_training_data`, `Answer::redacted`, `validate`). Cut from `docs/answer-collector-consent.md` in GameCult/Heimdall PR #6 at doc commit -`2364979`; the doc's tables and Representation section are the contract, and its rulings are on the PR -(comment 682). Specification only: no collector, tally, export or tombstone hashing lives here. +`4e99bfa`; the doc's tables and Representation section are the contract, and its rulings are on the PR +(comments 682 and 759). Specification only: no collector, tally, export or tombstone hashing lives here. Entry points: `src/lib.rs`; golden vectors `tests/vectors/consent_golden.csv`, run by `tests/golden.rs` (`cargo test`). Planned, not live: the collector, the tally, the training export. -## Where each ruling lands (doc lines at 2364979) +## Where each ruling lands (doc lines at 4e99bfa) | Ruling | Doc | Crate | |---|---|---| -| Timestamps are Unix ms UTC, signed 64-bit | L93-94 | `i64` on `accepted_at`, `received_at`, `terms_accepted_at` | -| Ids are non-empty opaque strings, `event_id` verbatim; versions unsigned 32-bit and positive | L95-98 | `validate()`; `notice_version: u32`; `check_sample_id` parses `u32` | -| Key parts joined with U+001F; empty part or part containing it refused | L99-100 | `key()` | -| Tombstone is the exited `Answer` with `text_hash` (32 bytes); HMAC-SHA-256 keyed by a per-member salt held by the member-side service; the collector cannot compute it | L74, L127-147, L186-187 | `Answer.text_hash`; `redacted([u8; 32])` takes the service's output and refuses an answer with no `text` (L143-144); `validate()` requires 32 bytes and `exited` | -| A reply with no established notice version or scope is stored with both absent, writes no acceptance, is excluded from every use | L82-89 | `notice_version`/`scope` optional and absent together (`validate()`); vector case `no-scope` | -| A reply's version is the one pinned at its `origin_server_ts`; changed version needs a fresh first reply | L32-34, L47, L87-89 | acceptance key includes the version; `is_training_data` accepts only the acceptance for the answer's own version; vector case `version-change` | +| Timestamps are Unix ms UTC, signed 64-bit | L98-99; `received_at` is collector-assigned, L75 | `i64` on `accepted_at`, `received_at`, `terms_accepted_at` | +| Ids are non-empty opaque strings, `event_id` verbatim; versions unsigned 32-bit and positive | L100-103 | `validate()`; `notice_version: u32`; `check_sample_id` parses `u32` | +| Key parts joined with U+001F; empty part or part containing it refused | L104-105 | `key()` | +| Tombstone is the exited `Answer` with `text_hash` (32 bytes); HMAC-SHA-256 keyed by a per-member salt held by the member-side service; the collector cannot compute it | L74, L132-152, L186-193 | `Answer.text_hash`; `redacted([u8; 32])` takes the service's output and refuses an answer with no `text` (L148-149); `validate()` requires 32 bytes and `exited` | +| A reply with no established notice version or scope is stored with both absent, writes no acceptance, is excluded from every use | L87-94 | `notice_version`/`scope` optional and absent together (`validate()`); vector case `no-scope` | +| A reply's version is the one pinned at its `origin_server_ts`; changed version needs a fresh first reply | L32-34, L47, L92-94 | acceptance key includes the version; `is_training_data` accepts only the acceptance for the answer's own version; vector case `version-change` | | `MemberTerms` is the typed shape of the member record's `terms_version`/`terms_accepted_at`, not a second store | L51-56 | `MemberTerms` doc comment; the collector does not hold it | -| Training needs acceptance `scope = train` equal to the answer's `scope`, and `accepted_at <= received_at` | L111-119 | `is_training_data` | -| One store per scope plus a store for scopeless replies | L120-125 | not modelled; the collector's layout owns it | +| Training needs acceptance `scope = train` equal to the answer's `scope`, and `accepted_at <= received_at` | L116-123 | `is_training_data` | +| One store per scope plus a store for scopeless replies | L125-130 | not modelled; the collector's layout owns it | -## Questions for Heimdall +Rulings of comment 759 (doc L72, L78-85; no open questions remain): -1. May `text` be an empty string? The doc says `text` is "string, or absent" (L72) and ids are non-empty - (L95), but says nothing of answer text. The crate accepts `Some("")` and `redacted` tombstones it. -2. The doc says a later answer to the same key replaces the earlier by "latest `received_at` wins" - (L78-80). Equal `received_at` for two different events is unspecified. The crate holds no replace - rule; the collector owns it. +| Ruling | Doc | Crate | +|---|---|---| +| A live answer must have non-empty `text` and is rejected otherwise; empty text is valid only on an exited answer (text cleared, hash-only tombstone) | L72, L83-85 | `Answer::validate()` refuses a non-exited answer whose `text` is absent or empty; an exited answer holds no text. Vector cases `empty-live-rejected`, `empty-revoked` | +| `received_at` is assigned by the collector; a tie on it breaks by `event_id` compared bytewise, the greater wins as the later reply | L75, L78-81 | `Answer::supersedes`: `(received_at, event_id bytes)` order; the vector harness replaces a stored answer only when the incoming one supersedes it. Vector cases `tie-greater-first`, `tie-greater-last` | diff --git a/packages/heimdall-consent/src/lib.rs b/packages/heimdall-consent/src/lib.rs index ee6be615..0119371f 100644 --- a/packages/heimdall-consent/src/lib.rs +++ b/packages/heimdall-consent/src/lib.rs @@ -1,8 +1,7 @@ //! Typed CultCache records for Heimdall's answer collector, cut from `docs/answer-collector-consent.md` -//! in GameCult/Heimdall PR #6 (doc commit 2364979): the notice acceptance record, the answer record, +//! in GameCult/Heimdall PR #6 (doc commit 4e99bfa): the notice acceptance record, the answer record, //! and the typed shape of the terms pair on the member record. Field names and types follow the -//! doc's tables and its Representation section; README.md cites the doc lines. Anything the doc -//! still leaves open is listed there as a question for Heimdall. +//! doc's tables and its Representation section; README.md cites the doc lines. use anyhow::{Result, bail}; use cultcache_rs::DatabaseEntry; @@ -108,8 +107,8 @@ pub struct Answer { /// One-way: set by redaction, never cleared. #[cultcache(key = 6, default)] pub exited: bool, - /// Assigned by the collector on receipt, unix milliseconds UTC (the reply's `origin_server_ts` - /// in practice); ties are broken by `supersedes`. + /// Assigned by the collector on receipt, unix milliseconds UTC, not copied from the reply (doc L75); + /// ties are broken by `supersedes`. #[cultcache(key = 7)] pub received_at: i64, #[cultcache(key = 8)] diff --git a/packages/heimdall-consent/tests/vectors/consent_golden.csv b/packages/heimdall-consent/tests/vectors/consent_golden.csv index 751f4586..f11db809 100644 --- a/packages/heimdall-consent/tests/vectors/consent_golden.csv +++ b/packages/heimdall-consent/tests/vectors/consent_golden.csv @@ -2,11 +2,11 @@ # Ops run in order against a real CultCache; each case starts empty. Timestamps are unix ms. # reply: first reply under a (room, member, notice_version) writes an acceptance, then the answer replaces any earlier one at (room, sample_id, member); payload is the text. # answer_raw: stores the answer with no acceptance written. redact: replaces the answer whose event_id matches with its tombstone; payload is the 32-byte tombstone hash as 64 hex chars (the member-side service's output; the crate never computes it). -# A reply with no established notice version or scope is stored with both absent (doc L82-89, Answer record), writes no acceptance, and is excluded. answer_raw with a version but no acceptance models a lost or never-written acceptance: still excluded (doc L120). +# A reply with no established notice version or scope is stored with both absent (doc L87-94, Answer record), writes no acceptance, and is excluded. answer_raw with a version but no acceptance models a lost or never-written acceptance: still excluded (doc L119-122). # reply_rejected: validate must refuse the answer (nothing stored). answer_revoked: stores an exited answer with no text; payload is the tombstone hash (empty means none). expect_winner: the stored answer's event_id and text must equal event and payload. -# Replacement at one key: the later reply is the greater (received_at, event_id), event_id compared bytewise; input order never decides (doc L78-80 and Heimdall PR #6 comment 759). +# Replacement at one key: the later reply is the greater (received_at, event_id), event_id compared bytewise; input order never decides (doc L78-81 and Heimdall PR #6 comment 759). # terms: member field terms_version (payload) and terms_accepted_at (ts). expect_*: payload is what must hold. -# Expected values are written by hand from docs/answer-collector-consent.md (Heimdall PR #6, doc commit 2364979), not computed by the crate. +# Expected values are written by hand from docs/answer-collector-consent.md (Heimdall PR #6, doc commit 4e99bfa), not computed by the crate. versioned-key,reply,m1,room1,1,train,s1@1,first,100,e1 versioned-key,reply,m1,room1,1,train,s1@2,second,200,e2 versioned-key,reply,m1,room1,1,train,s1@1,replaced,300,e3 From c4fae30677a21f8fd715e954c19aa2617d9e6d12 Mon Sep 17 00:00:00 2001 From: Libby Date: Fri, 6 Nov 2026 19:44:36 +0000 Subject: [PATCH 56/88] cultcache-rs: name, index and global lookups from the C# reference #[cultcache(name)], #[cultcache(index[ = alias])] and struct-level #[cultcache(global)] in the DatabaseEntry derive; get_by_name, get_by_index and get_global on CultCache; a second record of a global type under another key is a typed CultCacheSecondGlobal refusal on every write path and on load. Co-Authored-By: Claude Sonnet 5.5 --- .../cultcache-rs-derive/src/lib.rs | 163 +++++++++++++++++ packages/cultcache-rs/src/lib.rs | 165 ++++++++++++++++++ 2 files changed, 328 insertions(+) diff --git a/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs b/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs index 65e41b92..fe4e57c6 100644 --- a/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs +++ b/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs @@ -22,6 +22,7 @@ fn database_entry_codec(input: DeriveInput) -> syn::Result = None; let mut schema_name: Option = None; + let mut global = false; for attribute in input .attrs @@ -37,6 +38,12 @@ fn database_entry_codec(input: DeriveInput) -> syn::Result syn::Result ::std::option::Option<::std::string::String> { + ::cultcache_rs::CultLookupValue::lookup_value(&self.#member) + } + } + }); + let index_members = slots.iter().filter_map(|slot| slot.index.as_ref().map(|alias| (alias, &slot.ident))); + let index_pushes = index_members.map(|(alias, member)| { + quote! { + if let ::std::option::Option::Some(value) = ::cultcache_rs::CultLookupValue::lookup_value(&self.#member) { + held.push((#alias, value)); + } + } + }); + let index_pushes: Vec<_> = index_pushes.collect(); + let indexes_fn = (!index_pushes.is_empty()).then(|| { + quote! { + fn cult_indexes(&self) -> ::std::vec::Vec<(&'static str, ::std::string::String)> { + let mut held = ::std::vec::Vec::new(); + #(#index_pushes)* + held + } + } + }); Ok(quote! { impl ::cultcache_rs::DatabaseEntry for #ident { const TYPE: &'static str = #entry_type; const SCHEMA_NAME: &'static str = #schema_name; + const GLOBAL: bool = #global; + #name_fn + #indexes_fn } #codec @@ -133,6 +169,8 @@ fn named_slots(data: Data, ident: &syn::Ident, owner: &str) -> syn::Result = None; let mut default = false; let mut bytes = false; + let mut name = false; + let mut index: Option = None; for attribute in field .attrs .iter() @@ -157,6 +195,22 @@ fn named_slots(data: Data, ident: &syn::Ident, owner: &str) -> syn::Result()?.value() + } else { + String::new() + }; + index = Some(if alias.trim().is_empty() { field_ident.to_string() } else { alias }); + Ok(()) } else { Err(meta.error("unsupported field cultcache attribute")) } @@ -180,8 +234,28 @@ fn named_slots(data: Data, ident: &syn::Ident, owner: &str) -> syn::Result, } fn union_codec(input: DeriveInput) -> syn::Result { @@ -842,4 +920,89 @@ mod tests { let input: DeriveInput = syn::parse_str(source).expect("the fixture parses"); named_slots(input.data, &input.ident, "CultSlots").expect("the fixture is accepted"); } + fn entry_error(source: &str) -> String { + let input: DeriveInput = syn::parse_str(source).expect("the fixture parses"); + database_entry_codec(input).expect_err("the fixture is refused").to_string() + } + + fn entry_tokens(source: &str) -> String { + let input: DeriveInput = syn::parse_str(source).expect("the fixture parses"); + database_entry_codec(input).expect("the fixture is accepted").to_string().replace(' ', "") + } + + #[test] + fn name_index_and_global_generate_the_lookup_surface() { + let tokens = entry_tokens( + r#"#[cultcache(type = "t", global)] struct S { + #[cultcache(key = 0, name)] label: String, + #[cultcache(key = 1, index)] code: String, + #[cultcache(key = 2, index = "sku")] stock: Option, + }"#, + ); + assert!(tokens.contains("constGLOBAL:bool=true"), "{tokens}"); + assert!(tokens.contains("fncult_name"), "{tokens}"); + assert!(tokens.contains("(\"code\",value)"), "{tokens}"); + assert!(tokens.contains("(\"sku\",value)"), "{tokens}"); + } + + #[test] + fn a_record_without_lookups_generates_none_of_them() { + let tokens = entry_tokens(r#"#[cultcache(type = "t")] struct S { #[cultcache(key = 0)] a: String }"#); + assert!(tokens.contains("constGLOBAL:bool=false"), "{tokens}"); + assert!(!tokens.contains("fncult_name") && !tokens.contains("fncult_indexes"), "{tokens}"); + } + + // ResolveIndexAlias: a blank alias is the member's name. + #[test] + fn a_blank_index_alias_is_the_member_name() { + let tokens = entry_tokens( + r#"#[cultcache(type = "t")] struct S { #[cultcache(key = 0, index = " ")] code: String }"#, + ); + assert!(tokens.contains("(\"code\",value)"), "{tokens}"); + } + + #[test] + fn two_names_are_refused_for_that_reason() { + let message = entry_error( + r#"#[cultcache(type = "t")] struct S { #[cultcache(key = 0, name)] a: String, #[cultcache(key = 1, name)] b: String }"#, + ); + assert!(message.contains("more than one member"), "{message}"); + } + + // D10: two members that declare one alias; a bare index is the member name, so it collides with an explicit alias of it. + #[test] + fn two_members_with_one_index_alias_are_refused_for_that_reason() { + let message = entry_error( + r#"#[cultcache(type = "t")] struct S { #[cultcache(key = 0, index = "k")] a: String, #[cultcache(key = 1, index = "k")] b: String }"#, + ); + assert!(message.contains("both declare index alias \"k\""), "{message}"); + let message = entry_error( + r#"#[cultcache(type = "t")] struct S { #[cultcache(key = 0, index)] a: String, #[cultcache(key = 1, index = "a")] b: String }"#, + ); + assert!(message.contains("both declare index alias \"a\""), "{message}"); + } + + #[test] + fn a_repeated_lookup_attribute_is_refused() { + let message = entry_error( + r#"#[cultcache(type = "t")] struct S { #[cultcache(key = 0, name, name)] a: String }"#, + ); + assert!(message.contains("repeated `name`"), "{message}"); + let message = entry_error( + r#"#[cultcache(type = "t")] struct S { #[cultcache(key = 0, index, index = "x")] a: String }"#, + ); + assert!(message.contains("repeated `index`"), "{message}"); + let message = entry_error(r#"#[cultcache(type = "t", global, global)] struct S { #[cultcache(key = 0)] a: String }"#); + assert!(message.contains("repeated `global`"), "{message}"); + } + + // Only a DatabaseEntry is looked up: a nested slot record carrying the attributes is refused as before. + #[test] + fn lookup_attributes_are_refused_on_a_slots_record() { + let message = slots_error(r#"struct S { #[cultcache(key = 0, name)] a: String }"#); + assert!(message.contains("unsupported field cultcache attribute"), "{message}"); + let message = slots_error(r#"struct S { #[cultcache(key = 0, index)] a: String }"#); + assert!(message.contains("unsupported field cultcache attribute"), "{message}"); + } + } diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs index b77f4971..4ceb43f2 100644 --- a/packages/cultcache-rs/src/lib.rs +++ b/packages/cultcache-rs/src/lib.rs @@ -162,8 +162,68 @@ pub use cultcache_rs_derive::DatabaseEntry; pub trait DatabaseEntry: Serialize + DeserializeOwned + Clone + Send + 'static { const TYPE: &'static str; const SCHEMA_NAME: &'static str = "DatabaseEntry"; + /// `#[cultcache(global)]`: the cache holds at most one record of this type, found by `get_global`. + const GLOBAL: bool = false; + /// The `#[cultcache(name)]` member's value, which `get_by_name` finds the record by. `None` is no name. + fn cult_name(&self) -> Option { + None + } + /// Each `#[cultcache(index)]` member as (alias, value), which `get_by_index` finds the record by. + fn cult_indexes(&self) -> Vec<(&'static str, String)> { + Vec::new() + } } +/// What a `#[cultcache(name)]` or `#[cultcache(index)]` member says it is called: its text, or `None` for a member +/// that holds no value (an `Option` that is `None`). The cache treats an empty name and a blank index value as none. +pub trait CultLookupValue { + fn lookup_value(&self) -> Option; +} + +impl CultLookupValue for String { + fn lookup_value(&self) -> Option { + Some(self.clone()) + } +} + +impl CultLookupValue for Option { + fn lookup_value(&self) -> Option { + self.as_ref().and_then(CultLookupValue::lookup_value) + } +} + +macro_rules! lookup_value_by_display { + ($($ty:ty),* $(,)?) => {$( + impl CultLookupValue for $ty { + fn lookup_value(&self) -> Option { + Some(self.to_string()) + } + } + )*}; +} + +lookup_value_by_display!(bool, char, i8, i16, i32, i64, i128, isize, u8, u16, u32, u64, u128, usize); + +/// A global type already holds a record under another key; a second would leave `get_global` with two answers. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CultCacheSecondGlobal { + pub schema_name: String, + pub existing_key: String, + pub key: String, +} + +impl std::fmt::Display for CultCacheSecondGlobal { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + formatter, + "{} is a global and already has record {}; {} would be a second.", + self.schema_name, self.existing_key, self.key + ) + } +} + +impl std::error::Error for CultCacheSecondGlobal {} + /// A type written as a positional slots array: what the `CultSlots` and `DatabaseEntry` /// derives implement, and what a `CultUnion` arm must carry. #[diagnostic::on_unimplemented( @@ -2655,6 +2715,8 @@ struct CultCacheStoreRegistration { pub struct CultCache { definitions: BTreeMap, + /// The registered types declared `#[cultcache(global)]`. + globals: BTreeSet, schema_name_definitions: BTreeMap, entries: BTreeMap<(String, String), CultCacheEnvelope>, stores: Vec, @@ -2664,6 +2726,7 @@ impl CultCache { pub fn new() -> Self { Self { definitions: BTreeMap::new(), + globals: BTreeSet::new(), schema_name_definitions: BTreeMap::new(), entries: BTreeMap::new(), stores: Vec::new(), @@ -2694,6 +2757,11 @@ impl CultCache { )); } self.definitions.insert(T::TYPE.to_string(), T::SCHEMA_NAME); + if T::GLOBAL { + self.globals.insert(T::TYPE.to_string()); + } else { + self.globals.remove(T::TYPE); + } self.schema_name_definitions .insert(T::SCHEMA_NAME.to_string(), T::TYPE.to_string()); Ok(()) @@ -2800,6 +2868,7 @@ impl CultCache { loaded.insert(entry_id(&entry), entry); } } + self.refuse_second_globals(&loaded)?; self.entries = loaded; Ok(()) } @@ -2883,6 +2952,65 @@ impl CultCache { Ok(values) } + /// The one record of a `#[cultcache(global)]` type, or `None` before it is written (and for a type that is + /// not a global). + pub fn get_global(&self) -> Result> { + self.require_entry_type::()?; + if !T::GLOBAL { + return Ok(None); + } + self.find_record::(|_| true) + } + + /// The record whose `#[cultcache(name)]` member is `name`. An empty name is no name, so it finds nothing. Records + /// of one type sharing a name find the one with the smallest key (ordinal), whatever order they were written or + /// loaded in; removing it finds the next. + pub fn get_by_name(&self, name: &str) -> Result> { + self.require_entry_type::()?; + if name.is_empty() { + return Ok(None); + } + self.find_record::(|document| document.cult_name().as_deref() == Some(name)) + } + + /// The record whose `#[cultcache(index)]` member `alias` holds `value`. A blank value is not indexed, so it finds + /// nothing. Records of one type sharing a value find the one with the smallest key (ordinal), whatever order they + /// were written or loaded in; removing it finds the next. + pub fn get_by_index(&self, alias: &str, value: &str) -> Result> { + self.require_entry_type::()?; + if value.trim().is_empty() { + return Ok(None); + } + self.find_record::(|document| { + document + .cult_indexes() + .iter() + .any(|(held_alias, held_value)| *held_alias == alias && held_value == value) + }) + } + + /// The first record of `T`, in key order, that `matches`. + fn find_record(&self, matches: impl Fn(&T) -> bool) -> Result> { + let held = self + .entries + .range((T::TYPE.to_string(), String::new())..) + .take_while(|((held_type, _), _)| held_type == T::TYPE); + for (_, entry) in held { + let document: T = decode_payload(&entry.payload).with_context(|| { + format!( + "failed to decode CultCache entry {:?} at key {:?} as {}", + T::TYPE, + entry.key, + T::SCHEMA_NAME + ) + })?; + if matches(&document) { + return Ok(Some(document)); + } + } + Ok(None) + } + pub fn soa(&self) -> Result> { let keyed_rows = self.get_all_with_keys::()?; let (keys, documents): (Vec<_>, Vec<_>) = keyed_rows.into_iter().unzip(); @@ -2900,6 +3028,7 @@ impl CultCache { pub fn put(&mut self, key: impl Into, value: &T) -> Result { let (entry, parsed) = self.prepare_entry(key, value)?; + self.refuse_second_global(&self.entries, &entry.r#type, &entry.key)?; if let Some(home) = self.home_index(T::TYPE)? { self.stores[home].store.push(&entry)?; } @@ -3011,6 +3140,7 @@ impl CultCache { for entry in &staged { next.insert(entry_id(entry), entry.clone()); } + self.refuse_second_globals(&next)?; if let Some(store_index) = store_index { // The home store receives its own records only: those it already // holds in this cache plus the staged ones. @@ -3029,6 +3159,7 @@ impl CultCache { pub fn put_envelope(&mut self, entry: CultCacheEnvelope) -> Result { let parsed = self.validate_envelope::(&entry)?; + self.refuse_second_global(&self.entries, &entry.r#type, &entry.key)?; if let Some(home) = self.home_index(T::TYPE)? { self.stores[home].store.push(&entry)?; } @@ -3061,6 +3192,7 @@ impl CultCache { )); } require_payload_depth(&entry)?; + self.refuse_second_global(&self.entries, &entry.r#type, &entry.key)?; if let Some(home) = self.home_index(&entry.r#type)? { self.stores[home].store.push(&entry)?; } @@ -3146,10 +3278,43 @@ impl CultCache { self.home_index(&entry.r#type)?; loaded.insert(entry_id(&entry), entry); } + self.refuse_second_globals(&loaded)?; self.entries = loaded; Ok(()) } + fn refuse_second_global( + &self, + entries: &BTreeMap<(String, String), CultCacheEnvelope>, + type_id: &str, + key: &str, + ) -> Result<()> { + if !self.globals.contains(type_id) { + return Ok(()); + } + let held_other = entries + .range((type_id.to_string(), String::new())..) + .take_while(|((held_type, _), _)| held_type == type_id) + .map(|((_, held_key), _)| held_key) + .find(|held_key| held_key.as_str() != key); + match held_other { + Some(existing_key) => Err(CultCacheSecondGlobal { + schema_name: self.definitions.get(type_id).copied().unwrap_or(type_id).to_string(), + existing_key: existing_key.clone(), + key: key.to_string(), + } + .into()), + None => Ok(()), + } + } + + fn refuse_second_globals(&self, entries: &BTreeMap<(String, String), CultCacheEnvelope>) -> Result<()> { + for (type_id, key) in entries.keys() { + self.refuse_second_global(entries, type_id, key)?; + } + Ok(()) + } + fn require_entry_type(&self) -> Result<()> { match self.definitions.get(T::TYPE) { Some(schema_name) if *schema_name == T::SCHEMA_NAME => Ok(()), From 2528fbc090870356597051fa9f5d38dd097c4eef Mon Sep 17 00:00:00 2001 From: Libby Date: Fri, 6 Nov 2026 19:44:36 +0000 Subject: [PATCH 57/88] cultcache-rs: lookups parity tests ported from the C# suite Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-rs/tests/lookups.rs | 306 +++++++++++++++++++++++++ 1 file changed, 306 insertions(+) create mode 100644 packages/cultcache-rs/tests/lookups.rs diff --git a/packages/cultcache-rs/tests/lookups.rs b/packages/cultcache-rs/tests/lookups.rs new file mode 100644 index 00000000..2c3c4f7c --- /dev/null +++ b/packages/cultcache-rs/tests/lookups.rs @@ -0,0 +1,306 @@ +//! Lookups parity with the C# reference: `GetByName`, `GetByIndex` and `GetGlobal` +//! (`src/GameCult.Caching/CultCache.cs`) and the one-global rule its commit enforces. + +use cultcache_rs::{ + CacheBackingStore, CultCache, CultCacheSecondGlobal, DatabaseEntry, SingleFileMessagePackBackingStore, +}; + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "item")] +struct Item { + #[cultcache(key = 0, name)] + name: String, + #[cultcache(key = 1, index)] + category: String, + #[cultcache(key = 2, index = "sku")] + code: Option, + #[cultcache(key = 3, default)] + value: u32, +} + +fn item(name: &str, category: &str, code: Option<&str>) -> Item { + Item { name: name.into(), category: category.into(), code: code.map(Into::into), value: 0 } +} + +/// Same names and values as `Item`, a different type: lookups never cross types. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "gadget")] +struct Gadget { + #[cultcache(key = 0, name)] + name: String, + #[cultcache(key = 1, index)] + category: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "settings", global)] +struct Settings { + #[cultcache(key = 0)] + value: String, +} + +fn settings(value: &str) -> Settings { + Settings { value: value.into() } +} + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "plain")] +struct Plain { + #[cultcache(key = 0)] + value: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "numbered")] +struct Numbered { + #[cultcache(key = 0, name)] + id: u32, + #[cultcache(key = 1, index)] + owner: u64, +} + +fn cache() -> CultCache { + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + cache +} + +fn reopened(path: &std::path::Path) -> CultCache { + let mut reopened = cache(); + reopened.add_generic_backing_store(SingleFileMessagePackBackingStore::new(path)).unwrap(); + reopened.pull_all_backing_stores().unwrap(); + reopened +} + +fn stored_cache(path: &std::path::Path) -> CultCache { + let mut stored = cache(); + stored.add_generic_backing_store(SingleFileMessagePackBackingStore::new(path)).unwrap(); + stored.pull_all_backing_stores().unwrap(); + stored +} + +// The C# FirstCall snippet: a record written to a .cc file is found by its [CultName] through a second cache. +#[test] +fn a_record_is_found_by_name_and_by_index_after_a_reopen() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("lookups.cc"); + let mut written = stored_cache(&path); + written.put("item:potion", &Item { value: 50, ..item("Potion", "Consumable", Some("P-1")) }).unwrap(); + + let reopened = reopened(&path); + let by_name = reopened.get_by_name::("Potion").unwrap().unwrap(); + assert_eq!(by_name.value, 50); + assert_eq!(reopened.get_by_index::("category", "Consumable").unwrap(), Some(by_name.clone())); + assert_eq!(reopened.get_by_index::("sku", "P-1").unwrap(), Some(by_name)); +} + +#[test] +fn a_missing_name_or_value_is_none_and_an_unregistered_type_is_refused() { + let mut cache = cache(); + cache.put("item:a", &item("A", "cat", Some("a-1"))).unwrap(); + + assert_eq!(cache.get_by_name::("B").unwrap(), None); + assert_eq!(cache.get_by_name::("a").unwrap(), None, "names compare ordinally"); + assert_eq!(cache.get_by_index::("category", "other").unwrap(), None); + assert_eq!(cache.get_by_index::("nosuchalias", "cat").unwrap(), None); + assert_eq!(cache.get_by_index::("code", "a-1").unwrap(), None, "the alias is sku, not the member name"); + + let bare = CultCache::new(); + assert!(bare.get_by_name::("A").is_err()); + assert!(bare.get_by_index::("category", "cat").is_err()); + assert!(bare.get_global::().is_err()); +} + +#[test] +fn an_index_without_an_alias_is_named_by_its_member() { + let mut cache = cache(); + cache.put("item:a", &item("A", "cat", None)).unwrap(); + assert!(cache.get_by_index::("category", "cat").unwrap().is_some()); +} + +// A blank name is no name; a blank index value is not indexed; a nil member holds no value. +#[test] +fn blank_and_nil_values_are_not_found() { + let mut cache = cache(); + cache.put("item:blank", &item("", " ", None)).unwrap(); + assert_eq!(cache.get_by_name::("").unwrap(), None); + assert_eq!(cache.get_by_index::("category", " ").unwrap(), None); + assert_eq!(cache.get_by_index::("category", "").unwrap(), None); + assert_eq!(cache.get_by_index::("sku", "").unwrap(), None); + + // Only emptiness voids a name: whitespace is a name, as in C# (Length > 0). + cache.put("item:space", &item(" ", "x", None)).unwrap(); + assert_eq!(cache.get_by_name::(" ").unwrap().unwrap().category, "x"); +} + +#[test] +fn lookups_never_cross_types() { + let mut cache = cache(); + cache.put("item:a", &item("same", "same", None)).unwrap(); + cache.put("gadget:a", &Gadget { name: "same".into(), category: "same".into() }).unwrap(); + + assert_eq!(cache.get_by_name::("same").unwrap().unwrap().name, "same"); + cache.delete::("item:a").unwrap(); + assert_eq!(cache.get_by_name::("same").unwrap(), None); + assert!(cache.get_by_name::("same").unwrap().is_some()); + assert!(cache.get_by_index::("category", "same").unwrap().is_some()); +} + +#[test] +fn a_name_and_index_follow_a_rewritten_record_and_leave_a_removed_one() { + let mut cache = cache(); + cache.put("item:a", &item("old", "c1", None)).unwrap(); + cache.put("item:a", &item("new", "c2", None)).unwrap(); + + assert_eq!(cache.get_by_name::("old").unwrap(), None, "a stale name is not found"); + assert_eq!(cache.get_by_index::("category", "c1").unwrap(), None, "a stale value is not found"); + assert!(cache.get_by_name::("new").unwrap().is_some()); + assert!(cache.get_by_index::("category", "c2").unwrap().is_some()); + + cache.delete::("item:a").unwrap(); + assert_eq!(cache.get_by_name::("new").unwrap(), None); + assert_eq!(cache.get_by_index::("category", "c2").unwrap(), None); +} + +#[test] +fn non_string_members_are_found_by_their_text() { + let mut cache = cache(); + let owner = 161_u64; + cache.put("numbered:1", &Numbered { id: 42, owner }).unwrap(); + assert!(cache.get_by_name::("42").unwrap().is_some()); + assert!(cache.get_by_index::("owner", "161").unwrap().is_some()); +} + +// VariantTests.RemovingTheHolderADuplicateValueFindsLeavesTheOtherFindable: records sharing a value find the one with +// the smallest key, whatever the write order, live and after a reopen; removing the holder finds the other. +#[test] +fn records_sharing_a_value_find_the_smallest_key_and_removing_it_finds_the_next() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("shared.cc"); + { + let mut cache = stored_cache(&path); + cache.put("item:p2", &item("p2", "x", None)).unwrap(); + cache.put("item:p1", &item("p1", "x", None)).unwrap(); + let found = |cache: &CultCache| cache.get_by_index::("category", "x").unwrap().map(|found| found.name); + assert_eq!(found(&cache).as_deref(), Some("p1"), "the smallest key, though written second"); + assert_eq!(reopened(&path).get_by_index::("category", "x").unwrap().unwrap().name, "p1"); + + cache.delete::("item:p1").unwrap(); + assert_eq!(found(&cache).as_deref(), Some("p2"), "removing the holder finds the other one"); + + cache.put("item:p1", &item("p1", "x", None)).unwrap(); + assert_eq!(found(&cache).as_deref(), Some("p1")); + cache.delete::("item:p2").unwrap(); + assert_eq!(found(&cache).as_deref(), Some("p1"), "removing the other changes nothing"); + cache.delete::("item:p1").unwrap(); + assert_eq!(found(&cache), None); + } + assert_eq!(reopened(&path).get_by_index::("category", "x").unwrap(), None); +} + +#[test] +fn records_sharing_a_name_find_the_smallest_key() { + let mut cache = cache(); + cache.put("item:b", &Item { value: 2, ..item("twin", "c", None) }).unwrap(); + cache.put("item:a", &Item { value: 1, ..item("twin", "c", None) }).unwrap(); + assert_eq!(cache.get_by_name::("twin").unwrap().unwrap().value, 1); + cache.delete::("item:a").unwrap(); + assert_eq!(cache.get_by_name::("twin").unwrap().unwrap().value, 2); +} + +// BackingStoreTests.HydratesPersistedGlobal: opening a store invents no global; a written one survives a reopen. +#[test] +fn a_global_is_none_until_written_and_survives_a_reopen() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("global.cc"); + { + let mut seed = stored_cache(&path); + assert_eq!(seed.get_global::().unwrap(), None, "opening a store invented a global"); + seed.put("global:settings", &settings("durable authority")).unwrap(); + assert_eq!(seed.get_global::().unwrap(), Some(settings("durable authority"))); + } + assert_eq!(reopened(&path).get_global::().unwrap(), Some(settings("durable authority"))); +} + +#[test] +fn a_type_that_is_not_a_global_has_no_global() { + let mut cache = cache(); + cache.put("plain:one", &Plain { value: "v".into() }).unwrap(); + assert_eq!(cache.get_global::().unwrap(), None); +} + +#[test] +fn a_second_global_under_another_key_is_refused_and_writes_nothing() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("second.cc"); + let mut cache = stored_cache(&path); + cache.put("g1", &settings("one")).unwrap(); + + let refusal = cache.put("g2", &settings("two")).unwrap_err(); + let second = refusal.downcast_ref::().expect("a typed refusal"); + assert_eq!((second.existing_key.as_str(), second.key.as_str()), ("g1", "g2")); + assert!(refusal.to_string().contains("is a global and already has record g1; g2 would be a second.")); + assert_eq!(cache.get_global::().unwrap(), Some(settings("one"))); + assert_eq!(reopened(&path).get_global::().unwrap(), Some(settings("one")), "nothing reached the store"); + + // The same key is the same record, not a second. + cache.put("g1", &settings("one again")).unwrap(); + assert_eq!(cache.get_global::().unwrap(), Some(settings("one again"))); +} + +// RetypeEvictionTests.EvictingAGlobalFreesItsSlot: removing the global frees the slot for another key. +#[test] +fn removing_a_global_frees_its_slot() { + let mut cache = cache(); + cache.put("g1", &settings("one")).unwrap(); + assert!(cache.put("g2", &settings("two")).is_err()); + cache.delete::("g1").unwrap(); + assert_eq!(cache.get_global::().unwrap(), None); + cache.put("g2", &settings("two")).unwrap(); + assert_eq!(cache.get_global::().unwrap(), Some(settings("two"))); +} + +#[test] +fn every_write_path_refuses_a_second_global() { + let mut cache = cache(); + cache.put("g1", &settings("one")).unwrap(); + let (second, _) = cache.prepare_entry("g2", &settings("two")).unwrap(); + + assert!(cache.put_envelope::(second.clone()).unwrap_err().is::()); + assert!(cache.put_raw_envelope(second.clone()).unwrap_err().is::()); + let (plain, _) = cache.prepare_entry("plain:1", &Plain { value: "v".into() }).unwrap(); + assert!(cache.put_prepared_batch(vec![plain, second]).unwrap_err().is::()); + assert_eq!(cache.get::("plain:1").unwrap(), None, "a refused batch lands none of it"); + assert_eq!(cache.get_global::().unwrap(), Some(settings("one"))); +} + +#[test] +fn a_batch_that_stages_two_globals_is_refused_whole() { + let mut cache = cache(); + let (one, _) = cache.prepare_entry("g1", &settings("one")).unwrap(); + let (two, _) = cache.prepare_entry("g2", &settings("two")).unwrap(); + assert!(cache.put_prepared_batch(vec![one, two]).unwrap_err().is::()); + assert_eq!(cache.get_global::().unwrap(), None); +} + +// StoreRoutingTests: a store holding two globals is refused on load rather than answering one of them. +#[test] +fn loading_a_store_that_holds_two_globals_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("two.cc"); + let source = cache(); + let mut store = SingleFileMessagePackBackingStore::new(&path); + for (key, value) in [("g1", "one"), ("g2", "two")] { + let (entry, _) = source.prepare_entry(key, &settings(value)).unwrap(); + store.push(&entry).unwrap(); + } + + let mut loading = cache(); + loading.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap(); + assert!(loading.pull_all_backing_stores().unwrap_err().is::()); + assert_eq!(loading.get_global::().unwrap(), None, "a refused load installs nothing"); +} From 03b31f1f68f06a8db79e5a6ac2816b4d947639e6 Mon Sep 17 00:00:00 2001 From: Libby Date: Fri, 6 Nov 2026 19:44:57 +0000 Subject: [PATCH 58/88] cultnet-ts: CultNetErrorMessage carries routingHint, code (closed union) and details; decode tests on C#/Rust bytes Co-Authored-By: Claude Sonnet 5.5 --- packages/cultnet-ts/src/contracts.ts | 33 ++++++++ packages/cultnet-ts/src/index.ts | 4 + .../cultnet-ts/test/cultnet-error.test.ts | 83 +++++++++++++++++++ 3 files changed, 120 insertions(+) create mode 100644 packages/cultnet-ts/test/cultnet-error.test.ts diff --git a/packages/cultnet-ts/src/contracts.ts b/packages/cultnet-ts/src/contracts.ts index 31188790..fc225627 100644 --- a/packages/cultnet-ts/src/contracts.ts +++ b/packages/cultnet-ts/src/contracts.ts @@ -165,9 +165,38 @@ export interface CultNetLoginSuccessMessage { session: string; } +/** The closed refusal-code set of cultnet.error.v0 (contracts/cultnet/cultnet.error.schema.json). */ +export const CULTNET_ERROR_CODES = [ + "selection_invalid", + "cursor_stale", + "cursor_invalid", + "reference_outside_target", + "variant_unsupported", + "unowned_schema", + "write_not_authorized", +] as const; + +export type CultNetErrorCode = (typeof CULTNET_ERROR_CODES)[number]; + +export interface CultNetErrorRoutingHint { + shard: Record | null; + reason: string; +} + +export interface CultNetErrorDetails { + field: string | null; + value: string | null; + asOf: number | null; + current: number | null; +} + +/** Every key is written on the wire, null included (the C# map formatter never omits a key). */ export interface CultNetErrorMessage { schemaVersion: "cultnet.error.v0"; error: string; + routingHint: CultNetErrorRoutingHint | null; + code: CultNetErrorCode | null; + details: CultNetErrorDetails | null; } export interface CultNetSampleChangeNameMessage { @@ -842,6 +871,10 @@ function parseGameCultNetworkingMessage(input: unknown): CultNetMessage { return { schemaVersion: "cultnet.error.v0", error: requireLegacyString(payload[0], "ErrorMessage.Error"), + // The legacy union ErrorMessage carries only Error; the R-N fields exist on cultnet.schema.v0 alone. + routingHint: null, + code: null, + details: null, }; case 5: return { diff --git a/packages/cultnet-ts/src/index.ts b/packages/cultnet-ts/src/index.ts index c321bd07..af53a437 100644 --- a/packages/cultnet-ts/src/index.ts +++ b/packages/cultnet-ts/src/index.ts @@ -13,7 +13,11 @@ export { type CultNetDatabaseChangeRawMessage, type CultNetRawDocumentRecord, type CultNetRawPayloadEncoding, + CULTNET_ERROR_CODES, + type CultNetErrorCode, + type CultNetErrorDetails, type CultNetErrorMessage, + type CultNetErrorRoutingHint, type CultNetHelloMessage, type CultNetLoginMessage, type CultNetLoginSuccessMessage, diff --git a/packages/cultnet-ts/test/cultnet-error.test.ts b/packages/cultnet-ts/test/cultnet-error.test.ts new file mode 100644 index 00000000..d185de8b --- /dev/null +++ b/packages/cultnet-ts/test/cultnet-error.test.ts @@ -0,0 +1,83 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { decode, encode } from "@msgpack/msgpack"; + +import { + CULTNET_ERROR_CODES, + cultNetSchemas, + encodeCultNetMessageForWire, + parseCultNetMessage, + type CultNetErrorMessage, +} from "../src"; + +// contracts/cultnet/interop/error-vectors.json: bytes captured from MessagePack C# 3.1.7 and pinned +// by cultnet-rs/tests/error_message.rs (Rust encodes to exactly these bytes). +const VECTORS = join(__dirname, "..", "..", "..", "..", "contracts", "cultnet", "interop", "error-vectors.json"); +const vectors = (JSON.parse(readFileSync(VECTORS, "utf8")) as { + vectors: { name: string; messagePackHex: string; message: unknown }[]; +}).vectors; + +const bytesOf = (hex: string): Uint8Array => Uint8Array.from(Buffer.from(hex, "hex")); +const hexOf = (bytes: Uint8Array): string => Buffer.from(bytes).toString("hex").toUpperCase(); +const wireOf = (message: unknown): Uint8Array => encode(encodeCultNetMessageForWire(message as CultNetErrorMessage)); + +test("the error vector file carries a case for every closed code", () => { + const codes = new Set(vectors.map((v) => (v.message as CultNetErrorMessage).code)); + for (const code of CULTNET_ERROR_CODES) { + assert.ok(codes.has(code), `no vector for ${code}`); + } +}); + +test("a null code with null routingHint and details is a valid error message (no captured bytes carry one)", () => { + const message = parseCultNetMessage({ + schemaVersion: "cultnet.error.v0", error: "boom", routingHint: null, code: null, details: null, + }); + assert.equal((message as CultNetErrorMessage).code, null); +}); + +test("the closed code union equals the schema's enum", () => { + const schemaCodes = ((cultNetSchemas.errorSchema.properties.code as { enum?: (string | null)[] }).enum ?? []).filter((c) => c !== null); + assert.deepEqual([...CULTNET_ERROR_CODES].sort(), [...schemaCodes].sort()); +}); + +test("C#/Rust-written error bytes decode into the typed message with every key", () => { + for (const vector of vectors) { + const parsed = parseCultNetMessage(decode(bytesOf(vector.messagePackHex))); + assert.deepEqual(parsed, vector.message, vector.name); + assert.deepEqual(Object.keys(parsed), ["schemaVersion", "error", "routingHint", "code", "details"], vector.name); + } +}); + +test("a parsed error message re-encodes to the same bytes", () => { + for (const vector of vectors) { + const parsed = parseCultNetMessage(decode(bytesOf(vector.messagePackHex))); + assert.equal(hexOf(wireOf(parsed)), vector.messagePackHex, vector.name); + } +}); + +test("dropping code or details breaks the round trip and the decode", () => { + const vector = vectors.find((v) => v.name === "cursor_stale")!; + for (const dropped of ["code", "details", "routingHint"]) { + const copy = { ...(decode(bytesOf(vector.messagePackHex)) as Record) }; + delete copy[dropped]; + assert.throws(() => parseCultNetMessage(copy), /cultnet\.error\.v0/, `missing ${dropped} must be refused`); + assert.notEqual(hexOf(encode(copy)), vector.messagePackHex); + } +}); + +test("a code outside the closed enum is refused, as is a malformed details object", () => { + const base = decode(bytesOf(vectors.find((v) => v.name === "cursor_stale")!.messagePackHex)) as Record; + assert.throws(() => parseCultNetMessage({ ...base, code: "unsupported_schema_version" }), /cultnet\.error\.v0/); + assert.throws(() => parseCultNetMessage({ ...base, code: 7 }), /cultnet\.error\.v0/); + assert.throws( + () => parseCultNetMessage({ ...base, details: { field: null, value: null, asOf: 1, current: 2, extra: 1 } }), + /cultnet\.error\.v0/, + ); + assert.throws( + () => parseCultNetMessage({ ...base, details: { field: null, value: null, asOf: -1, current: 2 } }), + /cultnet\.error\.v0/, + ); +}); From 3250ba49d375bb2bc37ec3200782c638474ed6b5 Mon Sep 17 00:00:00 2001 From: Libby Date: Fri, 6 Nov 2026 19:45:52 +0000 Subject: [PATCH 59/88] cultnet-py: error_message builder and closed-code/full-key validation in parse_message Co-Authored-By: Claude Sonnet 5.5 --- .../cultnet-py/src/cultnet_py/__init__.py | 4 + .../cultnet-py/src/cultnet_py/messages.py | 75 ++++++++++++++++ .../cultnet-py/tests/test_cultnet_error.py | 90 +++++++++++++++++++ 3 files changed, 169 insertions(+) create mode 100644 packages/cultnet-py/tests/test_cultnet_error.py diff --git a/packages/cultnet-py/src/cultnet_py/__init__.py b/packages/cultnet-py/src/cultnet_py/__init__.py index 3fc36683..bbcc94f2 100644 --- a/packages/cultnet-py/src/cultnet_py/__init__.py +++ b/packages/cultnet-py/src/cultnet_py/__init__.py @@ -38,6 +38,7 @@ from .cultmesh_contracts import ( ) from .framing import decode_frame, encode_frame, read_frame, write_frame from .messages import ( + CULTNET_ERROR_CODES, CultNetMessage, CultNetWitnessArtifactBundle, compute_simulation_claim_hash, @@ -47,6 +48,7 @@ from .messages import ( document_delete, document_put_raw, encode_witness_artifact_bundle_payload, + error_message, hello, login, login_success, @@ -165,6 +167,7 @@ __all__ = [ "simulation_fact_document", "verse_catalog_request", "verse_from_wire", + "CULTNET_ERROR_CODES", "CultNetMessage", "CultNetWitnessArtifactBundle", "CultNetDatabaseSubscription", @@ -247,6 +250,7 @@ __all__ = [ "encode_witness_artifact_bundle_payload", "encode_frame", "encode_rudp_packet", + "error_message", "hello", "INTEROP_WIRE_CONTRACT", "login", diff --git a/packages/cultnet-py/src/cultnet_py/messages.py b/packages/cultnet-py/src/cultnet_py/messages.py index 4fa8d336..a5c0887e 100644 --- a/packages/cultnet-py/src/cultnet_py/messages.py +++ b/packages/cultnet-py/src/cultnet_py/messages.py @@ -28,9 +28,61 @@ def parse_message(payload: bytes) -> CultNetMessage: raise ValueError("CultNet schema-v0 messages must declare schemaVersion") body = dict(decoded) del body["schemaVersion"] + if schema_version == "cultnet.error.v0": + _validate_error_body(body) return CultNetMessage(schema_version=schema_version, body=body) +# Closed refusal-code set of cultnet.error.v0 (contracts/cultnet/cultnet.error.schema.json). Adding a +# code is one coordinated change across every runtime; a peer refuses a code it does not know. +CULTNET_ERROR_CODES: tuple[str, ...] = ( + "selection_invalid", + "cursor_stale", + "cursor_invalid", + "reference_outside_target", + "variant_unsupported", + "unowned_schema", + "write_not_authorized", +) + +_ERROR_DETAILS_KEYS = ("field", "value", "asOf", "current") + + +def _is_uint(value: Any) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and value >= 0 + + +def _validate_error_body(body: dict[str, Any]) -> None: + """Refuse a cultnet.error.v0 body that is not the full-key shape the schema requires.""" + expected = ("error", "routingHint", "code", "details") + if set(body) != set(expected): + raise ValueError(f"cultnet.error.v0 must carry exactly {list(expected)}, got {sorted(body)}") + if not isinstance(body["error"], str) or not body["error"]: + raise ValueError("cultnet.error.v0 error must be a non-empty string") + code = body["code"] + if code is not None and (not isinstance(code, str) or code not in CULTNET_ERROR_CODES): + raise ValueError(f"cultnet.error.v0 code {code!r} is not a recognized refusal code") + hint = body["routingHint"] + if hint is not None: + if ( + not isinstance(hint, dict) + or set(hint) != {"shard", "reason"} + or not isinstance(hint["reason"], str) + or not (hint["shard"] is None or isinstance(hint["shard"], dict)) + ): + raise ValueError("cultnet.error.v0 routingHint must be null or {shard, reason}") + details = body["details"] + if details is not None: + if not isinstance(details, dict) or set(details) != set(_ERROR_DETAILS_KEYS): + raise ValueError(f"cultnet.error.v0 details must be null or carry exactly {list(_ERROR_DETAILS_KEYS)}") + for key in ("field", "value"): + if details[key] is not None and not isinstance(details[key], str): + raise ValueError(f"cultnet.error.v0 details.{key} must be a string or null") + for key in ("asOf", "current"): + if details[key] is not None and not _is_uint(details[key]): + raise ValueError(f"cultnet.error.v0 details.{key} must be a non-negative integer or null") + + def hello( *, runtime_id: str, @@ -99,6 +151,29 @@ def verify_session( ) +def error_message( + error: str, + *, + code: str | None = None, + details: dict[str, Any] | None = None, + routing_hint: dict[str, Any] | None = None, +) -> CultNetMessage: + """Build cultnet.error.v0 with every key written (nil included), as the C# map formatter does. + + ``details`` may name any of field/value/asOf/current; the rest are written as null. + """ + body = { + "error": error, + "routingHint": routing_hint, + "code": code, + "details": None if details is None else {key: details.get(key) for key in _ERROR_DETAILS_KEYS}, + } + if details is not None and set(details) - set(_ERROR_DETAILS_KEYS): + raise ValueError(f"cultnet.error.v0 details has unknown keys {sorted(set(details) - set(_ERROR_DETAILS_KEYS))}") + _validate_error_body(body) + return CultNetMessage("cultnet.error.v0", body) + + def login_success( *, nonce: str, diff --git a/packages/cultnet-py/tests/test_cultnet_error.py b/packages/cultnet-py/tests/test_cultnet_error.py new file mode 100644 index 00000000..07f92cd9 --- /dev/null +++ b/packages/cultnet-py/tests/test_cultnet_error.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import json +import unittest +from pathlib import Path + +import msgpack + +from cultnet_py import CULTNET_ERROR_CODES, error_message, parse_message + +# contracts/cultnet/interop/error-vectors.json: bytes captured from MessagePack C# 3.1.7 and pinned +# by cultnet-rs/tests/error_message.rs (Rust encodes to exactly these bytes). +_REPO = Path(__file__).resolve().parents[3] +_VECTORS = json.loads((_REPO / "contracts/cultnet/interop/error-vectors.json").read_text(encoding="utf-8"))["vectors"] +_SCHEMA = json.loads((_REPO / "contracts/cultnet/cultnet.error.schema.json").read_text(encoding="utf-8")) + + +class CultNetErrorMessageTests(unittest.TestCase): + def test_closed_code_set_equals_the_schema_enum(self) -> None: + schema_codes = [c for c in _SCHEMA["properties"]["code"]["enum"] if c is not None] + self.assertEqual(sorted(CULTNET_ERROR_CODES), sorted(schema_codes)) + + def test_vectors_cover_every_code(self) -> None: + self.assertEqual({v["message"]["code"] for v in _VECTORS}, set(CULTNET_ERROR_CODES)) + + def test_decodes_csharp_and_rust_written_bytes_with_every_key(self) -> None: + for vector in _VECTORS: + message = parse_message(bytes.fromhex(vector["messagePackHex"])) + self.assertEqual(message.to_wire(), vector["message"], vector["name"]) + self.assertEqual( + list(message.to_wire()), ["schemaVersion", "error", "routingHint", "code", "details"], vector["name"] + ) + + def test_reencode_is_byte_identical(self) -> None: + for vector in _VECTORS: + expected = bytes.fromhex(vector["messagePackHex"]) + self.assertEqual(parse_message(expected).to_bytes(), expected, vector["name"]) + + def test_builder_produces_the_csharp_bytes(self) -> None: + by_name = {v["name"]: bytes.fromhex(v["messagePackHex"]) for v in _VECTORS} + stale = error_message( + "cursor_stale: cursor is stale", code="cursor_stale", details={"asOf": 1, "current": 2} + ) + self.assertEqual(stale.to_bytes(), by_name["cursor_stale"]) + invalid = error_message("cursor_invalid: cannot parse cursor", code="cursor_invalid") + self.assertEqual(invalid.to_bytes(), by_name["cursor_invalid"]) + selection = error_message( + "selection_invalid: blank key entry", + code="selection_invalid", + details={"field": "keys", "value": " "}, + ) + self.assertEqual(selection.to_bytes(), by_name["selection_invalid_field_value"]) + + def test_builder_writes_nil_keys_and_accepts_a_null_code(self) -> None: + wire = error_message("boom").to_wire() + self.assertEqual(wire, {"schemaVersion": "cultnet.error.v0", "error": "boom", "routingHint": None, "code": None, "details": None}) + + def test_unknown_code_is_refused_by_builder_and_decoder(self) -> None: + with self.assertRaises(ValueError): + error_message("x", code="unsupported_schema_version") + bad = msgpack.packb( + {"schemaVersion": "cultnet.error.v0", "error": "x", "routingHint": None, "code": "nope", "details": None}, + use_bin_type=True, + ) + with self.assertRaises(ValueError): + parse_message(bad) + + def test_dropping_code_or_details_is_refused(self) -> None: + vector = next(v for v in _VECTORS if v["name"] == "cursor_stale") + for dropped in ("code", "details", "routingHint"): + wire = dict(vector["message"]) + del wire[dropped] + with self.assertRaises(ValueError, msg=dropped): + parse_message(msgpack.packb(wire, use_bin_type=True)) + + def test_malformed_details_are_refused(self) -> None: + base = dict(next(v for v in _VECTORS if v["name"] == "cursor_stale")["message"]) + for details in ( + {"field": None, "value": None, "asOf": 1}, + {"field": None, "value": None, "asOf": -1, "current": 2}, + {"field": None, "value": None, "asOf": 1, "current": 2, "extra": 1}, + ): + with self.assertRaises(ValueError, msg=str(details)): + parse_message(msgpack.packb({**base, "details": details}, use_bin_type=True)) + with self.assertRaises(ValueError): + error_message("x", code="cursor_stale", details={"asOf": 1, "bogus": 2}) + + +if __name__ == "__main__": + unittest.main() From 95d8be34a76d091b8a5add7f9294a21100dd6bf0 Mon Sep 17 00:00:00 2001 From: Libby Date: Fri, 6 Nov 2026 19:45:56 +0000 Subject: [PATCH 60/88] cultcache-rs: lookups cross-type test, changelog and README entries Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-rs/CHANGELOG.md | 1 + packages/cultcache-rs/README.md | 3 +++ packages/cultcache-rs/tests/lookups.rs | 17 +++++++++++++++++ 3 files changed, 21 insertions(+) diff --git a/packages/cultcache-rs/CHANGELOG.md b/packages/cultcache-rs/CHANGELOG.md index f15fcee2..c417d535 100644 --- a/packages/cultcache-rs/CHANGELOG.md +++ b/packages/cultcache-rs/CHANGELOG.md @@ -13,6 +13,7 @@ only in git history. ### Added +- Lookups, matching the C# `GetByName`, `GetByIndex` and `GetGlobal`. `#[cultcache(name)]` and `#[cultcache(index)]` / `#[cultcache(index = "alias")]` on `DatabaseEntry` fields and `#[cultcache(global)]` on the struct give `CultCache::get_by_name`, `get_by_index` and `get_global`. An empty name and a blank index value are not found; records of one type sharing a name or value find the one with the smallest key. A second record of a global type under another key is refused on every write path and on load with `CultCacheSecondGlobal`. Two `name` members, or two members with one index alias, are refused at compile time. A member's text comes from `CultLookupValue` (`String`, `Option` of one, integers, `bool`, `char`). - `SchemaConflictError`, the typed refusal of a write the catalog cannot describe: records of different types under one schema id, or an arrived schema and a registered one that share an id and disagree on the schema name. A whole-store write (`push_all`) refuses with it, names the schema id, the names involved and a record key, writes nothing and leaves the file as it is. A caller tells it apart with `error.downcast_ref::()`. ## [0.3.1] diff --git a/packages/cultcache-rs/README.md b/packages/cultcache-rs/README.md index afae7285..2c5c8027 100644 --- a/packages/cultcache-rs/README.md +++ b/packages/cultcache-rs/README.md @@ -174,6 +174,9 @@ without pretending Rust has runtime reflection hiding under the floorboards. - `get_required_envelope::` - `get_all::` - `get_all_with_keys::` +- `get_by_name::` +- `get_by_index::` +- `get_global::` - `soa::` - `put::` - `put_envelope::` diff --git a/packages/cultcache-rs/tests/lookups.rs b/packages/cultcache-rs/tests/lookups.rs index 2c3c4f7c..2467fbc6 100644 --- a/packages/cultcache-rs/tests/lookups.rs +++ b/packages/cultcache-rs/tests/lookups.rs @@ -32,6 +32,20 @@ struct Gadget { category: String, } +/// Sorts after `Item` and has its exact shape: a lookup that ran past the end of its own type would answer from it. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "zed")] +struct Zed { + #[cultcache(key = 0, name)] + name: String, + #[cultcache(key = 1, index)] + category: String, + #[cultcache(key = 2, index = "sku")] + code: Option, + #[cultcache(key = 3, default)] + value: u32, +} + #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] #[cultcache(type = "settings", global)] struct Settings { @@ -63,6 +77,7 @@ fn cache() -> CultCache { let mut cache = CultCache::new(); cache.register_entry_type::().unwrap(); cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); cache.register_entry_type::().unwrap(); cache.register_entry_type::().unwrap(); cache.register_entry_type::().unwrap(); @@ -142,10 +157,12 @@ fn lookups_never_cross_types() { let mut cache = cache(); cache.put("item:a", &item("same", "same", None)).unwrap(); cache.put("gadget:a", &Gadget { name: "same".into(), category: "same".into() }).unwrap(); + cache.put("zed:a", &Zed { name: "same".into(), category: "same".into(), code: None, value: 0 }).unwrap(); assert_eq!(cache.get_by_name::("same").unwrap().unwrap().name, "same"); cache.delete::("item:a").unwrap(); assert_eq!(cache.get_by_name::("same").unwrap(), None); + assert_eq!(cache.get_by_index::("category", "same").unwrap(), None); assert!(cache.get_by_name::("same").unwrap().is_some()); assert!(cache.get_by_index::("category", "same").unwrap().is_some()); } From 9548d51a7108dbfb960d13b8024f7097b15a5876 Mon Sep 17 00:00:00 2001 From: Libby Date: Fri, 6 Nov 2026 20:47:07 +0000 Subject: [PATCH 61/88] cultcache-py: msgpack is a hard dependency; drop the optional-import fallbacks pyproject (and cultnet-py) already declare msgpack as required; the code treated it as optional behind a cultcache-py[msgpack] extra that does not exist. Co-Authored-By: Claude Sonnet 5.5 --- .../src/cultcache_py/documents.py | 29 +++---------------- .../cultcache-py/src/cultcache_py/leaves.py | 5 +--- 2 files changed, 5 insertions(+), 29 deletions(-) diff --git a/packages/cultcache-py/src/cultcache_py/documents.py b/packages/cultcache-py/src/cultcache_py/documents.py index 91d09ab8..23849713 100644 --- a/packages/cultcache-py/src/cultcache_py/documents.py +++ b/packages/cultcache-py/src/cultcache_py/documents.py @@ -6,6 +6,9 @@ from collections.abc import Mapping, Sequence, Set from itertools import chain from typing import Any, Callable, Generic, Iterable, TypeVar +import msgpack +from msgpack.exceptions import StackError as _MsgpackStackError + from .backing_store import CultCacheSchemaCatalogEntry, CultCacheSchemaCatalogMember from .leaves import Float32, SlotCodec from .leaves import any as _any_leaf @@ -24,12 +27,7 @@ or skipped, the outermost one is depth 1; depth 256 is accepted and 257 refused, MAX_SLOT_KEY = 65535 """The largest slot key a record may define: the largest MessagePack array16 length.""" -try: - from msgpack.exceptions import StackError as _MsgpackStackError - - _STACK_FAULTS: tuple[type[BaseException], ...] = (_MsgpackStackError,) -except ModuleNotFoundError: - _STACK_FAULTS = () +_STACK_FAULTS: tuple[type[BaseException], ...] = (_MsgpackStackError,) _CONTAINERS = (Mapping, Sequence, Set) @@ -85,22 +83,10 @@ def _json_payload_decoder(payload: bytes) -> Any: return json.loads(payload.decode("utf-8")) -def _msgpack() -> Any: - try: - import msgpack # type: ignore - except ModuleNotFoundError as exc: - raise RuntimeError( - "MessagePack payload encoding requires the optional 'msgpack' dependency. " - "Install with: python -m pip install cultcache-py[msgpack]" - ) from exc - return msgpack - - def _pack_tree(tree: Any) -> bytes: """The bytes of the tree the leaves produced. `packb` writes everything but a `Float32`, which it refuses with a TypeError; only then does an iterative walk pack each array and map header and each value, a `Float32` through a single-float packer.""" - msgpack = _msgpack() try: return msgpack.packb(tree, use_bin_type=True) # type: ignore[no-any-return] except TypeError: @@ -125,13 +111,6 @@ def _pack_tree(tree: Any) -> bytes: def _messagepack_payload_decoder(payload: bytes) -> Any: - try: - import msgpack # type: ignore - except ModuleNotFoundError as exc: - raise RuntimeError( - "MessagePack payload decoding requires the optional 'msgpack' dependency. " - "Install with: python -m pip install cultcache-py[msgpack]" - ) from exc return msgpack.unpackb(payload, raw=False) diff --git a/packages/cultcache-py/src/cultcache_py/leaves.py b/packages/cultcache-py/src/cultcache_py/leaves.py index 2d76078e..71bc6204 100644 --- a/packages/cultcache-py/src/cultcache_py/leaves.py +++ b/packages/cultcache-py/src/cultcache_py/leaves.py @@ -16,10 +16,7 @@ from collections.abc import Mapping from dataclasses import dataclass from typing import Any, Callable, NoReturn -try: - from msgpack import ExtType as _ExtType -except ModuleNotFoundError: - _ExtType = () # type: ignore[assignment,misc] +from msgpack import ExtType as _ExtType _bool, _bytes, _list, _int, _float, _str = bool, bytes, list, int, float, str _BYTE_LIKE = (_bytes, bytearray, memoryview) From b2434fc7fce01bbc6a64a41762972316bbcc4580 Mon Sep 17 00:00:00 2001 From: Libby Date: Fri, 6 Nov 2026 20:49:32 +0000 Subject: [PATCH 62/88] cultcache-py: single-file store takes the cross-process lock, stages uniquely and fsyncs push, push_all and delete hold .lock (flock; LockFileEx over the whole file on Windows, as cultcache-ts store-file.ts) across the read and the replace. The staged file is .tmp--, fsynced before os.replace, with a directory fsync after off Windows; a failed write removes its staged file and leaves the store. Tests fail when the rule breaks: a write waits while a flock holder (Rust's lock) and the TypeScript runtime's withStoreWriteLock hold the lock, three processes lose no writes, fsync precedes replace, staged names differ, a failed replace leaves no staged file. Co-Authored-By: Claude Sonnet 5.5 --- .../cultcache-py/src/cultcache_py/stores.py | 130 +++++++++++++---- .../cultcache-py/tests/test_store_lock.py | 135 ++++++++++++++++++ 2 files changed, 241 insertions(+), 24 deletions(-) create mode 100644 packages/cultcache-py/tests/test_store_lock.py diff --git a/packages/cultcache-py/src/cultcache_py/stores.py b/packages/cultcache-py/src/cultcache_py/stores.py index c9c76b06..77d671ad 100644 --- a/packages/cultcache-py/src/cultcache_py/stores.py +++ b/packages/cultcache-py/src/cultcache_py/stores.py @@ -3,9 +3,15 @@ import base64 import json import os +import sys import threading +import time +import uuid +from contextlib import contextmanager from pathlib import Path -from typing import Any, Callable, TypeVar +from typing import Any, Callable, Iterator, TypeVar + +import msgpack from .backing_store import ( CultCacheEnvelope, @@ -21,6 +27,8 @@ STORE_FORMAT_ELEMENT_IDS = "cultcache.store.v3" _READABLE_STORE_FORMATS = (STORE_FORMAT_VERSION, STORE_FORMAT_ELEMENT_IDS) _STORE_FORMAT_PREFIX = "cultcache.store." _PERSISTED_RECORD_SLOTS = 4 +_LOCK_POLL_SECONDS = 0.01 +LOCK_TIMEOUT_SECONDS = 30.0 _Contents = TypeVar("_Contents") @@ -114,14 +122,14 @@ class JsonLinesBackingStore: class SingleFileMessagePackBackingStore: - def __init__(self, path: str | os.PathLike[str]) -> None: + def __init__(self, path: str | os.PathLike[str], lock_timeout: float = LOCK_TIMEOUT_SECONDS) -> None: self.path = Path(path) + self.lock_timeout = lock_timeout self._lock = threading.RLock() # The header this store read; a rewrite keeps it, so a store that holds element ids never sheds its marker. self._format = STORE_FORMAT_VERSION def pull_all(self) -> list[CultCacheEnvelope]: - msgpack = self._msgpack() with self._lock: # An empty file is not a store (no writer leaves one), so it is decoded and refused. # The disk decides the header: a file that is gone is not marked. @@ -135,39 +143,34 @@ class SingleFileMessagePackBackingStore: if snapshot is not None: self._format, envelopes = snapshot return envelopes - return _decode_legacy_envelopes(decoded, msgpack) + return _decode_legacy_envelopes(decoded) except Exception as exc: raise StoreUnreadableError(self.path, exc) from exc def push(self, envelope: CultCacheEnvelope) -> None: - with self._lock: + with self._write_transaction(): self._replace_all([envelope], self.pull_all()) def push_all(self, envelopes: list[CultCacheEnvelope]) -> None: - with self._lock: + with self._write_transaction(): self._replace_all(list(envelopes), self.pull_all()) def delete(self, type: str, key: str) -> None: - with self._lock: + with self._write_transaction(): self._replace_all([], [item for item in self.pull_all() if not (item.type == type and item.key == key)]) - def _replace_all(self, supplied: list[CultCacheEnvelope], arrived: list[CultCacheEnvelope]) -> None: - msgpack = self._msgpack() - self.path.parent.mkdir(parents=True, exist_ok=True) - temp = self.path.with_suffix(self.path.suffix + ".tmp") - temp.write_bytes(msgpack.packb(_encode_snapshot(supplied, arrived, self._format), use_bin_type=True)) - temp.replace(self.path) + @contextmanager + def _write_transaction(self) -> Iterator[None]: + """The thread lock, then the cross-process lock on `.lock`, held across the read and the replace so two + processes never rewrite from the same stale read.""" + with self._lock: + self.path.parent.mkdir(parents=True, exist_ok=True) + with store_write_lock(self.path, self.lock_timeout): + yield + + def _replace_all(self, supplied: list[CultCacheEnvelope], arrived: list[CultCacheEnvelope]) -> None: + commit_store_file(self.path, msgpack.packb(_encode_snapshot(supplied, arrived, self._format), use_bin_type=True)) - @staticmethod - def _msgpack() -> Any: - try: - import msgpack # type: ignore - except ModuleNotFoundError as exc: - raise RuntimeError( - "SingleFileMessagePackBackingStore requires the optional 'msgpack' dependency. " - "Install with: python -m pip install cultcache-py[msgpack]" - ) from exc - return msgpack def canonical(candidates: Any) -> list[CultCacheSchemaCatalogEntry]: @@ -261,6 +264,85 @@ def _encode_snapshot( ] return [format_version, catalog, records] +def commit_store_file(path: Path, data: bytes) -> None: + """Writes `data` to a unique staged file beside the store, fsyncs it, replaces the store with it and, off Windows, + fsyncs the directory so the replace is durable. A failure before the replace removes the staged file and leaves the + store untouched. The caller holds the store lock.""" + staged = path.with_name(f"{path.name}.tmp-{os.getpid()}-{uuid.uuid4().hex}") + try: + fd = os.open(staged, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_BINARY", 0), 0o666) + try: + view = memoryview(data) + while view: + view = view[os.write(fd, view):] + os.fsync(fd) + finally: + os.close(fd) + os.replace(staged, path) + except BaseException: + try: + os.unlink(staged) + except OSError: + pass + raise + if sys.platform != "win32": + directory = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + + +@contextmanager +def store_write_lock(path: Path, timeout: float = LOCK_TIMEOUT_SECONDS) -> Iterator[None]: + """Holds the exclusive store lock on `.lock`, the lock the C#, Rust and TypeScript runtimes take: flock on Unix, + LockFileEx over the whole file on Windows, polled every 10 ms with no handle held between polls. The lock file is + created if absent and never written or deleted.""" + lock_path = path.with_name(path.name + ".lock") + deadline = time.monotonic() + timeout + while True: + release = _try_lock(lock_path) + if release is not None: + break + if time.monotonic() >= deadline: + raise TimeoutError(f"CultCache timed out after {timeout} s waiting for the store lock {lock_path}.") + time.sleep(_LOCK_POLL_SECONDS) + try: + yield + finally: + release() + + +def _try_lock(lock_path: Path) -> Callable[[], None] | None: + if sys.platform == "win32": + return _try_lock_windows(lock_path) + import fcntl + + fd = os.open(lock_path, os.O_RDWR | os.O_CREAT, 0o666) + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + os.close(fd) + return None + return lambda: os.close(fd) # closing drops the flock + + +def _try_lock_windows(lock_path: Path) -> Callable[[], None] | None: + import ctypes + import msvcrt + from ctypes import wintypes + + kernel = ctypes.WinDLL("kernel32", use_last_error=True) + kernel.LockFileEx.argtypes = [wintypes.HANDLE, wintypes.DWORD, wintypes.DWORD, wintypes.DWORD, wintypes.DWORD, ctypes.c_void_p] + fd = os.open(lock_path, os.O_RDWR | os.O_CREAT | os.O_BINARY, 0o666) + handle = msvcrt.get_osfhandle(fd) + overlapped = ctypes.create_string_buffer(32) + # LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY over the whole range, as the TypeScript runtime takes it. + if not kernel.LockFileEx(handle, 0x3, 0, 0xFFFFFFFF, 0xFFFFFFFF, overlapped): + os.close(fd) + return None + return lambda: os.close(fd) # closing drops the lock + def _read_store(path: Path, read: Callable[[Path], _Contents]) -> _Contents | None: """Reads a store file, or returns None when nothing is at its path. @@ -351,7 +433,7 @@ def _decode_snapshot(decoded: Any) -> tuple[str, list[CultCacheEnvelope]] | None return decoded[0], envelopes -def _decode_legacy_envelopes(decoded: Any, msgpack: Any) -> list[CultCacheEnvelope]: +def _decode_legacy_envelopes(decoded: Any) -> list[CultCacheEnvelope]: if not isinstance(decoded, list): raise ValueError("CultCache MessagePack store is not a recognized snapshot") envelopes: list[CultCacheEnvelope] = [] diff --git a/packages/cultcache-py/tests/test_store_lock.py b/packages/cultcache-py/tests/test_store_lock.py new file mode 100644 index 00000000..5ee29aa2 --- /dev/null +++ b/packages/cultcache-py/tests/test_store_lock.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +import tempfile +import time +import unittest +from pathlib import Path +from unittest import mock + +from cultcache_py import SingleFileMessagePackBackingStore, stores +from cultcache_py.backing_store import CultCacheEnvelope + +REPO = Path(__file__).resolve().parents[3] +TS_STORE_FILE = REPO / "packages" / "cultcache-ts" / "dist" / "store-file.js" +HOLD_SECONDS = 1.5 + +# A holder that takes the lock the way the Rust runtime does (an exclusive file lock), not through this package. +PY_HOLDER = """ +import fcntl, os, sys, time +fd = os.open(sys.argv[1] + '.lock', os.O_RDWR | os.O_CREAT, 0o666) +fcntl.flock(fd, fcntl.LOCK_EX) +print('HELD', flush=True) +time.sleep(float(sys.argv[2])) +""" +# A holder that is the TypeScript runtime's own withStoreWriteLock, built from packages/cultcache-ts. +TS_HOLDER = """ +const { withStoreWriteLock } = require(process.argv[1]); +withStoreWriteLock(process.argv[2], async () => { + console.log('HELD'); + await new Promise((r) => setTimeout(r, Number(process.argv[3]) * 1000)); +}); +""" + + +def envelope(key: str) -> CultCacheEnvelope: + return CultCacheEnvelope(key=key, type="t", payload=b"\x01", stored_at="2026-11-06T00:00:00Z") + + +def start_holder(argv: list[str]) -> subprocess.Popen[str]: + holder = subprocess.Popen(argv, stdout=subprocess.PIPE, text=True) + assert holder.stdout is not None and holder.stdout.readline().strip() == "HELD" + holder.held_at = time.monotonic() # type: ignore[attr-defined] + return holder + + +@unittest.skipIf(sys.platform == "win32", "the foreign holders here use flock; Windows is covered by the TS holder") +class StoreLockTests(unittest.TestCase): + def setUp(self) -> None: + self.dir = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.dir, ignore_errors=True) + self.path = self.dir / "state.cc" + + def assert_write_waits_for(self, holder: subprocess.Popen[str]) -> None: + self.addCleanup(holder.wait) + self.addCleanup(holder.stdout.close) # type: ignore[union-attr] + refused = SingleFileMessagePackBackingStore(self.path, lock_timeout=0.2) + with self.assertRaises(TimeoutError): + refused.push(envelope("blocked")) + self.assertFalse(self.path.exists(), "a write that could not take the lock must not touch the store") + + SingleFileMessagePackBackingStore(self.path, lock_timeout=10).push(envelope("after")) + self.assertGreaterEqual(time.monotonic() - holder.held_at, HOLD_SECONDS - 0.1, "the write finished while the holder still held the lock") # type: ignore[attr-defined] + self.assertEqual(["after"], [e.key for e in SingleFileMessagePackBackingStore(self.path).pull_all()]) + + def test_write_waits_while_a_foreign_process_holds_the_lock(self) -> None: + self.assert_write_waits_for(start_holder([sys.executable, "-c", PY_HOLDER, str(self.path), str(HOLD_SECONDS)])) + + @unittest.skipUnless(shutil.which("node") and TS_STORE_FILE.exists(), "needs node and a built packages/cultcache-ts (npm run build)") + def test_write_waits_while_the_typescript_runtime_holds_the_lock(self) -> None: + holder = start_holder(["node", "-e", TS_HOLDER, str(TS_STORE_FILE), str(self.path), str(HOLD_SECONDS)]) + self.assert_write_waits_for(holder) + + def test_the_lock_is_released_after_each_write(self) -> None: + store = SingleFileMessagePackBackingStore(self.path, lock_timeout=0.2) + for key in "abc": + store.push(envelope(key)) + self.assertEqual(["a", "b", "c"], sorted(e.key for e in store.pull_all())) + + def test_concurrent_processes_lose_no_writes(self) -> None: + script = ( + "import sys\n" + "from cultcache_py import SingleFileMessagePackBackingStore as S\n" + "from cultcache_py.backing_store import CultCacheEnvelope as E\n" + "s = S(sys.argv[1])\n" + "for i in range(25):\n" + " s.push(E(key=f'{sys.argv[2]}{i}', type='t', payload=b'\\x01', stored_at='x'))\n" + ) + env = {**os.environ, "PYTHONPATH": str(Path(stores.__file__).parents[1])} + procs = [subprocess.Popen([sys.executable, "-c", script, str(self.path), tag], env=env) for tag in "pqr"] + for proc in procs: + self.assertEqual(0, proc.wait()) + self.assertEqual(75, len(SingleFileMessagePackBackingStore(self.path).pull_all())) + + +class StagedWriteTests(unittest.TestCase): + def setUp(self) -> None: + self.dir = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.dir, ignore_errors=True) + self.path = self.dir / "state.cc" + + def test_the_staged_file_is_fsynced_before_the_replace(self) -> None: + order: list[str] = [] + real_fsync, real_replace = os.fsync, os.replace + with mock.patch("os.fsync", side_effect=lambda fd: (order.append("fsync"), real_fsync(fd))[1]), mock.patch( + "os.replace", side_effect=lambda a, b: (order.append("replace"), real_replace(a, b))[1] + ): + SingleFileMessagePackBackingStore(self.path).push(envelope("a")) + self.assertEqual("fsync", order[0]) + self.assertLess(order.index("fsync"), order.index("replace")) + + def test_the_staged_name_is_unique_per_write(self) -> None: + names: list[str] = [] + real_replace = os.replace + with mock.patch("os.replace", side_effect=lambda a, b: (names.append(os.fspath(a)), real_replace(a, b))[1]): + store = SingleFileMessagePackBackingStore(self.path) + store.push(envelope("a")) + store.push(envelope("b")) + self.assertEqual(2, len(set(names))) + self.assertNotIn(os.fspath(self.path) + ".tmp", names) + + def test_a_failed_replace_leaves_the_store_and_no_staged_file(self) -> None: + store = SingleFileMessagePackBackingStore(self.path) + store.push(envelope("a")) + before = self.path.read_bytes() + with mock.patch("os.replace", side_effect=OSError("boom")), self.assertRaises(OSError): + store.push(envelope("b")) + self.assertEqual(before, self.path.read_bytes()) + self.assertEqual([], [p.name for p in self.dir.iterdir() if ".tmp" in p.name]) + + +if __name__ == "__main__": + unittest.main() From 00ecdcf9e67bb0615472bddfd85a81ea7eee3ecf Mon Sep 17 00:00:00 2001 From: Libby Date: Fri, 6 Nov 2026 20:50:10 +0000 Subject: [PATCH 63/88] cultcache-py: read the slot-nil vector; changelog for the store lock and msgpack tests/vectors/slot-nil was read by TS only. The Python reader accepts the same 15 rows and refuses the same 2 as the C# reference and rewrites the reference's bytes. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-py/CHANGELOG.md | 5 ++ packages/cultcache-py/tests/test_slot_nil.py | 54 ++++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 packages/cultcache-py/tests/test_slot_nil.py diff --git a/packages/cultcache-py/CHANGELOG.md b/packages/cultcache-py/CHANGELOG.md index d2f76628..b460ab05 100644 --- a/packages/cultcache-py/CHANGELOG.md +++ b/packages/cultcache-py/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +### Changed + +- `msgpack` is a hard dependency, as `pyproject.toml` declared it. The optional-import fallbacks and the `cultcache-py[msgpack]` extra they named (which never existed) are gone: a missing msgpack is an `ImportError` at import. +- The single-file store's `push`, `push_all` and `delete` take the cross-process lock on `.lock` (flock; LockFileEx over the whole file on Windows) across the read and the replace, waiting up to `lock_timeout` seconds (30 by default, `TimeoutError` after), and exclude the C#, Rust and TypeScript runtimes' writers. The staged file is `.tmp--`, fsynced before the replace, with a directory fsync after it off Windows. A fixed `.tmp` is no longer used. + ### Breaking - A single-file store written as `cultcache.store.v3` is read and written: a record that holds an element id (an owned list element with an `[CultElementId]` member, carried through every write) marks the store v3, and a v3 store is carried through a rewrite with its ids. A runtime from before this change refuses a v3 store by name. diff --git a/packages/cultcache-py/tests/test_slot_nil.py b/packages/cultcache-py/tests/test_slot_nil.py new file mode 100644 index 00000000..dca207bb --- /dev/null +++ b/packages/cultcache-py/tests/test_slot_nil.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import unittest +from pathlib import Path + +import msgpack + +from cultcache_py import database_entry_field, define_database_entry_type, define_record_codec, leaves as t + +# tests/vectors/slot-nil is written only by the C# reference (SlotNilVectorTests): what it reads and rewrites for a nil, a +# str in a byte[] slot, and a record short of its last slots. This runtime accepts what the reference accepts and rewrites +# the same bytes. +VECTORS = Path(__file__).resolve().parents[3] / "tests" / "vectors" / "slot-nil" + + +def _probe(): + inner = define_record_codec("SlotNilInner", [database_entry_field("x", 0, default="", codec=t.string)]) + return define_database_entry_type( + "tests.slot-nil", + [ + database_entry_field("s", 0, default="", codec=t.nullable(t.string)), + database_entry_field("i", 1, default=0, codec=t.int32), + database_entry_field("b", 2, default=b"", codec=t.nullable(t.bytes)), + database_entry_field("r", 3, default=None, codec=t.nullable(inner)), + database_entry_field("l", 4, default=[], codec=t.nullable(t.list(t.string))), + ], + ) + + +class SlotNilVectorTests(unittest.TestCase): + def test_slots_read_and_rewrite_exactly_what_the_reference_does_for_nil_str_in_bytes_and_short_records(self) -> None: + rows = msgpack.unpackb((VECTORS / "reads.msgpack").read_bytes(), raw=False) + probe = _probe() + self.assertEqual(17, len(rows)) + accepted = 0 + for name, payload, was_accepted, rewritten in rows: + with self.subTest(name): + if was_accepted: + self.assertEqual(rewritten.hex(), probe.encode_payload(probe.decode_payload(payload)).hex()) + accepted += 1 + else: + with self.assertRaises(Exception): + probe.decode_payload(payload) + self.assertEqual(15, accepted) + + def test_a_character_outside_the_bmp_is_four_utf8_bytes(self) -> None: + rows = {name: (payload, rewritten) for name, payload, _, rewritten in msgpack.unpackb((VECTORS / "reads.msgpack").read_bytes(), raw=False)} + probe = _probe() + self.assertEqual("\U0001F600", probe.decode_payload(rows["string-astral"][0])["s"]) + self.assertEqual(b"\xf0\x9f\x98\x80", probe.decode_payload(rows["bytes-str-astral"][0])["b"]) + + +if __name__ == "__main__": + unittest.main() From 81aeb60c571a0999d40d6bfc245c402150d34a1c Mon Sep 17 00:00:00 2001 From: Libby Date: Fri, 6 Nov 2026 20:51:14 +0000 Subject: [PATCH 64/88] cultnet-py: schema catalog describes cultnet.error.v0 as the contract does Required is the contract's five keys (schemaVersion, error, routingHint, code, details), code is the closed enum (or null), details and routingHint are object-or-null, and the catalog has no messageId property, as cultnet.error.schema.json is closed. The test compares required, property names and the code enum to the contract file instead of pinning the stale two-key form. Co-Authored-By: Claude Sonnet 5.5 --- .../src/cultnet_py/schema_catalog.py | 21 ++++++++++++++----- packages/cultnet-py/tests/test_cultnet.py | 6 ++++-- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/packages/cultnet-py/src/cultnet_py/schema_catalog.py b/packages/cultnet-py/src/cultnet_py/schema_catalog.py index 6d71b3cd..908a777f 100644 --- a/packages/cultnet-py/src/cultnet_py/schema_catalog.py +++ b/packages/cultnet-py/src/cultnet_py/schema_catalog.py @@ -12,6 +12,17 @@ PEER_EXCHANGE_REQUEST = "cultmesh.peer_exchange_request.v0" TRANSPORT_PROFILE_SCHEMA_VERSION = "cultnet.transport_profile.v0" TRANSPORT_PROFILE_SCHEMA_ID = f"{CULTNET_SCHEMA_BASE}/cultnet.transport-profile.schema.json" +# The closed code set of contracts/cultnet/cultnet.error.schema.json; a code outside it is one coordinated change across every runtime. +CULTNET_ERROR_CODES = ( + "selection_invalid", + "cursor_stale", + "cursor_invalid", + "reference_outside_target", + "variant_unsupported", + "unowned_schema", + "write_not_authorized", +) + WIRE_MESSAGE_SCHEMA_VERSIONS = ( ("cultnet.hello.v0", "CultNet Hello Message", f"{CULTNET_SCHEMA_BASE}/cultnet.hello.schema.json"), ("cultnet.error.v0", "CultNet Error Message", f"{CULTNET_SCHEMA_BASE}/cultnet.error.schema.json"), @@ -269,7 +280,7 @@ def shared_contract_schema_json(schema_id: str, title: str, schema_version: str) def wire_message_required_fields(schema_version: str) -> list[str]: required = { "cultnet.hello.v0": ["schemaVersion", "runtimeId"], - "cultnet.error.v0": ["schemaVersion", "error"], + "cultnet.error.v0": ["schemaVersion", "error", "routingHint", "code", "details"], "cultnet.login.v0": ["schemaVersion", "nonce", "auth", "password"], "cultnet.register.v0": ["schemaVersion", "nonce", "email", "password", "name"], "cultnet.verify.v0": ["schemaVersion", "nonce", "session"], @@ -355,10 +366,10 @@ def wire_message_schema_properties(schema_version: str) -> dict[str, Any]: "supportsSchemaCatalog": {"type": "boolean"}, }, "cultnet.error.v0": { - "messageId": common["messageId"], - "error": {"type": "string"}, - "code": {"type": "string"}, - "details": {"type": "object", "additionalProperties": True}, + "error": {"type": "string", "minLength": 1}, + "routingHint": {"type": ["object", "null"]}, + "code": {"type": ["string", "null"], "enum": [None, *CULTNET_ERROR_CODES]}, + "details": {"type": ["object", "null"]}, }, "cultnet.login.v0": { "nonce": {"type": "string", "minLength": 1}, diff --git a/packages/cultnet-py/tests/test_cultnet.py b/packages/cultnet-py/tests/test_cultnet.py index 19c88b6a..a0378069 100644 --- a/packages/cultnet-py/tests/test_cultnet.py +++ b/packages/cultnet-py/tests/test_cultnet.py @@ -2235,9 +2235,11 @@ class CultNetTests(unittest.TestCase): self.assertEqual(delete_schema["properties"]["recordKey"]["type"], "string") error_schema = json.loads(by_version["cultnet.error.v0"]["schemaJson"]) - self.assertEqual(error_schema["required"], ["schemaVersion", "error"]) + contract = json.loads((Path(__file__).resolve().parents[3] / "contracts" / "cultnet" / "cultnet.error.schema.json").read_text(encoding="utf-8")) + self.assertEqual(error_schema["required"], contract["required"]) + self.assertEqual(sorted(error_schema["properties"]), sorted(contract["properties"])) self.assertEqual(error_schema["properties"]["error"]["type"], "string") - self.assertIn("details", error_schema["properties"]) + self.assertEqual(error_schema["properties"]["code"]["enum"], contract["properties"]["code"]["enum"]) login_schema = json.loads(by_version["cultnet.login.v0"]["schemaJson"]) self.assertEqual(login_schema["required"], ["schemaVersion", "nonce", "auth", "password"]) From 27d7e48cd6c38850ffea07025fdb3eefb09c9426 Mon Sep 17 00:00:00 2001 From: Libby Date: Fri, 6 Nov 2026 20:53:13 +0000 Subject: [PATCH 65/88] heimdall-consent: pin key(), MemberTerms read-back, acceptance member/experiment, scope rule; refuse leading-zero versions Answers Heimdall's review of PR #12 (review 4): tests for the exact U+001F joined key and the empty and separator refusals; an expect_terms vector op (newcomer case defends the MemberTerms pair) and expect_no_terms; a mismatched member or experiment acceptance is never training data; the three scope conditions kept, with a test of the rule over every scope pair; sample versions with a leading zero are refused (README notes Heimdall may rule normalise instead). Co-Authored-By: Claude Sonnet 5.5 --- packages/heimdall-consent/README.md | 2 +- packages/heimdall-consent/src/lib.rs | 9 +-- packages/heimdall-consent/tests/golden.rs | 65 ++++++++++++++++++- .../tests/vectors/consent_golden.csv | 4 +- 4 files changed, 73 insertions(+), 7 deletions(-) diff --git a/packages/heimdall-consent/README.md b/packages/heimdall-consent/README.md index bac41338..ed89b9d0 100644 --- a/packages/heimdall-consent/README.md +++ b/packages/heimdall-consent/README.md @@ -14,7 +14,7 @@ Entry points: `src/lib.rs`; golden vectors `tests/vectors/consent_golden.csv`, r | Ruling | Doc | Crate | |---|---|---| | Timestamps are Unix ms UTC, signed 64-bit | L98-99; `received_at` is collector-assigned, L75 | `i64` on `accepted_at`, `received_at`, `terms_accepted_at` | -| Ids are non-empty opaque strings, `event_id` verbatim; versions unsigned 32-bit and positive | L100-103 | `validate()`; `notice_version: u32`; `check_sample_id` parses `u32` | +| Ids are non-empty opaque strings, `event_id` verbatim; versions unsigned 32-bit and positive | L100-103 | `validate()`; `notice_version: u32`; `check_sample_id` parses `u32` and refuses a leading zero in the version (`s@01`) so one version has one key; the doc says only "positive integer", so Heimdall may instead rule that the crate normalises | | Key parts joined with U+001F; empty part or part containing it refused | L104-105 | `key()` | | Tombstone is the exited `Answer` with `text_hash` (32 bytes); HMAC-SHA-256 keyed by a per-member salt held by the member-side service; the collector cannot compute it | L74, L132-152, L186-193 | `Answer.text_hash`; `redacted([u8; 32])` takes the service's output and refuses an answer with no `text` (L148-149); `validate()` requires 32 bytes and `exited` | | A reply with no established notice version or scope is stored with both absent, writes no acceptance, is excluded from every use | L87-94 | `notice_version`/`scope` optional and absent together (`validate()`); vector case `no-scope` | diff --git a/packages/heimdall-consent/src/lib.rs b/packages/heimdall-consent/src/lib.rs index 0119371f..a9a75ebb 100644 --- a/packages/heimdall-consent/src/lib.rs +++ b/packages/heimdall-consent/src/lib.rs @@ -35,15 +35,16 @@ fn nonempty(name: &str, value: &str) -> Result<()> { Ok(()) } -/// `@`, version a positive unsigned 32-bit integer. The version is part of the id, -/// so it is part of every answer key and no key can name a sample without one. +/// `@`, version a positive unsigned 32-bit integer written without leading zeros +/// (`s@01` and `s@1` would be two keys for one version). The version is part of the id, so it is part +/// of every answer key and no key can name a sample without one. pub fn check_sample_id(sample_id: &str) -> Result<(&str, u32)> { let Some((sample, version)) = sample_id.rsplit_once('@') else { bail!("sample id {sample_id:?} has no @version"); }; match version.parse::() { - Ok(v) if v > 0 && !sample.is_empty() && !version.starts_with('+') => Ok((sample, v)), - _ => bail!("sample id {sample_id:?} needs a non-empty sample and a positive integer version"), + Ok(v) if v > 0 && !sample.is_empty() && !version.starts_with(['+', '0']) => Ok((sample, v)), + _ => bail!("sample id {sample_id:?} needs a non-empty sample and a positive integer version without leading zeros"), } } diff --git a/packages/heimdall-consent/tests/golden.rs b/packages/heimdall-consent/tests/golden.rs index 23a9161d..af6953b5 100644 --- a/packages/heimdall-consent/tests/golden.rs +++ b/packages/heimdall-consent/tests/golden.rs @@ -162,6 +162,11 @@ fn golden_vectors() { }) .unwrap(); } + "expect_terms" => { + let t = cache.get_required::(&MemberTerms::cache_key(member).unwrap()).unwrap(); + assert_eq!((t.member_id.as_str(), t.terms_version.as_str(), t.terms_accepted_at), (member, payload, ts), "{ctx}"); + } + "expect_no_terms" => assert!(cache.get::(&MemberTerms::cache_key(member).unwrap()).unwrap().is_none(), "{ctx}"), "expect_counts" => { let want = format!( "answers={};acceptances={}", @@ -214,7 +219,7 @@ fn records_survive_a_reopen() { #[test] fn key_refuses_unversioned_sample_ids() { - for bad in ["s", "s@", "s@0", "s@x", "@1", "s@-1", "s@+1", "s@4294967296"] { + for bad in ["s", "s@", "s@0", "s@x", "@1", "s@-1", "s@+1", "s@4294967296", "s@01", "s@00", "s@001"] { assert!(Answer::cache_key("r", bad, "m").is_err(), "{bad}"); } assert!(Answer::cache_key("r", "s@4294967295", "m").is_ok()); @@ -357,3 +362,61 @@ fn later_reply_is_greater_received_at_then_bytewise_event_id() { assert!(at(1, "z").supersedes(&at(1, "aa")), "bytewise, not by length"); assert!(!at(1, "e").supersedes(&at(1, "e"))); } + +#[test] +fn keys_join_the_parts_with_u001f_and_refuse_empty_or_separator_parts() { + assert_eq!(Answer::cache_key("r", "s@1", "m").unwrap(), "r\u{1f}s@1\u{1f}m"); + assert_eq!(NoticeAcceptance::cache_key("r", "m", 3).unwrap(), "r\u{1f}m\u{1f}3"); + assert_eq!(MemberTerms::cache_key("m").unwrap(), "m"); + for (experiment, sample, member) in [("", "s@1", "m"), ("r", "s@1", ""), ("r\u{1f}x", "s@1", "m"), ("r", "s\u{1f}x@1", "m"), ("r", "s@1", "m\u{1f}x")] { + assert!(Answer::cache_key(experiment, sample, member).is_err(), "{experiment:?} {sample:?} {member:?}"); + } + assert!(NoticeAcceptance::cache_key("", "m", 1).is_err()); + assert!(NoticeAcceptance::cache_key("r", "", 1).is_err()); + assert!(NoticeAcceptance::cache_key("r\u{1f}", "m", 1).is_err()); + assert!(NoticeAcceptance::cache_key("r", "m\u{1f}", 1).is_err()); + assert!(MemberTerms::cache_key("").is_err()); + assert!(MemberTerms::cache_key("m\u{1f}").is_err()); + // Parts that would collide if the separator were not refused or were `,` or empty. + assert_ne!(Answer::cache_key("a", "s@1", "bc").unwrap(), Answer::cache_key("ab", "s@1", "c").unwrap()); +} + +#[test] +fn an_acceptance_for_another_member_or_experiment_never_covers_the_answer() { + let acc = NoticeAcceptance { + member_id: "m".into(), + experiment_id: "r".into(), + notice_version: 1, + scope: Scope::Train, + accepted_at: 1, + event_id: "e0".into(), + }; + assert!(answer().is_training_data(Some(&acc))); + let other_member = NoticeAcceptance { member_id: "m2".into(), ..acc.clone() }; + let other_experiment = NoticeAcceptance { experiment_id: "r2".into(), ..acc }; + assert!(!answer().is_training_data(Some(&other_member))); + assert!(!answer().is_training_data(Some(&other_experiment))); +} + +/// The doc states the rule once: the answer's scope is `train`, the acceptance's own scope is `train`, +/// and the two are equal. The code keeps all three conditions (any two imply the third, so a single +/// removal is an equivalent mutant); this pins the rule itself over every pair of scopes. +#[test] +fn only_a_train_answer_under_a_train_acceptance_is_training_data() { + let scopes = [Scope::Show, Scope::Study, Scope::Train]; + for answer_scope in scopes { + for acceptance_scope in scopes { + let a = Answer { scope: Some(answer_scope), ..answer() }; + let acc = NoticeAcceptance { + member_id: "m".into(), + experiment_id: "r".into(), + notice_version: 1, + scope: acceptance_scope, + accepted_at: 1, + event_id: "e0".into(), + }; + let want = answer_scope == Scope::Train && acceptance_scope == Scope::Train; + assert_eq!(a.is_training_data(Some(&acc)), want, "answer {answer_scope:?} under acceptance {acceptance_scope:?}"); + } + } +} diff --git a/packages/heimdall-consent/tests/vectors/consent_golden.csv b/packages/heimdall-consent/tests/vectors/consent_golden.csv index f11db809..a7e5d553 100644 --- a/packages/heimdall-consent/tests/vectors/consent_golden.csv +++ b/packages/heimdall-consent/tests/vectors/consent_golden.csv @@ -5,7 +5,7 @@ # A reply with no established notice version or scope is stored with both absent (doc L87-94, Answer record), writes no acceptance, and is excluded. answer_raw with a version but no acceptance models a lost or never-written acceptance: still excluded (doc L119-122). # reply_rejected: validate must refuse the answer (nothing stored). answer_revoked: stores an exited answer with no text; payload is the tombstone hash (empty means none). expect_winner: the stored answer's event_id and text must equal event and payload. # Replacement at one key: the later reply is the greater (received_at, event_id), event_id compared bytewise; input order never decides (doc L78-81 and Heimdall PR #6 comment 759). -# terms: member field terms_version (payload) and terms_accepted_at (ts). expect_*: payload is what must hold. +# terms: member field terms_version (payload) and terms_accepted_at (ts). expect_terms: the stored pair for that member reads back as payload and ts; expect_no_terms: the member has none. expect_*: payload is what must hold. # Expected values are written by hand from docs/answer-collector-consent.md (Heimdall PR #6, doc commit 4e99bfa), not computed by the crate. versioned-key,reply,m1,room1,1,train,s1@1,first,100,e1 versioned-key,reply,m1,room1,1,train,s1@2,second,200,e2 @@ -26,6 +26,8 @@ redacted-reply,expect_exited,m6,room1,,,s1@1,a1b2c3d4e5f60718293a4b5c6d7e8f90a1b redacted-reply,expect_train,,,,,,e10,, redacted-reply,expect_counts,,,,,,answers=2;acceptances=2,, newcomer,terms,m8,,,,,2026-11,50, +newcomer,expect_terms,m8,,,,,2026-11,50, +newcomer,expect_no_terms,m9,,,,,,, newcomer,reply,m8,room1,3,train,s1@1,hello,100,e11 newcomer,reply,m9,room1,3,train,s1@1,veteran,110,e12 newcomer,reply,m8,room1,4,study,s1@1,again,200,e13 From af0b1b68172fa0240c52becaa35add4aeeb60e98 Mon Sep 17 00:00:00 2001 From: Libby Date: Sat, 7 Nov 2026 00:32:09 +0000 Subject: [PATCH 66/88] heimdall-consent: one version rule for every version field (positive, no leading zeros, <= 4294967295) check_version is the single rule; the sample id's @N and MemberTerms.terms_version both call it, so terms_version "2026-11" is now refused and the golden vector carries 4294967295 (the harness now validates the terms record). notice_version stays u32 and validate() refuses 0. Tests: boundary table (1, 4294967295 accepted; 0, 4294967296, 4294967297, 01, +1, -1, empty, 2026-11 refused) across all three; each rule checked by mutation (u64 wrap, leading-zero, sign, terms unchecked, returned value, empty sample, zero in both records) and killed. Co-Authored-By: Claude Sonnet 5.5 --- packages/heimdall-consent/README.md | 2 +- packages/heimdall-consent/src/lib.rs | 22 +++++++---- packages/heimdall-consent/tests/golden.rs | 39 +++++++++++++++---- .../tests/vectors/consent_golden.csv | 4 +- 4 files changed, 50 insertions(+), 17 deletions(-) diff --git a/packages/heimdall-consent/README.md b/packages/heimdall-consent/README.md index ed89b9d0..fb446e64 100644 --- a/packages/heimdall-consent/README.md +++ b/packages/heimdall-consent/README.md @@ -14,7 +14,7 @@ Entry points: `src/lib.rs`; golden vectors `tests/vectors/consent_golden.csv`, r | Ruling | Doc | Crate | |---|---|---| | Timestamps are Unix ms UTC, signed 64-bit | L98-99; `received_at` is collector-assigned, L75 | `i64` on `accepted_at`, `received_at`, `terms_accepted_at` | -| Ids are non-empty opaque strings, `event_id` verbatim; versions unsigned 32-bit and positive | L100-103 | `validate()`; `notice_version: u32`; `check_sample_id` parses `u32` and refuses a leading zero in the version (`s@01`) so one version has one key; the doc says only "positive integer", so Heimdall may instead rule that the crate normalises | +| Ids are non-empty opaque strings, `event_id` verbatim; every version field (`notice_version`, the `@N` of a sample id, `terms_version`) is a positive integer, no leading zeros, no sign, at most 4294967295 (Heimdall ruling 2026-11-07, one rule) | L100-103 | `check_version` is the single rule; `check_sample_id` and `MemberTerms::validate` call it; `notice_version: u32` with `validate()` refusing 0; golden vector `newcomer` carries `terms_version` 4294967295 | | Key parts joined with U+001F; empty part or part containing it refused | L104-105 | `key()` | | Tombstone is the exited `Answer` with `text_hash` (32 bytes); HMAC-SHA-256 keyed by a per-member salt held by the member-side service; the collector cannot compute it | L74, L132-152, L186-193 | `Answer.text_hash`; `redacted([u8; 32])` takes the service's output and refuses an answer with no `text` (L148-149); `validate()` requires 32 bytes and `exited` | | A reply with no established notice version or scope is stored with both absent, writes no acceptance, is excluded from every use | L87-94 | `notice_version`/`scope` optional and absent together (`validate()`); vector case `no-scope` | diff --git a/packages/heimdall-consent/src/lib.rs b/packages/heimdall-consent/src/lib.rs index a9a75ebb..508e3a67 100644 --- a/packages/heimdall-consent/src/lib.rs +++ b/packages/heimdall-consent/src/lib.rs @@ -35,17 +35,25 @@ fn nonempty(name: &str, value: &str) -> Result<()> { Ok(()) } -/// `@`, version a positive unsigned 32-bit integer written without leading zeros -/// (`s@01` and `s@1` would be two keys for one version). The version is part of the id, so it is part -/// of every answer key and no key can name a sample without one. +/// The one version rule (Heimdall ruling 2026-11-07): a positive integer written without leading +/// zeros or a sign, at most 4294967295 (`u32::MAX`). `01` and `1` would be two keys for one version. +pub fn check_version(version: &str) -> Result { + match version.parse::() { + Ok(v) if v > 0 && !version.starts_with(['+', '0']) => Ok(v), + _ => bail!("version {version:?} must be a positive integer, no leading zeros, at most {}", u32::MAX), + } +} + +/// `@`, the version under `check_version`. The version is part of the id, so it is +/// part of every answer key and no key can name a sample without one. pub fn check_sample_id(sample_id: &str) -> Result<(&str, u32)> { let Some((sample, version)) = sample_id.rsplit_once('@') else { bail!("sample id {sample_id:?} has no @version"); }; - match version.parse::() { - Ok(v) if v > 0 && !sample.is_empty() && !version.starts_with(['+', '0']) => Ok((sample, v)), - _ => bail!("sample id {sample_id:?} needs a non-empty sample and a positive integer version without leading zeros"), + if sample.is_empty() { + bail!("sample id {sample_id:?} needs a non-empty sample"); } + Ok((sample, check_version(version)?)) } /// The first reply a member posts under a notice version. Written once, never updated. @@ -207,7 +215,7 @@ pub struct MemberTerms { impl MemberTerms { pub fn validate(&self) -> Result<()> { nonempty("member_id", &self.member_id)?; - nonempty("terms_version", &self.terms_version) + check_version(&self.terms_version).map(drop) } /// `member_id`. diff --git a/packages/heimdall-consent/tests/golden.rs b/packages/heimdall-consent/tests/golden.rs index af6953b5..01e9477b 100644 --- a/packages/heimdall-consent/tests/golden.rs +++ b/packages/heimdall-consent/tests/golden.rs @@ -154,13 +154,9 @@ fn golden_vectors() { put_answer(&mut cache, &a.redacted(hash).unwrap()); } "terms" => { - cache - .put(MemberTerms::cache_key(member).unwrap(), &MemberTerms { - member_id: member.into(), - terms_version: payload.into(), - terms_accepted_at: ts, - }) - .unwrap(); + let terms = MemberTerms { member_id: member.into(), terms_version: payload.into(), terms_accepted_at: ts }; + terms.validate().unwrap(); + cache.put(MemberTerms::cache_key(member).unwrap(), &terms).unwrap(); } "expect_terms" => { let t = cache.get_required::(&MemberTerms::cache_key(member).unwrap()).unwrap(); @@ -226,6 +222,35 @@ fn key_refuses_unversioned_sample_ids() { assert!(Answer::cache_key("r", "s@1", "m").unwrap() != Answer::cache_key("r", "s@2", "m").unwrap()); } +#[test] +fn one_version_rule_everywhere() { + for ok in ["1", "2", "4294967295"] { + let n: u32 = ok.parse().unwrap(); + assert_eq!(check_version(ok).unwrap(), n, "{ok}"); + assert_eq!(check_sample_id(&format!("s@{ok}")).unwrap(), ("s", n), "s@{ok}"); + assert!(MemberTerms { member_id: "m".into(), terms_version: ok.into(), terms_accepted_at: 1 }.validate().is_ok(), "terms {ok}"); + } + for bad in ["0", "4294967296", "4294967297", "8589934593", "99999999999999999999", "01", "+1", "-1", "", "2026-11"] { + assert!(check_version(bad).is_err(), "{bad}"); + assert!(check_sample_id(&format!("s@{bad}")).is_err(), "s@{bad}"); + assert!(MemberTerms { member_id: "m".into(), terms_version: bad.into(), terms_accepted_at: 1 }.validate().is_err(), "terms {bad}"); + } +} + +#[test] +fn notice_version_zero_is_refused_in_both_records() { + let acc = NoticeAcceptance { + member_id: "m".into(), + experiment_id: "r".into(), + notice_version: 0, + scope: Scope::Train, + accepted_at: 1, + event_id: "e".into(), + }; + assert!(acc.validate().is_err()); + assert!(NoticeAcceptance { notice_version: u32::MAX, ..acc }.validate().is_ok()); +} + #[test] fn acceptance_after_the_answer_does_not_cover_it() { let a = Answer { diff --git a/packages/heimdall-consent/tests/vectors/consent_golden.csv b/packages/heimdall-consent/tests/vectors/consent_golden.csv index a7e5d553..1323aef8 100644 --- a/packages/heimdall-consent/tests/vectors/consent_golden.csv +++ b/packages/heimdall-consent/tests/vectors/consent_golden.csv @@ -25,8 +25,8 @@ redacted-reply,redact,m6,room1,,,s1@1,a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5 redacted-reply,expect_exited,m6,room1,,,s1@1,a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90,100,e9 redacted-reply,expect_train,,,,,,e10,, redacted-reply,expect_counts,,,,,,answers=2;acceptances=2,, -newcomer,terms,m8,,,,,2026-11,50, -newcomer,expect_terms,m8,,,,,2026-11,50, +newcomer,terms,m8,,,,,4294967295,50, +newcomer,expect_terms,m8,,,,,4294967295,50, newcomer,expect_no_terms,m9,,,,,,, newcomer,reply,m8,room1,3,train,s1@1,hello,100,e11 newcomer,reply,m9,room1,3,train,s1@1,veteran,110,e12 From a36a60b10604d2ba1b2fa325494cffc78a9732ac Mon Sep 17 00:00:00 2001 From: Libby Date: Sat, 7 Nov 2026 01:34:22 +0000 Subject: [PATCH 67/88] cultmesh-py: port CultMesh authority route verification ECDSA P-256 route certificate and session proof verification ported from CultMeshAuthorityProof.cs, with typed refusals, judged against the shared contracts/cultmesh authority route vectors (C# written and TS signed). Co-Authored-By: Claude Sonnet 5.5 --- packages/cultmesh-py/README.md | 11 + packages/cultmesh-py/pyproject.toml | 2 +- .../cultmesh-py/src/cultmesh_py/authority.py | 338 ++++++++++++++++++ .../tests/test_authority_route_vectors.py | 208 +++++++++++ 4 files changed, 558 insertions(+), 1 deletion(-) create mode 100644 packages/cultmesh-py/src/cultmesh_py/authority.py create mode 100644 packages/cultmesh-py/tests/test_authority_route_vectors.py diff --git a/packages/cultmesh-py/README.md b/packages/cultmesh-py/README.md index 63799e52..4a048dcc 100644 --- a/packages/cultmesh-py/README.md +++ b/packages/cultmesh-py/README.md @@ -228,6 +228,17 @@ prediction = session.predict(note_doc, "input:python:move", {"body": "predicted unsubscribe() ``` +## Authority route verification + +`cultmesh_py.authority` ports the C# reference +(`src/GameCult.Mesh/CultMeshAuthorityProof.cs`): `AuthorityTrustPolicy.validate` +judges an Odin-signed ECDSA P-256 route (channel protection, trusted root, +validity window, IEEE P1363 signature over the canonical bytes) and +`verify_session_proof` checks the provider nonce proof. Refusals raise +`AuthorityRouteRefusal` with the C# message text. It needs `cryptography`. +`tests/test_authority_route_vectors.py` judges +`contracts/cultmesh/authority-route-vectors*.json`. + ## Verify `cultmesh_py.verify` checks the public exports and `py.typed` markers of all diff --git a/packages/cultmesh-py/pyproject.toml b/packages/cultmesh-py/pyproject.toml index a1df5ee3..5f9bda4d 100644 --- a/packages/cultmesh-py/pyproject.toml +++ b/packages/cultmesh-py/pyproject.toml @@ -16,7 +16,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", ] -dependencies = ["cultcache-py>=0.2.0", "cultnet-py>=0.1.0"] +dependencies = ["cultcache-py>=0.2.0", "cultnet-py>=0.1.0", "cryptography>=41"] [project.scripts] cultmesh-py-daemon = "cultmesh_py.daemon:main" cultmesh-py-verify = "cultmesh_py.verify:main" diff --git a/packages/cultmesh-py/src/cultmesh_py/authority.py b/packages/cultmesh-py/src/cultmesh_py/authority.py new file mode 100644 index 00000000..3c6665ea --- /dev/null +++ b/packages/cultmesh-py/src/cultmesh_py/authority.py @@ -0,0 +1,338 @@ +"""CultMesh authority route verification. + +Port of `src/GameCult.Mesh/CultMeshAuthorityProof.cs` (the reference) and of the +route constructors in `CultMeshVerse.cs`. Rule for rule, in the same order, the +same input yields the same refusal text as `CultMeshAuthorityTrustPolicy.Verify`: +unsigned, channel protection, root lookup, validity window, signature. + +Shared vectors: `contracts/cultmesh/authority-route-vectors.json` (C# written). +""" + +from __future__ import annotations + +import base64 +import binascii +import re +import struct +from dataclasses import dataclass +from typing import Iterable, Mapping, Sequence + +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.hazmat.primitives.asymmetric.utils import encode_dss_signature + +ROUTE_DOMAIN = "gamecult.cultmesh.route-certificate.v1" +SESSION_DOMAIN = "gamecult.cultmesh.session-proof.v1" + +AUTHENTICATED_REMOTE = "authenticated-remote" +LOCAL_DEVELOPMENT = "local-development" + +# .NET `char.IsWhiteSpace`: Unicode Zs, Zl, Zp, U+0009-U+000D and U+0085. +# `str.strip()` is a different set (it also strips U+001C-U+001F). +_CSHARP_WHITESPACE = ( + "\u0009\u000a\u000b\u000c\u000d \u0085  " + "           " + "

  　" +) + + +def trim_csharp(value: str) -> str: + return value.strip(_CSHARP_WHITESPACE) + + +def is_blank_csharp(value: str | None) -> bool: + return value is None or trim_csharp(value) == "" + + +def _require(value: str | None, name: str) -> str: + if is_blank_csharp(value): + raise ValueError(f"Value must be non-empty. (Parameter '{name}')") + return trim_csharp(value) # type: ignore[arg-type] + + +class AuthorityRouteRefusal(Exception): + """A route or proof the consumer's trust policy refuses. + + Mirrors `CultMeshSessionException` carrying a `CultMeshSessionFailure` with + reason `Authentication`: `reason` is always "authentication". + """ + + reason = "authentication" + + def __init__(self, message: str, endpoint: str = "") -> None: + super().__init__(message) + self.message = message + self.endpoint = endpoint + + +def _b64_strict(value: str) -> bytes: + # `Convert.FromBase64String`: standard alphabet, padded, whitespace ignored. + stripped = re.sub(r"[ \t\r\n]", "", value) + return base64.b64decode(stripped, validate=True) + + +@dataclass(frozen=True) +class P256PublicKey: + key_id: str + x: str + y: str + + def __post_init__(self) -> None: + object.__setattr__(self, "key_id", _require(self.key_id, "keyId")) + for name in ("x", "y"): + value = _require(getattr(self, name), name) + try: + _b64_strict(value) + except (binascii.Error, ValueError) as error: + raise ValueError(f"Coordinate must be base64. (Parameter '{name}')") from error + object.__setattr__(self, name, value) + + def verify(self, payload: bytes, signature_base64: str) -> bool: + """IEEE P1363 (r || s) ECDSA-SHA256 verification; False on any malformation.""" + try: + x, y = _b64_strict(self.x), _b64_strict(self.y) + signature = _b64_strict(signature_base64) + if len(x) != 32 or len(y) != 32 or len(signature) != 64: + return False + key = ec.EllipticCurvePublicNumbers( + int.from_bytes(x, "big"), int.from_bytes(y, "big"), ec.SECP256R1() + ).public_key() + der = encode_dss_signature( + int.from_bytes(signature[:32], "big"), int.from_bytes(signature[32:], "big") + ) + key.verify(der, payload, ec.ECDSA(hashes.SHA256())) + return True + except (InvalidSignature, ValueError, binascii.Error): + return False + + +@dataclass(frozen=True) +class RouteCertificate: + provider_key: P256PublicKey + odin_key_id: str + issued_at_unix_milliseconds: int + expires_at_unix_milliseconds: int + signature: str = "" + + def __post_init__(self) -> None: + if is_blank_csharp(self.odin_key_id): + raise ValueError("Odin key identity is required. (Parameter 'odinKeyId')") + object.__setattr__(self, "odin_key_id", trim_csharp(self.odin_key_id)) + if self.issued_at_unix_milliseconds < 0: + raise ValueError("issuedAtUnixMilliseconds must not be negative.") + if self.expires_at_unix_milliseconds <= self.issued_at_unix_milliseconds: + raise ValueError("expiresAtUnixMilliseconds must be after issuedAtUnixMilliseconds.") + object.__setattr__(self, "signature", trim_csharp(self.signature or "")) + + +@dataclass(frozen=True) +class AuthorityRoute: + """`CultMeshAuthorityRoute`: construction cleans every field as the C# does.""" + + authority_runtime_id: str + endpoint: str + protocol_ids: Sequence[str] = () + priority: int = 0 + generation: str | None = None + certificate: RouteCertificate | None = None + + def __post_init__(self) -> None: + runtime_id = _require(self.authority_runtime_id, "authorityRuntimeId") + endpoint = _require(self.endpoint, "endpoint") + if self.priority < 0: + raise ValueError("priority must not be negative.") + object.__setattr__(self, "authority_runtime_id", runtime_id) + object.__setattr__(self, "endpoint", endpoint) + object.__setattr__( + self, + "protocol_ids", + tuple( + sorted( + {trim_csharp(v) for v in self.protocol_ids if not is_blank_csharp(v)}, + key=lambda v: v.encode("utf-16-be"), # StringComparer.Ordinal is UTF-16 code-unit order + ) + ), + ) + object.__setattr__( + self, + "generation", + f"{runtime_id}@{endpoint}" if is_blank_csharp(self.generation) else trim_csharp(self.generation), # type: ignore[arg-type] + ) + + +@dataclass(frozen=True) +class SessionOpen: + """The fields of `CultMeshSessionOpenMessage` the session proof covers.""" + + client_nonce: str | None = None + message_id: str | None = None + source_runtime_id: str | None = None + verse_id: str | None = None + authority_runtime_id: str | None = None + protocol_id: str | None = None + route_generation: str | None = None + + +@dataclass(frozen=True) +class SessionAccepted: + client_nonce: str | None + provider_key_id: str | None + provider_signature: str | None + + +@dataclass(frozen=True) +class VerifiedRoute: + route: AuthorityRoute + provider_key: P256PublicKey | None + is_local_development: bool + + +def canonical_fields(*values: str | None) -> bytes: + out = bytearray() + for value in values: + encoded = (value or "").encode("utf-8") + out += struct.pack(">I", len(encoded)) + encoded + return bytes(out) + + +def canonical_route(verse_id: str, route: AuthorityRoute) -> bytes: + certificate = route.certificate + if certificate is None: + raise ValueError("Route certificate is required.") + return canonical_fields( + ROUTE_DOMAIN, + verse_id, + route.authority_runtime_id, + route.endpoint, + "\u001f".join(route.protocol_ids), # already ordinally sorted; Python sorts by code point + str(route.priority), + route.generation, + certificate.provider_key.key_id, + certificate.provider_key.x, + certificate.provider_key.y, + certificate.odin_key_id, + str(certificate.issued_at_unix_milliseconds), + str(certificate.expires_at_unix_milliseconds), + ) + + +def canonical_session(request: SessionOpen, endpoint: str) -> bytes: + return canonical_fields( + SESSION_DOMAIN, + request.client_nonce, + request.message_id, + request.source_runtime_id, + request.verse_id, + request.authority_runtime_id, + request.protocol_id, + endpoint, + request.route_generation, + ) + + +# `System.Uri` refuses spellings WHATWG parsers repair; a lenient parse could call +# a refused endpoint loopback or protected. Same shape rule as the TypeScript port. +_AUTHORITY = re.compile(r"^[A-Za-z][A-Za-z0-9+\-.]*://([^/?#]+)") + + +def _split_endpoint(value: str) -> tuple[str, str] | None: + """Return (scheme, lowercase host) or None where System.Uri would refuse.""" + raw = re.sub(r"^[\x00-\x20]+|[\x00-\x20]+$", "", value) + if "\\" in raw: + return None + shape = _AUTHORITY.match(raw) + if not shape: + return None + authority = shape.group(1) + after_user = authority[authority.rfind("@") + 1 :] + if after_user.startswith("["): + host = after_user[: after_user.find("]") + 1] + else: + host = after_user.split(":")[0] + if not host or re.search(r"[^!-~]", host) or host.endswith("."): + return None + return raw.split("://", 1)[0].lower(), host.strip("[]").lower() + + +def is_loopback_endpoint(endpoint: str) -> bool: + parts = _split_endpoint(endpoint) + if parts is None: + return False + host = parts[1] + return host in ("localhost", "loopback", "::1", "::ffff:127.0.0.1") or bool( + re.fullmatch(r"127\.\d{1,3}\.\d{1,3}\.\d{1,3}", host) + ) + + +def is_protected_endpoint(endpoint: str) -> bool: + parts = _split_endpoint(endpoint) + return parts is not None and (parts[0] in ("wss", "https") or "quic" in parts[0]) + + +class AuthorityTrustPolicy: + """Consumer-owned policy. Fails closed unless a route is valid under it.""" + + def __init__(self, mode: str, odin_roots: Iterable[P256PublicKey] = ()) -> None: + if mode not in (AUTHENTICATED_REMOTE, LOCAL_DEVELOPMENT): + raise ValueError(f"Unknown trust mode '{mode}'.") + self.mode = mode + self._roots: dict[str, P256PublicKey] = {} + for root in odin_roots: + if root.key_id in self._roots: + raise ValueError(f"Odin root key id '{root.key_id}' is listed more than once in the trust policy.") + self._roots[root.key_id] = root + + def validate(self, verse_id: str, route: AuthorityRoute, now_unix_milliseconds: int) -> None: + self.verify(verse_id, route, now_unix_milliseconds) + + def verify(self, verse_id: str, route: AuthorityRoute, now_unix_milliseconds: int) -> VerifiedRoute: + local = self.mode == LOCAL_DEVELOPMENT and is_loopback_endpoint(route.endpoint) + + def refuse(message: str) -> AuthorityRouteRefusal: + return AuthorityRouteRefusal(message, route.endpoint) + + certificate = route.certificate + if certificate is None or is_blank_csharp(certificate.signature): + if local: + return VerifiedRoute(route, None, True) + raise refuse("Remote CultMesh routes require an Odin-signed authority certificate.") + if not is_protected_endpoint(route.endpoint) and not local: + raise refuse("Authenticated remote CultMesh routes require TLS or QUIC channel protection.") + root = self._roots.get(certificate.odin_key_id) + if root is None: + raise refuse(f"Odin key '{certificate.odin_key_id}' is not trusted by this consumer.") + if not (certificate.issued_at_unix_milliseconds <= now_unix_milliseconds < certificate.expires_at_unix_milliseconds): + raise refuse("The Odin route certificate is not currently valid.") + try: + signature = _b64_strict(certificate.signature) + except (binascii.Error, ValueError): + raise refuse("The Odin route signature is not base64.") from None + if len(signature) != 64: + raise refuse("The Odin route signature is not IEEE P1363 P-256.") + if not root.verify(canonical_route(verse_id, route), certificate.signature): + raise refuse("The Odin route certificate signature is invalid.") + return VerifiedRoute(route, certificate.provider_key, False) + + +def verify_session_proof( + request: SessionOpen, + response: SessionAccepted, + verse_id: str, + route: AuthorityRoute, + trust: AuthorityTrustPolicy, + now_unix_milliseconds: int, +) -> bool: + """Verifies the Odin route certificate (raising) and the provider nonce proof.""" + verified = trust.verify(verse_id, route, now_unix_milliseconds) + if verified.is_local_development: + return True + key = verified.provider_key + if ( + key is None + or response.provider_key_id != key.key_id + or response.client_nonce != request.client_nonce + or is_blank_csharp(response.provider_signature) + ): + return False + return key.verify(canonical_session(request, route.endpoint), response.provider_signature) # type: ignore[arg-type] diff --git a/packages/cultmesh-py/tests/test_authority_route_vectors.py b/packages/cultmesh-py/tests/test_authority_route_vectors.py new file mode 100644 index 00000000..8f3d2214 --- /dev/null +++ b/packages/cultmesh-py/tests/test_authority_route_vectors.py @@ -0,0 +1,208 @@ +"""Judges `contracts/cultmesh/authority-route-vectors*.json` in Python. + +`authority-route-vectors.json` is written by the C# reference (signatures are +captured C# bytes); the `.ts-signed.json` file is signed by TypeScript over awkward +inputs. Each negative case fails when the matching check is removed from +`cultmesh_py.authority`. +""" + +from __future__ import annotations + +import base64 +import json +import unittest +from dataclasses import replace +from pathlib import Path + +from cryptography.hazmat.primitives.asymmetric import ec + +from cultmesh_py.authority import ( + AUTHENTICATED_REMOTE, + LOCAL_DEVELOPMENT, + AuthorityRoute, + AuthorityRouteRefusal, + AuthorityTrustPolicy, + P256PublicKey, + RouteCertificate, + SessionAccepted, + SessionOpen, + canonical_route, + verify_session_proof, +) + +CONTRACTS = Path(__file__).resolve().parents[3] / "contracts" / "cultmesh" + + +def _key(element: dict) -> P256PublicKey: + return P256PublicKey(element["keyId"], element["x"], element["y"]) + + +class Vector: + def __init__(self, name: str) -> None: + doc = json.loads((CONTRACTS / name).read_text(encoding="utf-8")) + self.now = doc["nowUnixMilliseconds"] + self.root = _key(doc["odinRoot"]) + r = doc["route"] + self.verse_id = r["verseId"] + c = r["certificate"] + self.certificate = RouteCertificate( + _key(c["providerKey"]), c["odinKeyId"], c["issuedAtUnixMilliseconds"], + c["expiresAtUnixMilliseconds"], c["signature"], + ) + self.route = AuthorityRoute( + r["authorityRuntimeId"], r["endpoint"], r["protocolIds"], r["priority"], + r["generation"], self.certificate, + ) + p = doc["sessionProof"] + q = p["request"] + self.request = SessionOpen( + q["clientNonce"], q["messageId"], q["sourceRuntimeId"], q["verseId"], + q["authorityRuntimeId"], q["protocolId"], q["routeGeneration"], + ) + self.accepted = SessionAccepted(q["clientNonce"], p["providerKeyId"], p["providerSignature"]) + self.trust = AuthorityTrustPolicy(AUTHENTICATED_REMOTE, [self.root]) + + def with_certificate(self, **changes) -> AuthorityRoute: + return replace(self.route, certificate=replace(self.certificate, **changes)) + + +def refusal(test: unittest.TestCase, fn) -> str: + with test.assertRaises(AuthorityRouteRefusal) as raised: + fn() + test.assertEqual("authentication", raised.exception.reason) + return raised.exception.message + + +class AuthorityRouteVectorTests(unittest.TestCase): + def test_cs_written_vector_verifies_route_and_session_proof(self) -> None: + v = Vector("authority-route-vectors.json") + v.trust.validate(v.verse_id, v.route, v.now) + self.assertTrue(verify_session_proof(v.request, v.accepted, v.verse_id, v.route, v.trust, v.now)) + + def test_ts_signed_vector_verifies_and_tamper_is_refused(self) -> None: + v = Vector("authority-route-vectors.ts-signed.json") + v.trust.validate(v.verse_id, v.route, v.now) + self.assertTrue(verify_session_proof(v.request, v.accepted, v.verse_id, v.route, v.trust, v.now)) + tampered = replace(v.route, priority=v.route.priority + 1) + self.assertEqual( + "The Odin route certificate signature is invalid.", + refusal(self, lambda: v.trust.validate(v.verse_id, tampered, v.now)), + ) + + def test_canonical_route_is_length_prefixed_and_sorted(self) -> None: + v = Vector("authority-route-vectors.json") + data = canonical_route(v.verse_id, v.route) + self.assertTrue(data.startswith(b"\x00\x00\x00\x26gamecult.cultmesh.route-certificate.v1\x00\x00\x00\x08aetheria")) + self.assertIn(b"cultmesh.content.v1\x1fcultmesh.documents.v1", data) + + def test_tampered_payload_fields_are_refused(self) -> None: + v = Vector("authority-route-vectors.json") + invalid = "The Odin route certificate signature is invalid." + for tampered, verse in ( + (replace(v.route, priority=8), v.verse_id), + (replace(v.route, endpoint="wss://evil.example/mesh"), v.verse_id), + (replace(v.route, protocol_ids=("cultmesh.documents.v1",)), v.verse_id), + (replace(v.route, generation="other"), v.verse_id), + (v.with_certificate(expires_at_unix_milliseconds=v.certificate.expires_at_unix_milliseconds + 1), v.verse_id), + (v.route, "other-verse"), + ): + self.assertEqual(invalid, refusal(self, lambda: v.trust.validate(verse, tampered, v.now))) + + def test_wrong_root_key_is_refused(self) -> None: + v = Vector("authority-route-vectors.json") + numbers = ec.generate_private_key(ec.SECP256R1()).public_key().public_numbers() + impostor = P256PublicKey( + v.root.key_id, + base64.b64encode(numbers.x.to_bytes(32, "big")).decode(), + base64.b64encode(numbers.y.to_bytes(32, "big")).decode(), + ) + trust = AuthorityTrustPolicy(AUTHENTICATED_REMOTE, [impostor]) + self.assertEqual( + "The Odin route certificate signature is invalid.", + refusal(self, lambda: trust.validate(v.verse_id, v.route, v.now)), + ) + + def test_untrusted_odin_key_id_is_refused(self) -> None: + v = Vector("authority-route-vectors.json") + trust = AuthorityTrustPolicy(AUTHENTICATED_REMOTE, [replace(v.root, key_id="someone-else")]) + self.assertEqual( + "Odin key 'odin-vector' is not trusted by this consumer.", + refusal(self, lambda: trust.validate(v.verse_id, v.route, v.now)), + ) + + def test_expired_and_not_yet_valid_are_refused(self) -> None: + v = Vector("authority-route-vectors.json") + window = "The Odin route certificate is not currently valid." + expires = v.certificate.expires_at_unix_milliseconds + issued = v.certificate.issued_at_unix_milliseconds + v.trust.validate(v.verse_id, v.route, expires - 1) + v.trust.validate(v.verse_id, v.route, issued) + self.assertEqual(window, refusal(self, lambda: v.trust.validate(v.verse_id, v.route, expires))) + self.assertEqual(window, refusal(self, lambda: v.trust.validate(v.verse_id, v.route, issued - 1))) + + def test_unsigned_and_unprotected_routes_are_refused(self) -> None: + v = Vector("authority-route-vectors.json") + unsigned = replace(v.route, certificate=None) + self.assertEqual( + "Remote CultMesh routes require an Odin-signed authority certificate.", + refusal(self, lambda: v.trust.validate(v.verse_id, unsigned, v.now)), + ) + blank = v.with_certificate(signature=" ") + self.assertEqual( + "Remote CultMesh routes require an Odin-signed authority certificate.", + refusal(self, lambda: v.trust.validate(v.verse_id, blank, v.now)), + ) + plain = replace(v.route, endpoint="ws://provider.example/mesh") + self.assertEqual( + "Authenticated remote CultMesh routes require TLS or QUIC channel protection.", + refusal(self, lambda: v.trust.validate(v.verse_id, plain, v.now)), + ) + + def test_local_development_admits_only_unsigned_loopback(self) -> None: + v = Vector("authority-route-vectors.json") + dev = AuthorityTrustPolicy(LOCAL_DEVELOPMENT, []) + loop = AuthorityRoute("a", "ws://127.0.0.1:9/mesh") + dev.validate(v.verse_id, loop, v.now) + self.assertTrue(verify_session_proof(v.request, v.accepted, v.verse_id, loop, dev, v.now)) + for endpoint in ("ws://provider.example/mesh", "ws://127.0.0.1./mesh", "ws:127.0.0.1"): + refusal(self, lambda e=endpoint: dev.validate(v.verse_id, AuthorityRoute("a", e), v.now)) + authenticated = AuthorityTrustPolicy(AUTHENTICATED_REMOTE, []) + refusal(self, lambda: authenticated.validate(v.verse_id, loop, v.now)) + + def test_malformed_signatures_get_their_own_refusals(self) -> None: + v = Vector("authority-route-vectors.json") + not_base64 = v.with_certificate(signature="!!not base64!!") + self.assertEqual( + "The Odin route signature is not base64.", + refusal(self, lambda: v.trust.validate(v.verse_id, not_base64, v.now)), + ) + short = v.with_certificate(signature=base64.b64encode(b"x" * 63).decode()) + self.assertEqual( + "The Odin route signature is not IEEE P1363 P-256.", + refusal(self, lambda: v.trust.validate(v.verse_id, short, v.now)), + ) + + def test_session_proof_rejects_replay_wrong_key_and_forgery(self) -> None: + v = Vector("authority-route-vectors.json") + replayed = replace(v.request, client_nonce="replayed") + accepted = replace(v.accepted, client_nonce="replayed") + self.assertFalse(verify_session_proof(replayed, accepted, v.verse_id, v.route, v.trust, v.now)) + self.assertFalse(verify_session_proof( + v.request, replace(v.accepted, provider_key_id="other"), v.verse_id, v.route, v.trust, v.now)) + self.assertFalse(verify_session_proof( + v.request, replace(v.accepted, client_nonce="other"), v.verse_id, v.route, v.trust, v.now)) + self.assertFalse(verify_session_proof( + v.request, replace(v.accepted, provider_signature=""), v.verse_id, v.route, v.trust, v.now)) + raw = bytearray(base64.b64decode(v.accepted.provider_signature)) + raw[0] ^= 1 + forged = replace(v.accepted, provider_signature=base64.b64encode(bytes(raw)).decode()) + self.assertFalse(verify_session_proof(v.request, forged, v.verse_id, v.route, v.trust, v.now)) + moved = replace(v.request, protocol_id="cultmesh.content.v1") + self.assertFalse(verify_session_proof(moved, v.accepted, v.verse_id, v.route, v.trust, v.now)) + # An expired route fails closed even with a good proof. + refusal(self, lambda: verify_session_proof( + v.request, v.accepted, v.verse_id, v.route, v.trust, v.certificate.expires_at_unix_milliseconds)) + + +if __name__ == "__main__": + unittest.main() From 7b62cf4ddf9f24cfb4c5878260a2f0ea26d13f70 Mon Sep 17 00:00:00 2001 From: Libby Date: Sat, 7 Nov 2026 01:35:54 +0000 Subject: [PATCH 68/88] Capture the reference's selection/v1 carrier and error code/details bytes as interop vectors Co-Authored-By: Claude Sonnet 5.5 --- .../interop/carrier-vectors.cs-written.json | 350 ++++++++++++++++++ .../CarrierVectorTests.cs | 161 ++++++++ 2 files changed, 511 insertions(+) create mode 100644 contracts/cultnet/interop/carrier-vectors.cs-written.json create mode 100644 tests/GameCult.Networking.Tests/CarrierVectorTests.cs diff --git a/contracts/cultnet/interop/carrier-vectors.cs-written.json b/contracts/cultnet/interop/carrier-vectors.cs-written.json new file mode 100644 index 00000000..49f12ed0 --- /dev/null +++ b/contracts/cultnet/interop/carrier-vectors.cs-written.json @@ -0,0 +1,350 @@ +{ + "vectors": [ + { + "name": "snapshot_request_v1_defaults", + "schemaVersion": "cultnet.snapshot_request.v1", + "messagePackBase64": "ha1zY2hlbWFWZXJzaW9uu2N1bHRuZXQuc25hcHNob3RfcmVxdWVzdC52MaltZXNzYWdlSWShbalzZWxlY3Rpb26Jp3NjaGVtYXPApGtleXPApmZpZWxkc8ClY2l0ZXPApWNpdGVkwKpwcm9qZWN0aW9upmhlYWRlcqpkZXNjZW5kaW5nwqVsaW1pdMCmY3Vyc29ywKdzaGFyZElkwKpzaGFyZEVwb2NowA==", + "expected": { + "schemaVersion": "cultnet.snapshot_request.v1", + "messageId": "m", + "selection": { + "schemas": null, + "keys": null, + "fields": null, + "cites": null, + "cited": null, + "projection": "header", + "descending": false, + "limit": null, + "cursor": null, + "hasHop": false + }, + "shardId": null, + "shardEpoch": null + } + }, + { + "name": "snapshot_request_v1_full_selection_and_shard", + "schemaVersion": "cultnet.snapshot_request.v1", + "messagePackBase64": "ha1zY2hlbWFWZXJzaW9uu2N1bHRuZXQuc25hcHNob3RfcmVxdWVzdC52MaltZXNzYWdlSWSmbS1mdWxsqXNlbGVjdGlvbomnc2NoZW1hc5KpbGVhZl9hLnY52UdzaGEyNTY6ZmYzNWYzZDlkNmRlM2YxMDY2YjkzNjllNzNmZTZjNzY0OGU4ODM2OGQxOGVmYzIyNzVjOTk0NzMwY2Q4NGNmZqRrZXlzkqRhLWVxqPCfmIAtcm93pmZpZWxkc5KEpWluZGV4pGtpbmSib3CmYW55X29mpnZhbHVlc5Kmd2VhcG9upHRvb2ymbnVtYmVywISlaW5kZXikbWFzc6JvcKJnZaZ2YWx1ZXPApm51bWJlcqstMjIzMzc1OS4yNaVjaXRlc4KmdGFyZ2V0gqhzY2hlbWFJZNlHc2hhMjU2OmZmMzVmM2Q5ZDZkZTNmMTA2NmI5MzY5ZTczZmU2Yzc2NDhlODgzNjhkMThlZmMyMjc1Yzk5NDczMGNkODRjZmapcmVjb3JkS2V5pGEtZXGkcm9sZaZEZXNpZ26lY2l0ZWSCpHJvbGWmRGVzaWdupmV4aXN0c8KqcHJvamVjdGlvbqhkb2N1bWVudKpkZXNjZW5kaW5nw6VsaW1pdMzIpmN1cnNvcq1vcGFxdWUtY3Vyc29yp3NoYXJkSWSnc2hhcmQtMapzaGFyZEVwb2NoBw==", + "expected": { + "schemaVersion": "cultnet.snapshot_request.v1", + "messageId": "m-full", + "selection": { + "schemas": [ + "leaf_a.v9", + "sha256:ff35f3d9d6de3f1066b9369e73fe6c7648e88368d18efc2275c994730cd84cff" + ], + "keys": [ + "a-eq", + "\uD83D\uDE00-row" + ], + "fields": [ + { + "index": "kind", + "op": "any_of", + "values": [ + "weapon", + "tool" + ], + "number": null + }, + { + "index": "mass", + "op": "ge", + "values": null, + "number": "-2233759.25" + } + ], + "cites": { + "target": { + "schemaId": "sha256:ff35f3d9d6de3f1066b9369e73fe6c7648e88368d18efc2275c994730cd84cff", + "recordKey": "a-eq" + }, + "role": "Design" + }, + "cited": { + "role": "Design", + "exists": false + }, + "projection": "document", + "descending": true, + "limit": 200, + "cursor": "opaque-cursor", + "hasHop": true + }, + "shardId": "shard-1", + "shardEpoch": 7 + } + }, + { + "name": "database_subscribe_v1_defaults", + "schemaVersion": "cultnet.database_subscribe.v1", + "messagePackBase64": "iK1zY2hlbWFWZXJzaW9uvWN1bHRuZXQuZGF0YWJhc2Vfc3Vic2NyaWJlLnYxqW1lc3NhZ2VJZKFtrnN1YnNjcmlwdGlvbklko3N1YqlzZWxlY3Rpb26Jp3NjaGVtYXPApGtleXPApmZpZWxkc8ClY2l0ZXPApWNpdGVkwKpwcm9qZWN0aW9upmhlYWRlcqpkZXNjZW5kaW5nwqVsaW1pdMCmY3Vyc29ywK9pbmNsdWRlU25hcHNob3TDsWNvbnN1bWVyUnVudGltZUlkwKdib2R5SWRzwLdzdXBwb3J0ZWRCb2R5VHJhbnNwb3J0c8A=", + "expected": { + "schemaVersion": "cultnet.database_subscribe.v1", + "messageId": "m", + "subscriptionId": "sub", + "selection": { + "schemas": null, + "keys": null, + "fields": null, + "cites": null, + "cited": null, + "projection": "header", + "descending": false, + "limit": null, + "cursor": null, + "hasHop": false + }, + "includeSnapshot": true, + "consumerRuntimeId": null, + "bodyIds": null, + "supportedBodyTransports": null + } + }, + { + "name": "database_subscribe_v1_full", + "schemaVersion": "cultnet.database_subscribe.v1", + "messagePackBase64": "iK1zY2hlbWFWZXJzaW9uvWN1bHRuZXQuZGF0YWJhc2Vfc3Vic2NyaWJlLnYxqW1lc3NhZ2VJZKZtLWZ1bGyuc3Vic2NyaXB0aW9uSWSoc3ViLWZ1bGypc2VsZWN0aW9uiadzY2hlbWFzkqlsZWFmX2EudjnZR3NoYTI1NjpmZjM1ZjNkOWQ2ZGUzZjEwNjZiOTM2OWU3M2ZlNmM3NjQ4ZTg4MzY4ZDE4ZWZjMjI3NWM5OTQ3MzBjZDg0Y2ZmpGtleXOSpGEtZXGo8J\u002BYgC1yb3emZmllbGRzkoSlaW5kZXika2luZKJvcKZhbnlfb2amdmFsdWVzkqZ3ZWFwb26kdG9vbKZudW1iZXLAhKVpbmRleKRtYXNzom9womdlpnZhbHVlc8CmbnVtYmVyqy0yMjMzNzU5LjI1pWNpdGVzgqZ0YXJnZXSCqHNjaGVtYUlk2UdzaGEyNTY6ZmYzNWYzZDlkNmRlM2YxMDY2YjkzNjllNzNmZTZjNzY0OGU4ODM2OGQxOGVmYzIyNzVjOTk0NzMwY2Q4NGNmZqlyZWNvcmRLZXmkYS1lcaRyb2xlpkRlc2lnbqVjaXRlZIKkcm9sZaZEZXNpZ26mZXhpc3Rzwqpwcm9qZWN0aW9uqGRvY3VtZW50qmRlc2NlbmRpbmfDpWxpbWl0zMimY3Vyc29yrW9wYXF1ZS1jdXJzb3KvaW5jbHVkZVNuYXBzaG90wrFjb25zdW1lclJ1bnRpbWVJZKhjb25zdW1lcqdib2R5SWRzkaZib2R5LWG3c3VwcG9ydGVkQm9keVRyYW5zcG9ydHOSpHJ1ZHCkcXVpYw==", + "expected": { + "schemaVersion": "cultnet.database_subscribe.v1", + "messageId": "m-full", + "subscriptionId": "sub-full", + "selection": { + "schemas": [ + "leaf_a.v9", + "sha256:ff35f3d9d6de3f1066b9369e73fe6c7648e88368d18efc2275c994730cd84cff" + ], + "keys": [ + "a-eq", + "\uD83D\uDE00-row" + ], + "fields": [ + { + "index": "kind", + "op": "any_of", + "values": [ + "weapon", + "tool" + ], + "number": null + }, + { + "index": "mass", + "op": "ge", + "values": null, + "number": "-2233759.25" + } + ], + "cites": { + "target": { + "schemaId": "sha256:ff35f3d9d6de3f1066b9369e73fe6c7648e88368d18efc2275c994730cd84cff", + "recordKey": "a-eq" + }, + "role": "Design" + }, + "cited": { + "role": "Design", + "exists": false + }, + "projection": "document", + "descending": true, + "limit": 200, + "cursor": "opaque-cursor", + "hasHop": true + }, + "includeSnapshot": false, + "consumerRuntimeId": "consumer", + "bodyIds": [ + "body-a" + ], + "supportedBodyTransports": [ + "rudp", + "quic" + ] + } + }, + { + "name": "snapshot_response_raw_v1_headers", + "schemaVersion": "cultnet.snapshot_response_raw.v1", + "messagePackBase64": "i61zY2hlbWFWZXJzaW9u2SBjdWx0bmV0LnNuYXBzaG90X3Jlc3BvbnNlX3Jhdy52MaltZXNzYWdlSWShbadtYXRjaGVkAqRhc09mBaRuZXh0q25leHQtY3Vyc29yp2hlYWRlcnOSiqhzY2hlbWFJZNlHc2hhMjU2OmZmMzVmM2Q5ZDZkZTNmMTA2NmI5MzY5ZTczZmU2Yzc2NDhlODgzNjhkMThlZmMyMjc1Yzk5NDczMGNkODRjZmaqc2NoZW1hTmFtZaZsZWFmX2Gtc2NoZW1hVmVyc2lvbqExsXNjaGVtYUNvbnRlbnRIYXNoo2FiY6lyZWNvcmRLZXmkYS1lcahzdG9yZWRBdLgxOTcwLTAxLTAxVDAwOjAwOjAwLjAwMFqvc291cmNlUnVudGltZUlkonJ0rXNvdXJjZUFnZW50SWSlYWdlbnSqc291cmNlUm9sZaRyb2xlpHRhZ3PAiqhzY2hlbWFJZNlHc2hhMjU2OmZmMzVmM2Q5ZDZkZTNmMTA2NmI5MzY5ZTczZmU2Yzc2NDhlODgzNjhkMThlZmMyMjc1Yzk5NDczMGNkODRjZmaqc2NoZW1hTmFtZaZsZWFmX2Gtc2NoZW1hVmVyc2lvbqExsXNjaGVtYUNvbnRlbnRIYXNoo2FiY6lyZWNvcmRLZXmkYS1sb6hzdG9yZWRBdLgxOTcwLTAxLTAxVDAwOjAwOjAwLjAwMFqvc291cmNlUnVudGltZUlkonJ0rXNvdXJjZUFnZW50SWSlYWdlbnSqc291cmNlUm9sZaRyb2xlpHRhZ3OSpWFscGhhpGJldGGpZG9jdW1lbnRzwKVlZGdlc8Cnc2hhcmRJZKdzaGFyZC0xqnNoYXJkRXBvY2gDsHNoYXJkTG9nU2VxdWVuY2Up", + "expected": { + "schemaVersion": "cultnet.snapshot_response_raw.v1", + "messageId": "m", + "matched": 2, + "asOf": 5, + "next": "next-cursor", + "headers": [ + { + "schemaId": "sha256:ff35f3d9d6de3f1066b9369e73fe6c7648e88368d18efc2275c994730cd84cff", + "schemaName": "leaf_a", + "schemaVersion": "1", + "schemaContentHash": "abc", + "recordKey": "a-eq", + "storedAt": "1970-01-01T00:00:00.000Z", + "sourceRuntimeId": "rt", + "sourceAgentId": "agent", + "sourceRole": "role", + "tags": null + }, + { + "schemaId": "sha256:ff35f3d9d6de3f1066b9369e73fe6c7648e88368d18efc2275c994730cd84cff", + "schemaName": "leaf_a", + "schemaVersion": "1", + "schemaContentHash": "abc", + "recordKey": "a-lo", + "storedAt": "1970-01-01T00:00:00.000Z", + "sourceRuntimeId": "rt", + "sourceAgentId": "agent", + "sourceRole": "role", + "tags": [ + "alpha", + "beta" + ] + } + ], + "documents": null, + "edges": null, + "shardId": "shard-1", + "shardEpoch": 3, + "shardLogSequence": 41 + } + }, + { + "name": "snapshot_response_raw_v1_documents_and_edges", + "schemaVersion": "cultnet.snapshot_response_raw.v1", + "messagePackBase64": "i61zY2hlbWFWZXJzaW9u2SBjdWx0bmV0LnNuYXBzaG90X3Jlc3BvbnNlX3Jhdy52MaltZXNzYWdlSWShbadtYXRjaGVkAaRhc09mBaRuZXh0wKdoZWFkZXJzwKlkb2N1bWVudHORjKhzY2hlbWFJZNlHc2hhMjU2OmZmMzVmM2Q5ZDZkZTNmMTA2NmI5MzY5ZTczZmU2Yzc2NDhlODgzNjhkMThlZmMyMjc1Yzk5NDczMGNkODRjZmaqc2NoZW1hTmFtZcCtc2NoZW1hVmVyc2lvbsCxc2NoZW1hQ29udGVudEhhc2jAqXJlY29yZEtleaRhLWVxqHN0b3JlZEF0uDE5NzAtMDEtMDFUMDA6MDA6MDAuMDAwWq9wYXlsb2FkRW5jb2RpbmerbWVzc2FnZXBhY2uncGF5bG9hZMQEgaFrAa9zb3VyY2VSdW50aW1lSWTArXNvdXJjZUFnZW50SWTAqnNvdXJjZVJvbGXApHRhZ3PApWVkZ2VzkoWkZnJvbYKoc2NoZW1hSWTZR3NoYTI1Njo5ZDBiYzlhZWJjMzYxOWIwN2FhYmY1NjRmOTg2NjdhZmRlNDE0NDRjMWI4NjY2NjFkNzc3Njg3ZDQ0MzYwOGRjqXJlY29yZEtleadjaXRlci0xpHJvbGWmRGVzaWduonRvgqhzY2hlbWFJZNlHc2hhMjU2OmZmMzVmM2Q5ZDZkZTNmMTA2NmI5MzY5ZTczZmU2Yzc2NDhlODgzNjhkMThlZmMyMjc1Yzk5NDczMGNkODRjZmapcmVjb3JkS2V5pGEtZXGvcGF5bG9hZEVuY29kaW5nwKdwYXlsb2FkwIWkZnJvbYKoc2NoZW1hSWTZR3NoYTI1Njo5ZDBiYzlhZWJjMzYxOWIwN2FhYmY1NjRmOTg2NjdhZmRlNDE0NDRjMWI4NjY2NjFkNzc3Njg3ZDQ0MzYwOGRjqXJlY29yZEtleadjaXRlci0xpHJvbGWmRGVzaWduonRvgqhzY2hlbWFJZNlHc2hhMjU2OmZmMzVmM2Q5ZDZkZTNmMTA2NmI5MzY5ZTczZmU2Yzc2NDhlODgzNjhkMThlZmMyMjc1Yzk5NDczMGNkODRjZmapcmVjb3JkS2V5pGEtZXGvcGF5bG9hZEVuY29kaW5nq21lc3NhZ2VwYWNrp3BheWxvYWTEAwECA6dzaGFyZElkwKpzaGFyZEVwb2NowLBzaGFyZExvZ1NlcXVlbmNlwA==", + "expected": { + "schemaVersion": "cultnet.snapshot_response_raw.v1", + "messageId": "m", + "matched": 1, + "asOf": 5, + "next": null, + "headers": null, + "documents": [ + { + "schemaId": "sha256:ff35f3d9d6de3f1066b9369e73fe6c7648e88368d18efc2275c994730cd84cff", + "schemaName": null, + "schemaVersion": null, + "schemaContentHash": null, + "recordKey": "a-eq", + "storedAt": "1970-01-01T00:00:00.000Z", + "payloadEncoding": "messagepack", + "payload": "gaFrAQ==", + "sourceRuntimeId": null, + "sourceAgentId": null, + "sourceRole": null, + "tags": null + } + ], + "edges": [ + { + "from": { + "schemaId": "sha256:9d0bc9aebc3619b07aabf564f98667afde41444c1b866661d777687d443608dc", + "recordKey": "citer-1" + }, + "role": "Design", + "to": { + "schemaId": "sha256:ff35f3d9d6de3f1066b9369e73fe6c7648e88368d18efc2275c994730cd84cff", + "recordKey": "a-eq" + }, + "payloadEncoding": null, + "payload": null + }, + { + "from": { + "schemaId": "sha256:9d0bc9aebc3619b07aabf564f98667afde41444c1b866661d777687d443608dc", + "recordKey": "citer-1" + }, + "role": "Design", + "to": { + "schemaId": "sha256:ff35f3d9d6de3f1066b9369e73fe6c7648e88368d18efc2275c994730cd84cff", + "recordKey": "a-eq" + }, + "payloadEncoding": "messagepack", + "payload": "AQID" + } + ], + "shardId": null, + "shardEpoch": null, + "shardLogSequence": null + } + }, + { + "name": "error_plain", + "schemaVersion": "cultnet.error.v0", + "messagePackBase64": "ha1zY2hlbWFWZXJzaW9usGN1bHRuZXQuZXJyb3IudjClZXJyb3KkYm9vbatyb3V0aW5nSGludMCkY29kZcCnZGV0YWlsc8A=", + "expected": { + "schemaVersion": "cultnet.error.v0", + "error": "boom", + "routingHint": null, + "code": null, + "details": null + } + }, + { + "name": "error_selection_invalid", + "schemaVersion": "cultnet.error.v0", + "messagePackBase64": "ha1zY2hlbWFWZXJzaW9usGN1bHRuZXQuZXJyb3IudjClZXJyb3LZOnNlbGVjdGlvbl9pbnZhbGlkOiBmaWVsZHNbMF0uaW5kZXggImtpbmQiIGlzIG5vdCBkZWNsYXJlZC6rcm91dGluZ0hpbnTApGNvZGWxc2VsZWN0aW9uX2ludmFsaWSnZGV0YWlsc4SlZmllbGSvZmllbGRzWzBdLmluZGV4pXZhbHVlpGtpbmSkYXNPZsCnY3VycmVudMA=", + "expected": { + "schemaVersion": "cultnet.error.v0", + "error": "selection_invalid: fields[0].index \u0022kind\u0022 is not declared.", + "routingHint": null, + "code": "selection_invalid", + "details": { + "field": "fields[0].index", + "value": "kind", + "asOf": null, + "current": null + } + } + }, + { + "name": "error_cursor_stale", + "schemaVersion": "cultnet.error.v0", + "messagePackBase64": "ha1zY2hlbWFWZXJzaW9usGN1bHRuZXQuZXJyb3IudjClZXJyb3KzY3Vyc29yX3N0YWxlOiBzdGFsZatyb3V0aW5nSGludMCkY29kZaxjdXJzb3Jfc3RhbGWnZGV0YWlsc4SlZmllbGTApXZhbHVlwKRhc09mA6djdXJyZW50CQ==", + "expected": { + "schemaVersion": "cultnet.error.v0", + "error": "cursor_stale: stale", + "routingHint": null, + "code": "cursor_stale", + "details": { + "field": null, + "value": null, + "asOf": 3, + "current": 9 + } + } + }, + { + "name": "error_cursor_invalid_no_details", + "schemaVersion": "cultnet.error.v0", + "messagePackBase64": "ha1zY2hlbWFWZXJzaW9usGN1bHRuZXQuZXJyb3IudjClZXJyb3KzY3Vyc29yX2ludmFsaWQ6IGJhZKtyb3V0aW5nSGludMCkY29kZa5jdXJzb3JfaW52YWxpZKdkZXRhaWxzwA==", + "expected": { + "schemaVersion": "cultnet.error.v0", + "error": "cursor_invalid: bad", + "routingHint": null, + "code": "cursor_invalid", + "details": null + } + }, + { + "name": "error_write_not_authorized", + "schemaVersion": "cultnet.error.v0", + "messagePackBase64": "ha1zY2hlbWFWZXJzaW9usGN1bHRuZXQuZXJyb3IudjClZXJyb3K0d3JpdGVfbm90X2F1dGhvcml6ZWSrcm91dGluZ0hpbnTApGNvZGW0d3JpdGVfbm90X2F1dGhvcml6ZWSnZGV0YWlsc4SlZmllbGSpcmVjb3JkS2V5pXZhbHVloWukYXNPZsCnY3VycmVudMA=", + "expected": { + "schemaVersion": "cultnet.error.v0", + "error": "write_not_authorized", + "routingHint": null, + "code": "write_not_authorized", + "details": { + "field": "recordKey", + "value": "k", + "asOf": null, + "current": null + } + } + } + ] +} \ No newline at end of file diff --git a/tests/GameCult.Networking.Tests/CarrierVectorTests.cs b/tests/GameCult.Networking.Tests/CarrierVectorTests.cs new file mode 100644 index 00000000..f294bc0f --- /dev/null +++ b/tests/GameCult.Networking.Tests/CarrierVectorTests.cs @@ -0,0 +1,161 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Networking.Tests +{ + // The C# reference's own bytes for the selection/v1 message carriers and the cultnet.error.v0 + // code/details shape, written to contracts/cultnet/interop/carrier-vectors.cs-written.json for + // the runtimes that decode and re-encode them (packages/cultnet-ts reads it; a Rust or Python + // reader may follow). Each vector carries the whole message's MessagePack bytes plus `expected`, + // the reference's own camelCase JSON of the same object (byte[] as base64), so a reader judges + // against what the reference serialized rather than a second hand-written spec. + // + // Gated like SelectionParityVectorTests: CULTNET_WRITE_VECTORS=1 rewrites the committed file; + // otherwise the committed bytes are compared with a fresh serialization so reference drift fails + // here instead of in a reader. + public sealed class CarrierVectorTests + { + private const string LeafA = "sha256:ff35f3d9d6de3f1066b9369e73fe6c7648e88368d18efc2275c994730cd84cff"; + private const string CiterId = "sha256:9d0bc9aebc3619b07aabf564f98667afde41444c1b866661d777687d443608dc"; + + private static readonly JsonSerializerOptions JsonOptions = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; + + private static CultNetSelection FullSelection() => new() + { + Schemas = new[] { "leaf_a.v9", LeafA }, + Keys = new[] { "a-eq", "\U0001F600-row" }, + Fields = new[] + { + new CultNetFieldPredicate { Index = "kind", Op = "any_of", Values = new[] { "weapon", "tool" } }, + new CultNetFieldPredicate { Index = "mass", Op = "ge", Number = "-2233759.25" } + }, + Cites = new CultNetCitation { Target = new CultNetRecordRef { SchemaId = LeafA, RecordKey = "a-eq" }, Role = "Design" }, + Cited = new CultNetIncoming { Role = "Design", Exists = false }, + Projection = CultNetSelectionProjections.Document, + Descending = true, + Limit = 200, + Cursor = "opaque-cursor" + }; + + private static CultNetRawDocumentHeader Header(string key, params string[] tags) => new() + { + SchemaId = LeafA, + SchemaName = "leaf_a", + SchemaVersion = "1", + SchemaContentHash = "abc", + RecordKey = key, + StoredAt = "1970-01-01T00:00:00.000Z", + SourceRuntimeId = "rt", + SourceAgentId = "agent", + SourceRole = "role", + Tags = tags.Length == 0 ? null : tags + }; + + private static IEnumerable<(string Name, ICultNetSchemaMessage Message)> Cases() + { + yield return ("snapshot_request_v1_defaults", new CultNetSnapshotRequestV1Message { MessageId = "m" }); + yield return ("snapshot_request_v1_full_selection_and_shard", new CultNetSnapshotRequestV1Message + { + MessageId = "m-full", Selection = FullSelection(), ShardId = "shard-1", ShardEpoch = 7 + }); + yield return ("database_subscribe_v1_defaults", new CultNetDatabaseSubscribeV1Message { MessageId = "m", SubscriptionId = "sub" }); + yield return ("database_subscribe_v1_full", new CultNetDatabaseSubscribeV1Message + { + MessageId = "m-full", SubscriptionId = "sub-full", Selection = FullSelection(), IncludeSnapshot = false, + ConsumerRuntimeId = "consumer", BodyIds = new[] { "body-a" }, SupportedBodyTransports = new[] { "rudp", "quic" } + }); + yield return ("snapshot_response_raw_v1_headers", new CultNetSnapshotResponseRawV1Message + { + MessageId = "m", Matched = 2, AsOf = 5, Next = "next-cursor", + Headers = new[] { Header("a-eq"), Header("a-lo", "alpha", "beta") }, + ShardId = "shard-1", ShardEpoch = 3, ShardLogSequence = 41 + }); + yield return ("snapshot_response_raw_v1_documents_and_edges", new CultNetSnapshotResponseRawV1Message + { + MessageId = "m", Matched = 1, AsOf = 5, + Documents = new[] + { + new CultNetRawDocumentRecord { SchemaId = LeafA, RecordKey = "a-eq", StoredAt = "1970-01-01T00:00:00.000Z", Payload = new byte[] { 0x81, 0xA1, 0x6B, 0x01 } } + }, + Edges = new[] + { + new CultNetEdge + { + From = new CultNetRecordRef { SchemaId = CiterId, RecordKey = "citer-1" }, Role = "Design", + To = new CultNetRecordRef { SchemaId = LeafA, RecordKey = "a-eq" } + }, + new CultNetEdge + { + From = new CultNetRecordRef { SchemaId = CiterId, RecordKey = "citer-1" }, Role = "Design", + To = new CultNetRecordRef { SchemaId = LeafA, RecordKey = "a-eq" }, + PayloadEncoding = "messagepack", Payload = new byte[] { 1, 2, 3 } + } + } + }); + yield return ("error_plain", new CultNetErrorMessage { Error = "boom" }); + yield return ("error_selection_invalid", CultNetErrorMessage.ForSelectionInvalid( + new CultNetSelectionInvalidException("fields[0].index", "kind", "fields[0].index \"kind\" is not declared."))); + yield return ("error_cursor_stale", new CultNetErrorMessage + { + Error = "cursor_stale: stale", Code = "cursor_stale", Details = new CultNetErrorDetails { AsOf = 3, Current = 9 } + }); + yield return ("error_cursor_invalid_no_details", new CultNetErrorMessage { Error = "cursor_invalid: bad", Code = "cursor_invalid" }); + yield return ("error_write_not_authorized", new CultNetErrorMessage + { + Error = "write_not_authorized", Code = "write_not_authorized", Details = new CultNetErrorDetails { Field = "recordKey", Value = "k" } + }); + } + + private static object Vector(string name, ICultNetSchemaMessage message) + { + var bytes = MessagePackSerializer.Serialize(message.GetType(), message, CultNetSchemaMessageSerialization.Options); + return new + { + name, + schemaVersion = message.SchemaVersion, + messagePackBase64 = Convert.ToBase64String(bytes), + expected = JsonSerializer.SerializeToElement(message, message.GetType(), JsonOptions) + }; + } + + private static string VectorPath() + { + var directory = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); + while (directory != null && !File.Exists(Path.Combine(directory.FullName, "CultLib.sln"))) + directory = directory.Parent; + var root = directory?.FullName ?? throw new InvalidOperationException("CultLib.sln was not found above the test directory."); + return Path.Combine(root, "contracts", "cultnet", "interop", "carrier-vectors.cs-written.json"); + } + + [Test] + public void WriteCarrierVectors() + { + if (Environment.GetEnvironmentVariable("CULTNET_WRITE_VECTORS") != "1") + Assert.Ignore("Set CULTNET_WRITE_VECTORS=1 to (re)write the committed vector file."); + + var vectors = Cases().Select(c => Vector(c.Name, c.Message)).ToArray(); + File.WriteAllText(VectorPath(), JsonSerializer.Serialize(new { vectors }, new JsonSerializerOptions { WriteIndented = true })); + } + + [Test] + public void CommittedCarrierVectorsMatchAFreshSerialization() + { + using var committed = JsonDocument.Parse(File.ReadAllText(VectorPath())); + var byName = committed.RootElement.GetProperty("vectors").EnumerateArray() + .ToDictionary(v => v.GetProperty("name").GetString()!, v => v.GetProperty("messagePackBase64").GetString()!); + + Assert.That(byName.Count, Is.EqualTo(Cases().Count()), "the committed file must carry exactly the cases this writer produces"); + foreach (var (name, message) in Cases()) + { + var fresh = Convert.ToBase64String(MessagePackSerializer.Serialize(message.GetType(), message, CultNetSchemaMessageSerialization.Options)); + Assert.That(byName[name], Is.EqualTo(fresh), $"{name}: the committed bytes drifted from the reference serializer"); + } + } + } +} From 280ec74c9a84421e250b53747e9df850662b5752 Mon Sep 17 00:00:00 2001 From: Libby Date: Sat, 7 Nov 2026 01:36:05 +0000 Subject: [PATCH 69/88] cultmesh-rs: port CultMesh authority route verification ECDSA P-256 route certificate and session proof verification ported from CultMeshAuthorityProof.cs using the p256 crate (new dependency, pure Rust, no default features), judged against the shared contracts/cultmesh vectors. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultmesh-rs/Cargo.lock | 123 +++++ packages/cultmesh-rs/Cargo.toml | 3 + packages/cultmesh-rs/src/authority.rs | 424 ++++++++++++++++++ packages/cultmesh-rs/src/lib.rs | 1 + .../tests/authority_route_vectors.rs | 261 +++++++++++ 5 files changed, 812 insertions(+) create mode 100644 packages/cultmesh-rs/src/authority.rs create mode 100644 packages/cultmesh-rs/tests/authority_route_vectors.rs diff --git a/packages/cultmesh-rs/Cargo.lock b/packages/cultmesh-rs/Cargo.lock index d056b52e..de7c506e 100644 --- a/packages/cultmesh-rs/Cargo.lock +++ b/packages/cultmesh-rs/Cargo.lock @@ -67,6 +67,12 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + [[package]] name = "base64" version = "0.22.1" @@ -161,6 +167,18 @@ dependencies = [ "libc", ] +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + [[package]] name = "crypto-common" version = "0.1.7" @@ -214,11 +232,14 @@ name = "cultmesh-rs" version = "0.2.0" dependencies = [ "anyhow", + "base64", "cultcache-rs", "cultnet-rs", + "p256", "pretty_assertions", "rmp-serde", "serde", + "serde_json", "tempfile", ] @@ -298,10 +319,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", + "const-oid", "crypto-common", "subtle", ] +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + [[package]] name = "ed25519" version = "2.2.3" @@ -326,6 +362,25 @@ dependencies = [ "zeroize", ] +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest", + "ff", + "generic-array", + "group", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "errno" version = "0.3.14" @@ -342,6 +397,16 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "fiat-crypto" version = "0.2.9" @@ -396,6 +461,7 @@ checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", + "zeroize", ] [[package]] @@ -442,6 +508,17 @@ dependencies = [ "polyval", ] +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "hmac" version = "0.12.1" @@ -555,6 +632,18 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -602,6 +691,15 @@ dependencies = [ "yansi", ] +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -689,6 +787,16 @@ dependencies = [ "libc", ] +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", +] + [[package]] name = "rmp" version = "0.8.15" @@ -747,6 +855,20 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + [[package]] name = "semver" version = "1.0.28" @@ -840,6 +962,7 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ + "digest", "rand_core 0.6.4", ] diff --git a/packages/cultmesh-rs/Cargo.toml b/packages/cultmesh-rs/Cargo.toml index af75e419..984b74ab 100644 --- a/packages/cultmesh-rs/Cargo.toml +++ b/packages/cultmesh-rs/Cargo.toml @@ -13,9 +13,12 @@ path = "src/lib.rs" anyhow = "1" cultcache-rs = { path = "../cultcache-rs" } cultnet-rs = { path = "../cultnet-rs" } +base64 = "0.22" +p256 = { version = "0.13", default-features = false, features = ["ecdsa", "std"] } serde = { version = "1", features = ["derive"] } [dev-dependencies] pretty_assertions = "1.4.1" rmp-serde = "1" +serde_json = "1" tempfile = "3" diff --git a/packages/cultmesh-rs/src/authority.rs b/packages/cultmesh-rs/src/authority.rs new file mode 100644 index 00000000..f4368884 --- /dev/null +++ b/packages/cultmesh-rs/src/authority.rs @@ -0,0 +1,424 @@ +//! CultMesh authority route verification. +//! +//! Port of `src/GameCult.Mesh/CultMeshAuthorityProof.cs` (the reference) and of +//! the route constructors in `CultMeshVerse.cs`. Rule for rule, in the same +//! order, the same input yields the same refusal text as +//! `CultMeshAuthorityTrustPolicy.Verify`: unsigned, channel protection, root +//! lookup, validity window, signature. +//! +//! Shared vectors: `contracts/cultmesh/authority-route-vectors.json` (C# written). + +use base64::alphabet::STANDARD as STANDARD_ALPHABET; +use base64::engine::{DecodePaddingMode, GeneralPurpose, GeneralPurposeConfig}; +use base64::Engine; +use p256::ecdsa::signature::Verifier; +use p256::ecdsa::{Signature, VerifyingKey}; +use std::collections::BTreeMap; +use std::fmt; + +const ROUTE_DOMAIN: &str = "gamecult.cultmesh.route-certificate.v1"; +const SESSION_DOMAIN: &str = "gamecult.cultmesh.session-proof.v1"; + +// `Convert.FromBase64String`: standard alphabet, padded, tolerant of trailing bits. +const DOTNET_BASE64: GeneralPurpose = GeneralPurpose::new( + &STANDARD_ALPHABET, + GeneralPurposeConfig::new() + .with_decode_padding_mode(DecodePaddingMode::RequireCanonical) + .with_decode_allow_trailing_bits(true), +); + +fn base64_decode(value: &str) -> Option> { + let stripped: String = value.chars().filter(|c| !matches!(c, ' ' | '\t' | '\r' | '\n')).collect(); + DOTNET_BASE64.decode(stripped).ok() +} + +/// .NET `char.IsWhiteSpace` is the Unicode White_Space property, as is `char::is_whitespace`. +fn trim(value: &str) -> &str { + value.trim_matches(char::is_whitespace) +} + +fn is_blank(value: &str) -> bool { + trim(value).is_empty() +} + +fn require(value: &str, name: &str) -> Result { + if is_blank(value) { + Err(format!("Value must be non-empty. (Parameter '{name}')")) + } else { + Ok(trim(value).to_owned()) + } +} + +/// A route or proof the consumer's trust policy refuses. Mirrors +/// `CultMeshSessionFailure` with reason `Authentication`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AuthorityRouteRefusal { + pub message: String, + pub endpoint: String, +} + +impl AuthorityRouteRefusal { + pub const REASON: &'static str = "authentication"; +} + +impl fmt::Display for AuthorityRouteRefusal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.message) + } +} + +impl std::error::Error for AuthorityRouteRefusal {} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct P256PublicKey { + pub key_id: String, + pub x: String, + pub y: String, +} + +impl P256PublicKey { + pub fn new(key_id: &str, x: &str, y: &str) -> Result { + let key = Self { + key_id: require(key_id, "keyId")?, + x: require(x, "x")?, + y: require(y, "y")?, + }; + for (name, value) in [("x", &key.x), ("y", &key.y)] { + if base64_decode(value).is_none() { + return Err(format!("Coordinate must be base64. (Parameter '{name}')")); + } + } + Ok(key) + } + + /// IEEE P1363 (r || s) ECDSA-SHA256 verification; false on any malformation. + pub fn verify(&self, payload: &[u8], signature_base64: &str) -> bool { + let (Some(x), Some(y), Some(signature)) = ( + base64_decode(&self.x), + base64_decode(&self.y), + base64_decode(signature_base64), + ) else { + return false; + }; + if x.len() != 32 || y.len() != 32 || signature.len() != 64 { + return false; + } + let mut sec1 = Vec::with_capacity(65); + sec1.push(4); + sec1.extend_from_slice(&x); + sec1.extend_from_slice(&y); + let (Ok(key), Ok(signature)) = ( + VerifyingKey::from_sec1_bytes(&sec1), + Signature::from_slice(&signature), + ) else { + return false; + }; + key.verify(payload, &signature).is_ok() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RouteCertificate { + pub provider_key: P256PublicKey, + pub odin_key_id: String, + pub issued_at_unix_milliseconds: i64, + pub expires_at_unix_milliseconds: i64, + pub signature: String, +} + +impl RouteCertificate { + pub fn new( + provider_key: P256PublicKey, + odin_key_id: &str, + issued_at_unix_milliseconds: i64, + expires_at_unix_milliseconds: i64, + signature: &str, + ) -> Result { + if is_blank(odin_key_id) { + return Err("Odin key identity is required. (Parameter 'odinKeyId')".into()); + } + if issued_at_unix_milliseconds < 0 { + return Err("issuedAtUnixMilliseconds must not be negative.".into()); + } + if expires_at_unix_milliseconds <= issued_at_unix_milliseconds { + return Err("expiresAtUnixMilliseconds must be after issuedAtUnixMilliseconds.".into()); + } + Ok(Self { + provider_key, + odin_key_id: trim(odin_key_id).to_owned(), + issued_at_unix_milliseconds, + expires_at_unix_milliseconds, + signature: trim(signature).to_owned(), + }) + } +} + +/// `CultMeshAuthorityRoute`: construction cleans every field as the C# does. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AuthorityRoute { + pub authority_runtime_id: String, + pub endpoint: String, + pub protocol_ids: Vec, + pub priority: i32, + pub generation: String, + pub certificate: Option, +} + +impl AuthorityRoute { + pub fn new( + authority_runtime_id: &str, + endpoint: &str, + protocol_ids: &[&str], + priority: i32, + generation: Option<&str>, + certificate: Option, + ) -> Result { + let authority_runtime_id = require(authority_runtime_id, "authorityRuntimeId")?; + let endpoint = require(endpoint, "endpoint")?; + if priority < 0 { + return Err("priority must not be negative.".into()); + } + let mut protocols: Vec = protocol_ids + .iter() + .filter(|value| !is_blank(value)) + .map(|value| trim(value).to_owned()) + .collect(); + // StringComparer.Ordinal is UTF-16 code-unit order. + protocols.sort_by_key(|value| value.encode_utf16().collect::>()); + protocols.dedup(); + let generation = match generation { + Some(value) if !is_blank(value) => trim(value).to_owned(), + _ => format!("{authority_runtime_id}@{endpoint}"), + }; + Ok(Self { + authority_runtime_id, + endpoint, + protocol_ids: protocols, + priority, + generation, + certificate, + }) + } +} + +/// The fields of `CultMeshSessionOpenMessage` the session proof covers. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct SessionOpen { + pub client_nonce: String, + pub message_id: String, + pub source_runtime_id: String, + pub verse_id: String, + pub authority_runtime_id: String, + pub protocol_id: String, + pub route_generation: String, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct SessionAccepted { + pub client_nonce: String, + pub provider_key_id: String, + pub provider_signature: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TrustMode { + AuthenticatedRemote, + LocalDevelopment, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VerifiedRoute { + pub route: AuthorityRoute, + pub provider_key: Option, + pub is_local_development: bool, +} + +pub fn canonical_fields(values: &[&str]) -> Vec { + let mut out = Vec::new(); + for value in values { + out.extend_from_slice(&(value.len() as u32).to_be_bytes()); + out.extend_from_slice(value.as_bytes()); + } + out +} + +pub fn canonical_route(verse_id: &str, route: &AuthorityRoute) -> Result, String> { + let certificate = route.certificate.as_ref().ok_or("Route certificate is required.")?; + let protocols = route.protocol_ids.join("\u{1f}"); + let priority = route.priority.to_string(); + let issued = certificate.issued_at_unix_milliseconds.to_string(); + let expires = certificate.expires_at_unix_milliseconds.to_string(); + Ok(canonical_fields(&[ + ROUTE_DOMAIN, + verse_id, + &route.authority_runtime_id, + &route.endpoint, + &protocols, + &priority, + &route.generation, + &certificate.provider_key.key_id, + &certificate.provider_key.x, + &certificate.provider_key.y, + &certificate.odin_key_id, + &issued, + &expires, + ])) +} + +pub fn canonical_session(request: &SessionOpen, endpoint: &str) -> Vec { + canonical_fields(&[ + SESSION_DOMAIN, + &request.client_nonce, + &request.message_id, + &request.source_runtime_id, + &request.verse_id, + &request.authority_runtime_id, + &request.protocol_id, + endpoint, + &request.route_generation, + ]) +} + +/// `(scheme, lowercase host)`, or `None` where `System.Uri` refuses the spelling. +/// A lenient parse could call a refused endpoint loopback or protected, so the +/// shape rule is the same strict one the TypeScript port applies. +fn split_endpoint(value: &str) -> Option<(String, String)> { + let raw = value.trim_matches(|c: char| c <= ' '); + if raw.contains('\\') { + return None; + } + let (scheme, rest) = raw.split_once("://")?; + let mut scheme_chars = scheme.chars(); + if !scheme_chars.next()?.is_ascii_alphabetic() + || !scheme_chars.all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.')) + { + return None; + } + let authority = &rest[..rest.find(['/', '?', '#']).unwrap_or(rest.len())]; + if authority.is_empty() { + return None; + } + let after_user = &authority[authority.rfind('@').map_or(0, |i| i + 1)..]; + let host = if after_user.starts_with('[') { + &after_user[..after_user.find(']').map_or(after_user.len(), |i| i + 1)] + } else { + after_user.split(':').next().unwrap_or("") + }; + if host.is_empty() || host.chars().any(|c| !('\u{21}'..='\u{7e}').contains(&c)) || host.ends_with('.') { + return None; + } + Some(( + scheme.to_ascii_lowercase(), + host.trim_matches(['[', ']']).to_ascii_lowercase(), + )) +} + +pub fn is_loopback_endpoint(endpoint: &str) -> bool { + let Some((_, host)) = split_endpoint(endpoint) else { + return false; + }; + let octets: Vec<&str> = host.split('.').collect(); + matches!(host.as_str(), "localhost" | "loopback" | "::1" | "::ffff:127.0.0.1") + || (octets.len() == 4 + && octets[0] == "127" + && octets[1..].iter().all(|o| (1..=3).contains(&o.len()) && o.bytes().all(|b| b.is_ascii_digit()))) +} + +pub fn is_protected_endpoint(endpoint: &str) -> bool { + split_endpoint(endpoint) + .is_some_and(|(scheme, _)| scheme == "wss" || scheme == "https" || scheme.contains("quic")) +} + +/// Consumer-owned policy. Fails closed unless a route is valid under it. +pub struct AuthorityTrustPolicy { + mode: TrustMode, + roots: BTreeMap, +} + +impl AuthorityTrustPolicy { + pub fn new(mode: TrustMode, odin_roots: Vec) -> Result { + let mut roots = BTreeMap::new(); + for root in odin_roots { + let key_id = root.key_id.clone(); + if roots.insert(key_id.clone(), root).is_some() { + return Err(format!( + "Odin root key id '{key_id}' is listed more than once in the trust policy." + )); + } + } + Ok(Self { mode, roots }) + } + + pub fn validate(&self, verse_id: &str, route: &AuthorityRoute, now_unix_milliseconds: i64) -> Result<(), AuthorityRouteRefusal> { + self.verify(verse_id, route, now_unix_milliseconds).map(|_| ()) + } + + pub fn verify(&self, verse_id: &str, route: &AuthorityRoute, now_unix_milliseconds: i64) -> Result { + let refuse = |message: &str| AuthorityRouteRefusal { + message: message.to_owned(), + endpoint: route.endpoint.clone(), + }; + let local = self.mode == TrustMode::LocalDevelopment && is_loopback_endpoint(&route.endpoint); + let certificate = match &route.certificate { + Some(certificate) if !is_blank(&certificate.signature) => certificate, + _ if local => { + return Ok(VerifiedRoute { route: route.clone(), provider_key: None, is_local_development: true }); + } + _ => return Err(refuse("Remote CultMesh routes require an Odin-signed authority certificate.")), + }; + if !is_protected_endpoint(&route.endpoint) && !local { + return Err(refuse("Authenticated remote CultMesh routes require TLS or QUIC channel protection.")); + } + let Some(root) = self.roots.get(&certificate.odin_key_id) else { + return Err(refuse(&format!( + "Odin key '{}' is not trusted by this consumer.", + certificate.odin_key_id + ))); + }; + if now_unix_milliseconds < certificate.issued_at_unix_milliseconds + || now_unix_milliseconds >= certificate.expires_at_unix_milliseconds + { + return Err(refuse("The Odin route certificate is not currently valid.")); + } + let Some(signature) = base64_decode(&certificate.signature) else { + return Err(refuse("The Odin route signature is not base64.")); + }; + if signature.len() != 64 { + return Err(refuse("The Odin route signature is not IEEE P1363 P-256.")); + } + let canonical = canonical_route(verse_id, route).map_err(|message| refuse(&message))?; + if !root.verify(&canonical, &certificate.signature) { + return Err(refuse("The Odin route certificate signature is invalid.")); + } + Ok(VerifiedRoute { + route: route.clone(), + provider_key: Some(certificate.provider_key.clone()), + is_local_development: false, + }) + } +} + +/// Verifies the Odin route certificate (refusing) and the provider nonce proof. +pub fn verify_session_proof( + request: &SessionOpen, + response: &SessionAccepted, + verse_id: &str, + route: &AuthorityRoute, + trust: &AuthorityTrustPolicy, + now_unix_milliseconds: i64, +) -> Result { + let verified = trust.verify(verse_id, route, now_unix_milliseconds)?; + if verified.is_local_development { + return Ok(true); + } + let Some(key) = verified.provider_key else { + return Ok(false); + }; + if response.provider_key_id != key.key_id + || response.client_nonce != request.client_nonce + || is_blank(&response.provider_signature) + { + return Ok(false); + } + Ok(key.verify(&canonical_session(request, &route.endpoint), &response.provider_signature)) +} diff --git a/packages/cultmesh-rs/src/lib.rs b/packages/cultmesh-rs/src/lib.rs index e50719cd..5252fe09 100644 --- a/packages/cultmesh-rs/src/lib.rs +++ b/packages/cultmesh-rs/src/lib.rs @@ -25,6 +25,7 @@ use std::time::Duration; use std::time::Instant; mod rudp_document_server; +pub mod authority; pub use rudp_document_server::*; diff --git a/packages/cultmesh-rs/tests/authority_route_vectors.rs b/packages/cultmesh-rs/tests/authority_route_vectors.rs new file mode 100644 index 00000000..40cc5bba --- /dev/null +++ b/packages/cultmesh-rs/tests/authority_route_vectors.rs @@ -0,0 +1,261 @@ +//! Judges `contracts/cultmesh/authority-route-vectors*.json` in Rust. +//! +//! `authority-route-vectors.json` is written by the C# reference (its signatures +//! are captured C# bytes); the `.ts-signed.json` file is signed by TypeScript +//! over awkward inputs. Each negative case fails when the matching check is +//! removed from `cultmesh_rs::authority`. + +use base64::Engine; +use base64::engine::general_purpose::STANDARD; +use cultmesh_rs::authority::*; +use p256::ecdsa::Signature; +use serde_json::Value; +use std::path::PathBuf; + +struct Vector { + now: i64, + root: P256PublicKey, + verse_id: String, + route: AuthorityRoute, + request: SessionOpen, + accepted: SessionAccepted, + trust: AuthorityTrustPolicy, +} + +fn s<'a>(value: &'a Value, key: &str) -> &'a str { + value[key].as_str().unwrap_or_else(|| panic!("missing string {key}")) +} + +fn key(value: &Value) -> P256PublicKey { + P256PublicKey::new(s(value, "keyId"), s(value, "x"), s(value, "y")).unwrap() +} + +fn load(name: &str) -> Vector { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../contracts/cultmesh") + .join(name); + let doc: Value = serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap(); + let r = &doc["route"]; + let c = &r["certificate"]; + let certificate = RouteCertificate::new( + key(&c["providerKey"]), + s(c, "odinKeyId"), + c["issuedAtUnixMilliseconds"].as_i64().unwrap(), + c["expiresAtUnixMilliseconds"].as_i64().unwrap(), + s(c, "signature"), + ) + .unwrap(); + let protocols: Vec<&str> = r["protocolIds"].as_array().unwrap().iter().map(|v| v.as_str().unwrap()).collect(); + let route = AuthorityRoute::new( + s(r, "authorityRuntimeId"), + s(r, "endpoint"), + &protocols, + r["priority"].as_i64().unwrap() as i32, + Some(s(r, "generation")), + Some(certificate), + ) + .unwrap(); + let p = &doc["sessionProof"]; + let q = &p["request"]; + let root = key(&doc["odinRoot"]); + Vector { + now: doc["nowUnixMilliseconds"].as_i64().unwrap(), + verse_id: s(r, "verseId").to_owned(), + route, + request: SessionOpen { + client_nonce: s(q, "clientNonce").into(), + message_id: s(q, "messageId").into(), + source_runtime_id: s(q, "sourceRuntimeId").into(), + verse_id: s(q, "verseId").into(), + authority_runtime_id: s(q, "authorityRuntimeId").into(), + protocol_id: s(q, "protocolId").into(), + route_generation: s(q, "routeGeneration").into(), + }, + accepted: SessionAccepted { + client_nonce: s(q, "clientNonce").into(), + provider_key_id: s(p, "providerKeyId").into(), + provider_signature: s(p, "providerSignature").into(), + }, + trust: AuthorityTrustPolicy::new(TrustMode::AuthenticatedRemote, vec![root.clone()]).unwrap(), + root, + } +} + +impl Vector { + fn certificate(&self) -> &RouteCertificate { + self.route.certificate.as_ref().unwrap() + } + fn with_certificate(&self, edit: impl FnOnce(&mut RouteCertificate)) -> AuthorityRoute { + let mut route = self.route.clone(); + edit(route.certificate.as_mut().unwrap()); + route + } + fn refusal(&self, route: &AuthorityRoute, now: i64) -> String { + let error = self.trust.validate(&self.verse_id, route, now).expect_err("route must be refused"); + assert_eq!(AuthorityRouteRefusal::REASON, "authentication"); + assert_eq!(route.endpoint, error.endpoint); + error.message + } + fn proof(&self, request: &SessionOpen, accepted: &SessionAccepted) -> bool { + verify_session_proof(request, accepted, &self.verse_id, &self.route, &self.trust, self.now).unwrap() + } +} + +const INVALID: &str = "The Odin route certificate signature is invalid."; + +#[test] +fn cs_written_vector_verifies_route_and_session_proof() { + let v = load("authority-route-vectors.json"); + v.trust.validate(&v.verse_id, &v.route, v.now).unwrap(); + assert!(v.proof(&v.request, &v.accepted)); +} + +#[test] +fn ts_signed_vector_verifies_and_tamper_is_refused() { + let v = load("authority-route-vectors.ts-signed.json"); + v.trust.validate(&v.verse_id, &v.route, v.now).unwrap(); + assert!(v.proof(&v.request, &v.accepted)); + let mut tampered = v.route.clone(); + tampered.priority += 1; + assert_eq!(INVALID, v.refusal(&tampered, v.now)); +} + +#[test] +fn canonical_route_is_length_prefixed_and_sorted() { + let v = load("authority-route-vectors.json"); + let bytes = canonical_route(&v.verse_id, &v.route).unwrap(); + assert!(bytes.starts_with(b"\x00\x00\x00\x26gamecult.cultmesh.route-certificate.v1\x00\x00\x00\x08aetheria")); + let needle = b"cultmesh.content.v1\x1fcultmesh.documents.v1"; + assert!(bytes.windows(needle.len()).any(|w| w == needle)); +} + +#[test] +fn tampered_payload_fields_are_refused() { + let v = load("authority-route-vectors.json"); + let edits: Vec> = vec![ + Box::new(|r| r.priority = 8), + Box::new(|r| r.endpoint = "wss://evil.example/mesh".into()), + Box::new(|r| r.protocol_ids = vec!["cultmesh.documents.v1".into()]), + Box::new(|r| r.generation = "other".into()), + Box::new(|r| r.certificate.as_mut().unwrap().expires_at_unix_milliseconds += 1), + ]; + for edit in edits { + let mut route = v.route.clone(); + edit(&mut route); + assert_eq!(INVALID, v.refusal(&route, v.now)); + } + let error = v.trust.validate("other-verse", &v.route, v.now).unwrap_err(); + assert_eq!(INVALID, error.message); +} + +#[test] +fn wrong_root_key_is_refused() { + let v = load("authority-route-vectors.json"); + // The provider's key is a real P-256 key that did not sign this route. + let provider = &v.certificate().provider_key; + let impostor = P256PublicKey::new(&v.root.key_id, &provider.x, &provider.y).unwrap(); + let trust = AuthorityTrustPolicy::new(TrustMode::AuthenticatedRemote, vec![impostor]).unwrap(); + assert_eq!(INVALID, trust.validate(&v.verse_id, &v.route, v.now).unwrap_err().message); +} + +#[test] +fn untrusted_odin_key_id_is_refused() { + let v = load("authority-route-vectors.json"); + let other = P256PublicKey::new("someone-else", &v.root.x, &v.root.y).unwrap(); + let trust = AuthorityTrustPolicy::new(TrustMode::AuthenticatedRemote, vec![other]).unwrap(); + assert_eq!( + "Odin key 'odin-vector' is not trusted by this consumer.", + trust.validate(&v.verse_id, &v.route, v.now).unwrap_err().message + ); +} + +#[test] +fn expired_and_not_yet_valid_are_refused() { + let v = load("authority-route-vectors.json"); + let window = "The Odin route certificate is not currently valid."; + let (issued, expires) = (v.certificate().issued_at_unix_milliseconds, v.certificate().expires_at_unix_milliseconds); + v.trust.validate(&v.verse_id, &v.route, expires - 1).unwrap(); + v.trust.validate(&v.verse_id, &v.route, issued).unwrap(); + assert_eq!(window, v.refusal(&v.route, expires)); + assert_eq!(window, v.refusal(&v.route, issued - 1)); +} + +#[test] +fn unsigned_and_unprotected_routes_are_refused() { + let v = load("authority-route-vectors.json"); + let unsigned_message = "Remote CultMesh routes require an Odin-signed authority certificate."; + let mut unsigned = v.route.clone(); + unsigned.certificate = None; + assert_eq!(unsigned_message, v.refusal(&unsigned, v.now)); + let blank = v.with_certificate(|c| c.signature = " ".into()); + assert_eq!(unsigned_message, v.refusal(&blank, v.now)); + let mut plain = v.route.clone(); + plain.endpoint = "ws://provider.example/mesh".into(); + assert_eq!( + "Authenticated remote CultMesh routes require TLS or QUIC channel protection.", + v.refusal(&plain, v.now) + ); +} + +#[test] +fn local_development_admits_only_unsigned_loopback() { + let v = load("authority-route-vectors.json"); + let dev = AuthorityTrustPolicy::new(TrustMode::LocalDevelopment, vec![]).unwrap(); + let route = |endpoint: &str| AuthorityRoute::new("a", endpoint, &[], 0, None, None).unwrap(); + let loopback = route("ws://127.0.0.1:9/mesh"); + dev.validate(&v.verse_id, &loopback, v.now).unwrap(); + assert!(verify_session_proof(&v.request, &v.accepted, &v.verse_id, &loopback, &dev, v.now).unwrap()); + for endpoint in ["ws://provider.example/mesh", "ws://127.0.0.1./mesh", "ws:127.0.0.1"] { + assert!(dev.validate(&v.verse_id, &route(endpoint), v.now).is_err(), "{endpoint}"); + } + let remote = AuthorityTrustPolicy::new(TrustMode::AuthenticatedRemote, vec![]).unwrap(); + assert!(remote.validate(&v.verse_id, &loopback, v.now).is_err()); +} + +#[test] +fn malformed_signatures_get_their_own_refusals() { + let v = load("authority-route-vectors.json"); + let not_base64 = v.with_certificate(|c| c.signature = "!!not base64!!".into()); + assert_eq!("The Odin route signature is not base64.", v.refusal(¬_base64, v.now)); + let short = v.with_certificate(|c| c.signature = STANDARD.encode([b'x'; 63])); + assert_eq!("The Odin route signature is not IEEE P1363 P-256.", v.refusal(&short, v.now)); +} + +#[test] +fn high_s_signature_verifies_as_it_does_in_dotnet() { + let v = load("authority-route-vectors.json"); + let raw = STANDARD.decode(&v.certificate().signature).unwrap(); + let signature = Signature::from_slice(&raw).unwrap(); + let flipped = Signature::from_scalars(*signature.r().as_ref(), -*signature.s().as_ref()).unwrap(); + let high = v.with_certificate(|c| c.signature = STANDARD.encode(flipped.to_bytes())); + v.trust.validate(&v.verse_id, &high, v.now).unwrap(); +} + +#[test] +fn session_proof_rejects_replay_wrong_key_and_forgery() { + let v = load("authority-route-vectors.json"); + let mut replayed = v.request.clone(); + replayed.client_nonce = "replayed".into(); + let mut accepted = v.accepted.clone(); + accepted.client_nonce = "replayed".into(); + assert!(!v.proof(&replayed, &accepted)); + let mut wrong_id = v.accepted.clone(); + wrong_id.provider_key_id = "other".into(); + assert!(!v.proof(&v.request, &wrong_id)); + let mut wrong_nonce = v.accepted.clone(); + wrong_nonce.client_nonce = "other".into(); + assert!(!v.proof(&v.request, &wrong_nonce)); + let mut empty = v.accepted.clone(); + empty.provider_signature = String::new(); + assert!(!v.proof(&v.request, &empty)); + let mut raw = STANDARD.decode(&v.accepted.provider_signature).unwrap(); + raw[0] ^= 1; + let mut forged = v.accepted.clone(); + forged.provider_signature = STANDARD.encode(raw); + assert!(!v.proof(&v.request, &forged)); + let mut moved = v.request.clone(); + moved.protocol_id = "cultmesh.content.v1".into(); + assert!(!v.proof(&moved, &v.accepted)); + let expires = v.certificate().expires_at_unix_milliseconds; + assert!(verify_session_proof(&v.request, &v.accepted, &v.verse_id, &v.route, &v.trust, expires).is_err()); +} From 497ff201aaa6d86f067486090148b09bdb34cc6b Mon Sep 17 00:00:00 2001 From: Libby Date: Sat, 7 Nov 2026 01:36:05 +0000 Subject: [PATCH 70/88] cultmesh-py: pin high-S signature acceptance to the .NET reference Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-py/tests/test_authority_route_vectors.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/cultmesh-py/tests/test_authority_route_vectors.py b/packages/cultmesh-py/tests/test_authority_route_vectors.py index 8f3d2214..3f116e2d 100644 --- a/packages/cultmesh-py/tests/test_authority_route_vectors.py +++ b/packages/cultmesh-py/tests/test_authority_route_vectors.py @@ -182,6 +182,15 @@ class AuthorityRouteVectorTests(unittest.TestCase): refusal(self, lambda: v.trust.validate(v.verse_id, short, v.now)), ) + def test_high_s_signature_verifies_as_it_does_in_dotnet(self) -> None: + v = Vector("authority-route-vectors.json") + order = 0xFFFFFFFF00000000FFFFFFFFFFFFFFFFBCE6FAADA7179E84F3B9CAC2FC632551 + raw = base64.b64decode(v.certificate.signature) + s = int.from_bytes(raw[32:], "big") + flipped = raw[:32] + (order - s).to_bytes(32, "big") + high = v.with_certificate(signature=base64.b64encode(flipped).decode()) + v.trust.validate(v.verse_id, high, v.now) + def test_session_proof_rejects_replay_wrong_key_and_forgery(self) -> None: v = Vector("authority-route-vectors.json") replayed = replace(v.request, client_nonce="replayed") From 1d21a93a2d662b49ea4eb3d4e9f3770c04f2125d Mon Sep 17 00:00:00 2001 From: Libby Date: Sat, 7 Nov 2026 01:36:25 +0000 Subject: [PATCH 71/88] cultnet-ts: cultnet.error.v0 keeps routingHint, code and details, judged against the reference's bytes Co-Authored-By: Claude Sonnet 5.5 --- packages/cultnet-ts/src/contracts.ts | 32 ++++++++++++ packages/cultnet-ts/src/index.ts | 3 ++ .../cultnet-ts/test/carrier-vectors.test.ts | 49 +++++++++++++++++++ 3 files changed, 84 insertions(+) create mode 100644 packages/cultnet-ts/test/carrier-vectors.test.ts diff --git a/packages/cultnet-ts/src/contracts.ts b/packages/cultnet-ts/src/contracts.ts index 31188790..dcaff6f2 100644 --- a/packages/cultnet-ts/src/contracts.ts +++ b/packages/cultnet-ts/src/contracts.ts @@ -165,9 +165,37 @@ export interface CultNetLoginSuccessMessage { session: string; } +/** The closed `code` enum of `cultnet.error.schema.json`; the schema refuses any other string. */ +export type CultNetErrorCode = + | "selection_invalid" + | "cursor_stale" + | "cursor_invalid" + | "reference_outside_target" + | "variant_unsupported" + | "unowned_schema" + | "write_not_authorized"; + +/** Code-specific detail; each field is populated only for the code that defines it, null otherwise. */ +export interface CultNetErrorDetails { + field: string | null; + value: string | null; + asOf: number | null; + current: number | null; +} + +/** Shard routing detail for an authority failure. */ +export interface CultNetShardRoutingHint { + shard: Record | null; + reason: string; +} + +/** The reference writes all five keys, nil included, so a received error always carries them. */ export interface CultNetErrorMessage { schemaVersion: "cultnet.error.v0"; error: string; + routingHint: CultNetShardRoutingHint | null; + code: CultNetErrorCode | null; + details: CultNetErrorDetails | null; } export interface CultNetSampleChangeNameMessage { @@ -842,6 +870,9 @@ function parseGameCultNetworkingMessage(input: unknown): CultNetMessage { return { schemaVersion: "cultnet.error.v0", error: requireLegacyString(payload[0], "ErrorMessage.Error"), + routingHint: null, + code: null, + details: null, }; case 5: return { @@ -898,6 +929,7 @@ function encodeGameCultNetworkingMessage(message: CultNetMessage): [number, unkn decodeBase64Url(message.session, "LoginSuccessMessage.Session"), ]]; case "cultnet.error.v0": + // The gamecult.networking.v0 union's ErrorMessage carries only the text. return [4, [message.error]]; case "cultnet.sample.change_name.v0": return [5, [message.name]]; diff --git a/packages/cultnet-ts/src/index.ts b/packages/cultnet-ts/src/index.ts index c321bd07..e45a87f3 100644 --- a/packages/cultnet-ts/src/index.ts +++ b/packages/cultnet-ts/src/index.ts @@ -13,7 +13,10 @@ export { type CultNetDatabaseChangeRawMessage, type CultNetRawDocumentRecord, type CultNetRawPayloadEncoding, + type CultNetErrorCode, + type CultNetErrorDetails, type CultNetErrorMessage, + type CultNetShardRoutingHint, type CultNetHelloMessage, type CultNetLoginMessage, type CultNetLoginSuccessMessage, diff --git a/packages/cultnet-ts/test/carrier-vectors.test.ts b/packages/cultnet-ts/test/carrier-vectors.test.ts new file mode 100644 index 00000000..891e9575 --- /dev/null +++ b/packages/cultnet-ts/test/carrier-vectors.test.ts @@ -0,0 +1,49 @@ +// The reference's own bytes for the cultnet.error.v0 code/details shape and the selection/v1 +// carriers (`contracts/cultnet/interop/carrier-vectors.cs-written.json`, written by the C# +// CarrierVectorTests). Each vector is decoded, parsed, and re-encoded: the re-encoding must be the +// reference's exact bytes, and the decoded message must equal the reference's own JSON of the object. + +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { decode, encode } from "@msgpack/msgpack"; + +import { encodeCultNetMessageForWire, parseCultNetMessage } from "../src/contracts"; + +const FILE = join(__dirname, "..", "..", "..", "..", "contracts", "cultnet", "interop", "carrier-vectors.cs-written.json"); +type Vector = { name: string; schemaVersion: string; messagePackBase64: string; expected: unknown }; +const vectors: Vector[] = JSON.parse(readFileSync(FILE, "utf8")).vectors; +const vectorsFor = (prefix: string) => vectors.filter(v => v.name.startsWith(prefix)); +const bytesOf = (v: Vector) => new Uint8Array(Buffer.from(v.messagePackBase64, "base64")); + +/** Uint8Array to base64, recursively, so a decoded message compares with the reference's JSON. */ +function jsonShape(value: unknown): unknown { + if (value instanceof Uint8Array) return Buffer.from(value).toString("base64"); + if (Array.isArray(value)) return value.map(jsonShape); + if (value && typeof value === "object") { + return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, jsonShape(v)])); + } + return value; +} + +const reencode = (message: Parameters[0]) => + new Uint8Array(encode(encodeCultNetMessageForWire(message), { ignoreUndefined: true })); + +test("every error vector keeps the reference's code and details through parse and re-encode", () => { + const errors = vectorsFor("error_"); + assert.ok(errors.length >= 5); + for (const v of errors) { + const raw = decode(bytesOf(v)); + assert.deepEqual(jsonShape(raw), v.expected, `${v.name}: decoded bytes`); + const message = parseCultNetMessage(raw); + assert.equal(message.schemaVersion, "cultnet.error.v0"); + assert.deepEqual(jsonShape(message), v.expected, `${v.name}: parsed message`); + assert.deepEqual(reencode(message), bytesOf(v), `${v.name}: re-encoded bytes`); + } +}); + +test("an error whose code is outside the schema's closed enum is refused", () => { + const raw = decode(bytesOf(vectors.find(v => v.name === "error_cursor_stale")!)) as Record; + assert.throws(() => parseCultNetMessage({ ...raw, code: "made_up_code" }), /cultnet\.error\.v0/); +}); From f5c27711925fa850c0100fc461e7ef52e6b67f79 Mon Sep 17 00:00:00 2001 From: Libby Date: Sat, 7 Nov 2026 01:36:55 +0000 Subject: [PATCH 72/88] cultnet-py: multi-peer RUDP socket server, with C#-captured transcript parity Co-Authored-By: Claude Sonnet 5.5 --- packages/cultnet-py/CHANGELOG.md | 6 + .../cultnet-py/src/cultnet_py/__init__.py | 10 + .../cultnet-py/src/cultnet_py/rudp_server.py | 345 +++++++++++++++++ .../cultnet-py/src/cultnet_py/transport.py | 33 +- .../tests/test_rudp_multipeer_server.py | 350 ++++++++++++++++++ .../CultNetRudpMultiPeerVectorTests.cs | 155 ++++++++ tests/vectors/rudp-multipeer-server/README.md | 13 + .../rudp-multipeer-server/transcript.txt | 30 ++ 8 files changed, 937 insertions(+), 5 deletions(-) create mode 100644 packages/cultnet-py/src/cultnet_py/rudp_server.py create mode 100644 packages/cultnet-py/tests/test_rudp_multipeer_server.py create mode 100644 tests/GameCult.Networking.Tests/CultNetRudpMultiPeerVectorTests.cs create mode 100644 tests/vectors/rudp-multipeer-server/README.md create mode 100644 tests/vectors/rudp-multipeer-server/transcript.txt diff --git a/packages/cultnet-py/CHANGELOG.md b/packages/cultnet-py/CHANGELOG.md index cca85a79..04df5ddb 100644 --- a/packages/cultnet-py/CHANGELOG.md +++ b/packages/cultnet-py/CHANGELOG.md @@ -2,6 +2,12 @@ ## Unreleased (breaking; version 0.x, so the next minor) +- Added `CultNetRudpSocketTransportServer` (`cultnet_py.rudp_server`): one UDP socket serving many RUDP peers, one session per + remote endpoint, ported from the C# `CultNetRudpSocketTransportServer` and held to its bytes by + `tests/vectors/rudp-multipeer-server/transcript.txt`. The caller drives it (`try_receive_once`, `poll_resends`). Like the C# + listener it never reaps idle peers. `CultNetRudpSession` gained `next_sequence`, `withdraw_send` and `end_with`, and + `CultNetTransportStats` gained `send_failures`. + - Removed `resolve_document_and_schema_id_for_raw_record` from the package exports. A raw record resolves to one local schema id, the one its document registers, so there is no wire id to return. Use `resolve_document_for_raw_record(documents_by_schema_id, schema_id, record)`, which returns the document; take the schema id from `document.catalog_entry().schema_id`. diff --git a/packages/cultnet-py/src/cultnet_py/__init__.py b/packages/cultnet-py/src/cultnet_py/__init__.py index 3fc36683..d590759a 100644 --- a/packages/cultnet-py/src/cultnet_py/__init__.py +++ b/packages/cultnet-py/src/cultnet_py/__init__.py @@ -103,6 +103,12 @@ from .shard_replication import ( ) from .snapshot import CultNetRawDocumentRecord, CultNetRawSnapshotResponse from .subscription import CultNetDatabaseChange +from .rudp_server import ( + CultNetRudpSocketServerFrame, + CultNetRudpSocketServerOptions, + CultNetRudpSocketServerPeer, + CultNetRudpSocketTransportServer, +) from .transport import ( CultNetRudpDeliveredFrame, CultNetRudpPacket, @@ -211,6 +217,10 @@ __all__ = [ "CultNetRudpReceiveResult", "CultNetRudpSendOptions", "CultNetRudpSession", + "CultNetRudpSocketServerFrame", + "CultNetRudpSocketServerOptions", + "CultNetRudpSocketServerPeer", + "CultNetRudpSocketTransportServer", "is_permanent_send_error", "send_error_code", "CultNetRudpSessionOptions", diff --git a/packages/cultnet-py/src/cultnet_py/rudp_server.py b/packages/cultnet-py/src/cultnet_py/rudp_server.py new file mode 100644 index 00000000..2b162043 --- /dev/null +++ b/packages/cultnet-py/src/cultnet_py/rudp_server.py @@ -0,0 +1,345 @@ +"""Multi-peer RUDP listener: one UDP socket, one reliable session per remote endpoint. + +Matches the C# `CultNetRudpSocketTransportServer` (the reference): a Connect opens a peer's +session, a Connect from the same endpoint that is not a repeat replaces only that peer, and +a peer that ends, is refused or cannot be answered never disturbs the others. The caller +drives it: `try_receive_once` consumes one datagram, `poll_resends` sends what is due. +Everything runs on the caller's thread, so no session needs a lock. +""" +from __future__ import annotations + +from collections import deque +from dataclasses import dataclass +import socket + +from .transport import ( + REFUSED_PACKET_REASON, + CultNetRudpPacket, + CultNetRudpPacketType, + CultNetRudpSession, + CultNetRudpSessionOptions, + CultNetReconnectPolicy, + CultNetTransportFrame, + CultNetTransportStats, + _channel_send_options, + _now_ms, + create_rudp_transport_profile, + decode_rudp_packet, + encode_rudp_packet, + is_permanent_send_error, +) + +REPLACED_PEER_REASON = b"replaced by a new Connect generation" + + +@dataclass +class CultNetRudpSocketServerOptions: + runtime_id: str + socket: socket.socket + connection_id: int + initial_sequence: int | None = None + resend_delay_ms: int = 250 + transport_id: str = "rudp" + max_payload_bytes: int | None = None + max_fragment_bytes: int | None = None + max_pending_reliable_packets: int | None = None + reconnect_policy: CultNetReconnectPolicy | None = None + accept_payload: bytes = b"" + + +class CultNetRudpSocketServerPeer: + """One remote endpoint's session. A peer that reconnects from the same endpoint is a new object.""" + + def __init__(self, remote_addr: tuple, connect_payload: bytes, session: CultNetRudpSession) -> None: + self.remote_addr = remote_addr + self.connect_payload = connect_payload + self.session = session + # The last transport-level reason the session ended with, once it has. + self.disconnect_reason: bytes | None = None + + @property + def connected(self) -> bool: + return self.session.connected + + @property + def pending_reliable_packet_count(self) -> int: + return self.session.outstanding_reliable_packet_count + + +@dataclass(frozen=True) +class CultNetRudpSocketServerFrame: + peer: CultNetRudpSocketServerPeer + frame: CultNetTransportFrame + + +def _remote_key(addr: tuple) -> tuple: + return tuple(addr[:2]) + + +class CultNetRudpSocketTransportServer: + def __init__(self, options: CultNetRudpSocketServerOptions) -> None: + if not options.runtime_id.strip(): + raise ValueError("Runtime id is required.") + self.socket = options.socket + self._options = options + host, port = self.socket.getsockname()[:2] + self.profile = create_rudp_transport_profile( + options.runtime_id, + transport_id=options.transport_id, + host=host, + port=port, + max_payload_bytes=options.max_payload_bytes, + max_fragment_bytes=options.max_fragment_bytes, + max_pending_reliable_packets=options.max_pending_reliable_packets, + reconnect_policy=options.reconnect_policy, + ) + self._peers: dict[tuple, CultNetRudpSocketServerPeer] = {} + self._delivered: deque[CultNetRudpSocketServerFrame] = deque() + self._bytes_received = 0 + self._bytes_sent = 0 + self._frames_received = 0 + self._frames_sent = 0 + self._packets_dropped = 0 + self._send_failures = 0 + self._closed = False + self.peer_disconnected: list = [] # callbacks taking the peer + + @property + def peers(self) -> tuple[CultNetRudpSocketServerPeer, ...]: + return tuple(self._peers.values()) + + @property + def stats(self) -> CultNetTransportStats: + return CultNetTransportStats( + bytes_received=self._bytes_received, + bytes_sent=self._bytes_sent, + frames_received=self._frames_received, + frames_sent=self._frames_sent, + packets_dropped=self._packets_dropped, + send_failures=self._send_failures, + ) + + def send(self, peer: CultNetRudpSocketServerPeer, channel_id: str, payload: bytes) -> None: + """Sends to one peer. Raises when that peer's session has ended or was replaced, or when a + datagram can never be sent as built: the first fragment failing queues nothing, a later one + ends the session, as the peer has seen part of the frame.""" + if self._peers.get(_remote_key(peer.remote_addr)) is not peer: + raise ConnectionError("RUDP peer is not connected.") + first_sequence = peer.session.next_sequence + packets = peer.session.send_many( + channel_id, + payload, + _channel_send_options(channel_id, _now_ms()), + max_fragment_bytes=self._options.max_fragment_bytes, + ) + for index, packet in enumerate(packets): + error = self._send_packet(peer.remote_addr, packet) + if error is None: + continue + if index == 0: + peer.session.withdraw_send(first_sequence) + else: + self._end_unsendable_peer(peer, error) + raise error + self._frames_sent += 1 + + def send_schema(self, peer: CultNetRudpSocketServerPeer, payload: bytes) -> None: + self.send(peer, "schema", payload) + + def broadcast(self, channel_id: str, payload: bytes) -> list[CultNetRudpSocketServerPeer]: + """Sends to every connected peer and returns the peers that could not be sent to; one + peer's failure never stops the others.""" + failed = [] + for peer in self.peers: + try: + self.send(peer, channel_id, payload) + except (OSError, ConnectionError, ValueError): + failed.append(peer) + return failed + + def disconnect(self, peer: CultNetRudpSocketServerPeer, reason: bytes = b"") -> bool: + """Ends one peer's session, telling it why. False when the session had already ended.""" + return self.end_peer_session(peer, reason) + + def end_peer_session(self, peer: CultNetRudpSocketServerPeer, reason: bytes) -> bool: + """Ends the session of a peer whose delivered frame the application could not decode or + handle, with a goodbye carrying the application's reason. The frame is already + acknowledged, so the session cannot be kept; the peer's frames still queued are not + delivered. A session that already ended owes no goodbye, since the endpoint may belong + to a new session; the call then returns False.""" + self._packets_dropped += 1 + if peer.session.ended: + return False + self._send_packet(peer.remote_addr, peer.session.end_with(reason)) + self._delivered = deque(item for item in self._delivered if item.peer is not peer) + self._retire(peer, reason) + return True + + def receive_once(self) -> CultNetRudpSocketServerFrame | None: + return self.try_receive_once()[1] + + def try_receive_once(self) -> tuple[bool, CultNetRudpSocketServerFrame | None]: + """Consumes one transport work item. Returns whether progress was made, independently of + whether it completed an application frame.""" + if self._delivered: + return True, self._delivered.popleft() + try: + wire, remote = self.socket.recvfrom(65535) + except (TimeoutError, BlockingIOError, ConnectionResetError): + return False, None + self._bytes_received += len(wire) + + try: + packet = decode_rudp_packet(wire) + except ValueError: + self._packets_dropped += 1 + return True, None + if packet.connection_id != self._options.connection_id: + self._packets_dropped += 1 + return True, None + + key = _remote_key(remote) + if packet.packet_type == CultNetRudpPacketType.CONNECT: + self._receive_connect(packet, remote, key) + return True, None + peer = self._peers.get(key) + if peer is None: + self._packets_dropped += 1 + return True, None + self._receive_from_peer(peer, key, packet) + return True, (self._delivered.popleft() if self._delivered else None) + + def poll_resends(self) -> None: + """Sends the reliable packets whose resend timers are due, for every peer.""" + for peer in self.peers: + if self._peers.get(_remote_key(peer.remote_addr)) is not peer: + continue + for packet in peer.session.due_resends(_now_ms()): + error = self._send_packet(peer.remote_addr, packet) + if error is not None: + self._end_unsendable_peer(peer, error) + break + + def close(self) -> None: + if self._closed: + return + self._closed = True + self.socket.close() + + def __enter__(self) -> "CultNetRudpSocketTransportServer": + return self + + def __exit__(self, *_: object) -> None: + self.close() + + def _receive_connect(self, packet: CultNetRudpPacket, remote: tuple, key: tuple) -> None: + admitted = self._peers.get(key) + accept_payload = self._options.accept_payload + # The admitted session decides whether this Connect repeats the one it accepted: a repeat + # gets the Accept already owed and queues nothing. Any other Connect is a new generation + # and ends the admitted session here. + if admitted is not None: + if admitted.session.connect_repeats(packet): + try: + reply = admitted.session.accept_connect(packet, _now_ms(), accept_payload) + except ValueError: + self._packets_dropped += 1 + return + error = self._send_packet(admitted.remote_addr, reply) + if error is not None: + self._end_unsendable_peer(admitted, error) + return + admitted.session.reset_peer_state() + + peer = CultNetRudpSocketServerPeer( + tuple(remote), + packet.payload, + CultNetRudpSession( + CultNetRudpSessionOptions( + connection_id=self._options.connection_id, + initial_sequence=self._options.initial_sequence, + resend_delay_ms=self._options.resend_delay_ms, + max_pending_reliable_packets=self._options.max_pending_reliable_packets, + ) + ), + ) + try: + accept = peer.session.accept_connect(packet, _now_ms(), accept_payload) + except ValueError: + self._packets_dropped += 1 + accept = None + # A peer that cannot be answered gets no session, and the one it replaced has already ended. + if accept is None or self._send_packet(peer.remote_addr, accept) is not None: + self._peers.pop(key, None) + if accept is not None: + self._packets_dropped += 1 + else: + self._peers[key] = peer + if admitted is not None: + admitted.disconnect_reason = REPLACED_PEER_REASON + self._notify_disconnected(admitted) + + def _receive_from_peer(self, peer: CultNetRudpSocketServerPeer, key: tuple, packet: CultNetRudpPacket) -> None: + session = peer.session + try: + result = session.receive(packet, _now_ms()) + except ValueError: + # receive() has already recorded the packet's reliable sequence, so the session cannot + # be kept: a retransmit would be acknowledged and the frame silently lost. + self._packets_dropped += 1 + self._send_packet(peer.remote_addr, session.end_with(REFUSED_PACKET_REASON)) + self._retire(peer, REFUSED_PACKET_REASON) + return + error = None + if result.reply is not None: + error = self._send_packet(peer.remote_addr, result.reply) + for ready in result.ready_to_send: + if error is None: + error = self._send_packet(peer.remote_addr, ready) + if result.disconnected: + self._retire(peer, result.disconnect_reason) + return + for frame in result.delivered: + self._delivered.append( + CultNetRudpSocketServerFrame(peer, CultNetTransportFrame(frame.channel_id, frame.payload)) + ) + self._frames_received += 1 + if error is None and packet.packet_type == CultNetRudpPacketType.DATA: + error = self._send_packet(peer.remote_addr, session.create_ack_for_received(packet.sequence)) + if error is not None: + self._end_unsendable_peer(peer, error) + + def _end_unsendable_peer(self, peer: CultNetRudpSocketServerPeer, error: OSError) -> None: + """An admitted peer owes a packet that can never be sent as built: its session ends, never + the poll, the peer is told, and the disconnect reason names the error. A session that has + already ended owes no goodbye, since the endpoint may belong to a new session.""" + if peer.session.ended: + return + goodbye = peer.session.end_unsendable_session(error) + self._send_packet(peer.remote_addr, goodbye) + self._retire(peer, goodbye.payload) + + def _retire(self, peer: CultNetRudpSocketServerPeer, reason: bytes | None) -> None: + # A peer a new Connect already replaced was reported then; its key now names the successor. + key = _remote_key(peer.remote_addr) + if self._peers.get(key) is not peer: + return + del self._peers[key] + peer.disconnect_reason = reason + self._notify_disconnected(peer) + + def _notify_disconnected(self, peer: CultNetRudpSocketServerPeer) -> None: + for callback in list(self.peer_disconnected): + callback(peer) + + def _send_packet(self, remote_addr: tuple, packet: CultNetRudpPacket) -> OSError | None: + """A transient failure is that peer's lost datagram: counted, never raised. A permanent + one (`is_permanent_send_error`) is returned for the caller to refuse or end the session.""" + wire = encode_rudp_packet(packet) + try: + self._bytes_sent += self.socket.sendto(wire, remote_addr) + except OSError as error: + if is_permanent_send_error(error): + return error + self._send_failures += 1 + return None diff --git a/packages/cultnet-py/src/cultnet_py/transport.py b/packages/cultnet-py/src/cultnet_py/transport.py index 76b38bf5..3ecfa31b 100644 --- a/packages/cultnet-py/src/cultnet_py/transport.py +++ b/packages/cultnet-py/src/cultnet_py/transport.py @@ -29,6 +29,9 @@ class CultNetTransportStats: # transport: malformed frames, another session's connection id, a sender # that is not the peer, or a packet the session refuses. packets_dropped: int = 0 + # Datagrams a multi-peer server could not send to one peer and lost: each is + # a transient failure, never a reason to end a session. + send_failures: int = 0 @dataclass(frozen=True) @@ -437,6 +440,7 @@ def create_rudp_transport_profile( UNSENDABLE_PACKET_REASON = "packet could not be sent" +REFUSED_PACKET_REASON = b"session refused a packet" # The fixed name of each permanent send failure, keyed by errno, Winsock equivalents included. _PERMANENT_SEND_ERROR_CODES = { @@ -600,6 +604,28 @@ class CultNetRudpSession: self._fragment_buffers.clear() self._fragment_sets_evicted = 0 + @property + def next_sequence(self) -> int: + return self._next_sequence + + def withdraw_send(self, first_sequence: int) -> None: + """Undoes a send that never reached the wire: every packet it created at or after + `first_sequence`, pending or queued, leaves no trace and its sequences are reissued, + so the peer never sees a gap. Only valid while nothing else has been sent since.""" + for sequence in [s for s in self._pending_reliable if s >= first_sequence]: + del self._pending_reliable[sequence] + kept = [p for p in self._queued_reliable if p.sequence < first_sequence] + self._queued_reliable.clear() + self._queued_reliable.extend(kept) + self._next_sequence = first_sequence + + def end_with(self, reason: bytes) -> CultNetRudpPacket: + """Ends a session whose peer's frame was refused or failed after the session acknowledged + it. The peer's sequences are forgotten before the goodbye is built, or its ack field would + acknowledge the very frame that was refused.""" + self.reset_peer_state() + return self.create_disconnect(reason) + def create_connect(self, now_ms: int = 0, payload: bytes = b"") -> CultNetRudpPacket: # A session that has had a peer starts a new generation: nothing it # learned from that peer describes the one this Connect reaches, and @@ -1397,11 +1423,8 @@ class CultNetRudpSocketTransportConnection: self.socket.close() def _end_refused_session(self) -> None: - reason = b"session refused a packet" - # The goodbye is built after the reset, or its ack field would acknowledge - # the very frame the session refused. - self.session.reset_peer_state() - goodbye = self.session.create_disconnect(reason) + reason = REFUSED_PACKET_REASON + goodbye = self.session.end_with(reason) self.disconnect_reason = reason try: self._send_packet(goodbye) diff --git a/packages/cultnet-py/tests/test_rudp_multipeer_server.py b/packages/cultnet-py/tests/test_rudp_multipeer_server.py new file mode 100644 index 00000000..3bb14640 --- /dev/null +++ b/packages/cultnet-py/tests/test_rudp_multipeer_server.py @@ -0,0 +1,350 @@ +"""One UDP socket serves many peers, each in its own session, as the C# reference does. + +Sockets bind port 0, so no test races another for a port. +""" +from __future__ import annotations + +import errno +import socket +import time +import unittest +from pathlib import Path + +from cultnet_py.rudp_server import ( + CultNetRudpSocketServerOptions, + CultNetRudpSocketTransportServer, +) +from cultnet_py.transport import ( + CultNetRudpPacketType, + CultNetRudpSendOptions, + CultNetRudpSession, + CultNetRudpSessionOptions, + decode_rudp_packet, + encode_rudp_packet, +) + +CONNECTION_ID = 0x43554C54 +VECTORS = Path(__file__).resolve().parents[3] / "tests" / "vectors" / "rudp-multipeer-server" / "transcript.txt" + + +def bind() -> socket.socket: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.bind(("127.0.0.1", 0)) + sock.settimeout(0.03) + return sock + + +def make_server(**kwargs) -> CultNetRudpSocketTransportServer: + options = dict(runtime_id="py-test", socket=bind(), connection_id=CONNECTION_ID, resend_delay_ms=5) + options.update(kwargs) + return CultNetRudpSocketTransportServer(CultNetRudpSocketServerOptions(**options)) + + +def pump(server: CultNetRudpSocketTransportServer) -> list: + """Consumes datagrams until the socket is idle, returning the frames delivered.""" + frames = [] + while True: + progressed, frame = server.try_receive_once() + if frame is not None: + frames.append(frame) + if not progressed: + return frames + + +def drain(sock: socket.socket) -> list[bytes]: + wires = [] + while True: + try: + wires.append(sock.recvfrom(65535)[0]) + except TimeoutError: + return wires + + +class Client: + def __init__(self, server: CultNetRudpSocketTransportServer, initial_sequence: int) -> None: + self.server = server + self.socket = bind() + self.session = CultNetRudpSession( + CultNetRudpSessionOptions(connection_id=CONNECTION_ID, initial_sequence=initial_sequence) + ) + + @property + def addr(self) -> tuple: + return self.socket.getsockname() + + def transmit(self, packet) -> None: + self.socket.sendto(encode_rudp_packet(packet), self.server.socket.getsockname()) + + def connect(self, payload: bytes = b"") -> None: + self.transmit(self.session.create_connect(0, payload)) + pump(self.server) + self.receive() + + def receive(self) -> list: + packets = [decode_rudp_packet(wire) for wire in drain(self.socket)] + for packet in packets: + if self.session.connected or packet.packet_type == CultNetRudpPacketType.ACCEPT: + self.session.receive(packet, 0) + return packets + + def write(self, text: str) -> None: + self.transmit( + self.session.send("schema", text.encode(), CultNetRudpSendOptions(reliable=True, ordered=True)) + ) + + def peer(self): + return next(p for p in self.server.peers if p.remote_addr == self.addr) + + def close(self) -> None: + self.socket.close() + + +class MultiPeerServerTests(unittest.TestCase): + def setUp(self) -> None: + self.server = make_server(initial_sequence=500, accept_payload=b"acc") + self.addCleanup(self.server.close) + self.clients: list[Client] = [] + + def client(self, initial_sequence: int) -> Client: + client = Client(self.server, initial_sequence) + self.clients.append(client) + self.addCleanup(client.close) + return client + + def test_interleaved_peers_each_get_their_own_session_and_frames(self) -> None: + x, y = self.client(10), self.client(20) + x.connect(b"hx") + y.connect(b"hy") + self.assertEqual(len(self.server.peers), 2) + self.assertEqual(x.peer().connect_payload, b"hx") + self.assertEqual(y.peer().connect_payload, b"hy") + + x.write("x1") + y.write("y1") + x.write("x2") + y.write("y2") + frames = pump(self.server) + by_peer = {} + for item in frames: + by_peer.setdefault(item.peer.remote_addr, []).append(item.frame.payload) + self.assertEqual(by_peer, {x.addr: [b"x1", b"x2"], y.addr: [b"y1", b"y2"]}) + + self.server.send(x.peer(), "schema", b"to-x") + self.server.send(y.peer(), "schema", b"to-y") + for client, expected in ((x, b"to-x"), (y, b"to-y")): + data = [p for p in client.receive() if p.packet_type == CultNetRudpPacketType.DATA] + self.assertEqual([p.payload for p in data], [expected]) + + def test_one_peer_ending_does_not_disturb_the_other(self) -> None: + x, y = self.client(10), self.client(20) + x.connect() + y.connect() + y_peer = y.peer() + ended = [] + self.server.peer_disconnected.append(ended.append) + + x.transmit(x.session.create_disconnect(b"bye")) + pump(self.server) + self.assertEqual(len(self.server.peers), 1) + self.assertEqual([p.disconnect_reason for p in ended], [b"bye"]) + self.assertIs(y.peer(), y_peer) + + # Datagrams from the ended endpoint belong to no session. + before = self.server.stats.packets_dropped + x_late = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=CONNECTION_ID, initial_sequence=99)) + x_late.accept_connect(x.session.create_connect(0), 0) + x.transmit(x_late.send("schema", b"late", CultNetRudpSendOptions(reliable=True, ordered=True))) + pump(self.server) + self.assertEqual(self.server.stats.packets_dropped, before + 1) + + y.write("still here") + self.assertEqual([f.frame.payload for f in pump(self.server)], [b"still here"]) + self.server.send(y_peer, "schema", b"reply") + self.assertEqual( + [p.payload for p in y.receive() if p.packet_type == CultNetRudpPacketType.DATA], [b"reply"] + ) + + def test_a_new_connect_from_the_same_endpoint_replaces_only_that_peer(self) -> None: + x, y = self.client(10), self.client(20) + x.connect() + y.connect() + old_y = y.peer() + ended = [] + self.server.peer_disconnected.append(ended.append) + + y.session = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=CONNECTION_ID, initial_sequence=30)) + y.connect(b"again") + self.assertEqual(ended, [old_y]) + self.assertEqual(old_y.disconnect_reason, b"replaced by a new Connect generation") + self.assertIsNot(y.peer(), old_y) + self.assertEqual(y.peer().connect_payload, b"again") + with self.assertRaises(ConnectionError): + self.server.send(old_y, "schema", b"to the old session") + self.assertEqual(len(self.server.peers), 2) + + def test_a_repeated_connect_is_answered_with_the_owed_accept_and_starts_nothing(self) -> None: + x = self.client(10) + connect = x.session.create_connect(0, b"h") + x.transmit(connect) + pump(self.server) + first = x.peer() + x.transmit(connect) + pump(self.server) + self.assertIs(x.peer(), first) + accepts = [p for p in x.receive() if p.packet_type == CultNetRudpPacketType.ACCEPT] + self.assertEqual(len(accepts), 2) + self.assertEqual(accepts[0].sequence, accepts[1].sequence) + + def test_strays_are_dropped_and_counted(self) -> None: + x = self.client(10) + x.socket.sendto(b"not rudp", self.server.socket.getsockname()) + other = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=CONNECTION_ID + 1, initial_sequence=1)) + x.transmit(other.create_connect(0)) + x.transmit( + CultNetRudpSession(CultNetRudpSessionOptions(connection_id=CONNECTION_ID, initial_sequence=1)).create_ping() + ) + pump(self.server) + self.assertEqual(self.server.stats.packets_dropped, 3) + self.assertEqual(self.server.peers, ()) + + def test_resends_go_only_to_peers_with_unacknowledged_writes(self) -> None: + x, y = self.client(10), self.client(20) + x.connect() + y.connect() + self.server.send(x.peer(), "schema", b"owed") + x.receive() # the first transmission is lost to the test: x never acks + y.receive() + time.sleep(0.02) + self.server.poll_resends() + x_data = [p for p in x.receive() if p.packet_type == CultNetRudpPacketType.DATA] + self.assertEqual([p.payload for p in x_data], [b"owed"]) + self.assertEqual([p for p in y.receive() if p.packet_type == CultNetRudpPacketType.DATA], []) + + def test_broadcast_reaches_every_peer(self) -> None: + x, y = self.client(10), self.client(20) + x.connect() + y.connect() + self.assertEqual(self.server.broadcast("schema", b"all"), []) + for client in (x, y): + self.assertEqual( + [p.payload for p in client.receive() if p.packet_type == CultNetRudpPacketType.DATA], [b"all"] + ) + + def test_an_application_ending_a_session_tells_the_peer_and_drops_its_queued_frames(self) -> None: + x, y = self.client(10), self.client(20) + x.connect() + y.connect() + x.write("x1") + y.write("y1") + # Both frames are decoded into the server's queue; ending x discards x's. + progressed, first = self.server.try_receive_once() + self.assertTrue(progressed) + pump_rest = pump(self.server) + frames = [first] + pump_rest if first else pump_rest + peer_x = x.peer() + self.assertTrue(self.server.end_peer_session(peer_x, b"server could not handle a message")) + self.assertFalse(self.server.end_peer_session(peer_x, b"again"), "an ended session owes no second goodbye") + goodbye = [p for p in x.receive() if p.packet_type == CultNetRudpPacketType.DISCONNECT] + self.assertEqual([p.payload for p in goodbye], [b"server could not handle a message"]) + self.assertEqual(len(self.server.peers), 1) + self.assertEqual({f.peer.remote_addr for f in frames}, {x.addr, y.addr}) + + def test_a_datagram_too_large_to_send_is_that_callers_error_and_queues_nothing(self) -> None: + x = self.client(10) + x.connect() + peer = x.peer() + owed = peer.pending_reliable_packet_count # the Accept, not yet acknowledged + with self.assertRaises(OSError) as raised: + self.server.send(peer, "schema", bytes(70_000)) + self.assertEqual(raised.exception.errno, errno.EMSGSIZE) + self.assertEqual(peer.pending_reliable_packet_count, owed) + self.assertEqual(len(self.server.peers), 1) + # The refused write left no gap in the sequence space the peer is waiting on. + self.server.send(peer, "schema", b"fine") + data = [p for p in x.receive() if p.packet_type == CultNetRudpPacketType.DATA] + self.assertEqual([(p.sequence, p.payload) for p in data], [(501, b"fine")]) + + +class UnsendableAfterFirstFragmentSocket: + """Delegates to a real socket; fails permanently once its budget of sends to one address is spent.""" + + def __init__(self, real: socket.socket, addr: tuple, budget: int) -> None: + self._real, self._addr, self._budget = real, addr, budget + + def sendto(self, wire: bytes, addr: tuple) -> int: + if tuple(addr) == self._addr: + if self._budget == 0: + raise OSError(errno.EMSGSIZE, "Message too long") + self._budget -= 1 + return self._real.sendto(wire, addr) + + def __getattr__(self, name: str): + return getattr(self._real, name) + + +class UnsendableSessionTests(unittest.TestCase): + def test_a_later_fragment_that_cannot_be_sent_ends_that_session_only(self) -> None: + server = make_server(initial_sequence=500, max_fragment_bytes=8) + self.addCleanup(server.close) + x, y = Client(server, 10), Client(server, 20) + self.addCleanup(x.close) + self.addCleanup(y.close) + x.connect() + y.connect() + x_peer = x.peer() + server.socket = UnsendableAfterFirstFragmentSocket(server.socket, x.addr, budget=1) + ended = [] + server.peer_disconnected.append(ended.append) + + with self.assertRaises(OSError): + server.send(x_peer, "schema", bytes(24)) + self.assertEqual(ended, [x_peer]) + self.assertEqual(x_peer.disconnect_reason, b"packet could not be sent: EMSGSIZE") + self.assertEqual(len(server.peers), 1) + # The goodbye could not be sent either (same address), so the peer is only retired; y is unharmed. + server.send(y.peer(), "schema", b"ok") + self.assertEqual( + [p.payload for p in y.receive() if p.packet_type == CultNetRudpPacketType.DATA], [b"ok"] + ) + + +class ReferenceTranscriptTests(unittest.TestCase): + """Replays the datagrams the C# reference listener was captured answering, byte for byte.""" + + def test_the_listener_answers_with_the_bytes_the_reference_does(self) -> None: + server = make_server( + connection_id=0x43554C54, initial_sequence=500, max_fragment_bytes=8, accept_payload=b"acc", resend_delay_ms=250 + ) + self.addCleanup(server.close) + sockets = {label: bind() for label in ("X", "Y")} + for sock in sockets.values(): + self.addCleanup(sock.close) + target = server.socket.getsockname() + + steps: list[tuple[list[str], list[tuple[str, str]]]] = [] + for line in VECTORS.read_text().splitlines(): + if line.startswith("#") or not line.strip(): + continue + fields = line.split(" ") + if fields[0] == "out": + steps[-1][1].append((fields[1], fields[2])) + else: + steps.append((fields, [])) + self.assertGreater(len(steps), 10) + + for action, expected in steps: + if action[0] == "in": + sockets[action[1]].sendto(bytes.fromhex(action[2]), target) + while not server.try_receive_once()[0]: + time.sleep(0.001) + else: + peer = next(p for p in server.peers if p.remote_addr == sockets[action[1]].getsockname()) + server.send(peer, action[2], bytes.fromhex(action[3])) + actual = [(label, wire.hex()) for label in ("X", "Y") for wire in drain(sockets[label])] + self.assertEqual(actual, expected, f"after {' '.join(action[:2])}") + self.assertEqual(len(server.peers), 1) + self.assertEqual(server.stats.packets_dropped, 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/GameCult.Networking.Tests/CultNetRudpMultiPeerVectorTests.cs b/tests/GameCult.Networking.Tests/CultNetRudpMultiPeerVectorTests.cs new file mode 100644 index 00000000..0985d516 --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetRudpMultiPeerVectorTests.cs @@ -0,0 +1,155 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Text; +using NUnit.Framework; + +namespace GameCult.Networking.Tests +{ + /// + /// The multi-peer listener's golden in tests/vectors/rudp-multipeer-server: two peers on one socket, the + /// datagrams each sends and the datagrams the C# reference answers with. The C# reference is the only writer + /// (CULTNET_WRITE_VECTORS=1); every other run replays the scenario and compares it byte for byte. Other runtimes' + /// listeners replay the same transcript and are held to the same answers. + /// + public class CultNetRudpMultiPeerVectorTests + { + private const string WriteVariable = "CULTNET_WRITE_VECTORS"; + private const uint ConnectionId = 0x43554c54; + + private sealed class Client + { + public string Label = ""; + public Socket Socket = null!; + public CultNetRudpSession Session = null!; + } + + private static string Hex(byte[] bytes) => Convert.ToHexString(bytes).ToLowerInvariant(); + + private static Socket Bind() + { + var socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); + socket.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + return socket; + } + + [Test] + public void TheReferenceListenerAnswersTwoPeersWithThePinnedBytes() + { + var transcript = Run(); + var path = Path.Combine(VectorDirectory(), "transcript.txt"); + if (Environment.GetEnvironmentVariable(WriteVariable) == "1") + { + File.WriteAllText(path, transcript); + Assert.Pass("wrote " + path); + } + Assert.That(transcript, Is.EqualTo(File.ReadAllText(path))); + } + + private static string VectorDirectory() + { + var directory = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); + while (directory != null && !Directory.Exists(Path.Combine(directory.FullName, "tests", "vectors"))) + directory = directory.Parent; + if (directory == null) throw new InvalidOperationException("tests/vectors not found above the test directory."); + return Path.Combine(directory.FullName, "tests", "vectors", "rudp-multipeer-server"); + } + + private static string Run() + { + var log = new StringBuilder(); + log.Append("# config connection_id=0x43554c54 initial_sequence=500 max_fragment_bytes=8 accept_payload=616363\n"); + log.Append("# in | send | out ; peer labels name remote endpoints\n"); + var serverSocket = Bind(); + using var server = new CultNetRudpSocketTransportServer(new CultNetRudpSocketTransportServerOptions + { + RuntimeId = "csharp-vector", + Socket = serverSocket, + ConnectionId = ConnectionId, + InitialSequence = 500, + MaxFragmentBytes = 8, + AcceptPayload = Encoding.ASCII.GetBytes("acc") + }); + var target = serverSocket.LocalEndPoint!; + Client Make(string label, uint sequence) => new Client + { + Label = label, + Socket = Bind(), + Session = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId, InitialSequence = sequence }) + }; + var x = Make("X", 10); + var y = Make("Y", 20); + var clients = new[] { x, y }; + + void Deliver(Client client, CultNetRudpPacket packet) + { + var wire = CultNetRudpPacketCodec.Encode(packet); + log.Append($"in {client.Label} {Hex(wire)}\n"); + client.Socket.SendTo(wire, target); + while (!server.TryReceiveOnce(out _)) + System.Threading.Thread.Sleep(1); + Flush(); + } + + void Flush() + { + foreach (var client in clients) + { + var buffer = new byte[65535]; + EndPoint from = new IPEndPoint(IPAddress.Any, 0); + while (client.Socket.Poll(30_000, SelectMode.SelectRead)) + { + var received = client.Socket.ReceiveFrom(buffer, ref from); + var wire = buffer.AsSpan(0, received).ToArray(); + log.Append($"out {client.Label} {Hex(wire)}\n"); + // Clients that stay connected take the server's packets so their acks stay true. + var packet = CultNetRudpPacketCodec.Decode(wire); + if (client.Session.Connected || packet.PacketType == CultNetRudpPacketType.Accept) + client.Session.Receive(packet, 0); + } + } + } + + CultNetRudpSocketServerPeer PeerOf(Client client) => + server.Peers.Single(p => p.RemoteEndPoint.Equals(client.Socket.LocalEndPoint)); + + void Send(Client client, string channel, byte[] payload) + { + log.Append($"send {client.Label} {channel} {Hex(payload)}\n"); + server.Send(PeerOf(client), channel, payload); + Flush(); + } + + CultNetRudpPacket Data(Client client, string text) => + client.Session.Send("schema", Encoding.ASCII.GetBytes(text), new CultNetRudpSendOptions { Reliable = true, Ordered = true }); + + var xConnect = x.Session.CreateConnect(0, Encoding.ASCII.GetBytes("hx")); + Deliver(x, xConnect); + Deliver(y, y.Session.CreateConnect(0, Encoding.ASCII.GetBytes("hy"))); + // A repeat of X's accepted Connect is answered with the Accept owed, not a new session. + Deliver(x, xConnect); + Deliver(x, Data(x, "ping-x")); + Deliver(y, Data(y, "ping-y")); + Send(x, "schema", Encoding.ASCII.GetBytes("pong")); + // Larger than the fragment size: three fragments, one per datagram. + Send(y, "schema", Encoding.ASCII.GetBytes("0123456789abcdefghijklmn")); + Send(x, "latest", Encoding.ASCII.GetBytes("sample")); + // X leaves; Y is undisturbed, and what X sends afterwards belongs to no session. + var late = Data(x, "after-bye"); + Deliver(x, x.Session.CreateDisconnect(Encoding.ASCII.GetBytes("bye"))); + Deliver(x, late); + Send(y, "schema", Encoding.ASCII.GetBytes("still-here")); + // Y reconnects from the same endpoint: a new generation replaces only Y. + y.Session = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId, InitialSequence = 30 }); + Deliver(y, y.Session.CreateConnect(0, Encoding.ASCII.GetBytes("hy2"))); + Send(y, "schema", Encoding.ASCII.GetBytes("new-gen")); + log.Append($"# peers {server.Peers.Count} dropped {server.Stats.PacketsDropped}\n"); + foreach (var client in clients) client.Socket.Dispose(); + return log.ToString(); + } + } +} diff --git a/tests/vectors/rudp-multipeer-server/README.md b/tests/vectors/rudp-multipeer-server/README.md new file mode 100644 index 00000000..21231461 --- /dev/null +++ b/tests/vectors/rudp-multipeer-server/README.md @@ -0,0 +1,13 @@ +# rudp-multipeer-server vectors + +Written only by the C# reference, `tests/GameCult.Networking.Tests/CultNetRudpMultiPeerVectorTests.cs`, with +`CULTNET_WRITE_VECTORS=1`. Every other run replays the scenario and compares it byte for byte; a changed vector is a reviewed +commit. Other runtimes' multi-peer listeners replay the same transcript and are held to the same answers +(`packages/cultnet-py/tests/test_rudp_multipeer_server.py`). + +`transcript.txt` is one listener (`connection_id`, `initial_sequence`, `max_fragment_bytes` and `accept_payload` in its first line) +serving two loopback peers, X and Y. Each line is an action or what it caused: `in ` is a datagram the peer sends the +listener, `send ` is a payload the application sends that peer, and the `out ` lines after it are +every datagram the listener put on the wire in answer, X's before Y's. Peers are named by label because ports are ephemeral. The +scenario covers a Connect, a repeat of an accepted Connect, data and acks, a fragmented send, an unreliable `latest` send, one peer's +Disconnect and the datagram from it that follows, and the other peer's reconnect from the same endpoint. diff --git a/tests/vectors/rudp-multipeer-server/transcript.txt b/tests/vectors/rudp-multipeer-server/transcript.txt new file mode 100644 index 00000000..50f2f64f --- /dev/null +++ b/tests/vectors/rudp-multipeer-server/transcript.txt @@ -0,0 +1,30 @@ +# config connection_id=0x43554c54 initial_sequence=500 max_fragment_bytes=8 accept_payload=616363 +# in | send | out ; peer labels name remote endpoints +in X 434e52300001032b43554c540000000a0000000000000000000000000000000000020700636f6e74726f6c6878 +out X 434e52300002032b43554c54000001f40000000a00000000000000000000000000030700636f6e74726f6c616363 +in Y 434e52300001032b43554c54000000140000000000000000000000000000000000020700636f6e74726f6c6879 +out Y 434e52300002032b43554c54000001f40000001400000000000000000000000000030700636f6e74726f6c616363 +in X 434e52300001032b43554c540000000a0000000000000000000000000000000000020700636f6e74726f6c6878 +out X 434e52300002032b43554c54000001f40000000a00000000000000000000000000030700636f6e74726f6c616363 +in X 434e52300003032a43554c540000000b000001f400000000000000000000000000060600736368656d6170696e672d78 +out X 434e52300004002b43554c54000000000000000b00000001000000000000000000000700636f6e74726f6c +in Y 434e52300003032a43554c5400000015000001f400000000000000000000000000060600736368656d6170696e672d79 +out Y 434e52300004002b43554c54000000000000001500000001000000000000000000000700636f6e74726f6c +send X schema 706f6e67 +out X 434e52300003032a43554c54000001f50000000b00000001000000000000000000040600736368656d61706f6e67 +send Y schema 303132333435363738396162636465666768696a6b6c6d6e +out Y 434e523000030b2a43554c54000001f50000001500000001000100000003000000080600736368656d613031323334353637 +out Y 434e523000030b2a43554c54000001f60000001500000001000100010003000000080600736368656d613839616263646566 +out Y 434e523000030b2a43554c54000001f70000001500000001000100020003000000080600736368656d616768696a6b6c6d6e +send X latest 73616d706c65 +out X 434e52300003042a43554c54000000010000000b000000010000000000000000000606006c617465737473616d706c65 +in X 434e52300007002b43554c5400000000000001f500000001000000000000000000030700636f6e74726f6c627965 +in X 434e52300003032a43554c540000000c000001f500000001000000000000000000090600736368656d6161667465722d627965 +send Y schema 7374696c6c2d68657265 +out Y 434e523000030b2a43554c54000001f80000001500000001000200000002000000080600736368656d617374696c6c2d6865 +out Y 434e523000030b2a43554c54000001f90000001500000001000200010002000000020600736368656d617265 +in Y 434e52300001032b43554c540000001e0000000000000000000000000000000000030700636f6e74726f6c687932 +out Y 434e52300002032b43554c54000001f40000001e00000000000000000000000000030700636f6e74726f6c616363 +send Y schema 6e65772d67656e +out Y 434e52300003032a43554c54000001f50000001e00000000000000000000000000070600736368656d616e65772d67656e +# peers 1 dropped 1 From 51941fa61bef96959a0dabb85746b4e18b63eb6f Mon Sep 17 00:00:00 2001 From: Libby Date: Sat, 7 Nov 2026 01:39:57 +0000 Subject: [PATCH 73/88] Carrier vectors: drop the derived hasHop member from the reference's expected JSON Co-Authored-By: Claude Sonnet 5.5 --- .../interop/carrier-vectors.cs-written.json | 12 ++++-------- .../CarrierVectorTests.cs | 19 ++++++++++++++++++- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/contracts/cultnet/interop/carrier-vectors.cs-written.json b/contracts/cultnet/interop/carrier-vectors.cs-written.json index 49f12ed0..394724cf 100644 --- a/contracts/cultnet/interop/carrier-vectors.cs-written.json +++ b/contracts/cultnet/interop/carrier-vectors.cs-written.json @@ -16,8 +16,7 @@ "projection": "header", "descending": false, "limit": null, - "cursor": null, - "hasHop": false + "cursor": null }, "shardId": null, "shardEpoch": null @@ -70,8 +69,7 @@ "projection": "document", "descending": true, "limit": 200, - "cursor": "opaque-cursor", - "hasHop": true + "cursor": "opaque-cursor" }, "shardId": "shard-1", "shardEpoch": 7 @@ -94,8 +92,7 @@ "projection": "header", "descending": false, "limit": null, - "cursor": null, - "hasHop": false + "cursor": null }, "includeSnapshot": true, "consumerRuntimeId": null, @@ -151,8 +148,7 @@ "projection": "document", "descending": true, "limit": 200, - "cursor": "opaque-cursor", - "hasHop": true + "cursor": "opaque-cursor" }, "includeSnapshot": false, "consumerRuntimeId": "consumer", diff --git a/tests/GameCult.Networking.Tests/CarrierVectorTests.cs b/tests/GameCult.Networking.Tests/CarrierVectorTests.cs index f294bc0f..8d50bb76 100644 --- a/tests/GameCult.Networking.Tests/CarrierVectorTests.cs +++ b/tests/GameCult.Networking.Tests/CarrierVectorTests.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Text.Json; +using System.Text.Json.Nodes; using MessagePack; using NUnit.Framework; @@ -120,10 +121,26 @@ namespace GameCult.Networking.Tests name, schemaVersion = message.SchemaVersion, messagePackBase64 = Convert.ToBase64String(bytes), - expected = JsonSerializer.SerializeToElement(message, message.GetType(), JsonOptions) + expected = WithoutDerivedMembers(JsonSerializer.SerializeToNode(message, message.GetType(), JsonOptions)) }; } + // CultNetSelection.HasHop is an [IgnoreMember] convenience, not wire; System.Text.Json would emit it. + private static JsonNode? WithoutDerivedMembers(JsonNode? node) + { + switch (node) + { + case JsonObject obj: + obj.Remove("hasHop"); + foreach (var child in obj.Select(pair => pair.Value).ToArray()) WithoutDerivedMembers(child); + break; + case JsonArray array: + foreach (var child in array) WithoutDerivedMembers(child); + break; + } + return node; + } + private static string VectorPath() { var directory = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); From 11933ab65b4e2cb91c86db4adb4db3e6af1dc921 Mon Sep 17 00:00:00 2001 From: Libby Date: Sat, 7 Nov 2026 01:39:57 +0000 Subject: [PATCH 74/88] cultnet-ts: selection/v1 carriers (snapshot request, subscribe, raw response), judged against the reference's bytes Co-Authored-By: Claude Sonnet 5.5 --- packages/cultnet-ts/CHANGELOG.md | 4 + packages/cultnet-ts/README.md | 1 + packages/cultnet-ts/src/contracts.ts | 185 +++++++++++++++++- packages/cultnet-ts/src/index.ts | 16 ++ packages/cultnet-ts/src/selection-carriers.ts | 178 +++++++++++++++++ packages/cultnet-ts/src/selection.ts | 115 +++++++++++ .../cultnet-ts/test/carrier-vectors.test.ts | 122 ++++++++++++ 7 files changed, 620 insertions(+), 1 deletion(-) create mode 100644 packages/cultnet-ts/src/selection-carriers.ts create mode 100644 packages/cultnet-ts/src/selection.ts diff --git a/packages/cultnet-ts/CHANGELOG.md b/packages/cultnet-ts/CHANGELOG.md index eae823c4..a5b8bc4e 100644 --- a/packages/cultnet-ts/CHANGELOG.md +++ b/packages/cultnet-ts/CHANGELOG.md @@ -9,8 +9,12 @@ All notable changes to `cultnet-ts` are recorded here, following - `CultNetRudpSocketTransportServer`: one UDP socket serving many peers, each remote address and port with its own reliable session (`peerConnected`, `frame`, `peerDisconnected` events; `send(peer, ...)` answers on the peer a frame came from). A second client no longer replaces the first; Connects beyond `maxPeers` (default 256) are dropped, and `close()` resolves once every peer's goodbye has left the socket. The shape follows the Rust `CultNetRudpServerHub` and C# `CultNetRudpSocketTransportServer`, and the wire format is unchanged. The optional `peerTimeoutMs` (idle peers ended and told so; default off) exists only in TypeScript. `CultNetRudpSocketTransportConnection` in `server` mode still serves exactly one peer. +- The `cultnet.snapshot_request.v1`, `cultnet.database_subscribe.v1` and `cultnet.snapshot_response_raw.v1` carriers and the selection payload types (`CultNetSelection`, `CultNetRawDocumentHeader`, `CultNetEdge`). `parseCultNetMessage` accepts them (absent members arrive as absent keys; a page carries exactly one of `headers`/`documents`; an empty or blank `schemas`/`keys` entry throws `CultNetSelectionInvalidError` naming its field), and `encodeCultNetMessageForWire` writes the reference's nil-keyed, declaration-ordered map. Judged against bytes the C# reference wrote (`contracts/cultnet/interop/carrier-vectors.cs-written.json`) and the selection vectors C# and Rust wrote. The selection engine (evaluation, cursors, the declaration-dependent door checks) is not ported. + ### Breaking +- `CultNetErrorMessage` carries `routingHint`, `code` and `details` (null when absent), as the reference writes them and `cultnet.error.schema.json` requires; the interface previously dropped them. New `CultNetErrorCode`, `CultNetErrorDetails` and `CultNetShardRoutingHint` types. + - `CultMeshCdnChunkRef` carries `id` (slot 4), the reference's element id. `encodeManifest` writes five-slot chunk references and `packContent` sets the decimal offset as the id, as the reference does; a manifest written before ids reads `""` and is rewritten with the slot. Content vectors regenerated from the reference. - Needs the cultcache-ts release that ships with this change (its manifest range moves with that release): the Idunn runtime authority reader calls `readSingleFileStore`, which 0.14.x does not export. An absent authority file is now a plain `Error` without a code. diff --git a/packages/cultnet-ts/README.md b/packages/cultnet-ts/README.md index c766793b..acbf5a6a 100644 --- a/packages/cultnet-ts/README.md +++ b/packages/cultnet-ts/README.md @@ -65,6 +65,7 @@ Current message families: - `cultnet.snapshot_request.v0` - `cultnet.snapshot_response.v0` - `cultnet.snapshot_response_raw.v0` +- `cultnet.snapshot_request.v1`, `cultnet.database_subscribe.v1`, `cultnet.snapshot_response_raw.v1` (typed-selection carriers; no selection engine yet) - `cultnet.sample.change_name.v0` - `cultnet.sample.chat.v0` diff --git a/packages/cultnet-ts/src/contracts.ts b/packages/cultnet-ts/src/contracts.ts index dcaff6f2..ae531245 100644 --- a/packages/cultnet-ts/src/contracts.ts +++ b/packages/cultnet-ts/src/contracts.ts @@ -33,8 +33,25 @@ import verseCatalogRequestSchema from "../contracts/cultnet/cultmesh.verse-catal import verseCatalogResponseSchema from "../contracts/cultnet/cultmesh.verse-catalog-response.schema.json"; import sessionOpenSchema from "../contracts/cultnet/cultmesh.session-open.schema.json"; import sessionAcceptedSchema from "../contracts/cultnet/cultmesh.session-accepted.schema.json"; +import selectionSchema from "../contracts/cultnet/cultnet.selection.schema.json"; +import rawDocumentHeaderSchema from "../contracts/cultnet/cultnet.raw-document-header.schema.json"; +import snapshotRequestV1Schema from "../contracts/cultnet/cultnet.snapshot-request.v1.schema.json"; +import snapshotResponseRawV1Schema from "../contracts/cultnet/cultnet.snapshot-response-raw.v1.schema.json"; +import databaseSubscribeV1Schema from "../contracts/cultnet/cultnet.database-subscribe.v1.schema.json"; import ghostlightAgentStateSchema from "../contracts/cultnet/ghostlight.agent-state.schema.json"; +import { + encodeSelectionCarrierForWire, + normalizeSelectionCarrierNulls, + type CultNetSelectionCarrier, +} from "./selection-carriers"; +import { + validateSelectionShape, + type CultNetEdge, + type CultNetRawDocumentHeader, + type CultNetSelection, +} from "./selection"; + export type CultNetWireContract = "cultnet.schema.v0" | "gamecult.networking.v0"; export type CultNetSchemaVersion = @@ -53,6 +70,9 @@ export type CultNetSchemaVersion = | "cultnet.snapshot_response.v0" | "cultnet.snapshot_response_raw.v0" | "cultnet.database_subscribe.v0" + | "cultnet.snapshot_request.v1" + | "cultnet.snapshot_response_raw.v1" + | "cultnet.database_subscribe.v1" | "cultnet.database_unsubscribe.v0" | "cultnet.database_change_raw.v0" | "cultnet.schema_catalog_request.v0" @@ -275,6 +295,46 @@ export interface CultNetDatabaseSubscribeMessage { supportedBodyTransports?: string[]; } +/** A document snapshot through a typed selection (docs/cultnet-selection-cut.md). */ +export interface CultNetSnapshotRequestV1Message { + schemaVersion: "cultnet.snapshot_request.v1"; + messageId: string; + selection: CultNetSelection; + shardId?: string; + shardEpoch?: number; +} + +/** A live subscription through a typed selection. */ +export interface CultNetDatabaseSubscribeV1Message { + schemaVersion: "cultnet.database_subscribe.v1"; + messageId: string; + subscriptionId: string; + selection: CultNetSelection; + /** Default true. */ + includeSnapshot?: boolean; + consumerRuntimeId?: string; + bodyIds?: string[]; + supportedBodyTransports?: string[]; +} + +/** One selection page: exactly one of `headers` / `documents` is present, by projection. */ +export interface CultNetSnapshotResponseRawV1Message { + schemaVersion: "cultnet.snapshot_response_raw.v1"; + messageId: string; + /** Rows on this page. */ + matched: number; + /** The snapshot this page is exact as of. */ + asOf: number; + /** Cursor for the next page, absent on the last page. */ + next?: string; + headers?: CultNetRawDocumentHeader[]; + documents?: CultNetRawDocumentRecord[]; + edges?: CultNetEdge[]; + shardId?: string; + shardEpoch?: number; + shardLogSequence?: number; +} + export interface CultNetDatabaseUnsubscribeMessage { schemaVersion: "cultnet.database_unsubscribe.v0"; messageId: string; @@ -687,6 +747,9 @@ export type CultNetMessage = | CultNetSnapshotResponseMessage | CultNetSnapshotResponseRawMessage | CultNetDatabaseSubscribeMessage + | CultNetSnapshotRequestV1Message + | CultNetSnapshotResponseRawV1Message + | CultNetDatabaseSubscribeV1Message | CultNetDatabaseUnsubscribeMessage | CultNetDatabaseChangeRawMessage | CultNetSchemaCatalogRequestMessage @@ -721,6 +784,9 @@ const CULTNET_MESSAGE_SCHEMAS = [ snapshotResponseSchema, snapshotResponseRawSchema, databaseSubscribeSchema, + snapshotRequestV1Schema, + snapshotResponseRawV1Schema, + databaseSubscribeV1Schema, databaseUnsubscribeSchema, databaseChangeRawSchema, schemaCatalogRequestSchema, @@ -739,6 +805,8 @@ const cultNetValidators = new Map(); for (const schema of [ documentRecordSchema, rawDocumentRecordSchema, + rawDocumentHeaderSchema, + selectionSchema, documentMutationContractSchema, transportProfileSchema, schemaDescriptorSchema, @@ -785,6 +853,20 @@ export function parseCultNetMessage(input: unknown, wireContract: CultNetWireCon return input as CultNetSnapshotResponseRawMessage; } + if (schemaVersion === "cultnet.snapshot_response_raw.v1") { + validateSnapshotResponseRawV1Message(input); + return input as CultNetSnapshotResponseRawV1Message; + } + + if (schemaVersion === "cultnet.snapshot_request.v1" || schemaVersion === "cultnet.database_subscribe.v1") { + // The door first, so an empty or blank schemas/keys entry is a typed refusal naming its field + // rather than a generic schema-validation failure. + const selection = (input as { selection?: unknown }).selection; + if (selection && typeof selection === "object") { + validateSelectionShape(selection as CultNetSelection); + } + } + if (schemaVersion === "cultnet.database_change_raw.v0") { validateDatabaseChangeRawMessage(input); return input as CultNetDatabaseChangeRawMessage; @@ -806,7 +888,14 @@ export function encodeCultNetMessageForWire( return encodeGameCultNetworkingMessage(message); } - return message; + switch (message.schemaVersion) { + case "cultnet.snapshot_request.v1": + case "cultnet.database_subscribe.v1": + case "cultnet.snapshot_response_raw.v1": + return encodeSelectionCarrierForWire(message as CultNetSelectionCarrier); + default: + return message; + } } export function validateGhostlightAgentState(input: unknown): GhostlightAgentStateDocument { @@ -1134,6 +1223,89 @@ function validateSnapshotResponseRawMessage( }); } +function validateSnapshotResponseRawV1Message( + input: unknown, +): asserts input is CultNetSnapshotResponseRawV1Message { + const candidate = validateBinaryMessageRoot(input, "cultnet.snapshot_response_raw.v1"); + const fail = (path: string, problem: string): never => { + throw new Error(`Validation failed for cultnet.snapshot_response_raw.v1: ${path}: ${problem}`); + }; + for (const counter of ["matched", "asOf"] as const) { + const value = candidate[counter]; + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) { + fail(`/${counter}`, "must be a non-negative integer"); + } + } + if (typeof candidate.next !== "undefined" && typeof candidate.next !== "string") { + fail("/next", "must be a string"); + } + const hasHeaders = typeof candidate.headers !== "undefined"; + const hasDocuments = typeof candidate.documents !== "undefined"; + if (hasHeaders === hasDocuments) { + fail("/", "exactly one of headers and documents must be present"); + } + if (hasHeaders) { + if (!Array.isArray(candidate.headers)) fail("/headers", "must be an array"); + (candidate.headers as unknown[]).forEach((header, index) => validateRawDocumentHeader(header, `/headers/${index}`)); + } else { + if (!Array.isArray(candidate.documents)) fail("/documents", "must be an array"); + (candidate.documents as unknown[]).forEach((document, index) => validateRawDocumentRecord(document, `/documents/${index}`)); + } + if (typeof candidate.edges !== "undefined") { + if (!Array.isArray(candidate.edges)) fail("/edges", "must be an array"); + (candidate.edges as unknown[]).forEach((edge, index) => validateEdge(edge, `/edges/${index}`)); + } + requireOptionalNonEmptyString(candidate.shardId, "/shardId"); + for (const counter of ["shardEpoch", "shardLogSequence"] as const) { + const value = candidate[counter]; + if (typeof value !== "undefined" && (typeof value !== "number" || !Number.isSafeInteger(value))) { + fail(`/${counter}`, "must be an integer"); + } + } +} + +function validateRawDocumentHeader(input: unknown, instancePath: string): void { + if (!input || typeof input !== "object") { + throw new Error(`Validation failed for raw CultNet document header: ${instancePath}: must be an object`); + } + const candidate = input as Record; + requireNonEmptyString(candidate.schemaId, `${instancePath}/schemaId`); + requireOptionalNonEmptyString(candidate.schemaName, `${instancePath}/schemaName`); + requireOptionalNonEmptyString(candidate.schemaVersion, `${instancePath}/schemaVersion`); + requireOptionalNonEmptyString(candidate.schemaContentHash, `${instancePath}/schemaContentHash`); + requireNonEmptyString(candidate.recordKey, `${instancePath}/recordKey`); + requireNonEmptyString(candidate.storedAt, `${instancePath}/storedAt`); + requireOptionalNonEmptyString(candidate.sourceRuntimeId, `${instancePath}/sourceRuntimeId`); + requireOptionalNonEmptyString(candidate.sourceAgentId, `${instancePath}/sourceAgentId`); + requireOptionalNonEmptyString(candidate.sourceRole, `${instancePath}/sourceRole`); + requireOptionalStringArray(candidate.tags, `${instancePath}/tags`); +} + +function validateEdge(input: unknown, instancePath: string): void { + const fail = (path: string, problem: string): never => { + throw new Error(`Validation failed for cultnet.snapshot_response_raw.v1: ${path}: ${problem}`); + }; + if (!input || typeof input !== "object") { + fail(instancePath, "must be an object"); + } + const candidate = input as Record; + for (const end of ["from", "to"] as const) { + const ref = candidate[end] as Record | undefined; + if (!ref || typeof ref !== "object") { + fail(`${instancePath}/${end}`, "must be an object"); + } + requireNonEmptyString(ref!.schemaId, `${instancePath}/${end}/schemaId`); + requireNonEmptyString(ref!.recordKey, `${instancePath}/${end}/recordKey`); + } + requireNonEmptyString(candidate.role, `${instancePath}/role`); + if (typeof candidate.payloadEncoding !== "undefined" && candidate.payloadEncoding !== "messagepack") { + fail(`${instancePath}/payloadEncoding`, "must equal messagepack"); + } + if (typeof candidate.payload !== "undefined" && !(candidate.payload instanceof Uint8Array)) { + fail(`${instancePath}/payload`, "must be a Uint8Array"); + } +} + function validateDatabaseChangeRawMessage( input: unknown, ): asserts input is CultNetDatabaseChangeRawMessage { @@ -1166,6 +1338,7 @@ function validateBinaryMessageRoot( schemaVersion: | "cultnet.document_put_raw.v0" | "cultnet.snapshot_response_raw.v0" + | "cultnet.snapshot_response_raw.v1" | "cultnet.database_change_raw.v0", ): Record { if (!input || typeof input !== "object") { @@ -1268,6 +1441,11 @@ function normalizeCultNetOptionalNulls( "supportedBodyTransports", ]); return; + case "cultnet.snapshot_request.v1": + case "cultnet.database_subscribe.v1": + case "cultnet.snapshot_response_raw.v1": + normalizeSelectionCarrierNulls(schemaVersion, candidate); + return; case "cultnet.operation_request.v0": stripNullProperties(candidate, ["payloadEncoding", "sourceRuntimeId", "targetRuntimeId"]); return; @@ -1337,6 +1515,11 @@ export const cultNetSchemas = { snapshotResponseSchema, snapshotResponseRawSchema, databaseSubscribeSchema, + selectionSchema, + rawDocumentHeaderSchema, + snapshotRequestV1Schema, + snapshotResponseRawV1Schema, + databaseSubscribeV1Schema, databaseUnsubscribeSchema, databaseChangeRawSchema, schemaDescriptorSchema, diff --git a/packages/cultnet-ts/src/index.ts b/packages/cultnet-ts/src/index.ts index e45a87f3..d113fb49 100644 --- a/packages/cultnet-ts/src/index.ts +++ b/packages/cultnet-ts/src/index.ts @@ -3,6 +3,9 @@ export { encodeCultNetMessageForWire, parseCultNetMessage, validateGhostlightAgentState, + type CultNetDatabaseSubscribeV1Message, + type CultNetSnapshotRequestV1Message, + type CultNetSnapshotResponseRawV1Message, type CultNetDocumentDeleteMessage, type CultNetDocumentMutationContract, type CultNetDocumentPutMessage, @@ -65,6 +68,19 @@ export { type GhostlightVariableMap, type GhostlightWorldState, } from "./contracts"; +export { + CultNetSelectionInvalidError, + validateSelectionShape, + type CultNetCitation, + type CultNetEdge, + type CultNetFieldPredicate, + type CultNetIncoming, + type CultNetRawDocumentHeader, + type CultNetRecordRef, + type CultNetSelection, + type CultNetSelectionOperator, + type CultNetSelectionProjection, +} from "./selection"; export { base64ToBytes, bytesToBase64, diff --git a/packages/cultnet-ts/src/selection-carriers.ts b/packages/cultnet-ts/src/selection-carriers.ts new file mode 100644 index 00000000..21090f03 --- /dev/null +++ b/packages/cultnet-ts/src/selection-carriers.ts @@ -0,0 +1,178 @@ +// The two shape conversions the selection/v1 carriers need between the reference's wire and TypeScript: +// the reference writes every declared key, nil included, in declaration order (MessagePack-CSharp map +// mode; schema notes R-S/R-X), so a received message arrives with explicit nulls that are normalised +// to absent keys, and a sent message is rebuilt with every key in the reference's order. + +import type { + CultNetDatabaseSubscribeV1Message, + CultNetRawDocumentRecord, + CultNetSnapshotRequestV1Message, + CultNetSnapshotResponseRawV1Message, +} from "./contracts"; +import type { CultNetEdge, CultNetRawDocumentHeader, CultNetRecordRef, CultNetSelection } from "./selection"; + +export type CultNetSelectionCarrier = + | CultNetSnapshotRequestV1Message + | CultNetDatabaseSubscribeV1Message + | CultNetSnapshotResponseRawV1Message; + +type Bag = Record; + +const isBag = (value: unknown): value is Bag => !!value && typeof value === "object" && !Array.isArray(value); + +function stripNulls(candidate: unknown, keys: readonly string[]): void { + if (!isBag(candidate)) return; + for (const key of keys) { + if (candidate[key] === null) delete candidate[key]; + } +} + +function eachBag(list: unknown, visit: (item: Bag) => void): void { + if (!Array.isArray(list)) return; + for (const item of list) { + if (isBag(item)) visit(item); + } +} + +/** Drops the nil an absent selection member travels as, at every depth the selection carries one. */ +export function normalizeSelectionNulls(selection: unknown): void { + stripNulls(selection, ["schemas", "keys", "fields", "cites", "cited", "limit", "cursor"]); + if (!isBag(selection)) return; + eachBag(selection.fields, field => stripNulls(field, ["values", "number"])); + stripNulls(selection.cites, ["role"]); +} + +/** Drops the nil an absent member travels as, for one selection/v1 carrier message. */ +export function normalizeSelectionCarrierNulls( + schemaVersion: CultNetSelectionCarrier["schemaVersion"], + candidate: Bag, +): void { + switch (schemaVersion) { + case "cultnet.snapshot_request.v1": + stripNulls(candidate, ["shardId", "shardEpoch"]); + normalizeSelectionNulls(candidate.selection); + return; + case "cultnet.database_subscribe.v1": + stripNulls(candidate, ["consumerRuntimeId", "bodyIds", "supportedBodyTransports"]); + normalizeSelectionNulls(candidate.selection); + return; + case "cultnet.snapshot_response_raw.v1": + stripNulls(candidate, ["next", "headers", "documents", "edges", "shardId", "shardEpoch", "shardLogSequence"]); + eachBag(candidate.headers, header => stripNulls(header, HEADER_OPTIONAL)); + eachBag(candidate.documents, document => stripNulls(document, HEADER_OPTIONAL)); + eachBag(candidate.edges, edge => stripNulls(edge, ["payloadEncoding", "payload"])); + return; + } +} + +const HEADER_OPTIONAL = ["schemaName", "schemaVersion", "schemaContentHash", "sourceRuntimeId", "sourceAgentId", "sourceRole", "tags"] as const; + +const orNull = (value: T | undefined | null): T | null => value ?? null; + +function recordRefWire(ref: CultNetRecordRef) { + return { schemaId: ref.schemaId, recordKey: ref.recordKey }; +} + +function selectionWire(selection: CultNetSelection) { + return { + schemas: orNull(selection.schemas), + keys: orNull(selection.keys), + fields: selection.fields + ? selection.fields.map(field => ({ + index: field.index, + op: field.op, + values: orNull(field.values), + number: orNull(field.number), + })) + : null, + cites: selection.cites + ? { target: recordRefWire(selection.cites.target), role: orNull(selection.cites.role) } + : null, + cited: selection.cited ? { role: selection.cited.role, exists: selection.cited.exists } : null, + projection: selection.projection ?? "header", + descending: selection.descending ?? false, + limit: orNull(selection.limit), + cursor: orNull(selection.cursor), + }; +} + +function headerWire(header: CultNetRawDocumentHeader) { + return { + schemaId: header.schemaId, + schemaName: orNull(header.schemaName), + schemaVersion: orNull(header.schemaVersion), + schemaContentHash: orNull(header.schemaContentHash), + recordKey: header.recordKey, + storedAt: header.storedAt, + sourceRuntimeId: orNull(header.sourceRuntimeId), + sourceAgentId: orNull(header.sourceAgentId), + sourceRole: orNull(header.sourceRole), + tags: orNull(header.tags), + }; +} + +function documentWire(document: CultNetRawDocumentRecord) { + return { + schemaId: document.schemaId, + schemaName: orNull(document.schemaName), + schemaVersion: orNull(document.schemaVersion), + schemaContentHash: orNull(document.schemaContentHash), + recordKey: document.recordKey, + storedAt: document.storedAt, + payloadEncoding: document.payloadEncoding, + payload: document.payload, + sourceRuntimeId: orNull(document.sourceRuntimeId), + sourceAgentId: orNull(document.sourceAgentId), + sourceRole: orNull(document.sourceRole), + tags: orNull(document.tags), + }; +} + +function edgeWire(edge: CultNetEdge) { + return { + from: recordRefWire(edge.from), + role: edge.role, + to: recordRefWire(edge.to), + payloadEncoding: orNull(edge.payloadEncoding), + payload: orNull(edge.payload), + }; +} + +/** The reference's nil-keyed, declaration-ordered shape for one selection/v1 carrier message. */ +export function encodeSelectionCarrierForWire(message: CultNetSelectionCarrier): unknown { + switch (message.schemaVersion) { + case "cultnet.snapshot_request.v1": + return { + schemaVersion: message.schemaVersion, + messageId: message.messageId, + selection: selectionWire(message.selection), + shardId: orNull(message.shardId), + shardEpoch: orNull(message.shardEpoch), + }; + case "cultnet.database_subscribe.v1": + return { + schemaVersion: message.schemaVersion, + messageId: message.messageId, + subscriptionId: message.subscriptionId, + selection: selectionWire(message.selection), + includeSnapshot: message.includeSnapshot ?? true, + consumerRuntimeId: orNull(message.consumerRuntimeId), + bodyIds: orNull(message.bodyIds), + supportedBodyTransports: orNull(message.supportedBodyTransports), + }; + case "cultnet.snapshot_response_raw.v1": + return { + schemaVersion: message.schemaVersion, + messageId: message.messageId, + matched: message.matched, + asOf: message.asOf, + next: orNull(message.next), + headers: message.headers ? message.headers.map(headerWire) : null, + documents: message.documents ? message.documents.map(documentWire) : null, + edges: message.edges ? message.edges.map(edgeWire) : null, + shardId: orNull(message.shardId), + shardEpoch: orNull(message.shardEpoch), + shardLogSequence: orNull(message.shardLogSequence), + }; + } +} diff --git a/packages/cultnet-ts/src/selection.ts b/packages/cultnet-ts/src/selection.ts new file mode 100644 index 00000000..2bba1ebe --- /dev/null +++ b/packages/cultnet-ts/src/selection.ts @@ -0,0 +1,115 @@ +// cultnet.selection.v1 payload shapes (docs/cultnet-selection-cut.md, section 2), the carriers' common +// vocabulary: one typed selection, the header/edge rows a page answers with, and the door refusal. +// The messages that carry them live in contracts.ts; the nil-keyed wire encoding, null normalization and +// response validation live in selection-carriers.ts. + +export type CultNetSelectionOperator = "any_of" | "lt" | "le" | "ge" | "gt"; +export type CultNetSelectionProjection = "header" | "document"; + +/** One predicate over a single declared index alias. `values` iff any_of; `number` (canonical decimal string) iff a comparison. */ +export interface CultNetFieldPredicate { + index: string; + op: CultNetSelectionOperator; + values?: string[]; + number?: string; +} + +/** Identifies one row on the wire by schema and record key. */ +export interface CultNetRecordRef { + schemaId: string; + recordKey: string; +} + +/** Selects rows whose declared reference names `target`; an absent `role` matches any declared reference. */ +export interface CultNetCitation { + target: CultNetRecordRef; + role?: string; +} + +/** Selects rows some other row does (`exists`) or does not name in `role`. */ +export interface CultNetIncoming { + role: string; + exists: boolean; +} + +/** Absent `schemas`/`keys` mean every schema/key; a present empty list is refused at the door. */ +export interface CultNetSelection { + schemas?: string[]; + keys?: string[]; + fields?: CultNetFieldPredicate[]; + cites?: CultNetCitation; + cited?: CultNetIncoming; + /** Default "header". */ + projection?: CultNetSelectionProjection; + descending?: boolean; + /** Clamped to 1..=200 by the answering server. */ + limit?: number; + /** Opaque; minted by the answering server. */ + cursor?: string; +} + +/** A raw document record minus its payload and payloadEncoding. */ +export interface CultNetRawDocumentHeader { + schemaId: string; + schemaName?: string; + schemaVersion?: string; + schemaContentHash?: string; + recordKey: string; + storedAt: string; + sourceRuntimeId?: string; + sourceAgentId?: string; + sourceRole?: string; + tags?: string[]; +} + +/** One edge a hop traversed. `payload` is present under document projection only. */ +export interface CultNetEdge { + from: CultNetRecordRef; + role: string; + to: CultNetRecordRef; + payloadEncoding?: "messagepack"; + payload?: Uint8Array; +} + +/** + * Refused typed at the door: a selection the declared shape cannot support. Carried back as + * `cultnet.error.v0 { code: "selection_invalid", details: { field, value } }`. + */ +export class CultNetSelectionInvalidError extends Error { + constructor( + readonly field: string, + readonly value: string | null, + message: string, + ) { + super(message); + this.name = "CultNetSelectionInvalidError"; + } +} + +/** + * The declaration-independent half of the door, matching the reference's `ValidateShape`: an empty or + * blank `schemas`/`keys` entry or list, and an unrecognised `projection`. Checks that need the + * declared schemas (index aliases, reference roles, cites target) belong to the selection engine. + */ +export function validateSelectionShape(selection: CultNetSelection): void { + const list = (field: "schemas" | "keys") => { + const entries = selection[field]; + if (entries === undefined || entries === null) return; + if (entries.length === 0) { + throw new CultNetSelectionInvalidError( + field, null, `selection.${field} is present and empty; omit it to reach every ${field === "schemas" ? "schema" : "key"}.`); + } + for (const entry of entries) { + if (typeof entry !== "string" || entry.trim().length === 0) { + throw new CultNetSelectionInvalidError(field, entry, `selection.${field} carries an empty or whitespace entry.`); + } + } + }; + list("schemas"); + list("keys"); + const projection = selection.projection ?? "header"; + if (projection !== "header" && projection !== "document") { + throw new CultNetSelectionInvalidError( + "projection", String(projection), `selection.projection "${String(projection)}" is neither "header" nor "document".`); + } +} diff --git a/packages/cultnet-ts/test/carrier-vectors.test.ts b/packages/cultnet-ts/test/carrier-vectors.test.ts index 891e9575..63f5826f 100644 --- a/packages/cultnet-ts/test/carrier-vectors.test.ts +++ b/packages/cultnet-ts/test/carrier-vectors.test.ts @@ -47,3 +47,125 @@ test("an error whose code is outside the schema's closed enum is refused", () => const raw = decode(bytesOf(vectors.find(v => v.name === "error_cursor_stale")!)) as Record; assert.throws(() => parseCultNetMessage({ ...raw, code: "made_up_code" }), /cultnet\.error\.v0/); }); + +// ---- selection/v1 carriers ---- + +for (const prefix of ["snapshot_request_v1", "database_subscribe_v1", "snapshot_response_raw_v1"]) { + test(`${prefix} vectors: decode, parse, and re-encode to the reference's exact bytes`, () => { + const group = vectorsFor(prefix); + assert.ok(group.length >= 2, `${prefix}: vectors present`); + for (const v of group) { + const raw = decode(bytesOf(v)); + assert.deepEqual(jsonShape(raw), v.expected, `${v.name}: decoded bytes`); + const message = parseCultNetMessage(raw); + assert.equal(message.schemaVersion, v.schemaVersion); + assert.deepEqual(reencode(message), bytesOf(v), `${v.name}: re-encoded bytes`); + } + }); +} + +test("a parsed v1 message drops the nil an absent member travels as", () => { + const full = parseCultNetMessage(decode(bytesOf(vectors.find(v => v.name === "snapshot_request_v1_defaults")!))) as unknown as + { selection: Record; shardId?: unknown }; + assert.deepEqual(Object.keys(full.selection).sort(), ["descending", "projection"]); + assert.equal("shardId" in full, false); + const page = parseCultNetMessage(decode(bytesOf(vectors.find(v => v.name === "snapshot_response_raw_v1_headers")!))) as unknown as + { headers: Record[]; documents?: unknown }; + assert.equal("documents" in page, false); + assert.equal("tags" in page.headers[0], false); + assert.deepEqual(page.headers[1].tags, ["alpha", "beta"]); +}); + +test("edge payloads and document payloads arrive as bytes, not text", () => { + const page = parseCultNetMessage(decode(bytesOf(vectors.find(v => v.name === "snapshot_response_raw_v1_documents_and_edges")!))) as unknown as + { edges: { payload?: Uint8Array }[]; documents: { payload: Uint8Array }[] }; + assert.deepEqual(Array.from(page.edges[1].payload!), [1, 2, 3]); + assert.equal("payload" in page.edges[0], false); + assert.deepEqual(Array.from(page.documents[0].payload), [0x81, 0xa1, 0x6b, 0x01]); +}); + +const request = () => decode(bytesOf(vectors.find(v => v.name === "snapshot_request_v1_full_selection_and_shard")!)) as + { selection: Record } & Record; +const refusedField = (message: unknown): string | undefined => { + try { + parseCultNetMessage(message); + } catch (error) { + return (error as { field?: string }).field; + } + return "accepted"; +}; + +test("the door refuses an empty or blank schemas/keys entry with a typed field, never as every-schema", () => { + const withSelection = (patch: Record) => ({ ...request(), selection: { ...request().selection, ...patch } }); + assert.equal(refusedField(withSelection({ schemas: [] })), "schemas"); + assert.equal(refusedField(withSelection({ keys: ["a-eq", " "] })), "keys"); + assert.equal(refusedField(withSelection({ projection: "everything" })), "projection"); +}); + +test("the schema refuses a comparison whose number is not a canonical decimal, and a predicate that mixes values and number", () => { + const withField = (field: Record) => { + const message = request(); + return { ...message, selection: { ...message.selection, fields: [field] } }; + }; + for (const number of ["05", "1.50", "+1", "1e3", "-0", " 1"]) { + assert.throws(() => parseCultNetMessage(withField({ index: "mass", op: "ge", values: null, number })), /snapshot_request\.v1/, number); + } + assert.throws(() => parseCultNetMessage(withField({ index: "mass", op: "ge", values: ["1"], number: "1" })), /snapshot_request\.v1/); + assert.throws(() => parseCultNetMessage(withField({ index: "mass", op: "near", values: null, number: "1" })), /snapshot_request\.v1/); +}); + +test("a selection page with both or neither of headers and documents is refused", () => { + const page = decode(bytesOf(vectors.find(v => v.name === "snapshot_response_raw_v1_headers")!)) as Record; + assert.throws(() => parseCultNetMessage({ ...page, documents: [] }), /exactly one of headers and documents/); + assert.throws(() => parseCultNetMessage({ ...page, headers: null }), /exactly one of headers and documents/); +}); + +// ---- selection vectors: the reference's and Rust's selection bytes ---- + +const INTEROP = join(__dirname, "..", "..", "..", "..", "contracts", "cultnet", "interop"); +type SelectionVector = { name: string; selectionMessagePackBase64: string; expectedRefusalField?: string | null }; +const selectionVectors = (file: string): SelectionVector[] => + JSON.parse(readFileSync(join(INTEROP, file), "utf8")).vectors; + +import { CultNetSelectionInvalidError, validateSelectionShape, type CultNetSelection } from "../src/selection"; + +for (const file of ["selection-vectors.cs-written.json", "selection-vectors.rs-written.json"]) { + test(`${file}: every selection decodes inside a v1 request and the schema door agrees with the vector`, () => { + const all = selectionVectors(file); + assert.ok(all.length >= 20); + for (const v of all) { + const selection = decode(new Uint8Array(Buffer.from(v.selectionMessagePackBase64, "base64"))); + const message = { schemaVersion: "cultnet.snapshot_request.v1", messageId: "m", selection, shardId: null, shardEpoch: null }; + if (v.expectedRefusalField === "schemas" || v.expectedRefusalField === "keys") { + assert.equal(refusedField(message), v.expectedRefusalField, `${v.name}: shape refusal`); + continue; + } + if (v.expectedRefusalField) { + // Declaration-dependent refusals (cites target, index aliases) need the selection engine's schemas. + const parsed = parseCultNetMessage(message) as unknown as { selection: CultNetSelection }; + validateSelectionShape(parsed.selection); + continue; + } + const parsed = parseCultNetMessage(message) as unknown as { selection: CultNetSelection }; + validateSelectionShape(parsed.selection); + const rewritten = new Uint8Array(encode( + (encodeCultNetMessageForWire(parsed as never) as { selection: unknown }).selection)); + if (file.includes("cs-written")) { + // The reference's bytes are the nil-keyed shape: our encoder must reproduce them exactly. + assert.deepEqual(rewritten, new Uint8Array(Buffer.from(v.selectionMessagePackBase64, "base64")), + `${v.name}: selection re-encodes to the reference's bytes`); + } else { + // Rust omits absent members; what it means must survive our nil-keyed rewrite unchanged. + const again = parseCultNetMessage({ ...message, selection: decode(rewritten) }) as unknown as { selection: CultNetSelection }; + assert.deepEqual(again.selection, parsed.selection, `${v.name}: selection meaning survives re-encode`); + } + } + }); +} + +test("CultNetSelectionInvalidError carries the field and offending value", () => { + assert.throws( + () => validateSelectionShape({ keys: ["ok", ""] }), + (error: unknown) => error instanceof CultNetSelectionInvalidError && error.field === "keys" && error.value === "", + ); +}); From 420eb9e840d819d320fd88e641643c8967a08b64 Mon Sep 17 00:00:00 2001 From: Libby Date: Sat, 7 Nov 2026 02:40:57 +0000 Subject: [PATCH 75/88] heimdall-consent: pin the one version rule in the golden vectors (4294967295 ok; 4294967296 and 01 refused) Case one-version-rule runs version_ok/version_refused for notice_version, the sample id @N and terms_version; the Rust vector test handles both ops. README cites Heimdall PR #6 d6a8d42 for the rule. Co-Authored-By: Claude Sonnet 5.5 --- packages/heimdall-consent/README.md | 2 +- packages/heimdall-consent/tests/golden.rs | 25 ++++++++++++++++++- .../tests/vectors/consent_golden.csv | 10 ++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/packages/heimdall-consent/README.md b/packages/heimdall-consent/README.md index fb446e64..e38c8318 100644 --- a/packages/heimdall-consent/README.md +++ b/packages/heimdall-consent/README.md @@ -14,7 +14,7 @@ Entry points: `src/lib.rs`; golden vectors `tests/vectors/consent_golden.csv`, r | Ruling | Doc | Crate | |---|---|---| | Timestamps are Unix ms UTC, signed 64-bit | L98-99; `received_at` is collector-assigned, L75 | `i64` on `accepted_at`, `received_at`, `terms_accepted_at` | -| Ids are non-empty opaque strings, `event_id` verbatim; every version field (`notice_version`, the `@N` of a sample id, `terms_version`) is a positive integer, no leading zeros, no sign, at most 4294967295 (Heimdall ruling 2026-11-07, one rule) | L100-103 | `check_version` is the single rule; `check_sample_id` and `MemberTerms::validate` call it; `notice_version: u32` with `validate()` refusing 0; golden vector `newcomer` carries `terms_version` 4294967295 | +| Ids are non-empty opaque strings, `event_id` verbatim; every version field (`notice_version`, the `@N` of a sample id, `terms_version`) is a positive integer, no leading zeros, no sign, at most 4294967295 (Heimdall ruling 2026-11-07, one rule; doc line "Version rule" under Representation, Heimdall PR #6 commit d6a8d42) | L100-103 | `check_version` is the single rule; `check_sample_id` and `MemberTerms::validate` call it; `notice_version: u32` with `validate()` refusing 0; golden vector `newcomer` carries `terms_version` 4294967295, and case `one-version-rule` pins 4294967295 accepted, 4294967296 and `01` refused for `notice_version`, the sample id `@N` and `terms_version` | | Key parts joined with U+001F; empty part or part containing it refused | L104-105 | `key()` | | Tombstone is the exited `Answer` with `text_hash` (32 bytes); HMAC-SHA-256 keyed by a per-member salt held by the member-side service; the collector cannot compute it | L74, L132-152, L186-193 | `Answer.text_hash`; `redacted([u8; 32])` takes the service's output and refuses an answer with no `text` (L148-149); `validate()` requires 32 bytes and `exited` | | A reply with no established notice version or scope is stored with both absent, writes no acceptance, is excluded from every use | L87-94 | `notice_version`/`scope` optional and absent together (`validate()`); vector case `no-scope` | diff --git a/packages/heimdall-consent/tests/golden.rs b/packages/heimdall-consent/tests/golden.rs index 01e9477b..615d5cb4 100644 --- a/packages/heimdall-consent/tests/golden.rs +++ b/packages/heimdall-consent/tests/golden.rs @@ -68,7 +68,7 @@ fn golden_vectors() { _ => cases.push((f[0], vec![f])), } } - assert_eq!(cases.len(), 9, "fixture shrank"); + assert_eq!(cases.len(), 10, "fixture shrank"); for (name, rows) in cases { let dir = tempfile::tempdir().unwrap(); let mut cache = new_cache(&dir); @@ -185,6 +185,29 @@ fn golden_vectors() { assert_eq!((a.notice_version, Some(a.scope), a.accepted_at), (nv.unwrap(), scope(sc), ts), "{ctx}"); assert_eq!(a.event_id, payload, "{ctx}"); } + "version_ok" | "version_refused" => { + // member column names the field; payload is the version text as it arrives + let ok = match member { + "notice_version" => check_version(payload).is_ok_and(|v| { + NoticeAcceptance { + member_id: "m".into(), + experiment_id: room.into(), + notice_version: v, + scope: Scope::Train, + accepted_at: 1, + event_id: "e".into(), + } + .validate() + .is_ok() + }), + "sample_id" => check_sample_id(&format!("s@{payload}")).is_ok(), + "terms_version" => { + MemberTerms { member_id: "m".into(), terms_version: payload.into(), terms_accepted_at: 1 }.validate().is_ok() + } + other => panic!("unknown version field {other}"), + }; + assert_eq!(ok, op == "version_ok", "{ctx} field {member} version {payload}"); + } other => panic!("unknown op {other}"), } } diff --git a/packages/heimdall-consent/tests/vectors/consent_golden.csv b/packages/heimdall-consent/tests/vectors/consent_golden.csv index 1323aef8..4343b0b2 100644 --- a/packages/heimdall-consent/tests/vectors/consent_golden.csv +++ b/packages/heimdall-consent/tests/vectors/consent_golden.csv @@ -6,6 +6,7 @@ # reply_rejected: validate must refuse the answer (nothing stored). answer_revoked: stores an exited answer with no text; payload is the tombstone hash (empty means none). expect_winner: the stored answer's event_id and text must equal event and payload. # Replacement at one key: the later reply is the greater (received_at, event_id), event_id compared bytewise; input order never decides (doc L78-81 and Heimdall PR #6 comment 759). # terms: member field terms_version (payload) and terms_accepted_at (ts). expect_terms: the stored pair for that member reads back as payload and ts; expect_no_terms: the member has none. expect_*: payload is what must hold. +# version_ok / version_refused: the member column names the field (notice_version, sample_id, terms_version), payload is the version text; a sample id is checked as s@. One rule for all three: positive integer, no leading zeros, at most 4294967295 (Heimdall PR #6 cafdc52). # Expected values are written by hand from docs/answer-collector-consent.md (Heimdall PR #6, doc commit 4e99bfa), not computed by the crate. versioned-key,reply,m1,room1,1,train,s1@1,first,100,e1 versioned-key,reply,m1,room1,1,train,s1@2,second,200,e2 @@ -66,3 +67,12 @@ tie-greater-last,reply,m15,room1,,,s1@1,long-aa,500,aa tie-greater-last,reply,m15,room1,,,s1@1,short-z,500,z tie-greater-last,expect_winner,m14,room1,,,s1@1,lower-a,,a tie-greater-last,expect_winner,m15,room1,,,s1@1,short-z,,z +one-version-rule,version_ok,notice_version,room1,,,,4294967295,,version_ok-notice_version-4294967295 +one-version-rule,version_refused,notice_version,room1,,,,4294967296,,version_refused-notice_version-4294967296 +one-version-rule,version_refused,notice_version,room1,,,,01,,version_refused-notice_version-01 +one-version-rule,version_ok,sample_id,room1,,,,4294967295,,version_ok-sample_id-4294967295 +one-version-rule,version_refused,sample_id,room1,,,,4294967296,,version_refused-sample_id-4294967296 +one-version-rule,version_refused,sample_id,room1,,,,01,,version_refused-sample_id-01 +one-version-rule,version_ok,terms_version,room1,,,,4294967295,,version_ok-terms_version-4294967295 +one-version-rule,version_refused,terms_version,room1,,,,4294967296,,version_refused-terms_version-4294967296 +one-version-rule,version_refused,terms_version,room1,,,,01,,version_refused-terms_version-01 From 638e71f42724bb7a3f401acea17468aa029fefcf Mon Sep 17 00:00:00 2001 From: Libby Date: Sat, 7 Nov 2026 04:10:44 +0000 Subject: [PATCH 76/88] heimdall-consent: re-pin docs to Heimdall PR #6 aaeb175, fix doc line numbers, mark notice state event and registration-checkbox gap as not modelled No behaviour or vector rows changed. Co-Authored-By: Claude Sonnet 5.5 --- packages/heimdall-consent/README.md | 33 +++++++++++-------- packages/heimdall-consent/src/lib.rs | 4 +-- .../tests/vectors/consent_golden.csv | 8 ++--- 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/packages/heimdall-consent/README.md b/packages/heimdall-consent/README.md index e38c8318..3782e720 100644 --- a/packages/heimdall-consent/README.md +++ b/packages/heimdall-consent/README.md @@ -3,29 +3,34 @@ Typed CultCache records for Heimdall's answer collector: `NoticeAcceptance`, `Answer` and `MemberTerms`, plus the rules that need no store to state (`Answer::is_training_data`, `Answer::redacted`, `validate`). Cut from `docs/answer-collector-consent.md` in GameCult/Heimdall PR #6 at doc commit -`4e99bfa`; the doc's tables and Representation section are the contract, and its rulings are on the PR +`aaeb175` (PR head); the doc's tables and Representation section are the contract, and its rulings are on the PR (comments 682 and 759). Specification only: no collector, tally, export or tombstone hashing lives here. Entry points: `src/lib.rs`; golden vectors `tests/vectors/consent_golden.csv`, run by `tests/golden.rs` (`cargo test`). Planned, not live: the collector, the tally, the training export. -## Where each ruling lands (doc lines at 4e99bfa) +## Where each ruling lands (doc lines at aaeb175) | Ruling | Doc | Crate | |---|---|---| -| Timestamps are Unix ms UTC, signed 64-bit | L98-99; `received_at` is collector-assigned, L75 | `i64` on `accepted_at`, `received_at`, `terms_accepted_at` | -| Ids are non-empty opaque strings, `event_id` verbatim; every version field (`notice_version`, the `@N` of a sample id, `terms_version`) is a positive integer, no leading zeros, no sign, at most 4294967295 (Heimdall ruling 2026-11-07, one rule; doc line "Version rule" under Representation, Heimdall PR #6 commit d6a8d42) | L100-103 | `check_version` is the single rule; `check_sample_id` and `MemberTerms::validate` call it; `notice_version: u32` with `validate()` refusing 0; golden vector `newcomer` carries `terms_version` 4294967295, and case `one-version-rule` pins 4294967295 accepted, 4294967296 and `01` refused for `notice_version`, the sample id `@N` and `terms_version` | -| Key parts joined with U+001F; empty part or part containing it refused | L104-105 | `key()` | -| Tombstone is the exited `Answer` with `text_hash` (32 bytes); HMAC-SHA-256 keyed by a per-member salt held by the member-side service; the collector cannot compute it | L74, L132-152, L186-193 | `Answer.text_hash`; `redacted([u8; 32])` takes the service's output and refuses an answer with no `text` (L148-149); `validate()` requires 32 bytes and `exited` | -| A reply with no established notice version or scope is stored with both absent, writes no acceptance, is excluded from every use | L87-94 | `notice_version`/`scope` optional and absent together (`validate()`); vector case `no-scope` | -| A reply's version is the one pinned at its `origin_server_ts`; changed version needs a fresh first reply | L32-34, L47, L92-94 | acceptance key includes the version; `is_training_data` accepts only the acceptance for the answer's own version; vector case `version-change` | -| `MemberTerms` is the typed shape of the member record's `terms_version`/`terms_accepted_at`, not a second store | L51-56 | `MemberTerms` doc comment; the collector does not hold it | -| Training needs acceptance `scope = train` equal to the answer's `scope`, and `accepted_at <= received_at` | L116-123 | `is_training_data` | -| One store per scope plus a store for scopeless replies | L125-130 | not modelled; the collector's layout owns it | +| Timestamps are Unix ms UTC, signed 64-bit | L118-119; `received_at` is collector-assigned, L95, L208-209 | `i64` on `accepted_at`, `received_at`, `terms_accepted_at` | +| Ids are non-empty opaque strings, `event_id` verbatim; every version field (`notice_version`, the `@N` of a sample id, `terms_version`) is a positive integer, no leading zeros, no sign, at most 4294967295 (Heimdall ruling 2026-11-07, one rule; doc line "Version rule" under Representation, Heimdall PR #6 commit d6a8d42, now in the doc at aaeb175) | L120-136 | `check_version` is the single rule; `check_sample_id` and `MemberTerms::validate` call it; `notice_version: u32` with `validate()` refusing 0; golden vector `newcomer` carries `terms_version` 4294967295, and case `one-version-rule` pins 4294967295 accepted, 4294967296 and `01` refused for `notice_version`, the sample id `@N` and `terms_version` | +| Key parts joined with U+001F; empty part or part containing it refused | L137-138 | `key()` | +| Tombstone is the exited `Answer` with `text_hash` (32 bytes); HMAC-SHA-256 keyed by a per-member salt held by the member-side service; the collector cannot compute it | L94, L165-185, L221-226 | `Answer.text_hash`; `redacted([u8; 32])` takes the service's output and refuses an answer with no `text` (L181-182); `validate()` requires 32 bytes and `exited` | +| A reply with no established notice version or scope is stored with both absent, writes no acceptance, is excluded from every use | L107-114 | `notice_version`/`scope` optional and absent together (`validate()`); vector case `no-scope` | +| A reply's version is the one pinned at its `origin_server_ts`; changed version needs a fresh first reply | L19-20, L49-51, L112-114 | acceptance key includes the version; `is_training_data` accepts only the acceptance for the answer's own version; vector case `version-change` | +| `MemberTerms` is the typed shape of the member record's `terms_version`/`terms_accepted_at`, not a second store | L68-76 | `MemberTerms` doc comment; the collector does not hold it | +| Training needs acceptance `scope = train` equal to the answer's `scope`, and `accepted_at <= received_at` | L149-156 | `is_training_data` | +| One store per scope plus a store for scopeless replies | L158-163 | not modelled; the collector's layout owns it | -Rulings of comment 759 (doc L72, L78-85; no open questions remain): +Rulings of comment 759 (doc L92, L98-105; no open questions remain): | Ruling | Doc | Crate | |---|---|---| -| A live answer must have non-empty `text` and is rejected otherwise; empty text is valid only on an exited answer (text cleared, hash-only tombstone) | L72, L83-85 | `Answer::validate()` refuses a non-exited answer whose `text` is absent or empty; an exited answer holds no text. Vector cases `empty-live-rejected`, `empty-revoked` | -| `received_at` is assigned by the collector; a tie on it breaks by `event_id` compared bytewise, the greater wins as the later reply | L75, L78-81 | `Answer::supersedes`: `(received_at, event_id bytes)` order; the vector harness replaces a stored answer only when the incoming one supersedes it. Vector cases `tie-greater-first`, `tie-greater-last` | +| A live answer must have non-empty `text` and is rejected otherwise; empty text is valid only on an exited answer (text cleared, hash-only tombstone) | L92, L103-105 | `Answer::validate()` refuses a non-exited answer whose `text` is absent or empty; an exited answer holds no text. Vector cases `empty-live-rejected`, `empty-revoked` | +| `received_at` is assigned by the collector; a tie on it breaks by `event_id` compared bytewise, the greater wins as the later reply | L95, L98-101 | `Answer::supersedes`: `(received_at, event_id bytes)` order; the vector harness replaces a stored answer only when the incoming one supersedes it. Vector cases `tie-greater-first`, `tie-greater-last` | + +Not modelled (doc at aaeb175): + +- The `org.gamecult.notice` room state event (L24-37, L132-136): its content is `{version, scope}` with `version` a JSON number, and a string (including `"01"`) is malformed. The crate checks the version only as decimal text (`check_version`); reading, power-level checks and the malformed-means-`show` fallback belong to the collector. +- The registration-checkbox gap (L78-79): the `m.login.terms` checkbox records no version. The doc rules nothing on it; the crate takes `terms_version` as given. diff --git a/packages/heimdall-consent/src/lib.rs b/packages/heimdall-consent/src/lib.rs index 508e3a67..61ecb188 100644 --- a/packages/heimdall-consent/src/lib.rs +++ b/packages/heimdall-consent/src/lib.rs @@ -1,5 +1,5 @@ //! Typed CultCache records for Heimdall's answer collector, cut from `docs/answer-collector-consent.md` -//! in GameCult/Heimdall PR #6 (doc commit 4e99bfa): the notice acceptance record, the answer record, +//! in GameCult/Heimdall PR #6 (doc commit aaeb175): the notice acceptance record, the answer record, //! and the typed shape of the terms pair on the member record. Field names and types follow the //! doc's tables and its Representation section; README.md cites the doc lines. @@ -116,7 +116,7 @@ pub struct Answer { /// One-way: set by redaction, never cleared. #[cultcache(key = 6, default)] pub exited: bool, - /// Assigned by the collector on receipt, unix milliseconds UTC, not copied from the reply (doc L75); + /// Assigned by the collector on receipt, unix milliseconds UTC, not copied from the reply (doc L95); /// ties are broken by `supersedes`. #[cultcache(key = 7)] pub received_at: i64, diff --git a/packages/heimdall-consent/tests/vectors/consent_golden.csv b/packages/heimdall-consent/tests/vectors/consent_golden.csv index 4343b0b2..4f6e4048 100644 --- a/packages/heimdall-consent/tests/vectors/consent_golden.csv +++ b/packages/heimdall-consent/tests/vectors/consent_golden.csv @@ -2,12 +2,12 @@ # Ops run in order against a real CultCache; each case starts empty. Timestamps are unix ms. # reply: first reply under a (room, member, notice_version) writes an acceptance, then the answer replaces any earlier one at (room, sample_id, member); payload is the text. # answer_raw: stores the answer with no acceptance written. redact: replaces the answer whose event_id matches with its tombstone; payload is the 32-byte tombstone hash as 64 hex chars (the member-side service's output; the crate never computes it). -# A reply with no established notice version or scope is stored with both absent (doc L87-94, Answer record), writes no acceptance, and is excluded. answer_raw with a version but no acceptance models a lost or never-written acceptance: still excluded (doc L119-122). +# A reply with no established notice version or scope is stored with both absent (doc L107-114, Answer record), writes no acceptance, and is excluded. answer_raw with a version but no acceptance models a lost or never-written acceptance: still excluded (doc L152-158). # reply_rejected: validate must refuse the answer (nothing stored). answer_revoked: stores an exited answer with no text; payload is the tombstone hash (empty means none). expect_winner: the stored answer's event_id and text must equal event and payload. -# Replacement at one key: the later reply is the greater (received_at, event_id), event_id compared bytewise; input order never decides (doc L78-81 and Heimdall PR #6 comment 759). +# Replacement at one key: the later reply is the greater (received_at, event_id), event_id compared bytewise; input order never decides (doc L98-101 and Heimdall PR #6 comment 759). # terms: member field terms_version (payload) and terms_accepted_at (ts). expect_terms: the stored pair for that member reads back as payload and ts; expect_no_terms: the member has none. expect_*: payload is what must hold. -# version_ok / version_refused: the member column names the field (notice_version, sample_id, terms_version), payload is the version text; a sample id is checked as s@. One rule for all three: positive integer, no leading zeros, at most 4294967295 (Heimdall PR #6 cafdc52). -# Expected values are written by hand from docs/answer-collector-consent.md (Heimdall PR #6, doc commit 4e99bfa), not computed by the crate. +# version_ok / version_refused: the member column names the field (notice_version, sample_id, terms_version), payload is the version text; a sample id is checked as s@. One rule for all three: positive integer, no leading zeros, at most 4294967295 (Heimdall PR #6 d6a8d42, doc at aaeb175). +# Expected values are written by hand from docs/answer-collector-consent.md (Heimdall PR #6, doc commit aaeb175), not computed by the crate. versioned-key,reply,m1,room1,1,train,s1@1,first,100,e1 versioned-key,reply,m1,room1,1,train,s1@2,second,200,e2 versioned-key,reply,m1,room1,1,train,s1@1,replaced,300,e3 From 2897b4bad84b143136195e56e8ea6ff94c476d43 Mon Sep 17 00:00:00 2001 From: Libby Date: Sat, 7 Nov 2026 11:30:22 +0000 Subject: [PATCH 77/88] ci: Windows legs run on workflow_dispatch only; Ubuntu gates push and PR The forge runner is Ubuntu-only (Docker, one job), so Windows jobs queued on push and pull_request can never run. - cultnet-interop: the whole job is Windows-only; trigger is now workflow_dispatch. - cultmesh-portability: both matrices keep windows-latest on workflow_dispatch and run ubuntu-latest alone on push and pull_request. - quic-native: unchanged; Build win32-x64 already runs only on dispatch or a tag. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/cultmesh-portability.yml | 6 ++++-- .github/workflows/cultnet-interop.yml | 7 +++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cultmesh-portability.yml b/.github/workflows/cultmesh-portability.yml index f8a6e5da..3648ba2b 100644 --- a/.github/workflows/cultmesh-portability.yml +++ b/.github/workflows/cultmesh-portability.yml @@ -16,7 +16,8 @@ jobs: strategy: fail-fast: false matrix: - os: [windows-latest, ubuntu-latest] + # The forge runner is Ubuntu-only: Windows legs run on workflow_dispatch (GitHub or a Windows runner), Ubuntu gates push and PR. + os: ${{ fromJSON(github.event_name == 'workflow_dispatch' && '["windows-latest", "ubuntu-latest"]' || '["ubuntu-latest"]') }} steps: - name: Check out CultLib @@ -54,7 +55,8 @@ jobs: strategy: fail-fast: false matrix: - os: [windows-latest, ubuntu-latest] + # The forge runner is Ubuntu-only: Windows legs run on workflow_dispatch (GitHub or a Windows runner), Ubuntu gates push and PR. + os: ${{ fromJSON(github.event_name == 'workflow_dispatch' && '["windows-latest", "ubuntu-latest"]' || '["ubuntu-latest"]') }} steps: - name: Check out CultLib diff --git a/.github/workflows/cultnet-interop.yml b/.github/workflows/cultnet-interop.yml index 9072e0cb..118b496f 100644 --- a/.github/workflows/cultnet-interop.yml +++ b/.github/workflows/cultnet-interop.yml @@ -1,10 +1,9 @@ name: CultNet cross-runtime interop +# workflow_dispatch only: this job is Windows-only (pwsh Kotlin build, setup steps), and the forge +# runner is Ubuntu-only, so a push or pull_request trigger would queue a job that can never run. +# Run it from GitHub or any Windows runner. on: - push: - branches: - - main - pull_request: workflow_dispatch: jobs: From 7d654cc277c7c80e7bea21269f327a72c448c2b8 Mon Sep 17 00:00:00 2001 From: Libby Date: Sat, 7 Nov 2026 11:32:01 +0000 Subject: [PATCH 78/88] sln: drop EveBrowserNetworkSample so dotnet build CultLib.sln succeeds The sample ProjectReferences ../../../Eve, which is not checked out beside CultLib, and the only error in the build was Program.cs CS0234 (GameCult.Eve missing). The sample stays on disk and is built by scripts/verify-eve-browser-network.mjs with --eve-root; it is just no longer part of the default solution build. Co-Authored-By: Claude Sonnet 5.5 --- CultLib.sln | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/CultLib.sln b/CultLib.sln index a8baa383..b72f2d76 100644 --- a/CultLib.sln +++ b/CultLib.sln @@ -35,10 +35,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Networking.WebSock EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Networking.WebSockets.Tests", "tests\GameCult.Networking.WebSockets.Tests\GameCult.Networking.WebSockets.Tests.csproj", "{B9A441EC-DADE-4500-B092-C635F3EB296A}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "samples", "samples", "{5D20AA90-6969-D8BD-9DCD-8634F4692FDA}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EveBrowserNetworkSample", "samples\eve-browser-network\EveBrowserNetworkSample.csproj", "{5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}" -EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "packages", "packages", "{809F86A1-1C4C-B159-0CD4-DF9D33D876CE}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cultmath", "cultmath", "{B48EAE30-152F-01E3-6E31-82B350494D3C}" @@ -231,18 +227,6 @@ Global {B9A441EC-DADE-4500-B092-C635F3EB296A}.Release|x64.Build.0 = Release|Any CPU {B9A441EC-DADE-4500-B092-C635F3EB296A}.Release|x86.ActiveCfg = Release|Any CPU {B9A441EC-DADE-4500-B092-C635F3EB296A}.Release|x86.Build.0 = Release|Any CPU - {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Debug|Any CPU.Build.0 = Debug|Any CPU - {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Debug|x64.ActiveCfg = Debug|Any CPU - {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Debug|x64.Build.0 = Debug|Any CPU - {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Debug|x86.ActiveCfg = Debug|Any CPU - {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Debug|x86.Build.0 = Debug|Any CPU - {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Release|Any CPU.ActiveCfg = Release|Any CPU - {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Release|Any CPU.Build.0 = Release|Any CPU - {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Release|x64.ActiveCfg = Release|Any CPU - {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Release|x64.Build.0 = Release|Any CPU - {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Release|x86.ActiveCfg = Release|Any CPU - {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Release|x86.Build.0 = Release|Any CPU {54D543F7-8EB7-4A11-977E-22D3EE2D8C3F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {54D543F7-8EB7-4A11-977E-22D3EE2D8C3F}.Debug|Any CPU.Build.0 = Debug|Any CPU {54D543F7-8EB7-4A11-977E-22D3EE2D8C3F}.Debug|x64.ActiveCfg = Debug|Any CPU @@ -322,7 +306,6 @@ Global {21F26329-2119-4037-93CC-DD1801A1AB78} = {39D316AB-ABB5-49A2-8E83-CAF4C75B48F4} {7F32D96B-AB5D-4A72-9217-B5DCCFEFC7E5} = {E733CBFF-42AE-4B6A-B5B9-0071ECFB45F1} {B9A441EC-DADE-4500-B092-C635F3EB296A} = {39D316AB-ABB5-49A2-8E83-CAF4C75B48F4} - {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB} = {5D20AA90-6969-D8BD-9DCD-8634F4692FDA} {B48EAE30-152F-01E3-6E31-82B350494D3C} = {809F86A1-1C4C-B159-0CD4-DF9D33D876CE} {54D543F7-8EB7-4A11-977E-22D3EE2D8C3F} = {B48EAE30-152F-01E3-6E31-82B350494D3C} {FEF9DA88-2177-4FD0-885D-7096ACFA8A9D} = {B48EAE30-152F-01E3-6E31-82B350494D3C} From 3d7126d036f66e76da2ffb6eb0fe29ea8fe76128 Mon Sep 17 00:00:00 2001 From: Libby Date: Sat, 7 Nov 2026 11:33:51 +0000 Subject: [PATCH 79/88] docs: mark samples/eve-browser-network as needing a sibling Eve checkout The sample ProjectReferences ../Eve and is not in CultLib.sln (PR #22). README, getting-started 03/04 and browser-verse-transport now say so where they tell readers to run it. Pages 01 and 03:73 only link Program.cs as source to read, which needs nothing, so they are unchanged. Co-Authored-By: Claude Sonnet 5.5 --- README.md | 3 ++- src/GameCult.Mesh/docs/browser-verse-transport.md | 4 +++- .../docs/getting-started/03-publish-an-eve-surface.md | 4 +++- .../docs/getting-started/04-run-the-multiruntime-game.md | 3 ++- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6e6de55c..b916cabe 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,8 @@ start with the maintained series. Its first executable package-consumer checkpoint is [`samples/eve-two-runtime`](samples/eve-two-runtime/README.md). The real-process browser continuation is -[`samples/eve-browser-network`](samples/eve-browser-network/README.md). +[`samples/eve-browser-network`](samples/eve-browser-network/README.md); it +needs a sibling Eve checkout (`--eve-root`) and is not in `CultLib.sln`. One documented command runs both layers: ```powershell diff --git a/src/GameCult.Mesh/docs/browser-verse-transport.md b/src/GameCult.Mesh/docs/browser-verse-transport.md index fcfde998..e825a0d4 100644 --- a/src/GameCult.Mesh/docs/browser-verse-transport.md +++ b/src/GameCult.Mesh/docs/browser-verse-transport.md @@ -12,7 +12,9 @@ over binary loopback CultNet WebSocket routes under an explicit `local-development` trust policy. It proves one canonical command effect, idempotent retry, shared observation, provider restart on a new physical route, browser rediscovery, retained lease resubscription, and a second -canonical command after reconnection. +canonical command after reconnection. Running the sample needs a sibling Eve +checkout (`--eve-root`); it is not part of `CultLib.sln` and does not build +standalone. The current substrate has a browser client boundary and local live proof, but not a complete public Verse proof: diff --git a/src/GameCult.Mesh/docs/getting-started/03-publish-an-eve-surface.md b/src/GameCult.Mesh/docs/getting-started/03-publish-an-eve-surface.md index 5cb5b742..49a23f79 100644 --- a/src/GameCult.Mesh/docs/getting-started/03-publish-an-eve-surface.md +++ b/src/GameCult.Mesh/docs/getting-started/03-publish-an-eve-surface.md @@ -135,7 +135,9 @@ does **not** prove the network layer; the second program owns that evidence. The network program is [`samples/eve-browser-network`](../../../../samples/eve-browser-network/README.md). -Run it separately only while diagnosing that layer. A smoke against the +Running it needs a sibling Eve checkout (`--eve-root`); it is not part of +`CultLib.sln` and does not build standalone. Run it separately only while +diagnosing that layer. A smoke against the deployed Odin daemon remains a separate infrastructure gate. Its owner map and full acceptance contract are recorded in [`browser-verse-transport.md`](../browser-verse-transport.md). diff --git a/src/GameCult.Mesh/docs/getting-started/04-run-the-multiruntime-game.md b/src/GameCult.Mesh/docs/getting-started/04-run-the-multiruntime-game.md index ce18fdb3..581895a0 100644 --- a/src/GameCult.Mesh/docs/getting-started/04-run-the-multiruntime-game.md +++ b/src/GameCult.Mesh/docs/getting-started/04-run-the-multiruntime-game.md @@ -58,7 +58,8 @@ source-tree fallbacks into application code. The process harness lives in [`samples/eve-browser-network`](../../../../samples/eve-browser-network/README.md). -Application code should copy the typed state, operation, and Eve composition +Running it needs a sibling Eve checkout (`--eve-root`); it does not build +standalone. Application code should copy the typed state, operation, and Eve composition shape—not its benchmark flags or fixture orchestration. For a TypeScript provider daemon, continue with the From 23279b8e50dab4e2dc0c1b5a9117aa7bf0a1e2d1 Mon Sep 17 00:00:00 2001 From: Libby Date: Sat, 7 Nov 2026 11:37:03 +0000 Subject: [PATCH 80/88] cultcache-ts: browser-safe entry that excludes the node-only single-file store The package root imports node:fs, node:path and koffi through the single-file store, so esbuild platform=browser fails on it. src/browser.ts is the root surface minus that store; src/index.ts re-exports it and adds the store, so the Node entry is unchanged. Exposed as ./browser and as the 'browser' export condition on the root. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-ts/CHANGELOG.md | 2 + packages/cultcache-ts/README.md | 2 +- packages/cultcache-ts/package.json | 9 ++++ packages/cultcache-ts/src/browser.ts | 49 +++++++++++++++++++ packages/cultcache-ts/src/index.ts | 48 +----------------- .../cultcache-ts/test/browser-entry.test.ts | 45 +++++++++++++++++ 6 files changed, 107 insertions(+), 48 deletions(-) create mode 100644 packages/cultcache-ts/src/browser.ts create mode 100644 packages/cultcache-ts/test/browser-entry.test.ts diff --git a/packages/cultcache-ts/CHANGELOG.md b/packages/cultcache-ts/CHANGELOG.md index a2f10c54..93e17239 100644 --- a/packages/cultcache-ts/CHANGELOG.md +++ b/packages/cultcache-ts/CHANGELOG.md @@ -15,6 +15,8 @@ All notable changes to `@gamecult/cultcache-ts` are recorded here, following ### Added +- `@gamecult/cultcache-ts/browser`, also what a bundler resolves for the package root under the `browser` export condition: the root surface without `SingleFileMessagePackBackingStore` and `readSingleFileStore`, so it imports no `node:*` built-in and no `koffi` and bundles with esbuild `platform: "browser"`. The Node entry exports exactly what it did. + - `readSingleFileStore(path)`, the one synchronous reader of a single-file store, for consumers that must read a store file rather than decode it themselves. `pullAll`, `push`, `delete` and `pushAll` ask the same reader. - `StoreUnreadableError` for a store file this runtime cannot read, `SchemaConflictError` for a write the catalog cannot describe, and `schemaIdentityOf` with its result type `CultCacheSchemaIdentity` (`schemaId`, `schemaName`, `schemaVersion`, `compatibleSchemaIds`), the one derivation of a definition's identity. - `SingleFileStoreRead`, exported: the result type of `readSingleFileStore` (`absent`, `format`, `envelopes`, `repairedLegacyPayload`). diff --git a/packages/cultcache-ts/README.md b/packages/cultcache-ts/README.md index 192d4b5d..b3a85345 100644 --- a/packages/cultcache-ts/README.md +++ b/packages/cultcache-ts/README.md @@ -61,7 +61,7 @@ const note = decodeDocumentPayload(noteDocument, envelope.payload); // parsed No const payload = encodeDocumentPayload(noteDocument, note); // DocumentPayload ``` -Browsers import the same names from `@gamecult/cultcache-ts/document`, which holds no Node built-ins. +Browsers import the same names from `@gamecult/cultcache-ts/document`, which holds no Node built-ins. The whole cache without the single-file store is `@gamecult/cultcache-ts/browser`, which is also what bundlers resolve for the package root (`browser` condition); bring your own `CacheBackingStore`. ## `.cc` Inspection diff --git a/packages/cultcache-ts/package.json b/packages/cultcache-ts/package.json index c48e54e5..ffe1463d 100644 --- a/packages/cultcache-ts/package.json +++ b/packages/cultcache-ts/package.json @@ -7,9 +7,15 @@ "exports": { ".": { "types": "./dist/index.d.ts", + "browser": "./dist/browser.js", "require": "./dist/index.js", "default": "./dist/index.js" }, + "./browser": { + "types": "./dist/browser.d.ts", + "require": "./dist/browser.js", + "default": "./dist/browser.js" + }, "./document": { "types": "./dist/document.d.ts", "require": "./dist/document.js", @@ -25,6 +31,9 @@ "*": { "document": [ "dist/document.d.ts" + ], + "browser": [ + "dist/browser.d.ts" ] } }, diff --git a/packages/cultcache-ts/src/browser.ts b/packages/cultcache-ts/src/browser.ts new file mode 100644 index 00000000..bf62976f --- /dev/null +++ b/packages/cultcache-ts/src/browser.ts @@ -0,0 +1,49 @@ +// Browser-safe surface: everything the package root exports except the node-only single-file store +// (node:fs, koffi). Bring your own CacheBackingStore, or none. +export * as codecs from "./codec"; +export type { Codec, SlotsCodec } from "./codec"; +export { CultCache, CultCacheBuilder } from "./cult-cache"; +export { inspectCultCacheBytes } from "./cult-cache-inspector"; +export { + decodeDocumentPayload, + defineDocumentRegistry, + defineDocumentType, + documentPayloadBytes, + documentPayloadFromBytes, + encodeDocumentPayload, + schemaIdentityOf, + type CultCacheSchemaIdentity, +} from "./document"; +export { CultCachePayloadDepthError, MAX_PAYLOAD_DEPTH } from "./payload-depth"; +export type { DocumentPayload } from "./document"; +export { SchemaConflictError, StoreUnreadableError } from "./store-format"; +export type { + AnyCultCacheDocumentDefinition, + CacheBackingStore, + CultCacheDocumentAccessor, + CultCacheDocumentDefinition, + CultCacheDocumentFieldName, + CultCacheDocumentFormatter, + CultCacheDocumentIndexDefinition, + CultCacheDocumentRegistry, + CultCacheSchemaCatalogEntry, + CultCacheSchemaCatalogMember, + CultCacheSchema, + CultCacheDocumentValue, + CultCacheEnvelope, + CultCacheIndexScalar, + InferCultCacheSchemaValue, + CultCacheStoreRegistration, + PushAllOptions, +} from "./types"; +export type { + CultCacheInspection, + InspectedCatalogEntry, + InspectedCatalogMember, + InspectedRecord, +} from "./cult-cache-inspector"; +export { + defineSwarmDocuments, + type DefineDocumentType, + type SwarmDocumentCatalog, +} from "./swarm-documents"; diff --git a/packages/cultcache-ts/src/index.ts b/packages/cultcache-ts/src/index.ts index 1facf6e2..f55dd049 100644 --- a/packages/cultcache-ts/src/index.ts +++ b/packages/cultcache-ts/src/index.ts @@ -1,48 +1,2 @@ -export * as codecs from "./codec"; -export type { Codec, SlotsCodec } from "./codec"; -export { CultCache, CultCacheBuilder } from "./cult-cache"; -export { inspectCultCacheBytes } from "./cult-cache-inspector"; -export { - decodeDocumentPayload, - defineDocumentRegistry, - defineDocumentType, - documentPayloadBytes, - documentPayloadFromBytes, - encodeDocumentPayload, - schemaIdentityOf, - type CultCacheSchemaIdentity, -} from "./document"; -export { CultCachePayloadDepthError, MAX_PAYLOAD_DEPTH } from "./payload-depth"; -export type { DocumentPayload } from "./document"; +export * from "./browser"; export { SingleFileMessagePackBackingStore, readSingleFileStore, type SingleFileStoreRead } from "./single-file-messagepack-backing-store"; -export { SchemaConflictError, StoreUnreadableError } from "./store-format"; -export type { - AnyCultCacheDocumentDefinition, - CacheBackingStore, - CultCacheDocumentAccessor, - CultCacheDocumentDefinition, - CultCacheDocumentFieldName, - CultCacheDocumentFormatter, - CultCacheDocumentIndexDefinition, - CultCacheDocumentRegistry, - CultCacheSchemaCatalogEntry, - CultCacheSchemaCatalogMember, - CultCacheSchema, - CultCacheDocumentValue, - CultCacheEnvelope, - CultCacheIndexScalar, - InferCultCacheSchemaValue, - CultCacheStoreRegistration, - PushAllOptions, -} from "./types"; -export type { - CultCacheInspection, - InspectedCatalogEntry, - InspectedCatalogMember, - InspectedRecord, -} from "./cult-cache-inspector"; -export { - defineSwarmDocuments, - type DefineDocumentType, - type SwarmDocumentCatalog, -} from "./swarm-documents"; diff --git a/packages/cultcache-ts/test/browser-entry.test.ts b/packages/cultcache-ts/test/browser-entry.test.ts new file mode 100644 index 00000000..bbf2e2c4 --- /dev/null +++ b/packages/cultcache-ts/test/browser-entry.test.ts @@ -0,0 +1,45 @@ +import assert from "node:assert/strict"; +import { join } from "node:path"; +import test from "node:test"; +import { runInNewContext } from "node:vm"; +import { build } from "esbuild"; + +// dist-test/test/ -> dist-test/src/ +const srcDir = join(__dirname, "..", "src"); + +async function bundleForBrowser(entry: string): Promise { + const result = await build({ + entryPoints: [join(srcDir, entry)], + bundle: true, + platform: "browser", + format: "iife", + globalName: "cultcache", + write: false, + logLevel: "silent", + }); + return result.outputFiles[0].text; +} + +test("the browser entry bundles for platform=browser and runs without node", async () => { + const code = await bundleForBrowser("browser.js"); + assert.ok(!/node:fs|koffi/.test(code), "no node-only store in the browser bundle"); + // A context with no require/process/Buffer: only what a browser page has. + const sandbox: Record = { TextEncoder, TextDecoder }; + runInNewContext(code + "\nglobalThis.__api = cultcache;", sandbox); + const api = sandbox.__api as Record; + assert.equal(typeof api.CultCache, "function"); + assert.equal(typeof api.defineDocumentType, "function"); + assert.equal(api.SingleFileMessagePackBackingStore, undefined); +}); + +test("the node entry still exports the single-file store and everything the browser entry has", async () => { + const node = await import(join(srcDir, "index.js")); + const browser = await import(join(srcDir, "browser.js")); + assert.equal(typeof node.SingleFileMessagePackBackingStore, "function"); + assert.equal(typeof node.readSingleFileStore, "function"); + for (const name of Object.keys(browser)) assert.ok(name in node, `node entry lost ${name}`); +}); + +test("the package root keeps the node-only store out of the browser build", async () => { + await assert.rejects(bundleForBrowser("index.js"), /node:fs|node:path|koffi/); +}); From 4a407f9464623bc4d9401190ae21e7beab1e583f Mon Sep 17 00:00:00 2001 From: Libby Date: Sat, 7 Nov 2026 11:44:55 +0000 Subject: [PATCH 81/88] ci: drop the Eve getting-started job; Eve is deprecated and its repo no longer resolves cultmesh-portability's getting-started job checked out GameCult/Eve (404 on the forge) and ran verify-eve-getting-started.ps1 -EveRoot ./Eve, which is Eve end to end. Nothing in it runs without Eve, so the job goes. reactive-and-resource-lifetime has no Eve dependency and is unchanged. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/cultmesh-portability.yml | 39 ---------------------- 1 file changed, 39 deletions(-) diff --git a/.github/workflows/cultmesh-portability.yml b/.github/workflows/cultmesh-portability.yml index 3648ba2b..19bf1b34 100644 --- a/.github/workflows/cultmesh-portability.yml +++ b/.github/workflows/cultmesh-portability.yml @@ -9,45 +9,6 @@ on: workflow_dispatch: jobs: - getting-started: - name: ${{ matrix.os }} one-command multiplayer onboarding - runs-on: ${{ matrix.os }} - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - # The forge runner is Ubuntu-only: Windows legs run on workflow_dispatch (GitHub or a Windows runner), Ubuntu gates push and PR. - os: ${{ fromJSON(github.event_name == 'workflow_dispatch' && '["windows-latest", "ubuntu-latest"]' || '["ubuntu-latest"]') }} - - steps: - - name: Check out CultLib - uses: actions/checkout@v4 - with: - path: CultLib - - - name: Check out the matching Eve contract - uses: actions/checkout@v4 - with: - repository: GameCult/Eve - # Eve main is the contract this sample is written against; the codex - # branch this used to pin was never merged and stopped on 2026-08-25. - ref: ${{ vars.EVE_INTEGRATION_REF || 'main' }} - path: Eve - - - name: Set up .NET - uses: actions/setup-dotnet@v4 - with: - dotnet-version: 10.0.x - - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: 24 - - - name: Run the documented one-command checkpoint - shell: pwsh - run: ./CultLib/scripts/verify-eve-getting-started.ps1 -EveRoot ./Eve - reactive-and-resource-lifetime: name: ${{ matrix.os }} reactive semantics and leases runs-on: ${{ matrix.os }} From 0bffcf0f9fe1da5a44d629da6462a0057a71e3d3 Mon Sep 17 00:00:00 2001 From: Libby Date: Sat, 7 Nov 2026 12:23:39 +0000 Subject: [PATCH 82/88] cultnet: add unsupported_schema_version to the closed cultnet.error.v0 code set Ruling (steward): add the code across C#, Rust, TypeScript and Python, each with a captured-bytes case. details = { field: "schemaVersion", value: }, no routingHint. - schema (both copies), C# ForUnsupportedSchemaVersion + CultNetErrorCode.UnsupportedSchemaVersion in Rust, TS code list, Python CULTNET_ERROR_CODES and schema catalog. - bytes: error-vectors.json gains unsupported_schema_version (C# serializer output, checked by the C# test, the Rust error_message.rs case, and the TS/Python vector tests; the Python builder reproduces them). - cultnet-py interop_peer and cultmesh-py server now answer with the contract-shaped error (no extra messageId, details {field,value}) via messages.unsupported_schema_version_error. - tests that used unsupported_schema_version as the example of an unknown code now use not_a_code. Co-Authored-By: Claude Sonnet 5.5 --- .../cultnet/cultnet-distributed-database.md | 5 +++-- contracts/cultnet/cultnet.error.schema.json | 6 +++--- contracts/cultnet/interop/error-vectors.json | 16 ++++++++++++++++ packages/cultmesh-py/src/cultmesh_py/server.py | 8 ++------ packages/cultmesh-py/tests/test_cultmesh.py | 9 ++++----- packages/cultnet-py/src/cultnet_py/__init__.py | 2 ++ .../cultnet-py/src/cultnet_py/interop_peer.py | 10 ++-------- packages/cultnet-py/src/cultnet_py/messages.py | 11 +++++++++++ .../cultnet-py/src/cultnet_py/schema_catalog.py | 1 + packages/cultnet-py/tests/test_cultnet.py | 3 ++- packages/cultnet-py/tests/test_cultnet_error.py | 5 +++-- .../contracts/cultnet/cultnet.error.schema.json | 6 +++--- packages/cultnet-rs/src/contracts.rs | 3 +++ packages/cultnet-rs/tests/error_message.rs | 15 +++++++++++++++ packages/cultnet-ts/src/contracts.ts | 1 + packages/cultnet-ts/test/cultnet-error.test.ts | 2 +- src/GameCult.Networking/CultNetSchemaMessages.cs | 14 +++++++++++++- .../CultNetSchemaContractTests.cs | 13 +++++++++++++ 18 files changed, 98 insertions(+), 32 deletions(-) diff --git a/contracts/cultnet/cultnet-distributed-database.md b/contracts/cultnet/cultnet-distributed-database.md index a170835e..cc6a6a57 100644 --- a/contracts/cultnet/cultnet-distributed-database.md +++ b/contracts/cultnet/cultnet-distributed-database.md @@ -112,8 +112,9 @@ not guess. Adding a code is therefore a coordinated change across every runtime, copies), the C# `CultNetErrorMessage` factory, the Rust `CultNetErrorCode`, and a captured-bytes case in `packages/cultnet-rs/tests/error_message.rs` for it. TypeScript reads the schema. The current codes are `selection_invalid`, `cursor_stale`, `cursor_invalid`, `reference_outside_target`, `variant_unsupported`, -`unowned_schema` (a write, or a remote put or delete, for a schema and key no shard owns) and -`write_not_authorized` (a peer's put or delete its server's remote write authority did not grant). +`unowned_schema` (a write, or a remote put or delete, for a schema and key no shard owns), +`write_not_authorized` (a peer's put or delete its server's remote write authority did not grant) and +`unsupported_schema_version` (a message whose `schemaVersion` the peer does not speak; `details` names `schemaVersion` and the offered value). ## Live Invariants diff --git a/contracts/cultnet/cultnet.error.schema.json b/contracts/cultnet/cultnet.error.schema.json index e3b94c2b..341de5af 100644 --- a/contracts/cultnet/cultnet.error.schema.json +++ b/contracts/cultnet/cultnet.error.schema.json @@ -28,7 +28,7 @@ "code": { "description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it. The set is a closed enum: Rust and TypeScript refuse a message whose code is not listed, so adding a code is one coordinated change across every runtime (this schema, the C# factory, the Rust CultNetErrorCode, and a captured-bytes case for each).", "type": ["string", "null"], - "enum": [null, "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target", "variant_unsupported", "unowned_schema", "write_not_authorized"] + "enum": [null, "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target", "variant_unsupported", "unowned_schema", "write_not_authorized", "unsupported_schema_version"] }, "details": { "description": "Code-specific structured detail (R-N). Every field is populated only for the code that defines it; the others are null, not omitted.", @@ -36,8 +36,8 @@ "additionalProperties": false, "required": ["field", "value", "asOf", "current"], "properties": { - "field": { "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended. unowned_schema: \"schemaId\". write_not_authorized: \"recordKey\".", "type": ["string", "null"] }, - "value": { "description": "selection_invalid: the offending value, when there is one string worth naming. variant_unsupported: the id of the subscription that ended. unowned_schema: the schema id no shard owns. write_not_authorized: the key of the refused write.", "type": ["string", "null"] }, + "field": { "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended. unowned_schema: \"schemaId\". write_not_authorized: \"recordKey\". unsupported_schema_version: \"schemaVersion\".", "type": ["string", "null"] }, + "value": { "description": "selection_invalid: the offending value, when there is one string worth naming. variant_unsupported: the id of the subscription that ended. unowned_schema: the schema id no shard owns. write_not_authorized: the key of the refused write. unsupported_schema_version: the schemaVersion the peer does not speak.", "type": ["string", "null"] }, "asOf": { "description": "cursor_stale: the cursor's own minted asOf.", "type": ["integer", "null"], "minimum": 0 }, "current": { "description": "cursor_stale: the answering server's current asOf.", "type": ["integer", "null"], "minimum": 0 } } diff --git a/contracts/cultnet/interop/error-vectors.json b/contracts/cultnet/interop/error-vectors.json index 05422f9e..db5e79cd 100644 --- a/contracts/cultnet/interop/error-vectors.json +++ b/contracts/cultnet/interop/error-vectors.json @@ -129,6 +129,22 @@ "current": null } } + }, + { + "name": "unsupported_schema_version", + "messagePackHex": "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72D92C756E737570706F727465645F736368656D615F76657273696F6E3A2063756C746E65742E68656C6C6F2E7631AB726F7574696E6748696E74C0A4636F6465BA756E737570706F727465645F736368656D615F76657273696F6EA764657461696C7384A56669656C64AD736368656D6156657273696F6EA576616C7565B063756C746E65742E68656C6C6F2E7631A461734F66C0A763757272656E74C0", + "message": { + "schemaVersion": "cultnet.error.v0", + "error": "unsupported_schema_version: cultnet.hello.v1", + "routingHint": null, + "code": "unsupported_schema_version", + "details": { + "field": "schemaVersion", + "value": "cultnet.hello.v1", + "asOf": null, + "current": null + } + } } ] } diff --git a/packages/cultmesh-py/src/cultmesh_py/server.py b/packages/cultmesh-py/src/cultmesh_py/server.py index e50def4d..0fb400a0 100644 --- a/packages/cultmesh-py/src/cultmesh_py/server.py +++ b/packages/cultmesh-py/src/cultmesh_py/server.py @@ -10,6 +10,7 @@ from typing import Any import msgpack from cultnet_py import ( + unsupported_schema_version_error, CultNetRawSnapshotResponse, CultNetSimulationObservationHub, CultNetRudpPacket, @@ -229,12 +230,7 @@ class CultMeshLocalServer: if schema_version == "cultnet.simulation_observation.v0": candidates = self._candidate_responses(message) return candidates[0] if candidates else None - return self._error_response( - f"Unsupported CultNet message schema: {schema_version!r}.", - message_id=str(message.get("messageId") or ""), - code="unsupported_schema_version", - details={"schemaVersion": schema_version}, - ) + return unsupported_schema_version_error(schema_version).to_wire() def _accept_loop(self) -> None: while not self._stop.is_set(): diff --git a/packages/cultmesh-py/tests/test_cultmesh.py b/packages/cultmesh-py/tests/test_cultmesh.py index 67f5fe36..40c911b1 100644 --- a/packages/cultmesh-py/tests/test_cultmesh.py +++ b/packages/cultmesh-py/tests/test_cultmesh.py @@ -5048,17 +5048,16 @@ class CultMeshTests(unittest.TestCase): finally: server.stop() - self.assertEqual(unsupported["messageId"], "bad-schema") self.assertEqual(unsupported["code"], "unsupported_schema_version") - self.assertEqual(unsupported["details"]["schemaVersion"], "cultnet.nope.v0") - self.assertIn("Unsupported CultNet message schema", unsupported["error"]) + self.assertEqual(unsupported["details"]["field"], "schemaVersion") + self.assertEqual(unsupported["details"]["value"], "cultnet.nope.v0") + self.assertIn("unsupported_schema_version", unsupported["error"]) self.assertEqual(malformed_put["messageId"], "bad-put") self.assertEqual(malformed_put["code"], "malformed_document_put") self.assertEqual(malformed_delete["messageId"], "bad-delete") self.assertEqual(malformed_delete["code"], "malformed_document_delete") - self.assertEqual(unsupported_v1["messageId"], "bad-v1-selection") self.assertEqual(unsupported_v1["code"], "unsupported_schema_version") - self.assertEqual(unsupported_v1["details"]["schemaVersion"], "cultnet.snapshot_request.v1") + self.assertEqual(unsupported_v1["details"]["value"], "cultnet.snapshot_request.v1") def test_cultmesh_authority_lease_requires_live_matching_lease(self) -> None: peer = CultMeshPeerCard( diff --git a/packages/cultnet-py/src/cultnet_py/__init__.py b/packages/cultnet-py/src/cultnet_py/__init__.py index bbcc94f2..9bad58f6 100644 --- a/packages/cultnet-py/src/cultnet_py/__init__.py +++ b/packages/cultnet-py/src/cultnet_py/__init__.py @@ -49,6 +49,7 @@ from .messages import ( document_put_raw, encode_witness_artifact_bundle_payload, error_message, + unsupported_schema_version_error, hello, login, login_success, @@ -251,6 +252,7 @@ __all__ = [ "encode_frame", "encode_rudp_packet", "error_message", + "unsupported_schema_version_error", "hello", "INTEROP_WIRE_CONTRACT", "login", diff --git a/packages/cultnet-py/src/cultnet_py/interop_peer.py b/packages/cultnet-py/src/cultnet_py/interop_peer.py index e6cd4601..56dddec8 100644 --- a/packages/cultnet-py/src/cultnet_py/interop_peer.py +++ b/packages/cultnet-py/src/cultnet_py/interop_peer.py @@ -38,6 +38,7 @@ from .cultmesh_contracts import ( simulation_fact_document, ) +from .messages import unsupported_schema_version_error as _unsupported_schema_version_error from .schema_catalog import INTEROP_WIRE_CONTRACT, wire_message_schema_descriptors from .client import create_rudp_schema_transport from .transport import ( @@ -407,14 +408,7 @@ def is_unsupported_cultnet_version(schema_version: Any) -> bool: def unsupported_schema_version_error(message: dict[str, Any]) -> dict[str, Any]: - schema_version = message.get("schemaVersion") - return { - "schemaVersion": "cultnet.error.v0", - "messageId": str(message.get("messageId") or ""), - "error": f"Unsupported CultNet message schema: {schema_version!r}.", - "code": "unsupported_schema_version", - "details": {"schemaVersion": schema_version}, - } + return _unsupported_schema_version_error(message.get("schemaVersion")).to_wire() def handle_database_subscribe(state: PeerState, message: dict[str, Any], subscriptions: dict[str, DatabaseSubscription]) -> list[dict[str, Any]]: diff --git a/packages/cultnet-py/src/cultnet_py/messages.py b/packages/cultnet-py/src/cultnet_py/messages.py index a5c0887e..bf21b985 100644 --- a/packages/cultnet-py/src/cultnet_py/messages.py +++ b/packages/cultnet-py/src/cultnet_py/messages.py @@ -43,6 +43,7 @@ CULTNET_ERROR_CODES: tuple[str, ...] = ( "variant_unsupported", "unowned_schema", "write_not_authorized", + "unsupported_schema_version", ) _ERROR_DETAILS_KEYS = ("field", "value", "asOf", "current") @@ -174,6 +175,16 @@ def error_message( return CultNetMessage("cultnet.error.v0", body) +def unsupported_schema_version_error(schema_version: Any) -> CultNetMessage: + """The refusal for a message whose schemaVersion this peer does not speak (C# ForUnsupportedSchemaVersion).""" + value = None if schema_version is None else str(schema_version) + return error_message( + f"unsupported_schema_version: {value or ''}", + code="unsupported_schema_version", + details={"field": "schemaVersion", "value": value}, + ) + + def login_success( *, nonce: str, diff --git a/packages/cultnet-py/src/cultnet_py/schema_catalog.py b/packages/cultnet-py/src/cultnet_py/schema_catalog.py index 908a777f..fc03c8a2 100644 --- a/packages/cultnet-py/src/cultnet_py/schema_catalog.py +++ b/packages/cultnet-py/src/cultnet_py/schema_catalog.py @@ -21,6 +21,7 @@ CULTNET_ERROR_CODES = ( "variant_unsupported", "unowned_schema", "write_not_authorized", + "unsupported_schema_version", ) WIRE_MESSAGE_SCHEMA_VERSIONS = ( diff --git a/packages/cultnet-py/tests/test_cultnet.py b/packages/cultnet-py/tests/test_cultnet.py index a0378069..6e8d2b71 100644 --- a/packages/cultnet-py/tests/test_cultnet.py +++ b/packages/cultnet-py/tests/test_cultnet.py @@ -2580,7 +2580,8 @@ class CultNetTests(unittest.TestCase): self.assertEqual(len(responses), 1) self.assertEqual(responses[0]["schemaVersion"], "cultnet.error.v0") self.assertEqual(responses[0]["code"], "unsupported_schema_version") - self.assertEqual(responses[0]["messageId"], "v1-refused") + self.assertEqual(responses[0]["details"]["value"], schema_version) + self.assertNotIn("messageId", responses[0]) def test_cultnet_simulation_consensus_dedupes_witnesses_and_requires_quorum(self) -> None: from cultnet_py import CultNetSimulationConsensus diff --git a/packages/cultnet-py/tests/test_cultnet_error.py b/packages/cultnet-py/tests/test_cultnet_error.py index 07f92cd9..2904e679 100644 --- a/packages/cultnet-py/tests/test_cultnet_error.py +++ b/packages/cultnet-py/tests/test_cultnet_error.py @@ -6,7 +6,7 @@ from pathlib import Path import msgpack -from cultnet_py import CULTNET_ERROR_CODES, error_message, parse_message +from cultnet_py import CULTNET_ERROR_CODES, error_message, parse_message, unsupported_schema_version_error # contracts/cultnet/interop/error-vectors.json: bytes captured from MessagePack C# 3.1.7 and pinned # by cultnet-rs/tests/error_message.rs (Rust encodes to exactly these bytes). @@ -50,6 +50,7 @@ class CultNetErrorMessageTests(unittest.TestCase): details={"field": "keys", "value": " "}, ) self.assertEqual(selection.to_bytes(), by_name["selection_invalid_field_value"]) + self.assertEqual(unsupported_schema_version_error("cultnet.hello.v1").to_bytes(), by_name["unsupported_schema_version"]) def test_builder_writes_nil_keys_and_accepts_a_null_code(self) -> None: wire = error_message("boom").to_wire() @@ -57,7 +58,7 @@ class CultNetErrorMessageTests(unittest.TestCase): def test_unknown_code_is_refused_by_builder_and_decoder(self) -> None: with self.assertRaises(ValueError): - error_message("x", code="unsupported_schema_version") + error_message("x", code="not_a_code") bad = msgpack.packb( {"schemaVersion": "cultnet.error.v0", "error": "x", "routingHint": None, "code": "nope", "details": None}, use_bin_type=True, diff --git a/packages/cultnet-rs/contracts/cultnet/cultnet.error.schema.json b/packages/cultnet-rs/contracts/cultnet/cultnet.error.schema.json index e3b94c2b..341de5af 100644 --- a/packages/cultnet-rs/contracts/cultnet/cultnet.error.schema.json +++ b/packages/cultnet-rs/contracts/cultnet/cultnet.error.schema.json @@ -28,7 +28,7 @@ "code": { "description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it. The set is a closed enum: Rust and TypeScript refuse a message whose code is not listed, so adding a code is one coordinated change across every runtime (this schema, the C# factory, the Rust CultNetErrorCode, and a captured-bytes case for each).", "type": ["string", "null"], - "enum": [null, "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target", "variant_unsupported", "unowned_schema", "write_not_authorized"] + "enum": [null, "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target", "variant_unsupported", "unowned_schema", "write_not_authorized", "unsupported_schema_version"] }, "details": { "description": "Code-specific structured detail (R-N). Every field is populated only for the code that defines it; the others are null, not omitted.", @@ -36,8 +36,8 @@ "additionalProperties": false, "required": ["field", "value", "asOf", "current"], "properties": { - "field": { "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended. unowned_schema: \"schemaId\". write_not_authorized: \"recordKey\".", "type": ["string", "null"] }, - "value": { "description": "selection_invalid: the offending value, when there is one string worth naming. variant_unsupported: the id of the subscription that ended. unowned_schema: the schema id no shard owns. write_not_authorized: the key of the refused write.", "type": ["string", "null"] }, + "field": { "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended. unowned_schema: \"schemaId\". write_not_authorized: \"recordKey\". unsupported_schema_version: \"schemaVersion\".", "type": ["string", "null"] }, + "value": { "description": "selection_invalid: the offending value, when there is one string worth naming. variant_unsupported: the id of the subscription that ended. unowned_schema: the schema id no shard owns. write_not_authorized: the key of the refused write. unsupported_schema_version: the schemaVersion the peer does not speak.", "type": ["string", "null"] }, "asOf": { "description": "cursor_stale: the cursor's own minted asOf.", "type": ["integer", "null"], "minimum": 0 }, "current": { "description": "cursor_stale: the answering server's current asOf.", "type": ["integer", "null"], "minimum": 0 } } diff --git a/packages/cultnet-rs/src/contracts.rs b/packages/cultnet-rs/src/contracts.rs index 56afaa7c..bf661df2 100644 --- a/packages/cultnet-rs/src/contracts.rs +++ b/packages/cultnet-rs/src/contracts.rs @@ -247,6 +247,7 @@ pub enum CultNetErrorCode { VariantUnsupported, UnownedSchema, WriteNotAuthorized, + UnsupportedSchemaVersion, } impl CultNetErrorCode { @@ -259,6 +260,7 @@ impl CultNetErrorCode { Self::VariantUnsupported => "variant_unsupported", Self::UnownedSchema => "unowned_schema", Self::WriteNotAuthorized => "write_not_authorized", + Self::UnsupportedSchemaVersion => "unsupported_schema_version", } } @@ -271,6 +273,7 @@ impl CultNetErrorCode { "variant_unsupported" => Ok(Self::VariantUnsupported), "unowned_schema" => Ok(Self::UnownedSchema), "write_not_authorized" => Ok(Self::WriteNotAuthorized), + "unsupported_schema_version" => Ok(Self::UnsupportedSchemaVersion), _ => Err(anyhow!("ErrorMessage.Code is not a recognized cultnet.error.v0 code")), } } diff --git a/packages/cultnet-rs/tests/error_message.rs b/packages/cultnet-rs/tests/error_message.rs index 8b36d774..06e25935 100644 --- a/packages/cultnet-rs/tests/error_message.rs +++ b/packages/cultnet-rs/tests/error_message.rs @@ -173,6 +173,21 @@ fn cases() -> Vec { }), }, }, + Case { + name: "unsupported_schema_version", + // C#: ForUnsupportedSchemaVersion("cultnet.hello.v1") + csharp_hex: "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72D92C756E737570706F727465645F736368656D615F76657273696F6E3A2063756C746E65742E68656C6C6F2E7631AB726F7574696E6748696E74C0A4636F6465BA756E737570706F727465645F736368656D615F76657273696F6EA764657461696C7384A56669656C64AD736368656D6156657273696F6EA576616C7565B063756C746E65742E68656C6C6F2E7631A461734F66C0A763757272656E74C0", + message: || CultNetMessage::Error { + error: "unsupported_schema_version: cultnet.hello.v1".to_string(), + code: Some(CultNetErrorCode::UnsupportedSchemaVersion), + details: Some(CultNetErrorDetails { + field: Some("schemaVersion".to_string()), + value: Some("cultnet.hello.v1".to_string()), + as_of: None, + current: None, + }), + }, + }, ] } diff --git a/packages/cultnet-ts/src/contracts.ts b/packages/cultnet-ts/src/contracts.ts index fc225627..ee060a1b 100644 --- a/packages/cultnet-ts/src/contracts.ts +++ b/packages/cultnet-ts/src/contracts.ts @@ -174,6 +174,7 @@ export const CULTNET_ERROR_CODES = [ "variant_unsupported", "unowned_schema", "write_not_authorized", + "unsupported_schema_version", ] as const; export type CultNetErrorCode = (typeof CULTNET_ERROR_CODES)[number]; diff --git a/packages/cultnet-ts/test/cultnet-error.test.ts b/packages/cultnet-ts/test/cultnet-error.test.ts index d185de8b..42ef0753 100644 --- a/packages/cultnet-ts/test/cultnet-error.test.ts +++ b/packages/cultnet-ts/test/cultnet-error.test.ts @@ -70,7 +70,7 @@ test("dropping code or details breaks the round trip and the decode", () => { test("a code outside the closed enum is refused, as is a malformed details object", () => { const base = decode(bytesOf(vectors.find((v) => v.name === "cursor_stale")!.messagePackHex)) as Record; - assert.throws(() => parseCultNetMessage({ ...base, code: "unsupported_schema_version" }), /cultnet\.error\.v0/); + assert.throws(() => parseCultNetMessage({ ...base, code: "not_a_code" }), /cultnet\.error\.v0/); assert.throws(() => parseCultNetMessage({ ...base, code: 7 }), /cultnet\.error\.v0/); assert.throws( () => parseCultNetMessage({ ...base, details: { field: null, value: null, asOf: 1, current: 2, extra: 1 } }), diff --git a/src/GameCult.Networking/CultNetSchemaMessages.cs b/src/GameCult.Networking/CultNetSchemaMessages.cs index baf0b6d5..6b97c792 100644 --- a/src/GameCult.Networking/CultNetSchemaMessages.cs +++ b/src/GameCult.Networking/CultNetSchemaMessages.cs @@ -511,7 +511,7 @@ namespace GameCult.Networking [Key("routingHint")] public CultNetShardRoutingHint? RoutingHint { get; set; } /// /// Gets or sets the machine-readable refusal code (R-N), e.g. selection_invalid, - /// cursor_stale, cursor_invalid, reference_outside_target, unowned_schema, write_not_authorized. Additive to + /// cursor_stale, cursor_invalid, reference_outside_target, unowned_schema, write_not_authorized, unsupported_schema_version. Additive to /// cultnet.error.v0; a v0 peer that does not know this field ignores it. /// [Key("code")] public string? Code { get; set; } @@ -555,6 +555,18 @@ namespace GameCult.Networking Details = new CultNetErrorDetails { Field = "recordKey", Value = request.RecordKey.Value } }; + /// + /// Builds the wire error for a message whose schemaVersion this peer does not speak: code + /// unsupported_schema_version, details { field: "schemaVersion", value }. + /// + public static CultNetErrorMessage ForUnsupportedSchemaVersion(string? schemaVersion) => + new CultNetErrorMessage + { + Error = $"unsupported_schema_version: {schemaVersion}", + Code = "unsupported_schema_version", + Details = new CultNetErrorDetails { Field = "schemaVersion", Value = schemaVersion } + }; + /// /// Builds the wire error for a cursor refusal (R-N): code cursor_stale with details /// { asOf, current }, or code cursor_invalid with no details. diff --git a/tests/GameCult.Networking.Tests/CultNetSchemaContractTests.cs b/tests/GameCult.Networking.Tests/CultNetSchemaContractTests.cs index 8b989bb6..23bb5f5b 100644 --- a/tests/GameCult.Networking.Tests/CultNetSchemaContractTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetSchemaContractTests.cs @@ -220,6 +220,19 @@ namespace GameCult.Networking.Tests Assert.That(Convert.ToHexString(CultNetSchemaMessageSerialization.Serialize(message)), Is.EqualTo("85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72D92177726974655F6E6F745F617574686F72697A65643A206E6F74206772616E746564AB726F7574696E6748696E74C0A4636F6465B477726974655F6E6F745F617574686F72697A6564A764657461696C7384A56669656C64A97265636F72644B6579A576616C7565A8736F6D653A6B6579A461734F66C0A763757272656E74C0")); } + [Test] + public void ErrorMessage_UnsupportedSchemaVersionRealBytes_MatchesItsSchema() + { + var message = CultNetErrorMessage.ForUnsupportedSchemaVersion("cultnet.hello.v1"); + + Assert.That(message.Code, Is.EqualTo("unsupported_schema_version")); + Assert.That(message.Details!.Field, Is.EqualTo("schemaVersion")); + Assert.That(message.Details.Value, Is.EqualTo("cultnet.hello.v1")); + AssertErrorMatchesSchema(message); + // The bytes cultnet-rs's error_message.rs unsupported_schema_version case decodes and re-encodes. + Assert.That(Convert.ToHexString(CultNetSchemaMessageSerialization.Serialize(message)), Is.EqualTo("85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72D92C756E737570706F727465645F736368656D615F76657273696F6E3A2063756C746E65742E68656C6C6F2E7631AB726F7574696E6748696E74C0A4636F6465BA756E737570706F727465645F736368656D615F76657273696F6EA764657461696C7384A56669656C64AD736368656D6156657273696F6EA576616C7565B063756C746E65742E68656C6C6F2E7631A461734F66C0A763757272656E74C0")); + } + private static void AssertErrorMatchesSchema(CultNetErrorMessage message) { var validator = new MiniJsonSchemaValidator(SchemaDir()); From 08715ac55c783d02b1c7706ccb1ba22285ee1068 Mon Sep 17 00:00:00 2001 From: Libby Date: Sat, 7 Nov 2026 16:17:05 +0000 Subject: [PATCH 83/88] cultcache-rs: pin the one-global rule on load_soa load_soa installs a whole table and checks it with refuse_second_globals, but no test reached that call (PR #15 covered put paths and pull_all). A table holding two records of a global type is refused with CultCacheSecondGlobal and the prior view survives; a table with one global loads and replaces the view. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-rs/tests/lookups.rs | 31 +++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/packages/cultcache-rs/tests/lookups.rs b/packages/cultcache-rs/tests/lookups.rs index 2467fbc6..ca5b459d 100644 --- a/packages/cultcache-rs/tests/lookups.rs +++ b/packages/cultcache-rs/tests/lookups.rs @@ -2,7 +2,8 @@ //! (`src/GameCult.Caching/CultCache.cs`) and the one-global rule its commit enforces. use cultcache_rs::{ - CacheBackingStore, CultCache, CultCacheSecondGlobal, DatabaseEntry, SingleFileMessagePackBackingStore, + CacheBackingStore, CultCache, CultCacheSecondGlobal, CultCacheSoaTable, DatabaseEntry, + SingleFileMessagePackBackingStore, }; #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] @@ -321,3 +322,31 @@ fn loading_a_store_that_holds_two_globals_is_refused() { assert!(loading.pull_all_backing_stores().unwrap_err().is::()); assert_eq!(loading.get_global::().unwrap(), None, "a refused load installs nothing"); } + +// load_soa installs a whole table at once, so the one-global rule has to hold there too: a table holding two +// records of a global type is refused whole and the prior view survives. +#[test] +fn loading_an_soa_table_that_holds_two_globals_is_refused() { + let source = cache(); + let (first, _) = source.prepare_entry("g1", &settings("one")).unwrap(); + let (second, _) = source.prepare_entry("g2", &settings("two")).unwrap(); + + let mut loading = cache(); + loading.put("g0", &settings("prior")).unwrap(); + loading.put("plain:1", &Plain { value: "prior".into() }).unwrap(); + + let refusal = loading + .load_soa(CultCacheSoaTable::from_envelopes(vec![first.clone(), second])) + .unwrap_err(); + let second_global = refusal.downcast_ref::().expect("a typed refusal"); + let mut named = [second_global.existing_key.as_str(), second_global.key.as_str()]; + named.sort(); + assert_eq!(named, ["g1", "g2"], "the refusal names both records"); + assert_eq!(loading.get_global::().unwrap(), Some(settings("prior")), "a refused load keeps the prior view"); + assert_eq!(loading.get::("plain:1").unwrap(), Some(Plain { value: "prior".into() })); + + // One global in the table is the rule's own case, not a second. + loading.load_soa(CultCacheSoaTable::from_envelopes(vec![first])).unwrap(); + assert_eq!(loading.get_global::().unwrap(), Some(settings("one"))); + assert_eq!(loading.get::("plain:1").unwrap(), None, "a load replaces the view"); +} From 286a9af38969708c5e0e1e41dafbfbd90bd91de4 Mon Sep 17 00:00:00 2001 From: Libby Date: Sat, 7 Nov 2026 16:25:10 +0000 Subject: [PATCH 84/88] npm: scope cultnet-ts, cultmesh-ts and cultmesh-browser under @gamecult Unscoped npm names are unclaimed and open to dependency confusion; cultcache-ts was already @gamecult/cultcache-ts. package.json names, intra-repo dependencies, package-lock, import specifiers (src, tests, samples, docs/start, first-call docs test), the workspace list in run-typescript-workspaces, the verify-eve module map and package READMEs now use the scoped names. Directory names, workspace paths in CI and release tag prefixes are unchanged. Historical cut docs and past CHANGELOG entries keep the old names. Co-Authored-By: Claude Sonnet 5.5 --- docs/start/README.md | 4 +-- docs/start/io-game.md | 10 +++--- docs/start/io-game/client.ts | 2 +- docs/start/io-game/server.ts | 4 +-- package-lock.json | 33 ++++++++++--------- packages/cultmesh-browser/CHANGELOG.md | 8 ++++- packages/cultmesh-browser/README.md | 4 +-- packages/cultmesh-browser/package.json | 6 ++-- packages/cultmesh-browser/src/content.ts | 4 +-- packages/cultmesh-browser/src/index.ts | 6 ++-- .../cultmesh-browser/test/content.test.ts | 2 +- .../test/cultmesh-browser.test.ts | 4 +-- .../test/interop/content-https-provider.ts | 4 +-- .../test/interop/content-https-reader.ts | 2 +- .../test/interop/content-https.test.ts | 2 +- .../test/payload-type.test.ts | 4 +-- packages/cultmesh-py/README.md | 2 +- packages/cultmesh-ts/CHANGELOG.md | 3 +- packages/cultmesh-ts/README.md | 18 +++++----- packages/cultmesh-ts/package.json | 4 +-- packages/cultmesh-ts/src/content.ts | 2 +- packages/cultmesh-ts/src/index.ts | 4 +-- .../src/provider-rudp-transport.ts | 4 +-- .../src/provider-session-broker.ts | 4 +-- .../cultmesh-ts/src/provider-session-wire.ts | 4 +-- packages/cultmesh-ts/src/realtime-quic.ts | 4 +-- packages/cultmesh-ts/src/realtime-wire.ts | 4 +-- .../cultmesh-ts/test/codec-documents.test.ts | 2 +- packages/cultmesh-ts/test/content.test.ts | 2 +- packages/cultmesh-ts/test/cultmesh.test.ts | 4 +-- .../test/interop/cultmesh-quic-peer.ts | 4 +-- .../cultmesh-ts/test/payload-type.test.ts | 4 +-- .../test/provider-rudp-transport.test.ts | 4 +-- .../test/provider-session-wire.test.ts | 2 +- .../test/realtime-quic-consumer.test.ts | 4 +-- .../realtime-quic-session-manager.test.ts | 4 +-- .../rudp-document-server-send-failure.test.ts | 4 +-- packages/cultnet-py/README.md | 2 +- packages/cultnet-rs/README.md | 4 +-- packages/cultnet-ts/CHANGELOG.md | 3 +- packages/cultnet-ts/README.md | 18 +++++----- packages/cultnet-ts/package.json | 2 +- packages/cultnet-ts/src/cultmesh-authority.ts | 4 +-- packages/cultnet-ts/test/first-call.test.ts | 4 +-- .../test/idunn-runtime-authority.test.ts | 6 ++-- samples/eve-browser-network/README.md | 2 +- samples/eve-browser-network/browser/main.ts | 2 +- samples/eve-two-runtime/sample.mjs | 2 +- scripts/run-typescript-workspaces.mjs | 2 +- scripts/test-typescript-package-closure.mjs | 14 ++++---- scripts/verify-eve-browser-network.mjs | 4 +-- .../docs/browser-verse-transport.md | 12 +++---- .../03-publish-an-eve-surface.md | 2 +- .../docs/reliability-migration-plan.md | 2 +- src/GameCult.Networking/README.md | 2 +- 55 files changed, 142 insertions(+), 131 deletions(-) diff --git a/docs/start/README.md b/docs/start/README.md index 42adc5d9..e033c935 100644 --- a/docs/start/README.md +++ b/docs/start/README.md @@ -284,7 +284,7 @@ public static class CultNetFirstCall Source: [`packages/cultnet-ts/test/first-call.test.ts`](../../packages/cultnet-ts/test/first-call.test.ts), run by `npm test` in `packages/cultnet-ts` (build `packages/cultcache-ts` first: `npm run build` there). The test imports -from `../src` where the block reads `cultnet-ts`. More: `packages/cultnet-ts/README.md`. +from `../src` where the block reads `@gamecult/cultnet-ts`. More: `packages/cultnet-ts/README.md`. ```ts import { createServer, connect } from "node:net"; @@ -294,7 +294,7 @@ import { CultNetDocumentRegistry, createTcpFramedCultNetPeer, defineCultNetDocumentBinding, -} from "cultnet-ts"; +} from "@gamecult/cultnet-ts"; const noteDocument = defineDocumentType({ type: "note", diff --git a/docs/start/io-game.md b/docs/start/io-game.md index 4781d9bf..e7f701a7 100644 --- a/docs/start/io-game.md +++ b/docs/start/io-game.md @@ -9,7 +9,7 @@ leave because both lease the same document. | Piece | Status | | --- | --- | -| Two clients, real WebSockets, real `cultmesh-browser` client, one arena: join, see each other, move, leave | **tested** (`arena.test.ts`) | +| Two clients, real WebSockets, real `@gamecult/cultmesh-browser` client, one arena: join, see each other, move, leave | **tested** (`arena.test.ts`) | | Edge clamp, and the source below matching the files that ran | **tested** | | Browser bundle builds and the page is served | **tested** (HTTP fetch only) | | The page running in a real browser, two windows | **not yet**: no browser on the host that built this recipe | @@ -19,7 +19,7 @@ leave because both lease the same document. | Dropped players: a closed tab that never sent `leave` stays in the arena | **not yet**: operations do not carry a connection identity to the authority | | `CultCache` in the browser | **not yet**: the `@gamecult/cultcache-ts` entry imports Node built-ins, so the client keeps the decoded document in a variable and only `./document` is imported | -No `cultmesh-ts` or `cultnet-ts` TypeScript authority server exists. The server below is the CultNet +No `@gamecult/cultmesh-ts` or `@gamecult/cultnet-ts` TypeScript authority server exists. The server below is the CultNet message handling a provider owes, written in this recipe with the `ws` package. ## Run it @@ -61,8 +61,8 @@ export const arenaSize = 100; import { createServer, type RequestListener, type Server } from "node:http"; import { CultCache } from "@gamecult/cultcache-ts"; import { defineDocumentRegistry } from "@gamecult/cultcache-ts/document"; -import { decodeWireValue, encodeWireValue } from "cultnet-ts/wire"; -import { encodeCultNetMessageForWire, parseCultNetMessage, type CultNetMessage } from "cultnet-ts/contracts"; +import { decodeWireValue, encodeWireValue } from "@gamecult/cultnet-ts/wire"; +import { encodeCultNetMessageForWire, parseCultNetMessage, type CultNetMessage } from "@gamecult/cultnet-ts/contracts"; import { WebSocketServer, type WebSocket } from "ws"; import { arenaDocument, arenaKey, arenaSize, authorityRuntimeId, verseId, type ArenaState } from "./schema.ts"; @@ -163,7 +163,7 @@ export async function startArena(port = 0, host = "127.0.0.1", onRequest?: Reque ## client.ts ```ts -import { CultMeshBrowserClient, CultMeshBrowserOdinRendezvous, decodeCultNetOperationPayload, decodeCultNetPayload } from "cultmesh-browser"; +import { CultMeshBrowserClient, CultMeshBrowserOdinRendezvous, decodeCultNetOperationPayload, decodeCultNetPayload } from "@gamecult/cultmesh-browser"; import { arenaDocument, arenaKey, authorityRuntimeId, verseId, type ArenaState } from "./schema.ts"; // Runs unchanged in a browser or in Node 22+: both have a global WebSocket. diff --git a/docs/start/io-game/client.ts b/docs/start/io-game/client.ts index 8493e23c..10c4b4cd 100644 --- a/docs/start/io-game/client.ts +++ b/docs/start/io-game/client.ts @@ -1,4 +1,4 @@ -import { CultMeshBrowserClient, CultMeshBrowserOdinRendezvous, decodeCultNetOperationPayload, decodeCultNetPayload } from "cultmesh-browser"; +import { CultMeshBrowserClient, CultMeshBrowserOdinRendezvous, decodeCultNetOperationPayload, decodeCultNetPayload } from "@gamecult/cultmesh-browser"; import { arenaDocument, arenaKey, authorityRuntimeId, verseId, type ArenaState } from "./schema.ts"; // Runs unchanged in a browser or in Node 22+: both have a global WebSocket. diff --git a/docs/start/io-game/server.ts b/docs/start/io-game/server.ts index 39a1a618..7ef552b9 100644 --- a/docs/start/io-game/server.ts +++ b/docs/start/io-game/server.ts @@ -1,8 +1,8 @@ import { createServer, type RequestListener, type Server } from "node:http"; import { CultCache } from "@gamecult/cultcache-ts"; import { defineDocumentRegistry } from "@gamecult/cultcache-ts/document"; -import { decodeWireValue, encodeWireValue } from "cultnet-ts/wire"; -import { encodeCultNetMessageForWire, parseCultNetMessage, type CultNetMessage } from "cultnet-ts/contracts"; +import { decodeWireValue, encodeWireValue } from "@gamecult/cultnet-ts/wire"; +import { encodeCultNetMessageForWire, parseCultNetMessage, type CultNetMessage } from "@gamecult/cultnet-ts/contracts"; import { WebSocketServer, type WebSocket } from "ws"; import { arenaDocument, arenaKey, arenaSize, authorityRuntimeId, verseId, type ArenaState } from "./schema.ts"; diff --git a/package-lock.json b/package-lock.json index 9100d8db..abc8f6d1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -482,6 +482,18 @@ "resolved": "packages/cultcache-ts", "link": true }, + "node_modules/@gamecult/cultmesh-browser": { + "resolved": "packages/cultmesh-browser", + "link": true + }, + "node_modules/@gamecult/cultmesh-ts": { + "resolved": "packages/cultmesh-ts", + "link": true + }, + "node_modules/@gamecult/cultnet-ts": { + "resolved": "packages/cultnet-ts", + "link": true + }, "node_modules/@jsdevtools/ono": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/@jsdevtools/ono/-/ono-7.1.3.tgz", @@ -865,18 +877,6 @@ "dev": true, "license": "Python-2.0" }, - "node_modules/cultmesh-browser": { - "resolved": "packages/cultmesh-browser", - "link": true - }, - "node_modules/cultmesh-ts": { - "resolved": "packages/cultmesh-ts", - "link": true - }, - "node_modules/cultnet-ts": { - "resolved": "packages/cultnet-ts", - "link": true - }, "node_modules/esbuild": { "version": "0.28.1", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", @@ -1200,15 +1200,16 @@ } }, "packages/cultmesh-browser": { + "name": "@gamecult/cultmesh-browser", "version": "0.4.0", "license": "MIT", "dependencies": { "@gamecult/cultcache-ts": "^0.14.0", - "cultnet-ts": "^0.3.0" + "@gamecult/cultnet-ts": "^0.3.0" }, "devDependencies": { + "@gamecult/cultmesh-ts": "^0.2.0", "@types/node": "^24.0.0", - "cultmesh-ts": "^0.2.0", "typescript": "^5.9.0" }, "engines": { @@ -1216,11 +1217,12 @@ } }, "packages/cultmesh-ts": { + "name": "@gamecult/cultmesh-ts", "version": "0.2.0", "license": "MIT", "dependencies": { "@gamecult/cultcache-ts": "^0.14.0", - "cultnet-ts": "^0.3.0", + "@gamecult/cultnet-ts": "^0.3.0", "koffi": "3.3.0" }, "devDependencies": { @@ -1233,6 +1235,7 @@ } }, "packages/cultnet-ts": { + "name": "@gamecult/cultnet-ts", "version": "0.3.0", "license": "MIT", "dependencies": { diff --git a/packages/cultmesh-browser/CHANGELOG.md b/packages/cultmesh-browser/CHANGELOG.md index 0093ccf0..ccda16a0 100644 --- a/packages/cultmesh-browser/CHANGELOG.md +++ b/packages/cultmesh-browser/CHANGELOG.md @@ -1,8 +1,14 @@ # Changelog -All notable changes to `cultmesh-browser` are recorded here, following +All notable changes to `@gamecult/cultmesh-browser` are recorded here, following [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## Unreleased + +### Breaking + +- The package is renamed `cultmesh-browser` to `@gamecult/cultmesh-browser` (the repo already scopes `@gamecult/cultcache-ts`), so the unscoped name cannot be claimed by someone else. Import from `@gamecult/cultmesh-browser` and its subpaths; the API is unchanged. Depends on `@gamecult/cultnet-ts`; the interop test uses `@gamecult/cultmesh-ts`. + ## [0.4.0] ### Added diff --git a/packages/cultmesh-browser/README.md b/packages/cultmesh-browser/README.md index ab29d65f..500b858b 100644 --- a/packages/cultmesh-browser/README.md +++ b/packages/cultmesh-browser/README.md @@ -1,6 +1,6 @@ # CultMesh Browser -`cultmesh-browser` is the browser-safe CultMesh client boundary. It owns one +`@gamecult/cultmesh-browser` is the browser-safe CultMesh client boundary. It owns one WebSocket session for a stable Verse/authority-runtime target, retains explicit raw document leases across route replacement, and correlates typed CultNet operations with provider receipts. @@ -95,7 +95,7 @@ WebCrypto `crypto.subtle`; without WebCrypto it fails with endpoint CultMesh resolved is the only host the reader trusts. ```ts -import { fetchVerifiedContent } from "cultmesh-browser"; +import { fetchVerifiedContent } from "@gamecult/cultmesh-browser"; const bytes = await fetchVerifiedContent(endpoint, manifest, { signal: AbortSignal.timeout(30_000), diff --git a/packages/cultmesh-browser/package.json b/packages/cultmesh-browser/package.json index 6d52c9bf..480fb07e 100644 --- a/packages/cultmesh-browser/package.json +++ b/packages/cultmesh-browser/package.json @@ -1,5 +1,5 @@ { - "name": "cultmesh-browser", + "name": "@gamecult/cultmesh-browser", "version": "0.4.0", "description": "Browser-safe CultMesh document leases and typed operations over CultNet WebSocket transport.", "type": "module", @@ -42,11 +42,11 @@ }, "dependencies": { "@gamecult/cultcache-ts": "^0.14.0", - "cultnet-ts": "^0.3.0" + "@gamecult/cultnet-ts": "^0.3.0" }, "devDependencies": { "@types/node": "^24.0.0", - "cultmesh-ts": "^0.2.0", + "@gamecult/cultmesh-ts": "^0.2.0", "typescript": "^5.9.0" } } diff --git a/packages/cultmesh-browser/src/content.ts b/packages/cultmesh-browser/src/content.ts index d9e09b4f..c41fc904 100644 --- a/packages/cultmesh-browser/src/content.ts +++ b/packages/cultmesh-browser/src/content.ts @@ -3,8 +3,8 @@ // The response's declared length and coding are never consulted (Fetch decodes codings; the bytes received are the evidence). // Hashing, hex rules, record keys, the loopback rule and manifest validation belong to cultnet-ts and are only called here. -import { EMPTY_SHA256, chunkRecordKey, parseSha256Hex, sha256Hex, validateManifest, type CultMeshCdnArtifactManifest } from "cultnet-ts/content"; -import { isLoopbackEndpoint } from "cultnet-ts/authority"; +import { EMPTY_SHA256, chunkRecordKey, parseSha256Hex, sha256Hex, validateManifest, type CultMeshCdnArtifactManifest } from "@gamecult/cultnet-ts/content"; +import { isLoopbackEndpoint } from "@gamecult/cultnet-ts/authority"; export type CultMeshContentErrorCode = | "manifest-invalid" | "too-large" | "insecure-endpoint" | "webcrypto-unavailable" | "network" diff --git a/packages/cultmesh-browser/src/index.ts b/packages/cultmesh-browser/src/index.ts index beae9e36..3852615e 100644 --- a/packages/cultmesh-browser/src/index.ts +++ b/packages/cultmesh-browser/src/index.ts @@ -1,6 +1,6 @@ import { decodeDocumentPayload } from "@gamecult/cultcache-ts/document"; import type { AnyCultCacheDocumentDefinition, CultCacheDocumentValue } from "@gamecult/cultcache-ts"; -import { decodeWireValue, encodeWireValue } from "cultnet-ts/wire"; +import { decodeWireValue, encodeWireValue } from "@gamecult/cultnet-ts/wire"; import { encodeCultNetMessageForWire, parseCultNetMessage, @@ -16,7 +16,7 @@ import { type CultMeshVerseCatalogResponseMessage, type CultMeshSessionAcceptedMessage, type CultMeshSessionOpenMessage, -} from "cultnet-ts/contracts"; +} from "@gamecult/cultnet-ts/contracts"; import { base64ToBytes, bytesToBase64, @@ -31,7 +31,7 @@ import { type CultMeshAuthorityTrustMode, type CultMeshAuthorityTrustPolicy, type CultMeshP256PublicKey, -} from "cultnet-ts/authority"; +} from "@gamecult/cultnet-ts/authority"; export * from "./content.js"; export type CultMeshBrowserIdentity = CultMeshAuthorityIdentity; diff --git a/packages/cultmesh-browser/test/content.test.ts b/packages/cultmesh-browser/test/content.test.ts index cc0bb322..d4ec4d6e 100644 --- a/packages/cultmesh-browser/test/content.test.ts +++ b/packages/cultmesh-browser/test/content.test.ts @@ -11,7 +11,7 @@ import { gzipSync } from "node:zlib"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { build } from "esbuild"; -import { EMPTY_SHA256, packContent, sha256Hex, type CultMeshCdnArtifactManifest } from "cultnet-ts/content"; +import { EMPTY_SHA256, packContent, sha256Hex, type CultMeshCdnArtifactManifest } from "@gamecult/cultnet-ts/content"; import { CultMeshContentError, fetchVerifiedChunk, fetchVerifiedContent, type CultMeshContentErrorCode } from "../src/content.js"; diff --git a/packages/cultmesh-browser/test/cultmesh-browser.test.ts b/packages/cultmesh-browser/test/cultmesh-browser.test.ts index 1206b32d..17ffbc02 100644 --- a/packages/cultmesh-browser/test/cultmesh-browser.test.ts +++ b/packages/cultmesh-browser/test/cultmesh-browser.test.ts @@ -2,7 +2,7 @@ import assert from "node:assert/strict"; import test from "node:test"; import { codecs, defineDocumentType, documentPayloadFromBytes } from "@gamecult/cultcache-ts"; -import { decodeWireValue, encodeWireValue } from "cultnet-ts/wire"; +import { decodeWireValue, encodeWireValue } from "@gamecult/cultnet-ts/wire"; import { encodeCultNetMessageForWire, parseCultNetMessage, @@ -12,7 +12,7 @@ import { type CultNetRawDocumentRecord, type CultMeshVerseCatalogResponseMessage, type CultMeshSessionOpenMessage, -} from "cultnet-ts/contracts"; +} from "@gamecult/cultnet-ts/contracts"; import { CultMeshBrowserClient, diff --git a/packages/cultmesh-browser/test/interop/content-https-provider.ts b/packages/cultmesh-browser/test/interop/content-https-provider.ts index 50b5f223..f4b15aae 100644 --- a/packages/cultmesh-browser/test/interop/content-https-provider.ts +++ b/packages/cultmesh-browser/test/interop/content-https-provider.ts @@ -5,8 +5,8 @@ import { readFileSync } from "node:fs"; import { createServer } from "node:https"; import type { AddressInfo } from "node:net"; import { createInterface } from "node:readline"; -import { chunkRecordKey } from "cultnet-ts/content"; -import { createContentChunkHandler, createDirectoryChunkStore, toNodeListener, type ContentChunkAnswer, type ContentChunkRequest } from "cultmesh-ts/content"; +import { chunkRecordKey } from "@gamecult/cultnet-ts/content"; +import { createContentChunkHandler, createDirectoryChunkStore, toNodeListener, type ContentChunkAnswer, type ContentChunkRequest } from "@gamecult/cultmesh-ts/content"; const [storeDir, corruptDir, keyPem, certPem, escapeKey] = process.argv.slice(2) as [string, string, string, string, string]; const say = (line: object): void => { process.stdout.write(`${JSON.stringify(line)}\n`); }; diff --git a/packages/cultmesh-browser/test/interop/content-https-reader.ts b/packages/cultmesh-browser/test/interop/content-https-reader.ts index 693b22a8..fab02066 100644 --- a/packages/cultmesh-browser/test/interop/content-https-reader.ts +++ b/packages/cultmesh-browser/test/interop/content-https-reader.ts @@ -3,7 +3,7 @@ import { createHash } from "node:crypto"; import { readFileSync } from "node:fs"; import { join } from "node:path"; -import { decodeManifest } from "cultnet-ts/content"; +import { decodeManifest } from "@gamecult/cultnet-ts/content"; import { fetchVerifiedChunk, fetchVerifiedContent, CultMeshContentError } from "../../src/content.js"; const [endpointsJson, manifestsDir, only] = process.argv.slice(2) as [string, string, string | undefined]; diff --git a/packages/cultmesh-browser/test/interop/content-https.test.ts b/packages/cultmesh-browser/test/interop/content-https.test.ts index c86cf6ae..b26621eb 100644 --- a/packages/cultmesh-browser/test/interop/content-https.test.ts +++ b/packages/cultmesh-browser/test/interop/content-https.test.ts @@ -12,7 +12,7 @@ import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; -import { EMPTY_SHA256, encodeManifest, packContent } from "cultnet-ts/content"; +import { EMPTY_SHA256, encodeManifest, packContent } from "@gamecult/cultnet-ts/content"; const execFileAsync = promisify(execFile); const here = dirname(fileURLToPath(import.meta.url)); // dist-test/test/interop diff --git a/packages/cultmesh-browser/test/payload-type.test.ts b/packages/cultmesh-browser/test/payload-type.test.ts index 9cc95632..bdc6f250 100644 --- a/packages/cultmesh-browser/test/payload-type.test.ts +++ b/packages/cultmesh-browser/test/payload-type.test.ts @@ -2,8 +2,8 @@ import assert from "node:assert/strict"; import test from "node:test"; import { documentPayloadBytes, documentPayloadFromBytes, type DocumentPayload } from "@gamecult/cultcache-ts"; -import { decodeWireValue } from "cultnet-ts/wire"; -import type { CultNetRawDocumentRecord } from "cultnet-ts/contracts"; +import { decodeWireValue } from "@gamecult/cultnet-ts/wire"; +import type { CultNetRawDocumentRecord } from "@gamecult/cultnet-ts/contracts"; import { decodeCultNetPayload } from "../src/index.js"; diff --git a/packages/cultmesh-py/README.md b/packages/cultmesh-py/README.md index 4a048dcc..6a5339a5 100644 --- a/packages/cultmesh-py/README.md +++ b/packages/cultmesh-py/README.md @@ -7,7 +7,7 @@ game sessions with prediction reconciliation, committed simulation facts, the local TCP/RUDP server, and a package daemon. It depends on `cultcache-py` and `cultnet-py`, mirroring `cultmesh-rs` and -`cultmesh-ts`. The Verse catalog, peer exchange, authority lease, stream +`@gamecult/cultmesh-ts`. The Verse catalog, peer exchange, authority lease, stream negotiation, and simulation fact contracts are defined in `cultnet_py.cultmesh_contracts` and re-exported here. diff --git a/packages/cultmesh-ts/CHANGELOG.md b/packages/cultmesh-ts/CHANGELOG.md index 8b4e0ab5..9f66293c 100644 --- a/packages/cultmesh-ts/CHANGELOG.md +++ b/packages/cultmesh-ts/CHANGELOG.md @@ -1,12 +1,13 @@ # Changelog -All notable changes to `cultmesh-ts` are recorded here, following +All notable changes to `@gamecult/cultmesh-ts` are recorded here, following [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## Unreleased ### Breaking +- The package is renamed `cultmesh-ts` to `@gamecult/cultmesh-ts` (the repo already scopes `@gamecult/cultcache-ts`), so the unscoped name cannot be claimed by someone else. Import from `@gamecult/cultmesh-ts` and its subpaths; the API is unchanged. Depends on `@gamecult/cultnet-ts`. - `asSchemaAlias` is removed from `CultMeshDocumentHandle`, `CultMeshCollectionHandle` and their bound forms. Read the handle by the schema you mean. - `CultMeshDocumentCatalog` (`document`, `tryDocument`, `latest`, `replace`, `submitPrediction`, `authoritativeWriter`, `predictionWriter`, `watch`) and `CultMeshCollectionCatalog` (`collection`, `tryCollection`, `latest`, `watchChanges`) take `` and yield `CultMeshCatalogValue`: an explicit type argument wins, a CultCache document definition names its own value type, and a bare schema descriptor yields `unknown`. A value type is no longer inferred from a value or an expected result. The `parse` option is removed from these calls, and a lookup by type answers with that type's handle or none, not an alias of a compatible schema. - Adding a document or collection to a catalog under a schema id or schema name another type already holds is refused with `SchemaConflictError`. diff --git a/packages/cultmesh-ts/README.md b/packages/cultmesh-ts/README.md index 2f9eda51..9ec9e029 100644 --- a/packages/cultmesh-ts/README.md +++ b/packages/cultmesh-ts/README.md @@ -1,6 +1,6 @@ # CultMesh TS -`cultmesh-ts` is the TypeScript projection of CultMesh contracts for local +`@gamecult/cultmesh-ts` is the TypeScript projection of CultMesh contracts for local tools and browser-adjacent runtimes. The C# `GameCult.Mesh` package owns the native CultMesh stream implementation used by Mimir and Fensalir; this package mirrors the typed declaration and negotiation shapes without owning the hot @@ -21,7 +21,7 @@ WASM route. Runtime code can ask for a whole XY rect, then adapt it to the current wire schema at the edge: ```ts -import { CultMesh } from "cultmesh-ts"; +import { CultMesh } from "@gamecult/cultmesh-ts"; const viewport = CultMesh.rectFromBounds(-500, -300, 500, 300); const request = CultMesh.viewportRequest(viewport, [3, 8]); @@ -215,7 +215,7 @@ const stopFeed = viewportFeed.watch(request, "browser-starfire", render); ``` ```ts -import { CultMesh } from "cultmesh-ts"; +import { CultMesh } from "@gamecult/cultmesh-ts"; const verses = CultMesh.createVerseCatalog<{ verseId: string; label: string }>(); const stopWatching = verses.watch((verse) => { @@ -274,7 +274,7 @@ const noteShards = shards.list({ }); ``` -Schema and shard catalog factories delegate to `cultnet-ts`; the branded +Schema and shard catalog factories delegate to `@gamecult/cultnet-ts`; the branded `CultMesh` entrypoint does not create a second owner for schema discovery or topology truth. @@ -290,7 +290,7 @@ command transactions; the injected transport owns CultNet wire details. import { CultMeshProviderRudpTransport, CultMeshProviderSession, -} from "cultmesh-ts"; +} from "@gamecult/cultmesh-ts"; const odinTransport = new CultMeshProviderRudpTransport({ endpoint: "rudp://127.0.0.1:17871", @@ -347,7 +347,7 @@ evidence for the requested provider identity. Node runtimes can build the shared CultNet reliable-UDP transport from the branded CultMesh entrypoint while the transport semantics stay owned by -`cultnet-ts`: +`@gamecult/cultnet-ts`: ```ts const server = await CultMesh.createRudpServer("ts-server", 0x1020_3040); @@ -472,7 +472,7 @@ The split is deliberate: The mirrored API shape is `CultMeshStreamCatalog`: ```ts -import { CultMesh } from "cultmesh-ts"; +import { CultMesh } from "@gamecult/cultmesh-ts"; const streams = CultMesh.createStreamCatalog(); const stopWatchingStreams = streams.watch((stream) => { @@ -533,7 +533,7 @@ nonlocal fallback. ## Content Chunks over HTTP(S) -`cultmesh-ts/content` answers the C# HTTPS connector's request, `GET` or `HEAD` +`@gamecult/cultmesh-ts/content` answers the C# HTTPS connector's request, `GET` or `HEAD` `?chunkHash=&recordKey=`. The mount path is the host's; the handler reads only the query. Every chunk is hashed against its name before a byte is sent, so a corrupt or swapped file answers 500 and never leaves the @@ -550,7 +550,7 @@ host store may reuse its buffer). `onServerFault` reports every 500 (`hash-misma ```ts import { readFileSync } from "node:fs"; import { createServer } from "node:https"; -import { createContentChunkHandler, createDirectoryChunkStore, toNodeListener } from "cultmesh-ts/content"; +import { createContentChunkHandler, createDirectoryChunkStore, toNodeListener } from "@gamecult/cultmesh-ts/content"; const handler = createContentChunkHandler(createDirectoryChunkStore("/srv/content"), { onServerFault: fault => console.error("chunk fault", fault.kind, fault.chunkHash), diff --git a/packages/cultmesh-ts/package.json b/packages/cultmesh-ts/package.json index f6559fba..1ccaa421 100644 --- a/packages/cultmesh-ts/package.json +++ b/packages/cultmesh-ts/package.json @@ -1,5 +1,5 @@ { - "name": "cultmesh-ts", + "name": "@gamecult/cultmesh-ts", "version": "0.2.0", "description": "CultMesh local typed state and mesh catalog primitives for TypeScript runtimes.", "main": "dist/index.js", @@ -49,7 +49,7 @@ }, "dependencies": { "@gamecult/cultcache-ts": "^0.14.0", - "cultnet-ts": "^0.3.0", + "@gamecult/cultnet-ts": "^0.3.0", "koffi": "3.3.0" }, "devDependencies": { diff --git a/packages/cultmesh-ts/src/content.ts b/packages/cultmesh-ts/src/content.ts index a0f3f06d..b463f5b6 100644 --- a/packages/cultmesh-ts/src/content.ts +++ b/packages/cultmesh-ts/src/content.ts @@ -2,7 +2,7 @@ import { constants } from "node:fs"; import { open } from "node:fs/promises"; import type { IncomingMessage, ServerResponse } from "node:http"; import { join } from "node:path"; -import { DEFAULT_CHUNK_SIZE_BYTES, EMPTY_SHA256, chunkRecordKey, isNullOrWhiteSpaceCSharp, parseSha256Hex, sha256Hex } from "cultnet-ts/content"; +import { DEFAULT_CHUNK_SIZE_BYTES, EMPTY_SHA256, chunkRecordKey, isNullOrWhiteSpaceCSharp, parseSha256Hex, sha256Hex } from "@gamecult/cultnet-ts/content"; /** What the provider reads of a request. The path and the headers are never read: the route is the query alone. */ export interface ContentChunkRequest { method: string; url: string; } diff --git a/packages/cultmesh-ts/src/index.ts b/packages/cultmesh-ts/src/index.ts index 90331655..3e15eaac 100644 --- a/packages/cultmesh-ts/src/index.ts +++ b/packages/cultmesh-ts/src/index.ts @@ -1,5 +1,5 @@ import { createSocket, type Socket } from "node:dgram"; -import { decodeWireValue, encodeWireValue } from "cultnet-ts/wire"; +import { decodeWireValue, encodeWireValue } from "@gamecult/cultnet-ts/wire"; import { CultCache, @@ -40,7 +40,7 @@ import { type CultNetSchemaCatalogOptions, type CultNetSnapshotResponseRawMessage, type CultNetWireContract, -} from "cultnet-ts"; +} from "@gamecult/cultnet-ts"; import { CultMeshQuicRealtimeConnector, diff --git a/packages/cultmesh-ts/src/provider-rudp-transport.ts b/packages/cultmesh-ts/src/provider-rudp-transport.ts index fc0d75c0..b193825f 100644 --- a/packages/cultmesh-ts/src/provider-rudp-transport.ts +++ b/packages/cultmesh-ts/src/provider-rudp-transport.ts @@ -1,7 +1,7 @@ import { randomUUID } from "node:crypto"; import { createSocket, type Socket } from "node:dgram"; import { documentPayloadFromBytes } from "@gamecult/cultcache-ts"; -import { encodeWireValue } from "cultnet-ts/wire"; +import { encodeWireValue } from "@gamecult/cultnet-ts/wire"; import { CultNetPeer, CultNetRudpSocketTransportConnection, @@ -9,7 +9,7 @@ import { type CultNetOperationRequestMessage, type CultNetOperationResponseMessage, type CultNetRawDocumentRecord, -} from "cultnet-ts"; +} from "@gamecult/cultnet-ts"; import type { CultMeshProviderCommand, diff --git a/packages/cultmesh-ts/src/provider-session-broker.ts b/packages/cultmesh-ts/src/provider-session-broker.ts index 75240a14..c427b83a 100644 --- a/packages/cultmesh-ts/src/provider-session-broker.ts +++ b/packages/cultmesh-ts/src/provider-session-broker.ts @@ -1,11 +1,11 @@ import { encodeDocumentPayload } from "@gamecult/cultcache-ts"; -import { encodeWireValue } from "cultnet-ts/wire"; +import { encodeWireValue } from "@gamecult/cultnet-ts/wire"; import type { CultNetDocumentPutRawMessage, CultNetOperationRequestMessage, CultNetOperationResponseMessage, CultNetRawDocumentRecord, -} from "cultnet-ts"; +} from "@gamecult/cultnet-ts"; import type { CultMeshRudpServerSession } from "./index"; import { diff --git a/packages/cultmesh-ts/src/provider-session-wire.ts b/packages/cultmesh-ts/src/provider-session-wire.ts index d4884694..3cffb13e 100644 --- a/packages/cultmesh-ts/src/provider-session-wire.ts +++ b/packages/cultmesh-ts/src/provider-session-wire.ts @@ -1,6 +1,6 @@ import { decodeDocumentPayload, defineDocumentType } from "@gamecult/cultcache-ts"; -import { decodeWireValue, encodeWireValue } from "cultnet-ts/wire"; -import type { CultNetRawDocumentRecord } from "cultnet-ts"; +import { decodeWireValue, encodeWireValue } from "@gamecult/cultnet-ts/wire"; +import type { CultNetRawDocumentRecord } from "@gamecult/cultnet-ts"; export const CULTMESH_PROVIDER_SESSION_SERVICE_ID = "gamecult.mesh.provider_session"; diff --git a/packages/cultmesh-ts/src/realtime-quic.ts b/packages/cultmesh-ts/src/realtime-quic.ts index c368c96f..32b500d2 100644 --- a/packages/cultmesh-ts/src/realtime-quic.ts +++ b/packages/cultmesh-ts/src/realtime-quic.ts @@ -21,8 +21,8 @@ import { trimCSharp, verifyAuthorityRoute, type CultMeshAuthorityTrustPolicy, -} from "cultnet-ts"; -import type { CultMeshVerseDescriptorMessage } from "cultnet-ts/contracts"; +} from "@gamecult/cultnet-ts"; +import type { CultMeshVerseDescriptorMessage } from "@gamecult/cultnet-ts/contracts"; import { decodeRealtimeFrame, diff --git a/packages/cultmesh-ts/src/realtime-wire.ts b/packages/cultmesh-ts/src/realtime-wire.ts index 9fc99789..e87df641 100644 --- a/packages/cultmesh-ts/src/realtime-wire.ts +++ b/packages/cultmesh-ts/src/realtime-wire.ts @@ -16,7 +16,7 @@ // agree with itself. import { documentPayloadBytes, documentPayloadFromBytes, type DocumentPayload } from "@gamecult/cultcache-ts"; -import { isNullOrWhiteSpaceCSharp } from "cultnet-ts"; +import { isNullOrWhiteSpaceCSharp } from "@gamecult/cultnet-ts"; /** `CultMeshRealtimeWireProtocol.ApplicationProtocolName`: the QUIC ALPN. */ export const CULTMESH_REALTIME_ALPN = "cultmesh-state-v1"; @@ -70,7 +70,7 @@ const INT64_MAX = 2n ** 63n - 1n; * `CultMeshRealtimeFrame.Validate`. The C# reading of "non-empty" is * `string.IsNullOrWhiteSpace`, whose whitespace set is not * `String.prototype.trim`'s, so this borrows the one TypeScript implementation - * of that set from `cultnet-ts` rather than keeping a second reading here. + * of that set from `@gamecult/cultnet-ts` rather than keeping a second reading here. * * The range check has no C# counterpart because `long` gives it for free; a * `bigint` does not, and a `DataView` would otherwise throw a bare `RangeError` diff --git a/packages/cultmesh-ts/test/codec-documents.test.ts b/packages/cultmesh-ts/test/codec-documents.test.ts index 58c05568..eadc1d2f 100644 --- a/packages/cultmesh-ts/test/codec-documents.test.ts +++ b/packages/cultmesh-ts/test/codec-documents.test.ts @@ -19,7 +19,7 @@ import { CultNetDocumentRegistry, defineCultNetDocumentBinding, type CultNetPeer, -} from "cultnet-ts"; +} from "@gamecult/cultnet-ts"; import { CultMesh, cultMeshDocumentFromSingleFile, type CultMeshRudpDocumentPut } from "../src/index"; // A document written in the reference's positional shape, and a plain map document beside it. Every mesh reader of a diff --git a/packages/cultmesh-ts/test/content.test.ts b/packages/cultmesh-ts/test/content.test.ts index 9662bb86..7e3a357a 100644 --- a/packages/cultmesh-ts/test/content.test.ts +++ b/packages/cultmesh-ts/test/content.test.ts @@ -8,7 +8,7 @@ import { join } from "node:path"; import test, { mock } from "node:test"; import { execFileSync } from "node:child_process"; -import { EMPTY_SHA256, chunkRecordKey, packContent, sha256Hex } from "cultnet-ts/content"; +import { EMPTY_SHA256, chunkRecordKey, packContent, sha256Hex } from "@gamecult/cultnet-ts/content"; import { closeSync, constants as fsConstants, openSync, symlinkSync } from "node:fs"; import { diff --git a/packages/cultmesh-ts/test/cultmesh.test.ts b/packages/cultmesh-ts/test/cultmesh.test.ts index eccb4545..924ab697 100644 --- a/packages/cultmesh-ts/test/cultmesh.test.ts +++ b/packages/cultmesh-ts/test/cultmesh.test.ts @@ -4,7 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import test from "node:test"; import assert from "node:assert/strict"; -import { decodeWireValue, encodeWireValue } from "cultnet-ts/wire"; +import { decodeWireValue, encodeWireValue } from "@gamecult/cultnet-ts/wire"; import { z } from "zod"; import { SchemaConflictError, defineDocumentType, documentPayloadBytes, documentPayloadFromBytes } from "@gamecult/cultcache-ts"; import { @@ -19,7 +19,7 @@ import { type CultNetRudpPacket, type CultNetMessage, type CultNetSnapshotResponseRawMessage, -} from "cultnet-ts"; +} from "@gamecult/cultnet-ts"; import { CultMesh, CultMeshRudpUnservableDocumentError } from "../src/index"; // A record's payload as the bytes a peer would put on the wire. diff --git a/packages/cultmesh-ts/test/interop/cultmesh-quic-peer.ts b/packages/cultmesh-ts/test/interop/cultmesh-quic-peer.ts index e7e04c1a..f22d5af3 100644 --- a/packages/cultmesh-ts/test/interop/cultmesh-quic-peer.ts +++ b/packages/cultmesh-ts/test/interop/cultmesh-quic-peer.ts @@ -18,8 +18,8 @@ // harness can dial a peer late and exercise Cut C's retained-state seed // instead of the provider always tearing down the moment it is done sending. // -// Neither mode imports `cultmesh-ts` itself (the harness drives processes, -// and the dependency direction is `cultmesh-ts` -> `cultnet-ts`, never the +// Neither mode imports `@gamecult/cultmesh-ts` itself (the harness drives processes, +// and the dependency direction is `@gamecult/cultmesh-ts` -> `@gamecult/cultnet-ts`, never the // reverse); this script reaches the provider/connector through its own // package's `src/`. diff --git a/packages/cultmesh-ts/test/payload-type.test.ts b/packages/cultmesh-ts/test/payload-type.test.ts index cdba3505..f04541da 100644 --- a/packages/cultmesh-ts/test/payload-type.test.ts +++ b/packages/cultmesh-ts/test/payload-type.test.ts @@ -8,8 +8,8 @@ import { documentPayloadFromBytes, type DocumentPayload, } from "@gamecult/cultcache-ts"; -import { decodeWireValue } from "cultnet-ts/wire"; -import type { CultNetRawDocumentRecord } from "cultnet-ts"; +import { decodeWireValue } from "@gamecult/cultnet-ts/wire"; +import type { CultNetRawDocumentRecord } from "@gamecult/cultnet-ts"; import type { CultMeshRudpDocumentPut } from "../src/index"; import { decodeRealtimeFrame, encodeRealtimeFrame, type CultMeshRealtimeFrame } from "../src/realtime-wire"; diff --git a/packages/cultmesh-ts/test/provider-rudp-transport.test.ts b/packages/cultmesh-ts/test/provider-rudp-transport.test.ts index e9a9f1c5..d73ec1bd 100644 --- a/packages/cultmesh-ts/test/provider-rudp-transport.test.ts +++ b/packages/cultmesh-ts/test/provider-rudp-transport.test.ts @@ -2,7 +2,7 @@ import test from "node:test"; import assert from "node:assert/strict"; import { createSocket } from "node:dgram"; import { documentPayloadBytes } from "@gamecult/cultcache-ts"; -import { decodeWireValue } from "cultnet-ts/wire"; +import { decodeWireValue } from "@gamecult/cultnet-ts/wire"; import { CultNetDocumentRegistry, CultNetRudpSession, @@ -10,7 +10,7 @@ import { encodeRudpPacket, type CultNetOperationRequestMessage, type CultNetRudpPacket, -} from "cultnet-ts"; +} from "@gamecult/cultnet-ts"; import { CultMesh, diff --git a/packages/cultmesh-ts/test/provider-session-wire.test.ts b/packages/cultmesh-ts/test/provider-session-wire.test.ts index ee196f41..4905f161 100644 --- a/packages/cultmesh-ts/test/provider-session-wire.test.ts +++ b/packages/cultmesh-ts/test/provider-session-wire.test.ts @@ -2,7 +2,7 @@ import test from "node:test"; import assert from "node:assert/strict"; import { documentPayloadBytes, documentPayloadFromBytes, encodeDocumentPayload, type DocumentPayload } from "@gamecult/cultcache-ts"; -import { encodeWireValue } from "cultnet-ts/wire"; +import { encodeWireValue } from "@gamecult/cultnet-ts/wire"; import { cultMeshProviderSessionSchemas, diff --git a/packages/cultmesh-ts/test/realtime-quic-consumer.test.ts b/packages/cultmesh-ts/test/realtime-quic-consumer.test.ts index acf06a41..f48dec74 100644 --- a/packages/cultmesh-ts/test/realtime-quic-consumer.test.ts +++ b/packages/cultmesh-ts/test/realtime-quic-consumer.test.ts @@ -21,8 +21,8 @@ import { type CultMeshAuthorityRouteView, type CultMeshAuthorityTrustPolicy, type CultMeshP256PublicKey, -} from "cultnet-ts"; -import type { CultMeshAuthorityRouteMessage, CultMeshVerseDescriptorMessage } from "cultnet-ts/contracts"; +} from "@gamecult/cultnet-ts"; +import type { CultMeshAuthorityRouteMessage, CultMeshVerseDescriptorMessage } from "@gamecult/cultnet-ts/contracts"; import { CultMeshQuicNativeRuntime, diff --git a/packages/cultmesh-ts/test/realtime-quic-session-manager.test.ts b/packages/cultmesh-ts/test/realtime-quic-session-manager.test.ts index e1843466..f636fc95 100644 --- a/packages/cultmesh-ts/test/realtime-quic-session-manager.test.ts +++ b/packages/cultmesh-ts/test/realtime-quic-session-manager.test.ts @@ -14,8 +14,8 @@ import assert from "node:assert/strict"; import test from "node:test"; -import type { CultMeshAuthorityTrustPolicy } from "cultnet-ts"; -import type { CultMeshAuthorityRouteMessage, CultMeshVerseDescriptorMessage } from "cultnet-ts/contracts"; +import type { CultMeshAuthorityTrustPolicy } from "@gamecult/cultnet-ts"; +import type { CultMeshAuthorityRouteMessage, CultMeshVerseDescriptorMessage } from "@gamecult/cultnet-ts/contracts"; import { CultMeshQuicRealtimeSessionManager, diff --git a/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts b/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts index 713a3bee..c26c5a82 100644 --- a/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts +++ b/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts @@ -6,14 +6,14 @@ import assert from "node:assert/strict"; import { createSocket, type RemoteInfo, type Socket } from "node:dgram"; import test from "node:test"; import { documentPayloadFromBytes } from "@gamecult/cultcache-ts"; -import { encodeWireValue } from "cultnet-ts/wire"; +import { encodeWireValue } from "@gamecult/cultnet-ts/wire"; import { CultNetDocumentRegistry, CultNetRudpSession, decodeRudpPacket, encodeRudpPacket, type CultNetPeer, -} from "cultnet-ts"; +} from "@gamecult/cultnet-ts"; import { CultMesh } from "../src/index"; async function waitFor(predicate: () => boolean, description: string): Promise { diff --git a/packages/cultnet-py/README.md b/packages/cultnet-py/README.md index cca39ba7..98a6db93 100644 --- a/packages/cultnet-py/README.md +++ b/packages/cultnet-py/README.md @@ -8,7 +8,7 @@ wire contracts that both a node and a bare peer have to agree on. It depends on `cultcache-py` and `msgpack`; AES-GCM helpers on `CultNetSecret` light up with the optional `crypto` extra. `cultmesh-py` builds on it. The -layering mirrors `cultnet-rs` and `cultnet-ts`. +layering mirrors `cultnet-rs` and `@gamecult/cultnet-ts`. ## Example diff --git a/packages/cultnet-rs/README.md b/packages/cultnet-rs/README.md index 48ac2dd3..c356b97b 100644 --- a/packages/cultnet-rs/README.md +++ b/packages/cultnet-rs/README.md @@ -1,6 +1,6 @@ # CultNet RS -`cultnet-rs` is the Rust sibling of `cultnet-ts`: typed MessagePack messages, +`cultnet-rs` is the Rust sibling of `@gamecult/cultnet-ts`: typed MessagePack messages, 4-byte length-prefixed direct-pipe framing, CultLib-shaped security helpers, and CultCache replication without making callers paw through raw envelopes like we lost a bet. @@ -100,7 +100,7 @@ game-session authority remain separate responsibilities. ## Local Fast Lane -`cultnet-rs` now mirrors the raw replication seam from `cultnet-ts`: +`cultnet-rs` now mirrors the raw replication seam from `@gamecult/cultnet-ts`: - `cultnet.document_put_raw.v0` - `cultnet.snapshot_response_raw.v0` diff --git a/packages/cultnet-ts/CHANGELOG.md b/packages/cultnet-ts/CHANGELOG.md index a5b8bc4e..b2a16456 100644 --- a/packages/cultnet-ts/CHANGELOG.md +++ b/packages/cultnet-ts/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -All notable changes to `cultnet-ts` are recorded here, following +All notable changes to `@gamecult/cultnet-ts` are recorded here, following [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## Unreleased @@ -13,6 +13,7 @@ All notable changes to `cultnet-ts` are recorded here, following ### Breaking +- The package is renamed `cultnet-ts` to `@gamecult/cultnet-ts` (the repo already scopes `@gamecult/cultcache-ts`), so the unscoped name cannot be claimed by someone else. Import from `@gamecult/cultnet-ts` and its subpaths; the API is unchanged. - `CultNetErrorMessage` carries `routingHint`, `code` and `details` (null when absent), as the reference writes them and `cultnet.error.schema.json` requires; the interface previously dropped them. New `CultNetErrorCode`, `CultNetErrorDetails` and `CultNetShardRoutingHint` types. - `CultMeshCdnChunkRef` carries `id` (slot 4), the reference's element id. `encodeManifest` writes five-slot chunk references and `packContent` sets the decimal offset as the id, as the reference does; a manifest written before ids reads `""` and is rewritten with the slot. Content vectors regenerated from the reference. diff --git a/packages/cultnet-ts/README.md b/packages/cultnet-ts/README.md index acbf5a6a..1fb37a3f 100644 --- a/packages/cultnet-ts/README.md +++ b/packages/cultnet-ts/README.md @@ -1,6 +1,6 @@ # CultNet TS -`cultnet-ts` is the TypeScript port of the useful shape of GameCult's +`@gamecult/cultnet-ts` is the TypeScript port of the useful shape of GameCult's networking layer: typed MessagePack messages, direct-pipe framing, CultLib-style security/session helpers, and state replication that hands typed values into a `CultCache` instead of making consumer code paw raw database envelopes. @@ -42,7 +42,7 @@ We want: So the split is: - `cultcache-ts`: typed local persistence -- `cultnet-ts`: typed transport, auth, and state replication +- `@gamecult/cultnet-ts`: typed transport, auth, and state replication Same family. Different organs. Nobody needs a second spine growing out of the transport layer. @@ -80,7 +80,7 @@ CultNet without inventing a second ontology first. ## Generated Swarm Contracts -`cultnet-ts` can consume canonical JSON Schema files and generate: +`@gamecult/cultnet-ts` can consume canonical JSON Schema files and generate: - TypeScript contract classes/interfaces - runtime schema verifiers @@ -111,7 +111,7 @@ which exports: ## Wire Contracts -`cultnet-ts` does not sniff random payloads and pretend that counts as rigor. +`@gamecult/cultnet-ts` does not sniff random payloads and pretend that counts as rigor. You choose the contract explicitly. Current wire-contract options: @@ -129,7 +129,7 @@ away from the C# lineage. ## Content -`cultnet-ts/content` owns the TypeScript shapes of the CultMesh CDN +`@gamecult/cultnet-ts/content` owns the TypeScript shapes of the CultMesh CDN (`src/content.ts`, the twin of `CultMeshCdn.cs`): the artifact manifest and chunk reference, their MessagePack form, hash normalization, both record-key grammars, SHA-256 hashing through WebCrypto, and `packContent`. It is browser-safe: it @@ -160,7 +160,7 @@ imports no `node:` module, so a browser bundle can take it. ## Local Fast Lane For direct-pipe neighbors that already share the same CultCache formatter -contract, `cultnet-ts` now has a raw replication lane: +contract, `@gamecult/cultnet-ts` now has a raw replication lane: - `cultnet.document_put_raw.v0` - `cultnet.snapshot_response_raw.v0` @@ -185,7 +185,7 @@ cuts out the dumbest part of the trip. Apps sharing a CultNet space should be able to ask each other what they can exchange without resorting to folklore. -`cultnet-ts` now includes: +`@gamecult/cultnet-ts` now includes: - `cultnet.schema_catalog_request.v0` - `cultnet.schema_catalog_response.v0` @@ -232,7 +232,7 @@ import { createTcpFramedCultNetPeer, defineCultNetDocumentBinding, ghostlightAgentStateGeneratedContract, -} from "cultnet-ts"; +} from "@gamecult/cultnet-ts"; const ghostlightStateDefinition = defineDocumentType({ type: "ghostlight.agent-state", @@ -270,7 +270,7 @@ That means a TypeScript runtime can follow the same cryptographic and session story as the C# version instead of inventing one more house religion around "trust me, this pipe is probably fine." -For the legacy `gamecult.networking.v0` contract, `cultnet-ts` normalizes the +For the legacy `gamecult.networking.v0` contract, `@gamecult/cultnet-ts` normalizes the encrypted binary fields as base64url strings in the ergonomic TS surface and maps them back to the exact C# union payload shape at the wire boundary. diff --git a/packages/cultnet-ts/package.json b/packages/cultnet-ts/package.json index 1fc6abed..ebafac78 100644 --- a/packages/cultnet-ts/package.json +++ b/packages/cultnet-ts/package.json @@ -1,5 +1,5 @@ { - "name": "cultnet-ts", + "name": "@gamecult/cultnet-ts", "version": "0.3.0", "description": "CultNet-style typed direct-pipe messaging and state replication for TypeScript runtimes.", "main": "dist/index.js", diff --git a/packages/cultnet-ts/src/cultmesh-authority.ts b/packages/cultnet-ts/src/cultmesh-authority.ts index c260cf70..9d1c7438 100644 --- a/packages/cultnet-ts/src/cultmesh-authority.ts +++ b/packages/cultnet-ts/src/cultmesh-authority.ts @@ -39,8 +39,8 @@ export interface CultMeshAuthorityRouteCertificate { /** * Maps an advertised route's wire-shaped certificate to the view * `verifyAuthorityRoute` and the transports consume. Shared by the QUIC - * realtime session manager (`cultmesh-ts`) and the browser WebSocket client - * (`cultmesh-browser`), which advertise the same `CultMeshAuthorityRouteMessage` + * realtime session manager (`@gamecult/cultmesh-ts`) and the browser WebSocket client + * (`@gamecult/cultmesh-browser`), which advertise the same `CultMeshAuthorityRouteMessage` * certificate shape and were computing this mapping independently. */ export function routeCertificateView( diff --git a/packages/cultnet-ts/test/first-call.test.ts b/packages/cultnet-ts/test/first-call.test.ts index b389ca95..977a6e5b 100644 --- a/packages/cultnet-ts/test/first-call.test.ts +++ b/packages/cultnet-ts/test/first-call.test.ts @@ -6,7 +6,7 @@ import test from "node:test"; // The TypeScript CultNet first-call snippet that docs/start/README.md shows: define a typed document, send it // over a loopback TCP pipe as a cultnet.document_put_raw.v0 message, receive it into a second cache. The docs show // the region between the `snippet` markers verbatim, except that the package import reads -// "cultnet-ts" where this file reads "../src"; the docs test below fails when they differ. +// "@gamecult/cultnet-ts" where this file reads "../src"; the docs test below fails when they differ. // snippet:begin import { createServer, connect } from "node:net"; import { z } from "zod"; @@ -72,7 +72,7 @@ test("docs show the tested snippet", async () => { const snippet = source .split("// snippet:begin\n")[1] .split("// snippet:end\n")[0] - .replace('"../src"', '"cultnet-ts"'); + .replace('"../src"', '"@gamecult/cultnet-ts"'); const docs = (await readFile(join(__dirname, "..", "..", "..", "..", "docs", "start", "README.md"), "utf8")).replace(/\r\n/g, "\n"); assert.ok(docs.includes("```ts\n" + snippet + "```"), "docs/start/README.md no longer shows test/first-call.test.ts"); }); diff --git a/packages/cultnet-ts/test/idunn-runtime-authority.test.ts b/packages/cultnet-ts/test/idunn-runtime-authority.test.ts index cf1e4a26..2b45e3b1 100644 --- a/packages/cultnet-ts/test/idunn-runtime-authority.test.ts +++ b/packages/cultnet-ts/test/idunn-runtime-authority.test.ts @@ -461,13 +461,13 @@ function loadUnderDgramStub(modulePath: string): string { } test("the idunn-runtime subpath loads the signer and loader without loading dgram; the package root does load it", () => { - const subpath = require.resolve("cultnet-ts/idunn-runtime"); + const subpath = require.resolve("@gamecult/cultnet-ts/idunn-runtime"); assert.equal(loadUnderDgramStub(subpath), "loaded"); - assert.equal(loadUnderDgramStub(require.resolve("cultnet-ts")), "failed: dgram was loaded"); + assert.equal(loadUnderDgramStub(require.resolve("@gamecult/cultnet-ts")), "failed: dgram was loaded"); const exported = require(subpath) as Record; assert.equal(typeof exported.createIdunnRuntimeSigner, "function"); assert.equal(typeof exported.loadIdunnRuntimeAuthorityFromEnvironment, "function"); assert.equal(exported.createIdunnRuntimePresencePublisher, undefined); assert.equal(exported.signIdunnRuntimePresence, undefined); - assert.equal((require("cultnet-ts") as Record).signIdunnRuntimePresence, undefined); + assert.equal((require("@gamecult/cultnet-ts") as Record).signIdunnRuntimePresence, undefined); }); diff --git a/samples/eve-browser-network/README.md b/samples/eve-browser-network/README.md index 445bfb9d..62a7980b 100644 --- a/samples/eve-browser-network/README.md +++ b/samples/eve-browser-network/README.md @@ -4,7 +4,7 @@ This is the real-process continuation of `samples/eve-two-runtime`: - one C# provider owns a durable `.cc` counter, the Eve surface, command idempotency, and canonical receipts; -- one real Chromium page leases both documents through `cultmesh-browser` and +- one real Chromium page leases both documents through `@gamecult/cultmesh-browser` and lowers the provider-owned Eve surface; - one retained C# headless client discovers the provider through the same Odin identity, observes the typed counter and canonical receipt ids, survives the diff --git a/samples/eve-browser-network/browser/main.ts b/samples/eve-browser-network/browser/main.ts index fc57e364..76f8817a 100644 --- a/samples/eve-browser-network/browser/main.ts +++ b/samples/eve-browser-network/browser/main.ts @@ -5,7 +5,7 @@ import { CultMeshBrowserOdinRendezvous, decodeCultNetOperationPayload, decodeCultNetPayload, -} from "cultmesh-browser"; +} from "@gamecult/cultmesh-browser"; import { defineDocumentType } from "@gamecult/cultcache-ts/document"; // A payload is read through the definition that says how its bytes are laid out. These are plain-map definitions: the diff --git a/samples/eve-two-runtime/sample.mjs b/samples/eve-two-runtime/sample.mjs index 93ed1dc8..e7dc56c5 100644 --- a/samples/eve-two-runtime/sample.mjs +++ b/samples/eve-two-runtime/sample.mjs @@ -5,7 +5,7 @@ import { join } from "node:path"; import { JSDOM } from "jsdom"; import { z } from "zod"; import { defineDocumentType } from "@gamecult/cultcache-ts"; -import { CultMesh } from "cultmesh-ts"; +import { CultMesh } from "@gamecult/cultmesh-ts"; import { parseEveCommandReceipt, parseEveSurfaceDocument, diff --git a/scripts/run-typescript-workspaces.mjs b/scripts/run-typescript-workspaces.mjs index 0ef34005..3ed4514c 100644 --- a/scripts/run-typescript-workspaces.mjs +++ b/scripts/run-typescript-workspaces.mjs @@ -11,7 +11,7 @@ const npmCli = process.env.npm_execpath ?? // moves when a package is scoped: `8cb3b72` renamed `cultcache-ts` to // `@gamecult/cultcache-ts` and left this entry behind, which broke `build:ts` and // `test:ts` for every workspace from that commit on. -const workspaces = ["@gamecult/cultcache-ts", "cultnet-ts", "cultmesh-ts", "cultmesh-browser"]; +const workspaces = ["@gamecult/cultcache-ts", "@gamecult/cultnet-ts", "@gamecult/cultmesh-ts", "@gamecult/cultmesh-browser"]; const childEnv = process.platform === "win32" ? { ...process.env, diff --git a/scripts/test-typescript-package-closure.mjs b/scripts/test-typescript-package-closure.mjs index 6b0d7661..1fb6c140 100644 --- a/scripts/test-typescript-package-closure.mjs +++ b/scripts/test-typescript-package-closure.mjs @@ -79,8 +79,8 @@ try { const assert = require("node:assert/strict"); const cache = require("@gamecult/cultcache-ts"); const documentEntry = require("@gamecult/cultcache-ts/document"); -const net = require("cultnet-ts"); -const mesh = require("cultmesh-ts"); +const net = require("@gamecult/cultnet-ts"); +const mesh = require("@gamecult/cultmesh-ts"); assert.equal(typeof cache.defineDocumentType, "function"); // The document owner is also an entry of its own, which the browser client imports without the Node-only cache. assert.equal(documentEntry.decodeDocumentPayload, cache.decodeDocumentPayload); @@ -147,8 +147,8 @@ assert.deepEqual(mesh.CultMesh.vec2(3, 4), { x: 3, y: 4 }); writeFileSync(join(consumerRoot, "browser-runtime-smoke.mjs"), ` import assert from "node:assert/strict"; -import { CultMeshBrowserClient, decodeCultNetPayload } from "cultmesh-browser"; -import { parseCultNetMessage } from "cultnet-ts/contracts"; +import { CultMeshBrowserClient, decodeCultNetPayload } from "@gamecult/cultmesh-browser"; +import { parseCultNetMessage } from "@gamecult/cultnet-ts/contracts"; assert.equal(typeof CultMeshBrowserClient.connect, "function"); assert.equal(typeof decodeCultNetPayload, "function"); assert.equal(typeof parseCultNetMessage, "function"); @@ -159,14 +159,14 @@ assert.equal("putDocument" in CultMeshBrowserClient.prototype, false); writeFileSync(join(consumerRoot, "types-smoke.ts"), ` import { defineDocumentType } from "@gamecult/cultcache-ts"; import { decodeDocumentPayload, type DocumentPayload } from "@gamecult/cultcache-ts/document"; -import { CultNetPeer } from "cultnet-ts"; +import { CultNetPeer } from "@gamecult/cultnet-ts"; import { CultMesh, CultMeshProviderSession, type CultMeshProviderIdentity, type CultMeshProviderTransport, type CultMeshVec2, -} from "cultmesh-ts"; +} from "@gamecult/cultmesh-ts"; const point: CultMeshVec2 = CultMesh.vec2(3, 4); const defineDocument: typeof defineDocumentType = defineDocumentType; @@ -209,7 +209,7 @@ import { CultMeshBrowserClient, type CultMeshBrowserIdentity, type CultMeshRawDocumentLeaseOptions, -} from "cultmesh-browser"; +} from "@gamecult/cultmesh-browser"; const identity: CultMeshBrowserIdentity = { verseId: "smoke", diff --git a/scripts/verify-eve-browser-network.mjs b/scripts/verify-eve-browser-network.mjs index 401e4afb..998ee52c 100644 --- a/scripts/verify-eve-browser-network.mjs +++ b/scripts/verify-eve-browser-network.mjs @@ -46,8 +46,8 @@ try { const moduleMap = { "@gamecult/eve-contracts": join(eveRoot, "packages", "eve-contracts", "src", "index.ts"), "@gamecult/eve-browser-lowering": join(eveRoot, "packages", "eve-browser-lowering", "src", "index.ts"), - "cultmesh-browser": join(repoRoot, "packages", "cultmesh-browser", "src", "index.ts"), - "cultnet-ts/contracts": join(repoRoot, "packages", "cultnet-ts", "src", "contracts.ts"), + "@gamecult/cultmesh-browser": join(repoRoot, "packages", "cultmesh-browser", "src", "index.ts"), + "@gamecult/cultnet-ts/contracts": join(repoRoot, "packages", "cultnet-ts", "src", "contracts.ts"), }; const sampleTsconfig = join(workRoot, "tsconfig.json"); await writeFile(sampleTsconfig, JSON.stringify({ diff --git a/src/GameCult.Mesh/docs/browser-verse-transport.md b/src/GameCult.Mesh/docs/browser-verse-transport.md index e825a0d4..663acc06 100644 --- a/src/GameCult.Mesh/docs/browser-verse-transport.md +++ b/src/GameCult.Mesh/docs/browser-verse-transport.md @@ -19,14 +19,14 @@ standalone. The current substrate has a browser client boundary and local live proof, but not a complete public Verse proof: -- `cultmesh-ts` is a Node package and imports `node:dgram` at its root; -- `cultnet-ts/contracts` is the browser-safe contract entrypoint; the package +- `@gamecult/cultmesh-ts` is a Node package and imports `node:dgram` at its root; +- `@gamecult/cultnet-ts/contracts` is the browser-safe contract entrypoint; the package root still includes Node transport, filesystem, crypto, stream, and UDP modules and must not enter browser bundles; - C# and TypeScript now share `cultnet.database_subscribe.v0`, `cultnet.database_unsubscribe.v0`, and `cultnet.database_change_raw.v0`; - `cultmesh-browser` consumes those messages as explicit document leases; -- `cultmesh-browser` owns stable identity, route refresh, resubscription, and + `@gamecult/cultmesh-browser` consumes those messages as explicit document leases; +- `@gamecult/cultmesh-browser` owns stable identity, route refresh, resubscription, and typed operation correlation without exposing direct document writes; - `CultMeshBrowserOdinRendezvous` resolves stable Verse/provider identity with `cultmesh.verse_catalog_request.v0` and @@ -112,8 +112,8 @@ Shared paths: Deletion line: - the jsdom checkpoint remains labeled as a Node-hosted DOM proof; -- browser applications import `cultmesh-browser` and - `cultnet-ts/contracts`, never the Node-only package roots; +- browser applications import `@gamecult/cultmesh-browser` and + `@gamecult/cultnet-ts/contracts`, never the Node-only package roots; - subscription messages remain one shared schema contract across the C# and TypeScript catalogs; - the real provider adapter must consume the canonical WebSocket message lane; diff --git a/src/GameCult.Mesh/docs/getting-started/03-publish-an-eve-surface.md b/src/GameCult.Mesh/docs/getting-started/03-publish-an-eve-surface.md index 49a23f79..768a31a3 100644 --- a/src/GameCult.Mesh/docs/getting-started/03-publish-an-eve-surface.md +++ b/src/GameCult.Mesh/docs/getting-started/03-publish-an-eve-surface.md @@ -101,7 +101,7 @@ The real-process C# sample is then rebuilt from that same temporary NuGet feed. The source project references in its `.csproj` are contributor convenience only and are disabled by the checkpoint; they are not the consumer contract. -The first layer builds and packs `cultcache-ts`, `cultnet-ts`, `cultmesh-ts`, +The first layer builds and packs `cultcache-ts`, `@gamecult/cultnet-ts`, `@gamecult/cultmesh-ts`, `@gamecult/eve-contracts`, and `@gamecult/eve-browser-lowering`, installs the tarballs into an empty temporary consumer, and runs `samples/eve-two-runtime/sample.mjs`. Eve's generated runtime validators reject diff --git a/src/GameCult.Mesh/docs/reliability-migration-plan.md b/src/GameCult.Mesh/docs/reliability-migration-plan.md index 47e1b190..6763080f 100644 --- a/src/GameCult.Mesh/docs/reliability-migration-plan.md +++ b/src/GameCult.Mesh/docs/reliability-migration-plan.md @@ -574,7 +574,7 @@ registration, renewal, stable publication order, reconnect and replay, duplicate receipt reuse, explicit withdrawal, and the negative stop-during- registration timeline, durable receipt replay, observer isolation, conflicting command IDs, and exception-safe shutdown. The empty-consumer tarball smoke proves the session is -present in the installable `cultmesh-ts` package. A live Odin transport adapter +present in the installable `@gamecult/cultmesh-ts` package. A live Odin transport adapter and the VoidBot migration remain required for the production proof. ## Phase 3: Authority as a Mandatory Boundary diff --git a/src/GameCult.Networking/README.md b/src/GameCult.Networking/README.md index 7347d145..dea0e411 100644 --- a/src/GameCult.Networking/README.md +++ b/src/GameCult.Networking/README.md @@ -431,7 +431,7 @@ opt-in. Text frames and messages over the configured size bound are closed before schema dispatch. The matching `CultNetWebSocketSchemaClient` is suitable for C# headless and -service consumers. Browser applications use `cultmesh-browser`, which owns +service consumers. Browser applications use `@gamecult/cultmesh-browser`, which owns stable Verse/provider identity, document leases, resubscription, and operation correlation above the raw WebSocket route. From d0fcd3e36ec6a141b53757240eb1cc6f936a9f9e Mon Sep 17 00:00:00 2001 From: Libby Date: Sat, 7 Nov 2026 16:27:24 +0000 Subject: [PATCH 85/88] cultnet-ts, cultmesh-ts: one factory for the text-only error message PR #20 made routingHint, code and details required on CultNetErrorMessage; six call sites in cultmesh-ts still built the message without them, so build:ts failed with TS2345. cultnet-ts now owns cultNetTextError(error), the legacy-union decode uses it too, and the call sites and two test expectations call it. The types are unchanged. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultmesh-ts/src/index.ts | 25 ++++++---------------- packages/cultmesh-ts/test/cultmesh.test.ts | 5 +++-- packages/cultnet-ts/CHANGELOG.md | 1 + packages/cultnet-ts/src/contracts.ts | 15 ++++++------- packages/cultnet-ts/src/index.ts | 1 + 5 files changed, 19 insertions(+), 28 deletions(-) diff --git a/packages/cultmesh-ts/src/index.ts b/packages/cultmesh-ts/src/index.ts index 3e15eaac..cdf13483 100644 --- a/packages/cultmesh-ts/src/index.ts +++ b/packages/cultmesh-ts/src/index.ts @@ -20,6 +20,7 @@ import { CultNetRudpSocketTransportConnection, rudpClientBindHost, CultNetSchemaCatalog, + cultNetTextError, CultNetShardCatalog, cultNetBuiltinSchemaRegistry, decodeRudpPacket, @@ -5577,10 +5578,7 @@ export class CultMesh { switch (message.schemaVersion) { case "cultnet.snapshot_request.v0": { if (!options.getCache) { - sendSchemaMessage(record, { - schemaVersion: "cultnet.error.v0", - error: "CultMesh RUDP document server has no cache for snapshot requests.", - }); + sendSchemaMessage(record, cultNetTextError("CultMesh RUDP document server has no cache for snapshot requests.")); return; } let reply: CultNetMessage; @@ -5613,7 +5611,7 @@ export class CultMesh { const unservable = unservablePut(message); if (unservable) { reportError(unservable); - sendSchemaMessage(record, { schemaVersion: "cultnet.error.v0", error: unservable.message }); + sendSchemaMessage(record, cultNetTextError(unservable.message)); return; } const receipt = await options.onDocumentPutRaw(normalizeRudpDocumentPut(message, record.remote, options.documents)); @@ -5623,29 +5621,20 @@ export class CultMesh { } catch (error) { // The put is stored: the peer must hear that its receipt is not coming. reportError(error); - sendSchemaMessage(record, { - schemaVersion: "cultnet.error.v0", - error: `CultMesh RUDP document server stored a put but could not send its receipt.`, - }); + sendSchemaMessage(record, cultNetTextError(`CultMesh RUDP document server stored a put but could not send its receipt.`)); } } } return; case "cultnet.operation_request.v0": if (!options.onOperationRequest) { - sendSchemaMessage(record, { - schemaVersion: "cultnet.error.v0", - error: "CultMesh RUDP document server has no operation handler.", - }); + sendSchemaMessage(record, cultNetTextError("CultMesh RUDP document server has no operation handler.")); return; } sendSchemaMessage(record, await options.onOperationRequest(message, publicSession(record))); return; default: - sendSchemaMessage(record, { - schemaVersion: "cultnet.error.v0", - error: `Unsupported CultMesh RUDP document request ${message.schemaVersion}.`, - }); + sendSchemaMessage(record, cultNetTextError(`Unsupported CultMesh RUDP document request ${message.schemaVersion}.`)); } } @@ -5741,7 +5730,7 @@ export class CultMesh { */ function refuse(record: SessionRecord, text: string): void { if (record.closed) return; - const payload = encodeSchemaMessage({ schemaVersion: "cultnet.error.v0", error: text }); + const payload = encodeSchemaMessage(cultNetTextError(text)); for (const packet of record.session.sendMany("schema", payload, { reliable: false, ordered: false, diff --git a/packages/cultmesh-ts/test/cultmesh.test.ts b/packages/cultmesh-ts/test/cultmesh.test.ts index 924ab697..b73d97d3 100644 --- a/packages/cultmesh-ts/test/cultmesh.test.ts +++ b/packages/cultmesh-ts/test/cultmesh.test.ts @@ -12,6 +12,7 @@ import { CultNetPeer, CultNetRudpSession, cultNetBuiltinSchemaRegistry, + cultNetTextError, decodeRudpPacket, defineCultNetDocumentBinding, encodeRudpPacket, @@ -3708,7 +3709,7 @@ test("CultMesh TS RUDP document server answers a peer whose reliable queue is fu while (!refusalIn(rig.inbox) && Date.now() - startedAt < 3_000) await delay(5); const refusal = refusalIn(rig.inbox); assert.ok(refusal, "the refusal reaches the peer"); - assert.deepEqual(refusal.message, { schemaVersion: "cultnet.error.v0", error: "the snapshot response could not be queued" }); + assert.deepEqual(refusal.message, cultNetTextError("the snapshot response could not be queued")); for (const part of refusal.parts) { assert.ok(refused.every((sequence) => !acknowledges(part, sequence)), `ack ${part.ack} mask ${part.ackMask}`); } @@ -3725,7 +3726,7 @@ test("CultMesh TS RUDP document server refuses a snapshot with fixed text, never while (!refusalIn(rig.inbox) && Date.now() - startedAt < 3_000) await delay(5); const refusal = refusalIn(rig.inbox); assert.ok(refusal, "the refusal reaches the peer"); - assert.deepEqual(refusal.message, { schemaVersion: "cultnet.error.v0", error: "the snapshot source failed" }); + assert.deepEqual(refusal.message, cultNetTextError("the snapshot source failed")); assert.match(rig.errors[0]?.message ?? "", /CANARY-7f3a/); }); }); diff --git a/packages/cultnet-ts/CHANGELOG.md b/packages/cultnet-ts/CHANGELOG.md index b2a16456..3e8ad1fa 100644 --- a/packages/cultnet-ts/CHANGELOG.md +++ b/packages/cultnet-ts/CHANGELOG.md @@ -7,6 +7,7 @@ All notable changes to `@gamecult/cultnet-ts` are recorded here, following ### Added +- `cultNetTextError(error)`: the text-only `cultnet.error.v0` refusal, with `routingHint`, `code` and `details` null. The legacy-union error decode uses it, so there is one place that spells the null members. - `CultNetRudpSocketTransportServer`: one UDP socket serving many peers, each remote address and port with its own reliable session (`peerConnected`, `frame`, `peerDisconnected` events; `send(peer, ...)` answers on the peer a frame came from). A second client no longer replaces the first; Connects beyond `maxPeers` (default 256) are dropped, and `close()` resolves once every peer's goodbye has left the socket. The shape follows the Rust `CultNetRudpServerHub` and C# `CultNetRudpSocketTransportServer`, and the wire format is unchanged. The optional `peerTimeoutMs` (idle peers ended and told so; default off) exists only in TypeScript. `CultNetRudpSocketTransportConnection` in `server` mode still serves exactly one peer. - The `cultnet.snapshot_request.v1`, `cultnet.database_subscribe.v1` and `cultnet.snapshot_response_raw.v1` carriers and the selection payload types (`CultNetSelection`, `CultNetRawDocumentHeader`, `CultNetEdge`). `parseCultNetMessage` accepts them (absent members arrive as absent keys; a page carries exactly one of `headers`/`documents`; an empty or blank `schemas`/`keys` entry throws `CultNetSelectionInvalidError` naming its field), and `encodeCultNetMessageForWire` writes the reference's nil-keyed, declaration-ordered map. Judged against bytes the C# reference wrote (`contracts/cultnet/interop/carrier-vectors.cs-written.json`) and the selection vectors C# and Rust wrote. The selection engine (evaluation, cursors, the declaration-dependent door checks) is not ported. diff --git a/packages/cultnet-ts/src/contracts.ts b/packages/cultnet-ts/src/contracts.ts index f3bfb2e8..9ec805ff 100644 --- a/packages/cultnet-ts/src/contracts.ts +++ b/packages/cultnet-ts/src/contracts.ts @@ -199,6 +199,11 @@ export const CULTNET_ERROR_CODES = [ export type CultNetErrorCode = (typeof CULTNET_ERROR_CODES)[number]; +/** A refusal that carries text only: `routingHint`, `code` and `details` are written as null, as the reference writes an absent member. */ +export function cultNetTextError(error: string): CultNetErrorMessage { + return { schemaVersion: "cultnet.error.v0", error, routingHint: null, code: null, details: null }; +} + export interface CultNetErrorRoutingHint { shard: Record | null; reason: string; @@ -958,14 +963,8 @@ function parseGameCultNetworkingMessage(input: unknown): CultNetMessage { session: requireLegacyBytes(payload[1], "LoginSuccessMessage.Session"), }; case 4: - return { - schemaVersion: "cultnet.error.v0", - error: requireLegacyString(payload[0], "ErrorMessage.Error"), - // The legacy union ErrorMessage carries only Error; the R-N fields exist on cultnet.schema.v0 alone. - routingHint: null, - code: null, - details: null, - }; + // The legacy union ErrorMessage carries only Error; the R-N fields exist on cultnet.schema.v0 alone. + return cultNetTextError(requireLegacyString(payload[0], "ErrorMessage.Error")); case 5: return { schemaVersion: "cultnet.sample.change_name.v0", diff --git a/packages/cultnet-ts/src/index.ts b/packages/cultnet-ts/src/index.ts index 5e654c28..7857e72d 100644 --- a/packages/cultnet-ts/src/index.ts +++ b/packages/cultnet-ts/src/index.ts @@ -17,6 +17,7 @@ export { type CultNetRawDocumentRecord, type CultNetRawPayloadEncoding, CULTNET_ERROR_CODES, + cultNetTextError, type CultNetErrorCode, type CultNetErrorDetails, type CultNetErrorMessage, From 7ac9e60bdca3a7c196e7c555d67b3cfa8aa5c383 Mon Sep 17 00:00:00 2001 From: Libby Date: Sat, 7 Nov 2026 16:31:04 +0000 Subject: [PATCH 86/88] cultcache-ts 0.15.0: fold the untagged 0.14.1/0.14.2 and Unreleased into one entry 0.14.1 and 0.14.2 were never tagged; cultcache-ts-v0.14.0 is the last release. The Unreleased exports (SchemaConflictError, schemaIdentityOf, the document payload codec) are in no tagged release, which is what failed the declared-ranges check. Package and lockfile go to 0.15.0 and every consumer range (cultnet-ts, cultmesh-ts, cultmesh-browser) moves to ^0.15.0, since a 0.x caret does not admit a new minor. Co-Authored-By: Claude Sonnet 5.5 --- package-lock.json | 8 +++---- packages/cultcache-ts/CHANGELOG.md | 32 ++++++++------------------ packages/cultcache-ts/package.json | 2 +- packages/cultmesh-browser/package.json | 2 +- packages/cultmesh-ts/package.json | 2 +- packages/cultnet-ts/package.json | 2 +- 6 files changed, 18 insertions(+), 30 deletions(-) diff --git a/package-lock.json b/package-lock.json index abc8f6d1..0a1558da 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1184,7 +1184,7 @@ }, "packages/cultcache-ts": { "name": "@gamecult/cultcache-ts", - "version": "0.14.2", + "version": "0.15.0", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.3", @@ -1204,7 +1204,7 @@ "version": "0.4.0", "license": "MIT", "dependencies": { - "@gamecult/cultcache-ts": "^0.14.0", + "@gamecult/cultcache-ts": "^0.15.0", "@gamecult/cultnet-ts": "^0.3.0" }, "devDependencies": { @@ -1221,7 +1221,7 @@ "version": "0.2.0", "license": "MIT", "dependencies": { - "@gamecult/cultcache-ts": "^0.14.0", + "@gamecult/cultcache-ts": "^0.15.0", "@gamecult/cultnet-ts": "^0.3.0", "koffi": "3.3.0" }, @@ -1239,7 +1239,7 @@ "version": "0.3.0", "license": "MIT", "dependencies": { - "@gamecult/cultcache-ts": "^0.14.1", + "@gamecult/cultcache-ts": "^0.15.0", "@msgpack/msgpack": "^3.1.3", "ajv": "^8.17.1" }, diff --git a/packages/cultcache-ts/CHANGELOG.md b/packages/cultcache-ts/CHANGELOG.md index 93e17239..819df6c4 100644 --- a/packages/cultcache-ts/CHANGELOG.md +++ b/packages/cultcache-ts/CHANGELOG.md @@ -3,29 +3,26 @@ All notable changes to `@gamecult/cultcache-ts` are recorded here, following [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). -## Unreleased +## [0.15.0] + +0.14.1 and 0.14.2 were never tagged or published; this entry carries them with the Unreleased work, and `cultcache-ts-v0.14.0` is the previous release. ### Breaking - A single-file store written as `cultcache.store.v3` is read and written: a record that holds an element id (an owned list element with an `[CultElementId]` member, carried through every write) marks the store v3, and a v3 store is carried through a rewrite with its ids. A runtime from before this change refuses a v3 store by name. - A single-file store refuses a zero-byte file as `StoreUnreadableError`; only a path with nothing at it is an empty store. A header this runtime does not read is refused, and the refusal echoes the header only as `cultcache.store.v`. -- A write over a file the reader refuses (`StoreUnreadableError`, from `push`, `pushAll` and `delete`) refuses and leaves the file as it is. A flush used to replace such a file. A write set cannot merge with a disk it cannot read, so the file is repaired or removed by its owner before the store writes again. -- This reverses the 0.14.2 `pushAll` fix in two ways. A full `pushAll` over a file with no readable stamp no longer replaces it (healing it): the reader refuses it, and the file is left as it is. And a `cultcache.store.v3` stamp is read, not refused. The stamp rule has one owner, `refuseUnreadFormat`, asked only by the one reader that every write consults under the lock. The reader judges the stamp from the header before it decodes the body, so a newer store is refused by name whether or not its body is whole. +- A write over a file the reader refuses (`StoreUnreadableError`, from `push`, `pushAll` and `delete`) refuses and leaves the file as it is. A flush used to replace such a file. A write set cannot merge with a disk it cannot read, so the file is repaired or removed by its owner before the store writes again. A full `pushAll` over a file with no readable stamp does not replace it: the reader refuses it. The stamp rule has one owner, `refuseUnreadFormat`, asked only by the one reader that every write consults under the lock. The reader judges the stamp from the header before it decodes the body, so a newer store is refused by name whether or not its body is whole. - A dangling symbolic link at the store path is an I/O error with code `ENOENT`, not an empty store. ### Added - `@gamecult/cultcache-ts/browser`, also what a bundler resolves for the package root under the `browser` export condition: the root surface without `SingleFileMessagePackBackingStore` and `readSingleFileStore`, so it imports no `node:*` built-in and no `koffi` and bundles with esbuild `platform: "browser"`. The Node entry exports exactly what it did. - - `readSingleFileStore(path)`, the one synchronous reader of a single-file store, for consumers that must read a store file rather than decode it themselves. `pullAll`, `push`, `delete` and `pushAll` ask the same reader. - `StoreUnreadableError` for a store file this runtime cannot read, `SchemaConflictError` for a write the catalog cannot describe, and `schemaIdentityOf` with its result type `CultCacheSchemaIdentity` (`schemaId`, `schemaName`, `schemaVersion`, `compatibleSchemaIds`), the one derivation of a definition's identity. +- The document payload codec, exported from `@gamecult/cultcache-ts/document`: `decodeDocumentPayload`, `encodeDocumentPayload`, and the two named byte escapes `documentPayloadFromBytes` and `documentPayloadBytes`. - `SingleFileStoreRead`, exported: the result type of `readSingleFileStore` (`absent`, `format`, `envelopes`, `repairedLegacyPayload`). - `CultCacheInspection.format` is `StoreFormat | "legacy.envelope-array"`: it reports `cultcache.store.v3` for a v3 store. - A soft `pushAll` writes only where nothing is at the path, leaves a store it can read byte for byte, and refuses what the reader refuses. - -## [0.14.2] - -### Added - `SingleFileMessagePackBackingStore` takes a cross-process write lock on `.lock`, interoperable with the C# and Rust runtimes (`flock` on Unix, `LockFileEx` on Windows). `push` and `delete` read the current file, merge and replace it under the lock, so concurrent @@ -35,27 +32,18 @@ All notable changes to `@gamecult/cultcache-ts` are recorded here, following is, but C# writers can still overwrite records a concurrent TypeScript writer added. - The staged file is fsynced before the replace, and on Unix the directory after it. A directory sync failure after the replace is reported as replaced but not durable. +- `checkPayloadDepth` is re-exported from `@gamecult/cultcache-ts/document`, so dependents + (`cultnet-ts/content`) bound payload nesting with the same check the cache + applies on decode. ### Changed + - The Windows replace is a POSIX-semantics rename first (it lands while C# and Rust readers hold the store open), falling back to a write-through `MoveFileExW`, with 25 attempts 20 ms apart on errors 5, 32 and 33. Writers wait up to 30 s for the lock. - Opening a legacy store that needs healing rewrites it under the lock, from a read taken under the lock. -### Fixed -- A full `pushAll` reads the store's header under the lock first and refuses a store format - this runtime does not read (the same error a read gives), leaving the file as it was. A - file with no readable stamp is still replaced, so a whole write heals it. Soft `pushAll` - is unchanged. The stamp is decoded by the decoder a read uses, so a stamp followed by bytes - that are not UTF-8 is refused by a whole write as a read refuses it. - ### Dependencies + - `koffi` 3.3.0, loaded only on the first write. - -## [0.14.1] - -### Added -- `checkPayloadDepth` is re-exported from `@gamecult/cultcache-ts/document`, so dependents - (`cultnet-ts/content`) bound payload nesting with the same check the cache - applies on decode. diff --git a/packages/cultcache-ts/package.json b/packages/cultcache-ts/package.json index ffe1463d..b6803979 100644 --- a/packages/cultcache-ts/package.json +++ b/packages/cultcache-ts/package.json @@ -1,6 +1,6 @@ { "name": "@gamecult/cultcache-ts", - "version": "0.14.2", + "version": "0.15.0", "description": "CultCache-style typed persistence for TypeScript with schema-backed polymorphic MessagePack storage.", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/cultmesh-browser/package.json b/packages/cultmesh-browser/package.json index 480fb07e..0984ae6d 100644 --- a/packages/cultmesh-browser/package.json +++ b/packages/cultmesh-browser/package.json @@ -41,7 +41,7 @@ "node": ">=20" }, "dependencies": { - "@gamecult/cultcache-ts": "^0.14.0", + "@gamecult/cultcache-ts": "^0.15.0", "@gamecult/cultnet-ts": "^0.3.0" }, "devDependencies": { diff --git a/packages/cultmesh-ts/package.json b/packages/cultmesh-ts/package.json index 1ccaa421..92962cbc 100644 --- a/packages/cultmesh-ts/package.json +++ b/packages/cultmesh-ts/package.json @@ -48,7 +48,7 @@ "node": ">=20" }, "dependencies": { - "@gamecult/cultcache-ts": "^0.14.0", + "@gamecult/cultcache-ts": "^0.15.0", "@gamecult/cultnet-ts": "^0.3.0", "koffi": "3.3.0" }, diff --git a/packages/cultnet-ts/package.json b/packages/cultnet-ts/package.json index ebafac78..60c9e13f 100644 --- a/packages/cultnet-ts/package.json +++ b/packages/cultnet-ts/package.json @@ -73,7 +73,7 @@ "dependencies": { "@msgpack/msgpack": "^3.1.3", "ajv": "^8.17.1", - "@gamecult/cultcache-ts": "^0.14.1" + "@gamecult/cultcache-ts": "^0.15.0" }, "devDependencies": { "@types/node": "^24.0.0", From 85698256307fd670f56c9b6f69776112b3faf58a Mon Sep 17 00:00:00 2001 From: Libby Date: Sat, 7 Nov 2026 16:38:11 +0000 Subject: [PATCH 87/88] publish-packages: stop cultcache-ts-v* tags from triggering the npm publish Publishing @gamecult/cultcache-ts is deliberately disabled until ruled on, and whether NPM_TOKEN is set is unknown. The other tag patterns and workflow_dispatch are unchanged. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/publish-packages.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml index eb798bdf..f78c0bb2 100644 --- a/.github/workflows/publish-packages.yml +++ b/.github/workflows/publish-packages.yml @@ -16,7 +16,9 @@ name: Publish packages on: push: tags: - - "cultcache-ts-v*" + # cultcache-ts-v* is deliberately not a trigger: publishing @gamecult/cultcache-ts is disabled + # until it is ruled on, and whether NPM_TOKEN is set is unknown, so a pushed tag must not reach + # `npm publish`. The cultcache-ts jobs below still run from workflow_dispatch, which publishes nothing. - "cultcache-py-v*" - "cultnet-py-v*" - "cultmesh-py-v*" From bf5a09bd0452da40409e59c578862c8e43e97549 Mon Sep 17 00:00:00 2001 From: Libby Date: Sat, 7 Nov 2026 16:53:03 +0000 Subject: [PATCH 88/88] publish-packages: say that cultcache-ts publishing is off; keep the tag pin readable The header comment, semver-policy and F16 of the content map still listed cultcache-ts-v* as a trigger. They now say publishing is deliberately off and that a tag run uses the workflow at the tagged commit (cultcache-ts-v0.15.0 sits on 7d292085, before the guard). semver-policy gains a "Before pushing a tag" note. The guard's comment moves above `tags:`: inside the list it broke the three `workflow:` tests of check-changelog-semver.test.mjs, which read `tags:` followed directly by its items. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/publish-packages.yml | 23 ++++++++++++++++------- docs/cultmesh-ts-content-map.md | 2 +- docs/semver-policy.md | 15 +++++++++++++-- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml index f78c0bb2..caa24336 100644 --- a/.github/workflows/publish-packages.yml +++ b/.github/workflows/publish-packages.yml @@ -8,17 +8,26 @@ name: Publish packages # directly, which is already how Ghostlight, Epiphany, CodexConnector and Odin # consume the core stack. # -# Tag a release to publish, for example `cultcache-ts-v0.13.6` or -# `cultnet-py-v0.1.0`. The tag's version must match the version in the package -# manifest; scripts/release-after-suite.mjs refuses the release otherwise rather -# than shipping a mislabelled artifact. +# Tag a Python release to publish, for example `cultnet-py-v0.1.0`. The tag's +# version must match the version in the package manifest; +# scripts/release-after-suite.mjs refuses the release otherwise rather than +# shipping a mislabelled artifact. +# +# npm publishing of @gamecult/cultcache-ts is deliberately off: `cultcache-ts-v*` +# is not a tag trigger until it is ruled on (the cultcache-ts job below runs from +# workflow_dispatch only, which publishes nothing). A tag run uses the workflow +# file at the tagged commit, not the one on main, so a tag on an older commit +# (`cultcache-ts-v0.15.0` sits on 7d292085, before the guard) still runs that +# commit's triggers and jobs. Read this file at the tagged sha before pushing a tag. on: push: + # cultcache-ts-v* is deliberately not a trigger: publishing @gamecult/cultcache-ts is disabled + # until it is ruled on, and whether NPM_TOKEN is set is unknown, so a pushed tag must not reach + # `npm publish`. The cultcache-ts jobs below still run from workflow_dispatch, which publishes nothing. + # (The comment sits above `tags:` because the tag-prefix pin in scripts/check-changelog-semver.test.mjs + # reads the list as `tags:` followed directly by its items.) tags: - # cultcache-ts-v* is deliberately not a trigger: publishing @gamecult/cultcache-ts is disabled - # until it is ruled on, and whether NPM_TOKEN is set is unknown, so a pushed tag must not reach - # `npm publish`. The cultcache-ts jobs below still run from workflow_dispatch, which publishes nothing. - "cultcache-py-v*" - "cultnet-py-v*" - "cultmesh-py-v*" diff --git a/docs/cultmesh-ts-content-map.md b/docs/cultmesh-ts-content-map.md index 9aae6eba..de6b6d2f 100644 --- a/docs/cultmesh-ts-content-map.md +++ b/docs/cultmesh-ts-content-map.md @@ -34,7 +34,7 @@ ran under the session scratchpad (`imagination/`), not in the repo. | F13 | Cross-process precedent for C# against a Node provider exists. `packages/cultnet-ts/test/interop/cultnet-interop.test.ts:536-560` spawns the TS QUIC provider, then runs `dotnet test --filter NativeConnectorReceivesFromExternalProvider` with the endpoint in `CULTMESH_NATIVE_EXTERNAL_ENDPOINT`. The C# test skips without the variable (`tests/GameCult.Mesh.Quic.Native.Tests`, line 21). | source | | F14 | `cultnet-ts` is CommonJS. Its `./wire`, `./contracts` and `./authority` subpaths import only `@msgpack/msgpack`, `ajv` and `cultcache-ts/document`, so they are browser-safe. Its root `index` pulls `node:` modules (`peer`, `rudp`, `transport`, `security` and others). `cultmesh-browser` (ESM) already imports `cultnet-ts/wire`, `/contracts` and `/authority`. `cultmesh-ts` loads `koffi` lazily, inside a function (`realtime-quic-native.ts:139`), so a new subpath of it pays no native load. The root has `esbuild` ^0.28.1 as a devDependency. | source, grep | | F15 | TS packages build with `tsc` and test with `node --test dist-test/test/*.test.js` per workspace. `scripts/run-typescript-workspaces.mjs` runs them in order `cultcache-ts, cultnet-ts, cultmesh-ts, cultmesh-browser`, after `check-typescript-tests-wired.mjs`. No StrykerJS config exists in CultLib (`docs/mutation-testing.md:156-159`). | source | -| F16 | **No CultLib TS package except `cultcache-ts` has a publish job.** `publish-packages.yml` triggers on `cultcache-ts-v*`, `cultcache-py-v*`, `cultnet-py-v*` and `cultmesh-py-v*` only. Tags `cultnet-ts-v0.1.1` and `cultmesh-ts-v0.1.1` exist, but no workflow consumes them. `npm view` returns 404 anonymously for `cultnet-ts`, `cultmesh-ts`, `cultmesh-browser` and even `@gamecult/cultcache-ts` (`@gamecult/cultcache-ts` may be private; unverified). The doctrine's rule of at most three tags per push stands (SKILL.md:674). | source; `npm view` | +| F16 | **No CultLib TS package except `cultcache-ts` has a publish job.** `publish-packages.yml` triggered on `cultcache-ts-v*`, `cultcache-py-v*`, `cultnet-py-v*` and `cultmesh-py-v*` when this was written; it now triggers on the three `-py-v*` patterns only, because npm publishing is deliberately off until ruled on (a tag run uses the workflow at the tagged commit, so `cultcache-ts-v0.15.0` on `7d292085`, before that change, still ran the old one). Tags `cultnet-ts-v0.1.1` and `cultmesh-ts-v0.1.1` exist, but no workflow consumes them. `npm view` returns 404 anonymously for `cultnet-ts`, `cultmesh-ts`, `cultmesh-browser` and even `@gamecult/cultcache-ts` (`@gamecult/cultcache-ts` may be private; unverified). The doctrine's rule of at most three tags per push stands (SKILL.md:674). | source; `npm view` | | F17 | StreamPixels consumes CultLib as the git submodule `vendor/CultLib` (gitlink `30ee8b97`) with `file:` deps and `pnpm.overrides`. Its CI and Idunn recipes build the vendored workspaces (`npm ci --ignore-scripts`, then build `cultcache-ts`, `cultnet-ts` and `cultmesh-ts`). `apps/web` depends only on `cultnet-ts` today, not on `cultmesh-browser`. | StreamPixels `.gitmodules`, `apps/*/package.json`, CI workflow lines 62-95 | | F18 | The Rust reader sets two guards that the TS reader should match. A caller cap `max_bytes` is checked before any request. And a zero-size chunk is refused unless it is the only one, because a manifest of zero-size chunks would be a request flood. | source `cultnet-rs/src/content.rs:364-385` | | F19 | `BuildChunkUri` on more endpoint shapes, as `AbsoluteUri` (what `HttpClient` sends). `https://h` and `https://h/` both give `https://h/?chunkHash=…&recordKey=k`. `https://h:443/x` drops the default port. `https://H.Example/X/` lowercases the host and keeps the path's case. `https://h/x#frag` keeps the fragment after the query. `https://h/x?a=%20b+c` keeps the existing query verbatim and joins with `&`. **`https://h/x?` (an empty query) gives `https://h/x?&chunkHash=…`**, because `UriBuilder.Query` is `?`, which is not blank. `https://[::1]:8443/c` keeps the bracketed host. `Uri.EscapeDataString("!'()*~-._ +é𐐀")` is `%21%27%28%29%2A~-._%20%2B%C3%A9%F0%90%90%80`: unlike JS `encodeURIComponent`, it escapes `!'()*`. | probe: `dotnet fsi` on .NET 10, the three lines of `BuildChunkUri` (`CultMeshHttpsContentTransport.cs:114-122`) copied verbatim | diff --git a/docs/semver-policy.md b/docs/semver-policy.md index 324b1154..f1c2c170 100644 --- a/docs/semver-policy.md +++ b/docs/semver-policy.md @@ -352,10 +352,21 @@ own path into the DLL. No `global.json` pins the .NET SDK, so the same refusal at unchanged source and CRLF checkout means this SDK or host differs from the one that built the tracked DLL. +### Before pushing a tag + +A tag run uses the workflow files at the tagged commit, not the ones on main, and +the forge's GitHub mirror forwards the tag. Read `.github/workflows/publish-packages.yml` +(and any workflow that triggers on tags) at the tagged sha before pushing any tag: +a guard merged after that commit does not protect a tag that points before it. +`cultcache-ts-v0.15.0` sits on `7d292085`, which predates the guard, and its push +started a publish run (it failed; nothing reached npm). + ### Coverage gaps -`publish-packages.yml` only triggers on `cultcache-ts-v*`, `cultcache-py-v*`, -`cultnet-py-v*`, and `cultmesh-py-v*` tag pushes — it does not build or gate +`publish-packages.yml` only triggers on `cultcache-py-v*`, `cultnet-py-v*`, and +`cultmesh-py-v*` tag pushes (`cultcache-ts-v*` is deliberately not a trigger: npm +publishing is off until ruled on, and the cultcache-ts job runs from +`workflow_dispatch` only, publishing nothing) — it does not build or gate the Unity UPM packages at all; those are consumed directly by git tag (`unity/org.gamecult.cultlib`, `packages/cultmath/unity/org.gamecult.cultmath`, `src/GameCult.Unity/Assets/Caching`), so a workflow trigger on their tags