diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index 009fa696..4048eaf8 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -336,8 +336,11 @@ an element id, variants or not). A single-file store declares `v3` when any reco holds carries an element id: a non-empty `[CultElementId]` value in a record's document, or in a variant's override values. A store that cannot read a variant's override values (no codec attached) counts the variant as holding one. An element id is an ordinary keyed -string in the payload, so no reader can tell from a record's bytes whether it holds one; -a writer knows it only for the records it encodes. A write that copies any record forward +string in the payload, so only a reader with the record's type can tell; a C# write +decides for the records it encodes and, while the header it found is unmarked, for every +plain record it copies forward that its registry can read, decoding the bytes the file +holds under the store's lock. A copied record it cannot read, and a raw CultMesh payload, +decide nothing. A write that copies any record forward therefore keeps a `v3` header it found. Only a write that encodes every record the store will hold decides by content, and drops to `v1` (or `v2` with a variant) when none holds an id. A directory store's manifest says @@ -423,8 +426,8 @@ elements retargets nothing. the record's next write, or `CultCache.MintElementIds()`. - A store that holds an id declares the `v3` header (above), so a reader older than ids refuses it instead of skipping the id slot and rewriting the elements without ids. - A C# write also marks the store when it carries forward a record it read holding an - id, as long as the file still holds that record at the version it read. TypeScript, + A C# write also marks the store when a record it copies forward holds an id, judged + from the file's bytes at the write (above). TypeScript, Python and Rust read `v3` and keep a `v3` header they found when they rewrite a store; they carry ids and do not decide when a document carries them. - An object list's elements are written by their `[Key]` layout, because the id is a diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 4fca998b..45f1343f 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -3491,6 +3491,9 @@ namespace GameCult.Caching protected CultDocumentRegistry Registry => _registry ?? throw new InvalidOperationException("Backing store is not attached to a CultDocumentRegistry."); + // Null for a store no registry was attached to (CultMesh's raw single-file write). + private protected CultDocumentRegistry? AttachedRegistry => _registry; + protected ConcurrentDictionary Entries { get; } = new(StringComparer.Ordinal); @@ -3545,9 +3548,9 @@ namespace GameCult.Caching // The header a write leaves, for either store kind: HeaderFor decides. A record's bytes do not say whether it holds an // element id (an id is an ordinary keyed string), so only the store header carries that fact. A write that copies any // record forward keeps a marked header; a write that encodes every record the store will hold decides by content. An - // opaque payload (null) never marks an unmarked store, and keeps a marked one marked. A copied record this store read - // holding an id, at the version the file still holds, marks the store too: another runtime may have written ids under - // an unmarked header. Nothing a load records can unmark a store. + // opaque payload (null) never marks an unmarked store, and keeps a marked one marked. A copied record the writer decodes + // under the store's lock holding an id marks the store too (SingleFileBackingStore.CopiedHoldIds): another runtime may + // have written ids under an unmarked header. Nothing a load or a decode records can unmark a store. protected static string HeaderForWrite(string? durableHeader, IReadOnlyCollection written, bool copiesAny, bool copiedHoldIds, bool directoryStore, bool holdsVariants) => HeaderFor( holdsIds: copiedHoldIds || written.Any(value => value == true), @@ -3946,10 +3949,6 @@ namespace GameCult.Caching // A write into a store holding a variant lands only while the durable versions still equal these. private readonly Dictionary _lastRead = new(StringComparer.Ordinal); - // The version of each plain record a pull decoded holding an element id. Only a positive fact about those bytes: a - // write marks the header for a copied record still at that version, and nothing here ever unmarks a store. - private readonly Dictionary _readHoldingIds = new(StringComparer.Ordinal); - /// /// A store file from its parts, each already encoded: a catalog entry or record this store carries forward is the bytes its /// file held, one it writes is or of the object. @@ -3974,6 +3973,9 @@ namespace GameCult.Caching // Test seam: how many record encodings this store has hashed, to prove what a write hashes. internal int RecordsHashed; + // Test seam: how many copied records a write decoded to ask whether they hold an element id. + internal int CopiedRecordsDecoded; + private string VersionOfBytes(byte[] bytes) { RecordsHashed++; @@ -4038,7 +4040,6 @@ namespace GameCult.Caching if (durable == null) { _lastRead.Clear(); - _readHoldingIds.Clear(); SetLastSchemaMigrationReports(Array.Empty()); SetForeignRecords(Array.Empty()); return; @@ -4049,7 +4050,6 @@ namespace GameCult.Caching var persisted = new Dictionary(StringComparer.Ordinal); var foreign = new List(); var versions = new Dictionary(StringComparer.Ordinal); - var holdingIds = new Dictionary(StringComparer.Ordinal); foreach (var record in snapshot.Records) { var version = versions[record.Key] = durable.Version(record.Key); @@ -4062,8 +4062,6 @@ namespace GameCult.Caching var stored = ToStoredDocument(record, version, snapshot.SchemaCatalog, DeserializePayload, out var report); reports.Add(report); persisted[stored.Key.Value] = stored; - if (stored.Variant == null && CultElementIds.Holds(stored.Document)) - holdingIds[stored.Key.Value] = version; } var loaded = persisted.Values @@ -4081,9 +4079,6 @@ namespace GameCult.Caching _lastRead.Clear(); foreach (var (key, version) in versions) _lastRead[key] = version; - _readHoldingIds.Clear(); - foreach (var (key, version) in holdingIds) - _readHoldingIds[key] = version; SetLastSchemaMigrationReports(reports); SetForeignRecords(foreign); } @@ -4219,7 +4214,8 @@ namespace GameCult.Caching // write that then has nothing to write but keys it checked writes no file. Then a store that holds // a variant, or would after this write, is written only while its record versions are what this store last read: a variant // resolves against other records, and a write decodes none it did not stage, so it cannot judge another writer's. Versions - // only: a write hashes other records' bytes, never decodes them. Otherwise the whole write is refused and what it refused is + // only: a write hashes other records' bytes, never decodes them. That holds for the variant gate; the one decode of a copied + // record is the element-id check (CopiedHoldIds), which runs only on an unmarked store and can only mark it. Otherwise the whole write is refused and what it refused is // reloaded (Refuse). Every record the writer staged is encoded from its object now, and its version is the hash of that // encoding; every other record is copied as the slice the file holds. Returns the records written, each with the version of // the bytes written. @@ -4263,8 +4259,7 @@ namespace GameCult.Caching durable?.Snapshot.FormatVersion, upserts.Select(staged => staged.HoldsIds).ToArray(), copiesAny: copied.Length > 0, - copiedHoldIds: copied.Any(record => _readHoldingIds.TryGetValue(record.Key, out var read) && - SameStoredIdentity(read, durable!.Version(record.Key))), + copiedHoldIds: CopiedHoldIds(durable, copied, upserts), directoryStore: false, holdsVariants: records.Any(record => record.Variant != null)); @@ -4278,6 +4273,43 @@ namespace GameCult.Caching return versions; } + // Whether a record this write copies forward holds an element id, decided from the bytes the file holds under the store's + // lock, so every C# writer with a type for the record decides the same from the same bytes. Positive-only: it runs only + // while the header on disk is unmarked and can only mark the store, so a narrower type that decodes no ids changes nothing. + // It never refuses a write: a record that does not resolve or decode is copied as its bytes and decides nothing. Gates, in + // order, so the decode runs only where it can change the answer. + private bool CopiedHoldIds(Durable? durable, CultPersistedRecord[] copied, IReadOnlyCollection upserts) + { + if (durable == null || copied.Length == 0 || + string.Equals(durable.Snapshot.FormatVersion, CultPersistedStoreSnapshot.FormatV3, StringComparison.Ordinal) || + upserts.Any(staged => staged.HoldsIds == true) || + AttachedRegistry is not { } registry) + return false; + var catalog = durable.Snapshot.SchemaCatalog; + foreach (var record in copied) + { + if (record.Variant != null) + continue; + try + { + if (registry.Claimant(record.SchemaId, catalog, out _) == null) + continue; + var type = registry.ResolvePersistedSchemaDetailed(record.SchemaId, catalog).Descriptor.DocumentType; + if (!CultElementIds.CanHoldIds(type)) + continue; + CopiedRecordsDecoded++; + if (CultElementIds.Holds(DeserializePayload(type, record.Payload))) + return true; + } + catch (Exception e) when (e is not OperationCanceledException) + { + // Copied as its bytes: no type, or no decode, is no fact. + } + } + + return false; + } + private void RefuseWriteIntoMovedVariantStore(Durable? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals) { var records = durable?.Snapshot.Records ?? Array.Empty(); diff --git a/src/GameCult.Caching/CultElementIds.cs b/src/GameCult.Caching/CultElementIds.cs index 87223289..000bed40 100644 --- a/src/GameCult.Caching/CultElementIds.cs +++ b/src/GameCult.Caching/CultElementIds.cs @@ -110,6 +110,11 @@ namespace GameCult.Caching public List<(Type Owner, MemberInfo Member, Type Element, List Concrete)> Lists { get; } = new(); } + private static readonly ConcurrentDictionary IdCapable = new(); + + // A document type whose walk reaches no object list can hold no element id, so a write never decodes one to ask. + internal static bool CanHoldIds(Type documentType) => IdCapable.GetOrAdd(documentType, type => Inspect(type).IdPath.Count > 0); + // Every reason this document type cannot be registered under the id rule, one per offending concrete element type. internal static Survey Inspect(Type documentType) { diff --git a/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs b/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs index 495b2273..17ac6f59 100644 --- a/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs +++ b/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs @@ -108,7 +108,8 @@ namespace GameCult.Mesh descriptor.ToCatalogEntry(), storedAt, CultDocumentMessagePackSerialization.SerializeUntyped(document, typeof(TDocument)), - CultElementIds.Holds(document)); + CultElementIds.Holds(document), + resolvedRegistry); } catch { @@ -146,7 +147,7 @@ namespace GameCult.Mesh byte[] payload) { if (schema == null) throw new ArgumentNullException(nameof(schema)); - WriteSingleFileDocumentPayload(path, key, schema.ToCatalogEntry(payload), storedAt, payload, holdsIds: null); + WriteSingleFileDocumentPayload(path, key, schema.ToCatalogEntry(payload), storedAt, payload, holdsIds: null, registry: null); } /// @@ -229,7 +230,8 @@ namespace GameCult.Mesh CultSchemaCatalogEntry catalogEntry, string? storedAt, byte[] payload, - bool? holdsIds) + bool? holdsIds, + CultDocumentRegistry? registry) { if (string.IsNullOrWhiteSpace(path)) throw new ArgumentException("Value must be non-empty.", nameof(path)); if (catalogEntry == null) throw new ArgumentNullException(nameof(catalogEntry)); @@ -239,8 +241,13 @@ namespace GameCult.Mesh payload ??= Array.Empty(); // The write replaces this one key and copies every other record and catalog entry forward as the store holds them, under // the store's lock. A typed write sees its document, so its ids decide the header; a raw payload is opaque (holdsIds null), - // so a file already marked stays marked. A store this runtime cannot read refuses the write instead of being overwritten. - StoreAt(path).ApplyWriteSet( + // so a file already marked stays marked. A typed write attaches the registry its payload was encoded with, so the write + // decides for copied records that registry can read; a raw payload attaches none and decides nothing. A store this runtime + // cannot read refuses the write instead of being overwritten. + var store = StoreAt(path); + if (registry != null) + store.AttachRegistry(registry); + store.ApplyWriteSet( new[] { new CultStagedRecord(