From aa186513a5abd66b65b02c6347a3e9819de5168b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sat, 3 Oct 2026 03:05:08 +0200 Subject: [PATCH] C# stores: every flush, commit and CultMesh single-file write applies a write set to the durable store SingleFileBackingStore.ApplyWriteSet replaces WholeView and ReplaceDurable: under the lock, from the durable snapshot read there, it removes staged removals, writes staged upserts and copies every other record and the catalog entries they reference unchanged. PushAll, CommitBatchCore (conditional and unconditional, one apply) and CultMesh's write call only it. Catalog entries read from a store keep their exact bytes and are written verbatim while their decoded fields are unchanged. Derive no longer lets a registered entry publish an id nothing staged sits under. HeaderForWrite is the one header rule for the single-file and directory stores. A write is judged against variants another writer left in the file. Tests of the deleted rules are rewritten in the next commit. Co-Authored-By: Claude Sonnet 5.5 --- .../CultDocumentMessagePackSerialization.cs | 30 +- .../DirectoryMessagePackBackingStore.cs | 15 +- src/GameCult.Caching/CultCache.cs | 276 ++++++++++++------ .../CultMeshSingleFileDocuments.cs | 42 ++- 4 files changed, 242 insertions(+), 121 deletions(-) diff --git a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs index 82d27db8..3dec01d6 100644 --- a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs +++ b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs @@ -128,7 +128,7 @@ public static class CultDocumentMessagePackSerialization elements[slot] = value; } - var buffer = new ArrayBufferWriter(); + var buffer = new global::System.Buffers.ArrayBufferWriter(); var writer = new MessagePackWriter(buffer); writer.WriteArrayHeader(elements.Count); foreach (var element in elements) @@ -432,7 +432,33 @@ public static class CultDocumentMessagePackSerialization return record; } + // An entry read from a store is written as the bytes it was read from, so a fact this runtime does not decode survives a + // write that only carries the entry. That holds only while the decoded fields are the ones read: an entry changed since is + // encoded from them. private static void WriteSchemaCatalogEntry(ref MessagePackWriter writer, CultSchemaCatalogEntry entry) + { + if (entry.RawBytes is { } raw && EncodeSchemaCatalogEntry(entry).AsSpan().SequenceEqual(EncodeSchemaCatalogEntry(ReadRawSchemaCatalogEntry(raw)))) + writer.WriteRaw(new ReadOnlySpan(raw)); + else + WriteDecodedSchemaCatalogEntry(ref writer, entry); + } + + private static CultSchemaCatalogEntry ReadRawSchemaCatalogEntry(byte[] raw) + { + var reader = new MessagePackReader(raw); + return ReadSchemaCatalogEntry(ref reader); + } + + private static byte[] EncodeSchemaCatalogEntry(CultSchemaCatalogEntry entry) + { + var buffer = new global::System.Buffers.ArrayBufferWriter(); + var writer = new MessagePackWriter(buffer); + WriteDecodedSchemaCatalogEntry(ref writer, entry); + writer.Flush(); + return buffer.WrittenSpan.ToArray(); + } + + private static void WriteDecodedSchemaCatalogEntry(ref MessagePackWriter writer, CultSchemaCatalogEntry entry) { writer.WriteArrayHeader(SchemaCatalogEntryFieldCount); writer.Write(entry.SchemaId); @@ -455,6 +481,7 @@ public static class CultDocumentMessagePackSerialization private static CultSchemaCatalogEntry ReadSchemaCatalogEntry(ref MessagePackReader reader) { + var start = reader.Position; var fieldCount = reader.ReadArrayHeader(); var entry = new CultSchemaCatalogEntry(); @@ -508,6 +535,7 @@ public static class CultDocumentMessagePackSerialization reader.Skip(); } + entry.RawBytes = reader.Sequence.Slice(start, reader.Position).ToArray(); return entry; } diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index 41ccd76c..02b11213 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -293,20 +293,23 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore pagePayload); } + // The catalog and the header follow the shared write rules: the registered entry publishes only the ids this write put a record + // under, every other record and entry stays as the manifest holds it. + var written = keysToWrite.Where(key => Entries.ContainsKey(key)).ToHashSet(StringComparer.Ordinal); var targetCatalog = CultSchemaCatalogEntry.Derive( currentIndex.Values, - Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()).ToArray(), + written.Select(key => Entries[key].Descriptor).Distinct().Select(descriptor => descriptor.ToCatalogEntry()).ToArray(), currentManifest.SchemaCatalog) .OrderBy(entry => entry.SchemaName, StringComparer.Ordinal) .ThenBy(entry => entry.SchemaId, StringComparer.Ordinal) .ToArray(); - // The directory store writes only its dirty pages, so it keeps a manifest already marked. - var header = HeaderFor( - keysToWrite.Any(key => Entries.TryGetValue(key, out var written) && written.HoldsIds), + var header = HeaderForWrite( currentManifest.FormatVersion, - wholeStore: false, - directoryStore: true); + written.Select(key => (bool?)Entries[key].HoldsIds), + currentIndex.Values.Where(record => !written.Contains(record.Key)), + directoryStore: true, + holdsVariants: false); WriteManifest(targetCatalog, currentIndex.Values .OrderBy(record => record.Key, StringComparer.Ordinal) .ToArray(), header); diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 499ee0e0..c1cef4c2 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -29,11 +29,19 @@ namespace GameCult.Caching public CultSchemaMemberCatalogEntry[] Members { get; set; } = Array.Empty(); - // The catalog a write leaves, derived from the records being written and nothing else. For each schema id a record carries, one - // entry that publishes it: the entry that owns the id (a registered descriptor, else one that arrived with the records), else - // one that lists it as compatible (registered first). Entries are written as chosen, never merged; a registered descriptor - // wins over an arrived entry with the same own id. Two arrived entries with the same own id and different schema names, or a - // record no chosen entry publishes, refuse the write. + // The exact MessagePack bytes this entry was read from, when it was read from a store. A writer that carries the entry + // forward emits them as they are, so what this runtime does not decode (fields beyond the ones it knows, the width of an int, + // nil) survives; an entry whose decoded fields were changed since is encoded from them instead. Null for an entry a runtime + // built. + internal byte[]? RawBytes { get; set; } + + // The catalog a write leaves, derived from the records the written store holds and nothing else. For each schema id a record + // carries, one entry that publishes it: for an id the write stages a record under, the entry the writer registered (or passed) + // for it, which owns the id; for every other id the arrived (durable) entry that owns it, else the first that lists it as + // compatible, carried as the bytes it was read from. A registered entry never publishes an id nothing staged sits under, so + // the caller passes registered entries only for the ids it writes. Entries are written as chosen, never merged; a registered + // descriptor wins over an arrived entry with the same own id. Two arrived entries with the same own id and different schema + // names, or a record no chosen entry publishes, refuse the write. // Entries that tie are taken in one fixed order, so the catalog does not depend on the order they arrive in. private static IEnumerable Canonical(IEnumerable entries) => entries .OrderBy(entry => entry.SchemaName, StringComparer.Ordinal) @@ -78,9 +86,8 @@ namespace GameCult.Caching } else { - pick = Canonical(registered.Where(entry => entry.CompatibleSchemaIds.Contains(schemaId, StringComparer.Ordinal))).FirstOrDefault(); - isRegistered = pick != null; - pick ??= Canonical(arrived.Where(entry => entry.CompatibleSchemaIds.Contains(schemaId, StringComparer.Ordinal))).FirstOrDefault(); + pick = Canonical(arrived.Where(entry => entry.CompatibleSchemaIds.Contains(schemaId, StringComparer.Ordinal))).FirstOrDefault(); + isRegistered = false; } if (pick == null) @@ -136,6 +143,22 @@ namespace GameCult.Caching public CultVariantDelta? Variant { get; set; } } + // One record a write puts in a single-file store: encoded, with the catalog entry that publishes the id it is stored under and + // whether its document holds element ids (null when the writer cannot see, as for a raw payload). + internal sealed class CultStagedRecord + { + internal CultStagedRecord(CultPersistedRecord record, CultSchemaCatalogEntry entry, bool? holdsIds) + { + Record = record; + Entry = entry; + HoldsIds = holdsIds; + } + + internal CultPersistedRecord Record { get; } + internal CultSchemaCatalogEntry Entry { get; } + internal bool? HoldsIds { get; } + } + /// /// A record a store carries untouched because no registered type owns its schema id or lists it as compatible, for example a /// record of a type this build does not have, or one renamed without declaring its old id. The store keeps its bytes as the @@ -1999,10 +2022,10 @@ namespace GameCult.Caching // Read under the gate, by a store judging a merge. internal bool HoldsVariants => _variantKeys.Count > 0; - // A store wrote its whole view: every record it wrote is stored under its type's own id at the storedAt the write gave it, + // A store wrote: every record it wrote is stored under its type's own id at the storedAt the write gave it, // and a plain record carries the ids it held, so it is no longer in-memory only. A variant's ids are not cleared here: the // store wrote the delta it was handed, and only a write of the variant changes that. - internal void WroteWholeView(IEnumerable written) => Held(() => + internal void Wrote(IEnumerable written) => Held(() => { foreach (var record in written) { @@ -3444,6 +3467,25 @@ namespace GameCult.Caching return directoryStore ? DirectoryFormatV4 : holdsVariants ? CultPersistedStoreSnapshot.FormatV2 : CultPersistedStoreSnapshot.FormatV1; } + // The header a write leaves, for either store kind: HeaderFor decides, given what the write can see. Marked when a record it + // writes holds an id; kept marked while the store carries a record it cannot show holds none (an opaque payload it writes, or + // a copied record that is not known id-free); otherwise by content. A copied record is known id-free when this store holds + // that key under the same stored identity as the file and its document holds no id. + protected string HeaderForWrite(string? durableHeader, IEnumerable written, IEnumerable copied, bool directoryStore, bool holdsVariants) + { + var holds = written.ToArray(); + var holdsIds = holds.Any(value => value == true); + var seen = HeaderFor(holdsIds, durableHeader, wholeStore: true, directoryStore, holdsVariants); + var unseen = HeaderFor(holdsIds, durableHeader, wholeStore: false, directoryStore, holdsVariants); + return seen == unseen || (holds.All(value => value != null) && copied.All(KnownIdFree)) ? seen : unseen; + } + + private bool KnownIdFree(CultPersistedRecord record) => + Entries.TryGetValue(record.Key, out var held) && + held.StoredSchemaId == record.SchemaId && + held.StoredAt == record.StoredAt && + !held.HoldsIds; + // Once attached, a hold on the cache's gate: the store and its cache share one lock, so a load calling back into // the cache can never take the gate after the store lock, and a direct call publishes what it loaded when it // returns. File locks are always taken inside it. @@ -3725,9 +3767,10 @@ namespace GameCult.Caching }); } - // A plain flush writes this store's whole view under the lock: two writers never interleave bytes, but the last - // one wins over any file this runtime can read. Processes sharing a store must use conditional commit. A file it - // cannot read (truncated, trailing bytes, not a store, a format it does not read) is refused and left as it is. + // A flush applies this store's staged writes and removals onto what the file holds now, under the lock: two writers never + // interleave bytes, every record this cache did not stage stays as the file has it, and a plain flush is last-writer-wins + // only per staged key. A file it cannot read (truncated, trailing bytes, not a store, a format it does not read) is refused + // and left as it is. public override void PushAll() { ThrowIfReadOnly(); @@ -3736,17 +3779,11 @@ namespace GameCult.Caching using (AcquireLock(wait: true)) { var disk = ReadSnapshot(); - var foreign = ForeignOnDisk(disk); - RefuseForeign(_staged, foreign); - var view = WholeView(Entries.Values, disk, foreign); - WriteSnapshot( - view.Records, - Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()).ToArray(), - view.Arrived, - Entries.Values.Any(entry => entry.HoldsIds && !view.LaidBack.Contains(entry.Key.Value)), - disk?.FormatVersion, - wholeStore: foreign.Count == 0); - WroteWholeView(view.Records, view.LaidBack); + RefuseForeign(_staged, ForeignOnDisk(disk)); + JudgeMerge(Array.Empty(), Array.Empty(), disk ?? new CultPersistedStoreSnapshot()); + var (upserts, removals) = StagedWrites(Array.Empty(), Array.Empty()); + ApplyWriteSet(disk, upserts, removals); + Wrote(upserts.Select(staged => staged.Record)); } _staged.Clear(); @@ -3760,6 +3797,8 @@ namespace GameCult.Caching return Held(() => CommitBatchCore(request, wait)); } + // Conditions are checked against the file under the lock, and only the condition differs between a conditional and an + // unconditional commit: both apply the batch and every staged write onto the file as it is. private CultCommitOutcome CommitBatchCore(CultCommitRequest request, bool wait) { using var fileLock = AcquireLock(wait); @@ -3770,45 +3809,102 @@ namespace GameCult.Caching if (!request.ConditionsHold(disk.Records, Entries.Values)) return CultCommitOutcome.Mismatch; - // An unconditional commit writes what a flush would, this store's whole view plus the batch: last-writer-wins, - // and staged single writes land with it. A conditional commit (store clean) lands the batch onto the file as it is. - var ontoDisk = request.HasConditions && !IsDirty; - var foreign = ForeignOnDisk(disk); - RefuseForeign(request.Upserts.Concat(request.Deletes).Select(entry => entry.Key.Value).Concat(_staged), foreign); - - var view = ontoDisk ? null : WholeView(Entries.Values, disk, foreign); - var records = (view?.Records ?? disk.Records).ToDictionary(record => record.Key, StringComparer.Ordinal); - // The committer judged the batch against its own view; the file may have moved since. A merge is judged on the - // set the file will hold, exactly as that set would be judged loading: what other processes wrote (arriving), - // what they removed (departing), and this batch. - if (ontoDisk && Judging != null && MergeInvolvesVariants(request, disk)) - JudgeMerge(request, disk); - foreach (var entry in request.Deletes) - records.Remove(entry.Key.Value); - foreach (var entry in request.Upserts) - records[entry.Key.Value] = ToPersistedRecord(entry, SerializePayload); - - // Onto the file, only the batch's records are seen: the rest stay as they are, and the header they carry stays. An - // unconditional commit writes this store's whole view, so what its records hold decides. A record this commit - // replaces or removes no longer counts. - var replaced = request.Upserts.Concat(request.Deletes).Select(entry => entry.Key.Value).ToHashSet(StringComparer.Ordinal); - var holdsIds = request.Upserts.Any(entry => entry.HoldsIds) || - (!ontoDisk && Entries.Values.Any(entry => entry.HoldsIds && !replaced.Contains(entry.Key.Value) && !view!.LaidBack.Contains(entry.Key.Value))); - var registered = Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()) - .Concat(request.Upserts.Select(entry => entry.Descriptor.ToCatalogEntry())) - .ToArray(); - WriteSnapshot(records.Values, registered, view?.Arrived ?? disk.SchemaCatalog, holdsIds, disk.FormatVersion, wholeStore: !ontoDisk && foreign.Count == 0); + RefuseForeign(request.Upserts.Concat(request.Deletes).Select(entry => entry.Key.Value).Concat(_staged), ForeignOnDisk(disk)); + JudgeMerge(request.Upserts, request.Deletes, disk); + var (upserts, removals) = StagedWrites(request.Upserts, request.Deletes); + ApplyWriteSet(disk, upserts, removals); foreach (var entry in request.Deletes) Entries.TryRemove(entry.Key.Value, out _); foreach (var entry in request.Upserts) Entries[entry.Key.Value] = entry; - if (view != null) - WroteWholeView(records.Values, view.LaidBack); + Wrote(upserts.Select(staged => staged.Record)); _staged.Clear(); MarkFlushSucceeded(); return CultCommitOutcome.Committed; } + // What a write puts in the store and takes out of it: the staged keys as this cache holds them now (a key it no longer holds + // is a removal), and a commit's batch, which wins over a staged write of the same key. Records it did not stage are not here. + private (List Upserts, List Removals) StagedWrites( + IReadOnlyList batchUpserts, + IReadOnlyList batchDeletes) + { + var documents = new Dictionary(StringComparer.Ordinal); + var removals = new HashSet(StringComparer.Ordinal); + foreach (var key in _staged) + { + if (Entries.TryGetValue(key, out var held)) + documents[key] = held; + else + removals.Add(key); + } + + foreach (var entry in batchDeletes) + { + documents.Remove(entry.Key.Value); + removals.Add(entry.Key.Value); + } + + foreach (var entry in batchUpserts) + { + documents[entry.Key.Value] = entry; + removals.Remove(entry.Key.Value); + } + + var catalogEntries = new Dictionary(); + var upserts = documents.Values.Select(document => + { + if (!catalogEntries.TryGetValue(document.Descriptor, out var catalogEntry)) + catalogEntries[document.Descriptor] = catalogEntry = document.Descriptor.ToCatalogEntry(); + return new CultStagedRecord(ToPersistedRecord(document, SerializePayload), catalogEntry, document.HoldsIds); + }).ToList(); + return (upserts, removals.ToList()); + } + + /// + /// The one write of this store: under its lock, the durable store read there with the upserts written and the removals + /// and the upserts' keys taken out of what is copied, every other record copied forward as stored. CultMesh's + /// single-file document writes call this, with no cache attached. + /// + internal void ApplyWriteSet(IReadOnlyCollection upserts, IReadOnlyCollection removals) => Held(() => + { + using (AcquireLock(wait: true)) + ApplyWriteSet(ReadSnapshot(), upserts, removals); + }); + + // Flush, commit and Mesh's write all end here, with the durable store they read under the lock they hold. + private void ApplyWriteSet(CultPersistedStoreSnapshot? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals) + { + var written = upserts.Select(staged => staged.Record.Key).ToHashSet(StringComparer.Ordinal); + var removed = removals.ToHashSet(StringComparer.Ordinal); + var copied = (durable?.Records ?? Array.Empty()) + .Where(record => !written.Contains(record.Key) && !removed.Contains(record.Key)) + .ToArray(); + var records = copied.Concat(upserts.Select(staged => staged.Record)) + .OrderBy(record => record.Key, StringComparer.Ordinal) + .ToArray(); + var snapshot = new CultPersistedStoreSnapshot + { + FormatVersion = HeaderForWrite( + durable?.FormatVersion, + upserts.Select(staged => staged.HoldsIds), + copied, + directoryStore: false, + holdsVariants: records.Any(record => record.Variant != null)), + SchemaCatalog = CultSchemaCatalogEntry.Derive( + records, + upserts.Select(staged => staged.Entry).Distinct().ToArray(), + durable?.SchemaCatalog ?? Array.Empty()) + .OrderBy(entry => entry.SchemaName, StringComparer.Ordinal) + .ThenBy(entry => entry.SchemaId, StringComparer.Ordinal) + .ToArray(), + Records = records + }; + + Directory.CreateDirectory(FileInfo.DirectoryName!); + WriteSnapshotAtomically(FileInfo.FullName, SerializeSnapshot(snapshot)); + } + private void RefuseForeign( IEnumerable written, IReadOnlyDictionary foreign) => @@ -3834,21 +3930,43 @@ namespace GameCult.Caching return foreign; } + // The file now holds every record written, under the id and at the storedAt the write gave it: the store's entries and the + // cache's say so. A copied record is on disk as the file had it, so nothing learns anything new about it: in particular the + // ids it minted at load are still in memory only. + private void Wrote(IEnumerable written) + { + var records = written.ToArray(); + foreach (var record in records) + { + if (!Entries.TryGetValue(record.Key, out var entry)) + continue; + entry.StoredSchemaId = record.SchemaId; + entry.StoredAt = record.StoredAt; + } + + Cache?.Wrote(records); + } + // Only a variant makes a merge more than the committer's own judgement: a variant it lands, one another writer wrote // (read from the record's header, never its payload), or one this cache holds. A plain-only store never decodes - // another writer's records, so a schema this cache does not register cannot fail a commit that involves no variant. - private bool MergeInvolvesVariants(CultCommitRequest request, CultPersistedStoreSnapshot disk) => + // another writer's records, so a schema this cache does not register cannot fail a write that involves no variant. + private bool MergeInvolvesVariants(IReadOnlyList batchUpserts, CultPersistedStoreSnapshot disk) => Cache?.HoldsVariants == true || - request.Upserts.Any(entry => entry.Variant != null) || + batchUpserts.Any(entry => entry.Variant != null) || disk.Records.Any(record => record.Variant != null); - private void JudgeMerge(CultCommitRequest request, CultPersistedStoreSnapshot disk) + // A write lands on the file as it is now, so the set the file will hold is judged exactly as that set would be judged + // loading: what other processes wrote (arriving), what they removed (departing), and this write's batch. A staged key is this + // cache's own, already judged when it was staged, so it is neither arriving nor departing. + private void JudgeMerge(IReadOnlyList batchUpserts, IReadOnlyList batchDeletes, CultPersistedStoreSnapshot disk) { - var landing = request.Upserts.Select(entry => entry.Key.Value).ToHashSet(StringComparer.Ordinal); - var removing = request.Deletes.Select(entry => entry.Key.Value).ToHashSet(StringComparer.Ordinal); + if (Judging == null || !MergeInvolvesVariants(batchUpserts, disk)) + return; + var landing = batchUpserts.Select(entry => entry.Key.Value).Concat(_staged).ToHashSet(StringComparer.Ordinal); + var removing = batchDeletes.Select(entry => entry.Key.Value).ToHashSet(StringComparer.Ordinal); var onDisk = disk.Records.Select(record => record.Key).ToHashSet(StringComparer.Ordinal); - var arriving = new List(request.Upserts); - var departing = new List(request.Deletes); + var arriving = new List(batchUpserts); + var departing = new List(batchDeletes); foreach (var record in disk.Records) { if (landing.Contains(record.Key) || removing.Contains(record.Key)) @@ -3868,7 +3986,8 @@ namespace GameCult.Caching departing.AddRange(Entries.Values.Where(known => !onDisk.Contains(known.Key.Value) && !landing.Contains(known.Key.Value) && !removing.Contains(known.Key.Value))); - Judging!(arriving, departing); + if (arriving.Count > 0 || departing.Count > 0) + Judging(arriving, departing); } // Open, flush and commit read the file through this one call, so they agree on which files are stores. A file that is @@ -3888,20 +4007,10 @@ namespace GameCult.Caching } } - // CultMesh's single-file document helpers reach this store file only through these two members, so they share its one - // reader, its lock and its atomic replace. c2a-write-set replaces ReplaceDurable with ApplyWriteSet. + // CultMesh's single-file document helpers reach this store file only through this read and ApplyWriteSet, so they share + // its one reader, its lock and its atomic replace. internal CultPersistedStoreSnapshot? ReadDurable() => Held(ReadSnapshot); - internal void ReplaceDurable(Func next) => Held(() => - { - using (AcquireLock(wait: true)) - { - var snapshot = next(ReadSnapshot()); - Directory.CreateDirectory(FileInfo.DirectoryName!); - WriteSnapshotAtomically(FileInfo.FullName, SerializeSnapshot(snapshot)); - } - }); - private byte[]? ReadDisk() { FileInfo.Refresh(); @@ -3940,23 +4049,6 @@ namespace GameCult.Caching return (int)attributes != -1 && (attributes & FileAttributes.ReparsePoint) != 0; } - private void WriteSnapshot(IEnumerable records, IReadOnlyCollection registered, IReadOnlyCollection arrived, bool holdsElementIds, string? existingHeader, bool wholeStore) - { - var ordered = records.OrderBy(record => record.Key, StringComparer.Ordinal).ToArray(); - var snapshot = new CultPersistedStoreSnapshot - { - FormatVersion = HeaderFor(holdsElementIds, existingHeader, wholeStore, directoryStore: false, holdsVariants: ordered.Any(record => record.Variant != null)), - SchemaCatalog = CultSchemaCatalogEntry.Derive(ordered, registered, arrived) - .OrderBy(entry => entry.SchemaName, StringComparer.Ordinal) - .ThenBy(entry => entry.SchemaId, StringComparer.Ordinal) - .ToArray(), - Records = ordered - }; - - Directory.CreateDirectory(FileInfo.DirectoryName!); - WriteSnapshotAtomically(FileInfo.FullName, SerializeSnapshot(snapshot)); - } - // The lock is a sidecar opened exclusively, which excludes other handles in this process and in others alike. private FileStream? AcquireLock(bool wait) { diff --git a/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs b/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs index c6ce2167..495b2273 100644 --- a/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs +++ b/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs @@ -108,8 +108,7 @@ namespace GameCult.Mesh descriptor.ToCatalogEntry(), storedAt, CultDocumentMessagePackSerialization.SerializeUntyped(document, typeof(TDocument)), - CultElementIds.Holds(document), - contentKnown: true); + CultElementIds.Holds(document)); } catch { @@ -147,7 +146,7 @@ namespace GameCult.Mesh byte[] payload) { if (schema == null) throw new ArgumentNullException(nameof(schema)); - WriteSingleFileDocumentPayload(path, key, schema.ToCatalogEntry(payload), storedAt, payload, holdsIds: false, contentKnown: false); + WriteSingleFileDocumentPayload(path, key, schema.ToCatalogEntry(payload), storedAt, payload, holdsIds: null); } /// @@ -230,8 +229,7 @@ namespace GameCult.Mesh CultSchemaCatalogEntry catalogEntry, string? storedAt, byte[] payload, - bool holdsIds, - bool contentKnown) + bool? holdsIds) { if (string.IsNullOrWhiteSpace(path)) throw new ArgumentException("Value must be non-empty.", nameof(path)); if (catalogEntry == null) throw new ArgumentNullException(nameof(catalogEntry)); @@ -239,24 +237,24 @@ namespace GameCult.Mesh throw new ArgumentException("Catalog entry must include a schema id.", nameof(catalogEntry)); payload ??= Array.Empty(); - // A typed write sees the document it replaces the file with, so its ids decide. A raw payload is opaque: the writer - // cannot see whether it holds ids, so a file already marked stays marked. The store reads the file it replaces under - // its lock: one this runtime cannot read refuses the write instead of being overwritten. - StoreAt(path).ReplaceDurable(existing => new CultPersistedStoreSnapshot - { - FormatVersion = CacheBackingStore.HeaderFor(holdsIds, existing?.FormatVersion, wholeStore: contentKnown, directoryStore: false), - SchemaCatalog = new[] { catalogEntry }, - Records = new[] + // The write replaces this one key and copies every other record and catalog entry forward as the store holds them, under + // the store's lock. A typed write sees its document, so its ids decide the header; a raw payload is opaque (holdsIds null), + // so a file already marked stays marked. A store this runtime cannot read refuses the write instead of being overwritten. + StoreAt(path).ApplyWriteSet( + new[] { - new CultPersistedRecord - { - Key = key.Value, - SchemaId = catalogEntry.SchemaId, - StoredAt = string.IsNullOrWhiteSpace(storedAt) ? DateTimeOffset.UtcNow.ToString("O") : storedAt!, - Payload = payload - } - } - }); + new CultStagedRecord( + new CultPersistedRecord + { + Key = key.Value, + SchemaId = catalogEntry.SchemaId, + StoredAt = string.IsNullOrWhiteSpace(storedAt) ? DateTimeOffset.UtcNow.ToString("O") : storedAt!, + Payload = payload + }, + catalogEntry, + holdsIds) + }, + Array.Empty()); } private static SingleFileMessagePackBackingStore StoreAt(string path) => new SingleFileMessagePackBackingStore(path);