From 2a3773a4634e12533327093582f807fc32565e16 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sat, 3 Oct 2026 10:47:30 +0200 Subject: [PATCH] CultCache: the store owns a file's bytes, not the record object Delete RawBytes from CultPersistedRecord and CultSchemaCatalogEntry, with every drop-on-assign hook and the catalog's content guard. A decoded record or snapshot carries no stored bytes, so an in-place edit of its payload or overrides can never be shadowed by cached bytes, and the public snapshot and record serializers encode every record from its fields. SingleFileBackingStore now reads through ReadStore, which returns the snapshot and, parallel to it, the exact slice the file holds for each catalog entry and record. Its durable index keeps, by record key, that slice and its SHA-256 (hashed when a comparison asks), which is the record's version. A write re-encodes every record the writer staged and copies every other record, and every catalog entry it keeps, as the slice the file holds. The directory store keeps its manifest's catalog slices the same way. SerializeSnapshot/DeserializeSnapshot hooks become SerializeStore/ReadStore plus SerializeCatalogEntry. A staged but unchanged non-canonical record is now re-encoded canonically and gets a new version; an untouched one keeps its bytes and version. Tests: in-place edits through the snapshot API (payload, override) and through a held document (collection, nested object) are written as edited; an untouched non-canonical record survives a neighbour's write byte for byte; a staged one is re-encoded. The cost test counts record hashes. Co-Authored-By: Claude Sonnet 5.5 --- .../CultDocumentMessagePackSerialization.cs | 94 ++++---- .../DirectoryMessagePackBackingStore.cs | 58 ++--- .../Contracts/cultcache-persistence-format.md | 7 + src/GameCult.Caching/CultCache.cs | 207 +++++++++++------- .../CompatibleSchemaIdTests.cs | 3 +- .../ConditionalCommitTests.cs | 4 +- tests/GameCult.Caching.Tests/StoreSlices.cs | 27 +++ tests/GameCult.Caching.Tests/VariantTests.cs | 5 +- tests/GameCult.Caching.Tests/WriteSetTests.cs | 180 +++++++++++++-- 9 files changed, 401 insertions(+), 184 deletions(-) create mode 100644 tests/GameCult.Caching.Tests/StoreSlices.cs diff --git a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs index 68ec46e9..67db59f4 100644 --- a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs +++ b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs @@ -155,22 +155,33 @@ public static class CultDocumentMessagePackSerialization return ReadPersistedRecord(ref reader); } - public static byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot) + /// + /// Encodes the snapshot: every catalog entry and record from its fields, so what a caller edited is what is written. + /// + public static byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot) => + SerializeStore( + snapshot.FormatVersion, + snapshot.SchemaCatalog.Select(SerializeCatalogEntry).ToArray(), + snapshot.Records.Select(SerializePersistedRecord).ToArray()); + + // A store file from its parts, each already encoded: what a store carries forward is the bytes its file held, what it writes is + // an encoding of the object. + internal static byte[] SerializeStore(string formatVersion, IReadOnlyList catalogEntries, IReadOnlyList records) { var buffer = new global::System.Buffers.ArrayBufferWriter(); var writer = new MessagePackWriter(buffer); writer.WriteArrayHeader(StoreSnapshotFieldCount); - writer.Write(snapshot.FormatVersion); - writer.WriteArrayHeader(snapshot.SchemaCatalog.Length); - foreach (var entry in snapshot.SchemaCatalog) + writer.Write(formatVersion); + writer.WriteArrayHeader(catalogEntries.Count); + foreach (var entry in catalogEntries) { - WriteSchemaCatalogEntry(ref writer, entry); + writer.WriteRaw(new ReadOnlySpan(entry)); } - writer.WriteArrayHeader(snapshot.Records.Length); - foreach (var record in snapshot.Records) + writer.WriteArrayHeader(records.Count); + foreach (var record in records) { - WritePersistedRecord(ref writer, record); + writer.WriteRaw(new ReadOnlySpan(record)); } writer.Flush(); @@ -185,7 +196,12 @@ public static class CultDocumentMessagePackSerialization /// , so a reader never takes part of a file for the whole and never skips a slot /// it does not understand. The rewriting paths use this same reader as their verdict on whether a file may be replaced. /// - public static CultPersistedStoreSnapshot DeserializeSnapshot(byte[] payload) + public static CultPersistedStoreSnapshot DeserializeSnapshot(byte[] payload) => ReadStore(payload).Snapshot; + + // The reader behind DeserializeSnapshot, and beside the snapshot the exact bytes the file held for each catalog entry and each + // record (parallel to the snapshot's arrays). The snapshot's objects hold no bytes: a store that carries entries forward keeps + // these slices itself. + internal static (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) ReadStore(byte[] payload) { try { @@ -200,27 +216,33 @@ public static class CultDocumentMessagePackSerialization var snapshot = new CultPersistedStoreSnapshot(); if (fieldCount == 0) - return snapshot; + return (snapshot, Array.Empty(), Array.Empty()); snapshot.FormatVersion = reader.ReadString() ?? throw new CultStoreUnreadableException("Store snapshot declares no format version; this runtime reads " + CultPersistedStoreSnapshot.FormatV1 + ", " + CultPersistedStoreSnapshot.FormatV2 + " and " + CultPersistedStoreSnapshot.FormatV3 + "."); var catalogCount = reader.ReadArrayHeader(); snapshot.SchemaCatalog = new CultSchemaCatalogEntry[catalogCount]; + var catalogBytes = new byte[catalogCount][]; for (var index = 0; index < catalogCount; index++) { + var start = reader.Position; snapshot.SchemaCatalog[index] = ReadSchemaCatalogEntry(ref reader); + catalogBytes[index] = reader.Sequence.Slice(start, reader.Position).ToArray(); } var recordCount = reader.ReadArrayHeader(); snapshot.Records = new CultPersistedRecord[recordCount]; + var recordBytes = new byte[recordCount][]; for (var index = 0; index < recordCount; index++) { + var start = reader.Position; snapshot.Records[index] = ReadPersistedRecord(ref reader); + recordBytes[index] = reader.Sequence.Slice(start, reader.Position).ToArray(); } RequirePublishedSchemas(snapshot); - return snapshot; + return (snapshot, catalogBytes, recordBytes); } catch (CultStoreUnreadableException) { @@ -306,12 +328,6 @@ public static class CultDocumentMessagePackSerialization private static void WritePersistedRecord(ref MessagePackWriter writer, CultPersistedRecord record) { - if (record.RawBytes is { } raw) - { - writer.WriteRaw(new ReadOnlySpan(raw)); - return; - } - writer.WriteArrayHeader(record.Variant == null ? PersistedRecordFieldCount : VariantRecordFieldCount); writer.Write(record.Key); writer.Write(record.SchemaId); @@ -395,7 +411,6 @@ public static class CultDocumentMessagePackSerialization public static CultPersistedRecord ReadPersistedRecord(ref MessagePackReader reader) { - var start = reader.Position; var fieldCount = reader.ReadArrayHeader(); var record = new CultPersistedRecord(); @@ -436,37 +451,19 @@ public static class CultDocumentMessagePackSerialization } } - record.RawBytes = reader.Sequence.Slice(start, reader.Position).ToArray(); return record; } - // An entry read from a store is written as the bytes it was read from, so a fact this runtime does not decode survives a - // write that only carries the entry. That holds only while the decoded fields are the ones read: an entry changed since is - // encoded from them. - private static void WriteSchemaCatalogEntry(ref MessagePackWriter writer, CultSchemaCatalogEntry entry) - { - if (entry.RawBytes is { } raw && EncodeSchemaCatalogEntry(entry).AsSpan().SequenceEqual(EncodeSchemaCatalogEntry(ReadRawSchemaCatalogEntry(raw)))) - writer.WriteRaw(new ReadOnlySpan(raw)); - else - WriteDecodedSchemaCatalogEntry(ref writer, entry); - } - - private static CultSchemaCatalogEntry ReadRawSchemaCatalogEntry(byte[] raw) - { - var reader = new MessagePackReader(raw); - return ReadSchemaCatalogEntry(ref reader); - } - - private static byte[] EncodeSchemaCatalogEntry(CultSchemaCatalogEntry entry) + internal static byte[] SerializeCatalogEntry(CultSchemaCatalogEntry entry) { var buffer = new global::System.Buffers.ArrayBufferWriter(); var writer = new MessagePackWriter(buffer); - WriteDecodedSchemaCatalogEntry(ref writer, entry); + WriteSchemaCatalogEntry(ref writer, entry); writer.Flush(); return buffer.WrittenSpan.ToArray(); } - private static void WriteDecodedSchemaCatalogEntry(ref MessagePackWriter writer, CultSchemaCatalogEntry entry) + private static void WriteSchemaCatalogEntry(ref MessagePackWriter writer, CultSchemaCatalogEntry entry) { writer.WriteArrayHeader(SchemaCatalogEntryFieldCount); writer.Write(entry.SchemaId); @@ -489,7 +486,6 @@ public static class CultDocumentMessagePackSerialization private static CultSchemaCatalogEntry ReadSchemaCatalogEntry(ref MessagePackReader reader) { - var start = reader.Position; var fieldCount = reader.ReadArrayHeader(); var entry = new CultSchemaCatalogEntry(); @@ -543,7 +539,6 @@ public static class CultDocumentMessagePackSerialization reader.Skip(); } - entry.RawBytes = reader.Sequence.Slice(start, reader.Position).ToArray(); return entry; } @@ -620,9 +615,9 @@ public class SingleFileMessagePackBackingStore : SingleFileBackingStore { } - protected override byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot) + protected override byte[] SerializeStore(string formatVersion, IReadOnlyList catalogEntries, IReadOnlyList records) { - return CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot); + return CultDocumentMessagePackSerialization.SerializeStore(formatVersion, catalogEntries, records); } protected override byte[] SerializeRecord(CultPersistedRecord record) @@ -630,11 +625,16 @@ public class SingleFileMessagePackBackingStore : SingleFileBackingStore return CultDocumentMessagePackSerialization.SerializePersistedRecord(record); } - protected override CultPersistedStoreSnapshot DeserializeSnapshot(byte[] data) + protected override byte[] SerializeCatalogEntry(CultSchemaCatalogEntry entry) { - var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(data); - CultDocumentMessagePackSerialization.RequireSingleFileFormat(snapshot); - return snapshot; + return CultDocumentMessagePackSerialization.SerializeCatalogEntry(entry); + } + + protected override (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) ReadStore(byte[] data) + { + var store = CultDocumentMessagePackSerialization.ReadStore(data); + CultDocumentMessagePackSerialization.RequireSingleFileFormat(store.Snapshot); + return store; } protected override byte[] SerializePayload(object document) diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index 30ec5075..bfb7eca0 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -75,7 +75,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore // writer created. A page that fails under an unmoved manifest is corruption, not a race. using var lease = AcquireCommitLease(wait: true, create: false); var manifestBytes = ReadManifestBytes(); - var manifest = ParseManifest(manifestBytes); + var (manifest, _) = ParseManifest(manifestBytes); Trace($"manifest records={manifest.Records.Length}"); try { @@ -181,7 +181,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore using var commitLease = AcquireCommitLease(wait, create: true); if (commitLease == null) return CultCommitOutcome.Contended; - var manifest = ReadManifest(); + var (manifest, catalogBytes) = ReadManifest(); if (request.HasConditions && !request.ConditionsHold( manifest.Records.ToDictionary(record => record.Key, record => VersionOf(record.Payload), StringComparer.Ordinal), Entries.Values)) return CultCommitOutcome.Mismatch; @@ -206,7 +206,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore _deletedKeys.TryRemove(entry.Key.Value, out _); } IsDirty = true; - WriteGeneration(manifest); + WriteGeneration(manifest, catalogBytes); return CultCommitOutcome.Committed; } catch @@ -236,9 +236,9 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore Directory.CreateDirectory(_manifestFile.DirectoryName!); using var commitLease = AcquireCommitLease(wait: true, create: true); - var manifest = ReadManifest(); + var (manifest, catalogBytes) = ReadManifest(); RefuseForeign(manifest, Array.Empty()); - WriteGeneration(manifest); + WriteGeneration(manifest, catalogBytes); }); } @@ -258,7 +258,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore }); // Runs under the commit lease: pages first, then the manifest that names them. - private void WriteGeneration(CultPersistedStoreSnapshot currentManifest) + private void WriteGeneration(CultPersistedStoreSnapshot currentManifest, IReadOnlyDictionary catalogBytes) { Directory.CreateDirectory(_recordDirectory.FullName); var currentIndex = currentManifest.Records.ToDictionary(record => record.Key, record => record, StringComparer.Ordinal); @@ -316,7 +316,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore copiesAny: currentIndex.Values.Any(record => !written.Contains(record.Key)), directoryStore: true, holdsVariants: false); - WriteManifest(targetCatalog, currentIndex.Values + WriteManifest(targetCatalog, catalogBytes, currentIndex.Values .OrderBy(record => record.Key, StringComparer.Ordinal) .ToArray(), header); @@ -327,16 +327,19 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore MarkFlushSucceeded(); } - private void WriteManifest(CultSchemaCatalogEntry[] catalog, CultPersistedRecord[] index, string header) - { - var manifest = new CultPersistedStoreSnapshot - { - FormatVersion = header, - SchemaCatalog = catalog, - Records = index - }; - WriteFileAtomically(_manifestFile.FullName, CultDocumentMessagePackSerialization.SerializeSnapshot(manifest)); - } + // The index records are this store's own and are encoded from their fields; a catalog entry the manifest already held is written + // as the slice it held, one the writer built is encoded. + private void WriteManifest( + CultSchemaCatalogEntry[] catalog, + IReadOnlyDictionary catalogBytes, + CultPersistedRecord[] index, + string header) => + WriteFileAtomically( + _manifestFile.FullName, + CultDocumentMessagePackSerialization.SerializeStore( + header, + catalog.Select(entry => catalogBytes.TryGetValue(entry, out var held) ? held : CultDocumentMessagePackSerialization.SerializeCatalogEntry(entry)).ToArray(), + index.Select(CultDocumentMessagePackSerialization.SerializePersistedRecord).ToArray())); private void LoadRecordPages( CultPersistedRecord[] records, @@ -452,7 +455,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore Payload = HashPayload(pagePayload) }; - private CultPersistedStoreSnapshot ReadManifest() => ParseManifest(ReadManifestBytes()); + private (CultPersistedStoreSnapshot Snapshot, Dictionary CatalogBytes) ReadManifest() => ParseManifest(ReadManifestBytes()); private byte[]? ReadManifestBytes() => File.Exists(_manifestFile.FullName) ? ReadAllBytesShared(_manifestFile.FullName) : null; @@ -478,29 +481,31 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore private InvalidOperationException Unsettled(Exception? last) => new( $"Directory store {_manifestFile.FullName} changed under every one of {UnleasedLoadAttempts} unlocked loads; it did not settle.", last); - // A missing manifest is an empty store. An existing manifest in any other format is refused. - private CultPersistedStoreSnapshot ParseManifest(byte[]? manifestBytes) + // A missing manifest is an empty store. An existing manifest in any other format is refused. Beside the snapshot, the bytes the + // manifest held for each catalog entry, by the entry this read produced: the slices a write carries forward. + private (CultPersistedStoreSnapshot Snapshot, Dictionary CatalogBytes) ParseManifest(byte[]? manifestBytes) { if (manifestBytes == null) { - return new CultPersistedStoreSnapshot + return (new CultPersistedStoreSnapshot { FormatVersion = IndexedFormatVersion, SchemaCatalog = Array.Empty(), Records = Array.Empty() - }; + }, new Dictionary()); } - CultPersistedStoreSnapshot snapshot; + (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) read; try { - snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(manifestBytes); + read = CultDocumentMessagePackSerialization.ReadStore(manifestBytes); } catch (CultStoreUnreadableException ex) when (ex.Path == null) { throw new CultStoreUnreadableException(ex.Message, _manifestFile.FullName, ex.InnerException); } + var snapshot = read.Snapshot; if (!string.Equals(snapshot.FormatVersion, IndexedFormatVersion, StringComparison.Ordinal) && !string.Equals(snapshot.FormatVersion, IndexedFormatVersionWithIds, StringComparison.Ordinal)) { @@ -509,7 +514,10 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore _manifestFile.FullName); } - return snapshot; + var catalogBytes = new Dictionary(); + for (var index = 0; index < snapshot.SchemaCatalog.Length; index++) + catalogBytes[snapshot.SchemaCatalog[index]] = read.CatalogBytes[index]; + return (snapshot, catalogBytes); } private string ContentAddressedRecordPath(CultPersistedRecord metadata) diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index 761200d0..9065a377 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -68,6 +68,13 @@ and a catalog entry's exact bytes, unknown fields included, so a copied record keeps its version. No type interprets a copied record: a write may hash another record's bytes to compare versions, and reads its header (key, schema id, variant slot), but never decodes its payload. +A record is encoded from the object the writer hands in, and every other record is copied as the store holds it. The +store that read a file owns that file's bytes: for each stored record, its exact slice of the file and the SHA-256 of that +slice, which is the record's version; likewise each catalog entry's slice. A decoded record or snapshot object carries no +bytes, so an object a caller edited in place is never mistaken for what the file holds. Consequently a record the writer +staged is re-encoded canonically and has the version of its new encoding, even when its content is unchanged, and a +record nobody staged keeps its bytes and its version for as long as no writer stages it. The public snapshot and record +serializers encode every record from its fields. A record written over one the store holds gets a storedAt later than the one it replaces. A document changed in place and not staged is not written. diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 1f509ae1..b932a1da 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -29,16 +29,10 @@ namespace GameCult.Caching public CultSchemaMemberCatalogEntry[] Members { get; set; } = Array.Empty(); - // The exact MessagePack bytes this entry was read from, when it was read from a store. A writer that carries the entry - // forward emits them as they are, so what this runtime does not decode (fields beyond the ones it knows, the width of an int, - // nil) survives; an entry whose decoded fields were changed since is encoded from them instead. Null for an entry a runtime - // built. - internal byte[]? RawBytes { get; set; } - // The catalog a write leaves, derived from the records the written store holds and nothing else. For each schema id a record // carries, one entry that publishes it: for an id the write stages a record under, the entry the writer registered (or passed) // for it, which owns the id; for every other id the arrived (durable) entry that owns it, else the first that lists it as - // compatible, carried as the bytes it was read from. A registered entry never publishes an id nothing staged sits under, so + // compatible (the store that read it carries its bytes as read). A registered entry never publishes an id nothing staged sits under, so // the caller passes registered entries only for the ids it writes. Entries are written as chosen, never merged; a registered // descriptor wins over an arrived entry with the same own id. Two arrived entries with the same own id and different schema // names, or a record no chosen entry publishes, refuse the write. @@ -134,24 +128,13 @@ namespace GameCult.Caching public sealed class CultPersistedRecord { - private string _key = string.Empty; - private string _schemaId = string.Empty; - private string _storedAt = string.Empty; - private byte[] _payload = Array.Empty(); - private CultVariantDelta? _variant; - - public string Key { get => _key; set { _key = value; RawBytes = null; } } - public string SchemaId { get => _schemaId; set { _schemaId = value; RawBytes = null; } } - public string StoredAt { get => _storedAt; set { _storedAt = value; RawBytes = null; } } - public byte[] Payload { get => _payload; set { _payload = value; RawBytes = null; } } + public string Key { get; set; } = string.Empty; + public string SchemaId { get; set; } = string.Empty; + public string StoredAt { get; set; } = string.Empty; + public byte[] Payload { get; set; } = Array.Empty(); // Non-null for a variant record (slot 4): its base and overrides. Its payload is empty. - public CultVariantDelta? Variant { get => _variant; set { _variant = value; RawBytes = null; } } - - // The exact bytes this record was read from, as a store file held them. A record is written as the bytes it was read from - // while no field has been assigned since, so a copied record stays byte-identical and its version (the SHA-256 of its - // stored bytes) is the one the file's bytes name, whatever encoding their writer chose. Assigning any field drops them. - internal byte[]? RawBytes { get; set; } + public CultVariantDelta? Variant { get; set; } } // One record a write puts in a single-file store: encoded, with the catalog entry that publishes the id it is stored under and @@ -3750,24 +3733,75 @@ namespace GameCult.Caching // A write into a store holding a variant lands only while the durable versions still equal these. private readonly Dictionary _lastRead = new(StringComparer.Ordinal); - protected abstract byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot); + /// + /// A store file from its parts, each already encoded: a catalog entry or record this store carries forward is the bytes its + /// file held, one it writes is or of the object. + /// + protected abstract byte[] SerializeStore(string formatVersion, IReadOnlyList catalogEntries, IReadOnlyList records); - // One record's persisted encoding: the bytes a version names. + // One record's persisted encoding: the bytes a version names, and the bytes a written record is stored as. protected abstract byte[] SerializeRecord(CultPersistedRecord record); - private string Version(CultPersistedRecord record) - { - using var sha256 = SHA256.Create(); - return VersionOf(sha256.ComputeHash(SerializeRecord(record))); - } + // One catalog entry's persisted encoding, for an entry the writer built. + protected abstract byte[] SerializeCatalogEntry(CultSchemaCatalogEntry entry); /// /// This store's one reader of a non-empty store file, and so its one verdict on whether the file may be replaced: open, /// flush and commit all ask it. It must throw for anything that is not exactly one /// store this runtime reads (a truncated file, bytes after the store, a missing or extra slot, a format or record it does - /// not know), so a rewrite never overwrites a file it cannot see. + /// not know), so a rewrite never overwrites a file it cannot see. It returns the decoded snapshot and, parallel to the + /// snapshot's catalog and records, the exact bytes the file holds for each: the store keeps those, never the snapshot's objects. /// - protected abstract CultPersistedStoreSnapshot DeserializeSnapshot(byte[] data); + protected abstract (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) ReadStore(byte[] data); + + // Test seam: how many record encodings this store has hashed, to prove what a write hashes. + internal int RecordsHashed; + + private string VersionOfBytes(byte[] bytes) + { + RecordsHashed++; + using var sha256 = SHA256.Create(); + return VersionOf(sha256.ComputeHash(bytes)); + } + + // What the store's file holds, as the store owns it: the snapshot as read, and keyed by record key the exact slice the file + // holds for each record, whose version is the SHA-256 of that slice, hashed when a comparison first asks. Catalog entries + // keep their slices by the entry object this read produced. A write copies a record it did not stage from its slice here + // and re-encodes every record it was handed, so no object a caller edits is ever mistaken for the bytes in the file. + private sealed class Durable + { + private readonly SingleFileBackingStore _owner; + private readonly Dictionary _records; + private readonly Dictionary _versions = new(StringComparer.Ordinal); + private readonly Dictionary _catalog = new(); + + internal Durable(SingleFileBackingStore owner, (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) read) + { + _owner = owner; + Snapshot = read.Snapshot; + _records = new Dictionary(StringComparer.Ordinal); + for (var index = 0; index < Snapshot.Records.Length; index++) + _records[Snapshot.Records[index].Key] = read.RecordBytes[index]; + for (var index = 0; index < Snapshot.SchemaCatalog.Length; index++) + _catalog.Add(Snapshot.SchemaCatalog[index], read.CatalogBytes[index]); + } + + internal CultPersistedStoreSnapshot Snapshot { get; } + + internal byte[] Slice(string key) => _records[key]; + + internal string Version(string key) + { + if (!_versions.TryGetValue(key, out var version)) + _versions[key] = version = _owner.VersionOfBytes(_records[key]); + return version; + } + + // The bytes to write for a catalog entry the write chose: the slice the file held when it is one of this read's entries, + // otherwise an encoding of the entry the writer built. + internal byte[] CatalogBytes(CultSchemaCatalogEntry entry) => + _catalog.TryGetValue(entry, out var bytes) ? bytes : _owner.SerializeCatalogEntry(entry); + } protected abstract byte[] SerializePayload(object document); protected abstract object DeserializePayload(Type documentType, byte[] payload); @@ -3783,8 +3817,8 @@ namespace GameCult.Caching if (IsDirty) return; - var snapshot = ReadSnapshot(); - if (snapshot == null) + var durable = ReadSnapshot(); + if (durable == null) { _lastRead.Clear(); SetLastSchemaMigrationReports(Array.Empty()); @@ -3792,13 +3826,14 @@ namespace GameCult.Caching return; } + var snapshot = durable.Snapshot; var reports = new List(snapshot.Records.Length); var persisted = new Dictionary(StringComparer.Ordinal); var foreign = new List(); var versions = new Dictionary(StringComparer.Ordinal); foreach (var record in snapshot.Records) { - var version = versions[record.Key] = Version(record); + var version = versions[record.Key] = durable.Version(record.Key); if (Foreign(record, snapshot.SchemaCatalog) is { } carried) { foreign.Add(carried); @@ -3864,7 +3899,7 @@ namespace GameCult.Caching using (AcquireLock(wait: true)) { var disk = ReadSnapshot(); - RefuseForeign(_staged, ForeignOnDisk(disk)); + RefuseForeign(_staged, disk); var (upserts, removals) = StagedWrites(Array.Empty(), Array.Empty()); Wrote(ApplyWriteSet(disk, upserts, removals)); } @@ -3888,12 +3923,13 @@ namespace GameCult.Caching if (fileLock == null) return CultCommitOutcome.Contended; - var disk = ReadSnapshot() ?? new CultPersistedStoreSnapshot(); + var disk = ReadSnapshot(); if (request.HasConditions && !request.ConditionsHold( - disk.Records.ToDictionary(record => record.Key, Version, StringComparer.Ordinal), Entries.Values)) + (disk?.Snapshot.Records ?? Array.Empty()).ToDictionary(record => record.Key, record => disk!.Version(record.Key), StringComparer.Ordinal), + Entries.Values)) return CultCommitOutcome.Mismatch; - RefuseForeign(request.Upserts.Concat(request.Deletes).Select(entry => entry.Key.Value).Concat(_staged), ForeignOnDisk(disk)); + RefuseForeign(request.Upserts.Concat(request.Deletes).Select(entry => entry.Key.Value).Concat(_staged), disk); var (upserts, removals) = StagedWrites(request.Upserts, request.Deletes); var written = ApplyWriteSet(disk, upserts, removals); foreach (var entry in request.Deletes) @@ -3959,12 +3995,15 @@ namespace GameCult.Caching // a variant, or would after this write, is written only while its record versions are what this store last read: a variant // resolves against other records, and a write decodes none it did not stage, so it cannot judge another writer's. Versions // only: a write hashes other records' bytes, never decodes them. Otherwise the whole write is refused and what it refused is - // reloaded (Refuse). Returns the records written, each with the version of the bytes written. - private IReadOnlyCollection<(CultStagedRecord Staged, string Version)> ApplyWriteSet(CultPersistedStoreSnapshot? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals) + // reloaded (Refuse). Every record the writer staged is encoded from its object now, and its version is the hash of that + // encoding; every other record is copied as the slice the file holds. Returns the records written, each with the version of + // the bytes written. + private IReadOnlyCollection<(CultStagedRecord Staged, string Version)> ApplyWriteSet(Durable? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals) { RefuseWriteIntoMovedVariantStore(durable, upserts, removals); + var durableRecords = durable?.Snapshot.Records ?? Array.Empty(); // A record written over one the store holds is stored at a storedAt later than the one it replaces, whoever wrote that. - var replaced = (durable?.Records ?? Array.Empty()).ToDictionary(record => record.Key, record => record.StoredAt, StringComparer.Ordinal); + var replaced = durableRecords.ToDictionary(record => record.Key, record => record.StoredAt, StringComparer.Ordinal); foreach (var staged in upserts) { if (replaced.TryGetValue(staged.Record.Key, out var durableAt)) @@ -3973,33 +4012,34 @@ namespace GameCult.Caching var written = upserts.Select(staged => staged.Record.Key).ToHashSet(StringComparer.Ordinal); var removed = removals.ToHashSet(StringComparer.Ordinal); - var copied = (durable?.Records ?? Array.Empty()) + var copied = durableRecords .Where(record => !written.Contains(record.Key) && !removed.Contains(record.Key)) .ToArray(); - var records = copied.Concat(upserts.Select(staged => staged.Record)) - .OrderBy(record => record.Key, StringComparer.Ordinal) + var encoded = upserts.Select(staged => (Staged: staged, Bytes: SerializeRecord(staged.Record))).ToArray(); + var records = copied.Concat(upserts.Select(staged => staged.Record)).ToArray(); + var fileRecords = copied.Select(record => (record.Key, Bytes: durable!.Slice(record.Key))) + .Concat(encoded.Select(pair => (pair.Staged.Record.Key, pair.Bytes))) + .OrderBy(pair => pair.Key, StringComparer.Ordinal) + .Select(pair => pair.Bytes) .ToArray(); - var snapshot = new CultPersistedStoreSnapshot - { - FormatVersion = HeaderForWrite( - durable?.FormatVersion, - upserts.Select(staged => staged.HoldsIds).ToArray(), - copiesAny: copied.Length > 0, - directoryStore: false, - holdsVariants: records.Any(record => record.Variant != null)), - SchemaCatalog = CultSchemaCatalogEntry.Derive( - records, - upserts.Select(staged => staged.Entry).Distinct().ToArray(), - durable?.SchemaCatalog ?? Array.Empty()) - .OrderBy(entry => entry.SchemaName, StringComparer.Ordinal) - .ThenBy(entry => entry.SchemaId, StringComparer.Ordinal) - .ToArray(), - Records = records - }; + var catalog = CultSchemaCatalogEntry.Derive( + records, + upserts.Select(staged => staged.Entry).Distinct().ToArray(), + durable?.Snapshot.SchemaCatalog ?? Array.Empty()) + .OrderBy(entry => entry.SchemaName, StringComparer.Ordinal) + .ThenBy(entry => entry.SchemaId, StringComparer.Ordinal) + .Select(entry => durable?.CatalogBytes(entry) ?? SerializeCatalogEntry(entry)) + .ToArray(); + var format = HeaderForWrite( + durable?.Snapshot.FormatVersion, + upserts.Select(staged => staged.HoldsIds).ToArray(), + copiesAny: copied.Length > 0, + directoryStore: false, + holdsVariants: records.Any(record => record.Variant != null)); - var versions = upserts.Select(staged => (staged, Version(staged.Record))).ToArray(); + var versions = encoded.Select(pair => (pair.Staged, VersionOfBytes(pair.Bytes))).ToArray(); Directory.CreateDirectory(FileInfo.DirectoryName!); - WriteSnapshotAtomically(FileInfo.FullName, SerializeSnapshot(snapshot)); + WriteSnapshotAtomically(FileInfo.FullName, SerializeStore(format, catalog, fileRecords)); foreach (var key in removed) _lastRead.Remove(key); foreach (var (staged, version) in versions) @@ -4007,13 +4047,13 @@ namespace GameCult.Caching return versions; } - private void RefuseWriteIntoMovedVariantStore(CultPersistedStoreSnapshot? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals) + private void RefuseWriteIntoMovedVariantStore(Durable? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals) { - var records = durable?.Records ?? Array.Empty(); + var records = durable?.Snapshot.Records ?? Array.Empty(); if (records.All(record => record.Variant == null) && upserts.All(staged => staged.Record.Variant == null)) return; var changed = new HashSet(StringComparer.Ordinal); - var versions = records.ToDictionary(record => record.Key, Version, StringComparer.Ordinal); + var versions = records.ToDictionary(record => record.Key, record => durable!.Version(record.Key), StringComparer.Ordinal); foreach (var (key, version) in versions) { if (!_lastRead.TryGetValue(key, out var read) || !SameStoredIdentity(read, version)) @@ -4035,9 +4075,9 @@ namespace GameCult.Caching var changedKeys = changed.OrderBy(key => key, StringComparer.Ordinal).ToArray(); Refuse( recordKeys.Concat(changedKeys), - key => Cache == null || !byKey.TryGetValue(key, out var record) || Foreign(record, durable!.SchemaCatalog) != null + key => Cache == null || !byKey.TryGetValue(key, out var record) || Foreign(record, durable!.Snapshot.SchemaCatalog) != null ? null - : ToStoredDocument(record, versions[key], durable!.SchemaCatalog, DeserializePayload, out _), + : ToStoredDocument(record, versions[key], durable!.Snapshot.SchemaCatalog, DeserializePayload, out _), key => { _staged.Remove(key); @@ -4061,30 +4101,31 @@ namespace GameCult.Caching inner)); } - private void RefuseForeign( - IEnumerable written, - IReadOnlyDictionary foreign) => + private void RefuseForeign(IEnumerable written, Durable? disk) + { + var foreign = ForeignOnDisk(disk); RefuseForeign( written, - key => foreign.TryGetValue(key, out var carried) ? carried.Foreign : null, + key => foreign.TryGetValue(key, out var carried) ? carried : null, key => { _staged.Remove(key); IsDirty = _staged.Count > 0; - if (foreign.TryGetValue(key, out var carried)) - _lastRead[key] = Version(carried.Record); + if (foreign.ContainsKey(key)) + _lastRead[key] = disk!.Version(key); else _lastRead.Remove(key); }); + } // The records the file holds that no registered type claims, by key. - private Dictionary ForeignOnDisk(CultPersistedStoreSnapshot? disk) + private Dictionary ForeignOnDisk(Durable? disk) { - var foreign = new Dictionary(StringComparer.Ordinal); - foreach (var record in disk?.Records ?? Array.Empty()) + var foreign = new Dictionary(StringComparer.Ordinal); + foreach (var record in disk?.Snapshot.Records ?? Array.Empty()) { - if (Foreign(record, disk!.SchemaCatalog) is { } carried) - foreign[record.Key] = (carried, record); + if (Foreign(record, disk!.Snapshot.SchemaCatalog) is { } carried) + foreign[record.Key] = carried; } return foreign; @@ -4110,14 +4151,14 @@ namespace GameCult.Caching // Open, flush and commit read the file through this one call, so they agree on which files are stores. A file that is // gone is an empty store: nothing to read, nothing to lose. A zero-byte file is not a store; it is refused. - private CultPersistedStoreSnapshot? ReadSnapshot() + private Durable? ReadSnapshot() { var bytes = ReadDisk(); if (bytes == null) return null; try { - return DeserializeSnapshot(bytes); + return new Durable(this, ReadStore(bytes)); } catch (CultStoreUnreadableException ex) when (ex.Path == null) { @@ -4127,7 +4168,7 @@ namespace GameCult.Caching // CultMesh's single-file document helpers reach this store file only through this read and ApplyWriteSet, so they share // its one reader, its lock and its atomic replace. - internal CultPersistedStoreSnapshot? ReadDurable() => Held(ReadSnapshot); + internal CultPersistedStoreSnapshot? ReadDurable() => Held(() => ReadSnapshot()?.Snapshot); private byte[]? ReadDisk() { diff --git a/tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs b/tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs index 8fd1c73e..b87d5926 100644 --- a/tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs +++ b/tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs @@ -189,6 +189,7 @@ namespace GameCult.Caching.Tests var path = OldStore("copied.cc"); var descriptor = Declaring.GetRequired(); var before = Read(path); + var entryBefore = StoreSlices.CatalogEntry(path, OldId); using (var cache = Open(path, Declaring)) cache.Commit(batch => batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "e" }, new CultRecordKey("e"))); @@ -199,7 +200,7 @@ namespace GameCult.Caching.Tests var copied = written.Records.Single(record => record.Key == "d"); var original = before.Records.Single(); Assert.That((copied.StoredAt, copied.Payload), Is.EqualTo((original.StoredAt, original.Payload))); - Assert.That(written.SchemaCatalog.Single(entry => entry.SchemaId == OldId).RawBytes, Is.EqualTo(before.SchemaCatalog.Single().RawBytes), + Assert.That(StoreSlices.CatalogEntry(path, OldId), Is.EqualTo(entryBefore), "the entry that published the old id is carried as its bytes"); } diff --git a/tests/GameCult.Caching.Tests/ConditionalCommitTests.cs b/tests/GameCult.Caching.Tests/ConditionalCommitTests.cs index 0350e1e8..749fa66a 100644 --- a/tests/GameCult.Caching.Tests/ConditionalCommitTests.cs +++ b/tests/GameCult.Caching.Tests/ConditionalCommitTests.cs @@ -895,9 +895,9 @@ namespace GameCult.Caching.Tests base.Push(entry); } - protected override CultPersistedStoreSnapshot DeserializeSnapshot(byte[] data) + protected override (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) ReadStore(byte[] data) { - var snapshot = base.DeserializeSnapshot(data); + var snapshot = base.ReadStore(data); if (Pause is { } pause) { Paused = true; diff --git a/tests/GameCult.Caching.Tests/StoreSlices.cs b/tests/GameCult.Caching.Tests/StoreSlices.cs new file mode 100644 index 00000000..a27f2468 --- /dev/null +++ b/tests/GameCult.Caching.Tests/StoreSlices.cs @@ -0,0 +1,27 @@ +using System; +using System.IO; +using System.Linq; +using GameCult.Caching.MessagePack; + +namespace GameCult.Caching.Tests +{ + // The exact bytes a store file holds for its catalog entries and records. A read snapshot carries none: the store that read the + // file owns them, and these read them as the store's reader does. + internal static class StoreSlices + { + internal static byte[] CatalogEntry(string path, string schemaId) + { + var read = CultDocumentMessagePackSerialization.ReadStore(File.ReadAllBytes(path)); + return read.CatalogBytes[Array.FindIndex(read.Snapshot.SchemaCatalog, entry => entry.SchemaId == schemaId)]; + } + + internal static byte[][] CatalogEntries(string path) => + CultDocumentMessagePackSerialization.ReadStore(File.ReadAllBytes(path)).CatalogBytes; + + internal static byte[] Record(string path, string key) + { + var read = CultDocumentMessagePackSerialization.ReadStore(File.ReadAllBytes(path)); + return read.RecordBytes[Array.FindIndex(read.Snapshot.Records, record => record.Key == key)]; + } + } +} diff --git a/tests/GameCult.Caching.Tests/VariantTests.cs b/tests/GameCult.Caching.Tests/VariantTests.cs index 5c2ecc35..99304283 100644 --- a/tests/GameCult.Caching.Tests/VariantTests.cs +++ b/tests/GameCult.Caching.Tests/VariantTests.cs @@ -847,6 +847,7 @@ namespace GameCult.Caching.Tests var path = PathOf(viaFlush ? "drift-untouched-flush.cc" : "drift-untouched-commit.cc"); SeedDrift(path); var before = ReadStore(path); + var wideEntryBefore = StoreSlices.CatalogEntry(path, DriftWideId); using (var narrow = Open(path, registry: CultDocumentRegistry.ForTypes(new[] { typeof(DriftNarrow) }))) { @@ -870,8 +871,8 @@ namespace GameCult.Caching.Tests } Assert.That(OverriddenSlotsOn(path, DriftVariant.Value).OrderBy(slot => slot), Is.EqualTo(new[] { 0, 2 }), "the dropped slot's override is still stored"); - Assert.That(after.SchemaCatalog.Single(entry => entry.SchemaId == DriftWideId).RawBytes, - Is.EqualTo(before.SchemaCatalog.Single(entry => entry.SchemaId == DriftWideId).RawBytes)); + Assert.That(StoreSlices.CatalogEntry(path, DriftWideId), + Is.EqualTo(wideEntryBefore)); } // A type that lists Wide's id as compatible owns Wide's records: staging one writes it under the type's own id, at a later diff --git a/tests/GameCult.Caching.Tests/WriteSetTests.cs b/tests/GameCult.Caching.Tests/WriteSetTests.cs index 217b57ff..96c508fa 100644 --- a/tests/GameCult.Caching.Tests/WriteSetTests.cs +++ b/tests/GameCult.Caching.Tests/WriteSetTests.cs @@ -37,7 +37,29 @@ namespace GameCult.Caching.Tests public string Name = string.Empty; } - private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] { typeof(WsItem), typeof(WsOther) }); + [CultDocument("tests.write_set_nested", "tests.write_set_nested.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class WsNested + { + [Key(0)] + [CultName] + public string Name = string.Empty; + + [Key(1)] + public List Tags = new(); + + [Key(2)] + public WsInner Inner = new(); + } + + [MessagePackObject(AllowPrivate = true)] + internal sealed class WsInner + { + [Key(0)] + public int X; + } + + private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] { typeof(WsItem), typeof(WsOther), typeof(WsNested) }); private static readonly string ItemId = Registry.GetRequired(typeof(WsItem)).SchemaId; private static readonly CultRecordKey A = new("a"); private static readonly CultRecordKey X = new("x"); @@ -77,7 +99,7 @@ namespace GameCult.Caching.Tests } private static byte[] EntryBytes(string path, string schemaId) => - Read(path).SchemaCatalog.Single(entry => entry.SchemaId == schemaId).RawBytes!; + StoreSlices.CatalogEntry(path, schemaId); private static string[] Keys(string path) => Read(path).Records.Select(record => record.Key).OrderBy(key => key, StringComparer.Ordinal).ToArray(); @@ -289,6 +311,8 @@ namespace GameCult.Caching.Tests Assert.Ignore("CULTLIB_REAL_STORE does not name a copy of a real store."); var path = PathOf("real.cc"); File.Copy(source!, path); + var beforePath = PathOf("real-before.cc"); + File.Copy(source!, beforePath); var before = Read(path); using (var cache = Open(path, false)) @@ -298,11 +322,11 @@ namespace GameCult.Caching.Tests TestContext.Out.WriteLine($"real store: {before.Records.Length} records, {before.SchemaCatalog.Length} catalog entries, header {before.FormatVersion}"); Assert.That(after.Records.Length, Is.EqualTo(before.Records.Length + 1)); Assert.That(after.FormatVersion, Is.EqualTo(before.FormatVersion)); - byte[] Bytes(CultPersistedRecord record) => CultDocumentMessagePackSerialization.SerializeSnapshot(new CultPersistedStoreSnapshot { Records = new[] { record } }); - var changed = before.Records.Where(record => !Bytes(record).SequenceEqual(Bytes(after.Records.Single(kept => kept.Key == record.Key)))).Select(record => record.Key).ToArray(); + + var changed = before.Records.Where(record => !StoreSlices.Record(beforePath, record.Key).SequenceEqual(StoreSlices.Record(path, record.Key))).Select(record => record.Key).ToArray(); Assert.That(changed, Is.Empty, "records that are not byte-identical"); - var afterEntries = after.SchemaCatalog.Select(entry => entry.RawBytes!).ToList(); - var missing = before.SchemaCatalog.Where(entry => !afterEntries.Any(raw => raw.SequenceEqual(entry.RawBytes!))).Select(entry => entry.SchemaId).ToArray(); + var afterEntries = StoreSlices.CatalogEntries(path).ToList(); + var missing = StoreSlices.CatalogEntries(beforePath).Where(raw => !afterEntries.Any(other => other.SequenceEqual(raw))).Select(raw => BitConverter.ToString(raw)).ToArray(); Assert.That(missing, Is.Empty, "catalog entries that are not byte-identical"); } @@ -801,42 +825,30 @@ namespace GameCult.Caching.Tests }), Is.True, "the writer's own condition on the variant holds"); } - private sealed class CountingStore : SingleFileMessagePackBackingStore - { - public CountingStore(string path) : base(path) { } - - public int Encoded; - - protected override byte[] SerializeRecord(CultPersistedRecord record) - { - Encoded++; - return base.SerializeRecord(record); - } - } // A commit with no condition decides nothing about the other records, so it hashes none of them: only the record it writes - // is encoded to be named. A commit with a condition compares every durable record's version. + // is hashed to be named. A commit with a condition compares every durable record's version. [Test] public void AnUnconditionalCommitHashesOnlyTheRecordsItWrites() { var path = PathOf("cost.cc"); Seed(path, false, "a", "b", "c", "d", "e"); - var store = new CountingStore(path); + var store = new SingleFileMessagePackBackingStore(path); using var cache = new CultCache(Registry, CultCacheMessagePack.CreateCodec(Registry)); cache.AddBackingStore(store); cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); - store.Encoded = 0; + store.RecordsHashed = 0; Assert.That(cache.Commit(batch => batch.Upsert(typeof(WsItem), new WsItem { Name = "name-x" }, X)), Is.True); - Assert.That(store.Encoded, Is.EqualTo(1)); + Assert.That(store.RecordsHashed, Is.EqualTo(1)); - store.Encoded = 0; + store.RecordsHashed = 0; Assert.That(cache.Commit(batch => { batch.Expect(X, cache.Get(X)!); batch.Upsert(typeof(WsItem), new WsItem { Name = "name-x", Note = "again" }, X); }), Is.True); - Assert.That(store.Encoded, Is.GreaterThanOrEqualTo(6), "a condition hashes every durable record, which is how the count proves itself"); + Assert.That(store.RecordsHashed, Is.GreaterThanOrEqualTo(6), "a condition hashes every durable record, which is how the count proves itself"); } // Records no store has read or written are told apart however many are made in one clock tick. @@ -894,6 +906,126 @@ namespace GameCult.Caching.Tests private static string StoredAtOf(string path, string key) => Read(path).Records.Single(record => record.Key == key).StoredAt; + // The stored bytes belong to the store that read them, never to a record object: a record read through the public snapshot + // API is only a decoded copy, so an edit made to its payload in place is what a write of it stores. + [Test] + public void ARecordReadThroughTheSnapshotApiAndEditedInPlaceIsWrittenAsEdited() + { + var path = PathOf("edit-payload.cc"); + Seed(path, false, "k"); + var snapshot = Read(path); + OverwriteInPlace(snapshot.Records.Single().Payload, "seeded", "SEEDED"); + + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + + Assert.That(NoteOnDisk(path, false), Is.EqualTo("SEEDED")); + var record = CultDocumentMessagePackSerialization.DeserializePersistedRecord(StoreSlices.Record(path, "k")); + OverwriteInPlace(record.Payload, "SEEDED", "seeded"); + Assert.That( + CultDocumentMessagePackSerialization.DeserializePersistedRecord(CultDocumentMessagePackSerialization.SerializePersistedRecord(record)).Payload, + Is.EqualTo(record.Payload), "a single record is re-encoded from its fields too"); + } + + [Test] + public void AnOverrideReadThroughTheSnapshotApiAndEditedInPlaceIsWrittenAsEdited() + { + var path = PathOf("edit-override.cc"); + Seed(path, false, "a"); + using (var seed = Open(path, false)) + AddVariant(seed, "v", A); + var snapshot = Read(path); + var variant = snapshot.Records.Single(record => record.Key == "v").Variant!; + OverwriteInPlace(variant.Overrides.Single(entry => entry.Path[0].Slot == 1).Value, "variant-v", "VARIANT-v"); + + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + + using var fresh = Open(path, false); + Assert.That(fresh.Get(new CultRecordKey("v"))!.Note, Is.EqualTo("VARIANT-v")); + } + + // A held document edited in place (a list, a nested object) and put back as the same instance is written as edited, and a + // later write of a neighbour copies it as the file then holds it. + [Test] + public void AHeldDocumentEditedInPlaceAndPutBackIsWrittenAsEdited([Values] bool commit) + { + var path = PathOf("edit-held.cc"); + var n = new CultRecordKey("n"); + using (var seed = Open(path, false)) + seed.Commit(batch => batch.Upsert(typeof(WsNested), new WsNested { Name = "name-n", Tags = { "t1" }, Inner = { X = 1 } }, n)); + + using (var cache = Open(path, false)) + { + var held = cache.Get(n)!; + held.Tags.Add("t2"); + held.Inner.X = 9; + Land(cache, commit, typeof(WsNested), held, n); + } + + using (var fresh = Open(path, false)) + { + var read = fresh.Get(n)!; + Assert.That(read.Tags, Is.EqualTo(new[] { "t1", "t2" })); + Assert.That(read.Inner.X, Is.EqualTo(9)); + } + + var written = StoreSlices.Record(path, "n"); + using (var other = Open(path, false)) + Land(other, commit, typeof(WsItem), new WsItem { Name = "name-x" }, X); + Assert.That(StoreSlices.Record(path, "n"), Is.EqualTo(written), "the edit is copied as the file holds it"); + } + + // A record the writer did not stage keeps the bytes and the version its file holds, whatever encoding its writer chose, + // across any number of writes of its neighbours. + [Test] + public void AnUntouchedNonCanonicalRecordKeepsItsBytesAndVersionAcrossAWriteOfANeighbour([Values] bool commit) + { + var path = PathOf("untouched-wide.cc"); + Seed(path, false, "k"); + File.WriteAllBytes(path, WidenEveryRecord(File.ReadAllBytes(path), out var slices)); + using var cache = Open(path, false); + var version = cache.GetStored(K)!.StoredVersion; + Assert.That(version, Is.EqualTo(HexOfSha256(slices["k"]))); + + Land(cache, commit, typeof(WsItem), new WsItem { Name = "name-x" }, X); + Land(cache, commit, typeof(WsItem), new WsItem { Name = "name-y" }, new CultRecordKey("y")); + + Assert.That(StoreSlices.Record(path, "k"), Is.EqualTo(slices["k"])); + Assert.That(cache.GetStored(K)!.StoredVersion, Is.EqualTo(version)); + using var fresh = Open(path, false); + Assert.That(fresh.GetStored(K)!.StoredVersion, Is.EqualTo(version)); + } + + // A record the writer stages is encoded from its object, so a non-canonical record put back unchanged is written canonically + // and, being other bytes, has another version. + [Test] + public void AStagedNonCanonicalRecordPutBackUnchangedIsReEncodedAndGetsANewVersion([Values] bool commit) + { + var path = PathOf("staged-wide.cc"); + Seed(path, false, "k"); + File.WriteAllBytes(path, WidenEveryRecord(File.ReadAllBytes(path), out var slices)); + using var cache = Open(path, false); + var held = cache.Get(K)!; + + Land(cache, commit, typeof(WsItem), held, K); + + var written = StoreSlices.Record(path, "k"); + Assert.That(written, Is.Not.EqualTo(slices["k"])); + Assert.That(written, Is.EqualTo(CultDocumentMessagePackSerialization.SerializePersistedRecord(Read(path).Records.Single())), + "the record is in the canonical encoding"); + Assert.That(cache.GetStored(K)!.StoredVersion, Is.EqualTo(HexOfSha256(written))); + Assert.That(cache.GetStored(K)!.StoredVersion, Is.Not.EqualTo(HexOfSha256(slices["k"]))); + } + + private static void OverwriteInPlace(byte[] bytes, string from, string to) + { + var was = System.Text.Encoding.UTF8.GetBytes(from); + var now = System.Text.Encoding.UTF8.GetBytes(to); + Assert.That(now.Length, Is.EqualTo(was.Length)); + var at = IndexOf(bytes, was); + Assert.That(at, Is.GreaterThanOrEqualTo(0), $"'{from}' is in the bytes"); + Array.Copy(now, 0, bytes, at, now.Length); + } + private static void AddVariant(CultCache cache, string key, CultRecordKey baseKey) => cache.Commit(batch => batch.UpsertVariant(new CultRecordKey(key), baseKey, new[] {