diff --git a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs index 68ec46e9..67db59f4 100644 --- a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs +++ b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs @@ -155,22 +155,33 @@ public static class CultDocumentMessagePackSerialization return ReadPersistedRecord(ref reader); } - public static byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot) + /// + /// Encodes the snapshot: every catalog entry and record from its fields, so what a caller edited is what is written. + /// + public static byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot) => + SerializeStore( + snapshot.FormatVersion, + snapshot.SchemaCatalog.Select(SerializeCatalogEntry).ToArray(), + snapshot.Records.Select(SerializePersistedRecord).ToArray()); + + // A store file from its parts, each already encoded: what a store carries forward is the bytes its file held, what it writes is + // an encoding of the object. + internal static byte[] SerializeStore(string formatVersion, IReadOnlyList catalogEntries, IReadOnlyList records) { var buffer = new global::System.Buffers.ArrayBufferWriter(); var writer = new MessagePackWriter(buffer); writer.WriteArrayHeader(StoreSnapshotFieldCount); - writer.Write(snapshot.FormatVersion); - writer.WriteArrayHeader(snapshot.SchemaCatalog.Length); - foreach (var entry in snapshot.SchemaCatalog) + writer.Write(formatVersion); + writer.WriteArrayHeader(catalogEntries.Count); + foreach (var entry in catalogEntries) { - WriteSchemaCatalogEntry(ref writer, entry); + writer.WriteRaw(new ReadOnlySpan(entry)); } - writer.WriteArrayHeader(snapshot.Records.Length); - foreach (var record in snapshot.Records) + writer.WriteArrayHeader(records.Count); + foreach (var record in records) { - WritePersistedRecord(ref writer, record); + writer.WriteRaw(new ReadOnlySpan(record)); } writer.Flush(); @@ -185,7 +196,12 @@ public static class CultDocumentMessagePackSerialization /// , so a reader never takes part of a file for the whole and never skips a slot /// it does not understand. The rewriting paths use this same reader as their verdict on whether a file may be replaced. /// - public static CultPersistedStoreSnapshot DeserializeSnapshot(byte[] payload) + public static CultPersistedStoreSnapshot DeserializeSnapshot(byte[] payload) => ReadStore(payload).Snapshot; + + // The reader behind DeserializeSnapshot, and beside the snapshot the exact bytes the file held for each catalog entry and each + // record (parallel to the snapshot's arrays). The snapshot's objects hold no bytes: a store that carries entries forward keeps + // these slices itself. + internal static (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) ReadStore(byte[] payload) { try { @@ -200,27 +216,33 @@ public static class CultDocumentMessagePackSerialization var snapshot = new CultPersistedStoreSnapshot(); if (fieldCount == 0) - return snapshot; + return (snapshot, Array.Empty(), Array.Empty()); snapshot.FormatVersion = reader.ReadString() ?? throw new CultStoreUnreadableException("Store snapshot declares no format version; this runtime reads " + CultPersistedStoreSnapshot.FormatV1 + ", " + CultPersistedStoreSnapshot.FormatV2 + " and " + CultPersistedStoreSnapshot.FormatV3 + "."); var catalogCount = reader.ReadArrayHeader(); snapshot.SchemaCatalog = new CultSchemaCatalogEntry[catalogCount]; + var catalogBytes = new byte[catalogCount][]; for (var index = 0; index < catalogCount; index++) { + var start = reader.Position; snapshot.SchemaCatalog[index] = ReadSchemaCatalogEntry(ref reader); + catalogBytes[index] = reader.Sequence.Slice(start, reader.Position).ToArray(); } var recordCount = reader.ReadArrayHeader(); snapshot.Records = new CultPersistedRecord[recordCount]; + var recordBytes = new byte[recordCount][]; for (var index = 0; index < recordCount; index++) { + var start = reader.Position; snapshot.Records[index] = ReadPersistedRecord(ref reader); + recordBytes[index] = reader.Sequence.Slice(start, reader.Position).ToArray(); } RequirePublishedSchemas(snapshot); - return snapshot; + return (snapshot, catalogBytes, recordBytes); } catch (CultStoreUnreadableException) { @@ -306,12 +328,6 @@ public static class CultDocumentMessagePackSerialization private static void WritePersistedRecord(ref MessagePackWriter writer, CultPersistedRecord record) { - if (record.RawBytes is { } raw) - { - writer.WriteRaw(new ReadOnlySpan(raw)); - return; - } - writer.WriteArrayHeader(record.Variant == null ? PersistedRecordFieldCount : VariantRecordFieldCount); writer.Write(record.Key); writer.Write(record.SchemaId); @@ -395,7 +411,6 @@ public static class CultDocumentMessagePackSerialization public static CultPersistedRecord ReadPersistedRecord(ref MessagePackReader reader) { - var start = reader.Position; var fieldCount = reader.ReadArrayHeader(); var record = new CultPersistedRecord(); @@ -436,37 +451,19 @@ public static class CultDocumentMessagePackSerialization } } - record.RawBytes = reader.Sequence.Slice(start, reader.Position).ToArray(); return record; } - // An entry read from a store is written as the bytes it was read from, so a fact this runtime does not decode survives a - // write that only carries the entry. That holds only while the decoded fields are the ones read: an entry changed since is - // encoded from them. - private static void WriteSchemaCatalogEntry(ref MessagePackWriter writer, CultSchemaCatalogEntry entry) - { - if (entry.RawBytes is { } raw && EncodeSchemaCatalogEntry(entry).AsSpan().SequenceEqual(EncodeSchemaCatalogEntry(ReadRawSchemaCatalogEntry(raw)))) - writer.WriteRaw(new ReadOnlySpan(raw)); - else - WriteDecodedSchemaCatalogEntry(ref writer, entry); - } - - private static CultSchemaCatalogEntry ReadRawSchemaCatalogEntry(byte[] raw) - { - var reader = new MessagePackReader(raw); - return ReadSchemaCatalogEntry(ref reader); - } - - private static byte[] EncodeSchemaCatalogEntry(CultSchemaCatalogEntry entry) + internal static byte[] SerializeCatalogEntry(CultSchemaCatalogEntry entry) { var buffer = new global::System.Buffers.ArrayBufferWriter(); var writer = new MessagePackWriter(buffer); - WriteDecodedSchemaCatalogEntry(ref writer, entry); + WriteSchemaCatalogEntry(ref writer, entry); writer.Flush(); return buffer.WrittenSpan.ToArray(); } - private static void WriteDecodedSchemaCatalogEntry(ref MessagePackWriter writer, CultSchemaCatalogEntry entry) + private static void WriteSchemaCatalogEntry(ref MessagePackWriter writer, CultSchemaCatalogEntry entry) { writer.WriteArrayHeader(SchemaCatalogEntryFieldCount); writer.Write(entry.SchemaId); @@ -489,7 +486,6 @@ public static class CultDocumentMessagePackSerialization private static CultSchemaCatalogEntry ReadSchemaCatalogEntry(ref MessagePackReader reader) { - var start = reader.Position; var fieldCount = reader.ReadArrayHeader(); var entry = new CultSchemaCatalogEntry(); @@ -543,7 +539,6 @@ public static class CultDocumentMessagePackSerialization reader.Skip(); } - entry.RawBytes = reader.Sequence.Slice(start, reader.Position).ToArray(); return entry; } @@ -620,9 +615,9 @@ public class SingleFileMessagePackBackingStore : SingleFileBackingStore { } - protected override byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot) + protected override byte[] SerializeStore(string formatVersion, IReadOnlyList catalogEntries, IReadOnlyList records) { - return CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot); + return CultDocumentMessagePackSerialization.SerializeStore(formatVersion, catalogEntries, records); } protected override byte[] SerializeRecord(CultPersistedRecord record) @@ -630,11 +625,16 @@ public class SingleFileMessagePackBackingStore : SingleFileBackingStore return CultDocumentMessagePackSerialization.SerializePersistedRecord(record); } - protected override CultPersistedStoreSnapshot DeserializeSnapshot(byte[] data) + protected override byte[] SerializeCatalogEntry(CultSchemaCatalogEntry entry) { - var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(data); - CultDocumentMessagePackSerialization.RequireSingleFileFormat(snapshot); - return snapshot; + return CultDocumentMessagePackSerialization.SerializeCatalogEntry(entry); + } + + protected override (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) ReadStore(byte[] data) + { + var store = CultDocumentMessagePackSerialization.ReadStore(data); + CultDocumentMessagePackSerialization.RequireSingleFileFormat(store.Snapshot); + return store; } protected override byte[] SerializePayload(object document) diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index 30ec5075..bfb7eca0 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -75,7 +75,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore // writer created. A page that fails under an unmoved manifest is corruption, not a race. using var lease = AcquireCommitLease(wait: true, create: false); var manifestBytes = ReadManifestBytes(); - var manifest = ParseManifest(manifestBytes); + var (manifest, _) = ParseManifest(manifestBytes); Trace($"manifest records={manifest.Records.Length}"); try { @@ -181,7 +181,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore using var commitLease = AcquireCommitLease(wait, create: true); if (commitLease == null) return CultCommitOutcome.Contended; - var manifest = ReadManifest(); + var (manifest, catalogBytes) = ReadManifest(); if (request.HasConditions && !request.ConditionsHold( manifest.Records.ToDictionary(record => record.Key, record => VersionOf(record.Payload), StringComparer.Ordinal), Entries.Values)) return CultCommitOutcome.Mismatch; @@ -206,7 +206,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore _deletedKeys.TryRemove(entry.Key.Value, out _); } IsDirty = true; - WriteGeneration(manifest); + WriteGeneration(manifest, catalogBytes); return CultCommitOutcome.Committed; } catch @@ -236,9 +236,9 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore Directory.CreateDirectory(_manifestFile.DirectoryName!); using var commitLease = AcquireCommitLease(wait: true, create: true); - var manifest = ReadManifest(); + var (manifest, catalogBytes) = ReadManifest(); RefuseForeign(manifest, Array.Empty()); - WriteGeneration(manifest); + WriteGeneration(manifest, catalogBytes); }); } @@ -258,7 +258,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore }); // Runs under the commit lease: pages first, then the manifest that names them. - private void WriteGeneration(CultPersistedStoreSnapshot currentManifest) + private void WriteGeneration(CultPersistedStoreSnapshot currentManifest, IReadOnlyDictionary catalogBytes) { Directory.CreateDirectory(_recordDirectory.FullName); var currentIndex = currentManifest.Records.ToDictionary(record => record.Key, record => record, StringComparer.Ordinal); @@ -316,7 +316,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore copiesAny: currentIndex.Values.Any(record => !written.Contains(record.Key)), directoryStore: true, holdsVariants: false); - WriteManifest(targetCatalog, currentIndex.Values + WriteManifest(targetCatalog, catalogBytes, currentIndex.Values .OrderBy(record => record.Key, StringComparer.Ordinal) .ToArray(), header); @@ -327,16 +327,19 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore MarkFlushSucceeded(); } - private void WriteManifest(CultSchemaCatalogEntry[] catalog, CultPersistedRecord[] index, string header) - { - var manifest = new CultPersistedStoreSnapshot - { - FormatVersion = header, - SchemaCatalog = catalog, - Records = index - }; - WriteFileAtomically(_manifestFile.FullName, CultDocumentMessagePackSerialization.SerializeSnapshot(manifest)); - } + // The index records are this store's own and are encoded from their fields; a catalog entry the manifest already held is written + // as the slice it held, one the writer built is encoded. + private void WriteManifest( + CultSchemaCatalogEntry[] catalog, + IReadOnlyDictionary catalogBytes, + CultPersistedRecord[] index, + string header) => + WriteFileAtomically( + _manifestFile.FullName, + CultDocumentMessagePackSerialization.SerializeStore( + header, + catalog.Select(entry => catalogBytes.TryGetValue(entry, out var held) ? held : CultDocumentMessagePackSerialization.SerializeCatalogEntry(entry)).ToArray(), + index.Select(CultDocumentMessagePackSerialization.SerializePersistedRecord).ToArray())); private void LoadRecordPages( CultPersistedRecord[] records, @@ -452,7 +455,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore Payload = HashPayload(pagePayload) }; - private CultPersistedStoreSnapshot ReadManifest() => ParseManifest(ReadManifestBytes()); + private (CultPersistedStoreSnapshot Snapshot, Dictionary CatalogBytes) ReadManifest() => ParseManifest(ReadManifestBytes()); private byte[]? ReadManifestBytes() => File.Exists(_manifestFile.FullName) ? ReadAllBytesShared(_manifestFile.FullName) : null; @@ -478,29 +481,31 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore private InvalidOperationException Unsettled(Exception? last) => new( $"Directory store {_manifestFile.FullName} changed under every one of {UnleasedLoadAttempts} unlocked loads; it did not settle.", last); - // A missing manifest is an empty store. An existing manifest in any other format is refused. - private CultPersistedStoreSnapshot ParseManifest(byte[]? manifestBytes) + // A missing manifest is an empty store. An existing manifest in any other format is refused. Beside the snapshot, the bytes the + // manifest held for each catalog entry, by the entry this read produced: the slices a write carries forward. + private (CultPersistedStoreSnapshot Snapshot, Dictionary CatalogBytes) ParseManifest(byte[]? manifestBytes) { if (manifestBytes == null) { - return new CultPersistedStoreSnapshot + return (new CultPersistedStoreSnapshot { FormatVersion = IndexedFormatVersion, SchemaCatalog = Array.Empty(), Records = Array.Empty() - }; + }, new Dictionary()); } - CultPersistedStoreSnapshot snapshot; + (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) read; try { - snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(manifestBytes); + read = CultDocumentMessagePackSerialization.ReadStore(manifestBytes); } catch (CultStoreUnreadableException ex) when (ex.Path == null) { throw new CultStoreUnreadableException(ex.Message, _manifestFile.FullName, ex.InnerException); } + var snapshot = read.Snapshot; if (!string.Equals(snapshot.FormatVersion, IndexedFormatVersion, StringComparison.Ordinal) && !string.Equals(snapshot.FormatVersion, IndexedFormatVersionWithIds, StringComparison.Ordinal)) { @@ -509,7 +514,10 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore _manifestFile.FullName); } - return snapshot; + var catalogBytes = new Dictionary(); + for (var index = 0; index < snapshot.SchemaCatalog.Length; index++) + catalogBytes[snapshot.SchemaCatalog[index]] = read.CatalogBytes[index]; + return (snapshot, catalogBytes); } private string ContentAddressedRecordPath(CultPersistedRecord metadata) diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index 761200d0..9065a377 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -68,6 +68,13 @@ and a catalog entry's exact bytes, unknown fields included, so a copied record keeps its version. No type interprets a copied record: a write may hash another record's bytes to compare versions, and reads its header (key, schema id, variant slot), but never decodes its payload. +A record is encoded from the object the writer hands in, and every other record is copied as the store holds it. The +store that read a file owns that file's bytes: for each stored record, its exact slice of the file and the SHA-256 of that +slice, which is the record's version; likewise each catalog entry's slice. A decoded record or snapshot object carries no +bytes, so an object a caller edited in place is never mistaken for what the file holds. Consequently a record the writer +staged is re-encoded canonically and has the version of its new encoding, even when its content is unchanged, and a +record nobody staged keeps its bytes and its version for as long as no writer stages it. The public snapshot and record +serializers encode every record from its fields. A record written over one the store holds gets a storedAt later than the one it replaces. A document changed in place and not staged is not written. diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 1f509ae1..b932a1da 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -29,16 +29,10 @@ namespace GameCult.Caching public CultSchemaMemberCatalogEntry[] Members { get; set; } = Array.Empty(); - // The exact MessagePack bytes this entry was read from, when it was read from a store. A writer that carries the entry - // forward emits them as they are, so what this runtime does not decode (fields beyond the ones it knows, the width of an int, - // nil) survives; an entry whose decoded fields were changed since is encoded from them instead. Null for an entry a runtime - // built. - internal byte[]? RawBytes { get; set; } - // The catalog a write leaves, derived from the records the written store holds and nothing else. For each schema id a record // carries, one entry that publishes it: for an id the write stages a record under, the entry the writer registered (or passed) // for it, which owns the id; for every other id the arrived (durable) entry that owns it, else the first that lists it as - // compatible, carried as the bytes it was read from. A registered entry never publishes an id nothing staged sits under, so + // compatible (the store that read it carries its bytes as read). A registered entry never publishes an id nothing staged sits under, so // the caller passes registered entries only for the ids it writes. Entries are written as chosen, never merged; a registered // descriptor wins over an arrived entry with the same own id. Two arrived entries with the same own id and different schema // names, or a record no chosen entry publishes, refuse the write. @@ -134,24 +128,13 @@ namespace GameCult.Caching public sealed class CultPersistedRecord { - private string _key = string.Empty; - private string _schemaId = string.Empty; - private string _storedAt = string.Empty; - private byte[] _payload = Array.Empty(); - private CultVariantDelta? _variant; - - public string Key { get => _key; set { _key = value; RawBytes = null; } } - public string SchemaId { get => _schemaId; set { _schemaId = value; RawBytes = null; } } - public string StoredAt { get => _storedAt; set { _storedAt = value; RawBytes = null; } } - public byte[] Payload { get => _payload; set { _payload = value; RawBytes = null; } } + public string Key { get; set; } = string.Empty; + public string SchemaId { get; set; } = string.Empty; + public string StoredAt { get; set; } = string.Empty; + public byte[] Payload { get; set; } = Array.Empty(); // Non-null for a variant record (slot 4): its base and overrides. Its payload is empty. - public CultVariantDelta? Variant { get => _variant; set { _variant = value; RawBytes = null; } } - - // The exact bytes this record was read from, as a store file held them. A record is written as the bytes it was read from - // while no field has been assigned since, so a copied record stays byte-identical and its version (the SHA-256 of its - // stored bytes) is the one the file's bytes name, whatever encoding their writer chose. Assigning any field drops them. - internal byte[]? RawBytes { get; set; } + public CultVariantDelta? Variant { get; set; } } // One record a write puts in a single-file store: encoded, with the catalog entry that publishes the id it is stored under and @@ -3750,24 +3733,75 @@ namespace GameCult.Caching // A write into a store holding a variant lands only while the durable versions still equal these. private readonly Dictionary _lastRead = new(StringComparer.Ordinal); - protected abstract byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot); + /// + /// A store file from its parts, each already encoded: a catalog entry or record this store carries forward is the bytes its + /// file held, one it writes is or of the object. + /// + protected abstract byte[] SerializeStore(string formatVersion, IReadOnlyList catalogEntries, IReadOnlyList records); - // One record's persisted encoding: the bytes a version names. + // One record's persisted encoding: the bytes a version names, and the bytes a written record is stored as. protected abstract byte[] SerializeRecord(CultPersistedRecord record); - private string Version(CultPersistedRecord record) - { - using var sha256 = SHA256.Create(); - return VersionOf(sha256.ComputeHash(SerializeRecord(record))); - } + // One catalog entry's persisted encoding, for an entry the writer built. + protected abstract byte[] SerializeCatalogEntry(CultSchemaCatalogEntry entry); /// /// This store's one reader of a non-empty store file, and so its one verdict on whether the file may be replaced: open, /// flush and commit all ask it. It must throw for anything that is not exactly one /// store this runtime reads (a truncated file, bytes after the store, a missing or extra slot, a format or record it does - /// not know), so a rewrite never overwrites a file it cannot see. + /// not know), so a rewrite never overwrites a file it cannot see. It returns the decoded snapshot and, parallel to the + /// snapshot's catalog and records, the exact bytes the file holds for each: the store keeps those, never the snapshot's objects. /// - protected abstract CultPersistedStoreSnapshot DeserializeSnapshot(byte[] data); + protected abstract (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) ReadStore(byte[] data); + + // Test seam: how many record encodings this store has hashed, to prove what a write hashes. + internal int RecordsHashed; + + private string VersionOfBytes(byte[] bytes) + { + RecordsHashed++; + using var sha256 = SHA256.Create(); + return VersionOf(sha256.ComputeHash(bytes)); + } + + // What the store's file holds, as the store owns it: the snapshot as read, and keyed by record key the exact slice the file + // holds for each record, whose version is the SHA-256 of that slice, hashed when a comparison first asks. Catalog entries + // keep their slices by the entry object this read produced. A write copies a record it did not stage from its slice here + // and re-encodes every record it was handed, so no object a caller edits is ever mistaken for the bytes in the file. + private sealed class Durable + { + private readonly SingleFileBackingStore _owner; + private readonly Dictionary _records; + private readonly Dictionary _versions = new(StringComparer.Ordinal); + private readonly Dictionary _catalog = new(); + + internal Durable(SingleFileBackingStore owner, (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) read) + { + _owner = owner; + Snapshot = read.Snapshot; + _records = new Dictionary(StringComparer.Ordinal); + for (var index = 0; index < Snapshot.Records.Length; index++) + _records[Snapshot.Records[index].Key] = read.RecordBytes[index]; + for (var index = 0; index < Snapshot.SchemaCatalog.Length; index++) + _catalog.Add(Snapshot.SchemaCatalog[index], read.CatalogBytes[index]); + } + + internal CultPersistedStoreSnapshot Snapshot { get; } + + internal byte[] Slice(string key) => _records[key]; + + internal string Version(string key) + { + if (!_versions.TryGetValue(key, out var version)) + _versions[key] = version = _owner.VersionOfBytes(_records[key]); + return version; + } + + // The bytes to write for a catalog entry the write chose: the slice the file held when it is one of this read's entries, + // otherwise an encoding of the entry the writer built. + internal byte[] CatalogBytes(CultSchemaCatalogEntry entry) => + _catalog.TryGetValue(entry, out var bytes) ? bytes : _owner.SerializeCatalogEntry(entry); + } protected abstract byte[] SerializePayload(object document); protected abstract object DeserializePayload(Type documentType, byte[] payload); @@ -3783,8 +3817,8 @@ namespace GameCult.Caching if (IsDirty) return; - var snapshot = ReadSnapshot(); - if (snapshot == null) + var durable = ReadSnapshot(); + if (durable == null) { _lastRead.Clear(); SetLastSchemaMigrationReports(Array.Empty()); @@ -3792,13 +3826,14 @@ namespace GameCult.Caching return; } + var snapshot = durable.Snapshot; var reports = new List(snapshot.Records.Length); var persisted = new Dictionary(StringComparer.Ordinal); var foreign = new List(); var versions = new Dictionary(StringComparer.Ordinal); foreach (var record in snapshot.Records) { - var version = versions[record.Key] = Version(record); + var version = versions[record.Key] = durable.Version(record.Key); if (Foreign(record, snapshot.SchemaCatalog) is { } carried) { foreign.Add(carried); @@ -3864,7 +3899,7 @@ namespace GameCult.Caching using (AcquireLock(wait: true)) { var disk = ReadSnapshot(); - RefuseForeign(_staged, ForeignOnDisk(disk)); + RefuseForeign(_staged, disk); var (upserts, removals) = StagedWrites(Array.Empty(), Array.Empty()); Wrote(ApplyWriteSet(disk, upserts, removals)); } @@ -3888,12 +3923,13 @@ namespace GameCult.Caching if (fileLock == null) return CultCommitOutcome.Contended; - var disk = ReadSnapshot() ?? new CultPersistedStoreSnapshot(); + var disk = ReadSnapshot(); if (request.HasConditions && !request.ConditionsHold( - disk.Records.ToDictionary(record => record.Key, Version, StringComparer.Ordinal), Entries.Values)) + (disk?.Snapshot.Records ?? Array.Empty()).ToDictionary(record => record.Key, record => disk!.Version(record.Key), StringComparer.Ordinal), + Entries.Values)) return CultCommitOutcome.Mismatch; - RefuseForeign(request.Upserts.Concat(request.Deletes).Select(entry => entry.Key.Value).Concat(_staged), ForeignOnDisk(disk)); + RefuseForeign(request.Upserts.Concat(request.Deletes).Select(entry => entry.Key.Value).Concat(_staged), disk); var (upserts, removals) = StagedWrites(request.Upserts, request.Deletes); var written = ApplyWriteSet(disk, upserts, removals); foreach (var entry in request.Deletes) @@ -3959,12 +3995,15 @@ namespace GameCult.Caching // a variant, or would after this write, is written only while its record versions are what this store last read: a variant // resolves against other records, and a write decodes none it did not stage, so it cannot judge another writer's. Versions // only: a write hashes other records' bytes, never decodes them. Otherwise the whole write is refused and what it refused is - // reloaded (Refuse). Returns the records written, each with the version of the bytes written. - private IReadOnlyCollection<(CultStagedRecord Staged, string Version)> ApplyWriteSet(CultPersistedStoreSnapshot? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals) + // reloaded (Refuse). Every record the writer staged is encoded from its object now, and its version is the hash of that + // encoding; every other record is copied as the slice the file holds. Returns the records written, each with the version of + // the bytes written. + private IReadOnlyCollection<(CultStagedRecord Staged, string Version)> ApplyWriteSet(Durable? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals) { RefuseWriteIntoMovedVariantStore(durable, upserts, removals); + var durableRecords = durable?.Snapshot.Records ?? Array.Empty(); // A record written over one the store holds is stored at a storedAt later than the one it replaces, whoever wrote that. - var replaced = (durable?.Records ?? Array.Empty()).ToDictionary(record => record.Key, record => record.StoredAt, StringComparer.Ordinal); + var replaced = durableRecords.ToDictionary(record => record.Key, record => record.StoredAt, StringComparer.Ordinal); foreach (var staged in upserts) { if (replaced.TryGetValue(staged.Record.Key, out var durableAt)) @@ -3973,33 +4012,34 @@ namespace GameCult.Caching var written = upserts.Select(staged => staged.Record.Key).ToHashSet(StringComparer.Ordinal); var removed = removals.ToHashSet(StringComparer.Ordinal); - var copied = (durable?.Records ?? Array.Empty()) + var copied = durableRecords .Where(record => !written.Contains(record.Key) && !removed.Contains(record.Key)) .ToArray(); - var records = copied.Concat(upserts.Select(staged => staged.Record)) - .OrderBy(record => record.Key, StringComparer.Ordinal) + var encoded = upserts.Select(staged => (Staged: staged, Bytes: SerializeRecord(staged.Record))).ToArray(); + var records = copied.Concat(upserts.Select(staged => staged.Record)).ToArray(); + var fileRecords = copied.Select(record => (record.Key, Bytes: durable!.Slice(record.Key))) + .Concat(encoded.Select(pair => (pair.Staged.Record.Key, pair.Bytes))) + .OrderBy(pair => pair.Key, StringComparer.Ordinal) + .Select(pair => pair.Bytes) .ToArray(); - var snapshot = new CultPersistedStoreSnapshot - { - FormatVersion = HeaderForWrite( - durable?.FormatVersion, - upserts.Select(staged => staged.HoldsIds).ToArray(), - copiesAny: copied.Length > 0, - directoryStore: false, - holdsVariants: records.Any(record => record.Variant != null)), - SchemaCatalog = CultSchemaCatalogEntry.Derive( - records, - upserts.Select(staged => staged.Entry).Distinct().ToArray(), - durable?.SchemaCatalog ?? Array.Empty()) - .OrderBy(entry => entry.SchemaName, StringComparer.Ordinal) - .ThenBy(entry => entry.SchemaId, StringComparer.Ordinal) - .ToArray(), - Records = records - }; + var catalog = CultSchemaCatalogEntry.Derive( + records, + upserts.Select(staged => staged.Entry).Distinct().ToArray(), + durable?.Snapshot.SchemaCatalog ?? Array.Empty()) + .OrderBy(entry => entry.SchemaName, StringComparer.Ordinal) + .ThenBy(entry => entry.SchemaId, StringComparer.Ordinal) + .Select(entry => durable?.CatalogBytes(entry) ?? SerializeCatalogEntry(entry)) + .ToArray(); + var format = HeaderForWrite( + durable?.Snapshot.FormatVersion, + upserts.Select(staged => staged.HoldsIds).ToArray(), + copiesAny: copied.Length > 0, + directoryStore: false, + holdsVariants: records.Any(record => record.Variant != null)); - var versions = upserts.Select(staged => (staged, Version(staged.Record))).ToArray(); + var versions = encoded.Select(pair => (pair.Staged, VersionOfBytes(pair.Bytes))).ToArray(); Directory.CreateDirectory(FileInfo.DirectoryName!); - WriteSnapshotAtomically(FileInfo.FullName, SerializeSnapshot(snapshot)); + WriteSnapshotAtomically(FileInfo.FullName, SerializeStore(format, catalog, fileRecords)); foreach (var key in removed) _lastRead.Remove(key); foreach (var (staged, version) in versions) @@ -4007,13 +4047,13 @@ namespace GameCult.Caching return versions; } - private void RefuseWriteIntoMovedVariantStore(CultPersistedStoreSnapshot? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals) + private void RefuseWriteIntoMovedVariantStore(Durable? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals) { - var records = durable?.Records ?? Array.Empty(); + var records = durable?.Snapshot.Records ?? Array.Empty(); if (records.All(record => record.Variant == null) && upserts.All(staged => staged.Record.Variant == null)) return; var changed = new HashSet(StringComparer.Ordinal); - var versions = records.ToDictionary(record => record.Key, Version, StringComparer.Ordinal); + var versions = records.ToDictionary(record => record.Key, record => durable!.Version(record.Key), StringComparer.Ordinal); foreach (var (key, version) in versions) { if (!_lastRead.TryGetValue(key, out var read) || !SameStoredIdentity(read, version)) @@ -4035,9 +4075,9 @@ namespace GameCult.Caching var changedKeys = changed.OrderBy(key => key, StringComparer.Ordinal).ToArray(); Refuse( recordKeys.Concat(changedKeys), - key => Cache == null || !byKey.TryGetValue(key, out var record) || Foreign(record, durable!.SchemaCatalog) != null + key => Cache == null || !byKey.TryGetValue(key, out var record) || Foreign(record, durable!.Snapshot.SchemaCatalog) != null ? null - : ToStoredDocument(record, versions[key], durable!.SchemaCatalog, DeserializePayload, out _), + : ToStoredDocument(record, versions[key], durable!.Snapshot.SchemaCatalog, DeserializePayload, out _), key => { _staged.Remove(key); @@ -4061,30 +4101,31 @@ namespace GameCult.Caching inner)); } - private void RefuseForeign( - IEnumerable written, - IReadOnlyDictionary foreign) => + private void RefuseForeign(IEnumerable written, Durable? disk) + { + var foreign = ForeignOnDisk(disk); RefuseForeign( written, - key => foreign.TryGetValue(key, out var carried) ? carried.Foreign : null, + key => foreign.TryGetValue(key, out var carried) ? carried : null, key => { _staged.Remove(key); IsDirty = _staged.Count > 0; - if (foreign.TryGetValue(key, out var carried)) - _lastRead[key] = Version(carried.Record); + if (foreign.ContainsKey(key)) + _lastRead[key] = disk!.Version(key); else _lastRead.Remove(key); }); + } // The records the file holds that no registered type claims, by key. - private Dictionary ForeignOnDisk(CultPersistedStoreSnapshot? disk) + private Dictionary ForeignOnDisk(Durable? disk) { - var foreign = new Dictionary(StringComparer.Ordinal); - foreach (var record in disk?.Records ?? Array.Empty()) + var foreign = new Dictionary(StringComparer.Ordinal); + foreach (var record in disk?.Snapshot.Records ?? Array.Empty()) { - if (Foreign(record, disk!.SchemaCatalog) is { } carried) - foreign[record.Key] = (carried, record); + if (Foreign(record, disk!.Snapshot.SchemaCatalog) is { } carried) + foreign[record.Key] = carried; } return foreign; @@ -4110,14 +4151,14 @@ namespace GameCult.Caching // Open, flush and commit read the file through this one call, so they agree on which files are stores. A file that is // gone is an empty store: nothing to read, nothing to lose. A zero-byte file is not a store; it is refused. - private CultPersistedStoreSnapshot? ReadSnapshot() + private Durable? ReadSnapshot() { var bytes = ReadDisk(); if (bytes == null) return null; try { - return DeserializeSnapshot(bytes); + return new Durable(this, ReadStore(bytes)); } catch (CultStoreUnreadableException ex) when (ex.Path == null) { @@ -4127,7 +4168,7 @@ namespace GameCult.Caching // CultMesh's single-file document helpers reach this store file only through this read and ApplyWriteSet, so they share // its one reader, its lock and its atomic replace. - internal CultPersistedStoreSnapshot? ReadDurable() => Held(ReadSnapshot); + internal CultPersistedStoreSnapshot? ReadDurable() => Held(() => ReadSnapshot()?.Snapshot); private byte[]? ReadDisk() { diff --git a/tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs b/tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs index 8fd1c73e..b87d5926 100644 --- a/tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs +++ b/tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs @@ -189,6 +189,7 @@ namespace GameCult.Caching.Tests var path = OldStore("copied.cc"); var descriptor = Declaring.GetRequired(); var before = Read(path); + var entryBefore = StoreSlices.CatalogEntry(path, OldId); using (var cache = Open(path, Declaring)) cache.Commit(batch => batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "e" }, new CultRecordKey("e"))); @@ -199,7 +200,7 @@ namespace GameCult.Caching.Tests var copied = written.Records.Single(record => record.Key == "d"); var original = before.Records.Single(); Assert.That((copied.StoredAt, copied.Payload), Is.EqualTo((original.StoredAt, original.Payload))); - Assert.That(written.SchemaCatalog.Single(entry => entry.SchemaId == OldId).RawBytes, Is.EqualTo(before.SchemaCatalog.Single().RawBytes), + Assert.That(StoreSlices.CatalogEntry(path, OldId), Is.EqualTo(entryBefore), "the entry that published the old id is carried as its bytes"); } diff --git a/tests/GameCult.Caching.Tests/ConditionalCommitTests.cs b/tests/GameCult.Caching.Tests/ConditionalCommitTests.cs index 0350e1e8..749fa66a 100644 --- a/tests/GameCult.Caching.Tests/ConditionalCommitTests.cs +++ b/tests/GameCult.Caching.Tests/ConditionalCommitTests.cs @@ -895,9 +895,9 @@ namespace GameCult.Caching.Tests base.Push(entry); } - protected override CultPersistedStoreSnapshot DeserializeSnapshot(byte[] data) + protected override (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) ReadStore(byte[] data) { - var snapshot = base.DeserializeSnapshot(data); + var snapshot = base.ReadStore(data); if (Pause is { } pause) { Paused = true; diff --git a/tests/GameCult.Caching.Tests/StoreSlices.cs b/tests/GameCult.Caching.Tests/StoreSlices.cs new file mode 100644 index 00000000..a27f2468 --- /dev/null +++ b/tests/GameCult.Caching.Tests/StoreSlices.cs @@ -0,0 +1,27 @@ +using System; +using System.IO; +using System.Linq; +using GameCult.Caching.MessagePack; + +namespace GameCult.Caching.Tests +{ + // The exact bytes a store file holds for its catalog entries and records. A read snapshot carries none: the store that read the + // file owns them, and these read them as the store's reader does. + internal static class StoreSlices + { + internal static byte[] CatalogEntry(string path, string schemaId) + { + var read = CultDocumentMessagePackSerialization.ReadStore(File.ReadAllBytes(path)); + return read.CatalogBytes[Array.FindIndex(read.Snapshot.SchemaCatalog, entry => entry.SchemaId == schemaId)]; + } + + internal static byte[][] CatalogEntries(string path) => + CultDocumentMessagePackSerialization.ReadStore(File.ReadAllBytes(path)).CatalogBytes; + + internal static byte[] Record(string path, string key) + { + var read = CultDocumentMessagePackSerialization.ReadStore(File.ReadAllBytes(path)); + return read.RecordBytes[Array.FindIndex(read.Snapshot.Records, record => record.Key == key)]; + } + } +} diff --git a/tests/GameCult.Caching.Tests/VariantTests.cs b/tests/GameCult.Caching.Tests/VariantTests.cs index 5c2ecc35..99304283 100644 --- a/tests/GameCult.Caching.Tests/VariantTests.cs +++ b/tests/GameCult.Caching.Tests/VariantTests.cs @@ -847,6 +847,7 @@ namespace GameCult.Caching.Tests var path = PathOf(viaFlush ? "drift-untouched-flush.cc" : "drift-untouched-commit.cc"); SeedDrift(path); var before = ReadStore(path); + var wideEntryBefore = StoreSlices.CatalogEntry(path, DriftWideId); using (var narrow = Open(path, registry: CultDocumentRegistry.ForTypes(new[] { typeof(DriftNarrow) }))) { @@ -870,8 +871,8 @@ namespace GameCult.Caching.Tests } Assert.That(OverriddenSlotsOn(path, DriftVariant.Value).OrderBy(slot => slot), Is.EqualTo(new[] { 0, 2 }), "the dropped slot's override is still stored"); - Assert.That(after.SchemaCatalog.Single(entry => entry.SchemaId == DriftWideId).RawBytes, - Is.EqualTo(before.SchemaCatalog.Single(entry => entry.SchemaId == DriftWideId).RawBytes)); + Assert.That(StoreSlices.CatalogEntry(path, DriftWideId), + Is.EqualTo(wideEntryBefore)); } // A type that lists Wide's id as compatible owns Wide's records: staging one writes it under the type's own id, at a later diff --git a/tests/GameCult.Caching.Tests/WriteSetTests.cs b/tests/GameCult.Caching.Tests/WriteSetTests.cs index 217b57ff..96c508fa 100644 --- a/tests/GameCult.Caching.Tests/WriteSetTests.cs +++ b/tests/GameCult.Caching.Tests/WriteSetTests.cs @@ -37,7 +37,29 @@ namespace GameCult.Caching.Tests public string Name = string.Empty; } - private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] { typeof(WsItem), typeof(WsOther) }); + [CultDocument("tests.write_set_nested", "tests.write_set_nested.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class WsNested + { + [Key(0)] + [CultName] + public string Name = string.Empty; + + [Key(1)] + public List Tags = new(); + + [Key(2)] + public WsInner Inner = new(); + } + + [MessagePackObject(AllowPrivate = true)] + internal sealed class WsInner + { + [Key(0)] + public int X; + } + + private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] { typeof(WsItem), typeof(WsOther), typeof(WsNested) }); private static readonly string ItemId = Registry.GetRequired(typeof(WsItem)).SchemaId; private static readonly CultRecordKey A = new("a"); private static readonly CultRecordKey X = new("x"); @@ -77,7 +99,7 @@ namespace GameCult.Caching.Tests } private static byte[] EntryBytes(string path, string schemaId) => - Read(path).SchemaCatalog.Single(entry => entry.SchemaId == schemaId).RawBytes!; + StoreSlices.CatalogEntry(path, schemaId); private static string[] Keys(string path) => Read(path).Records.Select(record => record.Key).OrderBy(key => key, StringComparer.Ordinal).ToArray(); @@ -289,6 +311,8 @@ namespace GameCult.Caching.Tests Assert.Ignore("CULTLIB_REAL_STORE does not name a copy of a real store."); var path = PathOf("real.cc"); File.Copy(source!, path); + var beforePath = PathOf("real-before.cc"); + File.Copy(source!, beforePath); var before = Read(path); using (var cache = Open(path, false)) @@ -298,11 +322,11 @@ namespace GameCult.Caching.Tests TestContext.Out.WriteLine($"real store: {before.Records.Length} records, {before.SchemaCatalog.Length} catalog entries, header {before.FormatVersion}"); Assert.That(after.Records.Length, Is.EqualTo(before.Records.Length + 1)); Assert.That(after.FormatVersion, Is.EqualTo(before.FormatVersion)); - byte[] Bytes(CultPersistedRecord record) => CultDocumentMessagePackSerialization.SerializeSnapshot(new CultPersistedStoreSnapshot { Records = new[] { record } }); - var changed = before.Records.Where(record => !Bytes(record).SequenceEqual(Bytes(after.Records.Single(kept => kept.Key == record.Key)))).Select(record => record.Key).ToArray(); + + var changed = before.Records.Where(record => !StoreSlices.Record(beforePath, record.Key).SequenceEqual(StoreSlices.Record(path, record.Key))).Select(record => record.Key).ToArray(); Assert.That(changed, Is.Empty, "records that are not byte-identical"); - var afterEntries = after.SchemaCatalog.Select(entry => entry.RawBytes!).ToList(); - var missing = before.SchemaCatalog.Where(entry => !afterEntries.Any(raw => raw.SequenceEqual(entry.RawBytes!))).Select(entry => entry.SchemaId).ToArray(); + var afterEntries = StoreSlices.CatalogEntries(path).ToList(); + var missing = StoreSlices.CatalogEntries(beforePath).Where(raw => !afterEntries.Any(other => other.SequenceEqual(raw))).Select(raw => BitConverter.ToString(raw)).ToArray(); Assert.That(missing, Is.Empty, "catalog entries that are not byte-identical"); } @@ -801,42 +825,30 @@ namespace GameCult.Caching.Tests }), Is.True, "the writer's own condition on the variant holds"); } - private sealed class CountingStore : SingleFileMessagePackBackingStore - { - public CountingStore(string path) : base(path) { } - - public int Encoded; - - protected override byte[] SerializeRecord(CultPersistedRecord record) - { - Encoded++; - return base.SerializeRecord(record); - } - } // A commit with no condition decides nothing about the other records, so it hashes none of them: only the record it writes - // is encoded to be named. A commit with a condition compares every durable record's version. + // is hashed to be named. A commit with a condition compares every durable record's version. [Test] public void AnUnconditionalCommitHashesOnlyTheRecordsItWrites() { var path = PathOf("cost.cc"); Seed(path, false, "a", "b", "c", "d", "e"); - var store = new CountingStore(path); + var store = new SingleFileMessagePackBackingStore(path); using var cache = new CultCache(Registry, CultCacheMessagePack.CreateCodec(Registry)); cache.AddBackingStore(store); cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); - store.Encoded = 0; + store.RecordsHashed = 0; Assert.That(cache.Commit(batch => batch.Upsert(typeof(WsItem), new WsItem { Name = "name-x" }, X)), Is.True); - Assert.That(store.Encoded, Is.EqualTo(1)); + Assert.That(store.RecordsHashed, Is.EqualTo(1)); - store.Encoded = 0; + store.RecordsHashed = 0; Assert.That(cache.Commit(batch => { batch.Expect(X, cache.Get(X)!); batch.Upsert(typeof(WsItem), new WsItem { Name = "name-x", Note = "again" }, X); }), Is.True); - Assert.That(store.Encoded, Is.GreaterThanOrEqualTo(6), "a condition hashes every durable record, which is how the count proves itself"); + Assert.That(store.RecordsHashed, Is.GreaterThanOrEqualTo(6), "a condition hashes every durable record, which is how the count proves itself"); } // Records no store has read or written are told apart however many are made in one clock tick. @@ -894,6 +906,126 @@ namespace GameCult.Caching.Tests private static string StoredAtOf(string path, string key) => Read(path).Records.Single(record => record.Key == key).StoredAt; + // The stored bytes belong to the store that read them, never to a record object: a record read through the public snapshot + // API is only a decoded copy, so an edit made to its payload in place is what a write of it stores. + [Test] + public void ARecordReadThroughTheSnapshotApiAndEditedInPlaceIsWrittenAsEdited() + { + var path = PathOf("edit-payload.cc"); + Seed(path, false, "k"); + var snapshot = Read(path); + OverwriteInPlace(snapshot.Records.Single().Payload, "seeded", "SEEDED"); + + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + + Assert.That(NoteOnDisk(path, false), Is.EqualTo("SEEDED")); + var record = CultDocumentMessagePackSerialization.DeserializePersistedRecord(StoreSlices.Record(path, "k")); + OverwriteInPlace(record.Payload, "SEEDED", "seeded"); + Assert.That( + CultDocumentMessagePackSerialization.DeserializePersistedRecord(CultDocumentMessagePackSerialization.SerializePersistedRecord(record)).Payload, + Is.EqualTo(record.Payload), "a single record is re-encoded from its fields too"); + } + + [Test] + public void AnOverrideReadThroughTheSnapshotApiAndEditedInPlaceIsWrittenAsEdited() + { + var path = PathOf("edit-override.cc"); + Seed(path, false, "a"); + using (var seed = Open(path, false)) + AddVariant(seed, "v", A); + var snapshot = Read(path); + var variant = snapshot.Records.Single(record => record.Key == "v").Variant!; + OverwriteInPlace(variant.Overrides.Single(entry => entry.Path[0].Slot == 1).Value, "variant-v", "VARIANT-v"); + + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + + using var fresh = Open(path, false); + Assert.That(fresh.Get(new CultRecordKey("v"))!.Note, Is.EqualTo("VARIANT-v")); + } + + // A held document edited in place (a list, a nested object) and put back as the same instance is written as edited, and a + // later write of a neighbour copies it as the file then holds it. + [Test] + public void AHeldDocumentEditedInPlaceAndPutBackIsWrittenAsEdited([Values] bool commit) + { + var path = PathOf("edit-held.cc"); + var n = new CultRecordKey("n"); + using (var seed = Open(path, false)) + seed.Commit(batch => batch.Upsert(typeof(WsNested), new WsNested { Name = "name-n", Tags = { "t1" }, Inner = { X = 1 } }, n)); + + using (var cache = Open(path, false)) + { + var held = cache.Get(n)!; + held.Tags.Add("t2"); + held.Inner.X = 9; + Land(cache, commit, typeof(WsNested), held, n); + } + + using (var fresh = Open(path, false)) + { + var read = fresh.Get(n)!; + Assert.That(read.Tags, Is.EqualTo(new[] { "t1", "t2" })); + Assert.That(read.Inner.X, Is.EqualTo(9)); + } + + var written = StoreSlices.Record(path, "n"); + using (var other = Open(path, false)) + Land(other, commit, typeof(WsItem), new WsItem { Name = "name-x" }, X); + Assert.That(StoreSlices.Record(path, "n"), Is.EqualTo(written), "the edit is copied as the file holds it"); + } + + // A record the writer did not stage keeps the bytes and the version its file holds, whatever encoding its writer chose, + // across any number of writes of its neighbours. + [Test] + public void AnUntouchedNonCanonicalRecordKeepsItsBytesAndVersionAcrossAWriteOfANeighbour([Values] bool commit) + { + var path = PathOf("untouched-wide.cc"); + Seed(path, false, "k"); + File.WriteAllBytes(path, WidenEveryRecord(File.ReadAllBytes(path), out var slices)); + using var cache = Open(path, false); + var version = cache.GetStored(K)!.StoredVersion; + Assert.That(version, Is.EqualTo(HexOfSha256(slices["k"]))); + + Land(cache, commit, typeof(WsItem), new WsItem { Name = "name-x" }, X); + Land(cache, commit, typeof(WsItem), new WsItem { Name = "name-y" }, new CultRecordKey("y")); + + Assert.That(StoreSlices.Record(path, "k"), Is.EqualTo(slices["k"])); + Assert.That(cache.GetStored(K)!.StoredVersion, Is.EqualTo(version)); + using var fresh = Open(path, false); + Assert.That(fresh.GetStored(K)!.StoredVersion, Is.EqualTo(version)); + } + + // A record the writer stages is encoded from its object, so a non-canonical record put back unchanged is written canonically + // and, being other bytes, has another version. + [Test] + public void AStagedNonCanonicalRecordPutBackUnchangedIsReEncodedAndGetsANewVersion([Values] bool commit) + { + var path = PathOf("staged-wide.cc"); + Seed(path, false, "k"); + File.WriteAllBytes(path, WidenEveryRecord(File.ReadAllBytes(path), out var slices)); + using var cache = Open(path, false); + var held = cache.Get(K)!; + + Land(cache, commit, typeof(WsItem), held, K); + + var written = StoreSlices.Record(path, "k"); + Assert.That(written, Is.Not.EqualTo(slices["k"])); + Assert.That(written, Is.EqualTo(CultDocumentMessagePackSerialization.SerializePersistedRecord(Read(path).Records.Single())), + "the record is in the canonical encoding"); + Assert.That(cache.GetStored(K)!.StoredVersion, Is.EqualTo(HexOfSha256(written))); + Assert.That(cache.GetStored(K)!.StoredVersion, Is.Not.EqualTo(HexOfSha256(slices["k"]))); + } + + private static void OverwriteInPlace(byte[] bytes, string from, string to) + { + var was = System.Text.Encoding.UTF8.GetBytes(from); + var now = System.Text.Encoding.UTF8.GetBytes(to); + Assert.That(now.Length, Is.EqualTo(was.Length)); + var at = IndexOf(bytes, was); + Assert.That(at, Is.GreaterThanOrEqualTo(0), $"'{from}' is in the bytes"); + Array.Copy(now, 0, bytes, at, now.Length); + } + private static void AddVariant(CultCache cache, string key, CultRecordKey baseKey) => cache.Commit(batch => batch.UpsertVariant(new CultRecordKey(key), baseKey, new[] {