diff --git a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs
index 68ec46e9..67db59f4 100644
--- a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs
+++ b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs
@@ -155,22 +155,33 @@ public static class CultDocumentMessagePackSerialization
return ReadPersistedRecord(ref reader);
}
- public static byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot)
+ ///
+ /// Encodes the snapshot: every catalog entry and record from its fields, so what a caller edited is what is written.
+ ///
+ public static byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot) =>
+ SerializeStore(
+ snapshot.FormatVersion,
+ snapshot.SchemaCatalog.Select(SerializeCatalogEntry).ToArray(),
+ snapshot.Records.Select(SerializePersistedRecord).ToArray());
+
+ // A store file from its parts, each already encoded: what a store carries forward is the bytes its file held, what it writes is
+ // an encoding of the object.
+ internal static byte[] SerializeStore(string formatVersion, IReadOnlyList catalogEntries, IReadOnlyList records)
{
var buffer = new global::System.Buffers.ArrayBufferWriter();
var writer = new MessagePackWriter(buffer);
writer.WriteArrayHeader(StoreSnapshotFieldCount);
- writer.Write(snapshot.FormatVersion);
- writer.WriteArrayHeader(snapshot.SchemaCatalog.Length);
- foreach (var entry in snapshot.SchemaCatalog)
+ writer.Write(formatVersion);
+ writer.WriteArrayHeader(catalogEntries.Count);
+ foreach (var entry in catalogEntries)
{
- WriteSchemaCatalogEntry(ref writer, entry);
+ writer.WriteRaw(new ReadOnlySpan(entry));
}
- writer.WriteArrayHeader(snapshot.Records.Length);
- foreach (var record in snapshot.Records)
+ writer.WriteArrayHeader(records.Count);
+ foreach (var record in records)
{
- WritePersistedRecord(ref writer, record);
+ writer.WriteRaw(new ReadOnlySpan(record));
}
writer.Flush();
@@ -185,7 +196,12 @@ public static class CultDocumentMessagePackSerialization
/// , so a reader never takes part of a file for the whole and never skips a slot
/// it does not understand. The rewriting paths use this same reader as their verdict on whether a file may be replaced.
///
- public static CultPersistedStoreSnapshot DeserializeSnapshot(byte[] payload)
+ public static CultPersistedStoreSnapshot DeserializeSnapshot(byte[] payload) => ReadStore(payload).Snapshot;
+
+ // The reader behind DeserializeSnapshot, and beside the snapshot the exact bytes the file held for each catalog entry and each
+ // record (parallel to the snapshot's arrays). The snapshot's objects hold no bytes: a store that carries entries forward keeps
+ // these slices itself.
+ internal static (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) ReadStore(byte[] payload)
{
try
{
@@ -200,27 +216,33 @@ public static class CultDocumentMessagePackSerialization
var snapshot = new CultPersistedStoreSnapshot();
if (fieldCount == 0)
- return snapshot;
+ return (snapshot, Array.Empty(), Array.Empty());
snapshot.FormatVersion = reader.ReadString()
?? throw new CultStoreUnreadableException("Store snapshot declares no format version; this runtime reads " + CultPersistedStoreSnapshot.FormatV1 + ", " + CultPersistedStoreSnapshot.FormatV2 + " and " + CultPersistedStoreSnapshot.FormatV3 + ".");
var catalogCount = reader.ReadArrayHeader();
snapshot.SchemaCatalog = new CultSchemaCatalogEntry[catalogCount];
+ var catalogBytes = new byte[catalogCount][];
for (var index = 0; index < catalogCount; index++)
{
+ var start = reader.Position;
snapshot.SchemaCatalog[index] = ReadSchemaCatalogEntry(ref reader);
+ catalogBytes[index] = reader.Sequence.Slice(start, reader.Position).ToArray();
}
var recordCount = reader.ReadArrayHeader();
snapshot.Records = new CultPersistedRecord[recordCount];
+ var recordBytes = new byte[recordCount][];
for (var index = 0; index < recordCount; index++)
{
+ var start = reader.Position;
snapshot.Records[index] = ReadPersistedRecord(ref reader);
+ recordBytes[index] = reader.Sequence.Slice(start, reader.Position).ToArray();
}
RequirePublishedSchemas(snapshot);
- return snapshot;
+ return (snapshot, catalogBytes, recordBytes);
}
catch (CultStoreUnreadableException)
{
@@ -306,12 +328,6 @@ public static class CultDocumentMessagePackSerialization
private static void WritePersistedRecord(ref MessagePackWriter writer, CultPersistedRecord record)
{
- if (record.RawBytes is { } raw)
- {
- writer.WriteRaw(new ReadOnlySpan(raw));
- return;
- }
-
writer.WriteArrayHeader(record.Variant == null ? PersistedRecordFieldCount : VariantRecordFieldCount);
writer.Write(record.Key);
writer.Write(record.SchemaId);
@@ -395,7 +411,6 @@ public static class CultDocumentMessagePackSerialization
public static CultPersistedRecord ReadPersistedRecord(ref MessagePackReader reader)
{
- var start = reader.Position;
var fieldCount = reader.ReadArrayHeader();
var record = new CultPersistedRecord();
@@ -436,37 +451,19 @@ public static class CultDocumentMessagePackSerialization
}
}
- record.RawBytes = reader.Sequence.Slice(start, reader.Position).ToArray();
return record;
}
- // An entry read from a store is written as the bytes it was read from, so a fact this runtime does not decode survives a
- // write that only carries the entry. That holds only while the decoded fields are the ones read: an entry changed since is
- // encoded from them.
- private static void WriteSchemaCatalogEntry(ref MessagePackWriter writer, CultSchemaCatalogEntry entry)
- {
- if (entry.RawBytes is { } raw && EncodeSchemaCatalogEntry(entry).AsSpan().SequenceEqual(EncodeSchemaCatalogEntry(ReadRawSchemaCatalogEntry(raw))))
- writer.WriteRaw(new ReadOnlySpan(raw));
- else
- WriteDecodedSchemaCatalogEntry(ref writer, entry);
- }
-
- private static CultSchemaCatalogEntry ReadRawSchemaCatalogEntry(byte[] raw)
- {
- var reader = new MessagePackReader(raw);
- return ReadSchemaCatalogEntry(ref reader);
- }
-
- private static byte[] EncodeSchemaCatalogEntry(CultSchemaCatalogEntry entry)
+ internal static byte[] SerializeCatalogEntry(CultSchemaCatalogEntry entry)
{
var buffer = new global::System.Buffers.ArrayBufferWriter();
var writer = new MessagePackWriter(buffer);
- WriteDecodedSchemaCatalogEntry(ref writer, entry);
+ WriteSchemaCatalogEntry(ref writer, entry);
writer.Flush();
return buffer.WrittenSpan.ToArray();
}
- private static void WriteDecodedSchemaCatalogEntry(ref MessagePackWriter writer, CultSchemaCatalogEntry entry)
+ private static void WriteSchemaCatalogEntry(ref MessagePackWriter writer, CultSchemaCatalogEntry entry)
{
writer.WriteArrayHeader(SchemaCatalogEntryFieldCount);
writer.Write(entry.SchemaId);
@@ -489,7 +486,6 @@ public static class CultDocumentMessagePackSerialization
private static CultSchemaCatalogEntry ReadSchemaCatalogEntry(ref MessagePackReader reader)
{
- var start = reader.Position;
var fieldCount = reader.ReadArrayHeader();
var entry = new CultSchemaCatalogEntry();
@@ -543,7 +539,6 @@ public static class CultDocumentMessagePackSerialization
reader.Skip();
}
- entry.RawBytes = reader.Sequence.Slice(start, reader.Position).ToArray();
return entry;
}
@@ -620,9 +615,9 @@ public class SingleFileMessagePackBackingStore : SingleFileBackingStore
{
}
- protected override byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot)
+ protected override byte[] SerializeStore(string formatVersion, IReadOnlyList catalogEntries, IReadOnlyList records)
{
- return CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot);
+ return CultDocumentMessagePackSerialization.SerializeStore(formatVersion, catalogEntries, records);
}
protected override byte[] SerializeRecord(CultPersistedRecord record)
@@ -630,11 +625,16 @@ public class SingleFileMessagePackBackingStore : SingleFileBackingStore
return CultDocumentMessagePackSerialization.SerializePersistedRecord(record);
}
- protected override CultPersistedStoreSnapshot DeserializeSnapshot(byte[] data)
+ protected override byte[] SerializeCatalogEntry(CultSchemaCatalogEntry entry)
{
- var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(data);
- CultDocumentMessagePackSerialization.RequireSingleFileFormat(snapshot);
- return snapshot;
+ return CultDocumentMessagePackSerialization.SerializeCatalogEntry(entry);
+ }
+
+ protected override (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) ReadStore(byte[] data)
+ {
+ var store = CultDocumentMessagePackSerialization.ReadStore(data);
+ CultDocumentMessagePackSerialization.RequireSingleFileFormat(store.Snapshot);
+ return store;
}
protected override byte[] SerializePayload(object document)
diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs
index 30ec5075..bfb7eca0 100644
--- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs
+++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs
@@ -75,7 +75,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
// writer created. A page that fails under an unmoved manifest is corruption, not a race.
using var lease = AcquireCommitLease(wait: true, create: false);
var manifestBytes = ReadManifestBytes();
- var manifest = ParseManifest(manifestBytes);
+ var (manifest, _) = ParseManifest(manifestBytes);
Trace($"manifest records={manifest.Records.Length}");
try
{
@@ -181,7 +181,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
using var commitLease = AcquireCommitLease(wait, create: true);
if (commitLease == null)
return CultCommitOutcome.Contended;
- var manifest = ReadManifest();
+ var (manifest, catalogBytes) = ReadManifest();
if (request.HasConditions && !request.ConditionsHold(
manifest.Records.ToDictionary(record => record.Key, record => VersionOf(record.Payload), StringComparer.Ordinal), Entries.Values))
return CultCommitOutcome.Mismatch;
@@ -206,7 +206,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
_deletedKeys.TryRemove(entry.Key.Value, out _);
}
IsDirty = true;
- WriteGeneration(manifest);
+ WriteGeneration(manifest, catalogBytes);
return CultCommitOutcome.Committed;
}
catch
@@ -236,9 +236,9 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
Directory.CreateDirectory(_manifestFile.DirectoryName!);
using var commitLease = AcquireCommitLease(wait: true, create: true);
- var manifest = ReadManifest();
+ var (manifest, catalogBytes) = ReadManifest();
RefuseForeign(manifest, Array.Empty());
- WriteGeneration(manifest);
+ WriteGeneration(manifest, catalogBytes);
});
}
@@ -258,7 +258,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
});
// Runs under the commit lease: pages first, then the manifest that names them.
- private void WriteGeneration(CultPersistedStoreSnapshot currentManifest)
+ private void WriteGeneration(CultPersistedStoreSnapshot currentManifest, IReadOnlyDictionary catalogBytes)
{
Directory.CreateDirectory(_recordDirectory.FullName);
var currentIndex = currentManifest.Records.ToDictionary(record => record.Key, record => record, StringComparer.Ordinal);
@@ -316,7 +316,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
copiesAny: currentIndex.Values.Any(record => !written.Contains(record.Key)),
directoryStore: true,
holdsVariants: false);
- WriteManifest(targetCatalog, currentIndex.Values
+ WriteManifest(targetCatalog, catalogBytes, currentIndex.Values
.OrderBy(record => record.Key, StringComparer.Ordinal)
.ToArray(), header);
@@ -327,16 +327,19 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
MarkFlushSucceeded();
}
- private void WriteManifest(CultSchemaCatalogEntry[] catalog, CultPersistedRecord[] index, string header)
- {
- var manifest = new CultPersistedStoreSnapshot
- {
- FormatVersion = header,
- SchemaCatalog = catalog,
- Records = index
- };
- WriteFileAtomically(_manifestFile.FullName, CultDocumentMessagePackSerialization.SerializeSnapshot(manifest));
- }
+ // The index records are this store's own and are encoded from their fields; a catalog entry the manifest already held is written
+ // as the slice it held, one the writer built is encoded.
+ private void WriteManifest(
+ CultSchemaCatalogEntry[] catalog,
+ IReadOnlyDictionary catalogBytes,
+ CultPersistedRecord[] index,
+ string header) =>
+ WriteFileAtomically(
+ _manifestFile.FullName,
+ CultDocumentMessagePackSerialization.SerializeStore(
+ header,
+ catalog.Select(entry => catalogBytes.TryGetValue(entry, out var held) ? held : CultDocumentMessagePackSerialization.SerializeCatalogEntry(entry)).ToArray(),
+ index.Select(CultDocumentMessagePackSerialization.SerializePersistedRecord).ToArray()));
private void LoadRecordPages(
CultPersistedRecord[] records,
@@ -452,7 +455,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
Payload = HashPayload(pagePayload)
};
- private CultPersistedStoreSnapshot ReadManifest() => ParseManifest(ReadManifestBytes());
+ private (CultPersistedStoreSnapshot Snapshot, Dictionary CatalogBytes) ReadManifest() => ParseManifest(ReadManifestBytes());
private byte[]? ReadManifestBytes() => File.Exists(_manifestFile.FullName) ? ReadAllBytesShared(_manifestFile.FullName) : null;
@@ -478,29 +481,31 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
private InvalidOperationException Unsettled(Exception? last) => new(
$"Directory store {_manifestFile.FullName} changed under every one of {UnleasedLoadAttempts} unlocked loads; it did not settle.", last);
- // A missing manifest is an empty store. An existing manifest in any other format is refused.
- private CultPersistedStoreSnapshot ParseManifest(byte[]? manifestBytes)
+ // A missing manifest is an empty store. An existing manifest in any other format is refused. Beside the snapshot, the bytes the
+ // manifest held for each catalog entry, by the entry this read produced: the slices a write carries forward.
+ private (CultPersistedStoreSnapshot Snapshot, Dictionary CatalogBytes) ParseManifest(byte[]? manifestBytes)
{
if (manifestBytes == null)
{
- return new CultPersistedStoreSnapshot
+ return (new CultPersistedStoreSnapshot
{
FormatVersion = IndexedFormatVersion,
SchemaCatalog = Array.Empty(),
Records = Array.Empty()
- };
+ }, new Dictionary());
}
- CultPersistedStoreSnapshot snapshot;
+ (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) read;
try
{
- snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(manifestBytes);
+ read = CultDocumentMessagePackSerialization.ReadStore(manifestBytes);
}
catch (CultStoreUnreadableException ex) when (ex.Path == null)
{
throw new CultStoreUnreadableException(ex.Message, _manifestFile.FullName, ex.InnerException);
}
+ var snapshot = read.Snapshot;
if (!string.Equals(snapshot.FormatVersion, IndexedFormatVersion, StringComparison.Ordinal) &&
!string.Equals(snapshot.FormatVersion, IndexedFormatVersionWithIds, StringComparison.Ordinal))
{
@@ -509,7 +514,10 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
_manifestFile.FullName);
}
- return snapshot;
+ var catalogBytes = new Dictionary();
+ for (var index = 0; index < snapshot.SchemaCatalog.Length; index++)
+ catalogBytes[snapshot.SchemaCatalog[index]] = read.CatalogBytes[index];
+ return (snapshot, catalogBytes);
}
private string ContentAddressedRecordPath(CultPersistedRecord metadata)
diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md
index 761200d0..9065a377 100644
--- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md
+++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md
@@ -68,6 +68,13 @@ and a catalog entry's exact bytes, unknown fields included, so a copied record
keeps its version. No type interprets a copied record: a write may hash another
record's bytes to compare versions, and reads its header (key, schema id, variant
slot), but never decodes its payload.
+A record is encoded from the object the writer hands in, and every other record is copied as the store holds it. The
+store that read a file owns that file's bytes: for each stored record, its exact slice of the file and the SHA-256 of that
+slice, which is the record's version; likewise each catalog entry's slice. A decoded record or snapshot object carries no
+bytes, so an object a caller edited in place is never mistaken for what the file holds. Consequently a record the writer
+staged is re-encoded canonically and has the version of its new encoding, even when its content is unchanged, and a
+record nobody staged keeps its bytes and its version for as long as no writer stages it. The public snapshot and record
+serializers encode every record from its fields.
A record written over one the store holds gets a storedAt later than the one it
replaces. A document changed in place and not staged is not written.
diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs
index 1f509ae1..b932a1da 100644
--- a/src/GameCult.Caching/CultCache.cs
+++ b/src/GameCult.Caching/CultCache.cs
@@ -29,16 +29,10 @@ namespace GameCult.Caching
public CultSchemaMemberCatalogEntry[] Members { get; set; } = Array.Empty();
- // The exact MessagePack bytes this entry was read from, when it was read from a store. A writer that carries the entry
- // forward emits them as they are, so what this runtime does not decode (fields beyond the ones it knows, the width of an int,
- // nil) survives; an entry whose decoded fields were changed since is encoded from them instead. Null for an entry a runtime
- // built.
- internal byte[]? RawBytes { get; set; }
-
// The catalog a write leaves, derived from the records the written store holds and nothing else. For each schema id a record
// carries, one entry that publishes it: for an id the write stages a record under, the entry the writer registered (or passed)
// for it, which owns the id; for every other id the arrived (durable) entry that owns it, else the first that lists it as
- // compatible, carried as the bytes it was read from. A registered entry never publishes an id nothing staged sits under, so
+ // compatible (the store that read it carries its bytes as read). A registered entry never publishes an id nothing staged sits under, so
// the caller passes registered entries only for the ids it writes. Entries are written as chosen, never merged; a registered
// descriptor wins over an arrived entry with the same own id. Two arrived entries with the same own id and different schema
// names, or a record no chosen entry publishes, refuse the write.
@@ -134,24 +128,13 @@ namespace GameCult.Caching
public sealed class CultPersistedRecord
{
- private string _key = string.Empty;
- private string _schemaId = string.Empty;
- private string _storedAt = string.Empty;
- private byte[] _payload = Array.Empty();
- private CultVariantDelta? _variant;
-
- public string Key { get => _key; set { _key = value; RawBytes = null; } }
- public string SchemaId { get => _schemaId; set { _schemaId = value; RawBytes = null; } }
- public string StoredAt { get => _storedAt; set { _storedAt = value; RawBytes = null; } }
- public byte[] Payload { get => _payload; set { _payload = value; RawBytes = null; } }
+ public string Key { get; set; } = string.Empty;
+ public string SchemaId { get; set; } = string.Empty;
+ public string StoredAt { get; set; } = string.Empty;
+ public byte[] Payload { get; set; } = Array.Empty();
// Non-null for a variant record (slot 4): its base and overrides. Its payload is empty.
- public CultVariantDelta? Variant { get => _variant; set { _variant = value; RawBytes = null; } }
-
- // The exact bytes this record was read from, as a store file held them. A record is written as the bytes it was read from
- // while no field has been assigned since, so a copied record stays byte-identical and its version (the SHA-256 of its
- // stored bytes) is the one the file's bytes name, whatever encoding their writer chose. Assigning any field drops them.
- internal byte[]? RawBytes { get; set; }
+ public CultVariantDelta? Variant { get; set; }
}
// One record a write puts in a single-file store: encoded, with the catalog entry that publishes the id it is stored under and
@@ -3750,24 +3733,75 @@ namespace GameCult.Caching
// A write into a store holding a variant lands only while the durable versions still equal these.
private readonly Dictionary _lastRead = new(StringComparer.Ordinal);
- protected abstract byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot);
+ ///
+ /// A store file from its parts, each already encoded: a catalog entry or record this store carries forward is the bytes its
+ /// file held, one it writes is or of the object.
+ ///
+ protected abstract byte[] SerializeStore(string formatVersion, IReadOnlyList catalogEntries, IReadOnlyList records);
- // One record's persisted encoding: the bytes a version names.
+ // One record's persisted encoding: the bytes a version names, and the bytes a written record is stored as.
protected abstract byte[] SerializeRecord(CultPersistedRecord record);
- private string Version(CultPersistedRecord record)
- {
- using var sha256 = SHA256.Create();
- return VersionOf(sha256.ComputeHash(SerializeRecord(record)));
- }
+ // One catalog entry's persisted encoding, for an entry the writer built.
+ protected abstract byte[] SerializeCatalogEntry(CultSchemaCatalogEntry entry);
///
/// This store's one reader of a non-empty store file, and so its one verdict on whether the file may be replaced: open,
/// flush and commit all ask it. It must throw for anything that is not exactly one
/// store this runtime reads (a truncated file, bytes after the store, a missing or extra slot, a format or record it does
- /// not know), so a rewrite never overwrites a file it cannot see.
+ /// not know), so a rewrite never overwrites a file it cannot see. It returns the decoded snapshot and, parallel to the
+ /// snapshot's catalog and records, the exact bytes the file holds for each: the store keeps those, never the snapshot's objects.
///
- protected abstract CultPersistedStoreSnapshot DeserializeSnapshot(byte[] data);
+ protected abstract (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) ReadStore(byte[] data);
+
+ // Test seam: how many record encodings this store has hashed, to prove what a write hashes.
+ internal int RecordsHashed;
+
+ private string VersionOfBytes(byte[] bytes)
+ {
+ RecordsHashed++;
+ using var sha256 = SHA256.Create();
+ return VersionOf(sha256.ComputeHash(bytes));
+ }
+
+ // What the store's file holds, as the store owns it: the snapshot as read, and keyed by record key the exact slice the file
+ // holds for each record, whose version is the SHA-256 of that slice, hashed when a comparison first asks. Catalog entries
+ // keep their slices by the entry object this read produced. A write copies a record it did not stage from its slice here
+ // and re-encodes every record it was handed, so no object a caller edits is ever mistaken for the bytes in the file.
+ private sealed class Durable
+ {
+ private readonly SingleFileBackingStore _owner;
+ private readonly Dictionary _records;
+ private readonly Dictionary _versions = new(StringComparer.Ordinal);
+ private readonly Dictionary _catalog = new();
+
+ internal Durable(SingleFileBackingStore owner, (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) read)
+ {
+ _owner = owner;
+ Snapshot = read.Snapshot;
+ _records = new Dictionary(StringComparer.Ordinal);
+ for (var index = 0; index < Snapshot.Records.Length; index++)
+ _records[Snapshot.Records[index].Key] = read.RecordBytes[index];
+ for (var index = 0; index < Snapshot.SchemaCatalog.Length; index++)
+ _catalog.Add(Snapshot.SchemaCatalog[index], read.CatalogBytes[index]);
+ }
+
+ internal CultPersistedStoreSnapshot Snapshot { get; }
+
+ internal byte[] Slice(string key) => _records[key];
+
+ internal string Version(string key)
+ {
+ if (!_versions.TryGetValue(key, out var version))
+ _versions[key] = version = _owner.VersionOfBytes(_records[key]);
+ return version;
+ }
+
+ // The bytes to write for a catalog entry the write chose: the slice the file held when it is one of this read's entries,
+ // otherwise an encoding of the entry the writer built.
+ internal byte[] CatalogBytes(CultSchemaCatalogEntry entry) =>
+ _catalog.TryGetValue(entry, out var bytes) ? bytes : _owner.SerializeCatalogEntry(entry);
+ }
protected abstract byte[] SerializePayload(object document);
protected abstract object DeserializePayload(Type documentType, byte[] payload);
@@ -3783,8 +3817,8 @@ namespace GameCult.Caching
if (IsDirty)
return;
- var snapshot = ReadSnapshot();
- if (snapshot == null)
+ var durable = ReadSnapshot();
+ if (durable == null)
{
_lastRead.Clear();
SetLastSchemaMigrationReports(Array.Empty());
@@ -3792,13 +3826,14 @@ namespace GameCult.Caching
return;
}
+ var snapshot = durable.Snapshot;
var reports = new List(snapshot.Records.Length);
var persisted = new Dictionary(StringComparer.Ordinal);
var foreign = new List();
var versions = new Dictionary(StringComparer.Ordinal);
foreach (var record in snapshot.Records)
{
- var version = versions[record.Key] = Version(record);
+ var version = versions[record.Key] = durable.Version(record.Key);
if (Foreign(record, snapshot.SchemaCatalog) is { } carried)
{
foreign.Add(carried);
@@ -3864,7 +3899,7 @@ namespace GameCult.Caching
using (AcquireLock(wait: true))
{
var disk = ReadSnapshot();
- RefuseForeign(_staged, ForeignOnDisk(disk));
+ RefuseForeign(_staged, disk);
var (upserts, removals) = StagedWrites(Array.Empty(), Array.Empty());
Wrote(ApplyWriteSet(disk, upserts, removals));
}
@@ -3888,12 +3923,13 @@ namespace GameCult.Caching
if (fileLock == null)
return CultCommitOutcome.Contended;
- var disk = ReadSnapshot() ?? new CultPersistedStoreSnapshot();
+ var disk = ReadSnapshot();
if (request.HasConditions && !request.ConditionsHold(
- disk.Records.ToDictionary(record => record.Key, Version, StringComparer.Ordinal), Entries.Values))
+ (disk?.Snapshot.Records ?? Array.Empty()).ToDictionary(record => record.Key, record => disk!.Version(record.Key), StringComparer.Ordinal),
+ Entries.Values))
return CultCommitOutcome.Mismatch;
- RefuseForeign(request.Upserts.Concat(request.Deletes).Select(entry => entry.Key.Value).Concat(_staged), ForeignOnDisk(disk));
+ RefuseForeign(request.Upserts.Concat(request.Deletes).Select(entry => entry.Key.Value).Concat(_staged), disk);
var (upserts, removals) = StagedWrites(request.Upserts, request.Deletes);
var written = ApplyWriteSet(disk, upserts, removals);
foreach (var entry in request.Deletes)
@@ -3959,12 +3995,15 @@ namespace GameCult.Caching
// a variant, or would after this write, is written only while its record versions are what this store last read: a variant
// resolves against other records, and a write decodes none it did not stage, so it cannot judge another writer's. Versions
// only: a write hashes other records' bytes, never decodes them. Otherwise the whole write is refused and what it refused is
- // reloaded (Refuse). Returns the records written, each with the version of the bytes written.
- private IReadOnlyCollection<(CultStagedRecord Staged, string Version)> ApplyWriteSet(CultPersistedStoreSnapshot? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals)
+ // reloaded (Refuse). Every record the writer staged is encoded from its object now, and its version is the hash of that
+ // encoding; every other record is copied as the slice the file holds. Returns the records written, each with the version of
+ // the bytes written.
+ private IReadOnlyCollection<(CultStagedRecord Staged, string Version)> ApplyWriteSet(Durable? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals)
{
RefuseWriteIntoMovedVariantStore(durable, upserts, removals);
+ var durableRecords = durable?.Snapshot.Records ?? Array.Empty();
// A record written over one the store holds is stored at a storedAt later than the one it replaces, whoever wrote that.
- var replaced = (durable?.Records ?? Array.Empty()).ToDictionary(record => record.Key, record => record.StoredAt, StringComparer.Ordinal);
+ var replaced = durableRecords.ToDictionary(record => record.Key, record => record.StoredAt, StringComparer.Ordinal);
foreach (var staged in upserts)
{
if (replaced.TryGetValue(staged.Record.Key, out var durableAt))
@@ -3973,33 +4012,34 @@ namespace GameCult.Caching
var written = upserts.Select(staged => staged.Record.Key).ToHashSet(StringComparer.Ordinal);
var removed = removals.ToHashSet(StringComparer.Ordinal);
- var copied = (durable?.Records ?? Array.Empty())
+ var copied = durableRecords
.Where(record => !written.Contains(record.Key) && !removed.Contains(record.Key))
.ToArray();
- var records = copied.Concat(upserts.Select(staged => staged.Record))
- .OrderBy(record => record.Key, StringComparer.Ordinal)
+ var encoded = upserts.Select(staged => (Staged: staged, Bytes: SerializeRecord(staged.Record))).ToArray();
+ var records = copied.Concat(upserts.Select(staged => staged.Record)).ToArray();
+ var fileRecords = copied.Select(record => (record.Key, Bytes: durable!.Slice(record.Key)))
+ .Concat(encoded.Select(pair => (pair.Staged.Record.Key, pair.Bytes)))
+ .OrderBy(pair => pair.Key, StringComparer.Ordinal)
+ .Select(pair => pair.Bytes)
.ToArray();
- var snapshot = new CultPersistedStoreSnapshot
- {
- FormatVersion = HeaderForWrite(
- durable?.FormatVersion,
- upserts.Select(staged => staged.HoldsIds).ToArray(),
- copiesAny: copied.Length > 0,
- directoryStore: false,
- holdsVariants: records.Any(record => record.Variant != null)),
- SchemaCatalog = CultSchemaCatalogEntry.Derive(
- records,
- upserts.Select(staged => staged.Entry).Distinct().ToArray(),
- durable?.SchemaCatalog ?? Array.Empty())
- .OrderBy(entry => entry.SchemaName, StringComparer.Ordinal)
- .ThenBy(entry => entry.SchemaId, StringComparer.Ordinal)
- .ToArray(),
- Records = records
- };
+ var catalog = CultSchemaCatalogEntry.Derive(
+ records,
+ upserts.Select(staged => staged.Entry).Distinct().ToArray(),
+ durable?.Snapshot.SchemaCatalog ?? Array.Empty())
+ .OrderBy(entry => entry.SchemaName, StringComparer.Ordinal)
+ .ThenBy(entry => entry.SchemaId, StringComparer.Ordinal)
+ .Select(entry => durable?.CatalogBytes(entry) ?? SerializeCatalogEntry(entry))
+ .ToArray();
+ var format = HeaderForWrite(
+ durable?.Snapshot.FormatVersion,
+ upserts.Select(staged => staged.HoldsIds).ToArray(),
+ copiesAny: copied.Length > 0,
+ directoryStore: false,
+ holdsVariants: records.Any(record => record.Variant != null));
- var versions = upserts.Select(staged => (staged, Version(staged.Record))).ToArray();
+ var versions = encoded.Select(pair => (pair.Staged, VersionOfBytes(pair.Bytes))).ToArray();
Directory.CreateDirectory(FileInfo.DirectoryName!);
- WriteSnapshotAtomically(FileInfo.FullName, SerializeSnapshot(snapshot));
+ WriteSnapshotAtomically(FileInfo.FullName, SerializeStore(format, catalog, fileRecords));
foreach (var key in removed)
_lastRead.Remove(key);
foreach (var (staged, version) in versions)
@@ -4007,13 +4047,13 @@ namespace GameCult.Caching
return versions;
}
- private void RefuseWriteIntoMovedVariantStore(CultPersistedStoreSnapshot? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals)
+ private void RefuseWriteIntoMovedVariantStore(Durable? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals)
{
- var records = durable?.Records ?? Array.Empty();
+ var records = durable?.Snapshot.Records ?? Array.Empty();
if (records.All(record => record.Variant == null) && upserts.All(staged => staged.Record.Variant == null))
return;
var changed = new HashSet(StringComparer.Ordinal);
- var versions = records.ToDictionary(record => record.Key, Version, StringComparer.Ordinal);
+ var versions = records.ToDictionary(record => record.Key, record => durable!.Version(record.Key), StringComparer.Ordinal);
foreach (var (key, version) in versions)
{
if (!_lastRead.TryGetValue(key, out var read) || !SameStoredIdentity(read, version))
@@ -4035,9 +4075,9 @@ namespace GameCult.Caching
var changedKeys = changed.OrderBy(key => key, StringComparer.Ordinal).ToArray();
Refuse(
recordKeys.Concat(changedKeys),
- key => Cache == null || !byKey.TryGetValue(key, out var record) || Foreign(record, durable!.SchemaCatalog) != null
+ key => Cache == null || !byKey.TryGetValue(key, out var record) || Foreign(record, durable!.Snapshot.SchemaCatalog) != null
? null
- : ToStoredDocument(record, versions[key], durable!.SchemaCatalog, DeserializePayload, out _),
+ : ToStoredDocument(record, versions[key], durable!.Snapshot.SchemaCatalog, DeserializePayload, out _),
key =>
{
_staged.Remove(key);
@@ -4061,30 +4101,31 @@ namespace GameCult.Caching
inner));
}
- private void RefuseForeign(
- IEnumerable written,
- IReadOnlyDictionary foreign) =>
+ private void RefuseForeign(IEnumerable written, Durable? disk)
+ {
+ var foreign = ForeignOnDisk(disk);
RefuseForeign(
written,
- key => foreign.TryGetValue(key, out var carried) ? carried.Foreign : null,
+ key => foreign.TryGetValue(key, out var carried) ? carried : null,
key =>
{
_staged.Remove(key);
IsDirty = _staged.Count > 0;
- if (foreign.TryGetValue(key, out var carried))
- _lastRead[key] = Version(carried.Record);
+ if (foreign.ContainsKey(key))
+ _lastRead[key] = disk!.Version(key);
else
_lastRead.Remove(key);
});
+ }
// The records the file holds that no registered type claims, by key.
- private Dictionary ForeignOnDisk(CultPersistedStoreSnapshot? disk)
+ private Dictionary ForeignOnDisk(Durable? disk)
{
- var foreign = new Dictionary(StringComparer.Ordinal);
- foreach (var record in disk?.Records ?? Array.Empty())
+ var foreign = new Dictionary(StringComparer.Ordinal);
+ foreach (var record in disk?.Snapshot.Records ?? Array.Empty())
{
- if (Foreign(record, disk!.SchemaCatalog) is { } carried)
- foreign[record.Key] = (carried, record);
+ if (Foreign(record, disk!.Snapshot.SchemaCatalog) is { } carried)
+ foreign[record.Key] = carried;
}
return foreign;
@@ -4110,14 +4151,14 @@ namespace GameCult.Caching
// Open, flush and commit read the file through this one call, so they agree on which files are stores. A file that is
// gone is an empty store: nothing to read, nothing to lose. A zero-byte file is not a store; it is refused.
- private CultPersistedStoreSnapshot? ReadSnapshot()
+ private Durable? ReadSnapshot()
{
var bytes = ReadDisk();
if (bytes == null)
return null;
try
{
- return DeserializeSnapshot(bytes);
+ return new Durable(this, ReadStore(bytes));
}
catch (CultStoreUnreadableException ex) when (ex.Path == null)
{
@@ -4127,7 +4168,7 @@ namespace GameCult.Caching
// CultMesh's single-file document helpers reach this store file only through this read and ApplyWriteSet, so they share
// its one reader, its lock and its atomic replace.
- internal CultPersistedStoreSnapshot? ReadDurable() => Held(ReadSnapshot);
+ internal CultPersistedStoreSnapshot? ReadDurable() => Held(() => ReadSnapshot()?.Snapshot);
private byte[]? ReadDisk()
{
diff --git a/tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs b/tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs
index 8fd1c73e..b87d5926 100644
--- a/tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs
+++ b/tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs
@@ -189,6 +189,7 @@ namespace GameCult.Caching.Tests
var path = OldStore("copied.cc");
var descriptor = Declaring.GetRequired();
var before = Read(path);
+ var entryBefore = StoreSlices.CatalogEntry(path, OldId);
using (var cache = Open(path, Declaring))
cache.Commit(batch => batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "e" }, new CultRecordKey("e")));
@@ -199,7 +200,7 @@ namespace GameCult.Caching.Tests
var copied = written.Records.Single(record => record.Key == "d");
var original = before.Records.Single();
Assert.That((copied.StoredAt, copied.Payload), Is.EqualTo((original.StoredAt, original.Payload)));
- Assert.That(written.SchemaCatalog.Single(entry => entry.SchemaId == OldId).RawBytes, Is.EqualTo(before.SchemaCatalog.Single().RawBytes),
+ Assert.That(StoreSlices.CatalogEntry(path, OldId), Is.EqualTo(entryBefore),
"the entry that published the old id is carried as its bytes");
}
diff --git a/tests/GameCult.Caching.Tests/ConditionalCommitTests.cs b/tests/GameCult.Caching.Tests/ConditionalCommitTests.cs
index 0350e1e8..749fa66a 100644
--- a/tests/GameCult.Caching.Tests/ConditionalCommitTests.cs
+++ b/tests/GameCult.Caching.Tests/ConditionalCommitTests.cs
@@ -895,9 +895,9 @@ namespace GameCult.Caching.Tests
base.Push(entry);
}
- protected override CultPersistedStoreSnapshot DeserializeSnapshot(byte[] data)
+ protected override (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) ReadStore(byte[] data)
{
- var snapshot = base.DeserializeSnapshot(data);
+ var snapshot = base.ReadStore(data);
if (Pause is { } pause)
{
Paused = true;
diff --git a/tests/GameCult.Caching.Tests/StoreSlices.cs b/tests/GameCult.Caching.Tests/StoreSlices.cs
new file mode 100644
index 00000000..a27f2468
--- /dev/null
+++ b/tests/GameCult.Caching.Tests/StoreSlices.cs
@@ -0,0 +1,27 @@
+using System;
+using System.IO;
+using System.Linq;
+using GameCult.Caching.MessagePack;
+
+namespace GameCult.Caching.Tests
+{
+ // The exact bytes a store file holds for its catalog entries and records. A read snapshot carries none: the store that read the
+ // file owns them, and these read them as the store's reader does.
+ internal static class StoreSlices
+ {
+ internal static byte[] CatalogEntry(string path, string schemaId)
+ {
+ var read = CultDocumentMessagePackSerialization.ReadStore(File.ReadAllBytes(path));
+ return read.CatalogBytes[Array.FindIndex(read.Snapshot.SchemaCatalog, entry => entry.SchemaId == schemaId)];
+ }
+
+ internal static byte[][] CatalogEntries(string path) =>
+ CultDocumentMessagePackSerialization.ReadStore(File.ReadAllBytes(path)).CatalogBytes;
+
+ internal static byte[] Record(string path, string key)
+ {
+ var read = CultDocumentMessagePackSerialization.ReadStore(File.ReadAllBytes(path));
+ return read.RecordBytes[Array.FindIndex(read.Snapshot.Records, record => record.Key == key)];
+ }
+ }
+}
diff --git a/tests/GameCult.Caching.Tests/VariantTests.cs b/tests/GameCult.Caching.Tests/VariantTests.cs
index 5c2ecc35..99304283 100644
--- a/tests/GameCult.Caching.Tests/VariantTests.cs
+++ b/tests/GameCult.Caching.Tests/VariantTests.cs
@@ -847,6 +847,7 @@ namespace GameCult.Caching.Tests
var path = PathOf(viaFlush ? "drift-untouched-flush.cc" : "drift-untouched-commit.cc");
SeedDrift(path);
var before = ReadStore(path);
+ var wideEntryBefore = StoreSlices.CatalogEntry(path, DriftWideId);
using (var narrow = Open(path, registry: CultDocumentRegistry.ForTypes(new[] { typeof(DriftNarrow) })))
{
@@ -870,8 +871,8 @@ namespace GameCult.Caching.Tests
}
Assert.That(OverriddenSlotsOn(path, DriftVariant.Value).OrderBy(slot => slot), Is.EqualTo(new[] { 0, 2 }), "the dropped slot's override is still stored");
- Assert.That(after.SchemaCatalog.Single(entry => entry.SchemaId == DriftWideId).RawBytes,
- Is.EqualTo(before.SchemaCatalog.Single(entry => entry.SchemaId == DriftWideId).RawBytes));
+ Assert.That(StoreSlices.CatalogEntry(path, DriftWideId),
+ Is.EqualTo(wideEntryBefore));
}
// A type that lists Wide's id as compatible owns Wide's records: staging one writes it under the type's own id, at a later
diff --git a/tests/GameCult.Caching.Tests/WriteSetTests.cs b/tests/GameCult.Caching.Tests/WriteSetTests.cs
index 217b57ff..96c508fa 100644
--- a/tests/GameCult.Caching.Tests/WriteSetTests.cs
+++ b/tests/GameCult.Caching.Tests/WriteSetTests.cs
@@ -37,7 +37,29 @@ namespace GameCult.Caching.Tests
public string Name = string.Empty;
}
- private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] { typeof(WsItem), typeof(WsOther) });
+ [CultDocument("tests.write_set_nested", "tests.write_set_nested.v1")]
+ [MessagePackObject(AllowPrivate = true)]
+ internal sealed class WsNested
+ {
+ [Key(0)]
+ [CultName]
+ public string Name = string.Empty;
+
+ [Key(1)]
+ public List Tags = new();
+
+ [Key(2)]
+ public WsInner Inner = new();
+ }
+
+ [MessagePackObject(AllowPrivate = true)]
+ internal sealed class WsInner
+ {
+ [Key(0)]
+ public int X;
+ }
+
+ private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] { typeof(WsItem), typeof(WsOther), typeof(WsNested) });
private static readonly string ItemId = Registry.GetRequired(typeof(WsItem)).SchemaId;
private static readonly CultRecordKey A = new("a");
private static readonly CultRecordKey X = new("x");
@@ -77,7 +99,7 @@ namespace GameCult.Caching.Tests
}
private static byte[] EntryBytes(string path, string schemaId) =>
- Read(path).SchemaCatalog.Single(entry => entry.SchemaId == schemaId).RawBytes!;
+ StoreSlices.CatalogEntry(path, schemaId);
private static string[] Keys(string path) => Read(path).Records.Select(record => record.Key).OrderBy(key => key, StringComparer.Ordinal).ToArray();
@@ -289,6 +311,8 @@ namespace GameCult.Caching.Tests
Assert.Ignore("CULTLIB_REAL_STORE does not name a copy of a real store.");
var path = PathOf("real.cc");
File.Copy(source!, path);
+ var beforePath = PathOf("real-before.cc");
+ File.Copy(source!, beforePath);
var before = Read(path);
using (var cache = Open(path, false))
@@ -298,11 +322,11 @@ namespace GameCult.Caching.Tests
TestContext.Out.WriteLine($"real store: {before.Records.Length} records, {before.SchemaCatalog.Length} catalog entries, header {before.FormatVersion}");
Assert.That(after.Records.Length, Is.EqualTo(before.Records.Length + 1));
Assert.That(after.FormatVersion, Is.EqualTo(before.FormatVersion));
- byte[] Bytes(CultPersistedRecord record) => CultDocumentMessagePackSerialization.SerializeSnapshot(new CultPersistedStoreSnapshot { Records = new[] { record } });
- var changed = before.Records.Where(record => !Bytes(record).SequenceEqual(Bytes(after.Records.Single(kept => kept.Key == record.Key)))).Select(record => record.Key).ToArray();
+
+ var changed = before.Records.Where(record => !StoreSlices.Record(beforePath, record.Key).SequenceEqual(StoreSlices.Record(path, record.Key))).Select(record => record.Key).ToArray();
Assert.That(changed, Is.Empty, "records that are not byte-identical");
- var afterEntries = after.SchemaCatalog.Select(entry => entry.RawBytes!).ToList();
- var missing = before.SchemaCatalog.Where(entry => !afterEntries.Any(raw => raw.SequenceEqual(entry.RawBytes!))).Select(entry => entry.SchemaId).ToArray();
+ var afterEntries = StoreSlices.CatalogEntries(path).ToList();
+ var missing = StoreSlices.CatalogEntries(beforePath).Where(raw => !afterEntries.Any(other => other.SequenceEqual(raw))).Select(raw => BitConverter.ToString(raw)).ToArray();
Assert.That(missing, Is.Empty, "catalog entries that are not byte-identical");
}
@@ -801,42 +825,30 @@ namespace GameCult.Caching.Tests
}), Is.True, "the writer's own condition on the variant holds");
}
- private sealed class CountingStore : SingleFileMessagePackBackingStore
- {
- public CountingStore(string path) : base(path) { }
-
- public int Encoded;
-
- protected override byte[] SerializeRecord(CultPersistedRecord record)
- {
- Encoded++;
- return base.SerializeRecord(record);
- }
- }
// A commit with no condition decides nothing about the other records, so it hashes none of them: only the record it writes
- // is encoded to be named. A commit with a condition compares every durable record's version.
+ // is hashed to be named. A commit with a condition compares every durable record's version.
[Test]
public void AnUnconditionalCommitHashesOnlyTheRecordsItWrites()
{
var path = PathOf("cost.cc");
Seed(path, false, "a", "b", "c", "d", "e");
- var store = new CountingStore(path);
+ var store = new SingleFileMessagePackBackingStore(path);
using var cache = new CultCache(Registry, CultCacheMessagePack.CreateCodec(Registry));
cache.AddBackingStore(store);
cache.PullAllBackingStoresAsync().GetAwaiter().GetResult();
- store.Encoded = 0;
+ store.RecordsHashed = 0;
Assert.That(cache.Commit(batch => batch.Upsert(typeof(WsItem), new WsItem { Name = "name-x" }, X)), Is.True);
- Assert.That(store.Encoded, Is.EqualTo(1));
+ Assert.That(store.RecordsHashed, Is.EqualTo(1));
- store.Encoded = 0;
+ store.RecordsHashed = 0;
Assert.That(cache.Commit(batch =>
{
batch.Expect(X, cache.Get(X)!);
batch.Upsert(typeof(WsItem), new WsItem { Name = "name-x", Note = "again" }, X);
}), Is.True);
- Assert.That(store.Encoded, Is.GreaterThanOrEqualTo(6), "a condition hashes every durable record, which is how the count proves itself");
+ Assert.That(store.RecordsHashed, Is.GreaterThanOrEqualTo(6), "a condition hashes every durable record, which is how the count proves itself");
}
// Records no store has read or written are told apart however many are made in one clock tick.
@@ -894,6 +906,126 @@ namespace GameCult.Caching.Tests
private static string StoredAtOf(string path, string key) => Read(path).Records.Single(record => record.Key == key).StoredAt;
+ // The stored bytes belong to the store that read them, never to a record object: a record read through the public snapshot
+ // API is only a decoded copy, so an edit made to its payload in place is what a write of it stores.
+ [Test]
+ public void ARecordReadThroughTheSnapshotApiAndEditedInPlaceIsWrittenAsEdited()
+ {
+ var path = PathOf("edit-payload.cc");
+ Seed(path, false, "k");
+ var snapshot = Read(path);
+ OverwriteInPlace(snapshot.Records.Single().Payload, "seeded", "SEEDED");
+
+ File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot));
+
+ Assert.That(NoteOnDisk(path, false), Is.EqualTo("SEEDED"));
+ var record = CultDocumentMessagePackSerialization.DeserializePersistedRecord(StoreSlices.Record(path, "k"));
+ OverwriteInPlace(record.Payload, "SEEDED", "seeded");
+ Assert.That(
+ CultDocumentMessagePackSerialization.DeserializePersistedRecord(CultDocumentMessagePackSerialization.SerializePersistedRecord(record)).Payload,
+ Is.EqualTo(record.Payload), "a single record is re-encoded from its fields too");
+ }
+
+ [Test]
+ public void AnOverrideReadThroughTheSnapshotApiAndEditedInPlaceIsWrittenAsEdited()
+ {
+ var path = PathOf("edit-override.cc");
+ Seed(path, false, "a");
+ using (var seed = Open(path, false))
+ AddVariant(seed, "v", A);
+ var snapshot = Read(path);
+ var variant = snapshot.Records.Single(record => record.Key == "v").Variant!;
+ OverwriteInPlace(variant.Overrides.Single(entry => entry.Path[0].Slot == 1).Value, "variant-v", "VARIANT-v");
+
+ File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot));
+
+ using var fresh = Open(path, false);
+ Assert.That(fresh.Get(new CultRecordKey("v"))!.Note, Is.EqualTo("VARIANT-v"));
+ }
+
+ // A held document edited in place (a list, a nested object) and put back as the same instance is written as edited, and a
+ // later write of a neighbour copies it as the file then holds it.
+ [Test]
+ public void AHeldDocumentEditedInPlaceAndPutBackIsWrittenAsEdited([Values] bool commit)
+ {
+ var path = PathOf("edit-held.cc");
+ var n = new CultRecordKey("n");
+ using (var seed = Open(path, false))
+ seed.Commit(batch => batch.Upsert(typeof(WsNested), new WsNested { Name = "name-n", Tags = { "t1" }, Inner = { X = 1 } }, n));
+
+ using (var cache = Open(path, false))
+ {
+ var held = cache.Get(n)!;
+ held.Tags.Add("t2");
+ held.Inner.X = 9;
+ Land(cache, commit, typeof(WsNested), held, n);
+ }
+
+ using (var fresh = Open(path, false))
+ {
+ var read = fresh.Get(n)!;
+ Assert.That(read.Tags, Is.EqualTo(new[] { "t1", "t2" }));
+ Assert.That(read.Inner.X, Is.EqualTo(9));
+ }
+
+ var written = StoreSlices.Record(path, "n");
+ using (var other = Open(path, false))
+ Land(other, commit, typeof(WsItem), new WsItem { Name = "name-x" }, X);
+ Assert.That(StoreSlices.Record(path, "n"), Is.EqualTo(written), "the edit is copied as the file holds it");
+ }
+
+ // A record the writer did not stage keeps the bytes and the version its file holds, whatever encoding its writer chose,
+ // across any number of writes of its neighbours.
+ [Test]
+ public void AnUntouchedNonCanonicalRecordKeepsItsBytesAndVersionAcrossAWriteOfANeighbour([Values] bool commit)
+ {
+ var path = PathOf("untouched-wide.cc");
+ Seed(path, false, "k");
+ File.WriteAllBytes(path, WidenEveryRecord(File.ReadAllBytes(path), out var slices));
+ using var cache = Open(path, false);
+ var version = cache.GetStored(K)!.StoredVersion;
+ Assert.That(version, Is.EqualTo(HexOfSha256(slices["k"])));
+
+ Land(cache, commit, typeof(WsItem), new WsItem { Name = "name-x" }, X);
+ Land(cache, commit, typeof(WsItem), new WsItem { Name = "name-y" }, new CultRecordKey("y"));
+
+ Assert.That(StoreSlices.Record(path, "k"), Is.EqualTo(slices["k"]));
+ Assert.That(cache.GetStored(K)!.StoredVersion, Is.EqualTo(version));
+ using var fresh = Open(path, false);
+ Assert.That(fresh.GetStored(K)!.StoredVersion, Is.EqualTo(version));
+ }
+
+ // A record the writer stages is encoded from its object, so a non-canonical record put back unchanged is written canonically
+ // and, being other bytes, has another version.
+ [Test]
+ public void AStagedNonCanonicalRecordPutBackUnchangedIsReEncodedAndGetsANewVersion([Values] bool commit)
+ {
+ var path = PathOf("staged-wide.cc");
+ Seed(path, false, "k");
+ File.WriteAllBytes(path, WidenEveryRecord(File.ReadAllBytes(path), out var slices));
+ using var cache = Open(path, false);
+ var held = cache.Get(K)!;
+
+ Land(cache, commit, typeof(WsItem), held, K);
+
+ var written = StoreSlices.Record(path, "k");
+ Assert.That(written, Is.Not.EqualTo(slices["k"]));
+ Assert.That(written, Is.EqualTo(CultDocumentMessagePackSerialization.SerializePersistedRecord(Read(path).Records.Single())),
+ "the record is in the canonical encoding");
+ Assert.That(cache.GetStored(K)!.StoredVersion, Is.EqualTo(HexOfSha256(written)));
+ Assert.That(cache.GetStored(K)!.StoredVersion, Is.Not.EqualTo(HexOfSha256(slices["k"])));
+ }
+
+ private static void OverwriteInPlace(byte[] bytes, string from, string to)
+ {
+ var was = System.Text.Encoding.UTF8.GetBytes(from);
+ var now = System.Text.Encoding.UTF8.GetBytes(to);
+ Assert.That(now.Length, Is.EqualTo(was.Length));
+ var at = IndexOf(bytes, was);
+ Assert.That(at, Is.GreaterThanOrEqualTo(0), $"'{from}' is in the bytes");
+ Array.Copy(now, 0, bytes, at, now.Length);
+ }
+
private static void AddVariant(CultCache cache, string key, CultRecordKey baseKey) =>
cache.Commit(batch => batch.UpsertVariant(new CultRecordKey(key), baseKey, new[]
{