diff --git a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs index df7302de..823c82f2 100644 --- a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs +++ b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs @@ -182,7 +182,15 @@ public static class CultDocumentMessagePackSerialization return buffer.WrittenSpan.ToArray(); } - public static CultPersistedStoreSnapshot DeserializeSnapshot(byte[] payload) + /// + /// Reads a single-file snapshot. The stamp in slot 0 is checked against the formats this runtime reads (v1, v2) before + /// the catalog or any record is parsed, so a store stamped for another reader is refused, never mistaken for damage. + /// + public static CultPersistedStoreSnapshot DeserializeSnapshot(byte[] payload) => + DeserializeSnapshot(payload, RequireReadableSingleFileStamp); + + // The stamp decides readability and the body never does: requireReadable runs on slot 0 before anything else is parsed. + internal static CultPersistedStoreSnapshot DeserializeSnapshot(byte[] payload, Action requireReadable) { var reader = new MessagePackReader(payload); var fieldCount = reader.ReadArrayHeader(); @@ -192,6 +200,7 @@ public static class CultDocumentMessagePackSerialization { snapshot.FormatVersion = reader.ReadString() ?? throw new NotSupportedException("Store snapshot declares no format version; this runtime reads " + CultPersistedStoreSnapshot.FormatV1 + " and " + CultPersistedStoreSnapshot.FormatV2 + "."); + requireReadable(snapshot.FormatVersion); } if (fieldCount > 1) @@ -226,12 +235,7 @@ public static class CultDocumentMessagePackSerialization public static void RequireSingleFileFormat(CultPersistedStoreSnapshot snapshot) { var version = snapshot.FormatVersion; - if (!string.Equals(version, CultPersistedStoreSnapshot.FormatV1, StringComparison.Ordinal) && - !string.Equals(version, CultPersistedStoreSnapshot.FormatV2, StringComparison.Ordinal)) - { - throw new NotSupportedException( - $"Store format {DescribeHeader(version)} is not readable; this runtime reads {CultPersistedStoreSnapshot.FormatV1} and {CultPersistedStoreSnapshot.FormatV2}."); - } + RequireReadableSingleFileStamp(version); // Only a v2 store may hold a variant: a v1 header over a variant record is a store no reader could trust. if (string.Equals(version, CultPersistedStoreSnapshot.FormatV1, StringComparison.Ordinal)) @@ -245,6 +249,16 @@ public static class CultDocumentMessagePackSerialization } } + private static void RequireReadableSingleFileStamp(string version) + { + if (!string.Equals(version, CultPersistedStoreSnapshot.FormatV1, StringComparison.Ordinal) && + !string.Equals(version, CultPersistedStoreSnapshot.FormatV2, StringComparison.Ordinal)) + { + throw new NotSupportedException( + $"Store format {DescribeHeader(version)} is not readable; this runtime reads {CultPersistedStoreSnapshot.FormatV1} and {CultPersistedStoreSnapshot.FormatV2}."); + } + } + // A store header as a refusal may show it: echoed in the shape cultcache.store.v, and otherwise described // only by its length, since the bytes are the store's. private static string DescribeHeader(string header) diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index 436ec4f4..11a84591 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -448,22 +448,25 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore }; } - var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(manifestBytes); - if (CultPersistedStoreSnapshot.FormatNumber(snapshot.FormatVersion) is int number && number > 4) + // The stamp is judged before the manifest body is parsed, so a store stamped newer is refused whatever its body. + return CultDocumentMessagePackSerialization.DeserializeSnapshot(manifestBytes, RequireReadableStamp); + } + + private void RequireReadableStamp(string stamp) + { + if (CultPersistedStoreSnapshot.FormatNumber(stamp) is int number && number > 4) { // Every write path parses the manifest under its commit lease before it writes a page or the manifest, so a // store another process has since stamped newer is refused here, untouched. throw new NotSupportedException( - $"Directory store {_manifestFile.FullName} is {snapshot.FormatVersion}, a newer format than this runtime reads ({IndexedFormatVersion})."); + $"Directory store {_manifestFile.FullName} is {stamp}, a newer format than this runtime reads ({IndexedFormatVersion})."); } - if (!string.Equals(snapshot.FormatVersion, IndexedFormatVersion, StringComparison.Ordinal)) + if (!string.Equals(stamp, IndexedFormatVersion, StringComparison.Ordinal)) { throw new InvalidOperationException( - $"Directory store {_manifestFile.FullName} is {snapshot.FormatVersion}; only {IndexedFormatVersion} is readable."); + $"Directory store {_manifestFile.FullName} is {stamp}; only {IndexedFormatVersion} is readable."); } - - return snapshot; } private string ContentAddressedRecordPath(CultPersistedRecord metadata) diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 58231a49..d5fd0bbd 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -3542,39 +3542,34 @@ namespace GameCult.Caching // A plain flush replaces the file without merging it, so under the lock it first reads the disk the way attach and // commit do: a store this runtime does not read (a newer format, or a header the reader refuses) is refused and left - // as it is, and so is a read that fails on I/O. A file that merely will not parse (empty, corrupt) is not a newer - // store, and the flush proceeds as it always has, so it can still heal a damaged store. + // as it is, and so is a file that cannot be read (an I/O or access error). A file that is read but will not parse + // (empty, truncated, corrupt) is not a newer store, and the flush proceeds as it always has, so it can still heal. private void RefuseToRewriteAStoreItCannotRead() { + var bytes = ReadFileBytes(); + if (bytes == null) + return; try { - ReadSnapshot(); + DeserializeSnapshot(bytes); } catch (NotSupportedException) { throw; } - catch (IOException) - { - throw; - } - catch (UnauthorizedAccessException) - { - throw; - } catch (Exception) { } } - private CultPersistedStoreSnapshot? ReadSnapshot() + private byte[]? ReadFileBytes() { FileInfo.Refresh(); if (!FileInfo.Exists) return null; try { - return DeserializeSnapshot(ReadAllBytesShared(FileInfo.FullName)); + return ReadAllBytesShared(FileInfo.FullName); } catch (FileNotFoundException) { @@ -3582,6 +3577,12 @@ namespace GameCult.Caching } } + private CultPersistedStoreSnapshot? ReadSnapshot() + { + var bytes = ReadFileBytes(); + return bytes == null ? null : DeserializeSnapshot(bytes); + } + private void WriteSnapshot(IEnumerable records, IEnumerable catalog) { var ordered = records.OrderBy(record => record.Key, StringComparer.Ordinal).ToArray(); diff --git a/tests/GameCult.Caching.Tests/PreCut2StoreFormatTests.cs b/tests/GameCult.Caching.Tests/PreCut2StoreFormatTests.cs index 52746f84..7e339945 100644 --- a/tests/GameCult.Caching.Tests/PreCut2StoreFormatTests.cs +++ b/tests/GameCult.Caching.Tests/PreCut2StoreFormatTests.cs @@ -213,6 +213,113 @@ namespace GameCult.Caching.Tests } } + private static byte[] Raw(params object[] parts) + { + var bytes = new System.Collections.Generic.List(); + foreach (var part in parts) + { + switch (part) + { + case byte single: bytes.Add(single); break; + case string text: + var encoded = System.Text.Encoding.UTF8.GetBytes(text); + bytes.Add(0xd9); bytes.Add((byte)encoded.Length); bytes.AddRange(encoded); break; + case byte[] binary: + bytes.Add(0xc4); bytes.Add((byte)binary.Length); bytes.AddRange(binary); break; + default: throw new ArgumentException(part.GetType().Name); + } + } + return bytes.ToArray(); + } + + // The format decision comes from the header alone. A stamp this runtime does not read refuses whatever follows it, + // so a body that will not parse can never make a newer store look like damage a flush may heal. + private static System.Collections.Generic.IEnumerable NewerStampOverUnreadableBody() + { + byte garbage = 0xc1; + yield return new TestCaseData(Raw((byte)0x93, "cultcache.store.v3", garbage, garbage)).SetName("v3 header over a bad body"); + yield return new TestCaseData(Raw((byte)0x93, System.Text.Encoding.UTF8.GetBytes("cultcache.store.v3"), garbage, garbage)).SetName("bin header v3 over a bad body"); + yield return new TestCaseData(Raw((byte)0x93, System.Text.Encoding.UTF8.GetBytes("cultcache.store.v9"), garbage, garbage)).SetName("bin header v9 over a bad body"); + yield return new TestCaseData(Raw((byte)0x93, "cultcache.store.v3", (byte)0x91, garbage, (byte)0x90)).SetName("v3 header over a bad catalog"); + yield return new TestCaseData(Raw((byte)0x92, "cultcache.store.v4", garbage)).SetName("v4 header over garbage"); + } + + [TestCaseSource(nameof(NewerStampOverUnreadableBody))] + public async Task SingleFileFlushRefusesANewerStampWhateverItsBodyHolds(byte[] swapped) + { + var root = CopyFixture("single-file-v1"); + try + { + var file = Path.Combine(root, "store.msgpack"); + using var cache = new CultCache(); + cache.AddBackingStore(new SingleFileMessagePackBackingStore(file)); + await cache.PullAllBackingStoresAsync(); + File.WriteAllBytes(file, swapped); + + await cache.AddAsync(new PreCut2FixtureNote { Text = "must not overwrite" }); + Assert.Throws(() => cache.FlushAllBackingStores()); + Assert.That(File.ReadAllBytes(file), Is.EqualTo(swapped)); + Assert.Throws(() => new SingleFileMessagePackBackingStore(file).PullAll()); + } + finally + { + if (Directory.Exists(root)) Directory.Delete(root, recursive: true); + } + } + + [Test] + public async Task DirectoryWritesRefuseANewerStampWhateverTheManifestBodyHolds() + { + var root = CopyFixture("directory-v4"); + try + { + var manifest = Path.Combine(root, "store.cc"); + using var cache = new CultCache(); + cache.AddBackingStore(new DirectoryMessagePackBackingStore(manifest)); + await cache.PullAllBackingStoresAsync(); + var swapped = Raw((byte)0x93, "cultcache.store.v5.directory-content-addressed-pages", (byte)0xc1, (byte)0xc1); + File.WriteAllBytes(manifest, swapped); + + await cache.AddAsync(new PreCut2FixtureNote { Text = "must not overwrite" }); + Assert.Throws(() => cache.FlushAllBackingStores()); + Assert.That(File.ReadAllBytes(manifest), Is.EqualTo(swapped)); + } + finally + { + if (Directory.Exists(root)) Directory.Delete(root, recursive: true); + } + } + + // A file that is read but carries no readable stamp (empty, or cut off) is damage, not a newer store: a flush heals it. + [TestCase(0)] + [TestCase(1)] + [TestCase(10)] + [TestCase(-1)] + public async Task SingleFileFlushHealsAnEmptyOrTruncatedFile(int keep) + { + var root = CopyFixture("single-file-v1"); + try + { + var file = Path.Combine(root, "store.msgpack"); + using var cache = new CultCache(); + cache.AddBackingStore(new SingleFileMessagePackBackingStore(file)); + await cache.PullAllBackingStoresAsync(); + var whole = File.ReadAllBytes(file); + File.WriteAllBytes(file, whole.Take(keep < 0 ? whole.Length / 2 : keep).ToArray()); + + await cache.AddAsync(new PreCut2FixtureNote { Text = "heals" }); + cache.FlushAllBackingStores(); + + var healed = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(file)); + Assert.That(healed.FormatVersion, Is.EqualTo("cultcache.store.v1")); + Assert.That(healed.Records.Length, Is.GreaterThan(0)); + } + finally + { + if (Directory.Exists(root)) Directory.Delete(root, recursive: true); + } + } + // A store subclass that overrides nothing beyond its four abstract members is guarded by the base, not by an opt-in. private sealed class PlainCustomStore : SingleFileBackingStore { diff --git a/unity/org.gamecult.cultlib/CHANGELOG.md b/unity/org.gamecult.cultlib/CHANGELOG.md index c1e0eef5..34ec27f3 100644 --- a/unity/org.gamecult.cultlib/CHANGELOG.md +++ b/unity/org.gamecult.cultlib/CHANGELOG.md @@ -96,14 +96,15 @@ All notable changes to this package are documented in this file. 1.0.x. CultMesh single-file reads, the directory store and CultNet refuse a variant record by name; only the C# CultCache resolves variants. - A plain flush (`SingleFileBackingStore.PushAll`) reads the file under the lock before it - rewrites it, by the same rule attach and conditional commit use, and throws + rewrites it, by the same rule attach and conditional commit use (the format stamp in + the header decides, before the body is parsed), and throws `NotSupportedException` instead of writing when the store is one this runtime - refuses to read: stamped by another process with a newer format, or with a header the + refuses to read: stamped by another process with a newer format, or with a stamp the reader refuses. A read that fails with an I/O or access error also refuses. It used to overwrite the file from this cache's older view. A directory store throws the same exception, from the manifest parse every write already ran, for a manifest stamped - `cultcache.store.v` with N > 4 (it threw `InvalidOperationException`). A file that - merely cannot be parsed (empty, corrupt) is not a newer format, and a flush still + `cultcache.store.v` with N > 4 (it threw `InvalidOperationException`). A file with no + readable stamp at all (empty, truncated, or not a snapshot) is not a newer format, and a flush rewrites it. - `CultMesh.WriteSingleFileDocument` and `WriteSingleFileDocumentPayload` read the file they replace, under the store's `.lock`, and refuse (without touching