diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs index 1fb6c056..c3258c69 100644 --- a/packages/cultcache-rs/src/lib.rs +++ b/packages/cultcache-rs/src/lib.rs @@ -835,7 +835,14 @@ fn store_header(bytes: &[u8]) -> StoreHeader { let mut first = None; let mut cursor = std::io::Cursor::new(bytes); - let walked = serde::Deserializer::deserialize_seq(&mut rmp_serde::Deserializer::new(&mut cursor), HeaderVisitor(&mut first)); + // Walked on the grown stack, as every other decode of stored bytes is: a store may nest values deeper than the thread allows. + let walked = stacker::maybe_grow(DECODE_RED_ZONE, DECODE_STACK_SEGMENT, || { + let mut reader = rmp_serde::Deserializer::new(&mut cursor); + let mut grown = serde_stacker::Deserializer::new(&mut reader); + grown.red_zone = DECODE_RED_ZONE; + grown.stack_size = DECODE_STACK_SEGMENT; + serde::Deserializer::deserialize_seq(grown, HeaderVisitor(&mut first)) + }); let after = bytes.len() - cursor.position() as usize; let whole = walked .map_err(|error| opaque_decode_error(error).context("the store is not one complete MessagePack array")) diff --git a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs index 7e06a776..23c889f5 100644 --- a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs +++ b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs @@ -327,6 +327,9 @@ public static class CultDocumentMessagePackSerialization { try { + // The stamp is judged from the leading bytes alone, so a body that will not parse can never make a newer store look + // like damage. + JudgeStamp(payload, requireReadable); RequireOneArray(payload); var reader = new MessagePackReader(payload); var fieldCount = reader.ReadArrayHeader(); @@ -340,12 +343,7 @@ public static class CultDocumentMessagePackSerialization if (fieldCount == 0) return (snapshot, Array.Empty(), Array.Empty()); - // A stamp encoded as binary is not one this runtime wrote or reads; it is refused as such, not parsed as damage. - if (reader.NextMessagePackType == MessagePackType.Binary) - throw new CultStoreUnreadableException("Store snapshot declares its format version as binary; this runtime reads " + CultPersistedStoreSnapshot.FormatV1 + ", " + CultPersistedStoreSnapshot.FormatV2 + " and " + CultPersistedStoreSnapshot.FormatV3 + " as strings."); - snapshot.FormatVersion = reader.ReadString() - ?? throw new CultStoreUnreadableException("Store snapshot declares no format version; this runtime reads " + CultPersistedStoreSnapshot.FormatV1 + ", " + CultPersistedStoreSnapshot.FormatV2 + " and " + CultPersistedStoreSnapshot.FormatV3 + "."); - requireReadable?.Invoke(snapshot.FormatVersion); + snapshot.FormatVersion = reader.ReadString()!; var catalogCount = reader.ReadArrayHeader(); snapshot.SchemaCatalog = new CultSchemaCatalogEntry[catalogCount]; @@ -399,6 +397,31 @@ public static class CultDocumentMessagePackSerialization } } + // Slot 0 of a non-empty store, read before anything after it: a binary or nil stamp is refused as such, and a string stamp is + // judged by the caller's rule when it passes one. Bytes that do not open with an array and a string are left to the full read. + private static void JudgeStamp(byte[] payload, Action? requireReadable) + { + string? stamp; + try + { + var reader = new MessagePackReader(payload); + if (reader.NextMessagePackType != MessagePackType.Array || reader.ReadArrayHeader() == 0) + return; + if (reader.NextMessagePackType == MessagePackType.Binary) + throw new CultStoreUnreadableException("Store snapshot declares its format version as binary; this runtime reads " + CultPersistedStoreSnapshot.FormatV1 + ", " + CultPersistedStoreSnapshot.FormatV2 + " and " + CultPersistedStoreSnapshot.FormatV3 + " as strings."); + if (reader.TryReadNil()) + throw new CultStoreUnreadableException("Store snapshot declares no format version; this runtime reads " + CultPersistedStoreSnapshot.FormatV1 + ", " + CultPersistedStoreSnapshot.FormatV2 + " and " + CultPersistedStoreSnapshot.FormatV3 + "."); + stamp = reader.NextMessagePackType == MessagePackType.String ? reader.ReadString() : null; + } + catch (Exception ex) when (ex is EndOfStreamException or MessagePackSerializationException or InvalidOperationException) + { + return; + } + + if (stamp != null) + requireReadable?.Invoke(stamp); + } + private static void RequireOneArray(byte[] bytes) { var whole = new MessagePackReader(bytes); @@ -452,10 +475,20 @@ public static class CultDocumentMessagePackSerialization internal static string DescribeHeader(string header) { const string prefix = "cultcache.store.v"; - var version = header.StartsWith(prefix, StringComparison.Ordinal) ? header.Substring(prefix.Length) : ""; - return version.Length > 0 && version.All(character => character >= '0' && character <= '9') - ? header - : $"an unrecognised cultcache.store.* header of {System.Text.Encoding.UTF8.GetByteCount(header)} bytes"; + if (header.StartsWith(prefix, StringComparison.Ordinal)) + { + var end = prefix.Length; + while (end < header.Length && header[end] >= '0' && header[end] <= '9') + end++; + var digits = end - prefix.Length; + if (digits is > 0 and <= 6 && (end == header.Length || header[end] == '.')) + { + var named = header.Substring(0, end); + return end == header.Length ? named : $"{named} with a suffix of {System.Text.Encoding.UTF8.GetByteCount(header) - end} bytes"; + } + } + + return $"an unrecognised cultcache.store.* header of {System.Text.Encoding.UTF8.GetByteCount(header)} bytes"; } private static void WritePersistedRecord(ref MessagePackWriter writer, CultPersistedRecord record) diff --git a/tests/GameCult.Caching.Tests/ConditionalCommitTests.cs b/tests/GameCult.Caching.Tests/ConditionalCommitTests.cs index 1dfc6561..59fcd764 100644 --- a/tests/GameCult.Caching.Tests/ConditionalCommitTests.cs +++ b/tests/GameCult.Caching.Tests/ConditionalCommitTests.cs @@ -1008,6 +1008,10 @@ namespace GameCult.Caching.Tests private static StoreFactory DirectoryStore(string path, CultDocumentRegistry? registry = null) => new(path, () => new DirectoryMessagePackBackingStore(path), registry); + // The id Wide is stored under, which the later versions declare compatible: a cache reads and writes a record under another + // id only when its type declares it. SeedDrift checks that the constant is Wide's id. + private const string WideDriftSchemaId = "sha256:6033aeca8a784899aff1f8c43eb9df6bf3364e4fb6014e423c65afd51a9b8f18"; + private static readonly CultRecordKey DriftKey = new("drift"); private static readonly CultDocumentRegistry WideDrift = CultDocumentRegistry.ForTypes(new[] { typeof(ConditionalDriftWide) }); private static readonly CultDocumentRegistry NarrowDrift = CultDocumentRegistry.ForTypes(new[] { typeof(ConditionalDriftNarrow) }); @@ -1019,6 +1023,7 @@ namespace GameCult.Caching.Tests // The record at DriftKey, written under v1. private void SeedDrift(bool directory) { + Assert.That(WideDrift.GetRequired(typeof(ConditionalDriftWide)).SchemaId, Is.EqualTo(WideDriftSchemaId)); using var cache = Drift(directory, WideDrift).Open(); Assert.That(cache.Commit(batch => batch.Upsert( new ConditionalDriftWide { Name = "drift", Value = 1, Extra = "extra" }, @@ -1116,7 +1121,7 @@ namespace GameCult.Caching.Tests public string Extra = string.Empty; } - [CultDocument("tests.conditional_drift", "tests.conditional_drift.v2")] + [CultDocument("tests.conditional_drift", "tests.conditional_drift.v2", CompatibleSchemaIds = new[] { WideDriftSchemaId })] [MessagePackObject(AllowPrivate = true)] internal sealed class ConditionalDriftNarrow { @@ -1128,7 +1133,7 @@ namespace GameCult.Caching.Tests public int Value; } - [CultDocument("tests.conditional_drift", "tests.conditional_drift.v3")] + [CultDocument("tests.conditional_drift", "tests.conditional_drift.v3", CompatibleSchemaIds = new[] { WideDriftSchemaId })] [MessagePackObject(AllowPrivate = true)] internal sealed class ConditionalDriftReused { diff --git a/tests/GameCult.Caching.Tests/PreCut2StoreFormatTests.cs b/tests/GameCult.Caching.Tests/PreCut2StoreFormatTests.cs index 3298743d..ac6127f5 100644 --- a/tests/GameCult.Caching.Tests/PreCut2StoreFormatTests.cs +++ b/tests/GameCult.Caching.Tests/PreCut2StoreFormatTests.cs @@ -151,7 +151,7 @@ namespace GameCult.Caching.Tests var before = File.ReadAllBytes(file); using var cache = new CultCache(); - var error = Assert.Throws(() => + var error = Assert.Throws(() => cache.AddBackingStore(new SingleFileMessagePackBackingStore(file)))!; Assert.That(error.Message, Does.Contain(newerFormat)); Assert.That(cache.BackingStores, Is.Empty); @@ -186,11 +186,11 @@ namespace GameCult.Caching.Tests var before = File.ReadAllBytes(file); await cache.AddAsync(new PreCut2FixtureNote { Text = "staged after the other process wrote v4" }); - Assert.Throws(() => cache.FlushAllBackingStores()); + Assert.Throws(() => cache.FlushAllBackingStores()); Assert.That(File.ReadAllBytes(file), Is.EqualTo(before)); cache.Remove(new CultRecordKey("item:anvil")); - Assert.Throws(() => cache.BackingStores.Single().PushAll()); + Assert.Throws(() => cache.BackingStores.Single().PushAll()); Assert.That(File.ReadAllBytes(file), Is.EqualTo(before)); } finally @@ -270,9 +270,9 @@ namespace GameCult.Caching.Tests File.WriteAllBytes(file, swapped); await cache.AddAsync(new PreCut2FixtureNote { Text = "must not overwrite" }); - Assert.Throws(() => cache.FlushAllBackingStores()); + Assert.Throws(() => cache.FlushAllBackingStores()); Assert.That(File.ReadAllBytes(file), Is.EqualTo(swapped)); - Assert.Throws(() => new SingleFileMessagePackBackingStore(file).PullAll()); + Assert.Throws(() => new SingleFileMessagePackBackingStore(file).PullAll()); } finally { @@ -294,7 +294,7 @@ namespace GameCult.Caching.Tests File.WriteAllBytes(manifest, swapped); await cache.AddAsync(new PreCut2FixtureNote { Text = "must not overwrite" }); - Assert.Throws(() => cache.FlushAllBackingStores()); + Assert.Throws(() => cache.FlushAllBackingStores()); Assert.That(File.ReadAllBytes(manifest), Is.EqualTo(swapped)); } finally @@ -382,7 +382,7 @@ namespace GameCult.Caching.Tests var before = File.ReadAllBytes(file); await cache.AddAsync(new PreCut2FixtureNote { Text = "staged" }); - Assert.Throws(() => cache.FlushAllBackingStores()); + Assert.Throws(() => cache.FlushAllBackingStores()); Assert.That(File.ReadAllBytes(file), Is.EqualTo(before)); } finally @@ -517,12 +517,12 @@ namespace GameCult.Caching.Tests var before = files(); await cache.AddAsync(new PreCut2FixtureNote { Text = "staged after the other process wrote the newer format" }); - Assert.Throws(() => cache.FlushAllBackingStores()); + Assert.Throws(() => cache.FlushAllBackingStores()); Assert.That(files().Keys, Is.EquivalentTo(before.Keys)); foreach (var pair in before) Assert.That(File.ReadAllBytes(pair.Key), Is.EqualTo(pair.Value), pair.Key); cache.Remove(new CultRecordKey("item:anvil")); - Assert.Throws(() => cache.FlushAllBackingStores()); + Assert.Throws(() => cache.FlushAllBackingStores()); Assert.That(files().Keys, Is.EquivalentTo(before.Keys)); foreach (var pair in before) Assert.That(File.ReadAllBytes(pair.Key), Is.EqualTo(pair.Value), pair.Key); } diff --git a/tests/GameCult.Mesh.Tests/CultMeshSingleFileWriteGuardTests.cs b/tests/GameCult.Mesh.Tests/CultMeshSingleFileWriteGuardTests.cs index 2f41f52c..e1cb1cb5 100644 --- a/tests/GameCult.Mesh.Tests/CultMeshSingleFileWriteGuardTests.cs +++ b/tests/GameCult.Mesh.Tests/CultMeshSingleFileWriteGuardTests.cs @@ -45,7 +45,7 @@ public sealed class CultMeshSingleFileWriteGuardTests File.WriteAllBytes(_path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); var before = File.ReadAllBytes(_path); - var error = Assert.Throws(() => Write("item:a", 2))!; + var error = Assert.Throws(() => Write("item:a", 2))!; error.Message.Should().Contain(newer); File.ReadAllBytes(_path).Should().Equal(before); @@ -115,7 +115,7 @@ public sealed class CultMeshSingleFileWriteGuardTests File.WriteAllBytes(_path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); var before = File.ReadAllBytes(_path); - Assert.Throws(() => Write("item:a", 2)); + Assert.Throws(() => Write("item:a", 2)); File.ReadAllBytes(_path).Should().Equal(before); }