diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index 1155bbcb..f2c21616 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -15,8 +15,9 @@ namespace GameCult.Caching.MessagePack; // A small hot manifest indexes the store; each record lives in one cold content-addressed page. public sealed class DirectoryMessagePackBackingStore : CacheBackingStore { - private const string IndexedFormatVersion = CultPersistedStoreSnapshot.DirectoryFormatV4; - private const string IndexedFormatVersionWithIds = CultPersistedStoreSnapshot.DirectoryFormatV5; + // The two manifest headers; the shared header decision names them. + private static readonly string IndexedFormatVersion = HeaderFor(holdsIds: false, existingHeader: null, wholeStore: true, directoryStore: true); + private static readonly string IndexedFormatVersionWithIds = HeaderFor(holdsIds: true, existingHeader: null, wholeStore: true, directoryStore: true); // An unleased load that has not settled after this many attempts throws. private const int UnleasedLoadAttempts = 5; private readonly FileInfo _manifestFile; @@ -284,12 +285,15 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore pagePayload); } - // A manifest already marked stays marked: pages it names may hold ids this cache cannot read. - var carriesIds = string.Equals(currentManifest.FormatVersion, IndexedFormatVersionWithIds, StringComparison.Ordinal) || - keysToWrite.Any(key => Entries.TryGetValue(key, out var written) && written.HoldsIds); + // The directory store writes only its dirty pages, so it keeps a manifest already marked. + var header = HeaderFor( + keysToWrite.Any(key => Entries.TryGetValue(key, out var written) && written.HoldsIds), + currentManifest.FormatVersion, + wholeStore: false, + directoryStore: true); WriteManifest(targetCatalog, currentIndex.Values .OrderBy(record => record.Key, StringComparer.Ordinal) - .ToArray(), carriesIds); + .ToArray(), header); DeleteUnreferencedRecordPages(currentIndex.Values); @@ -299,11 +303,11 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore MarkFlushSucceeded(); } - private void WriteManifest(CultSchemaCatalogEntry[] catalog, CultPersistedRecord[] index, bool carriesElementIds) + private void WriteManifest(CultSchemaCatalogEntry[] catalog, CultPersistedRecord[] index, string header) { var manifest = new CultPersistedStoreSnapshot { - FormatVersion = CultPersistedStoreSnapshot.DirectoryFormatFor(carriesElementIds), + FormatVersion = header, SchemaCatalog = catalog, Records = index }; diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index beb57542..faf9df0d 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -65,21 +65,9 @@ namespace GameCult.Caching public const string FormatV1 = "cultcache.store.v1"; public const string FormatV2 = "cultcache.store.v2"; public const string FormatV3 = "cultcache.store.v3"; - // The directory store's manifest formats: v5 when the directory holds an element id, for the reason v3 exists. - public const string DirectoryFormatV4 = "cultcache.store.v4.directory-content-addressed-pages"; - public const string DirectoryFormatV5 = "cultcache.store.v5.directory-content-addressed-pages"; public string FormatVersion { get; set; } = FormatV1; - // The one header decision. v3 when the store holds an element id: a reader older than ids refuses the header, because it - // would skip the id slots and rewrite the elements without them. Otherwise v2 exactly when it holds a variant, and a store - // with neither stays byte-identical v1. A file already marked stays marked; the writer that knows that says so. - internal static string FormatFor(bool holdsIds, bool holdsVariants) => - holdsIds ? FormatV3 : holdsVariants ? FormatV2 : FormatV1; - - // The directory store's header decision (it refuses variants): the same owner as FormatFor. - internal static string DirectoryFormatFor(bool holdsIds) => holdsIds ? DirectoryFormatV5 : DirectoryFormatV4; - public CultSchemaCatalogEntry[] SchemaCatalog { get; set; } = Array.Empty(); public CultPersistedRecord[] Records { get; set; } = Array.Empty(); } @@ -2715,7 +2703,9 @@ namespace GameCult.Caching // decoded copy, and the delta this hold lands carries them. var bytes = entry.Value; var idPlan = CultElementIds.Plan(value, key + "." + slot.ToString(CultureInfo.InvariantCulture), deterministic: source != null); - holdsIds |= idPlan.HoldsIds; + // What the record persists: a write persists the ids it mints, a load keeps the delta as the store held it, so + // the ids a load mints exist only in memory. + holdsIds |= source == null ? idPlan.HoldsIds : idPlan.HeldIds; if (idPlan.Count > 0) { idPlan.Apply(); @@ -3042,6 +3032,23 @@ namespace GameCult.Caching private readonly object _detachedGate = new(); internal CultCache? Cache; + private const string DirectoryFormatV4 = "cultcache.store.v4.directory-content-addressed-pages"; + private const string DirectoryFormatV5 = "cultcache.store.v5.directory-content-addressed-pages"; + + // The one header decision, for every writer of every store. The marked header (v3, or the directory store's v5) says the + // store holds an element id: a reader older than ids refuses it, because it would skip the id slots and rewrite the + // elements without them. A writer that rewrites the whole store decides by content, so a store may drop back when nothing + // holds an id. A writer that touches only part of the store keeps a header already marked, because it cannot see what + // the rest holds. holdsIds is about the records this write knows are on disk. Unmarked, a single-file store is v2 exactly + // when it holds a variant, and otherwise v1; the directory store refuses variants and is v4. + protected internal static string HeaderFor(bool holdsIds, string? existingHeader, bool wholeStore, bool directoryStore, bool holdsVariants = false) + { + var marked = directoryStore ? DirectoryFormatV5 : CultPersistedStoreSnapshot.FormatV3; + if (holdsIds || (!wholeStore && string.Equals(existingHeader, marked, StringComparison.Ordinal))) + return marked; + return directoryStore ? DirectoryFormatV4 : holdsVariants ? CultPersistedStoreSnapshot.FormatV2 : CultPersistedStoreSnapshot.FormatV1; + } + // Once attached, a hold on the cache's gate: the store and its cache share one lock, so a load calling back into // the cache can never take the gate after the store lock, and a direct call publishes what it loaded when it // returns. File locks are always taken inside it. @@ -3249,7 +3256,9 @@ namespace GameCult.Caching WriteSnapshot( Entries.Values.Select(entry => ToPersistedRecord(entry, SerializePayload)), Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()), - Entries.Values.Any(entry => entry.HoldsIds)); + Entries.Values.Any(entry => entry.HoldsIds), + existingHeader: null, + wholeStore: true); Cache?.IdsPersisted(Entries.Keys); } @@ -3292,14 +3301,13 @@ namespace GameCult.Caching foreach (var entry in request.Upserts) records[entry.Key.Value] = ToPersistedRecord(entry, SerializePayload); - // One rule for every writer: the file holds ids when a record it will hold does. A record this commit replaces or removes - // no longer counts. Onto the file, only ids already persisted count (a load-minted id is on no disk), and a file already - // marked stays marked: it may hold records this cache cannot read, and a rewrite must not shed the marker. + // Onto the file, only the batch's records are seen: the rest stay as they are, and the header they carry stays. An + // unconditional commit writes this store's whole view, so what its records hold decides. A record this commit + // replaces or removes no longer counts. var replaced = request.Upserts.Concat(request.Deletes).Select(entry => entry.Key.Value).ToHashSet(StringComparer.Ordinal); var holdsIds = request.Upserts.Any(entry => entry.HoldsIds) || - Entries.Values.Any(entry => entry.HoldsIds && !replaced.Contains(entry.Key.Value) && !(ontoDisk && entry.IdsInMemoryOnly)) || - (ontoDisk && string.Equals(disk.FormatVersion, CultPersistedStoreSnapshot.FormatV3, StringComparison.Ordinal)); - WriteSnapshot(records.Values, catalog.Concat(request.Upserts.Select(entry => entry.Descriptor.ToCatalogEntry())), holdsIds); + (!ontoDisk && Entries.Values.Any(entry => entry.HoldsIds && !replaced.Contains(entry.Key.Value))); + WriteSnapshot(records.Values, catalog.Concat(request.Upserts.Select(entry => entry.Descriptor.ToCatalogEntry())), holdsIds, disk.FormatVersion, wholeStore: !ontoDisk); if (!ontoDisk) Cache?.IdsPersisted(records.Keys); foreach (var entry in request.Deletes) @@ -3354,13 +3362,13 @@ namespace GameCult.Caching } } - private void WriteSnapshot(IEnumerable records, IEnumerable catalog, bool holdsElementIds) + private void WriteSnapshot(IEnumerable records, IEnumerable catalog, bool holdsElementIds, string? existingHeader, bool wholeStore) { var ordered = records.OrderBy(record => record.Key, StringComparer.Ordinal).ToArray(); var used = new HashSet(ordered.Select(record => record.SchemaId), StringComparer.Ordinal); var snapshot = new CultPersistedStoreSnapshot { - FormatVersion = CultPersistedStoreSnapshot.FormatFor(holdsElementIds, ordered.Any(record => record.Variant != null)), + FormatVersion = HeaderFor(holdsElementIds, existingHeader, wholeStore, directoryStore: false, holdsVariants: ordered.Any(record => record.Variant != null)), SchemaCatalog = catalog .GroupBy(entry => entry.SchemaId, StringComparer.Ordinal) .Select(group => group.First()) diff --git a/src/GameCult.Caching/CultElementIds.cs b/src/GameCult.Caching/CultElementIds.cs index 3eacd7d2..29e574b9 100644 --- a/src/GameCult.Caching/CultElementIds.cs +++ b/src/GameCult.Caching/CultElementIds.cs @@ -64,9 +64,11 @@ namespace GameCult.Caching public int Count => _fills.Count; - // True when some element under the root holds a non-empty id once the plan is applied: one already there, or one this - // plan fills. It decides whether the store that writes the root carries the v3 header. - public bool HoldsIds { get; internal set; } + // True when some element under the root already holds a non-empty id: one a store persisted, or one the caller set. + public bool HeldIds { get; internal set; } + + // True when some element holds a non-empty id once the plan is applied: one already there, or one this plan fills. + public bool HoldsIds => HeldIds || _fills.Count > 0; internal void Add(object item, Shape shape, string? before, string id) => _fills.Add((item, shape, before, id)); @@ -253,7 +255,7 @@ namespace GameCult.Caching var id = getId(item); if (string.IsNullOrEmpty(id)) continue; - plan.HoldsIds = true; + plan.HeldIds = true; if (shape.Derive == null && !IsRandomId(id!)) throw new CultElementIdException( $"Record {rootPath}: element id '{id}' in the list at {path} is not 12 lowercase hex characters.", @@ -285,7 +287,6 @@ namespace GameCult.Caching else id = Mint(itemPath, deterministic, rootPath, taken); plan.Add(item, itemShape, get(item), id); - plan.HoldsIds = true; } Visit(item, itemPath); diff --git a/src/GameCult.Caching/GameCult.Caching.csproj b/src/GameCult.Caching/GameCult.Caching.csproj index 91c9133b..51253f8f 100644 --- a/src/GameCult.Caching/GameCult.Caching.csproj +++ b/src/GameCult.Caching/GameCult.Caching.csproj @@ -7,7 +7,6 @@ - diff --git a/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs b/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs index 8b62aaf6..5dbd7839 100644 --- a/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs +++ b/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs @@ -110,7 +110,8 @@ namespace GameCult.Mesh descriptor.ToCatalogEntry(), storedAt, CultDocumentMessagePackSerialization.SerializeUntyped(document, typeof(TDocument)), - ids.HoldsIds); + ids.HoldsIds, + contentKnown: true); } catch { @@ -148,7 +149,7 @@ namespace GameCult.Mesh byte[] payload) { if (schema == null) throw new ArgumentNullException(nameof(schema)); - WriteSingleFileDocumentPayload(path, key, schema.ToCatalogEntry(payload), storedAt, payload, holdsIds: false); + WriteSingleFileDocumentPayload(path, key, schema.ToCatalogEntry(payload), storedAt, payload, holdsIds: false, contentKnown: false); } /// @@ -231,7 +232,8 @@ namespace GameCult.Mesh CultSchemaCatalogEntry catalogEntry, string? storedAt, byte[] payload, - bool holdsIds) + bool holdsIds, + bool contentKnown) { if (string.IsNullOrWhiteSpace(path)) throw new ArgumentException("Value must be non-empty.", nameof(path)); if (catalogEntry == null) throw new ArgumentNullException(nameof(catalogEntry)); @@ -239,11 +241,11 @@ namespace GameCult.Mesh throw new ArgumentException("Catalog entry must include a schema id.", nameof(catalogEntry)); payload ??= Array.Empty(); - // A raw payload is opaque: whether it holds element ids is the typed caller's to say. A file already marked stays - // marked, as it does in a cache's own store. + // A typed write sees the document it replaces the file with, so its ids decide. A raw payload is opaque: the writer + // cannot see whether it holds ids, so a file already marked stays marked. var snapshot = new CultPersistedStoreSnapshot { - FormatVersion = CultPersistedStoreSnapshot.FormatFor(holdsIds || IsMarkedOnDisk(path), holdsVariants: false), + FormatVersion = CacheBackingStore.HeaderFor(holdsIds, contentKnown ? null : ExistingHeader(path), wholeStore: contentKnown, directoryStore: false), SchemaCatalog = new[] { catalogEntry }, Records = new[] { @@ -264,17 +266,15 @@ namespace GameCult.Mesh WriteFileAtomically(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); } - private static bool IsMarkedOnDisk(string path) + private static string? ExistingHeader(string path) { try { - var reader = new MessagePackReader(File.ReadAllBytes(path)); - reader.ReadArrayHeader(); - return string.Equals(reader.ReadString(), CultPersistedStoreSnapshot.FormatV3, StringComparison.Ordinal); + return CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).FormatVersion; } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or MessagePackSerializationException or EndOfStreamException or InvalidOperationException) + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or MessagePackSerializationException or EndOfStreamException or InvalidOperationException or NotSupportedException) { - return false; + return null; } } diff --git a/tests/GameCult.Caching.Tests/ElementIdTests.cs b/tests/GameCult.Caching.Tests/ElementIdTests.cs index 027f9bc5..60d1caba 100644 --- a/tests/GameCult.Caching.Tests/ElementIdTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdTests.cs @@ -810,29 +810,12 @@ namespace GameCult.Caching.Tests Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3"), "the file holds ids, so it is marked"); } - // A conditional commit lands onto the file as it is, and decides its header as every writer does: by the ids the file - // will hold. Ids on disk mark it whatever header the file carried; ids only this cache minted are on no disk. + // A conditional commit lands onto the file as it is: it sees only its own batch, so ids only this cache minted (they are on no + // disk) do not mark the file. [Test] - public void AConditionalCommitOntoAFileHoldingIdsMarksItAndOneOntoLoadMintedIdsDoesNot() + public void AConditionalCommitOntoLoadMintedIdsDoesNotMarkTheFile() { - var held = PathOf("conditional-held.cc"); Type[] types = { typeof(IdDeck), typeof(PreCut2FixtureItem) }; - using (var cache = OpenWith(held, false, types)) - cache.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("d"), new CultRecordKey("d"))); - var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(held)); - snapshot.FormatVersion = CultPersistedStoreSnapshot.FormatV1; - File.WriteAllBytes(held, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); - using (var cache = OpenWith(held, false, types)) - { - Assert.That(cache.Commit(batch => - { - batch.Expect(new CultRecordKey("plain"), null); - batch.Upsert(typeof(PreCut2FixtureItem), new PreCut2FixtureItem { Name = "plain" }, new CultRecordKey("plain")); - }), Is.True); - } - - Assert.That(HeaderOf(held), Is.EqualTo("cultcache.store.v3"), "the deck on disk holds ids"); - var preId = PathOf("conditional-minted.cc"); WritePreIdStore(preId, null, ("a", "a", 1)); using (var cache = OpenWith(preId, false, types)) @@ -871,6 +854,87 @@ namespace GameCult.Caching.Tests Assert.That(HeaderOf(replaced), Is.EqualTo("cultcache.store.v1")); } + // Soul's probe: a C1 v2 store whose variant override holds no ids. Loading mints them in memory; the store keeps the delta + // it read, so the file holds none and no write of a plain record beside it may mark the file. + private static void WriteV2StoreWithAnIdLessVariant(string path) + { + using (var seed = OpenWith(path, false, typeof(IdDeck), typeof(PreCut2FixtureItem))) + { + seed.Commit(batch => + { + batch.Upsert(typeof(IdDeck), EmptyDeck("base"), new CultRecordKey("base")); + batch.UpsertVariant(new CultRecordKey("v"), new CultRecordKey("base"), + new[] { seed.Override(nameof(IdDeck.Name), "v"), seed.Override(nameof(IdDeck.Reels), new List { new() }) }); + }); + } + + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + var record = snapshot.Records.Single(entry => entry.Key == "v"); + record.Variant = new CultVariantDelta(record.Variant!.BaseKey, record.Variant.Overrides + .Select(entry => entry.Path[0].Slot == 1 ? CultVariantOverride.Set(1, OldShapedReels("old")) : entry).ToArray()); + snapshot.FormatVersion = CultPersistedStoreSnapshot.FormatV2; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + } + + [TestCase("unconditional")] + [TestCase("conditional")] + [TestCase("flush")] + public void AVariantWhoseIdsExistOnlyInMemoryDoesNotMarkTheFileAPlainWriteLandsIn(string write) + { + var path = PathOf("variant-in-memory-" + write + ".cc"); + WriteV2StoreWithAnIdLessVariant(path); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v2")); + + using (var cache = OpenWith(path, false, typeof(IdDeck), typeof(PreCut2FixtureItem))) + { + Assert.That(AllIds(cache.Get(new CultRecordKey("v"))!), Has.All.Not.Empty, "the variant holds minted ids in memory"); + var key = new CultRecordKey("p"); + switch (write) + { + case "unconditional": + UpsertPlain(cache, "p"); + break; + case "conditional": + Assert.That(cache.Commit(batch => + { + batch.Expect(key, null); + batch.Upsert(typeof(PreCut2FixtureItem), new PreCut2FixtureItem { Name = "p" }, key); + }), Is.True); + break; + default: + cache.UpsertAsync(typeof(PreCut2FixtureItem), new PreCut2FixtureItem { Name = "p" }, key).GetAwaiter().GetResult(); + cache.FlushAllBackingStores(); + break; + } + } + + Assert.That(DiskRecord(path, "p"), Is.Not.Null); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v2"), "the file holds no id, whatever this cache minted in memory"); + } + + // Soul's A/B probe: a conditional commit lands onto the file as it is, so a stale view of what the file once held does not + // decide the header. A holds a deck with ids; B removes it; A commits a plain record conditionally. + [Test] + public void AConditionalCommitIsNotMarkedByIdsItsStaleViewHoldsAndTheFileNoLongerDoes() + { + var path = PathOf("stale-view.cc"); + Type[] types = { typeof(IdDeck), typeof(PreCut2FixtureItem) }; + using var a = OpenWith(path, false, types); + a.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("d"), new CultRecordKey("d"))); + using (var b = OpenWith(path, false, types)) + b.Commit(batch => batch.Remove(new CultRecordKey("d"))); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1"), "B's whole-store commit left no id"); + Assert.That(a.Get(new CultRecordKey("d")), Is.Not.Null, "A has not pulled: its view is stale"); + + Assert.That(a.Commit(batch => + { + batch.Expect(new CultRecordKey("plain"), null); + batch.Upsert(typeof(PreCut2FixtureItem), new PreCut2FixtureItem { Name = "plain" }, new CultRecordKey("plain")); + }), Is.True); + + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1"), "the merged file holds no id"); + } + // Any override holding an id marks the variant, wherever it sits among the overrides. [Test] public void AVariantWhoseIdBearingOverrideIsNotTheLastIsStillMarked() diff --git a/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs b/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs index 48944b41..a59677cb 100644 --- a/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs +++ b/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs @@ -62,8 +62,9 @@ public sealed class CultMeshSingleFileElementIdTests .Should().Equal(publication.Representations.Select(entry => entry.Id)); } + // A typed write replaces the whole file with a document it can see, so the ids that document holds decide the header. [Test] - public void TypedWriteOfADocumentHoldingNoIdStaysV1AndAMarkedFileStaysMarked() + public void TypedWriteDecidesTheHeaderByTheDocumentItWritesEvenOverAMarkedFile() { var path = Path_("none.cc"); CultMesh.WriteSingleFileDocument(path, Key, Publication()); @@ -72,7 +73,24 @@ public sealed class CultMeshSingleFileElementIdTests CultMesh.WriteSingleFileDocument(path, Key, Publication(Descriptor())); HeaderOf(path).Should().Be("cultcache.store.v3"); CultMesh.WriteSingleFileDocument(path, Key, Publication()); - HeaderOf(path).Should().Be("cultcache.store.v3", "a file already marked on disk stays marked"); + HeaderOf(path).Should().Be("cultcache.store.v1", "the whole file is replaced by a document that holds no id"); + } + + // A raw payload is opaque: the writer cannot see what it holds, so a marked file stays marked. + [Test] + public void RawPayloadWriteOverAMarkedFileKeepsItMarkedAndOverAnUnmarkedOneStaysV1() + { + var path = Path_("raw.cc"); + var schema = new CultMeshSingleFileDocumentSchema("raw:schema", "RawSchema", "1"); + var payload = new byte[] { 0x90 }; + + CultMesh.WriteSingleFileDocumentPayload(path, Key, schema, null, payload); + HeaderOf(path).Should().Be("cultcache.store.v1"); + + CultMesh.WriteSingleFileDocument(path, Key, Publication(Descriptor())); + HeaderOf(path).Should().Be("cultcache.store.v3"); + CultMesh.WriteSingleFileDocumentPayload(path, Key, schema, null, payload); + HeaderOf(path).Should().Be("cultcache.store.v3", "the payload's ids are not the writer's to see"); } [Test]