diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index 20f00c2e..d07704fa 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -63,10 +63,12 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore var reports = new List(); var loaded = new Dictionary(StringComparer.Ordinal); + var foreign = new List(); for (var attempt = 1; ; attempt++) { loaded.Clear(); reports.Clear(); + foreign.Clear(); // Only pages named by the manifest read under the lease are loaded; orphaned pages are never loaded. With no // lock to lease (none created yet, or a store copied without it) a writer may commit mid-load, so an unleased // load stands only if its manifest is still current; if the manifest moved it reloads, under the lease the @@ -83,7 +85,8 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore manifest.Records.OrderBy(record => record.Key, StringComparer.Ordinal).ToArray(), manifest.SchemaCatalog, loaded, - reports); + reports, + foreign); if (lease != null || !ManifestMoved(manifestBytes)) { Trace($"indexed-pages loaded={loaded.Count}"); @@ -129,6 +132,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore _hydratedKeys.Add(key); SetLastSchemaMigrationReports(reports); + SetForeignRecords(foreign); IsDirty = !_dirtyKeys.IsEmpty || !_deletedKeys.IsEmpty; Trace("publish"); } @@ -147,6 +151,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore RefuseVariant(entry); Held(() => { + RefuseOverwrite(entry.Key.Value); Entries[entry.Key.Value] = entry; _dirtyKeys[entry.Key.Value] = true; _deletedKeys.TryRemove(entry.Key.Value, out _); @@ -239,6 +244,14 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore { Directory.CreateDirectory(_recordDirectory.FullName); var currentIndex = currentManifest.Records.ToDictionary(record => record.Key, record => record, StringComparer.Ordinal); + // The manifest's other records stay as they are, foreign ones included; a write that would replace or remove a foreign + // record is refused before any page is written. + foreach (var key in _deletedKeys.Keys.Concat(_dirtyKeys.Keys)) + { + if (currentIndex.TryGetValue(key, out var held) && Foreign(held, currentManifest.SchemaCatalog) is { } carried) + throw Overwrites(carried); + } + foreach (var key in _deletedKeys.Keys) currentIndex.Remove(key); foreach (var key in _dirtyKeys.Keys) @@ -310,8 +323,20 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore CultPersistedRecord[] records, IReadOnlyCollection catalogEntries, Dictionary loaded, - List reports) + List reports, + List foreign) { + // A record no registered type claims stays as the manifest names it: its page is never read, and a commit keeps it. + var claimed = new List(records.Length); + foreach (var metadata in records) + { + if (Foreign(metadata, catalogEntries) is { } carried) + foreign.Add(carried); + else + claimed.Add(metadata); + } + + records = claimed.ToArray(); var tracePages = string.Equals( Environment.GetEnvironmentVariable("CULTCACHE_TRACE_STARTUP_PHASES"), "1", diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index dc1de3a6..df71b604 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -142,6 +142,14 @@ names a local type only when no local type has the id, for a store written by a schema ids this one cannot know (a Rust type's schema id is its entry type). A schema renamed under a stable id therefore opens in every runtime that holds the id. +A record that resolves to no local type is a foreign record, for example one of a type this build does not +have, or one renamed without declaring its old id. A store never destroys or relabels it: the store carries it +byte for byte under its own id and its own catalog entry, lists it as foreign (`CacheBackingStore.ForeignRecords` +in C#), and the cache never holds it. Writes of other records proceed and lay it back as the file holds it, a +whole-view write included. A write that would replace or remove it is refused with the typed schema conflict +naming its key and id, and nothing is written. A marked header stays marked while a store carries one, because +the writer cannot see whether it holds element ids. Declaring its id on a type claims it at the next load. + A reader resolves a record's schema id to the entry that owns it, that entry's own `schemaId`, and only when no entry owns the id to an entry that lists it as a compatible id (the first such entry in catalog order). diff --git a/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md b/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md index 9ae6f553..e1586163 100644 --- a/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md +++ b/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md @@ -102,10 +102,15 @@ lie about the payload: - target schema change - name/index lookup semantics change - missing embedded catalog entry -- no compatible local schema candidate This is not negotiable. Better a loud refusal than quiet bit-rot. +## Foreign records + +A record with no local schema candidate is not rejected: the store carries it untouched as a +foreign record and refuses any write that would replace or remove it. See "foreign record" in +`cultcache-persistence-format.md`. + ## Variant overrides A variant's override follows the same outcome as the same member on a plain record; there diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 9c422424..c53a7c7b 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -136,6 +136,32 @@ namespace GameCult.Caching public CultVariantDelta? Variant { get; set; } } + /// + /// A record a store carries untouched because no registered type owns its schema id or lists it as compatible, for example a + /// record of a type this build does not have, or one renamed without declaring its old id. The store keeps its bytes as the + /// file holds them, a write of another record keeps it, and a write that would replace or remove it is refused with a + /// . The cache never holds it. Declaring its schema id on a type claims it at the next load. + /// + public sealed class CultForeignRecord + { + internal CultForeignRecord(string key, string schemaId, string schemaName, string storedAt) + { + Key = key; + SchemaId = schemaId; + SchemaName = schemaName; + StoredAt = storedAt; + } + + public string Key { get; } + + public string SchemaId { get; } + + // The name the store's catalog gives the schema. + public string SchemaName { get; } + + public string StoredAt { get; } + } + public sealed class CultPersistedStoreSnapshot { public const string FormatV1 = "cultcache.store.v1"; @@ -605,24 +631,44 @@ namespace GameCult.Caching internal CultSchemaResolutionResult ResolvePersistedSchemaDetailed(string schemaId, IReadOnlyCollection catalog) { - var indexes = _indexes; - if (indexes.BySchemaId.TryGetValue(schemaId, out var exact)) + var local = Claimant(schemaId, catalog, out var persisted); + if (local == null) + throw new InvalidOperationException( + $"No local CultCache schema matches persisted schema '{persisted!.SchemaName}' ({schemaId})."); + if (persisted == null) { return new CultSchemaResolutionResult( - exact, + local, new CultSchemaMigrationReport { PersistedSchemaId = schemaId, - LocalSchemaId = exact.SchemaId, - PersistedSchemaName = exact.SchemaName, - LocalSchemaName = exact.SchemaName, + LocalSchemaId = local.SchemaId, + PersistedSchemaName = local.SchemaName, + LocalSchemaName = local.SchemaName, Kind = CultSchemaMigrationKind.Exact }); } + return BuildCompatibleResolutionResult(persisted, local); + } + + // The registered type a record under this schema id belongs to, or null when no registered type claims it: a foreign record, + // which a store carries untouched. Deciding this never decodes the record or compares shapes, so a store can classify what a + // file holds without reading a payload. persisted is the catalog entry the record resolves through, and null when a + // registered type owns the id. Claimants that cannot be told apart refuse. + internal CultDocumentDescriptor? Claimant( + string schemaId, + IReadOnlyCollection catalog, + out CultSchemaCatalogEntry? persisted) + { + var indexes = _indexes; + persisted = null; + if (indexes.BySchemaId.TryGetValue(schemaId, out var exact)) + return exact; + // A schema is published by its id or as a compatible id. - var persisted = catalog.FirstOrDefault(entry => string.Equals(entry.SchemaId, schemaId, StringComparison.Ordinal)) - ?? catalog.FirstOrDefault(entry => entry.CompatibleSchemaIds.Contains(schemaId, StringComparer.Ordinal)); + persisted = catalog.FirstOrDefault(entry => string.Equals(entry.SchemaId, schemaId, StringComparison.Ordinal)) + ?? catalog.FirstOrDefault(entry => entry.CompatibleSchemaIds.Contains(schemaId, StringComparer.Ordinal)); if (persisted == null) { throw new InvalidOperationException($"Persisted schema '{schemaId}' is not present in the embedded catalog."); @@ -639,23 +685,19 @@ namespace GameCult.Caching schemaId, listers.Select(lister => lister.SchemaName).ToArray(), string.Empty); - return BuildCompatibleResolutionResult(persisted, listers[0]); + return listers[0]; } foreach (var compatibleSchemaId in persisted.CompatibleSchemaIds.Where(candidate => !string.IsNullOrWhiteSpace(candidate))) { if (indexes.BySchemaId.TryGetValue(compatibleSchemaId, out var compatibleLocal)) - { - return BuildCompatibleResolutionResult(persisted, compatibleLocal); - } + return compatibleLocal; } if (indexes.BySchemaName.TryGetValue(persisted.SchemaName, out var localVersions)) { if (localVersions.Length == 1) - { - return BuildCompatibleResolutionResult(persisted, localVersions[0]); - } + return localVersions[0]; throw new InvalidOperationException( $"CultCache schema name '{persisted.SchemaName}' is ambiguous across local versions " + @@ -663,8 +705,7 @@ namespace GameCult.Caching "Persisted schema compatibility must identify an explicit schema id."); } - throw new InvalidOperationException( - $"No local CultCache schema matches persisted schema '{persisted.SchemaName}' ({schemaId})."); + return null; } private CultDocumentDescriptor RegisterDescriptor(CultDocumentDescriptor descriptor) @@ -3349,6 +3390,14 @@ namespace GameCult.Caching public IReadOnlyList LastSchemaMigrationReports => _lastSchemaMigrationReports; + /// + /// The records this store's last load found and carries untouched, ordered by key: no registered type owns their schema id or + /// lists it as compatible. See . + /// + public IReadOnlyList ForeignRecords => _foreignRecords; + + private CultForeignRecord[] _foreignRecords = Array.Empty(); + // Set by the cache at attach. A pull hands over everything it loaded and dropped in one call before it adopts // any of it; if the cache refuses a record the call throws and the store keeps its previous view. protected internal Action, IReadOnlyList>? Loaded; @@ -3505,6 +3554,34 @@ namespace GameCult.Caching _lastSchemaMigrationReports = reports?.ToArray() ?? Array.Empty(); } + // The one test of whether a stored record is foreign: no registered type claims its schema id through the catalog it was + // stored with. It reads the record's header only, never its payload. Null for a record the registry resolves. + protected CultForeignRecord? Foreign(CultPersistedRecord record, IReadOnlyCollection catalog) => + Registry.Claimant(record.SchemaId, catalog, out var persisted) == null + ? new CultForeignRecord(record.Key, record.SchemaId, persisted!.SchemaName, record.StoredAt) + : null; + + protected void SetForeignRecords(IEnumerable foreign) + { + _foreignRecords = foreign.OrderBy(record => record.Key, StringComparer.Ordinal).ToArray(); + } + + // A write never replaces or removes a record this store carries for a schema it does not know. + protected void RefuseOverwrite(string key) + { + var foreign = _foreignRecords.FirstOrDefault(record => string.Equals(record.Key, key, StringComparison.Ordinal)); + if (foreign != null) + throw Overwrites(foreign); + } + + protected CultSchemaConflictException Overwrites(CultForeignRecord foreign) => new( + $"Record '{foreign.Key}' in {this} is stored under schema id '{foreign.SchemaId}' ('{foreign.SchemaName}'), which " + + "no registered type owns or lists as compatible. The store carries it untouched and refuses a write that would replace or remove it; " + + "declare the id on a type to claim it.", + foreign.SchemaId, + new[] { foreign.SchemaName }, + foreign.Key); + protected void MarkFlushSucceeded() { IsDirty = false; @@ -3548,13 +3625,21 @@ namespace GameCult.Caching if (snapshot == null) { SetLastSchemaMigrationReports(Array.Empty()); + SetForeignRecords(Array.Empty()); return; } var reports = new List(snapshot.Records.Length); var persisted = new Dictionary(StringComparer.Ordinal); + var foreign = new List(); foreach (var record in snapshot.Records) { + if (Foreign(record, snapshot.SchemaCatalog) is { } carried) + { + foreign.Add(carried); + continue; + } + var stored = ToStoredDocument(record, snapshot.SchemaCatalog, DeserializePayload, out var report); reports.Add(report); persisted[stored.Key.Value] = stored; @@ -3574,6 +3659,7 @@ namespace GameCult.Caching foreach (var stored in loaded) Entries[stored.Key.Value] = stored; SetLastSchemaMigrationReports(reports); + SetForeignRecords(foreign); } public override void Push(CultStoredDocument entry) @@ -3581,6 +3667,7 @@ namespace GameCult.Caching ThrowIfReadOnly(); Held(() => { + RefuseOverwrite(entry.Key.Value); Entries[entry.Key.Value] = entry; IsDirty = true; }); @@ -3606,14 +3693,16 @@ namespace GameCult.Caching { using (AcquireLock(wait: true)) { - var records = WholeView(Entries.Values, ReadSnapshot()); + var disk = ReadSnapshot(); + var foreign = ForeignOnDisk(disk); + var records = WholeView(Entries.Values, disk, foreign); WriteSnapshot( records, Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()).ToArray(), - Array.Empty(), + disk?.SchemaCatalog ?? Array.Empty(), Entries.Values.Any(entry => entry.HoldsIds), - existingHeader: null, - wholeStore: true); + disk?.FormatVersion, + wholeStore: foreign.Count == 0); WroteWholeView(records); } @@ -3640,7 +3729,14 @@ namespace GameCult.Caching // An unconditional commit writes what a flush would, this store's whole view plus the batch: last-writer-wins, // and staged single writes land with it. A conditional commit (store clean) lands the batch onto the file as it is. var ontoDisk = request.HasConditions && !IsDirty; - var records = (ontoDisk ? disk.Records : WholeView(Entries.Values, disk)).ToDictionary(record => record.Key, StringComparer.Ordinal); + var foreign = ForeignOnDisk(disk); + foreach (var entry in request.Upserts.Concat(request.Deletes)) + { + if (foreign.TryGetValue(entry.Key.Value, out var carried)) + throw Overwrites(carried.Foreign); + } + + var records = (ontoDisk ? disk.Records : WholeView(Entries.Values, disk, foreign)).ToDictionary(record => record.Key, StringComparer.Ordinal); // The committer judged the batch against its own view; the file may have moved since. A merge is judged on the // set the file will hold, exactly as that set would be judged loading: what other processes wrote (arriving), // what they removed (departing), and this batch. @@ -3660,7 +3756,7 @@ namespace GameCult.Caching var registered = Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()) .Concat(request.Upserts.Select(entry => entry.Descriptor.ToCatalogEntry())) .ToArray(); - WriteSnapshot(records.Values, registered, ontoDisk ? disk.SchemaCatalog : Array.Empty(), holdsIds, disk.FormatVersion, wholeStore: !ontoDisk); + WriteSnapshot(records.Values, registered, disk.SchemaCatalog, holdsIds, disk.FormatVersion, wholeStore: !ontoDisk && foreign.Count == 0); foreach (var entry in request.Deletes) Entries.TryRemove(entry.Key.Value, out _); foreach (var entry in request.Upserts) @@ -3675,8 +3771,20 @@ namespace GameCult.Caching // that stores a record differently is a new store of it and mints it a later storedAt: under another id than it was loaded // under (a type re-encoding a record it read through an older or foreign schema, shedding what it lacks), or with other // bytes than the file holds at that storedAt. A record written exactly as it is stored keeps its storedAt. - private CultPersistedRecord[] WholeView(IEnumerable entries, CultPersistedStoreSnapshot? disk) + // The whole view also lays back every foreign record the file holds now, as the file holds it, and refuses an entry whose key + // the file holds as a foreign record. + private CultPersistedRecord[] WholeView( + IEnumerable view, + CultPersistedStoreSnapshot? disk, + IReadOnlyDictionary foreign) { + var entries = view.ToArray(); + foreach (var entry in entries) + { + if (foreign.TryGetValue(entry.Key.Value, out var carried)) + throw Overwrites(carried.Foreign); + } + var onDisk = (disk?.Records ?? Array.Empty()).ToDictionary(record => record.Key, StringComparer.Ordinal); return entries.Select(entry => { @@ -3687,7 +3795,20 @@ namespace GameCult.Caching if (!unchanged) record.StoredAt = CultCache.MintStoredAt(record.StoredAt); return record; - }).ToArray(); + }).Concat(foreign.Values.Select(carried => carried.Record)).ToArray(); + } + + // The records the file holds that no registered type claims, by key. + private Dictionary ForeignOnDisk(CultPersistedStoreSnapshot? disk) + { + var foreign = new Dictionary(StringComparer.Ordinal); + foreach (var record in disk?.Records ?? Array.Empty()) + { + if (Foreign(record, disk!.SchemaCatalog) is { } carried) + foreign[record.Key] = (carried, record); + } + + return foreign; } private static bool SameStoredBytes(CultPersistedRecord stored, CultPersistedRecord written) => @@ -3732,16 +3853,24 @@ namespace GameCult.Caching var removing = request.Deletes.Select(entry => entry.Key.Value).ToHashSet(StringComparer.Ordinal); var onDisk = disk.Records.Select(record => record.Key).ToHashSet(StringComparer.Ordinal); var arriving = new List(request.Upserts); + var departing = new List(request.Deletes); foreach (var record in disk.Records) { if (landing.Contains(record.Key) || removing.Contains(record.Key)) continue; + // A foreign record is not part of what the cache judges; one that replaced a record this store holds takes it away. + if (Foreign(record, disk.SchemaCatalog) != null) + { + if (Entries.TryGetValue(record.Key, out var replaced)) + departing.Add(replaced); + continue; + } + if (Entries.TryGetValue(record.Key, out var known) && known.StoredAt == record.StoredAt && known.StoredSchemaId == record.SchemaId) continue; arriving.Add(ToStoredDocument(record, disk.SchemaCatalog, DeserializePayload, out _)); } - var departing = new List(request.Deletes); departing.AddRange(Entries.Values.Where(known => !onDisk.Contains(known.Key.Value) && !landing.Contains(known.Key.Value) && !removing.Contains(known.Key.Value))); Judging!(arriving, departing); diff --git a/src/GameCult.Caching/CultSchemaConflictException.cs b/src/GameCult.Caching/CultSchemaConflictException.cs index 9419e2c5..30e0abb9 100644 --- a/src/GameCult.Caching/CultSchemaConflictException.cs +++ b/src/GameCult.Caching/CultSchemaConflictException.cs @@ -4,8 +4,9 @@ namespace GameCult.Caching { /// /// A write the catalog cannot describe: two schemas that share a schema id and disagree on the schema name (both arrived with the - /// records, neither registered), or a record whose schema id no chosen entry publishes. Nothing is written, so the store stays as - /// it was. It names the id, the schema names involved and a record key. + /// records, neither registered), a record whose schema id no chosen entry publishes, or a write that would replace or remove a + /// . Nothing is written, so the store stays as it was. It names the id, the schema names involved + /// and a record key. /// public sealed class CultSchemaConflictException : InvalidOperationException { diff --git a/tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs b/tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs index 8ac1b87c..39d6d767 100644 --- a/tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs +++ b/tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs @@ -171,11 +171,14 @@ namespace GameCult.Caching.Tests } [Test] - public void ARecordUnderAnOldIdNoTypeDeclaresIsNotOpenedByThatSchemaName() + public void ARecordUnderAnOldIdNoTypeDeclaresIsCarriedAsForeignNotOpenedByThatSchemaName() { var path = OldStore("undeclared.cc"); - Assert.That(() => Open(path, Bare), Throws.InstanceOf(), "the entry's name is metadata; only a declared id resolves it"); + using var cache = Open(path, Bare); + Assert.That(cache.Get(D), Is.Null, "the entry's name is metadata; only a declared id resolves it"); + Assert.That(cache.BackingStores.Single().ForeignRecords.Select(record => (record.Key, record.SchemaId, record.SchemaName)), + Is.EqualTo(new[] { (D.Value, OldId, "tests.legacy_deck") })); } // An unconditional commit writes the cache's whole view: the record the batch does not name is stamped with the registered diff --git a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs new file mode 100644 index 00000000..68493547 --- /dev/null +++ b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs @@ -0,0 +1,331 @@ +#nullable enable +using System; +using System.IO; +using System.Linq; +using GameCult.Caching.MessagePack; +using NUnit.Framework; +using static GameCult.Caching.Tests.EmittedDocumentTypes; + +namespace GameCult.Caching.Tests +{ + // A record under a schema id no registered type owns or lists as compatible is foreign. CultCache never destroys or relabels + // it: both store kinds carry it byte for byte, list it as foreign, and refuse any write that would replace or remove it. + public class ForeignRecordTests + { + private static readonly Type Deck = Emit("ForeignSuiteDeck", "tests.foreign_deck", "tests.foreign_deck.v1", new[] { new Field("Name", typeof(string), 0, IsName: true) }); + private static readonly Type Widget = Emit("ForeignSuiteWidget", "tests.foreign_widget", "tests.foreign_widget.v1", new[] { new Field("Label", typeof(string), 0, IsName: true) }); + + // A build that has both types, and one that has only the deck: to it the widget is foreign. + private static readonly CultDocumentRegistry Full = CultDocumentRegistry.ForTypes(new[] { Deck, Widget }); + private static readonly CultDocumentRegistry DeckOnly = CultDocumentRegistry.ForTypes(new[] { Deck }); + private static readonly string WidgetId = Full.GetRequired(Widget).SchemaId; + + private static readonly CultRecordKey D = new("d"); + private static readonly CultRecordKey E = new("e"); + private static readonly CultRecordKey W = new("w"); + private const string Canary = "CANARY-7f3c-payload-value"; + + public enum Write + { + Flush, + Commit, + ConditionalCommit + } + + private string _directory = ""; + + [SetUp] + public void SetUp() + { + _directory = Path.Combine(Path.GetTempPath(), "cultlib-foreign-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_directory); + } + + [TearDown] + public void TearDown() + { + if (Directory.Exists(_directory)) + Directory.Delete(_directory, true); + } + + private string PathOf(string name) => Path.Combine(_directory, name); + + private static CultCache Open(string path, CultDocumentRegistry registry, bool directory) => + CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry, UseDirectoryStore = directory }); + + private static object DeckOf(string name) => New(Deck, ("Name", name)); + + private static object WidgetOf(string label) => New(Widget, ("Label", label)); + + private static CultPersistedStoreSnapshot Read(string path) => + CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + + // Every file of the store, by name, with its bytes: a refused write leaves all of them as they were. + private static string[] Fingerprint(string path) + { + var files = new[] { path }.Concat(Directory.Exists(DirectoryMessagePackBackingStore.DefaultRecordDirectoryPath(path)) + ? Directory.GetFiles(DirectoryMessagePackBackingStore.DefaultRecordDirectoryPath(path), "*.msgpack") + : Array.Empty()) + .OrderBy(file => file, StringComparer.Ordinal); + return files.Select(file => Path.GetFileName(file) + ":" + Convert.ToBase64String(File.ReadAllBytes(file))).ToArray(); + } + + // What the store holds for one record as the file (single-file) or manifest (directory: the page hash) names it. + private static string Stored(string path, CultRecordKey key) + { + var record = Read(path).Records.Single(candidate => candidate.Key == key.Value); + return $"{record.SchemaId}|{record.StoredAt}|{Convert.ToBase64String(record.Payload)}|{record.Variant == null}"; + } + + // A store holding a deck d and a widget w, written by the build that has both types. + private string Seeded(string name, bool directory) + { + var path = PathOf(name); + using var cache = Open(path, Full, directory); + cache.Commit(batch => + { + batch.Upsert(Deck, DeckOf("d"), D); + batch.Upsert(Widget, WidgetOf("w"), W); + }); + return path; + } + + private static CacheBackingStore StoreOf(CultCache cache) => cache.BackingStores.Single(); + + private static void Land(CultCache cache, Write write, CultRecordKey key, object document) + { + switch (write) + { + case Write.Flush: + cache.UpsertAsync(document.GetType(), document, key).GetAwaiter().GetResult(); + cache.FlushAsync().GetAwaiter().GetResult(); + break; + case Write.Commit: + Assert.That(cache.Commit(batch => batch.Upsert(document.GetType(), document, key)), Is.True); + break; + case Write.ConditionalCommit: + Assert.That(cache.Commit(batch => + { + batch.Expect(D, cache.Get(D)); + batch.Upsert(document.GetType(), document, key); + }), Is.True); + break; + } + } + + private static void Refused(CultCache cache, Write write, CultRecordKey key, object document, CultRecordKey foreignKey) + { + var refusal = Assert.Throws(() => Land(cache, write, key, document))!; + Assert.Multiple(() => + { + Assert.That(refusal.RecordKey, Is.EqualTo(foreignKey.Value)); + Assert.That(refusal.SchemaId, Is.EqualTo(WidgetId)); + Assert.That(refusal.SchemaNames, Is.EqualTo(new[] { "tests.foreign_widget" })); + Assert.That(refusal.Message, Does.Contain(foreignKey.Value).And.Contain(WidgetId)); + Assert.That(refusal.Message, Does.Not.Contain(Canary), "a refusal never echoes the document it refused"); + }); + } + + [TestCase(false)] + [TestCase(true)] + public void AForeignRecordLoadsAsForeignAndTheCacheDoesNotHoldIt(bool directory) + { + var path = Seeded("seeded.cc", directory); + var storedAt = Read(path).Records.Single(record => record.Key == W.Value).StoredAt; + + using var cache = Open(path, DeckOnly, directory); + + var foreign = StoreOf(cache).ForeignRecords.Single(); + Assert.Multiple(() => + { + Assert.That(foreign.Key, Is.EqualTo(W.Value)); + Assert.That(foreign.SchemaId, Is.EqualTo(WidgetId)); + Assert.That(foreign.SchemaName, Is.EqualTo("tests.foreign_widget")); + Assert.That(foreign.StoredAt, Is.EqualTo(storedAt)); + Assert.That(cache.Get(W), Is.Null, "the cache never holds a foreign record"); + Assert.That(EmittedDocumentTypes.Read(cache.Get(D)!, "Name"), Is.EqualTo("d"), "the records it claims load as usual"); + }); + } + + // Every write path lays the foreign record back as the file holds it: the single-file store's whole view (a flush and an + // unconditional commit), its commit onto the file, and the directory store's generation. + [Test] + public void AWriteOfAnotherRecordKeepsTheForeignRecordByteForByte( + [Values(false, true)] bool directory, + [Values] Write write) + { + var path = Seeded("kept.cc", directory); + var before = Stored(path, W); + + using (var cache = Open(path, DeckOnly, directory)) + Land(cache, write, E, DeckOf("e")); + + Assert.That(Stored(path, W), Is.EqualTo(before)); + Assert.That(Read(path).Records.Select(record => record.Key), Is.EquivalentTo(new[] { "d", "e", "w" })); + using var full = Open(path, Full, directory); + Assert.That(EmittedDocumentTypes.Read(full.Get(W)!, "Label"), Is.EqualTo("w"), "the build that has the type still reads it"); + Assert.That(StoreOf(full).ForeignRecords, Is.Empty); + } + + [Test] + public void AWriteThatWouldReplaceAForeignRecordIsRefusedAndNothingIsWritten( + [Values(false, true)] bool directory, + [Values] Write write) + { + var path = Seeded("replace.cc", directory); + var before = Fingerprint(path); + + using var cache = Open(path, DeckOnly, directory); + Refused(cache, write, W, DeckOf(Canary), W); + + Assert.That(Fingerprint(path), Is.EqualTo(before)); + Assert.That(cache.Get(W), Is.Null); + } + + // Another writer replaced a record this cache holds with a record of a type it does not have. A write that would lay this + // cache's record over it is refused; the file keeps the other writer's record. + [TestCase(false)] + [TestCase(true)] + public void AWriteOverARecordAnotherWriterMadeForeignIsRefused(bool directory) + { + var path = Seeded("replaced.cc", directory); + using var cache = Open(path, DeckOnly, directory); + using (var other = Open(path, Full, directory)) + other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + var before = Fingerprint(path); + + // The single-file store writes its whole view, so any write reaches d; the directory store writes only d itself. + var refusal = Assert.Throws(() => Land(cache, Write.Flush, directory ? D : E, DeckOf(Canary)))!; + + Assert.That(refusal.RecordKey, Is.EqualTo(D.Value)); + Assert.That(refusal.Message, Does.Not.Contain(Canary)); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + } + + [TestCase(false)] + [TestCase(true)] + public void RemovingARecordAnotherWriterMadeForeignIsRefused(bool directory) + { + var path = Seeded("removed.cc", directory); + using var cache = Open(path, DeckOnly, directory); + cache.Commit(batch => batch.Upsert(Deck, DeckOf("e"), E)); + using (var other = Open(path, Full, directory)) + other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + var before = Fingerprint(path); + + // A condition keeps the single-file commit on the file as it is, so only the removal reaches d. + var refusal = Assert.Throws(() => cache.Commit(batch => + { + batch.Expect(E, cache.Get(E)); + batch.Remove(D); + }))!; + + Assert.That(refusal.RecordKey, Is.EqualTo(D.Value)); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + } + + [TestCase(false)] + [TestCase(true)] + public void ARecordAnotherWriterMadeForeignLeavesTheCacheAtThePull(bool directory) + { + var path = Seeded("pulled.cc", directory); + using var cache = Open(path, DeckOnly, directory); + using (var other = Open(path, Full, directory)) + other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + + cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + + Assert.That(cache.Get(D), Is.Null); + Assert.That(StoreOf(cache).ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "d", "w" })); + } + + // The file decides what is carried, not what this cache loaded: a foreign record another writer added since is kept too. + [TestCase(false)] + [TestCase(true)] + public void AForeignRecordAddedAfterTheLoadIsKept(bool directory) + { + var path = Seeded("added.cc", directory); + using var cache = Open(path, DeckOnly, directory); + var z = new CultRecordKey("z"); + using (var other = Open(path, Full, directory)) + other.Commit(batch => batch.Upsert(Widget, WidgetOf("z"), z)); + var before = Stored(path, z); + + Land(cache, Write.Flush, E, DeckOf("e")); + + Assert.That(Stored(path, z), Is.EqualTo(before)); + } + + // A writer cannot see whether a foreign record holds element ids, so a store marked as holding them stays marked while it + // carries one. Without one, a whole-view write decides by what it writes, and nothing here holds an id. + [Test] + public void AMarkedHeaderStaysMarkedWhileTheStoreCarriesAForeignRecord( + [Values(Write.Flush, Write.Commit)] Write write, + [Values] bool carriesForeign) + { + var path = Seeded("marked.cc", directory: false); + var snapshot = Read(path); + snapshot.FormatVersion = CultPersistedStoreSnapshot.FormatV3; + if (!carriesForeign) + { + snapshot.Records = snapshot.Records.Where(record => record.Key != W.Value).ToArray(); + snapshot.SchemaCatalog = snapshot.SchemaCatalog.Where(entry => entry.SchemaId != WidgetId).ToArray(); + } + + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + + using (var cache = Open(path, DeckOnly, directory: false)) + Land(cache, write, E, DeckOf("e")); + + Assert.That(Read(path).FormatVersion, Is.EqualTo(carriesForeign ? CultPersistedStoreSnapshot.FormatV3 : CultPersistedStoreSnapshot.FormatV1)); + } + + // A commit onto the file that involves a variant is judged on the set the file will hold. A foreign record is not decoded + // for that judgement, so it neither fails the commit nor joins the set. + [Test] + public void ACommitInvolvingAVariantIsNotRefusedByAForeignRecord() + { + var path = Seeded("variant.cc", directory: false); + var before = Stored(path, W); + using var cache = Open(path, DeckOnly, directory: false); + var v = new CultRecordKey("v"); + cache.UpsertVariantAsync(v, D, new[] { cache.Override(Deck, "Name", "v") }).GetAwaiter().GetResult(); + cache.FlushAsync().GetAwaiter().GetResult(); + + Land(cache, Write.ConditionalCommit, E, DeckOf("e")); + + Assert.That(Stored(path, W), Is.EqualTo(before)); + Assert.That(Read(path).Records.Select(record => record.Key), Is.EquivalentTo(new[] { "d", "e", "v", "w" })); + } + + // A variant's base that another writer replaced with a foreign record is gone from the set the file will hold, so a commit + // onto that file is refused as it would be if the base had been removed. + [Test] + public void ACommitOntoAFileWhereAForeignRecordReplacedAVariantsBaseIsRefused() + { + var path = Seeded("base.cc", directory: false); + using var cache = Open(path, DeckOnly, directory: false); + var v = new CultRecordKey("v"); + cache.UpsertVariantAsync(v, D, new[] { cache.Override(Deck, "Name", "v") }).GetAwaiter().GetResult(); + cache.FlushAsync().GetAwaiter().GetResult(); + var snapshot = Read(path); + var widget = snapshot.Records.Single(record => record.Key == W.Value); + snapshot.Records = snapshot.Records + .Select(record => record.Key == D.Value + ? new CultPersistedRecord { Key = D.Value, SchemaId = widget.SchemaId, StoredAt = widget.StoredAt, Payload = widget.Payload } + : record) + .ToArray(); + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + var before = Fingerprint(path); + + var refusal = Assert.Throws(Is.InstanceOf(), () => cache.Commit(batch => + { + batch.Expect(E, null); + batch.Upsert(Deck, DeckOf("e"), E); + }))!; + + Assert.That(refusal.Message, Does.Contain(v.Value).And.Contain(D.Value)); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + } + } +}