diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index 71140019..eeec3349 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -182,10 +182,17 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore return CultCommitOutcome.Contended; var (manifest, catalogBytes) = ReadManifest(); if (request.HasConditions && !request.ConditionsHold( - manifest.Records.ToDictionary(record => record.Key, record => VersionOf(record.Payload), StringComparer.Ordinal), Entries.Values)) + manifest.Records.ToDictionary(record => record.Key, record => VersionOf(record.Payload), StringComparer.Ordinal), Entries.Values, ForeignRecords)) return CultCommitOutcome.Mismatch; - RefuseUndeclaredOnDisk(manifest, request.Upserts.Concat(request.Deletes).Select(entry => entry.Key.Value).Concat(StagedKeys)); + RefuseUndeclaredOnDisk(manifest, request.Upserts.Concat(request.Deletes).Select(entry => entry.Key.Value).Concat(request.Checks).Concat(StagedKeys)); + if (request.Upserts.Count == 0 && request.Deletes.Count == 0 && !IsDirty) + { + // Nothing to write: the batch only judged keys it does not hold. + MarkFlushSucceeded(); + return CultCommitOutcome.Committed; + } + var previousEntries = Entries.ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal); var previousDirtyKeys = _dirtyKeys.Keys.ToArray(); var previousDeletedKeys = _deletedKeys.Keys.ToArray(); @@ -230,23 +237,52 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore ThrowIfReadOnly(); Held(() => { - if (!IsDirty) + if (!IsDirty && StagedChecks.Count == 0) return; Directory.CreateDirectory(_manifestFile.DirectoryName!); using var commitLease = AcquireCommitLease(wait: true, create: true); var (manifest, catalogBytes) = ReadManifest(); RefuseUndeclaredOnDisk(manifest, StagedKeys); - WriteGeneration(manifest, catalogBytes); + if (IsDirty) + WriteGeneration(manifest, catalogBytes); + else + MarkFlushSucceeded(); }); } - private IEnumerable StagedKeys => _deletedKeys.Keys.Concat(_dirtyKeys.Keys); + // Every key the next flush or commit judges: the staged writes and removals, and the staged checks. + private IEnumerable StagedKeys => _deletedKeys.Keys.Concat(_dirtyKeys.Keys).Concat(StagedChecks); - // The manifest as it is now, read without the commit lease: nothing is written after the read, the manifest is only ever - // replaced whole, and a read-only store must not create the lease. - protected internal override void RefuseRemovalOfUndeclared(IReadOnlyCollection keys) => - Held(() => RefuseUndeclaredOnDisk(ReadManifest().Snapshot, keys)); + // A writable store judges a removal of a key its cache does not hold at its next flush or commit, under the commit lease + // (StageChecks); a read-only store has neither, so it answers from its view and never creates the lease. + protected internal override void RefuseRemovalOfUndeclared(IReadOnlyCollection keys) + { + if (IsReadOnly) + base.RefuseRemovalOfUndeclared(keys); + else + StageChecks(keys); + } + + // A read-only store answers a batch's checks from its view and never creates the lease; a writable one reads the manifest + // under the commit lease. + protected internal override CultCommitOutcome JudgeChecks(IReadOnlyCollection keys, bool wait) + { + if (IsReadOnly) + return base.JudgeChecks(keys, wait); + return Held(() => + { + using var commitLease = AcquireCommitLease(wait, create: true); + if (commitLease == null) + return CultCommitOutcome.Contended; + RefuseUndeclaredOnDisk(ReadManifest().Snapshot, keys); + return CultCommitOutcome.Committed; + }); + } + + // A manifest record no registered type claims, listed at the version its index record names. + private CultForeignRecord? ForeignAt(CultPersistedRecord record, IReadOnlyCollection catalog) => + Foreign(record, catalog, VersionOf(record.Payload)); // The manifest's other records stay as they are, foreign and read-only ones included; a write that would replace or remove a // record the manifest holds under an id no registered type declares is refused before any page is written, by the one rule every @@ -258,10 +294,10 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore RefuseUndeclaredOnDisk( written, key => HeldAt(key) is { } held ? Undeclared(held, currentManifest.SchemaCatalog, Registry.Declares) : null, - key => HeldAt(key) is { } held && Foreign(held, currentManifest.SchemaCatalog) == null + key => HeldAt(key) is { } held && Foreign(held, currentManifest.SchemaCatalog, VersionOf(held.Payload)) == null ? LoadPage(held, currentManifest.SchemaCatalog, out _, out _) : null, - key => HeldAt(key) is { } held ? Foreign(held, currentManifest.SchemaCatalog) : null, + key => HeldAt(key) is { } held ? ForeignAt(held, currentManifest.SchemaCatalog) : null, key => { _dirtyKeys.TryRemove(key, out _); @@ -365,7 +401,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore var claimed = new List(records.Length); foreach (var metadata in records) { - if (Foreign(metadata, catalogEntries) is { } carried) + if (ForeignAt(metadata, catalogEntries) is { } carried) foreign.Add(carried); else claimed.Add(metadata); diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index b4509832..e1d5d0c0 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -206,7 +206,7 @@ A record that resolves to no local type is a foreign record, for example one of have, or one renamed under a new schema name without declaring its old id. A store never destroys or relabels it: the store carries it byte for byte under its own id and its own catalog entry, lists it as foreign (`CacheBackingStore.ForeignRecords` in C#), and the cache never holds it. Writes of other records proceed and copy it forward as the file holds it. A write that would replace or remove it is refused with the typed schema conflict -naming its key and id, and nothing is written. A removal of a key the cache does not hold is judged against the store as it is at the removal, so it is refused also when the foreign record arrived after the load. The refused keys are reloaded as the store holds them (a foreign record is not held, and is listed as foreign from then on, so its next write is refused before staging), and their staged changes are forgotten. A write that carries a record forward keeps a marked header marked. Declaring its id on a type claims it at the next load. +naming its key and id, and nothing is written. A removal of a key the cache does not hold removes nothing and is free; the store judges the key at its next flush or commit, under its lock, with the keys that write writes, and refuses the write if it holds a record there under an undeclared id, also when that record arrived after the load. A commit judges such a key in every writable store its cache attaches, each under its own lock: the stores it does not write first, then the store it writes, and a refusal in any of them writes nothing. A judged key is never removed. A read-only store answers at the removal, from its last load. The refused keys are reloaded as the store holds them (a foreign record is not held, and is listed as foreign from then on, so its next write is refused before staging), and their staged changes are forgotten. A write that carries a record forward keeps a marked header marked. Declaring its id on a type claims it at the next load. A reader resolves a record's schema id to the entry that owns it, that entry's own `schemaId`, and only when no entry owns the id to an entry that lists it as a compatible id (the first such entry diff --git a/src/GameCult.Caching/Contracts/cultcache-store-composition.md b/src/GameCult.Caching/Contracts/cultcache-store-composition.md index 1d9d709e..b03627ca 100644 --- a/src/GameCult.Caching/Contracts/cultcache-store-composition.md +++ b/src/GameCult.Caching/Contracts/cultcache-store-composition.md @@ -263,6 +263,14 @@ refuses to replace a file whose header it cannot read, and leaves it as it was. before anything is written, and that record is reloaded as the store holds it: dropped when foreign, held read-only when a type reads it (cultcache-persistence-format.md, ownership by declaration). +- A removal of a key the cache does not hold travels as a check-only key: a + commit carries it in the request, a flush stages it, and either store kind + judges it under its lock with the keys it writes, never removing it. Every + writable store judges a batch's checks under its own lock: the stores the + batch does not commit through first (`JudgeChecks`), the home last. A batch + that only checks keys writes no file. `ExpectUnchanged` compares the foreign + records a store carries at the versions it read them; a store kind lists a + foreign record through `Foreign` with the version it read. - A single-file store holding a variant (the directory store refuses variants) is written only from what its writer last read. A record that moved refuses the whole write with the typed write conflict, and the refused and moved keys diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 2af2c626..7f164e4c 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -163,12 +163,13 @@ namespace GameCult.Caching /// public sealed class CultForeignRecord { - internal CultForeignRecord(string key, string schemaId, string schemaName, string storedAt) + internal CultForeignRecord(string key, string schemaId, string schemaName, string storedAt, string storedVersion) { Key = key; SchemaId = schemaId; SchemaName = schemaName; StoredAt = storedAt; + StoredVersion = storedVersion; } public string Key { get; } @@ -179,6 +180,10 @@ namespace GameCult.Caching public string SchemaName { get; } public string StoredAt { get; } + + // The version of the record's stored bytes when its store read it, in that store's spelling, given by the store through + // CacheBackingStore.Foreign. ExpectUnchanged compares it with the version on disk at the commit. + internal string StoredVersion { get; } } public sealed class CultPersistedStoreSnapshot @@ -1860,24 +1865,42 @@ namespace GameCult.Caching internal CultCommitRequest( IReadOnlyList upserts, IReadOnlyList deletes, + IReadOnlyList checks, IReadOnlyList<(CultRecordKey Key, string? Version)> expected, bool expectUnchanged) { Upserts = upserts; Deletes = deletes; + Checks = checks; Expected = expected; ExpectUnchanged = expectUnchanged; } public IReadOnlyList Upserts { get; } public IReadOnlyList Deletes { get; } + + /// + /// Keys the batch removes that its cache does not hold. Such a removal removes nothing, so it is never in + /// ; but a store that holds a record at one of these keys under an id no registered type declares + /// refuses the commit, as it refuses a write of that record. The home's judges + /// these keys under its lock, with the keys it writes; every other attached store judges them first through + /// , each under its own lock, and a refusal anywhere writes nothing. A judged + /// key is never removed: a record a declared type reads stays where it is. + /// + public IReadOnlyList Checks { get; } public IReadOnlyList<(CultRecordKey Key, string? Version)> Expected { get; } public bool ExpectUnchanged { get; } public bool HasConditions => Expected.Count > 0 || ExpectUnchanged; - // durable is the version of each record the store holds on disk now, by key; observed is what its cache last loaded or - // committed. Identity is the record's version, the SHA-256 of its stored bytes, which names its schema id and storedAt too. - public bool ConditionsHold(IReadOnlyDictionary durable, IEnumerable observed) + /// + /// Whether the batch's conditions hold. is the version of each record the store holds on disk now, + /// by key; is what the store's cache last loaded or committed, and the + /// foreign records the store carries as it last read them (; empty for a home + /// with none). Under each foreign record compares against the version it had when the store + /// read it, as a record the cache holds compares against its own. Identity is the record's version, the SHA-256 of its stored + /// bytes, which names its schema id and storedAt too. + /// + public bool ConditionsHold(IReadOnlyDictionary durable, IEnumerable observed, IEnumerable foreign) { foreach (var (key, version) in Expected) { @@ -1893,6 +1916,7 @@ namespace GameCult.Caching .OrderBy(pair => pair.Key, StringComparer.Ordinal) .SequenceEqual(observed .Select(stored => (Key: stored.Key.Value, Version: stored.StoredVersion)) + .Concat(foreign.Select(record => (record.Key, Version: record.StoredVersion))) .OrderBy(pair => pair.Key, StringComparer.Ordinal)); } } @@ -2291,7 +2315,7 @@ namespace GameCult.Caching { foreach (var (store, _) in _stores) { - if (!store.IsReadOnly && store.IsDirty) + if (!store.IsReadOnly && (store.IsDirty || store.HasStagedChecks)) store.PushAll(); } return true; @@ -2352,8 +2376,15 @@ namespace GameCult.Caching return key; })); - // Refused with CultSchemaConflictException when the key holds a record stored under an id no registered type declares, - // read-only or foreign. False when the cache holds nothing there and no store carries a foreign record there. + /// + /// Removes the record at ; false when the cache holds nothing there. + /// + /// + /// A record stored under an id no registered type declares, read-only or foreign, is never removed: removing one the cache + /// holds is refused with now. A key the cache does not hold costs nothing here and + /// returns false; each writable store judges it at its next flush or commit, under its lock, and refuses that flush or + /// commit if it holds a record there under such an id (a read-only store answers now, from what it last loaded). + /// public bool Remove(CultRecordKey key) => Held(() => { if (!_entries.TryGetValue(key.Value, out var existing)) @@ -2467,7 +2498,7 @@ namespace GameCult.Caching { Held(() => { - if (FlushAttachedStoresOnDispose && IsDirty) + if (FlushAttachedStoresOnDispose && (IsDirty || _stores.Any(entry => entry.Store.HasStagedChecks))) FlushAllBackingStores(); return true; }); @@ -2633,18 +2664,33 @@ namespace GameCult.Caching : op.Variant != null ? StampVariant(op.Key, DescriptorOfBase(op.Key, op.Variant.BaseKey, batch), op.Variant) : StampFlatten(op.Key)) .ToArray(); - RefuseRemovalOfUndeclared(batch.Operations.Values + // A removal of a key the cache does not hold removes nothing: it is a check every store the commit reaches judges under its lock. + var checks = batch.Operations.Values .Where(op => op.IsRemoval && !_entries.ContainsKey(op.Key.Value)) .Select(op => op.Key.Value) - .ToArray()); + .ToArray(); var deletes = batch.Operations.Values .Where(op => op.IsRemoval && _entries.ContainsKey(op.Key.Value)) .Select(op => _entries[op.Key.Value]) .ToArray(); return Admit(upserts, deletes, null, (home, resolved) => { + // A batch that only checks keys names no home; it commits through the cache's one writable store, if it has one. + if (home == null && checks.Length > 0 && _stores.Where(entry => !entry.Store.IsReadOnly).Take(2).ToArray() is { Length: 1 } only) + home = only[0].Store; + // Every store the batch does not commit through judges its checks first, each under its own lock, so a refusal or + // contention writes nothing; the home judges its own in CommitBatch. + foreach (var (store, _) in _stores) + { + if (store == home || checks.Length == 0) + continue; + var judged = store.JudgeChecks(checks, wait); + if (judged != CultCommitOutcome.Committed) + return judged; + } + // What the store holds is what the hold resolved: a variant's delta beside its resolved view. - var request = new CultCommitRequest(resolved, deletes, batch.Expected.ToArray(), batch.ExpectsUnchanged); + var request = new CultCommitRequest(resolved, deletes, checks, batch.Expected.ToArray(), batch.ExpectsUnchanged); if (home != null) { if (request.HasConditions && home.IsDirty) @@ -2655,14 +2701,14 @@ namespace GameCult.Caching if (request.HasConditions && _stores.Count > 0) throw new InvalidOperationException("A conditional batch names its home store through a record it upserts or removes."); var inMemory = _entries.Values.ToDictionary(entry => entry.Key.Value, entry => entry.StoredVersion, StringComparer.Ordinal); - return request.ConditionsHold(inMemory, _entries.Values) ? CultCommitOutcome.Committed : CultCommitOutcome.Mismatch; + return request.ConditionsHold(inMemory, _entries.Values, Array.Empty()) ? CultCommitOutcome.Committed : CultCommitOutcome.Mismatch; }); }); } // A removal of a key the cache does not hold removes nothing, so no write carries it to a store; but a store that holds a // record there under an id no registered type declares refuses it, as it refuses every write of that record, rather than let - // the caller believe it was removed. + // the caller believe it was removed. Each store judges it its own way (CacheBackingStore.RefuseRemovalOfUndeclared). private void RefuseRemovalOfUndeclared(IReadOnlyCollection keys) { if (keys.Count == 0) @@ -3531,13 +3577,22 @@ namespace GameCult.Caching public abstract void Delete(CultStoredDocument entry); public abstract void PushAll(); - // One durable step under the store's exclusive lock; conditions are evaluated against what is on disk then. - // A failure restores the store's staged view. + /// + /// One durable step under the store's exclusive lock: the batch and every staged write land onto the store as it is then, + /// and the batch's conditions are evaluated against it. A failure restores the store's staged view. + /// + /// + /// An implementation judges , and its own , under that + /// lock with the keys it writes: a key whose record is under an id no registered type declares refuses the commit, and a + /// checked key is never removed. The cache calls on every other store before this commit. Each + /// foreign record a store lists comes from with the version it read, so pass to + /// . + /// public abstract CultCommitOutcome CommitBatch(CultCommitRequest request, bool wait); public virtual void Dispose() { - if (FlushOnDispose && IsDirty && !IsReadOnly) + if (FlushOnDispose && (IsDirty || HasStagedChecks) && !IsReadOnly) { PushAll(); } @@ -3644,9 +3699,11 @@ namespace GameCult.Caching // The one test of whether a stored record is foreign: no registered type claims its schema id through the catalog it was // stored with. It reads the record's header only, never its payload. Null for a record the registry resolves. - protected CultForeignRecord? Foreign(CultPersistedRecord record, IReadOnlyCollection catalog) => + // The one constructor path for a foreign record, so every store kind, one outside this library included, lists it at the + // version it read. + protected CultForeignRecord? Foreign(CultPersistedRecord record, IReadOnlyCollection catalog, string storedVersion) => Registry.Claimant(record.SchemaId, catalog, out var persisted) == null - ? new CultForeignRecord(record.Key, record.SchemaId, persisted!.SchemaName, record.StoredAt) + ? new CultForeignRecord(record.Key, record.SchemaId, persisted!.SchemaName, record.StoredAt, storedVersion) : null; protected void SetForeignRecords(IEnumerable foreign) @@ -3667,16 +3724,71 @@ namespace GameCult.Caching throw Overwrites(null, new[] { (key, held.StoredSchemaId, held.Descriptor.SchemaName) }); } - // The apply's refusal for removals of keys the cache does not hold. Such a removal writes nothing, so no write carries it to - // where the store is read; this reads it instead. A key whose durable record is under an id no registered type declares is - // refused and reloaded as the apply refuses and reloads it, including a record another writer put there after the load. - // A store kind that cannot read its durable state answers from its view (RefuseUndeclared). + /// + /// Called when the cache removes keys it does not hold, outside a batch that commits through this store. Such a removal + /// removes nothing, but a store that holds a record at one of these keys under an id no registered type declares must + /// refuse it rather than let the caller believe the record is gone. + /// + /// + /// + /// The default refuses now, from this store's view: the foreign records its last load listed and the records it holds + /// (). It reads nothing, so it cannot see a record another writer put there after that load. + /// + /// + /// A writable store kind that reads its durable state under a lock overrides this to call , and + /// judges at its next or as it judges + /// : under the lock, with the keys it writes, refused through the same refusal, never + /// removed. It calls when that step succeeds, which clears them. A read-only store has no + /// such step and keeps the default. Both shipped store kinds do this. A batch's own checks are judged through + /// . + /// + /// protected internal virtual void RefuseRemovalOfUndeclared(IReadOnlyCollection keys) { foreach (var key in keys) RefuseUndeclared(key); } + /// + /// Called for a batch's checks on every attached store that is not the batch's home, before the home commits. It judges + /// only the keys it is given: it never writes, stages, clears or calls + /// . + /// + /// + /// The default answers from this store's view () and reads nothing, which is right for a + /// read-only store. A writable store kind that reads its durable state under a lock overrides this to read under that + /// lock and refuse through , returning when it may not wait + /// and the lock is held. A refusal or contention here writes nothing anywhere, because the home has not yet committed. + /// + protected internal virtual CultCommitOutcome JudgeChecks(IReadOnlyCollection keys, bool wait) + { + foreach (var key in keys) + RefuseUndeclared(key); + return CultCommitOutcome.Committed; + } + + // Keys a removal named that this store's cache does not hold, waiting for this store's next flush or commit to judge them. + // A refusal takes back the keys it refused; a successful flush or commit clears them. + private readonly HashSet _checks = new(StringComparer.Ordinal); + + /// + /// The keys staged for this store's next flush or commit to judge. They are not writes: + /// does not count them. + /// + protected IReadOnlyCollection StagedChecks => _checks; + + /// + /// Stages keys for this store's next flush or commit to judge without removing them. See + /// . + /// + protected void StageChecks(IEnumerable keys) => Held(() => + { + foreach (var key in keys) + _checks.Add(key); + }); + + internal bool HasStagedChecks => _checks.Count > 0; + // A stored record under an id the writer does not declare, named by the catalog entry that publishes the id. Header only: no // payload is decoded. Null for a record under a declared id. protected static (string Key, string SchemaId, string SchemaName)? Undeclared( @@ -3752,7 +3864,10 @@ namespace GameCult.Caching } foreach (var key in refused) + { unstage(key); + _checks.Remove(key); + } foreach (var held in dropped) Entries.TryRemove(held.Key.Value, out _); foreach (var stored in loaded) @@ -3803,9 +3918,11 @@ namespace GameCult.Caching refused[0].Key, inner); + // A flush or commit succeeded: nothing staged is left, writes or checks. protected void MarkFlushSucceeded() { IsDirty = false; + _checks.Clear(); } } @@ -3928,7 +4045,7 @@ namespace GameCult.Caching foreach (var record in snapshot.Records) { var version = versions[record.Key] = durable.Version(record.Key); - if (Foreign(record, snapshot.SchemaCatalog) is { } carried) + if (Foreign(record, snapshot.SchemaCatalog, version) is { } carried) { foreign.Add(carried); continue; @@ -3994,7 +4111,7 @@ namespace GameCult.Caching { var disk = ReadSnapshot(); var (upserts, removals) = StagedWrites(Array.Empty(), Array.Empty()); - Wrote(ApplyWriteSet(disk, upserts, removals)); + Wrote(ApplyWriteSet(disk, upserts, removals, Array.Empty())); } _staged.Clear(); @@ -4019,11 +4136,12 @@ namespace GameCult.Caching var disk = ReadSnapshot(); if (request.HasConditions && !request.ConditionsHold( (disk?.Snapshot.Records ?? Array.Empty()).ToDictionary(record => record.Key, record => disk!.Version(record.Key), StringComparer.Ordinal), - Entries.Values)) + Entries.Values, + ForeignRecords)) return CultCommitOutcome.Mismatch; var (upserts, removals) = StagedWrites(request.Upserts, request.Deletes); - var written = ApplyWriteSet(disk, upserts, removals); + var written = ApplyWriteSet(disk, upserts, removals, request.Checks); foreach (var entry in request.Deletes) Entries.TryRemove(entry.Key.Value, out _); foreach (var entry in request.Upserts) @@ -4080,20 +4198,24 @@ namespace GameCult.Caching internal void ApplyWriteSet(IReadOnlyCollection upserts, IReadOnlyCollection removals) => Held(() => { using (AcquireLock(wait: true)) - _ = ApplyWriteSet(ReadSnapshot(), upserts, removals); + _ = ApplyWriteSet(ReadSnapshot(), upserts, removals, Array.Empty()); }); // Flush, commit and Mesh's write all end here, with the durable store they read under the lock they hold. First, a key the write - // names whose durable record is under an id the writer does not declare refuses the write (RefuseUndeclaredOnDisk). Then a store that holds + // names or checks whose durable record is under an id the writer does not declare refuses the write (RefuseUndeclaredOnDisk); a + // write that then has nothing to write but keys it checked writes no file. Then a store that holds // a variant, or would after this write, is written only while its record versions are what this store last read: a variant // resolves against other records, and a write decodes none it did not stage, so it cannot judge another writer's. Versions // only: a write hashes other records' bytes, never decodes them. Otherwise the whole write is refused and what it refused is // reloaded (Refuse). Every record the writer staged is encoded from its object now, and its version is the hash of that // encoding; every other record is copied as the slice the file holds. Returns the records written, each with the version of // the bytes written. - private IReadOnlyCollection<(CultStagedRecord Staged, string Version)> ApplyWriteSet(Durable? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals) + private IReadOnlyCollection<(CultStagedRecord Staged, string Version)> ApplyWriteSet( + Durable? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals, IEnumerable checks) { - RefuseUndeclaredOnDisk(durable, upserts, removals); + RefuseUndeclaredOnDisk(durable, upserts, removals, checks); + if (upserts.Count == 0 && removals.Count == 0 && (checks.Any() || StagedChecks.Count > 0)) + return Array.Empty<(CultStagedRecord, string)>(); RefuseWriteIntoMovedVariantStore(durable, upserts, removals); var durableRecords = durable?.Snapshot.Records ?? Array.Empty(); // A record written over one the store holds is stored at a storedAt later than the one it replaces, whoever wrote that. @@ -4193,17 +4315,38 @@ namespace GameCult.Caching inner)); } - // The file as it is now, read as a pull reads it, without the lock: nothing is written after the read, and the file is only - // ever replaced whole. - protected internal override void RefuseRemovalOfUndeclared(IReadOnlyCollection keys) => - Held(() => RefuseUndeclaredOnDisk(ReadSnapshot(), keys, Registry.Declares)); + // A writable store judges a removal of a key its cache does not hold at its next flush or commit, under the lock (StageChecks); + // a read-only store has neither, so it answers from its view and never creates the lock file. + protected internal override void RefuseRemovalOfUndeclared(IReadOnlyCollection keys) + { + if (IsReadOnly) + base.RefuseRemovalOfUndeclared(keys); + else + StageChecks(keys); + } + + // A read-only store answers a batch's checks from its view and never creates the lock file; a writable one reads the file + // under its lock. + protected internal override CultCommitOutcome JudgeChecks(IReadOnlyCollection keys, bool wait) + { + if (IsReadOnly) + return base.JudgeChecks(keys, wait); + return Held(() => + { + using var fileLock = AcquireLock(wait); + if (fileLock == null) + return CultCommitOutcome.Contended; + RefuseUndeclaredOnDisk(ReadSnapshot(), keys, Registry.Declares); + return CultCommitOutcome.Committed; + }); + } // A cache declares through its registry. A cacheless CultMesh write declares the ids of the catalog entry it writes: its own id - // and the ids it lists compatible. - private void RefuseUndeclaredOnDisk(Durable? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals) => + // and the ids it lists compatible. The keys judged are every key staged, written, removed or checked. + private void RefuseUndeclaredOnDisk(Durable? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals, IEnumerable checks) => RefuseUndeclaredOnDisk( durable, - _staged.Concat(upserts.Select(staged => staged.Record.Key)).Concat(removals), + _staged.Concat(upserts.Select(staged => staged.Record.Key)).Concat(removals).Concat(checks).Concat(StagedChecks), Cache != null ? Registry.Declares : id => upserts.Any(staged => @@ -4244,17 +4387,20 @@ namespace GameCult.Caching } // The durable record at a key as a load lists it when no registered type claims its id; null otherwise. - private CultForeignRecord? ForeignAt(Durable? durable, string key) => - durable?.Snapshot.Records.FirstOrDefault(candidate => string.Equals(candidate.Key, key, StringComparison.Ordinal)) is { } record - ? Foreign(record, durable.Snapshot.SchemaCatalog) - : null; + private CultForeignRecord? ForeignAt(Durable? durable, string key) + { + if (durable?.Snapshot.Records.FirstOrDefault(candidate => string.Equals(candidate.Key, key, StringComparison.Ordinal)) is not { } record || + Foreign(record, durable.Snapshot.SchemaCatalog, durable.Version(key)) is not { } carried) + return null; + return carried; + } // The durable record at a key decoded as a load decodes it, at its durable version: null when the store holds none there or // holds it foreign. The one decode both refusals reload through. private CultStoredDocument? DurableAt(Durable? durable, string key) { var record = durable?.Snapshot.Records.FirstOrDefault(candidate => string.Equals(candidate.Key, key, StringComparison.Ordinal)); - if (record == null || Foreign(record, durable!.Snapshot.SchemaCatalog) != null) + if (record == null || Foreign(record, durable!.Snapshot.SchemaCatalog, durable.Version(key)) != null) return null; return ToStoredDocument(record, durable.Version(key), durable.Snapshot.SchemaCatalog, DeserializePayload, out _); } diff --git a/tests/GameCult.Caching.ConsumerBuild/Program.cs b/tests/GameCult.Caching.ConsumerBuild/Program.cs index b0f15973..417da1a6 100644 --- a/tests/GameCult.Caching.ConsumerBuild/Program.cs +++ b/tests/GameCult.Caching.ConsumerBuild/Program.cs @@ -5,6 +5,42 @@ using MessagePack; // RefKeyedDictionaryCompilesInAConsumer: this project building is the test. System.Console.WriteLine(new ConsumerFaction().Standing.Count); +// An outside store kind versions its foreign records: this project has no InternalsVisibleTo, so building and running it pins that +// Foreign(record, catalog, version) is all a store kind outside the library needs for ExpectUnchanged to hold over them. +using var outside = new CultCache(CultDocumentRegistry.ForTypes(new[] { typeof(ConsumerFaction) })); +outside.AddBackingStore(new OutsideStore(), typeof(ConsumerFaction)); +var committed = outside.Commit(batch => +{ + batch.ExpectUnchanged(); + batch.Upsert(new ConsumerFaction { Name = "a" }, new CultRecordHandle(new CultRecordKey("a"))); +}); +if (!committed) + return 1; +return 0; + +// Lists one record under an id no type claims, at version "v1", and commits only when the conditions hold over it. +public sealed class OutsideStore : CacheBackingStore +{ + private static readonly CultSchemaCatalogEntry Entry = new() { SchemaId = "outside.unclaimed", SchemaName = "outside.unclaimed" }; + + public override void PullAll() + { + var record = new CultPersistedRecord { Key = "w", SchemaId = Entry.SchemaId, StoredAt = "t" }; + SetForeignRecords(new[] { Foreign(record, new[] { Entry }, "v1")! }); + } + + public override void Push(CultStoredDocument entry) { } + + public override void Delete(CultStoredDocument entry) { } + + public override void PushAll() { } + + public override CultCommitOutcome CommitBatch(CultCommitRequest request, bool wait) => + request.ConditionsHold(new Dictionary { ["w"] = "v1" }, Entries.Values, ForeignRecords) + ? CultCommitOutcome.Committed + : CultCommitOutcome.Mismatch; +} + [CultDocument("tests.consumer_faction", "tests.consumer_faction.v1")] [MessagePackObject] public sealed class ConsumerFaction diff --git a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs index 1f417f2c..4c30b9e0 100644 --- a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs +++ b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs @@ -18,6 +18,8 @@ namespace GameCult.Caching.Tests private static readonly Type ClaimingDeck = Emit("ForeignSuiteClaimingDeck", "tests.foreign_deck", "tests.foreign_deck.v1", new[] { new Field("Name", typeof(string), 0, IsName: true) }, new[] { "other.runtime.deck" }); private static readonly Type LateClaimingDeck = Emit("ForeignSuiteLateClaimingDeck", "tests.foreign_deck", "tests.foreign_deck.v2", new[] { new Field("Name", typeof(string), 0, IsName: true) }, new[] { "other.runtime.deck" }); private static readonly Type Widget = Emit("ForeignSuiteWidget", "tests.foreign_widget", "tests.foreign_widget.v1", new[] { new Field("Label", typeof(string), 0, IsName: true) }); + // A second type a cache registers, so that a second backing store has a home of its own. + private static readonly Type Gizmo = Emit("ForeignSuiteGizmo", "tests.foreign_gizmo", "tests.foreign_gizmo.v1", new[] { new Field("Tag", typeof(string), 0, IsName: true) }); // A build that has both types, and one that has only the deck: to it the widget is foreign. private static readonly CultDocumentRegistry Full = CultDocumentRegistry.ForTypes(new[] { Deck, Widget }); @@ -1011,10 +1013,10 @@ namespace GameCult.Caching.Tests store.PushAll(); break; case DirectWrite.CommitUpsert: - store.CommitBatch(new CultCommitRequest(new[] { mine }, Array.Empty(), Array.Empty<(CultRecordKey, string?)>(), false), wait: true); + store.CommitBatch(new CultCommitRequest(new[] { mine }, Array.Empty(), Array.Empty(), Array.Empty<(CultRecordKey, string?)>(), false), wait: true); break; case DirectWrite.CommitDelete: - store.CommitBatch(new CultCommitRequest(Array.Empty(), new[] { held }, Array.Empty<(CultRecordKey, string?)>(), false), wait: true); + store.CommitBatch(new CultCommitRequest(Array.Empty(), new[] { held }, Array.Empty(), Array.Empty<(CultRecordKey, string?)>(), false), wait: true); break; } })!; @@ -1026,9 +1028,10 @@ namespace GameCult.Caching.Tests Assert.That(EmittedDocumentTypes.Read(cache.Get(D)!, "Name"), Is.EqualTo("theirs")); } - // A removal of a foreign record, which the cache does not hold, is refused typed: the caller is never told it removed what the - // store still carries. The store is read as it is at the removal, so a foreign record another writer added after the load - // is refused too, and listed foreign from then on. + // A removal of a foreign record, which the cache does not hold, is refused typed where the store judges it under its lock: at + // the commit that carries it, or the flush that follows a Remove, which itself costs nothing and returns false. The caller is + // never told a write removed what the store still carries. The store is read at that judgement, so a foreign record another + // writer added after the load is refused too, and listed foreign from then on. [Test] public void RemovingAForeignRecordIsRefusedNotReportedAsDone([Values] bool directory, [Values] bool batch, [Values] bool appearedAfterLoad) { @@ -1054,14 +1057,573 @@ namespace GameCult.Caching.Tests if (batch) cache.Commit(stage => stage.Remove(W)); else - cache.Remove(W); + { + Assert.That(cache.Remove(W), Is.False, "a key the cache does not hold costs nothing at the call"); + cache.FlushAsync().GetAwaiter().GetResult(); + } })!; Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo((W.Value, WidgetId))); Assert.That(cache.IsDirty, Is.False); Assert.That(StoreOf(cache).ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "w" })); + Assert.DoesNotThrow(() => cache.FlushAsync().GetAwaiter().GetResult(), "a refusal forgets the check it refused"); Assert.That(cache.Remove(new CultRecordKey("absent")), Is.False, "a key nothing holds is still no removal"); Assert.That(Fingerprint(path), Is.EqualTo(before)); } } + + // A write that removes a key the cache does not hold is judged under the store's lock, after the load: a foreign record another + // writer committed in between refuses the whole commit, typed, and nothing the commit would have written lands. + [Test] + public void AForeignRecordThatLandsBetweenTheLoadAndTheCommitRefusesTheCommitThatRemovesIt([Values] bool directory) + { + var path = PathOf("window.cc"); + using (var seed = Open(path, Full, directory)) + seed.Commit(batch => batch.Upsert(Deck, DeckOf("d"), D)); + using var a = Open(path, DeckOnly, directory); + using (var b = Open(path, Full, directory)) + b.Commit(batch => batch.Upsert(Widget, WidgetOf("w"), W)); + var before = Fingerprint(path); + + var refusal = Assert.Throws(() => a.Commit(batch => + { + batch.Upsert(Deck, DeckOf(Canary), D); + batch.Remove(W); + }))!; + + Assert.Multiple(() => + { + Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo((W.Value, WidgetId))); + Assert.That(refusal.Message, Does.Not.Contain(Canary)); + Assert.That(Fingerprint(path), Is.EqualTo(before), "nothing was written, d included"); + Assert.That(a.IsDirty, Is.False); + }); + } + + // Removing a key nothing holds reads no store: the judgement is the flush's, so a store that cannot be read costs the removal + // nothing and the flush is where it is read. + [Test] + public void RemovingAnUnheldKeyReadsNothingUntilTheFlush([Values] bool directory) + { + var path = Seeded("unreadable.cc", directory); + using var cache = Open(path, DeckOnly, directory); + var original = File.ReadAllBytes(path); + File.WriteAllBytes(path, new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 }); + try + { + Assert.DoesNotThrow(() => Assert.That(cache.Remove(new CultRecordKey("absent")), Is.False)); + Assert.Throws(() => cache.FlushAsync().GetAwaiter().GetResult()); + } + finally + { + File.WriteAllBytes(path, original); + } + } + + // A checked key is judged and never removed: another writer's declared record at it stays where it is. + [Test] + public void ACheckedKeyIsNeverRemoved([Values] bool directory) + { + var path = PathOf("checked.cc"); + using (var seed = Open(path, Full, directory)) + seed.Commit(batch => batch.Upsert(Deck, DeckOf("d"), D)); + using var a = Open(path, DeckOnly, directory); + a.UpsertAsync(Deck, DeckOf("e"), E).GetAwaiter().GetResult(); + Assert.That(a.Remove(W), Is.False); + using (var b = Open(path, Full, directory)) + b.Commit(batch => batch.Upsert(Deck, DeckOf("theirs"), W)); + var theirs = Stored(path, W); + + Assert.DoesNotThrow(() => a.FlushAsync().GetAwaiter().GetResult()); + + Assert.That(Read(path).Records.Select(record => record.Key), Is.EquivalentTo(new[] { "d", "e", "w" }), "the staged write landed beside the checked key"); + Assert.That(Stored(path, W), Is.EqualTo(theirs), "the record at the checked key is still the other writer's bytes"); + using (var c = Open(path, Full, directory)) + c.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), W)); + Assert.DoesNotThrow(() => a.FlushAsync().GetAwaiter().GetResult(), "a flush clears what it judged: the next one has nothing to judge"); + } + + // ExpectUnchanged compares the foreign records a store carries at the versions it read them: it holds right after the load, + // and fails, writing nothing, once another writer changed one. + [Test] + public void ExpectUnchangedHoldsOverAStoreCarryingAForeignRecord([Values] bool directory) + { + var path = Seeded("expect-unchanged.cc", directory); + using var a = Open(path, DeckOnly, directory); + Assert.That(StoreOf(a).ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "w" })); + + Assert.That(a.Commit(batch => + { + batch.ExpectUnchanged(); + batch.Upsert(Deck, DeckOf("e"), E); + }), Is.True, "nothing changed since the load"); + + using (var b = Open(path, Full, directory)) + b.Commit(batch => batch.Upsert(Widget, WidgetOf("changed"), W)); + var before = Fingerprint(path); + + Assert.That(a.Commit(batch => + { + batch.ExpectUnchanged(); + batch.Upsert(Deck, DeckOf("f"), new CultRecordKey("f")); + }), Is.False, "another writer changed the foreign record"); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + } + + // A foreign key listed at the load that another writer then replaced with a record this cache reads but does not declare: the + // refused removal (judged at the flush) reloads it read-only and takes it off the foreign list. + [Test] + public void AListedForeignKeyBecomesReadOnlyWhenARemovalOfItIsRefused([Values] bool directory) + { + var path = Seeded("delist.cc", directory); + using var a = Open(path, DeckOnly, directory); + Assert.That(StoreOf(a).ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "w" })); + // The writer declares the widget's id, so it may replace the widget with a deck record under an id a reads but does not declare. + using (var other = Open(path, CultDocumentRegistry.ForTypes(new[] { MovingDeck, Widget }), directory)) + Assert.That(other.Commit(batch => batch.Upsert(MovingDeck, New(MovingDeck, ("Name", "moved")), W)), Is.True); + + Assert.That(a.Remove(W), Is.False); + var refusal = Assert.Throws(() => a.FlushAsync().GetAwaiter().GetResult())!; + RefusedOnDisk(refusal, W); + + Assert.That(StoreOf(a).ForeignRecords, Is.Empty, "no longer foreign"); + Assert.That(a.Get(W), Is.Not.Null, "held, read-only"); + Assert.Throws(() => a.Remove(W), "refused at the call now: the cache holds it"); + } + + // A single-file store holding a variant refuses a write after another writer changed it; a changed key whose record this cache + // does not read is listed as foreign by that refusal, as a load lists it. + [Test] + public void AVariantStoreRefusalListsTheRecordItRefusedAsForeign([Values(Write.Flush, Write.Commit)] Write write) + { + var path = PathOf("variant-refusal.cc"); + var gears = CultDocumentRegistry.ForTypes(new[] { typeof(VariantTests.VariantGear) }); + var withWidget = CultDocumentRegistry.ForTypes(new[] { typeof(VariantTests.VariantGear), Widget }); + using var a = Open(path, gears, directory: false); + var baseKey = new CultRecordKey("laser"); + a.Commit(batch => + { + batch.Upsert(typeof(VariantTests.VariantGear), new VariantTests.VariantGear { Name = "laser", Power = 10, Code = "l1" }, baseKey); + batch.UpsertVariant(new CultRecordKey("laser-big"), baseKey, new[] + { + a.Override(nameof(VariantTests.VariantGear.Name), "laser big"), + a.Override(nameof(VariantTests.VariantGear.Power), 25), + a.Override(nameof(VariantTests.VariantGear.Code), "laser big") + }); + }); + using (var b = Open(path, withWidget, directory: false)) + b.Commit(batch => batch.Upsert(Widget, WidgetOf("w"), W)); + Assert.That(StoreOf(a).ForeignRecords, Is.Empty); + + var conflict = Assert.Throws(() => + { + var next = new VariantTests.VariantGear { Name = "next", Power = 1, Code = "n" }; + var key = new CultRecordKey("n"); + if (write == Write.Flush) + { + a.UpsertAsync(typeof(VariantTests.VariantGear), next, key).GetAwaiter().GetResult(); + a.FlushAsync().GetAwaiter().GetResult(); + } + else + { + a.Commit(batch => batch.Upsert(typeof(VariantTests.VariantGear), next, key)); + } + })!; + + Assert.That(conflict.ChangedKeys, Does.Contain(W.Value)); + var listed = StoreOf(a).ForeignRecords.Single(); + Assert.That((listed.Key, listed.SchemaId), Is.EqualTo((W.Value, WidgetId)), "the refused record is listed as the store holds it"); + Assert.That(a.Get(W), Is.Null); + } + + private static CacheBackingStore StoreAt(string path, bool directory, bool readOnly) => directory + ? new DirectoryMessagePackBackingStore(path, readOnly: readOnly) + : new SingleFileMessagePackBackingStore(path, readOnly); + + // A batch with nothing to write and no key to check commits through no store: a staged write waits for its own flush. + [Test] + public void ABatchWithNothingToWriteOrCheckCommitsNothingToTheStore([Values] bool directory) + { + var path = Seeded("empty-batch.cc", directory); + using var cache = Open(path, DeckOnly, directory); + cache.UpsertAsync(Deck, DeckOf("e"), E).GetAwaiter().GetResult(); + + Assert.That(cache.Commit(batch => { }), Is.True); + + Assert.That(Read(path).Records.Select(record => record.Key), Is.EquivalentTo(new[] { "d", "w" }), "e is still only staged"); + cache.FlushAsync().GetAwaiter().GetResult(); + Assert.That(Read(path).Records.Select(record => record.Key), Is.EquivalentTo(new[] { "d", "e", "w" })); + } + + // A write that only checked keys has nothing to write: no file is replaced and no generation is written, by a commit or a flush. + [Test] + public void AWriteThatOnlyCheckedKeysWritesNoFile([Values] bool directory, [Values] bool viaCommit) + { + var path = Seeded("check-only.cc", directory); + using var cache = Open(path, DeckOnly, directory); + var past = new DateTime(2001, 1, 1, 0, 0, 0, DateTimeKind.Utc); + File.SetLastWriteTimeUtc(path, past); + var before = Fingerprint(path); + + if (viaCommit) + Assert.That(cache.Commit(batch => batch.Remove(new CultRecordKey("absent"))), Is.True); + else + { + Assert.That(cache.Remove(new CultRecordKey("absent")), Is.False); + cache.FlushAsync().GetAwaiter().GetResult(); + } + + Assert.That(Fingerprint(path), Is.EqualTo(before)); + Assert.That(File.GetLastWriteTimeUtc(path), Is.EqualTo(past), "the store was not replaced"); + Assert.That(cache.IsDirty, Is.False); + } + + // A flush or commit that only checked keys clears every check it judged: the next one has nothing to judge, so it does not + // refuse a record another writer put at a checked key afterwards. + [Test] + public void AWriteThatOnlyCheckedKeysClearsTheChecksItJudged([Values] bool directory, [Values] bool viaCommit) + { + var path = Seeded("check-cleared.cc", directory); + using var cache = Open(path, DeckOnly, directory); + var staged = new CultRecordKey("staged"); + Assert.That(cache.Remove(staged), Is.False); + if (viaCommit) + Assert.That(cache.Commit(batch => batch.Remove(new CultRecordKey("requested"))), Is.True); + else + cache.FlushAsync().GetAwaiter().GetResult(); + using (var other = Open(path, Full, directory)) + other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), staged)); + + Assert.DoesNotThrow(() => cache.FlushAsync().GetAwaiter().GetResult()); + } + + // A disposed cache and a disposed store judge the checks a removal staged, as their flush does. + [Test] + public void DisposingAStoreJudgesTheChecksARemovalStaged([Values] bool directory, [Values] bool viaCache) + { + var path = Seeded("dispose.cc", directory); + var cache = Open(path, DeckOnly, directory); + cache.FlushAttachedStoresOnDispose = true; + StoreOf(cache).FlushOnDispose = true; + Assert.That(cache.Remove(W), Is.False); + var before = Fingerprint(path); + + var refusal = Assert.Throws(() => + { + if (viaCache) + cache.Dispose(); + else + StoreOf(cache).Dispose(); + })!; + + Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo((W.Value, WidgetId))); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + try { cache.Dispose(); } catch (CultSchemaConflictException) { } + } + + // A read-only store has no lock to judge under, so it answers a removal of a key the cache does not hold at the call, from + // what it last loaded. + [Test] + public void AReadOnlyStoreRefusesTheRemovalOfAForeignRecordAtTheCall([Values] bool directory) + { + var path = Seeded("read-only.cc", directory); + using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = DeckOnly, UseDirectoryStore = directory, ReadOnly = true }); + + var refusal = Assert.Throws(() => cache.Remove(W))!; + + Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo((W.Value, WidgetId))); + Assert.That(cache.Remove(new CultRecordKey("absent")), Is.False); + } + + // A batch that only checks keys commits through the one writable store; a read-only store answers the checks from its last load. + [Test] + public void AReadOnlyStoreAnswersTheChecksFromItsView([Values] bool directory) + { + var writable = PathOf("home.cc"); + using (var seed = Open(writable, Full, directory)) + seed.Commit(batch => batch.Upsert(Deck, DeckOf("d"), D)); + var readOnly = PathOf("other.cc"); + using (var seed = Open(readOnly, CultDocumentRegistry.ForTypes(new[] { Gizmo, Widget }), directory)) + seed.Commit(batch => + { + batch.Upsert(Gizmo, New(Gizmo, ("Tag", "g")), new CultRecordKey("g")); + batch.Upsert(Widget, WidgetOf("w"), W); + }); + using var cache = new CultCache(CultDocumentRegistry.ForTypes(new[] { Deck, Gizmo })); + cache.AddBackingStore(StoreAt(writable, directory, readOnly: false), Deck); + cache.AddBackingStore(StoreAt(readOnly, directory, readOnly: true), Gizmo); + var before = Fingerprint(writable); + + var refusal = Assert.Throws(() => cache.Commit(batch => batch.Remove(W)))!; + + Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo((W.Value, WidgetId))); + Assert.That(Fingerprint(writable), Is.EqualTo(before)); + Assert.That(cache.Commit(batch => batch.Remove(new CultRecordKey("absent"))), Is.True, "a key no store holds is free"); + } + + // The home store judges a check under its lock, from the record it holds then: a foreign record it listed at the load that + // another writer since replaced with a declared one no longer refuses the commit, and is not removed. + [Test] + public void TheHomeStoreJudgesACheckUnderItsLockNotFromItsLoad([Values] bool directory) + { + var path = Seeded("home-judges.cc", directory); + using var cache = Open(path, DeckOnly, directory); + Assert.That(StoreOf(cache).ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "w" })); + using (var other = Open(path, Full, directory)) + other.Commit(batch => batch.Upsert(Deck, DeckOf("theirs"), W)); + var theirs = Stored(path, W); + + Assert.That(cache.Commit(batch => batch.Remove(W)), Is.True); + + Assert.That(Stored(path, W), Is.EqualTo(theirs), "the declared record the checked key now holds is not removed"); + } + + // A cache disposed with several stores flushes when any one of them has checks staged, not only when all do (a read-only store + // never has any). + [Test] + public void DisposingACacheFlushesWhenOnlyOneStoreHasChecksStaged([Values] bool directory) + { + var writable = Seeded("dispose-home.cc", directory); + var other = PathOf("dispose-other.cc"); + using (var seed = Open(other, CultDocumentRegistry.ForTypes(new[] { Gizmo }), directory)) + seed.Commit(batch => batch.Upsert(Gizmo, New(Gizmo, ("Tag", "g")), new CultRecordKey("g"))); + var cache = new CultCache(CultDocumentRegistry.ForTypes(new[] { Deck, Gizmo })) { FlushAttachedStoresOnDispose = true }; + cache.AddBackingStore(StoreAt(writable, directory, readOnly: false), Deck); + cache.AddBackingStore(StoreAt(other, directory, readOnly: true), Gizmo); + Assert.That(cache.Remove(W), Is.False); + + var refusal = Assert.Throws(() => cache.Dispose())!; + + Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo((W.Value, WidgetId))); + try { cache.Dispose(); } catch (CultSchemaConflictException) { } + } + + // Two writable stores: home.cc holds a deck and takes Deck, other.cc holds a gizmo and takes Gizmo, and, when asked, a widget at + // w that no registered type declares, so the cache lists it foreign at the load. + private (CultCache Cache, string Home, string Other) TwoWritableStores(bool directory, bool otherListsWidget) + { + var home = PathOf("home.cc"); + using (var seed = Open(home, DeckOnly, directory)) + seed.Commit(batch => batch.Upsert(Deck, DeckOf("d"), D)); + var other = PathOf("other.cc"); + using (var seed = Open(other, CultDocumentRegistry.ForTypes(new[] { Gizmo, Widget }), directory)) + { + seed.Commit(batch => batch.Upsert(Gizmo, New(Gizmo, ("Tag", "g")), new CultRecordKey("g"))); + if (otherListsWidget) + seed.Commit(batch => batch.Upsert(Widget, WidgetOf("w"), W)); + } + + var cache = new CultCache(CultDocumentRegistry.ForTypes(new[] { Deck, Gizmo })); + cache.AddBackingStore(StoreAt(home, directory, readOnly: false), Deck); + cache.AddBackingStore(StoreAt(other, directory, readOnly: false), Gizmo); + return (cache, home, other); + } + + // Every writable store judges a batch's checks under its own lock, whichever store the batch commits through: a record another + // writer put at a checked key in a store the batch does not write, after the load, refuses the whole commit and writes nothing. + [Test] + public void EveryWritableStoreRefusesACommitThatRemovesItsLateForeignRecord([Values] bool directory, [Values] bool withWrite) + { + var (cache, home, other) = TwoWritableStores(directory, otherListsWidget: false); + using var _ = cache; + using (var late = Open(other, CultDocumentRegistry.ForTypes(new[] { Gizmo, Widget }), directory)) + late.Commit(batch => batch.Upsert(Widget, WidgetOf("late"), W)); + var homeBefore = Fingerprint(home); + var otherBefore = Fingerprint(other); + + var refusal = Assert.Throws(() => cache.Commit(batch => + { + if (withWrite) + batch.Upsert(Deck, DeckOf("d2"), new CultRecordKey("d2")); + batch.Remove(W); + }))!; + + Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo((W.Value, WidgetId))); + Assert.That(Fingerprint(home), Is.EqualTo(homeBefore)); + Assert.That(Fingerprint(other), Is.EqualTo(otherBefore)); + Assert.That(Read(home).Records.Select(record => record.Key), Does.Not.Contain("d2")); + Assert.That(cache.BackingStores[1].ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "w" })); + } + + // The non-home judgement reads the store under its lock, not its listing: a foreign record that store listed at the load, + // since replaced by a declared one, no longer refuses the commit and is not removed. + [Test] + public void ANonHomeStoreJudgesACheckUnderItsLockNotFromItsLoad([Values] bool directory) + { + var (cache, home, other) = TwoWritableStores(directory, otherListsWidget: true); + using var _ = cache; + Assert.That(cache.BackingStores[1].ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "w" })); + using (var late = Open(other, CultDocumentRegistry.ForTypes(new[] { Gizmo, Widget }), directory)) + late.Commit(batch => batch.Upsert(Gizmo, New(Gizmo, ("Tag", "theirs")), W)); + var theirs = Stored(other, W); + + Assert.That(cache.Commit(batch => + { + batch.Upsert(Deck, DeckOf("d2"), new CultRecordKey("d2")); + batch.Remove(W); + }), Is.True); + + Assert.That(Read(home).Records.Select(record => record.Key), Does.Contain("d2")); + Assert.That(Stored(other, W), Is.EqualTo(theirs), "the declared record the checked key now holds is not removed"); + } + + // A store that is not the batch's home and cannot take its lock makes the commit contended before the home writes. + [Test] + public void ANonHomeStoreHeldByAnotherWriterMakesTryCommitContended() + { + var (cache, home, other) = TwoWritableStores(directory: false, otherListsWidget: false); + using var _ = cache; + var homeBefore = Fingerprint(home); + + using (new FileStream(other + ".lock", FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None)) + { + var outcome = cache.TryCommit(batch => + { + batch.Upsert(Deck, DeckOf("d2"), new CultRecordKey("d2")); + batch.Remove(W); + }); + + Assert.That(outcome, Is.EqualTo(CultCommitOutcome.Contended)); + } + + Assert.That(Fingerprint(home), Is.EqualTo(homeBefore)); + } + + private static void CopyStoreOver(string from, string to, bool directory) + { + if (directory) + { + var source = DirectoryMessagePackBackingStore.DefaultRecordDirectoryPath(from); + var target = DirectoryMessagePackBackingStore.DefaultRecordDirectoryPath(to); + Directory.CreateDirectory(target); + foreach (var file in Directory.GetFiles(source, "*.msgpack")) + File.Copy(file, Path.Combine(target, Path.GetFileName(file)), true); + } + File.Copy(from, to, true); + } + + private static FileStream HoldStoreLock(string path, bool directory) => directory + ? new FileStream(Path.Combine(DirectoryMessagePackBackingStore.DefaultRecordDirectoryPath(path), ".commit.lock"), FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None) + : new FileStream(path + ".lock", FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); + + // A non-home store is judged only after the commit holds that store's own lock: a foreign record another writer lands at the + // checked key while the commit waits for the lock is refused, whichever kind the store is and whether or not the batch writes. + [Test] + public void ANonHomeStoreIsReadOnlyAfterItsLockIsTaken([Values] bool directory, [Values] bool withWrite) + { + var (cache, home, other) = TwoWritableStores(directory, otherListsWidget: false); + using var _ = cache; + var copy = PathOf("copy.cc"); + CopyStoreOver(other, copy, directory); + using (var late = Open(copy, CultDocumentRegistry.ForTypes(new[] { Gizmo, Widget }), directory)) + late.Commit(batch => batch.Upsert(Widget, WidgetOf("late"), W)); + var homeBefore = Fingerprint(home); + + Exception? thrown = null; + var held = HoldStoreLock(other, directory); + var worker = new System.Threading.Thread(() => + { + try + { + cache.Commit(batch => + { + if (withWrite) + batch.Upsert(Deck, DeckOf("d2"), new CultRecordKey("d2")); + batch.Remove(W); + }); + } + catch (Exception ex) { thrown = ex; } + }); + worker.Start(); + // The lock wait is a Thread.Sleep retry loop, so the worker sleeping is the worker waiting for the held lock. + Assert.That(System.Threading.SpinWait.SpinUntil( + () => (worker.ThreadState & System.Threading.ThreadState.WaitSleepJoin) != 0, TimeSpan.FromSeconds(10)), Is.True, "the commit waits on the held lock"); + CopyStoreOver(copy, other, directory); + var otherAfterSwap = Fingerprint(other); + held.Dispose(); + + Assert.That(worker.Join(TimeSpan.FromSeconds(40)), Is.True, "the commit finished"); + Assert.That(thrown, Is.TypeOf()); + Assert.That(Fingerprint(home), Is.EqualTo(homeBefore), "the home is unwritten"); + Assert.That(Fingerprint(other), Is.EqualTo(otherAfterSwap), "the non-home store is unwritten"); + } + + // A directory store judging a check honours wait=false: its lease held by another writer makes TryCommit contended at once. + [Test] + public void ADirectoryNonHomeStoreWhoseLeaseIsHeldMakesTryCommitContended() + { + var (cache, home, other) = TwoWritableStores(directory: true, otherListsWidget: false); + using var _ = cache; + var homeBefore = Fingerprint(home); + + CultCommitOutcome outcome; + using (HoldStoreLock(other, directory: true)) + outcome = cache.TryCommit(batch => + { + batch.Upsert(Deck, DeckOf("d2"), new CultRecordKey("d2")); + batch.Remove(W); + }); + + Assert.That(outcome, Is.EqualTo(CultCommitOutcome.Contended)); + Assert.That(Fingerprint(home), Is.EqualTo(homeBefore)); + } + + // A non-home store the cache attached before anything was written to it has no directory and no lease yet: judging a check + // there creates what it locks, finds nothing, and writes no store. + [Test] + public void ANonHomeStoreNothingWasEverWrittenToJudgesAChecksOnlyCommit([Values] bool directory) + { + var home = PathOf("home.cc"); + using (var seed = Open(home, DeckOnly, directory)) + seed.Commit(batch => batch.Upsert(Deck, DeckOf("d"), D)); + var other = PathOf("fresh" + Path.DirectorySeparatorChar + "other.cc"); + using var cache = new CultCache(CultDocumentRegistry.ForTypes(new[] { Deck, Gizmo })); + cache.AddBackingStore(StoreAt(home, directory, readOnly: false), Deck); + cache.AddBackingStore(StoreAt(other, directory, readOnly: false), Gizmo); + + Assert.That(cache.Commit(batch => + { + batch.Upsert(Deck, DeckOf("d2"), new CultRecordKey("d2")); + batch.Remove(W); + }), Is.True); + + Assert.That(Read(home).Records.Select(record => record.Key), Does.Contain("d2")); + Assert.That(File.Exists(other), Is.False, "judging a check writes no store"); + } + + // A batch with no check asks no other store anything: a lock another writer holds on a store the batch does not write does + // not stand in its way. + [Test] + public void ABatchWithoutChecksIsNotHeldUpByAnotherStoreLock() + { + var (cache, home, other) = TwoWritableStores(directory: false, otherListsWidget: false); + using var _ = cache; + + using (new FileStream(other + ".lock", FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None)) + Assert.That(cache.TryCommit(batch => batch.Upsert(Deck, DeckOf("d2"), new CultRecordKey("d2"))), Is.EqualTo(CultCommitOutcome.Committed)); + + Assert.That(Read(home).Records.Select(record => record.Key), Does.Contain("d2")); + } + + // A foreign record a refusal lists carries the version the refusal read, so a conditional commit over the stores that + // list it holds when nothing has changed since. + [Test] + public void ExpectUnchangedHoldsAfterARefusalListedTheForeignRecord([Values] bool directory) + { + var path = PathOf("refusal-listed.cc"); + using (var seed = Open(path, Full, directory)) + seed.Commit(batch => batch.Upsert(Deck, DeckOf("d"), D)); + using var cache = Open(path, DeckOnly, directory); + using (var late = Open(path, Full, directory)) + late.Commit(batch => batch.Upsert(Widget, WidgetOf("late"), W)); + Assert.That(cache.Remove(W), Is.False); + Assert.Throws(() => cache.FlushAsync().GetAwaiter().GetResult()); + Assert.That(StoreOf(cache).ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "w" })); + + Assert.That(cache.Commit(batch => + { + batch.ExpectUnchanged(); + batch.Upsert(Deck, DeckOf("d2"), new CultRecordKey("d2")); + }), Is.True); + } } }