CultCache: a unique-index refusal names the least violation; shared refusals vector #99

Merged
libby merged 4 commits from hands/variants-pair-order-cs into main 2026-11-11 10:05:32 +00:00
13 changed files with 373 additions and 9 deletions

View file

@ -3495,10 +3495,11 @@ namespace GameCult.Caching
}
}
// The one index rule, judged on the commit's result whatever the write order: within one document type and one
// [CultIndex] alias, no two records hold one non-blank value. The holders of a value are the held key (unless this
// admission replaces or evicts it) plus every landing key projecting it; two distinct holders refuse, naming both
// keys and the index and never the value. One pass over what lands, one lookup per (index, value).
// The one index rule, a function of the state the admission would leave and never of staging, file or pull order:
// within one document type and one [CultIndex] alias, no two records hold one non-blank value. The holders of a
// value are the held key (unless this admission replaces or evicts it) plus every landing key projecting it. Over
// every (type, alias, value) held at two or more distinct keys, the refusal names the least by (alias, its least
// key, its second-least key, schema name), each compared by UTF-16 code unit, and never names the value.
private void RefuseIndexSharing(VariantPlan plan, IReadOnlyList<CultStoredDocument> evicted)
{
var landing = plan.Admitted.Concat(plan.Dependents).ToList();
@ -3520,17 +3521,34 @@ namespace GameCult.Caching
}
}
foreach (var ((index, value), landedKeys) in landed.OrderBy(pair => pair.Key.Index.Alias, StringComparer.Ordinal))
(string Alias, string First, string Second, string Schema)? least = null;
foreach (var ((index, value), landedKeys) in landed)
{
var holders = landedKeys.AsEnumerable();
if (_indexes.TryGetValue(index, out var held) && held.TryGetValue(value, out var heldKey) && !replaced.Contains(heldKey))
holders = holders.Append(heldKey);
var distinct = holders.Distinct(StringComparer.Ordinal).OrderBy(key => key, StringComparer.Ordinal).Take(2).ToArray();
if (distinct.Length > 1)
throw new InvalidOperationException(
$"Records {distinct[0]} and {distinct[1]} ({schemaNames[index.Type]}) " +
$"both hold unique index '{index.Alias}'; an index value names one record.");
if (distinct.Length < 2)
continue;
var candidate = (index.Alias, distinct[0], distinct[1], schemaNames[index.Type]);
if (least == null || CompareRefusals(candidate, least.Value) < 0)
least = candidate;
}
if (least is { } pair)
throw new InvalidOperationException(
$"Records {pair.First} and {pair.Second} ({pair.Schema}) " +
$"both hold unique index '{pair.Alias}'; an index value names one record.");
}
private static int CompareRefusals(
(string Alias, string First, string Second, string Schema) left,
(string Alias, string First, string Second, string Schema) right)
{
var order = StringComparer.Ordinal.Compare(left.Alias, right.Alias);
if (order == 0) order = StringComparer.Ordinal.Compare(left.First, right.First);
if (order == 0) order = StringComparer.Ordinal.Compare(left.Second, right.Second);
return order != 0 ? order : StringComparer.Ordinal.Compare(left.Schema, right.Schema);
}
private static InvalidOperationException CycleRefusal(IEnumerable<string> keys) =>

View file

@ -290,5 +290,306 @@ namespace GameCult.Caching.Tests
}
}
}
// ---- refusals.msgpack: [label, ops, refusal]; the pair a refusal names is a function of the resulting state ----
private sealed record Schema(string Name, Type Role, Type Index, string SecondField);
private static Schema Plain(string suffix, string name) => new(
name,
EmittedDocumentTypes.Emit("UqVec" + suffix + "Role", name, name + ".v1", RoleFields),
EmittedDocumentTypes.Emit("UqVec" + suffix + "Index", name, name + ".v1", IndexFields),
"Category");
private static readonly Schema[] Schemas =
{
new("tests.unique_item", ItemRole, ItemIndex, "Category"),
new("tests.uq_pair",
EmittedDocumentTypes.Emit("UqVecPairRole", "tests.uq_pair", "tests.uq_pair.v1",
new EmittedDocumentTypes.Field[] { new("Sku", typeof(string), 0, Role: "Sku"), new("Alt", typeof(string), 1, Role: "Alt") }),
EmittedDocumentTypes.Emit("UqVecPairIndex", "tests.uq_pair", "tests.uq_pair.v1",
new EmittedDocumentTypes.Field[] { new("Sku", typeof(string), 0, Index: "Sku"), new("Alt", typeof(string), 1, Index: "Alt") }),
"Alt"),
new("tests.uq_alias",
EmittedDocumentTypes.Emit("UqVecAliasRole", "tests.uq_alias", "tests.uq_alias.v1",
new EmittedDocumentTypes.Field[] { new("Sku", typeof(string), 0, Role: "Sku"), new("Alt", typeof(string), 1, Role: "alt") }),
EmittedDocumentTypes.Emit("UqVecAliasIndex", "tests.uq_alias", "tests.uq_alias.v1",
new EmittedDocumentTypes.Field[] { new("Sku", typeof(string), 0, Index: "Sku"), new("Alt", typeof(string), 1, Index: "alt") }),
"Alt"),
Plain("Alpha", "tests.uq_alpha"),
Plain("Beta", "tests.uq_beta"),
};
private static Schema SchemaNamed(string name) => Schemas.Single(schema => schema.Name == name);
private sealed record Rec(string Key, string Schema, string Sku, string Second = "");
private sealed record Op(string Kind, string? File = null, Rec[]? Records = null);
private sealed record Refusal(string K1, string K2, string Schema, string Alias);
private sealed record Case(string Label, Op[] Ops, Refusal? Refusal);
private const string Item = "tests.unique_item";
private const string Pair = "tests.uq_pair";
private const string Alias = "tests.uq_alias";
private const string Alpha = "tests.uq_alpha";
private const string Beta = "tests.uq_beta";
private static Rec[] Items_(params (string Key, string Sku)[] rows) => rows.Select(row => new Rec(row.Key, Item, row.Sku)).ToArray();
private static Op Open(string? file = null) => new("open", file);
private static Op Pull(string file) => new("pull", file);
private static Op Commit(params Rec[] records) => new("commit", Records: records);
private static readonly Rec[] PairRecords = { new("item:a", Pair, "S", "X"), new("item:b", Pair, "S", "Y"), new("item:c", Pair, "T", "Y") };
private static readonly Rec[] SurrogateRecords = Items_(("k\uFF01", "S"), ("k\U0001F600", "S"), ("k\uE000", "S"));
private static readonly Rec[] TypeRecords =
{
new("b:1", Alpha, "P"), new("a:1", Beta, "P"), new("b:2", Alpha, "P"), new("a:2", Beta, "P")
};
private static readonly Rec[] AliasRecords = { new("item:a", Alias, "S", "X"), new("item:b", Alias, "S", "Y"), new("item:c", Alias, "T", "Y") };
private static readonly Rec[] RepullBefore = Items_(("item:a1", "A"), ("item:b1", "C"));
private static readonly Rec[] RepullAfter = Items_(("item:a1", "A"), ("item:b1", "B"), ("item:b2", "B"), ("item:c1", "A"));
private static readonly Rec[] KeyCaseRecords = Items_(("a", "S"), ("z", "S"), ("B", "T"), ("C", "T"));
private static readonly Rec[] KeyPunctuationRecords = Items_(("_", "S"), ("a", "S"), ("Z", "T"), ("b", "T"));
private static readonly Rec[] SurrogateGroupRecords = Items_(("\uE000", "S"), ("\uE000b", "S"), ("\U0001F600", "T"), ("\U0001F600b", "T"));
private static readonly (string File, Rec[] Records)[] RefusalStores =
{
("pair-store.cc", PairRecords), ("surrogate-store.cc", SurrogateRecords), ("types-store.cc", TypeRecords),
("repull-before.cc", RepullBefore), ("repull-after.cc", RepullAfter), ("alias-store.cc", AliasRecords),
("key-case-store.cc", KeyCaseRecords), ("key-punctuation-store.cc", KeyPunctuationRecords),
("surrogate-groups-store.cc", SurrogateGroupRecords),
};
private static readonly Refusal SurrogateRefusal = new("k\U0001F600", "k\uE000", Item, "Sku");
private static readonly Refusal TypesRefusal = new("a:1", "a:2", Beta, "Sku");
private static readonly Refusal PairRefusal = new("item:b", "item:c", Pair, "Alt");
private static readonly Refusal AliasRefusal = new("item:a", "item:b", Alias, "Sku");
private static readonly Refusal RepullRefusal = new("item:a1", "item:c1", Item, "Sku");
private static readonly Refusal KeyCaseRefusal = new("B", "C", Item, "Sku");
private static readonly Refusal KeyPunctuationRefusal = new("Z", "b", Item, "Sku");
private static readonly Refusal SurrogateGroupRefusal = new("\U0001F600", "\U0001F600b", Item, "Sku");
private static readonly Case[] Cases =
{
new("batch-later-value-staged-first", new[] { Open(), Commit(Items_(("x", "A"), ("y", "A"), ("a", "B"), ("b", "B"))) },
new Refusal("a", "b", Item, "Sku")),
new("batch-surrogate-keys", new[] { Open(), Commit(SurrogateRecords) }, SurrogateRefusal),
new("batch-interleaved-types", new[] { Open(), Commit(TypeRecords) }, TypesRefusal),
new("batch-two-aliases", new[] { Open(), Commit(PairRecords) }, PairRefusal),
new("held-then-landing", new[] { Open(), Commit(Items_(("m", "S"))), Commit(Items_(("a", "S"))) },
new Refusal("a", "m", Item, "Sku")),
new("held-in-the-least-group", new[] { Open(), Commit(Items_(("m", "A"))), Commit(Items_(("z", "B"), ("y", "B"), ("a", "A"))) },
new Refusal("a", "m", Item, "Sku")),
new("swap-off-a-held-value", new[] { Open(), Commit(Items_(("a1", "A"))), Commit(Items_(("a1", "B"), ("c1", "A"))) }, null),
new("load-two-aliases", new[] { Open("pair-store.cc") }, PairRefusal),
new("load-surrogate-keys", new[] { Open("surrogate-store.cc") }, SurrogateRefusal),
new("load-interleaved-types", new[] { Open("types-store.cc") }, TypesRefusal),
new("repull-held-pair", new[] { Open("repull-before.cc"), Pull("repull-after.cc") }, RepullRefusal),
new("load-repull-after", new[] { Open("repull-after.cc") }, RepullRefusal),
new("commit-onto-a-loaded-holder", new[] { Open("repull-before.cc"), Commit(Items_(("item:z", "A"))) },
new Refusal("item:a1", "item:z", Item, "Sku")),
// Rows 14-19 extend the spec's floor of 13: orderings a code-point, case-insensitive or culture compare would flip.
new("batch-key-case", new[] { Open(), Commit(KeyCaseRecords) }, KeyCaseRefusal),
new("batch-key-punctuation", new[] { Open(), Commit(KeyPunctuationRecords) }, KeyPunctuationRefusal),
new("batch-key-case-in-a-group", new[] { Open(), Commit(Items_(("a", "S"), ("B", "S"))) },
new Refusal("B", "a", Item, "Sku")),
new("batch-surrogate-groups", new[] { Open(), Commit(SurrogateGroupRecords) }, SurrogateGroupRefusal),
new("batch-alias-case", new[] { Open(), Commit(AliasRecords) }, AliasRefusal),
new("load-alias-case", new[] { Open("alias-store.cc") }, AliasRefusal),
// Rows 20-22 load rows 14, 15 and 17's records, so a runtime with no batch commit still walks cross-group key order.
new("load-key-case", new[] { Open("key-case-store.cc") }, KeyCaseRefusal),
new("load-key-punctuation", new[] { Open("key-punctuation-store.cc") }, KeyPunctuationRefusal),
new("load-surrogate-groups", new[] { Open("surrogate-groups-store.cc") }, SurrogateGroupRefusal),
};
private static byte[] RefusalStoreBytes(Rec[] records)
{
var directory = Path.Combine(Path.GetTempPath(), $"cultlib-uidxr-{Guid.NewGuid():N}");
Directory.CreateDirectory(directory);
try
{
var path = Path.Combine(directory, "store.cc");
var roles = records.Select(record => SchemaNamed(record.Schema).Role).Distinct().ToArray();
using (var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions
{
Registry = CultDocumentRegistry.ForTypes(roles), StoreFlushOnDispose = true
}))
{
cache.Clock = () => StoredAt;
foreach (var record in records)
{
var schema = SchemaNamed(record.Schema);
cache.UpsertAsync(schema.Role, EmittedDocumentTypes.New(schema.Role, ("Sku", record.Sku), (schema.SecondField, record.Second)), new CultRecordKey(record.Key))
.GetAwaiter().GetResult();
}
cache.FlushAsync().GetAwaiter().GetResult();
}
return File.ReadAllBytes(path);
}
finally
{
Directory.Delete(directory, recursive: true);
}
}
private static byte[] RefusalsFile()
{
var buffer = new ArrayBufferWriter<byte>();
var writer = new MessagePackWriter(buffer);
writer.WriteArrayHeader(Cases.Length);
foreach (var row in Cases)
{
writer.WriteArrayHeader(3);
writer.Write(row.Label);
writer.WriteArrayHeader(row.Ops.Length);
foreach (var op in row.Ops)
{
writer.WriteArrayHeader(2);
writer.Write(op.Kind);
if (op.Records != null)
{
writer.WriteArrayHeader(op.Records.Length);
foreach (var record in op.Records)
{
writer.WriteArrayHeader(4);
writer.Write(record.Key);
writer.Write(record.Schema);
writer.Write(record.Sku);
writer.Write(record.Second);
}
}
else if (op.File == null) writer.WriteNil();
else writer.Write(op.File);
}
if (row.Refusal == null) writer.WriteNil();
else
{
writer.WriteArrayHeader(4);
writer.Write(row.Refusal.K1);
writer.Write(row.Refusal.K2);
writer.Write(row.Refusal.Schema);
writer.Write(row.Refusal.Alias);
}
}
writer.Flush();
return buffer.WrittenSpan.ToArray();
}
// Performs one op of a row against the open cache; the cache's own file is the only state it leaves behind.
private static void Perform(ref CultCache? cache, string path, Op op)
{
var indexTypes = Schemas.Select(schema => schema.Index).ToArray();
switch (op.Kind)
{
case "open":
if (op.File != null) File.WriteAllBytes(path, Committed(op.File));
cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = CultDocumentRegistry.ForTypes(indexTypes) });
break;
case "pull":
File.WriteAllBytes(path, Committed(op.File!));
cache!.PullAllBackingStoresAsync().GetAwaiter().GetResult();
break;
case "commit":
cache!.Commit(batch =>
{
foreach (var record in op.Records!)
{
var schema = SchemaNamed(record.Schema);
batch.Upsert(schema.Index, EmittedDocumentTypes.New(schema.Index, ("Sku", record.Sku), (schema.SecondField, record.Second)), new CultRecordKey(record.Key));
}
});
break;
default:
throw new InvalidOperationException(op.Kind);
}
}
[Test]
public void RefusalsHoldTheRule()
{
Check("refusals.msgpack", RefusalsFile());
foreach (var (file, records) in RefusalStores)
Check(file, RefusalStoreBytes(records));
// The committed file is what is walked, not the in-memory table.
var reader = new MessagePackReader(Committed("refusals.msgpack"));
var count = reader.ReadArrayHeader();
Assert.That(count, Is.EqualTo(Cases.Length));
var walked = 0;
for (var i = 0; i < count; i++)
{
Assert.That(reader.ReadArrayHeader(), Is.EqualTo(3));
var label = reader.ReadString()!;
var ops = new List<Op>();
for (var n = reader.ReadArrayHeader(); n > 0; n--)
{
Assert.That(reader.ReadArrayHeader(), Is.EqualTo(2));
var kind = reader.ReadString()!;
if (kind == "commit")
{
var records = new List<Rec>();
for (var m = reader.ReadArrayHeader(); m > 0; m--)
{
Assert.That(reader.ReadArrayHeader(), Is.EqualTo(4));
records.Add(new Rec(reader.ReadString()!, reader.ReadString()!, reader.ReadString()!, reader.ReadString()!));
}
ops.Add(new Op(kind, Records: records.ToArray()));
}
else ops.Add(new Op(kind, NextText(ref reader)));
}
Refusal? refusal = null;
if (!reader.TryReadNil())
{
Assert.That(reader.ReadArrayHeader(), Is.EqualTo(4));
refusal = new Refusal(reader.ReadString()!, reader.ReadString()!, reader.ReadString()!, reader.ReadString()!);
}
var directory = Path.Combine(Path.GetTempPath(), $"cultlib-uidxf-{Guid.NewGuid():N}");
Directory.CreateDirectory(directory);
CultCache? cache = null;
try
{
var path = Path.Combine(directory, "store.cc");
foreach (var op in ops.Take(ops.Count - 1))
Perform(ref cache, path, op);
var last = ops[^1];
if (refusal == null)
{
Assert.DoesNotThrow(() => Perform(ref cache, path, last), label);
}
else
{
cache?.FlushAsync().GetAwaiter().GetResult();
var before = last.Kind == "open" && last.File != null ? Committed(last.File)
: last.Kind == "pull" ? Committed(last.File!)
: File.Exists(path) ? File.ReadAllBytes(path) : Array.Empty<byte>();
CultCache? attempt = cache;
var refused = Assert.Throws<InvalidOperationException>(() => Perform(ref attempt, path, last), label)!;
Assert.That(refused.Message, Is.EqualTo(
$"Records {refusal.K1} and {refusal.K2} ({refusal.Schema}) both hold unique index '{refusal.Alias}'; an index value names one record."), label);
if (last.Kind != "open") cache?.FlushAsync().GetAwaiter().GetResult();
Assert.That(File.Exists(path) ? File.ReadAllBytes(path) : Array.Empty<byte>(), Is.EqualTo(before), $"{label}: the file is untouched");
}
walked++;
}
finally
{
cache?.Dispose();
Directory.Delete(directory, recursive: true);
}
}
Assert.That(walked, Is.EqualTo(22));
}
}
}

View file

@ -18,7 +18,52 @@ reviewed commit. Other runtimes are held to the same canonical text and the same
stored-at pinned to `2026-01-01T00:00:00Z`. Records `item:a` and `item:b` both hold Sku `S-1`; `item:c` holds Sku
`" "` and `item:d` holds Sku `U+0085`. Written through a role Sku.
- `blank-store.cc`: the same, only `item:c` and `item:d`.
- `pair-store.cc`, `surrogate-store.cc`, `types-store.cc`, `repull-before.cc`, `repull-after.cc`, `alias-store.cc`,
`key-case-store.cc`, `key-punctuation-store.cc`, `surrogate-groups-store.cc`: single-file stores written like
`duplicate-store.cc`, through roles, for the rows of `refusals.msgpack` that open or pull them. Schemas
`tests.unique_item`, `tests.uq_pair` (Sku and Alt both unique), `tests.uq_alias` (Sku and `alt` both unique),
`tests.uq_alpha` and `tests.uq_beta`, each slots `[0: Sku string, 1: second string]`, schema version the name plus
`.v1`. The schema ids C# writes in the store catalogs and on every record:
- `tests.unique_item`: `sha256:93bb890de83069061af696b23f94fd32fb3c2dad3c91ee645e8add56085f1882`
- `tests.uq_pair`: `sha256:2538cceafb6dcbce22ef11b932ecb804af5f96bc3aaedff692a52d3640433e60`
- `tests.uq_alias`: `sha256:77a77a394ebd8bf71e14b9b6f2cea4978c25d4e705fd12364f660edbf6c63b1f`
- `tests.uq_alpha`: `sha256:53aeaf2ea9b25e85ade6b15fc04c7653e7b206c87e258f1aa74dd32a840a1389`
- `tests.uq_beta`: `sha256:45ed62a64c91f6d815d3ec975c4a4a1b2ce07b6e7262a32d828673cf10899586`
- `outcomes.msgpack`: rows `[file, unique alias or nil, open|refused, k1 or nil, k2 or nil]`. Opening `file` with that
alias a unique index (nil: a role, no unique alias) yields the verdict. A refusal reads, byte for byte,
`Records <k1> and <k2> (tests.unique_item) both hold unique index '<alias>'; an index value names one record.`
with `k1 < k2` ordinal, never contains the value, and leaves the file byte-identical.
## refusals.msgpack
RULE: a unique-index refusal is a function of the state the admission would leave, never of staging, file or pull
order. Over every (type, unique alias, non-blank value) that state holds at two or more distinct keys, name the one
least by (alias, its least key, its second-least key, schema name), each compared by UTF-16 code unit (C#
`StringComparer.Ordinal`): `Records <k1> and <k2> (<schema>) both hold unique index '<alias>'; an index value names one
record.` with `k1 < k2` its two least keys. One rule for a single write, a batch, a load, a re-pull and an index
registration.
Rows `[label, ops, refusal]`. Each op is `["open", file-or-nil]` (copy that vector file to a fresh path, or nil for a
fresh path with no file, and open a cache on it), `["pull", file]` (overwrite the open cache's file with that vector
file, then pull every backing store again) or `["commit", [[key, schema, sku, second], ...]]` (one batch, staged in row
order; schema is the schema name). `refusal` is nil (every op succeeds) or `[k1, k2, schema, alias]`: every op but the
last succeeds and the last refuses with exactly the RULE text built from those fields, leaving the file
byte-identical. Schemas are those of the store files above; `second` is `Category` (plain) except in `tests.uq_pair`
and `tests.uq_alias`, where it is `Alt` (unique, alias `Alt` and `alt` respectively).
The 13 rows of the spec are a floor; rows 14-22 extend it with orderings that a code-point, case-insensitive or
culture compare would flip. Rows, in file order: 1 batch-later-value-staged-first, 2 batch-surrogate-keys, 3
batch-interleaved-types, 4 batch-two-aliases, 5 held-then-landing, 6 held-in-the-least-group, 7 swap-off-a-held-value,
8 load-two-aliases, 9 load-surrogate-keys, 10 load-interleaved-types, 11 repull-held-pair, 12 load-repull-after, 13
commit-onto-a-loaded-holder, 14 batch-key-case (`B` before `a`), 15 batch-key-punctuation (`Z` before `_` before `a`),
16 batch-key-case-in-a-group, 17 batch-surrogate-groups (U+1F600 before U+E000, the reverse of code-point order), 18
batch-alias-case (alias `Sku` before `alt`), 19 load-alias-case, 20 load-key-case, 21 load-key-punctuation, 22
load-surrogate-groups. Rows 19-22 open a store holding the records of rows 18, 14, 15 and 17, so a runtime with no batch
commit still walks alias and cross-group key order.
A runtime performs every row its API can perform and asserts the count it walked:
- C# and Rust: all 22.
- Python: 21, every row but 3 (`put_envelopes` takes one type).
- TypeScript: 11, rows 5, 8, 9, 10, 11, 12, 13, 19, 20, 21 and 22 (no batch commit, so no row 1-4, 6, 7 or 14-18;
`variants:idea:unique-index-runtimes.ts-batch-commit`).

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.