From 9b6253660b080c3a97454e0070ac4d180237bdeb Mon Sep 17 00:00:00 2001 From: Libby Date: Wed, 11 Nov 2026 04:29:15 +0000 Subject: [PATCH] cultcache-ts: indexes are unique at commit and load, roles name selection aliases, canonical index text Deletes the last-writer-wins derive and String(value) rendering; one judge (#refuseIndexSharing) for every write, load and registration. Co-Authored-By: Claude Sonnet 5.5 --- docs/start/README.md | 1 - packages/cultcache-ts/src/cult-cache.ts | 87 +++- packages/cultcache-ts/src/document.ts | 38 +- packages/cultcache-ts/src/types.ts | 12 + packages/cultcache-ts/test/first-call.test.ts | 1 - .../cultcache-ts/test/unique-index.test.ts | 403 ++++++++++++++++++ 6 files changed, 514 insertions(+), 28 deletions(-) create mode 100644 packages/cultcache-ts/test/unique-index.test.ts diff --git a/docs/start/README.md b/docs/start/README.md index c388f8b6..e52f7d73 100644 --- a/docs/start/README.md +++ b/docs/start/README.md @@ -94,7 +94,6 @@ const itemDocument = defineDocumentType({ schemaVersion: "gamecult.first_call_item.v1", schema: z.object({ name: z.string(), category: z.string(), value: z.number().int() }), name: "name", - indexes: { category: "category" }, }); const registry = defineDocumentRegistry(itemDocument); diff --git a/packages/cultcache-ts/src/cult-cache.ts b/packages/cultcache-ts/src/cult-cache.ts index 88ab94e9..b298b9a3 100644 --- a/packages/cultcache-ts/src/cult-cache.ts +++ b/packages/cultcache-ts/src/cult-cache.ts @@ -224,7 +224,7 @@ export class CultCache { return this.#serial(() => { const registered = this.#requireDefinition(definition); const nameAccessor = this.#compileAccessor(definition.type, "name", accessor); - const derived = this.#deriveTypeLookups(definition.type, nameAccessor, registered.indexAccessors); + const derived = this.#deriveTypeLookups(definition.type, registered.catalogEntry.schemaName, nameAccessor, registered.indexAccessors); this.#installAccessors(registered, nameAccessor, registered.indexAccessors, derived); return definition; }); @@ -243,7 +243,7 @@ export class CultCache { const registered = this.#requireDefinition(definition); const indexAccessors = new Map(registered.indexAccessors); indexAccessors.set(indexName, this.#compileAccessor(definition.type, `index "${indexName}"`, accessor)); - const derived = this.#deriveTypeLookups(definition.type, registered.nameAccessor, indexAccessors); + const derived = this.#deriveTypeLookups(definition.type, registered.catalogEntry.schemaName, registered.nameAccessor, indexAccessors); this.#installAccessors(registered, registered.nameAccessor, indexAccessors, derived); return definition; }); @@ -418,6 +418,7 @@ export class CultCache { return key ? this.get(definition, key) : undefined; } + // A unique index: one key per non-blank value. A roles alias is never indexed and answers undefined. getKeyByIndex( definition: TDefinition, indexName: string, @@ -579,7 +580,7 @@ export class CultCache { if (existingBySchemaName && existingBySchemaName !== registered) { throw refuse(existingBySchemaName, registered.catalogEntry, `schema name "${registered.catalogEntry.schemaName}"`); } - const derived = this.#deriveTypeLookups(definition.type, nameAccessor, indexAccessors); + const derived = this.#deriveTypeLookups(definition.type, registered.catalogEntry.schemaName, nameAccessor, indexAccessors); // Every check and accessor has run; install. this.#definitions.set(definition.type, registered); @@ -795,11 +796,13 @@ export class CultCache { switch (typeof value) { case "string": - return value; - case "number": + return isBlankIndexText(value) ? undefined : value; case "boolean": + return value ? "True" : "False"; case "bigint": - return String(value); + return value.toString(); + case "number": + return canonicalNumberText(value); default: throw new Error(`CultCache ${label} accessor for type "${type}" produced a non-scalar value.`); } @@ -894,6 +897,8 @@ export class CultCache { } } + const lookups = this.#lookupsFor(registered.nameAccessor, registered.indexAccessors, value); + this.#refuseIndexSharing(state, entry.type, registered.catalogEntry.schemaName, entry.key, lookups); return { ...entry, payload: this.#cloneBytes(entry.payload), @@ -902,10 +907,22 @@ export class CultCache { schemaId: registered.catalogEntry.schemaId, catalogEntry: registered.catalogEntry, value, - lookups: this.#lookupsFor(registered.nameAccessor, registered.indexAccessors, value), + lookups, }; } + // The one index rule (variants:ruling:unique-indexes): within one type and one unique alias, no two records hold one + // non-blank value. The held key at the landing record's own key is its previous self and keeps its value. The message + // names both keys, ordinal, and the alias, never the value. + #refuseIndexSharing(state: HydratedState, type: string, schemaName: string, key: string, lookups: EntryLookups): void { + for (const alias of [...lookups.indexes.keys()].sort(compareOrdinal)) { + const held = state.indexToKeyMaps.get(this.#indexId(type, alias))?.get(lookups.indexes.get(alias)!); + if (held !== undefined && held !== key) { + throw indexSharingRefusal(held, key, schemaName, alias); + } + } + } + // Infallible half of admission: replaces any held record at the same key and installs the admitted one. #installEntry(state: HydratedState, registered: RegisteredDefinition, hydrated: HydratedEntry): void { const entryId = this.#entryId(hydrated.type, hydrated.key); @@ -966,12 +983,14 @@ export class CultCache { // Runs the candidate accessors over every held record of the type into local maps. #deriveTypeLookups( type: string, + schemaName: string, nameAccessor: Accessor | undefined, indexAccessors: ReadonlyMap, ): DerivedTypeLookups { const entryLookups: Array<[HydratedEntry, EntryLookups]> = []; const nameMap = nameAccessor ? new Map() : undefined; const indexMaps = new Map>(); + const sharing = new Map(); for (const indexName of indexAccessors.keys()) { indexMaps.set(indexName, new Map()); } @@ -983,10 +1002,23 @@ export class CultCache { nameMap?.set(lookups.name, entry.key); } for (const [indexName, indexValue] of lookups.indexes) { - indexMaps.get(indexName)?.set(indexValue, entry.key); + const map = indexMaps.get(indexName)!; + const holder = map.get(indexValue); + if (holder === undefined) { + map.set(indexValue, entry.key); + } else { + sharing.set(`${indexName}\u0000${indexValue}`, [...(sharing.get(`${indexName}\u0000${indexValue}`) ?? [holder]), entry.key]); + } } } + // Judged whole: any value held by two records refuses, naming the two smallest keys of the first alias in ordinal order. + const shared = [...sharing].map(([id, keys]) => ({ alias: id.slice(0, id.indexOf("\u0000")), keys: keys.sort(compareOrdinal) })) + .sort((a, b) => compareOrdinal(a.alias, b.alias)); + if (shared.length > 0) { + throw indexSharingRefusal(shared[0].keys[0], shared[0].keys[1], schemaName, shared[0].alias); + } + return { entryLookups, nameMap, indexMaps }; } @@ -1057,3 +1089,42 @@ export class CultCache { return documentPayloadFromBytes(new Uint8Array(documentPayloadBytes(payload))); } } + +function compareOrdinal(a: string, b: string): number { + return a < b ? -1 : a > b ? 1 : 0; +} + +function indexSharingRefusal(keyA: string, keyB: string, schemaName: string, alias: string): Error { + const [first, second] = compareOrdinal(keyA, keyB) <= 0 ? [keyA, keyB] : [keyB, keyA]; + return new Error(`Records ${first} and ${second} (${schemaName}) both hold unique index '${alias}'; an index value names one record.`); +} + +// Blank is no value: empty or only C# White_Space characters. Not JavaScript's \s, which counts U+FEFF and omits U+0085. +const BLANK_INDEX_TEXT = /^[\t-\r \u0085\u00A0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000]*$/; + +function isBlankIndexText(value: string): boolean { + return BLANK_INDEX_TEXT.test(value); +} + +// The canonical text of a number, as C# RenderCanonicalNumber: integers as their digits (-0 is 0), a non-integral finite +// double as its exact decimal expansion, NaN and infinities as no value. +function canonicalNumberText(value: number): string | undefined { + if (!Number.isFinite(value)) { + return undefined; + } + if (Number.isInteger(value)) { + return BigInt(value).toString(); + } + + // value = mantissa * 2^exponent with exponent < 0, so value = mantissa * 5^k / 10^k for k = -exponent. + const view = new DataView(new ArrayBuffer(8)); + view.setFloat64(0, value); + const bits = view.getBigUint64(0); + const biasedExponent = Number((bits >> 52n) & 0x7ffn); + const fraction = bits & 0xfffffffffffffn; + const mantissa = biasedExponent === 0 ? fraction : fraction | (1n << 52n); + const k = -((biasedExponent === 0 ? 1 : biasedExponent) - 1075); + const digits = (mantissa * 5n ** BigInt(k)).toString().padStart(k + 1, "0"); + const text = `${digits.slice(0, digits.length - k)}.${digits.slice(digits.length - k)}`.replace(/0+$/, "").replace(/\.$/, ""); + return value < 0 ? `-${text}` : text; +} diff --git a/packages/cultcache-ts/src/document.ts b/packages/cultcache-ts/src/document.ts index 99ec170c..bc4ace88 100644 --- a/packages/cultcache-ts/src/document.ts +++ b/packages/cultcache-ts/src/document.ts @@ -32,23 +32,22 @@ export function defineDocumentType( throw new Error(`CultCache document type "${definition.type}" declares an invalid name accessor.`); } - if (Array.isArray(definition.indexes)) { - const seen = new Set(); - for (const index of definition.indexes) { - if (!index.name || index.name.trim().length === 0) { - throw new Error(`CultCache document type "${definition.type}" declares an index with an empty name.`); - } - if (seen.has(index.name)) { - throw new Error(`CultCache document type "${definition.type}" declares duplicate index "${index.name}".`); - } - seen.add(index.name); - } - } else if (definition.indexes) { - for (const name of Object.keys(definition.indexes)) { + const declared = new Set(); + for (const [kind, set] of [["index", definition.indexes], ["role", definition.roles]] as const) { + const names = new Set(); + for (const name of Array.isArray(set) ? set.map((entry) => entry.name) : set ? Object.keys(set) : []) { if (!name || name.trim().length === 0) { - throw new Error(`CultCache document type "${definition.type}" declares an index with an empty name.`); + throw new Error(`CultCache document type "${definition.type}" declares ${kind === "index" ? "an index" : "a role"} with an empty name.`); } + if (names.has(name)) { + throw new Error(`CultCache document type "${definition.type}" declares duplicate ${kind} "${name}".`); + } + if (kind === "role" && declared.has(name)) { + throw new Error(`CultCache document type "${definition.type}" declares "${name}" as both an index and a role; an index already names its role.`); + } + names.add(name); } + for (const name of names) declared.add(name); } if (definition.members) { @@ -138,10 +137,13 @@ export function resolveDocumentMembers(definition: AnyCultCacheDocumentDefinitio return definition.members ?? []; } const codec = definition.codec; - const indexAlias = (slotName: string) => - !Array.isArray(definition.indexes) && definition.indexes - ? Object.entries(definition.indexes).find(([, accessor]) => accessor === slotName)?.[0] ?? null - : null; + const indexAlias = (slotName: string) => { + for (const set of [definition.indexes, definition.roles]) { + const alias = !Array.isArray(set) && set ? Object.entries(set).find(([, accessor]) => accessor === slotName)?.[0] : undefined; + if (alias !== undefined) return alias; + } + return null; + }; return codec.members.map((member, index) => ({ ...member, isName: (definition.name as unknown) === codec.names[index], diff --git a/packages/cultcache-ts/src/types.ts b/packages/cultcache-ts/src/types.ts index 88c20f2e..6f6c0923 100644 --- a/packages/cultcache-ts/src/types.ts +++ b/packages/cultcache-ts/src/types.ts @@ -59,9 +59,21 @@ export interface CultCacheDocumentDefinition>; + /** + * Unique lookup aliases. A non-blank value names one record of the type: a write that would leave two records holding + * one value, and a store that already holds two, is refused naming both keys and the alias. Blank values (absent, + * empty, or only whitespace) are never indexed. `getByIndex` and `getKeyByIndex` answer from these. + */ indexes?: | Readonly>>> | readonly CultCacheDocumentIndexDefinition>[]; + /** + * Selection and reference aliases: the member's catalog `indexAlias`. A role implies nothing about uniqueness, many + * records may share a value, and it is never indexed for lookup. An alias is declared in `indexes` or `roles`, not both. + */ + roles?: + | Readonly>>> + | readonly CultCacheDocumentIndexDefinition>[]; formatter?: CultCacheDocumentFormatter>; codec?: SlotsCodec & object>; } diff --git a/packages/cultcache-ts/test/first-call.test.ts b/packages/cultcache-ts/test/first-call.test.ts index 3c0e22e5..45f6edd2 100644 --- a/packages/cultcache-ts/test/first-call.test.ts +++ b/packages/cultcache-ts/test/first-call.test.ts @@ -24,7 +24,6 @@ const itemDocument = defineDocumentType({ schemaVersion: "gamecult.first_call_item.v1", schema: z.object({ name: z.string(), category: z.string(), value: z.number().int() }), name: "name", - indexes: { category: "category" }, }); const registry = defineDocumentRegistry(itemDocument); diff --git a/packages/cultcache-ts/test/unique-index.test.ts b/packages/cultcache-ts/test/unique-index.test.ts new file mode 100644 index 00000000..db7fe875 --- /dev/null +++ b/packages/cultcache-ts/test/unique-index.test.ts @@ -0,0 +1,403 @@ +import assert from "node:assert/strict"; +import { existsSync } from "node:fs"; +import { copyFile, mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { test } from "node:test"; + +import { decode } from "@msgpack/msgpack"; +import { z } from "zod"; + +import { codecs } from "../src"; +import { CultCache } from "../src/cult-cache"; +import { defineDocumentType, resolveDocumentMembers } from "../src/document"; +import { SingleFileMessagePackBackingStore } from "../src/single-file-messagepack-backing-store"; +import type { AnyCultCacheDocumentDefinition } from "../src/types"; + +// variants:ruling:unique-indexes. `indexes` are unique within one type, `roles` are selection aliases. The vectors in +// tests/vectors/unique-index are written by the C# reference; the rest of the behaviour is pinned here. + +function findVectors(from: string): string { + for (let dir = from; ; dir = resolve(dir, "..")) { + if (existsSync(join(dir, "tests", "vectors", "unique-index"))) return join(dir, "tests", "vectors", "unique-index"); + if (resolve(dir, "..") === dir) throw new Error("tests/vectors/unique-index not found above the test."); + } +} +const vectors = findVectors(__dirname); + +const CANARY = "CANARY-uidx-5e1b"; + +const skuSchema = z.object({ sku: z.any().optional(), group: z.string().optional() }); + +function item(type: string, where: "indexes" | "roles" | "none" = "indexes", schemaName = `tests.${type}`) { + return defineDocumentType({ + type, + schemaId: `${schemaName}.v1`, + schemaName, + schemaVersion: `${schemaName}.v1`, + schema: skuSchema, + ...(where === "none" ? {} : { [where]: { sku: "sku" } }), + }); +} + +// A definition whose payload is never read back, so a value of any scalar kind (a bigint included) can be held and indexed. +function scalar(type: string) { + const held: unknown[] = []; + return defineDocumentType({ + type, + schemaId: `tests.${type}.v1`, + schemaName: `tests.${type}`, + schemaVersion: `tests.${type}.v1`, + schema: skuSchema, + formatter: { + encode: (value: unknown) => new Uint8Array([held.push(value) - 1]), + decode: (payload: Uint8Array) => held[payload[0]] as never, + }, + indexes: { sku: "sku" }, + }); +} + +function memory(...definitions: AnyCultCacheDocumentDefinition[]) { + let builder = CultCache.builder(); + for (const definition of definitions) builder = builder.withDocumentType(definition); + return builder.build(); +} + +function refusal(keyA: string, keyB: string, schemaName: string, alias: string) { + return `Records ${keyA} and ${keyB} (${schemaName}) both hold unique index '${alias}'; an index value names one record.`; +} + +test("a put of a held value at another key is refused naming both keys and the alias, and nothing moves", async () => { + const dir = await mkdtemp(join(tmpdir(), "uidx-")); + try { + const file = join(dir, "items.cc"); + const document = item("item"); + const cache = CultCache.builder().withDocumentType(document).withGenericStore(new SingleFileMessagePackBackingStore(file)).build(); + await cache.pullAllBackingStores(); + await cache.put(document, "zeta", { sku: CANARY }); + const before = await readFile(file); + const snapshot = cache.snapshot(); + + await assert.rejects(cache.put(document, "alpha", { sku: CANARY }), (error: Error) => { + assert.equal(error.message, refusal("alpha", "zeta", "tests.item", "sku")); + assert.ok(!error.message.includes(CANARY), "the value is never echoed"); + return true; + }); + assert.deepEqual(await readFile(file), before, "the store is byte-identical"); + assert.deepEqual(cache.snapshot(), snapshot); + assert.equal(cache.get(document, "alpha"), undefined); + assert.equal(cache.getKeyByIndex(document, "sku", CANARY), "zeta"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("rewriting a record's own key with its own value lands", async () => { + const document = item("item"); + const cache = memory(document); + await cache.put(document, "a", { sku: "S", group: "one" }); + await cache.put(document, "a", { sku: "S", group: "two" }); + assert.equal(cache.getKeyByIndex(document, "sku", "S"), "a"); + assert.equal(cache.get(document, "a")?.group, "two"); +}); + +test("a removed or changed holder releases its value", async () => { + const document = item("item"); + const cache = memory(document); + await cache.put(document, "b", { sku: "S" }); + await assert.rejects(cache.put(document, "a", { sku: "S" }), /Records a and b/u); + assert.equal(cache.getKeyByIndex(document, "sku", "S"), "b", "the refused write did not take the value"); + + await cache.delete(document, "b"); + assert.equal(cache.getKeyByIndex(document, "sku", "S"), undefined); + await cache.put(document, "a", { sku: "S" }); + assert.equal(cache.getKeyByIndex(document, "sku", "S"), "a"); + + await cache.put(document, "a", { sku: "T" }); + await cache.put(document, "b", { sku: "S" }); + assert.equal(cache.getKeyByIndex(document, "sku", "S"), "b"); + assert.equal(cache.getKeyByIndex(document, "sku", "T"), "a"); +}); + +test("the keys in a refusal are in ordinal order whichever one lands second", async () => { + const document = item("item"); + const cache = memory(document); + await cache.put(document, "m", { sku: "S" }); + await assert.rejects(cache.put(document, "z", { sku: "S" }), { message: refusal("m", "z", "tests.item", "sku") }); + await assert.rejects(cache.put(document, "B", { sku: "S" }), { message: refusal("B", "m", "tests.item", "sku") }); +}); + +// The C# White_Space set, and nothing else, is blank. Every UTF-16 code unit is judged, so a rule written for a few +// characters (or for JavaScript's \s, which counts U+FEFF and omits U+0085) cannot pass. +const CSHARP_WHITE_SPACE = (code: number) => + (code >= 0x09 && code <= 0x0d) || code === 0x20 || code === 0x85 || code === 0xa0 || code === 0x1680 || + (code >= 0x2000 && code <= 0x200a) || code === 0x2028 || code === 0x2029 || code === 0x202f || code === 0x205f || code === 0x3000; + +test("blank is exactly the C# White_Space set: every code unit, alone, repeated and padded", async () => { + const document = item("item"); + const cache = memory(document); + const disagreements: string[] = []; + for (let code = 0; code <= 0xffff; code++) { + if (code >= 0xd800 && code <= 0xdfff) continue; // lone surrogates do not survive MessagePack text + const character = String.fromCharCode(code); + await cache.put(document, "x", { sku: character }); + let blank = true; + try { + await cache.put(document, "y", { sku: character }); + } catch { + blank = false; + } + if (blank !== CSHARP_WHITE_SPACE(code)) disagreements.push(code.toString(16)); + await cache.delete(document, "x"); + await cache.delete(document, "y"); + } + assert.deepEqual(disagreements, []); + + // Not trimmed: a padded value is the value, distinct from its trimmed form. + await cache.put(document, "x", { sku: " a " }); + await cache.put(document, "y", { sku: "a" }); + assert.equal(cache.getKeyByIndex(document, "sku", " a "), "x"); + assert.equal(cache.getKeyByIndex(document, "sku", "a"), "y"); + await assert.rejects(cache.put(document, "z", { sku: " a " }), /Records x and z/u); +}); + +test("blank values are never indexed and never conflict: empty, spaces, absent", async () => { + const document = item("item"); + const cache = memory(document); + for (const [key, sku] of [["a", ""], ["b", " "], ["c", "\u0085"], ["d", undefined], ["e", " "]] as const) { + await cache.put(document, key, { sku }); + } + assert.equal(cache.getKeyByIndex(document, "sku", ""), undefined); + assert.equal(cache.getKeyByIndex(document, "sku", " "), undefined); + // U+FEFF and U+001F are values. + await cache.put(document, "f", { sku: "" }); + await assert.rejects(cache.put(document, "g", { sku: "" }), /Records f and g/u); + await cache.put(document, "h", { sku: "\u001F" }); + await assert.rejects(cache.put(document, "i", { sku: "\u001F" }), /Records h and i/u); +}); + +test("two types may share an alias and a value", async () => { + const first = item("first"); + const second = item("second"); + const cache = memory(first, second); + await cache.put(first, "a", { sku: "S" }); + await cache.put(second, "b", { sku: "S" }); + assert.equal(cache.getKeyByIndex(first, "sku", "S"), "a"); + assert.equal(cache.getKeyByIndex(second, "sku", "S"), "b"); +}); + +test("roles may share values and are not looked up", async () => { + const document = item("item", "roles"); + const cache = memory(document); + await cache.put(document, "a", { sku: "S" }); + await cache.put(document, "b", { sku: "S" }); + await cache.put(document, "c", { sku: "S" }); + assert.equal(cache.getKeyByIndex(document, "sku", "S"), undefined); + assert.equal(cache.getByIndex(document, "sku", "S"), undefined); +}); + +test("an alias declared in indexes and roles is refused at definition; each map is checked like indexes", () => { + const base = { type: "bad", schema: skuSchema, schemaId: "tests.bad.v1", schemaName: "tests.bad", schemaVersion: "tests.bad.v1" }; + assert.throws( + () => defineDocumentType({ ...base, indexes: { sku: "sku" }, roles: { sku: "sku" } }), + { message: 'CultCache document type "bad" declares "sku" as both an index and a role; an index already names its role.' }, + ); + assert.throws(() => defineDocumentType({ ...base, roles: { " ": "sku" } }), /declares a role with an empty name/u); + assert.throws( + () => defineDocumentType({ ...base, roles: [{ name: "r", accessor: "sku" }, { name: "r", accessor: "group" }] }), + /declares duplicate role "r"/u, + ); + assert.doesNotThrow(() => defineDocumentType({ ...base, indexes: { sku: "sku" }, roles: { group: "group" } })); +}); + +test("a codec definition's catalog entry is identical with an alias under indexes or roles", async () => { + const codec = codecs.slots<{ sku: string; group: string }>({ 0: ["sku", codecs.string], 1: ["group", codecs.string] }, "T.Item"); + const make = (where: "indexes" | "roles") => + defineDocumentType({ + type: "codec_item", + schemaId: "tests.codec_item.v1", + schemaName: "tests.codec_item", + schemaVersion: "tests.codec_item.v1", + schema: z.object({ sku: z.string(), group: z.string() }), + codec, + [where]: { sku: "sku" }, + }); + const catalog = async (definition: AnyCultCacheDocumentDefinition) => { + const cache = memory(definition); + await cache.put(definition, "a", { sku: "S", group: "g" } as never); + return cache.snapshot()[0].catalogEntry!; + }; + const members = (definition: AnyCultCacheDocumentDefinition) => resolveDocumentMembers(definition); + assert.deepEqual(members(make("indexes")), members(make("roles"))); + assert.equal(members(make("roles")).find((member) => member.memberName === "Sku")?.indexAlias, "sku"); + const [unique, role] = [await catalog(make("indexes")), await catalog(make("roles"))]; + assert.equal(unique.canonicalSchemaJson, role.canonicalSchemaJson); + assert.equal(unique.contentHash, role.contentHash); + assert.equal(unique.schemaId, role.schemaId); +}); + +test("vectors values.msgpack: every row's canonical text, blank rows no value", async () => { + const rows = decode(await readFile(join(vectors, "values.msgpack")), { useBigInt64: true }) as [string, string, unknown, string | null][]; + assert.ok(rows.length >= 20); + for (const [label, kind, value, text] of rows) { + const document = scalar(`value_${label.replaceAll(/[^a-z0-9]/gu, "_")}`); + const cache = memory(document); + void kind; + const normalized = value; + await cache.put(document, "one", { sku: normalized }); + if (text === null) { + await cache.put(document, "two", { sku: normalized }); + assert.equal(cache.get(document, "two") !== undefined, true, `${label}: a blank value commits twice`); + for (const probe of ["", " ", "NaN", "0", "Infinity", "-Infinity", "null", "undefined"]) { + assert.equal(cache.getKeyByIndex(document, "sku", probe), undefined, `${label}: found by '${probe}'`); + } + } else { + assert.equal(cache.getKeyByIndex(document, "sku", text), "one", `${label}: found by its text`); + await assert.rejects(cache.put(document, "two", { sku: normalized }), /Records one and two/u, label); + } + } +}); + +// An independent oracle for the double rule: the text must denote exactly the double, so N / 10^d == mantissa * 2^exponent +// in exact integer arithmetic, with no trailing zero and no bare point. A rule written to a few spellings cannot pass. +function exactlyDenotes(text: string, value: number): boolean { + const match = /^(-?)(\d+)(?:\.(\d*[1-9]))?$/u.exec(text); + if (!match) return false; + const decimals = match[3]?.length ?? 0; + const numerator = BigInt(match[2] + (match[3] ?? "")); + const view = new DataView(new ArrayBuffer(8)); + view.setFloat64(0, Math.abs(value)); + const bits = view.getBigUint64(0); + const biased = Number((bits >> 52n) & 0x7ffn); + const mantissa = biased === 0 ? bits & 0xfffffffffffffn : (bits & 0xfffffffffffffn) | (1n << 52n); + const exponent = (biased === 0 ? 1 : biased) - 1075; + const lhs = numerator * (exponent >= 0 ? 1n : 2n ** BigInt(-exponent)); + const rhs = mantissa * (exponent >= 0 ? 2n ** BigInt(exponent) : 1n) * 10n ** BigInt(decimals); + return (match[1] === "-") === (value < 0) && lhs === rhs; +} + +test("a number's index text is its exact value: integers as digits, doubles as the exact expansion, NaN and infinities none", async () => { + const document = scalar("number"); + const cache = memory(document); + const probes = [0.1, -0.1, 2.5e-7, 1.5, -123.456, 5e-324, 2.2250738585072014e-308, 1e-10, 123456789.123456789, Number.MIN_VALUE * 3, + ...Array.from({ length: 200 }, (_, i) => Math.sin(i + 1) * 10 ** ((i % 40) - 20)), 1e21 + 2 ** 20 + 0.5 * 0]; + // Round trip through lookups: the exact expansion of each probe is found, and it denotes the probe exactly. + for (const value of probes) { + if (Number.isInteger(value)) continue; + await cache.put(document, "k", { sku: value }); + const candidates = candidateTexts(value); + const found = candidates.filter((candidate) => cache.getKeyByIndex(document, "sku", candidate) === "k"); + assert.equal(found.length, 1, `${value}: exactly one spelling is its index text`); + assert.ok(exactlyDenotes(found[0], value), `${value}: ${found[0]} denotes it exactly`); + } + + await cache.put(document, "k", { sku: 1e21 }); + assert.equal(cache.getKeyByIndex(document, "sku", "1000000000000000000000"), "k"); + await cache.put(document, "k", { sku: -0 }); + assert.equal(cache.getKeyByIndex(document, "sku", "0"), "k"); + await cache.put(document, "k", { sku: 12n }); + assert.equal(cache.getKeyByIndex(document, "sku", "12"), "k"); + await cache.put(document, "k", { sku: true }); + assert.equal(cache.getKeyByIndex(document, "sku", "True"), "k"); + await cache.put(document, "k", { sku: false }); + assert.equal(cache.getKeyByIndex(document, "sku", "False"), "k"); + for (const value of [Number.NaN, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY]) { + await cache.put(document, "p", { sku: value }); + await cache.put(document, "q", { sku: value }); + } +}); + +// Spellings a lookup could be probed with: the exact decimal expansion computed by an arbitrary-precision route. +function candidateTexts(value: number): string[] { + const view = new DataView(new ArrayBuffer(8)); + view.setFloat64(0, Math.abs(value)); + const bits = view.getBigUint64(0); + const biased = Number((bits >> 52n) & 0x7ffn); + const mantissa = biased === 0 ? bits & 0xfffffffffffffn : (bits & 0xfffffffffffffn) | (1n << 52n); + const k = 1075 - (biased === 0 ? 1 : biased); + const scaled = (mantissa * 5n ** BigInt(k)).toString().padStart(k + 1, "0"); + const point = scaled.length - k; + const text = `${scaled.slice(0, point)}.${scaled.slice(point)}`.replace(/0+$/u, "").replace(/\.$/u, ""); + return [(value < 0 ? "-" : "") + text, String(value), value.toString()].filter((spelling, i, all) => all.indexOf(spelling) === i); +} + +function twoDefinitions(schemaId: string, where: "indexes" | "roles") { + return defineDocumentType({ + type: "unique_item", + schemaId, + schemaName: "tests.unique_item", + schemaVersion: "tests.unique_item.v1", + schema: z.object({ sku: z.string(), category: z.string() }), + codec: codecs.slots<{ sku: string; category: string }>({ 0: ["sku", codecs.string], 1: ["category", codecs.string] }, "UqVecItem"), + [where]: { Sku: "sku" }, + }); +} + +test("vectors duplicate-store.cc and blank-store.cc hold their verdicts", async () => { + const outcomes = decode(await readFile(join(vectors, "outcomes.msgpack"))) as [string, string | null, string, string | null, string | null][]; + assert.equal(outcomes.length, 3); + const dir = await mkdtemp(join(tmpdir(), "uidx-vec-")); + try { + for (const [name, alias, outcome, key1, key2] of outcomes) { + const file = join(dir, `${outcomes.indexOf(outcomes.find((row) => row[0] === name && row[1] === alias)!)}-${name}`); + await copyFile(join(vectors, name), file); + const before = await readFile(file); + const [envelope] = await new SingleFileMessagePackBackingStore(file).pullAll(); + const document = twoDefinitions(envelope.schemaId!, alias === null ? "roles" : "indexes"); + const cache = CultCache.builder().withDocumentType(document).withGenericStore(new SingleFileMessagePackBackingStore(file)).build(); + const stateBefore = cache.snapshot(); + if (outcome === "open") { + await cache.pullAllBackingStores(); + assert.equal(cache.snapshot().length, name === "blank-store.cc" ? 2 : 4); + if (alias === null) assert.equal(cache.getKeyByIndex(document, "Sku", "S-1"), undefined); + } else { + await assert.rejects(cache.pullAllBackingStores(), (error: Error) => { + assert.equal(error.message, refusal(key1!, key2!, "tests.unique_item", alias!)); + assert.ok(!error.message.includes("S-1")); + return true; + }); + assert.deepEqual(cache.snapshot(), stateBefore, "a refused load admits nothing"); + } + assert.deepEqual(await readFile(file), before, `${name} is untouched`); + } + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("a refused load leaves the cache's prior state as it was, and a later good load still works", async () => { + const dir = await mkdtemp(join(tmpdir(), "uidx-load-")); + try { + const bad = join(dir, "bad.cc"); + await copyFile(join(vectors, "duplicate-store.cc"), bad); + const [envelope] = await new SingleFileMessagePackBackingStore(bad).pullAll(); + const document = twoDefinitions(envelope.schemaId!, "indexes"); + const cache = CultCache.builder().withDocumentType(document).withGenericStore(new SingleFileMessagePackBackingStore(bad)).build(); + await assert.rejects(cache.pullAllBackingStores(), /Records item:a and item:b/u); + assert.equal(cache.get(document, "item:a"), undefined); + assert.equal(cache.getKeyByIndex(document, "Sku", "S-1"), undefined); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("registerIndex over held duplicates installs nothing", async () => { + const document = item("item", "roles"); + const cache = memory(document); + await cache.put(document, "a", { sku: "S", group: "g" }); + await cache.put(document, "b", { sku: "S", group: "h" }); + await assert.rejects(cache.registerIndex(document, "sku", "sku"), { message: refusal("a", "b", "tests.item", "sku") }); + assert.equal(cache.getKeyByIndex(document, "sku", "S"), undefined, "no map was installed"); + await cache.put(document, "c", { sku: "S", group: "i" }); + await cache.registerIndex(document, "group", "group"); + assert.equal(cache.getKeyByIndex(document, "group", "h"), "b"); + await assert.rejects(cache.put(document, "d", { sku: "x", group: "h" }), /Records b and d/u); +}); + +test("registerIndex over distinct held values installs the lookup", async () => { + const document = item("item", "none"); + const cache = memory(document); + await cache.put(document, "a", { sku: "S" }); + await cache.registerIndex(document, "sku", "sku"); + assert.equal(cache.getKeyByIndex(document, "sku", "S"), "a"); +});