Merge remote-tracking branch 'origin/hands/reader-dup-keys-ts' into reader-dup-keys

packages/cultcache-ts/src/single-file-messagepack-backing-store.ts

@@ -204,8 +204,9 @@ type DecodedSnapshot = {
 };
 
 function encodeSnapshot(supplied: CultCacheEnvelope[], arrived: CultCacheEnvelope[], format: StoreFormat): unknown[] {
-  const replaced = new Set(supplied.map((entry) => `${entry.type}::${entry.key}`));
-  const kept = arrived.filter((entry) => !replaced.has(`${entry.type}::${entry.key}`));
+  // A key is held once: a supplied record replaces the arrived record under its key whatever the type.
+  const replaced = new Set(supplied.map((entry) => entry.key));
+  const kept = arrived.filter((entry) => !replaced.has(entry.key));
   const entries = [...kept, ...supplied];
   const catalog = catalogEntriesFor([
     ...kept.map((envelope) => ({ envelope, registered: false })),
@@ -449,6 +450,13 @@ function decodeSnapshot(decoded: unknown): DecodedSnapshot | undefined {
   }
 
   const records = recordsRaw.map(decodeRecord);
+  const seen = new Set<string>();
+  for (const { key } of records) {
+    if (seen.has(key)) {
+      throw new Error(`CultCache record "${key}" appears more than once.`);
+    }
+    seen.add(key);
+  }
   return { format: decoded[0], catalogBySchemaId, records };
 }
 
@@ -479,12 +487,16 @@ function decodeCatalogEntry(value: unknown): CultCacheSchemaCatalogEntry {
     throw new Error(`CultCache schema catalog entry "${schemaId}" has invalid members.`);
   }
 
+  if (typeof canonicalSchemaJson !== "string") {
+    throw new Error(`CultCache schema catalog entry "${schemaId}" has an invalid canonical schema.`);
+  }
+
   return {
     schemaId,
     schemaName,
     schemaVersion: isNonEmptyString(schemaVersion) ? schemaVersion : `${schemaName}.v1`,
     contentHash: isNonEmptyString(contentHash) ? contentHash : schemaId,
-    canonicalSchemaJson: typeof canonicalSchemaJson === "string" ? canonicalSchemaJson : "",
+    canonicalSchemaJson,
     compatibleSchemaIds,
     members: members.map(decodeCatalogMember),
   };

packages/cultcache-ts/test/cult-cache.test.ts

@@ -2582,6 +2582,21 @@ test("SingleFileMessagePackBackingStore replaces a file exactly when it reads",
   }
 });
 
+// One key is held once: a record pushed under a key replaces the stored record under that key, whatever its type, so the writer cannot
+// emit a store its own reader refuses.
+test("SingleFileMessagePackBackingStore push replaces the stored record under a key whatever the type", async () => {
+  const seed = CultCache.builder().withDocumentType(ownsId).withDocumentType(listsId).build();
+  await seed.put(ownsId, "k", { name: "owner" });
+  await seed.put(listsId, "k", { name: "lister" });
+  const file = join(await mkdtemp(join(tmpdir(), "cultcache-push-key-")), "store.msgpack");
+  const store = new SingleFileMessagePackBackingStore(file);
+  await store.push(seed.getRequiredEnvelope(ownsId, "k"));
+  await store.push(seed.getRequiredEnvelope(listsId, "k"));
+  const records = (decode(await readFile(file)) as unknown[][])[2] as unknown[][];
+  assert.deepEqual(records.map((record) => [record[0], record[1]]), [["k", "id.lister"]]);
+  assert.equal((await store.pullAll()).length, 1);
+});
+
 // Owner beats lister: a definition that owns an id and one that lists it as compatible register together in either order. A record
 // under the id is the owner's, and a lister's only when nothing owns it; two listers and no owner name no single definition.
 const ownsId = defineDocumentType({ type: "tests.owns", schema: z.object({ name: z.string() }), schemaId: "id.owned", schemaName: "tests.owns" });

packages/cultcache-ts/test/store-file.test.ts

@@ -571,15 +571,17 @@ const snapshotOf = async (store: SingleFileMessagePackBackingStore): Promise<str
     .map((entry) => `${entry.type}/${entry.key}:${[...documentPayloadBytes(entry.payload)].at(-1)}`)
     .sort();
 
-test("push replaces the record at the same type and key and keeps every other", async () => {
+test("push replaces the record under the same key whatever its type and keeps every other", async () => {
   const directory = await scratch();
   try {
     const store = new SingleFileMessagePackBackingStore(join(directory, "store.cc"));
-    await store.pushAll([entryOf("a", "t", 1), entryOf("b", "t", 2), entryOf("a", "u", 3)]);
+    await store.pushAll([entryOf("a", "t", 1), entryOf("b", "t", 2), entryOf("c", "u", 3)]);
     await store.push(entryOf("a", "t", 9));
-    assert.deepEqual(await snapshotOf(store), ["t/a:9", "t/b:2", "u/a:3"]);
-    await store.push(entryOf("c", "t", 4));
-    assert.deepEqual(await snapshotOf(store), ["t/a:9", "t/b:2", "t/c:4", "u/a:3"]);
+    assert.deepEqual(await snapshotOf(store), ["t/a:9", "t/b:2", "u/c:3"]);
+    await store.push(entryOf("d", "t", 4));
+    assert.deepEqual(await snapshotOf(store), ["t/a:9", "t/b:2", "t/d:4", "u/c:3"]);
+    await store.push(entryOf("a", "u", 5));
+    assert.deepEqual(await snapshotOf(store), ["t/b:2", "t/d:4", "u/a:5", "u/c:3"]);
   } finally {
     await rm(directory, { recursive: true, force: true });
   }
@@ -589,11 +591,12 @@ test("delete removes only the record at that type and key", async () => {
   const directory = await scratch();
   try {
     const store = new SingleFileMessagePackBackingStore(join(directory, "store.cc"));
-    await store.pushAll([entryOf("a", "t", 1), entryOf("b", "t", 2), entryOf("a", "u", 3)]);
+    await store.pushAll([entryOf("a", "t", 1), entryOf("b", "t", 2), entryOf("c", "u", 3)]);
     await store.delete(entryOf("a", "t"));
-    assert.deepEqual(await snapshotOf(store), ["t/b:2", "u/a:3"]);
+    assert.deepEqual(await snapshotOf(store), ["t/b:2", "u/c:3"]);
     await store.delete(entryOf("zzz", "t"));
-    assert.deepEqual(await snapshotOf(store), ["t/b:2", "u/a:3"]);
+    await store.delete(entryOf("c", "t"));
+    assert.deepEqual(await snapshotOf(store), ["t/b:2", "u/c:3"]);
   } finally {
     await rm(directory, { recursive: true, force: true });
   }

tests/vectors/document-variants-c2a/readability/manifest.txt

@@ -36,3 +36,4 @@ dup-key-other-schema-v3.bin refuses refuses refuses refuses
 dup-key-different-payload-v3.bin refuses refuses refuses refuses
 distinct-keys-case-v3.bin reads reads reads reads
 distinct-keys-nfd-v3.bin reads reads reads reads
+../../document-variants-c0/current-catalog-bad-canonical.msgpack refuses refuses refuses refuses