grammar-reader fix 1: canonical keys through the one predicate, duplicate factions refused, bin/ext growth tolerated, guards pinned

The reader mints every faction and allegiance ImportKey and refuses a
non-canonical one as a typed NonCanonicalKey naming record and field (0 or 12);
two faction rows under one record key are DuplicateFaction. Payloads decode
through rmpv so a MessagePack bin or ext cell at a higher key (C# byte[],
DateTime) no longer fails the record. Tests pin the record-key sort and a
non-map Allegiance. Schema-version check (item 2) not done: CultLib gap.
All commits build (not yet run; the suite runs next).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Cargo.lock

@@ -668,6 +668,7 @@ dependencies = [
  "ghostlight-persona-projection",
  "reqwest",
  "rmp-serde",
+ "rmpv",
  "serde",
  "serde_bytes",
  "serde_json",

crates/ghostlight/Cargo.toml

@@ -12,6 +12,7 @@ cultcache-rs.workspace = true
 ghostlight-persona-projection = { path = "../ghostlight-persona-projection" }
 reqwest.workspace = true
 rmp-serde.workspace = true
+rmpv = "1.3"
 serde.workspace = true
 serde_bytes = "0.11"
 serde_json.workspace = true

crates/ghostlight/src/aetheria.rs

@@ -16,7 +16,7 @@ use crate::grammar::{
     GrammarBinding, GrammarError, GrammarReferentKind, GrammarRole, GrammarVerb, RenderPath,
 };
 use cultcache_rs::SingleFileMessagePackBackingStore;
-use serde_json::Value;
+use rmpv::Value;
 use std::collections::BTreeMap;
 use std::path::Path;
 
@@ -73,6 +73,10 @@ pub enum CatalogReadError {
     MultipleGrammars,
     #[error("record {record:?} field {field} is missing or not the mirrored type")]
     MalformedRecord { record: String, field: u8 },
+    #[error("record {record:?} field {field} is not a canonical import key")]
+    NonCanonicalKey { record: String, field: u8 },
+    #[error("record {record:?} is a second aetheria.faction under one record key")]
+    DuplicateFaction { record: String },
     #[error("record {record:?} field {field} is not a known render path")]
     UnknownRenderPath { record: String, field: u8 },
     #[error("record {record:?} field {field} is not a known referent kind")]
@@ -107,7 +111,16 @@ pub fn read_aetheria_catalog(path: impl AsRef<Path>) -> Result<AetheriaCatalog,
             }
             VERB_SCHEMA => verbs.push(read_verb(&Cells::of(&row.key, &row.payload)?)?),
             FACTION_SCHEMA => {
-                factions.push(read_faction(&row.key, &Cells::of(&row.key, &row.payload)?)?)
+                let faction = read_faction(&row.key, &Cells::of(&row.key, &row.payload)?)?;
+                if factions
+                    .last()
+                    .is_some_and(|seen: &AetheriaFaction| seen.key == faction.key)
+                {
+                    return Err(CatalogReadError::DuplicateFaction {
+                        record: row.key.clone(),
+                    });
+                }
+                factions.push(faction);
             }
             _ => {}
         }
@@ -127,7 +140,8 @@ struct Cells<'a> {
 
 impl<'a> Cells<'a> {
     fn of(record: &'a str, payload: &[u8]) -> Result<Self, CatalogReadError> {
-        match rmp_serde::from_slice::<Value>(payload) {
+        let mut payload = payload;
+        match rmpv::decode::read_value(&mut payload) {
             Ok(Value::Array(cells)) => Ok(Self { record, cells }),
             _ => Err(CatalogReadError::MalformedRecord {
                 record: record.to_owned(),
@@ -143,6 +157,13 @@ impl<'a> Cells<'a> {
         }
     }
 
+    fn non_canonical(&self, field: u8) -> CatalogReadError {
+        CatalogReadError::NonCanonicalKey {
+            record: self.record.to_owned(),
+            field,
+        }
+    }
+
     fn cell(&self, field: u8) -> Result<&Value, CatalogReadError> {
         self.cells
             .get(usize::from(field))
@@ -165,8 +186,11 @@ impl<'a> Cells<'a> {
     /// A string Aetheria may leave null; null reads as empty.
     fn text_or_empty(&self, field: u8) -> Result<String, CatalogReadError> {
         match self.cell(field)? {
-            Value::Null => Ok(String::new()),
-            Value::String(text) => Ok(text.clone()),
+            Value::Nil => Ok(String::new()),
+            Value::String(text) => text
+                .as_str()
+                .map(str::to_owned)
+                .ok_or_else(|| self.malformed(field)),
             _ => Err(self.malformed(field)),
         }
     }
@@ -243,19 +267,25 @@ fn canonical_name(name: &str, max_len: usize) -> bool {
 }
 
 fn read_faction(key: &str, cells: &Cells) -> Result<AetheriaFaction, CatalogReadError> {
-    let Some(allegiance) = cells.cell(12)?.as_object() else {
+    let own = ImportKey::new(FACTION_SCHEMA, key);
+    if !own.is_canonical() {
+        return Err(cells.non_canonical(0));
+    }
+    let Some(allegiance) = cells.cell(12)?.as_map() else {
         return Err(cells.malformed(12));
     };
     let mut standing = BTreeMap::new();
     for (other, weight) in allegiance {
+        let other = other.as_str().ok_or_else(|| cells.malformed(12))?;
         let weight = weight.as_f64().ok_or_else(|| cells.malformed(12))?;
-        standing.insert(
-            ImportKey::new(FACTION_SCHEMA, other.as_str()),
-            weight as f32,
-        );
+        let other = ImportKey::new(FACTION_SCHEMA, other);
+        if !other.is_canonical() {
+            return Err(cells.non_canonical(12));
+        }
+        standing.insert(other, weight as f32);
     }
     Ok(AetheriaFaction {
-        key: ImportKey::new(FACTION_SCHEMA, key),
+        key: own,
         name: cells.text(1)?,
         short_name: cells.text_or_empty(2)?,
         description: cells.text_or_empty(3)?,

crates/ghostlight/src/aetheria/tests.rs

@@ -366,7 +366,9 @@ fn every_referent_code_maps_to_its_own_kind() {
         (5, GrammarReferentKind::Cargo),
     ] {
         let mut rows = fixture_rows();
-        edit(&mut rows, "verb-attack", |cells| cells[3][0][2] = json!(code));
+        edit(&mut rows, "verb-attack", |cells| {
+            cells[3][0][2] = json!(code)
+        });
         let catalog = read(&rows).unwrap();
         assert_eq!(
             catalog.grammar.verb("attack"),
@@ -375,3 +377,121 @@ fn every_referent_code_maps_to_its_own_kind() {
         );
     }
 }
+
+fn faction_with_key(record: &str) -> Vec<CultCacheEnvelope> {
+    let mut rows = fixture_rows();
+    row_of(&mut rows, "faction-brannoch").key = record.into();
+    rows
+}
+
+#[test]
+fn a_faction_record_key_that_is_not_canonical_text_is_refused_by_key_and_field() {
+    for bad in [
+        "faction-cafe\u{301}",
+        "faction-brannoch ",
+        "faction-\u{202e}brannoch",
+        "",
+    ] {
+        assert_eq!(
+            read(&faction_with_key(bad)),
+            Err(CatalogReadError::NonCanonicalKey {
+                record: bad.into(),
+                field: 0
+            }),
+            "{bad:?}"
+        );
+    }
+}
+
+#[test]
+fn an_allegiance_key_that_is_not_canonical_text_is_refused_by_key_and_field() {
+    for bad in ["faction-cafe\u{301}", "faction-adrasteia ", "\u{202e}"] {
+        let mut rows = fixture_rows();
+        edit(&mut rows, "faction-brannoch", |cells| {
+            cells[12] = json!({ bad: 0.5 });
+        });
+        assert_eq!(
+            read(&rows),
+            Err(CatalogReadError::NonCanonicalKey {
+                record: "faction-brannoch".into(),
+                field: 12
+            }),
+            "{bad:?}"
+        );
+    }
+}
+
+#[test]
+fn two_faction_rows_under_one_record_key_are_refused() {
+    let mut rows = fixture_rows();
+    let twin = row_of(&mut rows, "faction-adrasteia").clone();
+    rows.push(twin);
+    assert_eq!(
+        read(&rows),
+        Err(CatalogReadError::DuplicateFaction {
+            record: "faction-adrasteia".into()
+        })
+    );
+}
+
+#[test]
+fn factions_are_read_in_record_key_order_whatever_the_file_order() {
+    let mut rows = fixture_rows();
+    rows.sort_by(|a, b| b.key.cmp(&a.key));
+    assert_eq!(rows.first().unwrap().key, "verb-speak");
+    let names: Vec<_> = read(&rows)
+        .unwrap()
+        .factions
+        .into_iter()
+        .map(|faction| faction.name)
+        .collect();
+    assert_eq!(names, ["Adrasteia", "Brannoch"]);
+}
+
+#[test]
+fn an_allegiance_that_is_not_a_map_is_malformed_not_empty() {
+    for not_a_map in [
+        json!([]),
+        json!(["faction-adrasteia"]),
+        Value::Null,
+        json!(1),
+    ] {
+        let mut rows = fixture_rows();
+        let shape = not_a_map.clone();
+        edit(&mut rows, "faction-brannoch", |cells| cells[12] = shape);
+        assert_eq!(
+            read(&rows),
+            Err(CatalogReadError::MalformedRecord {
+                record: "faction-brannoch".into(),
+                field: 12
+            }),
+            "{not_a_map}"
+        );
+    }
+}
+
+/// Appends cells to a faction as Aetheria's C# CultCache writes a `byte[]`
+/// (MessagePack bin) and a `DateTime` (MessagePack timestamp, ext -1).
+#[test]
+fn bin_and_ext_fields_aetheria_adds_at_higher_keys_are_ignored() {
+    let mut rows = fixture_rows();
+    let row = row_of(&mut rows, "faction-adrasteia");
+    let rmpv::Value::Array(mut cells) =
+        rmpv::decode::read_value(&mut row.payload.as_slice()).unwrap()
+    else {
+        panic!("not positional");
+    };
+    cells.push(rmpv::Value::Binary(vec![1, 0, 7]));
+    cells.push(rmpv::Value::Ext(
+        -1,
+        vec![0, 0, 0, 0, 0x65, 0x1f, 0x2a, 0x80],
+    ));
+    let mut payload = Vec::new();
+    rmpv::encode::write_value(&mut payload, &rmpv::Value::Array(cells)).unwrap();
+    row.payload = payload;
+
+    assert_eq!(
+        read(&rows).unwrap(),
+        read_aetheria_catalog(FIXTURE).unwrap()
+    );
+}