From 453b3d6c2fdaeddfa13a815c706c9e0d5d789579 Mon Sep 17 00:00:00 2001 From: Weksa Date: Mon, 9 Nov 2026 10:42:48 +0000 Subject: [PATCH 1/3] Corpus intake lows: S6 folds dotless i, kana and Arabic-Indic digits; CLI errors stop echoing identifiers; --admitted-on is a calendar date; marks-only texts refused FormError and StoreRefusal display no caller identifiers; clap value errors name the flag and reason only. One is_calendar_date serves admitted_on and the pronunciation date. A text with no base letter or digit (Lm or M only) is refused beside the no-alphanumeric refusal; modifier_letters.txt is the Lm table its test walks. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/form.rs | 32 +- crates/weksa/src/main.rs | 63 ++-- crates/weksa/src/store.rs | 196 +++++++++-- crates/weksa/tests/cli.rs | 228 ++++++++++++- crates/weksa/tests/forms.rs | 9 +- crates/weksa/tests/modifier_letters.txt | 411 ++++++++++++++++++++++++ crates/weksa/tests/stores.rs | 151 +++++++++ 7 files changed, 1043 insertions(+), 47 deletions(-) create mode 100644 crates/weksa/tests/modifier_letters.txt diff --git a/crates/weksa/src/form.rs b/crates/weksa/src/form.rs index 9de8eb8..3626fd1 100644 --- a/crates/weksa/src/form.rs +++ b/crates/weksa/src/form.rs @@ -22,7 +22,9 @@ impl FormError { impl fmt::Display for FormError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?} is not a {}", self.got, self.form) + // `got` stays a field for callers; the text does not echo it, because + // the value is often an identifier the caller supplied. + write!(f, "not a {}", self.form) } } @@ -61,6 +63,34 @@ macro_rules! string_form { } pub(crate) use string_form; +/// A real calendar date written `YYYY-MM-DD`: four ASCII digits, a month 1 to +/// 12 and a day that month has in that year (leap years by the Gregorian rule). +pub fn is_calendar_date(s: &str) -> bool { + let b = s.as_bytes(); + if b.len() != 10 || b[4] != b'-' || b[7] != b'-' { + return false; + } + let digits = |r: std::ops::Range| { + b[r.clone()] + .iter() + .all(u8::is_ascii_digit) + .then(|| s[r].parse::().ok()) + .flatten() + }; + let (Some(year), Some(month), Some(day)) = (digits(0..4), digits(5..7), digits(8..10)) else { + return false; + }; + let leap = year % 4 == 0 && (year % 100 != 0 || year % 400 == 0); + let days = match month { + 1 | 3 | 5 | 7 | 8 | 10 | 12 => 31, + 4 | 6 | 9 | 11 => 30, + 2 if leap => 29, + 2 => 28, + _ => return false, + }; + (1..=days).contains(&day) +} + /// A BCP 47-form language tag: 1 to 35 bytes of ASCII alphanumerics in /// hyphen-separated subtags of 1 to 8 bytes; the first is 2 to 8 letters, or the /// single letter `x` of a private-use tag. Form only: no registry is consulted. diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 5518823..fe997b3 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -4,6 +4,7 @@ use std::path::PathBuf; use anyhow::{Context, Result, bail}; +use clap::error::{ContextKind, ContextValue, ErrorKind}; use clap::{Args, Parser, Subcommand, ValueEnum}; use unicode_normalization::UnicodeNormalization; use weksa::corpus::{CorpusItem, ModelFamily, NameSpan, Source, Turn}; @@ -180,9 +181,7 @@ impl PutArgs { let lf = forms .iter_mut() .find(|lf| lf.language == language) - .with_context(|| { - format!("--pronunciation names {language}, which has no --form") - })?; + .context("--pronunciation names a language that has no --form")?; lf.pronunciation = Some(Pronunciation { ipa: ipa.nfc().collect(), ruled_by: need(self.ruled_by.clone(), "ruled-by")?, @@ -290,12 +289,47 @@ struct AddArgs { generator: Option, #[arg(long)] prompt_digest: Option, - #[arg(long)] + /// `YYYY-MM-DD`, a real calendar date. + #[arg(long, value_parser = parse_date)] admitted_on: String, #[arg(long)] supersedes: Option, } +fn parse_date(s: &str) -> std::result::Result { + if weksa::form::is_calendar_date(s) { + Ok(s.to_string()) + } else { + Err("not a YYYY-MM-DD calendar date".into()) + } +} + +/// Parses the command line. A rejected value is reported by the flag that took +/// it and the reason, never by the value: values are the caller's identifiers. +fn parse_cli() -> Cli { + use std::error::Error; + match Cli::try_parse() { + Ok(cli) => cli, + Err(e) + if matches!( + e.kind(), + ErrorKind::InvalidValue | ErrorKind::ValueValidation + ) => + { + let arg = match e.get(ContextKind::InvalidArg) { + Some(ContextValue::String(a)) => { + a.split_whitespace().next().unwrap_or("").to_string() + } + _ => String::new(), + }; + let reason = e.source().map(|s| format!(": {s}")).unwrap_or_default(); + eprintln!("error: invalid value for {arg}{reason}"); + std::process::exit(2); + } + Err(e) => e.exit(), + } +} + fn need(value: Option, flag: &str) -> Result { value.with_context(|| format!("this source needs --{flag}")) } @@ -413,17 +447,13 @@ fn refused(e: StoreError) -> anyhow::Error { } fn main() -> Result<()> { - let cli = Cli::parse(); + let cli = parse_cli(); match cli.command { Command::Validate { store, key } => { let inventory = InventoryStore::open(&cli.inventory)?.inventory()?; let packets = PacketStore::open(&store)?; let packets = match key { - Some(key) => vec![ - packets - .get(&key)? - .with_context(|| format!("no packet {key}"))?, - ], + Some(key) => vec![packets.get(&key)?.context("no such packet")?], None => packets.all()?, }; let mut bad = 0; @@ -531,9 +561,7 @@ fn main() -> Result<()> { let inventory_store = InventoryStore::open(&cli.inventory)?; let inventory = inventory_store.inventory()?; let mut packets = PacketStore::open(&store)?; - let packet = packets - .get(&key)? - .with_context(|| format!("no packet {key}"))?; + let packet = packets.get(&key)?.context("no such packet")?; // Admission is the only source of a LoweringInput; re-admitting the // stored packet is a no-op write that re-runs every door check. let view = packets @@ -562,12 +590,7 @@ fn main() -> Result<()> { let references = args .items .iter() - .map(|key| { - Ok(corpus - .get(key)? - .with_context(|| format!("no item {key}"))? - .text) - }) + .map(|key| Ok(corpus.get(key)?.context("no such item")?.text)) .collect::>>()?; let inputs: Vec = references .iter() @@ -587,7 +610,7 @@ fn main() -> Result<()> { Command::Split { item_key } => { let item = CorpusStore::open(&cli.corpus)? .get(&item_key)? - .with_context(|| format!("no item {item_key}"))?; + .context("no such item")?; println!("{}", split_text(&item)); } Command::Schema { out } => { diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index 764a7be..bc5e9fa 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -13,7 +13,9 @@ use unicode_segmentation::UnicodeSegmentation; use crate::corpus::{CorpusItem, CorpusItemId, Source, item_key, key_slug}; use crate::entity::{Entity, EntityOrigin, Pronunciation}; -use crate::form::{is_attestation, is_joining_only, is_language_tag, is_refused_format}; +use crate::form::{ + is_attestation, is_calendar_date, is_joining_only, is_language_tag, is_refused_format, +}; use crate::inventory::{Change, InvId, Inventory, InventoryEntry, InventoryRevision, revision_key}; use crate::packet::{EntityId, Packet}; use crate::validate::{LoweringInput, Refusal, admissible}; @@ -81,6 +83,44 @@ pub enum StoreRefusal { Malformed { key: String, why: &'static str }, } +/// What the CLI prints: the rule broken and the numbers and reasons that +/// explain it. Item keys, entity ids and groups are the caller's identifiers +/// and are left to the typed value, never to text. +impl fmt::Display for StoreRefusal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use StoreRefusal::*; + match self { + S1ItemImmutable { .. } => f.write_str("S1ItemImmutable"), + S2RevisionChain { expected, got } => { + write!(f, "S2RevisionChain expected {expected} got {got}") + } + S3Frozen { frozen } => write!(f, "S3Frozen at {frozen}"), + S4IntroducedRev { + introduced, + revision, + .. + } => write!(f, "S4IntroducedRev {introduced} in revision {revision}"), + S5SupersedeGroup { .. } => f.write_str("S5SupersedeGroup"), + S6TextHeld { .. } => f.write_str("S6TextHeld"), + UnknownCorpusItem { .. } => f.write_str("UnknownCorpusItem"), + PacketImmutable { .. } => f.write_str("PacketImmutable"), + UnknownEntity { .. } => f.write_str("UnknownEntity"), + NameSpanRange { start, end, .. } => write!(f, "NameSpanRange {start}..{end}"), + NameSpanSplitsCluster { start, end, .. } => { + write!(f, "NameSpanSplitsCluster {start}..{end}") + } + NotNfc { .. } => f.write_str("NotNfc"), + EntityBounds { bound, .. } => write!(f, "EntityBounds {bound:?}"), + NameSpanOverlap { .. } => f.write_str("NameSpanOverlap"), + NameSpanNotAForm { start, end, .. } => write!(f, "NameSpanNotAForm {start}..{end}"), + EntityNotAppendOnly { .. } => f.write_str("EntityNotAppendOnly"), + EntityNotOnItem { .. } => f.write_str("EntityNotOnItem"), + WithdrawnCorpusItem { .. } => f.write_str("WithdrawnCorpusItem"), + Malformed { why, .. } => write!(f, "Malformed: {why}"), + } + } +} + /// The capacity bound an entity broke. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum EntityBound { @@ -111,7 +151,7 @@ pub enum StoreError { impl fmt::Display for StoreError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - StoreError::Refused(r) => write!(f, "refused: {r:?}"), + StoreError::Refused(r) => write!(f, "refused: {r}"), StoreError::Invalid(rs) => { write!(f, "invalid packet:")?; rs.iter().try_for_each(|r| write!(f, " [{r}]")) @@ -244,6 +284,13 @@ impl CorpusStore { if !text.chars().any(char::is_alphanumeric) { return Err(malformed(&item.id, "text has no alphanumeric character")); } + // Read from the text as given: NFKC maps modifier letters (ʰ) to base letters. + if !item.text.chars().any(is_base) { + return Err(malformed(&item.id, "text has no base letter or digit")); + } + if !is_calendar_date(&item.admitted_on) { + return Err(malformed(&item.id, "admitted_on is not a YYYY-MM-DD date")); + } if let Some(why) = empty_attestation(&item.source) { return Err(malformed(&item.id, why)); } @@ -364,16 +411,140 @@ fn check_entities(item: &CorpusItem, entities: &EntityStore) -> Result<(), Store /// default case fold, NFKC) keeping letters, digits and combining marks /// (General_Category L, N, M) so tone marks and viramas keep words apart. Composed and decomposed, /// fullwidth, ligature and case-fold variants (straße, STRASSE) and spacing and -/// punctuation variants compare equal. Homoglyphs across scripts (Cyrillic a for -/// Latin a) stay apart: no key here reads what a character looks like. +/// punctuation variants compare equal. Three writings of one thing also fold: +/// Turkish dotless i and dotted capital I read as i, katakana reads as the +/// hiragana of the same syllable, and Arabic-Indic and Extended Arabic-Indic +/// digits read as 0-9. Homoglyphs across scripts (Cyrillic a for Latin a) stay +/// apart: no key here reads what a character looks like. fn normalized(text: &str) -> String { - let folded = default_case_fold_str(&text.nfkc().collect::()); - folded + let mapped: String = text.nfkc().map(fold_variant).collect(); + default_case_fold_str(&mapped) .nfkc() .filter(|c| c.is_alphanumeric() || is_combining_mark(*c)) .collect() } +/// The one-to-one folds `normalized` adds to NFKC and case folding, applied +/// before the case fold (U+0130 would otherwise fold to i plus a combining dot, +/// which the key keeps as a mark). +fn fold_variant(c: char) -> char { + match c { + '\u{131}' => 'i', + '\u{130}' => 'I', + // Katakana letters and small letters to hiragana; the iteration marks + // ヽヾ to ゝゞ. The prolonged sound mark U+30FC stays. + '\u{30a1}'..='\u{30f6}' | '\u{30fd}'..='\u{30fe}' => { + char::from_u32(c as u32 - 0x60).unwrap_or(c) + } + '\u{660}'..='\u{669}' => char::from(b'0' + (c as u32 - 0x660) as u8), + '\u{6f0}'..='\u{6f9}' => char::from(b'0' + (c as u32 - 0x6f0) as u8), + _ => c, + } +} + +/// A character that can carry a word: alphanumeric and neither a combining +/// mark (Mn, Mc, Me) nor a modifier letter (Lm). A text of none is marks and +/// modifiers only, like a text of punctuation. +fn is_base(c: char) -> bool { + c.is_alphanumeric() && !is_combining_mark(c) && !is_modifier_letter(c) +} + +/// General_Category Lm, Unicode 17 (the Rust 1.99 tables), as inclusive ranges. +fn is_modifier_letter(c: char) -> bool { + const LM: [(u32, u32); 79] = [ + (0x2b0, 0x2c1), + (0x2c6, 0x2d1), + (0x2e0, 0x2e4), + (0x2ec, 0x2ec), + (0x2ee, 0x2ee), + (0x374, 0x374), + (0x37a, 0x37a), + (0x559, 0x559), + (0x640, 0x640), + (0x6e5, 0x6e6), + (0x7f4, 0x7f5), + (0x7fa, 0x7fa), + (0x81a, 0x81a), + (0x824, 0x824), + (0x828, 0x828), + (0x8c9, 0x8c9), + (0x971, 0x971), + (0xe46, 0xe46), + (0xec6, 0xec6), + (0x10fc, 0x10fc), + (0x17d7, 0x17d7), + (0x1843, 0x1843), + (0x1aa7, 0x1aa7), + (0x1c78, 0x1c7d), + (0x1d2c, 0x1d6a), + (0x1d78, 0x1d78), + (0x1d9b, 0x1dbf), + (0x2071, 0x2071), + (0x207f, 0x207f), + (0x2090, 0x209c), + (0x2c7c, 0x2c7d), + (0x2d6f, 0x2d6f), + (0x2e2f, 0x2e2f), + (0x3005, 0x3005), + (0x3031, 0x3035), + (0x303b, 0x303b), + (0x309d, 0x309e), + (0x30fc, 0x30fe), + (0xa015, 0xa015), + (0xa4f8, 0xa4fd), + (0xa60c, 0xa60c), + (0xa67f, 0xa67f), + (0xa69c, 0xa69d), + (0xa717, 0xa71f), + (0xa770, 0xa770), + (0xa788, 0xa788), + (0xa7f1, 0xa7f4), + (0xa7f8, 0xa7f9), + (0xa9cf, 0xa9cf), + (0xa9e6, 0xa9e6), + (0xaa70, 0xaa70), + (0xaadd, 0xaadd), + (0xaaf3, 0xaaf4), + (0xab5c, 0xab5f), + (0xab69, 0xab69), + (0xff70, 0xff70), + (0xff9e, 0xff9f), + (0x10780, 0x10785), + (0x10787, 0x107b0), + (0x107b2, 0x107ba), + (0x10d4e, 0x10d4e), + (0x10d6f, 0x10d6f), + (0x10ec5, 0x10ec5), + (0x11dd9, 0x11dd9), + (0x16b40, 0x16b43), + (0x16d40, 0x16d42), + (0x16d6b, 0x16d6c), + (0x16f93, 0x16f9f), + (0x16fe0, 0x16fe1), + (0x16fe3, 0x16fe3), + (0x16ff2, 0x16ff3), + (0x1aff0, 0x1aff3), + (0x1aff5, 0x1affb), + (0x1affd, 0x1affe), + (0x1e030, 0x1e06d), + (0x1e137, 0x1e13d), + (0x1e4eb, 0x1e4eb), + (0x1e6ff, 0x1e6ff), + (0x1e94b, 0x1e94b), + ]; + let c = c as u32; + LM.binary_search_by(|&(lo, hi)| { + if hi < c { + std::cmp::Ordering::Less + } else if lo > c { + std::cmp::Ordering::Greater + } else { + std::cmp::Ordering::Equal + } + }) + .is_ok() +} + /// The field that holds an unattested provenance string, if any: a string with /// no alphanumeric, a control or format character, or over 200 bytes attests /// nothing. @@ -563,18 +734,7 @@ fn check_pronunciation(entity: &Entity, p: &Pronunciation) -> Result<(), StoreEr if !is_attestation(&p.ruled_by) || !is_attestation(&p.source) { return Err(bad("a pronunciation names who ruled it and where")); } - let b = p.date.as_bytes(); - let date_ok = b.len() == 10 - && b.iter().enumerate().all(|(i, c)| { - if i == 4 || i == 7 { - *c == b'-' - } else { - c.is_ascii_digit() - } - }) - && (1..=12).contains(&p.date[5..7].parse::().unwrap_or(0)) - && (1..=31).contains(&p.date[8..].parse::().unwrap_or(0)); - if !date_ok { + if !is_calendar_date(&p.date) { return Err(bad("a pronunciation is dated YYYY-MM-DD")); } if p.rejected.len() > EntityBound::MAX_FORMS { diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 72ad140..fda6901 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -836,7 +836,7 @@ fn proof_packet_lowers_and_the_lowering_scores_against_items() { assert!(text.contains("NIBU will stop"), "{text}"); let missing = weksa(d.path(), &["lower", "./p.cc", "packet:0000000000000000"]); - assert!(!missing.status.success() && out(&missing).contains("no packet")); + assert!(!missing.status.success() && out(&missing).contains("no such packet")); // IPA is the ruled pronunciation or nothing: spelling is not one. let args = ["lower", "./p.cc", "packet:6e6962752d303031", "--ipa"]; @@ -908,5 +908,229 @@ fn proof_packet_lowers_and_the_lowering_scores_against_items() { &url, ], ); - assert!(!unknown.status.success() && out(&unknown).contains("no item item:nibu:9")); + assert!(!unknown.status.success() && out(&unknown).contains("no such item")); + assert!(!out(&unknown).contains("item:nibu:9"), "{}", out(&unknown)); +} + +/// Runs `args` and demands a refusal whose text holds `reason` and none of the +/// identifiers the caller supplied. +fn refuses_without_echo(d: &std::path::Path, args: &[&str], reason: &str) { + let o = weksa(d, args); + let text = out(&o); + assert!(!o.status.success(), "{args:?}: {text}"); + assert!(text.contains(reason), "{args:?} lacks {reason:?}: {text}"); + for a in args.iter().filter(|a| a.to_lowercase().contains("canary")) { + // An argument may carry the canary beside flag syntax (`--x=...`); the + // identifier itself is the part from the canary on. + let id = &a[a.to_lowercase().find("canary").unwrap()..]; + assert!( + !text.to_lowercase().contains(&id.to_lowercase()), + "{args:?} echoes {id:?}: {text}" + ); + } +} + +#[test] +fn cli_errors_never_echo_identifiers() { + let d = tempfile::tempdir().unwrap(); + let p = d.path(); + cast_entities(p, &["nibu"]); + let add = |extra: &[&str]| -> Vec { + ["corpus", "add", "--admitted-on", "2026-10-29"] + .into_iter() + .chain(extra.iter().copied()) + .map(String::from) + .collect() + }; + let run = |args: Vec, reason: &str| { + let refs: Vec<&str> = args.iter().map(String::as_str).collect(); + refuses_without_echo(p, &refs, reason); + }; + // The positional entity id, malformed and well formed but unheld. + refuses_without_echo( + p, + &[ + "entity", + "put", + "canary-id", + "--form", + "en-US=X", + "--origin", + "live", + "--persona", + "k", + ], + "not a EntityId", + ); + // --speaker: a malformed id, and a well formed one no entity holds. + run( + add(&[ + "--slug", + "q", + "--text", + "Hello there.", + "--speaker", + "canary-speaker", + "--group", + "g", + "--source", + "operator-written", + ]), + "--speaker", + ); + run( + add(&[ + "--slug", + "q", + "--text", + "Hello there.", + "--speaker", + "entity:canary", + "--group", + "g", + "--source", + "operator-written", + ]), + "UnknownEntity", + ); + // clap enum values. + run( + add(&[ + "--slug", + "q", + "--text", + "Hello there.", + "--speaker", + "entity:nibu", + "--group", + "g", + "--source", + "canary-source", + ]), + "--source", + ); + refuses_without_echo( + p, + &[ + "entity", + "put", + "entity:kay", + "--form", + "en-US=X", + "--origin", + "canary-origin", + ], + "--origin", + ); + run( + add(&[ + "--slug", + "q", + "--text", + "Hello there.", + "--speaker", + "entity:nibu", + "--group", + "g", + "--source", + "model-written", + "--model-family", + "canary-family", + ]), + "--model-family", + ); + // Keys on withdraw, split, validate, lower and score. + refuses_without_echo(p, &["corpus", "withdraw", "item:canary:7"], "no such item"); + refuses_without_echo(p, &["split", "item:canary:7"], "no such item"); + std::fs::write(p.join("out.txt"), "text").unwrap(); + refuses_without_echo( + p, + &["score", "item:canary:7", "--output", "out.txt"], + "no such item", + ); + let packets = weksa(p, &["inventory", "seed"]); + assert!(packets.status.success(), "{}", out(&packets)); + let made = weksa(p, &["packet", "proof", "p.cc"]); + let _ = made; + refuses_without_echo( + p, + &["validate", "p.cc", "packet:canary:7"], + "no such packet", + ); + refuses_without_echo(p, &["lower", "p.cc", "packet:canary:7"], "no such packet"); + // S6 names no held key; S5 names no group. + let held = weksa( + p, + &add(&[ + "--slug", + "canarya", + "--text", + "Fresh words here.", + "--speaker", + "entity:nibu", + "--group", + "canary-first", + "--source", + "operator-written", + ]) + .iter() + .map(String::as_str) + .collect::>(), + ); + assert!(held.status.success(), "{}", out(&held)); + let copy = add(&[ + "--slug", + "other", + "--text", + "Fresh words here.", + "--speaker", + "entity:nibu", + "--group", + "g", + "--source", + "operator-written", + ]); + let copy = weksa(p, ©.iter().map(String::as_str).collect::>()); + assert!( + !copy.status.success() && out(©).contains("S6TextHeld"), + "{}", + out(©) + ); + assert!(!out(©).contains("canary"), "{}", out(©)); + let moved = add(&[ + "--slug", + "other", + "--text", + "Corrected words here.", + "--speaker", + "entity:nibu", + "--group", + "canary-second", + "--supersedes", + "item:canarya:1", + "--source", + "operator-written", + ]); + let moved = weksa(p, &moved.iter().map(String::as_str).collect::>()); + assert!( + !moved.status.success() && out(&moved).contains("S5SupersedeGroup"), + "{}", + out(&moved) + ); + assert!(!out(&moved).contains("canary"), "{}", out(&moved)); +} + +#[test] +fn admitted_on_must_be_a_real_calendar_date() { + let d = tempfile::tempdir().unwrap(); + cast_entities(d.path(), &["nibu"]); + for bad in ["2026-02-30", "", "junk", "2026-13-01", "2026-1-1"] { + let mut args = ADD.to_vec(); + let at = args.iter().position(|a| *a == "--admitted-on").unwrap(); + args[at + 1] = bad; + let o = weksa(d.path(), &args); + assert!(!o.status.success(), "{bad:?}: {}", out(&o)); + assert!(out(&o).contains("--admitted-on"), "{bad:?}: {}", out(&o)); + } + assert!(!out(&weksa(d.path(), &["corpus", "dump"])).contains("item:")); } diff --git a/crates/weksa/tests/forms.rs b/crates/weksa/tests/forms.rs index 61b62e1..f0b2469 100644 --- a/crates/weksa/tests/forms.rs +++ b/crates/weksa/tests/forms.rs @@ -51,15 +51,12 @@ fn forms_refuse_surface_text() { } #[test] -fn a_refusal_names_the_form_and_the_value_and_a_form_reads_back_as_written() { +fn a_refusal_names_the_form_and_keeps_the_value_out_of_its_text() { let e = NodeId::parse("n0").unwrap_err(); assert_eq!(e.form, "NodeId"); assert_eq!(e.got, "n0"); - assert_eq!(e.to_string(), "\"n0\" is not a NodeId"); - assert_eq!( - Grade::new(1001).unwrap_err().to_string(), - "\"1001\" is not a Grade" - ); + assert_eq!(e.to_string(), "not a NodeId"); + assert_eq!(Grade::new(1001).unwrap_err().to_string(), "not a Grade"); assert_eq!(EncoderId::parse(ENCODER_ID).unwrap().as_str(), ENCODER_ID); assert_eq!(PacketId::parse(PACKET_ID).unwrap().as_str(), PACKET_ID); assert_eq!( diff --git a/crates/weksa/tests/modifier_letters.txt b/crates/weksa/tests/modifier_letters.txt new file mode 100644 index 0000000..991618d --- /dev/null +++ b/crates/weksa/tests/modifier_letters.txt @@ -0,0 +1,411 @@ +# Lm, Node 24 (Unicode 17.0) over 0..10FFFF, one hex code point per line +2B0 +2B1 +2B2 +2B3 +2B4 +2B5 +2B6 +2B7 +2B8 +2B9 +2BA +2BB +2BC +2BD +2BE +2BF +2C0 +2C1 +2C6 +2C7 +2C8 +2C9 +2CA +2CB +2CC +2CD +2CE +2CF +2D0 +2D1 +2E0 +2E1 +2E2 +2E3 +2E4 +2EC +2EE +374 +37A +559 +640 +6E5 +6E6 +7F4 +7F5 +7FA +81A +824 +828 +8C9 +971 +E46 +EC6 +10FC +17D7 +1843 +1AA7 +1C78 +1C79 +1C7A +1C7B +1C7C +1C7D +1D2C +1D2D +1D2E +1D2F +1D30 +1D31 +1D32 +1D33 +1D34 +1D35 +1D36 +1D37 +1D38 +1D39 +1D3A +1D3B +1D3C +1D3D +1D3E +1D3F +1D40 +1D41 +1D42 +1D43 +1D44 +1D45 +1D46 +1D47 +1D48 +1D49 +1D4A +1D4B +1D4C +1D4D +1D4E +1D4F +1D50 +1D51 +1D52 +1D53 +1D54 +1D55 +1D56 +1D57 +1D58 +1D59 +1D5A +1D5B +1D5C +1D5D +1D5E +1D5F +1D60 +1D61 +1D62 +1D63 +1D64 +1D65 +1D66 +1D67 +1D68 +1D69 +1D6A +1D78 +1D9B +1D9C +1D9D +1D9E +1D9F +1DA0 +1DA1 +1DA2 +1DA3 +1DA4 +1DA5 +1DA6 +1DA7 +1DA8 +1DA9 +1DAA +1DAB +1DAC +1DAD +1DAE +1DAF +1DB0 +1DB1 +1DB2 +1DB3 +1DB4 +1DB5 +1DB6 +1DB7 +1DB8 +1DB9 +1DBA +1DBB +1DBC +1DBD +1DBE +1DBF +2071 +207F +2090 +2091 +2092 +2093 +2094 +2095 +2096 +2097 +2098 +2099 +209A +209B +209C +2C7C +2C7D +2D6F +2E2F +3005 +3031 +3032 +3033 +3034 +3035 +303B +309D +309E +30FC +30FD +30FE +A015 +A4F8 +A4F9 +A4FA +A4FB +A4FC +A4FD +A60C +A67F +A69C +A69D +A717 +A718 +A719 +A71A +A71B +A71C +A71D +A71E +A71F +A770 +A788 +A7F1 +A7F2 +A7F3 +A7F4 +A7F8 +A7F9 +A9CF +A9E6 +AA70 +AADD +AAF3 +AAF4 +AB5C +AB5D +AB5E +AB5F +AB69 +FF70 +FF9E +FF9F +10780 +10781 +10782 +10783 +10784 +10785 +10787 +10788 +10789 +1078A +1078B +1078C +1078D +1078E +1078F +10790 +10791 +10792 +10793 +10794 +10795 +10796 +10797 +10798 +10799 +1079A +1079B +1079C +1079D +1079E +1079F +107A0 +107A1 +107A2 +107A3 +107A4 +107A5 +107A6 +107A7 +107A8 +107A9 +107AA +107AB +107AC +107AD +107AE +107AF +107B0 +107B2 +107B3 +107B4 +107B5 +107B6 +107B7 +107B8 +107B9 +107BA +10D4E +10D6F +10EC5 +11DD9 +16B40 +16B41 +16B42 +16B43 +16D40 +16D41 +16D42 +16D6B +16D6C +16F93 +16F94 +16F95 +16F96 +16F97 +16F98 +16F99 +16F9A +16F9B +16F9C +16F9D +16F9E +16F9F +16FE0 +16FE1 +16FE3 +16FF2 +16FF3 +1AFF0 +1AFF1 +1AFF2 +1AFF3 +1AFF5 +1AFF6 +1AFF7 +1AFF8 +1AFF9 +1AFFA +1AFFB +1AFFD +1AFFE +1E030 +1E031 +1E032 +1E033 +1E034 +1E035 +1E036 +1E037 +1E038 +1E039 +1E03A +1E03B +1E03C +1E03D +1E03E +1E03F +1E040 +1E041 +1E042 +1E043 +1E044 +1E045 +1E046 +1E047 +1E048 +1E049 +1E04A +1E04B +1E04C +1E04D +1E04E +1E04F +1E050 +1E051 +1E052 +1E053 +1E054 +1E055 +1E056 +1E057 +1E058 +1E059 +1E05A +1E05B +1E05C +1E05D +1E05E +1E05F +1E060 +1E061 +1E062 +1E063 +1E064 +1E065 +1E066 +1E067 +1E068 +1E069 +1E06A +1E06B +1E06C +1E06D +1E137 +1E138 +1E139 +1E13A +1E13B +1E13C +1E13D +1E4EB +1E6FF +1E94B diff --git a/crates/weksa/tests/stores.rs b/crates/weksa/tests/stores.rs index cf5ba4a..8eb5f17 100644 --- a/crates/weksa/tests/stores.rs +++ b/crates/weksa/tests/stores.rs @@ -336,6 +336,157 @@ fn s6_reads_text_under_compatibility_case_folding() { .unwrap(); } +#[test] +fn s6_folds_dotless_i_kana_and_arabic_indic_digits() { + let d = dir(); + let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + // Turkish dotless and dotted capital i read as i; hiragana and katakana + // (small, voiced, iteration marks, halfwidth through NFKC) read as one + // syllabary; Arabic-Indic and Extended Arabic-Indic digits read as 0-9. + let pairs = [ + ("isi", "\u{131}s\u{131}"), + ("istanbul", "\u{130}stanbul"), + ( + "\u{3072}\u{3089}\u{304c}\u{306a}", + "\u{30d2}\u{30e9}\u{30ac}\u{30ca}", + ), + ("\u{3041}\u{3083}", "\u{30a1}\u{30e3}"), + ("\u{3042}\u{309d}\u{309e}", "\u{3042}\u{30fd}\u{30fe}"), + ("\u{306d}\u{3053}", "\u{ff88}\u{ff7a}"), + ( + "room 1234567890", + "room \u{661}\u{662}\u{663}\u{664}\u{665}\u{666}\u{667}\u{668}\u{669}\u{660}", + ), + ("gate 45", "gate \u{6f4}\u{6f5}"), + ]; + for (n, (a, b)) in pairs.into_iter().enumerate() { + let (first, second) = ( + format!("item:fold:{}", 2 * n + 1), + format!("item:fold:{}", 2 * n + 2), + ); + store.add(item(&first, a), &ents(&d)).unwrap(); + assert_eq!( + refusal(store.add(item(&second, b), &ents(&d))), + StoreRefusal::S6TextHeld { + key: second.clone(), + held_by: first.clone() + }, + "{a:?} vs {b:?}" + ); + } + // Nothing wider: the prolonged sound mark, Devanagari digits and the + // Cyrillic homoglyph of i stay apart from what they resemble. + store + .add(item("item:fold:90", "\u{30a2}\u{30fc}"), &ents(&d)) + .unwrap(); + store + .add(item("item:fold:91", "\u{30a2}"), &ents(&d)) + .unwrap(); + store + .add(item("item:fold:92", "room 12"), &ents(&d)) + .unwrap(); + store + .add(item("item:fold:93", "room \u{967}\u{968}"), &ents(&d)) + .unwrap(); + store.add(item("item:fold:94", "bit"), &ents(&d)).unwrap(); + store + .add(item("item:fold:95", "b\u{456}t"), &ents(&d)) + .unwrap(); +} + +#[test] +fn a_text_of_only_marks_is_refused() { + let d = dir(); + let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + // Prolonged sound mark, Thai maiyamok, fathatan, tatweel: alphabetic to + // Unicode, but none carries a word. + for (n, text) in [ + "\u{30fc}", + "\u{e46}", + "\u{64b}", + "\u{640}", + "\u{30fc}\u{30fc}", + "\u{640}\u{64b}", + ] + .into_iter() + .enumerate() + { + let key = format!("item:mo:{}", n + 1); + assert_eq!( + refusal(store.add(item(&key, text), &ents(&d))), + malformed(&key, "text has no base letter or digit"), + "{text:?}" + ); + } + // Every modifier letter (Lm) alone is refused: for having no base letter, + // or, where Unicode does not call it alphabetic, no alphanumeric at all, or + // (U+0374, U+037A) for not being NFC. + for line in include_str!("modifier_letters.txt") + .lines() + .filter(|l| !l.starts_with('#')) + { + let c = char::from_u32(u32::from_str_radix(line, 16).unwrap()).unwrap(); + let key = "item:mo:90"; + let got = refusal(store.add(item(key, &c.to_string()), &ents(&d))); + assert!( + got == StoreRefusal::NotNfc { key: key.into() } + || got == malformed(key, "text has no base letter or digit") + || got == malformed(key, "text has no alphanumeric character"), + "{c:?}: {got:?}" + ); + } + // With a base letter or digit the same marks are text. + store + .add(item("item:mo:91", "\u{30a2}\u{30fc}"), &ents(&d)) + .unwrap(); + store + .add(item("item:mo:92", "\u{e01}\u{e46}"), &ents(&d)) + .unwrap(); + store + .add(item("item:mo:93", "\u{640}7"), &ents(&d)) + .unwrap(); +} + +#[test] +fn admitted_on_is_a_real_calendar_date() { + let d = dir(); + let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + for (n, bad) in [ + "", + "junk", + "2026-02-30", + "2026-02-29", + "2026-13-01", + "2026-00-10", + "2026-04-31", + "2026-1-1", + "2026-10-29 ", + "20261029", + "2026-10-29T00:00Z", + "\u{661}\u{662}\u{663}\u{664}-10-29", + ] + .into_iter() + .enumerate() + { + let key = format!("item:dt:{}", n + 1); + let mut it = item(&key, "a date"); + it.admitted_on = bad.into(); + assert_eq!( + refusal(store.add(it, &ents(&d))), + malformed(&key, "admitted_on is not a YYYY-MM-DD date"), + "{bad:?}" + ); + } + for (n, good) in ["2026-10-29", "2024-02-29", "2000-02-29"] + .into_iter() + .enumerate() + { + let mut it = item(&format!("item:dt:{}", 90 + n), &format!("a date {n}")); + it.admitted_on = good.into(); + store.add(it, &ents(&d)).unwrap(); + } +} + #[test] fn s6_keeps_tone_marks_and_viramas_apart() { let d = dir(); -- 2.47.3 From 2ce6325b704f94f98a1dbd09e82ac0c65fdeccd2 Mon Sep 17 00:00:00 2001 From: Weksa Date: Mon, 9 Nov 2026 10:44:53 +0000 Subject: [PATCH 2/3] Test: pin the modifier-letter table to the toolchain's Unicode version Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/tests/stores.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/crates/weksa/tests/stores.rs b/crates/weksa/tests/stores.rs index 1eb6951..1922c44 100644 --- a/crates/weksa/tests/stores.rs +++ b/crates/weksa/tests/stores.rs @@ -394,6 +394,18 @@ fn s6_folds_dotless_i_kana_and_arabic_indic_digits() { .unwrap(); } +/// `modifier_letters.txt` is the Unicode 17 Lm list, the oracle for the test +/// below. When the toolchain's Unicode tables move, this fails first: regenerate +/// the file from the new tables, then bump the pin. +#[test] +fn the_modifier_letter_table_matches_the_toolchains_unicode_version() { + assert_eq!( + char::UNICODE_VERSION, + (17, 0, 0), + "regenerate crates/weksa/tests/modifier_letters.txt for the new Unicode version" + ); +} + #[test] fn a_text_of_only_marks_is_refused() { let d = dir(); -- 2.47.3 From 3a56d3180fbcde6a96338b1332f80ac7125be137 Mon Sep 17 00:00:00 2001 From: Weksa Date: Mon, 9 Nov 2026 11:03:58 +0000 Subject: [PATCH 3/3] =?UTF-8?q?Tests:=20pin=20the=20100-year=20rule,=20day?= =?UTF-8?q?=200,=20ASCII-only=20digits=20in=20dates,=20and=20the=20?= =?UTF-8?q?=E3=83=B6=20fold=20(Soul=20s19=20survivors)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/tests/forms.rs | 20 ++++++++++++++++++++ crates/weksa/tests/stores.rs | 1 + 2 files changed, 21 insertions(+) diff --git a/crates/weksa/tests/forms.rs b/crates/weksa/tests/forms.rs index f0b2469..dba73c8 100644 --- a/crates/weksa/tests/forms.rs +++ b/crates/weksa/tests/forms.rs @@ -147,3 +147,23 @@ fn grade_bound_is_the_type() { assert_eq!(grade["minimum"], 0, "{grade}"); assert_eq!(schema["$defs"]["NodeId"]["pattern"], "^n[1-9][0-9]{0,3}$"); } + +#[test] +fn calendar_dates_follow_the_gregorian_rule_and_ascii_digits_only() { + use weksa::form::is_calendar_date; + for ok in ["2024-02-29", "2000-02-29", "2026-12-31"] { + assert!(is_calendar_date(ok), "{ok}"); + } + for bad in [ + "1900-02-29", + "2023-02-29", + "2026-04-31", + "2026-10-00", + "2026-00-10", + "+026-10-29", + "2026-10-2\u{669}", + "\u{6f0}026-10-29", + ] { + assert!(!is_calendar_date(bad), "{bad}"); + } +} diff --git a/crates/weksa/tests/stores.rs b/crates/weksa/tests/stores.rs index 1922c44..a8486e7 100644 --- a/crates/weksa/tests/stores.rs +++ b/crates/weksa/tests/stores.rs @@ -358,6 +358,7 @@ fn s6_folds_dotless_i_kana_and_arabic_indic_digits() { "room \u{661}\u{662}\u{663}\u{664}\u{665}\u{666}\u{667}\u{668}\u{669}\u{660}", ), ("gate 45", "gate \u{6f4}\u{6f5}"), + ("\u{3096}", "\u{30f6}"), ]; for (n, (a, b)) in pairs.into_iter().enumerate() { let (first, second) = ( -- 2.47.3