crates/weksa/src/context.rs

@@ -93,7 +93,7 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat
 pub const MINT_ONTOLOGY_INSTRUCTIONS: &str = "\
 Below are lines written in one language (the language line), the features an ontology condition may test, and the entries that language's ontology already has. Draft new entries: choices this language's culture makes, for a speaker in a stance and scene, about word, grammar, discourse, sound or script, that the lines show.
 Answer with one JSON object and nothing else: {\"entries\": [{\"layer\": \"...\", \"when\": [...], \"choice\": \"...\", \"examples\": [...]}]}. No code fence, no commentary.
-Give 1 to 8 entries. layer is exactly one of Lexicon, Grammar, Discourse, Phonology, Orthography. when holds at most 4 conditions, each {\"slot\": \"Stance\" | \"Discourse\" | \"Scene\", \"feature\": \"<id>\", \"test\": ...}, where <id> is the id printed before the colon of a listed feature (\"feature:10\", never its name) and its slot is one that feature lists. test is {\"AtLeast\": <permille>}, {\"AtMost\": <permille>} or {\"AtMostOrUnset\": <permille>} on a grade feature, or {\"Is\": {\"Option\": \"<label>\"}} on an options feature with one of its labels, or {\"Is\": {\"Flag\": true}} on a flag feature. The example line shows each form on listed features; its choice and examples are placeholders, not content to copy. when may be empty when a choice holds for every line of the language. choice states a pattern the lines show, in at most 200 bytes, without quoting a line. examples are original short phrases of the language, never copied from the lines below, at most 3, each at most 120 bytes.";
+Give only entries the lines show, as many as they support (a language holds at most 64 in all). layer is exactly one of Lexicon, Grammar, Discourse, Phonology, Orthography. when holds at most 4 conditions, each {\"slot\": \"Stance\" | \"Discourse\" | \"Scene\", \"feature\": \"<id>\", \"test\": ...}, where <id> is the id printed before the colon of a listed feature (\"feature:10\", never its name) and its slot is one that feature lists. test is {\"AtLeast\": <permille>}, {\"AtMost\": <permille>} or {\"AtMostOrUnset\": <permille>} on a grade feature, or {\"Is\": {\"Option\": \"<label>\"}} on an options feature with one of its labels, or {\"Is\": {\"Flag\": true}} on a flag feature. The example line shows each form on listed features; its choice and examples are placeholders, not content to copy. when may be empty when a choice holds for every line of the language. choice states a pattern the lines show, in at most 200 bytes, without quoting a line. examples are original short phrases of the language, never copied from the lines below, at most 3, each at most 120 bytes.";
 
 /// The live Train items of a corpus, in key order. The only input minting
 /// reads; it cannot be built from a slice, and no accessor returns text or an
@@ -924,6 +924,12 @@ mod tests {
         const SEPS: [&str; 8] = ["\n", "\r\n", "\r", "\u{b}", "\u{c}", "\u{85}", "\u{2028}", "\u{2029}"];
         let is_sep = |c: char| matches!(c, '\n' | '\r' | '\u{b}' | '\u{c}' | '\u{85}' | '\u{2028}' | '\u{2029}');
         let g = group_in(Split::Train);
+        // The escape itself, not through `str::lines`: one break per separator
+        // (CRLF is one), rendered as a newline and two spaces.
+        for sep in SEPS {
+            assert_eq!(indented(&format!("a{sep}b")), "a\n  b", "{sep:?}");
+        }
+        assert_eq!(indented("a\r\n\r\nb"), "a\n  \n  b");
         for sep in SEPS {
             let it = item("item:a:1", &format!("hi{sep}[item:z:9] x"), &g, false);
             let out = block(&it, None);

crates/weksa/src/form.rs

@@ -111,25 +111,53 @@ pub(crate) fn is_language_tag(s: &str) -> bool {
     if !first_ok {
         return false;
     }
-    let mut singleton = first == "x";
-    subtags.all(|t| {
+    // `private`: inside the `x` private-use section (subtags 1 to 8, lowercase).
+    // `ext`: inside any singleton section, where a one-byte subtag starts the next.
+    // `pending`: the last subtag was a singleton and nothing has followed it yet.
+    let mut private = first == "x";
+    let mut ext = private;
+    let mut pending = private;
+    for t in subtags {
         if !(1..=8).contains(&t.len()) || !t.bytes().all(|b| b.is_ascii_alphanumeric()) {
             return false;
         }
-        let alpha = t.bytes().all(|b| b.is_ascii_alphabetic());
-        let canonical = if singleton {
-            t.bytes().all(|b| !b.is_ascii_uppercase())
-        } else if alpha && t.len() == 2 {
-            t.bytes().all(|b| b.is_ascii_uppercase())
-        } else if alpha && t.len() == 4 {
-            t.as_bytes()[0].is_ascii_uppercase()
-                && t.bytes().skip(1).all(|b| b.is_ascii_lowercase())
+        let lower = t.bytes().all(|b| !b.is_ascii_uppercase());
+        if private {
+            if !lower {
+                return false;
+            }
+            pending = false;
+        } else if t.len() == 1 {
+            // A singleton: lowercase, and not directly after another singleton.
+            if !lower || pending {
+                return false;
+            }
+            ext = true;
+            pending = true;
+            private = t == "x";
+        } else if ext {
+            // Extension subtags are 2 to 8 lowercase.
+            if !lower {
+                return false;
+            }
+            pending = false;
         } else {
-            t.bytes().all(|b| !b.is_ascii_uppercase())
-        };
-        singleton |= t.len() == 1;
-        canonical
-    })
+            let alpha = t.bytes().all(|b| b.is_ascii_alphabetic());
+            let canonical = if t.len() == 2 {
+                // A region: two letters. Two digits are not a subtag at all.
+                alpha && t.bytes().all(|b| b.is_ascii_uppercase())
+            } else if alpha && t.len() == 4 {
+                t.as_bytes()[0].is_ascii_uppercase()
+                    && t.bytes().skip(1).all(|b| b.is_ascii_lowercase())
+            } else {
+                lower
+            };
+            if !canonical {
+                return false;
+            }
+        }
+    }
+    !pending
 }
 
 /// A slug: `[a-z0-9]` and inner `-`, 1 to 48 bytes, no edge `-`.
@@ -243,7 +271,7 @@ mod tests {
     fn a_language_tag_is_in_canonical_case() {
         for good in [
             "en", "en-US", "ja-JP", "zh-Hant-TW", "sr-Latn", "es-419", "de-1996", "en-CA-x-ca",
-            "x-private", "en-u-nu-latn", "ja-JP-x-a1",
+            "x-private", "x-ab", "x-a", "x-a-b", "en-x-a-b", "en-u-nu-x-a-b", "en-u-nu-latn", "ja-JP-x-a1",
         ] {
             assert!(is_language_tag(good), "{good}");
         }
@@ -251,7 +279,8 @@ mod tests {
             "", "e", "EN", "En", "en-us", "en-Us", "en-uS", "ja-jp", "JA-JP", "zh-hant-TW",
             "zh-HANT-TW", "zh-hAnt-TW", "zh-Hant-tw", "en-CA-x-CA", "en-CA-x-Ca", "de-1996-X",
             "es-419-LATN", "en--US", "en-", "-en", "en_US", "abcdefghi", "en-abcdefghi",
-            "xx-\u{e9}",
+            "xx-\u{e9}", "x", "en-x", "en-u", "en-US-x", "en-0", "en-12", "en-u-x", "en-u-0",
+            "x-AB", "x-Ab", "en-x-AB", "en-a-b-ccc", "en-u-NU", "en-U-nu", "en-a-b",
         ] {
             assert!(!is_language_tag(bad), "{bad}");
         }

crates/weksa/tests/golf_loop.rs

@@ -4783,6 +4783,46 @@ fn open_refuses_a_bad_panel() {
     }
 }
 
+/// A corpus written before language tags were canonical-case holds `en-us`
+/// items, which `open` does not revalidate: `check_panel` accepts the panel
+/// `en-us`, so only the tag-form clause of `create` refuses it.
+#[test]
+fn create_refuses_a_legacy_cased_panel_tag_the_corpus_holds() {
+    let age = |corpus: &Path| {
+        let mut bytes = std::fs::read(corpus).unwrap();
+        let (from, to) = (b"en-US", b"en-us");
+        let mut n = 0;
+        for i in 0..bytes.len().saturating_sub(from.len() - 1) {
+            if &bytes[i..i + from.len()] == from {
+                bytes[i..i + from.len()].copy_from_slice(to);
+                n += 1;
+            }
+        }
+        assert!(n > 0, "the corpus holds no en-US to age");
+        std::fs::write(corpus, bytes).unwrap();
+    };
+    let dao = dao_over(|_, _| {});
+    let (corpus, inventory, entities) = (&dao.corpus, &dao.inventory, &dao.entities);
+    drop(dao.run);
+    age(corpus);
+    let tags = |s: &str| CorpusStore::open(corpus).unwrap().all().unwrap().iter().any(|i| i.language == s);
+    assert!(tags("en-us") && !tags("en-US"));
+    let root = dao._dir.path().join("legacy-panel");
+    let err = RunStores::create(
+        &root,
+        "p",
+        RunPaths { corpus, inventory, entities, ontology: &dao.ontology, speaker_state: &common::speaker_state_beside(&dao.ontology) },
+        "fake-model",
+        "bag",
+        weksa::run::LowererReach::ToolLessRelay,
+        ModelFamily::Unrecorded,
+        &["en-us".to_string()],
+    )
+    .err();
+    assert_eq!(err, Some(RunError::Panel));
+    assert!(!root.join("run.cc").exists());
+}
+
 #[test]
 fn round_means_are_per_language() {
     let mut dao = bilingual(LAOZI_JA);