crates/weksa/src/run.rs

@@ -35,12 +35,15 @@ use crate::form::{is_attestation, is_slug};
 use crate::lower::lower;
 use crate::model::{ModelPort, PortError};
 use crate::score::{CHRF_VERSION, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate};
-use crate::split::{Split, SplitScheme, fnv1a64, hex16};
+use crate::split::{Split, SplitScheme, fnv1a64, fnv1a64_of, hex16};
 use crate::store::{CorpusStore, EntityStore, InventoryStore, PacketStore, open_cache_over};
 
-/// The digest of the prompt builder's source. Any edit to `context.rs`,
-/// comments included, ends a run; code outside that file is not covered.
-pub const PROMPT_SOURCE_DIGEST: u64 = fnv1a64(include_bytes!("context.rs"));
+/// The digest of the prompt builder's source: `context.rs`, then `split.rs`,
+/// whose scheme and digest function decide which lines the prompt holds as
+/// Train and name every prompt. Any edit to either, comments included, ends a
+/// run; code outside those files is not covered.
+pub const PROMPT_SOURCE_DIGEST: u64 =
+    fnv1a64_of(&[include_bytes!("context.rs"), include_bytes!("split.rs")]);
 
 /// The digest of the lowerer's source. Any edit to `lower.rs`, comments
 /// included, ends a run; code outside that file is not covered.
@@ -387,6 +390,21 @@ pub struct RunStores {
 /// The run's own copy of the inventory, made at `create`.
 const INVENTORY_FILE: &str = "inventory.cc";
 
+/// The corpus, entity and head inventory files against the record's pins.
+fn check_pins(cache: &CultCache, root: &Path, record: &RunRecord) -> Result<(), RunError> {
+    if file_digest(&record.corpus).as_ref() != Some(&record.corpus_digest) {
+        return Err(RunError::Drift(DriftOf::Corpus));
+    }
+    if file_digest(&record.entities).as_ref() != Some(&record.entity_digest) {
+        return Err(RunError::Drift(DriftOf::Entities));
+    }
+    let (path, digest, _) = head_of(cache, root, record)?;
+    if file_digest_at(&path).as_ref() != Some(&digest) {
+        return Err(RunError::Drift(DriftOf::Inventory));
+    }
+    Ok(())
+}
+
 fn file_digest_at(path: &Path) -> Option<String> {
     std::fs::read(path).ok().map(|b| hex16(fnv1a64(&b)))
 }
@@ -453,6 +471,20 @@ impl RunStores {
         paths: RunPaths<'_>,
         model: &str,
         embedder: &str,
+    ) -> Result<Self, RunError> {
+        Self::create_between(root, label, paths, model, embedder, || {})
+    }
+
+    /// `create` with `between` run after the digests are taken and before the
+    /// stores are opened: the seam that lets a test change a store in that window.
+    #[doc(hidden)]
+    pub fn create_between(
+        root: &Path,
+        label: &str,
+        paths: RunPaths<'_>,
+        model: &str,
+        embedder: &str,
+        between: impl FnOnce(),
     ) -> Result<Self, RunError> {
         if !is_slug(label) {
             return Err(RunError::Malformed("label"));
@@ -463,7 +495,7 @@ impl RunStores {
         if !is_attestation(embedder) {
             return Err(RunError::Malformed("embedder"));
         }
-        let mut cache = run_cache(root)?;
+        let cache = run_cache(root)?;
         if one_run(&cache)?.is_some() {
             return Err(RunError::Exists);
         }
@@ -493,10 +525,16 @@ impl RunStores {
                 inventory_digest: file_digest_at(&head).ok_or(RunError::Store)?,
                 ..record
             };
-            cache
-                .put(format!("run:{label}"), &record)
+            between();
+            let mut run = Self::with(root, cache, record)?;
+            // The stores were read between the digests: unless they still match
+            // the pins, what was opened is not what was pinned. Nothing is
+            // recorded for a run that did not pin.
+            check_pins(&run.cache, root, &run.record)?;
+            run.cache
+                .put(format!("run:{label}"), &run.record)
                 .map_err(|_| RunError::Store)?;
-            Self::with(root, cache, record)
+            Ok(run)
         })();
         if pinned.is_err() {
             let _ = std::fs::remove_file(&head);
@@ -507,6 +545,13 @@ impl RunStores {
     /// Reopens the run at `root` from its record, refusing when a pinned store
     /// or the prompt or lowerer source no longer matches.
     pub fn open(root: &Path) -> Result<Self, RunError> {
+        Self::open_between(root, || {})
+    }
+
+    /// `open` with `between` run after the pins are checked and before the
+    /// stores are opened: the seam that lets a test change a store in that window.
+    #[doc(hidden)]
+    pub fn open_between(root: &Path, between: impl FnOnce()) -> Result<Self, RunError> {
         if !root.join("run.cc").is_file() {
             return Err(RunError::NoRun);
         }
@@ -518,17 +563,13 @@ impl RunStores {
         if record.lowerer_digest != hex16(LOWERER_SOURCE_DIGEST) {
             return Err(RunError::Drift(DriftOf::Lowerer));
         }
-        if file_digest(&record.corpus).as_ref() != Some(&record.corpus_digest) {
-            return Err(RunError::Drift(DriftOf::Corpus));
-        }
-        if file_digest(&record.entities).as_ref() != Some(&record.entity_digest) {
-            return Err(RunError::Drift(DriftOf::Entities));
-        }
-        let (path, digest, _) = head_of(&cache, root, &record)?;
-        if file_digest_at(&path).as_ref() != Some(&digest) {
-            return Err(RunError::Drift(DriftOf::Inventory));
-        }
-        Self::with(root, cache, record)
+        check_pins(&cache, root, &record)?;
+        between();
+        let run = Self::with(root, cache, record)?;
+        // Checking and opening are two reads of each file: the opened stores
+        // are the pinned ones only if the pins still hold after the open.
+        check_pins(&run.cache, root, &run.record)?;
+        Ok(run)
     }
 
     fn with(root: &Path, cache: CultCache, record: RunRecord) -> Result<Self, RunError> {

crates/weksa/src/split.rs

@@ -25,12 +25,22 @@ pub enum SplitScheme {
 }
 
 pub const fn fnv1a64(bytes: &[u8]) -> u64 {
+    fnv1a64_of(&[bytes])
+}
+
+/// FNV-1a 64 of the parts laid end to end.
+pub const fn fnv1a64_of(parts: &[&[u8]]) -> u64 {
     let mut h: u64 = 0xcbf2_9ce4_8422_2325;
-    let mut i = 0;
-    while i < bytes.len() {
-        h ^= bytes[i] as u64;
-        h = h.wrapping_mul(0x0000_0100_0000_01b3);
-        i += 1;
+    let mut p = 0;
+    while p < parts.len() {
+        let bytes = parts[p];
+        let mut i = 0;
+        while i < bytes.len() {
+            h ^= bytes[i] as u64;
+            h = h.wrapping_mul(0x0000_0100_0000_01b3);
+            i += 1;
+        }
+        p += 1;
     }
     h
 }

crates/weksa/tests/golf_loop.rs

@@ -349,11 +349,62 @@ fn a_run_from_before_the_copy_does_not_open() {
     assert_eq!(RunStores::open(&root).err(), Some(RunError::Store));
 }
 
+#[test]
+fn a_store_changed_between_its_digest_and_its_open_is_drift() {
+    let mut w = world();
+    let dir = w.dir.path().to_path_buf();
+    let root = dir.join("run");
+    run_over(&dir);
+    let more = |w: &mut common::World| {
+        let item = common::item_in(
+            "item:nibu:2",
+            "Another line.",
+            &common::group_in_bucket(10),
+            common::cast(),
+        );
+        w.corpus.add(item, &w.entities).unwrap();
+    };
+    // The pins hold at the check; the corpus changes before the stores open.
+    assert_eq!(
+        RunStores::open_between(&root, || more(&mut w)).err(),
+        Some(RunError::Drift(DriftOf::Corpus))
+    );
+
+    // The same window at create: nothing is pinned and nothing is left behind,
+    // so the retry creates.
+    let mut w = world();
+    let dir = w.dir.path().to_path_buf();
+    let root = dir.join("run");
+    for name in ["c.cc", "i.cc", "entities.cc"] {
+        assert!(dir.join(name).exists(), "{name}");
+    }
+    let (c, i, e) = (dir.join("c.cc"), dir.join("i.cc"), dir.join("entities.cc"));
+    let paths = || weksa::run::RunPaths {
+        corpus: &c,
+        inventory: &i,
+        entities: &e,
+    };
+    assert_eq!(
+        RunStores::create_between(&root, "t", paths(), "fake-model", "bag", || more(&mut w)).err(),
+        Some(RunError::Drift(DriftOf::Corpus))
+    );
+    assert!(!root.join("inventory.cc").exists());
+    assert_eq!(RunStores::open(&root).err(), Some(RunError::NoRun));
+    RunStores::create(&root, "t", paths(), "fake-model", "bag").unwrap();
+}
+
 #[test]
 fn source_digests_cover_their_files() {
     let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("src");
     let digest = |name: &str| fnv1a64(&std::fs::read(src.join(name)).unwrap());
-    assert_eq!(weksa::run::PROMPT_SOURCE_DIGEST, digest("context.rs"));
+    // The prompt's partition and names come from split.rs: it is covered too.
+    let both = [
+        std::fs::read(src.join("context.rs")).unwrap(),
+        std::fs::read(src.join("split.rs")).unwrap(),
+    ]
+    .concat();
+    assert_eq!(weksa::run::PROMPT_SOURCE_DIGEST, fnv1a64(&both));
+    assert_ne!(weksa::run::PROMPT_SOURCE_DIGEST, digest("context.rs"));
     assert_eq!(weksa::run::LOWERER_SOURCE_DIGEST, digest("lower.rs"));
     assert_ne!(
         weksa::run::PROMPT_SOURCE_DIGEST,