diff --git a/crates/odin-daemon/src/lib.rs b/crates/odin-daemon/src/lib.rs index 7549b75..7c2f115 100644 --- a/crates/odin-daemon/src/lib.rs +++ b/crates/odin-daemon/src/lib.rs @@ -4,15 +4,14 @@ //! read-only projection, a trusted receive time with any service-owned //! presence, and narrow clock/store/signer implementations. -use std::cell::{Cell, Ref, RefCell}; -use std::collections::{BTreeMap, BTreeSet}; -use std::path::PathBuf; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; use anyhow::{Context, Result, bail, ensure}; use chrono::{DateTime, SecondsFormat, Utc}; use cultcache_rs::{ - CacheBackingStore, CultCacheEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, + CultCacheEnvelope, CultCacheExpectedEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, }; use cultnet_rs::{ GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, GAMECULT_RUNTIME_PRESENCE_HEALTH_SIGNING_PURPOSE, @@ -30,6 +29,8 @@ use cultnet_rs::{ verify_service_identity_signature_with_public_key, }; +const CAS_ATTEMPTS: usize = 8; + /// Odin's durable high-water mark for one target's correlation sequence. /// /// The correlation's `publisher_sequence` must strictly increase for as long as @@ -177,9 +178,7 @@ pub trait IdunnProjectionSource { } } -/// Read-only adapter for the atomic projection file published by Idunn. It -/// answers no lookup itself: each [`snapshot`](Self::snapshot) is one read of -/// the file, and every lookup is made against a snapshot. +/// Read-only adapter for the atomic projection file published by Idunn. pub struct CultCacheIdunnProjectionSource { path: PathBuf, } @@ -189,61 +188,23 @@ impl CultCacheIdunnProjectionSource { Self { path: path.into() } } - /// Read the projection file once. An absent file projects nothing. - pub fn snapshot(&self) -> Result { - if !self.path.is_file() { - return Ok(IdunnProjectionSnapshot { - entries: Vec::new(), - }); - } - SingleFileMessagePackBackingStore::new(&self.path) - .pull_all_read_only_snapshot() - .map(|entries| IdunnProjectionSnapshot { entries }) - .map_err(|error| { - // The store's decode step reports its failures as text, so an - // I/O error in the chain here is from opening or reading the - // file and from nothing the file said. - if error.chain().any(|cause| cause.is::()) { - error.context(ProjectionUnreadable) - } else { - error - } - }) + pub fn path(&self) -> &Path { + &self.path } } -/// One reading of Idunn's projection file. Every lookup made against it is -/// answered from the same bytes. The default projects nothing. -#[derive(Default)] -pub struct IdunnProjectionSnapshot { - entries: Vec, -} - -impl IdunnProjectionSnapshot { - /// Every incarnation Idunn projects an Expected for, of every target. Only - /// the keys are read: decoding a record is the refresh of that one - /// incarnation, so a record that will not decode cannot make the list - /// unreadable. A record keyed by anything but an incarnation key is not - /// this contract's and is skipped. - pub fn incarnations(&self) -> BTreeSet { - self.entries - .iter() - .filter(|entry| entry.r#type == IdunnExpectedIncarnationRecord::TYPE) - .filter_map(|entry| IncarnationRef::parse_key(&entry.key)) - .collect() - } -} - -impl IdunnProjectionSource for IdunnProjectionSnapshot { +impl IdunnProjectionSource for CultCacheIdunnProjectionSource { fn projections(&self, target: &str) -> Result> { - let entries = &self.entries; - let anchor = provider_anchor(entries, target)?; + let Some(entries) = self.entries()? else { + return Ok(Vec::new()); + }; + let anchor = provider_anchor(&entries, target)?; // A record keyed by anything but its own incarnation key is not this // contract's. That includes the target-keyed single slot the previous // projection used; an Idunn still writing that shape projects nothing // this reader will act on, rather than something it will misread. let mut projections = Vec::new(); - for envelope in entries { + for envelope in &entries { if envelope.r#type != IdunnExpectedIncarnationRecord::TYPE { continue; } @@ -254,7 +215,7 @@ impl IdunnProjectionSource for IdunnProjectionSnapshot { continue; } projections.push(assemble_projection( - entries, + &entries, anchor.clone(), envelope, &incarnation, @@ -268,17 +229,40 @@ impl IdunnProjectionSource for IdunnProjectionSnapshot { /// record under another incarnation's key is that incarnation's fault, and /// never makes this one unreadable. fn projection(&self, incarnation: &IncarnationRef) -> Result> { - let entries = &self.entries; + let Some(entries) = self.entries()? else { + return Ok(None); + }; let Some(envelope) = unique_envelope( - entries, + &entries, IdunnExpectedIncarnationRecord::TYPE, &incarnation.key(), )? else { return Ok(None); }; - let anchor = provider_anchor(entries, &incarnation.target)?; - assemble_projection(entries, anchor, envelope, incarnation).map(Some) + let anchor = provider_anchor(&entries, &incarnation.target)?; + assemble_projection(&entries, anchor, envelope, incarnation).map(Some) + } +} + +impl CultCacheIdunnProjectionSource { + fn entries(&self) -> Result>> { + if !self.path.is_file() { + return Ok(None); + } + SingleFileMessagePackBackingStore::new(&self.path) + .pull_all_read_only_snapshot() + .map(Some) + .map_err(|error| { + // The store's decode step reports its failures as text, so an + // I/O error in the chain here is from opening or reading the + // file and from nothing the file said. + if error.chain().any(|cause| cause.is::()) { + error.context(ProjectionUnreadable) + } else { + error + } + }) } } @@ -426,261 +410,110 @@ pub struct OdinStoreSnapshot { pub trait OdinTopologyStore { fn read(&self, query: OdinStoreQuery) -> Result; - fn commit( + fn compare_exchange( &self, - query: &OdinStoreQuery, + observed: &OdinStoreSnapshot, replacement_presence: Option<&AdmittedPresence>, replacement_correlation: &StoredCorrelation, - ) -> Result<()>; + ) -> Result; fn withdraw_correlation(&self, incarnation: &IncarnationRef) -> Result<()>; /// Claim the next correlation sequence for this target, never returning a - /// value already published. `prior` is the sequence of the currently stored + /// value already used. `prior` is the sequence of the currently stored /// correlation, or 0 when there is none. /// - /// A sequence is published when the store is flushed. The mark is written - /// in the same atomic write as every correlation that uses it, so the file - /// never holds a correlation ahead of its mark. A sequence reserved and - /// never flushed was never published, and may be claimed again after a - /// restart. + /// The mark is advanced before the correlation that uses it is published, + /// so a crash in between leaves it ahead. That is the safe direction: + /// sequences may skip, they may never repeat. fn reserve_publisher_sequence(&self, target: &str, prior: u64) -> Result; } -impl OdinTopologyStore for &T { - fn read(&self, query: OdinStoreQuery) -> Result { - (*self).read(query) - } - - fn commit( - &self, - query: &OdinStoreQuery, - replacement_presence: Option<&AdmittedPresence>, - replacement_correlation: &StoredCorrelation, - ) -> Result<()> { - (*self).commit(query, replacement_presence, replacement_correlation) - } - - fn withdraw_correlation(&self, incarnation: &IncarnationRef) -> Result<()> { - (*self).withdraw_correlation(incarnation) - } - - fn reserve_publisher_sequence(&self, target: &str, prior: u64) -> Result { - (*self).reserve_publisher_sequence(target, prior) - } -} - -/// One record's identity in a CultCache store: its type, then its key. -pub type RecordId = (String, String); - -fn record_id(envelope: &CultCacheEnvelope) -> RecordId { - (envelope.r#type.clone(), envelope.key.clone()) -} - -/// Odin's store, held in memory for as long as Odin holds its write lease. It -/// persists existing CultNet documents directly instead of inventing a -/// parallel admission schema. -/// -/// The file is read once, by [`MemoryOdinTopologyStore::load`] when the lease -/// is granted, and never again while Odin serves: every lookup is answered -/// from memory. The file is Odin's durability checkpoint and Idunn's read-only -/// crossing. It is written by [`MemoryOdinTopologyStore::flush`] and nothing -/// else. -pub struct MemoryOdinTopologyStore { +/// Odin's durable replay and correlation store. It persists existing CultNet +/// documents directly instead of inventing a parallel admission schema. +pub struct CultCacheOdinTopologyStore { path: PathBuf, - records: RefCell>, - /// Exactly what the file held when it was loaded or last written. - flushed: RefCell>, - /// What every write since the last one that succeeded tried to write. A - /// write can fail after it replaced the file (syncing the directory) or - /// before (anything earlier), and the error does not say which, so the - /// file may hold any of these: each is Odin's own, not another writer's. - /// One is kept per failed write whose records differ from the last. - attempted: RefCell>>, - /// The records differ from what was last loaded or written. - dirty: Cell, } -impl MemoryOdinTopologyStore { - /// Read the store file under its lock. An absent file is an empty store. - /// Records are loaded as they are stored: one that does not decode is that - /// record's failure, met where the record is used, and never stops the - /// load. - pub fn load(path: impl Into) -> Result { - let path = path.into(); - let flushed = SingleFileMessagePackBackingStore::new(&path).pull_all()?; - let mut records = BTreeMap::new(); - for envelope in &flushed { - if records.insert(record_id(envelope), envelope.clone()).is_some() { - eprintln!( - "Odin's store holds {} {} more than once; the last is kept", - envelope.r#type, envelope.key - ); - } - } - Ok(Self { - path, - records: RefCell::new(records), - flushed: RefCell::new(flushed), - attempted: RefCell::new(Vec::new()), - dirty: Cell::new(false), - }) +impl CultCacheOdinTopologyStore { + pub fn new(path: impl Into) -> Self { + Self { path: path.into() } } - pub fn records(&self) -> Ref<'_, BTreeMap> { - self.records.borrow() - } - - /// Store one record in place of any with the same type and key. Storing - /// the record already held changes nothing. - pub fn put(&self, envelope: CultCacheEnvelope) { - let id = record_id(&envelope); - let mut records = self.records.borrow_mut(); - if records.get(&id) != Some(&envelope) { - records.insert(id, envelope); - self.dirty.set(true); - } - } - - /// Remove the record with this type and key, if one is held. - pub fn remove(&self, record_type: &str, key: &str) { - if self - .records - .borrow_mut() - .remove(&(record_type.to_owned(), key.to_owned())) - .is_some() - { - self.dirty.set(true); - } - } - - /// Whether Odin's correlation of this incarnation says it is Ready. No - /// correlation, or one that does not decode, is not Ready. - pub fn correlation_ready(&self, incarnation: &IncarnationRef) -> bool { - self.records - .borrow() - .get(&( - OdinRuntimeTopologyCorrelationRecord::TYPE.to_owned(), - incarnation.key(), - )) - .and_then(|envelope| decode_correlation(&envelope.payload).ok()) - .is_some_and(|correlation| correlation.ready) - } - - pub fn correlated_incarnations(&self) -> BTreeSet { - self.records - .borrow() - .keys() - .filter(|(record_type, _)| record_type == OdinRuntimeTopologyCorrelationRecord::TYPE) - .filter_map(|(_, key)| IncarnationRef::parse_key(key)) - .collect() + pub fn path(&self) -> &Path { + &self.path } /// Remove every correlation not keyed by an incarnation. Those were written /// by the previous, target-keyed Odin; nothing reads them under this - /// contract, and left in place the catalog would serve them to the Verse - /// as current. - pub fn retire_legacy_correlations(&self) -> usize { - let legacy: Vec = self - .records - .borrow() - .keys() - .filter(|(record_type, key)| { - record_type == OdinRuntimeTopologyCorrelationRecord::TYPE - && IncarnationRef::parse_key(key).is_none() - }) - .map(|(_, key)| key.clone()) - .collect(); - for key in &legacy { - self.remove(OdinRuntimeTopologyCorrelationRecord::TYPE, key); - } - legacy.len() - } - - pub fn is_dirty(&self) -> bool { - self.dirty.get() - } - - /// Write every record to the file as one atomic snapshot, if anything - /// changed since the last write; returns whether it wrote. - /// - /// The file must still hold exactly what Odin last loaded or wrote, or what - /// one of the failed writes since then tried to write. Anything else was - /// written by another process, and the write is refused with - /// [`ForeignStoreWrite`] rather than made over it. A failed write leaves - /// the records dirty, so the next write retries it. - pub fn flush(&self) -> Result { - if !self.dirty.get() { - return Ok(false); - } - let current: Vec = self.records.borrow().values().cloned().collect(); - let file = SingleFileMessagePackBackingStore::new(&self.path); - let mut exchanged = file.compare_exchange_snapshot(&self.flushed.borrow(), ¤t); - for attempted in self.attempted.borrow().iter().rev() { - if !matches!(exchanged, Ok(false)) { - break; + /// contract, and left in place `stored_snapshot` would serve them to the + /// Verse as current. Presence records are kept whatever their key: the + /// self publisher sequence is continued from them. + pub fn retire_legacy_correlations(&self) -> Result { + for _ in 0..CAS_ATTEMPTS { + if !self.path.is_file() { + return Ok(0); } - exchanged = file.compare_exchange_snapshot(attempted, ¤t); - } - match exchanged { - Ok(true) => {} - Ok(false) => return Err(ForeignStoreWrite.into()), - Err(error) => { - let mut attempted = self.attempted.borrow_mut(); - if attempted.last() != Some(¤t) { - attempted.push(current); - } - return Err(error); + let entries = + SingleFileMessagePackBackingStore::new(&self.path).pull_all_read_only_snapshot()?; + let retained = entries + .iter() + .filter(|entry| { + !(entry.r#type == OdinRuntimeTopologyCorrelationRecord::TYPE + && IncarnationRef::parse_key(&entry.key).is_none()) + }) + .cloned() + .collect::>(); + let removed = entries.len() - retained.len(); + if removed == 0 { + return Ok(0); + } + if SingleFileMessagePackBackingStore::new(&self.path) + .compare_exchange_snapshot(&entries, &retained)? + { + return Ok(removed); } } - *self.flushed.borrow_mut() = current; - self.attempted.borrow_mut().clear(); - self.dirty.set(false); - Ok(true) + bail!("Odin topology store changed repeatedly while retiring legacy correlations") } } -/// Odin's store file no longer holds what Odin last loaded or wrote, so another -/// process wrote it. While Odin holds its write lease it is the file's only -/// writer: this is a second writer, and Odin must not write over it. -#[derive(Debug)] -pub struct ForeignStoreWrite; - -impl std::fmt::Display for ForeignStoreWrite { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str("Odin's store file was written by another process") - } -} - -impl std::error::Error for ForeignStoreWrite {} - -impl OdinTopologyStore for MemoryOdinTopologyStore { +impl OdinTopologyStore for CultCacheOdinTopologyStore { fn read(&self, mut query: OdinStoreQuery) -> Result { query.dependencies.sort(); query.dependencies.dedup(); - let records = self.records.borrow(); - let find = - |record_type: &str, key: &str| records.get(&(record_type.to_owned(), key.to_owned())); - let presence = find( + let entries = if self.path.is_file() { + SingleFileMessagePackBackingStore::new(&self.path).pull_all_read_only_snapshot()? + } else { + Vec::new() + }; + let presence_key = + presence_store_key(&query.incarnation, &query.provider_signer_identity_id); + let presence = unique_envelope( + &entries, GameCultRuntimePresenceHealthRecord::TYPE, - &presence_store_key(&query.incarnation, &query.provider_signer_identity_id), - ) + &presence_key, + )? .map(decode_presence_envelope) .transpose()?; - let correlation = find( + let correlation = unique_envelope( + &entries, OdinRuntimeTopologyCorrelationRecord::TYPE, &query.incarnation.key(), - ) + )? .map(decode_correlation_envelope) .transpose()?; let mut dependency_correlations = BTreeMap::new(); for dependency in &query.dependencies { - let current = find(OdinRuntimeTopologyCorrelationRecord::TYPE, &dependency.key()) - .map(decode_correlation_envelope) - .transpose()?; + let current = unique_envelope( + &entries, + OdinRuntimeTopologyCorrelationRecord::TYPE, + &dependency.key(), + )? + .map(decode_correlation_envelope) + .transpose()?; dependency_correlations.insert(dependency.clone(), current); } - drop(records); Ok(OdinStoreSnapshot { query, presence, @@ -689,50 +522,128 @@ impl OdinTopologyStore for MemoryOdinTopologyStore { }) } - fn commit( + fn compare_exchange( &self, - query: &OdinStoreQuery, + observed: &OdinStoreSnapshot, replacement_presence: Option<&AdmittedPresence>, replacement_correlation: &StoredCorrelation, - ) -> Result<()> { - self.put(correlation_envelope( - &query.incarnation.key(), - replacement_correlation, - )); - if let Some(presence) = replacement_presence { - self.put(presence_envelope( - &presence_store_key(&query.incarnation, &query.provider_signer_identity_id), - presence, - )); + ) -> Result { + let presence_key = presence_store_key( + &observed.query.incarnation, + &observed.query.provider_signer_identity_id, + ); + let correlation_key = observed.query.incarnation.key(); + let mut conditions = BTreeMap::new(); + insert_condition( + &mut conditions, + CultCacheExpectedEnvelope { + key: presence_key.clone(), + r#type: GameCultRuntimePresenceHealthRecord::TYPE.into(), + current: observed + .presence + .as_ref() + .map(|presence| presence_envelope(&presence_key, presence)), + }, + )?; + insert_condition( + &mut conditions, + CultCacheExpectedEnvelope { + key: correlation_key.clone(), + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + current: observed + .correlation + .as_ref() + .map(|correlation| correlation_envelope(&correlation_key, correlation)), + }, + )?; + for (dependency, correlation) in &observed.dependency_correlations { + let key = dependency.key(); + insert_condition( + &mut conditions, + CultCacheExpectedEnvelope { + key: key.clone(), + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + current: correlation + .as_ref() + .map(|correlation| correlation_envelope(&key, correlation)), + }, + )?; } - Ok(()) + let mut replacements = vec![correlation_envelope( + &correlation_key, + replacement_correlation, + )]; + if let Some(presence) = replacement_presence { + replacements.push(presence_envelope(&presence_key, presence)); + } + if let Some(parent) = self.path.parent() { + std::fs::create_dir_all(parent)?; + } + SingleFileMessagePackBackingStore::new(&self.path) + .compare_exchange(&conditions.into_values().collect::>(), &replacements) } fn withdraw_correlation(&self, incarnation: &IncarnationRef) -> Result<()> { - self.remove(OdinRuntimeTopologyCorrelationRecord::TYPE, &incarnation.key()); - Ok(()) + let key = incarnation.key(); + for _ in 0..CAS_ATTEMPTS { + if !self.path.is_file() { + return Ok(()); + } + let entries = + SingleFileMessagePackBackingStore::new(&self.path).pull_all_read_only_snapshot()?; + let current = + unique_envelope(&entries, OdinRuntimeTopologyCorrelationRecord::TYPE, &key)?; + let Some(current) = current else { + return Ok(()); + }; + let replacement = entries + .iter() + .filter(|entry| *entry != current) + .cloned() + .collect::>(); + if SingleFileMessagePackBackingStore::new(&self.path) + .compare_exchange_snapshot(&entries, &replacement)? + { + return Ok(()); + } + } + bail!("Odin topology store changed repeatedly while withdrawing correlation") } fn reserve_publisher_sequence(&self, target: &str, prior: u64) -> Result { - let stored = self - .records - .borrow() - .get(&(TOPOLOGY_PUBLISHER_WATERMARK_TYPE.to_owned(), target.to_owned())) - .map(|envelope| decode_publisher_watermark(target, &envelope.payload)) - .transpose()? - .unwrap_or(0); - // The stored correlation can be ahead of the mark on the first pass - // after this record was introduced, so take whichever is higher. - let next = stored - .max(prior) - .checked_add(1) - .context("Odin topology publisher sequence exhausted")?; - self.put(publisher_watermark_envelope( - target, - next, - &Utc::now().to_rfc3339(), - )?); - Ok(next) + for _ in 0..CAS_ATTEMPTS { + let entries = if self.path.is_file() { + SingleFileMessagePackBackingStore::new(&self.path).pull_all_read_only_snapshot()? + } else { + Vec::new() + }; + let current = unique_envelope(&entries, TOPOLOGY_PUBLISHER_WATERMARK_TYPE, target)?; + let stored = current + .map(|envelope| decode_publisher_watermark(target, &envelope.payload)) + .transpose()? + .unwrap_or(0); + // The stored correlation can be ahead of the mark on the first pass + // after this record was introduced, so take whichever is higher. + let next = stored + .max(prior) + .checked_add(1) + .context("Odin topology publisher sequence exhausted")?; + let replacement = publisher_watermark_envelope(target, next, &Utc::now().to_rfc3339())?; + if let Some(parent) = self.path.parent() { + std::fs::create_dir_all(parent)?; + } + if SingleFileMessagePackBackingStore::new(&self.path).compare_exchange( + &[CultCacheExpectedEnvelope { + key: target.into(), + r#type: TOPOLOGY_PUBLISHER_WATERMARK_TYPE.into(), + current: current.cloned(), + }], + &[replacement], + )? { + return Ok(next); + } + } + bail!("Odin topology store changed repeatedly while reserving a publisher sequence") } } @@ -885,177 +796,184 @@ where dependencies: managed_dependency_incarnations(&projection.expected.dependencies), }; - let observed = self.store.read(query)?; - let admitted = self.select_presence(&observed, incoming, authority.as_ref())?; - let dependencies = - self.dependency_evidence(&projection.expected.dependencies, &observed, now)?; - let (presence, mut disagreements, replacement_presence) = match admitted { - SelectedPresence::None => (None, projection_disagreements.clone(), None), - SelectedPresence::Stale(stored) => { - let record = decode_presence(&stored.canonical_bytes)?; - let mut disagreements = projection_disagreements.clone(); + for _ in 0..CAS_ATTEMPTS { + let observed = self.store.read(query.clone())?; + let admitted = self.select_presence(&observed, incoming, authority.as_ref())?; + let dependencies = + self.dependency_evidence(&projection.expected.dependencies, &observed, now)?; + let (presence, mut disagreements, replacement_presence) = match admitted { + SelectedPresence::None => (None, projection_disagreements.clone(), None), + SelectedPresence::Stale(stored) => { + let record = decode_presence(&stored.canonical_bytes)?; + let mut disagreements = projection_disagreements.clone(); + disagreements.push(OdinTopologyDisagreement { + code: "stored-presence-not-current".into(), + expected: projection + .activation + .as_ref() + .map(IdunnRuntimeActivationRecord::canonical_sha256) + .transpose()?, + observed: Some(record.activation_witness_sha256), + }); + (None, disagreements, None) + } + SelectedPresence::Current { + correlation, + replacement, + } => { + let mut disagreements = projection_disagreements.clone(); + disagreements.extend_from_slice(correlation.disagreements()); + (Some(correlation), disagreements, replacement) + } + }; + if let Some(disagreement) = &lease_disagreement { + disagreements.push(disagreement.clone()); + } + if presence.is_none() { disagreements.push(OdinTopologyDisagreement { - code: "stored-presence-not-current".into(), - expected: projection - .activation - .as_ref() - .map(IdunnRuntimeActivationRecord::canonical_sha256) - .transpose()?, - observed: Some(record.activation_witness_sha256), - }); - (None, disagreements, None) - } - SelectedPresence::Current { - correlation, - replacement, - } => { - let mut disagreements = projection_disagreements.clone(); - disagreements.extend_from_slice(correlation.disagreements()); - (Some(correlation), disagreements, replacement) - } - }; - if let Some(disagreement) = &lease_disagreement { - disagreements.push(disagreement.clone()); - } - if presence.is_none() { - disagreements.push(OdinTopologyDisagreement { - code: "signed-presence-missing".into(), - expected: Some("dual-authenticated-runtime-presence".into()), - observed: None, - }); - disagreements.extend(projection.expected.capabilities.iter().enumerate().map( - |(index, capability)| OdinTopologyDisagreement { - code: format!("expected-capability-{index:03}-missing"), - expected: Some(format!( - "{}/{}/{} capacity>={}", - capability.capability, - capability.schema, - capability.compatibility, - capability.minimum_capacity - )), + code: "signed-presence-missing".into(), + expected: Some("dual-authenticated-runtime-presence".into()), observed: None, - }, - )); - } + }); + disagreements.extend(projection.expected.capabilities.iter().enumerate().map( + |(index, capability)| OdinTopologyDisagreement { + code: format!("expected-capability-{index:03}-missing"), + expected: Some(format!( + "{}/{}/{} capacity>={}", + capability.capability, + capability.schema, + capability.compatibility, + capability.minimum_capacity + )), + observed: None, + }, + )); + } - let observed_write_lease = if let Some(presence) = &presence { - correlate_write_lease( - &projection.expected, - lease_sha256.as_deref(), - presence.claim().record().write_lease_sha256.as_deref(), - &presence.claim().record().state, - &mut disagreements, - ) - } else { - None - }; - disagreements.sort_by(|left, right| left.code.cmp(&right.code)); + let observed_write_lease = if let Some(presence) = &presence { + correlate_write_lease( + &projection.expected, + lease_sha256.as_deref(), + presence.claim().record().write_lease_sha256.as_deref(), + &presence.claim().record().state, + &mut disagreements, + ) + } else { + None + }; + disagreements.sort_by(|left, right| left.code.cmp(&right.code)); - let prior = observed - .correlation - .as_ref() - .map(|stored| decode_correlation(&stored.canonical_bytes)) - .transpose()?; - let presence_record = presence.as_ref().map(|value| value.claim().record()); - let ready = presence_record.is_some_and(|record| record.state == "active") - && disagreements.is_empty() - && dependencies - .iter() - .all(|dependency| dependency_permits_ready(&dependency.kind, dependency.ready)); - let mut correlation = OdinRuntimeTopologyCorrelationRecord { - schema_version: ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into(), - target: projection.expected.target.clone(), - expected_projection_sha256: projection.expected.canonical_sha256()?, - expected: true, - current_activation_sha256: projection - .activation - .as_ref() - .map(IdunnRuntimeActivationRecord::canonical_sha256) - .transpose()?, - signed_presence_sha256: presence - .as_ref() - .map(|value| value.claim().signed_presence_sha256().into()), - observed_presence_state: presence_record.map(|record| record.state.clone()), - observed_presence_publisher_sequence: presence_record - .map(|record| record.publisher_sequence), - observed_write_lease_sha256: observed_write_lease, - observed_capabilities: presence_record - .map_or_else(Vec::new, |record| record.capabilities.clone()), - runtime_id: projection.expected.runtime_id.clone(), - runtime_instance_id: projection - .activation - .as_ref() - .map(|activation| activation.runtime_instance_id.clone()), - present: presence.is_some(), - ready, - dependencies, - disagreements, - signer_identity_id: self.signer.identity_id().into(), - // Placeholder. A sequence is only reserved once the facts are - // known to have changed, below; same_correlation_facts compares - // against the prior's sequence regardless of what sits here. - publisher_sequence: prior.as_ref().map_or(0, |record| record.publisher_sequence), - observed_at_unix_millis: now, - signature_algorithm: "ed25519".into(), - signature: Vec::new(), - }; - if let Some(prior) = &prior - && same_correlation_facts(prior, &correlation) - { - return Ok(observed + let prior = observed .correlation - .context("decoded prior correlation disappeared")? - .canonical_bytes); - } - // Reserved only after the dedupe: a refresh that observes nothing - // new must not burn a sequence or change the store. Reserved from a - // mark that survives withdrawal, not derived from the stored - // correlation, so a withdrawn correlation does not take the count - // with it. See TOPOLOGY_PUBLISHER_WATERMARK_SCHEMA. - correlation.publisher_sequence = self.store.reserve_publisher_sequence( - &observed.query.incarnation.target, - prior.as_ref().map_or(0, |record| record.publisher_sequence), - )?; - correlation.signature = self - .signer - .sign_correlation(&correlation.unsigned_signature_payload()?)?; - let canonical = correlation.canonical_bytes()?; - correlation.validate_against_expected(&projection.expected, lease_sha256.as_deref())?; - let (signed, unsigned) = - OdinRuntimeTopologyCorrelationRecord::decode_canonical_signed_payload(&canonical)?; - verify_service_identity_signature_with_public_key::< - OdinTopologyIdentity, - OdinRuntimeTopologyCorrelationPurpose, - >( - self.signer.public_key(), - &unsigned, - &ServiceIdentitySignature { - identity_id: signed.signer_identity_id, - signature: signed.signature, - }, - )?; - if let Some(authority) = &authority { - authenticate_odin_runtime_topology_correlation( - &canonical, - authority, - lease_sha256.as_deref(), + .as_ref() + .map(|stored| decode_correlation(&stored.canonical_bytes)) + .transpose()?; + let presence_record = presence.as_ref().map(|value| value.claim().record()); + let ready = presence_record.is_some_and(|record| record.state == "active") + && disagreements.is_empty() + && dependencies + .iter() + .all(|dependency| dependency_permits_ready(&dependency.kind, dependency.ready)); + let mut correlation = OdinRuntimeTopologyCorrelationRecord { + schema_version: ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into(), + target: projection.expected.target.clone(), + expected_projection_sha256: projection.expected.canonical_sha256()?, + expected: true, + current_activation_sha256: projection + .activation + .as_ref() + .map(IdunnRuntimeActivationRecord::canonical_sha256) + .transpose()?, + signed_presence_sha256: presence + .as_ref() + .map(|value| value.claim().signed_presence_sha256().into()), + observed_presence_state: presence_record.map(|record| record.state.clone()), + observed_presence_publisher_sequence: presence_record + .map(|record| record.publisher_sequence), + observed_write_lease_sha256: observed_write_lease, + observed_capabilities: presence_record + .map_or_else(Vec::new, |record| record.capabilities.clone()), + runtime_id: projection.expected.runtime_id.clone(), + runtime_instance_id: projection + .activation + .as_ref() + .map(|activation| activation.runtime_instance_id.clone()), + present: presence.is_some(), + ready, + dependencies, + disagreements, + signer_identity_id: self.signer.identity_id().into(), + // Placeholder. A sequence is only reserved once the facts are + // known to have changed, below; same_correlation_facts compares + // against the prior's sequence regardless of what sits here. + publisher_sequence: prior.as_ref().map_or(0, |record| record.publisher_sequence), + observed_at_unix_millis: now, + signature_algorithm: "ed25519".into(), + signature: Vec::new(), + }; + if let Some(prior) = &prior + && same_correlation_facts(prior, &correlation) + { + return Ok(observed + .correlation + .context("decoded prior correlation disappeared")? + .canonical_bytes); + } + // Reserved only after the dedupe: a refresh that observes nothing + // new must not burn a sequence or fsync the store. Reserved from a + // mark that survives withdrawal, not derived from the stored + // correlation, so a withdrawn correlation does not take the count + // with it. See TOPOLOGY_PUBLISHER_WATERMARK_SCHEMA. + correlation.publisher_sequence = self.store.reserve_publisher_sequence( + &observed.query.incarnation.target, + prior.as_ref().map_or(0, |record| record.publisher_sequence), + )?; + correlation.signature = self + .signer + .sign_correlation(&correlation.unsigned_signature_payload()?)?; + let canonical = correlation.canonical_bytes()?; + correlation.validate_against_expected(&projection.expected, lease_sha256.as_deref())?; + let (signed, unsigned) = + OdinRuntimeTopologyCorrelationRecord::decode_canonical_signed_payload(&canonical)?; + verify_service_identity_signature_with_public_key::< + OdinTopologyIdentity, + OdinRuntimeTopologyCorrelationPurpose, + >( self.signer.public_key(), - OdinTopologyAuthenticationContext { - trusted_received_at_unix_millis: now, - maximum_age_millis: self.policy.correlation_maximum_age_millis, - maximum_future_skew_millis: self - .policy - .correlation_maximum_future_skew_millis, + &unsigned, + &ServiceIdentitySignature { + identity_id: signed.signer_identity_id, + signature: signed.signature, }, )?; + if let Some(authority) = &authority { + authenticate_odin_runtime_topology_correlation( + &canonical, + authority, + lease_sha256.as_deref(), + self.signer.public_key(), + OdinTopologyAuthenticationContext { + trusted_received_at_unix_millis: now, + maximum_age_millis: self.policy.correlation_maximum_age_millis, + maximum_future_skew_millis: self + .policy + .correlation_maximum_future_skew_millis, + }, + )?; + } + let stored_correlation = StoredCorrelation { + canonical_bytes: canonical.clone(), + stored_at: rfc3339_millis(now)?, + }; + if self.store.compare_exchange( + &observed, + replacement_presence.as_ref(), + &stored_correlation, + )? { + return Ok(canonical); + } } - let stored_correlation = StoredCorrelation { - canonical_bytes: canonical.clone(), - stored_at: rfc3339_millis(now)?, - }; - self.store - .commit(&observed.query, replacement_presence.as_ref(), &stored_correlation)?; - Ok(canonical) + bail!("Odin topology store changed repeatedly during correlation") } fn select_presence( @@ -1701,6 +1619,22 @@ fn correlation_envelope(target: &str, correlation: &StoredCorrelation) -> CultCa } } +fn insert_condition( + conditions: &mut BTreeMap<(String, String), CultCacheExpectedEnvelope>, + condition: CultCacheExpectedEnvelope, +) -> Result<()> { + let identity = (condition.r#type.clone(), condition.key.clone()); + if let Some(current) = conditions.get(&identity) { + ensure!( + current.current == condition.current, + "store read set is incoherent" + ); + } else { + conditions.insert(identity, condition); + } + Ok(()) +} + fn rfc3339_millis(value: u64) -> Result { let value: i64 = value .try_into() @@ -1719,7 +1653,7 @@ fn parse_rfc3339_millis(value: &str) -> Result { #[cfg(test)] mod tests { - use std::path::{Path, PathBuf}; + use std::path::PathBuf; use cultnet_rs::{ GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA, GameCultProviderHealthIdentity, @@ -1765,9 +1699,7 @@ mod tests { ); } let projections = - CultCacheIdunnProjectionSource::new(&projection_path) - .snapshot()? - .projections(&target)?; + CultCacheIdunnProjectionSource::new(&projection_path).projections(&target)?; if projections.is_empty() { println!("== no incarnation-keyed projection for {target}"); return Ok(()); @@ -1868,12 +1800,10 @@ mod tests { candidate.activation = None; candidate.current_lease = None; world.publish_projection(&candidate)?; - let engine = world.engine(NOW); let candidate_incarnation = IncarnationRef::of(&candidate.expected)?; assert_ne!(incumbent.incarnation()?, candidate_incarnation); assert_eq!( CultCacheIdunnProjectionSource::new(&world.projection_path) - .snapshot()? .projections("odin")? .len(), 2 @@ -1926,7 +1856,7 @@ mod tests { }) .collect(); ensure!(store.compare_exchange_snapshot(¤t, &legacy)?); - let source = CultCacheIdunnProjectionSource::new(&world.projection_path).snapshot()?; + let source = CultCacheIdunnProjectionSource::new(&world.projection_path); assert!(source.projections("ghostlight")?.is_empty()); assert!(source.projection(&service.incarnation()?)?.is_none()); Ok(()) @@ -1949,7 +1879,7 @@ mod tests { schema_id: Some(IDUNN_EXPECTED_INCARNATION_SCHEMA.into()), }); ensure!(store.compare_exchange_snapshot(¤t, &with_ghost)?); - let source = CultCacheIdunnProjectionSource::new(&world.projection_path).snapshot()?; + let source = CultCacheIdunnProjectionSource::new(&world.projection_path); assert!(source.projection(&service.incarnation()?)?.is_some()); assert!( source @@ -1961,19 +1891,16 @@ mod tests { } /// A store the target-keyed Odin wrote holds correlations under bare - /// target keys. They are retired from the loaded set; presence history - /// under any key stays. + /// target keys. They are retired; presence history under any key stays. #[test] fn legacy_target_keyed_correlations_are_retired_and_presence_is_kept() -> Result<()> { let world = TestWorld::new()?; let service = world.service("ghostlight", Vec::new(), false)?; let presence = service.signed_presence(1, |_| {})?; - let correlation = world - .engine(NOW) - .admit_presence("ghostlight", &presence, NOW)?; - assert!(world.store.flush()?); - let file = SingleFileMessagePackBackingStore::new(&world.topology_path); - let current = file.pull_all_read_only_snapshot()?; + let engine = world.engine(NOW); + let correlation = engine.admit_presence("ghostlight", &presence, NOW)?; + let store = SingleFileMessagePackBackingStore::new(&world.topology_path); + let current = store.pull_all_read_only_snapshot()?; let mut with_legacy = current.clone(); with_legacy.push(CultCacheEnvelope { key: "ghostlight".into(), @@ -1989,21 +1916,23 @@ mod tests { stored_at: rfc3339_millis(NOW - 1_000)?, schema_id: Some(GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()), }); - ensure!(file.compare_exchange_snapshot(¤t, &with_legacy)?); + ensure!(store.compare_exchange_snapshot(¤t, &with_legacy)?); - let loaded = MemoryOdinTopologyStore::load(&world.topology_path)?; - assert_eq!(loaded.retire_legacy_correlations(), 1); - assert_eq!(loaded.retire_legacy_correlations(), 0); - let after = loaded.records(); - assert!(after.keys().all(|(record_type, key)| { - record_type != OdinRuntimeTopologyCorrelationRecord::TYPE - || IncarnationRef::parse_key(key).is_some() + let odin_store = CultCacheOdinTopologyStore::new(&world.topology_path); + assert_eq!(odin_store.retire_legacy_correlations()?, 1); + assert_eq!(odin_store.retire_legacy_correlations()?, 0); + let after = store.pull_all_read_only_snapshot()?; + assert!(after.iter().all(|entry| { + entry.r#type != OdinRuntimeTopologyCorrelationRecord::TYPE + || IncarnationRef::parse_key(&entry.key).is_some() })); - assert!(after.keys().any(|(_, key)| key == "ghostlight/legacy-signer")); - drop(after); assert!( - world - .engine_on(&loaded, NOW) + after + .iter() + .any(|entry| entry.key == "ghostlight/legacy-signer") + ); + assert!( + engine .current_signed_correlation(&service.incarnation()?)? .is_some() ); @@ -2013,7 +1942,7 @@ mod tests { #[test] fn withdrawing_a_correlation_does_not_reset_its_publisher_sequence() -> Result<()> { let temp = TempDir::new()?; - let store = MemoryOdinTopologyStore::load(temp.path().join("topology.cc"))?; + let store = CultCacheOdinTopologyStore::new(temp.path().join("topology.cc")); assert_eq!(store.reserve_publisher_sequence("heimdall", 0)?, 1); assert_eq!(store.reserve_publisher_sequence("heimdall", 1)?, 2); @@ -2036,275 +1965,6 @@ mod tests { Ok(()) } - /// The store file is read once, when the store is loaded. Replaced behind - /// the loaded store with bytes that do not decode, it changes nothing the - /// store answers: admission, refresh and withdrawal all run from memory. - #[test] - fn a_loaded_store_never_reads_its_file_again() -> Result<()> { - let world = TestWorld::new()?; - let service = world.service("ghostlight", Vec::new(), false)?; - let engine = world.engine(NOW); - let first = engine.admit_presence("ghostlight", &service.signed_presence(1, |_| {})?, NOW)?; - assert!(world.store.flush()?); - - std::fs::write(&world.topology_path, b"not a cultcache store")?; - assert_eq!( - engine.current_signed_correlation(&service.incarnation()?)?, - Some(first) - ); - let second = decode_signed(&engine.admit_presence( - "ghostlight", - &service.signed_presence(2, |_| {})?, - NOW + 1, - )?)?; - assert_eq!(second.publisher_sequence, 2); - assert_eq!( - std::fs::read(&world.topology_path)?, - b"not a cultcache store", - "nothing was written without a flush" - ); - Ok(()) - } - - /// A write is made only when something changed, and only over the file the - /// store last loaded or wrote. A file another process wrote is refused - /// and left as that process wrote it. - #[test] - fn a_flush_writes_only_changes_and_never_over_a_foreign_file() -> Result<()> { - let temp = TempDir::new()?; - let path = temp.path().join("topology.cc"); - let store = MemoryOdinTopologyStore::load(&path)?; - assert!(!store.flush()?); - assert!(!path.exists(), "an unchanged empty store writes nothing"); - - let document = CultCacheEnvelope { - key: "doc-1".into(), - r#type: "ghostlight.doc".into(), - payload: vec![1], - stored_at: rfc3339_millis(NOW)?, - schema_id: Some("ghostlight.doc.v1".into()), - }; - store.put(document.clone()); - assert!(store.flush()?); - store.put(document.clone()); - assert!(!store.flush()?, "storing the record already held is no change"); - - let file = SingleFileMessagePackBackingStore::new(&path); - let mut foreign = document.clone(); - foreign.key = "foreign".into(); - ensure!(file.compare_exchange_snapshot(&[document.clone()], &[foreign.clone()])?); - let mut changed = document; - changed.payload = vec![2]; - store.put(changed); - let error = store.flush().unwrap_err(); - assert!(error.downcast_ref::().is_some(), "{error:#}"); - assert_eq!(file.pull_all_read_only_snapshot()?, vec![foreign]); - Ok(()) - } - - fn peer_document(key: &str) -> Result { - Ok(CultCacheEnvelope { - key: key.into(), - r#type: "ghostlight.doc".into(), - payload: vec![1], - stored_at: rfc3339_millis(NOW)?, - schema_id: Some("ghostlight.doc.v1".into()), - }) - } - - /// A write that fails leaves its change to be written: the next write - /// lands it with nothing changed in between. - #[test] - fn a_failed_write_is_made_by_the_next_without_another_change() -> Result<()> { - let temp = TempDir::new()?; - let path = temp.path().join("topology.cc"); - let store = MemoryOdinTopologyStore::load(&path)?; - store.put(peer_document("doc-1")?); - std::fs::write(&path, b"not a cultcache store")?; - assert!(store.flush().is_err(), "the injected fault fails the write"); - - // The fault clears: the file holds what Odin loaded, which was nothing. - std::fs::remove_file(&path)?; - assert!(store.flush()?, "the failed change is written"); - assert_eq!( - SingleFileMessagePackBackingStore::new(&path).pull_all_read_only_snapshot()?, - vec![peer_document("doc-1")?] - ); - assert!(!store.flush()?, "and nothing is left to write"); - Ok(()) - } - - /// A write can fail after it replaced the file (the directory sync), and - /// the file then holds Odin's own records: the next write is made over - /// them. Once a write lands, those records are no longer Odin's to expect. - #[test] - fn a_write_that_failed_after_replacing_the_file_is_not_another_writer() -> Result<()> { - let temp = TempDir::new()?; - let path = temp.path().join("topology.cc"); - let file = SingleFileMessagePackBackingStore::new(&path); - let store = MemoryOdinTopologyStore::load(&path)?; - store.put(peer_document("doc-1")?); - std::fs::write(&path, b"not a cultcache store")?; - assert!(store.flush().is_err()); - // The state a failed directory sync leaves: the replace landed. - std::fs::remove_file(&path)?; - ensure!(file.compare_exchange_snapshot(&[], &[peer_document("doc-1")?])?); - - store.put(peer_document("doc-2")?); - assert!(store.flush()?, "Odin's own records are written over"); - let written = file.pull_all_read_only_snapshot()?; - assert_eq!(written, vec![peer_document("doc-1")?, peer_document("doc-2")?]); - - // Another writer puts back what the failed write had tried to write. - ensure!(file.compare_exchange_snapshot(&written, &[peer_document("doc-1")?])?); - store.put(peer_document("doc-3")?); - let error = store.flush().unwrap_err(); - assert!(error.downcast_ref::().is_some(), "{error:#}"); - assert_eq!(file.pull_all_read_only_snapshot()?, vec![peer_document("doc-1")?]); - Ok(()) - } - - /// Two failed writes in a row: the first after it replaced the file, the - /// second before (the lock cannot be taken), so the file still holds the - /// first one's records. Both are Odin's own, and the next write is made - /// over the first's. - #[test] - fn a_write_that_failed_before_replacing_keeps_every_earlier_attempt_odins() -> Result<()> { - let temp = TempDir::new()?; - let path = temp.path().join("topology.cc"); - let file = SingleFileMessagePackBackingStore::new(&path); - let store = MemoryOdinTopologyStore::load(&path)?; - store.put(peer_document("doc-1")?); - std::fs::write(&path, b"not a cultcache store")?; - assert!(store.flush().is_err()); - // The state a failed directory sync leaves: the replace landed. - std::fs::remove_file(&path)?; - ensure!(file.compare_exchange_snapshot(&[], &[peer_document("doc-1")?])?); - - store.put(peer_document("doc-2")?); - let lock = temp.path().join("topology.cc.lock"); - std::fs::remove_file(&lock)?; - std::fs::create_dir(&lock)?; - assert!(store.flush().is_err(), "the lock cannot be taken"); - std::fs::remove_dir(&lock)?; - assert_eq!(file.pull_all_read_only_snapshot()?, vec![peer_document("doc-1")?]); - - assert!(store.flush()?, "the first attempt is Odin's own, not a foreign write"); - assert_eq!( - file.pull_all_read_only_snapshot()?, - vec![peer_document("doc-1")?, peer_document("doc-2")?] - ); - Ok(()) - } - - /// Two writes fail before replacing the file, and the file then comes to - /// hold the later attempt's records (the state a failed directory sync of - /// that attempt would leave). Any failed attempt may be what the file - /// holds, not only the first: the next write is made over it. - #[test] - fn the_file_may_hold_a_later_failed_attempt_not_only_the_first() -> Result<()> { - let temp = TempDir::new()?; - let path = temp.path().join("topology.cc"); - let file = SingleFileMessagePackBackingStore::new(&path); - let store = MemoryOdinTopologyStore::load(&path)?; - std::fs::write(&path, b"not a cultcache store")?; - store.put(peer_document("doc-1")?); - assert!(store.flush().is_err()); - store.put(peer_document("doc-2")?); - assert!(store.flush().is_err()); - std::fs::remove_file(&path)?; - ensure!(file.compare_exchange_snapshot( - &[], - &[peer_document("doc-1")?, peer_document("doc-2")?] - )?); - - store.put(peer_document("doc-3")?); - assert!(store.flush()?, "the later attempt is Odin's own"); - assert_eq!( - file.pull_all_read_only_snapshot()?, - vec![peer_document("doc-1")?, peer_document("doc-2")?, peer_document("doc-3")?] - ); - Ok(()) - } - - /// Removing a record is a change of its own: the next flush writes the - /// file without it, with nothing else changed. - #[test] - fn a_removed_record_is_written_out() -> Result<()> { - let temp = TempDir::new()?; - let path = temp.path().join("topology.cc"); - let file = SingleFileMessagePackBackingStore::new(&path); - let store = MemoryOdinTopologyStore::load(&path)?; - store.put(peer_document("doc-1")?); - store.put(peer_document("doc-2")?); - assert!(store.flush()?); - - store.remove("ghostlight.doc", "doc-1"); - assert!(store.flush()?, "the removal is written"); - assert_eq!(file.pull_all_read_only_snapshot()?, vec![peer_document("doc-2")?]); - store.remove("ghostlight.doc", "doc-1"); - assert!(!store.flush()?, "removing what is not held changes nothing"); - Ok(()) - } - - fn file_sequences(path: &Path) -> Result<(u64, Vec)> { - let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; - let mark = unique_envelope(&entries, TOPOLOGY_PUBLISHER_WATERMARK_TYPE, "ghostlight")? - .map(|envelope| decode_publisher_watermark("ghostlight", &envelope.payload)) - .transpose()? - .unwrap_or(0); - let correlations = entries - .iter() - .filter(|entry| entry.r#type == OdinRuntimeTopologyCorrelationRecord::TYPE) - .map(|entry| Ok(decode_correlation(&entry.payload)?.publisher_sequence)) - .collect::>>()?; - Ok((mark, correlations)) - } - - /// A correlation sequence is published when it is written. The mark is - /// written with the correlations that use it, so the file never holds a - /// correlation ahead of its mark, and after a crash (the working set lost - /// without a final write) and a reload, the next correlation is above - /// every sequence the file holds -- even with the correlation withdrawn. - #[test] - fn a_crash_never_republishes_a_written_sequence() -> Result<()> { - let world = TestWorld::new()?; - let service = world.service("ghostlight", Vec::new(), false)?; - for sequence in 1..=2 { - world.engine(NOW).admit_presence( - "ghostlight", - &service.signed_presence(sequence, |_| {})?, - NOW + sequence, - )?; - } - world.withdraw_projection(&service.incarnation()?)?; - world.engine(NOW + 3).refresh(&service.incarnation()?)?; - assert!(world.store.flush()?); - let (mark, correlations) = file_sequences(&world.topology_path)?; - assert_eq!((mark, correlations.len()), (2, 0), "withdrawn, the mark stays"); - - // Changes made and never written, then the crash. - world.publish_projection(&service.projection)?; - world.engine(NOW + 4).admit_presence( - "ghostlight", - &service.signed_presence(3, |_| {})?, - NOW + 4, - )?; - let restarted = MemoryOdinTopologyStore::load(&world.topology_path)?; - assert_eq!(file_sequences(&world.topology_path)?.0, mark); - - let next = decode_signed(&world.engine_on(&restarted, NOW + 5).admit_presence( - "ghostlight", - &service.signed_presence(4, |_| {})?, - NOW + 5, - )?)?; - assert!(next.publisher_sequence > mark, "{} repeats", next.publisher_sequence); - assert!(restarted.flush()?); - let (mark, correlations) = file_sequences(&world.topology_path)?; - assert!(correlations.iter().all(|sequence| *sequence <= mark)); - Ok(()) - } - #[derive(Clone, Copy)] struct FixedClock(u64); @@ -2390,7 +2050,6 @@ mod tests { _temp: TempDir, projection_path: PathBuf, topology_path: PathBuf, - store: MemoryOdinTopologyStore, identity_root: PathBuf, idunn_signer: ServiceIdentitySigner, idunn_anchor: ServiceIdentityTrustAnchor, @@ -2407,11 +2066,9 @@ mod tests { let idunn_anchor = idunn_signer.trust_anchor()?; let odin_signer = enroll_service_identity_at::(&identity_root.join("odin.cc"))?; - let topology_path = temp.path().join("odin-topology.cc"); Ok(Self { projection_path: temp.path().join("idunn-projection.cc"), - store: MemoryOdinTopologyStore::load(&topology_path)?, - topology_path, + topology_path: temp.path().join("odin-topology.cc"), identity_root, idunn_signer, idunn_anchor, @@ -2602,18 +2259,18 @@ mod tests { Ok(()) } - fn engine(&self, now: u64) -> TestEngine<'_> { - self.engine_on(&self.store, now) - } - - /// An engine over another store, as a restarted Odin loads one. It reads - /// the projection once, when it is made, as each serving operation does. - fn engine_on<'a>(&'a self, store: &'a MemoryOdinTopologyStore, now: u64) -> TestEngine<'a> { + fn engine( + &self, + now: u64, + ) -> OdinTopologyAuthority< + CultCacheIdunnProjectionSource, + CultCacheOdinTopologyStore, + &ServiceIdentitySigner, + FixedClock, + > { OdinTopologyAuthority::new( - CultCacheIdunnProjectionSource::new(&self.projection_path) - .snapshot() - .expect("the test projection reads"), - store, + CultCacheIdunnProjectionSource::new(&self.projection_path), + CultCacheOdinTopologyStore::new(&self.topology_path), &self.odin_signer, FixedClock(now), self.idunn_anchor.clone(), @@ -2657,13 +2314,6 @@ mod tests { } } - type TestEngine<'a> = OdinTopologyAuthority< - IdunnProjectionSnapshot, - &'a MemoryOdinTopologyStore, - &'a ServiceIdentitySigner, - FixedClock, - >; - fn digest(byte: char) -> String { format!("sha256-{}", byte.to_string().repeat(64)) } @@ -2805,9 +2455,19 @@ mod tests { let reused = service.signed_presence(2, |record| record.detail = "different".into())?; assert!(engine.admit_presence("ghostlight", &reused, NOW).is_err()); - let (payload, stored_at) = stored_presence(&world, &service)?; - assert_eq!(payload, sequence_two); - assert_eq!(parse_rfc3339_millis(&stored_at)?, NOW); + let entries = SingleFileMessagePackBackingStore::new(&world.topology_path) + .pull_all_read_only_snapshot()?; + let stored = unique_envelope( + &entries, + GameCultRuntimePresenceHealthRecord::TYPE, + &presence_store_key( + &service.incarnation()?, + &service.projection.expected.expected_signer_identity_id, + ), + )? + .unwrap(); + assert_eq!(stored.payload, sequence_two); + assert_eq!(parse_rfc3339_millis(&stored.stored_at)?, NOW); Ok(()) } @@ -2815,21 +2475,20 @@ mod tests { /// received at `NOW`, beyond the skew window, as on yggdrasil 2026-09-29. const RESTART: u64 = NOW + 10_000; - /// The service's presence as the store holds it: payload and receipt time. - fn stored_presence(world: &TestWorld, service: &TestService) -> Result<(Vec, String)> { - let key = presence_store_key( - &service.incarnation()?, - &service.projection.expected.expected_signer_identity_id, - ); - let records = world.store.records(); - let stored = records - .get(&(GameCultRuntimePresenceHealthRecord::TYPE.to_owned(), key)) - .context("no stored presence")?; - Ok((stored.payload.clone(), stored.stored_at.clone())) - } - fn stored_presence_payload(world: &TestWorld, service: &TestService) -> Result> { - Ok(stored_presence(world, service)?.0) + let entries = SingleFileMessagePackBackingStore::new(&world.topology_path) + .pull_all_read_only_snapshot()?; + Ok(unique_envelope( + &entries, + GameCultRuntimePresenceHealthRecord::TYPE, + &presence_store_key( + &service.incarnation()?, + &service.projection.expected.expected_signer_identity_id, + ), + )? + .context("no stored presence")? + .payload + .clone()) } #[test] @@ -3141,11 +2800,9 @@ mod tests { let first = world .engine(NOW) .admit_presence("ghostlight", &presence, NOW)?; - assert!(world.store.flush()?); - let restarted = MemoryOdinTopologyStore::load(&world.topology_path)?; let after_restart = world - .engine_on(&restarted, NOW + 100) + .engine(NOW + 100) .refresh(&service.incarnation()?)? .unwrap(); assert_eq!(after_restart, first); @@ -3168,18 +2825,18 @@ mod tests { ); std::fs::remove_file(&world.projection_path)?; - let engine = world.engine(NOW); assert!( engine .current_signed_correlation(&service.incarnation()?)? .is_none() ); - let records = world.store.records(); - assert!(records.values().any(|entry| { + let entries = SingleFileMessagePackBackingStore::new(&world.topology_path) + .pull_all_read_only_snapshot()?; + assert!(entries.iter().any(|entry| { entry.r#type == GameCultRuntimePresenceHealthRecord::TYPE && entry.payload == presence })); - assert!(!records.values().any(|entry| { + assert!(!entries.iter().any(|entry| { entry.r#type == OdinRuntimeTopologyCorrelationRecord::TYPE && entry.key.starts_with("ghostlight@") })); @@ -3223,7 +2880,6 @@ mod tests { assert!(ready.dependencies[0].provider_evidence_sha256.is_some()); world.replace_provider_generation(&mut provider)?; - let engine = world.engine(NOW); let no_longer_exact = decode_signed(&engine.refresh(&consumer.incarnation()?)?.unwrap())?; assert!(no_longer_exact.present); assert!(!no_longer_exact.ready); diff --git a/crates/odin-daemon/src/main.rs b/crates/odin-daemon/src/main.rs index 509e499..d7d5c39 100644 --- a/crates/odin-daemon/src/main.rs +++ b/crates/odin-daemon/src/main.rs @@ -12,31 +12,31 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use anyhow::{Context, Result, bail, ensure}; use chrono::{DateTime, Utc}; -use cultcache_rs::{CultCacheEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore}; +use cultcache_rs::{ + CultCacheEnvelope, CultCacheExpectedEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, +}; use cultmesh_rs::{ CultMeshRudpDocumentServer, CultMeshRudpDocumentServerOptions, CultMeshRudpPollOutcome, CultMeshRudpRawDocumentReceipt, CultMeshRudpRawDocumentSink, CultMeshRudpSnapshotQuery, CultMeshRudpSnapshotSource, CultMeshSystemClock, }; use cultnet_rs::{ - CultNetMessage, CultNetRawDocumentRecord, CultNetRawPayloadEncoding, CultNetWireContract, - GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, + CultNetRawDocumentRecord, CultNetRawPayloadEncoding, GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, GameCultProviderHealthIdentity, GameCultRuntimeCapability, GameCultRuntimePresenceHealthPurpose, GameCultRuntimePresenceHealthRecord, IDUNN_EXPECTED_INCARNATION_SCHEMA, IDUNN_PROCESS_WRITE_LEASE_SCHEMA, IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME, IDUNN_RUNTIME_ACTIVATION_SCHEMA, IdunnExpectedIncarnationRecord, IdunnProcessWriteLeaseRecord, IdunnRuntimeActivationRecord, - IdunnRuntimeActivationSigner, IdunnServiceIdentity, OdinTopologyIdentity, - ServiceIdentityProfile, + IdunnRuntimeActivationSigner, IdunnServiceIdentity, + OdinRuntimeTopologyCorrelationRecord, OdinTopologyIdentity, ServiceIdentityProfile, ServiceIdentitySigner, ServiceIdentityTrustAnchor, derive_service_identity_id, - encode_cultnet_message_to_vec, open_service_identity_credential_reader, - verify_runtime_authority, + open_service_identity_credential_reader, verify_runtime_authority, }; use fs2::FileExt; use odin_daemon::{ - AuthenticationPolicy, CultCacheIdunnProjectionSource, ForeignStoreWrite, - IdunnProjectionSnapshot, IdunnProjectionSource, IncarnationRef, MemoryOdinTopologyStore, - OdinTopologyAuthority, PresenceAuthorityRefused, ProjectionUnreadable, SystemClock, + AuthenticationPolicy, CultCacheIdunnProjectionSource, CultCacheOdinTopologyStore, + IdunnProjectionSource, IncarnationRef, OdinTopologyAuthority, PresenceAuthorityRefused, + ProjectionUnreadable, SystemClock, }; const TARGET: &str = "odin"; @@ -70,15 +70,17 @@ const POLL_FAILURE_LIMIT: Duration = Duration::from_secs(30); const POLL_FAILURE_BACKOFF: Duration = Duration::from_millis(100); const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(5); const PROJECTION_REFRESH_INTERVAL: Duration = Duration::from_millis(250); -// Odin's own changes (its heartbeat, correlations, the watermark) are written -// at most once per interval, and not at all when nothing changed. A crash loses -// at most this much of them; a correlation sequence that was never written was -// never published. A peer's put is written before Odin accepts it -// (`accept_raw_document`). Operator ruling Q3 B, docs/write-pattern-cut.md. -const FLUSH_INTERVAL: Duration = Duration::from_secs(1); const IDLE_POLL_INTERVAL: Duration = Duration::from_millis(2); const MAX_RECENT_WARMING_PROOFS: usize = 64; const WARMING_PROOF_LIFETIME_MILLIS: u64 = 60_000; +const CAS_ATTEMPTS: usize = 8; + +type TopologyAuthority = OdinTopologyAuthority< + CultCacheIdunnProjectionSource, + CultCacheOdinTopologyStore, + ServiceIdentitySigner, + SystemClock, +>; /// Relative path of Odin's durable store inside the state root. The recipe's /// `topology` state slot in `deployment/idunn/recipe.toml` is the authority for @@ -108,53 +110,16 @@ struct ProcessWriteLeaseGuard { sha256: String, } -/// Odin's topology authority once its write lease is held: the store it loaded -/// and the credentials that sign for it. An `OdinTopologyAuthority` is built -/// over it for each operation, with the projection read for that operation. -struct Topology { - store: MemoryOdinTopologyStore, - signer: ServiceIdentitySigner, - idunn_anchor: ServiceIdentityTrustAnchor, -} - -impl Topology { - fn authority( - &self, - projections: P, - ) -> OdinTopologyAuthority< - P, - &MemoryOdinTopologyStore, - &ServiceIdentitySigner, - SystemClock, - > { - OdinTopologyAuthority::new( - projections, - &self.store, - &self.signer, - SystemClock, - self.idunn_anchor.clone(), - AuthenticationPolicy::default(), - ) - } -} - struct RuntimeState { options: Options, candidate: SocketAddr, authority_material: RuntimeAuthority, idunn_anchor: Option, topology_signer: Option>, - topology: Option, + topology: Option, write_lease: Option, write_lease_path: PathBuf, - /// The largest snapshot response the server sends: its own - /// `max_snapshot_response_bytes`, set by `serve_with` from the options it - /// builds the server with. `None` until then, and Odin cannot activate - /// without it. - max_snapshot_response_bytes: Option, recent_warming_proofs: VecDeque<(String, u64)>, - /// Counts from this launch. Every launch is a fresh activation, and - /// presences are ordered only within one activation. publisher_sequence: u64, log_gate: RefCell, } @@ -188,8 +153,7 @@ impl RuntimeState { require_expected_contract(&authority_material.expected, candidate)?; let idunn_anchor = read_trust_anchor::(&options.idunn_anchor)?; - let projection_source = - CultCacheIdunnProjectionSource::new(&options.idunn_projection).snapshot()?; + let projection_source = CultCacheIdunnProjectionSource::new(&options.idunn_projection); // This process is one exact incarnation, named by the Expected digest // in its immutable runtime bundle. The projection may carry other // incarnations of `odin` at the same time -- the one being replaced, @@ -234,6 +198,14 @@ impl RuntimeState { ) })?, )?; + let store_sequence = prior_self_publisher_sequence( + &options.store, + authority_material + .provider_signer + .entry() + .identity_id + .as_str(), + )?; Ok(Self { options, @@ -244,9 +216,8 @@ impl RuntimeState { topology: None, write_lease: None, write_lease_path: PathBuf::from(required_environment(PROCESS_WRITE_LEASE_ENVIRONMENT)?), - max_snapshot_response_bytes: None, recent_warming_proofs: VecDeque::new(), - publisher_sequence: 0, + publisher_sequence: store_sequence, log_gate: RefCell::default(), }) } @@ -259,9 +230,6 @@ impl RuntimeState { if self.activated() { return Ok(true); } - let limit = self - .max_snapshot_response_bytes - .context("Odin cannot activate before its server is built")?; let Some(lease) = acquire_process_write_lease( &self.write_lease_path, &self.authority_material, @@ -271,7 +239,6 @@ impl RuntimeState { return Ok(false); }; let Some(projected) = CultCacheIdunnProjectionSource::new(&self.options.idunn_projection) - .snapshot()? .projection(&self_incarnation(&self.authority_material))? else { return Ok(false); @@ -279,18 +246,6 @@ impl RuntimeState { if projected.current_lease.as_ref() != Some(&lease.record) { return Ok(false); } - // The one read of the store file while this process lives: from here - // on the working set is in memory, and the file is only written. - let store = MemoryOdinTopologyStore::load(&self.options.store)?; - // Correlations the previous, target-keyed Odin left in this store are - // not this contract's and would otherwise be served to the Verse as - // current. - store.retire_legacy_correlations(); - for (record_type, key, size) in drop_unservable_documents(&store, limit) { - eprintln!( - "Odin dropped {record_type} {key} from its store: its snapshot response is {size} bytes, over the {limit} the server sends" - ); - } let signer = self .topology_signer .take() @@ -300,55 +255,17 @@ impl RuntimeState { .take() .context("Idunn trust anchor was already consumed")?; self.write_lease = Some(lease); - self.topology = Some(Topology { - store, + self.topology = Some(OdinTopologyAuthority::new( + CultCacheIdunnProjectionSource::new(&self.options.idunn_projection), + CultCacheOdinTopologyStore::new(&self.options.store), signer, + SystemClock, idunn_anchor, - }); + AuthenticationPolicy::default(), + )); Ok(true) } - fn topology(&self) -> Result<&Topology> { - self.topology - .as_ref() - .context("Odin topology authority is absent") - } - - /// Write Odin's store file from the working set, if it changed; returns - /// whether it wrote. This is the only write of the file, made on the - /// interval, for a peer's put, and on the way out. The lease is - /// checked immediately before it, so nothing is written once the lease is - /// lost, and a file another process wrote is refused (`ForeignStoreWrite`) - /// rather than written over. Both end Odin (see `survive`). - fn flush(&self) -> Result { - let store = &self.topology()?.store; - if !store.is_dirty() { - return Ok(false); - } - self.require_current_write_lease()?; - store.flush() - } - - /// A peer's put is written before Odin accepts it: this returns `Ok` only - /// once the store holding the put is on disk. The write is the one `flush`, - /// so it carries everything else that changed too. A write that fails - /// refuses the put, and the server answers a refusal with an application - /// rejection naming the put. - /// - /// What this promises is Odin's answer, not the transport's. The RUDP - /// acknowledgement of the datagram carrying a put is sent when it is - /// received, before ordered delivery hands the put here (a later put can - /// be acknowledged while an earlier one is still being resent), so it says - /// nothing about durability. A put accepted here is on disk; a put refused - /// here is rejected by name. - /// - /// A refusal runs one way only. A put refused because its write failed may - /// still become durable: it stays in the working set, so the catalog - /// serves it and the next write lands it (or the failed write had already - /// replaced the file). That is sound because a put is the latest value for - /// its type and key, not an event: a publisher that retries a refused put - /// stores the same value, and one that does not has lost nothing it was - /// promised. fn accept_raw_document(&mut self, receipt: CultMeshRudpRawDocumentReceipt) -> Result<()> { ensure!( self.activated(), @@ -356,12 +273,10 @@ impl RuntimeState { ); validate_raw_document_shape(&receipt.document)?; if receipt.document.schema_id == GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA { - self.admit_presence_document(&receipt.document, receipt.received_at_unix_millis)?; - } else { - persist_generic_document(&self.topology()?.store, &receipt.document)?; + return self + .admit_presence_document(&receipt.document, receipt.received_at_unix_millis); } - self.flush()?; - Ok(()) + persist_generic_document(&self.options.store, &receipt.document) } /// The one admission path for a runtime-presence document, whether a @@ -376,13 +291,10 @@ impl RuntimeState { document.record_key == presence.target, "runtime-presence document key differs from its signed target" ); - let projections = - CultCacheIdunnProjectionSource::new(&self.options.idunn_projection).snapshot()?; - self.topology()?.authority(projections).admit_presence( - &presence.target, - &document.payload, - received_at_unix_millis, - )?; + self.topology + .as_ref() + .context("Odin topology authority is absent")? + .admit_presence(&presence.target, &document.payload, received_at_unix_millis)?; Ok(()) } @@ -524,8 +436,7 @@ impl RuntimeState { fn require_own_projection(&self) -> Result<()> { let path = &self.options.idunn_projection; match CultCacheIdunnProjectionSource::new(path) - .snapshot() - .and_then(|projections| projections.projection(&self_incarnation(&self.authority_material))) + .projection(&self_incarnation(&self.authority_material)) { Err(error) => Err(own_projection_failure(error)), // No file is an I/O condition (Idunn replaces it atomically, so it @@ -562,22 +473,24 @@ impl RuntimeState { Ok(()) } - /// Refresh every incarnation Idunn projects, and every incarnation Odin - /// holds a correlation for, projected or not (the latter are refreshed so - /// their correlations are withdrawn). The projection is read once for the - /// whole pass. - fn refresh_all_correlations(&self) -> Result<()> { + fn refresh_all_correlations(&mut self) -> Result<()> { self.require_current_write_lease()?; - let topology = self.topology()?; - let projections = - CultCacheIdunnProjectionSource::new(&self.options.idunn_projection).snapshot()?; - let mut incarnations = projections.incarnations(); - incarnations.extend(topology.store.correlated_incarnations()); - let authority = topology.authority(projections); + let topology = self + .topology + .as_ref() + .context("Odin topology authority is absent")?; + let mut incarnations = incarnation_keys( + &self.options.idunn_projection, + IdunnExpectedIncarnationRecord::TYPE, + )?; + incarnations.extend(incarnation_keys( + &self.options.store, + OdinRuntimeTopologyCorrelationRecord::TYPE, + )?); // Each incarnation is refreshed on its own: one whose records cannot be // read is that incarnation's failure, and never stops the others. for incarnation in incarnations { - if let Err(error) = authority.refresh(&incarnation) { + if let Err(error) = topology.refresh(&incarnation) { self.log_repeating( &format!("refresh of incarnation {}", incarnation.key()), format!("failed; the rest are unaffected: {error:#}"), @@ -588,14 +501,8 @@ impl RuntimeState { } /// The catalog serves what is stored, one record at a time: a record that - /// does not decode, or whose incarnation's projection does not, is skipped - /// and logged, and is never a reason to refuse every other record. Idunn's - /// projection is read once per query; a projection that cannot be read at - /// all is every presence's failure and no one else's, so the query is - /// answered as if Idunn projected nothing: peer documents are served and - /// presences skipped. Two incarnations of one target holding presences - /// under the same identity are never a refusal either: one is served (see - /// `presence_precedence`) and the other skipped. Odin's + /// does not decode, or whose projection cannot be read, is skipped and + /// logged, and is never a reason to refuse every other record. Odin's /// correlations are not part of the catalog: Idunn reads them from the /// store file, and a target has one per incarnation, so keying them by /// target would collide for the whole deploy window. @@ -603,18 +510,15 @@ impl RuntimeState { &self, query: &CultMeshRudpSnapshotQuery, ) -> Result> { - let records = self.topology()?.store.records(); - let projections = CultCacheIdunnProjectionSource::new(&self.options.idunn_projection) - .snapshot() - .unwrap_or_else(|error| { - self.log_repeating( - "catalog skips every presence", - format!("Idunn's projection cannot be read: {error:#}"), - ); - IdunnProjectionSnapshot::default() - }); + let entries = if self.options.store.is_file() { + SingleFileMessagePackBackingStore::new(&self.options.store) + .pull_all_read_only_snapshot()? + } else { + Vec::new() + }; + let projections = CultCacheIdunnProjectionSource::new(&self.options.idunn_projection); let mut selected = BTreeMap::new(); - for envelope in records.values() { + for envelope in &entries { let document = match self.public_document(&projections, envelope) { Ok(Some(document)) => document, Ok(None) => continue, @@ -630,56 +534,38 @@ impl RuntimeState { continue; } let identity = (document.schema_id.clone(), document.record_key.clone()); - let precedence = self.presence_precedence(envelope); - match selected.entry(identity) { - std::collections::btree_map::Entry::Vacant(slot) => { - slot.insert((document, precedence)); - } - std::collections::btree_map::Entry::Occupied(mut slot) => { - let (Some(this), Some(held)) = (precedence, slot.get().1) else { - bail!("Odin catalog contains duplicate public document identities"); - }; - self.log_repeating( - &format!("catalog serves one presence of {}", document.record_key), - "more than one incarnation holds an admitted presence; the Ready one, or else the latest, is served" - .into(), - ); - if this > held { - slot.insert((document, precedence)); - } - } - } + ensure!( + selected.insert(identity, document).is_none(), + "Odin catalog contains duplicate public document identities" + ); } - Ok(selected.into_values().map(|(document, _)| document).collect()) - } - - /// The order in which presences of one target are preferred when more than - /// one incarnation of it holds an admitted presence, as every target does - /// while it is being replaced: the incarnation whose correlation is Ready - /// first, then the latest observation. `None` for anything but a presence, - /// which has one record per identity and never competes. - fn presence_precedence(&self, envelope: &CultCacheEnvelope) -> Option<(bool, u64)> { - if envelope.r#type != GameCultRuntimePresenceHealthRecord::TYPE { - return None; - } - let presence = decode_presence(&envelope.payload).ok()?; - let incarnation = IncarnationRef::new(presence.target, presence.expected_projection_sha256); - let ready = self.topology().ok()?.store.correlation_ready(&incarnation); - Some((ready, presence.observed_at_unix_millis)) + Ok(selected.into_values().collect()) } /// One stored record as the Verse sees it; `None` when it is not public. fn public_document( &self, - projections: &IdunnProjectionSnapshot, + projections: &CultCacheIdunnProjectionSource, envelope: &CultCacheEnvelope, ) -> Result> { - if envelope.r#type != GameCultRuntimePresenceHealthRecord::TYPE { - return Ok(peer_document(envelope)); - } let Some(schema_id) = envelope.schema_id.clone() else { return Ok(None); }; + if envelope.r#type != GameCultRuntimePresenceHealthRecord::TYPE { + return Ok(is_peer_document_type(&envelope.r#type).then(|| { + CultNetRawDocumentRecord { + schema_id, + record_key: envelope.key.clone(), + stored_at: envelope.stored_at.clone(), + payload_encoding: CultNetRawPayloadEncoding::Messagepack, + payload: envelope.payload.clone(), + source_runtime_id: None, + source_agent_id: None, + source_role: None, + tags: None, + } + })); + } let presence = decode_presence(&envelope.payload)?; let Some(projection) = projections.projection(&IncarnationRef::new( presence.target.clone(), @@ -808,10 +694,13 @@ fn main() -> Result<()> { ); let socket = UdpSocket::bind(candidate) .with_context(|| format!("binding Odin CultNet RUDP candidate {candidate}"))?; - let (state, mut server) = serve_with( + let state = Rc::new(RefCell::new(RuntimeState::open(options, candidate)?)); + let mut server = CultMeshRudpDocumentServer::new( socket, + SinkHandle(state.clone()), + SnapshotHandle(state.clone()), + CultMeshSystemClock::default(), CultMeshRudpDocumentServerOptions::default(), - RuntimeState::open(options, candidate)?, )?; // Idunn runs this process as PID 1 of its own PID namespace, and a @@ -841,59 +730,20 @@ fn main() -> Result<()> { } } - serve(&state, &mut server, &mut ServingTimers::default(), &stopping) -} - -/// Build the server over `socket` and hand it `state`, from one set of -/// options: the largest response the server sends is the limit `state` -/// sizes stored documents against when it activates, so the two cannot -/// disagree. -fn serve_with( - socket: UdpSocket, - options: CultMeshRudpDocumentServerOptions, - mut state: RuntimeState, -) -> Result<(Rc>, OdinServer)> { - state.max_snapshot_response_bytes = Some(options.max_snapshot_response_bytes); - let state = Rc::new(RefCell::new(state)); - let server = CultMeshRudpDocumentServer::new( - socket, - SinkHandle(state.clone()), - SnapshotHandle(state.clone()), - CultMeshSystemClock::default(), - options, - )?; - Ok((state, server)) -} - -/// Serve until stopped on request or ended by a condition `survive` returns -/// (or a dead socket). Every way out makes the last write (`stop`). -fn serve( - state: &Rc>, - server: &mut OdinServer, - timers: &mut ServingTimers, - stopping: &AtomicBool, -) -> Result<()> { - let ended = loop { + // Correlations the previous, target-keyed Odin left in this store are not + // this contract's and would otherwise be served to the Verse as current. + // Presence history is kept: the self publisher sequence continues from it. + CultCacheOdinTopologyStore::new(&state.borrow().options.store).retire_legacy_correlations()?; + // Once serving, only losing the write lease ends Odin (see `survive`). + let mut timers = ServingTimers::default(); + loop { if stopping.load(Ordering::Relaxed) { eprintln!("Odin stopping on request"); - break Ok(()); + return Ok(()); } - match serving_pass(state, server, timers) { - Ok(true) => {} - Ok(false) => thread::sleep(IDLE_POLL_INTERVAL), - Err(error) => break Err(error), + if !serving_pass(&state, &mut server, &mut timers)? { + thread::sleep(IDLE_POLL_INTERVAL); } - }; - stop(&state.borrow()); - ended -} - -/// The last write before Odin ends, under `flush`'s own checks: after -/// `WriteLeaseLost` or `ForeignStoreWrite` it writes nothing. A failure is -/// logged, not returned: Odin is ending either way. -fn stop(state: &RuntimeState) { - if let Err(error) = state.flush() { - eprintln!("Odin's final store write was not made: {error:#}"); } } @@ -903,17 +753,14 @@ fn stop(state: &RuntimeState) { struct ServingTimers { last_heartbeat: Option, last_projection_refresh: Option, - /// When the store file was last written, or a write last failed. - last_flush: Option, poll_failing_since: Option, } type OdinServer = CultMeshRudpDocumentServer; -/// One turn of the serving loop: serve a datagram, refresh the topology, -/// publish Odin's own presence, and write the store file, when each is due. -/// Returns whether the poll made progress. It fails only for the conditions -/// `survive` returns, or a dead socket. +/// One turn of the serving loop: serve a datagram, refresh the topology, and +/// publish Odin's own presence when each is due. Returns whether the poll made +/// progress. It fails only for `WriteLeaseLost` or a dead socket. fn serving_pass( state: &Rc>, server: &mut OdinServer, @@ -930,15 +777,6 @@ fn serving_pass( survive(&state.borrow(), "self-presence publication", published)?; timers.last_heartbeat = Some(Instant::now()); } - if is_due(timers.last_flush, FLUSH_INTERVAL) { - let flushed = state.borrow().flush(); - // Only an attempt starts the interval: with nothing to write, the next - // change is written on the first pass after it. - if !matches!(flushed, Ok(false)) { - timers.last_flush = Some(Instant::now()); - } - survive(&state.borrow(), "store write", flushed)?; - } Ok(progressed) } @@ -962,10 +800,8 @@ impl std::error::Error for WriteLeaseLost {} /// The serving loop's failure policy. Odin's liveness is Odin's own: a failed /// refresh, publication or read is logged and retried on the next pass, never -/// allowed to end the daemon. Only `WriteLeaseLost` and `ForeignStoreWrite`, -/// wherever in the error chain they sit, are returned: each means another -/// writer holds Odin's state, and writing on would be a second writer. As a -/// named temporary rule, so is +/// allowed to end the daemon. Only `WriteLeaseLost`, wherever in the error chain +/// it sits, is returned -- and, as a named temporary rule, so is /// `PresenceAuthorityRefused`: a failure to establish Odin's OWN authority /// (`require_own_projection`), or a self-presence that authority will never /// admit (no verifiable authority, a signer that does not match the anchor, a @@ -981,7 +817,6 @@ fn survive(state: &RuntimeState, what: &str, result: Result) -> Result Ok(Some(value)), Err(error) if error.downcast_ref::().is_some() - || error.downcast_ref::().is_some() || error.downcast_ref::().is_some() => { Err(error) @@ -1342,6 +1177,39 @@ fn read_process_write_lease(path: &Path) -> Result Result { + if !path.is_file() { + return Ok(0); + } + Ok(SingleFileMessagePackBackingStore::new(path) + .pull_all_read_only_snapshot()? + .into_iter() + .filter(|entry| { + entry.r#type == GameCultRuntimePresenceHealthRecord::TYPE + && entry.key.starts_with(TARGET) + }) + .filter_map(|entry| match decode_presence(&entry.payload) { + Ok(presence) => Some(presence), + Err(error) => { + eprintln!( + "Odin startup skipped its stored presence {}: {error:#}", + entry.key + ); + None + } + }) + .filter(|presence| { + presence.target == TARGET && presence.signer_identity_id == signer_identity_id + }) + .map(|presence| presence.publisher_sequence) + .max() + .unwrap_or(0)) +} + /// The document namespaces providers publish to the rendezvous. A provider's /// document is stored and served back only under one of these; Odin's own /// records (correlations, the publisher watermark, every `odin.*` type) and @@ -1395,87 +1263,49 @@ fn is_peer_document_type(document_type: &str) -> bool { }) } -/// One stored peer document as the catalog serves it; `None` for any other -/// record. -fn peer_document(envelope: &CultCacheEnvelope) -> Option { - let schema_id = envelope.schema_id.clone()?; - is_peer_document_type(&envelope.r#type).then(|| CultNetRawDocumentRecord { - schema_id, - record_key: envelope.key.clone(), - stored_at: envelope.stored_at.clone(), - payload_encoding: CultNetRawPayloadEncoding::Messagepack, - payload: envelope.payload.clone(), - source_runtime_id: None, - source_agent_id: None, - source_role: None, - tags: None, - }) -} - -/// CultNet refuses an empty message id, so a response carries at least one -/// byte of it. -const SHORTEST_MESSAGE_ID: &str = "0"; - -/// Drop every stored peer document the server could never serve, and return -/// each one's type, key and response size. A document is unservable when a -/// snapshot response holding it alone, encoded as the server encodes one and -/// with the shortest message id CultNet admits, is larger than the largest -/// response the server sends (`limit`, the server's own -/// `max_snapshot_response_bytes`). Nobody can read such a document, and every -/// write of the store would carry it. Presences are Odin's own admitted -/// records, never this large, and are not checked; nor is anything the -/// catalog does not serve. -fn drop_unservable_documents( - store: &MemoryOdinTopologyStore, - limit: usize, -) -> Vec<(String, String, usize)> { - let unservable: Vec<(String, String, usize)> = store - .records() - .values() - .filter_map(|envelope| { - let response = CultNetMessage::SnapshotResponseRaw { - message_id: SHORTEST_MESSAGE_ID.into(), - documents: vec![peer_document(envelope)?], - }; - let size = - match encode_cultnet_message_to_vec(&response, CultNetWireContract::CultNetSchemaV0) { - Ok(payload) => payload.len(), - // Kept: the catalog meets the same failure when it serves - // the record, and logs it there. - Err(error) => { - eprintln!( - "Odin cannot size {} {} as a snapshot response; kept: {error:#}", - envelope.r#type, envelope.key - ); - return None; - } - }; - (size > limit).then(|| (envelope.r#type.clone(), envelope.key.clone(), size)) - }) - .collect(); - for (record_type, key, _) in &unservable { - store.remove(record_type, key); - } - unservable -} - -fn persist_generic_document( - store: &MemoryOdinTopologyStore, - document: &CultNetRawDocumentRecord, -) -> Result<()> { +fn persist_generic_document(path: &Path, document: &CultNetRawDocumentRecord) -> Result<()> { let document_type = document_type_for_schema(&document.schema_id)?; ensure!( is_peer_document_type(&document_type), "Odin accepts no {document_type} document from a provider" ); - store.put(CultCacheEnvelope { + let replacement = CultCacheEnvelope { key: document.record_key.clone(), - r#type: document_type, + r#type: document_type.clone(), payload: document.payload.clone(), stored_at: document.stored_at.clone(), schema_id: Some(document.schema_id.clone()), - }); - Ok(()) + }; + let store = SingleFileMessagePackBackingStore::new(path); + for _ in 0..CAS_ATTEMPTS { + let entries = if path.is_file() { + store.pull_all_read_only_snapshot()? + } else { + Vec::new() + }; + let mut matches = entries + .iter() + .filter(|entry| entry.r#type == document_type && entry.key == document.record_key); + let current = matches.next().cloned(); + ensure!( + matches.next().is_none(), + "generic CultCache identity is ambiguous" + ); + if current.as_ref() == Some(&replacement) { + return Ok(()); + } + if store.compare_exchange( + &[CultCacheExpectedEnvelope { + key: document.record_key.clone(), + r#type: document_type.clone(), + current, + }], + &[replacement.clone()], + )? { + return Ok(()); + } + } + bail!("Odin catalog changed repeatedly while persisting a provider document") } fn document_type_for_schema(schema_id: &str) -> Result { @@ -1519,6 +1349,28 @@ fn decode_presence(payload: &[u8]) -> Result Result> { + if !path.is_file() { + return Ok(BTreeSet::new()); + } + Ok(SingleFileMessagePackBackingStore::new(path) + .pull_all_read_only_snapshot()? + .into_iter() + .filter(|entry| entry.r#type == record_type) + .filter_map(|entry| IncarnationRef::parse_key(&entry.key)) + .collect()) +} + fn validate_snapshot_filters(query: &CultMeshRudpSnapshotQuery) -> Result<()> { ensure!( !query.message_id.is_empty() && query.message_id.trim() == query.message_id, @@ -1596,12 +1448,12 @@ fn rfc3339_millis(value: u64) -> Result { #[cfg(test)] mod tests { use cultnet_rs::{ - CultNetRudpReliableSendStatus, CultNetRudpSocketTransportConnection, - CultNetRudpSocketTransportOptions, GAMECULT_RUNTIME_PRESENCE_HEALTH_SIGNING_PURPOSE, - GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA, GameCultServiceTrustAnchorRecord, - IdunnExpectedCapability, IdunnExpectedRoute, IdunnRuntimeActivationLaunch, - ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA, OdinRuntimeTopologyCorrelationRecord, - enroll_service_identity_at, + CultNetMessage, CultNetRudpReliableSendStatus, CultNetRudpSocketTransportConnection, + CultNetRudpSocketTransportOptions, CultNetWireContract, + GAMECULT_RUNTIME_PRESENCE_HEALTH_SIGNING_PURPOSE, GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA, + GameCultServiceTrustAnchorRecord, IdunnExpectedCapability, IdunnExpectedRoute, + IdunnRuntimeActivationLaunch, ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA, + encode_cultnet_message_to_vec, enroll_service_identity_at, }; use super::*; @@ -1726,15 +1578,11 @@ mod tests { _temp: tempfile::TempDir, state: Rc>, server: OdinServer, - /// The server's own socket, shared: what is done to it is done to the server. - socket: UdpSocket, timers: ServingTimers, store: PathBuf, lease_path: PathBuf, provider_identity_id: String, lease: IdunnProcessWriteLeaseRecord, - /// Idunn's signer, for activations of further incarnations. - idunn_signer: ServiceIdentitySigner, } fn write_single_record(path: &Path, envelope: CultCacheEnvelope) -> Result<()> { @@ -1767,16 +1615,6 @@ mod tests { /// The same world with the production server bound under `options`. fn activated_odin_serving(options: CultMeshRudpDocumentServerOptions) -> Result { - activated_odin_with(options, |_| Ok(Vec::new())) - } - - /// The same world, with Odin's store file holding what `seed` returns when - /// Odin activates. `seed` runs before the lease is granted, with the - /// runtime that is about to activate, so it can sign Odin's own presences. - fn activated_odin_with( - options: CultMeshRudpDocumentServerOptions, - seed: impl FnOnce(&mut RuntimeState) -> Result>, - ) -> Result { let temp = tempfile::tempdir()?; let root = temp.path(); let socket = UdpSocket::bind("127.0.0.1:0")?; @@ -1870,18 +1708,10 @@ mod tests { topology: None, write_lease: None, write_lease_path: lease_path.clone(), - max_snapshot_response_bytes: None, recent_warming_proofs: VecDeque::new(), publisher_sequence: 0, log_gate: RefCell::default(), }; - let seeded = seed(&mut runtime)?; - if !seeded.is_empty() { - ensure!( - SingleFileMessagePackBackingStore::new(&store).compare_exchange_snapshot(&[], &seeded)?, - "test store seed failed" - ); - } // The lease names the Warming presence Odin signed for Idunn's probe. let warming = runtime.signed_presence_document("warming", "test warming")?; @@ -1937,31 +1767,27 @@ mod tests { "test projection CAS failed" ); - // Without its server Odin has no limit to size stored documents - // against, so it does not activate. ensure!( - runtime.try_activate().is_err(), - "Odin activated before its server was built" - ); - // Built as `main` builds it, so the activation below sizes stored - // documents against this server's own limit. - let server_socket = socket.try_clone()?; - let (state, server) = serve_with(socket, options, runtime)?; - ensure!( - state.borrow_mut().try_activate()?, + runtime.try_activate()?, "Odin did not activate in the fixture" ); + let state = Rc::new(RefCell::new(runtime)); + let server = CultMeshRudpDocumentServer::new( + socket, + SinkHandle(state.clone()), + SnapshotHandle(state.clone()), + CultMeshSystemClock::default(), + options, + )?; Ok(OdinWorld { _temp: temp, state, server, - socket: server_socket, timers: ServingTimers::default(), store, lease_path, provider_identity_id, lease, - idunn_signer, }) } @@ -1970,22 +1796,6 @@ mod tests { serving_pass(&self.state, &mut self.server, &mut self.timers) } - /// Serve as `main` does, with a stop already requested or not. - fn serve(&mut self, stop_requested: bool) -> Result<()> { - serve( - &self.state, - &mut self.server, - &mut self.timers, - &AtomicBool::new(stop_requested), - ) - } - - /// One heartbeat of Odin's own: a change to its bookkeeping, which is - /// written on the interval and not at once. - fn heartbeat(&self) -> Result<()> { - self.state.borrow_mut().publish_self_presence("test heartbeat") - } - /// Replace the lease under Odin with a later epoch. Odin holds a shared /// lock on the lease's lifetime lock, which is what stops Idunn from /// writing the lease under it; the test swaps the file underneath. @@ -2051,43 +1861,18 @@ mod tests { }) } - /// Every record in Odin's working set. - fn working_set(&self) -> Vec { - let state = self.state.borrow(); - let records = state.topology.as_ref().unwrap().store.records(); - records.values().cloned().collect() - } - - /// Every record in Odin's store file. - fn file_records(&self) -> Result> { - SingleFileMessagePackBackingStore::new(&self.store).pull_all_read_only_snapshot() - } - fn stored_keys(&self, record_type: &str) -> Result> { - Ok(self - .working_set() + Ok(SingleFileMessagePackBackingStore::new(&self.store) + .pull_all_read_only_snapshot()? .into_iter() .filter(|entry| entry.r#type == record_type) .map(|entry| entry.key) .collect()) } - /// The publisher sequence of Odin's own admitted presence, 0 when none - /// is held. + /// Odin's own publisher sequence as it is durably stored. fn stored_sequence(&self) -> Result { - let key = format!( - "{}/{}", - self_incarnation(&self.state.borrow().authority_material).key(), - self.provider_identity_id - ); - self.working_set() - .into_iter() - .find(|entry| { - entry.r#type == GameCultRuntimePresenceHealthRecord::TYPE && entry.key == key - }) - .map_or(Ok(0), |entry| { - Ok(decode_presence(&entry.payload)?.publisher_sequence) - }) + prior_self_publisher_sequence(&self.store, &self.provider_identity_id) } /// Run a peer on its own thread while Odin polls its socket, and nothing @@ -2198,30 +1983,28 @@ mod tests { Ok(()) } - /// A store write that fails is logged and retried; it never ends Odin, and - /// the next attempt lands once the fault clears. + /// A publication or refresh that fails is logged and retried; it never + /// ends Odin, and the next attempt lands once the fault clears. #[test] - fn a_failed_store_write_does_not_end_odin() -> Result<()> { + fn a_failed_self_publication_or_refresh_does_not_end_odin() -> Result<()> { let mut odin = activated_odin()?; odin.pass()?; - assert_eq!(odin.file_records()?, odin.working_set(), "the first pass writes"); + assert!(odin.stored_sequence()? > 0, "the first pass publishes"); - let written = std::fs::read(&odin.store)?; std::fs::write(&odin.store, b"not a cultcache store")?; assert!( - provider_put(&odin, "ghostlight.doc.v1", "doc-1", vec![1]).is_err(), - "a put whose write fails is refused" - ); - assert!( - odin.state.borrow().flush().is_err(), - "the injected fault must actually fail the write" + odin.state + .borrow_mut() + .publish_self_presence("probe") + .is_err(), + "the injected fault must actually fail the publication" ); odin.timers = ServingTimers::default(); odin.pass()?; let suppressed = |odin: &OdinWorld| odin.state.borrow().log_gate.borrow().suppressed_total; assert_eq!(suppressed(&odin), 0, "each distinct failure is logged"); - // The same fault on later passes is counted, not printed again. + // The same faults on later passes are counted, not printed again. for repeat in 1..=3 { odin.timers = ServingTimers::default(); odin.pass()?; @@ -2231,13 +2014,12 @@ mod tests { ); } - std::fs::write(&odin.store, written)?; + std::fs::remove_file(&odin.store)?; odin.timers = ServingTimers::default(); odin.pass()?; - assert_eq!( - odin.file_records()?, - odin.working_set(), - "the write after the fault clears lands" + assert!( + odin.stored_sequence()? > 0, + "the publication after the fault clears lands" ); Ok(()) } @@ -2453,25 +2235,24 @@ mod tests { /// A stored presence of Odin's own activation that no longer authenticates /// makes its publisher sequence unknowable; the claim is refused, closed. - /// The store is Odin's alone while it serves, so the bad record is in the - /// file when Odin activates. #[test] fn a_stored_presence_that_no_longer_authenticates_ends_odin() -> Result<()> { - let mut odin = activated_odin_with(CultMeshRudpDocumentServerOptions::default(), |runtime| { - let warming = runtime.signed_presence_document("warming", "earlier launch")?; - // Received two minutes after it was observed: outside the trusted window. - Ok(vec![CultCacheEnvelope { - key: format!( - "{}/{}", - self_incarnation(&runtime.authority_material).key(), - runtime.authority_material.provider_signer.entry().identity_id - ), - r#type: GameCultRuntimePresenceHealthRecord::TYPE.into(), - payload: warming.payload, - stored_at: rfc3339_millis(unix_millis()? + 120_000)?, - schema_id: Some(GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()), - }]) + let mut odin = activated_odin()?; + odin.pass()?; + // Received two minutes after it was observed: outside the trusted window. + let late = rfc3339_millis(unix_millis()? + 120_000)?; + odin.tamper_store(|entries| { + entries + .into_iter() + .map(|mut entry| { + if entry.r#type == GameCultRuntimePresenceHealthRecord::TYPE { + entry.stored_at = late.clone(); + } + entry + }) + .collect() })?; + odin.timers = ServingTimers::default(); assert_refused(&odin.pass().unwrap_err()); Ok(()) } @@ -2511,127 +2292,6 @@ mod tests { Ok(incarnation) } - /// A candidate incarnation of Odin that Idunn has activated, with its - /// Warming presence admitted beside the incumbent's, as every target has - /// for the moment it is being replaced. The candidate's presence is - /// observed after every presence already held. - fn admit_candidate_presence(odin: &OdinWorld) -> Result { - let now = unix_millis()?; - let mut candidate = odin.state.borrow().authority_material.expected.clone(); - candidate.incarnation_id = "odin/generation-2".into(); - candidate.sealed_release_id = digest('9'); - candidate.validate()?; - let incarnation = IncarnationRef::of(&candidate)?; - let launch = - IdunnRuntimeActivationLaunch::issue(&candidate, digest('8'), now - 10, &odin.idunn_signer)?; - let activation = launch.activation().clone(); - let mut credential = Vec::new(); - launch.write_credential(&mut credential)?; - odin.append_projection(vec![ - CultCacheEnvelope { - key: incarnation.key(), - r#type: IdunnExpectedIncarnationRecord::TYPE.into(), - payload: candidate.canonical_bytes()?, - stored_at: rfc3339_millis(now - 30)?, - schema_id: Some(IDUNN_EXPECTED_INCARNATION_SCHEMA.into()), - }, - CultCacheEnvelope { - key: incarnation.key(), - r#type: IdunnRuntimeActivationRecord::TYPE.into(), - payload: activation.canonical_bytes()?, - stored_at: rfc3339_millis(activation.issued_at_unix_millis)?, - schema_id: Some(IDUNN_RUNTIME_ACTIVATION_SCHEMA.into()), - }, - ])?; - // Signed as the candidate process would sign it: its own Expected and - // activation, no lease yet. - thread::sleep(Duration::from_millis(5)); - let document = { - let mut state = odin.state.borrow_mut(); - let authority = &mut state.authority_material; - let expected_sha256 = candidate.canonical_sha256()?; - let activation_sha256 = activation.canonical_sha256()?; - let signer = IdunnRuntimeActivationSigner::from_credential_reader(credential.as_slice())?; - let incumbent = ( - std::mem::replace(&mut authority.expected, candidate), - std::mem::replace(&mut authority.expected_sha256, expected_sha256), - std::mem::replace(&mut authority.activation, activation), - std::mem::replace(&mut authority.activation_sha256, activation_sha256), - std::mem::replace(&mut authority.activation_signer, signer), - ); - let lease = state.write_lease.take(); - let signed = state.signed_presence_document("warming", "test candidate"); - state.write_lease = lease; - let authority = &mut state.authority_material; - ( - authority.expected, - authority.expected_sha256, - authority.activation, - authority.activation_sha256, - authority.activation_signer, - ) = incumbent; - signed? - }; - provider_put_document(odin, document)?; - Ok(incarnation) - } - - /// The presence of `target` the catalog serves, by the incarnation it names. - fn served_presence_incarnation(odin: &OdinWorld) -> Result { - let catalog = odin.catalog()?; - let presences: Vec<_> = catalog - .iter() - .filter(|document| document.schema_id == GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA) - .collect(); - ensure!(presences.len() == 1, "{} presences served", presences.len()); - Ok(decode_presence(&presences[0].payload)?.expected_projection_sha256) - } - - /// While a target is being replaced, its incumbent and its candidate each - /// hold an admitted presence under the same catalog identity. The catalog - /// serves one: the incarnation whose correlation is Ready, even though the - /// other is newer. - #[test] - fn a_target_being_replaced_serves_the_presence_of_its_ready_incarnation() -> Result<()> { - let mut odin = activated_odin()?; - odin.pass()?; - let incumbent = self_incarnation(&odin.state.borrow().authority_material); - let candidate = admit_candidate_presence(&odin)?; - { - let state = odin.state.borrow(); - let store = &state.topology.as_ref().unwrap().store; - assert!(store.correlation_ready(&incumbent)); - assert!(!store.correlation_ready(&candidate), "the candidate is still Warming"); - } - assert_eq!(served_presence_incarnation(&odin)?, incumbent.expected_sha256); - Ok(()) - } - - /// With no correlation to tell them apart, the later presence is served, - /// whichever incarnation holds it. - #[test] - fn a_target_being_replaced_serves_the_later_presence_when_neither_is_ready() -> Result<()> { - let odin = activated_odin()?; - let incumbent = self_incarnation(&odin.state.borrow().authority_material); - odin.heartbeat()?; - let candidate = admit_candidate_presence(&odin)?; - let forget_correlations = || { - let state = odin.state.borrow(); - let store = &state.topology.as_ref().unwrap().store; - for incarnation in [&incumbent, &candidate] { - store.remove(OdinRuntimeTopologyCorrelationRecord::TYPE, &incarnation.key()); - } - }; - forget_correlations(); - assert_eq!(served_presence_incarnation(&odin)?, candidate.expected_sha256); - - thread::sleep(Duration::from_millis(5)); - odin.heartbeat()?; - forget_correlations(); - assert_eq!(served_presence_incarnation(&odin)?, incumbent.expected_sha256); - Ok(()) - } - /// Idunn projects the incumbent and the candidate of a target for the whole /// deploy window, and Odin writes a correlation for each. The catalog does /// not serve correlations, so two of one target cannot collide in it, and @@ -2675,103 +2335,44 @@ mod tests { Ok(()) } - /// One record that does not decode is skipped where it is met: Odin still - /// starts and serves, and the catalog serves everything else. The records - /// are in the file when Odin activates. + /// One record that does not decode is skipped where it is met: the catalog + /// still serves everything else, and startup still finds Odin's sequence. #[test] fn records_that_do_not_decode_never_stop_the_catalog_or_startup() -> Result<()> { - let mut odin = activated_odin_with(CultMeshRudpDocumentServerOptions::default(), |runtime| { - let own_signer = runtime - .authority_material - .provider_signer - .entry() - .identity_id - .clone(); - let stamp = rfc3339_millis(unix_millis()?)?; - let bad_presence = |key: String| CultCacheEnvelope { - key, - r#type: GameCultRuntimePresenceHealthRecord::TYPE.into(), - payload: vec![0xc1], - stored_at: stamp.clone(), - schema_id: Some(GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()), - }; - let earlier = runtime.signed_presence_document("warming", "earlier incarnation")?; - Ok(vec![ - bad_presence("ghost".into()), - bad_presence(format!("{TARGET}@{}/{own_signer}", digest('c'))), - CultCacheEnvelope { - key: format!("{TARGET}@{}/{own_signer}", digest('d')), + let mut odin = activated_odin()?; + odin.pass()?; + let before = odin.stored_sequence()?; + assert!(before > 0); + let stamp = rfc3339_millis(unix_millis()?)?; + let own_signer = odin.provider_identity_id.clone(); + odin.tamper_store(|mut entries| { + for key in [ + "ghost".to_owned(), + format!("{TARGET}@{}/{own_signer}", digest('c')), + ] { + entries.push(CultCacheEnvelope { + key, r#type: GameCultRuntimePresenceHealthRecord::TYPE.into(), - payload: earlier.payload, + payload: vec![0xc1], stored_at: stamp.clone(), schema_id: Some(GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()), - }, - generic_document()?, - ]) + }); + } + entries.push(generic_document().unwrap()); + entries })?; let catalog = odin.catalog()?; assert_eq!( documents_of(&catalog, GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA), 1, - "the presence that decodes is served" + "Odin's own presence is served" ); assert_eq!(documents_of(&catalog, "ghostlight.doc.v1"), 1); - odin.pass()?; - assert!(odin.stored_sequence()? > 0, "Odin publishes past them"); - Ok(()) - } - - /// The bytes this thread has read so far, as the kernel counts them. - fn thread_bytes_read() -> Result { - let io = std::fs::read_to_string("/proc/thread-self/io")?; - Ok(io - .lines() - .find_map(|line| line.strip_prefix("rchar: ")) - .context("no rchar in /proc/thread-self/io")? - .parse()?) - } - - /// A catalog query reads Idunn's projection once, however many stored - /// presences it has to look up there. - #[test] - fn a_catalog_query_reads_the_projection_once() -> Result<()> { - const PRESENCES: usize = 8; - let odin = activated_odin_with(CultMeshRudpDocumentServerOptions::default(), |runtime| { - let stamp = rfc3339_millis(unix_millis()?)?; - (0..PRESENCES) - .map(|index| { - Ok(CultCacheEnvelope { - key: format!("stored-{index}"), - r#type: GameCultRuntimePresenceHealthRecord::TYPE.into(), - payload: runtime.signed_presence_document("warming", "stored")?.payload, - stored_at: stamp.clone(), - schema_id: Some(GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()), - }) - }) - .collect() - })?; - let projection = odin.state.borrow().options.idunn_projection.clone(); - let projection_bytes = std::fs::metadata(&projection)?.len(); - // Every stored presence is looked up in the projection; the filter - // then keeps none of them, so none collides with another in the reply. - let query = CultMeshRudpSnapshotQuery { - session: cultmesh_rs::CultMeshRudpSessionKey { - remote_addr: "127.0.0.1:1".parse()?, - connection_id: 7, - }, - message_id: "lookups".into(), - requested_at_unix_millis: 1, - schema_ids: None, - record_keys: Some(vec!["absent".into()]), - }; - let before = thread_bytes_read()?; - assert!(odin.state.borrow_mut().raw_snapshot(&query)?.is_empty()); - let read = thread_bytes_read()? - before; - assert!(read >= projection_bytes, "the projection was read: {read} bytes"); - assert!( - read < 2 * projection_bytes, - "and only once: {read} bytes read, the projection is {projection_bytes}" + assert_eq!( + prior_self_publisher_sequence(&odin.store, &odin.provider_identity_id)?, + before, + "the undecodable presences do not change the sequence Odin continues from" ); Ok(()) } @@ -2782,7 +2383,10 @@ mod tests { fn a_presence_whose_projection_cannot_be_read_is_skipped_not_fatal() -> Result<()> { let mut odin = activated_odin()?; odin.pass()?; - provider_put(&odin, "ghostlight.doc.v1", "doc-1", vec![1])?; + odin.tamper_store(|mut entries| { + entries.push(generic_document().unwrap()); + entries + })?; odin.tamper_projection(|entries| { entries .into_iter() @@ -2804,26 +2408,6 @@ mod tests { Ok(()) } - /// A projection file that cannot be read at all skips every presence and - /// nothing else: the catalog still serves the peer documents. - #[test] - fn a_projection_file_that_cannot_be_read_skips_presences_not_the_catalog() -> Result<()> { - let mut odin = activated_odin()?; - odin.pass()?; - provider_put(&odin, "ghostlight.doc.v1", "doc-1", vec![1])?; - assert_eq!( - documents_of(&odin.catalog()?, GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA), - 1, - "Odin's own presence is served while the projection reads" - ); - let projection = odin.state.borrow().options.idunn_projection.clone(); - std::fs::write(projection, b"not a cultcache store")?; - let catalog = odin.catalog()?; - assert_eq!(documents_of(&catalog, "ghostlight.doc.v1"), 1); - assert_eq!(documents_of(&catalog, GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA), 0); - Ok(()) - } - /// Every failure to establish Odin's OWN authority ends Odin (temporary /// rule, see `survive`), whatever shape the failure takes: an anchor that /// does not decode or names another schema, an Expected Idunn no longer @@ -2995,23 +2579,6 @@ mod tests { key: &str, payload: Vec, ) -> Result<()> { - provider_put_document( - odin, - CultNetRawDocumentRecord { - schema_id: schema.into(), - record_key: key.into(), - stored_at: rfc3339_millis(unix_millis()?)?, - payload_encoding: CultNetRawPayloadEncoding::Messagepack, - payload, - source_runtime_id: None, - source_agent_id: None, - source_role: None, - tags: None, - }, - ) - } - - fn provider_put_document(odin: &OdinWorld, document: CultNetRawDocumentRecord) -> Result<()> { odin.state .borrow_mut() .accept_raw_document(CultMeshRudpRawDocumentReceipt { @@ -3022,7 +2589,17 @@ mod tests { message_id: "put".into(), transport_sequence: 1, received_at_unix_millis: unix_millis()?, - document, + document: CultNetRawDocumentRecord { + schema_id: schema.into(), + record_key: key.into(), + stored_at: rfc3339_millis(unix_millis()?)?, + payload_encoding: CultNetRawPayloadEncoding::Messagepack, + payload, + source_runtime_id: None, + source_agent_id: None, + source_role: None, + tags: None, + }, }) } @@ -3036,7 +2613,8 @@ mod tests { let mut odin = activated_odin()?; odin.pass()?; let before = odin.stored_sequence()?; - let stored_before = odin.working_set(); + let stored_before = + SingleFileMessagePackBackingStore::new(&odin.store).pull_all_read_only_snapshot()?; // Every kind Odin itself stored (its watermark and correlations, found // rather than named), and the kinds Idunn projects to it. @@ -3072,7 +2650,7 @@ mod tests { assert!(format!("{refused:#}").contains("accepts no"), "{refused:#}"); } assert_eq!( - odin.working_set(), + SingleFileMessagePackBackingStore::new(&odin.store).pull_all_read_only_snapshot()?, stored_before, "no refused put touched the store" ); @@ -3094,11 +2672,12 @@ mod tests { let mut odin = activated_odin()?; odin.pass()?; provider_put(&odin, "ghostlight.doc.v1", "doc-1", vec![1])?; - let stored: BTreeSet = odin - .working_set() - .into_iter() - .map(|entry| entry.r#type) - .collect(); + let stored: BTreeSet = + SingleFileMessagePackBackingStore::new(&odin.store) + .pull_all_read_only_snapshot()? + .into_iter() + .map(|entry| entry.r#type) + .collect(); assert!( stored.len() >= 4, "Odin's store holds more than what it serves: {stored:?}" @@ -3204,8 +2783,6 @@ mod tests { ); let lost = anyhow::Error::new(WriteLeaseLost("gone".into())).context("refreshing"); assert!(survive::<()>(&state, "x", Err(lost)).is_err()); - let foreign = anyhow::Error::new(ForeignStoreWrite).context("writing"); - assert!(survive::<()>(&state, "x", Err(foreign)).is_err()); Ok(()) } @@ -3335,524 +2912,4 @@ mod tests { assert!(!settle_poll(fault(), &mut since, restart).unwrap()); assert!(settle_poll(fault(), &mut since, restart + POLL_FAILURE_LIMIT).is_err()); } - - // ---- one write per interval ---------------------------------------------- - - /// The store file as a reader sees it: which file, and what it holds. Each - /// write replaces the file atomically, so it is a new file. - fn file_identity(path: &Path) -> Result<(u64, Vec)> { - use std::os::unix::fs::MetadataExt; - Ok((std::fs::metadata(path)?.ino(), std::fs::read(path)?)) - } - - fn ago(millis: u64) -> Option { - Instant::now().checked_sub(Duration::from_millis(millis)) - } - - /// Twenty-five of Odin's own changes (heartbeats) inside one interval are - /// one write, made once the interval has passed: not at 0.9 s after the - /// last write, at 1.1 s. - #[test] - fn changes_inside_one_interval_are_one_write() -> Result<()> { - let mut odin = activated_odin()?; - odin.pass()?; - let first = file_identity(&odin.store)?; - for _ in 0..25 { - odin.timers.last_heartbeat = None; - odin.timers.last_flush = Some(Instant::now()); - odin.pass()?; - } - assert!(odin.stored_sequence()? >= 26, "every heartbeat was admitted"); - assert_eq!(file_identity(&odin.store)?, first, "no write inside the interval"); - - odin.timers.last_flush = ago(900); - odin.pass()?; - assert_eq!(file_identity(&odin.store)?, first, "0.9 s is inside the interval"); - - odin.timers.last_flush = ago(1_100); - odin.pass()?; - let second = file_identity(&odin.store)?; - assert_ne!(second.0, first.0, "one write once the interval passed"); - assert_eq!(odin.file_records()?, odin.working_set()); - - // The write itself starts the next interval: a change right after it - // waits. - odin.heartbeat()?; - odin.pass()?; - assert_eq!(file_identity(&odin.store)?, second, "and only one"); - Ok(()) - } - - /// A peer's put is on disk when Odin accepts it, inside the interval, and - /// the write carries Odin's own unwritten changes with it. - #[test] - fn a_put_is_written_before_it_is_accepted() -> Result<()> { - let mut odin = activated_odin()?; - odin.pass()?; - odin.heartbeat()?; - odin.timers.last_flush = Some(Instant::now()); - provider_put(&odin, "ghostlight.doc.v1", "doc-1", vec![1])?; - assert!( - odin.file_records()? - .iter() - .any(|entry| entry.r#type == "ghostlight.doc" && entry.key == "doc-1") - ); - assert_eq!(odin.file_records()?, odin.working_set()); - Ok(()) - } - - /// Two puts pipelined on one session with the first datagram lost on the - /// way: the transport resends it and delivers both in order, and Odin - /// accepts, and so writes, both. The transport may acknowledge the second - /// before the first is resent (see `accept_raw_document`); what holds is - /// that neither put is lost and both are on disk once delivered. - #[test] - fn pipelined_puts_with_a_lost_datagram_are_both_written() -> Result<()> { - let mut odin = activated_odin()?; - odin.pass()?; - let server = odin.server.local_addr()?; - let acknowledged = odin.serve_while(move || -> Result { - // A relay that drops the client's first reliable data datagram. - let relay_socket = UdpSocket::bind("127.0.0.1:0")?; - relay_socket.set_read_timeout(Some(Duration::from_millis(5)))?; - let relay_addr = relay_socket.local_addr()?; - let stop = Arc::new(AtomicBool::new(false)); - let relay = thread::spawn({ - let stop = stop.clone(); - move || -> Result<()> { - let mut client = None; - let mut dropped = false; - let mut buffer = vec![0u8; 65_535]; - while !stop.load(Ordering::Relaxed) { - let Ok((length, from)) = relay_socket.recv_from(&mut buffer) else { - continue; - }; - let datagram = &buffer[..length]; - if from == server { - if let Some(client) = client { - relay_socket.send_to(datagram, client)?; - } - continue; - } - client = Some(from); - let reliable_data = cultnet_rs::decode_rudp_packet(datagram).is_ok_and( - |packet| { - packet.packet_type == cultnet_rs::CultNetRudpPacketType::Data - && packet.reliable - }, - ); - if reliable_data && !dropped { - dropped = true; - continue; - } - relay_socket.send_to(datagram, server)?; - } - Ok(()) - } - }); - let socket = UdpSocket::bind("127.0.0.1:0")?; - socket.set_read_timeout(Some(Duration::from_millis(5)))?; - let mut transport = CultNetRudpSocketTransportConnection::new( - CultNetRudpSocketTransportOptions::client("pipeliner", socket, relay_addr, 0x6100_0001), - )?; - transport.connect(Vec::new())?; - let deadline = Instant::now() + Duration::from_secs(3); - while !transport.connected() { - let _ = transport.receive_once()?; - transport.poll_resends()?; - ensure!(Instant::now() < deadline, "timed out connecting"); - } - let put = |key: &str| -> Result> { - let message = CultNetMessage::DocumentPutRaw { - message_id: format!("put-{key}"), - document: CultNetRawDocumentRecord { - schema_id: "ghostlight.doc.v1".into(), - record_key: key.into(), - stored_at: rfc3339_millis(unix_millis()?)?, - payload_encoding: CultNetRawPayloadEncoding::Messagepack, - payload: vec![7; 64], - source_runtime_id: None, - source_agent_id: None, - source_role: None, - tags: None, - }, - }; - Ok(encode_cultnet_message_to_vec(&message, CultNetWireContract::CultNetSchemaV0)?) - }; - let receipts = [ - transport.send_reliable("schema", put("first")?)?, - transport.send_reliable("schema", put("second")?)?, - ]; - let deadline = Instant::now() + Duration::from_secs(5); - let all_acknowledged = |transport: &CultNetRudpSocketTransportConnection| { - receipts.iter().all(|receipt| { - transport.reliable_send_status(receipt) - == CultNetRudpReliableSendStatus::Acknowledged - }) - }; - while !all_acknowledged(&transport) && Instant::now() < deadline { - let _ = transport.receive_once()?; - transport.poll_resends()?; - } - stop.store(true, Ordering::Relaxed); - relay.join().expect("relay thread panicked")?; - Ok(all_acknowledged(&transport)) - })??; - assert!(acknowledged, "both puts were delivered"); - let written: Vec = odin - .file_records()? - .into_iter() - .filter(|entry| entry.r#type == "ghostlight.doc") - .map(|entry| entry.key) - .collect(); - assert_eq!(written, vec!["first".to_owned(), "second".to_owned()]); - assert_eq!(odin.file_records()?, odin.working_set()); - Ok(()) - } - - /// A put whose write is not made is refused, so it is never accepted: - /// with the write failing, and with the lease lost. - #[test] - fn a_put_whose_write_is_not_made_is_refused() -> Result<()> { - let mut odin = activated_odin()?; - odin.pass()?; - std::fs::write(&odin.store, b"not a cultcache store")?; - assert!(provider_put(&odin, "ghostlight.doc.v1", "doc-1", vec![1]).is_err()); - - let mut odin = activated_odin()?; - odin.pass()?; - let written = file_identity(&odin.store)?; - odin.swap_lease()?; - let error = provider_put(&odin, "ghostlight.doc.v1", "doc-1", vec![1]).unwrap_err(); - assert!(error.downcast_ref::().is_some(), "{error:#}"); - assert_eq!(file_identity(&odin.store)?, written); - Ok(()) - } - - /// A correlation for an incarnation Idunn no longer projects is found - /// through the correlations Odin holds, and the refresh pass withdraws it. - #[test] - fn a_correlation_whose_incarnation_is_no_longer_projected_is_withdrawn() -> Result<()> { - let mut odin = activated_odin()?; - odin.pass()?; - let candidate = project_candidate_incarnation(&odin)?; - odin.state.borrow().refresh_all_correlations()?; - let correlations = || odin.stored_keys(OdinRuntimeTopologyCorrelationRecord::TYPE); - assert!(correlations()?.contains(&candidate.key())); - - odin.tamper_projection(|entries| { - entries - .into_iter() - .filter(|entry| entry.key != candidate.key()) - .collect() - })?; - odin.state.borrow().refresh_all_correlations()?; - assert!(!correlations()?.contains(&candidate.key())); - assert_eq!(correlations()?.len(), 1, "Odin's own stays"); - Ok(()) - } - - /// An interval in which nothing changed writes nothing, however often the - /// write is due. With nothing to write the write step reads nothing - /// either, not even the lease: it is due on every pass until something - /// changes. - #[test] - fn nothing_changed_is_no_write() -> Result<()> { - let mut odin = activated_odin()?; - odin.pass()?; - let written = file_identity(&odin.store)?; - for _ in 0..5 { - odin.timers.last_projection_refresh = None; - odin.timers.last_heartbeat = Some(Instant::now()); - odin.timers.last_flush = None; - odin.pass()?; - } - assert_eq!(file_identity(&odin.store)?, written); - - std::fs::write(&odin.lease_path, b"not a cultcache store")?; - odin.timers.last_projection_refresh = Some(Instant::now()); - odin.timers.last_heartbeat = Some(Instant::now()); - odin.timers.last_flush = None; - odin.pass()?; - assert_eq!(file_identity(&odin.store)?, written); - Ok(()) - } - - /// A peer document whose single-document snapshot response, encoded as the - /// server encodes one with a one-byte message id, is `response_bytes` long. - fn document_answering_in(key: &str, response_bytes: usize) -> Result { - let document = |payload_bytes: usize| -> Result { - Ok(CultCacheEnvelope { - key: key.into(), - r#type: "ghostlight.doc".into(), - payload: vec![7; payload_bytes], - stored_at: rfc3339_millis(unix_millis()?)?, - schema_id: Some("ghostlight.doc.v1".into()), - }) - }; - let response = |envelope: &CultCacheEnvelope| -> Result { - let served = CultNetRawDocumentRecord { - schema_id: "ghostlight.doc.v1".into(), - record_key: envelope.key.clone(), - stored_at: envelope.stored_at.clone(), - payload_encoding: CultNetRawPayloadEncoding::Messagepack, - payload: envelope.payload.clone(), - source_runtime_id: None, - source_agent_id: None, - source_role: None, - tags: None, - }; - let message = CultNetMessage::SnapshotResponseRaw { - message_id: "m".into(), - documents: vec![served], - }; - Ok(encode_cultnet_message_to_vec(&message, CultNetWireContract::CultNetSchemaV0)?.len()) - }; - let overhead = response(&document(1_000)?)? - 1_000; - let sized = document(response_bytes - overhead)?; - ensure!(response(&sized)? == response_bytes, "the document was not sized exactly"); - Ok(sized) - } - - /// A stored peer document the server could never serve is dropped when - /// Odin activates, everything else is served, and the first write removes - /// it from the file. The bound is the server's own limit, from the options - /// it was built with: a document answering in exactly that many bytes is - /// kept, one byte more is dropped. - #[test] - fn a_stored_document_the_server_could_never_serve_is_dropped_at_activation() -> Result<()> { - const LIMIT: usize = 4_096; - let options = CultMeshRudpDocumentServerOptions { - max_snapshot_response_bytes: LIMIT, - ..CultMeshRudpDocumentServerOptions::default() - }; - let mut odin = activated_odin_with(options, |_| { - Ok(vec![ - document_answering_in("at-limit", LIMIT)?, - document_answering_in("over-limit", LIMIT + 1)?, - document_answering_in("small", 2_000)?, - ]) - })?; - let held = odin.stored_keys("ghostlight.doc")?; - assert_eq!(held, vec!["at-limit".to_owned(), "small".to_owned()]); - let catalog = odin.catalog()?; - assert_eq!(documents_of(&catalog, "ghostlight.doc.v1"), 2); - assert!( - odin.file_records()?.iter().any(|entry| entry.key == "over-limit"), - "activation itself writes nothing" - ); - - odin.pass()?; - let written = odin.file_records()?; - assert!(!written.iter().any(|entry| entry.key == "over-limit")); - assert_eq!(written, odin.working_set()); - Ok(()) - } - - /// A due write with nothing to write does not start the interval: the next - /// change is written on the first pass after it. - #[test] - fn with_nothing_to_write_the_next_change_is_written_on_the_next_pass() -> Result<()> { - let mut odin = activated_odin()?; - odin.pass()?; - let written = file_identity(&odin.store)?; - odin.timers.last_projection_refresh = Some(Instant::now()); - odin.timers.last_heartbeat = Some(Instant::now()); - odin.timers.last_flush = ago(1_100); - odin.pass()?; - assert_eq!(file_identity(&odin.store)?, written, "nothing to write"); - - odin.heartbeat()?; - odin.pass()?; - assert_ne!(file_identity(&odin.store)?, written, "the change is written at once"); - assert_eq!(odin.file_records()?, odin.working_set()); - Ok(()) - } - - /// The lease is checked at the write itself: lost between a change and - /// its write, the write is not made and Odin ends. - #[test] - fn a_lease_lost_before_the_write_writes_nothing_and_ends_odin() -> Result<()> { - let mut odin = activated_odin()?; - odin.pass()?; - let written = file_identity(&odin.store)?; - odin.heartbeat()?; - odin.swap_lease()?; - odin.timers.last_projection_refresh = Some(Instant::now()); - odin.timers.last_heartbeat = Some(Instant::now()); - odin.timers.last_flush = None; - let error = odin.pass().unwrap_err(); - assert!( - error.downcast_ref::().is_some(), - "{error:#}" - ); - assert_eq!(file_identity(&odin.store)?, written); - Ok(()) - } - - /// A file another process wrote between Odin's writes ends Odin, and is - /// not written over. - #[test] - fn a_foreign_write_ends_odin_and_is_not_written_over() -> Result<()> { - let mut odin = activated_odin()?; - odin.pass()?; - odin.tamper_store(|entries| { - entries - .into_iter() - .filter(|entry| entry.r#type != GameCultRuntimePresenceHealthRecord::TYPE) - .collect() - })?; - let foreign = file_identity(&odin.store)?; - odin.heartbeat()?; - odin.timers.last_flush = None; - let error = odin.pass().unwrap_err(); - assert!( - error.downcast_ref::().is_some(), - "{error:#}" - ); - assert_eq!(file_identity(&odin.store)?, foreign); - Ok(()) - } - - /// Odin serving, written once, with a change of its own not yet written. - fn odin_with_an_unwritten_change() -> Result { - let mut odin = activated_odin()?; - odin.pass()?; - odin.heartbeat()?; - assert_ne!(odin.file_records()?, odin.working_set()); - Ok(odin) - } - - /// Stopping on request makes the last write. - #[test] - fn a_stop_request_writes_what_changed() -> Result<()> { - let mut odin = odin_with_an_unwritten_change()?; - odin.serve(true)?; - assert_eq!(odin.file_records()?, odin.working_set()); - Ok(()) - } - - /// Ending because Odin's own authority is refused, the lease is still - /// current: the last write is made. - #[test] - fn ending_on_a_refused_authority_writes_what_changed() -> Result<()> { - let mut odin = odin_with_an_unwritten_change()?; - odin.tamper_projection(|entries| { - entries - .into_iter() - .filter(|entry| entry.r#type != IdunnRuntimeActivationRecord::TYPE) - .collect() - })?; - odin.timers = ServingTimers::default(); - assert_refused(&odin.serve(false).unwrap_err()); - assert_eq!(odin.file_records()?, odin.working_set()); - Ok(()) - } - - /// Ending on a dead socket, the lease is still current: the last write is - /// made. The socket's own failure is real (a refused datagram, reported on - /// the server's next receive); only the length of the failing run is set. - #[test] - fn ending_on_a_dead_socket_writes_what_changed() -> Result<()> { - let mut odin = odin_with_an_unwritten_change()?; - let closed = UdpSocket::bind("127.0.0.1:0")?.local_addr()?; - odin.socket.connect(closed)?; - odin.socket.send(b"to nobody")?; - thread::sleep(Duration::from_millis(50)); - odin.timers.poll_failing_since = ago(POLL_FAILURE_LIMIT.as_millis() as u64 + 1_000); - let error = odin.serve(false).unwrap_err(); - assert!(format!("{error:#}").contains("failed every poll"), "{error:#}"); - assert_eq!(odin.file_records()?, odin.working_set()); - Ok(()) - } - - /// Ending because the lease was lost, nothing more is written. - #[test] - fn ending_on_a_lost_lease_writes_nothing() -> Result<()> { - let mut odin = odin_with_an_unwritten_change()?; - let written = file_identity(&odin.store)?; - odin.swap_lease()?; - odin.timers = ServingTimers::default(); - let error = odin.serve(false).unwrap_err(); - assert!(error.downcast_ref::().is_some(), "{error:#}"); - assert_eq!(file_identity(&odin.store)?, written); - Ok(()) - } - - /// Ending because another process wrote the file, nothing is written over - /// it. - #[test] - fn ending_on_a_foreign_write_writes_nothing() -> Result<()> { - let mut odin = odin_with_an_unwritten_change()?; - odin.tamper_store(|entries| { - entries - .into_iter() - .filter(|entry| entry.r#type != GameCultRuntimePresenceHealthRecord::TYPE) - .collect() - })?; - let foreign = file_identity(&odin.store)?; - odin.timers.last_flush = None; - let error = odin.serve(false).unwrap_err(); - assert!(error.downcast_ref::().is_some(), "{error:#}"); - assert_eq!(file_identity(&odin.store)?, foreign); - Ok(()) - } - - // ---- the store file is not working memory ----------------------------- - - /// Once Odin holds its lease, its working set is in memory. Rewriting the - /// store file behind its back changes nothing Odin serves or refreshes; - /// only a restart, which loads the file again, would see it. - #[test] - fn rewriting_the_store_file_behind_odin_changes_nothing_it_serves() -> Result<()> { - let mut odin = activated_odin()?; - provider_put(&odin, "ghostlight.doc.v1", "doc-1", vec![1])?; - odin.pass()?; - let served = odin.catalog()?; - let working = odin.working_set(); - assert_eq!(documents_of(&served, "ghostlight.doc.v1"), 1); - - odin.tamper_store(|entries| { - let mut entries: Vec<_> = entries - .into_iter() - .filter(|entry| entry.r#type == "odin.topology_publisher_watermark") - .collect(); - let mut foreign = generic_document().unwrap(); - foreign.key = "foreign".into(); - entries.push(foreign); - entries - })?; - for _ in 0..3 { - odin.state.borrow().refresh_all_correlations()?; - } - assert_eq!(odin.catalog()?, served); - assert_eq!(odin.working_set(), working); - assert_ne!( - MemoryOdinTopologyStore::load(&odin.store)?.records().len(), - working.len(), - "a restart would load what the file now holds" - ); - Ok(()) - } - - /// Refresh passes and catalog reads never read the store file. With the - /// file replaced by bytes that do not decode, a reader would fail at once; - /// every pass succeeds and logs nothing. - #[test] - fn refresh_passes_and_catalog_reads_never_read_the_store_file() -> Result<()> { - let mut odin = activated_odin()?; - provider_put(&odin, "ghostlight.doc.v1", "doc-1", vec![1])?; - odin.pass()?; - let served = odin.catalog()?; - std::fs::write(&odin.store, b"not a cultcache store")?; - for _ in 0..5 { - odin.state.borrow().refresh_all_correlations()?; - assert_eq!(odin.catalog()?, served); - } - assert!( - odin.state.borrow().log_gate.borrow().seen.is_empty(), - "a pass failed and was logged" - ); - Ok(()) - } } diff --git a/crates/odin-daemon/tests/daemon_process.rs b/crates/odin-daemon/tests/daemon_process.rs deleted file mode 100644 index e0bb98a..0000000 --- a/crates/odin-daemon/tests/daemon_process.rs +++ /dev/null @@ -1,350 +0,0 @@ -//! The real odin-daemon binary, run under an Idunn-shaped world: its bundle, -//! signer descriptors, projection and write lease on disk, and Idunn's warming -//! probe and lease grant made over RUDP as Idunn makes them. It reaches what -//! the in-process tests cannot: `main`'s signal wiring and a process that dies. -//! Unix only, like the daemon. - -use std::fs::File; -use std::net::{SocketAddr, UdpSocket}; -use std::path::{Path, PathBuf}; -use std::process::{Child, Command, ExitStatus, Stdio}; -use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; - -use anyhow::{Result, bail, ensure}; -use cultcache_rs::{CacheBackingStore, CultCacheEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore}; -use cultmesh_rs::{ - CultMeshRudpDocumentPublishOptions, CultMeshRudpSnapshotOptions, - publish_cultnet_message_to_rudp_catalog, request_raw_snapshot_from_rudp_catalog, -}; -use cultnet_rs::*; -use odin_daemon::IncarnationRef; - -fn digest(byte: char) -> String { - format!("sha256-{}", byte.to_string().repeat(64)) -} - -fn now() -> u64 { - SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis() as u64 -} - -fn rfc3339(millis: u64) -> String { - chrono::DateTime::::from_timestamp_millis(millis as i64) - .unwrap() - .to_rfc3339() -} - -/// Replace the whole store at `path` with `entries`. -fn write_store(path: &Path, entries: &[CultCacheEnvelope]) -> Result<()> { - let store = SingleFileMessagePackBackingStore::new(path); - let current = if path.is_file() { - store.pull_all_read_only_snapshot()? - } else { - Vec::new() - }; - ensure!(store.compare_exchange_snapshot(¤t, entries)?, "store write lost a race"); - Ok(()) -} - -fn credential_bytes(store: &Path) -> Result> { - Ok(rmp_serde::to_vec(&SingleFileMessagePackBackingStore::new(store).pull_all()?)?) -} - -struct Odin { - _temp: tempfile::TempDir, - root: PathBuf, - candidate: SocketAddr, - child: Child, - store: PathBuf, -} - -impl Odin { - fn log(&self) -> String { - std::fs::read_to_string(self.root.join("odin.log")).unwrap_or_default() - } - - /// Put one provider document and return once the server acknowledged it. - fn put(&self, key: &str) -> Result<()> { - let message = CultNetMessage::DocumentPutRaw { - message_id: format!("put-{key}"), - document: CultNetRawDocumentRecord { - schema_id: "ghostlight.doc.v1".into(), - record_key: key.into(), - stored_at: rfc3339(now()), - payload_encoding: CultNetRawPayloadEncoding::Messagepack, - payload: vec![1], - source_runtime_id: None, - source_agent_id: None, - source_role: None, - tags: None, - }, - }; - let mut options = CultMeshRudpDocumentPublishOptions::odin(self.candidate, "test-publisher"); - options.flush_timeout = Duration::from_secs(3); - publish_cultnet_message_to_rudp_catalog(&message, options) - } - - fn holds(&self, key: &str) -> Result { - Ok(SingleFileMessagePackBackingStore::new(&self.store) - .pull_all_read_only_snapshot()? - .iter() - .any(|entry| entry.r#type == "ghostlight.doc" && entry.key == key)) - } - - fn signal(&mut self, signal: &str) -> Result { - let pid = self.child.id().to_string(); - ensure!(Command::new("kill").args([signal, &pid]).status()?.success()); - let deadline = Instant::now() + Duration::from_secs(20); - loop { - if let Some(status) = self.child.try_wait()? { - return Ok(status); - } - ensure!(Instant::now() < deadline, "Odin did not exit on {signal}"); - std::thread::sleep(Duration::from_millis(10)); - } - } -} - -impl Drop for Odin { - fn drop(&mut self) { - let _ = self.child.kill(); - let _ = self.child.wait(); - } -} - -/// Launch Odin as Idunn does and grant it its lease; returns once Odin has -/// written its store, which it does on its first pass after activating. -fn start_odin() -> Result { - let temp = tempfile::tempdir()?; - let root = temp.path().to_path_buf(); - let candidate = UdpSocket::bind("127.0.0.1:0")?.local_addr()?; - let t0 = now(); - let idunn = enroll_service_identity_at::(&root.join("idunn.cc"))?; - enroll_service_identity_at::(&root.join("odin-topology.cc"))?; - let provider = - enroll_service_identity_at::(&root.join("odin-provider.cc"))?; - std::fs::write(root.join("topology-cred"), credential_bytes(&root.join("odin-topology.cc"))?)?; - std::fs::write(root.join("provider-cred"), credential_bytes(&root.join("odin-provider.cc"))?)?; - export_service_identity_trust_anchor::(&idunn, &root.join("idunn-anchor.cc"))?; - let expected = IdunnExpectedIncarnationRecord { - schema_version: IDUNN_EXPECTED_INCARNATION_SCHEMA.into(), - target: "odin".into(), - plan_id: digest('1'), - incarnation_id: "odin/generation-1".into(), - sealed_release_id: digest('2'), - source_repository: "github.com/GameCult/Odin".into(), - source_revision: "3".repeat(40), - recipe_sha256: digest('4'), - runtime_id: "odin-runtime".into(), - expected_signer_identity_id: provider.entry().identity_id.clone(), - health_contract: "odin.runtime-health.v1".into(), - artifact_sha256: digest('5'), - state_schema_generation: Some("odin-v2".into()), - state_contract_sha256: Some( - "sha256-4f2f2dcd931d16f6b02bf295f41227b867aa982765661d55fa9f29fb2db7e449".into(), - ), - write_lease_required: true, - route: Some(IdunnExpectedRoute { - route_id: "odin-route".into(), - transport: "rudp".into(), - stable_endpoint: "rudp://odin.internal:1000".into(), - candidate_endpoint: format!("rudp://{candidate}"), - }), - capabilities: vec![IdunnExpectedCapability { - capability: "odin.verse-rendezvous".into(), - schema: "odin.verse-topology.v1".into(), - compatibility: "v1".into(), - minimum_capacity: 1, - }], - dependencies: Vec::new(), - }; - expected.validate()?; - let launch = IdunnRuntimeActivationLaunch::issue(&expected, digest('7'), t0 - 20, &idunn)?; - let activation = launch.activation().clone(); - let mut activation_credential = Vec::new(); - launch.write_credential(&mut activation_credential)?; - std::fs::write(root.join("activation-cred"), &activation_credential)?; - - let expected_envelope = |key: &str| -> Result { - Ok(CultCacheEnvelope { - key: key.into(), - r#type: IdunnExpectedIncarnationRecord::TYPE.into(), - payload: expected.canonical_bytes()?, - stored_at: rfc3339(t0 - 30), - schema_id: Some(IDUNN_EXPECTED_INCARNATION_SCHEMA.into()), - }) - }; - let activation_envelope = |key: &str| -> Result { - Ok(CultCacheEnvelope { - key: key.into(), - r#type: IdunnRuntimeActivationRecord::TYPE.into(), - payload: activation.canonical_bytes()?, - stored_at: rfc3339(activation.issued_at_unix_millis), - schema_id: Some(IDUNN_RUNTIME_ACTIVATION_SCHEMA.into()), - }) - }; - let bundle = root.join("bundle"); - std::fs::create_dir_all(&bundle)?; - write_store(&bundle.join("expected.cc"), &[expected_envelope("odin")?])?; - write_store(&bundle.join("activation.cc"), &[activation_envelope("odin")?])?; - - let anchor = GameCultServiceTrustAnchorRecord { - schema_version: GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA.into(), - trust_anchor_id: "root/odin/runtime-presence".into(), - service_id: "odin".into(), - runtime_id: expected.runtime_id.clone(), - signer_identity_id: provider.entry().identity_id.clone(), - signer_public_key: provider.entry().public_key.clone(), - signature_algorithm: "ed25519".into(), - signing_purpose: GAMECULT_RUNTIME_PRESENCE_HEALTH_SIGNING_PURPOSE.into(), - signed_schema: GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), - binding_authority: "root".into(), - bound_at_unix_millis: t0 - 100, - expires_at_unix_millis: None, - private_state_exposed: false, - }; - let incarnation_key = IncarnationRef::of(&expected)?.key(); - let projection = root.join("projection.cc"); - let mut projected = vec![ - expected_envelope(&incarnation_key)?, - CultCacheEnvelope { - key: anchor.trust_anchor_id.clone(), - r#type: GameCultServiceTrustAnchorRecord::TYPE.into(), - payload: rmp_serde::to_vec(&anchor)?, - stored_at: rfc3339(anchor.bound_at_unix_millis), - schema_id: Some(GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA.into()), - }, - activation_envelope(&incarnation_key)?, - ]; - write_store(&projection, &projected)?; - let lease_path = root.join("lease.cc"); - File::create(root.join("lease.cc.lock"))?; - let state_root = root.join("state"); - std::fs::create_dir_all(&state_root)?; - - // Descriptors 3 and 4 carry the signer credentials, as systemd passes them. - let child = Command::new("bash") - .arg("-c") - .arg(format!( - "exec 3<{activation} 4<{provider}; exec {bin} --state-root {state} --idunn-projection {projection} --idunn-anchor {anchor}", - activation = root.join("activation-cred").display(), - provider = root.join("provider-cred").display(), - bin = env!("CARGO_BIN_EXE_odin-daemon"), - state = state_root.display(), - projection = projection.display(), - anchor = root.join("idunn-anchor.cc").display(), - )) - .env("GAMECULT_IDUNN_RUNTIME_BUNDLE", &bundle) - .env("GAMECULT_IDUNN_CANDIDATE_BIND", candidate.to_string()) - .env("GAMECULT_IDUNN_PROCESS_WRITE_LEASE", &lease_path) - .env("ODIN_TOPOLOGY_IDENTITY", root.join("topology-cred")) - .env("LISTEN_PID", "1") - .env("LISTEN_FDS", "2") - .env( - "LISTEN_FDNAMES", - format!("{IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME}:gamecult-runtime-presence-identity"), - ) - .stdout(Stdio::null()) - .stderr(Stdio::from(File::create(root.join("odin.log"))?)) - .spawn()?; - let mut odin = Odin { - _temp: temp, - root, - candidate, - child, - store: state_root.join("topology.cc"), - }; - - // Idunn's warming probe, then the lease naming that exact warming presence. - let deadline = Instant::now() + Duration::from_secs(20); - let response = loop { - match request_raw_snapshot_from_rudp_catalog(CultMeshRudpSnapshotOptions { - target: candidate, - runtime_id: "test-idunn".into(), - message_id: format!("warming-{}", now()), - schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), - record_keys: Some(vec!["odin".into()]), - ..Default::default() - }) { - Ok(response) => break response, - Err(error) => { - if let Some(status) = odin.child.try_wait()? { - bail!("Odin exited while warming ({status}): {}", odin.log()); - } - ensure!(Instant::now() < deadline, "Odin never answered warming: {error:#}"); - std::thread::sleep(Duration::from_millis(100)); - } - } - }; - let CultNetMessage::SnapshotResponseRaw { documents, .. } = response else { - bail!("the warming probe was not answered with raw documents") - }; - let presence: GameCultRuntimePresenceHealthRecord = rmp_serde::from_slice(&documents[0].payload)?; - ensure!(presence.state == "warming", "{}", presence.state); - let lease = IdunnProcessWriteLeaseRecord { - schema_version: IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into(), - target: "odin".into(), - expected_projection_sha256: expected.canonical_sha256()?, - plan_id: expected.plan_id.clone(), - incarnation_id: expected.incarnation_id.clone(), - sealed_release_id: expected.sealed_release_id.clone(), - activation_witness_sha256: activation.canonical_sha256()?, - state_schema_generation: "odin-v2".into(), - state_contract_sha256: expected.state_contract_sha256.clone().unwrap(), - runtime_id: expected.runtime_id.clone(), - runtime_instance_id: activation.runtime_instance_id.clone(), - warming_presence_sha256: presence.canonical_sha256()?, - lease_epoch: 1, - issued_at_unix_millis: now() - 5, - }; - let lease_envelope = |key: &str| -> Result { - Ok(CultCacheEnvelope { - key: key.into(), - r#type: IdunnProcessWriteLeaseRecord::TYPE.into(), - payload: lease.canonical_bytes()?, - stored_at: rfc3339(lease.issued_at_unix_millis), - schema_id: Some(IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into()), - }) - }; - write_store(&lease_path, &[lease_envelope("odin")?])?; - projected.push(lease_envelope(&incarnation_key)?); - write_store(&projection, &projected)?; - - while !odin.store.is_file() { - if let Some(status) = odin.child.try_wait()? { - bail!("Odin exited while activating ({status}): {}", odin.log()); - } - ensure!(Instant::now() < deadline, "Odin never activated: {}", odin.log()); - std::thread::sleep(Duration::from_millis(20)); - } - Ok(odin) -} - -/// A put Odin acknowledged is on disk: killed outright right after the -/// acknowledgement, inside the interval its own changes wait for, Odin loses -/// none of them. -#[test] -fn an_acknowledged_put_survives_sigkill() -> Result<()> { - for round in 0..3 { - let mut odin = start_odin()?; - // Inside the interval that Odin's activation write started. - std::thread::sleep(Duration::from_millis(300)); - let key = format!("doc-{round}"); - odin.put(&key)?; - odin.signal("-KILL")?; - ensure!(odin.holds(&key)?, "round {round}: the acknowledged put was lost: {}", odin.log()); - } - Ok(()) -} - -/// SIGTERM reaches the serving loop as a stop request: Odin makes its last -/// write and exits cleanly, rather than waiting to be killed. -#[test] -fn a_stop_signal_ends_odin_cleanly() -> Result<()> { - let mut odin = start_odin()?; - let status = odin.signal("-TERM")?; - let log = odin.log(); - ensure!(status.success(), "Odin exited {status}: {log}"); - ensure!(log.contains("Odin stopping on request"), "{log}"); - ensure!(!log.contains("final store write was not made"), "{log}"); - Ok(()) -} diff --git a/docs/store-recovery-cut.md b/docs/store-recovery-cut.md new file mode 100644 index 0000000..fca90d5 --- /dev/null +++ b/docs/store-recovery-cut.md @@ -0,0 +1,570 @@ +# Odin store repair and recovery: cut map + +**Operator rulings, 2026-09-30:** +- **Q-R1:** "You got it". A foreign file under a running Odin is set aside and Odin continues. +- **Q-R2:** "Yep". Keep the previous generation on disk. +- **Q-R3:** "Start empty". Nothing decodable at activation means Odin starts empty. +- **Q-R4:** "Yep, and Idunn needs better reporting capabilities so it can serve as watchdog. I ought to get a message + about this on Discord if it ever happens." + - Stop writing after the third set-aside in 10 minutes, and keep serving. + - **New requirement:** Idunn watchdog reporting that reaches the operator on Discord. It is mapped as its own + campaign and is not an Odin cut. +- **Q-R5:** "That's fine, Odin still runs after all, right?" Yes. Odin keeps serving, and the store condition is + published as its own typed record, not as a degraded presence. +- **Q-R6:** "Approved". Both drills. + +**R1 LANDED: CultLib main `daedfdd6` (2026-09-30).** The merged tree `0e6891b3` was verified on Yggdrasil in every +runtime, including the four-runtime store interop test. That test runs close to its 30 s per-command limit under +load (`cult-cache.test.ts:1407`), so it is fragile: follow-up. Consumer pins still gate their own bumps: Bifrost only +with R3b's TS fsync; Muninn and Idunn with their error-text matches. + +**R1 status (Self, 2026-09-30).** Built on CultLib `hands/cultcache-read-failures` (`c0b56e06`..`b7908668`), then +fix batch 2 (`..2d7a3c78`). Soul's verdict on batch 2 was to hold the merge. Batch 3 is in Hands, fixing: +- **Blocker, F1 (zero-byte stores on Yggdrasil).** + - Two zero-byte stores are live: `/srv/bifrost/persona-feedback/{inbox/observations,outbox/deliveries}.cc`. + - They are pre-created by `touch` in gamecult-ops `install-bifrost-persona-feedback-native-yggdrasil.sh:42,46` + and `bootstrap-bifrost-persona-feedback-yggdrasil.sh:22,29`. + - Bifrost `persona-feedback.mjs serve` and Epiphany `epiphany-swarm` crash-loop on them once their CultLib pins + move past R1. + - **Neither pin may move until the scripts stop pre-creating stores and Self has moved the live files aside.** +- **F4.** Rust errors echo record values through serde's text (`lib.rs:753-758`). The header is echoed without bound. +- **F6.** A dangling symlink reads as absent, and the writer then replaces it. +- **F7.** The same NotFound-only rule in Python's `JsonLinesBackingStore` and in TS `pullAll`'s repair write. +- **M7.** A header-prefix fixture. +Recorded for later cuts: +- **F3 → R2/R3b.** TS, Python and C# decode the whole buffer before reading the header. A truncated newer-format + store (20..n-1 bytes) is "unsupported format" in Rust and a raw decode error elsewhere. +- **F5 → R3.** C# `ReadSnapshot` reads ENOTDIR/EISDIR as an empty store (`FileInfo.Exists`, `CultCache.cs:3439`). + C# `DirectoryMessagePackBackingStore` `File.Exists` (`:409`, `:497`) is the same. This is a live disagreement between + runtimes until R3. +- **F8 → R3b.** Python leaves its staging file behind when a write fails (`stores.py:124-128`). +- **Tooling.** + - `packages/cultcache-rs` has no committed `Cargo.lock`, so the map's `cargo test --locked` cannot run. + - The cultcache-ts interop test needs `CARGO_TARGET_DIR` unset under ygg-verify. +- **Soul on batch 3 (`2d7a3c78..3c9e195b`, gamecult-ops `bca7978`), 2026-09-30: merge both.** Pre-merge batch 4 is in + Hands: fix the README symlink claim (S1), close value echoes in the redb and typed-decode paths (S2) and in the + Python/TS non-string key echo (S3), make the unlock policy coherent (S9), classify encode refusals separately from + `NotReplaced` (S10), and fix a test timeout (S11). The ops branch deletes the guard test (S6) and the dead compose + layout (S5). + - **Open for R3:** writes through a symlinked store path replace the link, so a store behind a link to a mounted + volume moves into the container layer. R3 decides resolve or refuse. + - **Open for R3b:** Python and TS never fsync. After F1 their first create-by-rename is not covered by ext4 + `auto_da_alloc`, so a crash can leave zero bytes, which every post-R1 reader refuses forever. **Bifrost's CultLib + pin moves past R1 only together with R3b's TS fsync.** + - **For W0 (Odin):** classify a write failure through `CultCacheStoreWriteFailed.kind` together with its `io::Error`, + never retry on `NotReplaced` alone (S10). + - Deployed Epiphany is `d2ca6630`, using a vendored cultcache-rs (`ba6a4874`) that reads a missing store as empty. + All Epiphany units have been inactive since 2026-08-22. + - **Done 2026-09-30 20:48 CEST (Self).** gamecult-ops merged at `3fa22aa`. The inbox is setgid 2750. All three + zero-byte stores were renamed in place and moved to `/root/zero-byte-stores-20260930/` on Yggdrasil, with + `manifest.txt` recording owner, mode and mtime. None had gained bytes. All three Bifrost units stayed active, and + persona-feedback logged nothing and did not restart. + - The compose/container lane (the compose file, two deploy scripts, the `.ps1` uploader, two runbooks) is dead: + the live Bifrost units run native `dotnet` and `node`. **Deleted 2026-09-30, operator: "delete the Bifrost lane".** + gamecult-ops `fdb4356`. Its live trigger, the Idunn legacy actuator route `deploy bifrost-persona-feedback`, was + retired in Idunn `7793b5b` and installed on Yggdrasil at 21:50 CEST (sha256 `d95fa08c...`; the previous binary is + at `/root/idunn-yggdrasil.pre-bifrost-retire-20260930`). The manifest and `/srv/bifrost/compose` were moved to + `/srv/retired/retired-2026-09-30/`. All three Bifrost units stayed active. **Follow-up:** persona-feedback has no + Idunn deploy route now. Give it a real Idunn binding. + - Live zero-byte stores to move aside after the ops merge, in Soul's order (setgid on inbox, then rename in place, + then relocate): `observations.cc`, `deliveries.cc`, and a third, `/var/lib/gamecult/epiphany/idunn-provider-health-admission.cc`. +- **Text matches.** gamecult-ops `scripts/idunn/start-idunn-local.ps1:143` matches the same removed text as Muninn + (below). It must change with Idunn's next CultLib bump. +- **Consumer follow-up that must travel with the next Muninn CultLib bump:** + `Muninn scripts/restart-starfire-muninn.ps1:144` matches `*failed to decode MessagePack*` to trigger + `Reset-CorruptMuninnStore`. R1 removes that text, so the reset would stop firing silently. The script must match + the typed `CultCacheStoreUnreadable` kind, or the new wording, in the same change. +- C#'s truncated-store exception (`EndOfStreamException`) is an `IOException`, the same I/O-versus-undecodable + overlap. It goes with R3's C# work. + +Status: cut map, Imagination pass 1 (Opus), 2026-09-30. Nothing has landed. This map owns the means for repairing +and recovering `/var/lib/gamecult/odin/topology.cc`. It follows `docs/write-pattern-cut.md` (Cuts 1-3) and starts +from that campaign's landed head. + +Anchors: +- Odin `hands/odin-write` at `cda2087` (Cuts 1-2 plus two test commits). Q3 B (peer puts fsynced before ACK) is + being finished on that branch now. **Re-anchor every Odin `file:line` below on the head that lands; the line + numbers here are `cda2087`.** +- CultLib `6d0cc963`. `packages/cultcache-rs` is byte-identical to Odin's pin `3bf1c0ce`, so the Rust line numbers + hold at both. +- Idunn `a96ad9d`, Heimdall `e8e2832`, gamecult-ops `runbooks/gamecult-state-backup.md` as of 2026-09-30. + +Operator, 2026-09-30, on the three options Self offered for an undecodable foreign write (degraded but loud, end +Odin, unchanged): **"None of these options involve repair and recovery, which is what ultimately needs to happen."** + +Open: Q-R1 to Q-R6 (section 8). D1 (section 6, O1) is a Self default the operator may overrule. + +## Short answer + +Odin already holds the one thing a repair needs: its in-memory working set, which is authoritative for everything +it owns while it holds the write lease. Today it never uses it for repair. It ends on a decodable foreign write, and +it silently stops persisting on an undecodable one. At activation it has no working set and no second copy, so it +crash-loops. + +The repair has one rule: **a store file that is not what Odin last wrote is set aside, never deleted and never +merged, and Odin writes its working set over it under the lease.** At activation Odin has no working set, so it +falls back to a previous generation kept on disk, and then to an empty store that providers refill. The +correlation sequence gets a time floor, so no rollback can recreate the watermark deadlock. Recovery is published as +a typed record in Odin's own catalog. The mechanism (typed read failures, a set-aside exchange, a retained previous +generation, no staging file left behind by a failed write) belongs to CultLib, because every consumer of the +single-file store has the same gap. + +## 1. Body facts (probed) + +### 1.1 Probe: cultcache-rs single-file store under fault + +The probe was a scratch crate at CultLib `3bf1c0ce`, `scratchpad/odin-recovery-probe` (Imagination session), commit +`b19eba9`, run through the stopgap on Yggdrasil in slot 2. It ran with `--tmpfs /small:size=64k --tmpfs /ro:ro`, +using `trap '' XFSZ; ulimit -f 40` for the file-size case. The log is `scratchpad/odin-recovery-probe.log`. + +| Fault | Result | +|---|---| +| P1: file truncated to half | `compare_exchange_snapshot` and `pull_all` return `Err`, **not** `Ok(false)`. There is no `io::Error` in the chain; the decode error is flattened to text (`lib.rs:720-725`). The message is wrong: "invalid type: string \"cultcache.store.v1\", expected struct CultCacheEnvelope". `store_header` (`lib.rs:317-341`) deserializes the *whole* array to find the header, so a truncated v1 file has no header and falls through to the legacy decoder. The file is unchanged by the failed exchange | +| P2: 12 bytes of garbage appended | `pull_all` returns `Ok`, all 3 records. `compare_exchange_snapshot(expected)` returns **`Ok(true)` and silently drops the tail**. `rmp_serde::from_slice` ignores trailing bytes. C#'s `DeserializeSnapshot` (`CultDocumentMessagePackSerialization.cs:179-215`) never checks for the end either | +| P2b: one byte flipped inside a payload | `pull_all` returns `Ok` (payloads are opaque bytes). `compare_exchange_snapshot` returns **`Ok(false)`**. To Odin, bit rot is a *decodable foreign write* | +| P3: header `cultcache.store.v4...` | `Err` with a distinct message, "not readable; this runtime reads v1 only". It is distinguishable only by its text | +| P4: another handle wrote a valid snapshot | `Ok(false)` | +| P5: `hard_link(topology.cc, topology.cc.displaced..x)`, then an exchange through the store | The link keeps the old bytes exactly, and the store writes the new file. The staging sweep (`remove_abandoned_staging_files`, `lib.rs:2730-2766`) removes a `..tmp` beside it and leaves the displaced name alone | +| read-only filesystem, no store | `pull_all` returns `Ok(0)`. The exchange fails opening the lock: `io::Error` `ReadOnlyFilesystem` (os 30) | +| `RLIMIT_FSIZE` 40 KiB, 200 KB write | `Err` `io::Error` `FileTooLarge` (os 27). The old file is intact and still decodes. **A 40,960 B partial staging file is left behind.** The next attempt sweeps it and leaves a new one | +| 64 KiB tmpfs, 200 KB write (ENOSPC) | `Err` `StorageFull` (os 28). The old file is intact. **A 61,440 B partial staging file is left behind: all of the filesystem's remaining space.** The same holds on every retry | + +What follows from the probe: +- **Decodable is not the same as legitimate.** A flipped payload byte is "decodable and foreign", and appended + garbage is "decodable and ours". The decodable/undecodable split cannot be what decides recovery. One rule has to + cover both: *not what Odin last wrote*. +- **A write that fails for lack of space holds the space it failed on.** `/var/lib/gamecult` is shared by every + service on Yggdrasil. Odin retries once per second (and, under Q3 B, on every put), so it re-takes whatever + another service frees. C# deletes its temp file in a `finally` (`CultCache.cs:3496-3531`); Rust does not + (`write_all_unlocked`, `lib.rs:741-768`). This is a Rust parity defect. +- **The atomic write keeps no previous generation.** `write_all_unlocked` stages, fsyncs, renames over the file and + fsyncs the directory. After the rename, the old inode is gone. The brief's "last good checkpoint kept by the + atomic write" does not exist on disk. The only last-good copy is Odin's in-memory `flushed` snapshot + (`lib.rs:506-510`), and it dies with the process. + +### 1.2 Odin today (`cda2087`) + +| Case | Path | Outcome | +|---|---|---| +| Decodable foreign write while serving | `MemoryOdinTopologyStore::flush` `lib.rs:613-626`: `compare_exchange_snapshot(flushed, current)` returns `false`, so `ForeignStoreWrite`. `survive` `main.rs:879-895` returns it, and Odin ends | Idunn restarts Odin (`control_plane.rs:68-71`: 6 restarts per hour, the wait doubling from 5 s). The new Odin **loads the foreign file as truth**. Odin's own unflushed changes, and every record of Odin's that the foreign file lacks, are lost without a trace. A flipped payload byte (P2b) takes this path | +| Undecodable foreign write while serving | the same exchange returns `Err` (decode). `survive` logs it and retries every second, forever. Nothing is persisted again | **Idunn's crossing breaks too.** `receive` (`Idunn drivers.rs:5005-5030`) reads the same file and propagates the error to `admit_latest_topology` (`control_plane.rs:7903-7907`) and `refresh_admitted_topology` (`:6040`). Every Odin-correlated deployment and refresh fails until the file is readable, which is never | +| Corrupt file at activation | `try_activate` `main.rs:246-297` calls `MemoryOdinTopologyStore::load` `lib.rs:519-538`, which calls `pull_all()?`. `main.rs:762` `while !try_activate()?` exits the process | Crash loop until Idunn's budget is spent. Then Idunn reports once, to its log only (`control_plane.rs:5709-5723`, `report_once` `:4599-4610` is `eprintln`), and stops. The Verse rendezvous is down | +| Disk full or read-only | the exchange returns `Err` (io). `survive` logs and retries every second | Odin serves from memory. Each attempt leaves a partial staging file (1.1). A stop or deploy in this state loses everything unflushed. Nothing typed says so | + +Detection latency for a foreign write: `flush` writes only when dirty (`main.rs:307-314`). Odin's own heartbeat +(`HEARTBEAT_INTERVAL` 5 s, `main.rs:69`) dirties the set, so a foreign write is met at the latest one heartbeat plus +one flush interval (about 6 s) after it happens. No polling read is needed, and Cut 1's rule (no reads while +serving) stands. + +### 1.3 Other sources of truth, and what they can recover + +| Source | Holds | Freshness | Recovers | +|---|---|---|---| +| Odin's working set (W) | everything, including unflushed changes | live | everything, but only while this process lives and holds the lease | +| `flushed` (in memory) | the last snapshot Odin wrote | at most one interval old | nothing more than W; it is the comparison base, not a store | +| previous generation on disk (new, O3 and R3) | the snapshot before the last one this store's owner verified and replaced | one write old | everything except the last write's delta, including publish-once documents, presence history and the watermark | +| nightly backup (`gamecult-state-backup.md`) | `/var/lib/gamecult/odin` whole, shared-lock copy | up to 24 h old, 3 days on Yggdrasil, 14 days plus 8 weekly on Raven | publish-once documents. **Never automatically**: "It never becomes live authority, and nothing restores it automatically"; "restore a store only after stopping its owner" | +| providers republishing | presences (2.7-12 s), Muninn documents (2 s), ads (about 30 s); Heimdall HEAD republishes `heimdall.command_boundary` and its three siblings every 60 s (`Heimdall src/index.ts:19,35-39`, gated on its write lease) | seconds to a minute | every document whose publisher republishes | +| Idunn's projection plus presences | correlations | one refresh pass (250 ms) after the presences return | correlations. They are a projection and never need restoring | +| Idunn's `odin_publisher_sequence_cursor` (`control_plane.rs:1456,1537,6081`) | the highest correlation sequence Idunn admitted, per target | live | nothing directly (Idunn owns it), but it is why the watermark must never roll back | + +Premise correction: `heimdall.command_boundary` is not publish-once at Heimdall HEAD. A `stored_at` of 09-23 +proves only that the publisher's own `stored_at` has not changed, because Odin stores the publisher's +`stored_at` (`persist_generic_document`, `main.rs:1330-1347`). Whether the deployed Heimdall re-puts is +unverified; O3's outstanding-republish list measures it. + +**The watermark is the one record whose loss is not self-healing.** Idunn refuses any correlation sequence at or +below its cursor (`control_plane.rs:1763,1785`). An Odin that starts from an empty store, from the previous +generation, or from a backup restarts a target's count below that cursor. That is the deadlock documented at +`lib.rs:33-49`: Idunn refuses every correlation until the count climbs past the old mark, and the refusal +withdraws the correlation before it can climb. Every recovery source except W has this defect. O1 removes it. + +Other facts the design leans on: +- **Idunn fences before it grants.** `CultCacheWriteLeaseDriver` (`Idunn drivers.rs:4050-4205`): "revocation + follows an exact incumbent stop", and grant is a nonblocking exchange from empty. A second *Odin* cannot hold + the lease while the first is still writing, so a foreign writer seen by a lease-holding Odin is not an Odin. +- **Idunn has a lifecycle brake** (`idunn-provision lifecycle-brake-engage|release|status`, + `provisioning.rs:95-97`; `odin-lifecycle-brake.cc` per `runbooks/odin-yggdrasil.md:29-30`). An engaged brake + denies continuity restarts (`control_plane.rs:5771-5774`, `:6219`). This is the operator's hold for a manual + restore. +- **A non-`active` Odin presence fans out.** `reconcile` sets `ready` only when the presence state is `active` + (`lib.rs:946`), and `dependency_evidence` (`lib.rs:1184ff`) makes every managed dependent of Odin not Ready. + The presence schema allows `degraded` (`cultnet-rs runtime_authority_contracts.rs:193`), but using it would + be a readiness statement about every dependent, not a store report. +- **The catalog serves only peer types and presences** (`public_document`, `main.rs:590-640`). No `odin.*` record + is served today, and neither Odin nor Idunn publishes an Eve surface. F3 in the write map (the unowned + `surface:gamecult.network.status`) is where an Eve lowering belongs. + +## 2. Ends + +1. No single fault, whether a foreign write of any kind, a corrupt file, or a failed write, ends Odin or leaves it + silently unpersisted. Only losing the write lease ends it. +2. A store file that is not what Odin last wrote is preserved byte for byte beside the store and never deleted, + merged or adopted by any automatic path. The owner (Odin under the lease) writes its own state over it. +3. At activation a corrupt store is recovered from the best source on the host (previous generation, then empty), + and never from the backup automatically. +4. No recovery can make a correlation sequence repeat or fall below one Idunn has admitted. +5. Every repair, and every stretch in which Odin is serving without persisting, is typed state that can be + queried through Odin's catalog while it is happening, including when the disk refuses writes. + +Invariants carried over: one writer (the lease, checked immediately before every write); Idunn's lockless +crossing reads a whole `.cc` snapshot at `topology.cc`; the path never becomes absent (Idunn reads an absent file +as an empty snapshot, which withdraws every correlation); no reads of the store while serving; one bad record +never stops Odin. + +## 3. Identity, lifecycle, authority + +| Persistent kind | Named by | Over time | Decided by | +|---|---|---|---| +| store file `topology.cc` | the state root plus `TOPOLOGY_SLOT` | replaced whole by the owner's exchange; never absent once created | Odin under the lease | +| displaced file (new) | `topology.cc.displaced..`, same directory (it must not end in `.tmp`) | created by a set-aside exchange as a hard link to the replaced inode; never modified; never deleted by any program. Travels to Raven with the nightly backup | created by Odin through CultLib; **disposed of only by the operator** | +| previous generation (new, Q-R2) | `topology.cc.previous` | replaced on every *verified* exchange (current == expected) by a hard link to the outgoing inode; **not** rotated by a set-aside exchange, so it always holds a generation that some writer verified before replacing | CultLib mechanism; Odin opts in | +| store condition record (new) | type `odin.store_condition`, schema `odin.store_condition.v1`, key `topology` | one record held in W; rewritten when the condition changes; flushed with the store; served by the catalog | Odin | +| correlation sequence floor (new rule, not a record) | per target, in the existing watermark | next = max(mark + 1, prior + 1, now in unix ms) | Odin | +| backup snapshot | `/var/backups/gamecult-state//` and Raven | nightly; never live authority | the operator restores it, with the lifecycle brake engaged (runbook) | + +## 4. Authority map (the whole campaign) + +- **Owner:** Odin, while it holds the current process-write lease, decides what `topology.cc` holds, when it is + set aside, and what activation recovers from. CultLib owns the mechanism (read classification, set-aside + exchange, previous generation, staging cleanup) and no policy. Idunn owns whether Odin runs (continuity budget, + lifecycle brake, deploys). The operator owns restores from backup and the disposal of displaced files. +- **Inputs:** W; `flushed`; the lease (read immediately before every write); at activation, `topology.cc` and then + `topology.cc.previous`; the clock (the sequence floor). +- **Outputs:** one atomic `topology.cc`; displaced files; `topology.cc.previous`; the `odin.store_condition` record + in the catalog. +- **Derived state:** `topology.cc` is a checkpoint and Idunn's crossing, never an input while serving. The + displaced files are evidence only: nothing reads them automatically. The previous generation is read only by + activation recovery. The condition record is a report and decides nothing. + - `ForeignStoreWrite` **is no longer an owner of Odin's life**. A foreign write becomes a set-aside event. +- **Forbidden writers and deciders:** + - Any path that ends Odin on a store condition. `survive`'s `ForeignStoreWrite` arm (`main.rs:884`) is + deleted; `WriteLeaseLost` stays, and so does the temporary `PresenceAuthorityRefused` rule. + - Any automatic delete of a displaced or previous file. + - Any merge of records from a foreign or displaced file into W. + - Any automatic restore from the backup. + - Idunn writing or repairing `topology.cc`. + - Any second writer of the file inside Odin: activation never writes. It prepares W and `flushed`, and the one + `flush` makes the first write through the same set-aside exchange. +- **Shared paths:** + - The interval flush, the Q3 B per-put flush, the stop flush and the activation-recovery first flush all go + through one primitive, `replace_snapshot_setting_aside`. + - Every path that reserves a correlation sequence goes through the one floored reservation. +- **Deletion line:** `ForeignStoreWrite` (`lib.rs:630-641`) and its `survive` arm are deleted before the set-aside + exchange is wired in. The `Err` escape in `load` (`lib.rs:521`, `pull_all()?`) is replaced by the recovery + ladder, not wrapped. + +## 5. Recovery by case + +| Case | Detected by | Action | Recovered from | Loss bound | Visible as | +|---|---|---|---|---|---| +| **A. Decodable foreign write while serving** (a hand edit, a tool, a restore made against the runbook, or bit rot in a payload per P2b) | the next flush: current != `flushed` | the lease is checked; the file is set aside (hard link); W is written; Odin keeps serving | W | none of Odin's own records. The foreign file's content is preserved, not adopted | condition event `displaced`, reason `mismatch`, with path, sha256, bytes and time | +| **B. Undecodable foreign write while serving** (truncation, garbage, a foreign format, and after R2 trailing bytes too) | the next flush: current does not decode | the same | W | none. Idunn's crossing is unreadable for at most one heartbeat plus one flush (about 6 s), instead of forever | the same, reason `undecodable` | +| **A/B repeated** (a live foreign writer) | a third set-aside inside 10 minutes in one activation | Odin stops writing and keeps serving; puts are not ACKed (Q3 B: ACK means durable); no new displaced files (Q-R4) | W stays in memory | nothing while the process lives | condition `contested`, since when, with the displaced files | +| **C1. Undecodable store at activation** | `load` gets `Undecodable` | W is built from `topology.cc.previous` if it decodes; `flushed` is set to "unreadable", so the first flush sets the corrupt file aside and writes W | the previous generation | the last write's delta: one interval of Odin bookkeeping, or one fsynced put. **A put ACKed as durable can be lost here; say so in Q3 B's record** | event `recovered-previous`, plus an outstanding-republish list: peer records in the previous generation not yet re-put by their publisher, cleared as they arrive | +| **C2. Nothing decodable at activation** | `.previous` absent or undecodable too | start empty (Q-R3); the first flush sets the corrupt file aside | providers republish; correlations re-derive in one pass; the sequence floor holds | every publish-once document; presence history (F4 makes this harmless) | event `started-empty`; outstanding list is `unknown`. The operator's path is the backup, with the lifecycle brake engaged | +| **C3. Unsupported format at activation** (a newer runtime wrote v4, or a downgrade) | `load` gets `UnsupportedFormat` | refuse to activate. The file is untouched and nothing is set aside | the deploy or rollback that caused it; Idunn owns that | none | Idunn's continuity budget and the Odin journal. This is a deployment fault, not a store fault | +| **C4. I/O error reading the store at activation** (EIO, EACCES after a restore with the wrong owner, EISDIR) | `load` gets an `io::Error` | refuse to activate. Nothing is set aside: **only bytes that were read are ever judged** | the operator fixes the host or the mode | none | as C3 | +| **D. Write fails while serving** (ENOSPC, EDQUOT, EFBIG, EROFS, EIO) | the exchange returns an `io::Error` while staging or renaming | nothing is set aside; the staging file is removed (R1); W is kept; retry at the flush interval; puts are not ACKed; Odin keeps serving | W, when the fault clears | nothing, unless Odin is stopped or redeployed before the fault clears (follow-up I-F1) | condition `unpersisted` since T, with the last error kind (`no-space`, `read-only`, `io`) and the dirty-record count, served from memory | +| **D'. Stop while unpersisted** | the stop flush fails | log it and stop, as today (`main.rs:794-799`) | the next activation loads the last good file | the unflushed delta | the final journal line only. I-F1 is the owner fix | + +Rollback safety, for C1, C2 and an operator restore: every source except W carries a watermark below Idunn's +cursor. The floor (O1) makes the next sequence exceed every sequence published before, whatever the store holds. + +## 6. Cuts + +Order: the Rust substrate gaps first (R1), then the watermark floor (O1, independent and small), the set-aside +primitive (R3), Odin's serving repair (O2), activation recovery (O3), parity hardening (R2, R3b), and finally the +runbook and drill (X1). Each cut is its own branch and its own Soul pass. Odin verification: +`cargo test --locked -p odin-daemon` from the Odin root. CultLib Rust: `cd packages/cultcache-rs && cargo test +--locked`. All of it runs on Yggdrasil through the stopgap. **A permission-based test is unproven there (root).** +The injections below avoid permissions: truncation, a directory at the store path (EISDIR), tmpfs size and +read-only options, and `RLIMIT_FSIZE`. + +### R1. cultcache-rs: typed read failures, no staging file left behind, a header that survives truncation (CultLib) + +- **Repo/branch:** CultLib `hands/cultcache-rs-store-faults` from `main`. Depends on nothing. +- **Deletes first:** the flattening `map_err(|error| anyhow!(...))` in `read_all_unlocked` (`lib.rs:720-725`); + `store_header`'s whole-array visitor (`lib.rs:317-341`). +- **Adds:** + - `pub struct CultCacheStoreUnreadable { path, kind: Undecodable | UnsupportedFormat }`, carried in the error + chain of every read of a single-file store whose bytes were read and not accepted, so consumers can + `downcast_ref`. An `io::Error` stays an `io::Error`, and the two never overlap. + - `store_header` reads only the array header and the first string, reusing `read_array_header` and + `read_string` (`lib.rs:2661-2700`). A truncated v1 file then reports as an undecodable v1 file, not as legacy. + - `write_all_unlocked` removes its own staging file on every failure after creating it. This is C# parity with + `CultCache.cs:3525-3530`. +- **Rules that must die under their own mutation:** + - a truncated store is `Undecodable`, and a v4 header is `UnsupportedFormat`; + - an `io::Error` read failure is never classified `Unreadable`; + - after a failed write, the directory holds no `.*.tmp`. +- **Verification:** + - Tests describe behaviour. The staging-cleanup test forces the failure without permissions. One option is a + child process under `RLIMIT_FSIZE` with SIGXFSZ ignored (the probe's method); another is a path whose parent + fills a size-capped tmpfs, if the test runner can mount one. Hands picks one and names it. + - `cargo mutants --in-diff`. +- **Estimate:** about -30 / +70. + +### O1. The correlation sequence has a time floor (Odin) — default D1 + +- **Repo/branch:** Odin `hands/odin-sequence-floor` from the landed `hands/odin-write` head. Depends on nothing + else. +- **Change:** `reserve_publisher_sequence` (`lib.rs:702-723`) returns max(mark + 1, prior + 1, now in unix ms), + and records it as the mark. The doc comment at `lib.rs:33-49` gains the rule: "A mark read from any older copy + of the store is a floor, never the next value; the clock makes every rollback safe." +- **Why a floor and not a read of Idunn's cursor:** it needs no new crossing. It holds for every recovery source, + including the backup, and the host clock is already trusted for presence freshness. +- **Assumption:** the clock does not step back across a rollback by more than the rollback's age. Soul checks that + Idunn accepts a jump: its rule is "strictly above the cursor" (`control_plane.rs:1763,1785`). No test may pin a + gap limit. +- **Verification:** + - Existing tests pinning counts (`withdrawing_a_correlation_does_not_reset_its_publisher_sequence` + `lib.rs:1996-2014`) are restated as order properties. + - New test: a store rolled back to an earlier snapshot, then one refresh, yields a correlation sequence greater + than every sequence published before the rollback. + - New test: two reservations inside one millisecond are strictly increasing. + - The clock must be injectable at the reservation. At least one mutant must be a function of the time (an offset + or a scale), with probe values where it would show. +- **Estimate:** about -5 / +25. + +### R3. The set-aside exchange and the previous generation (CultLib, Rust and C#; TS and Python in R3b) + +- **Repo/branch:** CultLib `hands/cultcache-set-aside` from R1. Q-R2 decides the previous-generation half. +- **Adds, on `SingleFileMessagePackBackingStore`:** + - `replace_snapshot_setting_aside(expected, replacements) -> Result`, with + `SnapshotReplacement::{Exchanged, SetAside { displaced: PathBuf, sha256, bytes, reason: Mismatch | + Undecodable }}`. It makes one exclusive lock hold, in this order: + 1. stage the replacement (write and fsync); + 2. read current; + 3. if current equals expected, rename the staged file over (rotating `.previous` first when enabled); + 4. if the read was an `io::Error`, remove the staging file, return the error and change nothing; + 5. otherwise hard-link current to the displaced name, fsync the directory, rename the staged file over, and + fsync the directory again. + + Staging comes first, so a full disk sets nothing aside. The path is never absent: a link, never a rename. A + set-aside never rotates `.previous`. + - Opt-in previous generation: a constructor option (`retaining_previous_generation`, or Hands' spelling of it) + plus `pull_previous_generation() -> Result>`, read under the shared lock and classified + like any read. Off by default, because existing consumers must not grow files they did not ask for. + - Filesystems without hard links: return an explicit error naming the filesystem. Do not fall back to a + copy-then-rename, which would make the path briefly hold new content while the evidence is still being + written. Windows NTFS links work; the Windows path is **not yet reached** by the Yggdrasil suite and is + recorded as such. + - C# reference: the same operation on `SingleFileBackingStore` (`CultCache.cs:3268ff`), with the same names + across runtimes. +- **Verification (Rust and C#):** + - a mismatch preserves the foreign bytes exactly and writes the replacement; + - an undecodable current is preserved and replaced; + - an I/O read failure (a directory at the path) changes nothing and leaves no staging file; + - a staging failure (size cap) changes nothing and sets nothing aside; + - the path is readable at every instant: a reader thread loops `pull_all_read_only_snapshot` through 1,000 + exchanges and never sees an absent file or an empty snapshot; + - `.previous` rotates only on a verified exchange; + - two set-asides in the same second get distinct names; + - mutation tools: `cargo mutants --in-diff`, and Stryker.NET on the C# diff. +- **R3b (parity, not blocking Odin):** cultcache-ts `single-file-messagepack-backing-store.ts:34` and cultcache-py + `stores.py:83` gain the same operation and option, with a shared fixture test. +- **Estimate:** Rust about +140, C# about +120, R3b about +200. + +### O2. A foreign store file while serving is set aside, not fatal (Odin) + +- **Repo/branch:** Odin `hands/odin-store-repair` from O1, with a CultLib pin bump to R3. Needs Q-R1 and Q-R4. +- **Deletes first:** `ForeignStoreWrite` (`lib.rs:630-641`) and its arm in `survive` (`main.rs:884`). The test + `a_flush_writes_only_changes_and_never_over_a_foreign_file` (`lib.rs:2055-2090`) and the `survive` tests at + `main.rs:2842-2844` and `:3110` pin the old rule: restate them, do not delete them. +- **Changes:** + - `MemoryOdinTopologyStore::flush` (`lib.rs:613-626`) calls `replace_snapshot_setting_aside(flushed, current)`. + On `SetAside` it records a `displaced` event and counts it toward the contested bound. On an I/O failure it + records `unpersisted` and returns the error, which `survive` logs and retries as today. + - `RuntimeState::flush` (`main.rs:307-314`) keeps its lease check immediately before the write. + - The Q3 B per-put path uses this same flush. Hands re-anchors on the Q3 B head. +- **Adds:** + - The `odin.store_condition` record (schema in `odin-core` `documents.rs`), held in W. It carries: + - `generation` and `written_by`, updated inside the flush before the snapshot is taken; + - `unpersisted_since` and `last_write_error`; + - `contested_since`; + - a bounded list of the last 16 recovery events: kind, time, displaced path, sha256 and bytes, source, records + loaded, and the outstanding list (bounded). + - One arm in `public_document` (`main.rs:590-608`) that serves exactly this type. Providers still cannot write + it: `is_peer_document_type` is unchanged. +- **Rules that must die under their own mutation:** + - a flush never leaves the store dirty (the condition record is written inside the flush, not after it); + - a set-aside is preceded by a lease check; + - the third set-aside inside the window stops writing, and the second does not; + - after `contested`, no file is written or created; + - on an I/O failure, nothing is set aside; + - the condition is served while writes fail. +- **Authority map:** as in section 4. The deletion line is the `ForeignStoreWrite` type and arm. +- **Verification (fault injection, each at the file layer, not a log line):** + - T-A: activate, serve, write a different valid snapshot behind Odin. After the next pass the file decodes to + W, the displaced file's bytes equal the foreign bytes, `serving_pass` returned `Ok`, the catalog is unchanged, + and the served condition names the displaced file and its sha256. + - T-A2 (P2b): flip one payload byte behind Odin; the same outcome, reason `mismatch`. + - T-B: truncate the file behind Odin; the same outcome, reason `undecodable`, and + `pull_all_read_only_snapshot` (Idunn's crossing) decodes the file after one pass. + - T-B-latency: with no puts, the set-aside happens within one heartbeat plus one flush interval. + - T-contested: three foreign writes inside the window make exactly two set-asides and a third `contested` + report. After that the file and directory are untouched by further passes, and Odin still serves. + - T-D: replace the store path with a directory (EISDIR, which works as root). Nothing is set aside, no staging + file remains, the condition is `unpersisted` and is served, and a put is not ACKed. Remove the directory, and + the next pass writes W, clears `unpersisted` and ACKs. + - T-lease: the lease withdrawn before a flush that would set aside means no link, no write, and Odin ends + (`WriteLeaseLost`). + - Negative: `rg -n "ForeignStoreWrite" crates` matches nothing; `rg -n "remove_file|remove_dir" crates/odin-daemon/src` + matches nothing new outside tests. + - `cargo mutants --in-diff`. +- **Estimate:** about -40 / +160. + +### O3. Activation recovers a corrupt store (Odin) + +- **Repo/branch:** Odin `hands/odin-store-activation` from O2. Needs Q-R2 and Q-R3. +- **Change:** `MemoryOdinTopologyStore::load` (`lib.rs:519-538`) becomes the recovery ladder: + + | `load` result | Outcome | + |---|---| + | `Ok` | as today | + | absent | empty, as today | + | `Unreadable::Undecodable` | try `pull_previous_generation()`, then empty (Q-R3); `flushed` becomes a sentinel meaning "the file is not a snapshot Odin can compare", so the first flush's exchange sets it aside | + | `Unreadable::UnsupportedFormat` | return the error: refuse | + | `io::Error` | return the error: refuse | + + Activation itself never writes. The first `serving_pass` flush (`main.rs:833-842`, due at once) writes W. That + set-aside is recorded under the activation event and is not counted toward the contested bound. +- **Adds:** the outstanding-republish list. It holds the peer-type records present in the recovered previous + generation, each cleared when its publisher re-puts it. After 10 minutes the remainder is marked "not + republished", which is the live answer to "who is publish-once". For `started-empty` the list is `unknown`. +- **Rules that must die under their own mutation:** + - Unsupported-format and I/O failures never set aside, never fall back, and never start. + - A recovered activation writes before it serves its first catalog snapshot, or it serves the recovered set and + marks the condition before the first write. Hands picks one and states it. +- **Verification:** + - T-C1: a truncated `topology.cc` and a valid `.previous` activate with the previous content. The first pass + writes it and sets the corrupt file aside byte-exact, and the condition is `recovered-previous` with its + outstanding list. Re-putting one listed document clears it. + - T-C2: both undecodable (Q-R3 A) start empty. Both files are preserved, the condition is `started-empty`, and + the next correlation sequence is above the floor. + - T-C3: a v4 header refuses to activate. The file is byte-identical afterwards, and there is no displaced file + and no `.previous` change. + - T-C4: a directory at the store path refuses, as C3. + - T-restart: a recovered Odin stopped and restarted loads its own first write normally, with no second + set-aside. + - `cargo mutants --in-diff`. +- **Estimate:** about -10 / +110. + +### R2. Single-file decoders reject trailing bytes, in every runtime (CultLib) + +- **Repo/branch:** CultLib `hands/cultcache-trailing-bytes`. Independent of Odin; land it before O3 ships so that + an appended file is caught as undecodable, not overwritten silently (P2). +- **Change:** C# `DeserializeSnapshot` (`CultDocumentMessagePackSerialization.cs:179-215`) checks `reader.End`. + Rust `decode_store_snapshot` (`lib.rs:2583ff`) and the legacy path check that the slice is consumed. TS and + Python do the same. **One release**: a runtime that accepts what another rejects is a parity split. +- **Verification:** one shared fixture (a valid store plus one byte) is rejected as undecodable by all four + runtimes, and the valid store is accepted by all four. +- **Estimate:** about +40 plus tests across four runtimes. + +### X1. Runbook and recovery drill (gamecult-ops, then live) + +- **Repo:** gamecult-ops `runbooks/odin-yggdrasil.md` gains a "Store recovery" section: + - how to read `odin.store_condition` from the catalog; + - what displaced and previous files mean, and that disposing of them is the operator's decision; + - the manual restore: engage Odin's lifecycle brake, stop the `idunn-odin-*` unit, set the live file aside with + `ln` (never `rm`), extract the snapshot's `topology.cc` with its owner, mode and `.lock`, release the brake, + and read the condition. The sequence floor makes an old watermark safe. +- **Drill D-1 (off-live, after O3; needs the Q2-style approval, Q-R6):** extract the newest Yggdrasil snapshot's + `topology.cc` into a container on Yggdrasil. Activate a test-harness Odin on it through the existing fixture + authority. Then truncate the copy and activate again with the extracted file as `.previous`. Report the records + per type, the outstanding list and the sequence floor. Delete the copy afterwards. +- **Drill D-2 (live, operator-scheduled, Q-R6), after O3 deploys:** + 1. Record the baseline: catalog types, Idunn correlated targets, route timeouts. + 2. Engage the lifecycle brake and stop Odin. + 3. `ln topology.cc /var/tmp/odin-drill-keep.cc` (the evidence of the evidence), then truncate `topology.cc` in + place to half. + 4. Release the brake and let Idunn restart Odin. + 5. Pass when all of these hold: + - Odin is serving within one restart; + - the condition is `recovered-previous`, with a displaced file whose sha256 equals the truncated file's; + - every Odin-correlated target is Ready again within 60 s; + - there are 0 route-challenge timeouts in the next hour; + - the catalog's type set equals the baseline after 10 minutes, or the outstanding list names every + difference. + 6. Then run D-2b, a foreign write while serving: `ln` the live file aside, then truncate it. It passes when the + condition shows `displaced` within about 6 s and Idunn's correlation reads never fail for longer than that + (the journal). + +## 7. Subtraction ledger (estimate) + +| Cut | Removed | Added | Formats and targets | +|---|---|---|---| +| R1 | about 30 | about 70 | none | +| O1 | about 5 | about 25 | none | +| R3 | 0 | about 260 (Rust 140, C# 120); R3b about 200 | two sibling-file conventions (`.displaced.*`, `.previous`); no format change | +| O2 | about 40 | about 160 | one Odin schema, `odin.store_condition.v1` | +| O3 | about 10 | about 110 | none | +| R2 | 0 | about 40 plus tests | none (it tightens the decoders) | + +The campaign is net additive, at about +570 without R3b. It buys ends 1-5, which the operator asked for, and +removes one fatal path and one silent path. Every addition in CultLib is a capability a general consumer of the +single-file store would expect, and two of them (staging cleanup, and the header that survives truncation) are +defects. + +## 8. Operator questions + +- **Q-R1. A store file changed underneath a running Odin (decodable or not).** + - Context: Idunn fences the old Odin before granting the lease, so the other writer is never an Odin. It is a + tool, a hand edit, a restore made against the runbook, or disk corruption; P2b shows a flipped byte looks + exactly like a foreign write. Today a decodable change ends Odin, and the restart adopts the foreign file + silently. An undecodable change stops persistence forever and breaks Idunn's crossing. + - A: set the file aside (kept byte for byte, never deleted), write Odin's working set over it, keep serving, + and report it as typed state. + - B: set it aside, write the working set, then end Odin so a fresh process starts from the rewritten file. + - **Recommended: A.** B's restart protects nothing that the lease check before the write does not already + protect, and costs a route outage. +- **Q-R2. Keep one previous generation on disk (`topology.cc.previous`)?** + - Context: without it, a store that is corrupt at activation has no on-host source except "empty", and the + backup is up to 24 h old and needs the operator. It costs one hard link per write (a directory update, no data + written) and one more file of the store's size. It is a CultLib option, off by default for other consumers. + - A: yes. B: no; activation recovery goes straight to empty. + - **Recommended: A.** It is the only source that recovers publish-once documents, presence history and + correlations without the operator. Its loss bound is one write. +- **Q-R3. Activation finds nothing decodable: no usable `topology.cc`, no usable previous generation.** + - Context: presences and most catalog documents return within seconds to a minute, and correlations re-derive + in one refresh pass. Publish-once documents (if any exist, see 1.3) stay missing until their publisher + re-puts them, or until the operator restores from the backup. + - A: start empty automatically, set the corrupt file aside, and report `started-empty`. + - B: refuse to start, so Odin crash-loops until the operator restores. Every service's rendezvous is down + meanwhile. + - **Recommended: A.** A down Odin loses everything the Verse needs; an empty one loses only what nobody + republishes, and the restore path stays open. +- **Q-R4. A foreign writer keeps writing (a third set-aside within 10 minutes).** + - Context: repeated repair against a live writer makes a displaced file every second. + - A: stop writing, keep serving from memory, withhold put ACKs, and report `contested` until the operator + finds the writer. The working set survives. + - B: end Odin. Idunn restarts it, and the restart loads whatever the foreign writer last wrote. + - **Recommended: A.** B hands the store to the unknown writer. A is loud and loses nothing that is in memory. +- **Q-R5. Should a store condition change Odin's presence to `degraded`?** + - Context: a non-`active` Odin presence makes every managed dependent of Odin not Ready (`lib.rs:946`, + `dependency_evidence`). The store condition does not affect serving. + - A: no. The presence stays `active`, and the condition record is the signal. + - B: `degraded` while `unpersisted` or `contested`. + - **Recommended: A.** Otherwise a full disk becomes an outage of every dependent, and a restart that Idunn might + take in response would destroy the only complete copy. +- **Q-R6. The drills.** + - D-1: decode a copy of last night's backup `topology.cc` in a container on Yggdrasil, the same terms as Q2 for + the write map, with the copy deleted afterwards. + - D-2: after O3 deploys, a scheduled live drill that truncates the real `topology.cc` behind the lifecycle brake, + keeping a hard link of the original. + - **Recommended: yes to both**, D-1 before O3 ships and D-2 in a window the operator names. + +## 9. Follow-ups outside this map + +- **I-F1 (Idunn):** a deploy or a stop of Odin while `odin.store_condition` says `unpersisted` or `contested` + destroys the only complete copy. Idunn could read the condition from the catalog and hold the deployment + (deployment brake, never continuity). This is Idunn's decision, raised with the Idunn map. +- **I-F2 (Idunn):** "Idunn stopped restarting admitted odin" is a log line (`report_once`, `control_plane.rs:4599`), + not typed state. It belongs to Idunn's own operator surface. +- **F3 (from the write map):** the Eve lowering of `odin.store_condition` belongs to Odin's operator surface. + This map publishes the typed record only. +- **H-F1 (Heimdall):** confirm whether the deployed Heimdall re-puts its Odin documents every 60 s. O3's + outstanding list measures it after any recovery. +- **Q3 B's record in the write map:** "an ACK means durable" holds against crashes, not against a corruption at + activation, which loses at most the last write (C1). diff --git a/docs/write-pattern-cut.md b/docs/write-pattern-cut.md index e174014..3b7d93b 100644 --- a/docs/write-pattern-cut.md +++ b/docs/write-pattern-cut.md @@ -3,7 +3,86 @@ **2026-09-30, Self.** Operator: "Let's fix Odin up, that's some furious bookkeeping right there. Look at Odin's write pattern and see if append only or per-document is the way to go." Answer: neither is the first fix. Cuts 1-2 go to Hands now. Q3 (a 1 s interval) and Q4 (the Rust directory store as a CultLib follow-up) are Self defaults, -and the operator may overrule them. Q1 and Q2 are put to the operator, and Cut 3 waits on Q1. +and the operator may overrule them. **Operator, 2026-09-30: "Q1: yes", so CultMesh refuses puts it could never +serve and Cut 3 goes ahead; "Q2: yeah, of course", so a read-only decode of a copy of live `topology.cc` is +approved. The copy is taken with `sudo cp` and decoded off the live path, and the copy is deleted afterwards.** + +**Soul on Cuts 1-2, and two operator rulings (2026-09-30).** +- **Q3 restated.** The premise "puts carry no application acknowledgement" was false. The cultmesh-rs server + ACKs a put only after `accept_raw_document` returns Ok, so after Cut 1 an ACK meant "in memory", and a SIGKILL + lost acknowledged puts in 6 of 6 probe runs. + - **Operator ruling: "Q3: B".** Peer puts are fsynced before they are acknowledged. Only Odin's own + bookkeeping (heartbeats, correlations) waits for the 1 s interval. An ACK means durable. +- **An undecodable foreign write.** Today Odin stays up, serves from memory and never persists again. Self offered + three options: degraded but loud, end Odin, or unchanged. + - **Operator: "None of these options involve repair and recovery, which is what ultimately needs to happen."** + - Recovery is mapped as its own pass. It covers an undecodable foreign write, a decodable one, and a failed + load at activation. + +**Soul on write batch 2 (2026-09-30).** Q3 B changes the expected effect. Cut 2's row in section 6 ("at most +3.5 MB/s, at most 1.05 writes/s") assumed Q3 A and is **superseded**. Under Q3 B, each peer put (presence +heartbeats included) writes the store once. +- With the live-shaped 3 MB store that is about 2.5-3 writes/s, about 7.5-9 MB/s, and 70-170 ms of the loop + blocked each second. +- Measured: route-challenge p99 461 ms and 11 kernel drops. +- After Cut 3's Odin half the store is about 80 KB and a put write is p50 about 4 ms. +- **Cut 2 therefore deploys only together with Cut 3's Odin half**, which is now being built on `hands/odin-write`. + +**Q2 decode result (Eyes, 2026-09-30; full table in the CultLib session scratchpad `odin-topology-decode.md`).** +- 77 records; 3,222,550 B of payload in a 3,240,078 B file. +- The two "~1.55 MB records" are per-type totals. Each type is dominated by one Ghostlight operator surface: + `eve:operator:ghostlight.campaign.27d444b3-...` (1,456,210 B) and its `operator-state` twin (1,456,341 B). + Together they are 89.9% of the file. +- Publisher: provider `gamecult.ghostlight.dungeon`. 99.5% of each record is one Eve `code` node holding a + pretty-printed JSON dump of the whole campaign state. +- These records are dead residue. They were stored 2026-09-23 and have not been rewritten since. Ghostlight `HEAD` + (`eeb3a68`) no longer contains the emitter (last touched at `6bb6869`, 2026-09-02). +- **So Q1's refusal breaks no live publisher.** Cut 3's Odin half drops exactly these two at activation. Every other + surface is 70 KB or less. +- Correction to section 3: the 1 MiB snapshot refusals in the journal are per-type responses, not per-record. + Cut 3's per-document bound is still right: each of these records is over 1 MiB alone. A type whose many small + records add up past 1 MiB would still be unservable as a whole-type snapshot. That is a follow-up for CultMesh + (paged snapshot responses), not part of this campaign. +- Presence history: 41 records, 42 KB, 37 of them withdrawn. F4 is a slow leak, not a size problem. +- The current `streampixels-web` incarnation has a correlation but no presence record; its last presence was + 09-29 19:53 UTC. This matches Idunn's "not Odin-correlated" state. + +**Soul on write batch 3 + Cut 3 Odin half (`f4a1f89..726e0f3`, 2026-09-30): not fit to deploy.** +- **S-1 (high; operator question Q5, below).** Every write, the fsync pair included, runs on the serving loop + (`main.rs:358` → `flush` → `compare_exchange_snapshot`; the interval flush at `:883` is on the same thread). + - A disk stall passes into route-challenge latency one for one. A probe that held the store lock measured: a 0 ms + stall gave 86 ms, 300 ms gave 288 ms, 1,500 ms gave 1,487 ms. + - Interleaved runs (ABBA) under IO pressure of 5-38% gave a challenge p99 no better with Cut 3 than without it. + - Cut 3 halves the bytes written, not the stalls. **The premise that the store's size caused the stalls was + wrong; they come from the disk.** +- **S-2 (high).** Q1's put refusal has not landed. Odin's pin `3bf1c0ce` has no put bound, so a 1.2 MB put is + ACKed, fsynced, and dropped at the next activation. It waits on the CultLib put-serve merge, then a pin bump with + a `served_record` override. +- **S-3 (medium).** `attempted` grows about 98 KB per failed write, roughly 0.9 GB/h on a full disk, which ends in + OOM. + - Fix in the owner: cultcache-rs returns a typed `NotReplaced`/`ReplacedNotDurable` write outcome (added to CultLib + R1 batch 3), and Odin keeps at most one candidate. +- **S-4 (medium).** Two admitted presences for one target, which a provider in a deploy window produces, refuse the + whole catalog (`main.rs:~628`). +- **S-5 (medium).** A RUDP transport ACK is recorded at receipt, so with pipelined puts a later put is ACKed while + an earlier one is still held. With the first datagram lost, the second is never delivered. That is data loss + under the QUIC-pivot rule; it is tracked in the CultLib ack map. `dcd36fc`'s doc line overstates. +- **S-6, S-8.** Test gaps (the limit taken through `main`; `remove_not_dirty`). +- **S-7.** The drop log echoes the key. Self ruled keys and schema ids are identities and may be logged, as in + CultLib R1. +- **Held:** F1's logic, F4's projection fallback, F5, F3's refusal, and Cut 3's activation bound, drop and removal. + Sizing uses `peer_document`, so it matches what is served. +- **Batch 4 in Hands:** S-4, S-6, S-8, the S-5 doc line, the attempted-list fixture gap, and committing Soul's + probes. S-1 waits on Q5; S-2 and S-3 wait on CultLib. + +**CultLib side landed (2026-09-30).** Ack Cut D was merged at `2f06ef84`, R1 at `daedfdd6` and the put-serve bound at +`9dcb3aa1`. That unblocks the Q5 sequence: +1. R1 batch 3's typed write outcome is on main. It carries `Rejected` as well as `NotReplaced`/`ReplacedNotDurable`. + W0 must classify by kind together with the `io::Error`, never by `NotReplaced` alone. +2. **C5 next:** cultmesh-rs deferred put reply, on main. +3. **W0:** one Odin pin bump. It closes S-2 (put refusal with a `served_record` override) and S-3 (delete + `attempted`), and it brings in Cut D's and the put-serve error texts. +4. **W1:** the writer thread. Status: cut map, Imagination pass 1 (Opus), 2026-09-30. Nothing has landed. Anchors are Odin `main` at `44951a1` (the deployed build, live on Yggdrasil since 2026-09-30 04:20:48 UTC) and CultLib `3bf1c0ce` @@ -352,6 +431,532 @@ records; update the check's type list in the same pass. - **Q4. CultLib parity: port `DirectoryMessagePackBackingStore` (v4) to cultcache-rs?** It is not needed by Odin after Cuts 1-3. A reasonable consumer of the Rust runtime would expect the reference's store. **Recommended: record it as CultLib follow-up F1 with no deadline.** It is independent of this map. +- **Q5 (2026-09-30, from Soul S-1). Where does the durable write run?** Today the serving loop runs the fsync, so + any disk stall stalls route challenges, catalog reads and every other put. + - A: a writer thread with group commit. + - The loop hands each accepted put to one writer thread. The writer writes and fsyncs everything queued since + its last write in one go, then the loop sends those puts' replies. + - Challenges and reads are never behind the disk. A stall delays only put replies, and Q3 B still holds (a + reply means durable). + - It needs cultmesh-rs to let a put handler reply later (a CultLib API change) plus one Odin thread. + - B: answer route challenges on their own path. Everything else still stalls, and the challenge would report + Odin healthy while its catalog is stuck. + - C: reopen Q3 and go back to the 1 s interval. That brings back lost acknowledged puts on a crash. + + **Recommended: A.** **RULED A by the operator, 2026-09-30: "agree on Odin writer thread".** Mapping is next + (Imagination): the cultmesh-rs deferred put reply, then the Odin writer thread with group commit. Mapped below in + "Q5 cut: the writer thread" (cuts C5, W0, W1; forks Q5-A and Q5-B). + +## Q5 cut: the writer thread + +**Imagination pass (Opus), 2026-09-30.** Maps the operator's ruling on Q5, "agree on Odin writer thread" (option A, +section 7). Anchors: +- Odin `hands/odin-write` at `434756e` (`crates/odin-daemon/src/main.rs`, `lib.rs`), CultLib pin `3bf1c0ce`. +- CultLib `hands/cultmesh-put-serve-bound` tip `e4e2a0b9` (`packages/cultmesh-rs/src/rudp_document_server.rs`). + Its merge base is `42ba0f08`, which already holds ack Cuts 1, 1b and 1c. +- CultLib `hands/ack-cutD` tip `e9d5ae2c` (lands on main first), and R1 batch 3 `hands/cultcache-read-failures` tip + `93ac5945` (`CultCacheStoreWriteFailed`). + +The mapping raises two forks, **Q5-A** and **Q5-B** (at the end of this section). Cuts C5 and W1 wait on Q5-A. + +**RULED, operator 2026-09-30: "Q5-A: A, but make a note of this for when we move on QUIC, Q5-B: A".** +- **Q5-A = A.** The server withholds the session's transport ACK until the put is answered. The wire is unchanged. + The note for QUIC (a put-result message) is in the QUIC campaign handoff. +- **Q5-B = A.** Puts are held during a writer stall, bounded by the existing session and byte limits. + +### Q5.1 Body facts (probed and read) + +**Probe O1, Odin `434756e` + scratch test, Yggdrasil slot 3 (`imag-q5-odin@31953c93`).** A test process holds the +store's sibling lock for 1,500 ms, the way Soul modelled a disk stall. Full serving passes run meanwhile. A route +challenge is sent at 200 ms, then a catalog read. + +| Case | Route challenge | Catalog read | +|---|---|---| +| nothing waiting to be written | served in **39 ms** | served in 29 ms | +| only Odin's own heartbeat waiting, and the interval write due | served in **1,342 ms** | 33 ms (sent after the release) | + +- No read path takes the store lock. At `434756e` the lock is taken only by `MemoryOdinTopologyStore::load` + (`lib.rs:509`, at activation) and by `flush` (`lib.rs:612` → `compare_exchange_snapshot`). +- The stall reaches the loop through **both** writes on it: a peer put (`main.rs:363`, Soul S-1) and the interval + write of Odin's own bookkeeping (`main.rs:933-941`). The interval write stalls the loop with no peer put at all. + So the bookkeeping write must move to the writer too, not only the put write. + +**The loop still reads two other files.** They are not the store and take no lock, but they are disk reads on the +loop: +- The lease file, on every route challenge and catalog read: `raw_snapshot` `main.rs:389-409` → + `require_current_write_lease` `:546-567` → `read_process_write_lease` `:1323` (`pull_all_read_only_snapshot`). +- Idunn's projection, on every catalog read (`:608`), refresh (`:572`) and heartbeat (`:524-544`). +- Both files are small and rewritten rarely by Idunn, so they are normally served from the page cache. Probe O1 + cannot show whether a real disk stall evicts them. Only the Yggdrasil rerun (Q5.6) can. If it does, the answer + is to hold the lease record in memory and re-check it where it is decided. That is its own follow-up, not this + cut. + +**What a put's reply is on the wire today.** CultNet has no put-acknowledgement message: the list in +`cultnet-rs/src/contracts.rs` has `document_put_raw` and `error.v0`, and no put result. +- A Rust publisher (`publish_cultnet_message_to_rudp_catalog`, `cultmesh-rs/src/lib.rs:871-924` at `e4e2a0b9`) + succeeds when its reliable packet is acknowledged by the RUDP transport, and fails on a `cultnet.error.v0`. + - Its default `flush_timeout` is 300 ms, with a resend every 50 ms (`:144-157`). +- The document server sends that acknowledgement at the end of `poll_once`, after the sink returns + (`rudp_document_server.rs:468-476`). A refusal ends the session with an Error frame that acknowledges nothing + (`end_rejected_session` `:760-793`, put-serve branch). +- **So today the put's reply is the transport acknowledgement.** That is how Q3 B's "an ACK means durable" holds: + the sink writes before it returns. + +**Every packet a session sends carries acknowledgement fields** (`ack`, `ack_mask`, from `ack_state`, cultnet-rs +`rudp.rs` at main). +- The fields report every *received* reliable sequence, not only delivered ones. +- `create_packet` stamps them on each new packet: data, Pong, the end-of-poll Ack. +- A reliable packet is stored with the fields it was created with, and resent unchanged (`track_reliable`, + `due_resends`). + +Probes P1-P4 (cultnet-rs sessions at `e4e2a0b9`, scratch test `imag_q5_probe.rs`, Yggdrasil, `imag-q5-cultlib@8dcfe36a`): + +| # | Case | Result | +|---|---|---| +| P1 | two pipelined ordered puts, the first lost | the second is delivered to nobody, yet `create_ack` acknowledges it. **S-5 is still open after ack Cuts 1-1c.** `ack_state` at main is the same code, so the ack map's to-do resolves to "not closed by the pin bump" | +| P2 | a put delivered, then a Ping | the Pong acknowledges the put | +| P3 | the resend of a reply created **before** the put | does not acknowledge it | +| P3 | a reliable reply created **after** the put | acknowledges it, and so does its resend | +| P4 | a retransmit of a delivered put | not delivered again (the server's end-of-poll ack still answers it, `:468`) | + +Consequence for a reply sent later: holding back only the end-of-poll ack is not enough. +- A Pong, or a snapshot response created on the same session after the put arrived, acknowledges the put too. +- The snapshot response keeps doing so on every resend, even if its first copy were patched. +- So a session with a put awaiting its answer must be **sent nothing created after that put arrived**. Resends of + earlier packets are safe: they carry the fields they were created with. + +**The other runtimes (parity).** +- **TS** (`cultmesh-ts/src/index.ts:5891-5914` at `e4e2a0b9`): + - It acknowledges each packet at receipt (`createAckForReceived`), before the handler runs. + - It runs each session's frames through one promise chain (`record.work`), so handling is serial per session. + - `onDocumentPutRaw` may return a Promise and an application-level receipt document, which is sent when it + resolves (`:5951-5972`). + - So TS already "replies later", but at the application level, and its transport ack does not mean handled. +- **Python** (`cultmesh-py/src/cultmesh_py/server.py:407`, `_handle_raw_put` `:415`): the handler is synchronous + and returns its responses. +- Neither has a durability meaning for its ack. Parity is follow-up **F-Q5a** (Q5.8). + +**Consumers of the sink trait** (`CultMeshRudpRawDocumentSink`): +- Odin's `SinkHandle` (`main.rs:168-172`). +- Ratatoskr's catalog test (`Ratatoskr/crates/ratatoskr-core/tests/catalog.rs:29`). +- CultLib's own tests, which use closures through the blanket impl (`rudp_document_server.rs:60-67`). +- Vendored copies (StreamPixels, Heimdall) do not implement it. + +**cultcache-rs write outcome (R1 batch 3, `93ac5945`).** +- Every failed single-file write carries `CultCacheStoreWriteFailed { kind: NotReplaced | ReplacedNotDurable }`. +- Any other error from `compare_exchange_snapshot` (lock, read, decode, validation) comes before the write step, + so the file is unchanged. +- One hole: `with_exclusive_lock` (`cultcache-rs/src/lib.rs:924-932` on that branch) returns the **unlock** error + after a write that succeeded, with no marker. Odin would read that as "not replaced". Its next write would then + find its own snapshot and report `ForeignStoreWrite`, and Odin would end. + - That is rare (unlock failing), and it belongs to R1: follow-up **F-Q5b**, not this cut. + +### Q5.2 Target shape + +1. **CultLib (cut C5).** A put's sink may answer later. + - While any put of a session awaits its answer, the server sends that session nothing new. + - The one exception is resends of packets created before the put arrived. + - A snapshot request behind the pending put waits in the session's order. + - An accepted answer releases one ordinary acknowledgement. + - A refusal ends the session with the Error frame the put-serve branch sends. + - The wire does not change, and existing publishers keep "acknowledged means the sink accepted", which for Odin + is durable (Q5-A). +2. **Odin (cut W1).** One writer thread owns the store file after activation. + - The loop owns the working set, decides when to write, and answers put replies from the writer's outcomes. + - Every put accepted while a write is in flight is covered by the next single write (group commit). + - Route challenges and catalog reads never wait on the writer. +3. **Odin (cut W0), before W1.** + - A pin bump. + - S-2: a `served_record` override so Odin's admission is exact. + - S-3: the `attempted` list dies, because the typed write outcome says what the file holds. + +### Q5.3 Identity, lifecycle, authority + +| Kind | Named by | Over time | Decided by | +|---|---|---|---| +| pending put (CultLib) | a server-minted `u64` put id, unique for the server's life, never reused across session generations | created when an admitted put is offered to the sink; ends when it is answered, or when its session ends (the answer is then discarded) | the sink answers; the server alone sends | +| put reply handle (CultLib) | the put id, inside a `CultMeshRudpPutReply` | owned by whoever the sink hands it to; consumed by `accept` or `refuse`; dropping it unanswered refuses | the sink's owner | +| working-set version (Odin) | a `u64` counter in `MemoryOdinTopologyStore` | +1 on every mutation that changes a record; never reset within an activation | the loop | +| written-through version (Odin) | a `u64` in `RuntimeState` | raised only by a writer outcome `Written { version }` | the writer's outcome, applied by the loop | +| write request (Odin) | its version | at most one in flight; replaced by nothing (the next request is built when the outcome arrives) | the loop submits; the writer executes | +| what the file holds (Odin, `flushed`) | none: one snapshot | set at activation from the load; replaced on `Ok(true)` and on `ReplacedNotDurable` | the writer alone | + +"Dirty" stops being a flag (`lib.rs:502`, `dirty: Cell`): it is derived as `version > written_through`. + +### Q5.4 Cuts + +#### Cut C5. A put's sink may answer later (CultLib, cultmesh-rs, behaviour) + +- **Repo/branch:** CultLib `hands/cultmesh-put-reply-later` from `main` after the put-serve branch merges (Q5.7). + Anchors below are at put-serve `e4e2a0b9`; they move by Cut D's diff (Q5.7 names the conflicts). +- **Deletes first:** the synchronous sink call and its error mapping in the put arm, `rudp_document_server.rs:621-633` + (they are replaced, not wrapped). Nothing else is deleted. +- **Public API** (default D-1 below): + + ```rust + pub trait CultMeshRudpRawDocumentSink { + /// Answer through `reply` now or later, from any thread. + fn accept_raw_document(&mut self, receipt: CultMeshRudpRawDocumentReceipt, reply: CultMeshRudpPutReply); + } + // Closures keep working and answer at once: + impl Result<()>> CultMeshRudpRawDocumentSink for F { .. } + + #[must_use] pub struct CultMeshRudpPutReply { /* put id, Sender to the server */ } // Send, !Clone + impl CultMeshRudpPutReply { + pub fn accept(self); + pub fn refuse(self, reason: impl std::fmt::Display); + } + // Drop without an answer refuses: "the put was not answered". + ``` + + - `CultMeshRudpRawDocumentReceipt` (`:37-43`) is unchanged; the reply is a separate argument, so the receipt stays + `Clone + PartialEq`. + - `CultMeshRudpPollOutcome` (`:286-290`) is unchanged. A later refusal is returned as `ApplicationRejected` with + `SinkRefused(reason)` from the `poll_once` that sends it. +- **Server changes, `rudp_document_server.rs`:** + - `CultMeshRudpDocumentServer` (`:303-311`) gains: + - The answer channel (`std::sync::mpsc`: the server keeps the receiver, each reply holds a sender). + - `answered: VecDeque`, the answers taken off the channel and not yet handled. + - A next put id. + - `pending: BTreeMap`. + - `SessionEntry` (`:292-297`) gains `pending_puts: BTreeSet` and `waiting: VecDeque<(u32, CultNetMessage)>`, + the delivered frames that may not be answered yet. + - **One predicate owns "withheld":** `fn acknowledgement_withheld(&self, key) -> bool`, true while + `pending_puts` is non-empty. The S-5 fix, if the ack map takes it here, adds its clause to this function and + nowhere else. + - `poll_once`: + - Drain answers first (before `maintain`, `:366`). Each answer is looked up in `pending`; an unknown id (its + session ended) is discarded. + - Refuse → `end_rejected_session(key, reason)`, and return one `ApplicationRejected`. Further answers stay in the + channel for the next poll. + - Accept → remove the id. When the session has none left, it is released: send its ack (the rule for which + poll sends it is under the put arm below), then handle `waiting` in order. + - `:436-438`: a `result.reply` (Pong) is not sent while withheld. + - `:468-476`: the end-of-poll ack is not sent while withheld. + - `deliver_application_message` (`:565`): + - Put arm (`:572`): admission is unchanged (put-serve `:587-619`). Then mint the id, record it, and call the sink + with a reply. + - `poll_once` looks at answers again after the delivered-frame loop (`:445-466`), before the end-of-poll ack, + but handles only those for this poll's session. The rest stay queued for the next poll's first step. (Answers + move from the channel into one server-side queue, so they can be taken selectively.) A sink that answered at + once is therefore acknowledged, or refused, in the same poll with the same datagrams as today, and another + session's refusal can never cut this session's ack short. + - A release sends `create_ack()` from the drain, except for the session whose packet this poll is handling: + that session gets its ordinary end-of-poll ack, so a synchronous sink's datagrams stay exactly as they were. + - Snapshot arm (`:634`): if the session is withheld, push the request onto `waiting` and return; otherwise + answer as today. + - A put that arrives while `waiting` is non-empty also queues, so a session's frames are handled in order. A + put behind another put that is only pending is offered at once, so pipelined puts can share one write. + - Session end, wherever a session is removed (`:441`, `:486-489`, `:523`, `end_refused_session`, + `end_rejected_session`, Cut D's `end_unsendable_session`): remove its ids from `pending`, so their answers are + discarded. + - **Put this in one `remove_session(key)` helper, so no removal path can forget it.** It is the one new helper + the cut earns: there are six removal sites today. + - Doc comments: the struct doc (`:300-302`, "no background thread") stays true. The sink trait doc says what + "acknowledged" means: the sink accepted. +- **Authority map:** + - Owner of what the peer is told about a put: the server, and only through `poll_once`. The sink decides accept or + refuse; it never sends. + - Inputs: sink answers (channel); transport receipts. + - Outputs: one ack per release; one refusal per refused put's session. + - Derived: "withheld" is derived from `pending_puts`; it is not a flag. + - Forbidden writers: + - No send path to a withheld session except `maintain`'s resends. + - The sink must not be able to acknowledge by returning. The return type carries no answer, so this holds by + construction. + - Shared paths: a synchronous closure sink and a deferred sink answer through the same reply and the same drain. + There is no second code path for "answered at once". +- **Verification (CultLib, Yggdrasil):** + - Wire-level test, the observer layer: a raw client socket records every datagram the server sends to its session. + - With the reply held across at least five resend intervals, including a Ping and a snapshot request on that + session, no datagram acknowledges the put's sequence. + - After `accept()`, the next poll sends an ack that does, and then the snapshot response. + - Mutations that must fail it: sending the end-of-poll ack regardless; sending the Pong; answering the snapshot + request at once. + - Another session's snapshot request is answered while the first session's put is held (the loop is not blocked). + - `refuse()` and dropping the reply: the peer receives `cultnet.error.v0` and a goodbye, and nothing acknowledges + the put (reuse the put-serve refusal assertions). + - Two pipelined puts on one session, each with its own held reply: + - Answering the second first releases nothing. + - Answering the first then releases one ack that covers both. + - This pins that a later put's acknowledgement never leaves before an earlier one's. + - Session ends while held (peer Disconnect; a new Connect from the same key): a later `accept()` sends nothing to + the new generation, and `pending` is empty. + - Closure sink: the datagram sequence is byte-identical to the pre-cut server for accept and refuse. This pins that + the synchronous path did not change. + - An unservable put is refused before the sink sees it: the sink is never called and no reply is minted. + - `cargo mutants --in-diff` on the cut's diff; every survivor is triaged. +- **Estimate:** about +170 source, -15 source, +350 tests. No new target, dependency or wire field. + +#### Cut W0. Pin bump, exact admission and the one-candidate store (Odin, subtraction first) + +- **Repo/branch:** Odin `hands/odin-write-w0` from `hands/odin-write` (`434756e`, or batch 4's tip). It bumps the + CultLib pin to the main commit holding Cut D, put-serve, R1 batch 3 and C5 (Q5.7). +- **Deletes first:** + - `attempted` (`lib.rs:494-499`, `:525`), the retry loop over it in `flush` (`:619-625`) and its push (`:628-633`). + - The tests that describe a file holding some older failed attempt: + - `a_write_that_failed_before_replacing_keeps_every_earlier_attempt_odins` (`lib.rs:2172`). + - `the_file_may_hold_a_later_failed_attempt_not_only_the_first` (`:2205`). + - Each models a `NotReplaced` failure that nonetheless changed the file, which the typed outcome now rules out. +- **Changes:** + - `flush` (`lib.rs:612-640`) matches the error's `CultCacheStoreWriteFailed` kind (`downcast_ref`): + - `ReplacedNotDurable` → `flushed = current`, and the store stays dirty so the next write re-syncs. + - Anything else → no change. + - `a_write_that_failed_after_replacing_the_file_is_not_another_writer` (`:2141`) is re-targeted to inject + `ReplacedNotDurable` as the real outcome, not a hand-made replace. + - `a_failed_write_is_made_by_the_next_without_another_change` (`:2118`) stays. + - **S-2:** Odin's `SnapshotHandle` (`main.rs:174-181`) overrides `served_record`. It builds the envelope as + `persist_generic_document` (`:1462`) would store it, then serves it through `peer_document` (`:1400`), the + function `drop_unservable_documents` (`:1428`) sizes with. Admission and activation then use one sizing. + - A runtime-presence put keeps the default (as received). Presences are served in `public_document`'s shape and + are never near the bound, and `drop_unservable_documents` does not check them either. + - A schema that `persist_generic_document` would refuse returns an error, which the server turns into a + refusal before the sink. + - `accept_raw_document`'s doc (`main.rs:332-351`) drops its S-5 paragraph; it moves to W1's writer doc. +- **Authority map:** what the file holds is decided by the one write outcome, not by a list of guesses. + `ForeignStoreWrite` means exactly "a snapshot Odin did not write". +- **Verification:** + - Test: a `ReplacedNotDurable` failure, then a change, then a write → made over the failed snapshot. + - Test: a `NotReplaced` failure, after which another writer puts the failed snapshot into the file → `ForeignStoreWrite`. + Mutation: keeping a candidate fails it. + - Test: 1,000 failed writes leave nothing retained. This pins S-3; observe the store's retained records, or the + absence of the field. + - S-2 test: a 1.2 MB put is refused as `DocumentUnservable` and is not in the working set. + - Mutation: sizing with the received record instead of `peer_document` survives nothing. + - Negative: `rg -n attempted crates/odin-daemon/src` matches nothing. + - The pin bump carries ack Cuts 1-3 and D into Odin. The full Odin suite runs on Yggdrasil, and Soul reads this cut + as a transport change, not only a subtraction. +- **Estimate:** about -150 (including the two tests) / +60. + +#### Cut W1. The writer thread (Odin, behaviour) + +- **Repo/branch:** Odin `hands/odin-writer` from W0. +- **Deletes first:** + - `MemoryOdinTopologyStore::flush` (`lib.rs:602-640`), its `path` (`:490`) and `flushed` (`:493`) fields, and the + `dirty` flag (`:502`, `:600-602`). + - `RuntimeState::flush` (`main.rs:317-330`). + - The `self.flush()?` in `accept_raw_document` (`main.rs:363`). + - `stop`'s direct write (`main.rs:891-898`). + - The flush step of `serving_pass` (`main.rs:933-941`). + - No path is left on the loop that can write the file. That holds before the writer is added. +- **Keeps:** + - `MemoryOdinTopologyStore::load` (`lib.rs:509`): the one locked read, at activation, on the loop, before any writer + exists. `load` returns the loaded snapshot beside the store, to become the writer's `flushed`. + - `ForeignStoreWrite`, `WriteLeaseLost`, `survive` (`main.rs:979-994`), unchanged in meaning. +- **Adds, `crates/odin-daemon/src/writer.rs`** (a module, not a crate): + - `StoreWriter`: `spawn(file, flushed, lease) -> StoreWriter`, `submit(WriteRequest)`, + `try_outcome() -> Option` and `finish(self)`. + - The channel is `sync_channel(1)`. The loop submits only when nothing is in flight, so `submit` never blocks. + - `WriteRequest { version, records }`. + - `WriteOutcome { version, result }`, where `result` is one of `Written`, `Failed(text)`, `Foreign` or + `LeaseLost(text)`. + - The thread: + - Checks the lease (the check extracted from `require_current_write_lease` `main.rs:546-567` into a free + function that the loop and the writer share), then `compare_exchange_snapshot(flushed, records)`. + - Applies W0's outcome rule to `flushed`, and sends the outcome. + - It never touches reply handles, the working set or the socket. + - Mock point: the thread writes through a one-method trait `StoreFile { compare_exchange_snapshot }`, implemented + for `SingleFileMessagePackBackingStore`. Tests gate it and count it. The lock-hold (Soul's method) is used where + the real file must be on the path. +- **Changes, `main.rs`:** + - `RuntimeState` (`:141-161`) gains `writer: Option` (spawned in `try_activate` after the load, + `:284-305`), `written_through: u64`, `in_flight: Option` and `replies: VecDeque<(u64, CultMeshRudpPutReply)>`. + - `SinkHandle::accept_raw_document` (`:168-172`) takes the reply. + - `RuntimeState::accept_raw_document` (`:352-367`): + - Validates and mutates as today. + - Pushes `(store.version(), reply)`, or accepts at once when `version <= written_through` (nothing new to make + durable). + - Refuses through the reply on a validation error. + - Submits if nothing is in flight. + - A new `serving_pass` step, in place of `:933-941`, run every pass: + - `try_outcome`. On `Written { v }`: `written_through = v`, and accept every reply with version `<= v`, in queue + order. + - On `Failed`: refuse the replies with version `<= v`; they remain in the working set (refusal runs one way, as + documented at `:344-350`). + - On `Foreign` / `LeaseLost`: return the error to `survive`, which ends Odin. + - Then, if nothing is in flight and `version > written_through`, submit when a reply is waiting or when + `FLUSH_INTERVAL` is due. + - So puts never wait for the interval; bookkeeping-only changes do. + - `stop` (`:894`): + - If the lease is still current and anything is unwritten or in flight, wait for the in-flight outcome, submit the + rest, and wait for that outcome too. This is a blocking wait; it is the only one, and it happens only on the + way out. + - Answer the replies, run one `poll_server` so the acks leave, then `finish` (drop the sender and join the + thread). + - After `WriteLeaseLost` or `ForeignStoreWrite`, write nothing. Dropping the replies refuses them. + - `FLUSH_INTERVAL`'s comment (`:73-77`) and the store's doc (`lib.rs:484-488`) say: the writer thread owns the + file; puts are covered by the next write; bookkeeping waits for the interval. +- **Authority map:** + - Owner of the store file after activation: the writer thread, the only holder of a + `SingleFileMessagePackBackingStore` for the store path. It also owns what the file holds (`flushed`). + - Owner of the working set, of when to write, and of put replies: the loop (`RuntimeState`). + - Inputs: + - Writer: one request at a time, and the lease file. + - Loop: provider puts, Idunn's projection, the lease file, and writer outcomes. + - Outputs: one atomic `.cc` snapshot per request; one outcome per request; replies answered in version order. + - Derived state: + - `dirty` is derived (`version > written_through`). + - `flushed` is no longer the loop's; it lives only in the writer. + - The interval decides only bookkeeping-only writes. + - Forbidden writers: no loop code path calls `compare_exchange_snapshot`, takes the store's lock, or answers a + put reply before the outcome covering its version. + - Shared paths: peer puts, self-presence, refresh, activation-time drops and the stop write all mutate the working + set, and reach the file only through `submit`. + - Deletion line: every loop-side write (above) is deleted before `writer.rs` is added. +- **Batching rule, stated once.** The loop keeps at most one write in flight. Everything that changed while it was in + flight, puts and bookkeeping alike, goes into the next request as one snapshot of the working set. N puts accepted + during one write cost one more write and one fsync pair, not N. +- **Backpressure** (Q5-B, recommended: hold): + - A stalled writer holds put replies. The loop keeps serving, and the working set keeps admitting (the catalog + serves admitted-not-durable puts, as it does today after a failed write). + - The queue is bounded by the server's existing limits: 64 sessions and 32 MiB of admitted payload. It adds no new + bound. + - A publisher that stops waiting (the Rust default is 300 ms) disconnects. Its session ends, its answer is + discarded, and its put is still written by the next write. That is what a publisher sees today during a stall, + minus the stalled challenges. +- **Ordering.** + - Replies are answered in version order from one queue, and the server sends acks in answer order. + - Within a session, C5 releases one ack only when every pending put of the session is answered. + - So no later put's reply leaves before an earlier one's. +- **Crash mid-batch.** + - The write is one atomic replace, so the file holds the old snapshot or the new one. + - No reply of a batch is answered before its `Written` outcome, so a SIGKILL anywhere loses no acknowledged put. + - A crash after the rename but before the directory sync is `ReplacedNotDurable` if the process lives; if it dies, + nothing of that batch was acknowledged. +- **Activation.** The store is read once, by `load`, before the writer is spawned. While the writer lives, the loop + never reads the file (Cut 1's rule, unchanged). +- **Verification (Odin, Yggdrasil):** + - **Loop latency under a stall.** This pins Q5's ruling; it is Soul's S-3 probe and O1, inverted into assertions. + - Hold the store's sibling lock for 1,500 ms, with a provider put in flight and Odin's heartbeat dirty with the + interval due. + - A route challenge and a catalog read are each served within 200 ms. The put's publisher sees no ack until the + lock is released, and then does, with the put on disk. + - Mutation that must fail it: running the write synchronously on the loop, by calling the `StoreFile` directly + from `serving_pass`. + - **Group commit.** Gated `StoreFile`, held on the first write, with 10 puts from 10 publishers accepted meanwhile. + - After release, exactly 2 writes are made and all 10 publishers are acknowledged. + - Mutation: one request per put gives 11 writes. + - **Ordering.** Gated `StoreFile`; put A's write is held and put B arrives. + - B's publisher is not acknowledged before A's; the answer order is observed at the server's answer channel. + - Mutation: answering the queue from the back. + - **Durability.** `daemon_process.rs`, extending `an_acknowledged_put_survives_sigkill` (`:326`). + - 20 rounds, each a stream of puts from 4 publishers while the test holds and releases the store's lock at random + offsets. SIGKILL at a random offset. + - Every put its publisher saw acknowledged is in the file after the kill. + - Mutation: accepting replies at submit instead of at `Written`. + - **One writer.** With the writer's `StoreFile` gated shut, N serving passes, a heartbeat and a stop request make no + write from the loop: the file's identity (`file_identity`, `main.rs:3343`) is unchanged until the gate opens. + - Negative grep: `rg -n "compare_exchange_snapshot|pull_all\(" crates/odin-daemon/src` matches only `writer.rs`, + `load` and the tests. + - **Lease lost with a write queued.** No write is made, Odin ends, and the queued replies are refused. The file's + identity is unchanged. + - **Foreign write.** Odin ends without writing over it (existing `:3696`, re-targeted through the writer). + - **Stop.** SIGTERM with a put held behind the lock: after the release Odin writes, the publisher is acknowledged, + and it exits 0. With the lease lost, it writes nothing. + - Re-targeted, not deleted: `changes_inside_one_interval_are_one_write` (`:3356`, now bookkeeping-only), + `a_put_is_written_before_it_is_accepted` (`:3389`, observed at the publisher's ack), + `a_put_whose_write_is_not_made_is_refused` (`:3516`), and the `ending_*` tests (`:3728-3800`). + - `cargo mutants --in-diff` on the cut's diff. +- **Estimate:** about -120 / +230 source, +300 tests. One new module, one thread, no new crate, dependency or target. + +### Q5.5 What stays on the loop, and what does not + +| Path | Before (`434756e`) | After W1 | +|---|---|---| +| route challenge (`raw_snapshot`, exact self query) | lease-file read, sign; blocked behind any write on the loop | the same reads; never behind a write | +| catalog read (`stored_snapshot`) | working set, projection read | unchanged; never behind a write | +| peer put | mutate, then write and fsync on the loop | mutate, queue the reply, submit; the reply is sent after `Written` | +| heartbeat, refresh | mutate; the interval write on the loop | mutate; the interval write on the writer | +| store lock | taken by `flush` on the loop | taken only by `load` (activation) and the writer | + +### Q5.6 What only Yggdrasil can show + +- **Rerun Soul's ABBA** (`soul-c3/latency_cmd.txt`: `soul_loop_stall_under_live_put_rate`, base against cut, in + interleaved rounds, with `/proc/pressure/io` before and after each). + - The base is `434756e`; the cut is W1's tip. Expect route-challenge p99 to stop tracking IO pressure, while put + round trip p99 still does. + - Soul's figures at `434756e` and its base over 8 rounds, under 5-38% pressure: challenge p99 310-2,129 ms, put + round-trip p99 1,121-2,593 ms (`soul-c3/latency.log`). +- Whether the loop's remaining disk reads (the lease and projection files) ever stall under real pressure. The ABBA + run shows it as challenge p99 outliers that coincide with pressure; if they appear, open the lease-in-memory + follow-up. +- After deploy, section 6's script, plus the count of "admitted odin route continuity: timed out" per hour. + +### Q5.7 Sequencing (recommended order) + +1. **Cut D merges to CultLib main** (already slated first). +2. **The put-serve branch rebases onto it and merges.** The conflicts: + - Cut D changes `send_packet` to return `Option` and adds `end_unsendable_session`. + - Its snapshot-send failure builds `reason: format!(..)`, where the put-serve branch made `reason` a typed + `CultMeshRudpRejectionReason`. It needs a variant (for example `ResponseSendFailed(String)`), and the put-serve + Hands owns it. + - Both branches edit `poll_once`'s ack site (`:468-476`) and `end_rejected_session`. +3. **R1 batch 3 merges** (cultcache-rs only; no conflict with 1-2). +4. **C5 is cut from that main** (it rewrites the lines 1-2 touched, so it goes after them), then Soul, then merge. +5. **One Odin pin bump to that commit, as W0**, carrying S-2, S-3 and the transport cuts. Then **W1** on top. + +W0 may start on the commit after step 3 if C5 slips, at the cost of a second bump before W1. One bump is recommended: +each bump moves Odin across ack Cuts 1, 1b, 1c, 3 and D, and a single Soul pass over that transport delta is cheaper +than two. + +### Q5.8 Follow-ups this map does not own + +- **F-Q5a (parity).** TS and Python acknowledge a put at receipt, and TS's reply-later is an application receipt. + - Either they adopt C5's rule ("acknowledged means the sink accepted"), or the QUIC campaign defines an + application-level put reply for every runtime. + - QUIC has no transport ack an application can read, so it needs one anyway. Recommended owner: the QUIC campaign. + Rust C5 is then the RUDP-era contract, and TS's receipt is the seed of the QUIC shape. +- **F-Q5b (R1).** `with_exclusive_lock` returns an unlock error after a successful write, with no write-outcome + marker (`cultcache-rs/src/lib.rs:924-932` at `93ac5945`). Odin would call its own next write foreign. Fix it in + cultcache-rs: tag it, or ignore the unlock error, since dropping the file releases the lock. +- **S-5 interface.** C5 adds `acknowledgement_withheld` as the one place that decides "send no ack to this session". + The ack map's S-5 decision (probe P1) plugs in there as a second clause if it is fixed in the document server. C5 + does not fix S-5. + +### Q5.9 Subtraction ledger (estimate) + +| Cut | Removed | Added | Targets, dependencies, wire | +|---|---|---|---| +| C5 (CultLib) | ~15 | ~170 source, ~350 tests | none; sink trait signature changes (Odin, Ratatoskr test) | +| W0 (Odin) | ~150 (the `attempted` list and 2 tests) | ~60 | pin bump | +| W1 (Odin) | ~120 (every loop-side write) | ~230 source, ~300 tests | one module, one thread | + +Net source is positive: about +175. It buys the ruled capability (challenges and reads independent of the disk, with +Q3 B kept), and it removes S-3's unbounded list. + +### Q5.10 Operator questions and defaults + +- **Q5-A. What does a put's reply mean on the RUDP wire once it can come later?** + - A: the document server withholds the transport acknowledgement until the sink answers. Nothing created after the + put is sent to that session until then. The wire is unchanged, and every existing publisher (Rust + `publish_cultnet_message_to_rudp_catalog`, TS and Python publishers) keeps "acknowledged means accepted", which + for Odin is durable. + - B: a new CultNet message, a put result, sent after durability; the transport ack means received. This is a wire + change in the C# reference and four runtimes. Every Odin publisher that waits for the transport ack today must + be changed to wait for it; until then Q3 B silently weakens to "received". + - **Recommended: A.** It keeps Q3 B for every publisher with a server-only change, and RUDP is frozen except for + crash and data-loss fixes. B is the right shape under QUIC (F-Q5a), not a reason to change the RUDP wire now. + - What depends on it: C5's whole design. + - A also narrows the ack map's Q-A2 meaning for the document server: acknowledged means delivered *and answered*. + That was already true of the synchronous sink. +- **Q5-B. A writer stalled for seconds: hold put replies, or refuse with a typed error?** + - A: hold. Replies wait, bounded by the server's session and payload limits. Publishers time out on their own + clock, and the put is written when the disk returns. + - B: refuse once more than N replies wait or the oldest waits longer than T. This needs a new `cultnet.error.v0` + code (the error-contract map owns the registry) and two tunables. + - **Recommended: A.** A refusal ends the publisher's session exactly as its own timeout does, but adds a code and + two numbers, and it tells the publisher less than the truth: the put may still become durable. B earns its place + only if publishers ever wait without a timeout. +- **Default D-1 (Self, overridable). The API is a completion handle** (`CultMeshRudpPutReply`: `accept`, `refuse`, + and Drop refuses). + - The alternatives were a token with `server.answer_put(token, result)`, and a reply method keyed by session and + message id. + - Keying by session and message id is ambiguous: message ids are publisher-chosen, and a reconnect reuses the + session key. + - A token cannot be answered from another thread without routing it back to the server, and a forgotten token + leaves a session withheld until it ends. + - The handle is the Rust expression of TS's Promise-returning `onDocumentPutRaw`, and it keeps closure sinks + source-compatible. ## 8. Related findings @@ -381,3 +986,23 @@ records; update the check's type list in the same pass. - **F7** `accept_raw_document` (`main.rs:269-280`) checks `activated()` but not the *current* lease before mutating. Cut 2 moves the lease check to the only write, which closes this for the file. Soul should confirm that nothing else acts on a lost lease. + +## C5 status (2026-10-01) + +**Hands done, Soul not yet run.** Drain mode: Soul waits for the operator's go. + +- **Branch:** CultLib `hands/cultmesh-deferred-put-reply`, commits `08e0c229..d708247e`, based on `9dcb3aa1`. +- **Mechanism:** the sink answers through `CultMeshRudpPutReply` with `accept()` or `refuse(reason)`. Dropping the reply unanswered refuses with `UNANSWERED_PUT_REASON`. `acknowledgement_withheld` is the single owner of the hold: while a put is unanswered, the session gets no end-of-poll ack, Pong, snapshot response or reply to a repeated Connect. +- **Verification (Hands):** + - cultmesh-rs: 21 unit and 32 integration tests pass. A byte-exact golden test also passes before the cut. + - cultnet-rs: 486 pass. + - Interop: 27 pass, 0 fail. + - cargo-mutants: 57 mutants; 0 missed, 5 timeouts, all in the `take_answers` loop. +- **Spec deviations Soul should attack:** + - A repeated Connect's ack is now withheld. + - Puts queued behind a waiting snapshot count as withheld. Snapshot responses are held in a new `responses_owed` queue: P1 pending, then S, then P2 must not acknowledge P2 early. + - Answers are checked after each message. + - A release that arrives during an unreliable packet sends its own ack. +- **Structural delta:** +406/−96 in `rudp_document_server.rs`, against a +170/−15 estimate. +- **Consumers:** Odin `SinkHandle` and Ratatoskr `tests/catalog.rs:29` break until the W0 pin bump. +- **Still open:** S-5. Its fix belongs in `acknowledgement_withheld`.