From 409d92f279e8b208be9f67c9ca5aa006dce9abcc Mon Sep 17 00:00:00 2001 From: soul-probe Date: Wed, 30 Sep 2026 12:51:36 +0200 Subject: [PATCH 01/43] Odin Cut 1: the store file stops being working memory Odin read and decoded its whole 3.24 MB store file about 30 times a second: every refresh pass, every admission, every catalog query and every CAS retry read it again. The working set now lives in memory under the write lease. Deleted: CultCacheOdinTopologyStore (every file read and CAS against the store, and its four CAS_ATTEMPTS retry loops, including the one in reconcile), persist_generic_document's file CAS loop, the store half of incarnation_keys, stored_snapshot's file read, and prior_self_publisher_sequence's file read in RuntimeState::open. With one writer under the lease an in-memory exchange cannot lose, so nothing retries. Added: MemoryOdinTopologyStore, keyed (type, key), implementing the OdinTopologyStore trait (still the mock point). It is loaded once, with the locked pull_all, in try_activate after the lease is verified; legacy retirement and the self publisher sequence are operations on the loaded set. The trait's compare_exchange collapses into commit. Refresh reads Idunn's projection once per pass (IdunnProjectionSnapshot) instead of once per incarnation; admission reads it once per admission. Not in this cut: the write policy. Each mutating operation still writes the whole file immediately, now from memory (flush after an admission, a put or a refresh pass that changed something). Tests: the two tests that corrupted the store while Odin served now put the bad record in the file at activation. New: rewriting the file behind Odin changes nothing it serves; refresh passes and catalog reads succeed with the file replaced by bytes that do not decode; a loaded store never reads its file again. Map: docs/write-pattern-cut.md, Cut 1. Co-Authored-By: Claude Opus 5.5 --- crates/odin-daemon/src/lib.rs | 996 ++++++++++++++++++--------------- crates/odin-daemon/src/main.rs | 450 +++++++++------ 2 files changed, 804 insertions(+), 642 deletions(-) diff --git a/crates/odin-daemon/src/lib.rs b/crates/odin-daemon/src/lib.rs index 7c2f115..40b44c1 100644 --- a/crates/odin-daemon/src/lib.rs +++ b/crates/odin-daemon/src/lib.rs @@ -4,14 +4,15 @@ //! read-only projection, a trusted receive time with any service-owned //! presence, and narrow clock/store/signer implementations. -use std::collections::BTreeMap; +use std::cell::{Cell, Ref, RefCell}; +use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; use anyhow::{Context, Result, bail, ensure}; use chrono::{DateTime, SecondsFormat, Utc}; use cultcache_rs::{ - CultCacheEnvelope, CultCacheExpectedEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, + CacheBackingStore, CultCacheEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, }; use cultnet_rs::{ GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, GAMECULT_RUNTIME_PRESENCE_HEALTH_SIGNING_PURPOSE, @@ -29,8 +30,6 @@ use cultnet_rs::{ verify_service_identity_signature_with_public_key, }; -const CAS_ATTEMPTS: usize = 8; - /// Odin's durable high-water mark for one target's correlation sequence. /// /// The correlation's `publisher_sequence` must strictly increase for as long as @@ -195,16 +194,70 @@ impl CultCacheIdunnProjectionSource { impl IdunnProjectionSource for CultCacheIdunnProjectionSource { fn projections(&self, target: &str) -> Result> { - let Some(entries) = self.entries()? else { - return Ok(Vec::new()); - }; - let anchor = provider_anchor(&entries, target)?; + self.snapshot()?.projections(target) + } + + fn projection(&self, incarnation: &IncarnationRef) -> Result> { + self.snapshot()?.projection(incarnation) + } +} + +impl CultCacheIdunnProjectionSource { + /// Read the projection file once. An absent file projects nothing. + pub fn snapshot(&self) -> Result { + if !self.path.is_file() { + return Ok(IdunnProjectionSnapshot { + entries: Vec::new(), + }); + } + SingleFileMessagePackBackingStore::new(&self.path) + .pull_all_read_only_snapshot() + .map(|entries| IdunnProjectionSnapshot { entries }) + .map_err(|error| { + // The store's decode step reports its failures as text, so an + // I/O error in the chain here is from opening or reading the + // file and from nothing the file said. + if error.chain().any(|cause| cause.is::()) { + error.context(ProjectionUnreadable) + } else { + error + } + }) + } +} + +/// One reading of Idunn's projection file. Every lookup made against it is +/// answered from the same bytes, so a pass that refreshes every incarnation +/// reads the file once. +pub struct IdunnProjectionSnapshot { + entries: Vec, +} + +impl IdunnProjectionSnapshot { + /// Every incarnation Idunn projects an Expected for, of every target. Only + /// the keys are read: decoding a record is the refresh of that one + /// incarnation, so a record that will not decode cannot make the list + /// unreadable. A record keyed by anything but an incarnation key is not + /// this contract's and is skipped. + pub fn incarnations(&self) -> BTreeSet { + self.entries + .iter() + .filter(|entry| entry.r#type == IdunnExpectedIncarnationRecord::TYPE) + .filter_map(|entry| IncarnationRef::parse_key(&entry.key)) + .collect() + } +} + +impl IdunnProjectionSource for IdunnProjectionSnapshot { + fn projections(&self, target: &str) -> Result> { + let entries = &self.entries; + let anchor = provider_anchor(entries, target)?; // A record keyed by anything but its own incarnation key is not this // contract's. That includes the target-keyed single slot the previous // projection used; an Idunn still writing that shape projects nothing // this reader will act on, rather than something it will misread. let mut projections = Vec::new(); - for envelope in &entries { + for envelope in entries { if envelope.r#type != IdunnExpectedIncarnationRecord::TYPE { continue; } @@ -215,7 +268,7 @@ impl IdunnProjectionSource for CultCacheIdunnProjectionSource { continue; } projections.push(assemble_projection( - &entries, + entries, anchor.clone(), envelope, &incarnation, @@ -229,40 +282,17 @@ impl IdunnProjectionSource for CultCacheIdunnProjectionSource { /// record under another incarnation's key is that incarnation's fault, and /// never makes this one unreadable. fn projection(&self, incarnation: &IncarnationRef) -> Result> { - let Some(entries) = self.entries()? else { - return Ok(None); - }; + let entries = &self.entries; let Some(envelope) = unique_envelope( - &entries, + entries, IdunnExpectedIncarnationRecord::TYPE, &incarnation.key(), )? else { return Ok(None); }; - let anchor = provider_anchor(&entries, &incarnation.target)?; - assemble_projection(&entries, anchor, envelope, incarnation).map(Some) - } -} - -impl CultCacheIdunnProjectionSource { - fn entries(&self) -> Result>> { - if !self.path.is_file() { - return Ok(None); - } - SingleFileMessagePackBackingStore::new(&self.path) - .pull_all_read_only_snapshot() - .map(Some) - .map_err(|error| { - // The store's decode step reports its failures as text, so an - // I/O error in the chain here is from opening or reading the - // file and from nothing the file said. - if error.chain().any(|cause| cause.is::()) { - error.context(ProjectionUnreadable) - } else { - error - } - }) + let anchor = provider_anchor(entries, &incarnation.target)?; + assemble_projection(entries, anchor, envelope, incarnation).map(Some) } } @@ -410,110 +440,233 @@ pub struct OdinStoreSnapshot { pub trait OdinTopologyStore { fn read(&self, query: OdinStoreQuery) -> Result; - fn compare_exchange( + /// Replace the queried incarnation's correlation, and its presence when + /// one is given. + fn commit( &self, - observed: &OdinStoreSnapshot, + query: &OdinStoreQuery, replacement_presence: Option<&AdmittedPresence>, replacement_correlation: &StoredCorrelation, - ) -> Result; + ) -> Result<()>; fn withdraw_correlation(&self, incarnation: &IncarnationRef) -> Result<()>; /// Claim the next correlation sequence for this target, never returning a - /// value already used. `prior` is the sequence of the currently stored + /// value already published. `prior` is the sequence of the currently stored /// correlation, or 0 when there is none. /// - /// The mark is advanced before the correlation that uses it is published, - /// so a crash in between leaves it ahead. That is the safe direction: - /// sequences may skip, they may never repeat. + /// A sequence is published when the store is flushed. The mark is written + /// in the same atomic write as every correlation that uses it, so the file + /// never holds a correlation ahead of its mark. A sequence reserved and + /// never flushed was never published, and may be claimed again after a + /// restart. fn reserve_publisher_sequence(&self, target: &str, prior: u64) -> Result; } -/// Odin's durable replay and correlation store. It persists existing CultNet -/// documents directly instead of inventing a parallel admission schema. -pub struct CultCacheOdinTopologyStore { - path: PathBuf, +impl OdinTopologyStore for &T { + fn read(&self, query: OdinStoreQuery) -> Result { + (*self).read(query) + } + + fn commit( + &self, + query: &OdinStoreQuery, + replacement_presence: Option<&AdmittedPresence>, + replacement_correlation: &StoredCorrelation, + ) -> Result<()> { + (*self).commit(query, replacement_presence, replacement_correlation) + } + + fn withdraw_correlation(&self, incarnation: &IncarnationRef) -> Result<()> { + (*self).withdraw_correlation(incarnation) + } + + fn reserve_publisher_sequence(&self, target: &str, prior: u64) -> Result { + (*self).reserve_publisher_sequence(target, prior) + } } -impl CultCacheOdinTopologyStore { - pub fn new(path: impl Into) -> Self { - Self { path: path.into() } +/// One record's identity in a CultCache store: its type, then its key. +pub type RecordId = (String, String); + +fn record_id(envelope: &CultCacheEnvelope) -> RecordId { + (envelope.r#type.clone(), envelope.key.clone()) +} + +/// Odin's store, held in memory for as long as Odin holds its write lease. It +/// persists existing CultNet documents directly instead of inventing a +/// parallel admission schema. +/// +/// The file is read once, by [`MemoryOdinTopologyStore::load`] when the lease +/// is granted, and never again while Odin serves: every lookup is answered +/// from memory. The file is Odin's durability checkpoint and Idunn's read-only +/// crossing. It is written by [`MemoryOdinTopologyStore::flush`] and nothing +/// else. +pub struct MemoryOdinTopologyStore { + path: PathBuf, + records: RefCell>, + /// Exactly what the file held when it was loaded or last written. + flushed: RefCell>, + /// The records differ from what was last loaded or written. + dirty: Cell, +} + +impl MemoryOdinTopologyStore { + /// Read the store file under its lock. An absent file is an empty store. + /// Records are loaded as they are stored: one that does not decode is that + /// record's failure, met where the record is used, and never stops the + /// load. + pub fn load(path: impl Into) -> Result { + let path = path.into(); + let flushed = SingleFileMessagePackBackingStore::new(&path).pull_all()?; + let mut records = BTreeMap::new(); + for envelope in &flushed { + if records.insert(record_id(envelope), envelope.clone()).is_some() { + eprintln!( + "Odin's store holds {} {} more than once; the last is kept", + envelope.r#type, envelope.key + ); + } + } + Ok(Self { + path, + records: RefCell::new(records), + flushed: RefCell::new(flushed), + dirty: Cell::new(false), + }) } pub fn path(&self) -> &Path { &self.path } + /// Every record, in the order the file stores them. + pub fn records(&self) -> Ref<'_, BTreeMap> { + self.records.borrow() + } + + /// Store one record in place of any with the same type and key. Storing + /// the record already held changes nothing. + pub fn put(&self, envelope: CultCacheEnvelope) { + let id = record_id(&envelope); + let mut records = self.records.borrow_mut(); + if records.get(&id) != Some(&envelope) { + records.insert(id, envelope); + self.dirty.set(true); + } + } + + fn remove(&self, record_type: &str, key: &str) { + if self + .records + .borrow_mut() + .remove(&(record_type.to_owned(), key.to_owned())) + .is_some() + { + self.dirty.set(true); + } + } + + /// Every incarnation this store holds a correlation for. + pub fn correlated_incarnations(&self) -> BTreeSet { + self.records + .borrow() + .keys() + .filter(|(record_type, _)| record_type == OdinRuntimeTopologyCorrelationRecord::TYPE) + .filter_map(|(_, key)| IncarnationRef::parse_key(key)) + .collect() + } + /// Remove every correlation not keyed by an incarnation. Those were written /// by the previous, target-keyed Odin; nothing reads them under this - /// contract, and left in place `stored_snapshot` would serve them to the - /// Verse as current. Presence records are kept whatever their key: the - /// self publisher sequence is continued from them. - pub fn retire_legacy_correlations(&self) -> Result { - for _ in 0..CAS_ATTEMPTS { - if !self.path.is_file() { - return Ok(0); - } - let entries = - SingleFileMessagePackBackingStore::new(&self.path).pull_all_read_only_snapshot()?; - let retained = entries - .iter() - .filter(|entry| { - !(entry.r#type == OdinRuntimeTopologyCorrelationRecord::TYPE - && IncarnationRef::parse_key(&entry.key).is_none()) - }) - .cloned() - .collect::>(); - let removed = entries.len() - retained.len(); - if removed == 0 { - return Ok(0); - } - if SingleFileMessagePackBackingStore::new(&self.path) - .compare_exchange_snapshot(&entries, &retained)? - { - return Ok(removed); - } + /// contract, and left in place the catalog would serve them to the Verse + /// as current. Presence records are kept whatever their key: the self + /// publisher sequence is continued from them. + pub fn retire_legacy_correlations(&self) -> usize { + let legacy: Vec = self + .records + .borrow() + .keys() + .filter(|(record_type, key)| { + record_type == OdinRuntimeTopologyCorrelationRecord::TYPE + && IncarnationRef::parse_key(key).is_none() + }) + .map(|(_, key)| key.clone()) + .collect(); + for key in &legacy { + self.remove(OdinRuntimeTopologyCorrelationRecord::TYPE, key); } - bail!("Odin topology store changed repeatedly while retiring legacy correlations") + legacy.len() + } + + /// Whether the records differ from what was last loaded or written. + pub fn is_dirty(&self) -> bool { + self.dirty.get() + } + + /// Write every record to the file as one atomic snapshot, if anything + /// changed since the last write; returns whether it wrote. + /// + /// The file must still hold exactly what Odin last loaded or wrote. Anything + /// else was written by another process, and the write is refused with + /// [`ForeignStoreWrite`] rather than made over it. + pub fn flush(&self) -> Result { + if !self.dirty.get() { + return Ok(false); + } + let current: Vec = self.records.borrow().values().cloned().collect(); + if !SingleFileMessagePackBackingStore::new(&self.path) + .compare_exchange_snapshot(&self.flushed.borrow(), ¤t)? + { + return Err(ForeignStoreWrite.into()); + } + *self.flushed.borrow_mut() = current; + self.dirty.set(false); + Ok(true) } } -impl OdinTopologyStore for CultCacheOdinTopologyStore { +/// Odin's store file no longer holds what Odin last loaded or wrote, so another +/// process wrote it. While Odin holds its write lease it is the file's only +/// writer: this is a second writer, and Odin must not write over it. +#[derive(Debug)] +pub struct ForeignStoreWrite; + +impl std::fmt::Display for ForeignStoreWrite { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("Odin's store file was written by another process") + } +} + +impl std::error::Error for ForeignStoreWrite {} + +impl OdinTopologyStore for MemoryOdinTopologyStore { fn read(&self, mut query: OdinStoreQuery) -> Result { query.dependencies.sort(); query.dependencies.dedup(); - let entries = if self.path.is_file() { - SingleFileMessagePackBackingStore::new(&self.path).pull_all_read_only_snapshot()? - } else { - Vec::new() - }; - let presence_key = - presence_store_key(&query.incarnation, &query.provider_signer_identity_id); - let presence = unique_envelope( - &entries, + let records = self.records.borrow(); + let find = + |record_type: &str, key: &str| records.get(&(record_type.to_owned(), key.to_owned())); + let presence = find( GameCultRuntimePresenceHealthRecord::TYPE, - &presence_key, - )? + &presence_store_key(&query.incarnation, &query.provider_signer_identity_id), + ) .map(decode_presence_envelope) .transpose()?; - let correlation = unique_envelope( - &entries, + let correlation = find( OdinRuntimeTopologyCorrelationRecord::TYPE, &query.incarnation.key(), - )? + ) .map(decode_correlation_envelope) .transpose()?; let mut dependency_correlations = BTreeMap::new(); for dependency in &query.dependencies { - let current = unique_envelope( - &entries, - OdinRuntimeTopologyCorrelationRecord::TYPE, - &dependency.key(), - )? - .map(decode_correlation_envelope) - .transpose()?; + let current = find(OdinRuntimeTopologyCorrelationRecord::TYPE, &dependency.key()) + .map(decode_correlation_envelope) + .transpose()?; dependency_correlations.insert(dependency.clone(), current); } + drop(records); Ok(OdinStoreSnapshot { query, presence, @@ -522,128 +675,50 @@ impl OdinTopologyStore for CultCacheOdinTopologyStore { }) } - fn compare_exchange( + fn commit( &self, - observed: &OdinStoreSnapshot, + query: &OdinStoreQuery, replacement_presence: Option<&AdmittedPresence>, replacement_correlation: &StoredCorrelation, - ) -> Result { - let presence_key = presence_store_key( - &observed.query.incarnation, - &observed.query.provider_signer_identity_id, - ); - let correlation_key = observed.query.incarnation.key(); - let mut conditions = BTreeMap::new(); - insert_condition( - &mut conditions, - CultCacheExpectedEnvelope { - key: presence_key.clone(), - r#type: GameCultRuntimePresenceHealthRecord::TYPE.into(), - current: observed - .presence - .as_ref() - .map(|presence| presence_envelope(&presence_key, presence)), - }, - )?; - insert_condition( - &mut conditions, - CultCacheExpectedEnvelope { - key: correlation_key.clone(), - r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), - current: observed - .correlation - .as_ref() - .map(|correlation| correlation_envelope(&correlation_key, correlation)), - }, - )?; - for (dependency, correlation) in &observed.dependency_correlations { - let key = dependency.key(); - insert_condition( - &mut conditions, - CultCacheExpectedEnvelope { - key: key.clone(), - r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), - current: correlation - .as_ref() - .map(|correlation| correlation_envelope(&key, correlation)), - }, - )?; - } - let mut replacements = vec![correlation_envelope( - &correlation_key, + ) -> Result<()> { + self.put(correlation_envelope( + &query.incarnation.key(), replacement_correlation, - )]; + )); if let Some(presence) = replacement_presence { - replacements.push(presence_envelope(&presence_key, presence)); + self.put(presence_envelope( + &presence_store_key(&query.incarnation, &query.provider_signer_identity_id), + presence, + )); } - if let Some(parent) = self.path.parent() { - std::fs::create_dir_all(parent)?; - } - SingleFileMessagePackBackingStore::new(&self.path) - .compare_exchange(&conditions.into_values().collect::>(), &replacements) + Ok(()) } fn withdraw_correlation(&self, incarnation: &IncarnationRef) -> Result<()> { - let key = incarnation.key(); - for _ in 0..CAS_ATTEMPTS { - if !self.path.is_file() { - return Ok(()); - } - let entries = - SingleFileMessagePackBackingStore::new(&self.path).pull_all_read_only_snapshot()?; - let current = - unique_envelope(&entries, OdinRuntimeTopologyCorrelationRecord::TYPE, &key)?; - let Some(current) = current else { - return Ok(()); - }; - let replacement = entries - .iter() - .filter(|entry| *entry != current) - .cloned() - .collect::>(); - if SingleFileMessagePackBackingStore::new(&self.path) - .compare_exchange_snapshot(&entries, &replacement)? - { - return Ok(()); - } - } - bail!("Odin topology store changed repeatedly while withdrawing correlation") + self.remove(OdinRuntimeTopologyCorrelationRecord::TYPE, &incarnation.key()); + Ok(()) } fn reserve_publisher_sequence(&self, target: &str, prior: u64) -> Result { - for _ in 0..CAS_ATTEMPTS { - let entries = if self.path.is_file() { - SingleFileMessagePackBackingStore::new(&self.path).pull_all_read_only_snapshot()? - } else { - Vec::new() - }; - let current = unique_envelope(&entries, TOPOLOGY_PUBLISHER_WATERMARK_TYPE, target)?; - let stored = current - .map(|envelope| decode_publisher_watermark(target, &envelope.payload)) - .transpose()? - .unwrap_or(0); - // The stored correlation can be ahead of the mark on the first pass - // after this record was introduced, so take whichever is higher. - let next = stored - .max(prior) - .checked_add(1) - .context("Odin topology publisher sequence exhausted")?; - let replacement = publisher_watermark_envelope(target, next, &Utc::now().to_rfc3339())?; - if let Some(parent) = self.path.parent() { - std::fs::create_dir_all(parent)?; - } - if SingleFileMessagePackBackingStore::new(&self.path).compare_exchange( - &[CultCacheExpectedEnvelope { - key: target.into(), - r#type: TOPOLOGY_PUBLISHER_WATERMARK_TYPE.into(), - current: current.cloned(), - }], - &[replacement], - )? { - return Ok(next); - } - } - bail!("Odin topology store changed repeatedly while reserving a publisher sequence") + let stored = self + .records + .borrow() + .get(&(TOPOLOGY_PUBLISHER_WATERMARK_TYPE.to_owned(), target.to_owned())) + .map(|envelope| decode_publisher_watermark(target, &envelope.payload)) + .transpose()? + .unwrap_or(0); + // The stored correlation can be ahead of the mark on the first pass + // after this record was introduced, so take whichever is higher. + let next = stored + .max(prior) + .checked_add(1) + .context("Odin topology publisher sequence exhausted")?; + self.put(publisher_watermark_envelope( + target, + next, + &Utc::now().to_rfc3339(), + )?); + Ok(next) } } @@ -796,184 +871,177 @@ where dependencies: managed_dependency_incarnations(&projection.expected.dependencies), }; - for _ in 0..CAS_ATTEMPTS { - let observed = self.store.read(query.clone())?; - let admitted = self.select_presence(&observed, incoming, authority.as_ref())?; - let dependencies = - self.dependency_evidence(&projection.expected.dependencies, &observed, now)?; - let (presence, mut disagreements, replacement_presence) = match admitted { - SelectedPresence::None => (None, projection_disagreements.clone(), None), - SelectedPresence::Stale(stored) => { - let record = decode_presence(&stored.canonical_bytes)?; - let mut disagreements = projection_disagreements.clone(); - disagreements.push(OdinTopologyDisagreement { - code: "stored-presence-not-current".into(), - expected: projection - .activation - .as_ref() - .map(IdunnRuntimeActivationRecord::canonical_sha256) - .transpose()?, - observed: Some(record.activation_witness_sha256), - }); - (None, disagreements, None) - } - SelectedPresence::Current { - correlation, - replacement, - } => { - let mut disagreements = projection_disagreements.clone(); - disagreements.extend_from_slice(correlation.disagreements()); - (Some(correlation), disagreements, replacement) - } - }; - if let Some(disagreement) = &lease_disagreement { - disagreements.push(disagreement.clone()); - } - if presence.is_none() { + let observed = self.store.read(query)?; + let admitted = self.select_presence(&observed, incoming, authority.as_ref())?; + let dependencies = + self.dependency_evidence(&projection.expected.dependencies, &observed, now)?; + let (presence, mut disagreements, replacement_presence) = match admitted { + SelectedPresence::None => (None, projection_disagreements.clone(), None), + SelectedPresence::Stale(stored) => { + let record = decode_presence(&stored.canonical_bytes)?; + let mut disagreements = projection_disagreements.clone(); disagreements.push(OdinTopologyDisagreement { - code: "signed-presence-missing".into(), - expected: Some("dual-authenticated-runtime-presence".into()), - observed: None, + code: "stored-presence-not-current".into(), + expected: projection + .activation + .as_ref() + .map(IdunnRuntimeActivationRecord::canonical_sha256) + .transpose()?, + observed: Some(record.activation_witness_sha256), }); - disagreements.extend(projection.expected.capabilities.iter().enumerate().map( - |(index, capability)| OdinTopologyDisagreement { - code: format!("expected-capability-{index:03}-missing"), - expected: Some(format!( - "{}/{}/{} capacity>={}", - capability.capability, - capability.schema, - capability.compatibility, - capability.minimum_capacity - )), - observed: None, - }, - )); + (None, disagreements, None) } + SelectedPresence::Current { + correlation, + replacement, + } => { + let mut disagreements = projection_disagreements.clone(); + disagreements.extend_from_slice(correlation.disagreements()); + (Some(correlation), disagreements, replacement) + } + }; + if let Some(disagreement) = &lease_disagreement { + disagreements.push(disagreement.clone()); + } + if presence.is_none() { + disagreements.push(OdinTopologyDisagreement { + code: "signed-presence-missing".into(), + expected: Some("dual-authenticated-runtime-presence".into()), + observed: None, + }); + disagreements.extend(projection.expected.capabilities.iter().enumerate().map( + |(index, capability)| OdinTopologyDisagreement { + code: format!("expected-capability-{index:03}-missing"), + expected: Some(format!( + "{}/{}/{} capacity>={}", + capability.capability, + capability.schema, + capability.compatibility, + capability.minimum_capacity + )), + observed: None, + }, + )); + } - let observed_write_lease = if let Some(presence) = &presence { - correlate_write_lease( - &projection.expected, - lease_sha256.as_deref(), - presence.claim().record().write_lease_sha256.as_deref(), - &presence.claim().record().state, - &mut disagreements, - ) - } else { - None - }; - disagreements.sort_by(|left, right| left.code.cmp(&right.code)); + let observed_write_lease = if let Some(presence) = &presence { + correlate_write_lease( + &projection.expected, + lease_sha256.as_deref(), + presence.claim().record().write_lease_sha256.as_deref(), + &presence.claim().record().state, + &mut disagreements, + ) + } else { + None + }; + disagreements.sort_by(|left, right| left.code.cmp(&right.code)); - let prior = observed - .correlation + let prior = observed + .correlation + .as_ref() + .map(|stored| decode_correlation(&stored.canonical_bytes)) + .transpose()?; + let presence_record = presence.as_ref().map(|value| value.claim().record()); + let ready = presence_record.is_some_and(|record| record.state == "active") + && disagreements.is_empty() + && dependencies + .iter() + .all(|dependency| dependency_permits_ready(&dependency.kind, dependency.ready)); + let mut correlation = OdinRuntimeTopologyCorrelationRecord { + schema_version: ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into(), + target: projection.expected.target.clone(), + expected_projection_sha256: projection.expected.canonical_sha256()?, + expected: true, + current_activation_sha256: projection + .activation .as_ref() - .map(|stored| decode_correlation(&stored.canonical_bytes)) - .transpose()?; - let presence_record = presence.as_ref().map(|value| value.claim().record()); - let ready = presence_record.is_some_and(|record| record.state == "active") - && disagreements.is_empty() - && dependencies - .iter() - .all(|dependency| dependency_permits_ready(&dependency.kind, dependency.ready)); - let mut correlation = OdinRuntimeTopologyCorrelationRecord { - schema_version: ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into(), - target: projection.expected.target.clone(), - expected_projection_sha256: projection.expected.canonical_sha256()?, - expected: true, - current_activation_sha256: projection - .activation - .as_ref() - .map(IdunnRuntimeActivationRecord::canonical_sha256) - .transpose()?, - signed_presence_sha256: presence - .as_ref() - .map(|value| value.claim().signed_presence_sha256().into()), - observed_presence_state: presence_record.map(|record| record.state.clone()), - observed_presence_publisher_sequence: presence_record - .map(|record| record.publisher_sequence), - observed_write_lease_sha256: observed_write_lease, - observed_capabilities: presence_record - .map_or_else(Vec::new, |record| record.capabilities.clone()), - runtime_id: projection.expected.runtime_id.clone(), - runtime_instance_id: projection - .activation - .as_ref() - .map(|activation| activation.runtime_instance_id.clone()), - present: presence.is_some(), - ready, - dependencies, - disagreements, - signer_identity_id: self.signer.identity_id().into(), - // Placeholder. A sequence is only reserved once the facts are - // known to have changed, below; same_correlation_facts compares - // against the prior's sequence regardless of what sits here. - publisher_sequence: prior.as_ref().map_or(0, |record| record.publisher_sequence), - observed_at_unix_millis: now, - signature_algorithm: "ed25519".into(), - signature: Vec::new(), - }; - if let Some(prior) = &prior - && same_correlation_facts(prior, &correlation) - { - return Ok(observed - .correlation - .context("decoded prior correlation disappeared")? - .canonical_bytes); - } - // Reserved only after the dedupe: a refresh that observes nothing - // new must not burn a sequence or fsync the store. Reserved from a - // mark that survives withdrawal, not derived from the stored - // correlation, so a withdrawn correlation does not take the count - // with it. See TOPOLOGY_PUBLISHER_WATERMARK_SCHEMA. - correlation.publisher_sequence = self.store.reserve_publisher_sequence( - &observed.query.incarnation.target, - prior.as_ref().map_or(0, |record| record.publisher_sequence), - )?; - correlation.signature = self - .signer - .sign_correlation(&correlation.unsigned_signature_payload()?)?; - let canonical = correlation.canonical_bytes()?; - correlation.validate_against_expected(&projection.expected, lease_sha256.as_deref())?; - let (signed, unsigned) = - OdinRuntimeTopologyCorrelationRecord::decode_canonical_signed_payload(&canonical)?; - verify_service_identity_signature_with_public_key::< - OdinTopologyIdentity, - OdinRuntimeTopologyCorrelationPurpose, - >( + .map(IdunnRuntimeActivationRecord::canonical_sha256) + .transpose()?, + signed_presence_sha256: presence + .as_ref() + .map(|value| value.claim().signed_presence_sha256().into()), + observed_presence_state: presence_record.map(|record| record.state.clone()), + observed_presence_publisher_sequence: presence_record + .map(|record| record.publisher_sequence), + observed_write_lease_sha256: observed_write_lease, + observed_capabilities: presence_record + .map_or_else(Vec::new, |record| record.capabilities.clone()), + runtime_id: projection.expected.runtime_id.clone(), + runtime_instance_id: projection + .activation + .as_ref() + .map(|activation| activation.runtime_instance_id.clone()), + present: presence.is_some(), + ready, + dependencies, + disagreements, + signer_identity_id: self.signer.identity_id().into(), + // Placeholder. A sequence is only reserved once the facts are + // known to have changed, below; same_correlation_facts compares + // against the prior's sequence regardless of what sits here. + publisher_sequence: prior.as_ref().map_or(0, |record| record.publisher_sequence), + observed_at_unix_millis: now, + signature_algorithm: "ed25519".into(), + signature: Vec::new(), + }; + if let Some(prior) = &prior + && same_correlation_facts(prior, &correlation) + { + return Ok(observed + .correlation + .context("decoded prior correlation disappeared")? + .canonical_bytes); + } + // Reserved only after the dedupe: a refresh that observes nothing + // new must not burn a sequence or change the store. Reserved from a + // mark that survives withdrawal, not derived from the stored + // correlation, so a withdrawn correlation does not take the count + // with it. See TOPOLOGY_PUBLISHER_WATERMARK_SCHEMA. + correlation.publisher_sequence = self.store.reserve_publisher_sequence( + &observed.query.incarnation.target, + prior.as_ref().map_or(0, |record| record.publisher_sequence), + )?; + correlation.signature = self + .signer + .sign_correlation(&correlation.unsigned_signature_payload()?)?; + let canonical = correlation.canonical_bytes()?; + correlation.validate_against_expected(&projection.expected, lease_sha256.as_deref())?; + let (signed, unsigned) = + OdinRuntimeTopologyCorrelationRecord::decode_canonical_signed_payload(&canonical)?; + verify_service_identity_signature_with_public_key::< + OdinTopologyIdentity, + OdinRuntimeTopologyCorrelationPurpose, + >( + self.signer.public_key(), + &unsigned, + &ServiceIdentitySignature { + identity_id: signed.signer_identity_id, + signature: signed.signature, + }, + )?; + if let Some(authority) = &authority { + authenticate_odin_runtime_topology_correlation( + &canonical, + authority, + lease_sha256.as_deref(), self.signer.public_key(), - &unsigned, - &ServiceIdentitySignature { - identity_id: signed.signer_identity_id, - signature: signed.signature, + OdinTopologyAuthenticationContext { + trusted_received_at_unix_millis: now, + maximum_age_millis: self.policy.correlation_maximum_age_millis, + maximum_future_skew_millis: self + .policy + .correlation_maximum_future_skew_millis, }, )?; - if let Some(authority) = &authority { - authenticate_odin_runtime_topology_correlation( - &canonical, - authority, - lease_sha256.as_deref(), - self.signer.public_key(), - OdinTopologyAuthenticationContext { - trusted_received_at_unix_millis: now, - maximum_age_millis: self.policy.correlation_maximum_age_millis, - maximum_future_skew_millis: self - .policy - .correlation_maximum_future_skew_millis, - }, - )?; - } - let stored_correlation = StoredCorrelation { - canonical_bytes: canonical.clone(), - stored_at: rfc3339_millis(now)?, - }; - if self.store.compare_exchange( - &observed, - replacement_presence.as_ref(), - &stored_correlation, - )? { - return Ok(canonical); - } } - bail!("Odin topology store changed repeatedly during correlation") + let stored_correlation = StoredCorrelation { + canonical_bytes: canonical.clone(), + stored_at: rfc3339_millis(now)?, + }; + self.store + .commit(&observed.query, replacement_presence.as_ref(), &stored_correlation)?; + Ok(canonical) } fn select_presence( @@ -1619,22 +1687,6 @@ fn correlation_envelope(target: &str, correlation: &StoredCorrelation) -> CultCa } } -fn insert_condition( - conditions: &mut BTreeMap<(String, String), CultCacheExpectedEnvelope>, - condition: CultCacheExpectedEnvelope, -) -> Result<()> { - let identity = (condition.r#type.clone(), condition.key.clone()); - if let Some(current) = conditions.get(&identity) { - ensure!( - current.current == condition.current, - "store read set is incoherent" - ); - } else { - conditions.insert(identity, condition); - } - Ok(()) -} - fn rfc3339_millis(value: u64) -> Result { let value: i64 = value .try_into() @@ -1891,16 +1943,19 @@ mod tests { } /// A store the target-keyed Odin wrote holds correlations under bare - /// target keys. They are retired; presence history under any key stays. + /// target keys. They are retired from the loaded set; presence history + /// under any key stays. #[test] fn legacy_target_keyed_correlations_are_retired_and_presence_is_kept() -> Result<()> { let world = TestWorld::new()?; let service = world.service("ghostlight", Vec::new(), false)?; let presence = service.signed_presence(1, |_| {})?; - let engine = world.engine(NOW); - let correlation = engine.admit_presence("ghostlight", &presence, NOW)?; - let store = SingleFileMessagePackBackingStore::new(&world.topology_path); - let current = store.pull_all_read_only_snapshot()?; + let correlation = world + .engine(NOW) + .admit_presence("ghostlight", &presence, NOW)?; + assert!(world.store.flush()?); + let file = SingleFileMessagePackBackingStore::new(&world.topology_path); + let current = file.pull_all_read_only_snapshot()?; let mut with_legacy = current.clone(); with_legacy.push(CultCacheEnvelope { key: "ghostlight".into(), @@ -1916,23 +1971,21 @@ mod tests { stored_at: rfc3339_millis(NOW - 1_000)?, schema_id: Some(GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()), }); - ensure!(store.compare_exchange_snapshot(¤t, &with_legacy)?); + ensure!(file.compare_exchange_snapshot(¤t, &with_legacy)?); - let odin_store = CultCacheOdinTopologyStore::new(&world.topology_path); - assert_eq!(odin_store.retire_legacy_correlations()?, 1); - assert_eq!(odin_store.retire_legacy_correlations()?, 0); - let after = store.pull_all_read_only_snapshot()?; - assert!(after.iter().all(|entry| { - entry.r#type != OdinRuntimeTopologyCorrelationRecord::TYPE - || IncarnationRef::parse_key(&entry.key).is_some() + let loaded = MemoryOdinTopologyStore::load(&world.topology_path)?; + assert_eq!(loaded.retire_legacy_correlations(), 1); + assert_eq!(loaded.retire_legacy_correlations(), 0); + let after = loaded.records(); + assert!(after.keys().all(|(record_type, key)| { + record_type != OdinRuntimeTopologyCorrelationRecord::TYPE + || IncarnationRef::parse_key(key).is_some() })); + assert!(after.keys().any(|(_, key)| key == "ghostlight/legacy-signer")); + drop(after); assert!( - after - .iter() - .any(|entry| entry.key == "ghostlight/legacy-signer") - ); - assert!( - engine + world + .engine_on(&loaded, NOW) .current_signed_correlation(&service.incarnation()?)? .is_some() ); @@ -1942,7 +1995,7 @@ mod tests { #[test] fn withdrawing_a_correlation_does_not_reset_its_publisher_sequence() -> Result<()> { let temp = TempDir::new()?; - let store = CultCacheOdinTopologyStore::new(temp.path().join("topology.cc")); + let store = MemoryOdinTopologyStore::load(temp.path().join("topology.cc"))?; assert_eq!(store.reserve_publisher_sequence("heimdall", 0)?, 1); assert_eq!(store.reserve_publisher_sequence("heimdall", 1)?, 2); @@ -1965,6 +2018,36 @@ mod tests { Ok(()) } + /// The store file is read once, when the store is loaded. Replaced behind + /// the loaded store with bytes that do not decode, it changes nothing the + /// store answers: admission, refresh and withdrawal all run from memory. + #[test] + fn a_loaded_store_never_reads_its_file_again() -> Result<()> { + let world = TestWorld::new()?; + let service = world.service("ghostlight", Vec::new(), false)?; + let engine = world.engine(NOW); + let first = engine.admit_presence("ghostlight", &service.signed_presence(1, |_| {})?, NOW)?; + assert!(world.store.flush()?); + + std::fs::write(&world.topology_path, b"not a cultcache store")?; + assert_eq!( + engine.current_signed_correlation(&service.incarnation()?)?, + Some(first) + ); + let second = decode_signed(&engine.admit_presence( + "ghostlight", + &service.signed_presence(2, |_| {})?, + NOW + 1, + )?)?; + assert_eq!(second.publisher_sequence, 2); + assert_eq!( + std::fs::read(&world.topology_path)?, + b"not a cultcache store", + "nothing was written without a flush" + ); + Ok(()) + } + #[derive(Clone, Copy)] struct FixedClock(u64); @@ -2050,6 +2133,7 @@ mod tests { _temp: TempDir, projection_path: PathBuf, topology_path: PathBuf, + store: MemoryOdinTopologyStore, identity_root: PathBuf, idunn_signer: ServiceIdentitySigner, idunn_anchor: ServiceIdentityTrustAnchor, @@ -2066,9 +2150,11 @@ mod tests { let idunn_anchor = idunn_signer.trust_anchor()?; let odin_signer = enroll_service_identity_at::(&identity_root.join("odin.cc"))?; + let topology_path = temp.path().join("odin-topology.cc"); Ok(Self { projection_path: temp.path().join("idunn-projection.cc"), - topology_path: temp.path().join("odin-topology.cc"), + store: MemoryOdinTopologyStore::load(&topology_path)?, + topology_path, identity_root, idunn_signer, idunn_anchor, @@ -2259,18 +2345,15 @@ mod tests { Ok(()) } - fn engine( - &self, - now: u64, - ) -> OdinTopologyAuthority< - CultCacheIdunnProjectionSource, - CultCacheOdinTopologyStore, - &ServiceIdentitySigner, - FixedClock, - > { + fn engine(&self, now: u64) -> TestEngine<'_> { + self.engine_on(&self.store, now) + } + + /// An engine over another store, as a restarted Odin loads one. + fn engine_on<'a>(&'a self, store: &'a MemoryOdinTopologyStore, now: u64) -> TestEngine<'a> { OdinTopologyAuthority::new( CultCacheIdunnProjectionSource::new(&self.projection_path), - CultCacheOdinTopologyStore::new(&self.topology_path), + store, &self.odin_signer, FixedClock(now), self.idunn_anchor.clone(), @@ -2314,6 +2397,13 @@ mod tests { } } + type TestEngine<'a> = OdinTopologyAuthority< + CultCacheIdunnProjectionSource, + &'a MemoryOdinTopologyStore, + &'a ServiceIdentitySigner, + FixedClock, + >; + fn digest(byte: char) -> String { format!("sha256-{}", byte.to_string().repeat(64)) } @@ -2455,19 +2545,9 @@ mod tests { let reused = service.signed_presence(2, |record| record.detail = "different".into())?; assert!(engine.admit_presence("ghostlight", &reused, NOW).is_err()); - let entries = SingleFileMessagePackBackingStore::new(&world.topology_path) - .pull_all_read_only_snapshot()?; - let stored = unique_envelope( - &entries, - GameCultRuntimePresenceHealthRecord::TYPE, - &presence_store_key( - &service.incarnation()?, - &service.projection.expected.expected_signer_identity_id, - ), - )? - .unwrap(); - assert_eq!(stored.payload, sequence_two); - assert_eq!(parse_rfc3339_millis(&stored.stored_at)?, NOW); + let (payload, stored_at) = stored_presence(&world, &service)?; + assert_eq!(payload, sequence_two); + assert_eq!(parse_rfc3339_millis(&stored_at)?, NOW); Ok(()) } @@ -2475,20 +2555,21 @@ mod tests { /// received at `NOW`, beyond the skew window, as on yggdrasil 2026-09-29. const RESTART: u64 = NOW + 10_000; + /// The service's presence as the store holds it: payload and receipt time. + fn stored_presence(world: &TestWorld, service: &TestService) -> Result<(Vec, String)> { + let key = presence_store_key( + &service.incarnation()?, + &service.projection.expected.expected_signer_identity_id, + ); + let records = world.store.records(); + let stored = records + .get(&(GameCultRuntimePresenceHealthRecord::TYPE.to_owned(), key)) + .context("no stored presence")?; + Ok((stored.payload.clone(), stored.stored_at.clone())) + } + fn stored_presence_payload(world: &TestWorld, service: &TestService) -> Result> { - let entries = SingleFileMessagePackBackingStore::new(&world.topology_path) - .pull_all_read_only_snapshot()?; - Ok(unique_envelope( - &entries, - GameCultRuntimePresenceHealthRecord::TYPE, - &presence_store_key( - &service.incarnation()?, - &service.projection.expected.expected_signer_identity_id, - ), - )? - .context("no stored presence")? - .payload - .clone()) + Ok(stored_presence(world, service)?.0) } #[test] @@ -2800,9 +2881,11 @@ mod tests { let first = world .engine(NOW) .admit_presence("ghostlight", &presence, NOW)?; + assert!(world.store.flush()?); + let restarted = MemoryOdinTopologyStore::load(&world.topology_path)?; let after_restart = world - .engine(NOW + 100) + .engine_on(&restarted, NOW + 100) .refresh(&service.incarnation()?)? .unwrap(); assert_eq!(after_restart, first); @@ -2831,12 +2914,11 @@ mod tests { .is_none() ); - let entries = SingleFileMessagePackBackingStore::new(&world.topology_path) - .pull_all_read_only_snapshot()?; - assert!(entries.iter().any(|entry| { + let records = world.store.records(); + assert!(records.values().any(|entry| { entry.r#type == GameCultRuntimePresenceHealthRecord::TYPE && entry.payload == presence })); - assert!(!entries.iter().any(|entry| { + assert!(!records.values().any(|entry| { entry.r#type == OdinRuntimeTopologyCorrelationRecord::TYPE && entry.key.starts_with("ghostlight@") })); diff --git a/crates/odin-daemon/src/main.rs b/crates/odin-daemon/src/main.rs index d7d5c39..92f3c96 100644 --- a/crates/odin-daemon/src/main.rs +++ b/crates/odin-daemon/src/main.rs @@ -12,9 +12,7 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use anyhow::{Context, Result, bail, ensure}; use chrono::{DateTime, Utc}; -use cultcache_rs::{ - CultCacheEnvelope, CultCacheExpectedEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, -}; +use cultcache_rs::{CultCacheEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore}; use cultmesh_rs::{ CultMeshRudpDocumentServer, CultMeshRudpDocumentServerOptions, CultMeshRudpPollOutcome, CultMeshRudpRawDocumentReceipt, CultMeshRudpRawDocumentSink, CultMeshRudpSnapshotQuery, @@ -34,9 +32,9 @@ use cultnet_rs::{ }; use fs2::FileExt; use odin_daemon::{ - AuthenticationPolicy, CultCacheIdunnProjectionSource, CultCacheOdinTopologyStore, - IdunnProjectionSource, IncarnationRef, OdinTopologyAuthority, PresenceAuthorityRefused, - ProjectionUnreadable, SystemClock, + AuthenticationPolicy, CultCacheIdunnProjectionSource, IdunnProjectionSource, IncarnationRef, + MemoryOdinTopologyStore, OdinTopologyAuthority, PresenceAuthorityRefused, ProjectionUnreadable, + SystemClock, }; const TARGET: &str = "odin"; @@ -73,14 +71,6 @@ const PROJECTION_REFRESH_INTERVAL: Duration = Duration::from_millis(250); const IDLE_POLL_INTERVAL: Duration = Duration::from_millis(2); const MAX_RECENT_WARMING_PROOFS: usize = 64; const WARMING_PROOF_LIFETIME_MILLIS: u64 = 60_000; -const CAS_ATTEMPTS: usize = 8; - -type TopologyAuthority = OdinTopologyAuthority< - CultCacheIdunnProjectionSource, - CultCacheOdinTopologyStore, - ServiceIdentitySigner, - SystemClock, ->; /// Relative path of Odin's durable store inside the state root. The recipe's /// `topology` state slot in `deployment/idunn/recipe.toml` is the authority for @@ -110,13 +100,43 @@ struct ProcessWriteLeaseGuard { sha256: String, } +/// Odin's topology authority once its write lease is held: the store it loaded +/// and the credentials that sign for it. An `OdinTopologyAuthority` is built +/// over it for each operation, with the projection read for that operation. +struct Topology { + store: MemoryOdinTopologyStore, + signer: ServiceIdentitySigner, + idunn_anchor: ServiceIdentityTrustAnchor, +} + +impl Topology { + fn authority( + &self, + projections: P, + ) -> OdinTopologyAuthority< + P, + &MemoryOdinTopologyStore, + &ServiceIdentitySigner, + SystemClock, + > { + OdinTopologyAuthority::new( + projections, + &self.store, + &self.signer, + SystemClock, + self.idunn_anchor.clone(), + AuthenticationPolicy::default(), + ) + } +} + struct RuntimeState { options: Options, candidate: SocketAddr, authority_material: RuntimeAuthority, idunn_anchor: Option, topology_signer: Option>, - topology: Option, + topology: Option, write_lease: Option, write_lease_path: PathBuf, recent_warming_proofs: VecDeque<(String, u64)>, @@ -198,14 +218,6 @@ impl RuntimeState { ) })?, )?; - let store_sequence = prior_self_publisher_sequence( - &options.store, - authority_material - .provider_signer - .entry() - .identity_id - .as_str(), - )?; Ok(Self { options, @@ -217,7 +229,7 @@ impl RuntimeState { write_lease: None, write_lease_path: PathBuf::from(required_environment(PROCESS_WRITE_LEASE_ENVIRONMENT)?), recent_warming_proofs: VecDeque::new(), - publisher_sequence: store_sequence, + publisher_sequence: 0, log_gate: RefCell::default(), }) } @@ -246,6 +258,18 @@ impl RuntimeState { if projected.current_lease.as_ref() != Some(&lease.record) { return Ok(false); } + // The one read of the store file while this process lives: from here + // on the working set is in memory, and the file is only written. + let store = MemoryOdinTopologyStore::load(&self.options.store)?; + // Correlations the previous, target-keyed Odin left in this store are + // not this contract's and would otherwise be served to the Verse as + // current. Presence history is kept: the self publisher sequence + // continues from it. + store.retire_legacy_correlations(); + let stored_sequence = prior_self_publisher_sequence( + store.records().values(), + &self.authority_material.provider_signer.entry().identity_id, + ); let signer = self .topology_signer .take() @@ -254,18 +278,28 @@ impl RuntimeState { .idunn_anchor .take() .context("Idunn trust anchor was already consumed")?; + self.publisher_sequence = self.publisher_sequence.max(stored_sequence); self.write_lease = Some(lease); - self.topology = Some(OdinTopologyAuthority::new( - CultCacheIdunnProjectionSource::new(&self.options.idunn_projection), - CultCacheOdinTopologyStore::new(&self.options.store), + self.topology = Some(Topology { + store, signer, - SystemClock, idunn_anchor, - AuthenticationPolicy::default(), - )); + }); Ok(true) } + fn topology(&self) -> Result<&Topology> { + self.topology + .as_ref() + .context("Odin topology authority is absent") + } + + /// Write Odin's store file from the working set, if it changed. + fn flush(&self) -> Result<()> { + self.topology()?.store.flush()?; + Ok(()) + } + fn accept_raw_document(&mut self, receipt: CultMeshRudpRawDocumentReceipt) -> Result<()> { ensure!( self.activated(), @@ -273,10 +307,11 @@ impl RuntimeState { ); validate_raw_document_shape(&receipt.document)?; if receipt.document.schema_id == GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA { - return self - .admit_presence_document(&receipt.document, receipt.received_at_unix_millis); + self.admit_presence_document(&receipt.document, receipt.received_at_unix_millis)?; + } else { + persist_generic_document(&self.topology()?.store, &receipt.document)?; } - persist_generic_document(&self.options.store, &receipt.document) + self.flush() } /// The one admission path for a runtime-presence document, whether a @@ -291,10 +326,13 @@ impl RuntimeState { document.record_key == presence.target, "runtime-presence document key differs from its signed target" ); - self.topology - .as_ref() - .context("Odin topology authority is absent")? - .admit_presence(&presence.target, &document.payload, received_at_unix_millis)?; + let projections = + CultCacheIdunnProjectionSource::new(&self.options.idunn_projection).snapshot()?; + self.topology()?.authority(projections).admit_presence( + &presence.target, + &document.payload, + received_at_unix_millis, + )?; Ok(()) } @@ -423,7 +461,8 @@ impl RuntimeState { self.require_current_write_lease()?; self.require_own_projection()?; let document = self.signed_presence_document("active", detail)?; - self.admit_presence_document(&document, unix_millis()?) + self.admit_presence_document(&document, unix_millis()?)?; + self.flush() } /// Idunn's projection of this process's own incarnation. A failure to @@ -473,31 +512,29 @@ impl RuntimeState { Ok(()) } - fn refresh_all_correlations(&mut self) -> Result<()> { + /// Refresh every incarnation Idunn projects, and every incarnation Odin + /// holds a correlation for, projected or not (the latter are refreshed so + /// their correlations are withdrawn). The projection is read once for the + /// whole pass. + fn refresh_all_correlations(&self) -> Result<()> { self.require_current_write_lease()?; - let topology = self - .topology - .as_ref() - .context("Odin topology authority is absent")?; - let mut incarnations = incarnation_keys( - &self.options.idunn_projection, - IdunnExpectedIncarnationRecord::TYPE, - )?; - incarnations.extend(incarnation_keys( - &self.options.store, - OdinRuntimeTopologyCorrelationRecord::TYPE, - )?); + let topology = self.topology()?; + let projections = + CultCacheIdunnProjectionSource::new(&self.options.idunn_projection).snapshot()?; + let mut incarnations = projections.incarnations(); + incarnations.extend(topology.store.correlated_incarnations()); + let authority = topology.authority(projections); // Each incarnation is refreshed on its own: one whose records cannot be // read is that incarnation's failure, and never stops the others. for incarnation in incarnations { - if let Err(error) = topology.refresh(&incarnation) { + if let Err(error) = authority.refresh(&incarnation) { self.log_repeating( &format!("refresh of incarnation {}", incarnation.key()), format!("failed; the rest are unaffected: {error:#}"), ); } } - Ok(()) + self.flush() } /// The catalog serves what is stored, one record at a time: a record that @@ -510,15 +547,10 @@ impl RuntimeState { &self, query: &CultMeshRudpSnapshotQuery, ) -> Result> { - let entries = if self.options.store.is_file() { - SingleFileMessagePackBackingStore::new(&self.options.store) - .pull_all_read_only_snapshot()? - } else { - Vec::new() - }; + let records = self.topology()?.store.records(); let projections = CultCacheIdunnProjectionSource::new(&self.options.idunn_projection); let mut selected = BTreeMap::new(); - for envelope in &entries { + for envelope in records.values() { let document = match self.public_document(&projections, envelope) { Ok(Some(document)) => document, Ok(None) => continue, @@ -730,10 +762,6 @@ fn main() -> Result<()> { } } - // Correlations the previous, target-keyed Odin left in this store are not - // this contract's and would otherwise be served to the Verse as current. - // Presence history is kept: the self publisher sequence continues from it. - CultCacheOdinTopologyStore::new(&state.borrow().options.store).retire_legacy_correlations()?; // Once serving, only losing the write lease ends Odin (see `survive`). let mut timers = ServingTimers::default(); loop { @@ -1181,12 +1209,11 @@ fn read_process_write_lease(path: &Path) -> Result Result { - if !path.is_file() { - return Ok(0); - } - Ok(SingleFileMessagePackBackingStore::new(path) - .pull_all_read_only_snapshot()? +fn prior_self_publisher_sequence<'a>( + records: impl IntoIterator, + signer_identity_id: &str, +) -> u64 { + records .into_iter() .filter(|entry| { entry.r#type == GameCultRuntimePresenceHealthRecord::TYPE @@ -1207,7 +1234,7 @@ fn prior_self_publisher_sequence(path: &Path, signer_identity_id: &str) -> Resul }) .map(|presence| presence.publisher_sequence) .max() - .unwrap_or(0)) + .unwrap_or(0) } /// The document namespaces providers publish to the rendezvous. A provider's @@ -1263,49 +1290,23 @@ fn is_peer_document_type(document_type: &str) -> bool { }) } -fn persist_generic_document(path: &Path, document: &CultNetRawDocumentRecord) -> Result<()> { +fn persist_generic_document( + store: &MemoryOdinTopologyStore, + document: &CultNetRawDocumentRecord, +) -> Result<()> { let document_type = document_type_for_schema(&document.schema_id)?; ensure!( is_peer_document_type(&document_type), "Odin accepts no {document_type} document from a provider" ); - let replacement = CultCacheEnvelope { + store.put(CultCacheEnvelope { key: document.record_key.clone(), - r#type: document_type.clone(), + r#type: document_type, payload: document.payload.clone(), stored_at: document.stored_at.clone(), schema_id: Some(document.schema_id.clone()), - }; - let store = SingleFileMessagePackBackingStore::new(path); - for _ in 0..CAS_ATTEMPTS { - let entries = if path.is_file() { - store.pull_all_read_only_snapshot()? - } else { - Vec::new() - }; - let mut matches = entries - .iter() - .filter(|entry| entry.r#type == document_type && entry.key == document.record_key); - let current = matches.next().cloned(); - ensure!( - matches.next().is_none(), - "generic CultCache identity is ambiguous" - ); - if current.as_ref() == Some(&replacement) { - return Ok(()); - } - if store.compare_exchange( - &[CultCacheExpectedEnvelope { - key: document.record_key.clone(), - r#type: document_type.clone(), - current, - }], - &[replacement.clone()], - )? { - return Ok(()); - } - } - bail!("Odin catalog changed repeatedly while persisting a provider document") + }); + Ok(()) } fn document_type_for_schema(schema_id: &str) -> Result { @@ -1349,28 +1350,6 @@ fn decode_presence(payload: &[u8]) -> Result Result> { - if !path.is_file() { - return Ok(BTreeSet::new()); - } - Ok(SingleFileMessagePackBackingStore::new(path) - .pull_all_read_only_snapshot()? - .into_iter() - .filter(|entry| entry.r#type == record_type) - .filter_map(|entry| IncarnationRef::parse_key(&entry.key)) - .collect()) -} - fn validate_snapshot_filters(query: &CultMeshRudpSnapshotQuery) -> Result<()> { ensure!( !query.message_id.is_empty() && query.message_id.trim() == query.message_id, @@ -1615,6 +1594,16 @@ mod tests { /// The same world with the production server bound under `options`. fn activated_odin_serving(options: CultMeshRudpDocumentServerOptions) -> Result { + activated_odin_with(options, |_| Ok(Vec::new())) + } + + /// The same world, with Odin's store file holding what `seed` returns when + /// Odin activates. `seed` runs before the lease is granted, with the + /// runtime that is about to activate, so it can sign Odin's own presences. + fn activated_odin_with( + options: CultMeshRudpDocumentServerOptions, + seed: impl FnOnce(&mut RuntimeState) -> Result>, + ) -> Result { let temp = tempfile::tempdir()?; let root = temp.path(); let socket = UdpSocket::bind("127.0.0.1:0")?; @@ -1712,6 +1701,13 @@ mod tests { publisher_sequence: 0, log_gate: RefCell::default(), }; + let seeded = seed(&mut runtime)?; + if !seeded.is_empty() { + ensure!( + SingleFileMessagePackBackingStore::new(&store).compare_exchange_snapshot(&[], &seeded)?, + "test store seed failed" + ); + } // The lease names the Warming presence Odin signed for Idunn's probe. let warming = runtime.signed_presence_document("warming", "test warming")?; @@ -1861,18 +1857,33 @@ mod tests { }) } + /// Every record in Odin's working set. + fn working_set(&self) -> Vec { + let state = self.state.borrow(); + let records = state.topology.as_ref().unwrap().store.records(); + records.values().cloned().collect() + } + + /// Every record in Odin's store file. + fn file_records(&self) -> Result> { + SingleFileMessagePackBackingStore::new(&self.store).pull_all_read_only_snapshot() + } + fn stored_keys(&self, record_type: &str) -> Result> { - Ok(SingleFileMessagePackBackingStore::new(&self.store) - .pull_all_read_only_snapshot()? + Ok(self + .working_set() .into_iter() .filter(|entry| entry.r#type == record_type) .map(|entry| entry.key) .collect()) } - /// Odin's own publisher sequence as it is durably stored. + /// Odin's own publisher sequence as its store holds it. fn stored_sequence(&self) -> Result { - prior_self_publisher_sequence(&self.store, &self.provider_identity_id) + Ok(prior_self_publisher_sequence( + &self.working_set(), + &self.provider_identity_id, + )) } /// Run a peer on its own thread while Odin polls its socket, and nothing @@ -1991,6 +2002,7 @@ mod tests { odin.pass()?; assert!(odin.stored_sequence()? > 0, "the first pass publishes"); + let written = std::fs::read(&odin.store)?; std::fs::write(&odin.store, b"not a cultcache store")?; assert!( odin.state @@ -2014,12 +2026,13 @@ mod tests { ); } - std::fs::remove_file(&odin.store)?; + std::fs::write(&odin.store, written)?; odin.timers = ServingTimers::default(); odin.pass()?; - assert!( - odin.stored_sequence()? > 0, - "the publication after the fault clears lands" + assert_eq!( + odin.file_records()?, + odin.working_set(), + "the write after the fault clears lands" ); Ok(()) } @@ -2235,24 +2248,25 @@ mod tests { /// A stored presence of Odin's own activation that no longer authenticates /// makes its publisher sequence unknowable; the claim is refused, closed. + /// The store is Odin's alone while it serves, so the bad record is in the + /// file when Odin activates. #[test] fn a_stored_presence_that_no_longer_authenticates_ends_odin() -> Result<()> { - let mut odin = activated_odin()?; - odin.pass()?; - // Received two minutes after it was observed: outside the trusted window. - let late = rfc3339_millis(unix_millis()? + 120_000)?; - odin.tamper_store(|entries| { - entries - .into_iter() - .map(|mut entry| { - if entry.r#type == GameCultRuntimePresenceHealthRecord::TYPE { - entry.stored_at = late.clone(); - } - entry - }) - .collect() + let mut odin = activated_odin_with(CultMeshRudpDocumentServerOptions::default(), |runtime| { + let warming = runtime.signed_presence_document("warming", "earlier launch")?; + // Received two minutes after it was observed: outside the trusted window. + Ok(vec![CultCacheEnvelope { + key: format!( + "{}/{}", + self_incarnation(&runtime.authority_material).key(), + runtime.authority_material.provider_signer.entry().identity_id + ), + r#type: GameCultRuntimePresenceHealthRecord::TYPE.into(), + payload: warming.payload, + stored_at: rfc3339_millis(unix_millis()? + 120_000)?, + schema_id: Some(GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()), + }]) })?; - odin.timers = ServingTimers::default(); assert_refused(&odin.pass().unwrap_err()); Ok(()) } @@ -2337,42 +2351,55 @@ mod tests { /// One record that does not decode is skipped where it is met: the catalog /// still serves everything else, and startup still finds Odin's sequence. + /// The records are in the file when Odin activates. #[test] fn records_that_do_not_decode_never_stop_the_catalog_or_startup() -> Result<()> { - let mut odin = activated_odin()?; - odin.pass()?; - let before = odin.stored_sequence()?; - assert!(before > 0); - let stamp = rfc3339_millis(unix_millis()?)?; - let own_signer = odin.provider_identity_id.clone(); - odin.tamper_store(|mut entries| { - for key in [ - "ghost".to_owned(), - format!("{TARGET}@{}/{own_signer}", digest('c')), - ] { - entries.push(CultCacheEnvelope { - key, + const EARLIER: u64 = 41; + let mut odin = activated_odin_with(CultMeshRudpDocumentServerOptions::default(), |runtime| { + let own_signer = runtime + .authority_material + .provider_signer + .entry() + .identity_id + .clone(); + let stamp = rfc3339_millis(unix_millis()?)?; + let bad_presence = |key: String| CultCacheEnvelope { + key, + r#type: GameCultRuntimePresenceHealthRecord::TYPE.into(), + payload: vec![0xc1], + stored_at: stamp.clone(), + schema_id: Some(GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()), + }; + // A presence an earlier incarnation of Odin stored, counted to 41. + runtime.publisher_sequence = EARLIER - 1; + let earlier = runtime.signed_presence_document("warming", "earlier incarnation")?; + runtime.publisher_sequence = 0; + Ok(vec![ + bad_presence("ghost".into()), + bad_presence(format!("{TARGET}@{}/{own_signer}", digest('c'))), + CultCacheEnvelope { + key: format!("{TARGET}@{}/{own_signer}", digest('d')), r#type: GameCultRuntimePresenceHealthRecord::TYPE.into(), - payload: vec![0xc1], + payload: earlier.payload, stored_at: stamp.clone(), schema_id: Some(GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()), - }); - } - entries.push(generic_document().unwrap()); - entries + }, + generic_document()?, + ]) })?; let catalog = odin.catalog()?; assert_eq!( documents_of(&catalog, GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA), 1, - "Odin's own presence is served" + "the presence that decodes is served" ); assert_eq!(documents_of(&catalog, "ghostlight.doc.v1"), 1); + odin.pass()?; assert_eq!( - prior_self_publisher_sequence(&odin.store, &odin.provider_identity_id)?, - before, - "the undecodable presences do not change the sequence Odin continues from" + odin.stored_sequence()?, + EARLIER + 1, + "Odin continues from the sequence it found past the undecodable presences" ); Ok(()) } @@ -2383,10 +2410,7 @@ mod tests { fn a_presence_whose_projection_cannot_be_read_is_skipped_not_fatal() -> Result<()> { let mut odin = activated_odin()?; odin.pass()?; - odin.tamper_store(|mut entries| { - entries.push(generic_document().unwrap()); - entries - })?; + provider_put(&odin, "ghostlight.doc.v1", "doc-1", vec![1])?; odin.tamper_projection(|entries| { entries .into_iter() @@ -2613,8 +2637,7 @@ mod tests { let mut odin = activated_odin()?; odin.pass()?; let before = odin.stored_sequence()?; - let stored_before = - SingleFileMessagePackBackingStore::new(&odin.store).pull_all_read_only_snapshot()?; + let stored_before = odin.working_set(); // Every kind Odin itself stored (its watermark and correlations, found // rather than named), and the kinds Idunn projects to it. @@ -2650,7 +2673,7 @@ mod tests { assert!(format!("{refused:#}").contains("accepts no"), "{refused:#}"); } assert_eq!( - SingleFileMessagePackBackingStore::new(&odin.store).pull_all_read_only_snapshot()?, + odin.working_set(), stored_before, "no refused put touched the store" ); @@ -2672,12 +2695,11 @@ mod tests { let mut odin = activated_odin()?; odin.pass()?; provider_put(&odin, "ghostlight.doc.v1", "doc-1", vec![1])?; - let stored: BTreeSet = - SingleFileMessagePackBackingStore::new(&odin.store) - .pull_all_read_only_snapshot()? - .into_iter() - .map(|entry| entry.r#type) - .collect(); + let stored: BTreeSet = odin + .working_set() + .into_iter() + .map(|entry| entry.r#type) + .collect(); assert!( stored.len() >= 4, "Odin's store holds more than what it serves: {stored:?}" @@ -2912,4 +2934,62 @@ mod tests { assert!(!settle_poll(fault(), &mut since, restart).unwrap()); assert!(settle_poll(fault(), &mut since, restart + POLL_FAILURE_LIMIT).is_err()); } + + // ---- the store file is not working memory ----------------------------- + + /// Once Odin holds its lease, its working set is in memory. Rewriting the + /// store file behind its back changes nothing Odin serves or refreshes; + /// only a restart, which loads the file again, would see it. + #[test] + fn rewriting_the_store_file_behind_odin_changes_nothing_it_serves() -> Result<()> { + let mut odin = activated_odin()?; + provider_put(&odin, "ghostlight.doc.v1", "doc-1", vec![1])?; + odin.pass()?; + let served = odin.catalog()?; + let working = odin.working_set(); + assert_eq!(documents_of(&served, "ghostlight.doc.v1"), 1); + + odin.tamper_store(|entries| { + let mut entries: Vec<_> = entries + .into_iter() + .filter(|entry| entry.r#type == "odin.topology_publisher_watermark") + .collect(); + let mut foreign = generic_document().unwrap(); + foreign.key = "foreign".into(); + entries.push(foreign); + entries + })?; + for _ in 0..3 { + odin.state.borrow().refresh_all_correlations()?; + } + assert_eq!(odin.catalog()?, served); + assert_eq!(odin.working_set(), working); + assert_ne!( + MemoryOdinTopologyStore::load(&odin.store)?.records().len(), + working.len(), + "a restart would load what the file now holds" + ); + Ok(()) + } + + /// Refresh passes and catalog reads never read the store file. With the + /// file replaced by bytes that do not decode, a reader would fail at once; + /// every pass succeeds and logs nothing. + #[test] + fn refresh_passes_and_catalog_reads_never_read_the_store_file() -> Result<()> { + let mut odin = activated_odin()?; + provider_put(&odin, "ghostlight.doc.v1", "doc-1", vec![1])?; + odin.pass()?; + let served = odin.catalog()?; + std::fs::write(&odin.store, b"not a cultcache store")?; + for _ in 0..5 { + odin.state.borrow().refresh_all_correlations()?; + assert_eq!(odin.catalog()?, served); + } + assert!( + odin.state.borrow().log_gate.borrow().seen.is_empty(), + "a pass failed and was logged" + ); + Ok(()) + } } From b5f13308e996363147b22630502266d63765baf2 Mon Sep 17 00:00:00 2001 From: soul-probe Date: Wed, 30 Sep 2026 13:00:53 +0200 Subject: [PATCH 02/43] Odin Cut 2: one write of the store file per interval, only when it changed Every admission, put and refresh that changed something rewrote and fsynced the whole store; a heartbeat did it twice (watermark, then correlation). Now the serving loop writes the file at most once per FLUSH_INTERVAL (1 s, operator default Q3), and only when the working set changed. Deleted first: the per-mutation writes Cut 1 left in accept_raw_document, publish_self_presence and refresh_all_correlations. The watermark's own write went with the file store in Cut 1; its mark now travels in the same atomic write as the correlations that use it. "Published" means written. Added: FLUSH_INTERVAL, the flush step in serving_pass, and a final write on SIGTERM/SIGINT. The dirty flag lives in MemoryOdinTopologyStore. RuntimeState:: flush is the only write of the file: it checks the lease immediately before writing, and CultLib's snapshot compare-exchange against what Odin last loaded or wrote refuses a file another process wrote (ForeignStoreWrite). survive treats ForeignStoreWrite like WriteLeaseLost: both end Odin. An attempt, failed or not, starts the interval; an interval with nothing to write does not. Tests: 50 changes inside one interval are one write, not made at 0.9 s and made at 1.1 s after the last; nothing changed is no write; a lease lost before the write writes nothing and ends Odin; a foreign write ends Odin and is not written over; stopping writes only under the lease; a crash and reload never republishes a written sequence, withdrawal included; a flush writes only changes and never over a foreign file. The failed-publication test becomes a failed-write test: the store is no longer read by publication. Map: docs/write-pattern-cut.md, Cut 2. Co-Authored-By: Claude Opus 5.5 --- crates/odin-daemon/src/lib.rs | 94 ++++++++++++++ crates/odin-daemon/src/main.rs | 215 +++++++++++++++++++++++++++++---- 2 files changed, 283 insertions(+), 26 deletions(-) diff --git a/crates/odin-daemon/src/lib.rs b/crates/odin-daemon/src/lib.rs index 40b44c1..53479ab 100644 --- a/crates/odin-daemon/src/lib.rs +++ b/crates/odin-daemon/src/lib.rs @@ -2048,6 +2048,100 @@ mod tests { Ok(()) } + /// A write is made only when something changed, and only over the file the + /// store last loaded or wrote. A file another process wrote is refused + /// and left as that process wrote it. + #[test] + fn a_flush_writes_only_changes_and_never_over_a_foreign_file() -> Result<()> { + let temp = TempDir::new()?; + let path = temp.path().join("topology.cc"); + let store = MemoryOdinTopologyStore::load(&path)?; + assert!(!store.flush()?); + assert!(!path.exists(), "an unchanged empty store writes nothing"); + + let document = CultCacheEnvelope { + key: "doc-1".into(), + r#type: "ghostlight.doc".into(), + payload: vec![1], + stored_at: rfc3339_millis(NOW)?, + schema_id: Some("ghostlight.doc.v1".into()), + }; + store.put(document.clone()); + assert!(store.flush()?); + store.put(document.clone()); + assert!(!store.flush()?, "storing the record already held is no change"); + + let file = SingleFileMessagePackBackingStore::new(&path); + let mut foreign = document.clone(); + foreign.key = "foreign".into(); + ensure!(file.compare_exchange_snapshot(&[document.clone()], &[foreign.clone()])?); + let mut changed = document; + changed.payload = vec![2]; + store.put(changed); + let error = store.flush().unwrap_err(); + assert!(error.downcast_ref::().is_some(), "{error:#}"); + assert_eq!(file.pull_all_read_only_snapshot()?, vec![foreign]); + Ok(()) + } + + fn file_sequences(path: &Path) -> Result<(u64, Vec)> { + let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; + let mark = unique_envelope(&entries, TOPOLOGY_PUBLISHER_WATERMARK_TYPE, "ghostlight")? + .map(|envelope| decode_publisher_watermark("ghostlight", &envelope.payload)) + .transpose()? + .unwrap_or(0); + let correlations = entries + .iter() + .filter(|entry| entry.r#type == OdinRuntimeTopologyCorrelationRecord::TYPE) + .map(|entry| Ok(decode_correlation(&entry.payload)?.publisher_sequence)) + .collect::>>()?; + Ok((mark, correlations)) + } + + /// A correlation sequence is published when it is written. The mark is + /// written with the correlations that use it, so the file never holds a + /// correlation ahead of its mark, and after a crash (the working set lost + /// without a final write) and a reload, the next correlation is above + /// every sequence the file holds -- even with the correlation withdrawn. + #[test] + fn a_crash_never_republishes_a_written_sequence() -> Result<()> { + let world = TestWorld::new()?; + let service = world.service("ghostlight", Vec::new(), false)?; + for sequence in 1..=2 { + world.engine(NOW).admit_presence( + "ghostlight", + &service.signed_presence(sequence, |_| {})?, + NOW + sequence, + )?; + } + world.withdraw_projection(&service.incarnation()?)?; + world.engine(NOW + 3).refresh(&service.incarnation()?)?; + assert!(world.store.flush()?); + let (mark, correlations) = file_sequences(&world.topology_path)?; + assert_eq!((mark, correlations.len()), (2, 0), "withdrawn, the mark stays"); + + // Changes made and never written, then the crash. + world.publish_projection(&service.projection)?; + world.engine(NOW + 4).admit_presence( + "ghostlight", + &service.signed_presence(3, |_| {})?, + NOW + 4, + )?; + let restarted = MemoryOdinTopologyStore::load(&world.topology_path)?; + assert_eq!(file_sequences(&world.topology_path)?.0, mark); + + let next = decode_signed(&world.engine_on(&restarted, NOW + 5).admit_presence( + "ghostlight", + &service.signed_presence(4, |_| {})?, + NOW + 5, + )?)?; + assert!(next.publisher_sequence > mark, "{} repeats", next.publisher_sequence); + assert!(restarted.flush()?); + let (mark, correlations) = file_sequences(&world.topology_path)?; + assert!(correlations.iter().all(|sequence| *sequence <= mark)); + Ok(()) + } + #[derive(Clone, Copy)] struct FixedClock(u64); diff --git a/crates/odin-daemon/src/main.rs b/crates/odin-daemon/src/main.rs index 92f3c96..f1eb8a2 100644 --- a/crates/odin-daemon/src/main.rs +++ b/crates/odin-daemon/src/main.rs @@ -32,9 +32,9 @@ use cultnet_rs::{ }; use fs2::FileExt; use odin_daemon::{ - AuthenticationPolicy, CultCacheIdunnProjectionSource, IdunnProjectionSource, IncarnationRef, - MemoryOdinTopologyStore, OdinTopologyAuthority, PresenceAuthorityRefused, ProjectionUnreadable, - SystemClock, + AuthenticationPolicy, CultCacheIdunnProjectionSource, ForeignStoreWrite, + IdunnProjectionSource, IncarnationRef, MemoryOdinTopologyStore, OdinTopologyAuthority, + PresenceAuthorityRefused, ProjectionUnreadable, SystemClock, }; const TARGET: &str = "odin"; @@ -68,6 +68,11 @@ const POLL_FAILURE_LIMIT: Duration = Duration::from_secs(30); const POLL_FAILURE_BACKOFF: Duration = Duration::from_millis(100); const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(5); const PROJECTION_REFRESH_INTERVAL: Duration = Duration::from_millis(250); +// At most one write of the store file per interval, and none when nothing +// changed. A crash loses at most this much of the catalog and presence; +// providers republish, and a correlation sequence that was never written was +// never published. Operator default Q3, docs/write-pattern-cut.md. +const FLUSH_INTERVAL: Duration = Duration::from_secs(1); const IDLE_POLL_INTERVAL: Duration = Duration::from_millis(2); const MAX_RECENT_WARMING_PROOFS: usize = 64; const WARMING_PROOF_LIFETIME_MILLIS: u64 = 60_000; @@ -294,10 +299,18 @@ impl RuntimeState { .context("Odin topology authority is absent") } - /// Write Odin's store file from the working set, if it changed. - fn flush(&self) -> Result<()> { - self.topology()?.store.flush()?; - Ok(()) + /// Write Odin's store file from the working set, if it changed; returns + /// whether it wrote. This is the only write of the file. The lease is + /// checked immediately before it, so nothing is written once the lease is + /// lost, and a file another process wrote is refused (`ForeignStoreWrite`) + /// rather than written over. Both end Odin (see `survive`). + fn flush(&self) -> Result { + let store = &self.topology()?.store; + if !store.is_dirty() { + return Ok(false); + } + self.require_current_write_lease()?; + store.flush() } fn accept_raw_document(&mut self, receipt: CultMeshRudpRawDocumentReceipt) -> Result<()> { @@ -311,7 +324,7 @@ impl RuntimeState { } else { persist_generic_document(&self.topology()?.store, &receipt.document)?; } - self.flush() + Ok(()) } /// The one admission path for a runtime-presence document, whether a @@ -461,8 +474,7 @@ impl RuntimeState { self.require_current_write_lease()?; self.require_own_projection()?; let document = self.signed_presence_document("active", detail)?; - self.admit_presence_document(&document, unix_millis()?)?; - self.flush() + self.admit_presence_document(&document, unix_millis()?) } /// Idunn's projection of this process's own incarnation. A failure to @@ -534,7 +546,7 @@ impl RuntimeState { ); } } - self.flush() + Ok(()) } /// The catalog serves what is stored, one record at a time: a record that @@ -767,6 +779,7 @@ fn main() -> Result<()> { loop { if stopping.load(Ordering::Relaxed) { eprintln!("Odin stopping on request"); + stop(&state.borrow()); return Ok(()); } if !serving_pass(&state, &mut server, &mut timers)? { @@ -775,20 +788,32 @@ fn main() -> Result<()> { } } +/// The last write before Odin stops on request. `flush` makes it only while +/// the lease is still current. A failure is logged and not returned: Odin is +/// stopping either way, and the next Odin loads what the file holds. +fn stop(state: &RuntimeState) { + if let Err(error) = state.flush() { + eprintln!("Odin's final store write was not made: {error:#}"); + } +} + /// A timer that has never fired is due, so the first pass refreshes and /// publishes. #[derive(Default)] struct ServingTimers { last_heartbeat: Option, last_projection_refresh: Option, + /// When the store file was last written, or a write last failed. + last_flush: Option, poll_failing_since: Option, } type OdinServer = CultMeshRudpDocumentServer; -/// One turn of the serving loop: serve a datagram, refresh the topology, and -/// publish Odin's own presence when each is due. Returns whether the poll made -/// progress. It fails only for `WriteLeaseLost` or a dead socket. +/// One turn of the serving loop: serve a datagram, refresh the topology, +/// publish Odin's own presence, and write the store file, when each is due. +/// Returns whether the poll made progress. It fails only for the conditions +/// `survive` returns, or a dead socket. fn serving_pass( state: &Rc>, server: &mut OdinServer, @@ -805,6 +830,15 @@ fn serving_pass( survive(&state.borrow(), "self-presence publication", published)?; timers.last_heartbeat = Some(Instant::now()); } + if is_due(timers.last_flush, FLUSH_INTERVAL) { + let flushed = state.borrow().flush(); + // Only an attempt starts the interval: with nothing to write, the next + // change is written on the first pass after it. + if !matches!(flushed, Ok(false)) { + timers.last_flush = Some(Instant::now()); + } + survive(&state.borrow(), "store write", flushed)?; + } Ok(progressed) } @@ -828,8 +862,10 @@ impl std::error::Error for WriteLeaseLost {} /// The serving loop's failure policy. Odin's liveness is Odin's own: a failed /// refresh, publication or read is logged and retried on the next pass, never -/// allowed to end the daemon. Only `WriteLeaseLost`, wherever in the error chain -/// it sits, is returned -- and, as a named temporary rule, so is +/// allowed to end the daemon. Only `WriteLeaseLost` and `ForeignStoreWrite`, +/// wherever in the error chain they sit, are returned: each means another +/// writer holds Odin's state, and writing on would be a second writer. As a +/// named temporary rule, so is /// `PresenceAuthorityRefused`: a failure to establish Odin's OWN authority /// (`require_own_projection`), or a self-presence that authority will never /// admit (no verifiable authority, a signer that does not match the anchor, a @@ -845,6 +881,7 @@ fn survive(state: &RuntimeState, what: &str, result: Result) -> Result Ok(Some(value)), Err(error) if error.downcast_ref::().is_some() + || error.downcast_ref::().is_some() || error.downcast_ref::().is_some() => { Err(error) @@ -1994,29 +2031,27 @@ mod tests { Ok(()) } - /// A publication or refresh that fails is logged and retried; it never - /// ends Odin, and the next attempt lands once the fault clears. + /// A store write that fails is logged and retried; it never ends Odin, and + /// the next attempt lands once the fault clears. #[test] - fn a_failed_self_publication_or_refresh_does_not_end_odin() -> Result<()> { + fn a_failed_store_write_does_not_end_odin() -> Result<()> { let mut odin = activated_odin()?; odin.pass()?; - assert!(odin.stored_sequence()? > 0, "the first pass publishes"); + assert_eq!(odin.file_records()?, odin.working_set(), "the first pass writes"); let written = std::fs::read(&odin.store)?; std::fs::write(&odin.store, b"not a cultcache store")?; + provider_put(&odin, "ghostlight.doc.v1", "doc-1", vec![1])?; assert!( - odin.state - .borrow_mut() - .publish_self_presence("probe") - .is_err(), - "the injected fault must actually fail the publication" + odin.state.borrow().flush().is_err(), + "the injected fault must actually fail the write" ); odin.timers = ServingTimers::default(); odin.pass()?; let suppressed = |odin: &OdinWorld| odin.state.borrow().log_gate.borrow().suppressed_total; assert_eq!(suppressed(&odin), 0, "each distinct failure is logged"); - // The same faults on later passes are counted, not printed again. + // The same fault on later passes is counted, not printed again. for repeat in 1..=3 { odin.timers = ServingTimers::default(); odin.pass()?; @@ -2805,6 +2840,8 @@ mod tests { ); let lost = anyhow::Error::new(WriteLeaseLost("gone".into())).context("refreshing"); assert!(survive::<()>(&state, "x", Err(lost)).is_err()); + let foreign = anyhow::Error::new(ForeignStoreWrite).context("writing"); + assert!(survive::<()>(&state, "x", Err(foreign)).is_err()); Ok(()) } @@ -2935,6 +2972,132 @@ mod tests { assert!(settle_poll(fault(), &mut since, restart + POLL_FAILURE_LIMIT).is_err()); } + // ---- one write per interval ---------------------------------------------- + + /// The store file as a reader sees it: which file, and what it holds. Each + /// write replaces the file atomically, so it is a new file. + fn file_identity(path: &Path) -> Result<(u64, Vec)> { + use std::os::unix::fs::MetadataExt; + Ok((std::fs::metadata(path)?.ino(), std::fs::read(path)?)) + } + + fn ago(millis: u64) -> Option { + Instant::now().checked_sub(Duration::from_millis(millis)) + } + + /// Fifty changes inside one interval are one write, made once the + /// interval has passed: not at 0.9 s after the last write, at 1.1 s. + #[test] + fn changes_inside_one_interval_are_one_write() -> Result<()> { + let mut odin = activated_odin()?; + odin.pass()?; + let first = file_identity(&odin.store)?; + for index in 0..25 { + provider_put(&odin, "ghostlight.doc.v1", &format!("doc-{index}"), vec![1])?; + odin.timers.last_heartbeat = None; + odin.timers.last_flush = Some(Instant::now()); + odin.pass()?; + } + assert!(odin.stored_sequence()? >= 26, "every heartbeat was admitted"); + assert_eq!(file_identity(&odin.store)?, first, "no write inside the interval"); + + odin.timers.last_flush = ago(900); + odin.pass()?; + assert_eq!(file_identity(&odin.store)?, first, "0.9 s is inside the interval"); + + odin.timers.last_flush = ago(1_100); + odin.pass()?; + let second = file_identity(&odin.store)?; + assert_ne!(second.0, first.0, "one write once the interval passed"); + assert_eq!(odin.file_records()?, odin.working_set()); + + odin.timers.last_flush = None; + odin.pass()?; + assert_eq!(file_identity(&odin.store)?, second, "and only one"); + Ok(()) + } + + /// An interval in which nothing changed writes nothing, however often the + /// write is due. + #[test] + fn nothing_changed_is_no_write() -> Result<()> { + let mut odin = activated_odin()?; + odin.pass()?; + let written = file_identity(&odin.store)?; + for _ in 0..5 { + odin.timers.last_projection_refresh = None; + odin.timers.last_heartbeat = Some(Instant::now()); + odin.timers.last_flush = None; + odin.pass()?; + } + assert_eq!(file_identity(&odin.store)?, written); + Ok(()) + } + + /// The lease is checked at the write itself: lost between a change and + /// its write, the write is not made and Odin ends. + #[test] + fn a_lease_lost_before_the_write_writes_nothing_and_ends_odin() -> Result<()> { + let mut odin = activated_odin()?; + odin.pass()?; + let written = file_identity(&odin.store)?; + provider_put(&odin, "ghostlight.doc.v1", "doc-1", vec![1])?; + odin.swap_lease()?; + odin.timers.last_projection_refresh = Some(Instant::now()); + odin.timers.last_heartbeat = Some(Instant::now()); + odin.timers.last_flush = None; + let error = odin.pass().unwrap_err(); + assert!( + error.downcast_ref::().is_some(), + "{error:#}" + ); + assert_eq!(file_identity(&odin.store)?, written); + Ok(()) + } + + /// A file another process wrote between Odin's writes ends Odin, and is + /// not written over. + #[test] + fn a_foreign_write_ends_odin_and_is_not_written_over() -> Result<()> { + let mut odin = activated_odin()?; + odin.pass()?; + odin.tamper_store(|entries| { + entries + .into_iter() + .filter(|entry| entry.r#type != GameCultRuntimePresenceHealthRecord::TYPE) + .collect() + })?; + let foreign = file_identity(&odin.store)?; + provider_put(&odin, "ghostlight.doc.v1", "doc-1", vec![1])?; + odin.timers.last_flush = None; + let error = odin.pass().unwrap_err(); + assert!( + error.downcast_ref::().is_some(), + "{error:#}" + ); + assert_eq!(file_identity(&odin.store)?, foreign); + Ok(()) + } + + /// Stopping on request writes what changed, while the lease is current, + /// and nothing once it is lost. + #[test] + fn stopping_writes_what_changed_only_under_the_lease() -> Result<()> { + let odin = activated_odin()?; + provider_put(&odin, "ghostlight.doc.v1", "doc-1", vec![1])?; + stop(&odin.state.borrow()); + assert_eq!(odin.file_records()?, odin.working_set()); + + let mut odin = activated_odin()?; + odin.pass()?; + let written = file_identity(&odin.store)?; + provider_put(&odin, "ghostlight.doc.v1", "doc-1", vec![1])?; + odin.swap_lease()?; + stop(&odin.state.borrow()); + assert_eq!(file_identity(&odin.store)?, written); + Ok(()) + } + // ---- the store file is not working memory ----------------------------- /// Once Odin holds its lease, its working set is in memory. Rewriting the From ae8fa5571ffd0aeeaf3dedefa1b6126dc9d608b0 Mon Sep 17 00:00:00 2001 From: soul-probe Date: Wed, 30 Sep 2026 13:09:31 +0200 Subject: [PATCH 03/43] Odin map: Q1 yes (refuse unservable puts), Q2 yes (decode a store copy) Co-Authored-By: Claude Opus 5.5 --- docs/write-pattern-cut.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/write-pattern-cut.md b/docs/write-pattern-cut.md index e174014..f5283dd 100644 --- a/docs/write-pattern-cut.md +++ b/docs/write-pattern-cut.md @@ -3,7 +3,9 @@ **2026-09-30, Self.** Operator: "Let's fix Odin up, that's some furious bookkeeping right there. Look at Odin's write pattern and see if append only or per-document is the way to go." Answer: neither is the first fix. Cuts 1-2 go to Hands now. Q3 (a 1 s interval) and Q4 (the Rust directory store as a CultLib follow-up) are Self defaults, -and the operator may overrule them. Q1 and Q2 are put to the operator, and Cut 3 waits on Q1. +and the operator may overrule them. **Operator, 2026-09-30: "Q1: yes", so CultMesh refuses puts it could never +serve and Cut 3 goes ahead; "Q2: yeah, of course", so a read-only decode of a copy of live `topology.cc` is +approved. The copy is taken with `sudo cp` and decoded off the live path, and the copy is deleted afterwards.** Status: cut map, Imagination pass 1 (Opus), 2026-09-30. Nothing has landed. Anchors are Odin `main` at `44951a1` (the deployed build, live on Yggdrasil since 2026-09-30 04:20:48 UTC) and CultLib `3bf1c0ce` From 8c84857670deee7baab0bcebec7f7d910167e5fb Mon Sep 17 00:00:00 2001 From: soul-probe Date: Wed, 30 Sep 2026 13:22:58 +0200 Subject: [PATCH 04/43] Odin map: Q2 decode result; the oversize records are dead Ghostlight residue Co-Authored-By: Claude Opus 5.5 --- docs/write-pattern-cut.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/write-pattern-cut.md b/docs/write-pattern-cut.md index f5283dd..42d5c83 100644 --- a/docs/write-pattern-cut.md +++ b/docs/write-pattern-cut.md @@ -7,6 +7,25 @@ and the operator may overrule them. **Operator, 2026-09-30: "Q1: yes", so CultMe serve and Cut 3 goes ahead; "Q2: yeah, of course", so a read-only decode of a copy of live `topology.cc` is approved. The copy is taken with `sudo cp` and decoded off the live path, and the copy is deleted afterwards.** +**Q2 decode result (Eyes, 2026-09-30; full table in the CultLib session scratchpad `odin-topology-decode.md`).** +- 77 records; 3,222,550 B of payload in a 3,240,078 B file. +- The two "~1.55 MB records" are per-type totals. Each type is dominated by one Ghostlight operator surface: + `eve:operator:ghostlight.campaign.27d444b3-...` (1,456,210 B) and its `operator-state` twin (1,456,341 B). + Together they are 89.9% of the file. +- Publisher: provider `gamecult.ghostlight.dungeon`. 99.5% of each record is one Eve `code` node holding a + pretty-printed JSON dump of the whole campaign state. +- These records are dead residue. They were stored 2026-09-23 and have not been rewritten since. Ghostlight `HEAD` + (`eeb3a68`) no longer contains the emitter (last touched at `6bb6869`, 2026-09-02). +- **So Q1's refusal breaks no live publisher.** Cut 3's Odin half drops exactly these two at activation. Every other + surface is 70 KB or less. +- Correction to section 3: the 1 MiB snapshot refusals in the journal are per-type responses, not per-record. + Cut 3's per-document bound is still right: each of these records is over 1 MiB alone. A type whose many small + records add up past 1 MiB would still be unservable as a whole-type snapshot. That is a follow-up for CultMesh + (paged snapshot responses), not part of this campaign. +- Presence history: 41 records, 42 KB, 37 of them withdrawn. F4 is a slow leak, not a size problem. +- The current `streampixels-web` incarnation has a correlation but no presence record; its last presence was + 09-29 19:53 UTC. This matches Idunn's "not Odin-correlated" state. + Status: cut map, Imagination pass 1 (Opus), 2026-09-30. Nothing has landed. Anchors are Odin `main` at `44951a1` (the deployed build, live on Yggdrasil since 2026-09-30 04:20:48 UTC) and CultLib `3bf1c0ce` (Odin's `Cargo.lock` pin). This map owns the means; there is no separate target document, because the ends are From 8189623411ae5c326e7e9154d95031417c3c8d95 Mon Sep 17 00:00:00 2001 From: soul-probe Date: Wed, 30 Sep 2026 13:32:11 +0200 Subject: [PATCH 05/43] Odin: the correlation record type is imported only where it is used Cut 1 left it imported at the top of main.rs, where only the tests use it; the release build warned. Co-Authored-By: Claude Opus 5.5 --- crates/odin-daemon/src/main.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/odin-daemon/src/main.rs b/crates/odin-daemon/src/main.rs index f1eb8a2..0615c3d 100644 --- a/crates/odin-daemon/src/main.rs +++ b/crates/odin-daemon/src/main.rs @@ -25,8 +25,8 @@ use cultnet_rs::{ IDUNN_EXPECTED_INCARNATION_SCHEMA, IDUNN_PROCESS_WRITE_LEASE_SCHEMA, IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME, IDUNN_RUNTIME_ACTIVATION_SCHEMA, IdunnExpectedIncarnationRecord, IdunnProcessWriteLeaseRecord, IdunnRuntimeActivationRecord, - IdunnRuntimeActivationSigner, IdunnServiceIdentity, - OdinRuntimeTopologyCorrelationRecord, OdinTopologyIdentity, ServiceIdentityProfile, + IdunnRuntimeActivationSigner, IdunnServiceIdentity, OdinTopologyIdentity, + ServiceIdentityProfile, ServiceIdentitySigner, ServiceIdentityTrustAnchor, derive_service_identity_id, open_service_identity_credential_reader, verify_runtime_authority, }; @@ -1469,7 +1469,8 @@ mod tests { GAMECULT_RUNTIME_PRESENCE_HEALTH_SIGNING_PURPOSE, GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA, GameCultServiceTrustAnchorRecord, IdunnExpectedCapability, IdunnExpectedRoute, IdunnRuntimeActivationLaunch, ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA, - encode_cultnet_message_to_vec, enroll_service_identity_at, + OdinRuntimeTopologyCorrelationRecord, encode_cultnet_message_to_vec, + enroll_service_identity_at, }; use super::*; From 35ea0d45fc7c716799da9e625a5b8710404ddc30 Mon Sep 17 00:00:00 2001 From: soul-probe Date: Wed, 30 Sep 2026 13:39:47 +0200 Subject: [PATCH 06/43] Odin test: with nothing to write, the write step reads nothing, not even the lease The flush step is due on every pass until something changes. Hand mutant M5b (the daemon skipping its own dirty check) left every write rule intact but read the lease file on every 2 ms pass; no test saw it. Co-Authored-By: Claude Opus 5.5 --- crates/odin-daemon/src/main.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/odin-daemon/src/main.rs b/crates/odin-daemon/src/main.rs index 0615c3d..b19eb48 100644 --- a/crates/odin-daemon/src/main.rs +++ b/crates/odin-daemon/src/main.rs @@ -3019,7 +3019,9 @@ mod tests { } /// An interval in which nothing changed writes nothing, however often the - /// write is due. + /// write is due. With nothing to write the write step reads nothing + /// either, not even the lease: it is due on every pass until something + /// changes. #[test] fn nothing_changed_is_no_write() -> Result<()> { let mut odin = activated_odin()?; @@ -3032,6 +3034,13 @@ mod tests { odin.pass()?; } assert_eq!(file_identity(&odin.store)?, written); + + std::fs::write(&odin.lease_path, b"not a cultcache store")?; + odin.timers.last_projection_refresh = Some(Instant::now()); + odin.timers.last_heartbeat = Some(Instant::now()); + odin.timers.last_flush = None; + odin.pass()?; + assert_eq!(file_identity(&odin.store)?, written); Ok(()) } From cda20870fd53e1f470a1af4b02a0631df150436d Mon Sep 17 00:00:00 2001 From: soul-probe Date: Wed, 30 Sep 2026 14:07:44 +0200 Subject: [PATCH 07/43] Odin tests: the write starts the next interval; unprojected correlations are withdrawn cargo-mutants survivors, both real gaps: - serving_pass with the timer condition inverted wrote every change on the next pass after a write; the interval test set the timer by hand on every pass and never saw the loop set it. A change right after a write now waits. - correlated_incarnations returning nothing left a correlation whose incarnation Idunn stopped projecting in place forever; the lib test refreshed that incarnation by name. The refresh pass now has to find it. Co-Authored-By: Claude Opus 5.5 --- crates/odin-daemon/src/main.rs | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/crates/odin-daemon/src/main.rs b/crates/odin-daemon/src/main.rs index b19eb48..ff75b2c 100644 --- a/crates/odin-daemon/src/main.rs +++ b/crates/odin-daemon/src/main.rs @@ -3012,12 +3012,37 @@ mod tests { assert_ne!(second.0, first.0, "one write once the interval passed"); assert_eq!(odin.file_records()?, odin.working_set()); - odin.timers.last_flush = None; + // The write itself starts the next interval: a change right after it + // waits. + provider_put(&odin, "ghostlight.doc.v1", "doc-late", vec![1])?; odin.pass()?; assert_eq!(file_identity(&odin.store)?, second, "and only one"); Ok(()) } + /// A correlation for an incarnation Idunn no longer projects is found + /// through the correlations Odin holds, and the refresh pass withdraws it. + #[test] + fn a_correlation_whose_incarnation_is_no_longer_projected_is_withdrawn() -> Result<()> { + let mut odin = activated_odin()?; + odin.pass()?; + let candidate = project_candidate_incarnation(&odin)?; + odin.state.borrow().refresh_all_correlations()?; + let correlations = || odin.stored_keys(OdinRuntimeTopologyCorrelationRecord::TYPE); + assert!(correlations()?.contains(&candidate.key())); + + odin.tamper_projection(|entries| { + entries + .into_iter() + .filter(|entry| entry.key != candidate.key()) + .collect() + })?; + odin.state.borrow().refresh_all_correlations()?; + assert!(!correlations()?.contains(&candidate.key())); + assert_eq!(correlations()?.len(), 1, "Odin's own stays"); + Ok(()) + } + /// An interval in which nothing changed writes nothing, however often the /// write is due. With nothing to write the write step reads nothing /// either, not even the lease: it is due on every pass until something From 77373cb5635c2a69e8c8efb6aa79a16c648a7ed4 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:06:06 +0200 Subject: [PATCH 08/43] Odin map: Q3 B (durable put ACK); foreign-write handling must repair and recover Co-Authored-By: Claude Opus 5.5 --- docs/write-pattern-cut.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/write-pattern-cut.md b/docs/write-pattern-cut.md index 42d5c83..e1334a2 100644 --- a/docs/write-pattern-cut.md +++ b/docs/write-pattern-cut.md @@ -7,6 +7,18 @@ and the operator may overrule them. **Operator, 2026-09-30: "Q1: yes", so CultMe serve and Cut 3 goes ahead; "Q2: yeah, of course", so a read-only decode of a copy of live `topology.cc` is approved. The copy is taken with `sudo cp` and decoded off the live path, and the copy is deleted afterwards.** +**Soul on Cuts 1-2, and two operator rulings (2026-09-30).** +- **Q3 restated.** The premise "puts carry no application acknowledgement" was false. The cultmesh-rs server + ACKs a put only after `accept_raw_document` returns Ok, so after Cut 1 an ACK meant "in memory", and a SIGKILL + lost acknowledged puts in 6 of 6 probe runs. + - **Operator ruling: "Q3: B".** Peer puts are fsynced before they are acknowledged. Only Odin's own + bookkeeping (heartbeats, correlations) waits for the 1 s interval. An ACK means durable. +- **An undecodable foreign write.** Today Odin stays up, serves from memory and never persists again. Self offered + three options: degraded but loud, end Odin, or unchanged. + - **Operator: "None of these options involve repair and recovery, which is what ultimately needs to happen."** + - Recovery is mapped as its own pass. It covers an undecodable foreign write, a decodable one, and a failed + load at activation. + **Q2 decode result (Eyes, 2026-09-30; full table in the CultLib session scratchpad `odin-topology-decode.md`).** - 77 records; 3,222,550 B of payload in a 3,240,078 B file. - The two "~1.55 MB records" are per-type totals. Each type is dominated by one Ghostlight operator surface: From 2ad60183a1650881ade00537eacb26cfa238cbec Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:14:25 +0200 Subject: [PATCH 09/43] Odin: the self presence counts from its own launch; the stored-sequence continuation is gone Every launch is issued a fresh activation (Idunn control_plane.rs, the continuity demotion), presences are ordered only within one activation (select_presence), and Idunn compares only the warming presence's digest. Continuing from a stored sequence protected nothing. Co-Authored-By: Claude Opus 5.5 --- crates/odin-daemon/src/lib.rs | 3 +- crates/odin-daemon/src/main.rs | 78 ++++++++++------------------------ 2 files changed, 23 insertions(+), 58 deletions(-) diff --git a/crates/odin-daemon/src/lib.rs b/crates/odin-daemon/src/lib.rs index 53479ab..beec1bc 100644 --- a/crates/odin-daemon/src/lib.rs +++ b/crates/odin-daemon/src/lib.rs @@ -580,8 +580,7 @@ impl MemoryOdinTopologyStore { /// Remove every correlation not keyed by an incarnation. Those were written /// by the previous, target-keyed Odin; nothing reads them under this /// contract, and left in place the catalog would serve them to the Verse - /// as current. Presence records are kept whatever their key: the self - /// publisher sequence is continued from them. + /// as current. pub fn retire_legacy_correlations(&self) -> usize { let legacy: Vec = self .records diff --git a/crates/odin-daemon/src/main.rs b/crates/odin-daemon/src/main.rs index ff75b2c..c9105d3 100644 --- a/crates/odin-daemon/src/main.rs +++ b/crates/odin-daemon/src/main.rs @@ -145,6 +145,8 @@ struct RuntimeState { write_lease: Option, write_lease_path: PathBuf, recent_warming_proofs: VecDeque<(String, u64)>, + /// Counts from this launch. Every launch is a fresh activation, and + /// presences are ordered only within one activation. publisher_sequence: u64, log_gate: RefCell, } @@ -268,13 +270,8 @@ impl RuntimeState { let store = MemoryOdinTopologyStore::load(&self.options.store)?; // Correlations the previous, target-keyed Odin left in this store are // not this contract's and would otherwise be served to the Verse as - // current. Presence history is kept: the self publisher sequence - // continues from it. + // current. store.retire_legacy_correlations(); - let stored_sequence = prior_self_publisher_sequence( - store.records().values(), - &self.authority_material.provider_signer.entry().identity_id, - ); let signer = self .topology_signer .take() @@ -283,7 +280,6 @@ impl RuntimeState { .idunn_anchor .take() .context("Idunn trust anchor was already consumed")?; - self.publisher_sequence = self.publisher_sequence.max(stored_sequence); self.write_lease = Some(lease); self.topology = Some(Topology { store, @@ -1242,38 +1238,6 @@ fn read_process_write_lease(path: &Path) -> Result( - records: impl IntoIterator, - signer_identity_id: &str, -) -> u64 { - records - .into_iter() - .filter(|entry| { - entry.r#type == GameCultRuntimePresenceHealthRecord::TYPE - && entry.key.starts_with(TARGET) - }) - .filter_map(|entry| match decode_presence(&entry.payload) { - Ok(presence) => Some(presence), - Err(error) => { - eprintln!( - "Odin startup skipped its stored presence {}: {error:#}", - entry.key - ); - None - } - }) - .filter(|presence| { - presence.target == TARGET && presence.signer_identity_id == signer_identity_id - }) - .map(|presence| presence.publisher_sequence) - .max() - .unwrap_or(0) -} - /// The document namespaces providers publish to the rendezvous. A provider's /// document is stored and served back only under one of these; Odin's own /// records (correlations, the publisher watermark, every `odin.*` type) and @@ -1916,12 +1880,22 @@ mod tests { .collect()) } - /// Odin's own publisher sequence as its store holds it. + /// The publisher sequence of Odin's own admitted presence, 0 when none + /// is held. fn stored_sequence(&self) -> Result { - Ok(prior_self_publisher_sequence( - &self.working_set(), - &self.provider_identity_id, - )) + let key = format!( + "{}/{}", + self_incarnation(&self.state.borrow().authority_material).key(), + self.provider_identity_id + ); + self.working_set() + .into_iter() + .find(|entry| { + entry.r#type == GameCultRuntimePresenceHealthRecord::TYPE && entry.key == key + }) + .map_or(Ok(0), |entry| { + Ok(decode_presence(&entry.payload)?.publisher_sequence) + }) } /// Run a peer on its own thread while Odin polls its socket, and nothing @@ -2385,12 +2359,11 @@ mod tests { Ok(()) } - /// One record that does not decode is skipped where it is met: the catalog - /// still serves everything else, and startup still finds Odin's sequence. - /// The records are in the file when Odin activates. + /// One record that does not decode is skipped where it is met: Odin still + /// starts and serves, and the catalog serves everything else. The records + /// are in the file when Odin activates. #[test] fn records_that_do_not_decode_never_stop_the_catalog_or_startup() -> Result<()> { - const EARLIER: u64 = 41; let mut odin = activated_odin_with(CultMeshRudpDocumentServerOptions::default(), |runtime| { let own_signer = runtime .authority_material @@ -2406,10 +2379,7 @@ mod tests { stored_at: stamp.clone(), schema_id: Some(GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()), }; - // A presence an earlier incarnation of Odin stored, counted to 41. - runtime.publisher_sequence = EARLIER - 1; let earlier = runtime.signed_presence_document("warming", "earlier incarnation")?; - runtime.publisher_sequence = 0; Ok(vec![ bad_presence("ghost".into()), bad_presence(format!("{TARGET}@{}/{own_signer}", digest('c'))), @@ -2432,11 +2402,7 @@ mod tests { ); assert_eq!(documents_of(&catalog, "ghostlight.doc.v1"), 1); odin.pass()?; - assert_eq!( - odin.stored_sequence()?, - EARLIER + 1, - "Odin continues from the sequence it found past the undecodable presences" - ); + assert!(odin.stored_sequence()? > 0, "Odin publishes past them"); Ok(()) } From 620efb85d81efab27b7f0ab61560464a10079b86 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:24:32 +0200 Subject: [PATCH 10/43] Odin: a catalog query reads Idunn's projection once The file-backed projection source no longer answers lookups itself, so no caller can read the file once per lookup: every lookup is made against one snapshot. The catalog query takes one per query, as the refresh pass does; startup, activation and the own-projection check take one each. Co-Authored-By: Claude Opus 5.5 --- crates/odin-daemon/src/lib.rs | 52 ++++++++++------------- crates/odin-daemon/src/main.rs | 75 ++++++++++++++++++++++++++++++---- 2 files changed, 88 insertions(+), 39 deletions(-) diff --git a/crates/odin-daemon/src/lib.rs b/crates/odin-daemon/src/lib.rs index beec1bc..67c8e7d 100644 --- a/crates/odin-daemon/src/lib.rs +++ b/crates/odin-daemon/src/lib.rs @@ -6,7 +6,7 @@ use std::cell::{Cell, Ref, RefCell}; use std::collections::{BTreeMap, BTreeSet}; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::time::{SystemTime, UNIX_EPOCH}; use anyhow::{Context, Result, bail, ensure}; @@ -177,7 +177,9 @@ pub trait IdunnProjectionSource { } } -/// Read-only adapter for the atomic projection file published by Idunn. +/// Read-only adapter for the atomic projection file published by Idunn. It +/// answers no lookup itself: each [`snapshot`](Self::snapshot) is one read of +/// the file, and every lookup is made against a snapshot. pub struct CultCacheIdunnProjectionSource { path: PathBuf, } @@ -187,22 +189,6 @@ impl CultCacheIdunnProjectionSource { Self { path: path.into() } } - pub fn path(&self) -> &Path { - &self.path - } -} - -impl IdunnProjectionSource for CultCacheIdunnProjectionSource { - fn projections(&self, target: &str) -> Result> { - self.snapshot()?.projections(target) - } - - fn projection(&self, incarnation: &IncarnationRef) -> Result> { - self.snapshot()?.projection(incarnation) - } -} - -impl CultCacheIdunnProjectionSource { /// Read the projection file once. An absent file projects nothing. pub fn snapshot(&self) -> Result { if !self.path.is_file() { @@ -227,8 +213,7 @@ impl CultCacheIdunnProjectionSource { } /// One reading of Idunn's projection file. Every lookup made against it is -/// answered from the same bytes, so a pass that refreshes every incarnation -/// reads the file once. +/// answered from the same bytes. pub struct IdunnProjectionSnapshot { entries: Vec, } @@ -536,10 +521,6 @@ impl MemoryOdinTopologyStore { }) } - pub fn path(&self) -> &Path { - &self.path - } - /// Every record, in the order the file stores them. pub fn records(&self) -> Ref<'_, BTreeMap> { self.records.borrow() @@ -1704,7 +1685,7 @@ fn parse_rfc3339_millis(value: &str) -> Result { #[cfg(test)] mod tests { - use std::path::PathBuf; + use std::path::{Path, PathBuf}; use cultnet_rs::{ GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA, GameCultProviderHealthIdentity, @@ -1750,7 +1731,9 @@ mod tests { ); } let projections = - CultCacheIdunnProjectionSource::new(&projection_path).projections(&target)?; + CultCacheIdunnProjectionSource::new(&projection_path) + .snapshot()? + .projections(&target)?; if projections.is_empty() { println!("== no incarnation-keyed projection for {target}"); return Ok(()); @@ -1851,10 +1834,12 @@ mod tests { candidate.activation = None; candidate.current_lease = None; world.publish_projection(&candidate)?; + let engine = world.engine(NOW); let candidate_incarnation = IncarnationRef::of(&candidate.expected)?; assert_ne!(incumbent.incarnation()?, candidate_incarnation); assert_eq!( CultCacheIdunnProjectionSource::new(&world.projection_path) + .snapshot()? .projections("odin")? .len(), 2 @@ -1907,7 +1892,7 @@ mod tests { }) .collect(); ensure!(store.compare_exchange_snapshot(¤t, &legacy)?); - let source = CultCacheIdunnProjectionSource::new(&world.projection_path); + let source = CultCacheIdunnProjectionSource::new(&world.projection_path).snapshot()?; assert!(source.projections("ghostlight")?.is_empty()); assert!(source.projection(&service.incarnation()?)?.is_none()); Ok(()) @@ -1930,7 +1915,7 @@ mod tests { schema_id: Some(IDUNN_EXPECTED_INCARNATION_SCHEMA.into()), }); ensure!(store.compare_exchange_snapshot(¤t, &with_ghost)?); - let source = CultCacheIdunnProjectionSource::new(&world.projection_path); + let source = CultCacheIdunnProjectionSource::new(&world.projection_path).snapshot()?; assert!(source.projection(&service.incarnation()?)?.is_some()); assert!( source @@ -2442,10 +2427,13 @@ mod tests { self.engine_on(&self.store, now) } - /// An engine over another store, as a restarted Odin loads one. + /// An engine over another store, as a restarted Odin loads one. It reads + /// the projection once, when it is made, as each serving operation does. fn engine_on<'a>(&'a self, store: &'a MemoryOdinTopologyStore, now: u64) -> TestEngine<'a> { OdinTopologyAuthority::new( - CultCacheIdunnProjectionSource::new(&self.projection_path), + CultCacheIdunnProjectionSource::new(&self.projection_path) + .snapshot() + .expect("the test projection reads"), store, &self.odin_signer, FixedClock(now), @@ -2491,7 +2479,7 @@ mod tests { } type TestEngine<'a> = OdinTopologyAuthority< - CultCacheIdunnProjectionSource, + IdunnProjectionSnapshot, &'a MemoryOdinTopologyStore, &'a ServiceIdentitySigner, FixedClock, @@ -3001,6 +2989,7 @@ mod tests { ); std::fs::remove_file(&world.projection_path)?; + let engine = world.engine(NOW); assert!( engine .current_signed_correlation(&service.incarnation()?)? @@ -3055,6 +3044,7 @@ mod tests { assert!(ready.dependencies[0].provider_evidence_sha256.is_some()); world.replace_provider_generation(&mut provider)?; + let engine = world.engine(NOW); let no_longer_exact = decode_signed(&engine.refresh(&consumer.incarnation()?)?.unwrap())?; assert!(no_longer_exact.present); assert!(!no_longer_exact.ready); diff --git a/crates/odin-daemon/src/main.rs b/crates/odin-daemon/src/main.rs index c9105d3..b1b461c 100644 --- a/crates/odin-daemon/src/main.rs +++ b/crates/odin-daemon/src/main.rs @@ -33,8 +33,8 @@ use cultnet_rs::{ use fs2::FileExt; use odin_daemon::{ AuthenticationPolicy, CultCacheIdunnProjectionSource, ForeignStoreWrite, - IdunnProjectionSource, IncarnationRef, MemoryOdinTopologyStore, OdinTopologyAuthority, - PresenceAuthorityRefused, ProjectionUnreadable, SystemClock, + IdunnProjectionSnapshot, IdunnProjectionSource, IncarnationRef, MemoryOdinTopologyStore, + OdinTopologyAuthority, PresenceAuthorityRefused, ProjectionUnreadable, SystemClock, }; const TARGET: &str = "odin"; @@ -180,7 +180,8 @@ impl RuntimeState { require_expected_contract(&authority_material.expected, candidate)?; let idunn_anchor = read_trust_anchor::(&options.idunn_anchor)?; - let projection_source = CultCacheIdunnProjectionSource::new(&options.idunn_projection); + let projection_source = + CultCacheIdunnProjectionSource::new(&options.idunn_projection).snapshot()?; // This process is one exact incarnation, named by the Expected digest // in its immutable runtime bundle. The projection may carry other // incarnations of `odin` at the same time -- the one being replaced, @@ -258,6 +259,7 @@ impl RuntimeState { return Ok(false); }; let Some(projected) = CultCacheIdunnProjectionSource::new(&self.options.idunn_projection) + .snapshot()? .projection(&self_incarnation(&self.authority_material))? else { return Ok(false); @@ -483,7 +485,8 @@ impl RuntimeState { fn require_own_projection(&self) -> Result<()> { let path = &self.options.idunn_projection; match CultCacheIdunnProjectionSource::new(path) - .projection(&self_incarnation(&self.authority_material)) + .snapshot() + .and_then(|projections| projections.projection(&self_incarnation(&self.authority_material))) { Err(error) => Err(own_projection_failure(error)), // No file is an I/O condition (Idunn replaces it atomically, so it @@ -546,8 +549,9 @@ impl RuntimeState { } /// The catalog serves what is stored, one record at a time: a record that - /// does not decode, or whose projection cannot be read, is skipped and - /// logged, and is never a reason to refuse every other record. Odin's + /// does not decode, or whose incarnation's projection does not, is skipped + /// and logged, and is never a reason to refuse every other record. Idunn's + /// projection is read once per query. Odin's /// correlations are not part of the catalog: Idunn reads them from the /// store file, and a target has one per incarnation, so keying them by /// target would collide for the whole deploy window. @@ -556,7 +560,8 @@ impl RuntimeState { query: &CultMeshRudpSnapshotQuery, ) -> Result> { let records = self.topology()?.store.records(); - let projections = CultCacheIdunnProjectionSource::new(&self.options.idunn_projection); + let projections = + CultCacheIdunnProjectionSource::new(&self.options.idunn_projection).snapshot()?; let mut selected = BTreeMap::new(); for envelope in records.values() { let document = match self.public_document(&projections, envelope) { @@ -585,7 +590,7 @@ impl RuntimeState { /// One stored record as the Verse sees it; `None` when it is not public. fn public_document( &self, - projections: &CultCacheIdunnProjectionSource, + projections: &IdunnProjectionSnapshot, envelope: &CultCacheEnvelope, ) -> Result> { let Some(schema_id) = envelope.schema_id.clone() else { @@ -2406,6 +2411,60 @@ mod tests { Ok(()) } + /// The bytes this thread has read so far, as the kernel counts them. + fn thread_bytes_read() -> Result { + let io = std::fs::read_to_string("/proc/thread-self/io")?; + Ok(io + .lines() + .find_map(|line| line.strip_prefix("rchar: ")) + .context("no rchar in /proc/thread-self/io")? + .parse()?) + } + + /// A catalog query reads Idunn's projection once, however many stored + /// presences it has to look up there. + #[test] + fn a_catalog_query_reads_the_projection_once() -> Result<()> { + const PRESENCES: usize = 8; + let odin = activated_odin_with(CultMeshRudpDocumentServerOptions::default(), |runtime| { + let stamp = rfc3339_millis(unix_millis()?)?; + (0..PRESENCES) + .map(|index| { + Ok(CultCacheEnvelope { + key: format!("stored-{index}"), + r#type: GameCultRuntimePresenceHealthRecord::TYPE.into(), + payload: runtime.signed_presence_document("warming", "stored")?.payload, + stored_at: stamp.clone(), + schema_id: Some(GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()), + }) + }) + .collect() + })?; + let projection = odin.state.borrow().options.idunn_projection.clone(); + let projection_bytes = std::fs::metadata(&projection)?.len(); + // Every stored presence is looked up in the projection; the filter + // then keeps none of them, so none collides with another in the reply. + let query = CultMeshRudpSnapshotQuery { + session: cultmesh_rs::CultMeshRudpSessionKey { + remote_addr: "127.0.0.1:1".parse()?, + connection_id: 7, + }, + message_id: "lookups".into(), + requested_at_unix_millis: 1, + schema_ids: None, + record_keys: Some(vec!["absent".into()]), + }; + let before = thread_bytes_read()?; + assert!(odin.state.borrow_mut().raw_snapshot(&query)?.is_empty()); + let read = thread_bytes_read()? - before; + assert!(read >= projection_bytes, "the projection was read: {read} bytes"); + assert!( + read < 2 * projection_bytes, + "and only once: {read} bytes read, the projection is {projection_bytes}" + ); + Ok(()) + } + /// A presence whose incarnation's projection cannot be read is skipped as /// that record's failure; the rest of the catalog is still served. #[test] From a4aa5fcf975ff0471311e8e1bafbefd284702b66 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:26:25 +0200 Subject: [PATCH 11/43] Odin: a failed write is retried as it stands, and its own replaced file is not a foreign write The records stay dirty until a write lands, so the next write makes the failed change without another mutation (tested with no heartbeat to re-dirty them). A write that fails after it replaced the file (the directory sync) leaves Odin's own records there; they are remembered as the write's attempt and accepted as a second baseline until a write lands. Co-Authored-By: Claude Opus 5.5 --- crates/odin-daemon/src/lib.rs | 94 ++++++++++++++++++++++++++++++++--- 1 file changed, 87 insertions(+), 7 deletions(-) diff --git a/crates/odin-daemon/src/lib.rs b/crates/odin-daemon/src/lib.rs index 67c8e7d..66ade96 100644 --- a/crates/odin-daemon/src/lib.rs +++ b/crates/odin-daemon/src/lib.rs @@ -492,6 +492,10 @@ pub struct MemoryOdinTopologyStore { records: RefCell>, /// Exactly what the file held when it was loaded or last written. flushed: RefCell>, + /// What the last write tried to write, when it failed. A write can fail + /// after the file was replaced (syncing the directory), and then the file + /// holds these records: Odin's own, not another writer's. + attempted: RefCell>>, /// The records differ from what was last loaded or written. dirty: Cell, } @@ -517,6 +521,7 @@ impl MemoryOdinTopologyStore { path, records: RefCell::new(records), flushed: RefCell::new(flushed), + attempted: RefCell::new(None), dirty: Cell::new(false), }) } @@ -587,20 +592,33 @@ impl MemoryOdinTopologyStore { /// Write every record to the file as one atomic snapshot, if anything /// changed since the last write; returns whether it wrote. /// - /// The file must still hold exactly what Odin last loaded or wrote. Anything - /// else was written by another process, and the write is refused with - /// [`ForeignStoreWrite`] rather than made over it. + /// The file must still hold exactly what Odin last loaded or wrote, or what + /// a failed write last tried to write. Anything else was written by another + /// process, and the write is refused with [`ForeignStoreWrite`] rather than + /// made over it. A failed write leaves the records dirty, so the next + /// write retries it. pub fn flush(&self) -> Result { if !self.dirty.get() { return Ok(false); } let current: Vec = self.records.borrow().values().cloned().collect(); - if !SingleFileMessagePackBackingStore::new(&self.path) - .compare_exchange_snapshot(&self.flushed.borrow(), ¤t)? - { - return Err(ForeignStoreWrite.into()); + let file = SingleFileMessagePackBackingStore::new(&self.path); + let exchanged = file + .compare_exchange_snapshot(&self.flushed.borrow(), ¤t) + .and_then(|exchanged| match &*self.attempted.borrow() { + Some(attempted) if !exchanged => file.compare_exchange_snapshot(attempted, ¤t), + _ => Ok(exchanged), + }); + match exchanged { + Ok(true) => {} + Ok(false) => return Err(ForeignStoreWrite.into()), + Err(error) => { + *self.attempted.borrow_mut() = Some(current); + return Err(error); + } } *self.flushed.borrow_mut() = current; + *self.attempted.borrow_mut() = None; self.dirty.set(false); Ok(true) } @@ -2068,6 +2086,68 @@ mod tests { Ok(()) } + fn peer_document(key: &str) -> Result { + Ok(CultCacheEnvelope { + key: key.into(), + r#type: "ghostlight.doc".into(), + payload: vec![1], + stored_at: rfc3339_millis(NOW)?, + schema_id: Some("ghostlight.doc.v1".into()), + }) + } + + /// A write that fails leaves its change to be written: the next write + /// lands it with nothing changed in between. + #[test] + fn a_failed_write_is_made_by_the_next_without_another_change() -> Result<()> { + let temp = TempDir::new()?; + let path = temp.path().join("topology.cc"); + let store = MemoryOdinTopologyStore::load(&path)?; + store.put(peer_document("doc-1")?); + std::fs::write(&path, b"not a cultcache store")?; + assert!(store.flush().is_err(), "the injected fault fails the write"); + + // The fault clears: the file holds what Odin loaded, which was nothing. + std::fs::remove_file(&path)?; + assert!(store.flush()?, "the failed change is written"); + assert_eq!( + SingleFileMessagePackBackingStore::new(&path).pull_all_read_only_snapshot()?, + vec![peer_document("doc-1")?] + ); + assert!(!store.flush()?, "and nothing is left to write"); + Ok(()) + } + + /// A write can fail after it replaced the file (the directory sync), and + /// the file then holds Odin's own records: the next write is made over + /// them. Once a write lands, those records are no longer Odin's to expect. + #[test] + fn a_write_that_failed_after_replacing_the_file_is_not_another_writer() -> Result<()> { + let temp = TempDir::new()?; + let path = temp.path().join("topology.cc"); + let file = SingleFileMessagePackBackingStore::new(&path); + let store = MemoryOdinTopologyStore::load(&path)?; + store.put(peer_document("doc-1")?); + std::fs::write(&path, b"not a cultcache store")?; + assert!(store.flush().is_err()); + // The state a failed directory sync leaves: the replace landed. + std::fs::remove_file(&path)?; + ensure!(file.compare_exchange_snapshot(&[], &[peer_document("doc-1")?])?); + + store.put(peer_document("doc-2")?); + assert!(store.flush()?, "Odin's own records are written over"); + let written = file.pull_all_read_only_snapshot()?; + assert_eq!(written, vec![peer_document("doc-1")?, peer_document("doc-2")?]); + + // Another writer puts back what the failed write had tried to write. + ensure!(file.compare_exchange_snapshot(&written, &[peer_document("doc-1")?])?); + store.put(peer_document("doc-3")?); + let error = store.flush().unwrap_err(); + assert!(error.downcast_ref::().is_some(), "{error:#}"); + assert_eq!(file.pull_all_read_only_snapshot()?, vec![peer_document("doc-1")?]); + Ok(()) + } + fn file_sequences(path: &Path) -> Result<(u64, Vec)> { let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; let mark = unique_envelope(&entries, TOPOLOGY_PUBLISHER_WATERMARK_TYPE, "ghostlight")? From 3afa3fb778a017a3ad7311134817e58bbf68a9a6 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:27:53 +0200 Subject: [PATCH 12/43] Odin: every way out of the serving loop makes the last write The loop is `serve`, and it ends in one place: a stop request, a condition `survive` returns, or a dead socket all reach `stop`, whose write is made only while the lease is current and the file is still Odin's. Tested for each exit kind: a stop request, a refused own authority and a dead socket write what changed; a lost lease and a foreign write write nothing. Co-Authored-By: Claude Opus 5.5 --- crates/odin-daemon/src/main.rs | 151 +++++++++++++++++++++++++++------ 1 file changed, 125 insertions(+), 26 deletions(-) diff --git a/crates/odin-daemon/src/main.rs b/crates/odin-daemon/src/main.rs index b1b461c..749c37c 100644 --- a/crates/odin-daemon/src/main.rs +++ b/crates/odin-daemon/src/main.rs @@ -775,23 +775,37 @@ fn main() -> Result<()> { } } - // Once serving, only losing the write lease ends Odin (see `survive`). - let mut timers = ServingTimers::default(); - loop { - if stopping.load(Ordering::Relaxed) { - eprintln!("Odin stopping on request"); - stop(&state.borrow()); - return Ok(()); - } - if !serving_pass(&state, &mut server, &mut timers)? { - thread::sleep(IDLE_POLL_INTERVAL); - } - } + serve(&state, &mut server, &mut ServingTimers::default(), &stopping) } -/// The last write before Odin stops on request. `flush` makes it only while -/// the lease is still current. A failure is logged and not returned: Odin is -/// stopping either way, and the next Odin loads what the file holds. +/// Serve until stopped on request or ended by a condition `survive` returns +/// (or a dead socket). Every way out makes the last write (`stop`). +fn serve( + state: &Rc>, + server: &mut OdinServer, + timers: &mut ServingTimers, + stopping: &AtomicBool, +) -> Result<()> { + let ended = loop { + if stopping.load(Ordering::Relaxed) { + eprintln!("Odin stopping on request"); + break Ok(()); + } + match serving_pass(state, server, timers) { + Ok(true) => {} + Ok(false) => thread::sleep(IDLE_POLL_INTERVAL), + Err(error) => break Err(error), + } + }; + stop(&state.borrow()); + ended +} + +/// The last write before Odin ends. `flush` makes it only while the lease is +/// still current and the file still holds what Odin wrote, so it writes +/// nothing after `WriteLeaseLost` or `ForeignStoreWrite`. A failure is logged +/// and not returned: Odin is ending either way, and the next Odin loads what +/// the file holds. fn stop(state: &RuntimeState) { if let Err(error) = state.flush() { eprintln!("Odin's final store write was not made: {error:#}"); @@ -1564,6 +1578,8 @@ mod tests { _temp: tempfile::TempDir, state: Rc>, server: OdinServer, + /// The server's own socket, shared: what is done to it is done to the server. + socket: UdpSocket, timers: ServingTimers, store: PathBuf, lease_path: PathBuf, @@ -1775,6 +1791,7 @@ mod tests { "Odin did not activate in the fixture" ); let state = Rc::new(RefCell::new(runtime)); + let server_socket = socket.try_clone()?; let server = CultMeshRudpDocumentServer::new( socket, SinkHandle(state.clone()), @@ -1786,6 +1803,7 @@ mod tests { _temp: temp, state, server, + socket: server_socket, timers: ServingTimers::default(), store, lease_path, @@ -1799,6 +1817,22 @@ mod tests { serving_pass(&self.state, &mut self.server, &mut self.timers) } + /// Serve as `main` does, with a stop already requested or not. + fn serve(&mut self, stop_requested: bool) -> Result<()> { + serve( + &self.state, + &mut self.server, + &mut self.timers, + &AtomicBool::new(stop_requested), + ) + } + + /// One heartbeat of Odin's own: a change to its bookkeeping, which is + /// written on the interval and not at once. + fn heartbeat(&self) -> Result<()> { + self.state.borrow_mut().publish_self_presence("test heartbeat") + } + /// Replace the lease under Odin with a later epoch. Odin holds a shared /// lock on the lease's lifetime lock, which is what stops Idunn from /// writing the lease under it; the test swaps the file underneath. @@ -3139,25 +3173,90 @@ mod tests { Ok(()) } - /// Stopping on request writes what changed, while the lease is current, - /// and nothing once it is lost. - #[test] - fn stopping_writes_what_changed_only_under_the_lease() -> Result<()> { - let odin = activated_odin()?; - provider_put(&odin, "ghostlight.doc.v1", "doc-1", vec![1])?; - stop(&odin.state.borrow()); - assert_eq!(odin.file_records()?, odin.working_set()); - + /// Odin serving, written once, with a change of its own not yet written. + fn odin_with_an_unwritten_change() -> Result { let mut odin = activated_odin()?; odin.pass()?; + odin.heartbeat()?; + assert_ne!(odin.file_records()?, odin.working_set()); + Ok(odin) + } + + /// Stopping on request makes the last write. + #[test] + fn a_stop_request_writes_what_changed() -> Result<()> { + let mut odin = odin_with_an_unwritten_change()?; + odin.serve(true)?; + assert_eq!(odin.file_records()?, odin.working_set()); + Ok(()) + } + + /// Ending because Odin's own authority is refused, the lease is still + /// current: the last write is made. + #[test] + fn ending_on_a_refused_authority_writes_what_changed() -> Result<()> { + let mut odin = odin_with_an_unwritten_change()?; + odin.tamper_projection(|entries| { + entries + .into_iter() + .filter(|entry| entry.r#type != IdunnRuntimeActivationRecord::TYPE) + .collect() + })?; + odin.timers = ServingTimers::default(); + assert_refused(&odin.serve(false).unwrap_err()); + assert_eq!(odin.file_records()?, odin.working_set()); + Ok(()) + } + + /// Ending on a dead socket, the lease is still current: the last write is + /// made. The socket's own failure is real (a refused datagram, reported on + /// the server's next receive); only the length of the failing run is set. + #[test] + fn ending_on_a_dead_socket_writes_what_changed() -> Result<()> { + let mut odin = odin_with_an_unwritten_change()?; + let closed = UdpSocket::bind("127.0.0.1:0")?.local_addr()?; + odin.socket.connect(closed)?; + odin.socket.send(b"to nobody")?; + thread::sleep(Duration::from_millis(50)); + odin.timers.poll_failing_since = ago(POLL_FAILURE_LIMIT.as_millis() as u64 + 1_000); + let error = odin.serve(false).unwrap_err(); + assert!(format!("{error:#}").contains("failed every poll"), "{error:#}"); + assert_eq!(odin.file_records()?, odin.working_set()); + Ok(()) + } + + /// Ending because the lease was lost, nothing more is written. + #[test] + fn ending_on_a_lost_lease_writes_nothing() -> Result<()> { + let mut odin = odin_with_an_unwritten_change()?; let written = file_identity(&odin.store)?; - provider_put(&odin, "ghostlight.doc.v1", "doc-1", vec![1])?; odin.swap_lease()?; - stop(&odin.state.borrow()); + odin.timers = ServingTimers::default(); + let error = odin.serve(false).unwrap_err(); + assert!(error.downcast_ref::().is_some(), "{error:#}"); assert_eq!(file_identity(&odin.store)?, written); Ok(()) } + /// Ending because another process wrote the file, nothing is written over + /// it. + #[test] + fn ending_on_a_foreign_write_writes_nothing() -> Result<()> { + let mut odin = odin_with_an_unwritten_change()?; + odin.tamper_store(|entries| { + entries + .into_iter() + .filter(|entry| entry.r#type != GameCultRuntimePresenceHealthRecord::TYPE) + .collect() + })?; + let foreign = file_identity(&odin.store)?; + odin.timers.last_flush = None; + let error = odin.serve(false).unwrap_err(); + assert!(error.downcast_ref::().is_some(), "{error:#}"); + assert_eq!(file_identity(&odin.store)?, foreign); + Ok(()) + } + // ---- the store file is not working memory ----------------------------- /// Once Odin holds its lease, its working set is in memory. Rewriting the From 0a09df9b7cb858b07a7bbbfaf12372d6e85cfcbc Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:30:28 +0200 Subject: [PATCH 13/43] Odin: a peer's put is written before it is acknowledged (Q3 B) Operator ruling Q3 B: `accept_raw_document` returns Ok, and so the server acknowledges the put, only once the store holding it is on disk. The write is the one `flush`, under the same lease check, so it also carries every other unwritten change. A write that is not made refuses the put. Odin's own bookkeeping stays on the one-second interval. The interval tests now change Odin's own bookkeeping (its heartbeat) instead of putting, since a put is no longer an interval change. Co-Authored-By: Claude Opus 5.5 --- crates/odin-daemon/src/main.rs | 74 ++++++++++++++++++++++++++++------ 1 file changed, 61 insertions(+), 13 deletions(-) diff --git a/crates/odin-daemon/src/main.rs b/crates/odin-daemon/src/main.rs index 749c37c..ccba290 100644 --- a/crates/odin-daemon/src/main.rs +++ b/crates/odin-daemon/src/main.rs @@ -68,10 +68,11 @@ const POLL_FAILURE_LIMIT: Duration = Duration::from_secs(30); const POLL_FAILURE_BACKOFF: Duration = Duration::from_millis(100); const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(5); const PROJECTION_REFRESH_INTERVAL: Duration = Duration::from_millis(250); -// At most one write of the store file per interval, and none when nothing -// changed. A crash loses at most this much of the catalog and presence; -// providers republish, and a correlation sequence that was never written was -// never published. Operator default Q3, docs/write-pattern-cut.md. +// Odin's own changes (its heartbeat, correlations, the watermark) are written +// at most once per interval, and not at all when nothing changed. A crash loses +// at most this much of them; a correlation sequence that was never written was +// never published. A peer's put is written before it is acknowledged +// (`accept_raw_document`). Operator ruling Q3 B, docs/write-pattern-cut.md. const FLUSH_INTERVAL: Duration = Duration::from_secs(1); const IDLE_POLL_INTERVAL: Duration = Duration::from_millis(2); const MAX_RECENT_WARMING_PROOFS: usize = 64; @@ -298,7 +299,8 @@ impl RuntimeState { } /// Write Odin's store file from the working set, if it changed; returns - /// whether it wrote. This is the only write of the file. The lease is + /// whether it wrote. This is the only write of the file, made on the + /// interval, for a peer's put, and on the way out. The lease is /// checked immediately before it, so nothing is written once the lease is /// lost, and a file another process wrote is refused (`ForeignStoreWrite`) /// rather than written over. Both end Odin (see `survive`). @@ -311,6 +313,11 @@ impl RuntimeState { store.flush() } + /// A peer's put is written before it is acknowledged: the server sends the + /// acknowledgement only when this returns `Ok`, and it returns `Ok` only + /// once the store holding the put is on disk. The write is the one `flush`, + /// so it carries everything else that changed too. A write that fails + /// refuses the put. fn accept_raw_document(&mut self, receipt: CultMeshRudpRawDocumentReceipt) -> Result<()> { ensure!( self.activated(), @@ -322,6 +329,7 @@ impl RuntimeState { } else { persist_generic_document(&self.topology()?.store, &receipt.document)?; } + self.flush()?; Ok(()) } @@ -2055,7 +2063,10 @@ mod tests { let written = std::fs::read(&odin.store)?; std::fs::write(&odin.store, b"not a cultcache store")?; - provider_put(&odin, "ghostlight.doc.v1", "doc-1", vec![1])?; + assert!( + provider_put(&odin, "ghostlight.doc.v1", "doc-1", vec![1]).is_err(), + "a put whose write fails is refused" + ); assert!( odin.state.borrow().flush().is_err(), "the injected fault must actually fail the write" @@ -3045,15 +3056,15 @@ mod tests { Instant::now().checked_sub(Duration::from_millis(millis)) } - /// Fifty changes inside one interval are one write, made once the - /// interval has passed: not at 0.9 s after the last write, at 1.1 s. + /// Twenty-five of Odin's own changes (heartbeats) inside one interval are + /// one write, made once the interval has passed: not at 0.9 s after the + /// last write, at 1.1 s. #[test] fn changes_inside_one_interval_are_one_write() -> Result<()> { let mut odin = activated_odin()?; odin.pass()?; let first = file_identity(&odin.store)?; - for index in 0..25 { - provider_put(&odin, "ghostlight.doc.v1", &format!("doc-{index}"), vec![1])?; + for _ in 0..25 { odin.timers.last_heartbeat = None; odin.timers.last_flush = Some(Instant::now()); odin.pass()?; @@ -3073,12 +3084,49 @@ mod tests { // The write itself starts the next interval: a change right after it // waits. - provider_put(&odin, "ghostlight.doc.v1", "doc-late", vec![1])?; + odin.heartbeat()?; odin.pass()?; assert_eq!(file_identity(&odin.store)?, second, "and only one"); Ok(()) } + /// A peer's put is on disk when Odin accepts it, inside the interval, and + /// the write carries Odin's own unwritten changes with it. + #[test] + fn a_put_is_written_before_it_is_accepted() -> Result<()> { + let mut odin = activated_odin()?; + odin.pass()?; + odin.heartbeat()?; + odin.timers.last_flush = Some(Instant::now()); + provider_put(&odin, "ghostlight.doc.v1", "doc-1", vec![1])?; + assert!( + odin.file_records()? + .iter() + .any(|entry| entry.r#type == "ghostlight.doc" && entry.key == "doc-1") + ); + assert_eq!(odin.file_records()?, odin.working_set()); + Ok(()) + } + + /// A put whose write is not made is refused, so it is never acknowledged: + /// with the write failing, and with the lease lost. + #[test] + fn a_put_whose_write_is_not_made_is_refused() -> Result<()> { + let mut odin = activated_odin()?; + odin.pass()?; + std::fs::write(&odin.store, b"not a cultcache store")?; + assert!(provider_put(&odin, "ghostlight.doc.v1", "doc-1", vec![1]).is_err()); + + let mut odin = activated_odin()?; + odin.pass()?; + let written = file_identity(&odin.store)?; + odin.swap_lease()?; + let error = provider_put(&odin, "ghostlight.doc.v1", "doc-1", vec![1]).unwrap_err(); + assert!(error.downcast_ref::().is_some(), "{error:#}"); + assert_eq!(file_identity(&odin.store)?, written); + Ok(()) + } + /// A correlation for an incarnation Idunn no longer projects is found /// through the correlations Odin holds, and the refresh pass withdraws it. #[test] @@ -3135,7 +3183,7 @@ mod tests { let mut odin = activated_odin()?; odin.pass()?; let written = file_identity(&odin.store)?; - provider_put(&odin, "ghostlight.doc.v1", "doc-1", vec![1])?; + odin.heartbeat()?; odin.swap_lease()?; odin.timers.last_projection_refresh = Some(Instant::now()); odin.timers.last_heartbeat = Some(Instant::now()); @@ -3162,7 +3210,7 @@ mod tests { .collect() })?; let foreign = file_identity(&odin.store)?; - provider_put(&odin, "ghostlight.doc.v1", "doc-1", vec![1])?; + odin.heartbeat()?; odin.timers.last_flush = None; let error = odin.pass().unwrap_err(); assert!( From c94b5c4f851eecc92afb16e2a2930321639a70af Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:30:32 +0200 Subject: [PATCH 14/43] Odin tests: the real binary; an acknowledged put survives SIGKILL, SIGTERM stops it cleanly From Soul's SIGTERM probe (scratch 5c379dd), made a test. It runs the built odin-daemon under an Idunn-shaped world with the warming probe and lease grant made over RUDP. With puts written before they are acknowledged, the SIGKILL round is the rule's test; the SIGTERM case pins `main`'s signal wiring, which no in-process test reaches. Co-Authored-By: Claude Opus 5.5 --- crates/odin-daemon/tests/daemon_process.rs | 350 +++++++++++++++++++++ 1 file changed, 350 insertions(+) create mode 100644 crates/odin-daemon/tests/daemon_process.rs diff --git a/crates/odin-daemon/tests/daemon_process.rs b/crates/odin-daemon/tests/daemon_process.rs new file mode 100644 index 0000000..e0bb98a --- /dev/null +++ b/crates/odin-daemon/tests/daemon_process.rs @@ -0,0 +1,350 @@ +//! The real odin-daemon binary, run under an Idunn-shaped world: its bundle, +//! signer descriptors, projection and write lease on disk, and Idunn's warming +//! probe and lease grant made over RUDP as Idunn makes them. It reaches what +//! the in-process tests cannot: `main`'s signal wiring and a process that dies. +//! Unix only, like the daemon. + +use std::fs::File; +use std::net::{SocketAddr, UdpSocket}; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, ExitStatus, Stdio}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use anyhow::{Result, bail, ensure}; +use cultcache_rs::{CacheBackingStore, CultCacheEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore}; +use cultmesh_rs::{ + CultMeshRudpDocumentPublishOptions, CultMeshRudpSnapshotOptions, + publish_cultnet_message_to_rudp_catalog, request_raw_snapshot_from_rudp_catalog, +}; +use cultnet_rs::*; +use odin_daemon::IncarnationRef; + +fn digest(byte: char) -> String { + format!("sha256-{}", byte.to_string().repeat(64)) +} + +fn now() -> u64 { + SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis() as u64 +} + +fn rfc3339(millis: u64) -> String { + chrono::DateTime::::from_timestamp_millis(millis as i64) + .unwrap() + .to_rfc3339() +} + +/// Replace the whole store at `path` with `entries`. +fn write_store(path: &Path, entries: &[CultCacheEnvelope]) -> Result<()> { + let store = SingleFileMessagePackBackingStore::new(path); + let current = if path.is_file() { + store.pull_all_read_only_snapshot()? + } else { + Vec::new() + }; + ensure!(store.compare_exchange_snapshot(¤t, entries)?, "store write lost a race"); + Ok(()) +} + +fn credential_bytes(store: &Path) -> Result> { + Ok(rmp_serde::to_vec(&SingleFileMessagePackBackingStore::new(store).pull_all()?)?) +} + +struct Odin { + _temp: tempfile::TempDir, + root: PathBuf, + candidate: SocketAddr, + child: Child, + store: PathBuf, +} + +impl Odin { + fn log(&self) -> String { + std::fs::read_to_string(self.root.join("odin.log")).unwrap_or_default() + } + + /// Put one provider document and return once the server acknowledged it. + fn put(&self, key: &str) -> Result<()> { + let message = CultNetMessage::DocumentPutRaw { + message_id: format!("put-{key}"), + document: CultNetRawDocumentRecord { + schema_id: "ghostlight.doc.v1".into(), + record_key: key.into(), + stored_at: rfc3339(now()), + payload_encoding: CultNetRawPayloadEncoding::Messagepack, + payload: vec![1], + source_runtime_id: None, + source_agent_id: None, + source_role: None, + tags: None, + }, + }; + let mut options = CultMeshRudpDocumentPublishOptions::odin(self.candidate, "test-publisher"); + options.flush_timeout = Duration::from_secs(3); + publish_cultnet_message_to_rudp_catalog(&message, options) + } + + fn holds(&self, key: &str) -> Result { + Ok(SingleFileMessagePackBackingStore::new(&self.store) + .pull_all_read_only_snapshot()? + .iter() + .any(|entry| entry.r#type == "ghostlight.doc" && entry.key == key)) + } + + fn signal(&mut self, signal: &str) -> Result { + let pid = self.child.id().to_string(); + ensure!(Command::new("kill").args([signal, &pid]).status()?.success()); + let deadline = Instant::now() + Duration::from_secs(20); + loop { + if let Some(status) = self.child.try_wait()? { + return Ok(status); + } + ensure!(Instant::now() < deadline, "Odin did not exit on {signal}"); + std::thread::sleep(Duration::from_millis(10)); + } + } +} + +impl Drop for Odin { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +/// Launch Odin as Idunn does and grant it its lease; returns once Odin has +/// written its store, which it does on its first pass after activating. +fn start_odin() -> Result { + let temp = tempfile::tempdir()?; + let root = temp.path().to_path_buf(); + let candidate = UdpSocket::bind("127.0.0.1:0")?.local_addr()?; + let t0 = now(); + let idunn = enroll_service_identity_at::(&root.join("idunn.cc"))?; + enroll_service_identity_at::(&root.join("odin-topology.cc"))?; + let provider = + enroll_service_identity_at::(&root.join("odin-provider.cc"))?; + std::fs::write(root.join("topology-cred"), credential_bytes(&root.join("odin-topology.cc"))?)?; + std::fs::write(root.join("provider-cred"), credential_bytes(&root.join("odin-provider.cc"))?)?; + export_service_identity_trust_anchor::(&idunn, &root.join("idunn-anchor.cc"))?; + let expected = IdunnExpectedIncarnationRecord { + schema_version: IDUNN_EXPECTED_INCARNATION_SCHEMA.into(), + target: "odin".into(), + plan_id: digest('1'), + incarnation_id: "odin/generation-1".into(), + sealed_release_id: digest('2'), + source_repository: "github.com/GameCult/Odin".into(), + source_revision: "3".repeat(40), + recipe_sha256: digest('4'), + runtime_id: "odin-runtime".into(), + expected_signer_identity_id: provider.entry().identity_id.clone(), + health_contract: "odin.runtime-health.v1".into(), + artifact_sha256: digest('5'), + state_schema_generation: Some("odin-v2".into()), + state_contract_sha256: Some( + "sha256-4f2f2dcd931d16f6b02bf295f41227b867aa982765661d55fa9f29fb2db7e449".into(), + ), + write_lease_required: true, + route: Some(IdunnExpectedRoute { + route_id: "odin-route".into(), + transport: "rudp".into(), + stable_endpoint: "rudp://odin.internal:1000".into(), + candidate_endpoint: format!("rudp://{candidate}"), + }), + capabilities: vec![IdunnExpectedCapability { + capability: "odin.verse-rendezvous".into(), + schema: "odin.verse-topology.v1".into(), + compatibility: "v1".into(), + minimum_capacity: 1, + }], + dependencies: Vec::new(), + }; + expected.validate()?; + let launch = IdunnRuntimeActivationLaunch::issue(&expected, digest('7'), t0 - 20, &idunn)?; + let activation = launch.activation().clone(); + let mut activation_credential = Vec::new(); + launch.write_credential(&mut activation_credential)?; + std::fs::write(root.join("activation-cred"), &activation_credential)?; + + let expected_envelope = |key: &str| -> Result { + Ok(CultCacheEnvelope { + key: key.into(), + r#type: IdunnExpectedIncarnationRecord::TYPE.into(), + payload: expected.canonical_bytes()?, + stored_at: rfc3339(t0 - 30), + schema_id: Some(IDUNN_EXPECTED_INCARNATION_SCHEMA.into()), + }) + }; + let activation_envelope = |key: &str| -> Result { + Ok(CultCacheEnvelope { + key: key.into(), + r#type: IdunnRuntimeActivationRecord::TYPE.into(), + payload: activation.canonical_bytes()?, + stored_at: rfc3339(activation.issued_at_unix_millis), + schema_id: Some(IDUNN_RUNTIME_ACTIVATION_SCHEMA.into()), + }) + }; + let bundle = root.join("bundle"); + std::fs::create_dir_all(&bundle)?; + write_store(&bundle.join("expected.cc"), &[expected_envelope("odin")?])?; + write_store(&bundle.join("activation.cc"), &[activation_envelope("odin")?])?; + + let anchor = GameCultServiceTrustAnchorRecord { + schema_version: GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA.into(), + trust_anchor_id: "root/odin/runtime-presence".into(), + service_id: "odin".into(), + runtime_id: expected.runtime_id.clone(), + signer_identity_id: provider.entry().identity_id.clone(), + signer_public_key: provider.entry().public_key.clone(), + signature_algorithm: "ed25519".into(), + signing_purpose: GAMECULT_RUNTIME_PRESENCE_HEALTH_SIGNING_PURPOSE.into(), + signed_schema: GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + binding_authority: "root".into(), + bound_at_unix_millis: t0 - 100, + expires_at_unix_millis: None, + private_state_exposed: false, + }; + let incarnation_key = IncarnationRef::of(&expected)?.key(); + let projection = root.join("projection.cc"); + let mut projected = vec![ + expected_envelope(&incarnation_key)?, + CultCacheEnvelope { + key: anchor.trust_anchor_id.clone(), + r#type: GameCultServiceTrustAnchorRecord::TYPE.into(), + payload: rmp_serde::to_vec(&anchor)?, + stored_at: rfc3339(anchor.bound_at_unix_millis), + schema_id: Some(GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA.into()), + }, + activation_envelope(&incarnation_key)?, + ]; + write_store(&projection, &projected)?; + let lease_path = root.join("lease.cc"); + File::create(root.join("lease.cc.lock"))?; + let state_root = root.join("state"); + std::fs::create_dir_all(&state_root)?; + + // Descriptors 3 and 4 carry the signer credentials, as systemd passes them. + let child = Command::new("bash") + .arg("-c") + .arg(format!( + "exec 3<{activation} 4<{provider}; exec {bin} --state-root {state} --idunn-projection {projection} --idunn-anchor {anchor}", + activation = root.join("activation-cred").display(), + provider = root.join("provider-cred").display(), + bin = env!("CARGO_BIN_EXE_odin-daemon"), + state = state_root.display(), + projection = projection.display(), + anchor = root.join("idunn-anchor.cc").display(), + )) + .env("GAMECULT_IDUNN_RUNTIME_BUNDLE", &bundle) + .env("GAMECULT_IDUNN_CANDIDATE_BIND", candidate.to_string()) + .env("GAMECULT_IDUNN_PROCESS_WRITE_LEASE", &lease_path) + .env("ODIN_TOPOLOGY_IDENTITY", root.join("topology-cred")) + .env("LISTEN_PID", "1") + .env("LISTEN_FDS", "2") + .env( + "LISTEN_FDNAMES", + format!("{IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME}:gamecult-runtime-presence-identity"), + ) + .stdout(Stdio::null()) + .stderr(Stdio::from(File::create(root.join("odin.log"))?)) + .spawn()?; + let mut odin = Odin { + _temp: temp, + root, + candidate, + child, + store: state_root.join("topology.cc"), + }; + + // Idunn's warming probe, then the lease naming that exact warming presence. + let deadline = Instant::now() + Duration::from_secs(20); + let response = loop { + match request_raw_snapshot_from_rudp_catalog(CultMeshRudpSnapshotOptions { + target: candidate, + runtime_id: "test-idunn".into(), + message_id: format!("warming-{}", now()), + schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), + record_keys: Some(vec!["odin".into()]), + ..Default::default() + }) { + Ok(response) => break response, + Err(error) => { + if let Some(status) = odin.child.try_wait()? { + bail!("Odin exited while warming ({status}): {}", odin.log()); + } + ensure!(Instant::now() < deadline, "Odin never answered warming: {error:#}"); + std::thread::sleep(Duration::from_millis(100)); + } + } + }; + let CultNetMessage::SnapshotResponseRaw { documents, .. } = response else { + bail!("the warming probe was not answered with raw documents") + }; + let presence: GameCultRuntimePresenceHealthRecord = rmp_serde::from_slice(&documents[0].payload)?; + ensure!(presence.state == "warming", "{}", presence.state); + let lease = IdunnProcessWriteLeaseRecord { + schema_version: IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into(), + target: "odin".into(), + expected_projection_sha256: expected.canonical_sha256()?, + plan_id: expected.plan_id.clone(), + incarnation_id: expected.incarnation_id.clone(), + sealed_release_id: expected.sealed_release_id.clone(), + activation_witness_sha256: activation.canonical_sha256()?, + state_schema_generation: "odin-v2".into(), + state_contract_sha256: expected.state_contract_sha256.clone().unwrap(), + runtime_id: expected.runtime_id.clone(), + runtime_instance_id: activation.runtime_instance_id.clone(), + warming_presence_sha256: presence.canonical_sha256()?, + lease_epoch: 1, + issued_at_unix_millis: now() - 5, + }; + let lease_envelope = |key: &str| -> Result { + Ok(CultCacheEnvelope { + key: key.into(), + r#type: IdunnProcessWriteLeaseRecord::TYPE.into(), + payload: lease.canonical_bytes()?, + stored_at: rfc3339(lease.issued_at_unix_millis), + schema_id: Some(IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into()), + }) + }; + write_store(&lease_path, &[lease_envelope("odin")?])?; + projected.push(lease_envelope(&incarnation_key)?); + write_store(&projection, &projected)?; + + while !odin.store.is_file() { + if let Some(status) = odin.child.try_wait()? { + bail!("Odin exited while activating ({status}): {}", odin.log()); + } + ensure!(Instant::now() < deadline, "Odin never activated: {}", odin.log()); + std::thread::sleep(Duration::from_millis(20)); + } + Ok(odin) +} + +/// A put Odin acknowledged is on disk: killed outright right after the +/// acknowledgement, inside the interval its own changes wait for, Odin loses +/// none of them. +#[test] +fn an_acknowledged_put_survives_sigkill() -> Result<()> { + for round in 0..3 { + let mut odin = start_odin()?; + // Inside the interval that Odin's activation write started. + std::thread::sleep(Duration::from_millis(300)); + let key = format!("doc-{round}"); + odin.put(&key)?; + odin.signal("-KILL")?; + ensure!(odin.holds(&key)?, "round {round}: the acknowledged put was lost: {}", odin.log()); + } + Ok(()) +} + +/// SIGTERM reaches the serving loop as a stop request: Odin makes its last +/// write and exits cleanly, rather than waiting to be killed. +#[test] +fn a_stop_signal_ends_odin_cleanly() -> Result<()> { + let mut odin = start_odin()?; + let status = odin.signal("-TERM")?; + let log = odin.log(); + ensure!(status.success(), "Odin exited {status}: {log}"); + ensure!(log.contains("Odin stopping on request"), "{log}"); + ensure!(!log.contains("final store write was not made"), "{log}"); + Ok(()) +} From 1d9bc0246d755653438a6522a75a221ee34672a8 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:30:37 +0200 Subject: [PATCH 15/43] Map Odin store repair and recovery Operator: "None of these options involve repair and recovery, which is what ultimately needs to happen." Q-R1..R6 open; R1 (CultLib typed read failures, temp cleanup) does not depend on them. Co-Authored-By: Claude Opus 5.5 --- docs/store-recovery-cut.md | 493 +++++++++++++++++++++++++++++++++++++ 1 file changed, 493 insertions(+) create mode 100644 docs/store-recovery-cut.md diff --git a/docs/store-recovery-cut.md b/docs/store-recovery-cut.md new file mode 100644 index 0000000..5efca27 --- /dev/null +++ b/docs/store-recovery-cut.md @@ -0,0 +1,493 @@ +# Odin store repair and recovery: cut map + +Status: cut map, Imagination pass 1 (Opus), 2026-09-30. Nothing has landed. This map owns the means for repairing +and recovering `/var/lib/gamecult/odin/topology.cc`. It follows `docs/write-pattern-cut.md` (Cuts 1-3) and starts +from that campaign's landed head. + +Anchors: +- Odin `hands/odin-write` at `cda2087` (Cuts 1-2 plus two test commits). Q3 B (peer puts fsynced before ACK) is + being finished on that branch now. **Re-anchor every Odin `file:line` below on the head that lands; the line + numbers here are `cda2087`.** +- CultLib `6d0cc963`. `packages/cultcache-rs` is byte-identical to Odin's pin `3bf1c0ce`, so the Rust line numbers + hold at both. +- Idunn `a96ad9d`, Heimdall `e8e2832`, gamecult-ops `runbooks/gamecult-state-backup.md` as of 2026-09-30. + +Operator, 2026-09-30, on the three options Self offered for an undecodable foreign write (degraded but loud, end +Odin, unchanged): **"None of these options involve repair and recovery, which is what ultimately needs to happen."** + +Open: Q-R1 to Q-R6 (section 8). D1 (section 6, O1) is a Self default the operator may overrule. + +## Short answer + +Odin already holds the one thing a repair needs: its in-memory working set, which is authoritative for everything +it owns while it holds the write lease. Today it never uses it for repair. It ends on a decodable foreign write, and +it silently stops persisting on an undecodable one. At activation it has no working set and no second copy, so it +crash-loops. + +The repair has one rule: **a store file that is not what Odin last wrote is set aside, never deleted and never +merged, and Odin writes its working set over it under the lease.** At activation Odin has no working set, so it +falls back to a previous generation kept on disk, and then to an empty store that providers refill. The +correlation sequence gets a time floor, so no rollback can recreate the watermark deadlock. Recovery is published as +a typed record in Odin's own catalog. The mechanism (typed read failures, a set-aside exchange, a retained previous +generation, no staging file left behind by a failed write) belongs to CultLib, because every consumer of the +single-file store has the same gap. + +## 1. Body facts (probed) + +### 1.1 Probe: cultcache-rs single-file store under fault + +The probe was a scratch crate at CultLib `3bf1c0ce`, `scratchpad/odin-recovery-probe` (Imagination session), commit +`b19eba9`, run through the stopgap on Yggdrasil in slot 2. It ran with `--tmpfs /small:size=64k --tmpfs /ro:ro`, +using `trap '' XFSZ; ulimit -f 40` for the file-size case. The log is `scratchpad/odin-recovery-probe.log`. + +| Fault | Result | +|---|---| +| P1: file truncated to half | `compare_exchange_snapshot` and `pull_all` return `Err`, **not** `Ok(false)`. There is no `io::Error` in the chain; the decode error is flattened to text (`lib.rs:720-725`). The message is wrong: "invalid type: string \"cultcache.store.v1\", expected struct CultCacheEnvelope". `store_header` (`lib.rs:317-341`) deserializes the *whole* array to find the header, so a truncated v1 file has no header and falls through to the legacy decoder. The file is unchanged by the failed exchange | +| P2: 12 bytes of garbage appended | `pull_all` returns `Ok`, all 3 records. `compare_exchange_snapshot(expected)` returns **`Ok(true)` and silently drops the tail**. `rmp_serde::from_slice` ignores trailing bytes. C#'s `DeserializeSnapshot` (`CultDocumentMessagePackSerialization.cs:179-215`) never checks for the end either | +| P2b: one byte flipped inside a payload | `pull_all` returns `Ok` (payloads are opaque bytes). `compare_exchange_snapshot` returns **`Ok(false)`**. To Odin, bit rot is a *decodable foreign write* | +| P3: header `cultcache.store.v4...` | `Err` with a distinct message, "not readable; this runtime reads v1 only". It is distinguishable only by its text | +| P4: another handle wrote a valid snapshot | `Ok(false)` | +| P5: `hard_link(topology.cc, topology.cc.displaced..x)`, then an exchange through the store | The link keeps the old bytes exactly, and the store writes the new file. The staging sweep (`remove_abandoned_staging_files`, `lib.rs:2730-2766`) removes a `..tmp` beside it and leaves the displaced name alone | +| read-only filesystem, no store | `pull_all` returns `Ok(0)`. The exchange fails opening the lock: `io::Error` `ReadOnlyFilesystem` (os 30) | +| `RLIMIT_FSIZE` 40 KiB, 200 KB write | `Err` `io::Error` `FileTooLarge` (os 27). The old file is intact and still decodes. **A 40,960 B partial staging file is left behind.** The next attempt sweeps it and leaves a new one | +| 64 KiB tmpfs, 200 KB write (ENOSPC) | `Err` `StorageFull` (os 28). The old file is intact. **A 61,440 B partial staging file is left behind: all of the filesystem's remaining space.** The same holds on every retry | + +What follows from the probe: +- **Decodable is not the same as legitimate.** A flipped payload byte is "decodable and foreign", and appended + garbage is "decodable and ours". The decodable/undecodable split cannot be what decides recovery. One rule has to + cover both: *not what Odin last wrote*. +- **A write that fails for lack of space holds the space it failed on.** `/var/lib/gamecult` is shared by every + service on Yggdrasil. Odin retries once per second (and, under Q3 B, on every put), so it re-takes whatever + another service frees. C# deletes its temp file in a `finally` (`CultCache.cs:3496-3531`); Rust does not + (`write_all_unlocked`, `lib.rs:741-768`). This is a Rust parity defect. +- **The atomic write keeps no previous generation.** `write_all_unlocked` stages, fsyncs, renames over the file and + fsyncs the directory. After the rename, the old inode is gone. The brief's "last good checkpoint kept by the + atomic write" does not exist on disk. The only last-good copy is Odin's in-memory `flushed` snapshot + (`lib.rs:506-510`), and it dies with the process. + +### 1.2 Odin today (`cda2087`) + +| Case | Path | Outcome | +|---|---|---| +| Decodable foreign write while serving | `MemoryOdinTopologyStore::flush` `lib.rs:613-626`: `compare_exchange_snapshot(flushed, current)` returns `false`, so `ForeignStoreWrite`. `survive` `main.rs:879-895` returns it, and Odin ends | Idunn restarts Odin (`control_plane.rs:68-71`: 6 restarts per hour, the wait doubling from 5 s). The new Odin **loads the foreign file as truth**. Odin's own unflushed changes, and every record of Odin's that the foreign file lacks, are lost without a trace. A flipped payload byte (P2b) takes this path | +| Undecodable foreign write while serving | the same exchange returns `Err` (decode). `survive` logs it and retries every second, forever. Nothing is persisted again | **Idunn's crossing breaks too.** `receive` (`Idunn drivers.rs:5005-5030`) reads the same file and propagates the error to `admit_latest_topology` (`control_plane.rs:7903-7907`) and `refresh_admitted_topology` (`:6040`). Every Odin-correlated deployment and refresh fails until the file is readable, which is never | +| Corrupt file at activation | `try_activate` `main.rs:246-297` calls `MemoryOdinTopologyStore::load` `lib.rs:519-538`, which calls `pull_all()?`. `main.rs:762` `while !try_activate()?` exits the process | Crash loop until Idunn's budget is spent. Then Idunn reports once, to its log only (`control_plane.rs:5709-5723`, `report_once` `:4599-4610` is `eprintln`), and stops. The Verse rendezvous is down | +| Disk full or read-only | the exchange returns `Err` (io). `survive` logs and retries every second | Odin serves from memory. Each attempt leaves a partial staging file (1.1). A stop or deploy in this state loses everything unflushed. Nothing typed says so | + +Detection latency for a foreign write: `flush` writes only when dirty (`main.rs:307-314`). Odin's own heartbeat +(`HEARTBEAT_INTERVAL` 5 s, `main.rs:69`) dirties the set, so a foreign write is met at the latest one heartbeat plus +one flush interval (about 6 s) after it happens. No polling read is needed, and Cut 1's rule (no reads while +serving) stands. + +### 1.3 Other sources of truth, and what they can recover + +| Source | Holds | Freshness | Recovers | +|---|---|---|---| +| Odin's working set (W) | everything, including unflushed changes | live | everything, but only while this process lives and holds the lease | +| `flushed` (in memory) | the last snapshot Odin wrote | at most one interval old | nothing more than W; it is the comparison base, not a store | +| previous generation on disk (new, O3 and R3) | the snapshot before the last one this store's owner verified and replaced | one write old | everything except the last write's delta, including publish-once documents, presence history and the watermark | +| nightly backup (`gamecult-state-backup.md`) | `/var/lib/gamecult/odin` whole, shared-lock copy | up to 24 h old, 3 days on Yggdrasil, 14 days plus 8 weekly on Raven | publish-once documents. **Never automatically**: "It never becomes live authority, and nothing restores it automatically"; "restore a store only after stopping its owner" | +| providers republishing | presences (2.7-12 s), Muninn documents (2 s), ads (about 30 s); Heimdall HEAD republishes `heimdall.command_boundary` and its three siblings every 60 s (`Heimdall src/index.ts:19,35-39`, gated on its write lease) | seconds to a minute | every document whose publisher republishes | +| Idunn's projection plus presences | correlations | one refresh pass (250 ms) after the presences return | correlations. They are a projection and never need restoring | +| Idunn's `odin_publisher_sequence_cursor` (`control_plane.rs:1456,1537,6081`) | the highest correlation sequence Idunn admitted, per target | live | nothing directly (Idunn owns it), but it is why the watermark must never roll back | + +Premise correction: `heimdall.command_boundary` is not publish-once at Heimdall HEAD. A `stored_at` of 09-23 +proves only that the publisher's own `stored_at` has not changed, because Odin stores the publisher's +`stored_at` (`persist_generic_document`, `main.rs:1330-1347`). Whether the deployed Heimdall re-puts is +unverified; O3's outstanding-republish list measures it. + +**The watermark is the one record whose loss is not self-healing.** Idunn refuses any correlation sequence at or +below its cursor (`control_plane.rs:1763,1785`). An Odin that starts from an empty store, from the previous +generation, or from a backup restarts a target's count below that cursor. That is the deadlock documented at +`lib.rs:33-49`: Idunn refuses every correlation until the count climbs past the old mark, and the refusal +withdraws the correlation before it can climb. Every recovery source except W has this defect. O1 removes it. + +Other facts the design leans on: +- **Idunn fences before it grants.** `CultCacheWriteLeaseDriver` (`Idunn drivers.rs:4050-4205`): "revocation + follows an exact incumbent stop", and grant is a nonblocking exchange from empty. A second *Odin* cannot hold + the lease while the first is still writing, so a foreign writer seen by a lease-holding Odin is not an Odin. +- **Idunn has a lifecycle brake** (`idunn-provision lifecycle-brake-engage|release|status`, + `provisioning.rs:95-97`; `odin-lifecycle-brake.cc` per `runbooks/odin-yggdrasil.md:29-30`). An engaged brake + denies continuity restarts (`control_plane.rs:5771-5774`, `:6219`). This is the operator's hold for a manual + restore. +- **A non-`active` Odin presence fans out.** `reconcile` sets `ready` only when the presence state is `active` + (`lib.rs:946`), and `dependency_evidence` (`lib.rs:1184ff`) makes every managed dependent of Odin not Ready. + The presence schema allows `degraded` (`cultnet-rs runtime_authority_contracts.rs:193`), but using it would + be a readiness statement about every dependent, not a store report. +- **The catalog serves only peer types and presences** (`public_document`, `main.rs:590-640`). No `odin.*` record + is served today, and neither Odin nor Idunn publishes an Eve surface. F3 in the write map (the unowned + `surface:gamecult.network.status`) is where an Eve lowering belongs. + +## 2. Ends + +1. No single fault, whether a foreign write of any kind, a corrupt file, or a failed write, ends Odin or leaves it + silently unpersisted. Only losing the write lease ends it. +2. A store file that is not what Odin last wrote is preserved byte for byte beside the store and never deleted, + merged or adopted by any automatic path. The owner (Odin under the lease) writes its own state over it. +3. At activation a corrupt store is recovered from the best source on the host (previous generation, then empty), + and never from the backup automatically. +4. No recovery can make a correlation sequence repeat or fall below one Idunn has admitted. +5. Every repair, and every stretch in which Odin is serving without persisting, is typed state that can be + queried through Odin's catalog while it is happening, including when the disk refuses writes. + +Invariants carried over: one writer (the lease, checked immediately before every write); Idunn's lockless +crossing reads a whole `.cc` snapshot at `topology.cc`; the path never becomes absent (Idunn reads an absent file +as an empty snapshot, which withdraws every correlation); no reads of the store while serving; one bad record +never stops Odin. + +## 3. Identity, lifecycle, authority + +| Persistent kind | Named by | Over time | Decided by | +|---|---|---|---| +| store file `topology.cc` | the state root plus `TOPOLOGY_SLOT` | replaced whole by the owner's exchange; never absent once created | Odin under the lease | +| displaced file (new) | `topology.cc.displaced..`, same directory (it must not end in `.tmp`) | created by a set-aside exchange as a hard link to the replaced inode; never modified; never deleted by any program. Travels to Raven with the nightly backup | created by Odin through CultLib; **disposed of only by the operator** | +| previous generation (new, Q-R2) | `topology.cc.previous` | replaced on every *verified* exchange (current == expected) by a hard link to the outgoing inode; **not** rotated by a set-aside exchange, so it always holds a generation that some writer verified before replacing | CultLib mechanism; Odin opts in | +| store condition record (new) | type `odin.store_condition`, schema `odin.store_condition.v1`, key `topology` | one record held in W; rewritten when the condition changes; flushed with the store; served by the catalog | Odin | +| correlation sequence floor (new rule, not a record) | per target, in the existing watermark | next = max(mark + 1, prior + 1, now in unix ms) | Odin | +| backup snapshot | `/var/backups/gamecult-state//` and Raven | nightly; never live authority | the operator restores it, with the lifecycle brake engaged (runbook) | + +## 4. Authority map (the whole campaign) + +- **Owner:** Odin, while it holds the current process-write lease, decides what `topology.cc` holds, when it is + set aside, and what activation recovers from. CultLib owns the mechanism (read classification, set-aside + exchange, previous generation, staging cleanup) and no policy. Idunn owns whether Odin runs (continuity budget, + lifecycle brake, deploys). The operator owns restores from backup and the disposal of displaced files. +- **Inputs:** W; `flushed`; the lease (read immediately before every write); at activation, `topology.cc` and then + `topology.cc.previous`; the clock (the sequence floor). +- **Outputs:** one atomic `topology.cc`; displaced files; `topology.cc.previous`; the `odin.store_condition` record + in the catalog. +- **Derived state:** `topology.cc` is a checkpoint and Idunn's crossing, never an input while serving. The + displaced files are evidence only: nothing reads them automatically. The previous generation is read only by + activation recovery. The condition record is a report and decides nothing. + - `ForeignStoreWrite` **is no longer an owner of Odin's life**. A foreign write becomes a set-aside event. +- **Forbidden writers and deciders:** + - Any path that ends Odin on a store condition. `survive`'s `ForeignStoreWrite` arm (`main.rs:884`) is + deleted; `WriteLeaseLost` stays, and so does the temporary `PresenceAuthorityRefused` rule. + - Any automatic delete of a displaced or previous file. + - Any merge of records from a foreign or displaced file into W. + - Any automatic restore from the backup. + - Idunn writing or repairing `topology.cc`. + - Any second writer of the file inside Odin: activation never writes. It prepares W and `flushed`, and the one + `flush` makes the first write through the same set-aside exchange. +- **Shared paths:** + - The interval flush, the Q3 B per-put flush, the stop flush and the activation-recovery first flush all go + through one primitive, `replace_snapshot_setting_aside`. + - Every path that reserves a correlation sequence goes through the one floored reservation. +- **Deletion line:** `ForeignStoreWrite` (`lib.rs:630-641`) and its `survive` arm are deleted before the set-aside + exchange is wired in. The `Err` escape in `load` (`lib.rs:521`, `pull_all()?`) is replaced by the recovery + ladder, not wrapped. + +## 5. Recovery by case + +| Case | Detected by | Action | Recovered from | Loss bound | Visible as | +|---|---|---|---|---|---| +| **A. Decodable foreign write while serving** (a hand edit, a tool, a restore made against the runbook, or bit rot in a payload per P2b) | the next flush: current != `flushed` | the lease is checked; the file is set aside (hard link); W is written; Odin keeps serving | W | none of Odin's own records. The foreign file's content is preserved, not adopted | condition event `displaced`, reason `mismatch`, with path, sha256, bytes and time | +| **B. Undecodable foreign write while serving** (truncation, garbage, a foreign format, and after R2 trailing bytes too) | the next flush: current does not decode | the same | W | none. Idunn's crossing is unreadable for at most one heartbeat plus one flush (about 6 s), instead of forever | the same, reason `undecodable` | +| **A/B repeated** (a live foreign writer) | a third set-aside inside 10 minutes in one activation | Odin stops writing and keeps serving; puts are not ACKed (Q3 B: ACK means durable); no new displaced files (Q-R4) | W stays in memory | nothing while the process lives | condition `contested`, since when, with the displaced files | +| **C1. Undecodable store at activation** | `load` gets `Undecodable` | W is built from `topology.cc.previous` if it decodes; `flushed` is set to "unreadable", so the first flush sets the corrupt file aside and writes W | the previous generation | the last write's delta: one interval of Odin bookkeeping, or one fsynced put. **A put ACKed as durable can be lost here; say so in Q3 B's record** | event `recovered-previous`, plus an outstanding-republish list: peer records in the previous generation not yet re-put by their publisher, cleared as they arrive | +| **C2. Nothing decodable at activation** | `.previous` absent or undecodable too | start empty (Q-R3); the first flush sets the corrupt file aside | providers republish; correlations re-derive in one pass; the sequence floor holds | every publish-once document; presence history (F4 makes this harmless) | event `started-empty`; outstanding list is `unknown`. The operator's path is the backup, with the lifecycle brake engaged | +| **C3. Unsupported format at activation** (a newer runtime wrote v4, or a downgrade) | `load` gets `UnsupportedFormat` | refuse to activate. The file is untouched and nothing is set aside | the deploy or rollback that caused it; Idunn owns that | none | Idunn's continuity budget and the Odin journal. This is a deployment fault, not a store fault | +| **C4. I/O error reading the store at activation** (EIO, EACCES after a restore with the wrong owner, EISDIR) | `load` gets an `io::Error` | refuse to activate. Nothing is set aside: **only bytes that were read are ever judged** | the operator fixes the host or the mode | none | as C3 | +| **D. Write fails while serving** (ENOSPC, EDQUOT, EFBIG, EROFS, EIO) | the exchange returns an `io::Error` while staging or renaming | nothing is set aside; the staging file is removed (R1); W is kept; retry at the flush interval; puts are not ACKed; Odin keeps serving | W, when the fault clears | nothing, unless Odin is stopped or redeployed before the fault clears (follow-up I-F1) | condition `unpersisted` since T, with the last error kind (`no-space`, `read-only`, `io`) and the dirty-record count, served from memory | +| **D'. Stop while unpersisted** | the stop flush fails | log it and stop, as today (`main.rs:794-799`) | the next activation loads the last good file | the unflushed delta | the final journal line only. I-F1 is the owner fix | + +Rollback safety, for C1, C2 and an operator restore: every source except W carries a watermark below Idunn's +cursor. The floor (O1) makes the next sequence exceed every sequence published before, whatever the store holds. + +## 6. Cuts + +Order: the Rust substrate gaps first (R1), then the watermark floor (O1, independent and small), the set-aside +primitive (R3), Odin's serving repair (O2), activation recovery (O3), parity hardening (R2, R3b), and finally the +runbook and drill (X1). Each cut is its own branch and its own Soul pass. Odin verification: +`cargo test --locked -p odin-daemon` from the Odin root. CultLib Rust: `cd packages/cultcache-rs && cargo test +--locked`. All of it runs on Yggdrasil through the stopgap. **A permission-based test is unproven there (root).** +The injections below avoid permissions: truncation, a directory at the store path (EISDIR), tmpfs size and +read-only options, and `RLIMIT_FSIZE`. + +### R1. cultcache-rs: typed read failures, no staging file left behind, a header that survives truncation (CultLib) + +- **Repo/branch:** CultLib `hands/cultcache-rs-store-faults` from `main`. Depends on nothing. +- **Deletes first:** the flattening `map_err(|error| anyhow!(...))` in `read_all_unlocked` (`lib.rs:720-725`); + `store_header`'s whole-array visitor (`lib.rs:317-341`). +- **Adds:** + - `pub struct CultCacheStoreUnreadable { path, kind: Undecodable | UnsupportedFormat }`, carried in the error + chain of every read of a single-file store whose bytes were read and not accepted, so consumers can + `downcast_ref`. An `io::Error` stays an `io::Error`, and the two never overlap. + - `store_header` reads only the array header and the first string, reusing `read_array_header` and + `read_string` (`lib.rs:2661-2700`). A truncated v1 file then reports as an undecodable v1 file, not as legacy. + - `write_all_unlocked` removes its own staging file on every failure after creating it. This is C# parity with + `CultCache.cs:3525-3530`. +- **Rules that must die under their own mutation:** + - a truncated store is `Undecodable`, and a v4 header is `UnsupportedFormat`; + - an `io::Error` read failure is never classified `Unreadable`; + - after a failed write, the directory holds no `.*.tmp`. +- **Verification:** + - Tests describe behaviour. The staging-cleanup test forces the failure without permissions. One option is a + child process under `RLIMIT_FSIZE` with SIGXFSZ ignored (the probe's method); another is a path whose parent + fills a size-capped tmpfs, if the test runner can mount one. Hands picks one and names it. + - `cargo mutants --in-diff`. +- **Estimate:** about -30 / +70. + +### O1. The correlation sequence has a time floor (Odin) — default D1 + +- **Repo/branch:** Odin `hands/odin-sequence-floor` from the landed `hands/odin-write` head. Depends on nothing + else. +- **Change:** `reserve_publisher_sequence` (`lib.rs:702-723`) returns max(mark + 1, prior + 1, now in unix ms), + and records it as the mark. The doc comment at `lib.rs:33-49` gains the rule: "A mark read from any older copy + of the store is a floor, never the next value; the clock makes every rollback safe." +- **Why a floor and not a read of Idunn's cursor:** it needs no new crossing. It holds for every recovery source, + including the backup, and the host clock is already trusted for presence freshness. +- **Assumption:** the clock does not step back across a rollback by more than the rollback's age. Soul checks that + Idunn accepts a jump: its rule is "strictly above the cursor" (`control_plane.rs:1763,1785`). No test may pin a + gap limit. +- **Verification:** + - Existing tests pinning counts (`withdrawing_a_correlation_does_not_reset_its_publisher_sequence` + `lib.rs:1996-2014`) are restated as order properties. + - New test: a store rolled back to an earlier snapshot, then one refresh, yields a correlation sequence greater + than every sequence published before the rollback. + - New test: two reservations inside one millisecond are strictly increasing. + - The clock must be injectable at the reservation. At least one mutant must be a function of the time (an offset + or a scale), with probe values where it would show. +- **Estimate:** about -5 / +25. + +### R3. The set-aside exchange and the previous generation (CultLib, Rust and C#; TS and Python in R3b) + +- **Repo/branch:** CultLib `hands/cultcache-set-aside` from R1. Q-R2 decides the previous-generation half. +- **Adds, on `SingleFileMessagePackBackingStore`:** + - `replace_snapshot_setting_aside(expected, replacements) -> Result`, with + `SnapshotReplacement::{Exchanged, SetAside { displaced: PathBuf, sha256, bytes, reason: Mismatch | + Undecodable }}`. It makes one exclusive lock hold, in this order: + 1. stage the replacement (write and fsync); + 2. read current; + 3. if current equals expected, rename the staged file over (rotating `.previous` first when enabled); + 4. if the read was an `io::Error`, remove the staging file, return the error and change nothing; + 5. otherwise hard-link current to the displaced name, fsync the directory, rename the staged file over, and + fsync the directory again. + + Staging comes first, so a full disk sets nothing aside. The path is never absent: a link, never a rename. A + set-aside never rotates `.previous`. + - Opt-in previous generation: a constructor option (`retaining_previous_generation`, or Hands' spelling of it) + plus `pull_previous_generation() -> Result>`, read under the shared lock and classified + like any read. Off by default, because existing consumers must not grow files they did not ask for. + - Filesystems without hard links: return an explicit error naming the filesystem. Do not fall back to a + copy-then-rename, which would make the path briefly hold new content while the evidence is still being + written. Windows NTFS links work; the Windows path is **not yet reached** by the Yggdrasil suite and is + recorded as such. + - C# reference: the same operation on `SingleFileBackingStore` (`CultCache.cs:3268ff`), with the same names + across runtimes. +- **Verification (Rust and C#):** + - a mismatch preserves the foreign bytes exactly and writes the replacement; + - an undecodable current is preserved and replaced; + - an I/O read failure (a directory at the path) changes nothing and leaves no staging file; + - a staging failure (size cap) changes nothing and sets nothing aside; + - the path is readable at every instant: a reader thread loops `pull_all_read_only_snapshot` through 1,000 + exchanges and never sees an absent file or an empty snapshot; + - `.previous` rotates only on a verified exchange; + - two set-asides in the same second get distinct names; + - mutation tools: `cargo mutants --in-diff`, and Stryker.NET on the C# diff. +- **R3b (parity, not blocking Odin):** cultcache-ts `single-file-messagepack-backing-store.ts:34` and cultcache-py + `stores.py:83` gain the same operation and option, with a shared fixture test. +- **Estimate:** Rust about +140, C# about +120, R3b about +200. + +### O2. A foreign store file while serving is set aside, not fatal (Odin) + +- **Repo/branch:** Odin `hands/odin-store-repair` from O1, with a CultLib pin bump to R3. Needs Q-R1 and Q-R4. +- **Deletes first:** `ForeignStoreWrite` (`lib.rs:630-641`) and its arm in `survive` (`main.rs:884`). The test + `a_flush_writes_only_changes_and_never_over_a_foreign_file` (`lib.rs:2055-2090`) and the `survive` tests at + `main.rs:2842-2844` and `:3110` pin the old rule: restate them, do not delete them. +- **Changes:** + - `MemoryOdinTopologyStore::flush` (`lib.rs:613-626`) calls `replace_snapshot_setting_aside(flushed, current)`. + On `SetAside` it records a `displaced` event and counts it toward the contested bound. On an I/O failure it + records `unpersisted` and returns the error, which `survive` logs and retries as today. + - `RuntimeState::flush` (`main.rs:307-314`) keeps its lease check immediately before the write. + - The Q3 B per-put path uses this same flush. Hands re-anchors on the Q3 B head. +- **Adds:** + - The `odin.store_condition` record (schema in `odin-core` `documents.rs`), held in W. It carries: + - `generation` and `written_by`, updated inside the flush before the snapshot is taken; + - `unpersisted_since` and `last_write_error`; + - `contested_since`; + - a bounded list of the last 16 recovery events: kind, time, displaced path, sha256 and bytes, source, records + loaded, and the outstanding list (bounded). + - One arm in `public_document` (`main.rs:590-608`) that serves exactly this type. Providers still cannot write + it: `is_peer_document_type` is unchanged. +- **Rules that must die under their own mutation:** + - a flush never leaves the store dirty (the condition record is written inside the flush, not after it); + - a set-aside is preceded by a lease check; + - the third set-aside inside the window stops writing, and the second does not; + - after `contested`, no file is written or created; + - on an I/O failure, nothing is set aside; + - the condition is served while writes fail. +- **Authority map:** as in section 4. The deletion line is the `ForeignStoreWrite` type and arm. +- **Verification (fault injection, each at the file layer, not a log line):** + - T-A: activate, serve, write a different valid snapshot behind Odin. After the next pass the file decodes to + W, the displaced file's bytes equal the foreign bytes, `serving_pass` returned `Ok`, the catalog is unchanged, + and the served condition names the displaced file and its sha256. + - T-A2 (P2b): flip one payload byte behind Odin; the same outcome, reason `mismatch`. + - T-B: truncate the file behind Odin; the same outcome, reason `undecodable`, and + `pull_all_read_only_snapshot` (Idunn's crossing) decodes the file after one pass. + - T-B-latency: with no puts, the set-aside happens within one heartbeat plus one flush interval. + - T-contested: three foreign writes inside the window make exactly two set-asides and a third `contested` + report. After that the file and directory are untouched by further passes, and Odin still serves. + - T-D: replace the store path with a directory (EISDIR, which works as root). Nothing is set aside, no staging + file remains, the condition is `unpersisted` and is served, and a put is not ACKed. Remove the directory, and + the next pass writes W, clears `unpersisted` and ACKs. + - T-lease: the lease withdrawn before a flush that would set aside means no link, no write, and Odin ends + (`WriteLeaseLost`). + - Negative: `rg -n "ForeignStoreWrite" crates` matches nothing; `rg -n "remove_file|remove_dir" crates/odin-daemon/src` + matches nothing new outside tests. + - `cargo mutants --in-diff`. +- **Estimate:** about -40 / +160. + +### O3. Activation recovers a corrupt store (Odin) + +- **Repo/branch:** Odin `hands/odin-store-activation` from O2. Needs Q-R2 and Q-R3. +- **Change:** `MemoryOdinTopologyStore::load` (`lib.rs:519-538`) becomes the recovery ladder: + + | `load` result | Outcome | + |---|---| + | `Ok` | as today | + | absent | empty, as today | + | `Unreadable::Undecodable` | try `pull_previous_generation()`, then empty (Q-R3); `flushed` becomes a sentinel meaning "the file is not a snapshot Odin can compare", so the first flush's exchange sets it aside | + | `Unreadable::UnsupportedFormat` | return the error: refuse | + | `io::Error` | return the error: refuse | + + Activation itself never writes. The first `serving_pass` flush (`main.rs:833-842`, due at once) writes W. That + set-aside is recorded under the activation event and is not counted toward the contested bound. +- **Adds:** the outstanding-republish list. It holds the peer-type records present in the recovered previous + generation, each cleared when its publisher re-puts it. After 10 minutes the remainder is marked "not + republished", which is the live answer to "who is publish-once". For `started-empty` the list is `unknown`. +- **Rules that must die under their own mutation:** + - Unsupported-format and I/O failures never set aside, never fall back, and never start. + - A recovered activation writes before it serves its first catalog snapshot, or it serves the recovered set and + marks the condition before the first write. Hands picks one and states it. +- **Verification:** + - T-C1: a truncated `topology.cc` and a valid `.previous` activate with the previous content. The first pass + writes it and sets the corrupt file aside byte-exact, and the condition is `recovered-previous` with its + outstanding list. Re-putting one listed document clears it. + - T-C2: both undecodable (Q-R3 A) start empty. Both files are preserved, the condition is `started-empty`, and + the next correlation sequence is above the floor. + - T-C3: a v4 header refuses to activate. The file is byte-identical afterwards, and there is no displaced file + and no `.previous` change. + - T-C4: a directory at the store path refuses, as C3. + - T-restart: a recovered Odin stopped and restarted loads its own first write normally, with no second + set-aside. + - `cargo mutants --in-diff`. +- **Estimate:** about -10 / +110. + +### R2. Single-file decoders reject trailing bytes, in every runtime (CultLib) + +- **Repo/branch:** CultLib `hands/cultcache-trailing-bytes`. Independent of Odin; land it before O3 ships so that + an appended file is caught as undecodable, not overwritten silently (P2). +- **Change:** C# `DeserializeSnapshot` (`CultDocumentMessagePackSerialization.cs:179-215`) checks `reader.End`. + Rust `decode_store_snapshot` (`lib.rs:2583ff`) and the legacy path check that the slice is consumed. TS and + Python do the same. **One release**: a runtime that accepts what another rejects is a parity split. +- **Verification:** one shared fixture (a valid store plus one byte) is rejected as undecodable by all four + runtimes, and the valid store is accepted by all four. +- **Estimate:** about +40 plus tests across four runtimes. + +### X1. Runbook and recovery drill (gamecult-ops, then live) + +- **Repo:** gamecult-ops `runbooks/odin-yggdrasil.md` gains a "Store recovery" section: + - how to read `odin.store_condition` from the catalog; + - what displaced and previous files mean, and that disposing of them is the operator's decision; + - the manual restore: engage Odin's lifecycle brake, stop the `idunn-odin-*` unit, set the live file aside with + `ln` (never `rm`), extract the snapshot's `topology.cc` with its owner, mode and `.lock`, release the brake, + and read the condition. The sequence floor makes an old watermark safe. +- **Drill D-1 (off-live, after O3; needs the Q2-style approval, Q-R6):** extract the newest Yggdrasil snapshot's + `topology.cc` into a container on Yggdrasil. Activate a test-harness Odin on it through the existing fixture + authority. Then truncate the copy and activate again with the extracted file as `.previous`. Report the records + per type, the outstanding list and the sequence floor. Delete the copy afterwards. +- **Drill D-2 (live, operator-scheduled, Q-R6), after O3 deploys:** + 1. Record the baseline: catalog types, Idunn correlated targets, route timeouts. + 2. Engage the lifecycle brake and stop Odin. + 3. `ln topology.cc /var/tmp/odin-drill-keep.cc` (the evidence of the evidence), then truncate `topology.cc` in + place to half. + 4. Release the brake and let Idunn restart Odin. + 5. Pass when all of these hold: + - Odin is serving within one restart; + - the condition is `recovered-previous`, with a displaced file whose sha256 equals the truncated file's; + - every Odin-correlated target is Ready again within 60 s; + - there are 0 route-challenge timeouts in the next hour; + - the catalog's type set equals the baseline after 10 minutes, or the outstanding list names every + difference. + 6. Then run D-2b, a foreign write while serving: `ln` the live file aside, then truncate it. It passes when the + condition shows `displaced` within about 6 s and Idunn's correlation reads never fail for longer than that + (the journal). + +## 7. Subtraction ledger (estimate) + +| Cut | Removed | Added | Formats and targets | +|---|---|---|---| +| R1 | about 30 | about 70 | none | +| O1 | about 5 | about 25 | none | +| R3 | 0 | about 260 (Rust 140, C# 120); R3b about 200 | two sibling-file conventions (`.displaced.*`, `.previous`); no format change | +| O2 | about 40 | about 160 | one Odin schema, `odin.store_condition.v1` | +| O3 | about 10 | about 110 | none | +| R2 | 0 | about 40 plus tests | none (it tightens the decoders) | + +The campaign is net additive, at about +570 without R3b. It buys ends 1-5, which the operator asked for, and +removes one fatal path and one silent path. Every addition in CultLib is a capability a general consumer of the +single-file store would expect, and two of them (staging cleanup, and the header that survives truncation) are +defects. + +## 8. Operator questions + +- **Q-R1. A store file changed underneath a running Odin (decodable or not).** + - Context: Idunn fences the old Odin before granting the lease, so the other writer is never an Odin. It is a + tool, a hand edit, a restore made against the runbook, or disk corruption; P2b shows a flipped byte looks + exactly like a foreign write. Today a decodable change ends Odin, and the restart adopts the foreign file + silently. An undecodable change stops persistence forever and breaks Idunn's crossing. + - A: set the file aside (kept byte for byte, never deleted), write Odin's working set over it, keep serving, + and report it as typed state. + - B: set it aside, write the working set, then end Odin so a fresh process starts from the rewritten file. + - **Recommended: A.** B's restart protects nothing that the lease check before the write does not already + protect, and costs a route outage. +- **Q-R2. Keep one previous generation on disk (`topology.cc.previous`)?** + - Context: without it, a store that is corrupt at activation has no on-host source except "empty", and the + backup is up to 24 h old and needs the operator. It costs one hard link per write (a directory update, no data + written) and one more file of the store's size. It is a CultLib option, off by default for other consumers. + - A: yes. B: no; activation recovery goes straight to empty. + - **Recommended: A.** It is the only source that recovers publish-once documents, presence history and + correlations without the operator. Its loss bound is one write. +- **Q-R3. Activation finds nothing decodable: no usable `topology.cc`, no usable previous generation.** + - Context: presences and most catalog documents return within seconds to a minute, and correlations re-derive + in one refresh pass. Publish-once documents (if any exist, see 1.3) stay missing until their publisher + re-puts them, or until the operator restores from the backup. + - A: start empty automatically, set the corrupt file aside, and report `started-empty`. + - B: refuse to start, so Odin crash-loops until the operator restores. Every service's rendezvous is down + meanwhile. + - **Recommended: A.** A down Odin loses everything the Verse needs; an empty one loses only what nobody + republishes, and the restore path stays open. +- **Q-R4. A foreign writer keeps writing (a third set-aside within 10 minutes).** + - Context: repeated repair against a live writer makes a displaced file every second. + - A: stop writing, keep serving from memory, withhold put ACKs, and report `contested` until the operator + finds the writer. The working set survives. + - B: end Odin. Idunn restarts it, and the restart loads whatever the foreign writer last wrote. + - **Recommended: A.** B hands the store to the unknown writer. A is loud and loses nothing that is in memory. +- **Q-R5. Should a store condition change Odin's presence to `degraded`?** + - Context: a non-`active` Odin presence makes every managed dependent of Odin not Ready (`lib.rs:946`, + `dependency_evidence`). The store condition does not affect serving. + - A: no. The presence stays `active`, and the condition record is the signal. + - B: `degraded` while `unpersisted` or `contested`. + - **Recommended: A.** Otherwise a full disk becomes an outage of every dependent, and a restart that Idunn might + take in response would destroy the only complete copy. +- **Q-R6. The drills.** + - D-1: decode a copy of last night's backup `topology.cc` in a container on Yggdrasil, the same terms as Q2 for + the write map, with the copy deleted afterwards. + - D-2: after O3 deploys, a scheduled live drill that truncates the real `topology.cc` behind the lifecycle brake, + keeping a hard link of the original. + - **Recommended: yes to both**, D-1 before O3 ships and D-2 in a window the operator names. + +## 9. Follow-ups outside this map + +- **I-F1 (Idunn):** a deploy or a stop of Odin while `odin.store_condition` says `unpersisted` or `contested` + destroys the only complete copy. Idunn could read the condition from the catalog and hold the deployment + (deployment brake, never continuity). This is Idunn's decision, raised with the Idunn map. +- **I-F2 (Idunn):** "Idunn stopped restarting admitted odin" is a log line (`report_once`, `control_plane.rs:4599`), + not typed state. It belongs to Idunn's own operator surface. +- **F3 (from the write map):** the Eve lowering of `odin.store_condition` belongs to Odin's operator surface. + This map publishes the typed record only. +- **H-F1 (Heimdall):** confirm whether the deployed Heimdall re-puts its Odin documents every 60 s. O3's + outstanding list measures it after any recovery. +- **Q3 B's record in the write map:** "an ACK means durable" holds against crashes, not against a corruption at + activation, which loses at most the last write (C1). From f4a1f8900a562b0eadb8ea64b532849940b4356a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:32:41 +0200 Subject: [PATCH 16/43] Odin: drop doc lines that restate the code they sit on One was also wrong: `records` is in key order, not the file's. Co-Authored-By: Claude Opus 5.5 --- crates/odin-daemon/src/lib.rs | 5 ----- crates/odin-daemon/src/main.rs | 8 +++----- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/crates/odin-daemon/src/lib.rs b/crates/odin-daemon/src/lib.rs index 66ade96..981c65a 100644 --- a/crates/odin-daemon/src/lib.rs +++ b/crates/odin-daemon/src/lib.rs @@ -425,8 +425,6 @@ pub struct OdinStoreSnapshot { pub trait OdinTopologyStore { fn read(&self, query: OdinStoreQuery) -> Result; - /// Replace the queried incarnation's correlation, and its presence when - /// one is given. fn commit( &self, query: &OdinStoreQuery, @@ -526,7 +524,6 @@ impl MemoryOdinTopologyStore { }) } - /// Every record, in the order the file stores them. pub fn records(&self) -> Ref<'_, BTreeMap> { self.records.borrow() } @@ -553,7 +550,6 @@ impl MemoryOdinTopologyStore { } } - /// Every incarnation this store holds a correlation for. pub fn correlated_incarnations(&self) -> BTreeSet { self.records .borrow() @@ -584,7 +580,6 @@ impl MemoryOdinTopologyStore { legacy.len() } - /// Whether the records differ from what was last loaded or written. pub fn is_dirty(&self) -> bool { self.dirty.get() } diff --git a/crates/odin-daemon/src/main.rs b/crates/odin-daemon/src/main.rs index ccba290..2405c82 100644 --- a/crates/odin-daemon/src/main.rs +++ b/crates/odin-daemon/src/main.rs @@ -809,11 +809,9 @@ fn serve( ended } -/// The last write before Odin ends. `flush` makes it only while the lease is -/// still current and the file still holds what Odin wrote, so it writes -/// nothing after `WriteLeaseLost` or `ForeignStoreWrite`. A failure is logged -/// and not returned: Odin is ending either way, and the next Odin loads what -/// the file holds. +/// The last write before Odin ends, under `flush`'s own checks: after +/// `WriteLeaseLost` or `ForeignStoreWrite` it writes nothing. A failure is +/// logged, not returned: Odin is ending either way. fn stop(state: &RuntimeState) { if let Err(error) = state.flush() { eprintln!("Odin's final store write was not made: {error:#}"); From cdc57bda2dc7b341a958137cebe0a5f6cc8b31a1 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:40:44 +0200 Subject: [PATCH 17/43] Store recovery map: Q-R1..R6 ruled; Idunn watchdog reporting to Discord is a new campaign Co-Authored-By: Claude Opus 5.5 --- docs/store-recovery-cut.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/store-recovery-cut.md b/docs/store-recovery-cut.md index 5efca27..1ce37e6 100644 --- a/docs/store-recovery-cut.md +++ b/docs/store-recovery-cut.md @@ -1,5 +1,18 @@ # Odin store repair and recovery: cut map +**Operator rulings, 2026-09-30:** +- **Q-R1:** "You got it". A foreign file under a running Odin is set aside and Odin continues. +- **Q-R2:** "Yep". Keep the previous generation on disk. +- **Q-R3:** "Start empty". Nothing decodable at activation means Odin starts empty. +- **Q-R4:** "Yep, and Idunn needs better reporting capabilities so it can serve as watchdog. I ought to get a message + about this on Discord if it ever happens." + - Stop writing after the third set-aside in 10 minutes, and keep serving. + - **New requirement:** Idunn watchdog reporting that reaches the operator on Discord. It is mapped as its own + campaign and is not an Odin cut. +- **Q-R5:** "That's fine, Odin still runs after all, right?" Yes. Odin keeps serving, and the store condition is + published as its own typed record, not as a degraded presence. +- **Q-R6:** "Approved". Both drills. + Status: cut map, Imagination pass 1 (Opus), 2026-09-30. Nothing has landed. This map owns the means for repairing and recovering `/var/lib/gamecult/odin/topology.cc`. It follows `docs/write-pattern-cut.md` (Cuts 1-3) and starts from that campaign's landed head. From e7fbbbcadeaf05c69a59f3c40e33323015db589d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 16:23:56 +0200 Subject: [PATCH 18/43] Store recovery map: R1 status; Muninn reset script must move with its next CultLib bump Co-Authored-By: Claude Opus 5.5 --- docs/store-recovery-cut.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/store-recovery-cut.md b/docs/store-recovery-cut.md index 1ce37e6..fc08905 100644 --- a/docs/store-recovery-cut.md +++ b/docs/store-recovery-cut.md @@ -13,6 +13,15 @@ published as its own typed record, not as a degraded presence. - **Q-R6:** "Approved". Both drills. +**R1 status (Self, 2026-09-30).** Built on CultLib `hands/cultcache-read-failures` (`c0b56e06`..`b7908668`); it +is in Soul. +- **Consumer follow-up that must travel with the next Muninn CultLib bump:** + `Muninn scripts/restart-starfire-muninn.ps1:144` matches `*failed to decode MessagePack*` to trigger + `Reset-CorruptMuninnStore`. R1 removes that text, so the reset would stop firing silently. The script must match + the typed `CultCacheStoreUnreadable` kind, or the new wording, in the same change. +- C#'s truncated-store exception (`EndOfStreamException`) is an `IOException`, the same I/O-versus-undecodable + overlap. It goes with R3's C# work. + Status: cut map, Imagination pass 1 (Opus), 2026-09-30. Nothing has landed. This map owns the means for repairing and recovering `/var/lib/gamecult/odin/topology.cc`. It follows `docs/write-pattern-cut.md` (Cuts 1-3) and starts from that campaign's landed head. From 80af7ef041dae37d3e28b457debf35710695747b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 16:43:41 +0200 Subject: [PATCH 19/43] Odin map: Q3 B supersedes the Cut 2 effect row; Cut 2 deploys with Cut 3 Odin half Co-Authored-By: Claude Opus 5.5 --- docs/write-pattern-cut.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/write-pattern-cut.md b/docs/write-pattern-cut.md index e1334a2..f8d5e42 100644 --- a/docs/write-pattern-cut.md +++ b/docs/write-pattern-cut.md @@ -19,6 +19,15 @@ approved. The copy is taken with `sudo cp` and decoded off the live path, and th - Recovery is mapped as its own pass. It covers an undecodable foreign write, a decodable one, and a failed load at activation. +**Soul on write batch 2 (2026-09-30).** Q3 B changes the expected effect. Cut 2's row in section 6 ("at most +3.5 MB/s, at most 1.05 writes/s") assumed Q3 A and is **superseded**. Under Q3 B, each peer put (presence +heartbeats included) writes the store once. +- With the live-shaped 3 MB store that is about 2.5-3 writes/s, about 7.5-9 MB/s, and 70-170 ms of the loop + blocked each second. +- Measured: route-challenge p99 461 ms and 11 kernel drops. +- After Cut 3's Odin half the store is about 80 KB and a put write is p50 about 4 ms. +- **Cut 2 therefore deploys only together with Cut 3's Odin half**, which is now being built on `hands/odin-write`. + **Q2 decode result (Eyes, 2026-09-30; full table in the CultLib session scratchpad `odin-topology-decode.md`).** - 77 records; 3,222,550 B of payload in a 3,240,078 B file. - The two "~1.55 MB records" are per-type totals. Each type is dominated by one Ghostlight operator surface: From 08ae1834441c0743ecec08eb307edb502e8b17dd Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 16:47:00 +0200 Subject: [PATCH 20/43] Odin: every failed write since the last success is Odin's own A write that fails does not say whether it replaced the file, so the file may hold what any failed write since the last success tried to write. The store kept only the last attempt: a failure after the replace followed by a failure before it left the file holding an attempt Odin no longer recognised, and the next write called Odin's own file foreign and ended it. Every attempt since the last successful write is now an accepted baseline, cleared when a write lands. Soul finding 1, with Soul's probe as the test. Co-Authored-By: Claude Opus 5.5 --- crates/odin-daemon/src/lib.rs | 73 +++++++++++++++++++++++++++-------- 1 file changed, 56 insertions(+), 17 deletions(-) diff --git a/crates/odin-daemon/src/lib.rs b/crates/odin-daemon/src/lib.rs index 981c65a..39bb32d 100644 --- a/crates/odin-daemon/src/lib.rs +++ b/crates/odin-daemon/src/lib.rs @@ -490,10 +490,12 @@ pub struct MemoryOdinTopologyStore { records: RefCell>, /// Exactly what the file held when it was loaded or last written. flushed: RefCell>, - /// What the last write tried to write, when it failed. A write can fail - /// after the file was replaced (syncing the directory), and then the file - /// holds these records: Odin's own, not another writer's. - attempted: RefCell>>, + /// What every write since the last one that succeeded tried to write. A + /// write can fail after it replaced the file (syncing the directory) or + /// before (anything earlier), and the error does not say which, so the + /// file may hold any of these: each is Odin's own, not another writer's. + /// One is kept per failed write whose records differ from the last. + attempted: RefCell>>, /// The records differ from what was last loaded or written. dirty: Cell, } @@ -519,7 +521,7 @@ impl MemoryOdinTopologyStore { path, records: RefCell::new(records), flushed: RefCell::new(flushed), - attempted: RefCell::new(None), + attempted: RefCell::new(Vec::new()), dirty: Cell::new(false), }) } @@ -588,32 +590,36 @@ impl MemoryOdinTopologyStore { /// changed since the last write; returns whether it wrote. /// /// The file must still hold exactly what Odin last loaded or wrote, or what - /// a failed write last tried to write. Anything else was written by another - /// process, and the write is refused with [`ForeignStoreWrite`] rather than - /// made over it. A failed write leaves the records dirty, so the next - /// write retries it. + /// one of the failed writes since then tried to write. Anything else was + /// written by another process, and the write is refused with + /// [`ForeignStoreWrite`] rather than made over it. A failed write leaves + /// the records dirty, so the next write retries it. pub fn flush(&self) -> Result { if !self.dirty.get() { return Ok(false); } let current: Vec = self.records.borrow().values().cloned().collect(); let file = SingleFileMessagePackBackingStore::new(&self.path); - let exchanged = file - .compare_exchange_snapshot(&self.flushed.borrow(), ¤t) - .and_then(|exchanged| match &*self.attempted.borrow() { - Some(attempted) if !exchanged => file.compare_exchange_snapshot(attempted, ¤t), - _ => Ok(exchanged), - }); + let mut exchanged = file.compare_exchange_snapshot(&self.flushed.borrow(), ¤t); + for attempted in self.attempted.borrow().iter().rev() { + if !matches!(exchanged, Ok(false)) { + break; + } + exchanged = file.compare_exchange_snapshot(attempted, ¤t); + } match exchanged { Ok(true) => {} Ok(false) => return Err(ForeignStoreWrite.into()), Err(error) => { - *self.attempted.borrow_mut() = Some(current); + let mut attempted = self.attempted.borrow_mut(); + if attempted.last() != Some(¤t) { + attempted.push(current); + } return Err(error); } } *self.flushed.borrow_mut() = current; - *self.attempted.borrow_mut() = None; + self.attempted.borrow_mut().clear(); self.dirty.set(false); Ok(true) } @@ -2143,6 +2149,39 @@ mod tests { Ok(()) } + /// Two failed writes in a row: the first after it replaced the file, the + /// second before (the lock cannot be taken), so the file still holds the + /// first one's records. Both are Odin's own, and the next write is made + /// over the first's. + #[test] + fn a_write_that_failed_before_replacing_keeps_every_earlier_attempt_odins() -> Result<()> { + let temp = TempDir::new()?; + let path = temp.path().join("topology.cc"); + let file = SingleFileMessagePackBackingStore::new(&path); + let store = MemoryOdinTopologyStore::load(&path)?; + store.put(peer_document("doc-1")?); + std::fs::write(&path, b"not a cultcache store")?; + assert!(store.flush().is_err()); + // The state a failed directory sync leaves: the replace landed. + std::fs::remove_file(&path)?; + ensure!(file.compare_exchange_snapshot(&[], &[peer_document("doc-1")?])?); + + store.put(peer_document("doc-2")?); + let lock = temp.path().join("topology.cc.lock"); + std::fs::remove_file(&lock)?; + std::fs::create_dir(&lock)?; + assert!(store.flush().is_err(), "the lock cannot be taken"); + std::fs::remove_dir(&lock)?; + assert_eq!(file.pull_all_read_only_snapshot()?, vec![peer_document("doc-1")?]); + + assert!(store.flush()?, "the first attempt is Odin's own, not a foreign write"); + assert_eq!( + file.pull_all_read_only_snapshot()?, + vec![peer_document("doc-1")?, peer_document("doc-2")?] + ); + Ok(()) + } + fn file_sequences(path: &Path) -> Result<(u64, Vec)> { let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; let mark = unique_envelope(&entries, TOPOLOGY_PUBLISHER_WATERMARK_TYPE, "ghostlight")? From 03f01ddba0451f9cf5f196964ee8e0ba27cb7f4f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 16:47:33 +0200 Subject: [PATCH 21/43] Odin: an unreadable projection file skips presences, not the catalog Reading Idunn's projection once per catalog query (620efb8) made a projection file that cannot be read refuse the whole catalog, where before it failed each presence lookup and left the peer documents served. The query now answers as if Idunn projected nothing, logged once per interval: peer documents are served and presences skipped. Odin's own projection check (require_own_projection) is unchanged and still retries or ends Odin as before. The new test restores file-level coverage: a projection that does not decode, and one whose lock cannot be opened. Soul finding 4. Co-Authored-By: Claude Opus 5.5 --- crates/odin-daemon/src/lib.rs | 3 ++- crates/odin-daemon/src/main.rs | 42 +++++++++++++++++++++++++++++++--- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/crates/odin-daemon/src/lib.rs b/crates/odin-daemon/src/lib.rs index 39bb32d..9a1d5ac 100644 --- a/crates/odin-daemon/src/lib.rs +++ b/crates/odin-daemon/src/lib.rs @@ -213,7 +213,8 @@ impl CultCacheIdunnProjectionSource { } /// One reading of Idunn's projection file. Every lookup made against it is -/// answered from the same bytes. +/// answered from the same bytes. The default projects nothing. +#[derive(Default)] pub struct IdunnProjectionSnapshot { entries: Vec, } diff --git a/crates/odin-daemon/src/main.rs b/crates/odin-daemon/src/main.rs index 2405c82..dd9f019 100644 --- a/crates/odin-daemon/src/main.rs +++ b/crates/odin-daemon/src/main.rs @@ -559,7 +559,10 @@ impl RuntimeState { /// The catalog serves what is stored, one record at a time: a record that /// does not decode, or whose incarnation's projection does not, is skipped /// and logged, and is never a reason to refuse every other record. Idunn's - /// projection is read once per query. Odin's + /// projection is read once per query; a projection that cannot be read at + /// all is every presence's failure and no one else's, so the query is + /// answered as if Idunn projected nothing: peer documents are served and + /// presences skipped. Odin's /// correlations are not part of the catalog: Idunn reads them from the /// store file, and a target has one per incarnation, so keying them by /// target would collide for the whole deploy window. @@ -568,8 +571,15 @@ impl RuntimeState { query: &CultMeshRudpSnapshotQuery, ) -> Result> { let records = self.topology()?.store.records(); - let projections = - CultCacheIdunnProjectionSource::new(&self.options.idunn_projection).snapshot()?; + let projections = CultCacheIdunnProjectionSource::new(&self.options.idunn_projection) + .snapshot() + .unwrap_or_else(|error| { + self.log_repeating( + "catalog skips every presence", + format!("Idunn's projection cannot be read: {error:#}"), + ); + IdunnProjectionSnapshot::default() + }); let mut selected = BTreeMap::new(); for envelope in records.values() { let document = match self.public_document(&projections, envelope) { @@ -2536,6 +2546,32 @@ mod tests { Ok(()) } + /// A projection file that cannot be read at all, because it does not + /// decode or cannot be opened, skips every presence and nothing else: the + /// catalog still serves the peer documents. + #[test] + fn a_projection_file_that_cannot_be_read_skips_presences_not_the_catalog() -> Result<()> { + let faults: [fn(&Path) -> Result<()>; 2] = [ + |projection| Ok(std::fs::write(projection, b"not a cultcache store")?), + |projection| Ok(std::fs::remove_file(sibling_lock_path(projection)?)?), + ]; + for fault in faults { + let mut odin = activated_odin()?; + odin.pass()?; + provider_put(&odin, "ghostlight.doc.v1", "doc-1", vec![1])?; + assert_eq!( + documents_of(&odin.catalog()?, GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA), + 1, + "Odin's own presence is served while the projection reads" + ); + fault(&odin.state.borrow().options.idunn_projection)?; + let catalog = odin.catalog()?; + assert_eq!(documents_of(&catalog, "ghostlight.doc.v1"), 1); + assert_eq!(documents_of(&catalog, GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA), 0); + } + Ok(()) + } + /// Every failure to establish Odin's OWN authority ends Odin (temporary /// rule, see `survive`), whatever shape the failure takes: an anchor that /// does not decode or names another schema, an Expected Idunn no longer From 749c6b9bbea2c52528e8fa5fa82be8cff4ad9f1c Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 16:48:12 +0200 Subject: [PATCH 22/43] Odin test: with nothing to write, the next change is written on the next pass Pins the rule that only an attempted write starts the flush interval: a due write with nothing to write leaves the next change to land on the first pass after it. Kills the mutant that starts the interval on every due pass. Soul finding 5. Co-Authored-By: Claude Opus 5.5 --- crates/odin-daemon/src/main.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/crates/odin-daemon/src/main.rs b/crates/odin-daemon/src/main.rs index dd9f019..5b54c42 100644 --- a/crates/odin-daemon/src/main.rs +++ b/crates/odin-daemon/src/main.rs @@ -3210,6 +3210,26 @@ mod tests { Ok(()) } + /// A due write with nothing to write does not start the interval: the next + /// change is written on the first pass after it. + #[test] + fn with_nothing_to_write_the_next_change_is_written_on_the_next_pass() -> Result<()> { + let mut odin = activated_odin()?; + odin.pass()?; + let written = file_identity(&odin.store)?; + odin.timers.last_projection_refresh = Some(Instant::now()); + odin.timers.last_heartbeat = Some(Instant::now()); + odin.timers.last_flush = ago(1_100); + odin.pass()?; + assert_eq!(file_identity(&odin.store)?, written, "nothing to write"); + + odin.heartbeat()?; + odin.pass()?; + assert_ne!(file_identity(&odin.store)?, written, "the change is written at once"); + assert_eq!(odin.file_records()?, odin.working_set()); + Ok(()) + } + /// The lease is checked at the write itself: lost between a change and /// its write, the write is not made and Odin ends. #[test] From dcd36fccb99ee6d0fbf454c0fbec9b47b4be84cf Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 16:48:12 +0200 Subject: [PATCH 23/43] Odin: say what a refused put promises An acknowledged put is always on disk. A put refused because its write failed may still become durable and be served: it stays in the working set, and the next write lands it, or the failed write had already replaced the file. Puts are latest-value per type and key, so that is sound; the promise is written down at accept_raw_document. Soul finding 3. Co-Authored-By: Claude Opus 5.5 --- crates/odin-daemon/src/main.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/odin-daemon/src/main.rs b/crates/odin-daemon/src/main.rs index 5b54c42..8ce0896 100644 --- a/crates/odin-daemon/src/main.rs +++ b/crates/odin-daemon/src/main.rs @@ -318,6 +318,14 @@ impl RuntimeState { /// once the store holding the put is on disk. The write is the one `flush`, /// so it carries everything else that changed too. A write that fails /// refuses the put. + /// + /// The promise runs one way only. An acknowledged put is always on disk. One + /// refused because its write failed may still be: it stays in the working set, so the catalog + /// serves it and the next write lands it (or the failed write had already + /// replaced the file). That is sound because a put is the latest value for + /// its type and key, not an event: a publisher that retries a refused put + /// stores the same value, and one that does not has lost nothing it was + /// promised. fn accept_raw_document(&mut self, receipt: CultMeshRudpRawDocumentReceipt) -> Result<()> { ensure!( self.activated(), From 6786faa4302903874bb9b272c5850900327e5e91 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 16:50:09 +0200 Subject: [PATCH 24/43] Odin Cut 3: a stored document the server could never serve is dropped at activation Odin's store carried two ~1.456 MB Eve records that no snapshot response could ever return (the server sends at most max_snapshot_response_bytes, 1 MiB), and every write rewrote them. Under Q3 B every put writes the store, so they cost a ~3 MB write per put. At activation, each stored peer document is sized as the server would answer a snapshot of it alone: the same encoder, the shortest message id. One larger than the server's limit is dropped from the working set and logged by type, key and size; the first write removes it from the file. The limit is the server's own option, handed to RuntimeState where main builds the server, so the two cannot disagree. Presences and records the catalog does not serve are not checked. Q1 A (refuse the unservable), Q2 (the two records are dead residue). The CultLib half, refusing such a put at the server, is its own cut. Co-Authored-By: Claude Opus 5.5 --- crates/odin-daemon/src/lib.rs | 3 +- crates/odin-daemon/src/main.rs | 184 ++++++++++++++++++++++++++++----- 2 files changed, 160 insertions(+), 27 deletions(-) diff --git a/crates/odin-daemon/src/lib.rs b/crates/odin-daemon/src/lib.rs index 9a1d5ac..74a86ce 100644 --- a/crates/odin-daemon/src/lib.rs +++ b/crates/odin-daemon/src/lib.rs @@ -542,7 +542,8 @@ impl MemoryOdinTopologyStore { } } - fn remove(&self, record_type: &str, key: &str) { + /// Remove the record with this type and key, if one is held. + pub fn remove(&self, record_type: &str, key: &str) { if self .records .borrow_mut() diff --git a/crates/odin-daemon/src/main.rs b/crates/odin-daemon/src/main.rs index 8ce0896..f43b463 100644 --- a/crates/odin-daemon/src/main.rs +++ b/crates/odin-daemon/src/main.rs @@ -19,7 +19,8 @@ use cultmesh_rs::{ CultMeshRudpSnapshotSource, CultMeshSystemClock, }; use cultnet_rs::{ - CultNetRawDocumentRecord, CultNetRawPayloadEncoding, GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, + CultNetMessage, CultNetRawDocumentRecord, CultNetRawPayloadEncoding, CultNetWireContract, + GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, GameCultProviderHealthIdentity, GameCultRuntimeCapability, GameCultRuntimePresenceHealthPurpose, GameCultRuntimePresenceHealthRecord, IDUNN_EXPECTED_INCARNATION_SCHEMA, IDUNN_PROCESS_WRITE_LEASE_SCHEMA, @@ -28,7 +29,8 @@ use cultnet_rs::{ IdunnRuntimeActivationSigner, IdunnServiceIdentity, OdinTopologyIdentity, ServiceIdentityProfile, ServiceIdentitySigner, ServiceIdentityTrustAnchor, derive_service_identity_id, - open_service_identity_credential_reader, verify_runtime_authority, + encode_cultnet_message_to_vec, open_service_identity_credential_reader, + verify_runtime_authority, }; use fs2::FileExt; use odin_daemon::{ @@ -145,6 +147,9 @@ struct RuntimeState { topology: Option, write_lease: Option, write_lease_path: PathBuf, + /// The largest snapshot response the server sends: its own + /// `max_snapshot_response_bytes`, handed over where the server is built. + max_snapshot_response_bytes: usize, recent_warming_proofs: VecDeque<(String, u64)>, /// Counts from this launch. Every launch is a fresh activation, and /// presences are ordered only within one activation. @@ -174,7 +179,11 @@ impl CultMeshRudpSnapshotSource for SnapshotHandle { } impl RuntimeState { - fn open(options: Options, candidate: SocketAddr) -> Result { + fn open( + options: Options, + candidate: SocketAddr, + max_snapshot_response_bytes: usize, + ) -> Result { let authority_material = load_runtime_authority(Path::new(&required_environment( RUNTIME_BUNDLE_ENVIRONMENT, )?))?; @@ -237,6 +246,7 @@ impl RuntimeState { topology: None, write_lease: None, write_lease_path: PathBuf::from(required_environment(PROCESS_WRITE_LEASE_ENVIRONMENT)?), + max_snapshot_response_bytes, recent_warming_proofs: VecDeque::new(), publisher_sequence: 0, log_gate: RefCell::default(), @@ -275,6 +285,14 @@ impl RuntimeState { // not this contract's and would otherwise be served to the Verse as // current. store.retire_legacy_correlations(); + for (record_type, key, size) in + drop_unservable_documents(&store, self.max_snapshot_response_bytes) + { + eprintln!( + "Odin dropped {record_type} {key} from its store: its snapshot response is {size} bytes, over the {} the server sends", + self.max_snapshot_response_bytes + ); + } let signer = self .topology_signer .take() @@ -619,24 +637,12 @@ impl RuntimeState { projections: &IdunnProjectionSnapshot, envelope: &CultCacheEnvelope, ) -> Result> { + if envelope.r#type != GameCultRuntimePresenceHealthRecord::TYPE { + return Ok(peer_document(envelope)); + } let Some(schema_id) = envelope.schema_id.clone() else { return Ok(None); }; - if envelope.r#type != GameCultRuntimePresenceHealthRecord::TYPE { - return Ok(is_peer_document_type(&envelope.r#type).then(|| { - CultNetRawDocumentRecord { - schema_id, - record_key: envelope.key.clone(), - stored_at: envelope.stored_at.clone(), - payload_encoding: CultNetRawPayloadEncoding::Messagepack, - payload: envelope.payload.clone(), - source_runtime_id: None, - source_agent_id: None, - source_role: None, - tags: None, - } - })); - } let presence = decode_presence(&envelope.payload)?; let Some(projection) = projections.projection(&IncarnationRef::new( presence.target.clone(), @@ -765,13 +771,18 @@ fn main() -> Result<()> { ); let socket = UdpSocket::bind(candidate) .with_context(|| format!("binding Odin CultNet RUDP candidate {candidate}"))?; - let state = Rc::new(RefCell::new(RuntimeState::open(options, candidate)?)); + let server_options = CultMeshRudpDocumentServerOptions::default(); + let state = Rc::new(RefCell::new(RuntimeState::open( + options, + candidate, + server_options.max_snapshot_response_bytes, + )?)); let mut server = CultMeshRudpDocumentServer::new( socket, SinkHandle(state.clone()), SnapshotHandle(state.clone()), CultMeshSystemClock::default(), - CultMeshRudpDocumentServerOptions::default(), + server_options, )?; // Idunn runs this process as PID 1 of its own PID namespace, and a @@ -1334,6 +1345,56 @@ fn is_peer_document_type(document_type: &str) -> bool { }) } +/// One stored peer document as the catalog serves it; `None` for any other +/// record. +fn peer_document(envelope: &CultCacheEnvelope) -> Option { + let schema_id = envelope.schema_id.clone()?; + is_peer_document_type(&envelope.r#type).then(|| CultNetRawDocumentRecord { + schema_id, + record_key: envelope.key.clone(), + stored_at: envelope.stored_at.clone(), + payload_encoding: CultNetRawPayloadEncoding::Messagepack, + payload: envelope.payload.clone(), + source_runtime_id: None, + source_agent_id: None, + source_role: None, + tags: None, + }) +} + +/// Drop every stored peer document the server could never serve, and return +/// each one's type, key and response size. A document is unservable when a +/// snapshot response holding it alone, encoded as the server encodes one and +/// with the shortest message id, is larger than the largest response the +/// server sends (`limit`, the server's own `max_snapshot_response_bytes`). +/// Nobody can read such a document, and every write of the store would carry +/// it. Presences are Odin's own admitted records, never this large, and are +/// not checked; nor is anything the catalog does not serve. +fn drop_unservable_documents( + store: &MemoryOdinTopologyStore, + limit: usize, +) -> Vec<(String, String, usize)> { + let unservable: Vec<(String, String, usize)> = store + .records() + .values() + .filter_map(|envelope| { + let response = CultNetMessage::SnapshotResponseRaw { + message_id: String::new(), + documents: vec![peer_document(envelope)?], + }; + let size = + encode_cultnet_message_to_vec(&response, CultNetWireContract::CultNetSchemaV0) + .ok()? + .len(); + (size > limit).then(|| (envelope.r#type.clone(), envelope.key.clone(), size)) + }) + .collect(); + for (record_type, key, _) in &unservable { + store.remove(record_type, key); + } + unservable +} + fn persist_generic_document( store: &MemoryOdinTopologyStore, document: &CultNetRawDocumentRecord, @@ -1471,12 +1532,11 @@ fn rfc3339_millis(value: u64) -> Result { #[cfg(test)] mod tests { use cultnet_rs::{ - CultNetMessage, CultNetRudpReliableSendStatus, CultNetRudpSocketTransportConnection, - CultNetRudpSocketTransportOptions, CultNetWireContract, - GAMECULT_RUNTIME_PRESENCE_HEALTH_SIGNING_PURPOSE, GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA, - GameCultServiceTrustAnchorRecord, IdunnExpectedCapability, IdunnExpectedRoute, - IdunnRuntimeActivationLaunch, ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA, - OdinRuntimeTopologyCorrelationRecord, encode_cultnet_message_to_vec, + CultNetRudpReliableSendStatus, CultNetRudpSocketTransportConnection, + CultNetRudpSocketTransportOptions, GAMECULT_RUNTIME_PRESENCE_HEALTH_SIGNING_PURPOSE, + GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA, GameCultServiceTrustAnchorRecord, + IdunnExpectedCapability, IdunnExpectedRoute, IdunnRuntimeActivationLaunch, + ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA, OdinRuntimeTopologyCorrelationRecord, enroll_service_identity_at, }; @@ -1744,6 +1804,7 @@ mod tests { topology: None, write_lease: None, write_lease_path: lease_path.clone(), + max_snapshot_response_bytes: options.max_snapshot_response_bytes, recent_warming_proofs: VecDeque::new(), publisher_sequence: 0, log_gate: RefCell::default(), @@ -3218,6 +3279,77 @@ mod tests { Ok(()) } + /// A peer document whose single-document snapshot response, encoded as the + /// server encodes one with an empty message id, is `response_bytes` long. + fn document_answering_in(key: &str, response_bytes: usize) -> Result { + let document = |payload_bytes: usize| -> Result { + Ok(CultCacheEnvelope { + key: key.into(), + r#type: "ghostlight.doc".into(), + payload: vec![7; payload_bytes], + stored_at: rfc3339_millis(unix_millis()?)?, + schema_id: Some("ghostlight.doc.v1".into()), + }) + }; + let response = |envelope: &CultCacheEnvelope| -> Result { + let served = CultNetRawDocumentRecord { + schema_id: "ghostlight.doc.v1".into(), + record_key: envelope.key.clone(), + stored_at: envelope.stored_at.clone(), + payload_encoding: CultNetRawPayloadEncoding::Messagepack, + payload: envelope.payload.clone(), + source_runtime_id: None, + source_agent_id: None, + source_role: None, + tags: None, + }; + let message = CultNetMessage::SnapshotResponseRaw { + message_id: String::new(), + documents: vec![served], + }; + Ok(encode_cultnet_message_to_vec(&message, CultNetWireContract::CultNetSchemaV0)?.len()) + }; + let overhead = response(&document(1_000)?)? - 1_000; + let sized = document(response_bytes - overhead)?; + ensure!(response(&sized)? == response_bytes, "the document was not sized exactly"); + Ok(sized) + } + + /// A stored peer document the server could never serve is dropped when + /// Odin activates, everything else is served, and the first write removes + /// it from the file. The bound is the server's own limit, from the options + /// it was built with: a document answering in exactly that many bytes is + /// kept, one byte more is dropped. + #[test] + fn a_stored_document_the_server_could_never_serve_is_dropped_at_activation() -> Result<()> { + const LIMIT: usize = 4_096; + let options = CultMeshRudpDocumentServerOptions { + max_snapshot_response_bytes: LIMIT, + ..CultMeshRudpDocumentServerOptions::default() + }; + let mut odin = activated_odin_with(options, |_| { + Ok(vec![ + document_answering_in("at-limit", LIMIT)?, + document_answering_in("over-limit", LIMIT + 1)?, + document_answering_in("small", 200)?, + ]) + })?; + let held = odin.stored_keys("ghostlight.doc")?; + assert_eq!(held, vec!["at-limit".to_owned(), "small".to_owned()]); + let catalog = odin.catalog()?; + assert_eq!(documents_of(&catalog, "ghostlight.doc.v1"), 2); + assert!( + odin.file_records()?.iter().any(|entry| entry.key == "over-limit"), + "activation itself writes nothing" + ); + + odin.pass()?; + let written = odin.file_records()?; + assert!(!written.iter().any(|entry| entry.key == "over-limit")); + assert_eq!(written, odin.working_set()); + Ok(()) + } + /// A due write with nothing to write does not start the interval: the next /// change is written on the first pass after it. #[test] From 723e74d5d1fc207ce66d0e42789fa4e13c9733f7 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 16:58:28 +0200 Subject: [PATCH 25/43] Odin test: the unreadable-projection case is a projection that does not decode The second fault in the test (removing the projection's lock file) did not make the read fail, so it asserted a fallback that never ran. The test keeps the fault that does: a projection file that does not decode. Co-Authored-By: Claude Opus 5.5 --- crates/odin-daemon/src/main.rs | 36 ++++++++++++++-------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/crates/odin-daemon/src/main.rs b/crates/odin-daemon/src/main.rs index f43b463..b16a736 100644 --- a/crates/odin-daemon/src/main.rs +++ b/crates/odin-daemon/src/main.rs @@ -2615,29 +2615,23 @@ mod tests { Ok(()) } - /// A projection file that cannot be read at all, because it does not - /// decode or cannot be opened, skips every presence and nothing else: the - /// catalog still serves the peer documents. + /// A projection file that cannot be read at all skips every presence and + /// nothing else: the catalog still serves the peer documents. #[test] fn a_projection_file_that_cannot_be_read_skips_presences_not_the_catalog() -> Result<()> { - let faults: [fn(&Path) -> Result<()>; 2] = [ - |projection| Ok(std::fs::write(projection, b"not a cultcache store")?), - |projection| Ok(std::fs::remove_file(sibling_lock_path(projection)?)?), - ]; - for fault in faults { - let mut odin = activated_odin()?; - odin.pass()?; - provider_put(&odin, "ghostlight.doc.v1", "doc-1", vec![1])?; - assert_eq!( - documents_of(&odin.catalog()?, GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA), - 1, - "Odin's own presence is served while the projection reads" - ); - fault(&odin.state.borrow().options.idunn_projection)?; - let catalog = odin.catalog()?; - assert_eq!(documents_of(&catalog, "ghostlight.doc.v1"), 1); - assert_eq!(documents_of(&catalog, GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA), 0); - } + let mut odin = activated_odin()?; + odin.pass()?; + provider_put(&odin, "ghostlight.doc.v1", "doc-1", vec![1])?; + assert_eq!( + documents_of(&odin.catalog()?, GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA), + 1, + "Odin's own presence is served while the projection reads" + ); + let projection = odin.state.borrow().options.idunn_projection.clone(); + std::fs::write(projection, b"not a cultcache store")?; + let catalog = odin.catalog()?; + assert_eq!(documents_of(&catalog, "ghostlight.doc.v1"), 1); + assert_eq!(documents_of(&catalog, GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA), 0); Ok(()) } From bdbecc3308816978b21b08de5a1d235d74e76b3e Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 16:58:28 +0200 Subject: [PATCH 26/43] Odin Cut 3: size a stored document with a message id CultNet admits CultNet refuses an empty message id, so sizing a document's snapshot response with one failed for every record, and the failure was read as "keep": nothing was ever dropped. The response is now sized with a one-byte message id, the shortest CultNet admits, and a record that cannot be sized is logged and kept rather than silently kept. Co-Authored-By: Claude Opus 5.5 --- crates/odin-daemon/src/main.rs | 36 +++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/crates/odin-daemon/src/main.rs b/crates/odin-daemon/src/main.rs index b16a736..2e4c490 100644 --- a/crates/odin-daemon/src/main.rs +++ b/crates/odin-daemon/src/main.rs @@ -1362,14 +1362,19 @@ fn peer_document(envelope: &CultCacheEnvelope) -> Option payload.len(), + // Kept: the catalog meets the same failure when it serves + // the record, and logs it there. + Err(error) => { + eprintln!( + "Odin cannot size {} {} as a snapshot response; kept: {error:#}", + envelope.r#type, envelope.key + ); + return None; + } + }; (size > limit).then(|| (envelope.r#type.clone(), envelope.key.clone(), size)) }) .collect(); @@ -3274,7 +3288,7 @@ mod tests { } /// A peer document whose single-document snapshot response, encoded as the - /// server encodes one with an empty message id, is `response_bytes` long. + /// server encodes one with a one-byte message id, is `response_bytes` long. fn document_answering_in(key: &str, response_bytes: usize) -> Result { let document = |payload_bytes: usize| -> Result { Ok(CultCacheEnvelope { @@ -3298,7 +3312,7 @@ mod tests { tags: None, }; let message = CultNetMessage::SnapshotResponseRaw { - message_id: String::new(), + message_id: "m".into(), documents: vec![served], }; Ok(encode_cultnet_message_to_vec(&message, CultNetWireContract::CultNetSchemaV0)?.len()) From 726e0f31a1b3cdb660c3f4c6731781a88536333a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 17:07:25 +0200 Subject: [PATCH 27/43] Odin test: size the small document inside the calibrated range The small document asked for a 200-byte response, less than the response's own fixed overhead, so sizing it underflowed. It now asks for 2,000 bytes, inside the range where the response grows one byte per payload byte. Co-Authored-By: Claude Opus 5.5 --- crates/odin-daemon/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/odin-daemon/src/main.rs b/crates/odin-daemon/src/main.rs index 2e4c490..191a456 100644 --- a/crates/odin-daemon/src/main.rs +++ b/crates/odin-daemon/src/main.rs @@ -3339,7 +3339,7 @@ mod tests { Ok(vec![ document_answering_in("at-limit", LIMIT)?, document_answering_in("over-limit", LIMIT + 1)?, - document_answering_in("small", 200)?, + document_answering_in("small", 2_000)?, ]) })?; let held = odin.stored_keys("ghostlight.doc")?; From 57cf8df1810443a44a73a2e991adb25ce3f3e53d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 18:20:40 +0200 Subject: [PATCH 28/43] Recovery map: R1 batch 2 Soul verdict (hold); zero-byte live stores block consumer pins Co-Authored-By: Claude Opus 5.5 --- docs/store-recovery-cut.md | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/docs/store-recovery-cut.md b/docs/store-recovery-cut.md index fc08905..f93dc5b 100644 --- a/docs/store-recovery-cut.md +++ b/docs/store-recovery-cut.md @@ -13,8 +13,31 @@ published as its own typed record, not as a degraded presence. - **Q-R6:** "Approved". Both drills. -**R1 status (Self, 2026-09-30).** Built on CultLib `hands/cultcache-read-failures` (`c0b56e06`..`b7908668`); it -is in Soul. +**R1 status (Self, 2026-09-30).** Built on CultLib `hands/cultcache-read-failures` (`c0b56e06`..`b7908668`), then +fix batch 2 (`..2d7a3c78`). Soul's verdict on batch 2 was to hold the merge. Batch 3 is in Hands, fixing: +- **Blocker, F1 (zero-byte stores on Yggdrasil).** + - Two zero-byte stores are live: `/srv/bifrost/persona-feedback/{inbox/observations,outbox/deliveries}.cc`. + - They are pre-created by `touch` in gamecult-ops `install-bifrost-persona-feedback-native-yggdrasil.sh:42,46` + and `bootstrap-bifrost-persona-feedback-yggdrasil.sh:22,29`. + - Bifrost `persona-feedback.mjs serve` and Epiphany `epiphany-swarm` crash-loop on them once their CultLib pins + move past R1. + - **Neither pin may move until the scripts stop pre-creating stores and Self has moved the live files aside.** +- **F4.** Rust errors echo record values through serde's text (`lib.rs:753-758`). The header is echoed without bound. +- **F6.** A dangling symlink reads as absent, and the writer then replaces it. +- **F7.** The same NotFound-only rule in Python's `JsonLinesBackingStore` and in TS `pullAll`'s repair write. +- **M7.** A header-prefix fixture. +Recorded for later cuts: +- **F3 → R2/R3b.** TS, Python and C# decode the whole buffer before reading the header. A truncated newer-format + store (20..n-1 bytes) is "unsupported format" in Rust and a raw decode error elsewhere. +- **F5 → R3.** C# `ReadSnapshot` reads ENOTDIR/EISDIR as an empty store (`FileInfo.Exists`, `CultCache.cs:3439`). + C# `DirectoryMessagePackBackingStore` `File.Exists` (`:409`, `:497`) is the same. This is a live disagreement between + runtimes until R3. +- **F8 → R3b.** Python leaves its staging file behind when a write fails (`stores.py:124-128`). +- **Tooling.** + - `packages/cultcache-rs` has no committed `Cargo.lock`, so the map's `cargo test --locked` cannot run. + - The cultcache-ts interop test needs `CARGO_TARGET_DIR` unset under ygg-verify. +- **Text matches.** gamecult-ops `scripts/idunn/start-idunn-local.ps1:143` matches the same removed text as Muninn + (below). It must change with Idunn's next CultLib bump. - **Consumer follow-up that must travel with the next Muninn CultLib bump:** `Muninn scripts/restart-starfire-muninn.ps1:144` matches `*failed to decode MessagePack*` to trigger `Reset-CorruptMuninnStore`. R1 removes that text, so the reset would stop firing silently. The script must match From 59ef4da9dd9f2fb73304783e36a78d40edf27423 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 18:34:36 +0200 Subject: [PATCH 29/43] Write map: Soul on batch 3 (not fit to deploy); Q5 where the durable write runs Co-Authored-By: Claude Opus 5.5 --- docs/write-pattern-cut.md | 41 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/docs/write-pattern-cut.md b/docs/write-pattern-cut.md index f8d5e42..e84fd71 100644 --- a/docs/write-pattern-cut.md +++ b/docs/write-pattern-cut.md @@ -47,6 +47,34 @@ heartbeats included) writes the store once. - The current `streampixels-web` incarnation has a correlation but no presence record; its last presence was 09-29 19:53 UTC. This matches Idunn's "not Odin-correlated" state. +**Soul on write batch 3 + Cut 3 Odin half (`f4a1f89..726e0f3`, 2026-09-30): not fit to deploy.** +- **S-1 (high; operator question Q5, below).** Every write, the fsync pair included, runs on the serving loop + (`main.rs:358` → `flush` → `compare_exchange_snapshot`; the interval flush at `:883` is on the same thread). + - A disk stall passes into route-challenge latency one for one. A probe that held the store lock measured: a 0 ms + stall gave 86 ms, 300 ms gave 288 ms, 1,500 ms gave 1,487 ms. + - Interleaved runs (ABBA) under IO pressure of 5-38% gave a challenge p99 no better with Cut 3 than without it. + - Cut 3 halves the bytes written, not the stalls. **The premise that the store's size caused the stalls was + wrong; they come from the disk.** +- **S-2 (high).** Q1's put refusal has not landed. Odin's pin `3bf1c0ce` has no put bound, so a 1.2 MB put is + ACKed, fsynced, and dropped at the next activation. It waits on the CultLib put-serve merge, then a pin bump with + a `served_record` override. +- **S-3 (medium).** `attempted` grows about 98 KB per failed write, roughly 0.9 GB/h on a full disk, which ends in + OOM. + - Fix in the owner: cultcache-rs returns a typed `NotReplaced`/`ReplacedNotDurable` write outcome (added to CultLib + R1 batch 3), and Odin keeps at most one candidate. +- **S-4 (medium).** Two admitted presences for one target, which a provider in a deploy window produces, refuse the + whole catalog (`main.rs:~628`). +- **S-5 (medium).** A RUDP transport ACK is recorded at receipt, so with pipelined puts a later put is ACKed while + an earlier one is still held. With the first datagram lost, the second is never delivered. That is data loss + under the QUIC-pivot rule; it is tracked in the CultLib ack map. `dcd36fc`'s doc line overstates. +- **S-6, S-8.** Test gaps (the limit taken through `main`; `remove_not_dirty`). +- **S-7.** The drop log echoes the key. Self ruled keys and schema ids are identities and may be logged, as in + CultLib R1. +- **Held:** F1's logic, F4's projection fallback, F5, F3's refusal, and Cut 3's activation bound, drop and removal. + Sizing uses `peer_document`, so it matches what is served. +- **Batch 4 in Hands:** S-4, S-6, S-8, the S-5 doc line, the attempted-list fixture gap, and committing Soul's + probes. S-1 waits on Q5; S-2 and S-3 wait on CultLib. + Status: cut map, Imagination pass 1 (Opus), 2026-09-30. Nothing has landed. Anchors are Odin `main` at `44951a1` (the deployed build, live on Yggdrasil since 2026-09-30 04:20:48 UTC) and CultLib `3bf1c0ce` (Odin's `Cargo.lock` pin). This map owns the means; there is no separate target document, because the ends are @@ -394,6 +422,19 @@ records; update the check's type list in the same pass. - **Q4. CultLib parity: port `DirectoryMessagePackBackingStore` (v4) to cultcache-rs?** It is not needed by Odin after Cuts 1-3. A reasonable consumer of the Rust runtime would expect the reference's store. **Recommended: record it as CultLib follow-up F1 with no deadline.** It is independent of this map. +- **Q5 (2026-09-30, from Soul S-1). Where does the durable write run?** Today the serving loop runs the fsync, so + any disk stall stalls route challenges, catalog reads and every other put. + - A: a writer thread with group commit. + - The loop hands each accepted put to one writer thread. The writer writes and fsyncs everything queued since + its last write in one go, then the loop sends those puts' replies. + - Challenges and reads are never behind the disk. A stall delays only put replies, and Q3 B still holds (a + reply means durable). + - It needs cultmesh-rs to let a put handler reply later (a CultLib API change) plus one Odin thread. + - B: answer route challenges on their own path. Everything else still stalls, and the challenge would report + Odin healthy while its catalog is stuck. + - C: reopen Q3 and go back to the 1 s interval. That brings back lost acknowledged puts on a crash. + + **Recommended: A.** ## 8. Related findings From a129da90fe50a036515d785a5a732cee0a38c2ea Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 18:37:09 +0200 Subject: [PATCH 30/43] Odin: the activation limit is set where the server is built, from its options Soul's mutant main_limit_halved survived: main passed the server's max_snapshot_response_bytes into RuntimeState::open by hand, and the tests built RuntimeState and the server separately, so nothing tied the two. One function, serve_with, now builds the server from its options and gives the state that same limit; main and the test fixture both go through it. The state has no limit until then, and Odin does not activate without one (the fixture checks it refuses before serve_with). Soul S-6. Co-Authored-By: Claude Opus 5.5 --- crates/odin-daemon/src/main.rs | 82 +++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 35 deletions(-) diff --git a/crates/odin-daemon/src/main.rs b/crates/odin-daemon/src/main.rs index 191a456..c81139f 100644 --- a/crates/odin-daemon/src/main.rs +++ b/crates/odin-daemon/src/main.rs @@ -148,8 +148,10 @@ struct RuntimeState { write_lease: Option, write_lease_path: PathBuf, /// The largest snapshot response the server sends: its own - /// `max_snapshot_response_bytes`, handed over where the server is built. - max_snapshot_response_bytes: usize, + /// `max_snapshot_response_bytes`, set by `serve_with` from the options it + /// builds the server with. `None` until then, and Odin cannot activate + /// without it. + max_snapshot_response_bytes: Option, recent_warming_proofs: VecDeque<(String, u64)>, /// Counts from this launch. Every launch is a fresh activation, and /// presences are ordered only within one activation. @@ -179,11 +181,7 @@ impl CultMeshRudpSnapshotSource for SnapshotHandle { } impl RuntimeState { - fn open( - options: Options, - candidate: SocketAddr, - max_snapshot_response_bytes: usize, - ) -> Result { + fn open(options: Options, candidate: SocketAddr) -> Result { let authority_material = load_runtime_authority(Path::new(&required_environment( RUNTIME_BUNDLE_ENVIRONMENT, )?))?; @@ -246,7 +244,7 @@ impl RuntimeState { topology: None, write_lease: None, write_lease_path: PathBuf::from(required_environment(PROCESS_WRITE_LEASE_ENVIRONMENT)?), - max_snapshot_response_bytes, + max_snapshot_response_bytes: None, recent_warming_proofs: VecDeque::new(), publisher_sequence: 0, log_gate: RefCell::default(), @@ -261,6 +259,9 @@ impl RuntimeState { if self.activated() { return Ok(true); } + let limit = self + .max_snapshot_response_bytes + .context("Odin cannot activate before its server is built")?; let Some(lease) = acquire_process_write_lease( &self.write_lease_path, &self.authority_material, @@ -285,12 +286,9 @@ impl RuntimeState { // not this contract's and would otherwise be served to the Verse as // current. store.retire_legacy_correlations(); - for (record_type, key, size) in - drop_unservable_documents(&store, self.max_snapshot_response_bytes) - { + for (record_type, key, size) in drop_unservable_documents(&store, limit) { eprintln!( - "Odin dropped {record_type} {key} from its store: its snapshot response is {size} bytes, over the {} the server sends", - self.max_snapshot_response_bytes + "Odin dropped {record_type} {key} from its store: its snapshot response is {size} bytes, over the {limit} the server sends" ); } let signer = self @@ -771,18 +769,10 @@ fn main() -> Result<()> { ); let socket = UdpSocket::bind(candidate) .with_context(|| format!("binding Odin CultNet RUDP candidate {candidate}"))?; - let server_options = CultMeshRudpDocumentServerOptions::default(); - let state = Rc::new(RefCell::new(RuntimeState::open( - options, - candidate, - server_options.max_snapshot_response_bytes, - )?)); - let mut server = CultMeshRudpDocumentServer::new( + let (state, mut server) = serve_with( socket, - SinkHandle(state.clone()), - SnapshotHandle(state.clone()), - CultMeshSystemClock::default(), - server_options, + CultMeshRudpDocumentServerOptions::default(), + RuntimeState::open(options, candidate)?, )?; // Idunn runs this process as PID 1 of its own PID namespace, and a @@ -815,6 +805,27 @@ fn main() -> Result<()> { serve(&state, &mut server, &mut ServingTimers::default(), &stopping) } +/// Build the server over `socket` and hand it `state`, from one set of +/// options: the largest response the server sends is the limit `state` +/// sizes stored documents against when it activates, so the two cannot +/// disagree. +fn serve_with( + socket: UdpSocket, + options: CultMeshRudpDocumentServerOptions, + mut state: RuntimeState, +) -> Result<(Rc>, OdinServer)> { + state.max_snapshot_response_bytes = Some(options.max_snapshot_response_bytes); + let state = Rc::new(RefCell::new(state)); + let server = CultMeshRudpDocumentServer::new( + socket, + SinkHandle(state.clone()), + SnapshotHandle(state.clone()), + CultMeshSystemClock::default(), + options, + )?; + Ok((state, server)) +} + /// Serve until stopped on request or ended by a condition `survive` returns /// (or a dead socket). Every way out makes the last write (`stop`). fn serve( @@ -1818,7 +1829,7 @@ mod tests { topology: None, write_lease: None, write_lease_path: lease_path.clone(), - max_snapshot_response_bytes: options.max_snapshot_response_bytes, + max_snapshot_response_bytes: None, recent_warming_proofs: VecDeque::new(), publisher_sequence: 0, log_gate: RefCell::default(), @@ -1885,19 +1896,20 @@ mod tests { "test projection CAS failed" ); + // Without its server Odin has no limit to size stored documents + // against, so it does not activate. ensure!( - runtime.try_activate()?, + runtime.try_activate().is_err(), + "Odin activated before its server was built" + ); + // Built as `main` builds it, so the activation below sizes stored + // documents against this server's own limit. + let server_socket = socket.try_clone()?; + let (state, server) = serve_with(socket, options, runtime)?; + ensure!( + state.borrow_mut().try_activate()?, "Odin did not activate in the fixture" ); - let state = Rc::new(RefCell::new(runtime)); - let server_socket = socket.try_clone()?; - let server = CultMeshRudpDocumentServer::new( - socket, - SinkHandle(state.clone()), - SnapshotHandle(state.clone()), - CultMeshSystemClock::default(), - options, - )?; Ok(OdinWorld { _temp: temp, state, From f9b0926830c94eb58902f6ea4fd37b442eeda46f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 18:38:13 +0200 Subject: [PATCH 31/43] Odin: a target being replaced has one presence in the catalog, not a refusal The catalog refused itself whole ("duplicate public document identities") whenever two projected incarnations of one target both held an admitted presence, which every provider has during a deploy window: presences are served under (schema, target), and the incumbent and the candidate share it. One is now served and the other skipped, logged once per interval: the incarnation whose Odin correlation is Ready, and between two that are both or neither Ready, the later observation. A duplicate identity between anything but two presences still refuses the query. Tests: the incumbent (Ready) is served over a newer Warming candidate; with no correlations to decide, the later presence is served, from either side. Soul S-4. Co-Authored-By: Claude Opus 5.5 --- crates/odin-daemon/src/lib.rs | 13 +++ crates/odin-daemon/src/main.rs | 197 ++++++++++++++++++++++++++++++--- 2 files changed, 193 insertions(+), 17 deletions(-) diff --git a/crates/odin-daemon/src/lib.rs b/crates/odin-daemon/src/lib.rs index 74a86ce..c9cae33 100644 --- a/crates/odin-daemon/src/lib.rs +++ b/crates/odin-daemon/src/lib.rs @@ -554,6 +554,19 @@ impl MemoryOdinTopologyStore { } } + /// Whether Odin's correlation of this incarnation says it is Ready. No + /// correlation, or one that does not decode, is not Ready. + pub fn correlation_ready(&self, incarnation: &IncarnationRef) -> bool { + self.records + .borrow() + .get(&( + OdinRuntimeTopologyCorrelationRecord::TYPE.to_owned(), + incarnation.key(), + )) + .and_then(|envelope| decode_correlation(&envelope.payload).ok()) + .is_some_and(|correlation| correlation.ready) + } + pub fn correlated_incarnations(&self) -> BTreeSet { self.records .borrow() diff --git a/crates/odin-daemon/src/main.rs b/crates/odin-daemon/src/main.rs index c81139f..fc074cf 100644 --- a/crates/odin-daemon/src/main.rs +++ b/crates/odin-daemon/src/main.rs @@ -586,7 +586,9 @@ impl RuntimeState { /// projection is read once per query; a projection that cannot be read at /// all is every presence's failure and no one else's, so the query is /// answered as if Idunn projected nothing: peer documents are served and - /// presences skipped. Odin's + /// presences skipped. Two incarnations of one target holding presences + /// under the same identity are never a refusal either: one is served (see + /// `presence_precedence`) and the other skipped. Odin's /// correlations are not part of the catalog: Idunn reads them from the /// store file, and a target has one per incarnation, so keying them by /// target would collide for the whole deploy window. @@ -621,12 +623,42 @@ impl RuntimeState { continue; } let identity = (document.schema_id.clone(), document.record_key.clone()); - ensure!( - selected.insert(identity, document).is_none(), - "Odin catalog contains duplicate public document identities" - ); + let precedence = self.presence_precedence(envelope); + match selected.entry(identity) { + std::collections::btree_map::Entry::Vacant(slot) => { + slot.insert((document, precedence)); + } + std::collections::btree_map::Entry::Occupied(mut slot) => { + let (Some(this), Some(held)) = (precedence, slot.get().1) else { + bail!("Odin catalog contains duplicate public document identities"); + }; + self.log_repeating( + &format!("catalog serves one presence of {}", document.record_key), + "more than one incarnation holds an admitted presence; the Ready one, or else the latest, is served" + .into(), + ); + if this > held { + slot.insert((document, precedence)); + } + } + } } - Ok(selected.into_values().collect()) + Ok(selected.into_values().map(|(document, _)| document).collect()) + } + + /// The order in which presences of one target are preferred when more than + /// one incarnation of it holds an admitted presence, as every target does + /// while it is being replaced: the incarnation whose correlation is Ready + /// first, then the latest observation. `None` for anything but a presence, + /// which has one record per identity and never competes. + fn presence_precedence(&self, envelope: &CultCacheEnvelope) -> Option<(bool, u64)> { + if envelope.r#type != GameCultRuntimePresenceHealthRecord::TYPE { + return None; + } + let presence = decode_presence(&envelope.payload).ok()?; + let incarnation = IncarnationRef::new(presence.target, presence.expected_projection_sha256); + let ready = self.topology().ok()?.store.correlation_ready(&incarnation); + Some((ready, presence.observed_at_unix_millis)) } /// One stored record as the Verse sees it; `None` when it is not public. @@ -1694,6 +1726,8 @@ mod tests { lease_path: PathBuf, provider_identity_id: String, lease: IdunnProcessWriteLeaseRecord, + /// Idunn's signer, for activations of further incarnations. + idunn_signer: ServiceIdentitySigner, } fn write_single_record(path: &Path, envelope: CultCacheEnvelope) -> Result<()> { @@ -1920,6 +1954,7 @@ mod tests { lease_path, provider_identity_id, lease, + idunn_signer, }) } @@ -2469,6 +2504,127 @@ mod tests { Ok(incarnation) } + /// A candidate incarnation of Odin that Idunn has activated, with its + /// Warming presence admitted beside the incumbent's, as every target has + /// for the moment it is being replaced. The candidate's presence is + /// observed after every presence already held. + fn admit_candidate_presence(odin: &OdinWorld) -> Result { + let now = unix_millis()?; + let mut candidate = odin.state.borrow().authority_material.expected.clone(); + candidate.incarnation_id = "odin/generation-2".into(); + candidate.sealed_release_id = digest('9'); + candidate.validate()?; + let incarnation = IncarnationRef::of(&candidate)?; + let launch = + IdunnRuntimeActivationLaunch::issue(&candidate, digest('8'), now - 10, &odin.idunn_signer)?; + let activation = launch.activation().clone(); + let mut credential = Vec::new(); + launch.write_credential(&mut credential)?; + odin.append_projection(vec![ + CultCacheEnvelope { + key: incarnation.key(), + r#type: IdunnExpectedIncarnationRecord::TYPE.into(), + payload: candidate.canonical_bytes()?, + stored_at: rfc3339_millis(now - 30)?, + schema_id: Some(IDUNN_EXPECTED_INCARNATION_SCHEMA.into()), + }, + CultCacheEnvelope { + key: incarnation.key(), + r#type: IdunnRuntimeActivationRecord::TYPE.into(), + payload: activation.canonical_bytes()?, + stored_at: rfc3339_millis(activation.issued_at_unix_millis)?, + schema_id: Some(IDUNN_RUNTIME_ACTIVATION_SCHEMA.into()), + }, + ])?; + // Signed as the candidate process would sign it: its own Expected and + // activation, no lease yet. + thread::sleep(Duration::from_millis(5)); + let document = { + let mut state = odin.state.borrow_mut(); + let authority = &mut state.authority_material; + let expected_sha256 = candidate.canonical_sha256()?; + let activation_sha256 = activation.canonical_sha256()?; + let signer = IdunnRuntimeActivationSigner::from_credential_reader(credential.as_slice())?; + let incumbent = ( + std::mem::replace(&mut authority.expected, candidate), + std::mem::replace(&mut authority.expected_sha256, expected_sha256), + std::mem::replace(&mut authority.activation, activation), + std::mem::replace(&mut authority.activation_sha256, activation_sha256), + std::mem::replace(&mut authority.activation_signer, signer), + ); + let lease = state.write_lease.take(); + let signed = state.signed_presence_document("warming", "test candidate"); + state.write_lease = lease; + let authority = &mut state.authority_material; + ( + authority.expected, + authority.expected_sha256, + authority.activation, + authority.activation_sha256, + authority.activation_signer, + ) = incumbent; + signed? + }; + provider_put_document(odin, document)?; + Ok(incarnation) + } + + /// The presence of `target` the catalog serves, by the incarnation it names. + fn served_presence_incarnation(odin: &OdinWorld) -> Result { + let catalog = odin.catalog()?; + let presences: Vec<_> = catalog + .iter() + .filter(|document| document.schema_id == GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA) + .collect(); + ensure!(presences.len() == 1, "{} presences served", presences.len()); + Ok(decode_presence(&presences[0].payload)?.expected_projection_sha256) + } + + /// While a target is being replaced, its incumbent and its candidate each + /// hold an admitted presence under the same catalog identity. The catalog + /// serves one: the incarnation whose correlation is Ready, even though the + /// other is newer. + #[test] + fn a_target_being_replaced_serves_the_presence_of_its_ready_incarnation() -> Result<()> { + let mut odin = activated_odin()?; + odin.pass()?; + let incumbent = self_incarnation(&odin.state.borrow().authority_material); + let candidate = admit_candidate_presence(&odin)?; + { + let state = odin.state.borrow(); + let store = &state.topology.as_ref().unwrap().store; + assert!(store.correlation_ready(&incumbent)); + assert!(!store.correlation_ready(&candidate), "the candidate is still Warming"); + } + assert_eq!(served_presence_incarnation(&odin)?, incumbent.expected_sha256); + Ok(()) + } + + /// With no correlation to tell them apart, the later presence is served, + /// whichever incarnation holds it. + #[test] + fn a_target_being_replaced_serves_the_later_presence_when_neither_is_ready() -> Result<()> { + let odin = activated_odin()?; + let incumbent = self_incarnation(&odin.state.borrow().authority_material); + odin.heartbeat()?; + let candidate = admit_candidate_presence(&odin)?; + let forget_correlations = || { + let state = odin.state.borrow(); + let store = &state.topology.as_ref().unwrap().store; + for incarnation in [&incumbent, &candidate] { + store.remove(OdinRuntimeTopologyCorrelationRecord::TYPE, &incarnation.key()); + } + }; + forget_correlations(); + assert_eq!(served_presence_incarnation(&odin)?, candidate.expected_sha256); + + thread::sleep(Duration::from_millis(5)); + odin.heartbeat()?; + forget_correlations(); + assert_eq!(served_presence_incarnation(&odin)?, incumbent.expected_sha256); + Ok(()) + } + /// Idunn projects the incumbent and the candidate of a target for the whole /// deploy window, and Odin writes a correlation for each. The catalog does /// not serve correlations, so two of one target cannot collide in it, and @@ -2832,6 +2988,23 @@ mod tests { key: &str, payload: Vec, ) -> Result<()> { + provider_put_document( + odin, + CultNetRawDocumentRecord { + schema_id: schema.into(), + record_key: key.into(), + stored_at: rfc3339_millis(unix_millis()?)?, + payload_encoding: CultNetRawPayloadEncoding::Messagepack, + payload, + source_runtime_id: None, + source_agent_id: None, + source_role: None, + tags: None, + }, + ) + } + + fn provider_put_document(odin: &OdinWorld, document: CultNetRawDocumentRecord) -> Result<()> { odin.state .borrow_mut() .accept_raw_document(CultMeshRudpRawDocumentReceipt { @@ -2842,17 +3015,7 @@ mod tests { message_id: "put".into(), transport_sequence: 1, received_at_unix_millis: unix_millis()?, - document: CultNetRawDocumentRecord { - schema_id: schema.into(), - record_key: key.into(), - stored_at: rfc3339_millis(unix_millis()?)?, - payload_encoding: CultNetRawPayloadEncoding::Messagepack, - payload, - source_runtime_id: None, - source_agent_id: None, - source_role: None, - tags: None, - }, + document, }) } From e2c0c8f6264c8fcfa2cd587e4e29ea00feee7987 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 18:38:30 +0200 Subject: [PATCH 32/43] Odin tests: a later failed attempt, and a removal on its own Two rules had no test of their own (Soul's mutants survived): - only_oldest_attempt_tried: every fixture's file held the oldest failed attempt. The new test leaves the file holding a later one. - remove_not_dirty: the heartbeat always dirtied the store, so a removal that did not mark it survived. The new test removes a record with nothing else changed and checks the write drops it (and that removing what is not held writes nothing). Soul S-8 and the attempted-list gap. Co-Authored-By: Claude Opus 5.5 --- crates/odin-daemon/src/lib.rs | 50 +++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/crates/odin-daemon/src/lib.rs b/crates/odin-daemon/src/lib.rs index c9cae33..7549b75 100644 --- a/crates/odin-daemon/src/lib.rs +++ b/crates/odin-daemon/src/lib.rs @@ -2197,6 +2197,56 @@ mod tests { Ok(()) } + /// Two writes fail before replacing the file, and the file then comes to + /// hold the later attempt's records (the state a failed directory sync of + /// that attempt would leave). Any failed attempt may be what the file + /// holds, not only the first: the next write is made over it. + #[test] + fn the_file_may_hold_a_later_failed_attempt_not_only_the_first() -> Result<()> { + let temp = TempDir::new()?; + let path = temp.path().join("topology.cc"); + let file = SingleFileMessagePackBackingStore::new(&path); + let store = MemoryOdinTopologyStore::load(&path)?; + std::fs::write(&path, b"not a cultcache store")?; + store.put(peer_document("doc-1")?); + assert!(store.flush().is_err()); + store.put(peer_document("doc-2")?); + assert!(store.flush().is_err()); + std::fs::remove_file(&path)?; + ensure!(file.compare_exchange_snapshot( + &[], + &[peer_document("doc-1")?, peer_document("doc-2")?] + )?); + + store.put(peer_document("doc-3")?); + assert!(store.flush()?, "the later attempt is Odin's own"); + assert_eq!( + file.pull_all_read_only_snapshot()?, + vec![peer_document("doc-1")?, peer_document("doc-2")?, peer_document("doc-3")?] + ); + Ok(()) + } + + /// Removing a record is a change of its own: the next flush writes the + /// file without it, with nothing else changed. + #[test] + fn a_removed_record_is_written_out() -> Result<()> { + let temp = TempDir::new()?; + let path = temp.path().join("topology.cc"); + let file = SingleFileMessagePackBackingStore::new(&path); + let store = MemoryOdinTopologyStore::load(&path)?; + store.put(peer_document("doc-1")?); + store.put(peer_document("doc-2")?); + assert!(store.flush()?); + + store.remove("ghostlight.doc", "doc-1"); + assert!(store.flush()?, "the removal is written"); + assert_eq!(file.pull_all_read_only_snapshot()?, vec![peer_document("doc-2")?]); + store.remove("ghostlight.doc", "doc-1"); + assert!(!store.flush()?, "removing what is not held changes nothing"); + Ok(()) + } + fn file_sequences(path: &Path) -> Result<(u64, Vec)> { let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; let mark = unique_envelope(&entries, TOPOLOGY_PUBLISHER_WATERMARK_TYPE, "ghostlight")? From 30156f21d2c046872adc0011a4703a632a42e9a9 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 18:39:00 +0200 Subject: [PATCH 33/43] Odin: Odin's answer to a put is the promise, not the transport's ack dcd36fc said an acknowledged put is always on disk. At the pinned cultmesh-rs that is false: RUDP acknowledges a reliable datagram when it is received, before ordered delivery hands the put to Odin, so a later put can be acknowledged while an earlier one is still being resent (Soul probed it with two pipelined puts and the first datagram dropped). What holds is Odin's own answer: a put accepted by accept_raw_document is on disk, and one it refuses is rejected by name. The comments now say that. Soul S-5. The transport defect is tracked by Self. Co-Authored-By: Claude Opus 5.5 --- crates/odin-daemon/src/main.rs | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/crates/odin-daemon/src/main.rs b/crates/odin-daemon/src/main.rs index fc074cf..5c2b03f 100644 --- a/crates/odin-daemon/src/main.rs +++ b/crates/odin-daemon/src/main.rs @@ -73,7 +73,7 @@ const PROJECTION_REFRESH_INTERVAL: Duration = Duration::from_millis(250); // Odin's own changes (its heartbeat, correlations, the watermark) are written // at most once per interval, and not at all when nothing changed. A crash loses // at most this much of them; a correlation sequence that was never written was -// never published. A peer's put is written before it is acknowledged +// never published. A peer's put is written before Odin accepts it // (`accept_raw_document`). Operator ruling Q3 B, docs/write-pattern-cut.md. const FLUSH_INTERVAL: Duration = Duration::from_secs(1); const IDLE_POLL_INTERVAL: Duration = Duration::from_millis(2); @@ -329,14 +329,21 @@ impl RuntimeState { store.flush() } - /// A peer's put is written before it is acknowledged: the server sends the - /// acknowledgement only when this returns `Ok`, and it returns `Ok` only + /// A peer's put is written before Odin accepts it: this returns `Ok` only /// once the store holding the put is on disk. The write is the one `flush`, /// so it carries everything else that changed too. A write that fails - /// refuses the put. + /// refuses the put, and the server answers a refusal with an application + /// rejection naming the put. /// - /// The promise runs one way only. An acknowledged put is always on disk. One - /// refused because its write failed may still be: it stays in the working set, so the catalog + /// What this promises is Odin's answer, not the transport's. The RUDP + /// acknowledgement of the datagram carrying a put is sent when it is + /// received, before ordered delivery hands the put here (a later put can + /// be acknowledged while an earlier one is still being resent), so it says + /// nothing about durability. A put accepted here is on disk; a put refused + /// here is rejected by name. + /// + /// A refusal runs one way only. A put refused because its write failed may + /// still become durable: it stays in the working set, so the catalog /// serves it and the next write lands it (or the failed write had already /// replaced the file). That is sound because a put is the latest value for /// its type and key, not an event: a publisher that retries a refused put @@ -3394,7 +3401,7 @@ mod tests { Ok(()) } - /// A put whose write is not made is refused, so it is never acknowledged: + /// A put whose write is not made is refused, so it is never accepted: /// with the write failing, and with the lease lost. #[test] fn a_put_whose_write_is_not_made_is_refused() -> Result<()> { From 434756e4f2b495ddd8d4cafbcab7cbfbef0a674b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 18:39:32 +0200 Subject: [PATCH 34/43] Odin test: pipelined puts with a lost datagram are both written Adapted from Soul's S-6 probe. Two puts go out on one session through a relay that drops the first reliable datagram. The transport resends it and delivers both in order; Odin accepts both, so both are on disk and the file matches the working set. The probe's other finding, that the transport acknowledges the second before the first is resent, is the transport's defect (tracked by Self) and is not pinned here. Co-Authored-By: Claude Opus 5.5 --- crates/odin-daemon/src/main.rs | 109 +++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/crates/odin-daemon/src/main.rs b/crates/odin-daemon/src/main.rs index 5c2b03f..509e499 100644 --- a/crates/odin-daemon/src/main.rs +++ b/crates/odin-daemon/src/main.rs @@ -3401,6 +3401,115 @@ mod tests { Ok(()) } + /// Two puts pipelined on one session with the first datagram lost on the + /// way: the transport resends it and delivers both in order, and Odin + /// accepts, and so writes, both. The transport may acknowledge the second + /// before the first is resent (see `accept_raw_document`); what holds is + /// that neither put is lost and both are on disk once delivered. + #[test] + fn pipelined_puts_with_a_lost_datagram_are_both_written() -> Result<()> { + let mut odin = activated_odin()?; + odin.pass()?; + let server = odin.server.local_addr()?; + let acknowledged = odin.serve_while(move || -> Result { + // A relay that drops the client's first reliable data datagram. + let relay_socket = UdpSocket::bind("127.0.0.1:0")?; + relay_socket.set_read_timeout(Some(Duration::from_millis(5)))?; + let relay_addr = relay_socket.local_addr()?; + let stop = Arc::new(AtomicBool::new(false)); + let relay = thread::spawn({ + let stop = stop.clone(); + move || -> Result<()> { + let mut client = None; + let mut dropped = false; + let mut buffer = vec![0u8; 65_535]; + while !stop.load(Ordering::Relaxed) { + let Ok((length, from)) = relay_socket.recv_from(&mut buffer) else { + continue; + }; + let datagram = &buffer[..length]; + if from == server { + if let Some(client) = client { + relay_socket.send_to(datagram, client)?; + } + continue; + } + client = Some(from); + let reliable_data = cultnet_rs::decode_rudp_packet(datagram).is_ok_and( + |packet| { + packet.packet_type == cultnet_rs::CultNetRudpPacketType::Data + && packet.reliable + }, + ); + if reliable_data && !dropped { + dropped = true; + continue; + } + relay_socket.send_to(datagram, server)?; + } + Ok(()) + } + }); + let socket = UdpSocket::bind("127.0.0.1:0")?; + socket.set_read_timeout(Some(Duration::from_millis(5)))?; + let mut transport = CultNetRudpSocketTransportConnection::new( + CultNetRudpSocketTransportOptions::client("pipeliner", socket, relay_addr, 0x6100_0001), + )?; + transport.connect(Vec::new())?; + let deadline = Instant::now() + Duration::from_secs(3); + while !transport.connected() { + let _ = transport.receive_once()?; + transport.poll_resends()?; + ensure!(Instant::now() < deadline, "timed out connecting"); + } + let put = |key: &str| -> Result> { + let message = CultNetMessage::DocumentPutRaw { + message_id: format!("put-{key}"), + document: CultNetRawDocumentRecord { + schema_id: "ghostlight.doc.v1".into(), + record_key: key.into(), + stored_at: rfc3339_millis(unix_millis()?)?, + payload_encoding: CultNetRawPayloadEncoding::Messagepack, + payload: vec![7; 64], + source_runtime_id: None, + source_agent_id: None, + source_role: None, + tags: None, + }, + }; + Ok(encode_cultnet_message_to_vec(&message, CultNetWireContract::CultNetSchemaV0)?) + }; + let receipts = [ + transport.send_reliable("schema", put("first")?)?, + transport.send_reliable("schema", put("second")?)?, + ]; + let deadline = Instant::now() + Duration::from_secs(5); + let all_acknowledged = |transport: &CultNetRudpSocketTransportConnection| { + receipts.iter().all(|receipt| { + transport.reliable_send_status(receipt) + == CultNetRudpReliableSendStatus::Acknowledged + }) + }; + while !all_acknowledged(&transport) && Instant::now() < deadline { + let _ = transport.receive_once()?; + transport.poll_resends()?; + } + stop.store(true, Ordering::Relaxed); + relay.join().expect("relay thread panicked")?; + Ok(all_acknowledged(&transport)) + })??; + assert!(acknowledged, "both puts were delivered"); + let written: Vec = odin + .file_records()? + .into_iter() + .filter(|entry| entry.r#type == "ghostlight.doc") + .map(|entry| entry.key) + .collect(); + assert_eq!(written, vec!["first".to_owned(), "second".to_owned()]); + assert_eq!(odin.file_records()?, odin.working_set()); + Ok(()) + } + /// A put whose write is not made is refused, so it is never accepted: /// with the write failing, and with the lease lost. #[test] From 47850dfcd6d9afe83811da5bf30a9be400e0fd9a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 18:43:46 +0200 Subject: [PATCH 35/43] Write map: Q5 ruled A (writer thread with group commit) Co-Authored-By: Claude Opus 5.5 --- docs/write-pattern-cut.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/write-pattern-cut.md b/docs/write-pattern-cut.md index e84fd71..ddbaea9 100644 --- a/docs/write-pattern-cut.md +++ b/docs/write-pattern-cut.md @@ -434,7 +434,8 @@ records; update the check's type list in the same pass. Odin healthy while its catalog is stuck. - C: reopen Q3 and go back to the 1 s interval. That brings back lost acknowledged puts on a crash. - **Recommended: A.** + **Recommended: A.** **RULED A by the operator, 2026-09-30: "agree on Odin writer thread".** Mapping is next + (Imagination): the cultmesh-rs deferred put reply, then the Odin writer thread with group commit. ## 8. Related findings From 95522dcd0ed7f11e0864254379ad1ab97301e00b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 19:03:15 +0200 Subject: [PATCH 36/43] Write map: Q5 cut map (C5 deferred put reply, W0 pin bump, W1 writer thread); Q5-A, Q5-B open Co-Authored-By: Claude Opus 5.5 --- docs/write-pattern-cut.md | 509 +++++++++++++++++++++++++++++++++++++- 1 file changed, 508 insertions(+), 1 deletion(-) diff --git a/docs/write-pattern-cut.md b/docs/write-pattern-cut.md index ddbaea9..60b48cc 100644 --- a/docs/write-pattern-cut.md +++ b/docs/write-pattern-cut.md @@ -435,7 +435,514 @@ records; update the check's type list in the same pass. - C: reopen Q3 and go back to the 1 s interval. That brings back lost acknowledged puts on a crash. **Recommended: A.** **RULED A by the operator, 2026-09-30: "agree on Odin writer thread".** Mapping is next - (Imagination): the cultmesh-rs deferred put reply, then the Odin writer thread with group commit. + (Imagination): the cultmesh-rs deferred put reply, then the Odin writer thread with group commit. Mapped below in + "Q5 cut: the writer thread" (cuts C5, W0, W1; forks Q5-A and Q5-B). + +## Q5 cut: the writer thread + +**Imagination pass (Opus), 2026-09-30.** Maps the operator's ruling on Q5, "agree on Odin writer thread" (option A, +section 7). Anchors: +- Odin `hands/odin-write` at `434756e` (`crates/odin-daemon/src/main.rs`, `lib.rs`), CultLib pin `3bf1c0ce`. +- CultLib `hands/cultmesh-put-serve-bound` tip `e4e2a0b9` (`packages/cultmesh-rs/src/rudp_document_server.rs`). + Its merge base is `42ba0f08`, which already holds ack Cuts 1, 1b and 1c. +- CultLib `hands/ack-cutD` tip `e9d5ae2c` (lands on main first), and R1 batch 3 `hands/cultcache-read-failures` tip + `93ac5945` (`CultCacheStoreWriteFailed`). + +The mapping raises two forks, **Q5-A** and **Q5-B** (at the end of this section). Cuts C5 and W1 wait on Q5-A. + +### Q5.1 Body facts (probed and read) + +**Probe O1, Odin `434756e` + scratch test, Yggdrasil slot 3 (`imag-q5-odin@31953c93`).** A test process holds the +store's sibling lock for 1,500 ms, the way Soul modelled a disk stall. Full serving passes run meanwhile. A route +challenge is sent at 200 ms, then a catalog read. + +| Case | Route challenge | Catalog read | +|---|---|---| +| nothing waiting to be written | served in **39 ms** | served in 29 ms | +| only Odin's own heartbeat waiting, and the interval write due | served in **1,342 ms** | 33 ms (sent after the release) | + +- No read path takes the store lock. At `434756e` the lock is taken only by `MemoryOdinTopologyStore::load` + (`lib.rs:509`, at activation) and by `flush` (`lib.rs:612` → `compare_exchange_snapshot`). +- The stall reaches the loop through **both** writes on it: a peer put (`main.rs:363`, Soul S-1) and the interval + write of Odin's own bookkeeping (`main.rs:933-941`). The interval write stalls the loop with no peer put at all. + So the bookkeeping write must move to the writer too, not only the put write. + +**The loop still reads two other files.** They are not the store and take no lock, but they are disk reads on the +loop: +- The lease file, on every route challenge and catalog read: `raw_snapshot` `main.rs:389-409` → + `require_current_write_lease` `:546-567` → `read_process_write_lease` `:1323` (`pull_all_read_only_snapshot`). +- Idunn's projection, on every catalog read (`:608`), refresh (`:572`) and heartbeat (`:524-544`). +- Both files are small and rewritten rarely by Idunn, so they are normally served from the page cache. Probe O1 + cannot show whether a real disk stall evicts them. Only the Yggdrasil rerun (Q5.6) can. If it does, the answer + is to hold the lease record in memory and re-check it where it is decided. That is its own follow-up, not this + cut. + +**What a put's reply is on the wire today.** CultNet has no put-acknowledgement message: the list in +`cultnet-rs/src/contracts.rs` has `document_put_raw` and `error.v0`, and no put result. +- A Rust publisher (`publish_cultnet_message_to_rudp_catalog`, `cultmesh-rs/src/lib.rs:871-924` at `e4e2a0b9`) + succeeds when its reliable packet is acknowledged by the RUDP transport, and fails on a `cultnet.error.v0`. + - Its default `flush_timeout` is 300 ms, with a resend every 50 ms (`:144-157`). +- The document server sends that acknowledgement at the end of `poll_once`, after the sink returns + (`rudp_document_server.rs:468-476`). A refusal ends the session with an Error frame that acknowledges nothing + (`end_rejected_session` `:760-793`, put-serve branch). +- **So today the put's reply is the transport acknowledgement.** That is how Q3 B's "an ACK means durable" holds: + the sink writes before it returns. + +**Every packet a session sends carries acknowledgement fields** (`ack`, `ack_mask`, from `ack_state`, cultnet-rs +`rudp.rs` at main). +- The fields report every *received* reliable sequence, not only delivered ones. +- `create_packet` stamps them on each new packet: data, Pong, the end-of-poll Ack. +- A reliable packet is stored with the fields it was created with, and resent unchanged (`track_reliable`, + `due_resends`). + +Probes P1-P4 (cultnet-rs sessions at `e4e2a0b9`, scratch test `imag_q5_probe.rs`, Yggdrasil, `imag-q5-cultlib@8dcfe36a`): + +| # | Case | Result | +|---|---|---| +| P1 | two pipelined ordered puts, the first lost | the second is delivered to nobody, yet `create_ack` acknowledges it. **S-5 is still open after ack Cuts 1-1c.** `ack_state` at main is the same code, so the ack map's to-do resolves to "not closed by the pin bump" | +| P2 | a put delivered, then a Ping | the Pong acknowledges the put | +| P3 | the resend of a reply created **before** the put | does not acknowledge it | +| P3 | a reliable reply created **after** the put | acknowledges it, and so does its resend | +| P4 | a retransmit of a delivered put | not delivered again (the server's end-of-poll ack still answers it, `:468`) | + +Consequence for a reply sent later: holding back only the end-of-poll ack is not enough. +- A Pong, or a snapshot response created on the same session after the put arrived, acknowledges the put too. +- The snapshot response keeps doing so on every resend, even if its first copy were patched. +- So a session with a put awaiting its answer must be **sent nothing created after that put arrived**. Resends of + earlier packets are safe: they carry the fields they were created with. + +**The other runtimes (parity).** +- **TS** (`cultmesh-ts/src/index.ts:5891-5914` at `e4e2a0b9`): + - It acknowledges each packet at receipt (`createAckForReceived`), before the handler runs. + - It runs each session's frames through one promise chain (`record.work`), so handling is serial per session. + - `onDocumentPutRaw` may return a Promise and an application-level receipt document, which is sent when it + resolves (`:5951-5972`). + - So TS already "replies later", but at the application level, and its transport ack does not mean handled. +- **Python** (`cultmesh-py/src/cultmesh_py/server.py:407`, `_handle_raw_put` `:415`): the handler is synchronous + and returns its responses. +- Neither has a durability meaning for its ack. Parity is follow-up **F-Q5a** (Q5.8). + +**Consumers of the sink trait** (`CultMeshRudpRawDocumentSink`): +- Odin's `SinkHandle` (`main.rs:168-172`). +- Ratatoskr's catalog test (`Ratatoskr/crates/ratatoskr-core/tests/catalog.rs:29`). +- CultLib's own tests, which use closures through the blanket impl (`rudp_document_server.rs:60-67`). +- Vendored copies (StreamPixels, Heimdall) do not implement it. + +**cultcache-rs write outcome (R1 batch 3, `93ac5945`).** +- Every failed single-file write carries `CultCacheStoreWriteFailed { kind: NotReplaced | ReplacedNotDurable }`. +- Any other error from `compare_exchange_snapshot` (lock, read, decode, validation) comes before the write step, + so the file is unchanged. +- One hole: `with_exclusive_lock` (`cultcache-rs/src/lib.rs:924-932` on that branch) returns the **unlock** error + after a write that succeeded, with no marker. Odin would read that as "not replaced". Its next write would then + find its own snapshot and report `ForeignStoreWrite`, and Odin would end. + - That is rare (unlock failing), and it belongs to R1: follow-up **F-Q5b**, not this cut. + +### Q5.2 Target shape + +1. **CultLib (cut C5).** A put's sink may answer later. + - While any put of a session awaits its answer, the server sends that session nothing new. + - The one exception is resends of packets created before the put arrived. + - A snapshot request behind the pending put waits in the session's order. + - An accepted answer releases one ordinary acknowledgement. + - A refusal ends the session with the Error frame the put-serve branch sends. + - The wire does not change, and existing publishers keep "acknowledged means the sink accepted", which for Odin + is durable (Q5-A). +2. **Odin (cut W1).** One writer thread owns the store file after activation. + - The loop owns the working set, decides when to write, and answers put replies from the writer's outcomes. + - Every put accepted while a write is in flight is covered by the next single write (group commit). + - Route challenges and catalog reads never wait on the writer. +3. **Odin (cut W0), before W1.** + - A pin bump. + - S-2: a `served_record` override so Odin's admission is exact. + - S-3: the `attempted` list dies, because the typed write outcome says what the file holds. + +### Q5.3 Identity, lifecycle, authority + +| Kind | Named by | Over time | Decided by | +|---|---|---|---| +| pending put (CultLib) | a server-minted `u64` put id, unique for the server's life, never reused across session generations | created when an admitted put is offered to the sink; ends when it is answered, or when its session ends (the answer is then discarded) | the sink answers; the server alone sends | +| put reply handle (CultLib) | the put id, inside a `CultMeshRudpPutReply` | owned by whoever the sink hands it to; consumed by `accept` or `refuse`; dropping it unanswered refuses | the sink's owner | +| working-set version (Odin) | a `u64` counter in `MemoryOdinTopologyStore` | +1 on every mutation that changes a record; never reset within an activation | the loop | +| written-through version (Odin) | a `u64` in `RuntimeState` | raised only by a writer outcome `Written { version }` | the writer's outcome, applied by the loop | +| write request (Odin) | its version | at most one in flight; replaced by nothing (the next request is built when the outcome arrives) | the loop submits; the writer executes | +| what the file holds (Odin, `flushed`) | none: one snapshot | set at activation from the load; replaced on `Ok(true)` and on `ReplacedNotDurable` | the writer alone | + +"Dirty" stops being a flag (`lib.rs:502`, `dirty: Cell`): it is derived as `version > written_through`. + +### Q5.4 Cuts + +#### Cut C5. A put's sink may answer later (CultLib, cultmesh-rs, behaviour) + +- **Repo/branch:** CultLib `hands/cultmesh-put-reply-later` from `main` after the put-serve branch merges (Q5.7). + Anchors below are at put-serve `e4e2a0b9`; they move by Cut D's diff (Q5.7 names the conflicts). +- **Deletes first:** the synchronous sink call and its error mapping in the put arm, `rudp_document_server.rs:621-633` + (they are replaced, not wrapped). Nothing else is deleted. +- **Public API** (default D-1 below): + + ```rust + pub trait CultMeshRudpRawDocumentSink { + /// Answer through `reply` now or later, from any thread. + fn accept_raw_document(&mut self, receipt: CultMeshRudpRawDocumentReceipt, reply: CultMeshRudpPutReply); + } + // Closures keep working and answer at once: + impl Result<()>> CultMeshRudpRawDocumentSink for F { .. } + + #[must_use] pub struct CultMeshRudpPutReply { /* put id, Sender to the server */ } // Send, !Clone + impl CultMeshRudpPutReply { + pub fn accept(self); + pub fn refuse(self, reason: impl std::fmt::Display); + } + // Drop without an answer refuses: "the put was not answered". + ``` + + - `CultMeshRudpRawDocumentReceipt` (`:37-43`) is unchanged; the reply is a separate argument, so the receipt stays + `Clone + PartialEq`. + - `CultMeshRudpPollOutcome` (`:286-290`) is unchanged. A later refusal is returned as `ApplicationRejected` with + `SinkRefused(reason)` from the `poll_once` that sends it. +- **Server changes, `rudp_document_server.rs`:** + - `CultMeshRudpDocumentServer` (`:303-311`) gains: + - The answer channel (`std::sync::mpsc`: the server keeps the receiver, each reply holds a sender). + - `answered: VecDeque`, the answers taken off the channel and not yet handled. + - A next put id. + - `pending: BTreeMap`. + - `SessionEntry` (`:292-297`) gains `pending_puts: BTreeSet` and `waiting: VecDeque<(u32, CultNetMessage)>`, + the delivered frames that may not be answered yet. + - **One predicate owns "withheld":** `fn acknowledgement_withheld(&self, key) -> bool`, true while + `pending_puts` is non-empty. The S-5 fix, if the ack map takes it here, adds its clause to this function and + nowhere else. + - `poll_once`: + - Drain answers first (before `maintain`, `:366`). Each answer is looked up in `pending`; an unknown id (its + session ended) is discarded. + - Refuse → `end_rejected_session(key, reason)`, and return one `ApplicationRejected`. Further answers stay in the + channel for the next poll. + - Accept → remove the id. When the session has none left, it is released: send its ack (the rule for which + poll sends it is under the put arm below), then handle `waiting` in order. + - `:436-438`: a `result.reply` (Pong) is not sent while withheld. + - `:468-476`: the end-of-poll ack is not sent while withheld. + - `deliver_application_message` (`:565`): + - Put arm (`:572`): admission is unchanged (put-serve `:587-619`). Then mint the id, record it, and call the sink + with a reply. + - `poll_once` looks at answers again after the delivered-frame loop (`:445-466`), before the end-of-poll ack, + but handles only those for this poll's session. The rest stay queued for the next poll's first step. (Answers + move from the channel into one server-side queue, so they can be taken selectively.) A sink that answered at + once is therefore acknowledged, or refused, in the same poll with the same datagrams as today, and another + session's refusal can never cut this session's ack short. + - A release sends `create_ack()` from the drain, except for the session whose packet this poll is handling: + that session gets its ordinary end-of-poll ack, so a synchronous sink's datagrams stay exactly as they were. + - Snapshot arm (`:634`): if the session is withheld, push the request onto `waiting` and return; otherwise + answer as today. + - A put that arrives while `waiting` is non-empty also queues, so a session's frames are handled in order. A + put behind another put that is only pending is offered at once, so pipelined puts can share one write. + - Session end, wherever a session is removed (`:441`, `:486-489`, `:523`, `end_refused_session`, + `end_rejected_session`, Cut D's `end_unsendable_session`): remove its ids from `pending`, so their answers are + discarded. + - **Put this in one `remove_session(key)` helper, so no removal path can forget it.** It is the one new helper + the cut earns: there are six removal sites today. + - Doc comments: the struct doc (`:300-302`, "no background thread") stays true. The sink trait doc says what + "acknowledged" means: the sink accepted. +- **Authority map:** + - Owner of what the peer is told about a put: the server, and only through `poll_once`. The sink decides accept or + refuse; it never sends. + - Inputs: sink answers (channel); transport receipts. + - Outputs: one ack per release; one refusal per refused put's session. + - Derived: "withheld" is derived from `pending_puts`; it is not a flag. + - Forbidden writers: + - No send path to a withheld session except `maintain`'s resends. + - The sink must not be able to acknowledge by returning. The return type carries no answer, so this holds by + construction. + - Shared paths: a synchronous closure sink and a deferred sink answer through the same reply and the same drain. + There is no second code path for "answered at once". +- **Verification (CultLib, Yggdrasil):** + - Wire-level test, the observer layer: a raw client socket records every datagram the server sends to its session. + - With the reply held across at least five resend intervals, including a Ping and a snapshot request on that + session, no datagram acknowledges the put's sequence. + - After `accept()`, the next poll sends an ack that does, and then the snapshot response. + - Mutations that must fail it: sending the end-of-poll ack regardless; sending the Pong; answering the snapshot + request at once. + - Another session's snapshot request is answered while the first session's put is held (the loop is not blocked). + - `refuse()` and dropping the reply: the peer receives `cultnet.error.v0` and a goodbye, and nothing acknowledges + the put (reuse the put-serve refusal assertions). + - Two pipelined puts on one session, each with its own held reply: + - Answering the second first releases nothing. + - Answering the first then releases one ack that covers both. + - This pins that a later put's acknowledgement never leaves before an earlier one's. + - Session ends while held (peer Disconnect; a new Connect from the same key): a later `accept()` sends nothing to + the new generation, and `pending` is empty. + - Closure sink: the datagram sequence is byte-identical to the pre-cut server for accept and refuse. This pins that + the synchronous path did not change. + - An unservable put is refused before the sink sees it: the sink is never called and no reply is minted. + - `cargo mutants --in-diff` on the cut's diff; every survivor is triaged. +- **Estimate:** about +170 source, -15 source, +350 tests. No new target, dependency or wire field. + +#### Cut W0. Pin bump, exact admission and the one-candidate store (Odin, subtraction first) + +- **Repo/branch:** Odin `hands/odin-write-w0` from `hands/odin-write` (`434756e`, or batch 4's tip). It bumps the + CultLib pin to the main commit holding Cut D, put-serve, R1 batch 3 and C5 (Q5.7). +- **Deletes first:** + - `attempted` (`lib.rs:494-499`, `:525`), the retry loop over it in `flush` (`:619-625`) and its push (`:628-633`). + - The tests that describe a file holding some older failed attempt: + - `a_write_that_failed_before_replacing_keeps_every_earlier_attempt_odins` (`lib.rs:2172`). + - `the_file_may_hold_a_later_failed_attempt_not_only_the_first` (`:2205`). + - Each models a `NotReplaced` failure that nonetheless changed the file, which the typed outcome now rules out. +- **Changes:** + - `flush` (`lib.rs:612-640`) matches the error's `CultCacheStoreWriteFailed` kind (`downcast_ref`): + - `ReplacedNotDurable` → `flushed = current`, and the store stays dirty so the next write re-syncs. + - Anything else → no change. + - `a_write_that_failed_after_replacing_the_file_is_not_another_writer` (`:2141`) is re-targeted to inject + `ReplacedNotDurable` as the real outcome, not a hand-made replace. + - `a_failed_write_is_made_by_the_next_without_another_change` (`:2118`) stays. + - **S-2:** Odin's `SnapshotHandle` (`main.rs:174-181`) overrides `served_record`. It builds the envelope as + `persist_generic_document` (`:1462`) would store it, then serves it through `peer_document` (`:1400`), the + function `drop_unservable_documents` (`:1428`) sizes with. Admission and activation then use one sizing. + - A runtime-presence put keeps the default (as received). Presences are served in `public_document`'s shape and + are never near the bound, and `drop_unservable_documents` does not check them either. + - A schema that `persist_generic_document` would refuse returns an error, which the server turns into a + refusal before the sink. + - `accept_raw_document`'s doc (`main.rs:332-351`) drops its S-5 paragraph; it moves to W1's writer doc. +- **Authority map:** what the file holds is decided by the one write outcome, not by a list of guesses. + `ForeignStoreWrite` means exactly "a snapshot Odin did not write". +- **Verification:** + - Test: a `ReplacedNotDurable` failure, then a change, then a write → made over the failed snapshot. + - Test: a `NotReplaced` failure, after which another writer puts the failed snapshot into the file → `ForeignStoreWrite`. + Mutation: keeping a candidate fails it. + - Test: 1,000 failed writes leave nothing retained. This pins S-3; observe the store's retained records, or the + absence of the field. + - S-2 test: a 1.2 MB put is refused as `DocumentUnservable` and is not in the working set. + - Mutation: sizing with the received record instead of `peer_document` survives nothing. + - Negative: `rg -n attempted crates/odin-daemon/src` matches nothing. + - The pin bump carries ack Cuts 1-3 and D into Odin. The full Odin suite runs on Yggdrasil, and Soul reads this cut + as a transport change, not only a subtraction. +- **Estimate:** about -150 (including the two tests) / +60. + +#### Cut W1. The writer thread (Odin, behaviour) + +- **Repo/branch:** Odin `hands/odin-writer` from W0. +- **Deletes first:** + - `MemoryOdinTopologyStore::flush` (`lib.rs:602-640`), its `path` (`:490`) and `flushed` (`:493`) fields, and the + `dirty` flag (`:502`, `:600-602`). + - `RuntimeState::flush` (`main.rs:317-330`). + - The `self.flush()?` in `accept_raw_document` (`main.rs:363`). + - `stop`'s direct write (`main.rs:891-898`). + - The flush step of `serving_pass` (`main.rs:933-941`). + - No path is left on the loop that can write the file. That holds before the writer is added. +- **Keeps:** + - `MemoryOdinTopologyStore::load` (`lib.rs:509`): the one locked read, at activation, on the loop, before any writer + exists. `load` returns the loaded snapshot beside the store, to become the writer's `flushed`. + - `ForeignStoreWrite`, `WriteLeaseLost`, `survive` (`main.rs:979-994`), unchanged in meaning. +- **Adds, `crates/odin-daemon/src/writer.rs`** (a module, not a crate): + - `StoreWriter`: `spawn(file, flushed, lease) -> StoreWriter`, `submit(WriteRequest)`, + `try_outcome() -> Option` and `finish(self)`. + - The channel is `sync_channel(1)`. The loop submits only when nothing is in flight, so `submit` never blocks. + - `WriteRequest { version, records }`. + - `WriteOutcome { version, result }`, where `result` is one of `Written`, `Failed(text)`, `Foreign` or + `LeaseLost(text)`. + - The thread: + - Checks the lease (the check extracted from `require_current_write_lease` `main.rs:546-567` into a free + function that the loop and the writer share), then `compare_exchange_snapshot(flushed, records)`. + - Applies W0's outcome rule to `flushed`, and sends the outcome. + - It never touches reply handles, the working set or the socket. + - Mock point: the thread writes through a one-method trait `StoreFile { compare_exchange_snapshot }`, implemented + for `SingleFileMessagePackBackingStore`. Tests gate it and count it. The lock-hold (Soul's method) is used where + the real file must be on the path. +- **Changes, `main.rs`:** + - `RuntimeState` (`:141-161`) gains `writer: Option` (spawned in `try_activate` after the load, + `:284-305`), `written_through: u64`, `in_flight: Option` and `replies: VecDeque<(u64, CultMeshRudpPutReply)>`. + - `SinkHandle::accept_raw_document` (`:168-172`) takes the reply. + - `RuntimeState::accept_raw_document` (`:352-367`): + - Validates and mutates as today. + - Pushes `(store.version(), reply)`, or accepts at once when `version <= written_through` (nothing new to make + durable). + - Refuses through the reply on a validation error. + - Submits if nothing is in flight. + - A new `serving_pass` step, in place of `:933-941`, run every pass: + - `try_outcome`. On `Written { v }`: `written_through = v`, and accept every reply with version `<= v`, in queue + order. + - On `Failed`: refuse the replies with version `<= v`; they remain in the working set (refusal runs one way, as + documented at `:344-350`). + - On `Foreign` / `LeaseLost`: return the error to `survive`, which ends Odin. + - Then, if nothing is in flight and `version > written_through`, submit when a reply is waiting or when + `FLUSH_INTERVAL` is due. + - So puts never wait for the interval; bookkeeping-only changes do. + - `stop` (`:894`): + - If the lease is still current and anything is unwritten or in flight, wait for the in-flight outcome, submit the + rest, and wait for that outcome too. This is a blocking wait; it is the only one, and it happens only on the + way out. + - Answer the replies, run one `poll_server` so the acks leave, then `finish` (drop the sender and join the + thread). + - After `WriteLeaseLost` or `ForeignStoreWrite`, write nothing. Dropping the replies refuses them. + - `FLUSH_INTERVAL`'s comment (`:73-77`) and the store's doc (`lib.rs:484-488`) say: the writer thread owns the + file; puts are covered by the next write; bookkeeping waits for the interval. +- **Authority map:** + - Owner of the store file after activation: the writer thread, the only holder of a + `SingleFileMessagePackBackingStore` for the store path. It also owns what the file holds (`flushed`). + - Owner of the working set, of when to write, and of put replies: the loop (`RuntimeState`). + - Inputs: + - Writer: one request at a time, and the lease file. + - Loop: provider puts, Idunn's projection, the lease file, and writer outcomes. + - Outputs: one atomic `.cc` snapshot per request; one outcome per request; replies answered in version order. + - Derived state: + - `dirty` is derived (`version > written_through`). + - `flushed` is no longer the loop's; it lives only in the writer. + - The interval decides only bookkeeping-only writes. + - Forbidden writers: no loop code path calls `compare_exchange_snapshot`, takes the store's lock, or answers a + put reply before the outcome covering its version. + - Shared paths: peer puts, self-presence, refresh, activation-time drops and the stop write all mutate the working + set, and reach the file only through `submit`. + - Deletion line: every loop-side write (above) is deleted before `writer.rs` is added. +- **Batching rule, stated once.** The loop keeps at most one write in flight. Everything that changed while it was in + flight, puts and bookkeeping alike, goes into the next request as one snapshot of the working set. N puts accepted + during one write cost one more write and one fsync pair, not N. +- **Backpressure** (Q5-B, recommended: hold): + - A stalled writer holds put replies. The loop keeps serving, and the working set keeps admitting (the catalog + serves admitted-not-durable puts, as it does today after a failed write). + - The queue is bounded by the server's existing limits: 64 sessions and 32 MiB of admitted payload. It adds no new + bound. + - A publisher that stops waiting (the Rust default is 300 ms) disconnects. Its session ends, its answer is + discarded, and its put is still written by the next write. That is what a publisher sees today during a stall, + minus the stalled challenges. +- **Ordering.** + - Replies are answered in version order from one queue, and the server sends acks in answer order. + - Within a session, C5 releases one ack only when every pending put of the session is answered. + - So no later put's reply leaves before an earlier one's. +- **Crash mid-batch.** + - The write is one atomic replace, so the file holds the old snapshot or the new one. + - No reply of a batch is answered before its `Written` outcome, so a SIGKILL anywhere loses no acknowledged put. + - A crash after the rename but before the directory sync is `ReplacedNotDurable` if the process lives; if it dies, + nothing of that batch was acknowledged. +- **Activation.** The store is read once, by `load`, before the writer is spawned. While the writer lives, the loop + never reads the file (Cut 1's rule, unchanged). +- **Verification (Odin, Yggdrasil):** + - **Loop latency under a stall.** This pins Q5's ruling; it is Soul's S-3 probe and O1, inverted into assertions. + - Hold the store's sibling lock for 1,500 ms, with a provider put in flight and Odin's heartbeat dirty with the + interval due. + - A route challenge and a catalog read are each served within 200 ms. The put's publisher sees no ack until the + lock is released, and then does, with the put on disk. + - Mutation that must fail it: running the write synchronously on the loop, by calling the `StoreFile` directly + from `serving_pass`. + - **Group commit.** Gated `StoreFile`, held on the first write, with 10 puts from 10 publishers accepted meanwhile. + - After release, exactly 2 writes are made and all 10 publishers are acknowledged. + - Mutation: one request per put gives 11 writes. + - **Ordering.** Gated `StoreFile`; put A's write is held and put B arrives. + - B's publisher is not acknowledged before A's; the answer order is observed at the server's answer channel. + - Mutation: answering the queue from the back. + - **Durability.** `daemon_process.rs`, extending `an_acknowledged_put_survives_sigkill` (`:326`). + - 20 rounds, each a stream of puts from 4 publishers while the test holds and releases the store's lock at random + offsets. SIGKILL at a random offset. + - Every put its publisher saw acknowledged is in the file after the kill. + - Mutation: accepting replies at submit instead of at `Written`. + - **One writer.** With the writer's `StoreFile` gated shut, N serving passes, a heartbeat and a stop request make no + write from the loop: the file's identity (`file_identity`, `main.rs:3343`) is unchanged until the gate opens. + - Negative grep: `rg -n "compare_exchange_snapshot|pull_all\(" crates/odin-daemon/src` matches only `writer.rs`, + `load` and the tests. + - **Lease lost with a write queued.** No write is made, Odin ends, and the queued replies are refused. The file's + identity is unchanged. + - **Foreign write.** Odin ends without writing over it (existing `:3696`, re-targeted through the writer). + - **Stop.** SIGTERM with a put held behind the lock: after the release Odin writes, the publisher is acknowledged, + and it exits 0. With the lease lost, it writes nothing. + - Re-targeted, not deleted: `changes_inside_one_interval_are_one_write` (`:3356`, now bookkeeping-only), + `a_put_is_written_before_it_is_accepted` (`:3389`, observed at the publisher's ack), + `a_put_whose_write_is_not_made_is_refused` (`:3516`), and the `ending_*` tests (`:3728-3800`). + - `cargo mutants --in-diff` on the cut's diff. +- **Estimate:** about -120 / +230 source, +300 tests. One new module, one thread, no new crate, dependency or target. + +### Q5.5 What stays on the loop, and what does not + +| Path | Before (`434756e`) | After W1 | +|---|---|---| +| route challenge (`raw_snapshot`, exact self query) | lease-file read, sign; blocked behind any write on the loop | the same reads; never behind a write | +| catalog read (`stored_snapshot`) | working set, projection read | unchanged; never behind a write | +| peer put | mutate, then write and fsync on the loop | mutate, queue the reply, submit; the reply is sent after `Written` | +| heartbeat, refresh | mutate; the interval write on the loop | mutate; the interval write on the writer | +| store lock | taken by `flush` on the loop | taken only by `load` (activation) and the writer | + +### Q5.6 What only Yggdrasil can show + +- **Rerun Soul's ABBA** (`soul-c3/latency_cmd.txt`: `soul_loop_stall_under_live_put_rate`, base against cut, in + interleaved rounds, with `/proc/pressure/io` before and after each). + - The base is `434756e`; the cut is W1's tip. Expect route-challenge p99 to stop tracking IO pressure, while put + round trip p99 still does. + - Soul's figures at `434756e` and its base over 8 rounds, under 5-38% pressure: challenge p99 310-2,129 ms, put + round-trip p99 1,121-2,593 ms (`soul-c3/latency.log`). +- Whether the loop's remaining disk reads (the lease and projection files) ever stall under real pressure. The ABBA + run shows it as challenge p99 outliers that coincide with pressure; if they appear, open the lease-in-memory + follow-up. +- After deploy, section 6's script, plus the count of "admitted odin route continuity: timed out" per hour. + +### Q5.7 Sequencing (recommended order) + +1. **Cut D merges to CultLib main** (already slated first). +2. **The put-serve branch rebases onto it and merges.** The conflicts: + - Cut D changes `send_packet` to return `Option` and adds `end_unsendable_session`. + - Its snapshot-send failure builds `reason: format!(..)`, where the put-serve branch made `reason` a typed + `CultMeshRudpRejectionReason`. It needs a variant (for example `ResponseSendFailed(String)`), and the put-serve + Hands owns it. + - Both branches edit `poll_once`'s ack site (`:468-476`) and `end_rejected_session`. +3. **R1 batch 3 merges** (cultcache-rs only; no conflict with 1-2). +4. **C5 is cut from that main** (it rewrites the lines 1-2 touched, so it goes after them), then Soul, then merge. +5. **One Odin pin bump to that commit, as W0**, carrying S-2, S-3 and the transport cuts. Then **W1** on top. + +W0 may start on the commit after step 3 if C5 slips, at the cost of a second bump before W1. One bump is recommended: +each bump moves Odin across ack Cuts 1, 1b, 1c, 3 and D, and a single Soul pass over that transport delta is cheaper +than two. + +### Q5.8 Follow-ups this map does not own + +- **F-Q5a (parity).** TS and Python acknowledge a put at receipt, and TS's reply-later is an application receipt. + - Either they adopt C5's rule ("acknowledged means the sink accepted"), or the QUIC campaign defines an + application-level put reply for every runtime. + - QUIC has no transport ack an application can read, so it needs one anyway. Recommended owner: the QUIC campaign. + Rust C5 is then the RUDP-era contract, and TS's receipt is the seed of the QUIC shape. +- **F-Q5b (R1).** `with_exclusive_lock` returns an unlock error after a successful write, with no write-outcome + marker (`cultcache-rs/src/lib.rs:924-932` at `93ac5945`). Odin would call its own next write foreign. Fix it in + cultcache-rs: tag it, or ignore the unlock error, since dropping the file releases the lock. +- **S-5 interface.** C5 adds `acknowledgement_withheld` as the one place that decides "send no ack to this session". + The ack map's S-5 decision (probe P1) plugs in there as a second clause if it is fixed in the document server. C5 + does not fix S-5. + +### Q5.9 Subtraction ledger (estimate) + +| Cut | Removed | Added | Targets, dependencies, wire | +|---|---|---|---| +| C5 (CultLib) | ~15 | ~170 source, ~350 tests | none; sink trait signature changes (Odin, Ratatoskr test) | +| W0 (Odin) | ~150 (the `attempted` list and 2 tests) | ~60 | pin bump | +| W1 (Odin) | ~120 (every loop-side write) | ~230 source, ~300 tests | one module, one thread | + +Net source is positive: about +175. It buys the ruled capability (challenges and reads independent of the disk, with +Q3 B kept), and it removes S-3's unbounded list. + +### Q5.10 Operator questions and defaults + +- **Q5-A. What does a put's reply mean on the RUDP wire once it can come later?** + - A: the document server withholds the transport acknowledgement until the sink answers. Nothing created after the + put is sent to that session until then. The wire is unchanged, and every existing publisher (Rust + `publish_cultnet_message_to_rudp_catalog`, TS and Python publishers) keeps "acknowledged means accepted", which + for Odin is durable. + - B: a new CultNet message, a put result, sent after durability; the transport ack means received. This is a wire + change in the C# reference and four runtimes. Every Odin publisher that waits for the transport ack today must + be changed to wait for it; until then Q3 B silently weakens to "received". + - **Recommended: A.** It keeps Q3 B for every publisher with a server-only change, and RUDP is frozen except for + crash and data-loss fixes. B is the right shape under QUIC (F-Q5a), not a reason to change the RUDP wire now. + - What depends on it: C5's whole design. + - A also narrows the ack map's Q-A2 meaning for the document server: acknowledged means delivered *and answered*. + That was already true of the synchronous sink. +- **Q5-B. A writer stalled for seconds: hold put replies, or refuse with a typed error?** + - A: hold. Replies wait, bounded by the server's session and payload limits. Publishers time out on their own + clock, and the put is written when the disk returns. + - B: refuse once more than N replies wait or the oldest waits longer than T. This needs a new `cultnet.error.v0` + code (the error-contract map owns the registry) and two tunables. + - **Recommended: A.** A refusal ends the publisher's session exactly as its own timeout does, but adds a code and + two numbers, and it tells the publisher less than the truth: the put may still become durable. B earns its place + only if publishers ever wait without a timeout. +- **Default D-1 (Self, overridable). The API is a completion handle** (`CultMeshRudpPutReply`: `accept`, `refuse`, + and Drop refuses). + - The alternatives were a token with `server.answer_put(token, result)`, and a reply method keyed by session and + message id. + - Keying by session and message id is ambiguous: message ids are publisher-chosen, and a reconnect reuses the + session key. + - A token cannot be answered from another thread without routing it back to the server, and a forgotten token + leaves a session withheld until it ends. + - The handle is the Rust expression of TS's Promise-returning `onDocumentPutRaw`, and it keeps closure sinks + source-compatible. ## 8. Related findings From 07501c7c6ea342e1a1c9b8d125cea97b608cf8f7 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 19:05:55 +0200 Subject: [PATCH 37/43] Write map: Q5-A and Q5-B ruled A (withhold ACK until answered; hold puts on a stall) Co-Authored-By: Claude Opus 5.5 --- docs/write-pattern-cut.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/write-pattern-cut.md b/docs/write-pattern-cut.md index 60b48cc..97036b3 100644 --- a/docs/write-pattern-cut.md +++ b/docs/write-pattern-cut.md @@ -450,6 +450,11 @@ section 7). Anchors: The mapping raises two forks, **Q5-A** and **Q5-B** (at the end of this section). Cuts C5 and W1 wait on Q5-A. +**RULED, operator 2026-09-30: "Q5-A: A, but make a note of this for when we move on QUIC, Q5-B: A".** +- **Q5-A = A.** The server withholds the session's transport ACK until the put is answered. The wire is unchanged. + The note for QUIC (a put-result message) is in the QUIC campaign handoff. +- **Q5-B = A.** Puts are held during a writer stall, bounded by the existing session and byte limits. + ### Q5.1 Body facts (probed and read) **Probe O1, Odin `434756e` + scratch test, Yggdrasil slot 3 (`imag-q5-odin@31953c93`).** A test process holds the From 3854bb5eaf3f1b2f5f7225312f5e05dbba2ea1ac Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 19:50:57 +0200 Subject: [PATCH 38/43] Recovery map: R1 batch 3 Soul (merge); symlink, fsync and encode-refusal follow-ups; live-file order Co-Authored-By: Claude Opus 5.5 --- docs/store-recovery-cut.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/store-recovery-cut.md b/docs/store-recovery-cut.md index f93dc5b..207f40f 100644 --- a/docs/store-recovery-cut.md +++ b/docs/store-recovery-cut.md @@ -36,6 +36,22 @@ Recorded for later cuts: - **Tooling.** - `packages/cultcache-rs` has no committed `Cargo.lock`, so the map's `cargo test --locked` cannot run. - The cultcache-ts interop test needs `CARGO_TARGET_DIR` unset under ygg-verify. +- **Soul on batch 3 (`2d7a3c78..3c9e195b`, gamecult-ops `bca7978`), 2026-09-30: merge both.** Pre-merge batch 4 is in + Hands: fix the README symlink claim (S1), close value echoes in the redb and typed-decode paths (S2) and in the + Python/TS non-string key echo (S3), make the unlock policy coherent (S9), classify encode refusals separately from + `NotReplaced` (S10), and fix a test timeout (S11). The ops branch deletes the guard test (S6) and the dead compose + layout (S5). + - **Open for R3:** writes through a symlinked store path replace the link, so a store behind a link to a mounted + volume moves into the container layer. R3 decides resolve or refuse. + - **Open for R3b:** Python and TS never fsync. After F1 their first create-by-rename is not covered by ext4 + `auto_da_alloc`, so a crash can leave zero bytes, which every post-R1 reader refuses forever. **Bifrost's CultLib + pin moves past R1 only together with R3b's TS fsync.** + - **For W0 (Odin):** classify a write failure through `CultCacheStoreWriteFailed.kind` together with its `io::Error`, + never retry on `NotReplaced` alone (S10). + - Deployed Epiphany is `d2ca6630`, using a vendored cultcache-rs (`ba6a4874`) that reads a missing store as empty. + All Epiphany units have been inactive since 2026-08-22. + - Live zero-byte stores to move aside after the ops merge, in Soul's order (setgid on inbox, then rename in place, + then relocate): `observations.cc`, `deliveries.cc`, and a third, `/var/lib/gamecult/epiphany/idunn-provider-health-admission.cc`. - **Text matches.** gamecult-ops `scripts/idunn/start-idunn-local.ps1:143` matches the same removed text as Muninn (below). It must change with Idunn's next CultLib bump. - **Consumer follow-up that must travel with the next Muninn CultLib bump:** From 1262fcdcb58358033aa3ab946b10088daac55481 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 20:49:11 +0200 Subject: [PATCH 39/43] Recovery map: live zero-byte stores moved aside; ops merged Co-Authored-By: Claude Opus 5.5 --- docs/store-recovery-cut.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/store-recovery-cut.md b/docs/store-recovery-cut.md index 207f40f..28f191a 100644 --- a/docs/store-recovery-cut.md +++ b/docs/store-recovery-cut.md @@ -50,6 +50,12 @@ Recorded for later cuts: never retry on `NotReplaced` alone (S10). - Deployed Epiphany is `d2ca6630`, using a vendored cultcache-rs (`ba6a4874`) that reads a missing store as empty. All Epiphany units have been inactive since 2026-08-22. + - **Done 2026-09-30 20:48 CEST (Self).** gamecult-ops merged at `3fa22aa`. The inbox is setgid 2750. All three + zero-byte stores were renamed in place and moved to `/root/zero-byte-stores-20260930/` on Yggdrasil, with + `manifest.txt` recording owner, mode and mtime. None had gained bytes. All three Bifrost units stayed active, and + persona-feedback logged nothing and did not restart. + - The compose/container lane (the compose file, two deploy scripts, the `.ps1` uploader, two runbooks) is dead: + the live Bifrost units run native `dotnet` and `node`. Deleting it is put to the operator. - Live zero-byte stores to move aside after the ops merge, in Soul's order (setgid on inbox, then rename in place, then relocate): `observations.cc`, `deliveries.cc`, and a third, `/var/lib/gamecult/epiphany/idunn-provider-health-admission.cc`. - **Text matches.** gamecult-ops `scripts/idunn/start-idunn-local.ps1:143` matches the same removed text as Muninn From 2fa1bffc38a0a798339956c7c87e3f6c51b43ce8 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 21:13:56 +0200 Subject: [PATCH 40/43] Recovery map: R1 landed at CultLib daedfdd6 Co-Authored-By: Claude Opus 5.5 --- docs/store-recovery-cut.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/store-recovery-cut.md b/docs/store-recovery-cut.md index 28f191a..0902471 100644 --- a/docs/store-recovery-cut.md +++ b/docs/store-recovery-cut.md @@ -13,6 +13,11 @@ published as its own typed record, not as a degraded presence. - **Q-R6:** "Approved". Both drills. +**R1 LANDED: CultLib main `daedfdd6` (2026-09-30).** The merged tree `0e6891b3` was verified on Yggdrasil in every +runtime, including the four-runtime store interop test. That test runs close to its 30 s per-command limit under +load (`cult-cache.test.ts:1407`), so it is fragile: follow-up. Consumer pins still gate their own bumps: Bifrost only +with R3b's TS fsync; Muninn and Idunn with their error-text matches. + **R1 status (Self, 2026-09-30).** Built on CultLib `hands/cultcache-read-failures` (`c0b56e06`..`b7908668`), then fix batch 2 (`..2d7a3c78`). Soul's verdict on batch 2 was to hold the merge. Batch 3 is in Hands, fixing: - **Blocker, F1 (zero-byte stores on Yggdrasil).** From 665dc0856c524b54b41f124ec0c42468573449b0 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 21:50:45 +0200 Subject: [PATCH 41/43] Recovery map: Bifrost container lane deleted and its legacy route retired on Yggdrasil Co-Authored-By: Claude Opus 5.5 --- docs/store-recovery-cut.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/store-recovery-cut.md b/docs/store-recovery-cut.md index 0902471..fca90d5 100644 --- a/docs/store-recovery-cut.md +++ b/docs/store-recovery-cut.md @@ -60,7 +60,12 @@ Recorded for later cuts: `manifest.txt` recording owner, mode and mtime. None had gained bytes. All three Bifrost units stayed active, and persona-feedback logged nothing and did not restart. - The compose/container lane (the compose file, two deploy scripts, the `.ps1` uploader, two runbooks) is dead: - the live Bifrost units run native `dotnet` and `node`. Deleting it is put to the operator. + the live Bifrost units run native `dotnet` and `node`. **Deleted 2026-09-30, operator: "delete the Bifrost lane".** + gamecult-ops `fdb4356`. Its live trigger, the Idunn legacy actuator route `deploy bifrost-persona-feedback`, was + retired in Idunn `7793b5b` and installed on Yggdrasil at 21:50 CEST (sha256 `d95fa08c...`; the previous binary is + at `/root/idunn-yggdrasil.pre-bifrost-retire-20260930`). The manifest and `/srv/bifrost/compose` were moved to + `/srv/retired/retired-2026-09-30/`. All three Bifrost units stayed active. **Follow-up:** persona-feedback has no + Idunn deploy route now. Give it a real Idunn binding. - Live zero-byte stores to move aside after the ops merge, in Soul's order (setgid on inbox, then rename in place, then relocate): `observations.cc`, `deliveries.cc`, and a third, `/var/lib/gamecult/epiphany/idunn-provider-health-admission.cc`. - **Text matches.** gamecult-ops `scripts/idunn/start-idunn-local.ps1:143` matches the same removed text as Muninn From f901b38fb72d074d68cdeb899291e40bf619b14c Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 23:41:04 +0200 Subject: [PATCH 42/43] Write map: CultLib side landed (Cut D, R1, put-serve); C5 next Co-Authored-By: Claude Opus 5.5 --- docs/write-pattern-cut.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/write-pattern-cut.md b/docs/write-pattern-cut.md index 97036b3..0c91331 100644 --- a/docs/write-pattern-cut.md +++ b/docs/write-pattern-cut.md @@ -75,6 +75,15 @@ heartbeats included) writes the store once. - **Batch 4 in Hands:** S-4, S-6, S-8, the S-5 doc line, the attempted-list fixture gap, and committing Soul's probes. S-1 waits on Q5; S-2 and S-3 wait on CultLib. +**CultLib side landed (2026-09-30).** Ack Cut D was merged at `2f06ef84`, R1 at `daedfdd6` and the put-serve bound at +`9dcb3aa1`. That unblocks the Q5 sequence: +1. R1 batch 3's typed write outcome is on main. It carries `Rejected` as well as `NotReplaced`/`ReplacedNotDurable`. + W0 must classify by kind together with the `io::Error`, never by `NotReplaced` alone. +2. **C5 next:** cultmesh-rs deferred put reply, on main. +3. **W0:** one Odin pin bump. It closes S-2 (put refusal with a `served_record` override) and S-3 (delete + `attempted`), and it brings in Cut D's and the put-serve error texts. +4. **W1:** the writer thread. + Status: cut map, Imagination pass 1 (Opus), 2026-09-30. Nothing has landed. Anchors are Odin `main` at `44951a1` (the deployed build, live on Yggdrasil since 2026-09-30 04:20:48 UTC) and CultLib `3bf1c0ce` (Odin's `Cargo.lock` pin). This map owns the means; there is no separate target document, because the ends are From 8e1de662c75e09e5b93005b600ccdad1cc0e8387 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 1 Oct 2026 01:04:24 +0200 Subject: [PATCH 43/43] Write map: C5 Hands landed on a branch; Soul pending the operator's go Co-Authored-By: Claude Opus 5.5 --- docs/write-pattern-cut.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/write-pattern-cut.md b/docs/write-pattern-cut.md index 0c91331..3b7d93b 100644 --- a/docs/write-pattern-cut.md +++ b/docs/write-pattern-cut.md @@ -986,3 +986,23 @@ Q3 B kept), and it removes S-3's unbounded list. - **F7** `accept_raw_document` (`main.rs:269-280`) checks `activated()` but not the *current* lease before mutating. Cut 2 moves the lease check to the only write, which closes this for the file. Soul should confirm that nothing else acts on a lost lease. + +## C5 status (2026-10-01) + +**Hands done, Soul not yet run.** Drain mode: Soul waits for the operator's go. + +- **Branch:** CultLib `hands/cultmesh-deferred-put-reply`, commits `08e0c229..d708247e`, based on `9dcb3aa1`. +- **Mechanism:** the sink answers through `CultMeshRudpPutReply` with `accept()` or `refuse(reason)`. Dropping the reply unanswered refuses with `UNANSWERED_PUT_REASON`. `acknowledgement_withheld` is the single owner of the hold: while a put is unanswered, the session gets no end-of-poll ack, Pong, snapshot response or reply to a repeated Connect. +- **Verification (Hands):** + - cultmesh-rs: 21 unit and 32 integration tests pass. A byte-exact golden test also passes before the cut. + - cultnet-rs: 486 pass. + - Interop: 27 pass, 0 fail. + - cargo-mutants: 57 mutants; 0 missed, 5 timeouts, all in the `take_answers` loop. +- **Spec deviations Soul should attack:** + - A repeated Connect's ack is now withheld. + - Puts queued behind a waiting snapshot count as withheld. Snapshot responses are held in a new `responses_owed` queue: P1 pending, then S, then P2 must not acknowledge P2 early. + - Answers are checked after each message. + - A release that arrives during an unreliable packet sends its own ack. +- **Structural delta:** +406/−96 in `rudp_document_server.rs`, against a +170/−15 estimate. +- **Consumers:** Odin `SinkHandle` and Ratatoskr `tests/catalog.rs:29` break until the W0 pin bump. +- **Still open:** S-5. Its fix belongs in `acknowledgement_withheld`.