diff --git a/Cargo.toml b/Cargo.toml index 76fabb0..09a0ec3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "crates/muninn-move-tracker", "crates/muninn-psmoveapi-tracker", "crates/odin-core", + "crates/odin-daemon", ] resolver = "3" exclude = [ diff --git a/crates/idunn-daemon/src/control_plane.rs b/crates/idunn-daemon/src/control_plane.rs index 225bc27..1f8361b 100644 --- a/crates/idunn-daemon/src/control_plane.rs +++ b/crates/idunn-daemon/src/control_plane.rs @@ -5,35 +5,36 @@ use std::path::{Path, PathBuf}; use std::thread; use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use anyhow::{anyhow, bail, ensure, Context, Result}; +use anyhow::{Context, Result, anyhow, bail, ensure}; use cultcache_rs::{ CultCacheEnvelope, CultCacheExpectedEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, }; use cultnet_rs::{ - authenticate_odin_runtime_topology_correlation, derive_service_identity_id, - evaluate_idunn_continuity_restart, evaluate_idunn_deployment_brake, open_service_identity_at, - verify_idunn_deployment_brake_authorization, verify_runtime_authority, AuthenticatedOdinRuntimeTopologyCorrelation, GameCultProviderHealthIdentity, + IDUNN_DEPLOYMENT_BRAKE_SCHEMA, IDUNN_LIFECYCLE_BRAKE_SCHEMA, IDUNN_PROCESS_WRITE_LEASE_SCHEMA, IdunnDeploymentBrakeObservation, IdunnDeploymentBrakeOperatorIdentity, IdunnDeploymentBrakeRecord, IdunnExpectedIncarnationRecord, IdunnLifecycleBrakeObservation, IdunnLifecycleBrakeRecord, IdunnProcessWriteLeaseRecord, IdunnRuntimeActivationLaunch, IdunnRuntimeActivationRecord, IdunnServiceIdentity, OdinTopologyAuthenticationContext, - OdinTopologyIdentity, ServiceIdentityProfile, ServiceIdentitySigner, - ServiceIdentityTrustAnchor, IDUNN_DEPLOYMENT_BRAKE_SCHEMA, IDUNN_LIFECYCLE_BRAKE_SCHEMA, - IDUNN_PROCESS_WRITE_LEASE_SCHEMA, + OdinTopologyDisagreement, OdinTopologyIdentity, RuntimePresenceAuthenticationContext, + ServiceIdentityProfile, ServiceIdentitySigner, ServiceIdentityTrustAnchor, + authenticate_odin_runtime_topology_correlation, authenticate_runtime_presence_claim, + correlate_runtime_presence_claim, derive_service_identity_id, + evaluate_idunn_continuity_restart, evaluate_idunn_deployment_brake, open_service_identity_at, + verify_idunn_deployment_brake_authorization, verify_runtime_authority, }; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use uuid::Uuid; -use crate::deployment::{DependencyKind, OperatorBinding, StartupOrder}; +use crate::deployment::{DependencyKind, OperatorBinding, RouteBinding, capability_compatible}; use crate::deployment_plan::{ - compile_deployment_plan, CompiledDeploymentPlan, DependencyProviderAuthority, SealedRelease, + CompiledDeploymentPlan, DependencyProviderAuthority, SealedRelease, compile_deployment_plan, }; use crate::drivers::{ CultCacheTopologyDriver, CultCacheWriteLeaseDriver, DockerRunnerDriver, FrozenSourceReceipt, GitSourceDriver, InstalledReleaseObservation, NginxRouteDriver, ProcessIdentity, - RouteObservation, RoutePort, RoutePreflightReceipt, RunnerPort, SourcePort, + RouteObservation, RoutePreflightReceipt, RunnerPort, SourcePort, SystemdTransientWorkloadDriver, TopologyPort, WorkloadObservation, WorkloadPort, WriteLeasePort, }; @@ -188,17 +189,127 @@ impl TopologyEvidence { } } -/// Opaque capability minted only after the transaction CAS advances the Odin -/// replay cursor and stores these exact bytes. Adjacent drivers may inspect the -/// authenticated receipt, but cannot construct replay admission themselves. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct RuntimePresenceEvidence { + canonical_bytes: Vec, + canonical_sha256: String, + message_id: String, + challenged_at_unix_millis: u64, + admitted_at_unix_millis: u64, +} + +impl RuntimePresenceEvidence { + fn from_present( + present: &cultnet_rs::VerifiedRuntimePresence, + message_id: String, + challenged_at_unix_millis: u64, + admitted_at_unix_millis: u64, + ) -> Result { + let evidence = Self { + canonical_bytes: present.canonical_bytes().to_vec(), + canonical_sha256: present.signed_presence_sha256().to_owned(), + message_id, + challenged_at_unix_millis, + admitted_at_unix_millis, + }; + evidence.validate_shape()?; + Ok(evidence) + } + + fn validate_shape(&self) -> Result<()> { + ensure!( + !self.canonical_bytes.is_empty() + && self.canonical_sha256 == sha256_id(&self.canonical_bytes), + "runtime presence evidence bytes or digest are invalid" + ); + require_id(&self.message_id, "runtime presence challenge")?; + ensure!( + self.challenged_at_unix_millis > 0 + && self.admitted_at_unix_millis >= self.challenged_at_unix_millis, + "runtime presence evidence timeline is invalid" + ); + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "source", rename_all = "kebab-case", deny_unknown_fields)] +enum WarmingEvidence { + OdinTopology { evidence: TopologyEvidence }, + FirstOdinDirect { evidence: RuntimePresenceEvidence }, +} + +impl WarmingEvidence { + fn validate_shape(&self) -> Result<()> { + match self { + Self::OdinTopology { evidence } => evidence.validate_shape(), + Self::FirstOdinDirect { evidence } => evidence.validate_shape(), + } + } + + fn canonical_sha256(&self) -> &str { + match self { + Self::OdinTopology { evidence } => &evidence.canonical_sha256, + Self::FirstOdinDirect { evidence } => &evidence.canonical_sha256, + } + } +} + +/// Opaque capability minted only after the transaction CAS stores exact +/// Warming evidence. Adjacent drivers receive only the candidate identity and +/// signed-presence digest needed to bind the write lease; they cannot inspect +/// or construct semantic admission. #[derive(Clone, Debug)] pub struct SequenceAdmittedWarming { transaction_id: String, - evidence: TopologyEvidence, - authenticated: AuthenticatedOdinRuntimeTopologyCorrelation, + evidence: WarmingEvidence, + signed_presence_sha256: String, + runtime_instance_id: String, } impl SequenceAdmittedWarming { + fn from_topology( + transaction_id: String, + evidence: TopologyEvidence, + authenticated: AuthenticatedOdinRuntimeTopologyCorrelation, + ) -> Result { + let record = authenticated.record(); + let signed_presence_sha256 = record + .signed_presence_sha256 + .clone() + .context("Warming topology has no signed presence")?; + let runtime_instance_id = record + .runtime_instance_id + .clone() + .context("Warming topology has no runtime instance")?; + Ok(Self { + transaction_id, + evidence: WarmingEvidence::OdinTopology { evidence }, + signed_presence_sha256, + runtime_instance_id, + }) + } + + fn from_first_odin_presence( + transaction_id: String, + evidence: RuntimePresenceEvidence, + present: cultnet_rs::VerifiedRuntimePresence, + ) -> Result { + ensure!( + present.canonical_bytes() == evidence.canonical_bytes + && present.signed_presence_sha256() == evidence.canonical_sha256, + "direct first-Odin Warming evidence differs from its authenticated presence" + ); + let runtime_instance_id = present.record().runtime_instance_id.clone(); + Ok(Self { + transaction_id, + signed_presence_sha256: evidence.canonical_sha256.clone(), + evidence: WarmingEvidence::FirstOdinDirect { evidence }, + runtime_instance_id, + }) + } + #[cfg(test)] pub(crate) fn for_test( transaction_id: impl Into, @@ -218,15 +329,21 @@ impl SequenceAdmittedWarming { require_id(&transaction_id, "test transaction id")?; let evidence = TopologyEvidence::from_authenticated(&authenticated, admitted_at_unix_millis)?; - Ok(Self { - transaction_id, - evidence, - authenticated, - }) - } - - pub(crate) fn authenticated(&self) -> &AuthenticatedOdinRuntimeTopologyCorrelation { - &self.authenticated + let signed_presence_sha256 = record + .signed_presence_sha256 + .clone() + .context("test Warming receipt has no signed presence")?; + let runtime_instance_id = record + .runtime_instance_id + .clone() + .context("test Warming receipt has no runtime instance")?; + let token = Self::from_topology(transaction_id, evidence, authenticated)?; + ensure!( + token.signed_presence_sha256 == signed_presence_sha256 + && token.runtime_instance_id == runtime_instance_id, + "test Warming token extraction changed" + ); + Ok(token) } pub(crate) fn transaction_id(&self) -> &str { @@ -234,7 +351,15 @@ impl SequenceAdmittedWarming { } pub(crate) fn evidence_sha256(&self) -> &str { - &self.evidence.canonical_sha256 + self.evidence.canonical_sha256() + } + + pub(crate) fn signed_presence_sha256(&self) -> &str { + &self.signed_presence_sha256 + } + + pub(crate) fn runtime_instance_id(&self) -> &str { + &self.runtime_instance_id } } @@ -352,14 +477,27 @@ impl LeasingEvidence { #[serde(tag = "result", rename_all = "kebab-case", deny_unknown_fields)] enum RoutingEvidence { SkippedUnrouted, - Promoted { observation: RouteObservation }, + Promoted { + observation: RouteObservation, + promoted_at_unix_millis: u64, + }, } impl RoutingEvidence { fn observation(&self) -> Option<&RouteObservation> { match self { Self::SkippedUnrouted => None, - Self::Promoted { observation } => Some(observation), + Self::Promoted { observation, .. } => Some(observation), + } + } + + fn promoted_at_unix_millis(&self) -> Option { + match self { + Self::SkippedUnrouted => None, + Self::Promoted { + promoted_at_unix_millis, + .. + } => Some(*promoted_at_unix_millis), } } } @@ -464,7 +602,7 @@ impl PostCommitCleanup { #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] #[cultcache( type = "idunn.deployment_transaction", - schema = "idunn.deployment_transaction.v1" + schema = "idunn.deployment_transaction.v2" )] struct DeploymentTransaction { #[cultcache(key = 0)] @@ -512,7 +650,7 @@ struct DeploymentTransaction { #[cultcache(key = 21)] latest_odin_observation: Option, #[cultcache(key = 22)] - warming: Option, + warming: Option, #[cultcache(key = 23)] route_preflight: Option, #[cultcache(key = 24)] @@ -641,6 +779,17 @@ impl DeploymentTransaction { self.phase != DeploymentPhase::Complete } + /// Admission has moved to the new generation at Complete, but an exact + /// draining incumbent still reserves its process and candidate endpoint. + /// No later mutation for the same target may overlap that cleanup. + fn blocks_new_target_mutation(&self) -> bool { + self.owns_target_authority() + || self + .post_commit_cleanup + .as_ref() + .is_some_and(|cleanup| !cleanup.is_complete()) + } + fn validate(&self) -> Result<()> { ensure!( self.schema_version == DEPLOYMENT_TRANSACTION_SCHEMA, @@ -712,7 +861,7 @@ impl DeploymentTransaction { self.workload.is_none() || self.activation.is_some(), "workload observation exists without its prepared activation" ); - for evidence in [&self.latest_odin_observation, &self.warming, &self.ready] + for evidence in [&self.latest_odin_observation, &self.ready] .into_iter() .flatten() { @@ -722,6 +871,19 @@ impl DeploymentTransaction { "topology evidence exceeds the transaction replay cursor" ); } + if let Some(warming) = &self.warming { + warming.validate_shape()?; + match warming { + WarmingEvidence::OdinTopology { evidence } => ensure!( + evidence.publisher_sequence <= self.odin_publisher_sequence_cursor, + "Warming topology evidence exceeds the transaction replay cursor" + ), + WarmingEvidence::FirstOdinDirect { .. } => ensure!( + self.target == "odin" && self.incumbent_generation_id.is_none(), + "direct Warming evidence is reserved for first Odin bootstrap" + ), + } + } if let Some(leasing) = &self.leasing { if let Some(expected) = &self.expected { ensure!( @@ -891,6 +1053,9 @@ impl DeploymentTransaction { == self.route_preflight.is_some(), "route preflight does not match routed Expected" ); + if let Some(preflight) = &self.route_preflight { + preflight.validate()?; + } } if self.phase >= DeploymentPhase::Leasing { ensure!(self.fencing.is_some(), "Leasing lacks fencing evidence"); @@ -911,6 +1076,18 @@ impl DeploymentTransaction { if self.phase >= DeploymentPhase::Committing { ensure!(self.routing.is_some(), "Committing lacks routing evidence"); } + if let Some(RoutingEvidence::Promoted { + observation, + promoted_at_unix_millis, + }) = &self.routing + { + observation.validate()?; + ensure!( + *promoted_at_unix_millis > 0 + && *promoted_at_unix_millis <= self.updated_at_unix_millis, + "route promotion time is outside the durable transaction timeline" + ); + } if self.phase == DeploymentPhase::Complete { ensure!( matches!( @@ -1010,7 +1187,7 @@ impl AdmittedOdinAuthority { #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] #[cultcache( type = "idunn.admitted_generation", - schema = "idunn.admitted_generation.v1" + schema = "idunn.admitted_generation.v2" )] struct AdmittedGeneration { #[cultcache(key = 0)] @@ -1049,6 +1226,8 @@ struct AdmittedGeneration { odin_authority: AdmittedOdinAuthority, #[cultcache(key = 17)] odin_publisher_sequence_cursor: u64, + #[cultcache(key = 18)] + route_repair_started_at_unix_millis: Option, } impl AdmittedGeneration { @@ -1085,6 +1264,7 @@ impl AdmittedGeneration { routing: required(&transaction.routing, "route disposition")?.clone(), odin_authority, odin_publisher_sequence_cursor: transaction.odin_publisher_sequence_cursor, + route_repair_started_at_unix_millis: None, }; generation.validate()?; Ok(generation) @@ -1127,6 +1307,20 @@ impl AdmittedGeneration { self.expected.route.is_some() == self.routing.observation().is_some(), "admitted route disposition differs from Expected" ); + if let Some(route) = self.routing.observation() { + route.validate()?; + ensure!( + route.runtime_instance_id == self.activation.runtime_instance_id, + "admitted route observation belongs to another runtime instance" + ); + } + if let Some(started_at) = self.route_repair_started_at_unix_millis { + ensure!(started_at > 0, "admitted route repair has no start time"); + ensure!( + matches!(&self.routing, RoutingEvidence::Promoted { .. }), + "only a promoted routed generation can own route repair intent" + ); + } Ok(()) } } @@ -1532,7 +1726,7 @@ impl ControlSnapshot { }) } - fn max_odin_sequence(&self, signer_identity_id: &str) -> u64 { + fn max_odin_sequence(&self, target: &str, signer_identity_id: &str) -> u64 { let transaction_max = self .transactions .iter() @@ -1541,7 +1735,10 @@ impl ControlSnapshot { .value .latest_odin_observation .as_ref() - .filter(|evidence| evidence.signer_identity_id == signer_identity_id) + .filter(|evidence| { + stored.value.target == target + && evidence.signer_identity_id == signer_identity_id + }) .map(|evidence| evidence.publisher_sequence) }) .max() @@ -1549,7 +1746,10 @@ impl ControlSnapshot { let admitted_max = self .admitted .iter() - .filter(|stored| stored.value.odin_authority.signer_identity_id == signer_identity_id) + .filter(|stored| { + stored.value.target == target + && stored.value.odin_authority.signer_identity_id == signer_identity_id + }) .map(|stored| stored.value.odin_publisher_sequence_cursor) .max() .unwrap_or(0); @@ -1672,9 +1872,85 @@ fn load_bindings(directory: &Path) -> Result> { ); } ensure!(!bindings.is_empty(), "Idunn binding directory is empty"); + validate_route_bindings(&bindings)?; Ok(bindings) } +fn validate_route_bindings(bindings: &BTreeMap) -> Result<()> { + let routes = bindings + .iter() + .filter_map(|(target, loaded)| { + loaded + .binding + .route + .as_ref() + .map(|route| (target.as_str(), route)) + }) + .collect::>(); + validate_route_binding_set(&routes) +} + +fn validate_route_binding_set(routes: &[(&str, &RouteBinding)]) -> Result<()> { + for (index, (target, route)) in routes.iter().enumerate() { + let (stable_host, stable_port) = route.stable_socket()?; + let private_host = route + .private_host + .parse::() + .context("validated route private host stopped being an IP address")?; + ensure!( + stable_host != private_host + || !(route.private_port_start..=route.private_port_end).contains(&stable_port), + "route {target} stable socket overlaps its candidate port range" + ); + + for (other_target, other) in &routes[index + 1..] { + ensure!( + route.route_id != other.route_id, + "route id {} is shared by targets {target} and {other_target}", + route.route_id + ); + ensure!( + route.config_path != other.config_path, + "route fragment {} is shared by targets {target} and {other_target}", + route.config_path.display() + ); + let other_stable = other.stable_socket()?; + ensure!( + route.driver != other.driver || (stable_host, stable_port) != other_stable, + "stable route socket {}:{} is shared by targets {target} and {other_target}", + stable_host, + stable_port + ); + if route.driver != other.driver { + continue; + } + let other_private_host = other + .private_host + .parse::() + .context("validated route private host stopped being an IP address")?; + let ranges_overlap = route.private_host == other.private_host + && route.private_port_start <= other.private_port_end + && other.private_port_start <= route.private_port_end; + ensure!( + !ranges_overlap, + "candidate port ranges overlap for targets {target} and {other_target}" + ); + ensure!( + stable_host != other_private_host + || !(other.private_port_start..=other.private_port_end).contains(&stable_port), + "stable route socket for {target} overlaps {other_target}'s candidate range" + ); + ensure!( + other_stable.0 != private_host + || !(route.private_port_start..=route.private_port_end) + .contains(&other_stable.1), + "stable route socket for {other_target} overlaps {target}'s candidate range" + ); + } + } + Ok(()) +} + fn resolve_selector( bindings: &BTreeMap, selector: &str, @@ -1978,18 +2254,38 @@ impl Engine { validate_authenticated_evidence(evidence, &authenticated)?; } if let Some(evidence) = &transaction.warming { - let authenticated = self.authenticate_topology_bytes( - snapshot, - transaction, - &evidence.canonical_bytes, - None, - evidence.admitted_at_unix_millis, - )?; - validate_authenticated_evidence(evidence, &authenticated)?; - ensure!( - is_semantic_warming(&authenticated), - "durable Warming label is not exact semantic Warming" - ); + match evidence { + WarmingEvidence::OdinTopology { evidence } => { + let authenticated = self.authenticate_topology_bytes( + snapshot, + transaction, + &evidence.canonical_bytes, + None, + evidence.admitted_at_unix_millis, + )?; + validate_authenticated_evidence(evidence, &authenticated)?; + let incumbent_lease_sha256 = + self.incumbent_lease_sha256_for_warming(snapshot, transaction)?; + ensure!( + is_semantic_warming( + required(&transaction.expected, "Warming Expected projection",)?, + required(&transaction.activation, "Warming activation")?, + incumbent_lease_sha256.as_deref(), + &authenticated, + )?, + "durable Warming gate is not supported by current runtime evidence" + ); + } + WarmingEvidence::FirstOdinDirect { evidence } => { + self.authenticate_first_odin_warming_presence( + transaction, + &evidence.message_id, + evidence.challenged_at_unix_millis, + evidence.admitted_at_unix_millis, + &evidence.canonical_bytes, + )?; + } + } } if let Some(evidence) = &transaction.ready { let authenticated = self.authenticate_topology_bytes( @@ -2062,14 +2358,9 @@ fn serve(options: RuntimeOptions) -> Result<()> { engine.validate_durable_authority(&ControlSnapshot::read(&engine.options.state_store)?)?; loop { - if engine.resume_one_transaction()? { - thread::sleep(Duration::from_millis(engine.options.poll_millis)); - continue; - } - if engine.supervise_one_admitted_generation()? { - continue; - } - if engine.freeze_one_queued_command()? { + let transaction_progress = engine.resume_one_transaction()?; + let continuity_progress = engine.supervise_one_admitted_generation()?; + if !transaction_progress && !continuity_progress && engine.freeze_one_queued_command()? { continue; } thread::sleep(Duration::from_millis(engine.options.poll_millis)); @@ -2083,6 +2374,7 @@ impl Engine { /// own command, so one target brake cannot suspend unrelated continuity. fn resume_one_transaction(&self) -> Result { let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let mut progressed = false; let mut candidates = snapshot .transactions .iter() @@ -2099,12 +2391,22 @@ impl Engine { if snapshot.has_earlier_authority_sibling(¤t.value) { continue; } - match self.advance_transaction(current) { - Ok(()) => {} - Err(error) if current.value.phase < DeploymentPhase::Fencing => { - self.begin_pre_fencing_abort(current, error)?; + if let Err(error) = self.advance_transaction(current) { + let latest_snapshot = ControlSnapshot::read(&self.options.state_store)?; + let latest = latest_snapshot + .transactions + .iter() + .find(|stored| stored.value.transaction_id == current.value.transaction_id) + .context("transaction disappeared while recording an execution error")?; + if latest.value.is_terminal() { + progressed = true; + } else if latest.value.phase < DeploymentPhase::Fencing + && latest.value.pre_fencing_abort.is_none() + { + self.begin_pre_fencing_abort(latest, error)?; + } else { + self.record_resumable_error(latest, &error)?; } - Err(error) => self.record_resumable_error(current, &error)?, } let after = ControlSnapshot::read(&self.options.state_store)?; let live = after @@ -2113,10 +2415,10 @@ impl Engine { .find(|stored| stored.value.transaction_id == current.value.transaction_id) .context("transaction disappeared while checking scheduler progress")?; if live.envelope != current.envelope { - return Ok(true); + progressed = true; } } - Ok(false) + Ok(progressed) } fn freeze_one_queued_command(&self) -> Result { @@ -2171,7 +2473,7 @@ impl Engine { let busy = snapshot .transactions .iter() - .filter(|stored| stored.value.owns_target_authority()) + .filter(|stored| stored.value.blocks_new_target_mutation()) .map(|stored| stored.value.target.as_str()) .collect::>(); if targets.iter().any(|target| busy.contains(target.as_str())) { @@ -2213,35 +2515,161 @@ impl Engine { fn supervise_one_admitted_generation(&self) -> Result { let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let busy = snapshot - .transactions - .iter() - .filter(|stored| stored.value.owns_target_authority()) - .map(|stored| stored.value.target.as_str()) - .collect::>(); + let mut progressed = false; let mut admitted = snapshot.admitted.iter().collect::>(); admitted.sort_by_key(|stored| stored.value.target.as_str()); for current in admitted { - if busy.contains(current.value.target.as_str()) { + let blocker = snapshot.transactions.iter().find(|stored| { + stored.value.target == current.value.target + && stored.value.blocks_new_target_mutation() + }); + if blocker.is_some_and(|stored| { + stored.value.phase >= DeploymentPhase::Fencing + && stored.value.phase < DeploymentPhase::Complete + }) { continue; } + match self.supervise_admitted_route(current) { + Ok(true) => { + progressed = true; + continue; + } + Ok(false) => {} + Err(error) => eprintln!( + "Idunn rejected admitted {} route continuity: {error:#}", + current.value.target + ), + } match self.refresh_admitted_topology(&snapshot, current) { - Ok(true) => return Ok(true), + Ok(true) => { + progressed = true; + continue; + } Ok(false) => {} Err(error) => eprintln!( "Idunn preserved admitted {} after rejecting topology observation: {error:#}", current.value.target ), } - if self - .workload - .observe( - ¤t.value.expected, - ¤t.value.activation, - ¤t.value.workload, - ) - .is_ok() - { + let mut operational_error = None; + let observation = match self.workload.observe( + ¤t.value.expected, + ¤t.value.activation, + ¤t.value.workload, + ) { + Ok(observation) => { + let lease_is_missing = if let Some(lease) = current.value.leasing.lease() { + let lease_health = (|| -> Result { + let lease_path = current + .value + .plan + .parsed_inputs()? + .1 + .process_write_lease + .context("admitted lease has no operator binding")? + .record_path; + let driver = + CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); + if driver.observe_exact(lease)? { + return Ok(true); + } + if driver.observe_empty()? { + return Ok(false); + } + bail!("write-lease store contains unexpected authority") + })(); + match lease_health { + Ok(true) => false, + Ok(false) => { + operational_error = + Some(anyhow!("admitted physical write lease is missing")); + true + } + Err(error) => { + eprintln!( + "Idunn preserved admitted {} after refusing an unsafe write-lease mutation: {error:#}", + current.value.target + ); + continue; + } + } + } else { + false + }; + if lease_is_missing { + None + } else { + Some(observation) + } + } + Err(error) => { + operational_error = Some(error); + None + } + }; + if let Some(observation) = observation { + if blocker.is_none() + || blocker.is_some_and(|stored| stored.value.phase == DeploymentPhase::Complete) + { + let repair = (|| -> Result<()> { + let topology = self.topology(); + let provider_anchor = self.provider_anchor_for_plan(¤t.value.plan)?; + if !topology.admitted_runtime_projection_is_exact( + ¤t.value.expected, + &provider_anchor, + ¤t.value.activation, + current.value.leasing.lease(), + )? { + ensure!( + topology + .publish_expected(¤t.value.expected, &provider_anchor,)? + == current.value.expected.canonical_sha256()?, + "admitted Expected projection repair differs" + ); + ensure!( + topology.publish_observed_activation( + ¤t.value.expected, + ¤t.value.activation, + &observation, + )? == current.value.activation.canonical_sha256()?, + "admitted activation projection repair differs" + ); + if let Some(lease) = current.value.leasing.lease() { + ensure!( + topology.publish_process_write_lease( + ¤t.value.expected, + ¤t.value.activation, + lease, + )? == lease.canonical_sha256()?, + "admitted write-lease projection repair differs" + ); + } + } + Ok(()) + })(); + if let Err(error) = repair { + eprintln!( + "Idunn preserved admitted {} after refusing topology projection repair: {error:#}", + current.value.target + ); + } + } + continue; + } + let workload_error = operational_error + .context("admitted operational state has neither observation nor error")?; + if let Some(blocker) = blocker { + if blocker.value.phase < DeploymentPhase::Fencing + && blocker.value.pre_fencing_abort.is_none() + { + self.begin_pre_fencing_abort( + blocker, + anyhow!( + "admitted incumbent failed before candidate fencing; deployment yielded to continuity: {workload_error:#}" + ), + )?; + progressed = true; + } continue; } let now = now_millis()?; @@ -2278,9 +2706,129 @@ impl Engine { )?, "continuity scheduling lost its command/transaction CAS" ); + progressed = true; + } + Ok(progressed) + } + + fn supervise_admitted_route(&self, current: &Stored) -> Result { + let Some(expected_route) = current.value.expected.route.as_ref() else { + ensure!( + matches!(¤t.value.routing, RoutingEvidence::SkippedUnrouted) + && current.value.route_repair_started_at_unix_millis.is_none(), + "unrouted admitted generation carries route authority" + ); + return Ok(false); + }; + let RoutingEvidence::Promoted { + observation, + promoted_at_unix_millis, + } = ¤t.value.routing + else { + bail!("routed admitted generation has no promoted route receipt") + }; + ensure!( + observation.route_id == expected_route.route_id + && observation.runtime_instance_id == current.value.activation.runtime_instance_id, + "admitted route receipt names another incarnation" + ); + let binding = current.value.plan.parsed_inputs()?.1; + let lease_driver = if let Some(lease) = current.value.leasing.lease() { + let lease_path = &binding + .process_write_lease + .as_ref() + .context("stateful admitted generation has no write-lease binding")? + .record_path; + let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); + ensure!( + driver.observe_exact(lease)?, + "admitted process write lease is no longer exact" + ); + Some(driver) + } else { + None + }; + let route_binding = binding + .route + .context("routed admitted generation has no operator route binding")?; + let driver = NginxRouteDriver::new(route_binding); + + let membership_is_exact = + driver.observe_membership(¤t.value.expected, &observation.membership_sha256)?; + let now = now_millis()?; + if membership_is_exact + && route_observation_is_current( + observation.observed_at_unix_millis, + now, + self.options.topology_maximum_age_millis, + self.options.topology_maximum_future_skew_millis, + ) + && current.value.route_repair_started_at_unix_millis.is_none() + { + return Ok(false); + } + if current.value.route_repair_started_at_unix_millis.is_none() { + let mut next = current.value.clone(); + next.route_repair_started_at_unix_millis = Some(now); + next.validate()?; + ensure!( + SingleFileMessagePackBackingStore::new(&self.options.state_store) + .compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: AdmittedGeneration::TYPE.into(), + key: current.value.target.clone(), + current: Some(current.envelope.clone()), + }], + &[admitted_envelope(&next, now)?], + )?, + "admitted generation changed before route repair intent CAS" + ); return Ok(true); } - Ok(false) + driver + .restore_admitted_membership(¤t.value.expected, &observation.membership_sha256)?; + let authority = self.runtime_authority_parts( + ¤t.value.plan, + ¤t.value.expected, + ¤t.value.activation, + )?; + let refreshed = self.prove_stable_route_against( + ¤t.value.expected, + ¤t.value.activation, + &authority, + current.value.leasing.lease_sha256(), + &driver, + observation.membership_sha256.clone(), + )?; + ensure!( + driver.observe_membership(¤t.value.expected, &refreshed.membership_sha256)?, + "admitted route membership changed during its continuity challenge" + ); + if let (Some(lease_driver), Some(lease)) = (&lease_driver, current.value.leasing.lease()) { + ensure!( + lease_driver.observe_exact(lease)?, + "admitted process write lease changed during its continuity challenge" + ); + } + let mut next = current.value.clone(); + next.routing = RoutingEvidence::Promoted { + observation: refreshed, + promoted_at_unix_millis: *promoted_at_unix_millis, + }; + next.route_repair_started_at_unix_millis = None; + next.validate()?; + ensure!( + SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: AdmittedGeneration::TYPE.into(), + key: current.value.target.clone(), + current: Some(current.envelope.clone()), + }], + &[admitted_envelope(&next, now)?], + )?, + "admitted generation changed before route continuity receipt CAS" + ); + Ok(true) } fn refresh_admitted_topology( @@ -2312,7 +2860,7 @@ impl Engine { let evidence = TopologyEvidence::from_authenticated(&authenticated, now)?; if !sequence_requires_admission( Some(¤t.value.latest_odin_observation), - snapshot.max_odin_sequence(&evidence.signer_identity_id), + snapshot.max_odin_sequence(¤t.value.target, &evidence.signer_identity_id), &evidence, )? { return Ok(false); @@ -2434,8 +2982,12 @@ impl Engine { if current.value.expected_publication_sha256.is_none() { let now = now_millis()?; + let plan = required(¤t.value.plan, "transaction plan")?; let expected = required(¤t.value.expected, "Expected projection")?; - let digest = self.topology().publish_expected(expected)?; + let provider_anchor = self.provider_anchor_for_plan(plan)?; + let digest = self + .topology() + .publish_expected(expected, &provider_anchor)?; return self.persist_same_phase(current, |next| { next.expected_publication_sha256 = Some(digest); next.updated_at_unix_millis = now; @@ -2541,35 +3093,64 @@ impl Engine { let activation = required(¤t.value.activation, "activation")?; let workload = required(¤t.value.workload, "workload")?; self.workload.observe(expected, activation, workload)?; - { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let incumbent_lease_sha256 = + self.incumbent_lease_sha256_for_warming(&snapshot, ¤t.value)?; + if current.value.warming.is_none() { + if current.value.target == "odin" && snapshot.admitted_for("odin").is_none() { + ensure!( + current.value.incumbent_generation_id.is_none() + && expected.write_lease_required, + "first Odin bootstrap must be a stateful first incarnation" + ); + let (evidence, present) = self.observe_first_odin_warming(¤t.value)?; + let _token = SequenceAdmittedWarming::from_first_odin_presence( + current.value.transaction_id.clone(), + evidence.clone(), + present, + )?; + return self.persist_same_phase(current, |next| { + next.warming = Some(WarmingEvidence::FirstOdinDirect { evidence }); + Ok(()) + }); + } let Some((admitted, authenticated)) = self.admit_latest_topology(current, None)? else { return Ok(()); }; if admitted.envelope != current.envelope { return Ok(()); } - let semantic_warming = is_semantic_warming(&authenticated); + let semantic_warming = is_semantic_warming( + expected, + activation, + incumbent_lease_sha256.as_deref(), + &authenticated, + )?; let latest = required( &admitted.value.latest_odin_observation, "sequence-admitted warming evidence", )?; - if current.value.warming.as_ref() == Some(latest) { - ensure!(semantic_warming, "stored Warming receipt changed meaning"); - } else if semantic_warming { + if semantic_warming { let evidence = latest.clone(); - let _token = SequenceAdmittedWarming { - transaction_id: admitted.value.transaction_id.clone(), - evidence: evidence.clone(), + let _token = SequenceAdmittedWarming::from_topology( + admitted.value.transaction_id.clone(), + evidence.clone(), authenticated, - }; + )?; return self.persist_same_phase(&admitted, |next| { - next.warming = Some(evidence); + next.warming = Some(WarmingEvidence::OdinTopology { evidence }); Ok(()) }); } else { return Ok(()); } } + let warming = self.rehydrate_warming_token(¤t.value, now_millis()?, false)?; + ensure!( + warming.transaction_id() == current.value.transaction_id + && warming.runtime_instance_id() == activation.runtime_instance_id.as_str(), + "durable Warming evidence belongs to another candidate" + ); if expected.route.is_some() && current.value.route_preflight.is_none() { let now = now_millis()?; @@ -2632,12 +3213,21 @@ impl Engine { }) .transpose()?; if let (Some(lease), Some(path)) = (incumbent_lease, &incumbent_lease_path) { + let incumbent = incumbent.context("incumbent lease lost its generation")?; + self.workload + .stop(&incumbent.value.workload) + .context("stopping the exact incumbent before revoking its lifetime-held write lease")?; let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, path); driver.revoke_exact(Some(lease))?; ensure!( driver.observe_empty()?, "incumbent write lease remained after exact fencing" ); + self.topology().withdraw_process_write_lease( + &incumbent.value.expected, + &incumbent.value.activation, + Some(lease), + )?; } let candidate_lease_path = if expected.write_lease_required { Some( @@ -2665,6 +3255,11 @@ impl Engine { ); } } + self.topology().withdraw_process_write_lease( + expected, + required(¤t.value.activation, "candidate activation")?, + None, + )?; FencingEvidence::Revoked { incumbent_lease_sha256: incumbent_lease .map(IdunnProcessWriteLeaseRecord::canonical_sha256) @@ -2684,74 +3279,64 @@ impl Engine { } fn advance_leasing(&self, current: &Stored) -> Result<()> { - if current.value.leasing.is_none() { - let now = now_millis()?; - let expected = required(¤t.value.expected, "Expected projection")?; - let evidence = if expected.write_lease_required { - let warming = self.rehydrate_warming_token(¤t.value, now)?; - ensure!( - warming.transaction_id() == current.value.transaction_id - && Some(warming.evidence_sha256()) - == current - .value - .warming - .as_ref() - .map(|value| value.canonical_sha256.as_str()), - "Warming token does not bind this transaction's durable admission" - ); - let activation = required(¤t.value.activation, "activation")?; - let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; - let _lease_path = binding - .process_write_lease - .context("stateful target has no write-lease binding")? - .record_path; - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let epoch = self - .exact_incumbent(&snapshot, ¤t.value)? - .and_then(|generation| generation.value.leasing.lease()) - .map_or(1, |lease| lease.lease_epoch.saturating_add(1)); - let warming_presence_sha256 = warming - .authenticated() - .record() - .signed_presence_sha256 - .clone() - .context("warming token has no signed presence")?; - let lease = IdunnProcessWriteLeaseRecord { - schema_version: IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into(), - target: expected.target.clone(), - expected_projection_sha256: expected.canonical_sha256()?, - plan_id: expected.plan_id.clone(), - incarnation_id: expected.incarnation_id.clone(), - sealed_release_id: expected.sealed_release_id.clone(), - activation_witness_sha256: activation.canonical_sha256()?, - state_schema_generation: expected - .state_schema_generation - .clone() - .context("stateful Expected has no schema generation")?, - state_contract_sha256: expected - .state_contract_sha256 - .clone() - .context("stateful Expected has no state contract")?, - runtime_id: expected.runtime_id.clone(), - runtime_instance_id: activation.runtime_instance_id.clone(), - warming_presence_sha256, - lease_epoch: epoch, - issued_at_unix_millis: now, - }; - lease.validate()?; - let lease_sha256 = lease.canonical_sha256()?; - LeasingEvidence::Prepared { - lease, - lease_sha256, - } - } else { - LeasingEvidence::SkippedStateless - }; - return self.persist_same_phase(current, |next| { - next.leasing = Some(evidence); - next.updated_at_unix_millis = now; - Ok(()) - }); + let expected = required(¤t.value.expected, "Expected projection")?; + if !expected.write_lease_required { + if current.value.leasing.is_none() { + let now = now_millis()?; + return self.persist_same_phase(current, |next| { + next.leasing = Some(LeasingEvidence::SkippedStateless); + next.updated_at_unix_millis = now; + Ok(()) + }); + } + return self.transition(current, DeploymentPhase::AwaitingReady); + } + + let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; + let lease_path = binding + .process_write_lease + .context("stateful target has no write-lease binding")? + .record_path; + let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); + + if matches!( + current.value.leasing.as_ref(), + Some(LeasingEvidence::Granted { .. }) + ) { + let activation = required(¤t.value.activation, "activation")?; + let warming = self.rehydrate_warming_token(¤t.value, now_millis()?, false)?; + let lease = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease) + .context("granted Leasing evidence has no write lease")?; + let recorded_sha256 = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256) + .context("granted Leasing evidence has no lease digest")?; + ensure!( + driver.observe_exact(lease)?, + "physical write lease disappeared after Granted became durable" + ); + self.workload.observe( + expected, + activation, + required(¤t.value.workload, "candidate workload")?, + )?; + ensure!( + driver.grant(expected, activation, &warming, lease)? == recorded_sha256, + "replayed physical write lease differs from Granted evidence" + ); + ensure!( + self.topology() + .publish_process_write_lease(expected, activation, lease)? + == recorded_sha256, + "replayed write-lease projection differs from Granted evidence" + ); + return self.transition(current, DeploymentPhase::AwaitingReady); } if let Some((lease, prepared_sha256)) = current @@ -2761,42 +3346,101 @@ impl Engine { .and_then(LeasingEvidence::prepared_lease) { let now = now_millis()?; - let expected = required(¤t.value.expected, "Expected projection")?; let activation = required(¤t.value.activation, "activation")?; - let warming = self.rehydrate_warming_token(¤t.value, now)?; - ensure!( - warming.transaction_id() == current.value.transaction_id - && Some(warming.evidence_sha256()) - == current - .value - .warming - .as_ref() - .map(|value| value.canonical_sha256.as_str()), - "Warming token does not bind this transaction's durable admission" - ); - let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; - let lease_path = binding - .process_write_lease - .context("stateful target has no write-lease binding")? - .record_path; - let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); + let historical_warming = self.rehydrate_warming_token(¤t.value, now, false)?; + let physical_is_exact = driver.observe_exact(lease)?; + let warming = if physical_is_exact { + historical_warming + } else { + ensure!( + driver.observe_empty()?, + "candidate write-lease store contains authority other than its durable prepared lease" + ); + match self.rehydrate_warming_token(¤t.value, now, true) { + Ok(warming) => warming, + Err(_) => { + let Some((admitted, fresh_evidence, fresh_warming)) = + self.fresh_warming_for_lease(current, now)? + else { + return Ok(()); + }; + let replacement = self.prepare_candidate_write_lease( + &admitted.value, + &fresh_warming, + now, + )?; + let replacement_sha256 = replacement.canonical_sha256()?; + return self.persist_same_phase(&admitted, |next| { + next.warming = Some(fresh_evidence); + next.leasing = Some(LeasingEvidence::Prepared { + lease: replacement, + lease_sha256: replacement_sha256, + }); + next.updated_at_unix_millis = now; + Ok(()) + }); + } + } + }; + self.workload.observe( + expected, + activation, + required(¤t.value.workload, "candidate workload")?, + )?; + if !physical_is_exact { + ensure!( + driver.observe_empty()?, + "candidate write-lease store changed before physical grant" + ); + } let granted_sha256 = driver.grant(expected, activation, &warming, lease)?; ensure!( granted_sha256 == prepared_sha256, "granted write lease differs from the durable prepared lease" ); + let projected_sha256 = self + .topology() + .publish_process_write_lease(expected, activation, lease)?; + ensure!( + projected_sha256 == granted_sha256, + "projected write lease differs from the granted process authority" + ); let lease = lease.clone(); return self.persist_same_phase(current, |next| { next.leasing = Some(LeasingEvidence::Granted { lease, - lease_sha256: granted_sha256, + lease_sha256: projected_sha256, }); next.updated_at_unix_millis = now; Ok(()) }); } - self.transition(current, DeploymentPhase::AwaitingReady) + ensure!( + current.value.leasing.is_none(), + "stateful Leasing phase carries invalid lease evidence" + ); + ensure!( + driver.observe_empty()?, + "candidate write-lease store contains authority before lease preparation" + ); + let now = now_millis()?; + let Some((admitted, fresh_evidence, fresh_warming)) = + self.fresh_warming_for_lease(current, now)? + else { + return Ok(()); + }; + let lease = self.prepare_candidate_write_lease(&admitted.value, &fresh_warming, now)?; + let lease_sha256 = lease.canonical_sha256()?; + self.persist_same_phase(&admitted, |next| { + next.warming = Some(fresh_evidence); + next.leasing = Some(LeasingEvidence::Prepared { + lease, + lease_sha256, + }); + next.updated_at_unix_millis = now; + Ok(()) + }) } fn advance_awaiting_ready(&self, current: &Stored) -> Result<()> { @@ -2804,7 +3448,7 @@ impl Engine { let expected = required(¤t.value.expected, "Expected projection")?; if expected.write_lease_required { let activation = required(¤t.value.activation, "activation")?; - let warming = self.rehydrate_warming_token(¤t.value, now)?; + let warming = self.rehydrate_warming_token(¤t.value, now, false)?; let lease = current .value .leasing @@ -2863,43 +3507,108 @@ impl Engine { } fn advance_routing(&self, current: &Stored) -> Result<()> { - let now = now_millis()?; let expected = required(¤t.value.expected, "Expected projection")?; let activation = required(¤t.value.activation, "activation")?; if current.value.routing.is_none() { - let ready = self.rehydrate_ready_token(¤t.value, now, false)?; + let current_lease = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256); + let Some((admitted, authenticated)) = + self.admit_latest_topology(current, current_lease)? + else { + return Ok(()); + }; + if admitted.envelope != current.envelope { + return Ok(()); + } ensure!( - ready.transaction_id() == current.value.transaction_id, + is_semantic_ready(&authenticated), + "latest Odin observation is not Ready at route admission" + ); + let latest = required( + &admitted.value.latest_odin_observation, + "current route-admission topology evidence", + )?; + if admitted.value.ready.as_ref() != Some(latest) { + let latest = latest.clone(); + return self.persist_same_phase(&admitted, |next| { + next.ready = Some(latest); + Ok(()) + }); + } + let ready = self.rehydrate_ready_token(&admitted.value, now_millis()?, true)?; + ensure!( + ready.transaction_id() == admitted.value.transaction_id, "Ready token belongs to another transaction" ); + validate_live_providers_for_deploy(admitted.value.command_kind, || { + self.validate_selected_providers_current( + required(&admitted.value.plan, "transaction plan")?, + now_millis()?, + ) + })?; + self.ensure_transaction_write_lease_current(&admitted.value, now_millis()?)?; let evidence = if expected.route.is_some() { - let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; + let binding = admitted.value.plan.as_ref().unwrap().parsed_inputs()?.1; let route_binding = binding.route.context("routed plan has no route binding")?; - let preflight = required(¤t.value.route_preflight, "route preflight")?; + let preflight = required(&admitted.value.route_preflight, "route preflight")?; let driver = NginxRouteDriver::new(route_binding); - let exact_candidate = RouteObservation { - route_id: preflight.route_id.clone(), - runtime_instance_id: preflight.candidate_runtime_instance_id.clone(), - membership_sha256: preflight.candidate_membership_sha256.clone(), - }; ensure!( - exact_candidate.runtime_instance_id == activation.runtime_instance_id, + preflight.candidate_runtime_instance_id == activation.runtime_instance_id, "route preflight belongs to another runtime instance" ); - let observation = if driver.observe(expected, &exact_candidate)? { - exact_candidate - } else { - driver.promote(expected, &activation.runtime_instance_id, preflight)? + let rollback_allowed = may_rollback_route_after_failed_proof(required( + &admitted.value.fencing, + "route fencing evidence", + )?); + let membership_sha256 = driver.install( + expected, + &activation.runtime_instance_id, + preflight, + rollback_allowed, + )?; + let observation = match self.prove_stable_route( + &admitted.value, + &driver, + membership_sha256, + ) { + Ok(observation) => observation, + Err(proof_error) if !rollback_allowed => { + return Err(proof_error).context( + "incumbent authority was fenced; candidate route remains installed for fail-closed retry", + ); + } + Err(proof_error) => { + return match driver.rollback( + expected, + &activation.runtime_instance_id, + preflight, + ) { + Ok(()) => Err(proof_error) + .context("candidate did not answer its stable route challenge"), + Err(rollback_error) => Err(proof_error).context(format!( + "candidate did not answer its stable route challenge; exact route rollback also failed: {rollback_error:#}" + )), + }; + } }; ensure!( - driver.observe(expected, &observation)?, - "promoted route is not the exact candidate membership" + driver.observe_membership(expected, &observation.membership_sha256)?, + "route membership changed during its signed stable-listener observation" ); - RoutingEvidence::Promoted { observation } + self.ensure_transaction_write_lease_current(&admitted.value, now_millis()?)?; + let promoted_at_unix_millis = observation.observed_at_unix_millis; + RoutingEvidence::Promoted { + observation, + promoted_at_unix_millis, + } } else { RoutingEvidence::SkippedUnrouted }; - return self.persist_same_phase(current, |next| { + let now = now_millis()?; + return self.persist_same_phase(&admitted, |next| { next.routing = Some(evidence); next.updated_at_unix_millis = now; Ok(()) @@ -2909,64 +3618,145 @@ impl Engine { } fn advance_committing(&self, current: &Stored) -> Result<()> { - let now = now_millis()?; - let expected = required(¤t.value.expected, "Expected projection")?; - let activation = required(¤t.value.activation, "activation")?; - let workload = required(¤t.value.workload, "workload")?; - self.workload.observe(expected, activation, workload)?; - let _ready = self.rehydrate_ready_token(¤t.value, now, false)?; - - if let Some(lease) = current + let current_lease_sha256 = current .value .leasing .as_ref() - .and_then(LeasingEvidence::lease) - { - let warming = self.rehydrate_warming_token(¤t.value, now)?; - let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; - let path = binding - .process_write_lease - .context("stateful target has no write-lease binding")? - .record_path; - ensure!( - CultCacheWriteLeaseDriver::new(¤t.value.target, path) - .observe(expected, activation, &warming, lease)?, - "write lease changed before admission commit" - ); + .and_then(LeasingEvidence::lease_sha256) + .map(str::to_owned); + let Some((ready_current, authenticated)) = + self.admit_latest_topology(current, current_lease_sha256.as_deref())? + else { + return Ok(()); + }; + if ready_current.envelope != current.envelope { + return Ok(()); } - if let Some(route) = current + ensure!( + is_semantic_ready(&authenticated), + "latest Odin observation is not Ready at admission commit" + ); + ensure!( + ready_current.value.latest_odin_observation.as_ref() + == ready_current.value.ready.as_ref(), + "latest Odin observation differs from the durable Ready receipt at admission commit" + ); + self.rehydrate_ready_token(&ready_current.value, now_millis()?, true)?; + validate_live_providers_for_deploy(ready_current.value.command_kind, || { + self.validate_selected_providers_current( + required(&ready_current.value.plan, "transaction plan")?, + now_millis()?, + ) + })?; + + let expected = required(&ready_current.value.expected, "Expected projection")?; + let activation = required(&ready_current.value.activation, "activation")?; + let workload = required(&ready_current.value.workload, "workload")?; + self.workload.observe(expected, activation, workload)?; + self.ensure_transaction_write_lease_current(&ready_current.value, now_millis()?)?; + if let Some(route) = ready_current .value .routing .as_ref() .and_then(RoutingEvidence::observation) { - let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; + let binding = ready_current + .value + .plan + .as_ref() + .unwrap() + .parsed_inputs()? + .1; let driver = NginxRouteDriver::new(binding.route.context("routed plan has no route binding")?); ensure!( - driver.observe(expected, route)?, - "route changed before admission commit" + driver.observe_membership(expected, &route.membership_sha256)?, + "route membership changed before admission commit" ); + let current_route = self.prove_stable_route( + &ready_current.value, + &driver, + route.membership_sha256.clone(), + )?; + ensure!( + current_route.route_id == route.route_id + && current_route.runtime_instance_id == route.runtime_instance_id, + "stable route changed incarnation before admission commit" + ); + ensure!( + driver.observe_membership(expected, &route.membership_sha256)?, + "route membership changed during the final signed admission challenge" + ); + self.ensure_transaction_write_lease_current(&ready_current.value, now_millis()?)?; } + let current_lease_sha256 = ready_current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256) + .map(str::to_owned); + let Some((commit_current, authenticated)) = + self.admit_latest_topology(&ready_current, current_lease_sha256.as_deref())? + else { + return Ok(()); + }; + if commit_current.envelope != ready_current.envelope { + return Ok(()); + } + ensure!( + is_semantic_ready(&authenticated), + "latest Odin observation is not Ready after the final admission challenge" + ); + ensure!( + commit_current.value.latest_odin_observation.as_ref() + == commit_current.value.ready.as_ref(), + "latest Odin observation differs from the durable Ready receipt after the final admission challenge" + ); + let now = now_millis()?; + self.rehydrate_ready_token(&commit_current.value, now, true)?; + validate_live_providers_for_deploy(commit_current.value.command_kind, || { + self.validate_selected_providers_current( + required(&commit_current.value.plan, "transaction plan")?, + now, + ) + })?; + self.workload.observe( + required(&commit_current.value.expected, "Expected projection")?, + required(&commit_current.value.activation, "activation")?, + required(&commit_current.value.workload, "workload")?, + )?; + self.ensure_transaction_write_lease_current(&commit_current.value, now)?; + let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let incumbent = self.exact_incumbent(&snapshot, ¤t.value)?; + let incumbent = self.exact_incumbent(&snapshot, &commit_current.value)?; let odin_authority = self.current_odin_authority(&snapshot)?; - let generation = AdmittedGeneration::from_transaction(¤t.value, odin_authority, now)?; + let generation = + AdmittedGeneration::from_transaction(&commit_current.value, odin_authority, now)?; let post_commit_cleanup = PostCommitCleanup { incumbent: match incumbent { + Some(incumbent) + if incumbent_was_stopped_during_fencing(required( + &commit_current.value.fencing, + "commit fencing evidence", + )?) => + { + IncumbentCleanupEvidence::Complete { + generation_id: incumbent.value.generation_id.clone(), + } + } Some(incumbent) => IncumbentCleanupEvidence::Pending { generation_id: incumbent.value.generation_id.clone(), workload: incumbent.value.workload.clone(), }, None => IncumbentCleanupEvidence::SkippedNoIncumbent, }, - source: match current.value.command_kind { + source: match commit_current.value.command_kind { CommandKind::Deploy => SourceCleanupEvidence::Pending, CommandKind::Continuity => SourceCleanupEvidence::SkippedContinuity, }, }; - let mut complete = current.value.clone(); + let mut complete = commit_current.value.clone(); complete.phase = DeploymentPhase::Complete; complete.updated_at_unix_millis = now; complete.last_error = None; @@ -2986,8 +3776,8 @@ impl Engine { &[ CultCacheExpectedEnvelope { r#type: DeploymentTransaction::TYPE.into(), - key: current.value.transaction_id.clone(), - current: Some(current.envelope.clone()), + key: commit_current.value.transaction_id.clone(), + current: Some(commit_current.envelope.clone()), }, admitted_expected, ], @@ -3016,6 +3806,187 @@ impl Engine { } } + fn incumbent_lease_sha256_for_warming( + &self, + snapshot: &ControlSnapshot, + transaction: &DeploymentTransaction, + ) -> Result> { + if let Some(fencing) = &transaction.fencing { + return match fencing { + FencingEvidence::SkippedStateless => Ok(None), + FencingEvidence::Revoked { + incumbent_lease_sha256, + .. + } => Ok(incumbent_lease_sha256.clone()), + }; + } + self.exact_incumbent(snapshot, transaction)? + .and_then(|incumbent| incumbent.value.leasing.lease()) + .map(IdunnProcessWriteLeaseRecord::canonical_sha256) + .transpose() + } + + fn fresh_warming_for_lease( + &self, + current: &Stored, + now: u64, + ) -> Result< + Option<( + Stored, + WarmingEvidence, + SequenceAdmittedWarming, + )>, + > { + let prior = self.rehydrate_warming_token(¤t.value, now, false)?; + match required(¤t.value.warming, "pre-fence Warming evidence")?.clone() { + WarmingEvidence::FirstOdinDirect { .. } => { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + ensure!( + snapshot.admitted_for("odin").is_none() + && self.exact_incumbent(&snapshot, ¤t.value)?.is_none(), + "direct first-Odin Warming refresh found an admitted Odin" + ); + let (evidence, present) = self.observe_first_odin_warming(¤t.value)?; + let token = SequenceAdmittedWarming::from_first_odin_presence( + current.value.transaction_id.clone(), + evidence.clone(), + present, + )?; + ensure!( + token.signed_presence_sha256() != prior.signed_presence_sha256(), + "first Odin replayed its pre-fence Warming presence" + ); + Ok(Some(( + current.clone(), + WarmingEvidence::FirstOdinDirect { evidence }, + token, + ))) + } + WarmingEvidence::OdinTopology { + evidence: prior_evidence, + } => { + let Some((admitted, authenticated)) = self.admit_latest_topology(current, None)? + else { + return Ok(None); + }; + let fresh_evidence = required( + &admitted.value.latest_odin_observation, + "post-fence Odin Warming observation", + )? + .clone(); + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let incumbent_lease_sha256 = + self.incumbent_lease_sha256_for_warming(&snapshot, &admitted.value)?; + if !is_semantic_warming( + required(&admitted.value.expected, "Warming Expected projection")?, + required(&admitted.value.activation, "Warming activation")?, + incumbent_lease_sha256.as_deref(), + &authenticated, + )? { + return Ok(None); + } + let token = SequenceAdmittedWarming::from_topology( + admitted.value.transaction_id.clone(), + fresh_evidence.clone(), + authenticated, + )?; + if !provider_warming_advanced( + prior_evidence.publisher_sequence, + prior.signed_presence_sha256(), + fresh_evidence.publisher_sequence, + token.signed_presence_sha256(), + ) { + return Ok(None); + } + Ok(Some(( + admitted, + WarmingEvidence::OdinTopology { + evidence: fresh_evidence, + }, + token, + ))) + } + } + } + + fn prepare_candidate_write_lease( + &self, + transaction: &DeploymentTransaction, + warming: &SequenceAdmittedWarming, + now: u64, + ) -> Result { + let expected = required(&transaction.expected, "Expected projection")?; + let activation = required(&transaction.activation, "activation")?; + ensure!( + expected.write_lease_required + && warming.transaction_id() == transaction.transaction_id + && warming.runtime_instance_id() == activation.runtime_instance_id, + "fresh Warming token does not own this stateful candidate" + ); + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let epoch = self + .exact_incumbent(&snapshot, transaction)? + .and_then(|generation| generation.value.leasing.lease()) + .map_or(1, |lease| lease.lease_epoch.saturating_add(1)); + let lease = IdunnProcessWriteLeaseRecord { + schema_version: IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into(), + target: expected.target.clone(), + expected_projection_sha256: expected.canonical_sha256()?, + plan_id: expected.plan_id.clone(), + incarnation_id: expected.incarnation_id.clone(), + sealed_release_id: expected.sealed_release_id.clone(), + activation_witness_sha256: activation.canonical_sha256()?, + state_schema_generation: expected + .state_schema_generation + .clone() + .context("stateful Expected has no schema generation")?, + state_contract_sha256: expected + .state_contract_sha256 + .clone() + .context("stateful Expected has no state contract")?, + runtime_id: expected.runtime_id.clone(), + runtime_instance_id: activation.runtime_instance_id.clone(), + warming_presence_sha256: warming.signed_presence_sha256().to_owned(), + lease_epoch: epoch, + issued_at_unix_millis: now, + }; + lease.validate()?; + Ok(lease) + } + + fn ensure_transaction_write_lease_current( + &self, + transaction: &DeploymentTransaction, + now: u64, + ) -> Result<()> { + let expected = required(&transaction.expected, "Expected projection")?; + let lease = transaction + .leasing + .as_ref() + .and_then(LeasingEvidence::lease); + ensure!( + expected.write_lease_required == lease.is_some(), + "transaction write-lease disposition differs from Expected" + ); + let Some(lease) = lease else { + return Ok(()); + }; + let activation = required(&transaction.activation, "activation")?; + let warming = self.rehydrate_warming_token(transaction, now, false)?; + let lease_path = required(&transaction.plan, "transaction plan")? + .parsed_inputs()? + .1 + .process_write_lease + .context("stateful target has no write-lease binding")? + .record_path; + ensure!( + CultCacheWriteLeaseDriver::new(&transaction.target, lease_path) + .observe(expected, activation, &warming, lease,)?, + "process write lease changed across the authority boundary" + ); + Ok(()) + } + fn select_candidate_port(&self, binding: &OperatorBinding) -> Result> { let Some(route) = &binding.route else { return Ok(None); @@ -3030,7 +4001,7 @@ impl Engine { snapshot .transactions .iter() - .filter(|stored| stored.value.owns_target_authority()) + .filter(|stored| stored.value.blocks_new_target_mutation()) .filter_map(|stored| stored.value.plan.as_ref()), ) { @@ -3143,16 +4114,203 @@ impl Engine { ) } + fn authenticate_routed_presence( + &self, + authority: &cultnet_rs::VerifiedRuntimeAuthority, + current_write_lease_sha256: Option<&str>, + message_id: &str, + challenged_at_unix_millis: u64, + received_at_unix_millis: u64, + canonical_presence: &[u8], + ) -> Result<(String, u64)> { + ensure!( + received_at_unix_millis >= challenged_at_unix_millis, + "route observation predates its challenge" + ); + let authenticated = authenticate_runtime_presence_claim( + canonical_presence, + authority, + RuntimePresenceAuthenticationContext { + trusted_received_at_unix_millis: received_at_unix_millis, + maximum_age_millis: self.options.topology_maximum_age_millis, + maximum_future_skew_millis: self.options.topology_maximum_future_skew_millis, + }, + )?; + let signed_presence_sha256 = authenticated.signed_presence_sha256().to_owned(); + let correlation = correlate_runtime_presence_claim(authenticated, authority)?; + ensure!( + correlation.disagreements().is_empty(), + "stable route answered with a runtime that disagrees with current authority" + ); + let present = correlation.into_undisputed_present()?; + let presence = present.record(); + ensure!( + presence.observed_at_unix_millis >= challenged_at_unix_millis, + "stable route returned a presence minted before the route challenge" + ); + ensure!( + presence.state == "active", + "stable route runtime is not Active" + ); + ensure!( + presence.detail == format!("route-observation:{message_id}"), + "stable route response is not bound to the exact challenge" + ); + ensure!( + presence.write_lease_sha256.as_deref() == current_write_lease_sha256, + "stable route runtime does not hold the exact current process write lease" + ); + Ok((signed_presence_sha256, received_at_unix_millis)) + } + + fn authenticate_first_odin_warming_presence( + &self, + transaction: &DeploymentTransaction, + message_id: &str, + challenged_at_unix_millis: u64, + received_at_unix_millis: u64, + canonical_presence: &[u8], + ) -> Result { + ensure!( + transaction.target == "odin" + && transaction.incumbent_generation_id.is_none() + && required(&transaction.expected, "first Odin Expected")?.write_lease_required, + "direct Warming presence is reserved for stateful first Odin bootstrap" + ); + ensure!( + received_at_unix_millis >= challenged_at_unix_millis, + "first Odin Warming observation predates its challenge" + ); + let authority = self.runtime_authority(transaction)?; + let authenticated = authenticate_runtime_presence_claim( + canonical_presence, + &authority, + RuntimePresenceAuthenticationContext { + trusted_received_at_unix_millis: received_at_unix_millis, + maximum_age_millis: self.options.topology_maximum_age_millis, + maximum_future_skew_millis: self.options.topology_maximum_future_skew_millis, + }, + )?; + let correlation = correlate_runtime_presence_claim(authenticated, &authority)?; + ensure!( + correlation.disagreements().is_empty(), + "first Odin Warming presence disagrees with its Expected activation" + ); + let present = correlation.into_undisputed_present()?; + let presence = present.record(); + ensure!( + presence.observed_at_unix_millis >= challenged_at_unix_millis + && presence.state == "warming" + && presence.write_lease_sha256.is_none() + && presence.detail == format!("idunn-warming:{message_id}"), + "first Odin candidate did not return exact fresh pre-lease Warming evidence" + ); + Ok(present) + } + + fn observe_first_odin_warming( + &self, + transaction: &DeploymentTransaction, + ) -> Result<(RuntimePresenceEvidence, cultnet_rs::VerifiedRuntimePresence)> { + let expected = required(&transaction.expected, "first Odin Expected")?; + let binding = required(&transaction.plan, "first Odin plan")? + .parsed_inputs()? + .1; + let driver = NginxRouteDriver::new( + binding + .route + .context("first Odin bootstrap has no candidate route binding")?, + ); + let message_id = format!("warming-{}", Uuid::new_v4().simple()); + let challenged_at_unix_millis = now_millis()?; + let response = driver.request_candidate_runtime_presence(expected, &message_id)?; + ensure!( + response.message_id == message_id, + "first Odin candidate transport substituted its challenge identity" + ); + let admitted_at_unix_millis = now_millis()?; + let present = self.authenticate_first_odin_warming_presence( + transaction, + &message_id, + challenged_at_unix_millis, + admitted_at_unix_millis, + &response.canonical_presence, + )?; + let evidence = RuntimePresenceEvidence::from_present( + &present, + message_id, + challenged_at_unix_millis, + admitted_at_unix_millis, + )?; + Ok((evidence, present)) + } + + fn prove_stable_route( + &self, + transaction: &DeploymentTransaction, + driver: &NginxRouteDriver, + membership_sha256: String, + ) -> Result { + let expected = required(&transaction.expected, "route Expected projection")?; + let activation = required(&transaction.activation, "route activation")?; + let authority = self.runtime_authority(transaction)?; + let current_write_lease_sha256 = transaction + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256); + self.prove_stable_route_against( + expected, + activation, + &authority, + current_write_lease_sha256, + driver, + membership_sha256, + ) + } + + fn prove_stable_route_against( + &self, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + authority: &cultnet_rs::VerifiedRuntimeAuthority, + current_write_lease_sha256: Option<&str>, + driver: &NginxRouteDriver, + membership_sha256: String, + ) -> Result { + let message_id = format!("route-{}", Uuid::new_v4().simple()); + let challenged_at_unix_millis = now_millis()?; + let response = driver.request_runtime_presence(expected, &message_id)?; + ensure!( + response.message_id == message_id, + "route transport substituted its challenge identity" + ); + let received_at_unix_millis = now_millis()?; + let (signed_presence_sha256, observed_at_unix_millis) = self.authenticate_routed_presence( + authority, + current_write_lease_sha256, + &message_id, + challenged_at_unix_millis, + received_at_unix_millis, + &response.canonical_presence, + )?; + let observation = RouteObservation { + route_id: driver.binding.route_id.clone(), + runtime_instance_id: activation.runtime_instance_id.clone(), + membership_sha256, + signed_presence_sha256, + observed_at_unix_millis, + }; + observation.validate()?; + Ok(observation) + } + fn runtime_authority_parts( &self, plan: &CompiledDeploymentPlan, expected: &IdunnExpectedIncarnationRecord, activation: &IdunnRuntimeActivationRecord, ) -> Result { - let binding = plan.parsed_inputs()?.1; - let provider_anchor = read_trust_anchor::( - &binding.runtime_identity.trust_anchor_store, - )?; + let provider_anchor = self.provider_anchor_for_plan(plan)?; verify_runtime_authority( expected, activation, @@ -3161,6 +4319,22 @@ impl Engine { ) } + fn provider_anchor_for_plan( + &self, + plan: &CompiledDeploymentPlan, + ) -> Result { + let binding = plan.parsed_inputs()?.1; + let provider_anchor = read_trust_anchor::( + &binding.runtime_identity.trust_anchor_store, + )?; + ensure!( + provider_anchor.schema_version + == ::TRUST_ANCHOR_SCHEMA, + "provider runtime presence trust anchor schema is unsupported" + ); + Ok(provider_anchor) + } + fn authenticate_topology_bytes( &self, snapshot: &ControlSnapshot, @@ -3220,7 +4394,7 @@ impl Engine { let evidence = TopologyEvidence::from_authenticated(&authenticated, now)?; if !sequence_requires_admission( live.value.latest_odin_observation.as_ref(), - snapshot.max_odin_sequence(&evidence.signer_identity_id), + snapshot.max_odin_sequence(&live.value.target, &evidence.signer_identity_id), &evidence, )? { return Ok(Some((live.clone(), authenticated))); @@ -3247,27 +4421,61 @@ impl Engine { fn rehydrate_warming_token( &self, transaction: &DeploymentTransaction, - _now: u64, + now: u64, + require_current: bool, ) -> Result { let evidence = required(&transaction.warming, "durable warming evidence")?.clone(); - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let authenticated = self.authenticate_topology_bytes( - &snapshot, - transaction, - &evidence.canonical_bytes, - None, - evidence.admitted_at_unix_millis, - )?; - validate_authenticated_evidence(&evidence, &authenticated)?; - ensure!( - is_semantic_warming(&authenticated), - "durable warming evidence no longer authenticates as Warming" - ); - Ok(SequenceAdmittedWarming { - transaction_id: transaction.transaction_id.clone(), - evidence, - authenticated, - }) + match evidence { + WarmingEvidence::OdinTopology { evidence } => { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let authenticated = self.authenticate_topology_bytes( + &snapshot, + transaction, + &evidence.canonical_bytes, + None, + if require_current { + now + } else { + evidence.admitted_at_unix_millis + }, + )?; + validate_authenticated_evidence(&evidence, &authenticated)?; + let incumbent_lease_sha256 = + self.incumbent_lease_sha256_for_warming(&snapshot, transaction)?; + ensure!( + is_semantic_warming( + required(&transaction.expected, "Warming Expected projection")?, + required(&transaction.activation, "Warming activation")?, + incumbent_lease_sha256.as_deref(), + &authenticated, + )?, + "durable warming evidence no longer satisfies the Warming gate" + ); + SequenceAdmittedWarming::from_topology( + transaction.transaction_id.clone(), + evidence, + authenticated, + ) + } + WarmingEvidence::FirstOdinDirect { evidence } => { + let present = self.authenticate_first_odin_warming_presence( + transaction, + &evidence.message_id, + evidence.challenged_at_unix_millis, + if require_current { + now + } else { + evidence.admitted_at_unix_millis + }, + &evidence.canonical_bytes, + )?; + SequenceAdmittedWarming::from_first_odin_presence( + transaction.transaction_id.clone(), + evidence, + present, + ) + } + } } fn rehydrate_ready_token( @@ -3399,16 +4607,32 @@ impl Engine { && admitted.value.plan.plan_id == *plan_id && admitted.value.sealed_release.sealed_release_id == *sealed_release_id && admitted.value.expected.canonical_sha256()? == *expected_projection_sha256 - && token.evidence_sha256() == odin_topology_correlation_sha256 - && token.publisher_sequence() == *odin_topology_publisher_sequence, + && token.publisher_sequence() >= *odin_topology_publisher_sequence, "selected dependency authority changed before actuation" ); - if selection.requirement.startup == StartupOrder::BeforeStart { - ensure!( - token.authenticated().record().ready, - "before-start dependency is not Ready" - ); - } + ensure!( + token.publisher_sequence() != *odin_topology_publisher_sequence + || token.evidence_sha256() == odin_topology_correlation_sha256, + "selected dependency Odin sequence changed evidence" + ); + ensure!( + token + .authenticated() + .record() + .observed_capabilities + .iter() + .any(|capability| { + capability_compatible( + &selection.requirement.capability, + &selection.requirement.schema, + &selection.requirement.compatibility, + &capability.capability, + &capability.schema, + &capability.compatibility, + ) && capability.capacity >= selection.requirement.minimum_capacity + }), + "selected dependency no longer provides its required capability" + ); } Ok(()) } @@ -3531,29 +4755,35 @@ impl Engine { let snapshot = ControlSnapshot::read(&self.options.state_store)?; if let Some(incumbent) = self.exact_incumbent(&snapshot, ¤t.value)? { let topology = self.topology(); - let expected_sha256 = topology.publish_expected(&incumbent.value.expected)?; + let failed_provider_anchor = self.provider_anchor_for_plan(required( + ¤t.value.plan, + "failed transaction plan", + )?)?; + let admitted_provider_anchor = + self.provider_anchor_for_plan(&incumbent.value.plan)?; + let expected_sha256 = topology.restore_admitted_expected_only( + expected, + &failed_provider_anchor, + current.value.activation.as_ref(), + &incumbent.value.expected, + &admitted_provider_anchor, + &incumbent.value.activation, + incumbent.value.leasing.lease(), + )?; ensure!( expected_sha256 == incumbent.value.expected.canonical_sha256()?, "restored incumbent Expected receipt differs" ); - if let Ok(observation) = self.workload.observe( - &incumbent.value.expected, - &incumbent.value.activation, - &incumbent.value.workload, - ) { - let activation_sha256 = topology.publish_observed_activation( - &incumbent.value.expected, - &incumbent.value.activation, - &observation, - )?; - ensure!( - activation_sha256 == incumbent.value.activation.canonical_sha256()?, - "restored incumbent activation receipt differs" - ); - } } else { + let plan = required(¤t.value.plan, "failed transaction plan")?; + let provider_anchor = self.provider_anchor_for_plan(plan)?; self.topology() - .withdraw_expected(expected) + .withdraw_expected( + expected, + &provider_anchor, + current.value.activation.as_ref(), + None, + ) .context("withdrawing exact failed Expected projection")?; } return self.persist_same_phase(current, |next| { @@ -3600,6 +4830,21 @@ impl Engine { workload, } = &cleanup.incumbent { + let binding = required(¤t.value.plan, "committed transaction plan")? + .parsed_inputs()? + .1; + if let Some(promoted_at_unix_millis) = required( + ¤t.value.routing, + "committed transaction routing evidence", + )? + .promoted_at_unix_millis() + { + let retire_not_before = + route_drain_deadline(promoted_at_unix_millis, binding.rollout.drain_seconds)?; + if now_millis()? < retire_not_before { + return Ok(()); + } + } self.workload .stop(workload) .with_context(|| format!("retiring admitted incumbent {generation_id}"))?; @@ -3627,6 +4872,50 @@ impl Engine { } } +fn route_drain_deadline(promoted_at_unix_millis: u64, drain_seconds: u32) -> Result { + promoted_at_unix_millis + .checked_add( + u64::from(drain_seconds) + .checked_mul(1_000) + .context("route drain duration overflows milliseconds")?, + ) + .context("route drain deadline overflows Unix milliseconds") +} + +fn route_observation_is_current( + observed_at_unix_millis: u64, + now_unix_millis: u64, + maximum_age_millis: u64, + maximum_future_skew_millis: u64, +) -> bool { + observed_at_unix_millis <= now_unix_millis.saturating_add(maximum_future_skew_millis) + && now_unix_millis.saturating_sub(observed_at_unix_millis) <= maximum_age_millis +} + +fn provider_warming_advanced( + prior_odin_sequence: u64, + prior_signed_presence_sha256: &str, + candidate_odin_sequence: u64, + candidate_signed_presence_sha256: &str, +) -> bool { + candidate_odin_sequence > prior_odin_sequence + && candidate_signed_presence_sha256 != prior_signed_presence_sha256 +} + +fn may_rollback_route_after_failed_proof(fencing: &FencingEvidence) -> bool { + matches!(fencing, FencingEvidence::SkippedStateless) +} + +fn incumbent_was_stopped_during_fencing(fencing: &FencingEvidence) -> bool { + matches!( + fencing, + FencingEvidence::Revoked { + incumbent_lease_sha256: Some(_), + .. + } + ) +} + fn candidate_cleanup_requirement( has_prepared_activation: bool, has_workload_observation: bool, @@ -3734,13 +5023,54 @@ fn validate_authenticated_evidence( Ok(()) } -fn is_semantic_warming(authenticated: &AuthenticatedOdinRuntimeTopologyCorrelation) -> bool { +fn is_semantic_warming( + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + incumbent_lease_sha256: Option<&str>, + authenticated: &AuthenticatedOdinRuntimeTopologyCorrelation, +) -> Result { let record = authenticated.record(); - record.present - && !record.ready - && record.observed_presence_state.as_deref() == Some("warming") - && record.observed_write_lease_sha256.is_none() - && record.disagreements.is_empty() + if !record.present || record.observed_write_lease_sha256.is_some() { + return Ok(false); + } + let expected_projection_detail = format!( + "expected:{};activation:{}", + expected.canonical_sha256()?, + activation.canonical_sha256()? + ); + if !warming_disagreements_match_incumbent( + &expected_projection_detail, + incumbent_lease_sha256, + &record.disagreements, + ) { + return Ok(false); + } + Ok(if expected.write_lease_required { + !record.ready && record.observed_presence_state.as_deref() == Some("warming") + } else { + matches!( + record.observed_presence_state.as_deref(), + Some("warming" | "active") + ) + }) +} + +fn warming_disagreements_match_incumbent( + expected_projection_detail: &str, + incumbent_lease_sha256: Option<&str>, + disagreements: &[OdinTopologyDisagreement], +) -> bool { + if disagreements.is_empty() { + true + } else if let (Some(incumbent_lease_sha256), [disagreement]) = + (incumbent_lease_sha256, disagreements) + { + disagreement.code == "projected-write-lease" + && disagreement.expected.as_deref() == Some(expected_projection_detail) + && disagreement.observed.as_deref() == Some(incumbent_lease_sha256) + } else { + false + } } fn is_semantic_ready(authenticated: &AuthenticatedOdinRuntimeTopologyCorrelation) -> bool { @@ -3928,6 +5258,28 @@ fn usage() -> &'static str { mod tests { use super::*; + fn digest(byte: char) -> String { + byte.to_string().repeat(64) + } + + fn route_binding( + route_id: &str, + stable_port: u16, + private_port_start: u16, + private_port_end: u16, + ) -> RouteBinding { + RouteBinding { + driver: crate::deployment::RouteDriver::NginxStreamTcp, + route_id: route_id.into(), + stable_endpoint: format!("tcp://127.0.0.1:{stable_port}"), + private_host: "127.0.0.1".into(), + private_port_start, + private_port_end, + config_path: PathBuf::from(format!("/etc/nginx/idunn-stream-routes/{route_id}.conf")), + reload_unit: "nginx.service".into(), + } + } + fn command(kind: CommandKind) -> DeploymentCommand { DeploymentCommand { schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), @@ -4055,6 +5407,37 @@ mod tests { } } + #[test] + fn route_bindings_are_one_global_socket_and_candidate_authority_map() { + let first = route_binding("first", 4103, 14103, 14111); + let second = route_binding("second", 8831, 18831, 18839); + validate_route_binding_set(&[("first", &first), ("second", &second)]).unwrap(); + + let mut duplicate_id = second.clone(); + duplicate_id.route_id = first.route_id.clone(); + assert!( + validate_route_binding_set(&[("first", &first), ("second", &duplicate_id)]).is_err() + ); + + let mut overlapping_candidates = second.clone(); + overlapping_candidates.private_port_start = 14111; + overlapping_candidates.private_port_end = 14120; + assert!( + validate_route_binding_set(&[("first", &first), ("second", &overlapping_candidates),]) + .is_err() + ); + + let mut stable_inside_other_range = second; + stable_inside_other_range.stable_endpoint = "tcp://127.0.0.1:14105".into(); + assert!( + validate_route_binding_set(&[ + ("first", &first), + ("second", &stable_inside_other_range), + ]) + .is_err() + ); + } + #[test] fn deployment_command_is_immutable_positional_fact() -> Result<()> { let value = command(CommandKind::Deploy); @@ -4079,9 +5462,11 @@ mod tests { DeploymentPhase::Committing, DeploymentPhase::Complete, ]; - assert!(phases - .windows(2) - .all(|pair| pair[1] as u8 == pair[0] as u8 + 1)); + assert!( + phases + .windows(2) + .all(|pair| pair[1] as u8 == pair[0] as u8 + 1) + ); } #[test] @@ -4103,6 +5488,44 @@ mod tests { Ok(()) } + #[test] + fn odin_sequence_cursors_are_scoped_by_target_and_signer() -> Result<()> { + let command = command(CommandKind::Deploy); + let mut ghostlight = + DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100)?; + ghostlight.latest_odin_observation = Some(topology(7, 1)); + ghostlight.odin_publisher_sequence_cursor = 7; + let mut odin = DeploymentTransaction::new(&command, "odin".into(), 1, None, 100)?; + odin.latest_odin_observation = Some(topology(41, 2)); + odin.odin_publisher_sequence_cursor = 41; + let envelope = |key: &str| CultCacheEnvelope { + key: key.into(), + r#type: DeploymentTransaction::TYPE.into(), + payload: Vec::new(), + stored_at: "1970-01-01T00:00:00.100Z".into(), + schema_id: Some(DEPLOYMENT_TRANSACTION_SCHEMA.into()), + }; + let snapshot = ControlSnapshot { + commands: Vec::new(), + transactions: vec![ + Stored { + envelope: envelope("ghostlight"), + value: ghostlight, + }, + Stored { + envelope: envelope("odin"), + value: odin, + }, + ], + admitted: Vec::new(), + }; + + assert_eq!(snapshot.max_odin_sequence("ghostlight", "odin-signer"), 7); + assert_eq!(snapshot.max_odin_sequence("odin", "odin-signer"), 41); + assert_eq!(snapshot.max_odin_sequence("ghostlight", "other-signer"), 0); + Ok(()) + } + #[test] fn stale_odin_provider_receipt_cannot_gate_continuity_restart() -> Result<()> { let touched = std::cell::Cell::new(false); @@ -4111,10 +5534,12 @@ mod tests { bail!("Odin provider receipt is stale") })?; assert!(!touched.get()); - assert!(validate_live_providers_for_deploy(CommandKind::Deploy, || { - bail!("Odin provider receipt is stale") - }) - .is_err()); + assert!( + validate_live_providers_for_deploy(CommandKind::Deploy, || { + bail!("Odin provider receipt is stale") + }) + .is_err() + ); Ok(()) } @@ -4228,6 +5653,103 @@ mod tests { Ok(()) } + #[test] + fn routed_incumbent_retirement_waits_for_the_declared_drain_deadline() -> Result<()> { + assert_eq!(route_drain_deadline(1_000, 30)?, 31_000); + assert!(route_drain_deadline(u64::MAX, 1).is_err()); + Ok(()) + } + + #[test] + fn admitted_route_receipt_refreshes_when_stale_or_implausibly_future_dated() { + assert!(route_observation_is_current(900, 1_000, 100, 10)); + assert!(route_observation_is_current(1_010, 1_000, 100, 10)); + assert!(!route_observation_is_current(899, 1_000, 100, 10)); + assert!(!route_observation_is_current(1_011, 1_000, 100, 10)); + } + + #[test] + fn lease_warming_requires_both_new_odin_and_new_provider_evidence() { + let old = digest('1'); + let fresh = digest('2'); + assert!(!provider_warming_advanced(7, &old, 7, &fresh)); + assert!(!provider_warming_advanced(7, &old, 8, &old)); + assert!(provider_warming_advanced(7, &old, 8, &fresh)); + } + + #[test] + fn failed_route_proof_never_restores_any_fenced_incumbent() { + assert!(!may_rollback_route_after_failed_proof( + &FencingEvidence::Revoked { + incumbent_lease_sha256: Some(digest('1')), + candidate_lease_path_verified_empty: false, + } + )); + assert!(!may_rollback_route_after_failed_proof( + &FencingEvidence::Revoked { + incumbent_lease_sha256: None, + candidate_lease_path_verified_empty: true, + } + )); + assert!(may_rollback_route_after_failed_proof( + &FencingEvidence::SkippedStateless + )); + } + + #[test] + fn fenced_writer_is_already_retired_before_post_commit_cleanup() { + assert!(incumbent_was_stopped_during_fencing( + &FencingEvidence::Revoked { + incumbent_lease_sha256: Some(digest('1')), + candidate_lease_path_verified_empty: false, + } + )); + assert!(!incumbent_was_stopped_during_fencing( + &FencingEvidence::Revoked { + incumbent_lease_sha256: None, + candidate_lease_path_verified_empty: true, + } + )); + assert!(!incumbent_was_stopped_during_fencing( + &FencingEvidence::SkippedStateless + )); + } + + #[test] + fn warming_accepts_only_the_exact_projected_incumbent_lease_disagreement() { + let expected_detail = format!("expected:{};activation:{}", digest('1'), digest('2')); + let incumbent = digest('3'); + let exact = OdinTopologyDisagreement { + code: "projected-write-lease".into(), + expected: Some(expected_detail.clone()), + observed: Some(incumbent.clone()), + }; + + assert!(warming_disagreements_match_incumbent( + &expected_detail, + Some(&incumbent), + &[exact.clone()], + )); + assert!(!warming_disagreements_match_incumbent( + &expected_detail, + None, + &[exact.clone()], + )); + + let mut substituted = exact.clone(); + substituted.observed = Some(digest('4')); + assert!(!warming_disagreements_match_incumbent( + &expected_detail, + Some(&incumbent), + &[substituted], + )); + assert!(!warming_disagreements_match_incumbent( + &expected_detail, + Some(&incumbent), + &[exact.clone(), exact], + )); + } + #[test] fn candidate_and_incumbent_must_differ_in_all_three_native_boundaries() { let incumbent = workload(1001, 40, 50); @@ -4305,6 +5827,7 @@ mod tests { assert!(!transaction.is_terminal()); assert!(!transaction.owns_target_authority()); + assert!(transaction.blocks_new_target_mutation()); Ok(()) } diff --git a/crates/idunn-daemon/src/deployment.rs b/crates/idunn-daemon/src/deployment.rs index ded7e2b..bffd316 100644 --- a/crates/idunn-daemon/src/deployment.rs +++ b/crates/idunn-daemon/src/deployment.rs @@ -1,7 +1,8 @@ use std::collections::{BTreeMap, BTreeSet}; +use std::net::IpAddr; use std::path::{Component, Path, PathBuf}; -use anyhow::{bail, ensure, Context, Result}; +use anyhow::{Context, Result, bail, ensure}; use serde::{Deserialize, Serialize}; pub const TARGET_DECLARATION_SCHEMA: &str = "gamecult.idunn.target_declaration.v1"; @@ -425,11 +426,45 @@ pub struct RouteBinding { pub reload_unit: String, } +impl RouteBinding { + pub(crate) fn stable_socket(&self) -> Result<(IpAddr, u16)> { + let authority = match self.driver { + RouteDriver::NginxStreamTcp => self + .stable_endpoint + .strip_prefix("http://") + .or_else(|| self.stable_endpoint.strip_prefix("tcp://")), + RouteDriver::NginxStreamUdp => self.stable_endpoint.strip_prefix("rudp://"), + } + .context("stable endpoint transport does not match its route driver")?; + ensure!( + !authority + .chars() + .any(|character| matches!(character, '/' | '?' | '#' | '@' | '[' | ']')), + "stable endpoint is not a plain IP address and port" + ); + let (host, port) = authority + .rsplit_once(':') + .context("stable endpoint has no port")?; + let host = host + .parse::() + .context("stable endpoint host is not an IP address")?; + ensure!( + matches!(host, IpAddr::V4(_)), + "route binding v2 stable endpoints use explicit IPv4 addresses" + ); + let port = port + .parse::() + .context("stable endpoint port is not a u16")?; + ensure!(port > 0, "stable endpoint port is zero"); + Ok((host, port)) + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub enum RouteDriver { - NginxHttp, NginxStreamTcp, + NginxStreamUdp, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -1027,6 +1062,14 @@ impl OperatorBinding { require_id(&route.route_id, "route id")?; require_value(&route.stable_endpoint, "stable endpoint")?; require_host(&route.private_host)?; + let private_host: IpAddr = route + .private_host + .parse() + .context("route private host is not an IP address")?; + ensure!( + private_host.is_loopback(), + "systemd-transient candidate routes must use a loopback private host" + ); ensure!( route.private_port_start > 0, "private port range starts at zero" @@ -1035,6 +1078,7 @@ impl OperatorBinding { route.private_port_start < route.private_port_end, "candidate rollout requires at least two private ports" ); + route.stable_socket()?; require_absolute_path(&route.config_path, "route config")?; require_unit(&route.reload_unit, "route reload unit")?; } @@ -1301,11 +1345,10 @@ impl OperatorBinding { self.route.as_ref(), ) { (true, ServiceTransport::Http, Some(route)) - if route.driver == RouteDriver::NginxHttp => + if route.driver == RouteDriver::NginxStreamTcp => { ensure!( - route.stable_endpoint.starts_with("http://") - || route.stable_endpoint.starts_with("https://"), + route.stable_endpoint.starts_with("http://"), "HTTP stable endpoint is not an HTTP URI" ); } @@ -1317,6 +1360,14 @@ impl OperatorBinding { "TCP stable endpoint is not a TCP URI" ); } + (true, ServiceTransport::Rudp, Some(route)) + if route.driver == RouteDriver::NginxStreamUdp => + { + ensure!( + route.stable_endpoint.starts_with("rudp://"), + "RUDP stable endpoint is not an RUDP URI" + ); + } (false, _, None) => {} _ => bail!("route binding does not match the launch contract"), } @@ -1681,13 +1732,13 @@ expected_signer_identity_id = "test-service-runtime-signer" trust_anchor_store = "/etc/gamecult/trust/test-service.cc" [route] -driver = "nginx-http" +driver = "nginx-stream-tcp" route_id = "test-service-public" -stable_endpoint = "https://yggdrasil.gamecult.org/test-service/" +stable_endpoint = "http://127.0.0.1:18830" private_host = "127.0.0.1" private_port_start = 18831 private_port_end = 18839 -config_path = "/etc/nginx/idunn-routes/test-service.conf" +config_path = "/etc/nginx/idunn-stream-routes/test-service.conf" reload_unit = "nginx.service" [process_write_lease] @@ -1765,16 +1816,33 @@ nodes = ["yggdrasil"] #[test] fn route_transport_mismatch_is_rejected() { let recipe = TargetDeclaration::parse(RECIPE).unwrap(); - let binding = - OperatorBinding::parse(&BINDING.replace("nginx-http", "nginx-stream-tcp")).unwrap(); - assert!(binding.admit(&recipe).is_err()); + assert!( + OperatorBinding::parse(&BINDING.replace("nginx-stream-tcp", "nginx-stream-udp")) + .is_err() + ); - let binding = OperatorBinding::parse(&BINDING.replace( - "https://yggdrasil.gamecult.org/test-service/", - "tcp://yggdrasil.gamecult.org:443", - )) + assert!( + OperatorBinding::parse( + &BINDING.replace("http://127.0.0.1:18830", "tcp://yggdrasil.gamecult.org:443",), + ) + .is_err() + ); + assert!(recipe.service.route_required); + } + + #[test] + fn rudp_route_uses_the_udp_stream_driver() { + let recipe = TargetDeclaration::parse( + &RECIPE.replace("transport = \"http\"", "transport = \"rudp\""), + ) .unwrap(); - assert!(binding.admit(&recipe).is_err()); + let binding = OperatorBinding::parse( + &BINDING + .replace("nginx-stream-tcp", "nginx-stream-udp") + .replace("http://127.0.0.1:18830", "rudp://127.0.0.1:17872"), + ) + .unwrap(); + assert!(binding.admit(&recipe).is_ok()); } #[test] @@ -1892,24 +1960,30 @@ nodes = ["yggdrasil"] #[test] fn v2_binding_is_explicitly_singleton() { - assert!(OperatorBinding::parse( - &BINDING.replace("desired_replicas = 1", "desired_replicas = 2") - ) - .is_err()); - assert!(OperatorBinding::parse(&BINDING.replace( - "nodes = [\"yggdrasil\"]", - "nodes = [\"yggdrasil\", \"raven\"]" - )) - .is_err()); + assert!( + OperatorBinding::parse( + &BINDING.replace("desired_replicas = 1", "desired_replicas = 2") + ) + .is_err() + ); + assert!( + OperatorBinding::parse(&BINDING.replace( + "nodes = [\"yggdrasil\"]", + "nodes = [\"yggdrasil\", \"raven\"]" + )) + .is_err() + ); } #[test] fn legacy_static_workload_identity_has_no_authority_in_v2() { - assert!(OperatorBinding::parse(&BINDING.replace( - "schema = \"gamecult.idunn.operator_binding.v2\"", - "schema = \"gamecult.idunn.operator_binding.v1\"", - ),) - .is_err()); + assert!( + OperatorBinding::parse(&BINDING.replace( + "schema = \"gamecult.idunn.operator_binding.v2\"", + "schema = \"gamecult.idunn.operator_binding.v1\"", + ),) + .is_err() + ); let with_static_identity = BINDING.replace( "driver = \"systemd-transient\"", "driver = \"systemd-transient\"\nuser = \"test-service\"\ngroup = \"test-service\"", @@ -1923,10 +1997,12 @@ nodes = ["yggdrasil"] OperatorBinding::parse(&BINDING.replace("state_group = \"test-service\"\n", ""),) .is_err() ); - assert!(OperatorBinding::parse( - &BINDING.replace("state_group = \"test-service\"", "state_group = \"root\""), - ) - .is_err()); + assert!( + OperatorBinding::parse( + &BINDING.replace("state_group = \"test-service\"", "state_group = \"root\""), + ) + .is_err() + ); } #[test] diff --git a/crates/idunn-daemon/src/deployment_plan.rs b/crates/idunn-daemon/src/deployment_plan.rs index ca4469a..9475701 100644 --- a/crates/idunn-daemon/src/deployment_plan.rs +++ b/crates/idunn-daemon/src/deployment_plan.rs @@ -1,20 +1,20 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::PathBuf; -use anyhow::{bail, ensure, Context, Result}; +use anyhow::{Context, Result, bail, ensure}; pub use cultnet_rs::IdunnExpectedIncarnationRecord as ExpectedIncarnation; use cultnet_rs::{ - IdunnExpectedCapability, IdunnExpectedDependency, IdunnExpectedRoute, - IDUNN_EXPECTED_INCARNATION_SCHEMA, + IDUNN_EXPECTED_INCARNATION_SCHEMA, IdunnExpectedCapability, IdunnExpectedDependency, + IdunnExpectedRoute, }; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use crate::control_plane::SequenceAdmittedReady; use crate::deployment::{ - capability_compatible, CapabilityDependency, DependencyKind, ExternalCapabilityBinding, - OperatorBinding, ServiceTransport, SourceSelectionPolicy, StartupOrder, StateDeclaration, - TargetDeclaration, + CapabilityDependency, DependencyKind, ExternalCapabilityBinding, OperatorBinding, + ServiceTransport, SourceSelectionPolicy, StartupOrder, StateDeclaration, TargetDeclaration, + capability_compatible, }; pub const SOURCE_SELECTION_FACTS_SCHEMA: &str = "idunn.source_selection_facts.v1"; @@ -286,11 +286,11 @@ fn managed_ready_provider_refs( let endpoint = expected .route .as_ref() - .map(|route| route.candidate_endpoint.clone()); + .map(|route| route.stable_endpoint.clone()); let mut providers = Vec::new(); for capability in &observed.observed_capabilities { providers.push(DependencyProviderRef { - provider_id: observed.runtime_id.clone(), + provider_id: expected.target.clone(), authority: DependencyProviderAuthority::ManagedReady { target: expected.target.clone(), incarnation_id: expected.incarnation_id.clone(), @@ -995,11 +995,12 @@ fn sha256_id(bytes: &[u8]) -> String { mod tests { use super::*; use cultnet_rs::{ + GameCultProviderHealthIdentity, GameCultRuntimeCapability, IdunnRuntimeActivationLaunch, + IdunnServiceIdentity, ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA, + OdinRuntimeTopologyCorrelationPurpose, OdinRuntimeTopologyCorrelationRecord, + OdinTopologyAuthenticationContext, OdinTopologyIdentity, authenticate_odin_runtime_topology_correlation, enroll_service_identity_at, - verify_runtime_authority, GameCultProviderHealthIdentity, GameCultRuntimeCapability, - IdunnRuntimeActivationLaunch, IdunnServiceIdentity, OdinRuntimeTopologyCorrelationPurpose, - OdinRuntimeTopologyCorrelationRecord, OdinTopologyAuthenticationContext, - OdinTopologyIdentity, ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA, + verify_runtime_authority, }; const RECIPE: &str = r#" @@ -1113,13 +1114,13 @@ expected_signer_identity_id = "service-runtime-signer" trust_anchor_store = "/etc/gamecult/trust/service.cc" [route] -driver = "nginx-http" +driver = "nginx-stream-tcp" route_id = "service" -stable_endpoint = "https://example.invalid/service/" +stable_endpoint = "http://127.0.0.1:17999" private_host = "127.0.0.1" private_port_start = 18000 private_port_end = 18009 -config_path = "/etc/nginx/idunn-routes/service.conf" +config_path = "/etc/nginx/idunn-stream-routes/service.conf" reload_unit = "nginx.service" [brakes] @@ -1189,7 +1190,7 @@ nodes = ["yggdrasil"] } } - fn ready_odin_provider(runtime_id: &str, capacity: u32) -> SequenceAdmittedReady { + fn ready_odin_provider(target: &str, runtime_id: &str, capacity: u32) -> SequenceAdmittedReady { (|| -> Result { let root = tempfile::tempdir()?; let provider = enroll_service_identity_at::( @@ -1200,6 +1201,7 @@ nodes = ["yggdrasil"] let odin_signer = enroll_service_identity_at::(&root.path().join("odin.cc"))?; let mut expected = odin(); + expected.target = target.into(); expected.runtime_id = runtime_id.into(); expected.expected_signer_identity_id = provider.entry().identity_id.clone(); let launch = IdunnRuntimeActivationLaunch::issue(&expected, digest(b'8'), 100, &idunn)?; @@ -1262,7 +1264,7 @@ nodes = ["yggdrasil"] } fn plan() -> CompiledDeploymentPlan { - let providers = [ready_odin_provider("odin-yggdrasil", 1)]; + let providers = [ready_odin_provider("odin", "odin-yggdrasil", 1)]; plan_with(&providers) } @@ -1302,7 +1304,7 @@ nodes = ["yggdrasil"] #[test] fn compiler_parses_the_exact_bytes_it_receipts() { - let providers = [ready_odin_provider("odin-yggdrasil", 1)]; + let providers = [ready_odin_provider("odin", "odin-yggdrasil", 1)]; let first = plan_with(&providers); let second = plan_with(&providers); assert_eq!(first, second); @@ -1316,17 +1318,19 @@ nodes = ["yggdrasil"] "contract = \"service.health\"", "contract = \"service.health.v2\"", ); - let providers = [ready_odin_provider("odin-yggdrasil", 1)]; - assert!(compile_deployment_plan( - changed.as_bytes(), - BINDING.as_bytes(), - source(RECIPE), - "service-incarnation-2", - Some(18002), - 111, - &providers, - ) - .is_err()); + let providers = [ready_odin_provider("odin", "odin-yggdrasil", 1)]; + assert!( + compile_deployment_plan( + changed.as_bytes(), + BINDING.as_bytes(), + source(RECIPE), + "service-incarnation-2", + Some(18002), + 111, + &providers, + ) + .is_err() + ); let mut corrupted = first; corrupted .binding_blob @@ -1382,8 +1386,8 @@ nodes = ["yggdrasil"] fn dependency_selection_uses_observed_ready_capacity_and_stable_identity_order() { let declaration = TargetDeclaration::parse(RECIPE).unwrap(); let admitted = [ - ready_odin_provider("odin-z", 4), - ready_odin_provider("odin-yggdrasil", 2), + ready_odin_provider("odin-z", "odin-runtime-z", 4), + ready_odin_provider("odin-yggdrasil", "odin-runtime-yggdrasil", 2), ]; let providers = admitted .iter() @@ -1395,6 +1399,10 @@ nodes = ["yggdrasil"] "odin-yggdrasil" ); assert_eq!(selected[0].provider.as_ref().unwrap().capacity, 2); + assert_eq!( + selected[0].provider.as_ref().unwrap().endpoint.as_deref(), + Some("tcp://10.77.0.1:17871") + ); assert!(matches!( &selected[0].provider.as_ref().unwrap().authority, DependencyProviderAuthority::ManagedReady { @@ -1459,7 +1467,7 @@ nodes = ["yggdrasil"] assert!(!expected.write_lease_required); assert_eq!( expected.dependencies[0].provider_id.as_deref(), - Some("odin-yggdrasil") + Some("odin") ); assert_eq!( expected.dependencies[0].provider_authority.as_deref(), @@ -1494,7 +1502,7 @@ nodes = ["yggdrasil"] "service-incarnation-sorted", Some(18002), 111, - &[ready_odin_provider("odin-yggdrasil", 1)], + &[ready_odin_provider("odin", "odin-yggdrasil", 1)], ) .unwrap(); let release = SealedRelease::new( diff --git a/crates/idunn-daemon/src/drivers.rs b/crates/idunn-daemon/src/drivers.rs index c4bfe5e..1554408 100644 --- a/crates/idunn-daemon/src/drivers.rs +++ b/crates/idunn-daemon/src/drivers.rs @@ -2,6 +2,7 @@ use std::collections::BTreeMap; use std::ffi::{OsStr, OsString}; use std::fs::{self, OpenOptions}; use std::io::{ErrorKind, Read, Write}; +use std::net::{SocketAddr, TcpStream}; use std::path::{Path, PathBuf}; use std::process::{Command, Output, Stdio}; use std::thread; @@ -10,8 +11,9 @@ use std::time::Duration; use anyhow::{Context, Result, bail, ensure}; use cultcache_rs::{ CacheBackingStore, CultCacheEnvelope, CultCacheExpectedEnvelope, DatabaseEntry, - SingleFileMessagePackBackingStore, + SingleFileMessagePackBackingStore, TryCompareExchangeSnapshotOutcome, }; +use cultmesh_rs::{CultMeshRudpSnapshotOptions, request_raw_snapshot_from_rudp_catalog}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use uuid::Uuid; @@ -29,11 +31,16 @@ use crate::deployment_plan::{ SOURCE_SELECTION_FACTS_SCHEMA, SealedRelease, SourceSelection, SourceSelectionFacts, }; use cultnet_rs::{ - GameCultProviderHealthIdentity, IDUNN_EXPECTED_INCARNATION_SCHEMA, + CultNetMessage, CultNetRawPayloadEncoding, CultNetWireContract, + GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, GAMECULT_RUNTIME_PRESENCE_HEALTH_SIGNING_PURPOSE, + GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA, GameCultProviderHealthIdentity, + GameCultServiceTrustAnchorRecord, IDUNN_EXPECTED_INCARNATION_SCHEMA, IDUNN_PROCESS_WRITE_LEASE_SCHEMA, IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME, IDUNN_RUNTIME_ACTIVATION_SCHEMA, IdunnExpectedIncarnationRecord, IdunnProcessWriteLeaseRecord, IdunnRuntimeActivationLaunch, IdunnRuntimeActivationRecord, IdunnRuntimeActivationSigner, ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA, OdinRuntimeTopologyCorrelationRecord, + ServiceIdentityProfile, ServiceIdentityTrustAnchor, decode_cultnet_message_from_slice, + derive_service_identity_id, encode_cultnet_message_to_vec, encode_frame, open_service_identity_credential_reader, }; @@ -293,14 +300,36 @@ pub trait WorkloadPort { } pub trait TopologyPort { - fn publish_expected(&self, expected: &IdunnExpectedIncarnationRecord) -> Result; - fn withdraw_expected(&self, expected: &IdunnExpectedIncarnationRecord) -> Result<()>; + fn publish_expected( + &self, + expected: &IdunnExpectedIncarnationRecord, + provider_anchor: &ServiceIdentityTrustAnchor, + ) -> Result; + fn withdraw_expected( + &self, + expected: &IdunnExpectedIncarnationRecord, + provider_anchor: &ServiceIdentityTrustAnchor, + activation: Option<&IdunnRuntimeActivationRecord>, + lease: Option<&IdunnProcessWriteLeaseRecord>, + ) -> Result<()>; fn publish_observed_activation( &self, expected: &IdunnExpectedIncarnationRecord, activation: &IdunnRuntimeActivationRecord, observation: &WorkloadObservation, ) -> Result; + fn publish_process_write_lease( + &self, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + lease: &IdunnProcessWriteLeaseRecord, + ) -> Result; + fn withdraw_process_write_lease( + &self, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + lease: Option<&IdunnProcessWriteLeaseRecord>, + ) -> Result<()>; fn receive(&self, target: &str) -> Result>; } @@ -313,6 +342,7 @@ pub struct ReceivedOdinTopologyCorrelation { pub trait WriteLeasePort { fn revoke_exact(&self, incumbent: Option<&IdunnProcessWriteLeaseRecord>) -> Result<()>; fn observe_empty(&self) -> Result; + fn observe_exact(&self, lease: &IdunnProcessWriteLeaseRecord) -> Result; fn grant( &self, expected: &IdunnExpectedIncarnationRecord, @@ -334,6 +364,22 @@ pub struct RouteObservation { pub route_id: String, pub runtime_instance_id: String, pub membership_sha256: String, + pub signed_presence_sha256: String, + pub observed_at_unix_millis: u64, +} + +impl RouteObservation { + pub fn validate(&self) -> Result<()> { + require_driver_id(&self.route_id, "route observation")?; + require_sha256_id(&self.runtime_instance_id, "route runtime instance")?; + require_sha256_id(&self.membership_sha256, "route membership")?; + require_sha256_id(&self.signed_presence_sha256, "route signed presence")?; + ensure!( + self.observed_at_unix_millis > 0, + "route observation has no trusted receipt time" + ); + Ok(()) + } } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -343,26 +389,53 @@ pub struct RoutePreflightReceipt { pub candidate_membership_sha256: String, pub incumbent_runtime_instance_id: Option, pub incumbent_membership_sha256: Option, + pub incumbent_configuration: Option>, } -pub trait RoutePort { - fn preflight( - &self, - expected: &IdunnExpectedIncarnationRecord, - runtime_instance_id: &str, - incumbent: Option<&RouteObservation>, - ) -> Result; - fn promote( - &self, - expected: &IdunnExpectedIncarnationRecord, - runtime_instance_id: &str, - preflight: &RoutePreflightReceipt, - ) -> Result; - fn observe( - &self, - expected: &IdunnExpectedIncarnationRecord, - observation: &RouteObservation, - ) -> Result; +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RouteSnapshotResponse { + pub message_id: String, + pub canonical_presence: Vec, +} + +const ROUTE_SNAPSHOT_TIMEOUT: Duration = Duration::from_secs(3); +const ROUTE_SNAPSHOT_MAX_BYTES: usize = 1024 * 1024; +const ROUTE_HTTP_MAX_HEADER_BYTES: usize = 32 * 1024; +const ROUTE_HTTP_SNAPSHOT_PATH: &str = "/cultnet/snapshot"; + +impl RoutePreflightReceipt { + pub fn validate(&self) -> Result<()> { + require_driver_id(&self.route_id, "route preflight")?; + require_sha256_id( + &self.candidate_runtime_instance_id, + "candidate route runtime instance", + )?; + require_sha256_id( + &self.candidate_membership_sha256, + "candidate route membership", + )?; + match ( + &self.incumbent_runtime_instance_id, + &self.incumbent_membership_sha256, + &self.incumbent_configuration, + ) { + (None, None, None) => {} + (Some(runtime), Some(membership), Some(configuration)) => { + require_sha256_id(runtime, "incumbent route runtime instance")?; + require_sha256_id(membership, "incumbent route membership")?; + ensure!( + !configuration.is_empty(), + "incumbent route configuration is empty" + ); + ensure!( + sha256_id(configuration) == *membership, + "incumbent route configuration differs from its membership digest" + ); + } + _ => bail!("incumbent route preflight evidence is partial"), + } + Ok(()) + } } /// Fixed-argv Git source driver. It never interprets recipe text as a command @@ -1295,7 +1368,7 @@ impl RunnerPort for DockerRunnerDriver { &self, source: &FrozenSource, plan: &CompiledDeploymentPlan, - staging_root: &Path, + staging_parent: &Path, sealed_at_unix_millis: u64, ) -> Result { plan.validate()?; @@ -1305,11 +1378,12 @@ impl RunnerPort for DockerRunnerDriver { "runner source differs from the compiled plan" ); let (declaration, binding) = plan.parsed_inputs()?; - ensure!( - !staging_root.exists(), - "release staging root already exists" - ); - fs::create_dir_all(staging_root)?; + fs::create_dir_all(staging_parent)?; + let staging_root = staging_parent.join(&source.receipt.transaction_id); + if staging_root.exists() { + remove_tree_inside(staging_parent, &staging_root)?; + } + fs::create_dir_all(&staging_root)?; let mut workspaces = BTreeMap::new(); for runner_id in binding.runners.keys() { @@ -1375,12 +1449,12 @@ impl RunnerPort for DockerRunnerDriver { &declaration, source, &workspaces, - staging_root, + &staging_root, &artifact.id, )?); } for workspace in workspaces.values() { - remove_tree_inside(staging_root, workspace)?; + remove_tree_inside(&staging_root, workspace)?; } let release = SealedRelease::new( plan, @@ -1390,7 +1464,7 @@ impl RunnerPort for DockerRunnerDriver { )?; Ok(MaterializedRelease { release, - root: staging_root.to_owned(), + root: staging_root, }) } } @@ -2848,9 +2922,10 @@ impl WorkloadPort for SystemdTransientWorkloadDriver { } /// One dedicated CultCache file is the write-lease authority for one target. -/// Its sibling lock is shared by the target only around durable commit edges; -/// Idunn's compare-exchange therefore fences the old writer before replacing -/// the record. Route membership never enters this store. +/// The admitted target holds its sibling lock shared for that process lifetime. +/// Grant and revocation therefore make one nonblocking CAS attempt; revocation +/// follows an exact incumbent stop. A foreign or stuck holder cannot freeze +/// Idunn's control loop. Route membership never enters this store. pub struct CultCacheWriteLeaseDriver { pub target: String, pub record_path: PathBuf, @@ -2914,11 +2989,6 @@ impl CultCacheWriteLeaseDriver { lease.validate()?; let expected_sha256 = expected.canonical_sha256()?; let activation_sha256 = activation.canonical_sha256()?; - let warming = warming.authenticated().record(); - let warming_presence_sha256 = warming - .signed_presence_sha256 - .as_deref() - .context("warming observation has no signed presence")?; ensure!( self.target == expected.target && lease.target == expected.target @@ -2939,14 +3009,8 @@ impl CultCacheWriteLeaseDriver { .state_contract_sha256 .as_deref() .context("write-lease Expected has no state contract")? - && lease.warming_presence_sha256 == warming_presence_sha256 - && warming.present - && !warming.ready - && warming.observed_presence_state.as_deref() == Some("warming") - && warming.runtime_instance_id.as_deref() - == Some(activation.runtime_instance_id.as_str()) - && warming.observed_write_lease_sha256.is_none() - && warming.disagreements.is_empty(), + && lease.warming_presence_sha256 == warming.signed_presence_sha256() + && warming.runtime_instance_id() == activation.runtime_instance_id.as_str(), "process write lease does not bind the exact warming candidate" ); Ok(()) @@ -2971,17 +3035,32 @@ impl WriteLeasePort for CultCacheWriteLeaseDriver { "refusing to revoke an unexpected process write lease" ); let store = SingleFileMessagePackBackingStore::new(&self.record_path); - ensure!( - store.compare_exchange_snapshot(std::slice::from_ref(&envelope), &[])?, - "process write lease changed while fencing the incumbent" - ); - Ok(()) + match store.try_compare_exchange_snapshot(std::slice::from_ref(&envelope), &[])? { + TryCompareExchangeSnapshotOutcome::Exchanged => Ok(()), + TryCompareExchangeSnapshotOutcome::Mismatch => { + bail!("process write lease changed while fencing the incumbent") + } + TryCompareExchangeSnapshotOutcome::LockContended => { + bail!("process write lease is held by a shared-lock consumer") + } + } } fn observe_empty(&self) -> Result { Ok(self.current()?.is_none()) } + fn observe_exact(&self, lease: &IdunnProcessWriteLeaseRecord) -> Result { + lease.validate()?; + ensure!( + lease.target == self.target, + "observed process write lease belongs to another target" + ); + Ok(self + .current()? + .is_some_and(|(_, current)| current == *lease)) + } + fn grant( &self, expected: &IdunnExpectedIncarnationRecord, @@ -2995,20 +3074,21 @@ impl WriteLeasePort for CultCacheWriteLeaseDriver { current == *lease, "another process already owns the write lease" ); + harden_root_authority_file(&self.record_path)?; + harden_root_authority_file(&authority_lock_path(&self.record_path))?; return lease.canonical_sha256(); } let envelope = self.envelope(lease)?; - ensure!( - SingleFileMessagePackBackingStore::new(&self.record_path).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: IdunnProcessWriteLeaseRecord::TYPE.into(), - key: lease.target.clone(), - current: None, - }], - &[envelope], - )?, - "process write-lease grant lost its empty-store compare-exchange" - ); + let store = SingleFileMessagePackBackingStore::new(&self.record_path); + match store.try_compare_exchange_snapshot(&[], &[envelope])? { + TryCompareExchangeSnapshotOutcome::Exchanged => {} + TryCompareExchangeSnapshotOutcome::Mismatch => { + bail!("process write-lease grant lost its empty-store compare-exchange") + } + TryCompareExchangeSnapshotOutcome::LockContended => { + bail!("process write lease is held by a shared-lock consumer") + } + } harden_root_authority_file(&self.record_path)?; harden_root_authority_file(&authority_lock_path(&self.record_path))?; lease.canonical_sha256() @@ -3028,43 +3108,581 @@ impl WriteLeasePort for CultCacheWriteLeaseDriver { } } -/// The local CultCache projection is Idunn's typed provider surface into -/// CultMesh. Odin writes a separate signed correlation store. Reading that -/// store never lets Idunn manufacture Present or Ready from its own config. +/// Idunn projects desired identity plus its own activation and current-lease +/// facts here. Service presence is absent by construction; only Odin may +/// correlate these records with signed runtime observation into Present/Ready. pub struct CultCacheTopologyDriver { pub projection_store: PathBuf, pub correlation_store: PathBuf, } -impl TopologyPort for CultCacheTopologyDriver { - fn publish_expected(&self, expected: &IdunnExpectedIncarnationRecord) -> Result { +impl CultCacheTopologyDriver { + /// Atomically demotes a failed pre-fence candidate back to the admitted + /// generation's Expected projection. Runtime activation and write-lease + /// records are deliberately absent until admitted-body supervision proves + /// the physical process and lease again. + pub fn restore_admitted_expected_only( + &self, + failed_expected: &IdunnExpectedIncarnationRecord, + failed_provider_anchor: &ServiceIdentityTrustAnchor, + failed_activation: Option<&IdunnRuntimeActivationRecord>, + admitted_expected: &IdunnExpectedIncarnationRecord, + admitted_provider_anchor: &ServiceIdentityTrustAnchor, + admitted_activation: &IdunnRuntimeActivationRecord, + admitted_lease: Option<&IdunnProcessWriteLeaseRecord>, + ) -> Result { + failed_expected.validate()?; + admitted_expected.validate()?; + ensure!( + failed_expected.target == admitted_expected.target, + "projection restoration crosses deployment targets" + ); + let failed_anchor = runtime_presence_trust_anchor(failed_expected, failed_provider_anchor)?; + let admitted_anchor = + runtime_presence_trust_anchor(admitted_expected, admitted_provider_anchor)?; + if let Some(activation) = failed_activation { + activation.validate()?; + ensure!( + activation.expected_projection_sha256 == failed_expected.canonical_sha256()?, + "failed activation does not bind the failed Expected projection" + ); + } + admitted_activation.validate()?; + ensure!( + admitted_activation.expected_projection_sha256 + == admitted_expected.canonical_sha256()?, + "admitted activation does not bind the admitted Expected projection" + ); + if let Some(lease) = admitted_lease { + validate_topology_lease(admitted_expected, admitted_activation, lease)?; + } + if let Some(parent) = self.projection_store.parent() { + fs::create_dir_all(parent)?; + } + let store = SingleFileMessagePackBackingStore::new(&self.projection_store); + let target = admitted_expected.target.as_str(); + let anchor_key = runtime_presence_trust_anchor_id(target); + for _ in 0..8 { + let entries = store.pull_all_read_only_snapshot()?; + let current_expected = + projection_entry(&entries, IdunnExpectedIncarnationRecord::TYPE, target)?; + if let Some(envelope) = current_expected { + ensure!( + envelope.schema_id.as_deref() == Some(IDUNN_EXPECTED_INCARNATION_SCHEMA), + "restored Expected projection schema is foreign" + ); + let current = IdunnExpectedIncarnationRecord::decode_canonical(&envelope.payload)?; + ensure!( + current == *failed_expected || current == *admitted_expected, + "refusing to replace an unknown Expected projection" + ); + } + let current_anchor = projection_entry( + &entries, + GameCultServiceTrustAnchorRecord::TYPE, + &anchor_key, + )?; + if let Some(envelope) = current_anchor { + let current = service_trust_anchor_from_envelope(envelope)?; + ensure!( + current == failed_anchor || current == admitted_anchor, + "refusing to replace an unknown runtime presence trust anchor" + ); + } + let current_activation = + projection_entry(&entries, IdunnRuntimeActivationRecord::TYPE, target)?; + if let Some(envelope) = current_activation { + ensure!( + envelope.schema_id.as_deref() == Some(IDUNN_RUNTIME_ACTIVATION_SCHEMA), + "restored activation projection schema is foreign" + ); + let current = IdunnRuntimeActivationRecord::decode_canonical(&envelope.payload)?; + ensure!( + failed_activation == Some(¤t) || admitted_activation == ¤t, + "refusing to remove an unknown runtime activation projection" + ); + } + let current_lease = + projection_entry(&entries, IdunnProcessWriteLeaseRecord::TYPE, target)?; + if let Some(envelope) = current_lease { + ensure!( + envelope.schema_id.as_deref() == Some(IDUNN_PROCESS_WRITE_LEASE_SCHEMA), + "restored write-lease projection schema is foreign" + ); + let current = IdunnProcessWriteLeaseRecord::decode_canonical(&envelope.payload)?; + ensure!( + admitted_lease == Some(¤t), + "refusing to remove an unknown process write-lease projection" + ); + } + if current_expected.is_some_and(|envelope| { + IdunnExpectedIncarnationRecord::decode_canonical(&envelope.payload) + .is_ok_and(|current| current == *admitted_expected) + }) && current_anchor.is_some_and(|envelope| { + service_trust_anchor_from_envelope(envelope) + .is_ok_and(|current| current == admitted_anchor) + }) && current_activation.is_none() + && current_lease.is_none() + { + return admitted_expected.canonical_sha256(); + } + let mut replacement = entries + .iter() + .filter(|envelope| { + !((envelope.key == target + && matches!( + envelope.r#type.as_str(), + IdunnExpectedIncarnationRecord::TYPE + | IdunnRuntimeActivationRecord::TYPE + | IdunnProcessWriteLeaseRecord::TYPE + )) + || (envelope.key == anchor_key + && envelope.r#type == GameCultServiceTrustAnchorRecord::TYPE)) + }) + .cloned() + .collect::>(); + replacement.extend([ + CultCacheEnvelope { + key: admitted_expected.target.clone(), + r#type: IdunnExpectedIncarnationRecord::TYPE.into(), + payload: admitted_expected.canonical_bytes()?, + stored_at: chrono::Utc::now().to_rfc3339(), + schema_id: Some(IDUNN_EXPECTED_INCARNATION_SCHEMA.into()), + }, + CultCacheEnvelope { + key: admitted_anchor.trust_anchor_id.clone(), + r#type: GameCultServiceTrustAnchorRecord::TYPE.into(), + payload: rmp_serde::to_vec(&admitted_anchor)?, + stored_at: rfc3339_millis(admitted_anchor.bound_at_unix_millis)?, + schema_id: Some(GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA.into()), + }, + ]); + if store.compare_exchange_snapshot(&entries, &replacement)? { + return admitted_expected.canonical_sha256(); + } + } + bail!("topology projection changed repeatedly during admitted restoration") + } + + pub fn admitted_runtime_projection_is_exact( + &self, + expected: &IdunnExpectedIncarnationRecord, + provider_anchor: &ServiceIdentityTrustAnchor, + activation: &IdunnRuntimeActivationRecord, + lease: Option<&IdunnProcessWriteLeaseRecord>, + ) -> Result { expected.validate()?; - upsert_record( - &self.projection_store, - CultCacheEnvelope { - key: expected.target.clone(), + activation.validate()?; + ensure!( + activation.expected_projection_sha256 == expected.canonical_sha256()?, + "admitted activation does not bind its Expected projection" + ); + if let Some(lease) = lease { + validate_topology_lease(expected, activation, lease)?; + } + if !self.projection_store.exists() { + return Ok(false); + } + let entries = SingleFileMessagePackBackingStore::new(&self.projection_store) + .pull_all_read_only_snapshot()?; + let anchor = runtime_presence_trust_anchor(expected, provider_anchor)?; + let expected_is_exact = match projection_entry( + &entries, + IdunnExpectedIncarnationRecord::TYPE, + &expected.target, + )? { + Some(envelope) => { + ensure!( + envelope.schema_id.as_deref() == Some(IDUNN_EXPECTED_INCARNATION_SCHEMA), + "admitted Expected projection schema is foreign" + ); + ensure!( + IdunnExpectedIncarnationRecord::decode_canonical(&envelope.payload)? + == *expected, + "admitted Expected projection was replaced by another incarnation" + ); + true + } + None => false, + }; + let anchor_is_exact = match projection_entry( + &entries, + GameCultServiceTrustAnchorRecord::TYPE, + &anchor.trust_anchor_id, + )? { + Some(envelope) => { + ensure!( + service_trust_anchor_from_envelope(envelope)? == anchor, + "admitted runtime presence trust anchor was replaced" + ); + true + } + None => false, + }; + let activation_is_exact = match projection_entry( + &entries, + IdunnRuntimeActivationRecord::TYPE, + &expected.target, + )? { + Some(envelope) => { + ensure!( + envelope.schema_id.as_deref() == Some(IDUNN_RUNTIME_ACTIVATION_SCHEMA), + "admitted activation projection schema is foreign" + ); + ensure!( + IdunnRuntimeActivationRecord::decode_canonical(&envelope.payload)? + == *activation, + "admitted activation projection was replaced" + ); + true + } + None => false, + }; + let lease_is_exact = match projection_entry( + &entries, + IdunnProcessWriteLeaseRecord::TYPE, + &expected.target, + )? { + Some(envelope) => { + ensure!( + envelope.schema_id.as_deref() == Some(IDUNN_PROCESS_WRITE_LEASE_SCHEMA), + "admitted write-lease projection schema is foreign" + ); + let current = IdunnProcessWriteLeaseRecord::decode_canonical(&envelope.payload)?; + ensure!( + lease == Some(¤t), + "admitted write-lease projection was replaced" + ); + true + } + None => lease.is_none(), + }; + Ok(expected_is_exact && anchor_is_exact && activation_is_exact && lease_is_exact) + } +} + +fn runtime_presence_trust_anchor( + expected: &IdunnExpectedIncarnationRecord, + provider_anchor: &ServiceIdentityTrustAnchor, +) -> Result { + expected.validate()?; + ensure!( + provider_anchor.schema_version + == ::TRUST_ANCHOR_SCHEMA, + "runtime presence trust anchor schema is unsupported" + ); + ensure!( + derive_service_identity_id::(&provider_anchor.public_key)? + == provider_anchor.identity_id + && provider_anchor.identity_id == expected.expected_signer_identity_id, + "runtime presence trust anchor does not bind the Expected signer" + ); + let bound_at_unix_millis: u64 = + chrono::DateTime::parse_from_rfc3339(&provider_anchor.identity_created_at)? + .timestamp_millis() + .try_into() + .context("runtime presence trust anchor creation time predates Unix epoch")?; + let anchor = GameCultServiceTrustAnchorRecord { + schema_version: GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA.into(), + trust_anchor_id: runtime_presence_trust_anchor_id(&expected.target), + service_id: expected.target.clone(), + runtime_id: expected.runtime_id.clone(), + signer_identity_id: provider_anchor.identity_id.clone(), + signer_public_key: provider_anchor.public_key.clone(), + signature_algorithm: "ed25519".into(), + signing_purpose: GAMECULT_RUNTIME_PRESENCE_HEALTH_SIGNING_PURPOSE.into(), + signed_schema: GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + binding_authority: "root".into(), + bound_at_unix_millis, + expires_at_unix_millis: None, + private_state_exposed: false, + }; + anchor.validate()?; + Ok(anchor) +} + +fn runtime_presence_trust_anchor_id(target: &str) -> String { + format!("root/{target}/runtime-presence") +} + +fn validate_topology_lease( + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + lease: &IdunnProcessWriteLeaseRecord, +) -> Result<()> { + expected.validate()?; + activation.validate()?; + lease.validate()?; + ensure!( + expected.write_lease_required + && lease.target == expected.target + && lease.expected_projection_sha256 == expected.canonical_sha256()? + && lease.plan_id == expected.plan_id + && lease.incarnation_id == expected.incarnation_id + && lease.sealed_release_id == expected.sealed_release_id + && lease.activation_witness_sha256 == activation.canonical_sha256()? + && lease.state_schema_generation + == expected + .state_schema_generation + .as_deref() + .context("write-lease Expected has no state generation")? + && lease.state_contract_sha256 + == expected + .state_contract_sha256 + .as_deref() + .context("write-lease Expected has no state contract")? + && lease.runtime_id == expected.runtime_id + && lease.runtime_instance_id == activation.runtime_instance_id, + "topology write lease does not bind the exact Expected activation" + ); + Ok(()) +} + +fn service_trust_anchor_from_envelope( + envelope: &CultCacheEnvelope, +) -> Result { + ensure!( + envelope.schema_id.as_deref() == Some(GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA), + "service trust-anchor projection schema is foreign" + ); + let anchor: GameCultServiceTrustAnchorRecord = rmp_serde::from_slice(&envelope.payload)?; + ensure!( + rmp_serde::to_vec(&anchor)? == envelope.payload, + "service trust-anchor projection is noncanonical" + ); + anchor.validate()?; + Ok(anchor) +} + +fn replace_records_atomically(path: &Path, replacements: &[CultCacheEnvelope]) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + ensure!( + replacements.iter().enumerate().all(|(index, entry)| { + replacements[index + 1..].iter().all(|other| { + (entry.r#type.as_str(), entry.key.as_str()) + != (other.r#type.as_str(), other.key.as_str()) + }) + }), + "atomic projection set contains a duplicate identity" + ); + let store = SingleFileMessagePackBackingStore::new(path); + for _ in 0..8 { + let entries = store.pull_all_read_only_snapshot()?; + let conditions = replacements + .iter() + .map(|replacement| { + Ok(CultCacheExpectedEnvelope { + r#type: replacement.r#type.clone(), + key: replacement.key.clone(), + current: projection_entry(&entries, &replacement.r#type, &replacement.key)? + .cloned(), + }) + }) + .collect::>>()?; + if conditions + .iter() + .zip(replacements) + .all(|(condition, replacement)| condition.current.as_ref() == Some(replacement)) + { + return Ok(()); + } + if store.compare_exchange(&conditions, replacements)? { + return Ok(()); + } + } + bail!("CultCache projection changed too often to publish atomic set") +} + +fn projection_entry<'a>( + entries: &'a [CultCacheEnvelope], + r#type: &str, + key: &str, +) -> Result> { + let mut matches = entries + .iter() + .filter(|entry| entry.r#type == r#type && entry.key == key); + let current = matches.next(); + ensure!( + matches.next().is_none(), + "CultCache projection identity is ambiguous" + ); + Ok(current) +} + +fn insert_exact_process_lease( + path: &Path, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + replacement: CultCacheEnvelope, +) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + let store = SingleFileMessagePackBackingStore::new(path); + for _ in 0..8 { + let entries = store.pull_all_read_only_snapshot()?; + let projected_expected = projection_entry( + &entries, + IdunnExpectedIncarnationRecord::TYPE, + &expected.target, + )? + .context("process write lease has no current Expected projection")?; + ensure!( + projected_expected.schema_id.as_deref() == Some(IDUNN_EXPECTED_INCARNATION_SCHEMA) + && IdunnExpectedIncarnationRecord::decode_canonical(&projected_expected.payload)? + == *expected, + "process write lease current Expected projection is substituted" + ); + let projected_activation = projection_entry( + &entries, + IdunnRuntimeActivationRecord::TYPE, + &expected.target, + )? + .context("process write lease has no observed activation projection")?; + ensure!( + projected_activation.schema_id.as_deref() == Some(IDUNN_RUNTIME_ACTIVATION_SCHEMA) + && IdunnRuntimeActivationRecord::decode_canonical(&projected_activation.payload)? + == *activation, + "process write lease observed activation projection is substituted" + ); + let projected_anchor = service_trust_anchor_from_envelope( + projection_entry( + &entries, + GameCultServiceTrustAnchorRecord::TYPE, + &runtime_presence_trust_anchor_id(&expected.target), + )? + .context("process write lease has no runtime presence trust anchor")?, + )?; + ensure!( + projected_anchor.service_id == expected.target + && projected_anchor.runtime_id == expected.runtime_id + && projected_anchor.signer_identity_id == expected.expected_signer_identity_id + && projected_anchor.signing_purpose + == GAMECULT_RUNTIME_PRESENCE_HEALTH_SIGNING_PURPOSE + && projected_anchor.signed_schema == GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, + "process write lease runtime presence trust anchor is substituted" + ); + let current = projection_entry(&entries, &replacement.r#type, &replacement.key)?; + if current == Some(&replacement) { + return Ok(()); + } + ensure!( + current.is_none(), + "refusing to replace a different current process write-lease projection" + ); + let conditions = [ + CultCacheExpectedEnvelope { r#type: IdunnExpectedIncarnationRecord::TYPE.into(), - payload: expected.canonical_bytes()?, - stored_at: chrono::Utc::now().to_rfc3339(), - schema_id: Some(IDUNN_EXPECTED_INCARNATION_SCHEMA.into()), + key: expected.target.clone(), + current: Some(projected_expected.clone()), }, + CultCacheExpectedEnvelope { + r#type: IdunnRuntimeActivationRecord::TYPE.into(), + key: expected.target.clone(), + current: Some(projected_activation.clone()), + }, + CultCacheExpectedEnvelope { + r#type: GameCultServiceTrustAnchorRecord::TYPE.into(), + key: runtime_presence_trust_anchor_id(&expected.target), + current: projection_entry( + &entries, + GameCultServiceTrustAnchorRecord::TYPE, + &runtime_presence_trust_anchor_id(&expected.target), + )? + .cloned(), + }, + CultCacheExpectedEnvelope { + r#type: replacement.r#type.clone(), + key: replacement.key.clone(), + current: None, + }, + ]; + if store.compare_exchange(&conditions, std::slice::from_ref(&replacement))? { + return Ok(()); + } + } + bail!("CultCache projection changed too often to publish exact record") +} + +impl TopologyPort for CultCacheTopologyDriver { + fn publish_expected( + &self, + expected: &IdunnExpectedIncarnationRecord, + provider_anchor: &ServiceIdentityTrustAnchor, + ) -> Result { + expected.validate()?; + let anchor = runtime_presence_trust_anchor(expected, provider_anchor)?; + replace_records_atomically( + &self.projection_store, + &[ + CultCacheEnvelope { + key: expected.target.clone(), + r#type: IdunnExpectedIncarnationRecord::TYPE.into(), + payload: expected.canonical_bytes()?, + stored_at: chrono::Utc::now().to_rfc3339(), + schema_id: Some(IDUNN_EXPECTED_INCARNATION_SCHEMA.into()), + }, + CultCacheEnvelope { + key: anchor.trust_anchor_id.clone(), + r#type: GameCultServiceTrustAnchorRecord::TYPE.into(), + payload: rmp_serde::to_vec(&anchor)?, + stored_at: rfc3339_millis(anchor.bound_at_unix_millis)?, + schema_id: Some(GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA.into()), + }, + ], )?; expected.canonical_sha256() } - fn withdraw_expected(&self, expected: &IdunnExpectedIncarnationRecord) -> Result<()> { + fn withdraw_expected( + &self, + expected: &IdunnExpectedIncarnationRecord, + provider_anchor: &ServiceIdentityTrustAnchor, + activation: Option<&IdunnRuntimeActivationRecord>, + lease: Option<&IdunnProcessWriteLeaseRecord>, + ) -> Result<()> { expected.validate()?; + let exact_anchor = runtime_presence_trust_anchor(expected, provider_anchor)?; + if let Some(activation) = activation { + activation.validate()?; + ensure!( + activation.expected_projection_sha256 == expected.canonical_sha256()? + && activation.runtime_id == expected.runtime_id, + "withdrawn activation does not bind the exact Expected projection" + ); + } + if let Some(lease) = lease { + validate_topology_lease( + expected, + activation.context("withdrawn lease has no exact activation")?, + lease, + )?; + } if !self.projection_store.exists() { return Ok(()); } - let expected_sha256 = expected.canonical_sha256()?; let store = SingleFileMessagePackBackingStore::new(&self.projection_store); for _ in 0..8 { let entries = store.pull_all_read_only_snapshot()?; let mut found_expected = false; + let mut found_anchor = false; let mut found_activation = false; + let mut found_lease = false; let mut retained = Vec::with_capacity(entries.len()); for envelope in &entries { + if envelope.r#type == GameCultServiceTrustAnchorRecord::TYPE + && envelope.key == exact_anchor.trust_anchor_id + { + ensure!( + !found_anchor + && service_trust_anchor_from_envelope(envelope)? == exact_anchor, + "refusing to withdraw a substituted runtime presence trust anchor" + ); + found_anchor = true; + continue; + } if envelope.key != expected.target { retained.push(envelope.clone()); continue; @@ -3082,22 +3700,34 @@ impl TopologyPort for CultCacheTopologyDriver { continue; } if envelope.r#type == IdunnRuntimeActivationRecord::TYPE { - let activation = + let current = IdunnRuntimeActivationRecord::decode_canonical(&envelope.payload)?; ensure!( !found_activation && envelope.schema_id.as_deref() == Some(IDUNN_RUNTIME_ACTIVATION_SCHEMA) - && activation.expected_projection_sha256 == expected_sha256 - && activation.runtime_id == expected.runtime_id, + && activation == Some(¤t), "refusing to withdraw a substituted runtime activation" ); found_activation = true; continue; } + if envelope.r#type == IdunnProcessWriteLeaseRecord::TYPE { + let current = + IdunnProcessWriteLeaseRecord::decode_canonical(&envelope.payload)?; + ensure!( + !found_lease + && envelope.schema_id.as_deref() + == Some(IDUNN_PROCESS_WRITE_LEASE_SCHEMA) + && lease == Some(¤t), + "refusing to withdraw a substituted process write lease" + ); + found_lease = true; + continue; + } retained.push(envelope.clone()); } - if !found_expected && !found_activation { + if !found_expected && !found_anchor && !found_activation && !found_lease { return Ok(()); } if store.compare_exchange_snapshot(&entries, &retained)? { @@ -3134,6 +3764,73 @@ impl TopologyPort for CultCacheTopologyDriver { activation.canonical_sha256() } + fn publish_process_write_lease( + &self, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + lease: &IdunnProcessWriteLeaseRecord, + ) -> Result { + validate_topology_lease(expected, activation, lease)?; + insert_exact_process_lease( + &self.projection_store, + expected, + activation, + CultCacheEnvelope { + key: expected.target.clone(), + r#type: IdunnProcessWriteLeaseRecord::TYPE.into(), + payload: lease.canonical_bytes()?, + stored_at: rfc3339_millis(lease.issued_at_unix_millis)?, + schema_id: Some(IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into()), + }, + )?; + lease.canonical_sha256() + } + + fn withdraw_process_write_lease( + &self, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + lease: Option<&IdunnProcessWriteLeaseRecord>, + ) -> Result<()> { + expected.validate()?; + activation.validate()?; + if let Some(lease) = lease { + validate_topology_lease(expected, activation, lease)?; + } + if !self.projection_store.exists() { + return Ok(()); + } + let store = SingleFileMessagePackBackingStore::new(&self.projection_store); + for _ in 0..8 { + let entries = store.pull_all_read_only_snapshot()?; + let mut found = false; + let mut retained = Vec::with_capacity(entries.len()); + for envelope in &entries { + if envelope.key != expected.target + || envelope.r#type != IdunnProcessWriteLeaseRecord::TYPE + { + retained.push(envelope.clone()); + continue; + } + let current = IdunnProcessWriteLeaseRecord::decode_canonical(&envelope.payload)?; + ensure!( + !found + && envelope.schema_id.as_deref() == Some(IDUNN_PROCESS_WRITE_LEASE_SCHEMA) + && lease == Some(¤t), + "refusing to withdraw an unexpected process write-lease projection" + ); + found = true; + } + if !found { + return Ok(()); + } + if store.compare_exchange_snapshot(&entries, &retained)? { + return Ok(()); + } + } + bail!("process write-lease projection changed repeatedly while withdrawing it") + } + fn receive(&self, target: &str) -> Result> { require_driver_id(target, "topology target")?; if !self.correlation_store.exists() { @@ -3163,8 +3860,9 @@ impl TopologyPort for CultCacheTopologyDriver { } /// nginx owns proxy mechanics. Idunn supplies one exact backend membership, -/// validates the complete nginx configuration, reloads it, and then observes -/// that nginx's loaded configuration contains those exact bytes. +/// validates the complete nginx configuration, and reloads it. Configuration +/// bytes are actuator state, not proof that the selected runtime answered on +/// the stable route; the control plane admits that proof separately. pub struct NginxRouteDriver { pub binding: RouteBinding, pub nginx_program: PathBuf, @@ -3211,12 +3909,9 @@ impl NginxRouteDriver { Ok(output) } - fn render( - &self, - expected: &IdunnExpectedIncarnationRecord, - runtime_instance_id: &str, - ) -> Result> { + fn render(&self, expected: &IdunnExpectedIncarnationRecord) -> Result> { expected.validate()?; + let expected_projection_sha256 = expected.canonical_sha256()?; let route = expected .route .as_ref() @@ -3226,13 +3921,14 @@ impl NginxRouteDriver { && route.stable_endpoint == self.binding.stable_endpoint, "route driver binding differs from Expected" ); - let (candidate_host, candidate_port) = endpoint_host_port( - &route.candidate_endpoint, - match self.binding.driver { - RouteDriver::NginxHttp => "http://", - RouteDriver::NginxStreamTcp => "tcp://", - }, - )?; + let endpoint_prefix = match (self.binding.driver, route.transport.as_str()) { + (RouteDriver::NginxStreamTcp, "http") => "http://", + (RouteDriver::NginxStreamTcp, "tcp") => "tcp://", + (RouteDriver::NginxStreamUdp, "rudp") => "rudp://", + _ => bail!("route driver cannot carry the Expected transport"), + }; + let (candidate_host, candidate_port) = + endpoint_host_port(&route.candidate_endpoint, endpoint_prefix)?; ensure!( candidate_host == self.binding.private_host && (self.binding.private_port_start..=self.binding.private_port_end) @@ -3241,40 +3937,30 @@ impl NginxRouteDriver { ); let upstream = nginx_identifier(&self.binding.route_id)?; let rendered = match self.binding.driver { - RouteDriver::NginxHttp => format!( - "# Idunn runtime {runtime_instance_id}\nupstream {upstream} {{\n server {candidate_host}:{candidate_port};\n keepalive 16;\n}}\n" - ), RouteDriver::NginxStreamTcp => { let (stable_host, stable_port) = - endpoint_host_port(&route.stable_endpoint, "tcp://")?; + endpoint_host_port(&route.stable_endpoint, endpoint_prefix)?; format!( - "# Idunn runtime {runtime_instance_id}\nupstream {upstream} {{\n server {candidate_host}:{candidate_port};\n}}\nserver {{\n listen {stable_host}:{stable_port};\n proxy_pass {upstream};\n}}\n" + "# Idunn Expected {expected_projection_sha256}\nupstream {upstream} {{\n server {candidate_host}:{candidate_port};\n}}\nserver {{\n listen {stable_host}:{stable_port};\n proxy_pass {upstream};\n}}\n" + ) + } + RouteDriver::NginxStreamUdp => { + let (stable_host, stable_port) = + endpoint_host_port(&route.stable_endpoint, "rudp://")?; + format!( + "# Idunn Expected {expected_projection_sha256}\nupstream {upstream} {{\n server {candidate_host}:{candidate_port};\n}}\nserver {{\n listen {stable_host}:{stable_port} udp reuseport;\n proxy_pass {upstream};\n}}\n" ) } }; Ok(rendered.into_bytes()) } - fn loaded_matches(&self, rendered: &[u8]) -> Result { - let output = self.command(&self.nginx_program, [OsString::from("-T")])?; - let loaded = std::str::from_utf8(&output.stdout) - .context("nginx loaded configuration is not UTF-8")?; - Ok(nginx_config_section(loaded, &self.binding.config_path)? - .is_some_and(|section| section.as_bytes() == rendered)) - } - - fn current_and_loaded(&self) -> Result<(Option>, Option>)> { - let current = match fs::read(&self.binding.config_path) { + fn current_configuration(&self) -> Result>> { + Ok(match fs::read(&self.binding.config_path) { Ok(bytes) => Some(bytes), Err(error) if error.kind() == ErrorKind::NotFound => None, Err(error) => return Err(error).context("reading route fragment"), - }; - let output = self.command(&self.nginx_program, [OsString::from("-T")])?; - let dump = std::str::from_utf8(&output.stdout) - .context("nginx loaded configuration is not UTF-8")?; - let loaded = nginx_config_section(dump, &self.binding.config_path)? - .map(|section| section.as_bytes().to_vec()); - Ok((current, loaded)) + }) } fn write_fragment(&self, content: Option<&[u8]>) -> Result<()> { @@ -3344,12 +4030,12 @@ impl NginxRouteDriver { self.reload() } - fn fail_after_rollback( + fn fail_after_rollback( &self, prior: Option<&[u8]>, failure: anyhow::Error, context: &str, - ) -> Result<()> { + ) -> Result { match self.restore(prior) { Ok(()) => Err(failure).context(context.to_owned()), Err(rollback) => Err(failure).context(format!( @@ -3357,22 +4043,16 @@ impl NginxRouteDriver { )), } } -} -impl RoutePort for NginxRouteDriver { - fn preflight( + pub fn preflight( &self, expected: &IdunnExpectedIncarnationRecord, runtime_instance_id: &str, incumbent: Option<&RouteObservation>, ) -> Result { - let rendered = self.render(expected, runtime_instance_id)?; + let rendered = self.render(expected)?; ensure!(!rendered.is_empty(), "candidate route rendered empty"); - let (current, loaded) = self.current_and_loaded()?; - ensure!( - current == loaded, - "nginx route baseline differs from its loaded configuration" - ); + let current = self.current_configuration()?; let incumbent_membership_sha256 = current.as_ref().map(|bytes| sha256_id(bytes)); match incumbent { Some(incumbent) => ensure!( @@ -3387,86 +4067,379 @@ impl RoutePort for NginxRouteDriver { ), } self.validate_candidate_in_private_mount(&rendered)?; - let after = self.current_and_loaded()?; + let after = self.current_configuration()?; ensure!( - after.0 == current && after.1 == current, + after == current, "nginx route baseline changed during candidate validation" ); - Ok(RoutePreflightReceipt { + let receipt = RoutePreflightReceipt { route_id: self.binding.route_id.clone(), candidate_runtime_instance_id: runtime_instance_id.to_owned(), candidate_membership_sha256: sha256_id(&rendered), incumbent_runtime_instance_id: incumbent .map(|observation| observation.runtime_instance_id.clone()), incumbent_membership_sha256, - }) + incumbent_configuration: current, + }; + receipt.validate()?; + Ok(receipt) } - fn promote( + /// Install the candidate route membership and require nginx to accept the + /// complete configuration. Success is not route admission: callers must + /// still obtain a fresh signed runtime observation through the stable + /// listener. + pub fn install( &self, expected: &IdunnExpectedIncarnationRecord, runtime_instance_id: &str, preflight: &RoutePreflightReceipt, - ) -> Result { - let rendered = self.render(expected, runtime_instance_id)?; + rollback_allowed: bool, + ) -> Result { + preflight.validate()?; + let rendered = self.render(expected)?; ensure!( preflight.route_id == self.binding.route_id && preflight.candidate_runtime_instance_id == runtime_instance_id && preflight.candidate_membership_sha256 == sha256_id(&rendered), "route preflight does not authorize this candidate membership" ); - let (prior, loaded) = self.current_and_loaded()?; + let prior = self.current_configuration()?; let prior_sha256 = prior.as_ref().map(|bytes| sha256_id(bytes)); + let candidate_already_written = prior.as_deref() == Some(rendered.as_slice()); ensure!( - prior == loaded && prior_sha256 == preflight.incumbent_membership_sha256, + candidate_already_written + || (prior == preflight.incumbent_configuration + && prior_sha256 == preflight.incumbent_membership_sha256), "route baseline changed after preflight" ); - atomic_replace(&self.binding.config_path, &rendered)?; + if !candidate_already_written { + atomic_replace(&self.binding.config_path, &rendered)?; + } if let Err(error) = self.reload() { - self.fail_after_rollback( - prior.as_deref(), - error, - "candidate route validation or reload failed", - )?; + if rollback_allowed { + return self.fail_after_rollback( + preflight.incumbent_configuration.as_deref(), + error, + "candidate route validation or reload failed", + ); + } + return Err(error).context( + "candidate route reload failed after incumbent route authority was fenced; candidate fragment retained for retry", + ); } - match self.loaded_matches(&rendered) { - Ok(true) => {} - Ok(false) => self.fail_after_rollback( - prior.as_deref(), - anyhow::anyhow!("nginx loaded configuration omitted the admitted membership"), - "candidate route observation failed", - )?, - Err(error) => self.fail_after_rollback( - prior.as_deref(), - error, - "candidate route observation failed", - )?, - } - Ok(RouteObservation { - route_id: self.binding.route_id.clone(), - runtime_instance_id: runtime_instance_id.to_owned(), - membership_sha256: sha256_id(&rendered), - }) + ensure!( + self.current_configuration()?.as_deref() == Some(rendered.as_slice()), + "candidate route fragment changed during reload" + ); + Ok(sha256_id(&rendered)) } - fn observe( + pub fn observe_membership( &self, expected: &IdunnExpectedIncarnationRecord, - observation: &RouteObservation, + membership_sha256: &str, ) -> Result { - let rendered = self.render(expected, &observation.runtime_instance_id)?; + let rendered = self.render(expected)?; ensure!( - observation.route_id == self.binding.route_id - && observation.membership_sha256 == sha256_id(&rendered), + membership_sha256 == sha256_id(&rendered), "route observation does not describe the expected membership" ); - let current = match fs::read(&self.binding.config_path) { - Ok(bytes) => bytes, - Err(error) if error.kind() == ErrorKind::NotFound => return Ok(false), - Err(error) => return Err(error).context("reading route observation"), - }; - Ok(current == rendered && self.loaded_matches(&rendered)?) + Ok(self.current_configuration()?.as_deref() == Some(rendered.as_slice())) } + + /// Restore the exact membership owned by an admitted generation. Any + /// different fragment is drift, not a rollback baseline, so it is never + /// restored after a failed reload. A failed reload removes the newly + /// written fragment so the next continuity pass cannot mistake disk bytes + /// for an adopted route. The caller must still challenge the stable + /// listener to prove that nginx workers adopted this membership. + pub fn restore_admitted_membership( + &self, + expected: &IdunnExpectedIncarnationRecord, + membership_sha256: &str, + ) -> Result<()> { + let rendered = self.render(expected)?; + ensure!( + membership_sha256 == sha256_id(&rendered), + "admitted route receipt does not describe the expected membership" + ); + self.validate_candidate_in_private_mount(&rendered)?; + if self.current_configuration()?.as_deref() != Some(rendered.as_slice()) { + atomic_replace(&self.binding.config_path, &rendered)?; + } + if let Err(reload) = self.reload() { + return match self.write_fragment(None) { + Ok(()) => Err(reload).context("reloading the exact admitted route membership"), + Err(cleanup) => Err(reload).context(format!( + "reloading the exact admitted route membership; removing its unproved fragment also failed: {cleanup:#}" + )), + }; + } + ensure!( + self.current_configuration()?.as_deref() == Some(rendered.as_slice()), + "admitted route fragment changed during restoration" + ); + Ok(()) + } + + /// Ask the stable listener for one freshly minted provider-owned presence + /// statement. This method validates the CultNet wire exchange and exact + /// document selection only; Idunn's control plane authenticates and + /// correlates the signed payload against current authority. + pub fn request_runtime_presence( + &self, + expected: &IdunnExpectedIncarnationRecord, + message_id: &str, + ) -> Result { + let route = expected + .route + .as_ref() + .context("route challenge has no Expected route")?; + ensure!( + route.route_id == self.binding.route_id + && route.stable_endpoint == self.binding.stable_endpoint, + "route challenge binding differs from Expected" + ); + self.request_runtime_presence_at(expected, message_id, &route.stable_endpoint) + } + + /// The bootstrap exception for the first managed Odin observes Warming on + /// the exact candidate endpoint. It does not install or admit a route. + pub fn request_candidate_runtime_presence( + &self, + expected: &IdunnExpectedIncarnationRecord, + message_id: &str, + ) -> Result { + let route = expected + .route + .as_ref() + .context("candidate challenge has no Expected route")?; + ensure!( + route.route_id == self.binding.route_id, + "candidate challenge binding differs from Expected" + ); + self.render(expected)?; + self.request_runtime_presence_at(expected, message_id, &route.candidate_endpoint) + } + + fn request_runtime_presence_at( + &self, + expected: &IdunnExpectedIncarnationRecord, + message_id: &str, + endpoint: &str, + ) -> Result { + expected.validate()?; + require_driver_id(message_id, "route challenge message")?; + let route = expected + .route + .as_ref() + .context("route challenge has no Expected route")?; + let request = CultNetMessage::SnapshotRequest { + message_id: message_id.to_owned(), + schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), + record_keys: Some(vec![expected.target.clone()]), + }; + let endpoint_prefix = match route.transport.as_str() { + "http" => "http://", + "tcp" => "tcp://", + "rudp" => "rudp://", + _ => bail!("route challenge transport is unsupported"), + }; + let (host, port) = endpoint_host_port(endpoint, endpoint_prefix)?; + let target = SocketAddr::new( + host.parse() + .context("route challenge endpoint host is not an IP address")?, + port, + ); + let response = match route.transport.as_str() { + "http" => { + let payload = + encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0)?; + let response = request_http_snapshot(target, &payload)?; + decode_cultnet_message_from_slice(&response, CultNetWireContract::CultNetSchemaV0)? + } + "tcp" => { + let payload = + encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0)?; + let response = request_tcp_snapshot(target, &payload)?; + decode_cultnet_message_from_slice(&response, CultNetWireContract::CultNetSchemaV0)? + } + "rudp" => request_raw_snapshot_from_rudp_catalog(CultMeshRudpSnapshotOptions { + target, + runtime_id: "idunn-route-observer".into(), + message_id: message_id.to_owned(), + schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), + record_keys: Some(vec![expected.target.clone()]), + ..CultMeshRudpSnapshotOptions::default() + })?, + _ => bail!("route challenge transport is unsupported"), + }; + exact_route_snapshot_response(response, message_id, &expected.target) + } + + /// Restore the exact preflight baseline. This may only replace the exact + /// candidate authorized by the receipt; an unrelated writer is never + /// overwritten as a side effect of rollback. + pub fn rollback( + &self, + expected: &IdunnExpectedIncarnationRecord, + runtime_instance_id: &str, + preflight: &RoutePreflightReceipt, + ) -> Result<()> { + preflight.validate()?; + let rendered = self.render(expected)?; + ensure!( + preflight.route_id == self.binding.route_id + && preflight.candidate_runtime_instance_id == runtime_instance_id + && preflight.candidate_membership_sha256 == sha256_id(&rendered), + "route preflight does not authorize rollback of this candidate membership" + ); + ensure!( + self.current_configuration()?.as_deref() == Some(rendered.as_slice()), + "candidate route changed before rollback" + ); + self.restore(preflight.incumbent_configuration.as_deref()) + } +} + +fn request_tcp_snapshot(target: SocketAddr, payload: &[u8]) -> Result> { + let mut stream = TcpStream::connect_timeout(&target, ROUTE_SNAPSHOT_TIMEOUT) + .with_context(|| format!("connecting stable CultNet TCP route {target}"))?; + stream.set_read_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT))?; + stream.set_write_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT))?; + stream.write_all(&encode_frame(payload)?)?; + stream.flush()?; + + let mut header = [0_u8; 4]; + stream + .read_exact(&mut header) + .context("reading stable CultNet TCP response frame")?; + let length = u32::from_be_bytes(header) as usize; + ensure!( + (1..=ROUTE_SNAPSHOT_MAX_BYTES).contains(&length), + "stable CultNet TCP response exceeds the route observation bound" + ); + let mut response = vec![0_u8; length]; + stream + .read_exact(&mut response) + .context("reading stable CultNet TCP response payload")?; + Ok(response) +} + +fn request_http_snapshot(target: SocketAddr, payload: &[u8]) -> Result> { + let mut stream = TcpStream::connect_timeout(&target, ROUTE_SNAPSHOT_TIMEOUT) + .with_context(|| format!("connecting stable CultNet HTTP route {target}"))?; + stream.set_read_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT))?; + stream.set_write_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT))?; + let head = format!( + "POST {ROUTE_HTTP_SNAPSHOT_PATH} HTTP/1.1\r\nHost: {target}\r\nContent-Type: application/msgpack\r\nAccept: application/msgpack\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + payload.len() + ); + stream.write_all(head.as_bytes())?; + stream.write_all(payload)?; + stream.flush()?; + + let maximum = ROUTE_HTTP_MAX_HEADER_BYTES + .checked_add(ROUTE_SNAPSHOT_MAX_BYTES) + .context("route HTTP response bound overflow")?; + let mut response = Vec::new(); + (&mut stream) + .take((maximum + 1) as u64) + .read_to_end(&mut response) + .context("reading stable CultNet HTTP response")?; + ensure!( + response.len() <= maximum, + "stable CultNet HTTP response exceeds the route observation bound" + ); + let header_end = response + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map(|index| index + 4) + .context("stable CultNet HTTP response has no header boundary")?; + ensure!( + header_end <= ROUTE_HTTP_MAX_HEADER_BYTES, + "stable CultNet HTTP response headers exceed their bound" + ); + let headers = std::str::from_utf8(&response[..header_end]) + .context("stable CultNet HTTP response headers are not UTF-8")?; + let mut lines = headers.split("\r\n"); + let status = lines + .next() + .context("stable CultNet HTTP response has no status")?; + ensure!( + matches!(status, "HTTP/1.1 200 OK" | "HTTP/1.0 200 OK"), + "stable CultNet HTTP route rejected the snapshot challenge: {status}" + ); + let mut content_length = None; + for line in lines.filter(|line| !line.is_empty()) { + let (name, value) = line + .split_once(':') + .context("stable CultNet HTTP response contains a malformed header")?; + if name.eq_ignore_ascii_case("transfer-encoding") { + bail!("stable CultNet HTTP response uses unsupported transfer encoding"); + } + if name.eq_ignore_ascii_case("content-length") { + ensure!( + content_length.is_none(), + "stable CultNet HTTP response repeats Content-Length" + ); + content_length = Some( + value + .trim() + .parse::() + .context("stable CultNet HTTP Content-Length is invalid")?, + ); + } + } + let content_length = + content_length.context("stable CultNet HTTP response has no Content-Length")?; + ensure!( + content_length <= ROUTE_SNAPSHOT_MAX_BYTES + && response.len().saturating_sub(header_end) == content_length, + "stable CultNet HTTP response body differs from its bounded Content-Length" + ); + Ok(response[header_end..].to_vec()) +} + +fn exact_route_snapshot_response( + response: CultNetMessage, + message_id: &str, + target: &str, +) -> Result { + let CultNetMessage::SnapshotResponseRaw { + message_id: response_id, + documents, + } = response + else { + bail!("stable route returned a non-raw snapshot response") + }; + ensure!( + response_id == message_id, + "stable route snapshot response belongs to another challenge" + ); + ensure!( + documents.len() == 1, + "stable route snapshot response is not the exact singleton presence document" + ); + let document = documents + .into_iter() + .next() + .expect("singleton response length was checked"); + ensure!( + document.schema_id == GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA + && document.record_key == target + && document.payload_encoding == CultNetRawPayloadEncoding::Messagepack, + "stable route substituted the runtime presence document" + ); + ensure!( + !document.payload.is_empty() && document.payload.len() <= ROUTE_SNAPSHOT_MAX_BYTES, + "stable route presence payload is empty or exceeds its bound" + ); + Ok(RouteSnapshotResponse { + message_id: response_id, + canonical_presence: document.payload, + }) } fn release_artifact<'a>( @@ -4654,24 +5627,6 @@ fn nginx_identifier(route_id: &str) -> Result { Ok(identifier) } -fn nginx_config_section<'a>(dump: &'a str, config_path: &Path) -> Result> { - let marker = format!("# configuration file {}:\n", config_path.display()); - let matches = dump.match_indices(&marker).collect::>(); - ensure!( - matches.len() <= 1, - "nginx loaded configuration contains duplicate sections for {}", - config_path.display() - ); - let Some((start, _)) = matches.first().copied() else { - return Ok(None); - }; - let content = &dump[start + marker.len()..]; - let end = content - .find("\n# configuration file ") - .map_or(content.len(), |index| index + 1); - Ok(Some(&content[..end])) -} - fn atomic_replace(path: &Path, bytes: &[u8]) -> Result<()> { #[cfg(unix)] use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; @@ -5539,6 +6494,7 @@ mod tests { IdunnExpectedIncarnationRecord, IdunnRuntimeActivationRecord, SequenceAdmittedWarming, + ServiceIdentityTrustAnchor, )> { let provider = enroll_service_identity_at::( &root.join("provider.cc"), @@ -5594,7 +6550,7 @@ mod tests { }, )?; let warming = SequenceAdmittedWarming::for_test("test-transaction", warming, 120)?; - Ok((expected, activation, warming)) + Ok((expected, activation, warming, provider.trust_anchor()?)) } fn lease( @@ -5614,52 +6570,81 @@ mod tests { state_contract_sha256: expected.state_contract_sha256.clone().unwrap(), runtime_id: expected.runtime_id.clone(), runtime_instance_id: activation.runtime_instance_id.clone(), - warming_presence_sha256: warming - .authenticated() - .record() - .signed_presence_sha256 - .clone() - .unwrap(), + warming_presence_sha256: warming.signed_presence_sha256().to_owned(), lease_epoch: 1, issued_at_unix_millis: 200, } } + fn topology_driver(root: &Path, name: &str) -> CultCacheTopologyDriver { + CultCacheTopologyDriver { + projection_store: root.join(format!("{name}.cc")), + correlation_store: root.join(format!("{name}-correlation.cc")), + } + } + + fn project_activation( + driver: &CultCacheTopologyDriver, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result<()> { + ensure!( + activation.expected_projection_sha256 == expected.canonical_sha256()?, + "test activation belongs to another Expected" + ); + upsert_record( + &driver.projection_store, + CultCacheEnvelope { + key: expected.target.clone(), + r#type: IdunnRuntimeActivationRecord::TYPE.into(), + payload: activation.canonical_bytes()?, + stored_at: rfc3339_millis(activation.issued_at_unix_millis)?, + schema_id: Some(IDUNN_RUNTIME_ACTIVATION_SCHEMA.into()), + }, + ) + } + #[test] - fn nginx_loaded_observation_is_bound_to_the_exact_file_section() -> Result<()> { - let path = Path::new("/etc/nginx/idunn-routes/service.conf"); - let rendered = "upstream idunn_service {\n server 127.0.0.1:4104;\n}\n"; - let dump = format!( - "# configuration file /etc/nginx/nginx.conf:\nevents {{}}\n# configuration file {}:\n{rendered}# configuration file /etc/nginx/other.conf:\n{rendered}", - path.display() - ); - assert_eq!(nginx_config_section(&dump, path)?, Some(rendered)); - assert_eq!( - nginx_config_section(&dump, Path::new("/etc/nginx/missing.conf"))?, - None - ); - let duplicate = format!("{dump}# configuration file {}:\n{rendered}", path.display()); - assert!(nginx_config_section(&duplicate, path).is_err()); + fn nginx_rudp_route_is_rendered_as_a_udp_stream_proxy() -> Result<()> { + let driver = NginxRouteDriver::new(RouteBinding { + driver: RouteDriver::NginxStreamUdp, + route_id: "odin-rudp".into(), + stable_endpoint: "rudp://127.0.0.1:17872".into(), + private_host: "127.0.0.1".into(), + private_port_start: 27872, + private_port_end: 27879, + config_path: "/etc/nginx/idunn-stream-routes/odin-rudp.conf".into(), + reload_unit: "nginx.service".into(), + }); + let mut candidate = expected(); + candidate.route = Some(cultnet_rs::IdunnExpectedRoute { + route_id: "odin-rudp".into(), + transport: "rudp".into(), + stable_endpoint: "rudp://127.0.0.1:17872".into(), + candidate_endpoint: "rudp://127.0.0.1:27872".into(), + }); + let rendered = String::from_utf8(driver.render(&candidate)?)?; + assert!(rendered.contains("listen 127.0.0.1:17872 udp reuseport;")); + assert!(rendered.contains("server 127.0.0.1:27872;")); Ok(()) } #[cfg(unix)] #[test] - fn nginx_preflight_validates_candidate_and_binds_the_loaded_incumbent() -> Result<()> { + fn nginx_preflight_binds_exact_incumbent_configuration_without_claiming_live_route() + -> Result<()> { use std::os::unix::fs::PermissionsExt; let temp = tempfile::tempdir()?; let config = temp.path().join("route.conf"); - let loaded = temp.path().join("loaded.conf"); let nginx = temp.path().join("nginx"); let systemd_run = temp.path().join("systemd-run"); let systemctl = temp.path().join("systemctl"); fs::write( &nginx, format!( - "#!/bin/sh\nconfig='{}'\nloaded='{}'\nif [ \"$1\" = \"-T\" ]; then\n if [ -f \"$loaded\" ]; then\n printf '# configuration file %s:\\n' \"$config\"\n /bin/cat \"$loaded\"\n fi\n exit 0\nfi\nif [ \"$1\" = \"-t\" ]; then\n candidate=${{IDUNN_TEST_SHADOW:-$config}}\n /bin/grep -q 'server 127.0.0.1:4104' \"$candidate\"\n exit $?\nfi\nexit 64\n", + "#!/bin/sh\nconfig='{}'\nif [ \"$1\" = \"-t\" ]; then\n candidate=${{IDUNN_TEST_SHADOW:-$config}}\n /bin/grep -q 'server 127.0.0.1:4104' \"$candidate\"\n exit $?\nfi\nexit 64\n", config.display(), - loaded.display(), ), )?; fs::write( @@ -5670,18 +6655,14 @@ mod tests { )?; fs::write( &systemctl, - format!( - "#!/bin/sh\nif [ \"$1\" = \"reload\" ]; then\n /bin/cp '{}' '{}'\n exit 0\nfi\nexit 64\n", - config.display(), - loaded.display(), - ), + "#!/bin/sh\nif [ -e \"$0.fail\" ]; then exit 65; fi\nif [ \"$1\" = \"reload\" ]; then echo reload >> \"$0.calls\"; exit 0; fi\nexit 64\n", )?; fs::set_permissions(&nginx, fs::Permissions::from_mode(0o755))?; fs::set_permissions(&systemd_run, fs::Permissions::from_mode(0o755))?; fs::set_permissions(&systemctl, fs::Permissions::from_mode(0o755))?; let binding = RouteBinding { - driver: RouteDriver::NginxHttp, + driver: RouteDriver::NginxStreamTcp, route_id: "service-route".into(), stable_endpoint: "http://127.0.0.1:4103".into(), private_host: "127.0.0.1".into(), @@ -5694,7 +6675,7 @@ mod tests { binding, nginx_program: nginx, systemd_run_program: systemd_run, - systemctl_program: systemctl, + systemctl_program: systemctl.clone(), preflight_root: temp.path().join("preflight"), }; let mut candidate = expected(); @@ -5706,23 +6687,58 @@ mod tests { }); let preflight = driver.preflight(&candidate, &digest('a'), None)?; assert!(!config.exists()); - assert!(!loaded.exists()); assert_eq!(fs::read_dir(&driver.preflight_root)?.count(), 0); - let admitted = driver.promote(&candidate, &digest('a'), &preflight)?; - assert!(driver.observe(&candidate, &admitted)?); + let candidate_bytes = driver.render(&candidate)?; + fs::write(&config, &candidate_bytes)?; + let membership_sha256 = driver.install(&candidate, &digest('a'), &preflight, true)?; + assert_eq!( + fs::read_to_string(systemctl.with_extension("calls"))?, + "reload\n" + ); + assert!(driver.observe_membership(&candidate, &membership_sha256)?); let admitted_bytes = fs::read(&config)?; - assert_eq!(fs::read(&loaded)?, admitted_bytes); + assert!(std::str::from_utf8(&admitted_bytes)?.contains("listen 127.0.0.1:4103;")); + let admitted = RouteObservation { + route_id: "service-route".into(), + runtime_instance_id: digest('a'), + membership_sha256, + signed_presence_sha256: digest('c'), + observed_at_unix_millis: 1, + }; let next_preflight = driver.preflight(&candidate, &digest('b'), Some(&admitted))?; assert_eq!(fs::read(&config)?, admitted_bytes); + fs::write(systemctl.with_extension("fail"), b"fail\n")?; + assert!( + driver + .install(&candidate, &digest('b'), &next_preflight, false) + .is_err() + ); + assert_eq!(fs::read(&config)?, admitted_bytes); + fs::remove_file(systemctl.with_extension("fail"))?; + driver.install(&candidate, &digest('b'), &next_preflight, false)?; fs::write(&config, b"foreign route\n")?; assert!( driver - .promote(&candidate, &digest('b'), &next_preflight) + .install(&candidate, &digest('b'), &next_preflight, true) .is_err() ); - assert_eq!(fs::read(&loaded)?, admitted_bytes); + driver.restore_admitted_membership(&candidate, &admitted.membership_sha256)?; + assert_eq!(fs::read(&config)?, admitted_bytes); + assert!(driver.observe_membership(&candidate, &admitted.membership_sha256)?); + + fs::write(&config, b"foreign route after admission\n")?; + fs::write(systemctl.with_extension("fail"), b"fail\n")?; + assert!( + driver + .restore_admitted_membership(&candidate, &admitted.membership_sha256) + .is_err() + ); + assert!(!config.exists()); + fs::remove_file(systemctl.with_extension("fail"))?; + driver.restore_admitted_membership(&candidate, &admitted.membership_sha256)?; + assert_eq!(fs::read(&config)?, admitted_bytes); Ok(()) } @@ -5773,65 +6789,70 @@ mod tests { #[test] fn expected_withdrawal_is_exact_atomic_and_idempotent() -> Result<()> { let temp = tempfile::tempdir()?; - let (expected, activation, _) = authenticated_warming(temp.path())?; - let projection_store = temp.path().join("projection.cc"); - let driver = CultCacheTopologyDriver { - projection_store: projection_store.clone(), - correlation_store: temp.path().join("correlation.cc"), - }; - driver.publish_expected(&expected)?; - upsert_record( - &projection_store, - CultCacheEnvelope { - key: expected.target.clone(), - r#type: IdunnRuntimeActivationRecord::TYPE.into(), - payload: activation.canonical_bytes()?, - stored_at: "2026-09-03T00:00:00Z".into(), - schema_id: Some(IDUNN_RUNTIME_ACTIVATION_SCHEMA.into()), - }, - )?; - upsert_record( - &projection_store, - CultCacheEnvelope { - key: "unrelated".into(), - r#type: "test.unrelated".into(), - payload: vec![0x90], - stored_at: "2026-09-03T00:00:00Z".into(), - schema_id: Some("test.unrelated.v1".into()), - }, - )?; - - driver.withdraw_expected(&expected)?; - let remaining = SingleFileMessagePackBackingStore::new(&projection_store) + let (expected, activation, warming, provider_anchor) = authenticated_warming(temp.path())?; + let lease = lease(&expected, &activation, &warming); + let driver = topology_driver(temp.path(), "projection"); + driver.publish_expected(&expected, &provider_anchor)?; + let published = SingleFileMessagePackBackingStore::new(&driver.projection_store) .pull_all_read_only_snapshot()?; - assert_eq!(remaining.len(), 1); - assert_eq!(remaining[0].key, "unrelated"); - driver.withdraw_expected(&expected)?; + assert_eq!(published.len(), 2); + assert!(published.iter().any(|entry| { + entry.key == expected.target && entry.r#type == IdunnExpectedIncarnationRecord::TYPE + })); + let published_anchor = published + .iter() + .find(|entry| entry.r#type == GameCultServiceTrustAnchorRecord::TYPE) + .context("atomic Expected set has no provider trust anchor")?; + assert_eq!( + service_trust_anchor_from_envelope(published_anchor)?, + runtime_presence_trust_anchor(&expected, &provider_anchor)? + ); + project_activation(&driver, &expected, &activation)?; + driver.publish_process_write_lease(&expected, &activation, &lease)?; - let substituted_store = temp.path().join("substituted-expected.cc"); - let substituted_driver = CultCacheTopologyDriver { - projection_store: substituted_store.clone(), - correlation_store: temp.path().join("substituted-correlation.cc"), - }; - substituted_driver.publish_expected(&expected)?; let mut substituted_expected = expected.clone(); substituted_expected.incarnation_id = "another-incarnation".into(); + upsert_record( + &driver.projection_store, + CultCacheEnvelope { + key: expected.target.clone(), + r#type: IdunnExpectedIncarnationRecord::TYPE.into(), + payload: substituted_expected.canonical_bytes()?, + stored_at: "2026-09-03T00:00:00Z".into(), + schema_id: Some(IDUNN_EXPECTED_INCARNATION_SCHEMA.into()), + }, + )?; assert!( - substituted_driver - .withdraw_expected(&substituted_expected) + driver + .withdraw_expected(&expected, &provider_anchor, Some(&activation), Some(&lease)) .is_err() ); + driver.publish_expected(&expected, &provider_anchor)?; - let substituted_activation_store = temp.path().join("substituted-activation.cc"); - let substituted_activation_driver = CultCacheTopologyDriver { - projection_store: substituted_activation_store.clone(), - correlation_store: temp.path().join("substituted-activation-correlation.cc"), - }; - substituted_activation_driver.publish_expected(&expected)?; - let mut substituted_activation = activation; + let mut substituted_anchor = runtime_presence_trust_anchor(&expected, &provider_anchor)?; + substituted_anchor.runtime_id = "another-runtime".into(); + substituted_anchor.validate()?; + upsert_record( + &driver.projection_store, + CultCacheEnvelope { + key: substituted_anchor.trust_anchor_id.clone(), + r#type: GameCultServiceTrustAnchorRecord::TYPE.into(), + payload: rmp_serde::to_vec(&substituted_anchor)?, + stored_at: "2026-09-03T00:00:00Z".into(), + schema_id: Some(GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA.into()), + }, + )?; + assert!( + driver + .withdraw_expected(&expected, &provider_anchor, Some(&activation), Some(&lease)) + .is_err() + ); + driver.publish_expected(&expected, &provider_anchor)?; + + let mut substituted_activation = activation.clone(); substituted_activation.expected_projection_sha256 = digest('a'); upsert_record( - &substituted_activation_store, + &driver.projection_store, CultCacheEnvelope { key: expected.target.clone(), r#type: IdunnRuntimeActivationRecord::TYPE.into(), @@ -5841,10 +6862,199 @@ mod tests { }, )?; assert!( - substituted_activation_driver - .withdraw_expected(&expected) + driver + .withdraw_expected(&expected, &provider_anchor, Some(&activation), None) .is_err() ); + project_activation(&driver, &expected, &activation)?; + + let mut substituted_lease = lease.clone(); + substituted_lease.lease_epoch += 1; + assert!( + driver + .publish_process_write_lease(&expected, &activation, &substituted_lease) + .is_err() + ); + assert!( + driver + .withdraw_expected( + &expected, + &provider_anchor, + Some(&activation), + Some(&substituted_lease), + ) + .is_err() + ); + + upsert_record( + &driver.projection_store, + CultCacheEnvelope { + key: "unrelated".into(), + r#type: "test.unrelated".into(), + payload: vec![0x90], + stored_at: "2026-09-03T00:00:00Z".into(), + schema_id: Some("test.unrelated.v1".into()), + }, + )?; + driver.withdraw_expected(&expected, &provider_anchor, Some(&activation), Some(&lease))?; + let remaining = SingleFileMessagePackBackingStore::new(&driver.projection_store) + .pull_all_read_only_snapshot()?; + assert_eq!(remaining.len(), 1); + assert_eq!(remaining[0].key, "unrelated"); + driver.withdraw_expected(&expected, &provider_anchor, Some(&activation), Some(&lease))?; + Ok(()) + } + + #[test] + fn incumbent_projection_restoration_demotes_to_expected_only_and_rejects_unknown_authority() + -> Result<()> { + let temp = tempfile::tempdir()?; + let (incumbent, activation, warming, provider_anchor) = authenticated_warming(temp.path())?; + let lease = lease(&incumbent, &activation, &warming); + let driver = topology_driver(temp.path(), "restoration"); + driver.publish_expected(&incumbent, &provider_anchor)?; + project_activation(&driver, &incumbent, &activation)?; + driver.publish_process_write_lease(&incumbent, &activation, &lease)?; + + let mut candidate = incumbent.clone(); + candidate.plan_id = digest('a'); + candidate.incarnation_id = "candidate-incarnation".into(); + candidate.sealed_release_id = digest('b'); + candidate.artifact_sha256 = digest('c'); + let mut candidate_activation = activation.clone(); + candidate_activation.expected_projection_sha256 = candidate.canonical_sha256()?; + candidate_activation.runtime_instance_id = digest('d'); + driver.publish_expected(&candidate, &provider_anchor)?; + project_activation(&driver, &candidate, &candidate_activation)?; + + upsert_record( + &driver.projection_store, + CultCacheEnvelope { + key: "unrelated".into(), + r#type: "test.unrelated".into(), + payload: vec![0x90], + stored_at: "2026-09-03T00:00:00Z".into(), + schema_id: Some("test.unrelated.v1".into()), + }, + )?; + + assert_eq!( + driver.restore_admitted_expected_only( + &candidate, + &provider_anchor, + Some(&candidate_activation), + &incumbent, + &provider_anchor, + &activation, + Some(&lease), + )?, + incumbent.canonical_sha256()? + ); + + let restored = SingleFileMessagePackBackingStore::new(&driver.projection_store) + .pull_all_read_only_snapshot()?; + assert_eq!(restored.len(), 3); + assert_eq!( + IdunnExpectedIncarnationRecord::decode_canonical( + &restored + .iter() + .find(|entry| entry.r#type == IdunnExpectedIncarnationRecord::TYPE) + .unwrap() + .payload, + )?, + incumbent + ); + assert_eq!( + service_trust_anchor_from_envelope( + restored + .iter() + .find(|entry| entry.r#type == GameCultServiceTrustAnchorRecord::TYPE) + .unwrap(), + )?, + runtime_presence_trust_anchor(&incumbent, &provider_anchor)? + ); + assert!( + restored + .iter() + .all(|entry| entry.r#type != IdunnRuntimeActivationRecord::TYPE + && entry.r#type != IdunnProcessWriteLeaseRecord::TYPE) + ); + assert!( + restored + .iter() + .any(|entry| entry.key == "unrelated" && entry.r#type == "test.unrelated") + ); + let idempotent_bytes = fs::read(&driver.projection_store)?; + driver.restore_admitted_expected_only( + &candidate, + &provider_anchor, + Some(&candidate_activation), + &incumbent, + &provider_anchor, + &activation, + Some(&lease), + )?; + assert_eq!(fs::read(&driver.projection_store)?, idempotent_bytes); + + let admitted_anchor = runtime_presence_trust_anchor(&incumbent, &provider_anchor)?; + let mut unknown_expected = incumbent.clone(); + unknown_expected.incarnation_id = "unknown-incarnation".into(); + let mut unknown_anchor = admitted_anchor.clone(); + unknown_anchor.runtime_id = "unknown-runtime".into(); + unknown_anchor.validate()?; + let mut unknown_activation = activation.clone(); + unknown_activation.runtime_instance_id = digest('e'); + let mut unknown_lease = lease.clone(); + unknown_lease.lease_epoch += 1; + let substitutions = [ + CultCacheEnvelope { + key: incumbent.target.clone(), + r#type: IdunnExpectedIncarnationRecord::TYPE.into(), + payload: unknown_expected.canonical_bytes()?, + stored_at: "2026-09-03T00:00:00Z".into(), + schema_id: Some(IDUNN_EXPECTED_INCARNATION_SCHEMA.into()), + }, + CultCacheEnvelope { + key: admitted_anchor.trust_anchor_id.clone(), + r#type: GameCultServiceTrustAnchorRecord::TYPE.into(), + payload: rmp_serde::to_vec(&unknown_anchor)?, + stored_at: "2026-09-03T00:00:00Z".into(), + schema_id: Some(GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA.into()), + }, + CultCacheEnvelope { + key: incumbent.target.clone(), + r#type: IdunnRuntimeActivationRecord::TYPE.into(), + payload: unknown_activation.canonical_bytes()?, + stored_at: "2026-09-03T00:00:00Z".into(), + schema_id: Some(IDUNN_RUNTIME_ACTIVATION_SCHEMA.into()), + }, + CultCacheEnvelope { + key: incumbent.target.clone(), + r#type: IdunnProcessWriteLeaseRecord::TYPE.into(), + payload: unknown_lease.canonical_bytes()?, + stored_at: "2026-09-03T00:00:00Z".into(), + schema_id: Some(IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into()), + }, + ]; + for substitution in substitutions { + fs::write(&driver.projection_store, &idempotent_bytes)?; + upsert_record(&driver.projection_store, substitution)?; + let before = fs::read(&driver.projection_store)?; + assert!( + driver + .restore_admitted_expected_only( + &candidate, + &provider_anchor, + Some(&candidate_activation), + &incumbent, + &provider_anchor, + &activation, + Some(&lease), + ) + .is_err() + ); + assert_eq!(fs::read(&driver.projection_store)?, before); + } Ok(()) } @@ -5886,19 +7096,75 @@ mod tests { let temp = tempfile::tempdir()?; let path = temp.path().join("lease.cc"); let driver = CultCacheWriteLeaseDriver::new("service", &path); - let (expected, activation, warming) = authenticated_warming(temp.path())?; + let (expected, activation, warming, provider_anchor) = authenticated_warming(temp.path())?; let lease = lease(&expected, &activation, &warming); + let topology = topology_driver(temp.path(), "lease-projection"); + topology.publish_expected(&expected, &provider_anchor)?; + project_activation(&topology, &expected, &activation)?; + let lease_sha256 = driver.grant(&expected, &activation, &warming, &lease)?; assert_eq!(lease_sha256, lease.canonical_sha256()?); assert!(driver.observe(&expected, &activation, &warming, &lease)?); + assert!( + !SingleFileMessagePackBackingStore::new(&topology.projection_store) + .pull_all_read_only_snapshot()? + .iter() + .any(|entry| entry.r#type == IdunnProcessWriteLeaseRecord::TYPE) + ); + assert_eq!( + topology.publish_process_write_lease(&expected, &activation, &lease)?, + lease_sha256 + ); let mut surprise = lease.clone(); surprise.lease_epoch = 2; assert!(driver.revoke_exact(Some(&surprise)).is_err()); assert!(driver.observe(&expected, &activation, &warming, &lease)?); + SingleFileMessagePackBackingStore::new(&path).with_read_only_shared_snapshot(|_| { + let denial = driver + .revoke_exact(Some(&lease)) + .expect_err("shared lifetime holder must deny without blocking"); + assert!( + denial + .to_string() + .contains("held by a shared-lock consumer") + ); + Ok(()) + })?; + assert!(driver.observe(&expected, &activation, &warming, &lease)?); + driver.revoke_exact(Some(&lease))?; assert!(driver.observe_empty()?); + topology.withdraw_process_write_lease(&expected, &activation, Some(&lease))?; + assert!( + !SingleFileMessagePackBackingStore::new(&topology.projection_store) + .pull_all_read_only_snapshot()? + .iter() + .any(|entry| entry.r#type == IdunnProcessWriteLeaseRecord::TYPE) + ); + topology.withdraw_process_write_lease(&expected, &activation, Some(&lease))?; + driver.revoke_exact(Some(&lease))?; + + SingleFileMessagePackBackingStore::new(&path).with_read_only_shared_snapshot( + |snapshot| { + assert!(snapshot.is_empty()); + let denial = driver + .grant(&expected, &activation, &warming, &lease) + .expect_err("shared lifetime holder must deny grant without blocking"); + assert!( + denial + .to_string() + .contains("held by a shared-lock consumer") + ); + Ok(()) + }, + )?; + assert!(driver.observe_empty()?); + assert_eq!( + driver.grant(&expected, &activation, &warming, &lease)?, + lease.canonical_sha256()? + ); driver.revoke_exact(Some(&lease))?; Ok(()) } diff --git a/crates/idunn-daemon/src/provisioning.rs b/crates/idunn-daemon/src/provisioning.rs index 33612b8..1487768 100644 --- a/crates/idunn-daemon/src/provisioning.rs +++ b/crates/idunn-daemon/src/provisioning.rs @@ -5,10 +5,12 @@ use cultcache_rs::{ use cultnet_rs::{ GameCultProviderHealthIdentity, IDUNN_DEPLOYMENT_BRAKE_AUTHORITY, IDUNN_DEPLOYMENT_BRAKE_ID, IDUNN_DEPLOYMENT_BRAKE_SCHEMA, IDUNN_DEPLOYMENT_BRAKE_SCOPE, IDUNN_DEPLOYMENT_RELEASE_PURPOSE, + IDUNN_LIFECYCLE_BRAKE_AUTHORITY, IDUNN_LIFECYCLE_BRAKE_SCHEMA, IDUNN_LIFECYCLE_BRAKE_SCOPE, IdunnDeploymentBrakeObservation, IdunnDeploymentBrakeOperatorIdentity, - IdunnDeploymentBrakeRecord, IdunnDeploymentBrakeReleasePurpose, IdunnServiceIdentity, - ServiceIdentityProfile, ServiceIdentityTrustAnchor, derive_service_identity_id, - enroll_service_identity_at, evaluate_idunn_deployment_brake, + IdunnDeploymentBrakeRecord, IdunnDeploymentBrakeReleasePurpose, IdunnLifecycleBrakeObservation, + IdunnLifecycleBrakeRecord, IdunnServiceIdentity, OdinTopologyIdentity, ServiceIdentityProfile, + ServiceIdentityTrustAnchor, derive_service_identity_id, enroll_service_identity_at, + evaluate_idunn_continuity_restart, evaluate_idunn_deployment_brake, export_service_identity_trust_anchor, open_service_identity_at, }; use odin_core::{ @@ -49,39 +51,34 @@ pub fn run(args: impl IntoIterator) -> Result<()> { let public_key = provider_health_public_key(&options)?; println!("{public_key}"); } - "enroll-idunn-identity" => { - require_only(&options, &["private-store"])?; - enroll_service_identity_at::(&path(&options, "private-store")?)?; - } + "export-provider-health-public-anchor" => export_identity_anchor::< + GameCultProviderHealthIdentity, + >( + &options, "provider health public anchor" + )?, + "enroll-idunn-identity" => enroll_identity::(&options)?, "export-idunn-public-anchor" => { - require_only(&options, &["private-store", "public-anchor"])?; - let private = path(&options, "private-store")?; - let public = path(&options, "public-anchor")?; - reject_alias(&private, &public)?; - refuse_existing(&public, "public anchor")?; - let signer = open_service_identity_at::(&private)?; - export_service_identity_trust_anchor(&signer, &public)?; + export_identity_anchor::(&options, "Idunn public anchor")? + } + "enroll-odin-topology-identity" => enroll_identity::(&options)?, + "export-odin-topology-public-anchor" => { + export_identity_anchor::(&options, "Odin topology public anchor")? } "enroll-deployment-brake-operator" => { - require_only(&options, &["private-store"])?; - enroll_service_identity_at::(&path( - &options, - "private-store", - )?)?; + enroll_identity::(&options)? } "export-deployment-brake-operator-anchor" => { - require_only(&options, &["private-store", "public-anchor"])?; - let private = path(&options, "private-store")?; - let public = path(&options, "public-anchor")?; - reject_alias(&private, &public)?; - refuse_existing(&public, "deployment brake operator anchor")?; - let signer = - open_service_identity_at::(&private)?; - export_service_identity_trust_anchor(&signer, &public)?; + export_identity_anchor::( + &options, + "deployment brake operator anchor", + )? } "deployment-brake-engage" => deployment_brake_engage(&options)?, "deployment-brake-release" => deployment_brake_release(&options)?, "deployment-brake-status" => deployment_brake_status(&options)?, + "lifecycle-brake-engage" => lifecycle_brake_engage(&options)?, + "lifecycle-brake-release" => lifecycle_brake_release(&options)?, + "lifecycle-brake-status" => lifecycle_brake_status(&options)?, "create-daemon-health-trust-binding" => create_health_binding(&options)?, "add-daemon-health-trust-binding" => add_health_binding(&options)?, "require-daemon-health-release-binding" => require_health_binding_release(&options)?, @@ -97,6 +94,26 @@ pub fn run(args: impl IntoIterator) -> Result<()> { Ok(()) } +fn enroll_identity(options: &BTreeMap) -> Result<()> { + require_only(options, &["private-store"])?; + enroll_service_identity_at::

(&path(options, "private-store")?)?; + Ok(()) +} + +fn export_identity_anchor( + options: &BTreeMap, + label: &str, +) -> Result<()> { + require_only(options, &["private-store", "public-anchor"])?; + let private = path(options, "private-store")?; + let public = path(options, "public-anchor")?; + reject_alias(&private, &public)?; + refuse_existing(&public, label)?; + let signer = open_service_identity_at::

(&private)?; + export_service_identity_trust_anchor(&signer, &public)?; + Ok(()) +} + fn enroll_provider_health_identity(options: &BTreeMap) -> Result { require_only(options, &["private-store"])?; let signer = enroll_service_identity_at::(&path( @@ -262,6 +279,184 @@ fn deployment_brake_status(options: &BTreeMap) -> Result<()> { Ok(()) } +fn lifecycle_brake_engage(options: &BTreeMap) -> Result<()> { + require_only( + options, + &[ + "store", + "runtime-id", + "target", + "reason", + "updated-at-unix-millis", + ], + )?; + let updated = parse_u64(options, "updated-at-unix-millis")?; + let record = IdunnLifecycleBrakeRecord { + schema_version: IDUNN_LIFECYCLE_BRAKE_SCHEMA.into(), + authority: IDUNN_LIFECYCLE_BRAKE_AUTHORITY.into(), + runtime_id: required(options, "runtime-id")?.into(), + target: required(options, "target")?.into(), + scope: IDUNN_LIFECYCLE_BRAKE_SCOPE.into(), + status: "engaged".into(), + reason: required(options, "reason")?.into(), + updated_at_unix_millis: updated, + released_until_unix_millis: None, + }; + replace_lifecycle_brake(&path(options, "store")?, record) +} + +fn lifecycle_brake_release(options: &BTreeMap) -> Result<()> { + allow_only( + options, + &[ + "store", + "runtime-id", + "target", + "reason", + "updated-at-unix-millis", + "released-until-unix-millis", + ], + )?; + let updated = parse_u64(options, "updated-at-unix-millis")?; + let record = IdunnLifecycleBrakeRecord { + schema_version: IDUNN_LIFECYCLE_BRAKE_SCHEMA.into(), + authority: IDUNN_LIFECYCLE_BRAKE_AUTHORITY.into(), + runtime_id: required(options, "runtime-id")?.into(), + target: required(options, "target")?.into(), + scope: IDUNN_LIFECYCLE_BRAKE_SCOPE.into(), + status: "released".into(), + reason: required(options, "reason")?.into(), + updated_at_unix_millis: updated, + released_until_unix_millis: options + .get("released-until-unix-millis") + .map(|value| { + value + .parse() + .context("--released-until-unix-millis must be u64") + }) + .transpose()?, + }; + replace_lifecycle_brake(&path(options, "store")?, record) +} + +fn lifecycle_brake_status(options: &BTreeMap) -> Result<()> { + require_only( + options, + &["store", "runtime-id", "target", "now-unix-millis"], + )?; + let runtime_id = lifecycle_identifier(options, "runtime-id")?; + let target = lifecycle_identifier(options, "target")?; + let now = parse_u64(options, "now-unix-millis")?; + match read_lifecycle_brake(&path(options, "store")?)? { + None => { + evaluate_idunn_continuity_restart( + IdunnLifecycleBrakeObservation::Missing, + runtime_id, + target, + now, + ) + .map_err(|denial| anyhow!("lifecycle brake denied continuity restart: {denial:?}"))?; + println!("allowed runtime={runtime_id} target={target} brake=absent"); + } + Some(record) => { + evaluate_idunn_continuity_restart( + IdunnLifecycleBrakeObservation::Present(&record), + runtime_id, + target, + now, + ) + .map_err(|denial| anyhow!("lifecycle brake denied continuity restart: {denial:?}"))?; + if let Some(expires) = record.released_until_unix_millis { + println!( + "allowed runtime={} target={} brake=released expires={expires}", + record.runtime_id, record.target + ); + } else { + println!( + "allowed runtime={} target={} brake=released expires=never", + record.runtime_id, record.target + ); + } + } + } + Ok(()) +} + +fn replace_lifecycle_brake(path: &Path, record: IdunnLifecycleBrakeRecord) -> Result<()> { + record.validate()?; + let store = SingleFileMessagePackBackingStore::new(path); + let entries = store.pull_all_read_only_snapshot()?; + let current = match entries.as_slice() { + [] => None, + [entry] + if entry.r#type == IdunnLifecycleBrakeRecord::TYPE + && entry.schema_id.as_deref() == Some(IDUNN_LIFECYCLE_BRAKE_SCHEMA) => + { + let current_record = IdunnLifecycleBrakeRecord::decode_canonical(&entry.payload)?; + if entry.key != current_record.target { + bail!("lifecycle brake envelope key does not match its target"); + } + if current_record.runtime_id != record.runtime_id + || current_record.target != record.target + { + bail!("lifecycle brake store is bound to another runtime or target"); + } + if record.updated_at_unix_millis <= current_record.updated_at_unix_millis { + bail!("lifecycle brake update time must advance"); + } + Some(entry.clone()) + } + _ => bail!("lifecycle brake store is corrupt or ambiguous"), + }; + let next = typed_envelope( + &record.target, + &record, + IDUNN_LIFECYCLE_BRAKE_SCHEMA, + record.updated_at_unix_millis, + )?; + if !store.compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: IdunnLifecycleBrakeRecord::TYPE.into(), + key: record.target.clone(), + current, + }], + &[next], + )? { + bail!("lifecycle brake changed during transition"); + } + Ok(()) +} + +fn read_lifecycle_brake(path: &Path) -> Result> { + if !path.exists() { + return Ok(None); + } + let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; + let entry = match entries.as_slice() { + [] => return Ok(None), + [entry] => entry, + _ => bail!("lifecycle brake store is ambiguous"), + }; + if entry.r#type != IdunnLifecycleBrakeRecord::TYPE + || entry.schema_id.as_deref() != Some(IDUNN_LIFECYCLE_BRAKE_SCHEMA) + { + bail!("lifecycle brake store contains a foreign record"); + } + let record = IdunnLifecycleBrakeRecord::decode_canonical(&entry.payload)?; + if entry.key != record.target { + bail!("lifecycle brake envelope key does not match its target"); + } + Ok(Some(record)) +} + +fn lifecycle_identifier<'a>(options: &'a BTreeMap, name: &str) -> Result<&'a str> { + let value = required(options, name)?; + if value.trim().is_empty() || value.len() > 256 || value.chars().any(char::is_control) { + bail!("--{name} is empty, oversized, or contains control characters"); + } + Ok(value) +} + fn replace_brake(path: &Path, record: IdunnDeploymentBrakeRecord, millis: u64) -> Result<()> { let store = SingleFileMessagePackBackingStore::new(path); let entries = store.pull_all_read_only_snapshot()?; @@ -907,7 +1102,7 @@ fn normalized(path: &Path) -> Result { } fn usage() -> &'static str { - "Usage: idunn-provision enroll-provider-health-identity --private-store \n idunn-provision provider-health-public-key --private-store \n idunn-provision enroll-idunn-identity --private-store \n idunn-provision export-idunn-public-anchor --private-store --public-anchor \n idunn-provision enroll-deployment-brake-operator --private-store \n idunn-provision export-deployment-brake-operator-anchor --private-store --public-anchor \n idunn-provision deployment-brake-engage --store --runtime-id --owner --reason --observed-at-unix-millis \n idunn-provision deployment-brake-release --store --private-store --runtime-id --owner --reason --authorization-id --release-id --deployment-id --issued-at-unix-millis --expires-at-unix-millis \n idunn-provision deployment-brake-status --store --operator-anchor --runtime-id [--release-id --deployment-id --now-unix-millis ]\n idunn-provision create-daemon-health-trust-binding --output --binding-id --daemon --health-contract --source-runtime --signer-public-key-hex --bound-at-unix-millis --release-binding-required \n idunn-provision add-daemon-health-trust-binding --output --binding-id --daemon --health-contract --source-runtime --signer-public-key-hex --bound-at-unix-millis --release-binding-required \n idunn-provision require-daemon-health-release-binding --store --binding-id --daemon --health-contract --source-runtime --signer-public-key-hex \n idunn-provision rotate-daemon-health-trust-signer --output --binding-id --signer-public-key-hex --bound-at-unix-millis \n idunn-provision validate-daemon-health-trust-binding --input \n idunn-provision create-provider-projection-trust-anchor --output --trust-anchor-id --runtime-id --idunn-public-anchor --bound-at-unix-millis --expires-at-unix-millis \n idunn-provision validate-provider-projection-trust-anchor --input --idunn-public-anchor " + "Usage: idunn-provision enroll-provider-health-identity --private-store \n idunn-provision provider-health-public-key --private-store \n idunn-provision export-provider-health-public-anchor --private-store --public-anchor \n idunn-provision enroll-idunn-identity --private-store \n idunn-provision export-idunn-public-anchor --private-store --public-anchor \n idunn-provision enroll-odin-topology-identity --private-store \n idunn-provision export-odin-topology-public-anchor --private-store --public-anchor \n idunn-provision enroll-deployment-brake-operator --private-store \n idunn-provision export-deployment-brake-operator-anchor --private-store --public-anchor \n idunn-provision deployment-brake-engage --store --runtime-id --owner --reason --observed-at-unix-millis \n idunn-provision deployment-brake-release --store --private-store --runtime-id --owner --reason --authorization-id --release-id --deployment-id --issued-at-unix-millis --expires-at-unix-millis \n idunn-provision deployment-brake-status --store --operator-anchor --runtime-id [--release-id --deployment-id --now-unix-millis ]\n idunn-provision lifecycle-brake-engage --store --runtime-id --target --reason --updated-at-unix-millis \n idunn-provision lifecycle-brake-release --store --runtime-id --target --reason --updated-at-unix-millis [--released-until-unix-millis ]\n idunn-provision lifecycle-brake-status --store --runtime-id --target --now-unix-millis \n idunn-provision create-daemon-health-trust-binding --output --binding-id --daemon --health-contract --source-runtime --signer-public-key-hex --bound-at-unix-millis --release-binding-required \n idunn-provision add-daemon-health-trust-binding --output --binding-id --daemon --health-contract --source-runtime --signer-public-key-hex --bound-at-unix-millis --release-binding-required \n idunn-provision require-daemon-health-release-binding --store --binding-id --daemon --health-contract --source-runtime --signer-public-key-hex \n idunn-provision rotate-daemon-health-trust-signer --output --binding-id --signer-public-key-hex --bound-at-unix-millis \n idunn-provision validate-daemon-health-trust-binding --input \n idunn-provision create-provider-projection-trust-anchor --output --trust-anchor-id --runtime-id --idunn-public-anchor --bound-at-unix-millis --expires-at-unix-millis \n idunn-provision validate-provider-projection-trust-anchor --input --idunn-public-anchor " } #[cfg(test)] @@ -1476,6 +1671,144 @@ mod tests { Ok(()) } + #[test] + fn provider_and_odin_anchor_exports_are_profile_exact_immutable_and_non_aliasing() -> Result<()> + { + let temp = TempDir::new()?; + let provider_private = temp.path().join("provider-private.cc"); + let provider_anchor = temp.path().join("provider-anchor.cc"); + invoke(&[ + "enroll-provider-health-identity", + "--private-store", + provider_private.to_str().unwrap(), + ])?; + assert!( + invoke(&[ + "export-provider-health-public-anchor", + "--private-store", + provider_private.to_str().unwrap(), + "--public-anchor", + provider_private.to_str().unwrap(), + ]) + .is_err() + ); + invoke(&[ + "export-provider-health-public-anchor", + "--private-store", + provider_private.to_str().unwrap(), + "--public-anchor", + provider_anchor.to_str().unwrap(), + ])?; + let provider_entries = store_snapshot(&provider_anchor)?; + let [provider_envelope] = provider_entries.as_slice() else { + panic!("provider anchor store must contain exactly one envelope"); + }; + assert_eq!( + provider_envelope.r#type, + GameCultProviderHealthIdentity::TRUST_ANCHOR_TYPE + ); + assert_eq!( + provider_envelope.key, + GameCultProviderHealthIdentity::TRUST_ANCHOR_KEY + ); + assert_eq!( + provider_envelope.schema_id.as_deref(), + Some(GameCultProviderHealthIdentity::TRUST_ANCHOR_SCHEMA) + ); + let provider: ServiceIdentityTrustAnchor = + rmp_serde::from_slice(&provider_envelope.payload)?; + assert_eq!(rmp_serde::to_vec(&provider)?, provider_envelope.payload); + assert_eq!( + provider.identity_id, + derive_service_identity_id::(&provider.public_key)? + ); + assert!(open_service_identity_at::(&provider_private).is_err()); + let provider_before = std::fs::read(&provider_anchor)?; + assert!( + invoke(&[ + "export-provider-health-public-anchor", + "--private-store", + provider_private.to_str().unwrap(), + "--public-anchor", + provider_anchor.to_str().unwrap(), + ]) + .is_err() + ); + assert_eq!(std::fs::read(&provider_anchor)?, provider_before); + + let odin_private = temp.path().join("odin-private.cc"); + let odin_anchor = temp.path().join("odin-anchor.cc"); + invoke(&[ + "enroll-odin-topology-identity", + "--private-store", + odin_private.to_str().unwrap(), + ])?; + assert!(open_service_identity_at::(&odin_private).is_err()); + assert!( + invoke(&[ + "export-odin-topology-public-anchor", + "--private-store", + odin_private.to_str().unwrap(), + "--public-anchor", + odin_private.to_str().unwrap(), + ]) + .is_err() + ); + invoke(&[ + "export-odin-topology-public-anchor", + "--private-store", + odin_private.to_str().unwrap(), + "--public-anchor", + odin_anchor.to_str().unwrap(), + ])?; + let odin_entries = store_snapshot(&odin_anchor)?; + let [odin_envelope] = odin_entries.as_slice() else { + panic!("Odin anchor store must contain exactly one envelope"); + }; + assert_eq!( + odin_envelope.r#type, + OdinTopologyIdentity::TRUST_ANCHOR_TYPE + ); + assert_eq!(odin_envelope.key, OdinTopologyIdentity::TRUST_ANCHOR_KEY); + assert_eq!( + odin_envelope.schema_id.as_deref(), + Some(OdinTopologyIdentity::TRUST_ANCHOR_SCHEMA) + ); + let odin: ServiceIdentityTrustAnchor = rmp_serde::from_slice(&odin_envelope.payload)?; + assert_eq!(rmp_serde::to_vec(&odin)?, odin_envelope.payload); + assert_eq!( + odin.identity_id, + derive_service_identity_id::(&odin.public_key)? + ); + assert_ne!(provider.identity_id, odin.identity_id); + let odin_before = std::fs::read(&odin_anchor)?; + assert!( + invoke(&[ + "export-odin-topology-public-anchor", + "--private-store", + odin_private.to_str().unwrap(), + "--public-anchor", + odin_anchor.to_str().unwrap(), + ]) + .is_err() + ); + assert_eq!(std::fs::read(&odin_anchor)?, odin_before); + + let wrong_profile_output = temp.path().join("wrong-profile.cc"); + assert!( + invoke(&[ + "export-provider-health-public-anchor", + "--private-store", + odin_private.to_str().unwrap(), + "--public-anchor", + wrong_profile_output.to_str().unwrap(), + ]) + .is_err() + ); + assert!(!wrong_profile_output.exists()); + Ok(()) + } + #[test] fn identity_enrollment_and_export_are_immutable_and_paths_cannot_alias() -> Result<()> { let temp = TempDir::new()?; @@ -1916,6 +2249,159 @@ mod tests { Ok(()) } + #[test] + fn lifecycle_brake_is_absence_allow_and_exact_to_runtime_target_and_expiry() -> Result<()> { + let temp = TempDir::new()?; + let missing = temp.path().join("missing-lifecycle.cc"); + invoke(&[ + "lifecycle-brake-status", + "--store", + missing.to_str().unwrap(), + "--runtime-id", + "idunn-yggdrasil", + "--target", + "ghostlight", + "--now-unix-millis", + "100", + ])?; + assert!(!missing.exists()); + + let store = temp.path().join("ghostlight-lifecycle.cc"); + invoke(&[ + "lifecycle-brake-engage", + "--store", + store.to_str().unwrap(), + "--runtime-id", + "idunn-yggdrasil", + "--target", + "ghostlight", + "--reason", + "operator maintenance", + "--updated-at-unix-millis", + "200", + ])?; + let engaged = read_lifecycle_brake(&store)?.context("lifecycle brake must exist")?; + assert_eq!(engaged.status, "engaged"); + assert_eq!(engaged.runtime_id, "idunn-yggdrasil"); + assert_eq!(engaged.target, "ghostlight"); + assert!(read_brake(&store).is_err()); + assert!( + invoke(&[ + "lifecycle-brake-status", + "--store", + store.to_str().unwrap(), + "--runtime-id", + "idunn-yggdrasil", + "--target", + "ghostlight", + "--now-unix-millis", + "201", + ]) + .is_err() + ); + + let engaged_bytes = std::fs::read(&store)?; + assert!( + invoke(&[ + "lifecycle-brake-release", + "--store", + store.to_str().unwrap(), + "--runtime-id", + "idunn-nightwing", + "--target", + "ghostlight", + "--reason", + "wrong runtime", + "--updated-at-unix-millis", + "300", + ]) + .is_err() + ); + assert!( + invoke(&[ + "lifecycle-brake-release", + "--store", + store.to_str().unwrap(), + "--runtime-id", + "idunn-yggdrasil", + "--target", + "odin", + "--reason", + "wrong target", + "--updated-at-unix-millis", + "300", + ]) + .is_err() + ); + assert_eq!(std::fs::read(&store)?, engaged_bytes); + + invoke(&[ + "lifecycle-brake-release", + "--store", + store.to_str().unwrap(), + "--runtime-id", + "idunn-yggdrasil", + "--target", + "ghostlight", + "--reason", + "bounded continuity window", + "--updated-at-unix-millis", + "300", + "--released-until-unix-millis", + "500", + ])?; + invoke(&[ + "lifecycle-brake-status", + "--store", + store.to_str().unwrap(), + "--runtime-id", + "idunn-yggdrasil", + "--target", + "ghostlight", + "--now-unix-millis", + "499", + ])?; + for (runtime, target, now) in [ + ("idunn-nightwing", "ghostlight", "499"), + ("idunn-yggdrasil", "odin", "499"), + ("idunn-yggdrasil", "ghostlight", "500"), + ] { + assert!( + invoke(&[ + "lifecycle-brake-status", + "--store", + store.to_str().unwrap(), + "--runtime-id", + runtime, + "--target", + target, + "--now-unix-millis", + now, + ]) + .is_err() + ); + } + let released_bytes = std::fs::read(&store)?; + assert!( + invoke(&[ + "lifecycle-brake-engage", + "--store", + store.to_str().unwrap(), + "--runtime-id", + "idunn-yggdrasil", + "--target", + "ghostlight", + "--reason", + "stale replay", + "--updated-at-unix-millis", + "300", + ]) + .is_err() + ); + assert_eq!(std::fs::read(&store)?, released_bytes); + Ok(()) + } + #[test] fn brake_compare_exchange_rejects_stale_transition_snapshot() -> Result<()> { let temp = TempDir::new()?; diff --git a/crates/odin-daemon/Cargo.toml b/crates/odin-daemon/Cargo.toml new file mode 100644 index 0000000..e50553d --- /dev/null +++ b/crates/odin-daemon/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "odin-daemon" +version = "0.1.0" +edition.workspace = true +license.workspace = true +publish.workspace = true + +[lib] +name = "odin_daemon" +path = "src/lib.rs" + +[dependencies] +anyhow = "1" +chrono = "0.4" +cultcache-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "cbde940df2f5ee0ccac50e8dbfe9f36ffbebb585", package = "cultcache-rs" } +cultmesh-rs = { path = "../../vendor/cultmesh-rs" } +cultnet-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "cbde940df2f5ee0ccac50e8dbfe9f36ffbebb585", package = "cultnet-rs" } +fs2 = "0.4" +rmp-serde = "1.3" +serde = "1" + +[dev-dependencies] +tempfile = "3" diff --git a/crates/odin-daemon/src/lib.rs b/crates/odin-daemon/src/lib.rs new file mode 100644 index 0000000..bfd38e6 --- /dev/null +++ b/crates/odin-daemon/src/lib.rs @@ -0,0 +1,1832 @@ +//! Deterministic Odin authority for runtime-topology correlation. +//! +//! This library owns no listener or lifecycle loop. Callers provide Idunn's +//! read-only projection, a trusted receive time with any service-owned +//! presence, and narrow clock/store/signer implementations. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, Result, bail, ensure}; +use chrono::{DateTime, SecondsFormat, Utc}; +use cultcache_rs::{ + CultCacheEnvelope, CultCacheExpectedEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, +}; +use cultnet_rs::{ + GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, GAMECULT_RUNTIME_PRESENCE_HEALTH_SIGNING_PURPOSE, + GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA, GameCultRuntimePresenceHealthRecord, + GameCultServiceTrustAnchorRecord, IDUNN_EXPECTED_INCARNATION_SCHEMA, + IDUNN_PROCESS_WRITE_LEASE_SCHEMA, IDUNN_RUNTIME_ACTIVATION_SCHEMA, IdunnExpectedDependency, + IdunnExpectedIncarnationRecord, IdunnProcessWriteLeaseRecord, IdunnRuntimeActivationPurpose, + IdunnRuntimeActivationRecord, IdunnServiceIdentity, ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA, + OdinRuntimeDependencyEvidence, OdinRuntimeTopologyCorrelationPurpose, + OdinRuntimeTopologyCorrelationRecord, OdinTopologyAuthenticationContext, + OdinTopologyDisagreement, OdinTopologyIdentity, RuntimePresenceAuthenticationContext, + ServiceIdentitySignature, ServiceIdentitySigner, ServiceIdentityTrustAnchor, + authenticate_odin_runtime_topology_correlation, authenticate_runtime_presence_claim, + correlate_runtime_presence_claim, verify_runtime_authority, verify_service_identity_signature, + verify_service_identity_signature_with_public_key, +}; + +const CAS_ATTEMPTS: usize = 8; + +#[derive(Clone, Copy, Debug)] +pub struct AuthenticationPolicy { + pub presence_maximum_age_millis: u64, + pub presence_maximum_future_skew_millis: u64, + pub correlation_maximum_age_millis: u64, + pub correlation_maximum_future_skew_millis: u64, +} + +impl Default for AuthenticationPolicy { + fn default() -> Self { + Self { + presence_maximum_age_millis: 30_000, + presence_maximum_future_skew_millis: 5_000, + correlation_maximum_age_millis: 30_000, + correlation_maximum_future_skew_millis: 5_000, + } + } +} + +pub trait Clock { + fn now_unix_millis(&self) -> Result; +} + +#[derive(Clone, Copy, Debug, Default)] +pub struct SystemClock; + +impl Clock for SystemClock { + fn now_unix_millis(&self) -> Result { + Ok(SystemTime::now() + .duration_since(UNIX_EPOCH) + .context("system clock predates the Unix epoch")? + .as_millis() + .try_into() + .context("system clock does not fit the topology timestamp")?) + } +} + +/// One exact Idunn-owned projection. The service anchor is provider lookup +/// material selected by Expected; it is not runtime observation. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct IdunnRuntimeProjection { + pub expected: IdunnExpectedIncarnationRecord, + pub provider_anchor: Option, + pub activation: Option, + pub current_lease: Option, +} + +impl IdunnRuntimeProjection { + pub fn validate(&self) -> Result<()> { + self.expected.validate()?; + if let Some(anchor) = &self.provider_anchor { + anchor.validate()?; + } + if let Some(activation) = &self.activation { + activation.validate()?; + } + if let Some(lease) = &self.current_lease { + lease.validate()?; + } + Ok(()) + } +} + +pub trait IdunnProjectionSource { + fn current_projection(&self, target: &str) -> Result>; +} + +/// Read-only adapter for the atomic projection file published by Idunn. +pub struct CultCacheIdunnProjectionSource { + path: PathBuf, +} + +impl CultCacheIdunnProjectionSource { + pub fn new(path: impl Into) -> Self { + Self { path: path.into() } + } + + pub fn path(&self) -> &Path { + &self.path + } +} + +impl IdunnProjectionSource for CultCacheIdunnProjectionSource { + fn current_projection(&self, target: &str) -> Result> { + if !self.path.is_file() { + return Ok(None); + } + let entries = + SingleFileMessagePackBackingStore::new(&self.path).pull_all_read_only_snapshot()?; + let expected_envelope = + unique_envelope(&entries, IdunnExpectedIncarnationRecord::TYPE, target)?; + let anchor_key = runtime_presence_anchor_id(target); + let anchor_envelope = unique_envelope( + &entries, + GameCultServiceTrustAnchorRecord::TYPE, + &anchor_key, + )?; + let activation_envelope = + unique_envelope(&entries, IdunnRuntimeActivationRecord::TYPE, target)?; + let lease_envelope = unique_envelope(&entries, IdunnProcessWriteLeaseRecord::TYPE, target)?; + + let Some(expected_envelope) = expected_envelope else { + ensure!( + anchor_envelope.is_none() + && activation_envelope.is_none() + && lease_envelope.is_none(), + "Idunn projection has runtime authority without Expected" + ); + return Ok(None); + }; + ensure_schema(expected_envelope, IDUNN_EXPECTED_INCARNATION_SCHEMA)?; + let expected = + IdunnExpectedIncarnationRecord::decode_canonical(&expected_envelope.payload)?; + ensure!( + expected.target == target, + "Expected projection key is substituted" + ); + + let provider_anchor = anchor_envelope + .map(|envelope| { + ensure_schema(envelope, GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA)?; + decode_canonical(&envelope.payload, "provider trust anchor") + }) + .transpose()?; + + let activation = activation_envelope + .map(|envelope| { + ensure_schema(envelope, IDUNN_RUNTIME_ACTIVATION_SCHEMA)?; + IdunnRuntimeActivationRecord::decode_canonical(&envelope.payload) + }) + .transpose()?; + + let current_lease = lease_envelope + .map(|envelope| { + ensure_schema(envelope, IDUNN_PROCESS_WRITE_LEASE_SCHEMA)?; + IdunnProcessWriteLeaseRecord::decode_canonical(&envelope.payload) + }) + .transpose()?; + let projection = IdunnRuntimeProjection { + expected, + provider_anchor, + activation, + current_lease, + }; + projection.validate()?; + Ok(Some(projection)) + } +} + +pub trait OdinCorrelationSigner { + fn identity_id(&self) -> &str; + fn public_key(&self) -> &[u8]; + fn sign_correlation(&self, unsigned_payload: &[u8]) -> Result>; +} + +impl OdinCorrelationSigner for ServiceIdentitySigner { + fn identity_id(&self) -> &str { + &self.entry().identity_id + } + + fn public_key(&self) -> &[u8] { + &self.entry().public_key + } + + fn sign_correlation(&self, unsigned_payload: &[u8]) -> Result> { + Ok(self + .sign::(unsigned_payload) + .signature) + } +} + +impl OdinCorrelationSigner for &T { + fn identity_id(&self) -> &str { + (*self).identity_id() + } + + fn public_key(&self) -> &[u8] { + (*self).public_key() + } + + fn sign_correlation(&self, unsigned_payload: &[u8]) -> Result> { + (*self).sign_correlation(unsigned_payload) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AdmittedPresence { + pub canonical_bytes: Vec, + pub trusted_received_at_unix_millis: u64, + pub stored_at: String, +} + +impl AdmittedPresence { + fn new(canonical_bytes: Vec, trusted_received_at_unix_millis: u64) -> Result { + Ok(Self { + canonical_bytes, + trusted_received_at_unix_millis, + stored_at: rfc3339_millis(trusted_received_at_unix_millis)?, + }) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StoredCorrelation { + pub canonical_bytes: Vec, + pub stored_at: String, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct OdinStoreQuery { + pub target: String, + pub provider_signer_identity_id: String, + pub dependency_targets: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct OdinStoreSnapshot { + pub query: OdinStoreQuery, + pub presence: Option, + pub correlation: Option, + pub dependency_correlations: BTreeMap>, +} + +pub trait OdinTopologyStore { + fn read(&self, query: OdinStoreQuery) -> Result; + + fn compare_exchange( + &self, + observed: &OdinStoreSnapshot, + replacement_presence: Option<&AdmittedPresence>, + replacement_correlation: &StoredCorrelation, + ) -> Result; + + fn withdraw_correlation(&self, target: &str) -> Result<()>; +} + +/// Odin's durable replay and correlation store. It persists existing CultNet +/// documents directly instead of inventing a parallel admission schema. +pub struct CultCacheOdinTopologyStore { + path: PathBuf, +} + +impl CultCacheOdinTopologyStore { + pub fn new(path: impl Into) -> Self { + Self { path: path.into() } + } + + pub fn path(&self) -> &Path { + &self.path + } +} + +impl OdinTopologyStore for CultCacheOdinTopologyStore { + fn read(&self, mut query: OdinStoreQuery) -> Result { + query.dependency_targets.sort(); + query.dependency_targets.dedup(); + let entries = if self.path.is_file() { + SingleFileMessagePackBackingStore::new(&self.path).pull_all_read_only_snapshot()? + } else { + Vec::new() + }; + let presence_key = presence_store_key(&query.target, &query.provider_signer_identity_id); + let presence = unique_envelope( + &entries, + GameCultRuntimePresenceHealthRecord::TYPE, + &presence_key, + )? + .map(decode_presence_envelope) + .transpose()?; + let correlation = unique_envelope( + &entries, + OdinRuntimeTopologyCorrelationRecord::TYPE, + &query.target, + )? + .map(decode_correlation_envelope) + .transpose()?; + let mut dependency_correlations = BTreeMap::new(); + for dependency_target in &query.dependency_targets { + let current = unique_envelope( + &entries, + OdinRuntimeTopologyCorrelationRecord::TYPE, + dependency_target, + )? + .map(decode_correlation_envelope) + .transpose()?; + dependency_correlations.insert(dependency_target.clone(), current); + } + Ok(OdinStoreSnapshot { + query, + presence, + correlation, + dependency_correlations, + }) + } + + fn compare_exchange( + &self, + observed: &OdinStoreSnapshot, + replacement_presence: Option<&AdmittedPresence>, + replacement_correlation: &StoredCorrelation, + ) -> Result { + let presence_key = presence_store_key( + &observed.query.target, + &observed.query.provider_signer_identity_id, + ); + let mut conditions = BTreeMap::new(); + insert_condition( + &mut conditions, + CultCacheExpectedEnvelope { + key: presence_key.clone(), + r#type: GameCultRuntimePresenceHealthRecord::TYPE.into(), + current: observed + .presence + .as_ref() + .map(|presence| presence_envelope(&presence_key, presence)), + }, + )?; + insert_condition( + &mut conditions, + CultCacheExpectedEnvelope { + key: observed.query.target.clone(), + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + current: observed + .correlation + .as_ref() + .map(|correlation| correlation_envelope(&observed.query.target, correlation)), + }, + )?; + for (target, correlation) in &observed.dependency_correlations { + insert_condition( + &mut conditions, + CultCacheExpectedEnvelope { + key: target.clone(), + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + current: correlation + .as_ref() + .map(|correlation| correlation_envelope(target, correlation)), + }, + )?; + } + let mut replacements = vec![correlation_envelope( + &observed.query.target, + replacement_correlation, + )]; + if let Some(presence) = replacement_presence { + replacements.push(presence_envelope(&presence_key, presence)); + } + if let Some(parent) = self.path.parent() { + std::fs::create_dir_all(parent)?; + } + SingleFileMessagePackBackingStore::new(&self.path) + .compare_exchange(&conditions.into_values().collect::>(), &replacements) + } + + fn withdraw_correlation(&self, target: &str) -> Result<()> { + for _ in 0..CAS_ATTEMPTS { + if !self.path.is_file() { + return Ok(()); + } + let entries = + SingleFileMessagePackBackingStore::new(&self.path).pull_all_read_only_snapshot()?; + let current = + unique_envelope(&entries, OdinRuntimeTopologyCorrelationRecord::TYPE, target)?; + let Some(current) = current else { + return Ok(()); + }; + let replacement = entries + .iter() + .filter(|entry| *entry != current) + .cloned() + .collect::>(); + if SingleFileMessagePackBackingStore::new(&self.path) + .compare_exchange_snapshot(&entries, &replacement)? + { + return Ok(()); + } + } + bail!("Odin topology store changed repeatedly while withdrawing correlation") + } +} + +pub struct OdinTopologyAuthority { + projections: P, + store: S, + signer: K, + clock: C, + idunn_anchor: ServiceIdentityTrustAnchor, + policy: AuthenticationPolicy, +} + +impl OdinTopologyAuthority +where + P: IdunnProjectionSource, + S: OdinTopologyStore, + K: OdinCorrelationSigner, + C: Clock, +{ + pub fn new( + projections: P, + store: S, + signer: K, + clock: C, + idunn_anchor: ServiceIdentityTrustAnchor, + policy: AuthenticationPolicy, + ) -> Self { + Self { + projections, + store, + signer, + clock, + idunn_anchor, + policy, + } + } + + /// Correlate current Expected with any previously admitted raw observation. + /// With no admitted observation this can only produce Expected, never Present. + pub fn refresh(&self, target: &str) -> Result>> { + let Some(projection) = self.projections.current_projection(target)? else { + self.store.withdraw_correlation(target)?; + return Ok(None); + }; + self.reconcile(projection, None).map(Some) + } + + /// Authenticate and monotonically admit one exact provider-owned presence. + pub fn admit_presence( + &self, + target: &str, + canonical_presence: &[u8], + trusted_received_at_unix_millis: u64, + ) -> Result> { + let projection = self + .projections + .current_projection(target)? + .context("runtime presence has no current Expected projection")?; + self.reconcile( + projection, + Some((canonical_presence, trusted_received_at_unix_millis)), + ) + } + + /// Return the exact signed bytes admitted by Odin for transport. + pub fn current_signed_correlation(&self, target: &str) -> Result>> { + self.refresh(target) + } + + fn reconcile( + &self, + projection: IdunnRuntimeProjection, + incoming: Option<(&[u8], u64)>, + ) -> Result> { + projection.validate()?; + let now = self.clock.now_unix_millis()?; + let (authority, projection_disagreements) = + classify_runtime_authority(&projection, &self.idunn_anchor, now)?; + let (lease_sha256, lease_disagreement) = classify_current_lease(&projection)?; + let query = OdinStoreQuery { + target: projection.expected.target.clone(), + provider_signer_identity_id: projection.expected.expected_signer_identity_id.clone(), + dependency_targets: managed_dependency_targets(&projection.expected.dependencies), + }; + + for _ in 0..CAS_ATTEMPTS { + let observed = self.store.read(query.clone())?; + let admitted = self.select_presence(&observed, incoming, authority.as_ref())?; + let dependencies = + self.dependency_evidence(&projection.expected.dependencies, &observed, now)?; + let (presence, mut disagreements, replacement_presence) = match admitted { + SelectedPresence::None => (None, projection_disagreements.clone(), None), + SelectedPresence::Stale(stored) => { + let record = decode_presence(&stored.canonical_bytes)?; + let mut disagreements = projection_disagreements.clone(); + disagreements.push(OdinTopologyDisagreement { + code: "stored-presence-not-current".into(), + expected: projection + .activation + .as_ref() + .map(IdunnRuntimeActivationRecord::canonical_sha256) + .transpose()?, + observed: Some(record.activation_witness_sha256), + }); + (None, disagreements, None) + } + SelectedPresence::Current { + correlation, + replacement, + } => { + let mut disagreements = projection_disagreements.clone(); + disagreements.extend_from_slice(correlation.disagreements()); + (Some(correlation), disagreements, replacement) + } + }; + if let Some(disagreement) = &lease_disagreement { + disagreements.push(disagreement.clone()); + } + if presence.is_none() { + disagreements.push(OdinTopologyDisagreement { + code: "signed-presence-missing".into(), + expected: Some("dual-authenticated-runtime-presence".into()), + observed: None, + }); + disagreements.extend(projection.expected.capabilities.iter().enumerate().map( + |(index, capability)| OdinTopologyDisagreement { + code: format!("expected-capability-{index:03}-missing"), + expected: Some(format!( + "{}/{}/{} capacity>={}", + capability.capability, + capability.schema, + capability.compatibility, + capability.minimum_capacity + )), + observed: None, + }, + )); + } + + let observed_write_lease = if let Some(presence) = &presence { + correlate_write_lease( + &projection.expected, + lease_sha256.as_deref(), + presence.claim().record().write_lease_sha256.as_deref(), + &presence.claim().record().state, + &mut disagreements, + ) + } else { + None + }; + disagreements.sort_by(|left, right| left.code.cmp(&right.code)); + + let prior = observed + .correlation + .as_ref() + .map(|stored| decode_correlation(&stored.canonical_bytes)) + .transpose()?; + let publisher_sequence = prior.as_ref().map_or(Ok(1), |record| { + record + .publisher_sequence + .checked_add(1) + .context("Odin topology publisher sequence exhausted") + })?; + let presence_record = presence.as_ref().map(|value| value.claim().record()); + let ready = presence_record.is_some_and(|record| record.state == "active") + && disagreements.is_empty() + && dependencies + .iter() + .all(|dependency| dependency.kind == "optional" || dependency.ready); + let mut correlation = OdinRuntimeTopologyCorrelationRecord { + schema_version: ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into(), + target: projection.expected.target.clone(), + expected_projection_sha256: projection.expected.canonical_sha256()?, + expected: true, + current_activation_sha256: projection + .activation + .as_ref() + .map(IdunnRuntimeActivationRecord::canonical_sha256) + .transpose()?, + signed_presence_sha256: presence + .as_ref() + .map(|value| value.claim().signed_presence_sha256().into()), + observed_presence_state: presence_record.map(|record| record.state.clone()), + observed_presence_publisher_sequence: presence_record + .map(|record| record.publisher_sequence), + observed_write_lease_sha256: observed_write_lease, + observed_capabilities: presence_record + .map_or_else(Vec::new, |record| record.capabilities.clone()), + runtime_id: projection.expected.runtime_id.clone(), + runtime_instance_id: projection + .activation + .as_ref() + .map(|activation| activation.runtime_instance_id.clone()), + present: presence.is_some(), + ready, + dependencies, + disagreements, + signer_identity_id: self.signer.identity_id().into(), + publisher_sequence, + observed_at_unix_millis: now, + signature_algorithm: "ed25519".into(), + signature: Vec::new(), + }; + if let Some(prior) = &prior + && same_correlation_facts(prior, &correlation) + { + return Ok(observed + .correlation + .context("decoded prior correlation disappeared")? + .canonical_bytes); + } + correlation.signature = self + .signer + .sign_correlation(&correlation.unsigned_signature_payload()?)?; + let canonical = correlation.canonical_bytes()?; + correlation.validate_against_expected(&projection.expected, lease_sha256.as_deref())?; + let (signed, unsigned) = + OdinRuntimeTopologyCorrelationRecord::decode_canonical_signed_payload(&canonical)?; + verify_service_identity_signature_with_public_key::< + OdinTopologyIdentity, + OdinRuntimeTopologyCorrelationPurpose, + >( + self.signer.public_key(), + &unsigned, + &ServiceIdentitySignature { + identity_id: signed.signer_identity_id, + signature: signed.signature, + }, + )?; + if let Some(authority) = &authority { + authenticate_odin_runtime_topology_correlation( + &canonical, + authority, + lease_sha256.as_deref(), + self.signer.public_key(), + OdinTopologyAuthenticationContext { + trusted_received_at_unix_millis: now, + maximum_age_millis: self.policy.correlation_maximum_age_millis, + maximum_future_skew_millis: self + .policy + .correlation_maximum_future_skew_millis, + }, + )?; + } + let stored_correlation = StoredCorrelation { + canonical_bytes: canonical.clone(), + stored_at: rfc3339_millis(now)?, + }; + if self.store.compare_exchange( + &observed, + replacement_presence.as_ref(), + &stored_correlation, + )? { + return Ok(canonical); + } + } + bail!("Odin topology store changed repeatedly during correlation") + } + + fn select_presence( + &self, + observed: &OdinStoreSnapshot, + incoming: Option<(&[u8], u64)>, + authority: Option<&cultnet_rs::VerifiedRuntimeAuthority>, + ) -> Result { + let stored_record = observed + .presence + .as_ref() + .map(|stored| decode_presence(&stored.canonical_bytes)) + .transpose()?; + let Some(authority) = authority else { + ensure!( + incoming.is_none(), + "runtime presence has no exact current activation and provider anchor" + ); + return Ok(observed + .presence + .as_ref() + .map_or(SelectedPresence::None, |stored| { + SelectedPresence::Stale(stored.clone()) + })); + }; + let selected = if let Some((bytes, received_at)) = incoming { + if let Some(stored_presence) = observed + .presence + .as_ref() + .filter(|stored| stored.canonical_bytes == bytes) + { + let stored_claim = authenticate_runtime_presence_claim( + &stored_presence.canonical_bytes, + authority, + self.presence_context(stored_presence.trusted_received_at_unix_millis), + )?; + return Ok(SelectedPresence::Current { + correlation: correlate_runtime_presence_claim(stored_claim, authority)?, + replacement: None, + }); + } + let claim = authenticate_runtime_presence_claim( + bytes, + authority, + self.presence_context(received_at), + )?; + if let Some(stored) = &stored_record { + if claim.record().publisher_sequence < stored.publisher_sequence { + bail!("runtime presence publisher sequence was reordered"); + } + if claim.record().publisher_sequence == stored.publisher_sequence { + bail!("runtime presence publisher sequence was reused with different bytes"); + } + } + let replacement = AdmittedPresence::new(bytes.to_vec(), received_at)?; + Some((claim, Some(replacement))) + } else if let Some(stored) = &observed.presence { + match authenticate_runtime_presence_claim( + &stored.canonical_bytes, + authority, + self.presence_context(stored.trusted_received_at_unix_millis), + ) { + Ok(claim) => Some((claim, None)), + Err(error) => { + let prior_matches_current = + observed.correlation.as_ref().is_some_and(|value| { + decode_correlation(&value.canonical_bytes).is_ok_and(|record| { + record.expected_projection_sha256 == authority.expected_sha256() + && record.current_activation_sha256.as_deref() + == Some(authority.activation_sha256()) + }) + }); + if prior_matches_current { + return Err(error).context("stored current presence failed authentication"); + } + return Ok(SelectedPresence::Stale(stored.clone())); + } + } + } else { + None + }; + match selected { + Some((claim, replacement)) => Ok(SelectedPresence::Current { + correlation: correlate_runtime_presence_claim(claim, authority)?, + replacement, + }), + None => Ok(SelectedPresence::None), + } + } + + fn dependency_evidence( + &self, + requirements: &[IdunnExpectedDependency], + observed: &OdinStoreSnapshot, + now: u64, + ) -> Result> { + requirements + .iter() + .map(|requirement| { + let mut evidence = OdinRuntimeDependencyEvidence { + kind: requirement.kind.clone(), + capability: requirement.capability.clone(), + schema: requirement.schema.clone(), + compatibility: requirement.compatibility.clone(), + provider_id: requirement.provider_id.clone(), + provider_authority: requirement.provider_authority.clone(), + provider_expected_projection_sha256: requirement + .provider_expected_projection_sha256 + .clone(), + provider_endpoint: requirement.provider_endpoint.clone(), + observed_capacity: None, + provider_evidence_sha256: None, + ready: false, + }; + let Some(provider_id) = requirement.provider_id.as_deref() else { + return Ok(evidence); + }; + if requirement.provider_authority.as_deref() != Some("managed-incarnation") { + return Ok(evidence); + } + let Some(provider_projection) = self.projections.current_projection(provider_id)? + else { + return Ok(evidence); + }; + provider_projection.validate()?; + if provider_projection.expected.canonical_sha256()? + != requirement + .provider_expected_projection_sha256 + .as_deref() + .unwrap_or_default() + { + return Ok(evidence); + } + if let Some(endpoint) = requirement.provider_endpoint.as_deref() + && provider_projection + .expected + .route + .as_ref() + .map(|route| route.stable_endpoint.as_str()) + != Some(endpoint) + { + return Ok(evidence); + } + let (provider_authority, authority_disagreements) = + classify_runtime_authority(&provider_projection, &self.idunn_anchor, now)?; + let Some(provider_authority) = provider_authority else { + return Ok(evidence); + }; + if !authority_disagreements.is_empty() { + return Ok(evidence); + } + let Some(stored) = observed + .dependency_correlations + .get(provider_id) + .and_then(Option::as_ref) + else { + return Ok(evidence); + }; + let (current_lease_sha256, _) = classify_current_lease(&provider_projection)?; + let Ok(authenticated) = authenticate_odin_runtime_topology_correlation( + &stored.canonical_bytes, + &provider_authority, + current_lease_sha256.as_deref(), + self.signer.public_key(), + OdinTopologyAuthenticationContext { + trusted_received_at_unix_millis: now, + maximum_age_millis: self.policy.correlation_maximum_age_millis, + maximum_future_skew_millis: self + .policy + .correlation_maximum_future_skew_millis, + }, + ) else { + return Ok(evidence); + }; + let provider = authenticated.record(); + let capacity = provider + .observed_capabilities + .iter() + .find(|capability| { + capability.capability == requirement.capability + && capability.schema == requirement.schema + && capability.compatibility == requirement.compatibility + }) + .map(|capability| capability.capacity); + evidence.observed_capacity = capacity; + evidence.provider_evidence_sha256 = Some(provider.canonical_sha256()?); + evidence.ready = provider.ready + && capacity.is_some_and(|capacity| capacity >= requirement.minimum_capacity); + Ok(evidence) + }) + .collect() + } + + fn presence_context( + &self, + trusted_received_at_unix_millis: u64, + ) -> RuntimePresenceAuthenticationContext { + RuntimePresenceAuthenticationContext { + trusted_received_at_unix_millis, + maximum_age_millis: self.policy.presence_maximum_age_millis, + maximum_future_skew_millis: self.policy.presence_maximum_future_skew_millis, + } + } +} + +enum SelectedPresence { + None, + Stale(AdmittedPresence), + Current { + correlation: cultnet_rs::RuntimePresenceCorrelation, + replacement: Option, + }, +} + +fn correlate_write_lease( + expected: &IdunnExpectedIncarnationRecord, + current: Option<&str>, + observed: Option<&str>, + presence_state: &str, + disagreements: &mut Vec, +) -> Option { + if presence_state == "warming" && observed.is_none() { + return None; + } + if expected.write_lease_required && current.is_some() && current == observed { + return observed.map(str::to_string); + } + if expected.write_lease_required { + disagreements.push(OdinTopologyDisagreement { + code: "write-lease".into(), + expected: Some( + current + .unwrap_or("current-idunn-process-write-lease") + .into(), + ), + observed: observed.map(str::to_string), + }); + } else if observed.is_some() { + disagreements.push(OdinTopologyDisagreement { + code: "unexpected-write-lease".into(), + expected: None, + observed: observed.map(str::to_string), + }); + } + None +} + +fn same_correlation_facts( + prior: &OdinRuntimeTopologyCorrelationRecord, + next: &OdinRuntimeTopologyCorrelationRecord, +) -> bool { + let mut normalized = next.clone(); + normalized.publisher_sequence = prior.publisher_sequence; + normalized.observed_at_unix_millis = prior.observed_at_unix_millis; + normalized.signature = prior.signature.clone(); + &normalized == prior +} + +fn managed_dependency_targets(requirements: &[IdunnExpectedDependency]) -> Vec { + requirements + .iter() + .filter(|dependency| { + dependency.provider_authority.as_deref() == Some("managed-incarnation") + }) + .filter_map(|dependency| dependency.provider_id.clone()) + .collect() +} + +fn classify_runtime_authority( + projection: &IdunnRuntimeProjection, + idunn_anchor: &ServiceIdentityTrustAnchor, + now: u64, +) -> Result<( + Option, + Vec, +)> { + let expected = &projection.expected; + let expected_sha256 = expected.canonical_sha256()?; + let mut disagreements = Vec::new(); + match &projection.provider_anchor { + None => disagreements.push(OdinTopologyDisagreement { + code: "provider-trust-anchor-missing".into(), + expected: Some(expected.expected_signer_identity_id.clone()), + observed: None, + }), + Some(anchor) => { + push_disagreement( + &mut disagreements, + "provider-trust-anchor-id", + Some(runtime_presence_anchor_id(&expected.target)), + Some(anchor.trust_anchor_id.clone()), + ); + push_disagreement( + &mut disagreements, + "provider-service-id", + Some(expected.target.clone()), + Some(anchor.service_id.clone()), + ); + push_disagreement( + &mut disagreements, + "provider-runtime-id", + Some(expected.runtime_id.clone()), + Some(anchor.runtime_id.clone()), + ); + push_disagreement( + &mut disagreements, + "provider-signer-identity", + Some(expected.expected_signer_identity_id.clone()), + Some(anchor.signer_identity_id.clone()), + ); + push_disagreement( + &mut disagreements, + "provider-signing-purpose", + Some(GAMECULT_RUNTIME_PRESENCE_HEALTH_SIGNING_PURPOSE.into()), + Some(anchor.signing_purpose.clone()), + ); + push_disagreement( + &mut disagreements, + "provider-signed-schema", + Some(GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()), + Some(anchor.signed_schema.clone()), + ); + if anchor.bound_at_unix_millis > now + || anchor + .expires_at_unix_millis + .is_some_and(|expires_at| now >= expires_at) + { + disagreements.push(OdinTopologyDisagreement { + code: "provider-trust-anchor-time".into(), + expected: Some(format!("current-at:{now}")), + observed: Some(format!( + "bound:{};expires:{}", + anchor.bound_at_unix_millis, + anchor + .expires_at_unix_millis + .map_or_else(|| "none".into(), |value| value.to_string()) + )), + }); + } + } + } + match &projection.activation { + None => disagreements.push(OdinTopologyDisagreement { + code: "current-activation-missing".into(), + expected: Some(expected_sha256.clone()), + observed: None, + }), + Some(activation) => { + if activation.idunn_signer_identity_id != idunn_anchor.identity_id { + disagreements.push(OdinTopologyDisagreement { + code: "activation-idunn-signer".into(), + expected: Some(idunn_anchor.identity_id.clone()), + observed: Some(activation.idunn_signer_identity_id.clone()), + }); + } else { + verify_service_identity_signature::< + IdunnServiceIdentity, + IdunnRuntimeActivationPurpose, + >( + idunn_anchor, + &activation.unsigned_signature_payload()?, + &ServiceIdentitySignature { + identity_id: activation.idunn_signer_identity_id.clone(), + signature: activation.signature.clone(), + }, + ) + .context("verifying projected Idunn runtime activation")?; + } + push_disagreement( + &mut disagreements, + "activation-expected-projection", + Some(expected_sha256.clone()), + Some(activation.expected_projection_sha256.clone()), + ); + push_disagreement( + &mut disagreements, + "activation-runtime-id", + Some(expected.runtime_id.clone()), + Some(activation.runtime_id.clone()), + ); + } + } + disagreements.sort_by(|left, right| left.code.cmp(&right.code)); + let authority = if disagreements.is_empty() { + let anchor = projection.provider_anchor.as_ref().unwrap(); + let activation = projection.activation.as_ref().unwrap(); + Some(verify_runtime_authority( + expected, + activation, + idunn_anchor, + &anchor.signer_public_key, + )?) + } else { + None + }; + Ok((authority, disagreements)) +} + +fn classify_current_lease( + projection: &IdunnRuntimeProjection, +) -> Result<(Option, Option)> { + let Some(lease) = &projection.current_lease else { + return Ok((None, None)); + }; + let lease_sha256 = lease.canonical_sha256()?; + let activation = projection.activation.as_ref(); + let binds = activation + .map(|activation| lease_binds(&projection.expected, activation, lease)) + .transpose()? + .unwrap_or(false); + if projection.expected.write_lease_required && binds { + return Ok((Some(lease_sha256), None)); + } + let expected_activation = activation + .map(IdunnRuntimeActivationRecord::canonical_sha256) + .transpose()? + .unwrap_or_else(|| "none".into()); + Ok(( + None, + Some(OdinTopologyDisagreement { + code: "projected-write-lease".into(), + expected: Some(format!( + "expected:{};activation:{expected_activation}", + projection.expected.canonical_sha256()? + )), + observed: Some(lease_sha256), + }), + )) +} + +fn lease_binds( + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + lease: &IdunnProcessWriteLeaseRecord, +) -> Result { + Ok(lease.target == expected.target + && lease.expected_projection_sha256 == expected.canonical_sha256()? + && lease.plan_id == expected.plan_id + && lease.incarnation_id == expected.incarnation_id + && lease.sealed_release_id == expected.sealed_release_id + && lease.activation_witness_sha256 == activation.canonical_sha256()? + && Some(lease.state_schema_generation.as_str()) + == expected.state_schema_generation.as_deref() + && Some(lease.state_contract_sha256.as_str()) == expected.state_contract_sha256.as_deref() + && lease.runtime_id == expected.runtime_id + && lease.runtime_instance_id == activation.runtime_instance_id) +} + +fn push_disagreement( + disagreements: &mut Vec, + code: &str, + expected: Option, + observed: Option, +) { + if expected != observed { + disagreements.push(OdinTopologyDisagreement { + code: code.into(), + expected, + observed, + }); + } +} + +fn runtime_presence_anchor_id(target: &str) -> String { + format!("root/{target}/runtime-presence") +} + +fn presence_store_key(target: &str, signer_identity_id: &str) -> String { + format!("{target}/{signer_identity_id}") +} + +fn unique_envelope<'a>( + entries: &'a [CultCacheEnvelope], + record_type: &str, + key: &str, +) -> Result> { + let mut matches = entries + .iter() + .filter(|entry| entry.r#type == record_type && entry.key == key); + let current = matches.next(); + ensure!(matches.next().is_none(), "CultCache identity is ambiguous"); + Ok(current) +} + +fn ensure_schema(envelope: &CultCacheEnvelope, schema: &str) -> Result<()> { + ensure!( + envelope.schema_id.as_deref() == Some(schema), + "CultCache envelope schema is substituted" + ); + Ok(()) +} + +fn decode_canonical(bytes: &[u8], label: &str) -> Result +where + T: serde::de::DeserializeOwned + serde::Serialize, +{ + let value = rmp_serde::from_slice(bytes).with_context(|| format!("decoding {label}"))?; + ensure!( + rmp_serde::to_vec(&value)? == bytes, + "{label} is not canonical positional MessagePack" + ); + Ok(value) +} + +fn decode_presence(bytes: &[u8]) -> Result { + let record: GameCultRuntimePresenceHealthRecord = decode_canonical(bytes, "runtime presence")?; + record.validate()?; + Ok(record) +} + +fn decode_correlation(bytes: &[u8]) -> Result { + Ok(OdinRuntimeTopologyCorrelationRecord::decode_canonical_signed_payload(bytes)?.0) +} + +fn decode_presence_envelope(envelope: &CultCacheEnvelope) -> Result { + ensure_schema(envelope, GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA)?; + decode_presence(&envelope.payload)?; + Ok(AdmittedPresence { + canonical_bytes: envelope.payload.clone(), + trusted_received_at_unix_millis: parse_rfc3339_millis(&envelope.stored_at)?, + stored_at: envelope.stored_at.clone(), + }) +} + +fn decode_correlation_envelope(envelope: &CultCacheEnvelope) -> Result { + ensure_schema(envelope, ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA)?; + let record = decode_correlation(&envelope.payload)?; + ensure!( + record.target == envelope.key, + "stored correlation target is substituted" + ); + ensure!( + parse_rfc3339_millis(&envelope.stored_at)? == record.observed_at_unix_millis, + "stored correlation receipt time is substituted" + ); + Ok(StoredCorrelation { + canonical_bytes: envelope.payload.clone(), + stored_at: envelope.stored_at.clone(), + }) +} + +fn presence_envelope(key: &str, presence: &AdmittedPresence) -> CultCacheEnvelope { + CultCacheEnvelope { + key: key.into(), + r#type: GameCultRuntimePresenceHealthRecord::TYPE.into(), + payload: presence.canonical_bytes.clone(), + stored_at: presence.stored_at.clone(), + schema_id: Some(GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()), + } +} + +fn correlation_envelope(target: &str, correlation: &StoredCorrelation) -> CultCacheEnvelope { + CultCacheEnvelope { + key: target.into(), + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + payload: correlation.canonical_bytes.clone(), + stored_at: correlation.stored_at.clone(), + schema_id: Some(ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into()), + } +} + +fn insert_condition( + conditions: &mut BTreeMap<(String, String), CultCacheExpectedEnvelope>, + condition: CultCacheExpectedEnvelope, +) -> Result<()> { + let identity = (condition.r#type.clone(), condition.key.clone()); + if let Some(current) = conditions.get(&identity) { + ensure!( + current.current == condition.current, + "store read set is incoherent" + ); + } else { + conditions.insert(identity, condition); + } + Ok(()) +} + +fn rfc3339_millis(value: u64) -> Result { + let value: i64 = value + .try_into() + .context("timestamp exceeds RFC3339 range")?; + Ok(DateTime::::from_timestamp_millis(value) + .context("timestamp exceeds RFC3339 range")? + .to_rfc3339_opts(SecondsFormat::Millis, true)) +} + +fn parse_rfc3339_millis(value: &str) -> Result { + DateTime::parse_from_rfc3339(value)? + .timestamp_millis() + .try_into() + .context("CultCache timestamp predates Unix epoch") +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use cultnet_rs::{ + GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA, GameCultProviderHealthIdentity, + GameCultRuntimeCapability, GameCultRuntimePresenceHealthPurpose, IdunnExpectedCapability, + IdunnExpectedRoute, IdunnRuntimeActivationLaunch, IdunnRuntimeActivationSigner, + IdunnServiceIdentity, enroll_service_identity_at, + }; + use tempfile::TempDir; + + use super::*; + + const NOW: u64 = 1_000_000; + + #[derive(Clone, Copy)] + struct FixedClock(u64); + + impl Clock for FixedClock { + fn now_unix_millis(&self) -> Result { + Ok(self.0) + } + } + + struct TestService { + projection: IdunnRuntimeProjection, + provider_signer: ServiceIdentitySigner, + activation_signer: IdunnRuntimeActivationSigner, + } + + impl TestService { + fn signed_presence(&self, sequence: u64, edit: F) -> Result> + where + F: FnOnce(&mut GameCultRuntimePresenceHealthRecord), + { + let expected = &self.projection.expected; + let activation = self.projection.activation.as_ref().unwrap(); + let mut record = GameCultRuntimePresenceHealthRecord { + schema_version: GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + target: expected.target.clone(), + expected_projection_sha256: expected.canonical_sha256()?, + plan_id: expected.plan_id.clone(), + incarnation_id: expected.incarnation_id.clone(), + sealed_release_id: expected.sealed_release_id.clone(), + activation_witness_sha256: activation.canonical_sha256()?, + state_schema_generation: expected.state_schema_generation.clone(), + state_contract_sha256: expected.state_contract_sha256.clone(), + runtime_id: expected.runtime_id.clone(), + runtime_instance_id: activation.runtime_instance_id.clone(), + bound_endpoint: expected + .route + .as_ref() + .map(|route| route.candidate_endpoint.clone()), + capabilities: expected + .capabilities + .iter() + .map(|capability| GameCultRuntimeCapability { + capability: capability.capability.clone(), + schema: capability.schema.clone(), + compatibility: capability.compatibility.clone(), + capacity: capability.minimum_capacity, + }) + .collect(), + health_contract: expected.health_contract.clone(), + state: "active".into(), + detail: "ready".into(), + write_lease_sha256: self + .projection + .current_lease + .as_ref() + .map(IdunnProcessWriteLeaseRecord::canonical_sha256) + .transpose()?, + signer_identity_id: self.provider_signer.entry().identity_id.clone(), + publisher_sequence: sequence, + observed_at_unix_millis: NOW - 10, + signature_algorithm: "ed25519".into(), + signature: Vec::new(), + activation_signer_identity_id: activation.activation_signer_identity_id.clone(), + activation_signature: Vec::new(), + }; + edit(&mut record); + let proof_payload = record.canonical_proof_payload()?; + record.signature = self + .provider_signer + .sign::(&proof_payload) + .signature; + record.activation_signature = self.activation_signer.sign_presence_proof(&record)?; + record.validate()?; + Ok(rmp_serde::to_vec(&record)?) + } + } + + struct TestWorld { + _temp: TempDir, + projection_path: PathBuf, + topology_path: PathBuf, + identity_root: PathBuf, + idunn_signer: ServiceIdentitySigner, + idunn_anchor: ServiceIdentityTrustAnchor, + odin_signer: ServiceIdentitySigner, + } + + impl TestWorld { + fn new() -> Result { + let temp = tempfile::tempdir()?; + let identity_root = temp.path().join("identities"); + let idunn_signer = enroll_service_identity_at::( + &identity_root.join("idunn.cc"), + )?; + let idunn_anchor = idunn_signer.trust_anchor()?; + let odin_signer = + enroll_service_identity_at::(&identity_root.join("odin.cc"))?; + Ok(Self { + projection_path: temp.path().join("idunn-projection.cc"), + topology_path: temp.path().join("odin-topology.cc"), + identity_root, + idunn_signer, + idunn_anchor, + odin_signer, + _temp: temp, + }) + } + + fn service( + &self, + target: &str, + dependencies: Vec, + stateful: bool, + ) -> Result { + let provider_signer = enroll_service_identity_at::( + &self.identity_root.join(format!("{target}-provider.cc")), + )?; + let expected = IdunnExpectedIncarnationRecord { + schema_version: IDUNN_EXPECTED_INCARNATION_SCHEMA.into(), + target: target.into(), + plan_id: digest('1'), + incarnation_id: format!("{target}/generation-1"), + sealed_release_id: digest('2'), + source_repository: format!("github.com/GameCult/{target}"), + source_revision: "3".repeat(40), + recipe_sha256: digest('4'), + runtime_id: format!("{target}-runtime"), + expected_signer_identity_id: provider_signer.entry().identity_id.clone(), + health_contract: format!("{target}.runtime-health.v1"), + artifact_sha256: digest('5'), + state_schema_generation: stateful.then(|| "state-v1".into()), + state_contract_sha256: stateful.then(|| digest('6')), + write_lease_required: stateful, + route: Some(IdunnExpectedRoute { + route_id: format!("{target}-route"), + transport: "tcp".into(), + stable_endpoint: format!("tcp://{target}.internal:1000"), + candidate_endpoint: format!("tcp://127.0.0.1:{}", target.len() + 10_000), + }), + capabilities: vec![IdunnExpectedCapability { + capability: "data".into(), + schema: "data.v1".into(), + compatibility: "v1".into(), + minimum_capacity: 1, + }], + dependencies, + }; + expected.validate()?; + let launch = IdunnRuntimeActivationLaunch::issue( + &expected, + digest('7'), + NOW - 20, + &self.idunn_signer, + )?; + let activation = launch.activation().clone(); + let mut credential = Vec::new(); + let emitted = launch.write_credential(&mut credential)?; + ensure!( + emitted == activation, + "activation launch changed while emitted" + ); + let activation_signer = + IdunnRuntimeActivationSigner::from_credential_reader(credential.as_slice())?; + let provider_anchor = GameCultServiceTrustAnchorRecord { + schema_version: GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA.into(), + trust_anchor_id: runtime_presence_anchor_id(target), + service_id: target.into(), + runtime_id: expected.runtime_id.clone(), + signer_identity_id: provider_signer.entry().identity_id.clone(), + signer_public_key: provider_signer.entry().public_key.clone(), + signature_algorithm: "ed25519".into(), + signing_purpose: GAMECULT_RUNTIME_PRESENCE_HEALTH_SIGNING_PURPOSE.into(), + signed_schema: GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + binding_authority: "root".into(), + bound_at_unix_millis: NOW - 100, + expires_at_unix_millis: None, + private_state_exposed: false, + }; + let current_lease = if stateful { + Some(IdunnProcessWriteLeaseRecord { + schema_version: IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into(), + target: target.into(), + expected_projection_sha256: expected.canonical_sha256()?, + plan_id: expected.plan_id.clone(), + incarnation_id: expected.incarnation_id.clone(), + sealed_release_id: expected.sealed_release_id.clone(), + activation_witness_sha256: activation.canonical_sha256()?, + state_schema_generation: expected.state_schema_generation.clone().unwrap(), + state_contract_sha256: expected.state_contract_sha256.clone().unwrap(), + runtime_id: expected.runtime_id.clone(), + runtime_instance_id: activation.runtime_instance_id.clone(), + warming_presence_sha256: digest('8'), + lease_epoch: 1, + issued_at_unix_millis: NOW - 5, + }) + } else { + None + }; + let projection = IdunnRuntimeProjection { + expected, + provider_anchor: Some(provider_anchor), + activation: Some(activation), + current_lease, + }; + projection.validate()?; + self.publish_projection(&projection)?; + Ok(TestService { + projection, + provider_signer, + activation_signer, + }) + } + + fn publish_projection(&self, projection: &IdunnRuntimeProjection) -> Result<()> { + let store = SingleFileMessagePackBackingStore::new(&self.projection_path); + let current = if self.projection_path.is_file() { + store.pull_all_read_only_snapshot()? + } else { + Vec::new() + }; + let anchor_key = runtime_presence_anchor_id(&projection.expected.target); + let mut next: Vec<_> = current + .iter() + .filter(|envelope| { + !((envelope.r#type == IdunnExpectedIncarnationRecord::TYPE + || envelope.r#type == IdunnRuntimeActivationRecord::TYPE + || envelope.r#type == IdunnProcessWriteLeaseRecord::TYPE) + && envelope.key == projection.expected.target) + && !(envelope.r#type == GameCultServiceTrustAnchorRecord::TYPE + && envelope.key == anchor_key) + }) + .cloned() + .collect(); + next.push(CultCacheEnvelope { + key: projection.expected.target.clone(), + r#type: IdunnExpectedIncarnationRecord::TYPE.into(), + payload: projection.expected.canonical_bytes()?, + stored_at: rfc3339_millis(NOW - 30)?, + schema_id: Some(IDUNN_EXPECTED_INCARNATION_SCHEMA.into()), + }); + if let Some(anchor) = &projection.provider_anchor { + next.push(CultCacheEnvelope { + key: anchor_key, + r#type: GameCultServiceTrustAnchorRecord::TYPE.into(), + payload: rmp_serde::to_vec(anchor)?, + stored_at: rfc3339_millis(anchor.bound_at_unix_millis)?, + schema_id: Some(GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA.into()), + }); + } + if let Some(activation) = &projection.activation { + next.push(CultCacheEnvelope { + key: projection.expected.target.clone(), + r#type: IdunnRuntimeActivationRecord::TYPE.into(), + payload: activation.canonical_bytes()?, + stored_at: rfc3339_millis(activation.issued_at_unix_millis)?, + schema_id: Some(IDUNN_RUNTIME_ACTIVATION_SCHEMA.into()), + }); + } + if let Some(lease) = &projection.current_lease { + next.push(CultCacheEnvelope { + key: projection.expected.target.clone(), + r#type: IdunnProcessWriteLeaseRecord::TYPE.into(), + payload: lease.canonical_bytes()?, + stored_at: rfc3339_millis(lease.issued_at_unix_millis)?, + schema_id: Some(IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into()), + }); + } + ensure!( + store.compare_exchange_snapshot(¤t, &next)?, + "test projection CAS failed" + ); + Ok(()) + } + + fn engine( + &self, + now: u64, + ) -> OdinTopologyAuthority< + CultCacheIdunnProjectionSource, + CultCacheOdinTopologyStore, + &ServiceIdentitySigner, + FixedClock, + > { + OdinTopologyAuthority::new( + CultCacheIdunnProjectionSource::new(&self.projection_path), + CultCacheOdinTopologyStore::new(&self.topology_path), + &self.odin_signer, + FixedClock(now), + self.idunn_anchor.clone(), + AuthenticationPolicy::default(), + ) + } + + fn replace_provider_generation(&self, service: &mut TestService) -> Result<()> { + service.projection.expected.plan_id = digest('9'); + service.projection.expected.incarnation_id = + format!("{}/generation-2", service.projection.expected.target); + service.projection.expected.validate()?; + let launch = IdunnRuntimeActivationLaunch::issue( + &service.projection.expected, + digest('a'), + NOW + 10, + &self.idunn_signer, + )?; + service.projection.activation = Some(launch.activation().clone()); + service.projection.current_lease = None; + self.publish_projection(&service.projection) + } + } + + fn digest(byte: char) -> String { + format!("sha256-{}", byte.to_string().repeat(64)) + } + + fn decode_signed(bytes: &[u8]) -> Result { + Ok(OdinRuntimeTopologyCorrelationRecord::decode_canonical_signed_payload(bytes)?.0) + } + + #[test] + fn partial_or_mismatched_projection_remains_signed_expected_only() -> Result<()> { + let world = TestWorld::new()?; + let mut service = world.service("ghostlight", Vec::new(), false)?; + service.projection.activation = None; + let anchor = service.projection.provider_anchor.as_mut().unwrap(); + anchor.trust_anchor_id = "root/other/runtime-presence".into(); + anchor.service_id = "other".into(); + world.publish_projection(&service.projection)?; + + let bytes = world.engine(NOW).refresh("ghostlight")?.unwrap(); + let record = decode_signed(&bytes)?; + assert!(record.expected); + assert!(!record.present); + assert!(!record.ready); + assert!(record.signed_presence_sha256.is_none()); + let codes: Vec<_> = record + .disagreements + .iter() + .map(|value| value.code.as_str()) + .collect(); + assert!(codes.contains(&"current-activation-missing")); + assert!(codes.contains(&"provider-service-id")); + assert!(codes.contains(&"provider-trust-anchor-id")); + assert!(codes.contains(&"signed-presence-missing")); + Ok(()) + } + + #[test] + fn authenticated_disagreement_is_present_but_not_ready() -> Result<()> { + let world = TestWorld::new()?; + let service = world.service("ghostlight", Vec::new(), false)?; + let presence = service.signed_presence(1, |record| { + record.health_contract = "ghostlight.other-health.v1".into(); + })?; + + let record = decode_signed(&world.engine(NOW).admit_presence( + "ghostlight", + &presence, + NOW, + )?)?; + assert!(record.present); + assert!(!record.ready); + assert!( + record + .disagreements + .iter() + .any(|value| value.code == "health-contract") + ); + Ok(()) + } + + #[test] + fn exact_active_presence_and_current_lease_are_ready() -> Result<()> { + let world = TestWorld::new()?; + let service = world.service("ghostlight", Vec::new(), true)?; + let presence = service.signed_presence(1, |_| {})?; + + let record = decode_signed(&world.engine(NOW).admit_presence( + "ghostlight", + &presence, + NOW, + )?)?; + assert!(record.present); + assert!(record.ready); + assert_eq!( + record.observed_write_lease_sha256, + service + .projection + .current_lease + .as_ref() + .map(IdunnProcessWriteLeaseRecord::canonical_sha256) + .transpose()? + ); + Ok(()) + } + + #[test] + fn stale_projected_lease_is_visible_while_candidate_warming_is_present() -> Result<()> { + let world = TestWorld::new()?; + let mut service = world.service("ghostlight", Vec::new(), true)?; + let stale = service.projection.current_lease.as_mut().unwrap(); + stale.expected_projection_sha256 = digest('a'); + stale.plan_id = digest('b'); + stale.incarnation_id = "ghostlight/incumbent".into(); + stale.sealed_release_id = digest('c'); + stale.activation_witness_sha256 = digest('d'); + stale.runtime_instance_id = digest('e'); + world.publish_projection(&service.projection)?; + let warming = service.signed_presence(1, |record| { + record.state = "warming".into(); + record.detail = "starting".into(); + record.write_lease_sha256 = None; + })?; + + let record = decode_signed(&world.engine(NOW).admit_presence( + "ghostlight", + &warming, + NOW, + )?)?; + assert!(record.present); + assert!(!record.ready); + assert_eq!(record.observed_presence_state.as_deref(), Some("warming")); + assert!(record.observed_write_lease_sha256.is_none()); + assert!( + record + .disagreements + .iter() + .any(|value| value.code == "projected-write-lease") + ); + Ok(()) + } + + #[test] + fn provider_sequence_admission_is_monotonic_and_exact_retry_is_idempotent() -> Result<()> { + let world = TestWorld::new()?; + let service = world.service("ghostlight", Vec::new(), false)?; + let sequence_two = service.signed_presence(2, |_| {})?; + let engine = world.engine(NOW); + + let first = engine.admit_presence("ghostlight", &sequence_two, NOW)?; + let duplicate = engine.admit_presence("ghostlight", &sequence_two, NOW + 60_000)?; + assert_eq!(duplicate, first); + assert_eq!( + engine.current_signed_correlation("ghostlight")?, + Some(first) + ); + + let lower = service.signed_presence(1, |_| {})?; + assert!(engine.admit_presence("ghostlight", &lower, NOW).is_err()); + let reused = service.signed_presence(2, |record| record.detail = "different".into())?; + assert!(engine.admit_presence("ghostlight", &reused, NOW).is_err()); + + let entries = SingleFileMessagePackBackingStore::new(&world.topology_path) + .pull_all_read_only_snapshot()?; + let stored = unique_envelope( + &entries, + GameCultRuntimePresenceHealthRecord::TYPE, + &presence_store_key( + "ghostlight", + &service.projection.expected.expected_signer_identity_id, + ), + )? + .unwrap(); + assert_eq!(stored.payload, sequence_two); + assert_eq!(parse_rfc3339_millis(&stored.stored_at)?, NOW); + Ok(()) + } + + #[test] + fn restart_rehydrates_exact_bytes_without_retimestamping() -> Result<()> { + let world = TestWorld::new()?; + let service = world.service("ghostlight", Vec::new(), false)?; + let presence = service.signed_presence(1, |_| {})?; + let first = world + .engine(NOW) + .admit_presence("ghostlight", &presence, NOW)?; + + let after_restart = world.engine(NOW + 100).refresh("ghostlight")?.unwrap(); + assert_eq!(after_restart, first); + assert_eq!(decode_signed(&after_restart)?.observed_at_unix_millis, NOW); + Ok(()) + } + + #[test] + fn withdrawn_expected_removes_current_correlation_but_preserves_presence_history() -> Result<()> + { + let world = TestWorld::new()?; + let service = world.service("ghostlight", Vec::new(), false)?; + let presence = service.signed_presence(1, |_| {})?; + let engine = world.engine(NOW); + engine.admit_presence("ghostlight", &presence, NOW)?; + assert!(engine.current_signed_correlation("ghostlight")?.is_some()); + + std::fs::remove_file(&world.projection_path)?; + assert!(engine.current_signed_correlation("ghostlight")?.is_none()); + + let entries = SingleFileMessagePackBackingStore::new(&world.topology_path) + .pull_all_read_only_snapshot()?; + assert!(entries.iter().any(|entry| { + entry.r#type == GameCultRuntimePresenceHealthRecord::TYPE && entry.payload == presence + })); + assert!(!entries.iter().any(|entry| { + entry.r#type == OdinRuntimeTopologyCorrelationRecord::TYPE && entry.key == "ghostlight" + })); + Ok(()) + } + + #[test] + fn dependencies_consume_only_exact_current_admitted_correlations() -> Result<()> { + let world = TestWorld::new()?; + let mut provider = world.service("provider", Vec::new(), false)?; + let provider_expected_sha256 = provider.projection.expected.canonical_sha256()?; + let provider_endpoint = provider + .projection + .expected + .route + .as_ref() + .unwrap() + .stable_endpoint + .clone(); + let requirement = IdunnExpectedDependency { + kind: "required".into(), + capability: "data".into(), + schema: "data.v1".into(), + compatibility: "v1".into(), + minimum_capacity: 1, + startup: "before-promotion".into(), + provider_id: Some("provider".into()), + provider_authority: Some("managed-incarnation".into()), + provider_expected_projection_sha256: Some(provider_expected_sha256), + provider_endpoint: Some(provider_endpoint), + }; + let consumer = world.service("consumer", vec![requirement], false)?; + let provider_presence = provider.signed_presence(1, |_| {})?; + let consumer_presence = consumer.signed_presence(1, |_| {})?; + let engine = world.engine(NOW); + + engine.admit_presence("provider", &provider_presence, NOW)?; + let ready = decode_signed(&engine.admit_presence("consumer", &consumer_presence, NOW)?)?; + assert!(ready.ready); + assert!(ready.dependencies[0].ready); + assert!(ready.dependencies[0].provider_evidence_sha256.is_some()); + + world.replace_provider_generation(&mut provider)?; + let no_longer_exact = decode_signed(&engine.refresh("consumer")?.unwrap())?; + assert!(no_longer_exact.present); + assert!(!no_longer_exact.ready); + assert!(!no_longer_exact.dependencies[0].ready); + assert!( + no_longer_exact.dependencies[0] + .provider_evidence_sha256 + .is_none() + ); + Ok(()) + } +} diff --git a/crates/odin-daemon/src/main.rs b/crates/odin-daemon/src/main.rs new file mode 100644 index 0000000..99094d3 --- /dev/null +++ b/crates/odin-daemon/src/main.rs @@ -0,0 +1,1248 @@ +use std::cell::RefCell; +use std::collections::{BTreeMap, BTreeSet, VecDeque}; +use std::fs::{File, OpenOptions}; +use std::net::{SocketAddr, UdpSocket}; +use std::os::fd::{FromRawFd, RawFd}; +use std::path::{Path, PathBuf}; +use std::rc::Rc; +use std::thread; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, Result, anyhow, bail, ensure}; +use chrono::{DateTime, Utc}; +use cultcache_rs::{ + CultCacheEnvelope, CultCacheExpectedEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, +}; +use cultmesh_rs::{ + CultMeshRudpDocumentPublishOptions, CultMeshRudpDocumentServer, + CultMeshRudpDocumentServerOptions, CultMeshRudpPollOutcome, CultMeshRudpRawDocumentReceipt, + CultMeshRudpRawDocumentSink, CultMeshRudpSnapshotQuery, CultMeshRudpSnapshotSource, + CultMeshSystemClock, publish_cultnet_message_to_rudp_catalog, +}; +use cultnet_rs::{ + CultNetMessage, CultNetRawDocumentRecord, CultNetRawPayloadEncoding, + GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, GameCultProviderHealthIdentity, + GameCultRuntimeCapability, GameCultRuntimePresenceHealthPurpose, + GameCultRuntimePresenceHealthRecord, IDUNN_EXPECTED_INCARNATION_SCHEMA, + IDUNN_PROCESS_WRITE_LEASE_SCHEMA, IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME, + IDUNN_RUNTIME_ACTIVATION_SCHEMA, IdunnExpectedIncarnationRecord, IdunnProcessWriteLeaseRecord, + IdunnRuntimeActivationRecord, IdunnRuntimeActivationSigner, IdunnServiceIdentity, + ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA, OdinRuntimeTopologyCorrelationRecord, + OdinTopologyIdentity, ServiceIdentityProfile, ServiceIdentitySigner, + ServiceIdentityTrustAnchor, derive_service_identity_id, + open_service_identity_credential_reader, verify_runtime_authority, +}; +use fs2::FileExt; +use odin_daemon::{ + AuthenticationPolicy, CultCacheIdunnProjectionSource, CultCacheOdinTopologyStore, + IdunnProjectionSource, OdinTopologyAuthority, SystemClock, +}; + +const TARGET: &str = "odin"; +const HEALTH_CONTRACT: &str = "odin.runtime-health.v1"; +const RENDEZVOUS_CAPABILITY: &str = "odin.verse-rendezvous"; +const RENDEZVOUS_SCHEMA: &str = "odin.verse-topology.v1"; +const RENDEZVOUS_COMPATIBILITY: &str = "v1"; +const STATE_SCHEMA_GENERATION: &str = "odin-v2"; +// rmp-serde SHA-256 of deployment/idunn/recipe.toml's exact [state] value. +const STATE_CONTRACT_SHA256: &str = + "sha256-4f2f2dcd931d16f6b02bf295f41227b867aa982765661d55fa9f29fb2db7e449"; + +const RUNTIME_BUNDLE_ENVIRONMENT: &str = "GAMECULT_IDUNN_RUNTIME_BUNDLE"; +const CANDIDATE_BIND_ENVIRONMENT: &str = "GAMECULT_IDUNN_CANDIDATE_BIND"; +const PROCESS_WRITE_LEASE_ENVIRONMENT: &str = "GAMECULT_IDUNN_PROCESS_WRITE_LEASE"; +const TOPOLOGY_IDENTITY_ENVIRONMENT: &str = "ODIN_TOPOLOGY_IDENTITY"; +const SYSTEMD_LISTEN_PID_ENVIRONMENT: &str = "LISTEN_PID"; +const SYSTEMD_LISTEN_FDS_ENVIRONMENT: &str = "LISTEN_FDS"; +const SYSTEMD_LISTEN_FDNAMES_ENVIRONMENT: &str = "LISTEN_FDNAMES"; +const ACTIVATION_SIGNER_FD_NAME: &str = IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME; +const PROVIDER_SIGNER_FD_NAME: &str = "gamecult-runtime-presence-identity"; +const SYSTEMD_LISTEN_FDS_START: RawFd = 3; + +const BOOTSTRAP_LEASE_TIMEOUT: Duration = Duration::from_secs(60); +const SELF_PUBLISH_TIMEOUT: Duration = Duration::from_secs(6); +const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(5); +const PROJECTION_REFRESH_INTERVAL: Duration = Duration::from_millis(250); +const IDLE_POLL_INTERVAL: Duration = Duration::from_millis(2); +const MAX_RECENT_WARMING_PROOFS: usize = 64; +const WARMING_PROOF_LIFETIME_MILLIS: u64 = 60_000; +const CAS_ATTEMPTS: usize = 8; + +type TopologyAuthority = OdinTopologyAuthority< + CultCacheIdunnProjectionSource, + CultCacheOdinTopologyStore, + ServiceIdentitySigner, + SystemClock, +>; + +#[derive(Clone, Debug)] +struct Options { + store: PathBuf, + idunn_projection: PathBuf, + idunn_anchor: PathBuf, +} + +struct RuntimeAuthority { + expected: IdunnExpectedIncarnationRecord, + expected_sha256: String, + activation: IdunnRuntimeActivationRecord, + activation_sha256: String, + activation_signer: IdunnRuntimeActivationSigner, + provider_signer: ServiceIdentitySigner, +} + +struct ProcessWriteLeaseGuard { + _lock: File, + record: IdunnProcessWriteLeaseRecord, + sha256: String, +} + +struct RuntimeState { + options: Options, + candidate: SocketAddr, + authority_material: RuntimeAuthority, + idunn_anchor: Option, + topology_signer: Option>, + topology: Option, + write_lease: Option, + write_lease_path: PathBuf, + recent_warming_proofs: VecDeque<(String, u64)>, + publisher_sequence: u64, +} + +#[derive(Clone)] +struct SinkHandle(Rc>); + +#[derive(Clone)] +struct SnapshotHandle(Rc>); + +impl CultMeshRudpRawDocumentSink for SinkHandle { + fn accept_raw_document(&mut self, receipt: CultMeshRudpRawDocumentReceipt) -> Result<()> { + self.0.borrow_mut().accept_raw_document(receipt) + } +} + +impl CultMeshRudpSnapshotSource for SnapshotHandle { + fn raw_snapshot( + &mut self, + query: &CultMeshRudpSnapshotQuery, + ) -> Result> { + self.0.borrow_mut().raw_snapshot(query) + } +} + +impl RuntimeState { + fn open(options: Options, candidate: SocketAddr) -> Result { + let authority_material = load_runtime_authority(Path::new(&required_environment( + RUNTIME_BUNDLE_ENVIRONMENT, + )?))?; + require_expected_contract(&authority_material.expected, candidate)?; + + let idunn_anchor = read_trust_anchor::(&options.idunn_anchor)?; + let projection_source = CultCacheIdunnProjectionSource::new(&options.idunn_projection); + let projection = projection_source + .current_projection(TARGET)? + .context("Idunn projection has no Expected Odin incarnation")?; + ensure!( + projection.expected == authority_material.expected + && projection.activation.as_ref() == Some(&authority_material.activation), + "Idunn live projection differs from Odin's immutable runtime bundle" + ); + let provider_anchor = projection + .provider_anchor + .as_ref() + .context("Idunn projection has no Odin runtime-presence trust anchor")?; + ensure!( + provider_anchor.signer_identity_id + == authority_material.provider_signer.entry().identity_id + && provider_anchor.signer_public_key + == authority_material.provider_signer.entry().public_key, + "Odin provider signer differs from Idunn's projected trust anchor" + ); + verify_runtime_authority( + &authority_material.expected, + &authority_material.activation, + &idunn_anchor, + &provider_anchor.signer_public_key, + )?; + + let topology_identity_path = + PathBuf::from(required_environment(TOPOLOGY_IDENTITY_ENVIRONMENT)?); + let topology_signer = open_service_identity_credential_reader::( + File::open(&topology_identity_path).with_context(|| { + format!( + "opening Odin topology identity credential {}", + topology_identity_path.display() + ) + })?, + )?; + let store_sequence = prior_self_publisher_sequence( + &options.store, + authority_material + .provider_signer + .entry() + .identity_id + .as_str(), + )?; + + Ok(Self { + options, + candidate, + authority_material, + idunn_anchor: Some(idunn_anchor), + topology_signer: Some(topology_signer), + topology: None, + write_lease: None, + write_lease_path: PathBuf::from(required_environment(PROCESS_WRITE_LEASE_ENVIRONMENT)?), + recent_warming_proofs: VecDeque::new(), + publisher_sequence: store_sequence, + }) + } + + fn activated(&self) -> bool { + self.topology.is_some() && self.write_lease.is_some() + } + + fn try_activate(&mut self) -> Result { + if self.activated() { + return Ok(true); + } + let Some(lease) = acquire_process_write_lease( + &self.write_lease_path, + &self.authority_material, + &self.recent_warming_proofs, + )? + else { + return Ok(false); + }; + let Some(projected) = CultCacheIdunnProjectionSource::new(&self.options.idunn_projection) + .current_projection(TARGET)? + else { + return Ok(false); + }; + if projected.current_lease.as_ref() != Some(&lease.record) { + return Ok(false); + } + let signer = self + .topology_signer + .take() + .context("Odin topology signer was already consumed")?; + let idunn_anchor = self + .idunn_anchor + .take() + .context("Idunn trust anchor was already consumed")?; + self.write_lease = Some(lease); + self.topology = Some(OdinTopologyAuthority::new( + CultCacheIdunnProjectionSource::new(&self.options.idunn_projection), + CultCacheOdinTopologyStore::new(&self.options.store), + signer, + SystemClock, + idunn_anchor, + AuthenticationPolicy::default(), + )); + Ok(true) + } + + fn accept_raw_document(&mut self, receipt: CultMeshRudpRawDocumentReceipt) -> Result<()> { + ensure!( + self.activated(), + "Odin does not admit or persist provider documents before its process-write lease" + ); + validate_raw_document_shape(&receipt.document)?; + if receipt.document.schema_id == GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA { + let presence = decode_presence(&receipt.document.payload)?; + ensure!( + receipt.document.record_key == presence.target, + "runtime-presence document key differs from its signed target" + ); + self.topology + .as_ref() + .context("Odin topology authority is absent")? + .admit_presence( + &presence.target, + &receipt.document.payload, + receipt.received_at_unix_millis, + )?; + return Ok(()); + } + persist_generic_document(&self.options.store, &receipt.document) + } + + fn raw_snapshot( + &mut self, + query: &CultMeshRudpSnapshotQuery, + ) -> Result> { + validate_snapshot_filters(query)?; + if exact_self_presence_query(query) { + let (state, detail) = if self.activated() { + self.require_current_write_lease()?; + ("active", format!("route-observation:{}", query.message_id)) + } else { + ("warming", format!("idunn-warming:{}", query.message_id)) + }; + return Ok(vec![self.signed_presence_document(state, &detail)?]); + } + ensure!( + self.activated(), + "Odin catalog is unavailable until Idunn grants its process-write lease" + ); + self.refresh_all_correlations()?; + self.stored_snapshot(query) + } + + fn signed_presence_document( + &mut self, + state: &str, + detail: &str, + ) -> Result { + let now = unix_millis()?; + self.publisher_sequence = self + .publisher_sequence + .checked_add(1) + .context("Odin runtime-presence publisher sequence exhausted")?; + let expected = &self.authority_material.expected; + let activation = &self.authority_material.activation; + let source_runtime_id = expected.runtime_id.clone(); + let mut record = GameCultRuntimePresenceHealthRecord { + schema_version: GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + target: expected.target.clone(), + expected_projection_sha256: self.authority_material.expected_sha256.clone(), + plan_id: expected.plan_id.clone(), + incarnation_id: expected.incarnation_id.clone(), + sealed_release_id: expected.sealed_release_id.clone(), + activation_witness_sha256: self.authority_material.activation_sha256.clone(), + state_schema_generation: expected.state_schema_generation.clone(), + state_contract_sha256: expected.state_contract_sha256.clone(), + runtime_id: expected.runtime_id.clone(), + runtime_instance_id: activation.runtime_instance_id.clone(), + bound_endpoint: Some(format!("rudp://{}", self.candidate)), + capabilities: expected + .capabilities + .iter() + .map(|capability| GameCultRuntimeCapability { + capability: capability.capability.clone(), + schema: capability.schema.clone(), + compatibility: capability.compatibility.clone(), + capacity: capability.minimum_capacity, + }) + .collect(), + health_contract: expected.health_contract.clone(), + state: state.into(), + detail: detail.into(), + write_lease_sha256: self.write_lease.as_ref().map(|lease| lease.sha256.clone()), + signer_identity_id: self + .authority_material + .provider_signer + .entry() + .identity_id + .clone(), + publisher_sequence: self.publisher_sequence, + observed_at_unix_millis: now, + signature_algorithm: "ed25519".into(), + signature: Vec::new(), + activation_signer_identity_id: activation.activation_signer_identity_id.clone(), + activation_signature: Vec::new(), + }; + ensure!( + (state == "warming" && record.write_lease_sha256.is_none()) + || (state == "active" && record.write_lease_sha256.is_some()), + "Odin presence state does not match its process-write authority" + ); + let proof_payload = record.canonical_proof_payload()?; + record.signature = self + .authority_material + .provider_signer + .sign::(&proof_payload) + .signature; + record.activation_signature = self + .authority_material + .activation_signer + .sign_presence_proof(&record)?; + record.validate()?; + let payload = rmp_serde::to_vec(&record)?; + ensure!( + rmp_serde::from_slice::(&payload)? == record, + "Odin runtime presence is not canonical MessagePack" + ); + if state == "warming" { + remember_recent_warming_proof( + &mut self.recent_warming_proofs, + record.canonical_sha256()?, + now, + ); + } + Ok(CultNetRawDocumentRecord { + schema_id: GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + record_key: TARGET.into(), + stored_at: rfc3339_millis(now)?, + payload_encoding: CultNetRawPayloadEncoding::Messagepack, + payload, + source_runtime_id: Some(source_runtime_id), + source_agent_id: Some(record.signer_identity_id), + source_role: Some("runtime-presence-health-publisher".into()), + tags: Some(vec!["cultnet.transport.rudp.v0".into()]), + }) + } + + fn active_document_put(&mut self, detail: &str) -> Result { + self.require_current_write_lease()?; + let document = self.signed_presence_document("active", detail)?; + let sequence = self.publisher_sequence; + Ok(CultNetMessage::DocumentPutRaw { + message_id: format!( + "odin-presence:{}:{}:{sequence}", + self.authority_material.expected.runtime_id, + self.authority_material.activation.runtime_instance_id + ), + document, + }) + } + + fn require_current_write_lease(&self) -> Result<()> { + let held = self + .write_lease + .as_ref() + .context("Odin has no process-write lease")?; + let current = read_process_write_lease(&self.write_lease_path)? + .context("Odin process-write lease was withdrawn")?; + ensure!( + current == held.record && current.canonical_sha256()? == held.sha256, + "Odin process-write lease changed after admission" + ); + Ok(()) + } + + fn refresh_all_correlations(&mut self) -> Result<()> { + self.require_current_write_lease()?; + let topology = self + .topology + .as_ref() + .context("Odin topology authority is absent")?; + let mut targets = projection_targets(&self.options.idunn_projection)?; + targets.extend(correlation_targets(&self.options.store)?); + for target in targets { + topology.refresh(&target)?; + } + Ok(()) + } + + fn stored_snapshot( + &self, + query: &CultMeshRudpSnapshotQuery, + ) -> Result> { + let entries = if self.options.store.is_file() { + SingleFileMessagePackBackingStore::new(&self.options.store) + .pull_all_read_only_snapshot()? + } else { + Vec::new() + }; + let projections = CultCacheIdunnProjectionSource::new(&self.options.idunn_projection); + let mut selected = BTreeMap::new(); + for envelope in entries { + let Some(schema_id) = envelope.schema_id.clone() else { + continue; + }; + let document = if envelope.r#type == GameCultRuntimePresenceHealthRecord::TYPE { + let presence = decode_presence(&envelope.payload)?; + let Some(projection) = projections.current_projection(&presence.target)? else { + continue; + }; + if projection.expected.expected_signer_identity_id != presence.signer_identity_id + || projection.expected.canonical_sha256()? + != presence.expected_projection_sha256 + || projection + .activation + .as_ref() + .map(IdunnRuntimeActivationRecord::canonical_sha256) + .transpose()? + .as_deref() + != Some(presence.activation_witness_sha256.as_str()) + || projection + .activation + .as_ref() + .map(|activation| activation.runtime_instance_id.as_str()) + != Some(presence.runtime_instance_id.as_str()) + { + continue; + } + CultNetRawDocumentRecord { + schema_id, + record_key: presence.target.clone(), + stored_at: envelope.stored_at, + payload_encoding: CultNetRawPayloadEncoding::Messagepack, + payload: envelope.payload, + source_runtime_id: Some(presence.runtime_id), + source_agent_id: Some(presence.signer_identity_id), + source_role: Some("runtime-presence-health-publisher".into()), + tags: Some(vec!["odin-observed".into()]), + } + } else if envelope.r#type == OdinRuntimeTopologyCorrelationRecord::TYPE { + let (correlation, _) = + OdinRuntimeTopologyCorrelationRecord::decode_canonical_signed_payload( + &envelope.payload, + )?; + CultNetRawDocumentRecord { + schema_id, + record_key: correlation.target, + stored_at: envelope.stored_at, + payload_encoding: CultNetRawPayloadEncoding::Messagepack, + payload: envelope.payload, + source_runtime_id: Some(self.authority_material.expected.runtime_id.clone()), + source_agent_id: Some(correlation.signer_identity_id), + source_role: Some("odin-topology-correlation".into()), + tags: Some(vec!["odin-owned".into()]), + } + } else { + CultNetRawDocumentRecord { + schema_id, + record_key: envelope.key, + stored_at: envelope.stored_at, + payload_encoding: CultNetRawPayloadEncoding::Messagepack, + payload: envelope.payload, + source_runtime_id: None, + source_agent_id: None, + source_role: None, + tags: None, + } + }; + if !query_allows(query, &document) { + continue; + } + let identity = (document.schema_id.clone(), document.record_key.clone()); + ensure!( + selected.insert(identity, document).is_none(), + "Odin catalog contains duplicate public document identities" + ); + } + Ok(selected.into_values().collect()) + } +} + +fn main() -> Result<()> { + let options = parse_options(std::env::args().skip(1))?; + let candidate: SocketAddr = required_environment(CANDIDATE_BIND_ENVIRONMENT)? + .parse() + .context("parsing Idunn candidate bind")?; + ensure!( + candidate.ip().is_loopback() && candidate.port() != 0, + "Odin candidate bind must be one fixed loopback socket" + ); + let socket = UdpSocket::bind(candidate) + .with_context(|| format!("binding Odin CultNet RUDP candidate {candidate}"))?; + let state = Rc::new(RefCell::new(RuntimeState::open(options, candidate)?)); + let mut server = CultMeshRudpDocumentServer::new( + socket, + SinkHandle(state.clone()), + SnapshotHandle(state.clone()), + CultMeshSystemClock::default(), + CultMeshRudpDocumentServerOptions::default(), + )?; + + let bootstrap_started = Instant::now(); + while !state.borrow_mut().try_activate()? { + let progressed = poll_server(&mut server)?; + ensure!( + bootstrap_started.elapsed() < BOOTSTRAP_LEASE_TIMEOUT, + "timed out waiting for Idunn to admit Odin's exact Warming proof and grant its lease" + ); + if !progressed { + thread::sleep(IDLE_POLL_INTERVAL); + } + } + + publish_self_presence(&state, &mut server, "ready")?; + state.borrow_mut().refresh_all_correlations()?; + let mut last_heartbeat = Instant::now(); + let mut last_projection_refresh = Instant::now(); + loop { + let progressed = poll_server(&mut server)?; + if last_projection_refresh.elapsed() >= PROJECTION_REFRESH_INTERVAL { + state.borrow_mut().refresh_all_correlations()?; + last_projection_refresh = Instant::now(); + } + if last_heartbeat.elapsed() >= HEARTBEAT_INTERVAL { + publish_self_presence(&state, &mut server, "ready")?; + last_heartbeat = Instant::now(); + } + if !progressed { + thread::sleep(IDLE_POLL_INTERVAL); + } + } +} + +fn publish_self_presence( + state: &Rc>, + server: &mut CultMeshRudpDocumentServer, + detail: &str, +) -> Result<()> { + let (message, options) = { + let mut state = state.borrow_mut(); + let message = state.active_document_put(detail)?; + let mut options = CultMeshRudpDocumentPublishOptions::odin( + state.candidate, + state.authority_material.expected.runtime_id.clone(), + ); + options.connect_timeout = Duration::from_secs(2); + options.flush_timeout = Duration::from_secs(2); + options.source_agent_id = Some( + state + .authority_material + .provider_signer + .entry() + .identity_id + .clone(), + ); + options.source_role = Some("runtime-presence-health-publisher".into()); + options.tags = vec!["cultnet.transport.rudp.v0".into()]; + (message, options) + }; + let publisher = + thread::spawn(move || publish_cultnet_message_to_rudp_catalog(&message, options)); + let started = Instant::now(); + while !publisher.is_finished() { + let progressed = poll_server(server)?; + ensure!( + started.elapsed() < SELF_PUBLISH_TIMEOUT, + "Odin self-presence publication did not complete" + ); + if !progressed { + thread::sleep(IDLE_POLL_INTERVAL); + } + } + publisher + .join() + .map_err(|_| anyhow!("Odin self-presence publisher panicked"))??; + Ok(()) +} + +fn poll_server( + server: &mut CultMeshRudpDocumentServer, +) -> Result { + match server.poll_once()? { + CultMeshRudpPollOutcome::Idle => Ok(false), + CultMeshRudpPollOutcome::Handled => Ok(true), + CultMeshRudpPollOutcome::ApplicationRejected(rejection) => { + eprintln!( + "Odin rejected CultMesh RUDP application message {:?} {} from {:?}: {}", + rejection.operation, rejection.message_id, rejection.session, rejection.reason + ); + Ok(true) + } + } +} + +fn parse_options(args: impl Iterator) -> Result { + let mut args = args.peekable(); + let mut values = BTreeMap::new(); + while let Some(name) = args.next() { + let name = name + .strip_prefix("--") + .with_context(|| format!("expected --option, got {name:?}"))?; + ensure!( + matches!(name, "store" | "idunn-projection" | "idunn-anchor"), + "unsupported Odin option --{name}" + ); + let value = args + .next() + .with_context(|| format!("missing value for --{name}"))?; + ensure!( + values + .insert(name.to_owned(), PathBuf::from(value)) + .is_none(), + "duplicate Odin option --{name}" + ); + } + let take = |name: &str| -> Result { + let path = values + .get(name) + .cloned() + .with_context(|| format!("--{name} is required"))?; + ensure!(path.is_absolute(), "--{name} must be an absolute path"); + Ok(path) + }; + let options = Options { + store: take("store")?, + idunn_projection: take("idunn-projection")?, + idunn_anchor: take("idunn-anchor")?, + }; + ensure!( + options.store != options.idunn_projection + && options.store != options.idunn_anchor + && options.idunn_projection != options.idunn_anchor, + "Odin store, Idunn projection, and Idunn anchor paths must be distinct" + ); + Ok(options) +} + +fn load_runtime_authority(bundle: &Path) -> Result { + ensure!(bundle.is_absolute(), "Idunn runtime bundle is not absolute"); + let (expected_key, expected_payload) = read_single_runtime_record( + &bundle.join("expected.cc"), + IdunnExpectedIncarnationRecord::TYPE, + IDUNN_EXPECTED_INCARNATION_SCHEMA, + )?; + let expected = IdunnExpectedIncarnationRecord::decode_canonical(&expected_payload)?; + ensure!( + expected_key == expected.target, + "Expected key is substituted" + ); + let (activation_key, activation_payload) = read_single_runtime_record( + &bundle.join("activation.cc"), + IdunnRuntimeActivationRecord::TYPE, + IDUNN_RUNTIME_ACTIVATION_SCHEMA, + )?; + let activation = IdunnRuntimeActivationRecord::decode_canonical(&activation_payload)?; + ensure!( + activation_key == expected.target, + "activation key is substituted" + ); + let expected_sha256 = expected.canonical_sha256()?; + ensure!( + activation.expected_projection_sha256 == expected_sha256 + && activation.runtime_id == expected.runtime_id, + "activation does not bind Odin's Expected projection" + ); + let (activation_credential, provider_identity) = take_runtime_signer_descriptors()?; + let activation_signer = + IdunnRuntimeActivationSigner::from_credential_reader(activation_credential)?; + ensure!( + activation_signer.identity_id() == activation.activation_signer_identity_id + && activation_signer.public_key() == activation.activation_signer_public_key, + "activation credential does not belong to Odin's activation" + ); + let provider_signer = open_service_identity_credential_reader::( + provider_identity, + )?; + ensure!( + provider_signer.entry().identity_id == expected.expected_signer_identity_id, + "provider credential is not the signer selected by Expected" + ); + Ok(RuntimeAuthority { + expected, + expected_sha256, + activation_sha256: activation.canonical_sha256()?, + activation, + activation_signer, + provider_signer, + }) +} + +fn require_expected_contract( + expected: &IdunnExpectedIncarnationRecord, + bind: SocketAddr, +) -> Result<()> { + ensure!( + expected.target == TARGET + && expected.health_contract == HEALTH_CONTRACT + && expected.state_schema_generation.as_deref() == Some(STATE_SCHEMA_GENERATION) + && expected.state_contract_sha256.as_deref() == Some(STATE_CONTRACT_SHA256) + && expected.write_lease_required, + "Odin Expected projection differs from its compiled runtime/state contract" + ); + let route = expected + .route + .as_ref() + .context("Odin Expected projection has no admitted route")?; + ensure!( + route.transport == "rudp" && route.candidate_endpoint == format!("rudp://{bind}"), + "Odin candidate bind differs from Expected" + ); + let capability = expected + .capabilities + .iter() + .find(|capability| { + capability.capability == RENDEZVOUS_CAPABILITY + && capability.schema == RENDEZVOUS_SCHEMA + && capability.compatibility == RENDEZVOUS_COMPATIBILITY + }) + .context("Odin Expected projection omits its rendezvous capability")?; + ensure!( + capability.minimum_capacity > 0 && expected.capabilities.len() == 1, + "Odin Expected capability set differs from its compiled contract" + ); + ensure!( + expected.dependencies.is_empty(), + "Odin bootstrap cannot depend on a managed daemon" + ); + Ok(()) +} + +fn read_single_runtime_record( + path: &Path, + expected_type: &str, + expected_schema: &str, +) -> Result<(String, Vec)> { + let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; + let [envelope] = entries.as_slice() else { + bail!("runtime authority store must contain exactly one record"); + }; + ensure!( + envelope.r#type == expected_type && envelope.schema_id.as_deref() == Some(expected_schema), + "runtime authority store has the wrong typed envelope" + ); + Ok((envelope.key.clone(), envelope.payload.clone())) +} + +fn take_runtime_signer_descriptors() -> Result<(File, File)> { + let pid = required_environment(SYSTEMD_LISTEN_PID_ENVIRONMENT)?; + let count = required_environment(SYSTEMD_LISTEN_FDS_ENVIRONMENT)?; + let names = required_environment(SYSTEMD_LISTEN_FDNAMES_ENVIRONMENT)?; + ensure!( + pid == std::process::id().to_string(), + "systemd signer descriptors belong to another process" + ); + ensure!(count == "2", "Odin requires exactly two signer descriptors"); + ensure!( + names == format!("{ACTIVATION_SIGNER_FD_NAME}:{PROVIDER_SIGNER_FD_NAME}"), + "systemd signer descriptor names or order differ from Idunn's contract" + ); + // SAFETY: the exact systemd LISTEN_* contract above assigns sole ownership + // of descriptors 3 and 4 to this process, and this is the first FD consumer. + let activation = unsafe { File::from_raw_fd(SYSTEMD_LISTEN_FDS_START) }; + // SAFETY: descriptor 4 is distinct and covered by the same exact contract. + let provider = unsafe { File::from_raw_fd(SYSTEMD_LISTEN_FDS_START + 1) }; + Ok((activation, provider)) +} + +fn read_trust_anchor(path: &Path) -> Result { + let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; + let [envelope] = entries.as_slice() else { + bail!("service trust-anchor store must contain exactly one document"); + }; + ensure!( + envelope.r#type == P::TRUST_ANCHOR_TYPE + && envelope.key == P::TRUST_ANCHOR_KEY + && envelope.schema_id.as_deref() == Some(P::TRUST_ANCHOR_SCHEMA), + "service trust anchor belongs to another identity profile" + ); + let anchor: ServiceIdentityTrustAnchor = rmp_serde::from_slice(&envelope.payload)?; + ensure!( + rmp_serde::to_vec(&anchor)? == envelope.payload + && anchor.schema_version == P::TRUST_ANCHOR_SCHEMA + && derive_service_identity_id::

(&anchor.public_key)? == anchor.identity_id, + "service trust anchor is noncanonical or self-inconsistent" + ); + Ok(anchor) +} + +fn acquire_process_write_lease( + path: &Path, + authority: &RuntimeAuthority, + recent_warming: &VecDeque<(String, u64)>, +) -> Result> { + let Some(observed) = read_process_write_lease(path)? else { + return Ok(None); + }; + let now = unix_millis()?; + let warming_is_ours = + recent_warming_proof(recent_warming, &observed.warming_presence_sha256, now); + let exact_incarnation = observed.target == authority.expected.target + && observed.expected_projection_sha256 == authority.expected_sha256 + && observed.plan_id == authority.expected.plan_id + && observed.incarnation_id == authority.expected.incarnation_id + && observed.sealed_release_id == authority.expected.sealed_release_id + && observed.activation_witness_sha256 == authority.activation_sha256 + && Some(observed.state_schema_generation.as_str()) + == authority.expected.state_schema_generation.as_deref() + && Some(observed.state_contract_sha256.as_str()) + == authority.expected.state_contract_sha256.as_deref() + && observed.runtime_id == authority.expected.runtime_id + && observed.runtime_instance_id == authority.activation.runtime_instance_id; + if !warming_is_ours || !exact_incarnation { + return Ok(None); + } + let lock_path = sibling_lock_path(path)?; + let lock = OpenOptions::new() + .read(true) + .open(&lock_path) + .with_context(|| format!("opening process-write-lease lock {}", lock_path.display()))?; + FileExt::lock_shared(&lock)?; + let current = read_process_write_lease(path)? + .context("process-write lease disappeared while acquiring its lifetime lock")?; + ensure!( + current == observed, + "process-write lease changed while acquiring its lifetime lock" + ); + Ok(Some(ProcessWriteLeaseGuard { + _lock: lock, + sha256: current.canonical_sha256()?, + record: current, + })) +} + +fn remember_recent_warming_proof( + recent_warming: &mut VecDeque<(String, u64)>, + sha256: String, + now: u64, +) { + while recent_warming + .front() + .is_some_and(|(_, issued)| now.saturating_sub(*issued) > WARMING_PROOF_LIFETIME_MILLIS) + { + recent_warming.pop_front(); + } + recent_warming.push_back((sha256, now)); + while recent_warming.len() > MAX_RECENT_WARMING_PROOFS { + recent_warming.pop_front(); + } +} + +fn recent_warming_proof( + recent_warming: &VecDeque<(String, u64)>, + expected_sha256: &str, + now: u64, +) -> bool { + recent_warming.iter().any(|(sha256, issued)| { + sha256 == expected_sha256 && now.saturating_sub(*issued) <= WARMING_PROOF_LIFETIME_MILLIS + }) +} + +fn read_process_write_lease(path: &Path) -> Result> { + if !path.is_file() { + return Ok(None); + } + let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; + let [envelope] = entries.as_slice() else { + ensure!(entries.is_empty(), "process-write-lease store is ambiguous"); + return Ok(None); + }; + ensure!( + envelope.r#type == IdunnProcessWriteLeaseRecord::TYPE + && envelope.schema_id.as_deref() == Some(IDUNN_PROCESS_WRITE_LEASE_SCHEMA), + "process-write-lease store has the wrong typed envelope" + ); + let lease = IdunnProcessWriteLeaseRecord::decode_canonical(&envelope.payload)?; + ensure!( + envelope.key == lease.target, + "process-write-lease key differs from its target" + ); + Ok(Some(lease)) +} + +fn prior_self_publisher_sequence(path: &Path, signer_identity_id: &str) -> Result { + if !path.is_file() { + return Ok(0); + } + SingleFileMessagePackBackingStore::new(path) + .pull_all_read_only_snapshot()? + .into_iter() + .filter(|entry| entry.r#type == GameCultRuntimePresenceHealthRecord::TYPE) + .try_fold(0, |maximum, entry| { + let presence = decode_presence(&entry.payload)?; + Ok( + if presence.target == TARGET && presence.signer_identity_id == signer_identity_id { + maximum.max(presence.publisher_sequence) + } else { + maximum + }, + ) + }) +} + +fn persist_generic_document(path: &Path, document: &CultNetRawDocumentRecord) -> Result<()> { + let document_type = document_type_for_schema(&document.schema_id)?; + ensure!( + !matches!( + document_type.as_str(), + "gamecult.runtime_presence_health" + | "odin.runtime_topology_correlation" + | "idunn.expected_incarnation" + | "idunn.runtime_activation" + | "idunn.process_write_lease" + | "gamecult.service_trust_anchor" + ), + "generic provider traffic cannot write an authority-owned Odin/Idunn document" + ); + let replacement = CultCacheEnvelope { + key: document.record_key.clone(), + r#type: document_type.clone(), + payload: document.payload.clone(), + stored_at: document.stored_at.clone(), + schema_id: Some(document.schema_id.clone()), + }; + let store = SingleFileMessagePackBackingStore::new(path); + for _ in 0..CAS_ATTEMPTS { + let entries = if path.is_file() { + store.pull_all_read_only_snapshot()? + } else { + Vec::new() + }; + let mut matches = entries + .iter() + .filter(|entry| entry.r#type == document_type && entry.key == document.record_key); + let current = matches.next().cloned(); + ensure!( + matches.next().is_none(), + "generic CultCache identity is ambiguous" + ); + if current.as_ref() == Some(&replacement) { + return Ok(()); + } + if store.compare_exchange( + &[CultCacheExpectedEnvelope { + key: document.record_key.clone(), + r#type: document_type.clone(), + current, + }], + &[replacement.clone()], + )? { + return Ok(()); + } + } + bail!("Odin catalog changed repeatedly while persisting a provider document") +} + +fn document_type_for_schema(schema_id: &str) -> Result { + ensure!( + !schema_id.is_empty() && schema_id.trim() == schema_id, + "provider schema id is empty or padded" + ); + let Some((prefix, version)) = schema_id.rsplit_once(".v") else { + bail!("provider schema id has no explicit version suffix"); + }; + ensure!( + !prefix.is_empty() + && !version.is_empty() + && version.chars().all(|value| value.is_ascii_digit()), + "provider schema id has an invalid version suffix" + ); + Ok(prefix.into()) +} + +fn validate_raw_document_shape(document: &CultNetRawDocumentRecord) -> Result<()> { + ensure!( + document.payload_encoding == CultNetRawPayloadEncoding::Messagepack + && !document.record_key.is_empty() + && document.record_key.trim() == document.record_key + && !document.payload.is_empty(), + "raw provider document has an invalid key, encoding, or payload" + ); + document_type_for_schema(&document.schema_id)?; + DateTime::parse_from_rfc3339(&document.stored_at) + .context("raw provider document has an invalid stored_at")?; + Ok(()) +} + +fn decode_presence(payload: &[u8]) -> Result { + let presence: GameCultRuntimePresenceHealthRecord = rmp_serde::from_slice(payload)?; + ensure!( + rmp_serde::to_vec(&presence)? == payload, + "runtime presence is not canonical MessagePack" + ); + presence.validate()?; + Ok(presence) +} + +fn projection_targets(path: &Path) -> Result> { + let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; + entries + .into_iter() + .filter(|entry| entry.r#type == IdunnExpectedIncarnationRecord::TYPE) + .map(|entry| { + ensure!( + entry.schema_id.as_deref() == Some(IDUNN_EXPECTED_INCARNATION_SCHEMA), + "Idunn projection contains an Expected record under the wrong schema" + ); + let expected = IdunnExpectedIncarnationRecord::decode_canonical(&entry.payload)?; + ensure!( + entry.key == expected.target, + "Idunn Expected key is substituted" + ); + Ok(expected.target) + }) + .collect() +} + +fn correlation_targets(path: &Path) -> Result> { + if !path.is_file() { + return Ok(BTreeSet::new()); + } + SingleFileMessagePackBackingStore::new(path) + .pull_all_read_only_snapshot()? + .into_iter() + .filter(|entry| entry.r#type == OdinRuntimeTopologyCorrelationRecord::TYPE) + .map(|entry| { + ensure!( + entry.schema_id.as_deref() == Some(ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA), + "Odin correlation is stored under the wrong schema" + ); + let (record, _) = + OdinRuntimeTopologyCorrelationRecord::decode_canonical_signed_payload( + &entry.payload, + )?; + ensure!( + entry.key == record.target, + "Odin correlation key is substituted" + ); + Ok(record.target) + }) + .collect() +} + +fn validate_snapshot_filters(query: &CultMeshRudpSnapshotQuery) -> Result<()> { + ensure!( + !query.message_id.is_empty() && query.message_id.trim() == query.message_id, + "snapshot message id is empty or padded" + ); + for (label, values) in [ + ("schema", query.schema_ids.as_ref()), + ("record key", query.record_keys.as_ref()), + ] { + if let Some(values) = values { + ensure!(!values.is_empty(), "snapshot {label} filter is empty"); + let unique = values.iter().collect::>(); + ensure!( + unique.len() == values.len() + && values + .iter() + .all(|value| !value.is_empty() && value.trim() == value), + "snapshot {label} filter is duplicated, empty, or padded" + ); + } + } + Ok(()) +} + +fn exact_self_presence_query(query: &CultMeshRudpSnapshotQuery) -> bool { + query.schema_ids.as_deref() == Some(&[GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.to_owned()]) + && query.record_keys.as_deref() == Some(&[TARGET.to_owned()]) +} + +fn query_allows(query: &CultMeshRudpSnapshotQuery, document: &CultNetRawDocumentRecord) -> bool { + query + .schema_ids + .as_ref() + .is_none_or(|values| values.contains(&document.schema_id)) + && query + .record_keys + .as_ref() + .is_none_or(|values| values.contains(&document.record_key)) +} + +fn sibling_lock_path(path: &Path) -> Result { + let mut name = path + .file_name() + .context("CultCache authority path has no filename")? + .to_os_string(); + name.push(".lock"); + Ok(path.with_file_name(name)) +} + +fn required_environment(name: &str) -> Result { + std::env::var(name) + .ok() + .filter(|value| !value.is_empty() && value.trim() == value) + .with_context(|| format!("{name} is required for an Idunn-managed Odin")) +} + +fn unix_millis() -> Result { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .context("system clock predates Unix epoch")? + .as_millis() + .try_into() + .context("Unix time exceeds u64") +} + +fn rfc3339_millis(value: u64) -> Result { + let millis: i64 = value + .try_into() + .context("timestamp exceeds RFC3339 range")?; + Ok(DateTime::::from_timestamp_millis(millis) + .context("timestamp exceeds RFC3339 range")? + .to_rfc3339()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn options_are_exact_and_absolute() { + let parsed = parse_options( + [ + "--store", + "/var/lib/gamecult/odin-v2/topology.cc", + "--idunn-projection", + "/var/lib/gamecult/idunn-projection/topology.cc", + "--idunn-anchor", + "/etc/gamecult/idunn/idunn-public-anchor.cc", + ] + .into_iter() + .map(str::to_owned), + ) + .unwrap(); + assert_eq!( + parsed.store, + Path::new("/var/lib/gamecult/odin-v2/topology.cc") + ); + assert!( + parse_options( + [ + "--store", + "relative", + "--idunn-projection", + "/p", + "--idunn-anchor", + "/a" + ] + .into_iter() + .map(str::to_owned) + ) + .is_err() + ); + } + + #[test] + fn schema_to_document_type_is_versioned_and_deterministic() { + assert_eq!( + document_type_for_schema("heimdall.command_boundary.v1").unwrap(), + "heimdall.command_boundary" + ); + for value in ["heimdall.command_boundary", "heimdall.v", ".v1", " x.v1"] { + assert!(document_type_for_schema(value).is_err()); + } + } + + #[test] + fn only_exact_singleton_self_presence_query_is_a_route_challenge() { + let exact = CultMeshRudpSnapshotQuery { + session: cultmesh_rs::CultMeshRudpSessionKey { + remote_addr: "127.0.0.1:1".parse().unwrap(), + connection_id: 7, + }, + message_id: "challenge".into(), + requested_at_unix_millis: 1, + schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), + record_keys: Some(vec![TARGET.into()]), + }; + assert!(exact_self_presence_query(&exact)); + let mut broad = exact.clone(); + broad.record_keys = None; + assert!(!exact_self_presence_query(&broad)); + } + + #[test] + fn first_odin_accepts_only_recent_provider_owned_warming_proofs() { + let mut proofs = VecDeque::new(); + remember_recent_warming_proof(&mut proofs, "warming-one".into(), 100); + assert!(recent_warming_proof( + &proofs, + "warming-one", + 100 + WARMING_PROOF_LIFETIME_MILLIS, + )); + assert!(!recent_warming_proof( + &proofs, + "warming-one", + 101 + WARMING_PROOF_LIFETIME_MILLIS, + )); + assert!(!recent_warming_proof(&proofs, "foreign-proof", 100)); + + for index in 0..=MAX_RECENT_WARMING_PROOFS { + remember_recent_warming_proof(&mut proofs, format!("warming-{index}"), 200); + } + assert_eq!(proofs.len(), MAX_RECENT_WARMING_PROOFS); + assert_eq!(proofs.back().unwrap().0, "warming-64"); + assert!(!proofs.iter().any(|(proof, _)| proof == "warming-one")); + } +} diff --git a/deployment/idunn/recipe.toml b/deployment/idunn/recipe.toml new file mode 100644 index 0000000..f4455df --- /dev/null +++ b/deployment/idunn/recipe.toml @@ -0,0 +1,72 @@ +schema = "gamecult.idunn.target_declaration.v1" +target = "odin" +source_stamp_environment = "ODIN_BUILD_COMMIT" +required_gitlinks = [] + +[[steps]] +id = "test-odin-daemon" +phase = "test" +runner = "rust-build" +argv = ["cargo", "test", "--locked", "-p", "odin-daemon"] + +[[steps]] +id = "build-odin-daemon" +phase = "build" +runner = "rust-build" +argv = [ + "cargo", + "build", + "--locked", + "--release", + "-p", + "odin-daemon", + "--bin", + "odin-daemon", +] + +[[artifacts]] +id = "odin-daemon" +source_kind = "runner-output" +runner = "rust-build" +source = "target/release/odin-daemon" +destination = "odin-daemon" +executable = true + +[service] +executable_artifact = "odin-daemon" +arguments = [ + { kind = "literal", value = "--store" }, + { kind = "binding", name = "store_path" }, + { kind = "literal", value = "--idunn-projection" }, + { kind = "binding", name = "idunn_projection_path" }, + { kind = "literal", value = "--idunn-anchor" }, + { kind = "binding", name = "idunn_anchor_path" }, +] +transport = "rudp" +route_required = true +required_environment = [ + "GAMECULT_IDUNN_CANDIDATE_BIND", + "GAMECULT_IDUNN_PROCESS_WRITE_LEASE", + "GAMECULT_IDUNN_RUNTIME_BUNDLE", + "ODIN_TOPOLOGY_IDENTITY", +] + +[service.health] +contract = "odin.runtime-health.v1" + +[state] +schema_generation = "odin-v2" + +[[state.slots]] +id = "topology" +relative_path = "topology.cc" +kind = "cultcache-file" +schema = "odin.runtime_topology_correlation.v2" +writer = "process-bound-single-writer" +recovery = "preserve" +startup = "create-or-open-after-write-lease" + +[[provides]] +capability = "odin.verse-rendezvous" +schema = "odin.verse-topology.v1" +compatibility = "v1"