From eedce4f3de2aefccaa6e74bb5770b57cd8c6a87b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 4 Sep 2026 15:13:38 +0200 Subject: [PATCH] Land the Idunn projection, UDP route actuation, and the Odin daemon This had been sitting uncommitted in the worktree. It is real work rather than stray edits: RouteBinding gains stable_socket and an NginxStreamUdp driver so a route can be actuated over UDP, control_plane and drivers grow the Idunn-owned projection and bounded RUDP transport, and crates/odin-daemon arrives as a new member of the workspace. The deployment recipe comes with it, including the required_gitlinks fix the core-stack repoint made necessary: it named vendor/cultnet-rs, which no longer exists now that CultNet comes from CultLib, so Idunn would have refused to admit the target. The list is empty because Odin has no submodules left; the core stack is pinned by revision in Cargo.lock inside the frozen source tree. Everything except odin-daemon builds here. odin-daemon reads systemd listen file descriptors through std::os::fd and cannot compile on Windows by design, so Linux CI is its first real witness. muninn and sleipnir pass 162 tests; the one failure expects an Unreliable media channel that no CultNet lineage has ever declared, and predates this. Committed rather than left in flight. rustfmt reports 94 files across the repo, which predates this work, so formatting is left for its own pass instead of being buried in a 4,000-line diff. Co-Authored-By: Claude Opus 5 --- Cargo.toml | 1 + crates/idunn-daemon/src/control_plane.rs | 2111 +++++++++++++++++--- crates/idunn-daemon/src/deployment.rs | 144 +- crates/idunn-daemon/src/deployment_plan.rs | 74 +- crates/idunn-daemon/src/drivers.rs | 1826 ++++++++++++++--- crates/idunn-daemon/src/provisioning.rs | 542 ++++- crates/odin-daemon/Cargo.toml | 23 + crates/odin-daemon/src/lib.rs | 1832 +++++++++++++++++ crates/odin-daemon/src/main.rs | 1248 ++++++++++++ deployment/idunn/recipe.toml | 72 + 10 files changed, 7204 insertions(+), 669 deletions(-) create mode 100644 crates/odin-daemon/Cargo.toml create mode 100644 crates/odin-daemon/src/lib.rs create mode 100644 crates/odin-daemon/src/main.rs create mode 100644 deployment/idunn/recipe.toml diff --git a/Cargo.toml b/Cargo.toml index 76fabb0..09a0ec3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "crates/muninn-move-tracker", "crates/muninn-psmoveapi-tracker", "crates/odin-core", + "crates/odin-daemon", ] resolver = "3" exclude = [ diff --git a/crates/idunn-daemon/src/control_plane.rs b/crates/idunn-daemon/src/control_plane.rs index 225bc27..1f8361b 100644 --- a/crates/idunn-daemon/src/control_plane.rs +++ b/crates/idunn-daemon/src/control_plane.rs @@ -5,35 +5,36 @@ use std::path::{Path, PathBuf}; use std::thread; use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use anyhow::{anyhow, bail, ensure, Context, Result}; +use anyhow::{Context, Result, anyhow, bail, ensure}; use cultcache_rs::{ CultCacheEnvelope, CultCacheExpectedEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, }; use cultnet_rs::{ - authenticate_odin_runtime_topology_correlation, derive_service_identity_id, - evaluate_idunn_continuity_restart, evaluate_idunn_deployment_brake, open_service_identity_at, - verify_idunn_deployment_brake_authorization, verify_runtime_authority, AuthenticatedOdinRuntimeTopologyCorrelation, GameCultProviderHealthIdentity, + IDUNN_DEPLOYMENT_BRAKE_SCHEMA, IDUNN_LIFECYCLE_BRAKE_SCHEMA, IDUNN_PROCESS_WRITE_LEASE_SCHEMA, IdunnDeploymentBrakeObservation, IdunnDeploymentBrakeOperatorIdentity, IdunnDeploymentBrakeRecord, IdunnExpectedIncarnationRecord, IdunnLifecycleBrakeObservation, IdunnLifecycleBrakeRecord, IdunnProcessWriteLeaseRecord, IdunnRuntimeActivationLaunch, IdunnRuntimeActivationRecord, IdunnServiceIdentity, OdinTopologyAuthenticationContext, - OdinTopologyIdentity, ServiceIdentityProfile, ServiceIdentitySigner, - ServiceIdentityTrustAnchor, IDUNN_DEPLOYMENT_BRAKE_SCHEMA, IDUNN_LIFECYCLE_BRAKE_SCHEMA, - IDUNN_PROCESS_WRITE_LEASE_SCHEMA, + OdinTopologyDisagreement, OdinTopologyIdentity, RuntimePresenceAuthenticationContext, + ServiceIdentityProfile, ServiceIdentitySigner, ServiceIdentityTrustAnchor, + authenticate_odin_runtime_topology_correlation, authenticate_runtime_presence_claim, + correlate_runtime_presence_claim, derive_service_identity_id, + evaluate_idunn_continuity_restart, evaluate_idunn_deployment_brake, open_service_identity_at, + verify_idunn_deployment_brake_authorization, verify_runtime_authority, }; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use uuid::Uuid; -use crate::deployment::{DependencyKind, OperatorBinding, StartupOrder}; +use crate::deployment::{DependencyKind, OperatorBinding, RouteBinding, capability_compatible}; use crate::deployment_plan::{ - compile_deployment_plan, CompiledDeploymentPlan, DependencyProviderAuthority, SealedRelease, + CompiledDeploymentPlan, DependencyProviderAuthority, SealedRelease, compile_deployment_plan, }; use crate::drivers::{ CultCacheTopologyDriver, CultCacheWriteLeaseDriver, DockerRunnerDriver, FrozenSourceReceipt, GitSourceDriver, InstalledReleaseObservation, NginxRouteDriver, ProcessIdentity, - RouteObservation, RoutePort, RoutePreflightReceipt, RunnerPort, SourcePort, + RouteObservation, RoutePreflightReceipt, RunnerPort, SourcePort, SystemdTransientWorkloadDriver, TopologyPort, WorkloadObservation, WorkloadPort, WriteLeasePort, }; @@ -188,17 +189,127 @@ impl TopologyEvidence { } } -/// Opaque capability minted only after the transaction CAS advances the Odin -/// replay cursor and stores these exact bytes. Adjacent drivers may inspect the -/// authenticated receipt, but cannot construct replay admission themselves. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct RuntimePresenceEvidence { + canonical_bytes: Vec, + canonical_sha256: String, + message_id: String, + challenged_at_unix_millis: u64, + admitted_at_unix_millis: u64, +} + +impl RuntimePresenceEvidence { + fn from_present( + present: &cultnet_rs::VerifiedRuntimePresence, + message_id: String, + challenged_at_unix_millis: u64, + admitted_at_unix_millis: u64, + ) -> Result { + let evidence = Self { + canonical_bytes: present.canonical_bytes().to_vec(), + canonical_sha256: present.signed_presence_sha256().to_owned(), + message_id, + challenged_at_unix_millis, + admitted_at_unix_millis, + }; + evidence.validate_shape()?; + Ok(evidence) + } + + fn validate_shape(&self) -> Result<()> { + ensure!( + !self.canonical_bytes.is_empty() + && self.canonical_sha256 == sha256_id(&self.canonical_bytes), + "runtime presence evidence bytes or digest are invalid" + ); + require_id(&self.message_id, "runtime presence challenge")?; + ensure!( + self.challenged_at_unix_millis > 0 + && self.admitted_at_unix_millis >= self.challenged_at_unix_millis, + "runtime presence evidence timeline is invalid" + ); + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "source", rename_all = "kebab-case", deny_unknown_fields)] +enum WarmingEvidence { + OdinTopology { evidence: TopologyEvidence }, + FirstOdinDirect { evidence: RuntimePresenceEvidence }, +} + +impl WarmingEvidence { + fn validate_shape(&self) -> Result<()> { + match self { + Self::OdinTopology { evidence } => evidence.validate_shape(), + Self::FirstOdinDirect { evidence } => evidence.validate_shape(), + } + } + + fn canonical_sha256(&self) -> &str { + match self { + Self::OdinTopology { evidence } => &evidence.canonical_sha256, + Self::FirstOdinDirect { evidence } => &evidence.canonical_sha256, + } + } +} + +/// Opaque capability minted only after the transaction CAS stores exact +/// Warming evidence. Adjacent drivers receive only the candidate identity and +/// signed-presence digest needed to bind the write lease; they cannot inspect +/// or construct semantic admission. #[derive(Clone, Debug)] pub struct SequenceAdmittedWarming { transaction_id: String, - evidence: TopologyEvidence, - authenticated: AuthenticatedOdinRuntimeTopologyCorrelation, + evidence: WarmingEvidence, + signed_presence_sha256: String, + runtime_instance_id: String, } impl SequenceAdmittedWarming { + fn from_topology( + transaction_id: String, + evidence: TopologyEvidence, + authenticated: AuthenticatedOdinRuntimeTopologyCorrelation, + ) -> Result { + let record = authenticated.record(); + let signed_presence_sha256 = record + .signed_presence_sha256 + .clone() + .context("Warming topology has no signed presence")?; + let runtime_instance_id = record + .runtime_instance_id + .clone() + .context("Warming topology has no runtime instance")?; + Ok(Self { + transaction_id, + evidence: WarmingEvidence::OdinTopology { evidence }, + signed_presence_sha256, + runtime_instance_id, + }) + } + + fn from_first_odin_presence( + transaction_id: String, + evidence: RuntimePresenceEvidence, + present: cultnet_rs::VerifiedRuntimePresence, + ) -> Result { + ensure!( + present.canonical_bytes() == evidence.canonical_bytes + && present.signed_presence_sha256() == evidence.canonical_sha256, + "direct first-Odin Warming evidence differs from its authenticated presence" + ); + let runtime_instance_id = present.record().runtime_instance_id.clone(); + Ok(Self { + transaction_id, + signed_presence_sha256: evidence.canonical_sha256.clone(), + evidence: WarmingEvidence::FirstOdinDirect { evidence }, + runtime_instance_id, + }) + } + #[cfg(test)] pub(crate) fn for_test( transaction_id: impl Into, @@ -218,15 +329,21 @@ impl SequenceAdmittedWarming { require_id(&transaction_id, "test transaction id")?; let evidence = TopologyEvidence::from_authenticated(&authenticated, admitted_at_unix_millis)?; - Ok(Self { - transaction_id, - evidence, - authenticated, - }) - } - - pub(crate) fn authenticated(&self) -> &AuthenticatedOdinRuntimeTopologyCorrelation { - &self.authenticated + let signed_presence_sha256 = record + .signed_presence_sha256 + .clone() + .context("test Warming receipt has no signed presence")?; + let runtime_instance_id = record + .runtime_instance_id + .clone() + .context("test Warming receipt has no runtime instance")?; + let token = Self::from_topology(transaction_id, evidence, authenticated)?; + ensure!( + token.signed_presence_sha256 == signed_presence_sha256 + && token.runtime_instance_id == runtime_instance_id, + "test Warming token extraction changed" + ); + Ok(token) } pub(crate) fn transaction_id(&self) -> &str { @@ -234,7 +351,15 @@ impl SequenceAdmittedWarming { } pub(crate) fn evidence_sha256(&self) -> &str { - &self.evidence.canonical_sha256 + self.evidence.canonical_sha256() + } + + pub(crate) fn signed_presence_sha256(&self) -> &str { + &self.signed_presence_sha256 + } + + pub(crate) fn runtime_instance_id(&self) -> &str { + &self.runtime_instance_id } } @@ -352,14 +477,27 @@ impl LeasingEvidence { #[serde(tag = "result", rename_all = "kebab-case", deny_unknown_fields)] enum RoutingEvidence { SkippedUnrouted, - Promoted { observation: RouteObservation }, + Promoted { + observation: RouteObservation, + promoted_at_unix_millis: u64, + }, } impl RoutingEvidence { fn observation(&self) -> Option<&RouteObservation> { match self { Self::SkippedUnrouted => None, - Self::Promoted { observation } => Some(observation), + Self::Promoted { observation, .. } => Some(observation), + } + } + + fn promoted_at_unix_millis(&self) -> Option { + match self { + Self::SkippedUnrouted => None, + Self::Promoted { + promoted_at_unix_millis, + .. + } => Some(*promoted_at_unix_millis), } } } @@ -464,7 +602,7 @@ impl PostCommitCleanup { #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] #[cultcache( type = "idunn.deployment_transaction", - schema = "idunn.deployment_transaction.v1" + schema = "idunn.deployment_transaction.v2" )] struct DeploymentTransaction { #[cultcache(key = 0)] @@ -512,7 +650,7 @@ struct DeploymentTransaction { #[cultcache(key = 21)] latest_odin_observation: Option, #[cultcache(key = 22)] - warming: Option, + warming: Option, #[cultcache(key = 23)] route_preflight: Option, #[cultcache(key = 24)] @@ -641,6 +779,17 @@ impl DeploymentTransaction { self.phase != DeploymentPhase::Complete } + /// Admission has moved to the new generation at Complete, but an exact + /// draining incumbent still reserves its process and candidate endpoint. + /// No later mutation for the same target may overlap that cleanup. + fn blocks_new_target_mutation(&self) -> bool { + self.owns_target_authority() + || self + .post_commit_cleanup + .as_ref() + .is_some_and(|cleanup| !cleanup.is_complete()) + } + fn validate(&self) -> Result<()> { ensure!( self.schema_version == DEPLOYMENT_TRANSACTION_SCHEMA, @@ -712,7 +861,7 @@ impl DeploymentTransaction { self.workload.is_none() || self.activation.is_some(), "workload observation exists without its prepared activation" ); - for evidence in [&self.latest_odin_observation, &self.warming, &self.ready] + for evidence in [&self.latest_odin_observation, &self.ready] .into_iter() .flatten() { @@ -722,6 +871,19 @@ impl DeploymentTransaction { "topology evidence exceeds the transaction replay cursor" ); } + if let Some(warming) = &self.warming { + warming.validate_shape()?; + match warming { + WarmingEvidence::OdinTopology { evidence } => ensure!( + evidence.publisher_sequence <= self.odin_publisher_sequence_cursor, + "Warming topology evidence exceeds the transaction replay cursor" + ), + WarmingEvidence::FirstOdinDirect { .. } => ensure!( + self.target == "odin" && self.incumbent_generation_id.is_none(), + "direct Warming evidence is reserved for first Odin bootstrap" + ), + } + } if let Some(leasing) = &self.leasing { if let Some(expected) = &self.expected { ensure!( @@ -891,6 +1053,9 @@ impl DeploymentTransaction { == self.route_preflight.is_some(), "route preflight does not match routed Expected" ); + if let Some(preflight) = &self.route_preflight { + preflight.validate()?; + } } if self.phase >= DeploymentPhase::Leasing { ensure!(self.fencing.is_some(), "Leasing lacks fencing evidence"); @@ -911,6 +1076,18 @@ impl DeploymentTransaction { if self.phase >= DeploymentPhase::Committing { ensure!(self.routing.is_some(), "Committing lacks routing evidence"); } + if let Some(RoutingEvidence::Promoted { + observation, + promoted_at_unix_millis, + }) = &self.routing + { + observation.validate()?; + ensure!( + *promoted_at_unix_millis > 0 + && *promoted_at_unix_millis <= self.updated_at_unix_millis, + "route promotion time is outside the durable transaction timeline" + ); + } if self.phase == DeploymentPhase::Complete { ensure!( matches!( @@ -1010,7 +1187,7 @@ impl AdmittedOdinAuthority { #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] #[cultcache( type = "idunn.admitted_generation", - schema = "idunn.admitted_generation.v1" + schema = "idunn.admitted_generation.v2" )] struct AdmittedGeneration { #[cultcache(key = 0)] @@ -1049,6 +1226,8 @@ struct AdmittedGeneration { odin_authority: AdmittedOdinAuthority, #[cultcache(key = 17)] odin_publisher_sequence_cursor: u64, + #[cultcache(key = 18)] + route_repair_started_at_unix_millis: Option, } impl AdmittedGeneration { @@ -1085,6 +1264,7 @@ impl AdmittedGeneration { routing: required(&transaction.routing, "route disposition")?.clone(), odin_authority, odin_publisher_sequence_cursor: transaction.odin_publisher_sequence_cursor, + route_repair_started_at_unix_millis: None, }; generation.validate()?; Ok(generation) @@ -1127,6 +1307,20 @@ impl AdmittedGeneration { self.expected.route.is_some() == self.routing.observation().is_some(), "admitted route disposition differs from Expected" ); + if let Some(route) = self.routing.observation() { + route.validate()?; + ensure!( + route.runtime_instance_id == self.activation.runtime_instance_id, + "admitted route observation belongs to another runtime instance" + ); + } + if let Some(started_at) = self.route_repair_started_at_unix_millis { + ensure!(started_at > 0, "admitted route repair has no start time"); + ensure!( + matches!(&self.routing, RoutingEvidence::Promoted { .. }), + "only a promoted routed generation can own route repair intent" + ); + } Ok(()) } } @@ -1532,7 +1726,7 @@ impl ControlSnapshot { }) } - fn max_odin_sequence(&self, signer_identity_id: &str) -> u64 { + fn max_odin_sequence(&self, target: &str, signer_identity_id: &str) -> u64 { let transaction_max = self .transactions .iter() @@ -1541,7 +1735,10 @@ impl ControlSnapshot { .value .latest_odin_observation .as_ref() - .filter(|evidence| evidence.signer_identity_id == signer_identity_id) + .filter(|evidence| { + stored.value.target == target + && evidence.signer_identity_id == signer_identity_id + }) .map(|evidence| evidence.publisher_sequence) }) .max() @@ -1549,7 +1746,10 @@ impl ControlSnapshot { let admitted_max = self .admitted .iter() - .filter(|stored| stored.value.odin_authority.signer_identity_id == signer_identity_id) + .filter(|stored| { + stored.value.target == target + && stored.value.odin_authority.signer_identity_id == signer_identity_id + }) .map(|stored| stored.value.odin_publisher_sequence_cursor) .max() .unwrap_or(0); @@ -1672,9 +1872,85 @@ fn load_bindings(directory: &Path) -> Result> { ); } ensure!(!bindings.is_empty(), "Idunn binding directory is empty"); + validate_route_bindings(&bindings)?; Ok(bindings) } +fn validate_route_bindings(bindings: &BTreeMap) -> Result<()> { + let routes = bindings + .iter() + .filter_map(|(target, loaded)| { + loaded + .binding + .route + .as_ref() + .map(|route| (target.as_str(), route)) + }) + .collect::>(); + validate_route_binding_set(&routes) +} + +fn validate_route_binding_set(routes: &[(&str, &RouteBinding)]) -> Result<()> { + for (index, (target, route)) in routes.iter().enumerate() { + let (stable_host, stable_port) = route.stable_socket()?; + let private_host = route + .private_host + .parse::() + .context("validated route private host stopped being an IP address")?; + ensure!( + stable_host != private_host + || !(route.private_port_start..=route.private_port_end).contains(&stable_port), + "route {target} stable socket overlaps its candidate port range" + ); + + for (other_target, other) in &routes[index + 1..] { + ensure!( + route.route_id != other.route_id, + "route id {} is shared by targets {target} and {other_target}", + route.route_id + ); + ensure!( + route.config_path != other.config_path, + "route fragment {} is shared by targets {target} and {other_target}", + route.config_path.display() + ); + let other_stable = other.stable_socket()?; + ensure!( + route.driver != other.driver || (stable_host, stable_port) != other_stable, + "stable route socket {}:{} is shared by targets {target} and {other_target}", + stable_host, + stable_port + ); + if route.driver != other.driver { + continue; + } + let other_private_host = other + .private_host + .parse::() + .context("validated route private host stopped being an IP address")?; + let ranges_overlap = route.private_host == other.private_host + && route.private_port_start <= other.private_port_end + && other.private_port_start <= route.private_port_end; + ensure!( + !ranges_overlap, + "candidate port ranges overlap for targets {target} and {other_target}" + ); + ensure!( + stable_host != other_private_host + || !(other.private_port_start..=other.private_port_end).contains(&stable_port), + "stable route socket for {target} overlaps {other_target}'s candidate range" + ); + ensure!( + other_stable.0 != private_host + || !(route.private_port_start..=route.private_port_end) + .contains(&other_stable.1), + "stable route socket for {other_target} overlaps {target}'s candidate range" + ); + } + } + Ok(()) +} + fn resolve_selector( bindings: &BTreeMap, selector: &str, @@ -1978,18 +2254,38 @@ impl Engine { validate_authenticated_evidence(evidence, &authenticated)?; } if let Some(evidence) = &transaction.warming { - let authenticated = self.authenticate_topology_bytes( - snapshot, - transaction, - &evidence.canonical_bytes, - None, - evidence.admitted_at_unix_millis, - )?; - validate_authenticated_evidence(evidence, &authenticated)?; - ensure!( - is_semantic_warming(&authenticated), - "durable Warming label is not exact semantic Warming" - ); + match evidence { + WarmingEvidence::OdinTopology { evidence } => { + let authenticated = self.authenticate_topology_bytes( + snapshot, + transaction, + &evidence.canonical_bytes, + None, + evidence.admitted_at_unix_millis, + )?; + validate_authenticated_evidence(evidence, &authenticated)?; + let incumbent_lease_sha256 = + self.incumbent_lease_sha256_for_warming(snapshot, transaction)?; + ensure!( + is_semantic_warming( + required(&transaction.expected, "Warming Expected projection",)?, + required(&transaction.activation, "Warming activation")?, + incumbent_lease_sha256.as_deref(), + &authenticated, + )?, + "durable Warming gate is not supported by current runtime evidence" + ); + } + WarmingEvidence::FirstOdinDirect { evidence } => { + self.authenticate_first_odin_warming_presence( + transaction, + &evidence.message_id, + evidence.challenged_at_unix_millis, + evidence.admitted_at_unix_millis, + &evidence.canonical_bytes, + )?; + } + } } if let Some(evidence) = &transaction.ready { let authenticated = self.authenticate_topology_bytes( @@ -2062,14 +2358,9 @@ fn serve(options: RuntimeOptions) -> Result<()> { engine.validate_durable_authority(&ControlSnapshot::read(&engine.options.state_store)?)?; loop { - if engine.resume_one_transaction()? { - thread::sleep(Duration::from_millis(engine.options.poll_millis)); - continue; - } - if engine.supervise_one_admitted_generation()? { - continue; - } - if engine.freeze_one_queued_command()? { + let transaction_progress = engine.resume_one_transaction()?; + let continuity_progress = engine.supervise_one_admitted_generation()?; + if !transaction_progress && !continuity_progress && engine.freeze_one_queued_command()? { continue; } thread::sleep(Duration::from_millis(engine.options.poll_millis)); @@ -2083,6 +2374,7 @@ impl Engine { /// own command, so one target brake cannot suspend unrelated continuity. fn resume_one_transaction(&self) -> Result { let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let mut progressed = false; let mut candidates = snapshot .transactions .iter() @@ -2099,12 +2391,22 @@ impl Engine { if snapshot.has_earlier_authority_sibling(¤t.value) { continue; } - match self.advance_transaction(current) { - Ok(()) => {} - Err(error) if current.value.phase < DeploymentPhase::Fencing => { - self.begin_pre_fencing_abort(current, error)?; + if let Err(error) = self.advance_transaction(current) { + let latest_snapshot = ControlSnapshot::read(&self.options.state_store)?; + let latest = latest_snapshot + .transactions + .iter() + .find(|stored| stored.value.transaction_id == current.value.transaction_id) + .context("transaction disappeared while recording an execution error")?; + if latest.value.is_terminal() { + progressed = true; + } else if latest.value.phase < DeploymentPhase::Fencing + && latest.value.pre_fencing_abort.is_none() + { + self.begin_pre_fencing_abort(latest, error)?; + } else { + self.record_resumable_error(latest, &error)?; } - Err(error) => self.record_resumable_error(current, &error)?, } let after = ControlSnapshot::read(&self.options.state_store)?; let live = after @@ -2113,10 +2415,10 @@ impl Engine { .find(|stored| stored.value.transaction_id == current.value.transaction_id) .context("transaction disappeared while checking scheduler progress")?; if live.envelope != current.envelope { - return Ok(true); + progressed = true; } } - Ok(false) + Ok(progressed) } fn freeze_one_queued_command(&self) -> Result { @@ -2171,7 +2473,7 @@ impl Engine { let busy = snapshot .transactions .iter() - .filter(|stored| stored.value.owns_target_authority()) + .filter(|stored| stored.value.blocks_new_target_mutation()) .map(|stored| stored.value.target.as_str()) .collect::>(); if targets.iter().any(|target| busy.contains(target.as_str())) { @@ -2213,35 +2515,161 @@ impl Engine { fn supervise_one_admitted_generation(&self) -> Result { let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let busy = snapshot - .transactions - .iter() - .filter(|stored| stored.value.owns_target_authority()) - .map(|stored| stored.value.target.as_str()) - .collect::>(); + let mut progressed = false; let mut admitted = snapshot.admitted.iter().collect::>(); admitted.sort_by_key(|stored| stored.value.target.as_str()); for current in admitted { - if busy.contains(current.value.target.as_str()) { + let blocker = snapshot.transactions.iter().find(|stored| { + stored.value.target == current.value.target + && stored.value.blocks_new_target_mutation() + }); + if blocker.is_some_and(|stored| { + stored.value.phase >= DeploymentPhase::Fencing + && stored.value.phase < DeploymentPhase::Complete + }) { continue; } + match self.supervise_admitted_route(current) { + Ok(true) => { + progressed = true; + continue; + } + Ok(false) => {} + Err(error) => eprintln!( + "Idunn rejected admitted {} route continuity: {error:#}", + current.value.target + ), + } match self.refresh_admitted_topology(&snapshot, current) { - Ok(true) => return Ok(true), + Ok(true) => { + progressed = true; + continue; + } Ok(false) => {} Err(error) => eprintln!( "Idunn preserved admitted {} after rejecting topology observation: {error:#}", current.value.target ), } - if self - .workload - .observe( - ¤t.value.expected, - ¤t.value.activation, - ¤t.value.workload, - ) - .is_ok() - { + let mut operational_error = None; + let observation = match self.workload.observe( + ¤t.value.expected, + ¤t.value.activation, + ¤t.value.workload, + ) { + Ok(observation) => { + let lease_is_missing = if let Some(lease) = current.value.leasing.lease() { + let lease_health = (|| -> Result { + let lease_path = current + .value + .plan + .parsed_inputs()? + .1 + .process_write_lease + .context("admitted lease has no operator binding")? + .record_path; + let driver = + CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); + if driver.observe_exact(lease)? { + return Ok(true); + } + if driver.observe_empty()? { + return Ok(false); + } + bail!("write-lease store contains unexpected authority") + })(); + match lease_health { + Ok(true) => false, + Ok(false) => { + operational_error = + Some(anyhow!("admitted physical write lease is missing")); + true + } + Err(error) => { + eprintln!( + "Idunn preserved admitted {} after refusing an unsafe write-lease mutation: {error:#}", + current.value.target + ); + continue; + } + } + } else { + false + }; + if lease_is_missing { + None + } else { + Some(observation) + } + } + Err(error) => { + operational_error = Some(error); + None + } + }; + if let Some(observation) = observation { + if blocker.is_none() + || blocker.is_some_and(|stored| stored.value.phase == DeploymentPhase::Complete) + { + let repair = (|| -> Result<()> { + let topology = self.topology(); + let provider_anchor = self.provider_anchor_for_plan(¤t.value.plan)?; + if !topology.admitted_runtime_projection_is_exact( + ¤t.value.expected, + &provider_anchor, + ¤t.value.activation, + current.value.leasing.lease(), + )? { + ensure!( + topology + .publish_expected(¤t.value.expected, &provider_anchor,)? + == current.value.expected.canonical_sha256()?, + "admitted Expected projection repair differs" + ); + ensure!( + topology.publish_observed_activation( + ¤t.value.expected, + ¤t.value.activation, + &observation, + )? == current.value.activation.canonical_sha256()?, + "admitted activation projection repair differs" + ); + if let Some(lease) = current.value.leasing.lease() { + ensure!( + topology.publish_process_write_lease( + ¤t.value.expected, + ¤t.value.activation, + lease, + )? == lease.canonical_sha256()?, + "admitted write-lease projection repair differs" + ); + } + } + Ok(()) + })(); + if let Err(error) = repair { + eprintln!( + "Idunn preserved admitted {} after refusing topology projection repair: {error:#}", + current.value.target + ); + } + } + continue; + } + let workload_error = operational_error + .context("admitted operational state has neither observation nor error")?; + if let Some(blocker) = blocker { + if blocker.value.phase < DeploymentPhase::Fencing + && blocker.value.pre_fencing_abort.is_none() + { + self.begin_pre_fencing_abort( + blocker, + anyhow!( + "admitted incumbent failed before candidate fencing; deployment yielded to continuity: {workload_error:#}" + ), + )?; + progressed = true; + } continue; } let now = now_millis()?; @@ -2278,9 +2706,129 @@ impl Engine { )?, "continuity scheduling lost its command/transaction CAS" ); + progressed = true; + } + Ok(progressed) + } + + fn supervise_admitted_route(&self, current: &Stored) -> Result { + let Some(expected_route) = current.value.expected.route.as_ref() else { + ensure!( + matches!(¤t.value.routing, RoutingEvidence::SkippedUnrouted) + && current.value.route_repair_started_at_unix_millis.is_none(), + "unrouted admitted generation carries route authority" + ); + return Ok(false); + }; + let RoutingEvidence::Promoted { + observation, + promoted_at_unix_millis, + } = ¤t.value.routing + else { + bail!("routed admitted generation has no promoted route receipt") + }; + ensure!( + observation.route_id == expected_route.route_id + && observation.runtime_instance_id == current.value.activation.runtime_instance_id, + "admitted route receipt names another incarnation" + ); + let binding = current.value.plan.parsed_inputs()?.1; + let lease_driver = if let Some(lease) = current.value.leasing.lease() { + let lease_path = &binding + .process_write_lease + .as_ref() + .context("stateful admitted generation has no write-lease binding")? + .record_path; + let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); + ensure!( + driver.observe_exact(lease)?, + "admitted process write lease is no longer exact" + ); + Some(driver) + } else { + None + }; + let route_binding = binding + .route + .context("routed admitted generation has no operator route binding")?; + let driver = NginxRouteDriver::new(route_binding); + + let membership_is_exact = + driver.observe_membership(¤t.value.expected, &observation.membership_sha256)?; + let now = now_millis()?; + if membership_is_exact + && route_observation_is_current( + observation.observed_at_unix_millis, + now, + self.options.topology_maximum_age_millis, + self.options.topology_maximum_future_skew_millis, + ) + && current.value.route_repair_started_at_unix_millis.is_none() + { + return Ok(false); + } + if current.value.route_repair_started_at_unix_millis.is_none() { + let mut next = current.value.clone(); + next.route_repair_started_at_unix_millis = Some(now); + next.validate()?; + ensure!( + SingleFileMessagePackBackingStore::new(&self.options.state_store) + .compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: AdmittedGeneration::TYPE.into(), + key: current.value.target.clone(), + current: Some(current.envelope.clone()), + }], + &[admitted_envelope(&next, now)?], + )?, + "admitted generation changed before route repair intent CAS" + ); return Ok(true); } - Ok(false) + driver + .restore_admitted_membership(¤t.value.expected, &observation.membership_sha256)?; + let authority = self.runtime_authority_parts( + ¤t.value.plan, + ¤t.value.expected, + ¤t.value.activation, + )?; + let refreshed = self.prove_stable_route_against( + ¤t.value.expected, + ¤t.value.activation, + &authority, + current.value.leasing.lease_sha256(), + &driver, + observation.membership_sha256.clone(), + )?; + ensure!( + driver.observe_membership(¤t.value.expected, &refreshed.membership_sha256)?, + "admitted route membership changed during its continuity challenge" + ); + if let (Some(lease_driver), Some(lease)) = (&lease_driver, current.value.leasing.lease()) { + ensure!( + lease_driver.observe_exact(lease)?, + "admitted process write lease changed during its continuity challenge" + ); + } + let mut next = current.value.clone(); + next.routing = RoutingEvidence::Promoted { + observation: refreshed, + promoted_at_unix_millis: *promoted_at_unix_millis, + }; + next.route_repair_started_at_unix_millis = None; + next.validate()?; + ensure!( + SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: AdmittedGeneration::TYPE.into(), + key: current.value.target.clone(), + current: Some(current.envelope.clone()), + }], + &[admitted_envelope(&next, now)?], + )?, + "admitted generation changed before route continuity receipt CAS" + ); + Ok(true) } fn refresh_admitted_topology( @@ -2312,7 +2860,7 @@ impl Engine { let evidence = TopologyEvidence::from_authenticated(&authenticated, now)?; if !sequence_requires_admission( Some(¤t.value.latest_odin_observation), - snapshot.max_odin_sequence(&evidence.signer_identity_id), + snapshot.max_odin_sequence(¤t.value.target, &evidence.signer_identity_id), &evidence, )? { return Ok(false); @@ -2434,8 +2982,12 @@ impl Engine { if current.value.expected_publication_sha256.is_none() { let now = now_millis()?; + let plan = required(¤t.value.plan, "transaction plan")?; let expected = required(¤t.value.expected, "Expected projection")?; - let digest = self.topology().publish_expected(expected)?; + let provider_anchor = self.provider_anchor_for_plan(plan)?; + let digest = self + .topology() + .publish_expected(expected, &provider_anchor)?; return self.persist_same_phase(current, |next| { next.expected_publication_sha256 = Some(digest); next.updated_at_unix_millis = now; @@ -2541,35 +3093,64 @@ impl Engine { let activation = required(¤t.value.activation, "activation")?; let workload = required(¤t.value.workload, "workload")?; self.workload.observe(expected, activation, workload)?; - { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let incumbent_lease_sha256 = + self.incumbent_lease_sha256_for_warming(&snapshot, ¤t.value)?; + if current.value.warming.is_none() { + if current.value.target == "odin" && snapshot.admitted_for("odin").is_none() { + ensure!( + current.value.incumbent_generation_id.is_none() + && expected.write_lease_required, + "first Odin bootstrap must be a stateful first incarnation" + ); + let (evidence, present) = self.observe_first_odin_warming(¤t.value)?; + let _token = SequenceAdmittedWarming::from_first_odin_presence( + current.value.transaction_id.clone(), + evidence.clone(), + present, + )?; + return self.persist_same_phase(current, |next| { + next.warming = Some(WarmingEvidence::FirstOdinDirect { evidence }); + Ok(()) + }); + } let Some((admitted, authenticated)) = self.admit_latest_topology(current, None)? else { return Ok(()); }; if admitted.envelope != current.envelope { return Ok(()); } - let semantic_warming = is_semantic_warming(&authenticated); + let semantic_warming = is_semantic_warming( + expected, + activation, + incumbent_lease_sha256.as_deref(), + &authenticated, + )?; let latest = required( &admitted.value.latest_odin_observation, "sequence-admitted warming evidence", )?; - if current.value.warming.as_ref() == Some(latest) { - ensure!(semantic_warming, "stored Warming receipt changed meaning"); - } else if semantic_warming { + if semantic_warming { let evidence = latest.clone(); - let _token = SequenceAdmittedWarming { - transaction_id: admitted.value.transaction_id.clone(), - evidence: evidence.clone(), + let _token = SequenceAdmittedWarming::from_topology( + admitted.value.transaction_id.clone(), + evidence.clone(), authenticated, - }; + )?; return self.persist_same_phase(&admitted, |next| { - next.warming = Some(evidence); + next.warming = Some(WarmingEvidence::OdinTopology { evidence }); Ok(()) }); } else { return Ok(()); } } + let warming = self.rehydrate_warming_token(¤t.value, now_millis()?, false)?; + ensure!( + warming.transaction_id() == current.value.transaction_id + && warming.runtime_instance_id() == activation.runtime_instance_id.as_str(), + "durable Warming evidence belongs to another candidate" + ); if expected.route.is_some() && current.value.route_preflight.is_none() { let now = now_millis()?; @@ -2632,12 +3213,21 @@ impl Engine { }) .transpose()?; if let (Some(lease), Some(path)) = (incumbent_lease, &incumbent_lease_path) { + let incumbent = incumbent.context("incumbent lease lost its generation")?; + self.workload + .stop(&incumbent.value.workload) + .context("stopping the exact incumbent before revoking its lifetime-held write lease")?; let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, path); driver.revoke_exact(Some(lease))?; ensure!( driver.observe_empty()?, "incumbent write lease remained after exact fencing" ); + self.topology().withdraw_process_write_lease( + &incumbent.value.expected, + &incumbent.value.activation, + Some(lease), + )?; } let candidate_lease_path = if expected.write_lease_required { Some( @@ -2665,6 +3255,11 @@ impl Engine { ); } } + self.topology().withdraw_process_write_lease( + expected, + required(¤t.value.activation, "candidate activation")?, + None, + )?; FencingEvidence::Revoked { incumbent_lease_sha256: incumbent_lease .map(IdunnProcessWriteLeaseRecord::canonical_sha256) @@ -2684,74 +3279,64 @@ impl Engine { } fn advance_leasing(&self, current: &Stored) -> Result<()> { - if current.value.leasing.is_none() { - let now = now_millis()?; - let expected = required(¤t.value.expected, "Expected projection")?; - let evidence = if expected.write_lease_required { - let warming = self.rehydrate_warming_token(¤t.value, now)?; - ensure!( - warming.transaction_id() == current.value.transaction_id - && Some(warming.evidence_sha256()) - == current - .value - .warming - .as_ref() - .map(|value| value.canonical_sha256.as_str()), - "Warming token does not bind this transaction's durable admission" - ); - let activation = required(¤t.value.activation, "activation")?; - let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; - let _lease_path = binding - .process_write_lease - .context("stateful target has no write-lease binding")? - .record_path; - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let epoch = self - .exact_incumbent(&snapshot, ¤t.value)? - .and_then(|generation| generation.value.leasing.lease()) - .map_or(1, |lease| lease.lease_epoch.saturating_add(1)); - let warming_presence_sha256 = warming - .authenticated() - .record() - .signed_presence_sha256 - .clone() - .context("warming token has no signed presence")?; - let lease = IdunnProcessWriteLeaseRecord { - schema_version: IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into(), - target: expected.target.clone(), - expected_projection_sha256: expected.canonical_sha256()?, - plan_id: expected.plan_id.clone(), - incarnation_id: expected.incarnation_id.clone(), - sealed_release_id: expected.sealed_release_id.clone(), - activation_witness_sha256: activation.canonical_sha256()?, - state_schema_generation: expected - .state_schema_generation - .clone() - .context("stateful Expected has no schema generation")?, - state_contract_sha256: expected - .state_contract_sha256 - .clone() - .context("stateful Expected has no state contract")?, - runtime_id: expected.runtime_id.clone(), - runtime_instance_id: activation.runtime_instance_id.clone(), - warming_presence_sha256, - lease_epoch: epoch, - issued_at_unix_millis: now, - }; - lease.validate()?; - let lease_sha256 = lease.canonical_sha256()?; - LeasingEvidence::Prepared { - lease, - lease_sha256, - } - } else { - LeasingEvidence::SkippedStateless - }; - return self.persist_same_phase(current, |next| { - next.leasing = Some(evidence); - next.updated_at_unix_millis = now; - Ok(()) - }); + let expected = required(¤t.value.expected, "Expected projection")?; + if !expected.write_lease_required { + if current.value.leasing.is_none() { + let now = now_millis()?; + return self.persist_same_phase(current, |next| { + next.leasing = Some(LeasingEvidence::SkippedStateless); + next.updated_at_unix_millis = now; + Ok(()) + }); + } + return self.transition(current, DeploymentPhase::AwaitingReady); + } + + let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; + let lease_path = binding + .process_write_lease + .context("stateful target has no write-lease binding")? + .record_path; + let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); + + if matches!( + current.value.leasing.as_ref(), + Some(LeasingEvidence::Granted { .. }) + ) { + let activation = required(¤t.value.activation, "activation")?; + let warming = self.rehydrate_warming_token(¤t.value, now_millis()?, false)?; + let lease = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease) + .context("granted Leasing evidence has no write lease")?; + let recorded_sha256 = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256) + .context("granted Leasing evidence has no lease digest")?; + ensure!( + driver.observe_exact(lease)?, + "physical write lease disappeared after Granted became durable" + ); + self.workload.observe( + expected, + activation, + required(¤t.value.workload, "candidate workload")?, + )?; + ensure!( + driver.grant(expected, activation, &warming, lease)? == recorded_sha256, + "replayed physical write lease differs from Granted evidence" + ); + ensure!( + self.topology() + .publish_process_write_lease(expected, activation, lease)? + == recorded_sha256, + "replayed write-lease projection differs from Granted evidence" + ); + return self.transition(current, DeploymentPhase::AwaitingReady); } if let Some((lease, prepared_sha256)) = current @@ -2761,42 +3346,101 @@ impl Engine { .and_then(LeasingEvidence::prepared_lease) { let now = now_millis()?; - let expected = required(¤t.value.expected, "Expected projection")?; let activation = required(¤t.value.activation, "activation")?; - let warming = self.rehydrate_warming_token(¤t.value, now)?; - ensure!( - warming.transaction_id() == current.value.transaction_id - && Some(warming.evidence_sha256()) - == current - .value - .warming - .as_ref() - .map(|value| value.canonical_sha256.as_str()), - "Warming token does not bind this transaction's durable admission" - ); - let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; - let lease_path = binding - .process_write_lease - .context("stateful target has no write-lease binding")? - .record_path; - let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); + let historical_warming = self.rehydrate_warming_token(¤t.value, now, false)?; + let physical_is_exact = driver.observe_exact(lease)?; + let warming = if physical_is_exact { + historical_warming + } else { + ensure!( + driver.observe_empty()?, + "candidate write-lease store contains authority other than its durable prepared lease" + ); + match self.rehydrate_warming_token(¤t.value, now, true) { + Ok(warming) => warming, + Err(_) => { + let Some((admitted, fresh_evidence, fresh_warming)) = + self.fresh_warming_for_lease(current, now)? + else { + return Ok(()); + }; + let replacement = self.prepare_candidate_write_lease( + &admitted.value, + &fresh_warming, + now, + )?; + let replacement_sha256 = replacement.canonical_sha256()?; + return self.persist_same_phase(&admitted, |next| { + next.warming = Some(fresh_evidence); + next.leasing = Some(LeasingEvidence::Prepared { + lease: replacement, + lease_sha256: replacement_sha256, + }); + next.updated_at_unix_millis = now; + Ok(()) + }); + } + } + }; + self.workload.observe( + expected, + activation, + required(¤t.value.workload, "candidate workload")?, + )?; + if !physical_is_exact { + ensure!( + driver.observe_empty()?, + "candidate write-lease store changed before physical grant" + ); + } let granted_sha256 = driver.grant(expected, activation, &warming, lease)?; ensure!( granted_sha256 == prepared_sha256, "granted write lease differs from the durable prepared lease" ); + let projected_sha256 = self + .topology() + .publish_process_write_lease(expected, activation, lease)?; + ensure!( + projected_sha256 == granted_sha256, + "projected write lease differs from the granted process authority" + ); let lease = lease.clone(); return self.persist_same_phase(current, |next| { next.leasing = Some(LeasingEvidence::Granted { lease, - lease_sha256: granted_sha256, + lease_sha256: projected_sha256, }); next.updated_at_unix_millis = now; Ok(()) }); } - self.transition(current, DeploymentPhase::AwaitingReady) + ensure!( + current.value.leasing.is_none(), + "stateful Leasing phase carries invalid lease evidence" + ); + ensure!( + driver.observe_empty()?, + "candidate write-lease store contains authority before lease preparation" + ); + let now = now_millis()?; + let Some((admitted, fresh_evidence, fresh_warming)) = + self.fresh_warming_for_lease(current, now)? + else { + return Ok(()); + }; + let lease = self.prepare_candidate_write_lease(&admitted.value, &fresh_warming, now)?; + let lease_sha256 = lease.canonical_sha256()?; + self.persist_same_phase(&admitted, |next| { + next.warming = Some(fresh_evidence); + next.leasing = Some(LeasingEvidence::Prepared { + lease, + lease_sha256, + }); + next.updated_at_unix_millis = now; + Ok(()) + }) } fn advance_awaiting_ready(&self, current: &Stored) -> Result<()> { @@ -2804,7 +3448,7 @@ impl Engine { let expected = required(¤t.value.expected, "Expected projection")?; if expected.write_lease_required { let activation = required(¤t.value.activation, "activation")?; - let warming = self.rehydrate_warming_token(¤t.value, now)?; + let warming = self.rehydrate_warming_token(¤t.value, now, false)?; let lease = current .value .leasing @@ -2863,43 +3507,108 @@ impl Engine { } fn advance_routing(&self, current: &Stored) -> Result<()> { - let now = now_millis()?; let expected = required(¤t.value.expected, "Expected projection")?; let activation = required(¤t.value.activation, "activation")?; if current.value.routing.is_none() { - let ready = self.rehydrate_ready_token(¤t.value, now, false)?; + let current_lease = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256); + let Some((admitted, authenticated)) = + self.admit_latest_topology(current, current_lease)? + else { + return Ok(()); + }; + if admitted.envelope != current.envelope { + return Ok(()); + } ensure!( - ready.transaction_id() == current.value.transaction_id, + is_semantic_ready(&authenticated), + "latest Odin observation is not Ready at route admission" + ); + let latest = required( + &admitted.value.latest_odin_observation, + "current route-admission topology evidence", + )?; + if admitted.value.ready.as_ref() != Some(latest) { + let latest = latest.clone(); + return self.persist_same_phase(&admitted, |next| { + next.ready = Some(latest); + Ok(()) + }); + } + let ready = self.rehydrate_ready_token(&admitted.value, now_millis()?, true)?; + ensure!( + ready.transaction_id() == admitted.value.transaction_id, "Ready token belongs to another transaction" ); + validate_live_providers_for_deploy(admitted.value.command_kind, || { + self.validate_selected_providers_current( + required(&admitted.value.plan, "transaction plan")?, + now_millis()?, + ) + })?; + self.ensure_transaction_write_lease_current(&admitted.value, now_millis()?)?; let evidence = if expected.route.is_some() { - let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; + let binding = admitted.value.plan.as_ref().unwrap().parsed_inputs()?.1; let route_binding = binding.route.context("routed plan has no route binding")?; - let preflight = required(¤t.value.route_preflight, "route preflight")?; + let preflight = required(&admitted.value.route_preflight, "route preflight")?; let driver = NginxRouteDriver::new(route_binding); - let exact_candidate = RouteObservation { - route_id: preflight.route_id.clone(), - runtime_instance_id: preflight.candidate_runtime_instance_id.clone(), - membership_sha256: preflight.candidate_membership_sha256.clone(), - }; ensure!( - exact_candidate.runtime_instance_id == activation.runtime_instance_id, + preflight.candidate_runtime_instance_id == activation.runtime_instance_id, "route preflight belongs to another runtime instance" ); - let observation = if driver.observe(expected, &exact_candidate)? { - exact_candidate - } else { - driver.promote(expected, &activation.runtime_instance_id, preflight)? + let rollback_allowed = may_rollback_route_after_failed_proof(required( + &admitted.value.fencing, + "route fencing evidence", + )?); + let membership_sha256 = driver.install( + expected, + &activation.runtime_instance_id, + preflight, + rollback_allowed, + )?; + let observation = match self.prove_stable_route( + &admitted.value, + &driver, + membership_sha256, + ) { + Ok(observation) => observation, + Err(proof_error) if !rollback_allowed => { + return Err(proof_error).context( + "incumbent authority was fenced; candidate route remains installed for fail-closed retry", + ); + } + Err(proof_error) => { + return match driver.rollback( + expected, + &activation.runtime_instance_id, + preflight, + ) { + Ok(()) => Err(proof_error) + .context("candidate did not answer its stable route challenge"), + Err(rollback_error) => Err(proof_error).context(format!( + "candidate did not answer its stable route challenge; exact route rollback also failed: {rollback_error:#}" + )), + }; + } }; ensure!( - driver.observe(expected, &observation)?, - "promoted route is not the exact candidate membership" + driver.observe_membership(expected, &observation.membership_sha256)?, + "route membership changed during its signed stable-listener observation" ); - RoutingEvidence::Promoted { observation } + self.ensure_transaction_write_lease_current(&admitted.value, now_millis()?)?; + let promoted_at_unix_millis = observation.observed_at_unix_millis; + RoutingEvidence::Promoted { + observation, + promoted_at_unix_millis, + } } else { RoutingEvidence::SkippedUnrouted }; - return self.persist_same_phase(current, |next| { + let now = now_millis()?; + return self.persist_same_phase(&admitted, |next| { next.routing = Some(evidence); next.updated_at_unix_millis = now; Ok(()) @@ -2909,64 +3618,145 @@ impl Engine { } fn advance_committing(&self, current: &Stored) -> Result<()> { - let now = now_millis()?; - let expected = required(¤t.value.expected, "Expected projection")?; - let activation = required(¤t.value.activation, "activation")?; - let workload = required(¤t.value.workload, "workload")?; - self.workload.observe(expected, activation, workload)?; - let _ready = self.rehydrate_ready_token(¤t.value, now, false)?; - - if let Some(lease) = current + let current_lease_sha256 = current .value .leasing .as_ref() - .and_then(LeasingEvidence::lease) - { - let warming = self.rehydrate_warming_token(¤t.value, now)?; - let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; - let path = binding - .process_write_lease - .context("stateful target has no write-lease binding")? - .record_path; - ensure!( - CultCacheWriteLeaseDriver::new(¤t.value.target, path) - .observe(expected, activation, &warming, lease)?, - "write lease changed before admission commit" - ); + .and_then(LeasingEvidence::lease_sha256) + .map(str::to_owned); + let Some((ready_current, authenticated)) = + self.admit_latest_topology(current, current_lease_sha256.as_deref())? + else { + return Ok(()); + }; + if ready_current.envelope != current.envelope { + return Ok(()); } - if let Some(route) = current + ensure!( + is_semantic_ready(&authenticated), + "latest Odin observation is not Ready at admission commit" + ); + ensure!( + ready_current.value.latest_odin_observation.as_ref() + == ready_current.value.ready.as_ref(), + "latest Odin observation differs from the durable Ready receipt at admission commit" + ); + self.rehydrate_ready_token(&ready_current.value, now_millis()?, true)?; + validate_live_providers_for_deploy(ready_current.value.command_kind, || { + self.validate_selected_providers_current( + required(&ready_current.value.plan, "transaction plan")?, + now_millis()?, + ) + })?; + + let expected = required(&ready_current.value.expected, "Expected projection")?; + let activation = required(&ready_current.value.activation, "activation")?; + let workload = required(&ready_current.value.workload, "workload")?; + self.workload.observe(expected, activation, workload)?; + self.ensure_transaction_write_lease_current(&ready_current.value, now_millis()?)?; + if let Some(route) = ready_current .value .routing .as_ref() .and_then(RoutingEvidence::observation) { - let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; + let binding = ready_current + .value + .plan + .as_ref() + .unwrap() + .parsed_inputs()? + .1; let driver = NginxRouteDriver::new(binding.route.context("routed plan has no route binding")?); ensure!( - driver.observe(expected, route)?, - "route changed before admission commit" + driver.observe_membership(expected, &route.membership_sha256)?, + "route membership changed before admission commit" ); + let current_route = self.prove_stable_route( + &ready_current.value, + &driver, + route.membership_sha256.clone(), + )?; + ensure!( + current_route.route_id == route.route_id + && current_route.runtime_instance_id == route.runtime_instance_id, + "stable route changed incarnation before admission commit" + ); + ensure!( + driver.observe_membership(expected, &route.membership_sha256)?, + "route membership changed during the final signed admission challenge" + ); + self.ensure_transaction_write_lease_current(&ready_current.value, now_millis()?)?; } + let current_lease_sha256 = ready_current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256) + .map(str::to_owned); + let Some((commit_current, authenticated)) = + self.admit_latest_topology(&ready_current, current_lease_sha256.as_deref())? + else { + return Ok(()); + }; + if commit_current.envelope != ready_current.envelope { + return Ok(()); + } + ensure!( + is_semantic_ready(&authenticated), + "latest Odin observation is not Ready after the final admission challenge" + ); + ensure!( + commit_current.value.latest_odin_observation.as_ref() + == commit_current.value.ready.as_ref(), + "latest Odin observation differs from the durable Ready receipt after the final admission challenge" + ); + let now = now_millis()?; + self.rehydrate_ready_token(&commit_current.value, now, true)?; + validate_live_providers_for_deploy(commit_current.value.command_kind, || { + self.validate_selected_providers_current( + required(&commit_current.value.plan, "transaction plan")?, + now, + ) + })?; + self.workload.observe( + required(&commit_current.value.expected, "Expected projection")?, + required(&commit_current.value.activation, "activation")?, + required(&commit_current.value.workload, "workload")?, + )?; + self.ensure_transaction_write_lease_current(&commit_current.value, now)?; + let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let incumbent = self.exact_incumbent(&snapshot, ¤t.value)?; + let incumbent = self.exact_incumbent(&snapshot, &commit_current.value)?; let odin_authority = self.current_odin_authority(&snapshot)?; - let generation = AdmittedGeneration::from_transaction(¤t.value, odin_authority, now)?; + let generation = + AdmittedGeneration::from_transaction(&commit_current.value, odin_authority, now)?; let post_commit_cleanup = PostCommitCleanup { incumbent: match incumbent { + Some(incumbent) + if incumbent_was_stopped_during_fencing(required( + &commit_current.value.fencing, + "commit fencing evidence", + )?) => + { + IncumbentCleanupEvidence::Complete { + generation_id: incumbent.value.generation_id.clone(), + } + } Some(incumbent) => IncumbentCleanupEvidence::Pending { generation_id: incumbent.value.generation_id.clone(), workload: incumbent.value.workload.clone(), }, None => IncumbentCleanupEvidence::SkippedNoIncumbent, }, - source: match current.value.command_kind { + source: match commit_current.value.command_kind { CommandKind::Deploy => SourceCleanupEvidence::Pending, CommandKind::Continuity => SourceCleanupEvidence::SkippedContinuity, }, }; - let mut complete = current.value.clone(); + let mut complete = commit_current.value.clone(); complete.phase = DeploymentPhase::Complete; complete.updated_at_unix_millis = now; complete.last_error = None; @@ -2986,8 +3776,8 @@ impl Engine { &[ CultCacheExpectedEnvelope { r#type: DeploymentTransaction::TYPE.into(), - key: current.value.transaction_id.clone(), - current: Some(current.envelope.clone()), + key: commit_current.value.transaction_id.clone(), + current: Some(commit_current.envelope.clone()), }, admitted_expected, ], @@ -3016,6 +3806,187 @@ impl Engine { } } + fn incumbent_lease_sha256_for_warming( + &self, + snapshot: &ControlSnapshot, + transaction: &DeploymentTransaction, + ) -> Result> { + if let Some(fencing) = &transaction.fencing { + return match fencing { + FencingEvidence::SkippedStateless => Ok(None), + FencingEvidence::Revoked { + incumbent_lease_sha256, + .. + } => Ok(incumbent_lease_sha256.clone()), + }; + } + self.exact_incumbent(snapshot, transaction)? + .and_then(|incumbent| incumbent.value.leasing.lease()) + .map(IdunnProcessWriteLeaseRecord::canonical_sha256) + .transpose() + } + + fn fresh_warming_for_lease( + &self, + current: &Stored, + now: u64, + ) -> Result< + Option<( + Stored, + WarmingEvidence, + SequenceAdmittedWarming, + )>, + > { + let prior = self.rehydrate_warming_token(¤t.value, now, false)?; + match required(¤t.value.warming, "pre-fence Warming evidence")?.clone() { + WarmingEvidence::FirstOdinDirect { .. } => { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + ensure!( + snapshot.admitted_for("odin").is_none() + && self.exact_incumbent(&snapshot, ¤t.value)?.is_none(), + "direct first-Odin Warming refresh found an admitted Odin" + ); + let (evidence, present) = self.observe_first_odin_warming(¤t.value)?; + let token = SequenceAdmittedWarming::from_first_odin_presence( + current.value.transaction_id.clone(), + evidence.clone(), + present, + )?; + ensure!( + token.signed_presence_sha256() != prior.signed_presence_sha256(), + "first Odin replayed its pre-fence Warming presence" + ); + Ok(Some(( + current.clone(), + WarmingEvidence::FirstOdinDirect { evidence }, + token, + ))) + } + WarmingEvidence::OdinTopology { + evidence: prior_evidence, + } => { + let Some((admitted, authenticated)) = self.admit_latest_topology(current, None)? + else { + return Ok(None); + }; + let fresh_evidence = required( + &admitted.value.latest_odin_observation, + "post-fence Odin Warming observation", + )? + .clone(); + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let incumbent_lease_sha256 = + self.incumbent_lease_sha256_for_warming(&snapshot, &admitted.value)?; + if !is_semantic_warming( + required(&admitted.value.expected, "Warming Expected projection")?, + required(&admitted.value.activation, "Warming activation")?, + incumbent_lease_sha256.as_deref(), + &authenticated, + )? { + return Ok(None); + } + let token = SequenceAdmittedWarming::from_topology( + admitted.value.transaction_id.clone(), + fresh_evidence.clone(), + authenticated, + )?; + if !provider_warming_advanced( + prior_evidence.publisher_sequence, + prior.signed_presence_sha256(), + fresh_evidence.publisher_sequence, + token.signed_presence_sha256(), + ) { + return Ok(None); + } + Ok(Some(( + admitted, + WarmingEvidence::OdinTopology { + evidence: fresh_evidence, + }, + token, + ))) + } + } + } + + fn prepare_candidate_write_lease( + &self, + transaction: &DeploymentTransaction, + warming: &SequenceAdmittedWarming, + now: u64, + ) -> Result { + let expected = required(&transaction.expected, "Expected projection")?; + let activation = required(&transaction.activation, "activation")?; + ensure!( + expected.write_lease_required + && warming.transaction_id() == transaction.transaction_id + && warming.runtime_instance_id() == activation.runtime_instance_id, + "fresh Warming token does not own this stateful candidate" + ); + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let epoch = self + .exact_incumbent(&snapshot, transaction)? + .and_then(|generation| generation.value.leasing.lease()) + .map_or(1, |lease| lease.lease_epoch.saturating_add(1)); + let lease = IdunnProcessWriteLeaseRecord { + schema_version: IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into(), + target: expected.target.clone(), + expected_projection_sha256: expected.canonical_sha256()?, + plan_id: expected.plan_id.clone(), + incarnation_id: expected.incarnation_id.clone(), + sealed_release_id: expected.sealed_release_id.clone(), + activation_witness_sha256: activation.canonical_sha256()?, + state_schema_generation: expected + .state_schema_generation + .clone() + .context("stateful Expected has no schema generation")?, + state_contract_sha256: expected + .state_contract_sha256 + .clone() + .context("stateful Expected has no state contract")?, + runtime_id: expected.runtime_id.clone(), + runtime_instance_id: activation.runtime_instance_id.clone(), + warming_presence_sha256: warming.signed_presence_sha256().to_owned(), + lease_epoch: epoch, + issued_at_unix_millis: now, + }; + lease.validate()?; + Ok(lease) + } + + fn ensure_transaction_write_lease_current( + &self, + transaction: &DeploymentTransaction, + now: u64, + ) -> Result<()> { + let expected = required(&transaction.expected, "Expected projection")?; + let lease = transaction + .leasing + .as_ref() + .and_then(LeasingEvidence::lease); + ensure!( + expected.write_lease_required == lease.is_some(), + "transaction write-lease disposition differs from Expected" + ); + let Some(lease) = lease else { + return Ok(()); + }; + let activation = required(&transaction.activation, "activation")?; + let warming = self.rehydrate_warming_token(transaction, now, false)?; + let lease_path = required(&transaction.plan, "transaction plan")? + .parsed_inputs()? + .1 + .process_write_lease + .context("stateful target has no write-lease binding")? + .record_path; + ensure!( + CultCacheWriteLeaseDriver::new(&transaction.target, lease_path) + .observe(expected, activation, &warming, lease,)?, + "process write lease changed across the authority boundary" + ); + Ok(()) + } + fn select_candidate_port(&self, binding: &OperatorBinding) -> Result> { let Some(route) = &binding.route else { return Ok(None); @@ -3030,7 +4001,7 @@ impl Engine { snapshot .transactions .iter() - .filter(|stored| stored.value.owns_target_authority()) + .filter(|stored| stored.value.blocks_new_target_mutation()) .filter_map(|stored| stored.value.plan.as_ref()), ) { @@ -3143,16 +4114,203 @@ impl Engine { ) } + fn authenticate_routed_presence( + &self, + authority: &cultnet_rs::VerifiedRuntimeAuthority, + current_write_lease_sha256: Option<&str>, + message_id: &str, + challenged_at_unix_millis: u64, + received_at_unix_millis: u64, + canonical_presence: &[u8], + ) -> Result<(String, u64)> { + ensure!( + received_at_unix_millis >= challenged_at_unix_millis, + "route observation predates its challenge" + ); + let authenticated = authenticate_runtime_presence_claim( + canonical_presence, + authority, + RuntimePresenceAuthenticationContext { + trusted_received_at_unix_millis: received_at_unix_millis, + maximum_age_millis: self.options.topology_maximum_age_millis, + maximum_future_skew_millis: self.options.topology_maximum_future_skew_millis, + }, + )?; + let signed_presence_sha256 = authenticated.signed_presence_sha256().to_owned(); + let correlation = correlate_runtime_presence_claim(authenticated, authority)?; + ensure!( + correlation.disagreements().is_empty(), + "stable route answered with a runtime that disagrees with current authority" + ); + let present = correlation.into_undisputed_present()?; + let presence = present.record(); + ensure!( + presence.observed_at_unix_millis >= challenged_at_unix_millis, + "stable route returned a presence minted before the route challenge" + ); + ensure!( + presence.state == "active", + "stable route runtime is not Active" + ); + ensure!( + presence.detail == format!("route-observation:{message_id}"), + "stable route response is not bound to the exact challenge" + ); + ensure!( + presence.write_lease_sha256.as_deref() == current_write_lease_sha256, + "stable route runtime does not hold the exact current process write lease" + ); + Ok((signed_presence_sha256, received_at_unix_millis)) + } + + fn authenticate_first_odin_warming_presence( + &self, + transaction: &DeploymentTransaction, + message_id: &str, + challenged_at_unix_millis: u64, + received_at_unix_millis: u64, + canonical_presence: &[u8], + ) -> Result { + ensure!( + transaction.target == "odin" + && transaction.incumbent_generation_id.is_none() + && required(&transaction.expected, "first Odin Expected")?.write_lease_required, + "direct Warming presence is reserved for stateful first Odin bootstrap" + ); + ensure!( + received_at_unix_millis >= challenged_at_unix_millis, + "first Odin Warming observation predates its challenge" + ); + let authority = self.runtime_authority(transaction)?; + let authenticated = authenticate_runtime_presence_claim( + canonical_presence, + &authority, + RuntimePresenceAuthenticationContext { + trusted_received_at_unix_millis: received_at_unix_millis, + maximum_age_millis: self.options.topology_maximum_age_millis, + maximum_future_skew_millis: self.options.topology_maximum_future_skew_millis, + }, + )?; + let correlation = correlate_runtime_presence_claim(authenticated, &authority)?; + ensure!( + correlation.disagreements().is_empty(), + "first Odin Warming presence disagrees with its Expected activation" + ); + let present = correlation.into_undisputed_present()?; + let presence = present.record(); + ensure!( + presence.observed_at_unix_millis >= challenged_at_unix_millis + && presence.state == "warming" + && presence.write_lease_sha256.is_none() + && presence.detail == format!("idunn-warming:{message_id}"), + "first Odin candidate did not return exact fresh pre-lease Warming evidence" + ); + Ok(present) + } + + fn observe_first_odin_warming( + &self, + transaction: &DeploymentTransaction, + ) -> Result<(RuntimePresenceEvidence, cultnet_rs::VerifiedRuntimePresence)> { + let expected = required(&transaction.expected, "first Odin Expected")?; + let binding = required(&transaction.plan, "first Odin plan")? + .parsed_inputs()? + .1; + let driver = NginxRouteDriver::new( + binding + .route + .context("first Odin bootstrap has no candidate route binding")?, + ); + let message_id = format!("warming-{}", Uuid::new_v4().simple()); + let challenged_at_unix_millis = now_millis()?; + let response = driver.request_candidate_runtime_presence(expected, &message_id)?; + ensure!( + response.message_id == message_id, + "first Odin candidate transport substituted its challenge identity" + ); + let admitted_at_unix_millis = now_millis()?; + let present = self.authenticate_first_odin_warming_presence( + transaction, + &message_id, + challenged_at_unix_millis, + admitted_at_unix_millis, + &response.canonical_presence, + )?; + let evidence = RuntimePresenceEvidence::from_present( + &present, + message_id, + challenged_at_unix_millis, + admitted_at_unix_millis, + )?; + Ok((evidence, present)) + } + + fn prove_stable_route( + &self, + transaction: &DeploymentTransaction, + driver: &NginxRouteDriver, + membership_sha256: String, + ) -> Result { + let expected = required(&transaction.expected, "route Expected projection")?; + let activation = required(&transaction.activation, "route activation")?; + let authority = self.runtime_authority(transaction)?; + let current_write_lease_sha256 = transaction + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256); + self.prove_stable_route_against( + expected, + activation, + &authority, + current_write_lease_sha256, + driver, + membership_sha256, + ) + } + + fn prove_stable_route_against( + &self, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + authority: &cultnet_rs::VerifiedRuntimeAuthority, + current_write_lease_sha256: Option<&str>, + driver: &NginxRouteDriver, + membership_sha256: String, + ) -> Result { + let message_id = format!("route-{}", Uuid::new_v4().simple()); + let challenged_at_unix_millis = now_millis()?; + let response = driver.request_runtime_presence(expected, &message_id)?; + ensure!( + response.message_id == message_id, + "route transport substituted its challenge identity" + ); + let received_at_unix_millis = now_millis()?; + let (signed_presence_sha256, observed_at_unix_millis) = self.authenticate_routed_presence( + authority, + current_write_lease_sha256, + &message_id, + challenged_at_unix_millis, + received_at_unix_millis, + &response.canonical_presence, + )?; + let observation = RouteObservation { + route_id: driver.binding.route_id.clone(), + runtime_instance_id: activation.runtime_instance_id.clone(), + membership_sha256, + signed_presence_sha256, + observed_at_unix_millis, + }; + observation.validate()?; + Ok(observation) + } + fn runtime_authority_parts( &self, plan: &CompiledDeploymentPlan, expected: &IdunnExpectedIncarnationRecord, activation: &IdunnRuntimeActivationRecord, ) -> Result { - let binding = plan.parsed_inputs()?.1; - let provider_anchor = read_trust_anchor::( - &binding.runtime_identity.trust_anchor_store, - )?; + let provider_anchor = self.provider_anchor_for_plan(plan)?; verify_runtime_authority( expected, activation, @@ -3161,6 +4319,22 @@ impl Engine { ) } + fn provider_anchor_for_plan( + &self, + plan: &CompiledDeploymentPlan, + ) -> Result { + let binding = plan.parsed_inputs()?.1; + let provider_anchor = read_trust_anchor::( + &binding.runtime_identity.trust_anchor_store, + )?; + ensure!( + provider_anchor.schema_version + == ::TRUST_ANCHOR_SCHEMA, + "provider runtime presence trust anchor schema is unsupported" + ); + Ok(provider_anchor) + } + fn authenticate_topology_bytes( &self, snapshot: &ControlSnapshot, @@ -3220,7 +4394,7 @@ impl Engine { let evidence = TopologyEvidence::from_authenticated(&authenticated, now)?; if !sequence_requires_admission( live.value.latest_odin_observation.as_ref(), - snapshot.max_odin_sequence(&evidence.signer_identity_id), + snapshot.max_odin_sequence(&live.value.target, &evidence.signer_identity_id), &evidence, )? { return Ok(Some((live.clone(), authenticated))); @@ -3247,27 +4421,61 @@ impl Engine { fn rehydrate_warming_token( &self, transaction: &DeploymentTransaction, - _now: u64, + now: u64, + require_current: bool, ) -> Result { let evidence = required(&transaction.warming, "durable warming evidence")?.clone(); - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let authenticated = self.authenticate_topology_bytes( - &snapshot, - transaction, - &evidence.canonical_bytes, - None, - evidence.admitted_at_unix_millis, - )?; - validate_authenticated_evidence(&evidence, &authenticated)?; - ensure!( - is_semantic_warming(&authenticated), - "durable warming evidence no longer authenticates as Warming" - ); - Ok(SequenceAdmittedWarming { - transaction_id: transaction.transaction_id.clone(), - evidence, - authenticated, - }) + match evidence { + WarmingEvidence::OdinTopology { evidence } => { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let authenticated = self.authenticate_topology_bytes( + &snapshot, + transaction, + &evidence.canonical_bytes, + None, + if require_current { + now + } else { + evidence.admitted_at_unix_millis + }, + )?; + validate_authenticated_evidence(&evidence, &authenticated)?; + let incumbent_lease_sha256 = + self.incumbent_lease_sha256_for_warming(&snapshot, transaction)?; + ensure!( + is_semantic_warming( + required(&transaction.expected, "Warming Expected projection")?, + required(&transaction.activation, "Warming activation")?, + incumbent_lease_sha256.as_deref(), + &authenticated, + )?, + "durable warming evidence no longer satisfies the Warming gate" + ); + SequenceAdmittedWarming::from_topology( + transaction.transaction_id.clone(), + evidence, + authenticated, + ) + } + WarmingEvidence::FirstOdinDirect { evidence } => { + let present = self.authenticate_first_odin_warming_presence( + transaction, + &evidence.message_id, + evidence.challenged_at_unix_millis, + if require_current { + now + } else { + evidence.admitted_at_unix_millis + }, + &evidence.canonical_bytes, + )?; + SequenceAdmittedWarming::from_first_odin_presence( + transaction.transaction_id.clone(), + evidence, + present, + ) + } + } } fn rehydrate_ready_token( @@ -3399,16 +4607,32 @@ impl Engine { && admitted.value.plan.plan_id == *plan_id && admitted.value.sealed_release.sealed_release_id == *sealed_release_id && admitted.value.expected.canonical_sha256()? == *expected_projection_sha256 - && token.evidence_sha256() == odin_topology_correlation_sha256 - && token.publisher_sequence() == *odin_topology_publisher_sequence, + && token.publisher_sequence() >= *odin_topology_publisher_sequence, "selected dependency authority changed before actuation" ); - if selection.requirement.startup == StartupOrder::BeforeStart { - ensure!( - token.authenticated().record().ready, - "before-start dependency is not Ready" - ); - } + ensure!( + token.publisher_sequence() != *odin_topology_publisher_sequence + || token.evidence_sha256() == odin_topology_correlation_sha256, + "selected dependency Odin sequence changed evidence" + ); + ensure!( + token + .authenticated() + .record() + .observed_capabilities + .iter() + .any(|capability| { + capability_compatible( + &selection.requirement.capability, + &selection.requirement.schema, + &selection.requirement.compatibility, + &capability.capability, + &capability.schema, + &capability.compatibility, + ) && capability.capacity >= selection.requirement.minimum_capacity + }), + "selected dependency no longer provides its required capability" + ); } Ok(()) } @@ -3531,29 +4755,35 @@ impl Engine { let snapshot = ControlSnapshot::read(&self.options.state_store)?; if let Some(incumbent) = self.exact_incumbent(&snapshot, ¤t.value)? { let topology = self.topology(); - let expected_sha256 = topology.publish_expected(&incumbent.value.expected)?; + let failed_provider_anchor = self.provider_anchor_for_plan(required( + ¤t.value.plan, + "failed transaction plan", + )?)?; + let admitted_provider_anchor = + self.provider_anchor_for_plan(&incumbent.value.plan)?; + let expected_sha256 = topology.restore_admitted_expected_only( + expected, + &failed_provider_anchor, + current.value.activation.as_ref(), + &incumbent.value.expected, + &admitted_provider_anchor, + &incumbent.value.activation, + incumbent.value.leasing.lease(), + )?; ensure!( expected_sha256 == incumbent.value.expected.canonical_sha256()?, "restored incumbent Expected receipt differs" ); - if let Ok(observation) = self.workload.observe( - &incumbent.value.expected, - &incumbent.value.activation, - &incumbent.value.workload, - ) { - let activation_sha256 = topology.publish_observed_activation( - &incumbent.value.expected, - &incumbent.value.activation, - &observation, - )?; - ensure!( - activation_sha256 == incumbent.value.activation.canonical_sha256()?, - "restored incumbent activation receipt differs" - ); - } } else { + let plan = required(¤t.value.plan, "failed transaction plan")?; + let provider_anchor = self.provider_anchor_for_plan(plan)?; self.topology() - .withdraw_expected(expected) + .withdraw_expected( + expected, + &provider_anchor, + current.value.activation.as_ref(), + None, + ) .context("withdrawing exact failed Expected projection")?; } return self.persist_same_phase(current, |next| { @@ -3600,6 +4830,21 @@ impl Engine { workload, } = &cleanup.incumbent { + let binding = required(¤t.value.plan, "committed transaction plan")? + .parsed_inputs()? + .1; + if let Some(promoted_at_unix_millis) = required( + ¤t.value.routing, + "committed transaction routing evidence", + )? + .promoted_at_unix_millis() + { + let retire_not_before = + route_drain_deadline(promoted_at_unix_millis, binding.rollout.drain_seconds)?; + if now_millis()? < retire_not_before { + return Ok(()); + } + } self.workload .stop(workload) .with_context(|| format!("retiring admitted incumbent {generation_id}"))?; @@ -3627,6 +4872,50 @@ impl Engine { } } +fn route_drain_deadline(promoted_at_unix_millis: u64, drain_seconds: u32) -> Result { + promoted_at_unix_millis + .checked_add( + u64::from(drain_seconds) + .checked_mul(1_000) + .context("route drain duration overflows milliseconds")?, + ) + .context("route drain deadline overflows Unix milliseconds") +} + +fn route_observation_is_current( + observed_at_unix_millis: u64, + now_unix_millis: u64, + maximum_age_millis: u64, + maximum_future_skew_millis: u64, +) -> bool { + observed_at_unix_millis <= now_unix_millis.saturating_add(maximum_future_skew_millis) + && now_unix_millis.saturating_sub(observed_at_unix_millis) <= maximum_age_millis +} + +fn provider_warming_advanced( + prior_odin_sequence: u64, + prior_signed_presence_sha256: &str, + candidate_odin_sequence: u64, + candidate_signed_presence_sha256: &str, +) -> bool { + candidate_odin_sequence > prior_odin_sequence + && candidate_signed_presence_sha256 != prior_signed_presence_sha256 +} + +fn may_rollback_route_after_failed_proof(fencing: &FencingEvidence) -> bool { + matches!(fencing, FencingEvidence::SkippedStateless) +} + +fn incumbent_was_stopped_during_fencing(fencing: &FencingEvidence) -> bool { + matches!( + fencing, + FencingEvidence::Revoked { + incumbent_lease_sha256: Some(_), + .. + } + ) +} + fn candidate_cleanup_requirement( has_prepared_activation: bool, has_workload_observation: bool, @@ -3734,13 +5023,54 @@ fn validate_authenticated_evidence( Ok(()) } -fn is_semantic_warming(authenticated: &AuthenticatedOdinRuntimeTopologyCorrelation) -> bool { +fn is_semantic_warming( + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + incumbent_lease_sha256: Option<&str>, + authenticated: &AuthenticatedOdinRuntimeTopologyCorrelation, +) -> Result { let record = authenticated.record(); - record.present - && !record.ready - && record.observed_presence_state.as_deref() == Some("warming") - && record.observed_write_lease_sha256.is_none() - && record.disagreements.is_empty() + if !record.present || record.observed_write_lease_sha256.is_some() { + return Ok(false); + } + let expected_projection_detail = format!( + "expected:{};activation:{}", + expected.canonical_sha256()?, + activation.canonical_sha256()? + ); + if !warming_disagreements_match_incumbent( + &expected_projection_detail, + incumbent_lease_sha256, + &record.disagreements, + ) { + return Ok(false); + } + Ok(if expected.write_lease_required { + !record.ready && record.observed_presence_state.as_deref() == Some("warming") + } else { + matches!( + record.observed_presence_state.as_deref(), + Some("warming" | "active") + ) + }) +} + +fn warming_disagreements_match_incumbent( + expected_projection_detail: &str, + incumbent_lease_sha256: Option<&str>, + disagreements: &[OdinTopologyDisagreement], +) -> bool { + if disagreements.is_empty() { + true + } else if let (Some(incumbent_lease_sha256), [disagreement]) = + (incumbent_lease_sha256, disagreements) + { + disagreement.code == "projected-write-lease" + && disagreement.expected.as_deref() == Some(expected_projection_detail) + && disagreement.observed.as_deref() == Some(incumbent_lease_sha256) + } else { + false + } } fn is_semantic_ready(authenticated: &AuthenticatedOdinRuntimeTopologyCorrelation) -> bool { @@ -3928,6 +5258,28 @@ fn usage() -> &'static str { mod tests { use super::*; + fn digest(byte: char) -> String { + byte.to_string().repeat(64) + } + + fn route_binding( + route_id: &str, + stable_port: u16, + private_port_start: u16, + private_port_end: u16, + ) -> RouteBinding { + RouteBinding { + driver: crate::deployment::RouteDriver::NginxStreamTcp, + route_id: route_id.into(), + stable_endpoint: format!("tcp://127.0.0.1:{stable_port}"), + private_host: "127.0.0.1".into(), + private_port_start, + private_port_end, + config_path: PathBuf::from(format!("/etc/nginx/idunn-stream-routes/{route_id}.conf")), + reload_unit: "nginx.service".into(), + } + } + fn command(kind: CommandKind) -> DeploymentCommand { DeploymentCommand { schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), @@ -4055,6 +5407,37 @@ mod tests { } } + #[test] + fn route_bindings_are_one_global_socket_and_candidate_authority_map() { + let first = route_binding("first", 4103, 14103, 14111); + let second = route_binding("second", 8831, 18831, 18839); + validate_route_binding_set(&[("first", &first), ("second", &second)]).unwrap(); + + let mut duplicate_id = second.clone(); + duplicate_id.route_id = first.route_id.clone(); + assert!( + validate_route_binding_set(&[("first", &first), ("second", &duplicate_id)]).is_err() + ); + + let mut overlapping_candidates = second.clone(); + overlapping_candidates.private_port_start = 14111; + overlapping_candidates.private_port_end = 14120; + assert!( + validate_route_binding_set(&[("first", &first), ("second", &overlapping_candidates),]) + .is_err() + ); + + let mut stable_inside_other_range = second; + stable_inside_other_range.stable_endpoint = "tcp://127.0.0.1:14105".into(); + assert!( + validate_route_binding_set(&[ + ("first", &first), + ("second", &stable_inside_other_range), + ]) + .is_err() + ); + } + #[test] fn deployment_command_is_immutable_positional_fact() -> Result<()> { let value = command(CommandKind::Deploy); @@ -4079,9 +5462,11 @@ mod tests { DeploymentPhase::Committing, DeploymentPhase::Complete, ]; - assert!(phases - .windows(2) - .all(|pair| pair[1] as u8 == pair[0] as u8 + 1)); + assert!( + phases + .windows(2) + .all(|pair| pair[1] as u8 == pair[0] as u8 + 1) + ); } #[test] @@ -4103,6 +5488,44 @@ mod tests { Ok(()) } + #[test] + fn odin_sequence_cursors_are_scoped_by_target_and_signer() -> Result<()> { + let command = command(CommandKind::Deploy); + let mut ghostlight = + DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100)?; + ghostlight.latest_odin_observation = Some(topology(7, 1)); + ghostlight.odin_publisher_sequence_cursor = 7; + let mut odin = DeploymentTransaction::new(&command, "odin".into(), 1, None, 100)?; + odin.latest_odin_observation = Some(topology(41, 2)); + odin.odin_publisher_sequence_cursor = 41; + let envelope = |key: &str| CultCacheEnvelope { + key: key.into(), + r#type: DeploymentTransaction::TYPE.into(), + payload: Vec::new(), + stored_at: "1970-01-01T00:00:00.100Z".into(), + schema_id: Some(DEPLOYMENT_TRANSACTION_SCHEMA.into()), + }; + let snapshot = ControlSnapshot { + commands: Vec::new(), + transactions: vec![ + Stored { + envelope: envelope("ghostlight"), + value: ghostlight, + }, + Stored { + envelope: envelope("odin"), + value: odin, + }, + ], + admitted: Vec::new(), + }; + + assert_eq!(snapshot.max_odin_sequence("ghostlight", "odin-signer"), 7); + assert_eq!(snapshot.max_odin_sequence("odin", "odin-signer"), 41); + assert_eq!(snapshot.max_odin_sequence("ghostlight", "other-signer"), 0); + Ok(()) + } + #[test] fn stale_odin_provider_receipt_cannot_gate_continuity_restart() -> Result<()> { let touched = std::cell::Cell::new(false); @@ -4111,10 +5534,12 @@ mod tests { bail!("Odin provider receipt is stale") })?; assert!(!touched.get()); - assert!(validate_live_providers_for_deploy(CommandKind::Deploy, || { - bail!("Odin provider receipt is stale") - }) - .is_err()); + assert!( + validate_live_providers_for_deploy(CommandKind::Deploy, || { + bail!("Odin provider receipt is stale") + }) + .is_err() + ); Ok(()) } @@ -4228,6 +5653,103 @@ mod tests { Ok(()) } + #[test] + fn routed_incumbent_retirement_waits_for_the_declared_drain_deadline() -> Result<()> { + assert_eq!(route_drain_deadline(1_000, 30)?, 31_000); + assert!(route_drain_deadline(u64::MAX, 1).is_err()); + Ok(()) + } + + #[test] + fn admitted_route_receipt_refreshes_when_stale_or_implausibly_future_dated() { + assert!(route_observation_is_current(900, 1_000, 100, 10)); + assert!(route_observation_is_current(1_010, 1_000, 100, 10)); + assert!(!route_observation_is_current(899, 1_000, 100, 10)); + assert!(!route_observation_is_current(1_011, 1_000, 100, 10)); + } + + #[test] + fn lease_warming_requires_both_new_odin_and_new_provider_evidence() { + let old = digest('1'); + let fresh = digest('2'); + assert!(!provider_warming_advanced(7, &old, 7, &fresh)); + assert!(!provider_warming_advanced(7, &old, 8, &old)); + assert!(provider_warming_advanced(7, &old, 8, &fresh)); + } + + #[test] + fn failed_route_proof_never_restores_any_fenced_incumbent() { + assert!(!may_rollback_route_after_failed_proof( + &FencingEvidence::Revoked { + incumbent_lease_sha256: Some(digest('1')), + candidate_lease_path_verified_empty: false, + } + )); + assert!(!may_rollback_route_after_failed_proof( + &FencingEvidence::Revoked { + incumbent_lease_sha256: None, + candidate_lease_path_verified_empty: true, + } + )); + assert!(may_rollback_route_after_failed_proof( + &FencingEvidence::SkippedStateless + )); + } + + #[test] + fn fenced_writer_is_already_retired_before_post_commit_cleanup() { + assert!(incumbent_was_stopped_during_fencing( + &FencingEvidence::Revoked { + incumbent_lease_sha256: Some(digest('1')), + candidate_lease_path_verified_empty: false, + } + )); + assert!(!incumbent_was_stopped_during_fencing( + &FencingEvidence::Revoked { + incumbent_lease_sha256: None, + candidate_lease_path_verified_empty: true, + } + )); + assert!(!incumbent_was_stopped_during_fencing( + &FencingEvidence::SkippedStateless + )); + } + + #[test] + fn warming_accepts_only_the_exact_projected_incumbent_lease_disagreement() { + let expected_detail = format!("expected:{};activation:{}", digest('1'), digest('2')); + let incumbent = digest('3'); + let exact = OdinTopologyDisagreement { + code: "projected-write-lease".into(), + expected: Some(expected_detail.clone()), + observed: Some(incumbent.clone()), + }; + + assert!(warming_disagreements_match_incumbent( + &expected_detail, + Some(&incumbent), + &[exact.clone()], + )); + assert!(!warming_disagreements_match_incumbent( + &expected_detail, + None, + &[exact.clone()], + )); + + let mut substituted = exact.clone(); + substituted.observed = Some(digest('4')); + assert!(!warming_disagreements_match_incumbent( + &expected_detail, + Some(&incumbent), + &[substituted], + )); + assert!(!warming_disagreements_match_incumbent( + &expected_detail, + Some(&incumbent), + &[exact.clone(), exact], + )); + } + #[test] fn candidate_and_incumbent_must_differ_in_all_three_native_boundaries() { let incumbent = workload(1001, 40, 50); @@ -4305,6 +5827,7 @@ mod tests { assert!(!transaction.is_terminal()); assert!(!transaction.owns_target_authority()); + assert!(transaction.blocks_new_target_mutation()); Ok(()) } diff --git a/crates/idunn-daemon/src/deployment.rs b/crates/idunn-daemon/src/deployment.rs index ded7e2b..bffd316 100644 --- a/crates/idunn-daemon/src/deployment.rs +++ b/crates/idunn-daemon/src/deployment.rs @@ -1,7 +1,8 @@ use std::collections::{BTreeMap, BTreeSet}; +use std::net::IpAddr; use std::path::{Component, Path, PathBuf}; -use anyhow::{bail, ensure, Context, Result}; +use anyhow::{Context, Result, bail, ensure}; use serde::{Deserialize, Serialize}; pub const TARGET_DECLARATION_SCHEMA: &str = "gamecult.idunn.target_declaration.v1"; @@ -425,11 +426,45 @@ pub struct RouteBinding { pub reload_unit: String, } +impl RouteBinding { + pub(crate) fn stable_socket(&self) -> Result<(IpAddr, u16)> { + let authority = match self.driver { + RouteDriver::NginxStreamTcp => self + .stable_endpoint + .strip_prefix("http://") + .or_else(|| self.stable_endpoint.strip_prefix("tcp://")), + RouteDriver::NginxStreamUdp => self.stable_endpoint.strip_prefix("rudp://"), + } + .context("stable endpoint transport does not match its route driver")?; + ensure!( + !authority + .chars() + .any(|character| matches!(character, '/' | '?' | '#' | '@' | '[' | ']')), + "stable endpoint is not a plain IP address and port" + ); + let (host, port) = authority + .rsplit_once(':') + .context("stable endpoint has no port")?; + let host = host + .parse::() + .context("stable endpoint host is not an IP address")?; + ensure!( + matches!(host, IpAddr::V4(_)), + "route binding v2 stable endpoints use explicit IPv4 addresses" + ); + let port = port + .parse::() + .context("stable endpoint port is not a u16")?; + ensure!(port > 0, "stable endpoint port is zero"); + Ok((host, port)) + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub enum RouteDriver { - NginxHttp, NginxStreamTcp, + NginxStreamUdp, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -1027,6 +1062,14 @@ impl OperatorBinding { require_id(&route.route_id, "route id")?; require_value(&route.stable_endpoint, "stable endpoint")?; require_host(&route.private_host)?; + let private_host: IpAddr = route + .private_host + .parse() + .context("route private host is not an IP address")?; + ensure!( + private_host.is_loopback(), + "systemd-transient candidate routes must use a loopback private host" + ); ensure!( route.private_port_start > 0, "private port range starts at zero" @@ -1035,6 +1078,7 @@ impl OperatorBinding { route.private_port_start < route.private_port_end, "candidate rollout requires at least two private ports" ); + route.stable_socket()?; require_absolute_path(&route.config_path, "route config")?; require_unit(&route.reload_unit, "route reload unit")?; } @@ -1301,11 +1345,10 @@ impl OperatorBinding { self.route.as_ref(), ) { (true, ServiceTransport::Http, Some(route)) - if route.driver == RouteDriver::NginxHttp => + if route.driver == RouteDriver::NginxStreamTcp => { ensure!( - route.stable_endpoint.starts_with("http://") - || route.stable_endpoint.starts_with("https://"), + route.stable_endpoint.starts_with("http://"), "HTTP stable endpoint is not an HTTP URI" ); } @@ -1317,6 +1360,14 @@ impl OperatorBinding { "TCP stable endpoint is not a TCP URI" ); } + (true, ServiceTransport::Rudp, Some(route)) + if route.driver == RouteDriver::NginxStreamUdp => + { + ensure!( + route.stable_endpoint.starts_with("rudp://"), + "RUDP stable endpoint is not an RUDP URI" + ); + } (false, _, None) => {} _ => bail!("route binding does not match the launch contract"), } @@ -1681,13 +1732,13 @@ expected_signer_identity_id = "test-service-runtime-signer" trust_anchor_store = "/etc/gamecult/trust/test-service.cc" [route] -driver = "nginx-http" +driver = "nginx-stream-tcp" route_id = "test-service-public" -stable_endpoint = "https://yggdrasil.gamecult.org/test-service/" +stable_endpoint = "http://127.0.0.1:18830" private_host = "127.0.0.1" private_port_start = 18831 private_port_end = 18839 -config_path = "/etc/nginx/idunn-routes/test-service.conf" +config_path = "/etc/nginx/idunn-stream-routes/test-service.conf" reload_unit = "nginx.service" [process_write_lease] @@ -1765,16 +1816,33 @@ nodes = ["yggdrasil"] #[test] fn route_transport_mismatch_is_rejected() { let recipe = TargetDeclaration::parse(RECIPE).unwrap(); - let binding = - OperatorBinding::parse(&BINDING.replace("nginx-http", "nginx-stream-tcp")).unwrap(); - assert!(binding.admit(&recipe).is_err()); + assert!( + OperatorBinding::parse(&BINDING.replace("nginx-stream-tcp", "nginx-stream-udp")) + .is_err() + ); - let binding = OperatorBinding::parse(&BINDING.replace( - "https://yggdrasil.gamecult.org/test-service/", - "tcp://yggdrasil.gamecult.org:443", - )) + assert!( + OperatorBinding::parse( + &BINDING.replace("http://127.0.0.1:18830", "tcp://yggdrasil.gamecult.org:443",), + ) + .is_err() + ); + assert!(recipe.service.route_required); + } + + #[test] + fn rudp_route_uses_the_udp_stream_driver() { + let recipe = TargetDeclaration::parse( + &RECIPE.replace("transport = \"http\"", "transport = \"rudp\""), + ) .unwrap(); - assert!(binding.admit(&recipe).is_err()); + let binding = OperatorBinding::parse( + &BINDING + .replace("nginx-stream-tcp", "nginx-stream-udp") + .replace("http://127.0.0.1:18830", "rudp://127.0.0.1:17872"), + ) + .unwrap(); + assert!(binding.admit(&recipe).is_ok()); } #[test] @@ -1892,24 +1960,30 @@ nodes = ["yggdrasil"] #[test] fn v2_binding_is_explicitly_singleton() { - assert!(OperatorBinding::parse( - &BINDING.replace("desired_replicas = 1", "desired_replicas = 2") - ) - .is_err()); - assert!(OperatorBinding::parse(&BINDING.replace( - "nodes = [\"yggdrasil\"]", - "nodes = [\"yggdrasil\", \"raven\"]" - )) - .is_err()); + assert!( + OperatorBinding::parse( + &BINDING.replace("desired_replicas = 1", "desired_replicas = 2") + ) + .is_err() + ); + assert!( + OperatorBinding::parse(&BINDING.replace( + "nodes = [\"yggdrasil\"]", + "nodes = [\"yggdrasil\", \"raven\"]" + )) + .is_err() + ); } #[test] fn legacy_static_workload_identity_has_no_authority_in_v2() { - assert!(OperatorBinding::parse(&BINDING.replace( - "schema = \"gamecult.idunn.operator_binding.v2\"", - "schema = \"gamecult.idunn.operator_binding.v1\"", - ),) - .is_err()); + assert!( + OperatorBinding::parse(&BINDING.replace( + "schema = \"gamecult.idunn.operator_binding.v2\"", + "schema = \"gamecult.idunn.operator_binding.v1\"", + ),) + .is_err() + ); let with_static_identity = BINDING.replace( "driver = \"systemd-transient\"", "driver = \"systemd-transient\"\nuser = \"test-service\"\ngroup = \"test-service\"", @@ -1923,10 +1997,12 @@ nodes = ["yggdrasil"] OperatorBinding::parse(&BINDING.replace("state_group = \"test-service\"\n", ""),) .is_err() ); - assert!(OperatorBinding::parse( - &BINDING.replace("state_group = \"test-service\"", "state_group = \"root\""), - ) - .is_err()); + assert!( + OperatorBinding::parse( + &BINDING.replace("state_group = \"test-service\"", "state_group = \"root\""), + ) + .is_err() + ); } #[test] diff --git a/crates/idunn-daemon/src/deployment_plan.rs b/crates/idunn-daemon/src/deployment_plan.rs index ca4469a..9475701 100644 --- a/crates/idunn-daemon/src/deployment_plan.rs +++ b/crates/idunn-daemon/src/deployment_plan.rs @@ -1,20 +1,20 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::PathBuf; -use anyhow::{bail, ensure, Context, Result}; +use anyhow::{Context, Result, bail, ensure}; pub use cultnet_rs::IdunnExpectedIncarnationRecord as ExpectedIncarnation; use cultnet_rs::{ - IdunnExpectedCapability, IdunnExpectedDependency, IdunnExpectedRoute, - IDUNN_EXPECTED_INCARNATION_SCHEMA, + IDUNN_EXPECTED_INCARNATION_SCHEMA, IdunnExpectedCapability, IdunnExpectedDependency, + IdunnExpectedRoute, }; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use crate::control_plane::SequenceAdmittedReady; use crate::deployment::{ - capability_compatible, CapabilityDependency, DependencyKind, ExternalCapabilityBinding, - OperatorBinding, ServiceTransport, SourceSelectionPolicy, StartupOrder, StateDeclaration, - TargetDeclaration, + CapabilityDependency, DependencyKind, ExternalCapabilityBinding, OperatorBinding, + ServiceTransport, SourceSelectionPolicy, StartupOrder, StateDeclaration, TargetDeclaration, + capability_compatible, }; pub const SOURCE_SELECTION_FACTS_SCHEMA: &str = "idunn.source_selection_facts.v1"; @@ -286,11 +286,11 @@ fn managed_ready_provider_refs( let endpoint = expected .route .as_ref() - .map(|route| route.candidate_endpoint.clone()); + .map(|route| route.stable_endpoint.clone()); let mut providers = Vec::new(); for capability in &observed.observed_capabilities { providers.push(DependencyProviderRef { - provider_id: observed.runtime_id.clone(), + provider_id: expected.target.clone(), authority: DependencyProviderAuthority::ManagedReady { target: expected.target.clone(), incarnation_id: expected.incarnation_id.clone(), @@ -995,11 +995,12 @@ fn sha256_id(bytes: &[u8]) -> String { mod tests { use super::*; use cultnet_rs::{ + GameCultProviderHealthIdentity, GameCultRuntimeCapability, IdunnRuntimeActivationLaunch, + IdunnServiceIdentity, ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA, + OdinRuntimeTopologyCorrelationPurpose, OdinRuntimeTopologyCorrelationRecord, + OdinTopologyAuthenticationContext, OdinTopologyIdentity, authenticate_odin_runtime_topology_correlation, enroll_service_identity_at, - verify_runtime_authority, GameCultProviderHealthIdentity, GameCultRuntimeCapability, - IdunnRuntimeActivationLaunch, IdunnServiceIdentity, OdinRuntimeTopologyCorrelationPurpose, - OdinRuntimeTopologyCorrelationRecord, OdinTopologyAuthenticationContext, - OdinTopologyIdentity, ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA, + verify_runtime_authority, }; const RECIPE: &str = r#" @@ -1113,13 +1114,13 @@ expected_signer_identity_id = "service-runtime-signer" trust_anchor_store = "/etc/gamecult/trust/service.cc" [route] -driver = "nginx-http" +driver = "nginx-stream-tcp" route_id = "service" -stable_endpoint = "https://example.invalid/service/" +stable_endpoint = "http://127.0.0.1:17999" private_host = "127.0.0.1" private_port_start = 18000 private_port_end = 18009 -config_path = "/etc/nginx/idunn-routes/service.conf" +config_path = "/etc/nginx/idunn-stream-routes/service.conf" reload_unit = "nginx.service" [brakes] @@ -1189,7 +1190,7 @@ nodes = ["yggdrasil"] } } - fn ready_odin_provider(runtime_id: &str, capacity: u32) -> SequenceAdmittedReady { + fn ready_odin_provider(target: &str, runtime_id: &str, capacity: u32) -> SequenceAdmittedReady { (|| -> Result { let root = tempfile::tempdir()?; let provider = enroll_service_identity_at::( @@ -1200,6 +1201,7 @@ nodes = ["yggdrasil"] let odin_signer = enroll_service_identity_at::(&root.path().join("odin.cc"))?; let mut expected = odin(); + expected.target = target.into(); expected.runtime_id = runtime_id.into(); expected.expected_signer_identity_id = provider.entry().identity_id.clone(); let launch = IdunnRuntimeActivationLaunch::issue(&expected, digest(b'8'), 100, &idunn)?; @@ -1262,7 +1264,7 @@ nodes = ["yggdrasil"] } fn plan() -> CompiledDeploymentPlan { - let providers = [ready_odin_provider("odin-yggdrasil", 1)]; + let providers = [ready_odin_provider("odin", "odin-yggdrasil", 1)]; plan_with(&providers) } @@ -1302,7 +1304,7 @@ nodes = ["yggdrasil"] #[test] fn compiler_parses_the_exact_bytes_it_receipts() { - let providers = [ready_odin_provider("odin-yggdrasil", 1)]; + let providers = [ready_odin_provider("odin", "odin-yggdrasil", 1)]; let first = plan_with(&providers); let second = plan_with(&providers); assert_eq!(first, second); @@ -1316,17 +1318,19 @@ nodes = ["yggdrasil"] "contract = \"service.health\"", "contract = \"service.health.v2\"", ); - let providers = [ready_odin_provider("odin-yggdrasil", 1)]; - assert!(compile_deployment_plan( - changed.as_bytes(), - BINDING.as_bytes(), - source(RECIPE), - "service-incarnation-2", - Some(18002), - 111, - &providers, - ) - .is_err()); + let providers = [ready_odin_provider("odin", "odin-yggdrasil", 1)]; + assert!( + compile_deployment_plan( + changed.as_bytes(), + BINDING.as_bytes(), + source(RECIPE), + "service-incarnation-2", + Some(18002), + 111, + &providers, + ) + .is_err() + ); let mut corrupted = first; corrupted .binding_blob @@ -1382,8 +1386,8 @@ nodes = ["yggdrasil"] fn dependency_selection_uses_observed_ready_capacity_and_stable_identity_order() { let declaration = TargetDeclaration::parse(RECIPE).unwrap(); let admitted = [ - ready_odin_provider("odin-z", 4), - ready_odin_provider("odin-yggdrasil", 2), + ready_odin_provider("odin-z", "odin-runtime-z", 4), + ready_odin_provider("odin-yggdrasil", "odin-runtime-yggdrasil", 2), ]; let providers = admitted .iter() @@ -1395,6 +1399,10 @@ nodes = ["yggdrasil"] "odin-yggdrasil" ); assert_eq!(selected[0].provider.as_ref().unwrap().capacity, 2); + assert_eq!( + selected[0].provider.as_ref().unwrap().endpoint.as_deref(), + Some("tcp://10.77.0.1:17871") + ); assert!(matches!( &selected[0].provider.as_ref().unwrap().authority, DependencyProviderAuthority::ManagedReady { @@ -1459,7 +1467,7 @@ nodes = ["yggdrasil"] assert!(!expected.write_lease_required); assert_eq!( expected.dependencies[0].provider_id.as_deref(), - Some("odin-yggdrasil") + Some("odin") ); assert_eq!( expected.dependencies[0].provider_authority.as_deref(), @@ -1494,7 +1502,7 @@ nodes = ["yggdrasil"] "service-incarnation-sorted", Some(18002), 111, - &[ready_odin_provider("odin-yggdrasil", 1)], + &[ready_odin_provider("odin", "odin-yggdrasil", 1)], ) .unwrap(); let release = SealedRelease::new( diff --git a/crates/idunn-daemon/src/drivers.rs b/crates/idunn-daemon/src/drivers.rs index c4bfe5e..1554408 100644 --- a/crates/idunn-daemon/src/drivers.rs +++ b/crates/idunn-daemon/src/drivers.rs @@ -2,6 +2,7 @@ use std::collections::BTreeMap; use std::ffi::{OsStr, OsString}; use std::fs::{self, OpenOptions}; use std::io::{ErrorKind, Read, Write}; +use std::net::{SocketAddr, TcpStream}; use std::path::{Path, PathBuf}; use std::process::{Command, Output, Stdio}; use std::thread; @@ -10,8 +11,9 @@ use std::time::Duration; use anyhow::{Context, Result, bail, ensure}; use cultcache_rs::{ CacheBackingStore, CultCacheEnvelope, CultCacheExpectedEnvelope, DatabaseEntry, - SingleFileMessagePackBackingStore, + SingleFileMessagePackBackingStore, TryCompareExchangeSnapshotOutcome, }; +use cultmesh_rs::{CultMeshRudpSnapshotOptions, request_raw_snapshot_from_rudp_catalog}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use uuid::Uuid; @@ -29,11 +31,16 @@ use crate::deployment_plan::{ SOURCE_SELECTION_FACTS_SCHEMA, SealedRelease, SourceSelection, SourceSelectionFacts, }; use cultnet_rs::{ - GameCultProviderHealthIdentity, IDUNN_EXPECTED_INCARNATION_SCHEMA, + CultNetMessage, CultNetRawPayloadEncoding, CultNetWireContract, + GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, GAMECULT_RUNTIME_PRESENCE_HEALTH_SIGNING_PURPOSE, + GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA, GameCultProviderHealthIdentity, + GameCultServiceTrustAnchorRecord, IDUNN_EXPECTED_INCARNATION_SCHEMA, IDUNN_PROCESS_WRITE_LEASE_SCHEMA, IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME, IDUNN_RUNTIME_ACTIVATION_SCHEMA, IdunnExpectedIncarnationRecord, IdunnProcessWriteLeaseRecord, IdunnRuntimeActivationLaunch, IdunnRuntimeActivationRecord, IdunnRuntimeActivationSigner, ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA, OdinRuntimeTopologyCorrelationRecord, + ServiceIdentityProfile, ServiceIdentityTrustAnchor, decode_cultnet_message_from_slice, + derive_service_identity_id, encode_cultnet_message_to_vec, encode_frame, open_service_identity_credential_reader, }; @@ -293,14 +300,36 @@ pub trait WorkloadPort { } pub trait TopologyPort { - fn publish_expected(&self, expected: &IdunnExpectedIncarnationRecord) -> Result; - fn withdraw_expected(&self, expected: &IdunnExpectedIncarnationRecord) -> Result<()>; + fn publish_expected( + &self, + expected: &IdunnExpectedIncarnationRecord, + provider_anchor: &ServiceIdentityTrustAnchor, + ) -> Result; + fn withdraw_expected( + &self, + expected: &IdunnExpectedIncarnationRecord, + provider_anchor: &ServiceIdentityTrustAnchor, + activation: Option<&IdunnRuntimeActivationRecord>, + lease: Option<&IdunnProcessWriteLeaseRecord>, + ) -> Result<()>; fn publish_observed_activation( &self, expected: &IdunnExpectedIncarnationRecord, activation: &IdunnRuntimeActivationRecord, observation: &WorkloadObservation, ) -> Result; + fn publish_process_write_lease( + &self, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + lease: &IdunnProcessWriteLeaseRecord, + ) -> Result; + fn withdraw_process_write_lease( + &self, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + lease: Option<&IdunnProcessWriteLeaseRecord>, + ) -> Result<()>; fn receive(&self, target: &str) -> Result>; } @@ -313,6 +342,7 @@ pub struct ReceivedOdinTopologyCorrelation { pub trait WriteLeasePort { fn revoke_exact(&self, incumbent: Option<&IdunnProcessWriteLeaseRecord>) -> Result<()>; fn observe_empty(&self) -> Result; + fn observe_exact(&self, lease: &IdunnProcessWriteLeaseRecord) -> Result; fn grant( &self, expected: &IdunnExpectedIncarnationRecord, @@ -334,6 +364,22 @@ pub struct RouteObservation { pub route_id: String, pub runtime_instance_id: String, pub membership_sha256: String, + pub signed_presence_sha256: String, + pub observed_at_unix_millis: u64, +} + +impl RouteObservation { + pub fn validate(&self) -> Result<()> { + require_driver_id(&self.route_id, "route observation")?; + require_sha256_id(&self.runtime_instance_id, "route runtime instance")?; + require_sha256_id(&self.membership_sha256, "route membership")?; + require_sha256_id(&self.signed_presence_sha256, "route signed presence")?; + ensure!( + self.observed_at_unix_millis > 0, + "route observation has no trusted receipt time" + ); + Ok(()) + } } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -343,26 +389,53 @@ pub struct RoutePreflightReceipt { pub candidate_membership_sha256: String, pub incumbent_runtime_instance_id: Option, pub incumbent_membership_sha256: Option, + pub incumbent_configuration: Option>, } -pub trait RoutePort { - fn preflight( - &self, - expected: &IdunnExpectedIncarnationRecord, - runtime_instance_id: &str, - incumbent: Option<&RouteObservation>, - ) -> Result; - fn promote( - &self, - expected: &IdunnExpectedIncarnationRecord, - runtime_instance_id: &str, - preflight: &RoutePreflightReceipt, - ) -> Result; - fn observe( - &self, - expected: &IdunnExpectedIncarnationRecord, - observation: &RouteObservation, - ) -> Result; +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RouteSnapshotResponse { + pub message_id: String, + pub canonical_presence: Vec, +} + +const ROUTE_SNAPSHOT_TIMEOUT: Duration = Duration::from_secs(3); +const ROUTE_SNAPSHOT_MAX_BYTES: usize = 1024 * 1024; +const ROUTE_HTTP_MAX_HEADER_BYTES: usize = 32 * 1024; +const ROUTE_HTTP_SNAPSHOT_PATH: &str = "/cultnet/snapshot"; + +impl RoutePreflightReceipt { + pub fn validate(&self) -> Result<()> { + require_driver_id(&self.route_id, "route preflight")?; + require_sha256_id( + &self.candidate_runtime_instance_id, + "candidate route runtime instance", + )?; + require_sha256_id( + &self.candidate_membership_sha256, + "candidate route membership", + )?; + match ( + &self.incumbent_runtime_instance_id, + &self.incumbent_membership_sha256, + &self.incumbent_configuration, + ) { + (None, None, None) => {} + (Some(runtime), Some(membership), Some(configuration)) => { + require_sha256_id(runtime, "incumbent route runtime instance")?; + require_sha256_id(membership, "incumbent route membership")?; + ensure!( + !configuration.is_empty(), + "incumbent route configuration is empty" + ); + ensure!( + sha256_id(configuration) == *membership, + "incumbent route configuration differs from its membership digest" + ); + } + _ => bail!("incumbent route preflight evidence is partial"), + } + Ok(()) + } } /// Fixed-argv Git source driver. It never interprets recipe text as a command @@ -1295,7 +1368,7 @@ impl RunnerPort for DockerRunnerDriver { &self, source: &FrozenSource, plan: &CompiledDeploymentPlan, - staging_root: &Path, + staging_parent: &Path, sealed_at_unix_millis: u64, ) -> Result { plan.validate()?; @@ -1305,11 +1378,12 @@ impl RunnerPort for DockerRunnerDriver { "runner source differs from the compiled plan" ); let (declaration, binding) = plan.parsed_inputs()?; - ensure!( - !staging_root.exists(), - "release staging root already exists" - ); - fs::create_dir_all(staging_root)?; + fs::create_dir_all(staging_parent)?; + let staging_root = staging_parent.join(&source.receipt.transaction_id); + if staging_root.exists() { + remove_tree_inside(staging_parent, &staging_root)?; + } + fs::create_dir_all(&staging_root)?; let mut workspaces = BTreeMap::new(); for runner_id in binding.runners.keys() { @@ -1375,12 +1449,12 @@ impl RunnerPort for DockerRunnerDriver { &declaration, source, &workspaces, - staging_root, + &staging_root, &artifact.id, )?); } for workspace in workspaces.values() { - remove_tree_inside(staging_root, workspace)?; + remove_tree_inside(&staging_root, workspace)?; } let release = SealedRelease::new( plan, @@ -1390,7 +1464,7 @@ impl RunnerPort for DockerRunnerDriver { )?; Ok(MaterializedRelease { release, - root: staging_root.to_owned(), + root: staging_root, }) } } @@ -2848,9 +2922,10 @@ impl WorkloadPort for SystemdTransientWorkloadDriver { } /// One dedicated CultCache file is the write-lease authority for one target. -/// Its sibling lock is shared by the target only around durable commit edges; -/// Idunn's compare-exchange therefore fences the old writer before replacing -/// the record. Route membership never enters this store. +/// The admitted target holds its sibling lock shared for that process lifetime. +/// Grant and revocation therefore make one nonblocking CAS attempt; revocation +/// follows an exact incumbent stop. A foreign or stuck holder cannot freeze +/// Idunn's control loop. Route membership never enters this store. pub struct CultCacheWriteLeaseDriver { pub target: String, pub record_path: PathBuf, @@ -2914,11 +2989,6 @@ impl CultCacheWriteLeaseDriver { lease.validate()?; let expected_sha256 = expected.canonical_sha256()?; let activation_sha256 = activation.canonical_sha256()?; - let warming = warming.authenticated().record(); - let warming_presence_sha256 = warming - .signed_presence_sha256 - .as_deref() - .context("warming observation has no signed presence")?; ensure!( self.target == expected.target && lease.target == expected.target @@ -2939,14 +3009,8 @@ impl CultCacheWriteLeaseDriver { .state_contract_sha256 .as_deref() .context("write-lease Expected has no state contract")? - && lease.warming_presence_sha256 == warming_presence_sha256 - && warming.present - && !warming.ready - && warming.observed_presence_state.as_deref() == Some("warming") - && warming.runtime_instance_id.as_deref() - == Some(activation.runtime_instance_id.as_str()) - && warming.observed_write_lease_sha256.is_none() - && warming.disagreements.is_empty(), + && lease.warming_presence_sha256 == warming.signed_presence_sha256() + && warming.runtime_instance_id() == activation.runtime_instance_id.as_str(), "process write lease does not bind the exact warming candidate" ); Ok(()) @@ -2971,17 +3035,32 @@ impl WriteLeasePort for CultCacheWriteLeaseDriver { "refusing to revoke an unexpected process write lease" ); let store = SingleFileMessagePackBackingStore::new(&self.record_path); - ensure!( - store.compare_exchange_snapshot(std::slice::from_ref(&envelope), &[])?, - "process write lease changed while fencing the incumbent" - ); - Ok(()) + match store.try_compare_exchange_snapshot(std::slice::from_ref(&envelope), &[])? { + TryCompareExchangeSnapshotOutcome::Exchanged => Ok(()), + TryCompareExchangeSnapshotOutcome::Mismatch => { + bail!("process write lease changed while fencing the incumbent") + } + TryCompareExchangeSnapshotOutcome::LockContended => { + bail!("process write lease is held by a shared-lock consumer") + } + } } fn observe_empty(&self) -> Result { Ok(self.current()?.is_none()) } + fn observe_exact(&self, lease: &IdunnProcessWriteLeaseRecord) -> Result { + lease.validate()?; + ensure!( + lease.target == self.target, + "observed process write lease belongs to another target" + ); + Ok(self + .current()? + .is_some_and(|(_, current)| current == *lease)) + } + fn grant( &self, expected: &IdunnExpectedIncarnationRecord, @@ -2995,20 +3074,21 @@ impl WriteLeasePort for CultCacheWriteLeaseDriver { current == *lease, "another process already owns the write lease" ); + harden_root_authority_file(&self.record_path)?; + harden_root_authority_file(&authority_lock_path(&self.record_path))?; return lease.canonical_sha256(); } let envelope = self.envelope(lease)?; - ensure!( - SingleFileMessagePackBackingStore::new(&self.record_path).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: IdunnProcessWriteLeaseRecord::TYPE.into(), - key: lease.target.clone(), - current: None, - }], - &[envelope], - )?, - "process write-lease grant lost its empty-store compare-exchange" - ); + let store = SingleFileMessagePackBackingStore::new(&self.record_path); + match store.try_compare_exchange_snapshot(&[], &[envelope])? { + TryCompareExchangeSnapshotOutcome::Exchanged => {} + TryCompareExchangeSnapshotOutcome::Mismatch => { + bail!("process write-lease grant lost its empty-store compare-exchange") + } + TryCompareExchangeSnapshotOutcome::LockContended => { + bail!("process write lease is held by a shared-lock consumer") + } + } harden_root_authority_file(&self.record_path)?; harden_root_authority_file(&authority_lock_path(&self.record_path))?; lease.canonical_sha256() @@ -3028,43 +3108,581 @@ impl WriteLeasePort for CultCacheWriteLeaseDriver { } } -/// The local CultCache projection is Idunn's typed provider surface into -/// CultMesh. Odin writes a separate signed correlation store. Reading that -/// store never lets Idunn manufacture Present or Ready from its own config. +/// Idunn projects desired identity plus its own activation and current-lease +/// facts here. Service presence is absent by construction; only Odin may +/// correlate these records with signed runtime observation into Present/Ready. pub struct CultCacheTopologyDriver { pub projection_store: PathBuf, pub correlation_store: PathBuf, } -impl TopologyPort for CultCacheTopologyDriver { - fn publish_expected(&self, expected: &IdunnExpectedIncarnationRecord) -> Result { +impl CultCacheTopologyDriver { + /// Atomically demotes a failed pre-fence candidate back to the admitted + /// generation's Expected projection. Runtime activation and write-lease + /// records are deliberately absent until admitted-body supervision proves + /// the physical process and lease again. + pub fn restore_admitted_expected_only( + &self, + failed_expected: &IdunnExpectedIncarnationRecord, + failed_provider_anchor: &ServiceIdentityTrustAnchor, + failed_activation: Option<&IdunnRuntimeActivationRecord>, + admitted_expected: &IdunnExpectedIncarnationRecord, + admitted_provider_anchor: &ServiceIdentityTrustAnchor, + admitted_activation: &IdunnRuntimeActivationRecord, + admitted_lease: Option<&IdunnProcessWriteLeaseRecord>, + ) -> Result { + failed_expected.validate()?; + admitted_expected.validate()?; + ensure!( + failed_expected.target == admitted_expected.target, + "projection restoration crosses deployment targets" + ); + let failed_anchor = runtime_presence_trust_anchor(failed_expected, failed_provider_anchor)?; + let admitted_anchor = + runtime_presence_trust_anchor(admitted_expected, admitted_provider_anchor)?; + if let Some(activation) = failed_activation { + activation.validate()?; + ensure!( + activation.expected_projection_sha256 == failed_expected.canonical_sha256()?, + "failed activation does not bind the failed Expected projection" + ); + } + admitted_activation.validate()?; + ensure!( + admitted_activation.expected_projection_sha256 + == admitted_expected.canonical_sha256()?, + "admitted activation does not bind the admitted Expected projection" + ); + if let Some(lease) = admitted_lease { + validate_topology_lease(admitted_expected, admitted_activation, lease)?; + } + if let Some(parent) = self.projection_store.parent() { + fs::create_dir_all(parent)?; + } + let store = SingleFileMessagePackBackingStore::new(&self.projection_store); + let target = admitted_expected.target.as_str(); + let anchor_key = runtime_presence_trust_anchor_id(target); + for _ in 0..8 { + let entries = store.pull_all_read_only_snapshot()?; + let current_expected = + projection_entry(&entries, IdunnExpectedIncarnationRecord::TYPE, target)?; + if let Some(envelope) = current_expected { + ensure!( + envelope.schema_id.as_deref() == Some(IDUNN_EXPECTED_INCARNATION_SCHEMA), + "restored Expected projection schema is foreign" + ); + let current = IdunnExpectedIncarnationRecord::decode_canonical(&envelope.payload)?; + ensure!( + current == *failed_expected || current == *admitted_expected, + "refusing to replace an unknown Expected projection" + ); + } + let current_anchor = projection_entry( + &entries, + GameCultServiceTrustAnchorRecord::TYPE, + &anchor_key, + )?; + if let Some(envelope) = current_anchor { + let current = service_trust_anchor_from_envelope(envelope)?; + ensure!( + current == failed_anchor || current == admitted_anchor, + "refusing to replace an unknown runtime presence trust anchor" + ); + } + let current_activation = + projection_entry(&entries, IdunnRuntimeActivationRecord::TYPE, target)?; + if let Some(envelope) = current_activation { + ensure!( + envelope.schema_id.as_deref() == Some(IDUNN_RUNTIME_ACTIVATION_SCHEMA), + "restored activation projection schema is foreign" + ); + let current = IdunnRuntimeActivationRecord::decode_canonical(&envelope.payload)?; + ensure!( + failed_activation == Some(¤t) || admitted_activation == ¤t, + "refusing to remove an unknown runtime activation projection" + ); + } + let current_lease = + projection_entry(&entries, IdunnProcessWriteLeaseRecord::TYPE, target)?; + if let Some(envelope) = current_lease { + ensure!( + envelope.schema_id.as_deref() == Some(IDUNN_PROCESS_WRITE_LEASE_SCHEMA), + "restored write-lease projection schema is foreign" + ); + let current = IdunnProcessWriteLeaseRecord::decode_canonical(&envelope.payload)?; + ensure!( + admitted_lease == Some(¤t), + "refusing to remove an unknown process write-lease projection" + ); + } + if current_expected.is_some_and(|envelope| { + IdunnExpectedIncarnationRecord::decode_canonical(&envelope.payload) + .is_ok_and(|current| current == *admitted_expected) + }) && current_anchor.is_some_and(|envelope| { + service_trust_anchor_from_envelope(envelope) + .is_ok_and(|current| current == admitted_anchor) + }) && current_activation.is_none() + && current_lease.is_none() + { + return admitted_expected.canonical_sha256(); + } + let mut replacement = entries + .iter() + .filter(|envelope| { + !((envelope.key == target + && matches!( + envelope.r#type.as_str(), + IdunnExpectedIncarnationRecord::TYPE + | IdunnRuntimeActivationRecord::TYPE + | IdunnProcessWriteLeaseRecord::TYPE + )) + || (envelope.key == anchor_key + && envelope.r#type == GameCultServiceTrustAnchorRecord::TYPE)) + }) + .cloned() + .collect::>(); + replacement.extend([ + CultCacheEnvelope { + key: admitted_expected.target.clone(), + r#type: IdunnExpectedIncarnationRecord::TYPE.into(), + payload: admitted_expected.canonical_bytes()?, + stored_at: chrono::Utc::now().to_rfc3339(), + schema_id: Some(IDUNN_EXPECTED_INCARNATION_SCHEMA.into()), + }, + CultCacheEnvelope { + key: admitted_anchor.trust_anchor_id.clone(), + r#type: GameCultServiceTrustAnchorRecord::TYPE.into(), + payload: rmp_serde::to_vec(&admitted_anchor)?, + stored_at: rfc3339_millis(admitted_anchor.bound_at_unix_millis)?, + schema_id: Some(GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA.into()), + }, + ]); + if store.compare_exchange_snapshot(&entries, &replacement)? { + return admitted_expected.canonical_sha256(); + } + } + bail!("topology projection changed repeatedly during admitted restoration") + } + + pub fn admitted_runtime_projection_is_exact( + &self, + expected: &IdunnExpectedIncarnationRecord, + provider_anchor: &ServiceIdentityTrustAnchor, + activation: &IdunnRuntimeActivationRecord, + lease: Option<&IdunnProcessWriteLeaseRecord>, + ) -> Result { expected.validate()?; - upsert_record( - &self.projection_store, - CultCacheEnvelope { - key: expected.target.clone(), + activation.validate()?; + ensure!( + activation.expected_projection_sha256 == expected.canonical_sha256()?, + "admitted activation does not bind its Expected projection" + ); + if let Some(lease) = lease { + validate_topology_lease(expected, activation, lease)?; + } + if !self.projection_store.exists() { + return Ok(false); + } + let entries = SingleFileMessagePackBackingStore::new(&self.projection_store) + .pull_all_read_only_snapshot()?; + let anchor = runtime_presence_trust_anchor(expected, provider_anchor)?; + let expected_is_exact = match projection_entry( + &entries, + IdunnExpectedIncarnationRecord::TYPE, + &expected.target, + )? { + Some(envelope) => { + ensure!( + envelope.schema_id.as_deref() == Some(IDUNN_EXPECTED_INCARNATION_SCHEMA), + "admitted Expected projection schema is foreign" + ); + ensure!( + IdunnExpectedIncarnationRecord::decode_canonical(&envelope.payload)? + == *expected, + "admitted Expected projection was replaced by another incarnation" + ); + true + } + None => false, + }; + let anchor_is_exact = match projection_entry( + &entries, + GameCultServiceTrustAnchorRecord::TYPE, + &anchor.trust_anchor_id, + )? { + Some(envelope) => { + ensure!( + service_trust_anchor_from_envelope(envelope)? == anchor, + "admitted runtime presence trust anchor was replaced" + ); + true + } + None => false, + }; + let activation_is_exact = match projection_entry( + &entries, + IdunnRuntimeActivationRecord::TYPE, + &expected.target, + )? { + Some(envelope) => { + ensure!( + envelope.schema_id.as_deref() == Some(IDUNN_RUNTIME_ACTIVATION_SCHEMA), + "admitted activation projection schema is foreign" + ); + ensure!( + IdunnRuntimeActivationRecord::decode_canonical(&envelope.payload)? + == *activation, + "admitted activation projection was replaced" + ); + true + } + None => false, + }; + let lease_is_exact = match projection_entry( + &entries, + IdunnProcessWriteLeaseRecord::TYPE, + &expected.target, + )? { + Some(envelope) => { + ensure!( + envelope.schema_id.as_deref() == Some(IDUNN_PROCESS_WRITE_LEASE_SCHEMA), + "admitted write-lease projection schema is foreign" + ); + let current = IdunnProcessWriteLeaseRecord::decode_canonical(&envelope.payload)?; + ensure!( + lease == Some(¤t), + "admitted write-lease projection was replaced" + ); + true + } + None => lease.is_none(), + }; + Ok(expected_is_exact && anchor_is_exact && activation_is_exact && lease_is_exact) + } +} + +fn runtime_presence_trust_anchor( + expected: &IdunnExpectedIncarnationRecord, + provider_anchor: &ServiceIdentityTrustAnchor, +) -> Result { + expected.validate()?; + ensure!( + provider_anchor.schema_version + == ::TRUST_ANCHOR_SCHEMA, + "runtime presence trust anchor schema is unsupported" + ); + ensure!( + derive_service_identity_id::(&provider_anchor.public_key)? + == provider_anchor.identity_id + && provider_anchor.identity_id == expected.expected_signer_identity_id, + "runtime presence trust anchor does not bind the Expected signer" + ); + let bound_at_unix_millis: u64 = + chrono::DateTime::parse_from_rfc3339(&provider_anchor.identity_created_at)? + .timestamp_millis() + .try_into() + .context("runtime presence trust anchor creation time predates Unix epoch")?; + let anchor = GameCultServiceTrustAnchorRecord { + schema_version: GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA.into(), + trust_anchor_id: runtime_presence_trust_anchor_id(&expected.target), + service_id: expected.target.clone(), + runtime_id: expected.runtime_id.clone(), + signer_identity_id: provider_anchor.identity_id.clone(), + signer_public_key: provider_anchor.public_key.clone(), + signature_algorithm: "ed25519".into(), + signing_purpose: GAMECULT_RUNTIME_PRESENCE_HEALTH_SIGNING_PURPOSE.into(), + signed_schema: GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + binding_authority: "root".into(), + bound_at_unix_millis, + expires_at_unix_millis: None, + private_state_exposed: false, + }; + anchor.validate()?; + Ok(anchor) +} + +fn runtime_presence_trust_anchor_id(target: &str) -> String { + format!("root/{target}/runtime-presence") +} + +fn validate_topology_lease( + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + lease: &IdunnProcessWriteLeaseRecord, +) -> Result<()> { + expected.validate()?; + activation.validate()?; + lease.validate()?; + ensure!( + expected.write_lease_required + && lease.target == expected.target + && lease.expected_projection_sha256 == expected.canonical_sha256()? + && lease.plan_id == expected.plan_id + && lease.incarnation_id == expected.incarnation_id + && lease.sealed_release_id == expected.sealed_release_id + && lease.activation_witness_sha256 == activation.canonical_sha256()? + && lease.state_schema_generation + == expected + .state_schema_generation + .as_deref() + .context("write-lease Expected has no state generation")? + && lease.state_contract_sha256 + == expected + .state_contract_sha256 + .as_deref() + .context("write-lease Expected has no state contract")? + && lease.runtime_id == expected.runtime_id + && lease.runtime_instance_id == activation.runtime_instance_id, + "topology write lease does not bind the exact Expected activation" + ); + Ok(()) +} + +fn service_trust_anchor_from_envelope( + envelope: &CultCacheEnvelope, +) -> Result { + ensure!( + envelope.schema_id.as_deref() == Some(GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA), + "service trust-anchor projection schema is foreign" + ); + let anchor: GameCultServiceTrustAnchorRecord = rmp_serde::from_slice(&envelope.payload)?; + ensure!( + rmp_serde::to_vec(&anchor)? == envelope.payload, + "service trust-anchor projection is noncanonical" + ); + anchor.validate()?; + Ok(anchor) +} + +fn replace_records_atomically(path: &Path, replacements: &[CultCacheEnvelope]) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + ensure!( + replacements.iter().enumerate().all(|(index, entry)| { + replacements[index + 1..].iter().all(|other| { + (entry.r#type.as_str(), entry.key.as_str()) + != (other.r#type.as_str(), other.key.as_str()) + }) + }), + "atomic projection set contains a duplicate identity" + ); + let store = SingleFileMessagePackBackingStore::new(path); + for _ in 0..8 { + let entries = store.pull_all_read_only_snapshot()?; + let conditions = replacements + .iter() + .map(|replacement| { + Ok(CultCacheExpectedEnvelope { + r#type: replacement.r#type.clone(), + key: replacement.key.clone(), + current: projection_entry(&entries, &replacement.r#type, &replacement.key)? + .cloned(), + }) + }) + .collect::>>()?; + if conditions + .iter() + .zip(replacements) + .all(|(condition, replacement)| condition.current.as_ref() == Some(replacement)) + { + return Ok(()); + } + if store.compare_exchange(&conditions, replacements)? { + return Ok(()); + } + } + bail!("CultCache projection changed too often to publish atomic set") +} + +fn projection_entry<'a>( + entries: &'a [CultCacheEnvelope], + r#type: &str, + key: &str, +) -> Result> { + let mut matches = entries + .iter() + .filter(|entry| entry.r#type == r#type && entry.key == key); + let current = matches.next(); + ensure!( + matches.next().is_none(), + "CultCache projection identity is ambiguous" + ); + Ok(current) +} + +fn insert_exact_process_lease( + path: &Path, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + replacement: CultCacheEnvelope, +) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + let store = SingleFileMessagePackBackingStore::new(path); + for _ in 0..8 { + let entries = store.pull_all_read_only_snapshot()?; + let projected_expected = projection_entry( + &entries, + IdunnExpectedIncarnationRecord::TYPE, + &expected.target, + )? + .context("process write lease has no current Expected projection")?; + ensure!( + projected_expected.schema_id.as_deref() == Some(IDUNN_EXPECTED_INCARNATION_SCHEMA) + && IdunnExpectedIncarnationRecord::decode_canonical(&projected_expected.payload)? + == *expected, + "process write lease current Expected projection is substituted" + ); + let projected_activation = projection_entry( + &entries, + IdunnRuntimeActivationRecord::TYPE, + &expected.target, + )? + .context("process write lease has no observed activation projection")?; + ensure!( + projected_activation.schema_id.as_deref() == Some(IDUNN_RUNTIME_ACTIVATION_SCHEMA) + && IdunnRuntimeActivationRecord::decode_canonical(&projected_activation.payload)? + == *activation, + "process write lease observed activation projection is substituted" + ); + let projected_anchor = service_trust_anchor_from_envelope( + projection_entry( + &entries, + GameCultServiceTrustAnchorRecord::TYPE, + &runtime_presence_trust_anchor_id(&expected.target), + )? + .context("process write lease has no runtime presence trust anchor")?, + )?; + ensure!( + projected_anchor.service_id == expected.target + && projected_anchor.runtime_id == expected.runtime_id + && projected_anchor.signer_identity_id == expected.expected_signer_identity_id + && projected_anchor.signing_purpose + == GAMECULT_RUNTIME_PRESENCE_HEALTH_SIGNING_PURPOSE + && projected_anchor.signed_schema == GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, + "process write lease runtime presence trust anchor is substituted" + ); + let current = projection_entry(&entries, &replacement.r#type, &replacement.key)?; + if current == Some(&replacement) { + return Ok(()); + } + ensure!( + current.is_none(), + "refusing to replace a different current process write-lease projection" + ); + let conditions = [ + CultCacheExpectedEnvelope { r#type: IdunnExpectedIncarnationRecord::TYPE.into(), - payload: expected.canonical_bytes()?, - stored_at: chrono::Utc::now().to_rfc3339(), - schema_id: Some(IDUNN_EXPECTED_INCARNATION_SCHEMA.into()), + key: expected.target.clone(), + current: Some(projected_expected.clone()), }, + CultCacheExpectedEnvelope { + r#type: IdunnRuntimeActivationRecord::TYPE.into(), + key: expected.target.clone(), + current: Some(projected_activation.clone()), + }, + CultCacheExpectedEnvelope { + r#type: GameCultServiceTrustAnchorRecord::TYPE.into(), + key: runtime_presence_trust_anchor_id(&expected.target), + current: projection_entry( + &entries, + GameCultServiceTrustAnchorRecord::TYPE, + &runtime_presence_trust_anchor_id(&expected.target), + )? + .cloned(), + }, + CultCacheExpectedEnvelope { + r#type: replacement.r#type.clone(), + key: replacement.key.clone(), + current: None, + }, + ]; + if store.compare_exchange(&conditions, std::slice::from_ref(&replacement))? { + return Ok(()); + } + } + bail!("CultCache projection changed too often to publish exact record") +} + +impl TopologyPort for CultCacheTopologyDriver { + fn publish_expected( + &self, + expected: &IdunnExpectedIncarnationRecord, + provider_anchor: &ServiceIdentityTrustAnchor, + ) -> Result { + expected.validate()?; + let anchor = runtime_presence_trust_anchor(expected, provider_anchor)?; + replace_records_atomically( + &self.projection_store, + &[ + CultCacheEnvelope { + key: expected.target.clone(), + r#type: IdunnExpectedIncarnationRecord::TYPE.into(), + payload: expected.canonical_bytes()?, + stored_at: chrono::Utc::now().to_rfc3339(), + schema_id: Some(IDUNN_EXPECTED_INCARNATION_SCHEMA.into()), + }, + CultCacheEnvelope { + key: anchor.trust_anchor_id.clone(), + r#type: GameCultServiceTrustAnchorRecord::TYPE.into(), + payload: rmp_serde::to_vec(&anchor)?, + stored_at: rfc3339_millis(anchor.bound_at_unix_millis)?, + schema_id: Some(GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA.into()), + }, + ], )?; expected.canonical_sha256() } - fn withdraw_expected(&self, expected: &IdunnExpectedIncarnationRecord) -> Result<()> { + fn withdraw_expected( + &self, + expected: &IdunnExpectedIncarnationRecord, + provider_anchor: &ServiceIdentityTrustAnchor, + activation: Option<&IdunnRuntimeActivationRecord>, + lease: Option<&IdunnProcessWriteLeaseRecord>, + ) -> Result<()> { expected.validate()?; + let exact_anchor = runtime_presence_trust_anchor(expected, provider_anchor)?; + if let Some(activation) = activation { + activation.validate()?; + ensure!( + activation.expected_projection_sha256 == expected.canonical_sha256()? + && activation.runtime_id == expected.runtime_id, + "withdrawn activation does not bind the exact Expected projection" + ); + } + if let Some(lease) = lease { + validate_topology_lease( + expected, + activation.context("withdrawn lease has no exact activation")?, + lease, + )?; + } if !self.projection_store.exists() { return Ok(()); } - let expected_sha256 = expected.canonical_sha256()?; let store = SingleFileMessagePackBackingStore::new(&self.projection_store); for _ in 0..8 { let entries = store.pull_all_read_only_snapshot()?; let mut found_expected = false; + let mut found_anchor = false; let mut found_activation = false; + let mut found_lease = false; let mut retained = Vec::with_capacity(entries.len()); for envelope in &entries { + if envelope.r#type == GameCultServiceTrustAnchorRecord::TYPE + && envelope.key == exact_anchor.trust_anchor_id + { + ensure!( + !found_anchor + && service_trust_anchor_from_envelope(envelope)? == exact_anchor, + "refusing to withdraw a substituted runtime presence trust anchor" + ); + found_anchor = true; + continue; + } if envelope.key != expected.target { retained.push(envelope.clone()); continue; @@ -3082,22 +3700,34 @@ impl TopologyPort for CultCacheTopologyDriver { continue; } if envelope.r#type == IdunnRuntimeActivationRecord::TYPE { - let activation = + let current = IdunnRuntimeActivationRecord::decode_canonical(&envelope.payload)?; ensure!( !found_activation && envelope.schema_id.as_deref() == Some(IDUNN_RUNTIME_ACTIVATION_SCHEMA) - && activation.expected_projection_sha256 == expected_sha256 - && activation.runtime_id == expected.runtime_id, + && activation == Some(¤t), "refusing to withdraw a substituted runtime activation" ); found_activation = true; continue; } + if envelope.r#type == IdunnProcessWriteLeaseRecord::TYPE { + let current = + IdunnProcessWriteLeaseRecord::decode_canonical(&envelope.payload)?; + ensure!( + !found_lease + && envelope.schema_id.as_deref() + == Some(IDUNN_PROCESS_WRITE_LEASE_SCHEMA) + && lease == Some(¤t), + "refusing to withdraw a substituted process write lease" + ); + found_lease = true; + continue; + } retained.push(envelope.clone()); } - if !found_expected && !found_activation { + if !found_expected && !found_anchor && !found_activation && !found_lease { return Ok(()); } if store.compare_exchange_snapshot(&entries, &retained)? { @@ -3134,6 +3764,73 @@ impl TopologyPort for CultCacheTopologyDriver { activation.canonical_sha256() } + fn publish_process_write_lease( + &self, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + lease: &IdunnProcessWriteLeaseRecord, + ) -> Result { + validate_topology_lease(expected, activation, lease)?; + insert_exact_process_lease( + &self.projection_store, + expected, + activation, + CultCacheEnvelope { + key: expected.target.clone(), + r#type: IdunnProcessWriteLeaseRecord::TYPE.into(), + payload: lease.canonical_bytes()?, + stored_at: rfc3339_millis(lease.issued_at_unix_millis)?, + schema_id: Some(IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into()), + }, + )?; + lease.canonical_sha256() + } + + fn withdraw_process_write_lease( + &self, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + lease: Option<&IdunnProcessWriteLeaseRecord>, + ) -> Result<()> { + expected.validate()?; + activation.validate()?; + if let Some(lease) = lease { + validate_topology_lease(expected, activation, lease)?; + } + if !self.projection_store.exists() { + return Ok(()); + } + let store = SingleFileMessagePackBackingStore::new(&self.projection_store); + for _ in 0..8 { + let entries = store.pull_all_read_only_snapshot()?; + let mut found = false; + let mut retained = Vec::with_capacity(entries.len()); + for envelope in &entries { + if envelope.key != expected.target + || envelope.r#type != IdunnProcessWriteLeaseRecord::TYPE + { + retained.push(envelope.clone()); + continue; + } + let current = IdunnProcessWriteLeaseRecord::decode_canonical(&envelope.payload)?; + ensure!( + !found + && envelope.schema_id.as_deref() == Some(IDUNN_PROCESS_WRITE_LEASE_SCHEMA) + && lease == Some(¤t), + "refusing to withdraw an unexpected process write-lease projection" + ); + found = true; + } + if !found { + return Ok(()); + } + if store.compare_exchange_snapshot(&entries, &retained)? { + return Ok(()); + } + } + bail!("process write-lease projection changed repeatedly while withdrawing it") + } + fn receive(&self, target: &str) -> Result> { require_driver_id(target, "topology target")?; if !self.correlation_store.exists() { @@ -3163,8 +3860,9 @@ impl TopologyPort for CultCacheTopologyDriver { } /// nginx owns proxy mechanics. Idunn supplies one exact backend membership, -/// validates the complete nginx configuration, reloads it, and then observes -/// that nginx's loaded configuration contains those exact bytes. +/// validates the complete nginx configuration, and reloads it. Configuration +/// bytes are actuator state, not proof that the selected runtime answered on +/// the stable route; the control plane admits that proof separately. pub struct NginxRouteDriver { pub binding: RouteBinding, pub nginx_program: PathBuf, @@ -3211,12 +3909,9 @@ impl NginxRouteDriver { Ok(output) } - fn render( - &self, - expected: &IdunnExpectedIncarnationRecord, - runtime_instance_id: &str, - ) -> Result> { + fn render(&self, expected: &IdunnExpectedIncarnationRecord) -> Result> { expected.validate()?; + let expected_projection_sha256 = expected.canonical_sha256()?; let route = expected .route .as_ref() @@ -3226,13 +3921,14 @@ impl NginxRouteDriver { && route.stable_endpoint == self.binding.stable_endpoint, "route driver binding differs from Expected" ); - let (candidate_host, candidate_port) = endpoint_host_port( - &route.candidate_endpoint, - match self.binding.driver { - RouteDriver::NginxHttp => "http://", - RouteDriver::NginxStreamTcp => "tcp://", - }, - )?; + let endpoint_prefix = match (self.binding.driver, route.transport.as_str()) { + (RouteDriver::NginxStreamTcp, "http") => "http://", + (RouteDriver::NginxStreamTcp, "tcp") => "tcp://", + (RouteDriver::NginxStreamUdp, "rudp") => "rudp://", + _ => bail!("route driver cannot carry the Expected transport"), + }; + let (candidate_host, candidate_port) = + endpoint_host_port(&route.candidate_endpoint, endpoint_prefix)?; ensure!( candidate_host == self.binding.private_host && (self.binding.private_port_start..=self.binding.private_port_end) @@ -3241,40 +3937,30 @@ impl NginxRouteDriver { ); let upstream = nginx_identifier(&self.binding.route_id)?; let rendered = match self.binding.driver { - RouteDriver::NginxHttp => format!( - "# Idunn runtime {runtime_instance_id}\nupstream {upstream} {{\n server {candidate_host}:{candidate_port};\n keepalive 16;\n}}\n" - ), RouteDriver::NginxStreamTcp => { let (stable_host, stable_port) = - endpoint_host_port(&route.stable_endpoint, "tcp://")?; + endpoint_host_port(&route.stable_endpoint, endpoint_prefix)?; format!( - "# Idunn runtime {runtime_instance_id}\nupstream {upstream} {{\n server {candidate_host}:{candidate_port};\n}}\nserver {{\n listen {stable_host}:{stable_port};\n proxy_pass {upstream};\n}}\n" + "# Idunn Expected {expected_projection_sha256}\nupstream {upstream} {{\n server {candidate_host}:{candidate_port};\n}}\nserver {{\n listen {stable_host}:{stable_port};\n proxy_pass {upstream};\n}}\n" + ) + } + RouteDriver::NginxStreamUdp => { + let (stable_host, stable_port) = + endpoint_host_port(&route.stable_endpoint, "rudp://")?; + format!( + "# Idunn Expected {expected_projection_sha256}\nupstream {upstream} {{\n server {candidate_host}:{candidate_port};\n}}\nserver {{\n listen {stable_host}:{stable_port} udp reuseport;\n proxy_pass {upstream};\n}}\n" ) } }; Ok(rendered.into_bytes()) } - fn loaded_matches(&self, rendered: &[u8]) -> Result { - let output = self.command(&self.nginx_program, [OsString::from("-T")])?; - let loaded = std::str::from_utf8(&output.stdout) - .context("nginx loaded configuration is not UTF-8")?; - Ok(nginx_config_section(loaded, &self.binding.config_path)? - .is_some_and(|section| section.as_bytes() == rendered)) - } - - fn current_and_loaded(&self) -> Result<(Option>, Option>)> { - let current = match fs::read(&self.binding.config_path) { + fn current_configuration(&self) -> Result>> { + Ok(match fs::read(&self.binding.config_path) { Ok(bytes) => Some(bytes), Err(error) if error.kind() == ErrorKind::NotFound => None, Err(error) => return Err(error).context("reading route fragment"), - }; - let output = self.command(&self.nginx_program, [OsString::from("-T")])?; - let dump = std::str::from_utf8(&output.stdout) - .context("nginx loaded configuration is not UTF-8")?; - let loaded = nginx_config_section(dump, &self.binding.config_path)? - .map(|section| section.as_bytes().to_vec()); - Ok((current, loaded)) + }) } fn write_fragment(&self, content: Option<&[u8]>) -> Result<()> { @@ -3344,12 +4030,12 @@ impl NginxRouteDriver { self.reload() } - fn fail_after_rollback( + fn fail_after_rollback( &self, prior: Option<&[u8]>, failure: anyhow::Error, context: &str, - ) -> Result<()> { + ) -> Result { match self.restore(prior) { Ok(()) => Err(failure).context(context.to_owned()), Err(rollback) => Err(failure).context(format!( @@ -3357,22 +4043,16 @@ impl NginxRouteDriver { )), } } -} -impl RoutePort for NginxRouteDriver { - fn preflight( + pub fn preflight( &self, expected: &IdunnExpectedIncarnationRecord, runtime_instance_id: &str, incumbent: Option<&RouteObservation>, ) -> Result { - let rendered = self.render(expected, runtime_instance_id)?; + let rendered = self.render(expected)?; ensure!(!rendered.is_empty(), "candidate route rendered empty"); - let (current, loaded) = self.current_and_loaded()?; - ensure!( - current == loaded, - "nginx route baseline differs from its loaded configuration" - ); + let current = self.current_configuration()?; let incumbent_membership_sha256 = current.as_ref().map(|bytes| sha256_id(bytes)); match incumbent { Some(incumbent) => ensure!( @@ -3387,86 +4067,379 @@ impl RoutePort for NginxRouteDriver { ), } self.validate_candidate_in_private_mount(&rendered)?; - let after = self.current_and_loaded()?; + let after = self.current_configuration()?; ensure!( - after.0 == current && after.1 == current, + after == current, "nginx route baseline changed during candidate validation" ); - Ok(RoutePreflightReceipt { + let receipt = RoutePreflightReceipt { route_id: self.binding.route_id.clone(), candidate_runtime_instance_id: runtime_instance_id.to_owned(), candidate_membership_sha256: sha256_id(&rendered), incumbent_runtime_instance_id: incumbent .map(|observation| observation.runtime_instance_id.clone()), incumbent_membership_sha256, - }) + incumbent_configuration: current, + }; + receipt.validate()?; + Ok(receipt) } - fn promote( + /// Install the candidate route membership and require nginx to accept the + /// complete configuration. Success is not route admission: callers must + /// still obtain a fresh signed runtime observation through the stable + /// listener. + pub fn install( &self, expected: &IdunnExpectedIncarnationRecord, runtime_instance_id: &str, preflight: &RoutePreflightReceipt, - ) -> Result { - let rendered = self.render(expected, runtime_instance_id)?; + rollback_allowed: bool, + ) -> Result { + preflight.validate()?; + let rendered = self.render(expected)?; ensure!( preflight.route_id == self.binding.route_id && preflight.candidate_runtime_instance_id == runtime_instance_id && preflight.candidate_membership_sha256 == sha256_id(&rendered), "route preflight does not authorize this candidate membership" ); - let (prior, loaded) = self.current_and_loaded()?; + let prior = self.current_configuration()?; let prior_sha256 = prior.as_ref().map(|bytes| sha256_id(bytes)); + let candidate_already_written = prior.as_deref() == Some(rendered.as_slice()); ensure!( - prior == loaded && prior_sha256 == preflight.incumbent_membership_sha256, + candidate_already_written + || (prior == preflight.incumbent_configuration + && prior_sha256 == preflight.incumbent_membership_sha256), "route baseline changed after preflight" ); - atomic_replace(&self.binding.config_path, &rendered)?; + if !candidate_already_written { + atomic_replace(&self.binding.config_path, &rendered)?; + } if let Err(error) = self.reload() { - self.fail_after_rollback( - prior.as_deref(), - error, - "candidate route validation or reload failed", - )?; + if rollback_allowed { + return self.fail_after_rollback( + preflight.incumbent_configuration.as_deref(), + error, + "candidate route validation or reload failed", + ); + } + return Err(error).context( + "candidate route reload failed after incumbent route authority was fenced; candidate fragment retained for retry", + ); } - match self.loaded_matches(&rendered) { - Ok(true) => {} - Ok(false) => self.fail_after_rollback( - prior.as_deref(), - anyhow::anyhow!("nginx loaded configuration omitted the admitted membership"), - "candidate route observation failed", - )?, - Err(error) => self.fail_after_rollback( - prior.as_deref(), - error, - "candidate route observation failed", - )?, - } - Ok(RouteObservation { - route_id: self.binding.route_id.clone(), - runtime_instance_id: runtime_instance_id.to_owned(), - membership_sha256: sha256_id(&rendered), - }) + ensure!( + self.current_configuration()?.as_deref() == Some(rendered.as_slice()), + "candidate route fragment changed during reload" + ); + Ok(sha256_id(&rendered)) } - fn observe( + pub fn observe_membership( &self, expected: &IdunnExpectedIncarnationRecord, - observation: &RouteObservation, + membership_sha256: &str, ) -> Result { - let rendered = self.render(expected, &observation.runtime_instance_id)?; + let rendered = self.render(expected)?; ensure!( - observation.route_id == self.binding.route_id - && observation.membership_sha256 == sha256_id(&rendered), + membership_sha256 == sha256_id(&rendered), "route observation does not describe the expected membership" ); - let current = match fs::read(&self.binding.config_path) { - Ok(bytes) => bytes, - Err(error) if error.kind() == ErrorKind::NotFound => return Ok(false), - Err(error) => return Err(error).context("reading route observation"), - }; - Ok(current == rendered && self.loaded_matches(&rendered)?) + Ok(self.current_configuration()?.as_deref() == Some(rendered.as_slice())) } + + /// Restore the exact membership owned by an admitted generation. Any + /// different fragment is drift, not a rollback baseline, so it is never + /// restored after a failed reload. A failed reload removes the newly + /// written fragment so the next continuity pass cannot mistake disk bytes + /// for an adopted route. The caller must still challenge the stable + /// listener to prove that nginx workers adopted this membership. + pub fn restore_admitted_membership( + &self, + expected: &IdunnExpectedIncarnationRecord, + membership_sha256: &str, + ) -> Result<()> { + let rendered = self.render(expected)?; + ensure!( + membership_sha256 == sha256_id(&rendered), + "admitted route receipt does not describe the expected membership" + ); + self.validate_candidate_in_private_mount(&rendered)?; + if self.current_configuration()?.as_deref() != Some(rendered.as_slice()) { + atomic_replace(&self.binding.config_path, &rendered)?; + } + if let Err(reload) = self.reload() { + return match self.write_fragment(None) { + Ok(()) => Err(reload).context("reloading the exact admitted route membership"), + Err(cleanup) => Err(reload).context(format!( + "reloading the exact admitted route membership; removing its unproved fragment also failed: {cleanup:#}" + )), + }; + } + ensure!( + self.current_configuration()?.as_deref() == Some(rendered.as_slice()), + "admitted route fragment changed during restoration" + ); + Ok(()) + } + + /// Ask the stable listener for one freshly minted provider-owned presence + /// statement. This method validates the CultNet wire exchange and exact + /// document selection only; Idunn's control plane authenticates and + /// correlates the signed payload against current authority. + pub fn request_runtime_presence( + &self, + expected: &IdunnExpectedIncarnationRecord, + message_id: &str, + ) -> Result { + let route = expected + .route + .as_ref() + .context("route challenge has no Expected route")?; + ensure!( + route.route_id == self.binding.route_id + && route.stable_endpoint == self.binding.stable_endpoint, + "route challenge binding differs from Expected" + ); + self.request_runtime_presence_at(expected, message_id, &route.stable_endpoint) + } + + /// The bootstrap exception for the first managed Odin observes Warming on + /// the exact candidate endpoint. It does not install or admit a route. + pub fn request_candidate_runtime_presence( + &self, + expected: &IdunnExpectedIncarnationRecord, + message_id: &str, + ) -> Result { + let route = expected + .route + .as_ref() + .context("candidate challenge has no Expected route")?; + ensure!( + route.route_id == self.binding.route_id, + "candidate challenge binding differs from Expected" + ); + self.render(expected)?; + self.request_runtime_presence_at(expected, message_id, &route.candidate_endpoint) + } + + fn request_runtime_presence_at( + &self, + expected: &IdunnExpectedIncarnationRecord, + message_id: &str, + endpoint: &str, + ) -> Result { + expected.validate()?; + require_driver_id(message_id, "route challenge message")?; + let route = expected + .route + .as_ref() + .context("route challenge has no Expected route")?; + let request = CultNetMessage::SnapshotRequest { + message_id: message_id.to_owned(), + schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), + record_keys: Some(vec![expected.target.clone()]), + }; + let endpoint_prefix = match route.transport.as_str() { + "http" => "http://", + "tcp" => "tcp://", + "rudp" => "rudp://", + _ => bail!("route challenge transport is unsupported"), + }; + let (host, port) = endpoint_host_port(endpoint, endpoint_prefix)?; + let target = SocketAddr::new( + host.parse() + .context("route challenge endpoint host is not an IP address")?, + port, + ); + let response = match route.transport.as_str() { + "http" => { + let payload = + encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0)?; + let response = request_http_snapshot(target, &payload)?; + decode_cultnet_message_from_slice(&response, CultNetWireContract::CultNetSchemaV0)? + } + "tcp" => { + let payload = + encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0)?; + let response = request_tcp_snapshot(target, &payload)?; + decode_cultnet_message_from_slice(&response, CultNetWireContract::CultNetSchemaV0)? + } + "rudp" => request_raw_snapshot_from_rudp_catalog(CultMeshRudpSnapshotOptions { + target, + runtime_id: "idunn-route-observer".into(), + message_id: message_id.to_owned(), + schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), + record_keys: Some(vec![expected.target.clone()]), + ..CultMeshRudpSnapshotOptions::default() + })?, + _ => bail!("route challenge transport is unsupported"), + }; + exact_route_snapshot_response(response, message_id, &expected.target) + } + + /// Restore the exact preflight baseline. This may only replace the exact + /// candidate authorized by the receipt; an unrelated writer is never + /// overwritten as a side effect of rollback. + pub fn rollback( + &self, + expected: &IdunnExpectedIncarnationRecord, + runtime_instance_id: &str, + preflight: &RoutePreflightReceipt, + ) -> Result<()> { + preflight.validate()?; + let rendered = self.render(expected)?; + ensure!( + preflight.route_id == self.binding.route_id + && preflight.candidate_runtime_instance_id == runtime_instance_id + && preflight.candidate_membership_sha256 == sha256_id(&rendered), + "route preflight does not authorize rollback of this candidate membership" + ); + ensure!( + self.current_configuration()?.as_deref() == Some(rendered.as_slice()), + "candidate route changed before rollback" + ); + self.restore(preflight.incumbent_configuration.as_deref()) + } +} + +fn request_tcp_snapshot(target: SocketAddr, payload: &[u8]) -> Result> { + let mut stream = TcpStream::connect_timeout(&target, ROUTE_SNAPSHOT_TIMEOUT) + .with_context(|| format!("connecting stable CultNet TCP route {target}"))?; + stream.set_read_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT))?; + stream.set_write_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT))?; + stream.write_all(&encode_frame(payload)?)?; + stream.flush()?; + + let mut header = [0_u8; 4]; + stream + .read_exact(&mut header) + .context("reading stable CultNet TCP response frame")?; + let length = u32::from_be_bytes(header) as usize; + ensure!( + (1..=ROUTE_SNAPSHOT_MAX_BYTES).contains(&length), + "stable CultNet TCP response exceeds the route observation bound" + ); + let mut response = vec![0_u8; length]; + stream + .read_exact(&mut response) + .context("reading stable CultNet TCP response payload")?; + Ok(response) +} + +fn request_http_snapshot(target: SocketAddr, payload: &[u8]) -> Result> { + let mut stream = TcpStream::connect_timeout(&target, ROUTE_SNAPSHOT_TIMEOUT) + .with_context(|| format!("connecting stable CultNet HTTP route {target}"))?; + stream.set_read_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT))?; + stream.set_write_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT))?; + let head = format!( + "POST {ROUTE_HTTP_SNAPSHOT_PATH} HTTP/1.1\r\nHost: {target}\r\nContent-Type: application/msgpack\r\nAccept: application/msgpack\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + payload.len() + ); + stream.write_all(head.as_bytes())?; + stream.write_all(payload)?; + stream.flush()?; + + let maximum = ROUTE_HTTP_MAX_HEADER_BYTES + .checked_add(ROUTE_SNAPSHOT_MAX_BYTES) + .context("route HTTP response bound overflow")?; + let mut response = Vec::new(); + (&mut stream) + .take((maximum + 1) as u64) + .read_to_end(&mut response) + .context("reading stable CultNet HTTP response")?; + ensure!( + response.len() <= maximum, + "stable CultNet HTTP response exceeds the route observation bound" + ); + let header_end = response + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map(|index| index + 4) + .context("stable CultNet HTTP response has no header boundary")?; + ensure!( + header_end <= ROUTE_HTTP_MAX_HEADER_BYTES, + "stable CultNet HTTP response headers exceed their bound" + ); + let headers = std::str::from_utf8(&response[..header_end]) + .context("stable CultNet HTTP response headers are not UTF-8")?; + let mut lines = headers.split("\r\n"); + let status = lines + .next() + .context("stable CultNet HTTP response has no status")?; + ensure!( + matches!(status, "HTTP/1.1 200 OK" | "HTTP/1.0 200 OK"), + "stable CultNet HTTP route rejected the snapshot challenge: {status}" + ); + let mut content_length = None; + for line in lines.filter(|line| !line.is_empty()) { + let (name, value) = line + .split_once(':') + .context("stable CultNet HTTP response contains a malformed header")?; + if name.eq_ignore_ascii_case("transfer-encoding") { + bail!("stable CultNet HTTP response uses unsupported transfer encoding"); + } + if name.eq_ignore_ascii_case("content-length") { + ensure!( + content_length.is_none(), + "stable CultNet HTTP response repeats Content-Length" + ); + content_length = Some( + value + .trim() + .parse::() + .context("stable CultNet HTTP Content-Length is invalid")?, + ); + } + } + let content_length = + content_length.context("stable CultNet HTTP response has no Content-Length")?; + ensure!( + content_length <= ROUTE_SNAPSHOT_MAX_BYTES + && response.len().saturating_sub(header_end) == content_length, + "stable CultNet HTTP response body differs from its bounded Content-Length" + ); + Ok(response[header_end..].to_vec()) +} + +fn exact_route_snapshot_response( + response: CultNetMessage, + message_id: &str, + target: &str, +) -> Result { + let CultNetMessage::SnapshotResponseRaw { + message_id: response_id, + documents, + } = response + else { + bail!("stable route returned a non-raw snapshot response") + }; + ensure!( + response_id == message_id, + "stable route snapshot response belongs to another challenge" + ); + ensure!( + documents.len() == 1, + "stable route snapshot response is not the exact singleton presence document" + ); + let document = documents + .into_iter() + .next() + .expect("singleton response length was checked"); + ensure!( + document.schema_id == GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA + && document.record_key == target + && document.payload_encoding == CultNetRawPayloadEncoding::Messagepack, + "stable route substituted the runtime presence document" + ); + ensure!( + !document.payload.is_empty() && document.payload.len() <= ROUTE_SNAPSHOT_MAX_BYTES, + "stable route presence payload is empty or exceeds its bound" + ); + Ok(RouteSnapshotResponse { + message_id: response_id, + canonical_presence: document.payload, + }) } fn release_artifact<'a>( @@ -4654,24 +5627,6 @@ fn nginx_identifier(route_id: &str) -> Result { Ok(identifier) } -fn nginx_config_section<'a>(dump: &'a str, config_path: &Path) -> Result> { - let marker = format!("# configuration file {}:\n", config_path.display()); - let matches = dump.match_indices(&marker).collect::>(); - ensure!( - matches.len() <= 1, - "nginx loaded configuration contains duplicate sections for {}", - config_path.display() - ); - let Some((start, _)) = matches.first().copied() else { - return Ok(None); - }; - let content = &dump[start + marker.len()..]; - let end = content - .find("\n# configuration file ") - .map_or(content.len(), |index| index + 1); - Ok(Some(&content[..end])) -} - fn atomic_replace(path: &Path, bytes: &[u8]) -> Result<()> { #[cfg(unix)] use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; @@ -5539,6 +6494,7 @@ mod tests { IdunnExpectedIncarnationRecord, IdunnRuntimeActivationRecord, SequenceAdmittedWarming, + ServiceIdentityTrustAnchor, )> { let provider = enroll_service_identity_at::( &root.join("provider.cc"), @@ -5594,7 +6550,7 @@ mod tests { }, )?; let warming = SequenceAdmittedWarming::for_test("test-transaction", warming, 120)?; - Ok((expected, activation, warming)) + Ok((expected, activation, warming, provider.trust_anchor()?)) } fn lease( @@ -5614,52 +6570,81 @@ mod tests { state_contract_sha256: expected.state_contract_sha256.clone().unwrap(), runtime_id: expected.runtime_id.clone(), runtime_instance_id: activation.runtime_instance_id.clone(), - warming_presence_sha256: warming - .authenticated() - .record() - .signed_presence_sha256 - .clone() - .unwrap(), + warming_presence_sha256: warming.signed_presence_sha256().to_owned(), lease_epoch: 1, issued_at_unix_millis: 200, } } + fn topology_driver(root: &Path, name: &str) -> CultCacheTopologyDriver { + CultCacheTopologyDriver { + projection_store: root.join(format!("{name}.cc")), + correlation_store: root.join(format!("{name}-correlation.cc")), + } + } + + fn project_activation( + driver: &CultCacheTopologyDriver, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result<()> { + ensure!( + activation.expected_projection_sha256 == expected.canonical_sha256()?, + "test activation belongs to another Expected" + ); + upsert_record( + &driver.projection_store, + CultCacheEnvelope { + key: expected.target.clone(), + r#type: IdunnRuntimeActivationRecord::TYPE.into(), + payload: activation.canonical_bytes()?, + stored_at: rfc3339_millis(activation.issued_at_unix_millis)?, + schema_id: Some(IDUNN_RUNTIME_ACTIVATION_SCHEMA.into()), + }, + ) + } + #[test] - fn nginx_loaded_observation_is_bound_to_the_exact_file_section() -> Result<()> { - let path = Path::new("/etc/nginx/idunn-routes/service.conf"); - let rendered = "upstream idunn_service {\n server 127.0.0.1:4104;\n}\n"; - let dump = format!( - "# configuration file /etc/nginx/nginx.conf:\nevents {{}}\n# configuration file {}:\n{rendered}# configuration file /etc/nginx/other.conf:\n{rendered}", - path.display() - ); - assert_eq!(nginx_config_section(&dump, path)?, Some(rendered)); - assert_eq!( - nginx_config_section(&dump, Path::new("/etc/nginx/missing.conf"))?, - None - ); - let duplicate = format!("{dump}# configuration file {}:\n{rendered}", path.display()); - assert!(nginx_config_section(&duplicate, path).is_err()); + fn nginx_rudp_route_is_rendered_as_a_udp_stream_proxy() -> Result<()> { + let driver = NginxRouteDriver::new(RouteBinding { + driver: RouteDriver::NginxStreamUdp, + route_id: "odin-rudp".into(), + stable_endpoint: "rudp://127.0.0.1:17872".into(), + private_host: "127.0.0.1".into(), + private_port_start: 27872, + private_port_end: 27879, + config_path: "/etc/nginx/idunn-stream-routes/odin-rudp.conf".into(), + reload_unit: "nginx.service".into(), + }); + let mut candidate = expected(); + candidate.route = Some(cultnet_rs::IdunnExpectedRoute { + route_id: "odin-rudp".into(), + transport: "rudp".into(), + stable_endpoint: "rudp://127.0.0.1:17872".into(), + candidate_endpoint: "rudp://127.0.0.1:27872".into(), + }); + let rendered = String::from_utf8(driver.render(&candidate)?)?; + assert!(rendered.contains("listen 127.0.0.1:17872 udp reuseport;")); + assert!(rendered.contains("server 127.0.0.1:27872;")); Ok(()) } #[cfg(unix)] #[test] - fn nginx_preflight_validates_candidate_and_binds_the_loaded_incumbent() -> Result<()> { + fn nginx_preflight_binds_exact_incumbent_configuration_without_claiming_live_route() + -> Result<()> { use std::os::unix::fs::PermissionsExt; let temp = tempfile::tempdir()?; let config = temp.path().join("route.conf"); - let loaded = temp.path().join("loaded.conf"); let nginx = temp.path().join("nginx"); let systemd_run = temp.path().join("systemd-run"); let systemctl = temp.path().join("systemctl"); fs::write( &nginx, format!( - "#!/bin/sh\nconfig='{}'\nloaded='{}'\nif [ \"$1\" = \"-T\" ]; then\n if [ -f \"$loaded\" ]; then\n printf '# configuration file %s:\\n' \"$config\"\n /bin/cat \"$loaded\"\n fi\n exit 0\nfi\nif [ \"$1\" = \"-t\" ]; then\n candidate=${{IDUNN_TEST_SHADOW:-$config}}\n /bin/grep -q 'server 127.0.0.1:4104' \"$candidate\"\n exit $?\nfi\nexit 64\n", + "#!/bin/sh\nconfig='{}'\nif [ \"$1\" = \"-t\" ]; then\n candidate=${{IDUNN_TEST_SHADOW:-$config}}\n /bin/grep -q 'server 127.0.0.1:4104' \"$candidate\"\n exit $?\nfi\nexit 64\n", config.display(), - loaded.display(), ), )?; fs::write( @@ -5670,18 +6655,14 @@ mod tests { )?; fs::write( &systemctl, - format!( - "#!/bin/sh\nif [ \"$1\" = \"reload\" ]; then\n /bin/cp '{}' '{}'\n exit 0\nfi\nexit 64\n", - config.display(), - loaded.display(), - ), + "#!/bin/sh\nif [ -e \"$0.fail\" ]; then exit 65; fi\nif [ \"$1\" = \"reload\" ]; then echo reload >> \"$0.calls\"; exit 0; fi\nexit 64\n", )?; fs::set_permissions(&nginx, fs::Permissions::from_mode(0o755))?; fs::set_permissions(&systemd_run, fs::Permissions::from_mode(0o755))?; fs::set_permissions(&systemctl, fs::Permissions::from_mode(0o755))?; let binding = RouteBinding { - driver: RouteDriver::NginxHttp, + driver: RouteDriver::NginxStreamTcp, route_id: "service-route".into(), stable_endpoint: "http://127.0.0.1:4103".into(), private_host: "127.0.0.1".into(), @@ -5694,7 +6675,7 @@ mod tests { binding, nginx_program: nginx, systemd_run_program: systemd_run, - systemctl_program: systemctl, + systemctl_program: systemctl.clone(), preflight_root: temp.path().join("preflight"), }; let mut candidate = expected(); @@ -5706,23 +6687,58 @@ mod tests { }); let preflight = driver.preflight(&candidate, &digest('a'), None)?; assert!(!config.exists()); - assert!(!loaded.exists()); assert_eq!(fs::read_dir(&driver.preflight_root)?.count(), 0); - let admitted = driver.promote(&candidate, &digest('a'), &preflight)?; - assert!(driver.observe(&candidate, &admitted)?); + let candidate_bytes = driver.render(&candidate)?; + fs::write(&config, &candidate_bytes)?; + let membership_sha256 = driver.install(&candidate, &digest('a'), &preflight, true)?; + assert_eq!( + fs::read_to_string(systemctl.with_extension("calls"))?, + "reload\n" + ); + assert!(driver.observe_membership(&candidate, &membership_sha256)?); let admitted_bytes = fs::read(&config)?; - assert_eq!(fs::read(&loaded)?, admitted_bytes); + assert!(std::str::from_utf8(&admitted_bytes)?.contains("listen 127.0.0.1:4103;")); + let admitted = RouteObservation { + route_id: "service-route".into(), + runtime_instance_id: digest('a'), + membership_sha256, + signed_presence_sha256: digest('c'), + observed_at_unix_millis: 1, + }; let next_preflight = driver.preflight(&candidate, &digest('b'), Some(&admitted))?; assert_eq!(fs::read(&config)?, admitted_bytes); + fs::write(systemctl.with_extension("fail"), b"fail\n")?; + assert!( + driver + .install(&candidate, &digest('b'), &next_preflight, false) + .is_err() + ); + assert_eq!(fs::read(&config)?, admitted_bytes); + fs::remove_file(systemctl.with_extension("fail"))?; + driver.install(&candidate, &digest('b'), &next_preflight, false)?; fs::write(&config, b"foreign route\n")?; assert!( driver - .promote(&candidate, &digest('b'), &next_preflight) + .install(&candidate, &digest('b'), &next_preflight, true) .is_err() ); - assert_eq!(fs::read(&loaded)?, admitted_bytes); + driver.restore_admitted_membership(&candidate, &admitted.membership_sha256)?; + assert_eq!(fs::read(&config)?, admitted_bytes); + assert!(driver.observe_membership(&candidate, &admitted.membership_sha256)?); + + fs::write(&config, b"foreign route after admission\n")?; + fs::write(systemctl.with_extension("fail"), b"fail\n")?; + assert!( + driver + .restore_admitted_membership(&candidate, &admitted.membership_sha256) + .is_err() + ); + assert!(!config.exists()); + fs::remove_file(systemctl.with_extension("fail"))?; + driver.restore_admitted_membership(&candidate, &admitted.membership_sha256)?; + assert_eq!(fs::read(&config)?, admitted_bytes); Ok(()) } @@ -5773,65 +6789,70 @@ mod tests { #[test] fn expected_withdrawal_is_exact_atomic_and_idempotent() -> Result<()> { let temp = tempfile::tempdir()?; - let (expected, activation, _) = authenticated_warming(temp.path())?; - let projection_store = temp.path().join("projection.cc"); - let driver = CultCacheTopologyDriver { - projection_store: projection_store.clone(), - correlation_store: temp.path().join("correlation.cc"), - }; - driver.publish_expected(&expected)?; - upsert_record( - &projection_store, - CultCacheEnvelope { - key: expected.target.clone(), - r#type: IdunnRuntimeActivationRecord::TYPE.into(), - payload: activation.canonical_bytes()?, - stored_at: "2026-09-03T00:00:00Z".into(), - schema_id: Some(IDUNN_RUNTIME_ACTIVATION_SCHEMA.into()), - }, - )?; - upsert_record( - &projection_store, - CultCacheEnvelope { - key: "unrelated".into(), - r#type: "test.unrelated".into(), - payload: vec![0x90], - stored_at: "2026-09-03T00:00:00Z".into(), - schema_id: Some("test.unrelated.v1".into()), - }, - )?; - - driver.withdraw_expected(&expected)?; - let remaining = SingleFileMessagePackBackingStore::new(&projection_store) + let (expected, activation, warming, provider_anchor) = authenticated_warming(temp.path())?; + let lease = lease(&expected, &activation, &warming); + let driver = topology_driver(temp.path(), "projection"); + driver.publish_expected(&expected, &provider_anchor)?; + let published = SingleFileMessagePackBackingStore::new(&driver.projection_store) .pull_all_read_only_snapshot()?; - assert_eq!(remaining.len(), 1); - assert_eq!(remaining[0].key, "unrelated"); - driver.withdraw_expected(&expected)?; + assert_eq!(published.len(), 2); + assert!(published.iter().any(|entry| { + entry.key == expected.target && entry.r#type == IdunnExpectedIncarnationRecord::TYPE + })); + let published_anchor = published + .iter() + .find(|entry| entry.r#type == GameCultServiceTrustAnchorRecord::TYPE) + .context("atomic Expected set has no provider trust anchor")?; + assert_eq!( + service_trust_anchor_from_envelope(published_anchor)?, + runtime_presence_trust_anchor(&expected, &provider_anchor)? + ); + project_activation(&driver, &expected, &activation)?; + driver.publish_process_write_lease(&expected, &activation, &lease)?; - let substituted_store = temp.path().join("substituted-expected.cc"); - let substituted_driver = CultCacheTopologyDriver { - projection_store: substituted_store.clone(), - correlation_store: temp.path().join("substituted-correlation.cc"), - }; - substituted_driver.publish_expected(&expected)?; let mut substituted_expected = expected.clone(); substituted_expected.incarnation_id = "another-incarnation".into(); + upsert_record( + &driver.projection_store, + CultCacheEnvelope { + key: expected.target.clone(), + r#type: IdunnExpectedIncarnationRecord::TYPE.into(), + payload: substituted_expected.canonical_bytes()?, + stored_at: "2026-09-03T00:00:00Z".into(), + schema_id: Some(IDUNN_EXPECTED_INCARNATION_SCHEMA.into()), + }, + )?; assert!( - substituted_driver - .withdraw_expected(&substituted_expected) + driver + .withdraw_expected(&expected, &provider_anchor, Some(&activation), Some(&lease)) .is_err() ); + driver.publish_expected(&expected, &provider_anchor)?; - let substituted_activation_store = temp.path().join("substituted-activation.cc"); - let substituted_activation_driver = CultCacheTopologyDriver { - projection_store: substituted_activation_store.clone(), - correlation_store: temp.path().join("substituted-activation-correlation.cc"), - }; - substituted_activation_driver.publish_expected(&expected)?; - let mut substituted_activation = activation; + let mut substituted_anchor = runtime_presence_trust_anchor(&expected, &provider_anchor)?; + substituted_anchor.runtime_id = "another-runtime".into(); + substituted_anchor.validate()?; + upsert_record( + &driver.projection_store, + CultCacheEnvelope { + key: substituted_anchor.trust_anchor_id.clone(), + r#type: GameCultServiceTrustAnchorRecord::TYPE.into(), + payload: rmp_serde::to_vec(&substituted_anchor)?, + stored_at: "2026-09-03T00:00:00Z".into(), + schema_id: Some(GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA.into()), + }, + )?; + assert!( + driver + .withdraw_expected(&expected, &provider_anchor, Some(&activation), Some(&lease)) + .is_err() + ); + driver.publish_expected(&expected, &provider_anchor)?; + + let mut substituted_activation = activation.clone(); substituted_activation.expected_projection_sha256 = digest('a'); upsert_record( - &substituted_activation_store, + &driver.projection_store, CultCacheEnvelope { key: expected.target.clone(), r#type: IdunnRuntimeActivationRecord::TYPE.into(), @@ -5841,10 +6862,199 @@ mod tests { }, )?; assert!( - substituted_activation_driver - .withdraw_expected(&expected) + driver + .withdraw_expected(&expected, &provider_anchor, Some(&activation), None) .is_err() ); + project_activation(&driver, &expected, &activation)?; + + let mut substituted_lease = lease.clone(); + substituted_lease.lease_epoch += 1; + assert!( + driver + .publish_process_write_lease(&expected, &activation, &substituted_lease) + .is_err() + ); + assert!( + driver + .withdraw_expected( + &expected, + &provider_anchor, + Some(&activation), + Some(&substituted_lease), + ) + .is_err() + ); + + upsert_record( + &driver.projection_store, + CultCacheEnvelope { + key: "unrelated".into(), + r#type: "test.unrelated".into(), + payload: vec![0x90], + stored_at: "2026-09-03T00:00:00Z".into(), + schema_id: Some("test.unrelated.v1".into()), + }, + )?; + driver.withdraw_expected(&expected, &provider_anchor, Some(&activation), Some(&lease))?; + let remaining = SingleFileMessagePackBackingStore::new(&driver.projection_store) + .pull_all_read_only_snapshot()?; + assert_eq!(remaining.len(), 1); + assert_eq!(remaining[0].key, "unrelated"); + driver.withdraw_expected(&expected, &provider_anchor, Some(&activation), Some(&lease))?; + Ok(()) + } + + #[test] + fn incumbent_projection_restoration_demotes_to_expected_only_and_rejects_unknown_authority() + -> Result<()> { + let temp = tempfile::tempdir()?; + let (incumbent, activation, warming, provider_anchor) = authenticated_warming(temp.path())?; + let lease = lease(&incumbent, &activation, &warming); + let driver = topology_driver(temp.path(), "restoration"); + driver.publish_expected(&incumbent, &provider_anchor)?; + project_activation(&driver, &incumbent, &activation)?; + driver.publish_process_write_lease(&incumbent, &activation, &lease)?; + + let mut candidate = incumbent.clone(); + candidate.plan_id = digest('a'); + candidate.incarnation_id = "candidate-incarnation".into(); + candidate.sealed_release_id = digest('b'); + candidate.artifact_sha256 = digest('c'); + let mut candidate_activation = activation.clone(); + candidate_activation.expected_projection_sha256 = candidate.canonical_sha256()?; + candidate_activation.runtime_instance_id = digest('d'); + driver.publish_expected(&candidate, &provider_anchor)?; + project_activation(&driver, &candidate, &candidate_activation)?; + + upsert_record( + &driver.projection_store, + CultCacheEnvelope { + key: "unrelated".into(), + r#type: "test.unrelated".into(), + payload: vec![0x90], + stored_at: "2026-09-03T00:00:00Z".into(), + schema_id: Some("test.unrelated.v1".into()), + }, + )?; + + assert_eq!( + driver.restore_admitted_expected_only( + &candidate, + &provider_anchor, + Some(&candidate_activation), + &incumbent, + &provider_anchor, + &activation, + Some(&lease), + )?, + incumbent.canonical_sha256()? + ); + + let restored = SingleFileMessagePackBackingStore::new(&driver.projection_store) + .pull_all_read_only_snapshot()?; + assert_eq!(restored.len(), 3); + assert_eq!( + IdunnExpectedIncarnationRecord::decode_canonical( + &restored + .iter() + .find(|entry| entry.r#type == IdunnExpectedIncarnationRecord::TYPE) + .unwrap() + .payload, + )?, + incumbent + ); + assert_eq!( + service_trust_anchor_from_envelope( + restored + .iter() + .find(|entry| entry.r#type == GameCultServiceTrustAnchorRecord::TYPE) + .unwrap(), + )?, + runtime_presence_trust_anchor(&incumbent, &provider_anchor)? + ); + assert!( + restored + .iter() + .all(|entry| entry.r#type != IdunnRuntimeActivationRecord::TYPE + && entry.r#type != IdunnProcessWriteLeaseRecord::TYPE) + ); + assert!( + restored + .iter() + .any(|entry| entry.key == "unrelated" && entry.r#type == "test.unrelated") + ); + let idempotent_bytes = fs::read(&driver.projection_store)?; + driver.restore_admitted_expected_only( + &candidate, + &provider_anchor, + Some(&candidate_activation), + &incumbent, + &provider_anchor, + &activation, + Some(&lease), + )?; + assert_eq!(fs::read(&driver.projection_store)?, idempotent_bytes); + + let admitted_anchor = runtime_presence_trust_anchor(&incumbent, &provider_anchor)?; + let mut unknown_expected = incumbent.clone(); + unknown_expected.incarnation_id = "unknown-incarnation".into(); + let mut unknown_anchor = admitted_anchor.clone(); + unknown_anchor.runtime_id = "unknown-runtime".into(); + unknown_anchor.validate()?; + let mut unknown_activation = activation.clone(); + unknown_activation.runtime_instance_id = digest('e'); + let mut unknown_lease = lease.clone(); + unknown_lease.lease_epoch += 1; + let substitutions = [ + CultCacheEnvelope { + key: incumbent.target.clone(), + r#type: IdunnExpectedIncarnationRecord::TYPE.into(), + payload: unknown_expected.canonical_bytes()?, + stored_at: "2026-09-03T00:00:00Z".into(), + schema_id: Some(IDUNN_EXPECTED_INCARNATION_SCHEMA.into()), + }, + CultCacheEnvelope { + key: admitted_anchor.trust_anchor_id.clone(), + r#type: GameCultServiceTrustAnchorRecord::TYPE.into(), + payload: rmp_serde::to_vec(&unknown_anchor)?, + stored_at: "2026-09-03T00:00:00Z".into(), + schema_id: Some(GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA.into()), + }, + CultCacheEnvelope { + key: incumbent.target.clone(), + r#type: IdunnRuntimeActivationRecord::TYPE.into(), + payload: unknown_activation.canonical_bytes()?, + stored_at: "2026-09-03T00:00:00Z".into(), + schema_id: Some(IDUNN_RUNTIME_ACTIVATION_SCHEMA.into()), + }, + CultCacheEnvelope { + key: incumbent.target.clone(), + r#type: IdunnProcessWriteLeaseRecord::TYPE.into(), + payload: unknown_lease.canonical_bytes()?, + stored_at: "2026-09-03T00:00:00Z".into(), + schema_id: Some(IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into()), + }, + ]; + for substitution in substitutions { + fs::write(&driver.projection_store, &idempotent_bytes)?; + upsert_record(&driver.projection_store, substitution)?; + let before = fs::read(&driver.projection_store)?; + assert!( + driver + .restore_admitted_expected_only( + &candidate, + &provider_anchor, + Some(&candidate_activation), + &incumbent, + &provider_anchor, + &activation, + Some(&lease), + ) + .is_err() + ); + assert_eq!(fs::read(&driver.projection_store)?, before); + } Ok(()) } @@ -5886,19 +7096,75 @@ mod tests { let temp = tempfile::tempdir()?; let path = temp.path().join("lease.cc"); let driver = CultCacheWriteLeaseDriver::new("service", &path); - let (expected, activation, warming) = authenticated_warming(temp.path())?; + let (expected, activation, warming, provider_anchor) = authenticated_warming(temp.path())?; let lease = lease(&expected, &activation, &warming); + let topology = topology_driver(temp.path(), "lease-projection"); + topology.publish_expected(&expected, &provider_anchor)?; + project_activation(&topology, &expected, &activation)?; + let lease_sha256 = driver.grant(&expected, &activation, &warming, &lease)?; assert_eq!(lease_sha256, lease.canonical_sha256()?); assert!(driver.observe(&expected, &activation, &warming, &lease)?); + assert!( + !SingleFileMessagePackBackingStore::new(&topology.projection_store) + .pull_all_read_only_snapshot()? + .iter() + .any(|entry| entry.r#type == IdunnProcessWriteLeaseRecord::TYPE) + ); + assert_eq!( + topology.publish_process_write_lease(&expected, &activation, &lease)?, + lease_sha256 + ); let mut surprise = lease.clone(); surprise.lease_epoch = 2; assert!(driver.revoke_exact(Some(&surprise)).is_err()); assert!(driver.observe(&expected, &activation, &warming, &lease)?); + SingleFileMessagePackBackingStore::new(&path).with_read_only_shared_snapshot(|_| { + let denial = driver + .revoke_exact(Some(&lease)) + .expect_err("shared lifetime holder must deny without blocking"); + assert!( + denial + .to_string() + .contains("held by a shared-lock consumer") + ); + Ok(()) + })?; + assert!(driver.observe(&expected, &activation, &warming, &lease)?); + driver.revoke_exact(Some(&lease))?; assert!(driver.observe_empty()?); + topology.withdraw_process_write_lease(&expected, &activation, Some(&lease))?; + assert!( + !SingleFileMessagePackBackingStore::new(&topology.projection_store) + .pull_all_read_only_snapshot()? + .iter() + .any(|entry| entry.r#type == IdunnProcessWriteLeaseRecord::TYPE) + ); + topology.withdraw_process_write_lease(&expected, &activation, Some(&lease))?; + driver.revoke_exact(Some(&lease))?; + + SingleFileMessagePackBackingStore::new(&path).with_read_only_shared_snapshot( + |snapshot| { + assert!(snapshot.is_empty()); + let denial = driver + .grant(&expected, &activation, &warming, &lease) + .expect_err("shared lifetime holder must deny grant without blocking"); + assert!( + denial + .to_string() + .contains("held by a shared-lock consumer") + ); + Ok(()) + }, + )?; + assert!(driver.observe_empty()?); + assert_eq!( + driver.grant(&expected, &activation, &warming, &lease)?, + lease.canonical_sha256()? + ); driver.revoke_exact(Some(&lease))?; Ok(()) } diff --git a/crates/idunn-daemon/src/provisioning.rs b/crates/idunn-daemon/src/provisioning.rs index 33612b8..1487768 100644 --- a/crates/idunn-daemon/src/provisioning.rs +++ b/crates/idunn-daemon/src/provisioning.rs @@ -5,10 +5,12 @@ use cultcache_rs::{ use cultnet_rs::{ GameCultProviderHealthIdentity, IDUNN_DEPLOYMENT_BRAKE_AUTHORITY, IDUNN_DEPLOYMENT_BRAKE_ID, IDUNN_DEPLOYMENT_BRAKE_SCHEMA, IDUNN_DEPLOYMENT_BRAKE_SCOPE, IDUNN_DEPLOYMENT_RELEASE_PURPOSE, + IDUNN_LIFECYCLE_BRAKE_AUTHORITY, IDUNN_LIFECYCLE_BRAKE_SCHEMA, IDUNN_LIFECYCLE_BRAKE_SCOPE, IdunnDeploymentBrakeObservation, IdunnDeploymentBrakeOperatorIdentity, - IdunnDeploymentBrakeRecord, IdunnDeploymentBrakeReleasePurpose, IdunnServiceIdentity, - ServiceIdentityProfile, ServiceIdentityTrustAnchor, derive_service_identity_id, - enroll_service_identity_at, evaluate_idunn_deployment_brake, + IdunnDeploymentBrakeRecord, IdunnDeploymentBrakeReleasePurpose, IdunnLifecycleBrakeObservation, + IdunnLifecycleBrakeRecord, IdunnServiceIdentity, OdinTopologyIdentity, ServiceIdentityProfile, + ServiceIdentityTrustAnchor, derive_service_identity_id, enroll_service_identity_at, + evaluate_idunn_continuity_restart, evaluate_idunn_deployment_brake, export_service_identity_trust_anchor, open_service_identity_at, }; use odin_core::{ @@ -49,39 +51,34 @@ pub fn run(args: impl IntoIterator) -> Result<()> { let public_key = provider_health_public_key(&options)?; println!("{public_key}"); } - "enroll-idunn-identity" => { - require_only(&options, &["private-store"])?; - enroll_service_identity_at::(&path(&options, "private-store")?)?; - } + "export-provider-health-public-anchor" => export_identity_anchor::< + GameCultProviderHealthIdentity, + >( + &options, "provider health public anchor" + )?, + "enroll-idunn-identity" => enroll_identity::(&options)?, "export-idunn-public-anchor" => { - require_only(&options, &["private-store", "public-anchor"])?; - let private = path(&options, "private-store")?; - let public = path(&options, "public-anchor")?; - reject_alias(&private, &public)?; - refuse_existing(&public, "public anchor")?; - let signer = open_service_identity_at::(&private)?; - export_service_identity_trust_anchor(&signer, &public)?; + export_identity_anchor::(&options, "Idunn public anchor")? + } + "enroll-odin-topology-identity" => enroll_identity::(&options)?, + "export-odin-topology-public-anchor" => { + export_identity_anchor::(&options, "Odin topology public anchor")? } "enroll-deployment-brake-operator" => { - require_only(&options, &["private-store"])?; - enroll_service_identity_at::(&path( - &options, - "private-store", - )?)?; + enroll_identity::(&options)? } "export-deployment-brake-operator-anchor" => { - require_only(&options, &["private-store", "public-anchor"])?; - let private = path(&options, "private-store")?; - let public = path(&options, "public-anchor")?; - reject_alias(&private, &public)?; - refuse_existing(&public, "deployment brake operator anchor")?; - let signer = - open_service_identity_at::(&private)?; - export_service_identity_trust_anchor(&signer, &public)?; + export_identity_anchor::( + &options, + "deployment brake operator anchor", + )? } "deployment-brake-engage" => deployment_brake_engage(&options)?, "deployment-brake-release" => deployment_brake_release(&options)?, "deployment-brake-status" => deployment_brake_status(&options)?, + "lifecycle-brake-engage" => lifecycle_brake_engage(&options)?, + "lifecycle-brake-release" => lifecycle_brake_release(&options)?, + "lifecycle-brake-status" => lifecycle_brake_status(&options)?, "create-daemon-health-trust-binding" => create_health_binding(&options)?, "add-daemon-health-trust-binding" => add_health_binding(&options)?, "require-daemon-health-release-binding" => require_health_binding_release(&options)?, @@ -97,6 +94,26 @@ pub fn run(args: impl IntoIterator) -> Result<()> { Ok(()) } +fn enroll_identity(options: &BTreeMap) -> Result<()> { + require_only(options, &["private-store"])?; + enroll_service_identity_at::

(&path(options, "private-store")?)?; + Ok(()) +} + +fn export_identity_anchor( + options: &BTreeMap, + label: &str, +) -> Result<()> { + require_only(options, &["private-store", "public-anchor"])?; + let private = path(options, "private-store")?; + let public = path(options, "public-anchor")?; + reject_alias(&private, &public)?; + refuse_existing(&public, label)?; + let signer = open_service_identity_at::

(&private)?; + export_service_identity_trust_anchor(&signer, &public)?; + Ok(()) +} + fn enroll_provider_health_identity(options: &BTreeMap) -> Result { require_only(options, &["private-store"])?; let signer = enroll_service_identity_at::(&path( @@ -262,6 +279,184 @@ fn deployment_brake_status(options: &BTreeMap) -> Result<()> { Ok(()) } +fn lifecycle_brake_engage(options: &BTreeMap) -> Result<()> { + require_only( + options, + &[ + "store", + "runtime-id", + "target", + "reason", + "updated-at-unix-millis", + ], + )?; + let updated = parse_u64(options, "updated-at-unix-millis")?; + let record = IdunnLifecycleBrakeRecord { + schema_version: IDUNN_LIFECYCLE_BRAKE_SCHEMA.into(), + authority: IDUNN_LIFECYCLE_BRAKE_AUTHORITY.into(), + runtime_id: required(options, "runtime-id")?.into(), + target: required(options, "target")?.into(), + scope: IDUNN_LIFECYCLE_BRAKE_SCOPE.into(), + status: "engaged".into(), + reason: required(options, "reason")?.into(), + updated_at_unix_millis: updated, + released_until_unix_millis: None, + }; + replace_lifecycle_brake(&path(options, "store")?, record) +} + +fn lifecycle_brake_release(options: &BTreeMap) -> Result<()> { + allow_only( + options, + &[ + "store", + "runtime-id", + "target", + "reason", + "updated-at-unix-millis", + "released-until-unix-millis", + ], + )?; + let updated = parse_u64(options, "updated-at-unix-millis")?; + let record = IdunnLifecycleBrakeRecord { + schema_version: IDUNN_LIFECYCLE_BRAKE_SCHEMA.into(), + authority: IDUNN_LIFECYCLE_BRAKE_AUTHORITY.into(), + runtime_id: required(options, "runtime-id")?.into(), + target: required(options, "target")?.into(), + scope: IDUNN_LIFECYCLE_BRAKE_SCOPE.into(), + status: "released".into(), + reason: required(options, "reason")?.into(), + updated_at_unix_millis: updated, + released_until_unix_millis: options + .get("released-until-unix-millis") + .map(|value| { + value + .parse() + .context("--released-until-unix-millis must be u64") + }) + .transpose()?, + }; + replace_lifecycle_brake(&path(options, "store")?, record) +} + +fn lifecycle_brake_status(options: &BTreeMap) -> Result<()> { + require_only( + options, + &["store", "runtime-id", "target", "now-unix-millis"], + )?; + let runtime_id = lifecycle_identifier(options, "runtime-id")?; + let target = lifecycle_identifier(options, "target")?; + let now = parse_u64(options, "now-unix-millis")?; + match read_lifecycle_brake(&path(options, "store")?)? { + None => { + evaluate_idunn_continuity_restart( + IdunnLifecycleBrakeObservation::Missing, + runtime_id, + target, + now, + ) + .map_err(|denial| anyhow!("lifecycle brake denied continuity restart: {denial:?}"))?; + println!("allowed runtime={runtime_id} target={target} brake=absent"); + } + Some(record) => { + evaluate_idunn_continuity_restart( + IdunnLifecycleBrakeObservation::Present(&record), + runtime_id, + target, + now, + ) + .map_err(|denial| anyhow!("lifecycle brake denied continuity restart: {denial:?}"))?; + if let Some(expires) = record.released_until_unix_millis { + println!( + "allowed runtime={} target={} brake=released expires={expires}", + record.runtime_id, record.target + ); + } else { + println!( + "allowed runtime={} target={} brake=released expires=never", + record.runtime_id, record.target + ); + } + } + } + Ok(()) +} + +fn replace_lifecycle_brake(path: &Path, record: IdunnLifecycleBrakeRecord) -> Result<()> { + record.validate()?; + let store = SingleFileMessagePackBackingStore::new(path); + let entries = store.pull_all_read_only_snapshot()?; + let current = match entries.as_slice() { + [] => None, + [entry] + if entry.r#type == IdunnLifecycleBrakeRecord::TYPE + && entry.schema_id.as_deref() == Some(IDUNN_LIFECYCLE_BRAKE_SCHEMA) => + { + let current_record = IdunnLifecycleBrakeRecord::decode_canonical(&entry.payload)?; + if entry.key != current_record.target { + bail!("lifecycle brake envelope key does not match its target"); + } + if current_record.runtime_id != record.runtime_id + || current_record.target != record.target + { + bail!("lifecycle brake store is bound to another runtime or target"); + } + if record.updated_at_unix_millis <= current_record.updated_at_unix_millis { + bail!("lifecycle brake update time must advance"); + } + Some(entry.clone()) + } + _ => bail!("lifecycle brake store is corrupt or ambiguous"), + }; + let next = typed_envelope( + &record.target, + &record, + IDUNN_LIFECYCLE_BRAKE_SCHEMA, + record.updated_at_unix_millis, + )?; + if !store.compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: IdunnLifecycleBrakeRecord::TYPE.into(), + key: record.target.clone(), + current, + }], + &[next], + )? { + bail!("lifecycle brake changed during transition"); + } + Ok(()) +} + +fn read_lifecycle_brake(path: &Path) -> Result> { + if !path.exists() { + return Ok(None); + } + let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; + let entry = match entries.as_slice() { + [] => return Ok(None), + [entry] => entry, + _ => bail!("lifecycle brake store is ambiguous"), + }; + if entry.r#type != IdunnLifecycleBrakeRecord::TYPE + || entry.schema_id.as_deref() != Some(IDUNN_LIFECYCLE_BRAKE_SCHEMA) + { + bail!("lifecycle brake store contains a foreign record"); + } + let record = IdunnLifecycleBrakeRecord::decode_canonical(&entry.payload)?; + if entry.key != record.target { + bail!("lifecycle brake envelope key does not match its target"); + } + Ok(Some(record)) +} + +fn lifecycle_identifier<'a>(options: &'a BTreeMap, name: &str) -> Result<&'a str> { + let value = required(options, name)?; + if value.trim().is_empty() || value.len() > 256 || value.chars().any(char::is_control) { + bail!("--{name} is empty, oversized, or contains control characters"); + } + Ok(value) +} + fn replace_brake(path: &Path, record: IdunnDeploymentBrakeRecord, millis: u64) -> Result<()> { let store = SingleFileMessagePackBackingStore::new(path); let entries = store.pull_all_read_only_snapshot()?; @@ -907,7 +1102,7 @@ fn normalized(path: &Path) -> Result { } fn usage() -> &'static str { - "Usage: idunn-provision enroll-provider-health-identity --private-store \n idunn-provision provider-health-public-key --private-store \n idunn-provision enroll-idunn-identity --private-store \n idunn-provision export-idunn-public-anchor --private-store --public-anchor \n idunn-provision enroll-deployment-brake-operator --private-store \n idunn-provision export-deployment-brake-operator-anchor --private-store --public-anchor \n idunn-provision deployment-brake-engage --store --runtime-id --owner --reason --observed-at-unix-millis \n idunn-provision deployment-brake-release --store --private-store --runtime-id --owner --reason --authorization-id --release-id --deployment-id --issued-at-unix-millis --expires-at-unix-millis \n idunn-provision deployment-brake-status --store --operator-anchor --runtime-id [--release-id --deployment-id --now-unix-millis ]\n idunn-provision create-daemon-health-trust-binding --output --binding-id --daemon --health-contract --source-runtime --signer-public-key-hex --bound-at-unix-millis --release-binding-required \n idunn-provision add-daemon-health-trust-binding --output --binding-id --daemon --health-contract --source-runtime --signer-public-key-hex --bound-at-unix-millis --release-binding-required \n idunn-provision require-daemon-health-release-binding --store --binding-id --daemon --health-contract --source-runtime --signer-public-key-hex \n idunn-provision rotate-daemon-health-trust-signer --output --binding-id --signer-public-key-hex --bound-at-unix-millis \n idunn-provision validate-daemon-health-trust-binding --input \n idunn-provision create-provider-projection-trust-anchor --output --trust-anchor-id --runtime-id --idunn-public-anchor --bound-at-unix-millis --expires-at-unix-millis \n idunn-provision validate-provider-projection-trust-anchor --input --idunn-public-anchor " + "Usage: idunn-provision enroll-provider-health-identity --private-store \n idunn-provision provider-health-public-key --private-store \n idunn-provision export-provider-health-public-anchor --private-store --public-anchor \n idunn-provision enroll-idunn-identity --private-store \n idunn-provision export-idunn-public-anchor --private-store --public-anchor \n idunn-provision enroll-odin-topology-identity --private-store \n idunn-provision export-odin-topology-public-anchor --private-store --public-anchor \n idunn-provision enroll-deployment-brake-operator --private-store \n idunn-provision export-deployment-brake-operator-anchor --private-store --public-anchor \n idunn-provision deployment-brake-engage --store --runtime-id --owner --reason --observed-at-unix-millis \n idunn-provision deployment-brake-release --store --private-store --runtime-id --owner --reason --authorization-id --release-id --deployment-id --issued-at-unix-millis --expires-at-unix-millis \n idunn-provision deployment-brake-status --store --operator-anchor --runtime-id [--release-id --deployment-id --now-unix-millis ]\n idunn-provision lifecycle-brake-engage --store --runtime-id --target --reason --updated-at-unix-millis \n idunn-provision lifecycle-brake-release --store --runtime-id --target --reason --updated-at-unix-millis [--released-until-unix-millis ]\n idunn-provision lifecycle-brake-status --store --runtime-id --target --now-unix-millis \n idunn-provision create-daemon-health-trust-binding --output --binding-id --daemon --health-contract --source-runtime --signer-public-key-hex --bound-at-unix-millis --release-binding-required \n idunn-provision add-daemon-health-trust-binding --output --binding-id --daemon --health-contract --source-runtime --signer-public-key-hex --bound-at-unix-millis --release-binding-required \n idunn-provision require-daemon-health-release-binding --store --binding-id --daemon --health-contract --source-runtime --signer-public-key-hex \n idunn-provision rotate-daemon-health-trust-signer --output --binding-id --signer-public-key-hex --bound-at-unix-millis \n idunn-provision validate-daemon-health-trust-binding --input \n idunn-provision create-provider-projection-trust-anchor --output --trust-anchor-id --runtime-id --idunn-public-anchor --bound-at-unix-millis --expires-at-unix-millis \n idunn-provision validate-provider-projection-trust-anchor --input --idunn-public-anchor " } #[cfg(test)] @@ -1476,6 +1671,144 @@ mod tests { Ok(()) } + #[test] + fn provider_and_odin_anchor_exports_are_profile_exact_immutable_and_non_aliasing() -> Result<()> + { + let temp = TempDir::new()?; + let provider_private = temp.path().join("provider-private.cc"); + let provider_anchor = temp.path().join("provider-anchor.cc"); + invoke(&[ + "enroll-provider-health-identity", + "--private-store", + provider_private.to_str().unwrap(), + ])?; + assert!( + invoke(&[ + "export-provider-health-public-anchor", + "--private-store", + provider_private.to_str().unwrap(), + "--public-anchor", + provider_private.to_str().unwrap(), + ]) + .is_err() + ); + invoke(&[ + "export-provider-health-public-anchor", + "--private-store", + provider_private.to_str().unwrap(), + "--public-anchor", + provider_anchor.to_str().unwrap(), + ])?; + let provider_entries = store_snapshot(&provider_anchor)?; + let [provider_envelope] = provider_entries.as_slice() else { + panic!("provider anchor store must contain exactly one envelope"); + }; + assert_eq!( + provider_envelope.r#type, + GameCultProviderHealthIdentity::TRUST_ANCHOR_TYPE + ); + assert_eq!( + provider_envelope.key, + GameCultProviderHealthIdentity::TRUST_ANCHOR_KEY + ); + assert_eq!( + provider_envelope.schema_id.as_deref(), + Some(GameCultProviderHealthIdentity::TRUST_ANCHOR_SCHEMA) + ); + let provider: ServiceIdentityTrustAnchor = + rmp_serde::from_slice(&provider_envelope.payload)?; + assert_eq!(rmp_serde::to_vec(&provider)?, provider_envelope.payload); + assert_eq!( + provider.identity_id, + derive_service_identity_id::(&provider.public_key)? + ); + assert!(open_service_identity_at::(&provider_private).is_err()); + let provider_before = std::fs::read(&provider_anchor)?; + assert!( + invoke(&[ + "export-provider-health-public-anchor", + "--private-store", + provider_private.to_str().unwrap(), + "--public-anchor", + provider_anchor.to_str().unwrap(), + ]) + .is_err() + ); + assert_eq!(std::fs::read(&provider_anchor)?, provider_before); + + let odin_private = temp.path().join("odin-private.cc"); + let odin_anchor = temp.path().join("odin-anchor.cc"); + invoke(&[ + "enroll-odin-topology-identity", + "--private-store", + odin_private.to_str().unwrap(), + ])?; + assert!(open_service_identity_at::(&odin_private).is_err()); + assert!( + invoke(&[ + "export-odin-topology-public-anchor", + "--private-store", + odin_private.to_str().unwrap(), + "--public-anchor", + odin_private.to_str().unwrap(), + ]) + .is_err() + ); + invoke(&[ + "export-odin-topology-public-anchor", + "--private-store", + odin_private.to_str().unwrap(), + "--public-anchor", + odin_anchor.to_str().unwrap(), + ])?; + let odin_entries = store_snapshot(&odin_anchor)?; + let [odin_envelope] = odin_entries.as_slice() else { + panic!("Odin anchor store must contain exactly one envelope"); + }; + assert_eq!( + odin_envelope.r#type, + OdinTopologyIdentity::TRUST_ANCHOR_TYPE + ); + assert_eq!(odin_envelope.key, OdinTopologyIdentity::TRUST_ANCHOR_KEY); + assert_eq!( + odin_envelope.schema_id.as_deref(), + Some(OdinTopologyIdentity::TRUST_ANCHOR_SCHEMA) + ); + let odin: ServiceIdentityTrustAnchor = rmp_serde::from_slice(&odin_envelope.payload)?; + assert_eq!(rmp_serde::to_vec(&odin)?, odin_envelope.payload); + assert_eq!( + odin.identity_id, + derive_service_identity_id::(&odin.public_key)? + ); + assert_ne!(provider.identity_id, odin.identity_id); + let odin_before = std::fs::read(&odin_anchor)?; + assert!( + invoke(&[ + "export-odin-topology-public-anchor", + "--private-store", + odin_private.to_str().unwrap(), + "--public-anchor", + odin_anchor.to_str().unwrap(), + ]) + .is_err() + ); + assert_eq!(std::fs::read(&odin_anchor)?, odin_before); + + let wrong_profile_output = temp.path().join("wrong-profile.cc"); + assert!( + invoke(&[ + "export-provider-health-public-anchor", + "--private-store", + odin_private.to_str().unwrap(), + "--public-anchor", + wrong_profile_output.to_str().unwrap(), + ]) + .is_err() + ); + assert!(!wrong_profile_output.exists()); + Ok(()) + } + #[test] fn identity_enrollment_and_export_are_immutable_and_paths_cannot_alias() -> Result<()> { let temp = TempDir::new()?; @@ -1916,6 +2249,159 @@ mod tests { Ok(()) } + #[test] + fn lifecycle_brake_is_absence_allow_and_exact_to_runtime_target_and_expiry() -> Result<()> { + let temp = TempDir::new()?; + let missing = temp.path().join("missing-lifecycle.cc"); + invoke(&[ + "lifecycle-brake-status", + "--store", + missing.to_str().unwrap(), + "--runtime-id", + "idunn-yggdrasil", + "--target", + "ghostlight", + "--now-unix-millis", + "100", + ])?; + assert!(!missing.exists()); + + let store = temp.path().join("ghostlight-lifecycle.cc"); + invoke(&[ + "lifecycle-brake-engage", + "--store", + store.to_str().unwrap(), + "--runtime-id", + "idunn-yggdrasil", + "--target", + "ghostlight", + "--reason", + "operator maintenance", + "--updated-at-unix-millis", + "200", + ])?; + let engaged = read_lifecycle_brake(&store)?.context("lifecycle brake must exist")?; + assert_eq!(engaged.status, "engaged"); + assert_eq!(engaged.runtime_id, "idunn-yggdrasil"); + assert_eq!(engaged.target, "ghostlight"); + assert!(read_brake(&store).is_err()); + assert!( + invoke(&[ + "lifecycle-brake-status", + "--store", + store.to_str().unwrap(), + "--runtime-id", + "idunn-yggdrasil", + "--target", + "ghostlight", + "--now-unix-millis", + "201", + ]) + .is_err() + ); + + let engaged_bytes = std::fs::read(&store)?; + assert!( + invoke(&[ + "lifecycle-brake-release", + "--store", + store.to_str().unwrap(), + "--runtime-id", + "idunn-nightwing", + "--target", + "ghostlight", + "--reason", + "wrong runtime", + "--updated-at-unix-millis", + "300", + ]) + .is_err() + ); + assert!( + invoke(&[ + "lifecycle-brake-release", + "--store", + store.to_str().unwrap(), + "--runtime-id", + "idunn-yggdrasil", + "--target", + "odin", + "--reason", + "wrong target", + "--updated-at-unix-millis", + "300", + ]) + .is_err() + ); + assert_eq!(std::fs::read(&store)?, engaged_bytes); + + invoke(&[ + "lifecycle-brake-release", + "--store", + store.to_str().unwrap(), + "--runtime-id", + "idunn-yggdrasil", + "--target", + "ghostlight", + "--reason", + "bounded continuity window", + "--updated-at-unix-millis", + "300", + "--released-until-unix-millis", + "500", + ])?; + invoke(&[ + "lifecycle-brake-status", + "--store", + store.to_str().unwrap(), + "--runtime-id", + "idunn-yggdrasil", + "--target", + "ghostlight", + "--now-unix-millis", + "499", + ])?; + for (runtime, target, now) in [ + ("idunn-nightwing", "ghostlight", "499"), + ("idunn-yggdrasil", "odin", "499"), + ("idunn-yggdrasil", "ghostlight", "500"), + ] { + assert!( + invoke(&[ + "lifecycle-brake-status", + "--store", + store.to_str().unwrap(), + "--runtime-id", + runtime, + "--target", + target, + "--now-unix-millis", + now, + ]) + .is_err() + ); + } + let released_bytes = std::fs::read(&store)?; + assert!( + invoke(&[ + "lifecycle-brake-engage", + "--store", + store.to_str().unwrap(), + "--runtime-id", + "idunn-yggdrasil", + "--target", + "ghostlight", + "--reason", + "stale replay", + "--updated-at-unix-millis", + "300", + ]) + .is_err() + ); + assert_eq!(std::fs::read(&store)?, released_bytes); + Ok(()) + } + #[test] fn brake_compare_exchange_rejects_stale_transition_snapshot() -> Result<()> { let temp = TempDir::new()?; diff --git a/crates/odin-daemon/Cargo.toml b/crates/odin-daemon/Cargo.toml new file mode 100644 index 0000000..e50553d --- /dev/null +++ b/crates/odin-daemon/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "odin-daemon" +version = "0.1.0" +edition.workspace = true +license.workspace = true +publish.workspace = true + +[lib] +name = "odin_daemon" +path = "src/lib.rs" + +[dependencies] +anyhow = "1" +chrono = "0.4" +cultcache-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "cbde940df2f5ee0ccac50e8dbfe9f36ffbebb585", package = "cultcache-rs" } +cultmesh-rs = { path = "../../vendor/cultmesh-rs" } +cultnet-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "cbde940df2f5ee0ccac50e8dbfe9f36ffbebb585", package = "cultnet-rs" } +fs2 = "0.4" +rmp-serde = "1.3" +serde = "1" + +[dev-dependencies] +tempfile = "3" diff --git a/crates/odin-daemon/src/lib.rs b/crates/odin-daemon/src/lib.rs new file mode 100644 index 0000000..bfd38e6 --- /dev/null +++ b/crates/odin-daemon/src/lib.rs @@ -0,0 +1,1832 @@ +//! Deterministic Odin authority for runtime-topology correlation. +//! +//! This library owns no listener or lifecycle loop. Callers provide Idunn's +//! read-only projection, a trusted receive time with any service-owned +//! presence, and narrow clock/store/signer implementations. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, Result, bail, ensure}; +use chrono::{DateTime, SecondsFormat, Utc}; +use cultcache_rs::{ + CultCacheEnvelope, CultCacheExpectedEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, +}; +use cultnet_rs::{ + GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, GAMECULT_RUNTIME_PRESENCE_HEALTH_SIGNING_PURPOSE, + GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA, GameCultRuntimePresenceHealthRecord, + GameCultServiceTrustAnchorRecord, IDUNN_EXPECTED_INCARNATION_SCHEMA, + IDUNN_PROCESS_WRITE_LEASE_SCHEMA, IDUNN_RUNTIME_ACTIVATION_SCHEMA, IdunnExpectedDependency, + IdunnExpectedIncarnationRecord, IdunnProcessWriteLeaseRecord, IdunnRuntimeActivationPurpose, + IdunnRuntimeActivationRecord, IdunnServiceIdentity, ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA, + OdinRuntimeDependencyEvidence, OdinRuntimeTopologyCorrelationPurpose, + OdinRuntimeTopologyCorrelationRecord, OdinTopologyAuthenticationContext, + OdinTopologyDisagreement, OdinTopologyIdentity, RuntimePresenceAuthenticationContext, + ServiceIdentitySignature, ServiceIdentitySigner, ServiceIdentityTrustAnchor, + authenticate_odin_runtime_topology_correlation, authenticate_runtime_presence_claim, + correlate_runtime_presence_claim, verify_runtime_authority, verify_service_identity_signature, + verify_service_identity_signature_with_public_key, +}; + +const CAS_ATTEMPTS: usize = 8; + +#[derive(Clone, Copy, Debug)] +pub struct AuthenticationPolicy { + pub presence_maximum_age_millis: u64, + pub presence_maximum_future_skew_millis: u64, + pub correlation_maximum_age_millis: u64, + pub correlation_maximum_future_skew_millis: u64, +} + +impl Default for AuthenticationPolicy { + fn default() -> Self { + Self { + presence_maximum_age_millis: 30_000, + presence_maximum_future_skew_millis: 5_000, + correlation_maximum_age_millis: 30_000, + correlation_maximum_future_skew_millis: 5_000, + } + } +} + +pub trait Clock { + fn now_unix_millis(&self) -> Result; +} + +#[derive(Clone, Copy, Debug, Default)] +pub struct SystemClock; + +impl Clock for SystemClock { + fn now_unix_millis(&self) -> Result { + Ok(SystemTime::now() + .duration_since(UNIX_EPOCH) + .context("system clock predates the Unix epoch")? + .as_millis() + .try_into() + .context("system clock does not fit the topology timestamp")?) + } +} + +/// One exact Idunn-owned projection. The service anchor is provider lookup +/// material selected by Expected; it is not runtime observation. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct IdunnRuntimeProjection { + pub expected: IdunnExpectedIncarnationRecord, + pub provider_anchor: Option, + pub activation: Option, + pub current_lease: Option, +} + +impl IdunnRuntimeProjection { + pub fn validate(&self) -> Result<()> { + self.expected.validate()?; + if let Some(anchor) = &self.provider_anchor { + anchor.validate()?; + } + if let Some(activation) = &self.activation { + activation.validate()?; + } + if let Some(lease) = &self.current_lease { + lease.validate()?; + } + Ok(()) + } +} + +pub trait IdunnProjectionSource { + fn current_projection(&self, target: &str) -> Result>; +} + +/// Read-only adapter for the atomic projection file published by Idunn. +pub struct CultCacheIdunnProjectionSource { + path: PathBuf, +} + +impl CultCacheIdunnProjectionSource { + pub fn new(path: impl Into) -> Self { + Self { path: path.into() } + } + + pub fn path(&self) -> &Path { + &self.path + } +} + +impl IdunnProjectionSource for CultCacheIdunnProjectionSource { + fn current_projection(&self, target: &str) -> Result> { + if !self.path.is_file() { + return Ok(None); + } + let entries = + SingleFileMessagePackBackingStore::new(&self.path).pull_all_read_only_snapshot()?; + let expected_envelope = + unique_envelope(&entries, IdunnExpectedIncarnationRecord::TYPE, target)?; + let anchor_key = runtime_presence_anchor_id(target); + let anchor_envelope = unique_envelope( + &entries, + GameCultServiceTrustAnchorRecord::TYPE, + &anchor_key, + )?; + let activation_envelope = + unique_envelope(&entries, IdunnRuntimeActivationRecord::TYPE, target)?; + let lease_envelope = unique_envelope(&entries, IdunnProcessWriteLeaseRecord::TYPE, target)?; + + let Some(expected_envelope) = expected_envelope else { + ensure!( + anchor_envelope.is_none() + && activation_envelope.is_none() + && lease_envelope.is_none(), + "Idunn projection has runtime authority without Expected" + ); + return Ok(None); + }; + ensure_schema(expected_envelope, IDUNN_EXPECTED_INCARNATION_SCHEMA)?; + let expected = + IdunnExpectedIncarnationRecord::decode_canonical(&expected_envelope.payload)?; + ensure!( + expected.target == target, + "Expected projection key is substituted" + ); + + let provider_anchor = anchor_envelope + .map(|envelope| { + ensure_schema(envelope, GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA)?; + decode_canonical(&envelope.payload, "provider trust anchor") + }) + .transpose()?; + + let activation = activation_envelope + .map(|envelope| { + ensure_schema(envelope, IDUNN_RUNTIME_ACTIVATION_SCHEMA)?; + IdunnRuntimeActivationRecord::decode_canonical(&envelope.payload) + }) + .transpose()?; + + let current_lease = lease_envelope + .map(|envelope| { + ensure_schema(envelope, IDUNN_PROCESS_WRITE_LEASE_SCHEMA)?; + IdunnProcessWriteLeaseRecord::decode_canonical(&envelope.payload) + }) + .transpose()?; + let projection = IdunnRuntimeProjection { + expected, + provider_anchor, + activation, + current_lease, + }; + projection.validate()?; + Ok(Some(projection)) + } +} + +pub trait OdinCorrelationSigner { + fn identity_id(&self) -> &str; + fn public_key(&self) -> &[u8]; + fn sign_correlation(&self, unsigned_payload: &[u8]) -> Result>; +} + +impl OdinCorrelationSigner for ServiceIdentitySigner { + fn identity_id(&self) -> &str { + &self.entry().identity_id + } + + fn public_key(&self) -> &[u8] { + &self.entry().public_key + } + + fn sign_correlation(&self, unsigned_payload: &[u8]) -> Result> { + Ok(self + .sign::(unsigned_payload) + .signature) + } +} + +impl OdinCorrelationSigner for &T { + fn identity_id(&self) -> &str { + (*self).identity_id() + } + + fn public_key(&self) -> &[u8] { + (*self).public_key() + } + + fn sign_correlation(&self, unsigned_payload: &[u8]) -> Result> { + (*self).sign_correlation(unsigned_payload) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AdmittedPresence { + pub canonical_bytes: Vec, + pub trusted_received_at_unix_millis: u64, + pub stored_at: String, +} + +impl AdmittedPresence { + fn new(canonical_bytes: Vec, trusted_received_at_unix_millis: u64) -> Result { + Ok(Self { + canonical_bytes, + trusted_received_at_unix_millis, + stored_at: rfc3339_millis(trusted_received_at_unix_millis)?, + }) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StoredCorrelation { + pub canonical_bytes: Vec, + pub stored_at: String, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct OdinStoreQuery { + pub target: String, + pub provider_signer_identity_id: String, + pub dependency_targets: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct OdinStoreSnapshot { + pub query: OdinStoreQuery, + pub presence: Option, + pub correlation: Option, + pub dependency_correlations: BTreeMap>, +} + +pub trait OdinTopologyStore { + fn read(&self, query: OdinStoreQuery) -> Result; + + fn compare_exchange( + &self, + observed: &OdinStoreSnapshot, + replacement_presence: Option<&AdmittedPresence>, + replacement_correlation: &StoredCorrelation, + ) -> Result; + + fn withdraw_correlation(&self, target: &str) -> Result<()>; +} + +/// Odin's durable replay and correlation store. It persists existing CultNet +/// documents directly instead of inventing a parallel admission schema. +pub struct CultCacheOdinTopologyStore { + path: PathBuf, +} + +impl CultCacheOdinTopologyStore { + pub fn new(path: impl Into) -> Self { + Self { path: path.into() } + } + + pub fn path(&self) -> &Path { + &self.path + } +} + +impl OdinTopologyStore for CultCacheOdinTopologyStore { + fn read(&self, mut query: OdinStoreQuery) -> Result { + query.dependency_targets.sort(); + query.dependency_targets.dedup(); + let entries = if self.path.is_file() { + SingleFileMessagePackBackingStore::new(&self.path).pull_all_read_only_snapshot()? + } else { + Vec::new() + }; + let presence_key = presence_store_key(&query.target, &query.provider_signer_identity_id); + let presence = unique_envelope( + &entries, + GameCultRuntimePresenceHealthRecord::TYPE, + &presence_key, + )? + .map(decode_presence_envelope) + .transpose()?; + let correlation = unique_envelope( + &entries, + OdinRuntimeTopologyCorrelationRecord::TYPE, + &query.target, + )? + .map(decode_correlation_envelope) + .transpose()?; + let mut dependency_correlations = BTreeMap::new(); + for dependency_target in &query.dependency_targets { + let current = unique_envelope( + &entries, + OdinRuntimeTopologyCorrelationRecord::TYPE, + dependency_target, + )? + .map(decode_correlation_envelope) + .transpose()?; + dependency_correlations.insert(dependency_target.clone(), current); + } + Ok(OdinStoreSnapshot { + query, + presence, + correlation, + dependency_correlations, + }) + } + + fn compare_exchange( + &self, + observed: &OdinStoreSnapshot, + replacement_presence: Option<&AdmittedPresence>, + replacement_correlation: &StoredCorrelation, + ) -> Result { + let presence_key = presence_store_key( + &observed.query.target, + &observed.query.provider_signer_identity_id, + ); + let mut conditions = BTreeMap::new(); + insert_condition( + &mut conditions, + CultCacheExpectedEnvelope { + key: presence_key.clone(), + r#type: GameCultRuntimePresenceHealthRecord::TYPE.into(), + current: observed + .presence + .as_ref() + .map(|presence| presence_envelope(&presence_key, presence)), + }, + )?; + insert_condition( + &mut conditions, + CultCacheExpectedEnvelope { + key: observed.query.target.clone(), + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + current: observed + .correlation + .as_ref() + .map(|correlation| correlation_envelope(&observed.query.target, correlation)), + }, + )?; + for (target, correlation) in &observed.dependency_correlations { + insert_condition( + &mut conditions, + CultCacheExpectedEnvelope { + key: target.clone(), + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + current: correlation + .as_ref() + .map(|correlation| correlation_envelope(target, correlation)), + }, + )?; + } + let mut replacements = vec![correlation_envelope( + &observed.query.target, + replacement_correlation, + )]; + if let Some(presence) = replacement_presence { + replacements.push(presence_envelope(&presence_key, presence)); + } + if let Some(parent) = self.path.parent() { + std::fs::create_dir_all(parent)?; + } + SingleFileMessagePackBackingStore::new(&self.path) + .compare_exchange(&conditions.into_values().collect::>(), &replacements) + } + + fn withdraw_correlation(&self, target: &str) -> Result<()> { + for _ in 0..CAS_ATTEMPTS { + if !self.path.is_file() { + return Ok(()); + } + let entries = + SingleFileMessagePackBackingStore::new(&self.path).pull_all_read_only_snapshot()?; + let current = + unique_envelope(&entries, OdinRuntimeTopologyCorrelationRecord::TYPE, target)?; + let Some(current) = current else { + return Ok(()); + }; + let replacement = entries + .iter() + .filter(|entry| *entry != current) + .cloned() + .collect::>(); + if SingleFileMessagePackBackingStore::new(&self.path) + .compare_exchange_snapshot(&entries, &replacement)? + { + return Ok(()); + } + } + bail!("Odin topology store changed repeatedly while withdrawing correlation") + } +} + +pub struct OdinTopologyAuthority { + projections: P, + store: S, + signer: K, + clock: C, + idunn_anchor: ServiceIdentityTrustAnchor, + policy: AuthenticationPolicy, +} + +impl OdinTopologyAuthority +where + P: IdunnProjectionSource, + S: OdinTopologyStore, + K: OdinCorrelationSigner, + C: Clock, +{ + pub fn new( + projections: P, + store: S, + signer: K, + clock: C, + idunn_anchor: ServiceIdentityTrustAnchor, + policy: AuthenticationPolicy, + ) -> Self { + Self { + projections, + store, + signer, + clock, + idunn_anchor, + policy, + } + } + + /// Correlate current Expected with any previously admitted raw observation. + /// With no admitted observation this can only produce Expected, never Present. + pub fn refresh(&self, target: &str) -> Result>> { + let Some(projection) = self.projections.current_projection(target)? else { + self.store.withdraw_correlation(target)?; + return Ok(None); + }; + self.reconcile(projection, None).map(Some) + } + + /// Authenticate and monotonically admit one exact provider-owned presence. + pub fn admit_presence( + &self, + target: &str, + canonical_presence: &[u8], + trusted_received_at_unix_millis: u64, + ) -> Result> { + let projection = self + .projections + .current_projection(target)? + .context("runtime presence has no current Expected projection")?; + self.reconcile( + projection, + Some((canonical_presence, trusted_received_at_unix_millis)), + ) + } + + /// Return the exact signed bytes admitted by Odin for transport. + pub fn current_signed_correlation(&self, target: &str) -> Result>> { + self.refresh(target) + } + + fn reconcile( + &self, + projection: IdunnRuntimeProjection, + incoming: Option<(&[u8], u64)>, + ) -> Result> { + projection.validate()?; + let now = self.clock.now_unix_millis()?; + let (authority, projection_disagreements) = + classify_runtime_authority(&projection, &self.idunn_anchor, now)?; + let (lease_sha256, lease_disagreement) = classify_current_lease(&projection)?; + let query = OdinStoreQuery { + target: projection.expected.target.clone(), + provider_signer_identity_id: projection.expected.expected_signer_identity_id.clone(), + dependency_targets: managed_dependency_targets(&projection.expected.dependencies), + }; + + for _ in 0..CAS_ATTEMPTS { + let observed = self.store.read(query.clone())?; + let admitted = self.select_presence(&observed, incoming, authority.as_ref())?; + let dependencies = + self.dependency_evidence(&projection.expected.dependencies, &observed, now)?; + let (presence, mut disagreements, replacement_presence) = match admitted { + SelectedPresence::None => (None, projection_disagreements.clone(), None), + SelectedPresence::Stale(stored) => { + let record = decode_presence(&stored.canonical_bytes)?; + let mut disagreements = projection_disagreements.clone(); + disagreements.push(OdinTopologyDisagreement { + code: "stored-presence-not-current".into(), + expected: projection + .activation + .as_ref() + .map(IdunnRuntimeActivationRecord::canonical_sha256) + .transpose()?, + observed: Some(record.activation_witness_sha256), + }); + (None, disagreements, None) + } + SelectedPresence::Current { + correlation, + replacement, + } => { + let mut disagreements = projection_disagreements.clone(); + disagreements.extend_from_slice(correlation.disagreements()); + (Some(correlation), disagreements, replacement) + } + }; + if let Some(disagreement) = &lease_disagreement { + disagreements.push(disagreement.clone()); + } + if presence.is_none() { + disagreements.push(OdinTopologyDisagreement { + code: "signed-presence-missing".into(), + expected: Some("dual-authenticated-runtime-presence".into()), + observed: None, + }); + disagreements.extend(projection.expected.capabilities.iter().enumerate().map( + |(index, capability)| OdinTopologyDisagreement { + code: format!("expected-capability-{index:03}-missing"), + expected: Some(format!( + "{}/{}/{} capacity>={}", + capability.capability, + capability.schema, + capability.compatibility, + capability.minimum_capacity + )), + observed: None, + }, + )); + } + + let observed_write_lease = if let Some(presence) = &presence { + correlate_write_lease( + &projection.expected, + lease_sha256.as_deref(), + presence.claim().record().write_lease_sha256.as_deref(), + &presence.claim().record().state, + &mut disagreements, + ) + } else { + None + }; + disagreements.sort_by(|left, right| left.code.cmp(&right.code)); + + let prior = observed + .correlation + .as_ref() + .map(|stored| decode_correlation(&stored.canonical_bytes)) + .transpose()?; + let publisher_sequence = prior.as_ref().map_or(Ok(1), |record| { + record + .publisher_sequence + .checked_add(1) + .context("Odin topology publisher sequence exhausted") + })?; + let presence_record = presence.as_ref().map(|value| value.claim().record()); + let ready = presence_record.is_some_and(|record| record.state == "active") + && disagreements.is_empty() + && dependencies + .iter() + .all(|dependency| dependency.kind == "optional" || dependency.ready); + let mut correlation = OdinRuntimeTopologyCorrelationRecord { + schema_version: ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into(), + target: projection.expected.target.clone(), + expected_projection_sha256: projection.expected.canonical_sha256()?, + expected: true, + current_activation_sha256: projection + .activation + .as_ref() + .map(IdunnRuntimeActivationRecord::canonical_sha256) + .transpose()?, + signed_presence_sha256: presence + .as_ref() + .map(|value| value.claim().signed_presence_sha256().into()), + observed_presence_state: presence_record.map(|record| record.state.clone()), + observed_presence_publisher_sequence: presence_record + .map(|record| record.publisher_sequence), + observed_write_lease_sha256: observed_write_lease, + observed_capabilities: presence_record + .map_or_else(Vec::new, |record| record.capabilities.clone()), + runtime_id: projection.expected.runtime_id.clone(), + runtime_instance_id: projection + .activation + .as_ref() + .map(|activation| activation.runtime_instance_id.clone()), + present: presence.is_some(), + ready, + dependencies, + disagreements, + signer_identity_id: self.signer.identity_id().into(), + publisher_sequence, + observed_at_unix_millis: now, + signature_algorithm: "ed25519".into(), + signature: Vec::new(), + }; + if let Some(prior) = &prior + && same_correlation_facts(prior, &correlation) + { + return Ok(observed + .correlation + .context("decoded prior correlation disappeared")? + .canonical_bytes); + } + correlation.signature = self + .signer + .sign_correlation(&correlation.unsigned_signature_payload()?)?; + let canonical = correlation.canonical_bytes()?; + correlation.validate_against_expected(&projection.expected, lease_sha256.as_deref())?; + let (signed, unsigned) = + OdinRuntimeTopologyCorrelationRecord::decode_canonical_signed_payload(&canonical)?; + verify_service_identity_signature_with_public_key::< + OdinTopologyIdentity, + OdinRuntimeTopologyCorrelationPurpose, + >( + self.signer.public_key(), + &unsigned, + &ServiceIdentitySignature { + identity_id: signed.signer_identity_id, + signature: signed.signature, + }, + )?; + if let Some(authority) = &authority { + authenticate_odin_runtime_topology_correlation( + &canonical, + authority, + lease_sha256.as_deref(), + self.signer.public_key(), + OdinTopologyAuthenticationContext { + trusted_received_at_unix_millis: now, + maximum_age_millis: self.policy.correlation_maximum_age_millis, + maximum_future_skew_millis: self + .policy + .correlation_maximum_future_skew_millis, + }, + )?; + } + let stored_correlation = StoredCorrelation { + canonical_bytes: canonical.clone(), + stored_at: rfc3339_millis(now)?, + }; + if self.store.compare_exchange( + &observed, + replacement_presence.as_ref(), + &stored_correlation, + )? { + return Ok(canonical); + } + } + bail!("Odin topology store changed repeatedly during correlation") + } + + fn select_presence( + &self, + observed: &OdinStoreSnapshot, + incoming: Option<(&[u8], u64)>, + authority: Option<&cultnet_rs::VerifiedRuntimeAuthority>, + ) -> Result { + let stored_record = observed + .presence + .as_ref() + .map(|stored| decode_presence(&stored.canonical_bytes)) + .transpose()?; + let Some(authority) = authority else { + ensure!( + incoming.is_none(), + "runtime presence has no exact current activation and provider anchor" + ); + return Ok(observed + .presence + .as_ref() + .map_or(SelectedPresence::None, |stored| { + SelectedPresence::Stale(stored.clone()) + })); + }; + let selected = if let Some((bytes, received_at)) = incoming { + if let Some(stored_presence) = observed + .presence + .as_ref() + .filter(|stored| stored.canonical_bytes == bytes) + { + let stored_claim = authenticate_runtime_presence_claim( + &stored_presence.canonical_bytes, + authority, + self.presence_context(stored_presence.trusted_received_at_unix_millis), + )?; + return Ok(SelectedPresence::Current { + correlation: correlate_runtime_presence_claim(stored_claim, authority)?, + replacement: None, + }); + } + let claim = authenticate_runtime_presence_claim( + bytes, + authority, + self.presence_context(received_at), + )?; + if let Some(stored) = &stored_record { + if claim.record().publisher_sequence < stored.publisher_sequence { + bail!("runtime presence publisher sequence was reordered"); + } + if claim.record().publisher_sequence == stored.publisher_sequence { + bail!("runtime presence publisher sequence was reused with different bytes"); + } + } + let replacement = AdmittedPresence::new(bytes.to_vec(), received_at)?; + Some((claim, Some(replacement))) + } else if let Some(stored) = &observed.presence { + match authenticate_runtime_presence_claim( + &stored.canonical_bytes, + authority, + self.presence_context(stored.trusted_received_at_unix_millis), + ) { + Ok(claim) => Some((claim, None)), + Err(error) => { + let prior_matches_current = + observed.correlation.as_ref().is_some_and(|value| { + decode_correlation(&value.canonical_bytes).is_ok_and(|record| { + record.expected_projection_sha256 == authority.expected_sha256() + && record.current_activation_sha256.as_deref() + == Some(authority.activation_sha256()) + }) + }); + if prior_matches_current { + return Err(error).context("stored current presence failed authentication"); + } + return Ok(SelectedPresence::Stale(stored.clone())); + } + } + } else { + None + }; + match selected { + Some((claim, replacement)) => Ok(SelectedPresence::Current { + correlation: correlate_runtime_presence_claim(claim, authority)?, + replacement, + }), + None => Ok(SelectedPresence::None), + } + } + + fn dependency_evidence( + &self, + requirements: &[IdunnExpectedDependency], + observed: &OdinStoreSnapshot, + now: u64, + ) -> Result> { + requirements + .iter() + .map(|requirement| { + let mut evidence = OdinRuntimeDependencyEvidence { + kind: requirement.kind.clone(), + capability: requirement.capability.clone(), + schema: requirement.schema.clone(), + compatibility: requirement.compatibility.clone(), + provider_id: requirement.provider_id.clone(), + provider_authority: requirement.provider_authority.clone(), + provider_expected_projection_sha256: requirement + .provider_expected_projection_sha256 + .clone(), + provider_endpoint: requirement.provider_endpoint.clone(), + observed_capacity: None, + provider_evidence_sha256: None, + ready: false, + }; + let Some(provider_id) = requirement.provider_id.as_deref() else { + return Ok(evidence); + }; + if requirement.provider_authority.as_deref() != Some("managed-incarnation") { + return Ok(evidence); + } + let Some(provider_projection) = self.projections.current_projection(provider_id)? + else { + return Ok(evidence); + }; + provider_projection.validate()?; + if provider_projection.expected.canonical_sha256()? + != requirement + .provider_expected_projection_sha256 + .as_deref() + .unwrap_or_default() + { + return Ok(evidence); + } + if let Some(endpoint) = requirement.provider_endpoint.as_deref() + && provider_projection + .expected + .route + .as_ref() + .map(|route| route.stable_endpoint.as_str()) + != Some(endpoint) + { + return Ok(evidence); + } + let (provider_authority, authority_disagreements) = + classify_runtime_authority(&provider_projection, &self.idunn_anchor, now)?; + let Some(provider_authority) = provider_authority else { + return Ok(evidence); + }; + if !authority_disagreements.is_empty() { + return Ok(evidence); + } + let Some(stored) = observed + .dependency_correlations + .get(provider_id) + .and_then(Option::as_ref) + else { + return Ok(evidence); + }; + let (current_lease_sha256, _) = classify_current_lease(&provider_projection)?; + let Ok(authenticated) = authenticate_odin_runtime_topology_correlation( + &stored.canonical_bytes, + &provider_authority, + current_lease_sha256.as_deref(), + self.signer.public_key(), + OdinTopologyAuthenticationContext { + trusted_received_at_unix_millis: now, + maximum_age_millis: self.policy.correlation_maximum_age_millis, + maximum_future_skew_millis: self + .policy + .correlation_maximum_future_skew_millis, + }, + ) else { + return Ok(evidence); + }; + let provider = authenticated.record(); + let capacity = provider + .observed_capabilities + .iter() + .find(|capability| { + capability.capability == requirement.capability + && capability.schema == requirement.schema + && capability.compatibility == requirement.compatibility + }) + .map(|capability| capability.capacity); + evidence.observed_capacity = capacity; + evidence.provider_evidence_sha256 = Some(provider.canonical_sha256()?); + evidence.ready = provider.ready + && capacity.is_some_and(|capacity| capacity >= requirement.minimum_capacity); + Ok(evidence) + }) + .collect() + } + + fn presence_context( + &self, + trusted_received_at_unix_millis: u64, + ) -> RuntimePresenceAuthenticationContext { + RuntimePresenceAuthenticationContext { + trusted_received_at_unix_millis, + maximum_age_millis: self.policy.presence_maximum_age_millis, + maximum_future_skew_millis: self.policy.presence_maximum_future_skew_millis, + } + } +} + +enum SelectedPresence { + None, + Stale(AdmittedPresence), + Current { + correlation: cultnet_rs::RuntimePresenceCorrelation, + replacement: Option, + }, +} + +fn correlate_write_lease( + expected: &IdunnExpectedIncarnationRecord, + current: Option<&str>, + observed: Option<&str>, + presence_state: &str, + disagreements: &mut Vec, +) -> Option { + if presence_state == "warming" && observed.is_none() { + return None; + } + if expected.write_lease_required && current.is_some() && current == observed { + return observed.map(str::to_string); + } + if expected.write_lease_required { + disagreements.push(OdinTopologyDisagreement { + code: "write-lease".into(), + expected: Some( + current + .unwrap_or("current-idunn-process-write-lease") + .into(), + ), + observed: observed.map(str::to_string), + }); + } else if observed.is_some() { + disagreements.push(OdinTopologyDisagreement { + code: "unexpected-write-lease".into(), + expected: None, + observed: observed.map(str::to_string), + }); + } + None +} + +fn same_correlation_facts( + prior: &OdinRuntimeTopologyCorrelationRecord, + next: &OdinRuntimeTopologyCorrelationRecord, +) -> bool { + let mut normalized = next.clone(); + normalized.publisher_sequence = prior.publisher_sequence; + normalized.observed_at_unix_millis = prior.observed_at_unix_millis; + normalized.signature = prior.signature.clone(); + &normalized == prior +} + +fn managed_dependency_targets(requirements: &[IdunnExpectedDependency]) -> Vec { + requirements + .iter() + .filter(|dependency| { + dependency.provider_authority.as_deref() == Some("managed-incarnation") + }) + .filter_map(|dependency| dependency.provider_id.clone()) + .collect() +} + +fn classify_runtime_authority( + projection: &IdunnRuntimeProjection, + idunn_anchor: &ServiceIdentityTrustAnchor, + now: u64, +) -> Result<( + Option, + Vec, +)> { + let expected = &projection.expected; + let expected_sha256 = expected.canonical_sha256()?; + let mut disagreements = Vec::new(); + match &projection.provider_anchor { + None => disagreements.push(OdinTopologyDisagreement { + code: "provider-trust-anchor-missing".into(), + expected: Some(expected.expected_signer_identity_id.clone()), + observed: None, + }), + Some(anchor) => { + push_disagreement( + &mut disagreements, + "provider-trust-anchor-id", + Some(runtime_presence_anchor_id(&expected.target)), + Some(anchor.trust_anchor_id.clone()), + ); + push_disagreement( + &mut disagreements, + "provider-service-id", + Some(expected.target.clone()), + Some(anchor.service_id.clone()), + ); + push_disagreement( + &mut disagreements, + "provider-runtime-id", + Some(expected.runtime_id.clone()), + Some(anchor.runtime_id.clone()), + ); + push_disagreement( + &mut disagreements, + "provider-signer-identity", + Some(expected.expected_signer_identity_id.clone()), + Some(anchor.signer_identity_id.clone()), + ); + push_disagreement( + &mut disagreements, + "provider-signing-purpose", + Some(GAMECULT_RUNTIME_PRESENCE_HEALTH_SIGNING_PURPOSE.into()), + Some(anchor.signing_purpose.clone()), + ); + push_disagreement( + &mut disagreements, + "provider-signed-schema", + Some(GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()), + Some(anchor.signed_schema.clone()), + ); + if anchor.bound_at_unix_millis > now + || anchor + .expires_at_unix_millis + .is_some_and(|expires_at| now >= expires_at) + { + disagreements.push(OdinTopologyDisagreement { + code: "provider-trust-anchor-time".into(), + expected: Some(format!("current-at:{now}")), + observed: Some(format!( + "bound:{};expires:{}", + anchor.bound_at_unix_millis, + anchor + .expires_at_unix_millis + .map_or_else(|| "none".into(), |value| value.to_string()) + )), + }); + } + } + } + match &projection.activation { + None => disagreements.push(OdinTopologyDisagreement { + code: "current-activation-missing".into(), + expected: Some(expected_sha256.clone()), + observed: None, + }), + Some(activation) => { + if activation.idunn_signer_identity_id != idunn_anchor.identity_id { + disagreements.push(OdinTopologyDisagreement { + code: "activation-idunn-signer".into(), + expected: Some(idunn_anchor.identity_id.clone()), + observed: Some(activation.idunn_signer_identity_id.clone()), + }); + } else { + verify_service_identity_signature::< + IdunnServiceIdentity, + IdunnRuntimeActivationPurpose, + >( + idunn_anchor, + &activation.unsigned_signature_payload()?, + &ServiceIdentitySignature { + identity_id: activation.idunn_signer_identity_id.clone(), + signature: activation.signature.clone(), + }, + ) + .context("verifying projected Idunn runtime activation")?; + } + push_disagreement( + &mut disagreements, + "activation-expected-projection", + Some(expected_sha256.clone()), + Some(activation.expected_projection_sha256.clone()), + ); + push_disagreement( + &mut disagreements, + "activation-runtime-id", + Some(expected.runtime_id.clone()), + Some(activation.runtime_id.clone()), + ); + } + } + disagreements.sort_by(|left, right| left.code.cmp(&right.code)); + let authority = if disagreements.is_empty() { + let anchor = projection.provider_anchor.as_ref().unwrap(); + let activation = projection.activation.as_ref().unwrap(); + Some(verify_runtime_authority( + expected, + activation, + idunn_anchor, + &anchor.signer_public_key, + )?) + } else { + None + }; + Ok((authority, disagreements)) +} + +fn classify_current_lease( + projection: &IdunnRuntimeProjection, +) -> Result<(Option, Option)> { + let Some(lease) = &projection.current_lease else { + return Ok((None, None)); + }; + let lease_sha256 = lease.canonical_sha256()?; + let activation = projection.activation.as_ref(); + let binds = activation + .map(|activation| lease_binds(&projection.expected, activation, lease)) + .transpose()? + .unwrap_or(false); + if projection.expected.write_lease_required && binds { + return Ok((Some(lease_sha256), None)); + } + let expected_activation = activation + .map(IdunnRuntimeActivationRecord::canonical_sha256) + .transpose()? + .unwrap_or_else(|| "none".into()); + Ok(( + None, + Some(OdinTopologyDisagreement { + code: "projected-write-lease".into(), + expected: Some(format!( + "expected:{};activation:{expected_activation}", + projection.expected.canonical_sha256()? + )), + observed: Some(lease_sha256), + }), + )) +} + +fn lease_binds( + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + lease: &IdunnProcessWriteLeaseRecord, +) -> Result { + Ok(lease.target == expected.target + && lease.expected_projection_sha256 == expected.canonical_sha256()? + && lease.plan_id == expected.plan_id + && lease.incarnation_id == expected.incarnation_id + && lease.sealed_release_id == expected.sealed_release_id + && lease.activation_witness_sha256 == activation.canonical_sha256()? + && Some(lease.state_schema_generation.as_str()) + == expected.state_schema_generation.as_deref() + && Some(lease.state_contract_sha256.as_str()) == expected.state_contract_sha256.as_deref() + && lease.runtime_id == expected.runtime_id + && lease.runtime_instance_id == activation.runtime_instance_id) +} + +fn push_disagreement( + disagreements: &mut Vec, + code: &str, + expected: Option, + observed: Option, +) { + if expected != observed { + disagreements.push(OdinTopologyDisagreement { + code: code.into(), + expected, + observed, + }); + } +} + +fn runtime_presence_anchor_id(target: &str) -> String { + format!("root/{target}/runtime-presence") +} + +fn presence_store_key(target: &str, signer_identity_id: &str) -> String { + format!("{target}/{signer_identity_id}") +} + +fn unique_envelope<'a>( + entries: &'a [CultCacheEnvelope], + record_type: &str, + key: &str, +) -> Result> { + let mut matches = entries + .iter() + .filter(|entry| entry.r#type == record_type && entry.key == key); + let current = matches.next(); + ensure!(matches.next().is_none(), "CultCache identity is ambiguous"); + Ok(current) +} + +fn ensure_schema(envelope: &CultCacheEnvelope, schema: &str) -> Result<()> { + ensure!( + envelope.schema_id.as_deref() == Some(schema), + "CultCache envelope schema is substituted" + ); + Ok(()) +} + +fn decode_canonical(bytes: &[u8], label: &str) -> Result +where + T: serde::de::DeserializeOwned + serde::Serialize, +{ + let value = rmp_serde::from_slice(bytes).with_context(|| format!("decoding {label}"))?; + ensure!( + rmp_serde::to_vec(&value)? == bytes, + "{label} is not canonical positional MessagePack" + ); + Ok(value) +} + +fn decode_presence(bytes: &[u8]) -> Result { + let record: GameCultRuntimePresenceHealthRecord = decode_canonical(bytes, "runtime presence")?; + record.validate()?; + Ok(record) +} + +fn decode_correlation(bytes: &[u8]) -> Result { + Ok(OdinRuntimeTopologyCorrelationRecord::decode_canonical_signed_payload(bytes)?.0) +} + +fn decode_presence_envelope(envelope: &CultCacheEnvelope) -> Result { + ensure_schema(envelope, GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA)?; + decode_presence(&envelope.payload)?; + Ok(AdmittedPresence { + canonical_bytes: envelope.payload.clone(), + trusted_received_at_unix_millis: parse_rfc3339_millis(&envelope.stored_at)?, + stored_at: envelope.stored_at.clone(), + }) +} + +fn decode_correlation_envelope(envelope: &CultCacheEnvelope) -> Result { + ensure_schema(envelope, ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA)?; + let record = decode_correlation(&envelope.payload)?; + ensure!( + record.target == envelope.key, + "stored correlation target is substituted" + ); + ensure!( + parse_rfc3339_millis(&envelope.stored_at)? == record.observed_at_unix_millis, + "stored correlation receipt time is substituted" + ); + Ok(StoredCorrelation { + canonical_bytes: envelope.payload.clone(), + stored_at: envelope.stored_at.clone(), + }) +} + +fn presence_envelope(key: &str, presence: &AdmittedPresence) -> CultCacheEnvelope { + CultCacheEnvelope { + key: key.into(), + r#type: GameCultRuntimePresenceHealthRecord::TYPE.into(), + payload: presence.canonical_bytes.clone(), + stored_at: presence.stored_at.clone(), + schema_id: Some(GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()), + } +} + +fn correlation_envelope(target: &str, correlation: &StoredCorrelation) -> CultCacheEnvelope { + CultCacheEnvelope { + key: target.into(), + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + payload: correlation.canonical_bytes.clone(), + stored_at: correlation.stored_at.clone(), + schema_id: Some(ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into()), + } +} + +fn insert_condition( + conditions: &mut BTreeMap<(String, String), CultCacheExpectedEnvelope>, + condition: CultCacheExpectedEnvelope, +) -> Result<()> { + let identity = (condition.r#type.clone(), condition.key.clone()); + if let Some(current) = conditions.get(&identity) { + ensure!( + current.current == condition.current, + "store read set is incoherent" + ); + } else { + conditions.insert(identity, condition); + } + Ok(()) +} + +fn rfc3339_millis(value: u64) -> Result { + let value: i64 = value + .try_into() + .context("timestamp exceeds RFC3339 range")?; + Ok(DateTime::::from_timestamp_millis(value) + .context("timestamp exceeds RFC3339 range")? + .to_rfc3339_opts(SecondsFormat::Millis, true)) +} + +fn parse_rfc3339_millis(value: &str) -> Result { + DateTime::parse_from_rfc3339(value)? + .timestamp_millis() + .try_into() + .context("CultCache timestamp predates Unix epoch") +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use cultnet_rs::{ + GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA, GameCultProviderHealthIdentity, + GameCultRuntimeCapability, GameCultRuntimePresenceHealthPurpose, IdunnExpectedCapability, + IdunnExpectedRoute, IdunnRuntimeActivationLaunch, IdunnRuntimeActivationSigner, + IdunnServiceIdentity, enroll_service_identity_at, + }; + use tempfile::TempDir; + + use super::*; + + const NOW: u64 = 1_000_000; + + #[derive(Clone, Copy)] + struct FixedClock(u64); + + impl Clock for FixedClock { + fn now_unix_millis(&self) -> Result { + Ok(self.0) + } + } + + struct TestService { + projection: IdunnRuntimeProjection, + provider_signer: ServiceIdentitySigner, + activation_signer: IdunnRuntimeActivationSigner, + } + + impl TestService { + fn signed_presence(&self, sequence: u64, edit: F) -> Result> + where + F: FnOnce(&mut GameCultRuntimePresenceHealthRecord), + { + let expected = &self.projection.expected; + let activation = self.projection.activation.as_ref().unwrap(); + let mut record = GameCultRuntimePresenceHealthRecord { + schema_version: GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + target: expected.target.clone(), + expected_projection_sha256: expected.canonical_sha256()?, + plan_id: expected.plan_id.clone(), + incarnation_id: expected.incarnation_id.clone(), + sealed_release_id: expected.sealed_release_id.clone(), + activation_witness_sha256: activation.canonical_sha256()?, + state_schema_generation: expected.state_schema_generation.clone(), + state_contract_sha256: expected.state_contract_sha256.clone(), + runtime_id: expected.runtime_id.clone(), + runtime_instance_id: activation.runtime_instance_id.clone(), + bound_endpoint: expected + .route + .as_ref() + .map(|route| route.candidate_endpoint.clone()), + capabilities: expected + .capabilities + .iter() + .map(|capability| GameCultRuntimeCapability { + capability: capability.capability.clone(), + schema: capability.schema.clone(), + compatibility: capability.compatibility.clone(), + capacity: capability.minimum_capacity, + }) + .collect(), + health_contract: expected.health_contract.clone(), + state: "active".into(), + detail: "ready".into(), + write_lease_sha256: self + .projection + .current_lease + .as_ref() + .map(IdunnProcessWriteLeaseRecord::canonical_sha256) + .transpose()?, + signer_identity_id: self.provider_signer.entry().identity_id.clone(), + publisher_sequence: sequence, + observed_at_unix_millis: NOW - 10, + signature_algorithm: "ed25519".into(), + signature: Vec::new(), + activation_signer_identity_id: activation.activation_signer_identity_id.clone(), + activation_signature: Vec::new(), + }; + edit(&mut record); + let proof_payload = record.canonical_proof_payload()?; + record.signature = self + .provider_signer + .sign::(&proof_payload) + .signature; + record.activation_signature = self.activation_signer.sign_presence_proof(&record)?; + record.validate()?; + Ok(rmp_serde::to_vec(&record)?) + } + } + + struct TestWorld { + _temp: TempDir, + projection_path: PathBuf, + topology_path: PathBuf, + identity_root: PathBuf, + idunn_signer: ServiceIdentitySigner, + idunn_anchor: ServiceIdentityTrustAnchor, + odin_signer: ServiceIdentitySigner, + } + + impl TestWorld { + fn new() -> Result { + let temp = tempfile::tempdir()?; + let identity_root = temp.path().join("identities"); + let idunn_signer = enroll_service_identity_at::( + &identity_root.join("idunn.cc"), + )?; + let idunn_anchor = idunn_signer.trust_anchor()?; + let odin_signer = + enroll_service_identity_at::(&identity_root.join("odin.cc"))?; + Ok(Self { + projection_path: temp.path().join("idunn-projection.cc"), + topology_path: temp.path().join("odin-topology.cc"), + identity_root, + idunn_signer, + idunn_anchor, + odin_signer, + _temp: temp, + }) + } + + fn service( + &self, + target: &str, + dependencies: Vec, + stateful: bool, + ) -> Result { + let provider_signer = enroll_service_identity_at::( + &self.identity_root.join(format!("{target}-provider.cc")), + )?; + let expected = IdunnExpectedIncarnationRecord { + schema_version: IDUNN_EXPECTED_INCARNATION_SCHEMA.into(), + target: target.into(), + plan_id: digest('1'), + incarnation_id: format!("{target}/generation-1"), + sealed_release_id: digest('2'), + source_repository: format!("github.com/GameCult/{target}"), + source_revision: "3".repeat(40), + recipe_sha256: digest('4'), + runtime_id: format!("{target}-runtime"), + expected_signer_identity_id: provider_signer.entry().identity_id.clone(), + health_contract: format!("{target}.runtime-health.v1"), + artifact_sha256: digest('5'), + state_schema_generation: stateful.then(|| "state-v1".into()), + state_contract_sha256: stateful.then(|| digest('6')), + write_lease_required: stateful, + route: Some(IdunnExpectedRoute { + route_id: format!("{target}-route"), + transport: "tcp".into(), + stable_endpoint: format!("tcp://{target}.internal:1000"), + candidate_endpoint: format!("tcp://127.0.0.1:{}", target.len() + 10_000), + }), + capabilities: vec![IdunnExpectedCapability { + capability: "data".into(), + schema: "data.v1".into(), + compatibility: "v1".into(), + minimum_capacity: 1, + }], + dependencies, + }; + expected.validate()?; + let launch = IdunnRuntimeActivationLaunch::issue( + &expected, + digest('7'), + NOW - 20, + &self.idunn_signer, + )?; + let activation = launch.activation().clone(); + let mut credential = Vec::new(); + let emitted = launch.write_credential(&mut credential)?; + ensure!( + emitted == activation, + "activation launch changed while emitted" + ); + let activation_signer = + IdunnRuntimeActivationSigner::from_credential_reader(credential.as_slice())?; + let provider_anchor = GameCultServiceTrustAnchorRecord { + schema_version: GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA.into(), + trust_anchor_id: runtime_presence_anchor_id(target), + service_id: target.into(), + runtime_id: expected.runtime_id.clone(), + signer_identity_id: provider_signer.entry().identity_id.clone(), + signer_public_key: provider_signer.entry().public_key.clone(), + signature_algorithm: "ed25519".into(), + signing_purpose: GAMECULT_RUNTIME_PRESENCE_HEALTH_SIGNING_PURPOSE.into(), + signed_schema: GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + binding_authority: "root".into(), + bound_at_unix_millis: NOW - 100, + expires_at_unix_millis: None, + private_state_exposed: false, + }; + let current_lease = if stateful { + Some(IdunnProcessWriteLeaseRecord { + schema_version: IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into(), + target: target.into(), + expected_projection_sha256: expected.canonical_sha256()?, + plan_id: expected.plan_id.clone(), + incarnation_id: expected.incarnation_id.clone(), + sealed_release_id: expected.sealed_release_id.clone(), + activation_witness_sha256: activation.canonical_sha256()?, + state_schema_generation: expected.state_schema_generation.clone().unwrap(), + state_contract_sha256: expected.state_contract_sha256.clone().unwrap(), + runtime_id: expected.runtime_id.clone(), + runtime_instance_id: activation.runtime_instance_id.clone(), + warming_presence_sha256: digest('8'), + lease_epoch: 1, + issued_at_unix_millis: NOW - 5, + }) + } else { + None + }; + let projection = IdunnRuntimeProjection { + expected, + provider_anchor: Some(provider_anchor), + activation: Some(activation), + current_lease, + }; + projection.validate()?; + self.publish_projection(&projection)?; + Ok(TestService { + projection, + provider_signer, + activation_signer, + }) + } + + fn publish_projection(&self, projection: &IdunnRuntimeProjection) -> Result<()> { + let store = SingleFileMessagePackBackingStore::new(&self.projection_path); + let current = if self.projection_path.is_file() { + store.pull_all_read_only_snapshot()? + } else { + Vec::new() + }; + let anchor_key = runtime_presence_anchor_id(&projection.expected.target); + let mut next: Vec<_> = current + .iter() + .filter(|envelope| { + !((envelope.r#type == IdunnExpectedIncarnationRecord::TYPE + || envelope.r#type == IdunnRuntimeActivationRecord::TYPE + || envelope.r#type == IdunnProcessWriteLeaseRecord::TYPE) + && envelope.key == projection.expected.target) + && !(envelope.r#type == GameCultServiceTrustAnchorRecord::TYPE + && envelope.key == anchor_key) + }) + .cloned() + .collect(); + next.push(CultCacheEnvelope { + key: projection.expected.target.clone(), + r#type: IdunnExpectedIncarnationRecord::TYPE.into(), + payload: projection.expected.canonical_bytes()?, + stored_at: rfc3339_millis(NOW - 30)?, + schema_id: Some(IDUNN_EXPECTED_INCARNATION_SCHEMA.into()), + }); + if let Some(anchor) = &projection.provider_anchor { + next.push(CultCacheEnvelope { + key: anchor_key, + r#type: GameCultServiceTrustAnchorRecord::TYPE.into(), + payload: rmp_serde::to_vec(anchor)?, + stored_at: rfc3339_millis(anchor.bound_at_unix_millis)?, + schema_id: Some(GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA.into()), + }); + } + if let Some(activation) = &projection.activation { + next.push(CultCacheEnvelope { + key: projection.expected.target.clone(), + r#type: IdunnRuntimeActivationRecord::TYPE.into(), + payload: activation.canonical_bytes()?, + stored_at: rfc3339_millis(activation.issued_at_unix_millis)?, + schema_id: Some(IDUNN_RUNTIME_ACTIVATION_SCHEMA.into()), + }); + } + if let Some(lease) = &projection.current_lease { + next.push(CultCacheEnvelope { + key: projection.expected.target.clone(), + r#type: IdunnProcessWriteLeaseRecord::TYPE.into(), + payload: lease.canonical_bytes()?, + stored_at: rfc3339_millis(lease.issued_at_unix_millis)?, + schema_id: Some(IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into()), + }); + } + ensure!( + store.compare_exchange_snapshot(¤t, &next)?, + "test projection CAS failed" + ); + Ok(()) + } + + fn engine( + &self, + now: u64, + ) -> OdinTopologyAuthority< + CultCacheIdunnProjectionSource, + CultCacheOdinTopologyStore, + &ServiceIdentitySigner, + FixedClock, + > { + OdinTopologyAuthority::new( + CultCacheIdunnProjectionSource::new(&self.projection_path), + CultCacheOdinTopologyStore::new(&self.topology_path), + &self.odin_signer, + FixedClock(now), + self.idunn_anchor.clone(), + AuthenticationPolicy::default(), + ) + } + + fn replace_provider_generation(&self, service: &mut TestService) -> Result<()> { + service.projection.expected.plan_id = digest('9'); + service.projection.expected.incarnation_id = + format!("{}/generation-2", service.projection.expected.target); + service.projection.expected.validate()?; + let launch = IdunnRuntimeActivationLaunch::issue( + &service.projection.expected, + digest('a'), + NOW + 10, + &self.idunn_signer, + )?; + service.projection.activation = Some(launch.activation().clone()); + service.projection.current_lease = None; + self.publish_projection(&service.projection) + } + } + + fn digest(byte: char) -> String { + format!("sha256-{}", byte.to_string().repeat(64)) + } + + fn decode_signed(bytes: &[u8]) -> Result { + Ok(OdinRuntimeTopologyCorrelationRecord::decode_canonical_signed_payload(bytes)?.0) + } + + #[test] + fn partial_or_mismatched_projection_remains_signed_expected_only() -> Result<()> { + let world = TestWorld::new()?; + let mut service = world.service("ghostlight", Vec::new(), false)?; + service.projection.activation = None; + let anchor = service.projection.provider_anchor.as_mut().unwrap(); + anchor.trust_anchor_id = "root/other/runtime-presence".into(); + anchor.service_id = "other".into(); + world.publish_projection(&service.projection)?; + + let bytes = world.engine(NOW).refresh("ghostlight")?.unwrap(); + let record = decode_signed(&bytes)?; + assert!(record.expected); + assert!(!record.present); + assert!(!record.ready); + assert!(record.signed_presence_sha256.is_none()); + let codes: Vec<_> = record + .disagreements + .iter() + .map(|value| value.code.as_str()) + .collect(); + assert!(codes.contains(&"current-activation-missing")); + assert!(codes.contains(&"provider-service-id")); + assert!(codes.contains(&"provider-trust-anchor-id")); + assert!(codes.contains(&"signed-presence-missing")); + Ok(()) + } + + #[test] + fn authenticated_disagreement_is_present_but_not_ready() -> Result<()> { + let world = TestWorld::new()?; + let service = world.service("ghostlight", Vec::new(), false)?; + let presence = service.signed_presence(1, |record| { + record.health_contract = "ghostlight.other-health.v1".into(); + })?; + + let record = decode_signed(&world.engine(NOW).admit_presence( + "ghostlight", + &presence, + NOW, + )?)?; + assert!(record.present); + assert!(!record.ready); + assert!( + record + .disagreements + .iter() + .any(|value| value.code == "health-contract") + ); + Ok(()) + } + + #[test] + fn exact_active_presence_and_current_lease_are_ready() -> Result<()> { + let world = TestWorld::new()?; + let service = world.service("ghostlight", Vec::new(), true)?; + let presence = service.signed_presence(1, |_| {})?; + + let record = decode_signed(&world.engine(NOW).admit_presence( + "ghostlight", + &presence, + NOW, + )?)?; + assert!(record.present); + assert!(record.ready); + assert_eq!( + record.observed_write_lease_sha256, + service + .projection + .current_lease + .as_ref() + .map(IdunnProcessWriteLeaseRecord::canonical_sha256) + .transpose()? + ); + Ok(()) + } + + #[test] + fn stale_projected_lease_is_visible_while_candidate_warming_is_present() -> Result<()> { + let world = TestWorld::new()?; + let mut service = world.service("ghostlight", Vec::new(), true)?; + let stale = service.projection.current_lease.as_mut().unwrap(); + stale.expected_projection_sha256 = digest('a'); + stale.plan_id = digest('b'); + stale.incarnation_id = "ghostlight/incumbent".into(); + stale.sealed_release_id = digest('c'); + stale.activation_witness_sha256 = digest('d'); + stale.runtime_instance_id = digest('e'); + world.publish_projection(&service.projection)?; + let warming = service.signed_presence(1, |record| { + record.state = "warming".into(); + record.detail = "starting".into(); + record.write_lease_sha256 = None; + })?; + + let record = decode_signed(&world.engine(NOW).admit_presence( + "ghostlight", + &warming, + NOW, + )?)?; + assert!(record.present); + assert!(!record.ready); + assert_eq!(record.observed_presence_state.as_deref(), Some("warming")); + assert!(record.observed_write_lease_sha256.is_none()); + assert!( + record + .disagreements + .iter() + .any(|value| value.code == "projected-write-lease") + ); + Ok(()) + } + + #[test] + fn provider_sequence_admission_is_monotonic_and_exact_retry_is_idempotent() -> Result<()> { + let world = TestWorld::new()?; + let service = world.service("ghostlight", Vec::new(), false)?; + let sequence_two = service.signed_presence(2, |_| {})?; + let engine = world.engine(NOW); + + let first = engine.admit_presence("ghostlight", &sequence_two, NOW)?; + let duplicate = engine.admit_presence("ghostlight", &sequence_two, NOW + 60_000)?; + assert_eq!(duplicate, first); + assert_eq!( + engine.current_signed_correlation("ghostlight")?, + Some(first) + ); + + let lower = service.signed_presence(1, |_| {})?; + assert!(engine.admit_presence("ghostlight", &lower, NOW).is_err()); + let reused = service.signed_presence(2, |record| record.detail = "different".into())?; + assert!(engine.admit_presence("ghostlight", &reused, NOW).is_err()); + + let entries = SingleFileMessagePackBackingStore::new(&world.topology_path) + .pull_all_read_only_snapshot()?; + let stored = unique_envelope( + &entries, + GameCultRuntimePresenceHealthRecord::TYPE, + &presence_store_key( + "ghostlight", + &service.projection.expected.expected_signer_identity_id, + ), + )? + .unwrap(); + assert_eq!(stored.payload, sequence_two); + assert_eq!(parse_rfc3339_millis(&stored.stored_at)?, NOW); + Ok(()) + } + + #[test] + fn restart_rehydrates_exact_bytes_without_retimestamping() -> Result<()> { + let world = TestWorld::new()?; + let service = world.service("ghostlight", Vec::new(), false)?; + let presence = service.signed_presence(1, |_| {})?; + let first = world + .engine(NOW) + .admit_presence("ghostlight", &presence, NOW)?; + + let after_restart = world.engine(NOW + 100).refresh("ghostlight")?.unwrap(); + assert_eq!(after_restart, first); + assert_eq!(decode_signed(&after_restart)?.observed_at_unix_millis, NOW); + Ok(()) + } + + #[test] + fn withdrawn_expected_removes_current_correlation_but_preserves_presence_history() -> Result<()> + { + let world = TestWorld::new()?; + let service = world.service("ghostlight", Vec::new(), false)?; + let presence = service.signed_presence(1, |_| {})?; + let engine = world.engine(NOW); + engine.admit_presence("ghostlight", &presence, NOW)?; + assert!(engine.current_signed_correlation("ghostlight")?.is_some()); + + std::fs::remove_file(&world.projection_path)?; + assert!(engine.current_signed_correlation("ghostlight")?.is_none()); + + let entries = SingleFileMessagePackBackingStore::new(&world.topology_path) + .pull_all_read_only_snapshot()?; + assert!(entries.iter().any(|entry| { + entry.r#type == GameCultRuntimePresenceHealthRecord::TYPE && entry.payload == presence + })); + assert!(!entries.iter().any(|entry| { + entry.r#type == OdinRuntimeTopologyCorrelationRecord::TYPE && entry.key == "ghostlight" + })); + Ok(()) + } + + #[test] + fn dependencies_consume_only_exact_current_admitted_correlations() -> Result<()> { + let world = TestWorld::new()?; + let mut provider = world.service("provider", Vec::new(), false)?; + let provider_expected_sha256 = provider.projection.expected.canonical_sha256()?; + let provider_endpoint = provider + .projection + .expected + .route + .as_ref() + .unwrap() + .stable_endpoint + .clone(); + let requirement = IdunnExpectedDependency { + kind: "required".into(), + capability: "data".into(), + schema: "data.v1".into(), + compatibility: "v1".into(), + minimum_capacity: 1, + startup: "before-promotion".into(), + provider_id: Some("provider".into()), + provider_authority: Some("managed-incarnation".into()), + provider_expected_projection_sha256: Some(provider_expected_sha256), + provider_endpoint: Some(provider_endpoint), + }; + let consumer = world.service("consumer", vec![requirement], false)?; + let provider_presence = provider.signed_presence(1, |_| {})?; + let consumer_presence = consumer.signed_presence(1, |_| {})?; + let engine = world.engine(NOW); + + engine.admit_presence("provider", &provider_presence, NOW)?; + let ready = decode_signed(&engine.admit_presence("consumer", &consumer_presence, NOW)?)?; + assert!(ready.ready); + assert!(ready.dependencies[0].ready); + assert!(ready.dependencies[0].provider_evidence_sha256.is_some()); + + world.replace_provider_generation(&mut provider)?; + let no_longer_exact = decode_signed(&engine.refresh("consumer")?.unwrap())?; + assert!(no_longer_exact.present); + assert!(!no_longer_exact.ready); + assert!(!no_longer_exact.dependencies[0].ready); + assert!( + no_longer_exact.dependencies[0] + .provider_evidence_sha256 + .is_none() + ); + Ok(()) + } +} diff --git a/crates/odin-daemon/src/main.rs b/crates/odin-daemon/src/main.rs new file mode 100644 index 0000000..99094d3 --- /dev/null +++ b/crates/odin-daemon/src/main.rs @@ -0,0 +1,1248 @@ +use std::cell::RefCell; +use std::collections::{BTreeMap, BTreeSet, VecDeque}; +use std::fs::{File, OpenOptions}; +use std::net::{SocketAddr, UdpSocket}; +use std::os::fd::{FromRawFd, RawFd}; +use std::path::{Path, PathBuf}; +use std::rc::Rc; +use std::thread; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, Result, anyhow, bail, ensure}; +use chrono::{DateTime, Utc}; +use cultcache_rs::{ + CultCacheEnvelope, CultCacheExpectedEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, +}; +use cultmesh_rs::{ + CultMeshRudpDocumentPublishOptions, CultMeshRudpDocumentServer, + CultMeshRudpDocumentServerOptions, CultMeshRudpPollOutcome, CultMeshRudpRawDocumentReceipt, + CultMeshRudpRawDocumentSink, CultMeshRudpSnapshotQuery, CultMeshRudpSnapshotSource, + CultMeshSystemClock, publish_cultnet_message_to_rudp_catalog, +}; +use cultnet_rs::{ + CultNetMessage, CultNetRawDocumentRecord, CultNetRawPayloadEncoding, + GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, GameCultProviderHealthIdentity, + GameCultRuntimeCapability, GameCultRuntimePresenceHealthPurpose, + GameCultRuntimePresenceHealthRecord, IDUNN_EXPECTED_INCARNATION_SCHEMA, + IDUNN_PROCESS_WRITE_LEASE_SCHEMA, IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME, + IDUNN_RUNTIME_ACTIVATION_SCHEMA, IdunnExpectedIncarnationRecord, IdunnProcessWriteLeaseRecord, + IdunnRuntimeActivationRecord, IdunnRuntimeActivationSigner, IdunnServiceIdentity, + ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA, OdinRuntimeTopologyCorrelationRecord, + OdinTopologyIdentity, ServiceIdentityProfile, ServiceIdentitySigner, + ServiceIdentityTrustAnchor, derive_service_identity_id, + open_service_identity_credential_reader, verify_runtime_authority, +}; +use fs2::FileExt; +use odin_daemon::{ + AuthenticationPolicy, CultCacheIdunnProjectionSource, CultCacheOdinTopologyStore, + IdunnProjectionSource, OdinTopologyAuthority, SystemClock, +}; + +const TARGET: &str = "odin"; +const HEALTH_CONTRACT: &str = "odin.runtime-health.v1"; +const RENDEZVOUS_CAPABILITY: &str = "odin.verse-rendezvous"; +const RENDEZVOUS_SCHEMA: &str = "odin.verse-topology.v1"; +const RENDEZVOUS_COMPATIBILITY: &str = "v1"; +const STATE_SCHEMA_GENERATION: &str = "odin-v2"; +// rmp-serde SHA-256 of deployment/idunn/recipe.toml's exact [state] value. +const STATE_CONTRACT_SHA256: &str = + "sha256-4f2f2dcd931d16f6b02bf295f41227b867aa982765661d55fa9f29fb2db7e449"; + +const RUNTIME_BUNDLE_ENVIRONMENT: &str = "GAMECULT_IDUNN_RUNTIME_BUNDLE"; +const CANDIDATE_BIND_ENVIRONMENT: &str = "GAMECULT_IDUNN_CANDIDATE_BIND"; +const PROCESS_WRITE_LEASE_ENVIRONMENT: &str = "GAMECULT_IDUNN_PROCESS_WRITE_LEASE"; +const TOPOLOGY_IDENTITY_ENVIRONMENT: &str = "ODIN_TOPOLOGY_IDENTITY"; +const SYSTEMD_LISTEN_PID_ENVIRONMENT: &str = "LISTEN_PID"; +const SYSTEMD_LISTEN_FDS_ENVIRONMENT: &str = "LISTEN_FDS"; +const SYSTEMD_LISTEN_FDNAMES_ENVIRONMENT: &str = "LISTEN_FDNAMES"; +const ACTIVATION_SIGNER_FD_NAME: &str = IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME; +const PROVIDER_SIGNER_FD_NAME: &str = "gamecult-runtime-presence-identity"; +const SYSTEMD_LISTEN_FDS_START: RawFd = 3; + +const BOOTSTRAP_LEASE_TIMEOUT: Duration = Duration::from_secs(60); +const SELF_PUBLISH_TIMEOUT: Duration = Duration::from_secs(6); +const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(5); +const PROJECTION_REFRESH_INTERVAL: Duration = Duration::from_millis(250); +const IDLE_POLL_INTERVAL: Duration = Duration::from_millis(2); +const MAX_RECENT_WARMING_PROOFS: usize = 64; +const WARMING_PROOF_LIFETIME_MILLIS: u64 = 60_000; +const CAS_ATTEMPTS: usize = 8; + +type TopologyAuthority = OdinTopologyAuthority< + CultCacheIdunnProjectionSource, + CultCacheOdinTopologyStore, + ServiceIdentitySigner, + SystemClock, +>; + +#[derive(Clone, Debug)] +struct Options { + store: PathBuf, + idunn_projection: PathBuf, + idunn_anchor: PathBuf, +} + +struct RuntimeAuthority { + expected: IdunnExpectedIncarnationRecord, + expected_sha256: String, + activation: IdunnRuntimeActivationRecord, + activation_sha256: String, + activation_signer: IdunnRuntimeActivationSigner, + provider_signer: ServiceIdentitySigner, +} + +struct ProcessWriteLeaseGuard { + _lock: File, + record: IdunnProcessWriteLeaseRecord, + sha256: String, +} + +struct RuntimeState { + options: Options, + candidate: SocketAddr, + authority_material: RuntimeAuthority, + idunn_anchor: Option, + topology_signer: Option>, + topology: Option, + write_lease: Option, + write_lease_path: PathBuf, + recent_warming_proofs: VecDeque<(String, u64)>, + publisher_sequence: u64, +} + +#[derive(Clone)] +struct SinkHandle(Rc>); + +#[derive(Clone)] +struct SnapshotHandle(Rc>); + +impl CultMeshRudpRawDocumentSink for SinkHandle { + fn accept_raw_document(&mut self, receipt: CultMeshRudpRawDocumentReceipt) -> Result<()> { + self.0.borrow_mut().accept_raw_document(receipt) + } +} + +impl CultMeshRudpSnapshotSource for SnapshotHandle { + fn raw_snapshot( + &mut self, + query: &CultMeshRudpSnapshotQuery, + ) -> Result> { + self.0.borrow_mut().raw_snapshot(query) + } +} + +impl RuntimeState { + fn open(options: Options, candidate: SocketAddr) -> Result { + let authority_material = load_runtime_authority(Path::new(&required_environment( + RUNTIME_BUNDLE_ENVIRONMENT, + )?))?; + require_expected_contract(&authority_material.expected, candidate)?; + + let idunn_anchor = read_trust_anchor::(&options.idunn_anchor)?; + let projection_source = CultCacheIdunnProjectionSource::new(&options.idunn_projection); + let projection = projection_source + .current_projection(TARGET)? + .context("Idunn projection has no Expected Odin incarnation")?; + ensure!( + projection.expected == authority_material.expected + && projection.activation.as_ref() == Some(&authority_material.activation), + "Idunn live projection differs from Odin's immutable runtime bundle" + ); + let provider_anchor = projection + .provider_anchor + .as_ref() + .context("Idunn projection has no Odin runtime-presence trust anchor")?; + ensure!( + provider_anchor.signer_identity_id + == authority_material.provider_signer.entry().identity_id + && provider_anchor.signer_public_key + == authority_material.provider_signer.entry().public_key, + "Odin provider signer differs from Idunn's projected trust anchor" + ); + verify_runtime_authority( + &authority_material.expected, + &authority_material.activation, + &idunn_anchor, + &provider_anchor.signer_public_key, + )?; + + let topology_identity_path = + PathBuf::from(required_environment(TOPOLOGY_IDENTITY_ENVIRONMENT)?); + let topology_signer = open_service_identity_credential_reader::( + File::open(&topology_identity_path).with_context(|| { + format!( + "opening Odin topology identity credential {}", + topology_identity_path.display() + ) + })?, + )?; + let store_sequence = prior_self_publisher_sequence( + &options.store, + authority_material + .provider_signer + .entry() + .identity_id + .as_str(), + )?; + + Ok(Self { + options, + candidate, + authority_material, + idunn_anchor: Some(idunn_anchor), + topology_signer: Some(topology_signer), + topology: None, + write_lease: None, + write_lease_path: PathBuf::from(required_environment(PROCESS_WRITE_LEASE_ENVIRONMENT)?), + recent_warming_proofs: VecDeque::new(), + publisher_sequence: store_sequence, + }) + } + + fn activated(&self) -> bool { + self.topology.is_some() && self.write_lease.is_some() + } + + fn try_activate(&mut self) -> Result { + if self.activated() { + return Ok(true); + } + let Some(lease) = acquire_process_write_lease( + &self.write_lease_path, + &self.authority_material, + &self.recent_warming_proofs, + )? + else { + return Ok(false); + }; + let Some(projected) = CultCacheIdunnProjectionSource::new(&self.options.idunn_projection) + .current_projection(TARGET)? + else { + return Ok(false); + }; + if projected.current_lease.as_ref() != Some(&lease.record) { + return Ok(false); + } + let signer = self + .topology_signer + .take() + .context("Odin topology signer was already consumed")?; + let idunn_anchor = self + .idunn_anchor + .take() + .context("Idunn trust anchor was already consumed")?; + self.write_lease = Some(lease); + self.topology = Some(OdinTopologyAuthority::new( + CultCacheIdunnProjectionSource::new(&self.options.idunn_projection), + CultCacheOdinTopologyStore::new(&self.options.store), + signer, + SystemClock, + idunn_anchor, + AuthenticationPolicy::default(), + )); + Ok(true) + } + + fn accept_raw_document(&mut self, receipt: CultMeshRudpRawDocumentReceipt) -> Result<()> { + ensure!( + self.activated(), + "Odin does not admit or persist provider documents before its process-write lease" + ); + validate_raw_document_shape(&receipt.document)?; + if receipt.document.schema_id == GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA { + let presence = decode_presence(&receipt.document.payload)?; + ensure!( + receipt.document.record_key == presence.target, + "runtime-presence document key differs from its signed target" + ); + self.topology + .as_ref() + .context("Odin topology authority is absent")? + .admit_presence( + &presence.target, + &receipt.document.payload, + receipt.received_at_unix_millis, + )?; + return Ok(()); + } + persist_generic_document(&self.options.store, &receipt.document) + } + + fn raw_snapshot( + &mut self, + query: &CultMeshRudpSnapshotQuery, + ) -> Result> { + validate_snapshot_filters(query)?; + if exact_self_presence_query(query) { + let (state, detail) = if self.activated() { + self.require_current_write_lease()?; + ("active", format!("route-observation:{}", query.message_id)) + } else { + ("warming", format!("idunn-warming:{}", query.message_id)) + }; + return Ok(vec![self.signed_presence_document(state, &detail)?]); + } + ensure!( + self.activated(), + "Odin catalog is unavailable until Idunn grants its process-write lease" + ); + self.refresh_all_correlations()?; + self.stored_snapshot(query) + } + + fn signed_presence_document( + &mut self, + state: &str, + detail: &str, + ) -> Result { + let now = unix_millis()?; + self.publisher_sequence = self + .publisher_sequence + .checked_add(1) + .context("Odin runtime-presence publisher sequence exhausted")?; + let expected = &self.authority_material.expected; + let activation = &self.authority_material.activation; + let source_runtime_id = expected.runtime_id.clone(); + let mut record = GameCultRuntimePresenceHealthRecord { + schema_version: GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + target: expected.target.clone(), + expected_projection_sha256: self.authority_material.expected_sha256.clone(), + plan_id: expected.plan_id.clone(), + incarnation_id: expected.incarnation_id.clone(), + sealed_release_id: expected.sealed_release_id.clone(), + activation_witness_sha256: self.authority_material.activation_sha256.clone(), + state_schema_generation: expected.state_schema_generation.clone(), + state_contract_sha256: expected.state_contract_sha256.clone(), + runtime_id: expected.runtime_id.clone(), + runtime_instance_id: activation.runtime_instance_id.clone(), + bound_endpoint: Some(format!("rudp://{}", self.candidate)), + capabilities: expected + .capabilities + .iter() + .map(|capability| GameCultRuntimeCapability { + capability: capability.capability.clone(), + schema: capability.schema.clone(), + compatibility: capability.compatibility.clone(), + capacity: capability.minimum_capacity, + }) + .collect(), + health_contract: expected.health_contract.clone(), + state: state.into(), + detail: detail.into(), + write_lease_sha256: self.write_lease.as_ref().map(|lease| lease.sha256.clone()), + signer_identity_id: self + .authority_material + .provider_signer + .entry() + .identity_id + .clone(), + publisher_sequence: self.publisher_sequence, + observed_at_unix_millis: now, + signature_algorithm: "ed25519".into(), + signature: Vec::new(), + activation_signer_identity_id: activation.activation_signer_identity_id.clone(), + activation_signature: Vec::new(), + }; + ensure!( + (state == "warming" && record.write_lease_sha256.is_none()) + || (state == "active" && record.write_lease_sha256.is_some()), + "Odin presence state does not match its process-write authority" + ); + let proof_payload = record.canonical_proof_payload()?; + record.signature = self + .authority_material + .provider_signer + .sign::(&proof_payload) + .signature; + record.activation_signature = self + .authority_material + .activation_signer + .sign_presence_proof(&record)?; + record.validate()?; + let payload = rmp_serde::to_vec(&record)?; + ensure!( + rmp_serde::from_slice::(&payload)? == record, + "Odin runtime presence is not canonical MessagePack" + ); + if state == "warming" { + remember_recent_warming_proof( + &mut self.recent_warming_proofs, + record.canonical_sha256()?, + now, + ); + } + Ok(CultNetRawDocumentRecord { + schema_id: GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + record_key: TARGET.into(), + stored_at: rfc3339_millis(now)?, + payload_encoding: CultNetRawPayloadEncoding::Messagepack, + payload, + source_runtime_id: Some(source_runtime_id), + source_agent_id: Some(record.signer_identity_id), + source_role: Some("runtime-presence-health-publisher".into()), + tags: Some(vec!["cultnet.transport.rudp.v0".into()]), + }) + } + + fn active_document_put(&mut self, detail: &str) -> Result { + self.require_current_write_lease()?; + let document = self.signed_presence_document("active", detail)?; + let sequence = self.publisher_sequence; + Ok(CultNetMessage::DocumentPutRaw { + message_id: format!( + "odin-presence:{}:{}:{sequence}", + self.authority_material.expected.runtime_id, + self.authority_material.activation.runtime_instance_id + ), + document, + }) + } + + fn require_current_write_lease(&self) -> Result<()> { + let held = self + .write_lease + .as_ref() + .context("Odin has no process-write lease")?; + let current = read_process_write_lease(&self.write_lease_path)? + .context("Odin process-write lease was withdrawn")?; + ensure!( + current == held.record && current.canonical_sha256()? == held.sha256, + "Odin process-write lease changed after admission" + ); + Ok(()) + } + + fn refresh_all_correlations(&mut self) -> Result<()> { + self.require_current_write_lease()?; + let topology = self + .topology + .as_ref() + .context("Odin topology authority is absent")?; + let mut targets = projection_targets(&self.options.idunn_projection)?; + targets.extend(correlation_targets(&self.options.store)?); + for target in targets { + topology.refresh(&target)?; + } + Ok(()) + } + + fn stored_snapshot( + &self, + query: &CultMeshRudpSnapshotQuery, + ) -> Result> { + let entries = if self.options.store.is_file() { + SingleFileMessagePackBackingStore::new(&self.options.store) + .pull_all_read_only_snapshot()? + } else { + Vec::new() + }; + let projections = CultCacheIdunnProjectionSource::new(&self.options.idunn_projection); + let mut selected = BTreeMap::new(); + for envelope in entries { + let Some(schema_id) = envelope.schema_id.clone() else { + continue; + }; + let document = if envelope.r#type == GameCultRuntimePresenceHealthRecord::TYPE { + let presence = decode_presence(&envelope.payload)?; + let Some(projection) = projections.current_projection(&presence.target)? else { + continue; + }; + if projection.expected.expected_signer_identity_id != presence.signer_identity_id + || projection.expected.canonical_sha256()? + != presence.expected_projection_sha256 + || projection + .activation + .as_ref() + .map(IdunnRuntimeActivationRecord::canonical_sha256) + .transpose()? + .as_deref() + != Some(presence.activation_witness_sha256.as_str()) + || projection + .activation + .as_ref() + .map(|activation| activation.runtime_instance_id.as_str()) + != Some(presence.runtime_instance_id.as_str()) + { + continue; + } + CultNetRawDocumentRecord { + schema_id, + record_key: presence.target.clone(), + stored_at: envelope.stored_at, + payload_encoding: CultNetRawPayloadEncoding::Messagepack, + payload: envelope.payload, + source_runtime_id: Some(presence.runtime_id), + source_agent_id: Some(presence.signer_identity_id), + source_role: Some("runtime-presence-health-publisher".into()), + tags: Some(vec!["odin-observed".into()]), + } + } else if envelope.r#type == OdinRuntimeTopologyCorrelationRecord::TYPE { + let (correlation, _) = + OdinRuntimeTopologyCorrelationRecord::decode_canonical_signed_payload( + &envelope.payload, + )?; + CultNetRawDocumentRecord { + schema_id, + record_key: correlation.target, + stored_at: envelope.stored_at, + payload_encoding: CultNetRawPayloadEncoding::Messagepack, + payload: envelope.payload, + source_runtime_id: Some(self.authority_material.expected.runtime_id.clone()), + source_agent_id: Some(correlation.signer_identity_id), + source_role: Some("odin-topology-correlation".into()), + tags: Some(vec!["odin-owned".into()]), + } + } else { + CultNetRawDocumentRecord { + schema_id, + record_key: envelope.key, + stored_at: envelope.stored_at, + payload_encoding: CultNetRawPayloadEncoding::Messagepack, + payload: envelope.payload, + source_runtime_id: None, + source_agent_id: None, + source_role: None, + tags: None, + } + }; + if !query_allows(query, &document) { + continue; + } + let identity = (document.schema_id.clone(), document.record_key.clone()); + ensure!( + selected.insert(identity, document).is_none(), + "Odin catalog contains duplicate public document identities" + ); + } + Ok(selected.into_values().collect()) + } +} + +fn main() -> Result<()> { + let options = parse_options(std::env::args().skip(1))?; + let candidate: SocketAddr = required_environment(CANDIDATE_BIND_ENVIRONMENT)? + .parse() + .context("parsing Idunn candidate bind")?; + ensure!( + candidate.ip().is_loopback() && candidate.port() != 0, + "Odin candidate bind must be one fixed loopback socket" + ); + let socket = UdpSocket::bind(candidate) + .with_context(|| format!("binding Odin CultNet RUDP candidate {candidate}"))?; + let state = Rc::new(RefCell::new(RuntimeState::open(options, candidate)?)); + let mut server = CultMeshRudpDocumentServer::new( + socket, + SinkHandle(state.clone()), + SnapshotHandle(state.clone()), + CultMeshSystemClock::default(), + CultMeshRudpDocumentServerOptions::default(), + )?; + + let bootstrap_started = Instant::now(); + while !state.borrow_mut().try_activate()? { + let progressed = poll_server(&mut server)?; + ensure!( + bootstrap_started.elapsed() < BOOTSTRAP_LEASE_TIMEOUT, + "timed out waiting for Idunn to admit Odin's exact Warming proof and grant its lease" + ); + if !progressed { + thread::sleep(IDLE_POLL_INTERVAL); + } + } + + publish_self_presence(&state, &mut server, "ready")?; + state.borrow_mut().refresh_all_correlations()?; + let mut last_heartbeat = Instant::now(); + let mut last_projection_refresh = Instant::now(); + loop { + let progressed = poll_server(&mut server)?; + if last_projection_refresh.elapsed() >= PROJECTION_REFRESH_INTERVAL { + state.borrow_mut().refresh_all_correlations()?; + last_projection_refresh = Instant::now(); + } + if last_heartbeat.elapsed() >= HEARTBEAT_INTERVAL { + publish_self_presence(&state, &mut server, "ready")?; + last_heartbeat = Instant::now(); + } + if !progressed { + thread::sleep(IDLE_POLL_INTERVAL); + } + } +} + +fn publish_self_presence( + state: &Rc>, + server: &mut CultMeshRudpDocumentServer, + detail: &str, +) -> Result<()> { + let (message, options) = { + let mut state = state.borrow_mut(); + let message = state.active_document_put(detail)?; + let mut options = CultMeshRudpDocumentPublishOptions::odin( + state.candidate, + state.authority_material.expected.runtime_id.clone(), + ); + options.connect_timeout = Duration::from_secs(2); + options.flush_timeout = Duration::from_secs(2); + options.source_agent_id = Some( + state + .authority_material + .provider_signer + .entry() + .identity_id + .clone(), + ); + options.source_role = Some("runtime-presence-health-publisher".into()); + options.tags = vec!["cultnet.transport.rudp.v0".into()]; + (message, options) + }; + let publisher = + thread::spawn(move || publish_cultnet_message_to_rudp_catalog(&message, options)); + let started = Instant::now(); + while !publisher.is_finished() { + let progressed = poll_server(server)?; + ensure!( + started.elapsed() < SELF_PUBLISH_TIMEOUT, + "Odin self-presence publication did not complete" + ); + if !progressed { + thread::sleep(IDLE_POLL_INTERVAL); + } + } + publisher + .join() + .map_err(|_| anyhow!("Odin self-presence publisher panicked"))??; + Ok(()) +} + +fn poll_server( + server: &mut CultMeshRudpDocumentServer, +) -> Result { + match server.poll_once()? { + CultMeshRudpPollOutcome::Idle => Ok(false), + CultMeshRudpPollOutcome::Handled => Ok(true), + CultMeshRudpPollOutcome::ApplicationRejected(rejection) => { + eprintln!( + "Odin rejected CultMesh RUDP application message {:?} {} from {:?}: {}", + rejection.operation, rejection.message_id, rejection.session, rejection.reason + ); + Ok(true) + } + } +} + +fn parse_options(args: impl Iterator) -> Result { + let mut args = args.peekable(); + let mut values = BTreeMap::new(); + while let Some(name) = args.next() { + let name = name + .strip_prefix("--") + .with_context(|| format!("expected --option, got {name:?}"))?; + ensure!( + matches!(name, "store" | "idunn-projection" | "idunn-anchor"), + "unsupported Odin option --{name}" + ); + let value = args + .next() + .with_context(|| format!("missing value for --{name}"))?; + ensure!( + values + .insert(name.to_owned(), PathBuf::from(value)) + .is_none(), + "duplicate Odin option --{name}" + ); + } + let take = |name: &str| -> Result { + let path = values + .get(name) + .cloned() + .with_context(|| format!("--{name} is required"))?; + ensure!(path.is_absolute(), "--{name} must be an absolute path"); + Ok(path) + }; + let options = Options { + store: take("store")?, + idunn_projection: take("idunn-projection")?, + idunn_anchor: take("idunn-anchor")?, + }; + ensure!( + options.store != options.idunn_projection + && options.store != options.idunn_anchor + && options.idunn_projection != options.idunn_anchor, + "Odin store, Idunn projection, and Idunn anchor paths must be distinct" + ); + Ok(options) +} + +fn load_runtime_authority(bundle: &Path) -> Result { + ensure!(bundle.is_absolute(), "Idunn runtime bundle is not absolute"); + let (expected_key, expected_payload) = read_single_runtime_record( + &bundle.join("expected.cc"), + IdunnExpectedIncarnationRecord::TYPE, + IDUNN_EXPECTED_INCARNATION_SCHEMA, + )?; + let expected = IdunnExpectedIncarnationRecord::decode_canonical(&expected_payload)?; + ensure!( + expected_key == expected.target, + "Expected key is substituted" + ); + let (activation_key, activation_payload) = read_single_runtime_record( + &bundle.join("activation.cc"), + IdunnRuntimeActivationRecord::TYPE, + IDUNN_RUNTIME_ACTIVATION_SCHEMA, + )?; + let activation = IdunnRuntimeActivationRecord::decode_canonical(&activation_payload)?; + ensure!( + activation_key == expected.target, + "activation key is substituted" + ); + let expected_sha256 = expected.canonical_sha256()?; + ensure!( + activation.expected_projection_sha256 == expected_sha256 + && activation.runtime_id == expected.runtime_id, + "activation does not bind Odin's Expected projection" + ); + let (activation_credential, provider_identity) = take_runtime_signer_descriptors()?; + let activation_signer = + IdunnRuntimeActivationSigner::from_credential_reader(activation_credential)?; + ensure!( + activation_signer.identity_id() == activation.activation_signer_identity_id + && activation_signer.public_key() == activation.activation_signer_public_key, + "activation credential does not belong to Odin's activation" + ); + let provider_signer = open_service_identity_credential_reader::( + provider_identity, + )?; + ensure!( + provider_signer.entry().identity_id == expected.expected_signer_identity_id, + "provider credential is not the signer selected by Expected" + ); + Ok(RuntimeAuthority { + expected, + expected_sha256, + activation_sha256: activation.canonical_sha256()?, + activation, + activation_signer, + provider_signer, + }) +} + +fn require_expected_contract( + expected: &IdunnExpectedIncarnationRecord, + bind: SocketAddr, +) -> Result<()> { + ensure!( + expected.target == TARGET + && expected.health_contract == HEALTH_CONTRACT + && expected.state_schema_generation.as_deref() == Some(STATE_SCHEMA_GENERATION) + && expected.state_contract_sha256.as_deref() == Some(STATE_CONTRACT_SHA256) + && expected.write_lease_required, + "Odin Expected projection differs from its compiled runtime/state contract" + ); + let route = expected + .route + .as_ref() + .context("Odin Expected projection has no admitted route")?; + ensure!( + route.transport == "rudp" && route.candidate_endpoint == format!("rudp://{bind}"), + "Odin candidate bind differs from Expected" + ); + let capability = expected + .capabilities + .iter() + .find(|capability| { + capability.capability == RENDEZVOUS_CAPABILITY + && capability.schema == RENDEZVOUS_SCHEMA + && capability.compatibility == RENDEZVOUS_COMPATIBILITY + }) + .context("Odin Expected projection omits its rendezvous capability")?; + ensure!( + capability.minimum_capacity > 0 && expected.capabilities.len() == 1, + "Odin Expected capability set differs from its compiled contract" + ); + ensure!( + expected.dependencies.is_empty(), + "Odin bootstrap cannot depend on a managed daemon" + ); + Ok(()) +} + +fn read_single_runtime_record( + path: &Path, + expected_type: &str, + expected_schema: &str, +) -> Result<(String, Vec)> { + let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; + let [envelope] = entries.as_slice() else { + bail!("runtime authority store must contain exactly one record"); + }; + ensure!( + envelope.r#type == expected_type && envelope.schema_id.as_deref() == Some(expected_schema), + "runtime authority store has the wrong typed envelope" + ); + Ok((envelope.key.clone(), envelope.payload.clone())) +} + +fn take_runtime_signer_descriptors() -> Result<(File, File)> { + let pid = required_environment(SYSTEMD_LISTEN_PID_ENVIRONMENT)?; + let count = required_environment(SYSTEMD_LISTEN_FDS_ENVIRONMENT)?; + let names = required_environment(SYSTEMD_LISTEN_FDNAMES_ENVIRONMENT)?; + ensure!( + pid == std::process::id().to_string(), + "systemd signer descriptors belong to another process" + ); + ensure!(count == "2", "Odin requires exactly two signer descriptors"); + ensure!( + names == format!("{ACTIVATION_SIGNER_FD_NAME}:{PROVIDER_SIGNER_FD_NAME}"), + "systemd signer descriptor names or order differ from Idunn's contract" + ); + // SAFETY: the exact systemd LISTEN_* contract above assigns sole ownership + // of descriptors 3 and 4 to this process, and this is the first FD consumer. + let activation = unsafe { File::from_raw_fd(SYSTEMD_LISTEN_FDS_START) }; + // SAFETY: descriptor 4 is distinct and covered by the same exact contract. + let provider = unsafe { File::from_raw_fd(SYSTEMD_LISTEN_FDS_START + 1) }; + Ok((activation, provider)) +} + +fn read_trust_anchor(path: &Path) -> Result { + let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; + let [envelope] = entries.as_slice() else { + bail!("service trust-anchor store must contain exactly one document"); + }; + ensure!( + envelope.r#type == P::TRUST_ANCHOR_TYPE + && envelope.key == P::TRUST_ANCHOR_KEY + && envelope.schema_id.as_deref() == Some(P::TRUST_ANCHOR_SCHEMA), + "service trust anchor belongs to another identity profile" + ); + let anchor: ServiceIdentityTrustAnchor = rmp_serde::from_slice(&envelope.payload)?; + ensure!( + rmp_serde::to_vec(&anchor)? == envelope.payload + && anchor.schema_version == P::TRUST_ANCHOR_SCHEMA + && derive_service_identity_id::

(&anchor.public_key)? == anchor.identity_id, + "service trust anchor is noncanonical or self-inconsistent" + ); + Ok(anchor) +} + +fn acquire_process_write_lease( + path: &Path, + authority: &RuntimeAuthority, + recent_warming: &VecDeque<(String, u64)>, +) -> Result> { + let Some(observed) = read_process_write_lease(path)? else { + return Ok(None); + }; + let now = unix_millis()?; + let warming_is_ours = + recent_warming_proof(recent_warming, &observed.warming_presence_sha256, now); + let exact_incarnation = observed.target == authority.expected.target + && observed.expected_projection_sha256 == authority.expected_sha256 + && observed.plan_id == authority.expected.plan_id + && observed.incarnation_id == authority.expected.incarnation_id + && observed.sealed_release_id == authority.expected.sealed_release_id + && observed.activation_witness_sha256 == authority.activation_sha256 + && Some(observed.state_schema_generation.as_str()) + == authority.expected.state_schema_generation.as_deref() + && Some(observed.state_contract_sha256.as_str()) + == authority.expected.state_contract_sha256.as_deref() + && observed.runtime_id == authority.expected.runtime_id + && observed.runtime_instance_id == authority.activation.runtime_instance_id; + if !warming_is_ours || !exact_incarnation { + return Ok(None); + } + let lock_path = sibling_lock_path(path)?; + let lock = OpenOptions::new() + .read(true) + .open(&lock_path) + .with_context(|| format!("opening process-write-lease lock {}", lock_path.display()))?; + FileExt::lock_shared(&lock)?; + let current = read_process_write_lease(path)? + .context("process-write lease disappeared while acquiring its lifetime lock")?; + ensure!( + current == observed, + "process-write lease changed while acquiring its lifetime lock" + ); + Ok(Some(ProcessWriteLeaseGuard { + _lock: lock, + sha256: current.canonical_sha256()?, + record: current, + })) +} + +fn remember_recent_warming_proof( + recent_warming: &mut VecDeque<(String, u64)>, + sha256: String, + now: u64, +) { + while recent_warming + .front() + .is_some_and(|(_, issued)| now.saturating_sub(*issued) > WARMING_PROOF_LIFETIME_MILLIS) + { + recent_warming.pop_front(); + } + recent_warming.push_back((sha256, now)); + while recent_warming.len() > MAX_RECENT_WARMING_PROOFS { + recent_warming.pop_front(); + } +} + +fn recent_warming_proof( + recent_warming: &VecDeque<(String, u64)>, + expected_sha256: &str, + now: u64, +) -> bool { + recent_warming.iter().any(|(sha256, issued)| { + sha256 == expected_sha256 && now.saturating_sub(*issued) <= WARMING_PROOF_LIFETIME_MILLIS + }) +} + +fn read_process_write_lease(path: &Path) -> Result> { + if !path.is_file() { + return Ok(None); + } + let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; + let [envelope] = entries.as_slice() else { + ensure!(entries.is_empty(), "process-write-lease store is ambiguous"); + return Ok(None); + }; + ensure!( + envelope.r#type == IdunnProcessWriteLeaseRecord::TYPE + && envelope.schema_id.as_deref() == Some(IDUNN_PROCESS_WRITE_LEASE_SCHEMA), + "process-write-lease store has the wrong typed envelope" + ); + let lease = IdunnProcessWriteLeaseRecord::decode_canonical(&envelope.payload)?; + ensure!( + envelope.key == lease.target, + "process-write-lease key differs from its target" + ); + Ok(Some(lease)) +} + +fn prior_self_publisher_sequence(path: &Path, signer_identity_id: &str) -> Result { + if !path.is_file() { + return Ok(0); + } + SingleFileMessagePackBackingStore::new(path) + .pull_all_read_only_snapshot()? + .into_iter() + .filter(|entry| entry.r#type == GameCultRuntimePresenceHealthRecord::TYPE) + .try_fold(0, |maximum, entry| { + let presence = decode_presence(&entry.payload)?; + Ok( + if presence.target == TARGET && presence.signer_identity_id == signer_identity_id { + maximum.max(presence.publisher_sequence) + } else { + maximum + }, + ) + }) +} + +fn persist_generic_document(path: &Path, document: &CultNetRawDocumentRecord) -> Result<()> { + let document_type = document_type_for_schema(&document.schema_id)?; + ensure!( + !matches!( + document_type.as_str(), + "gamecult.runtime_presence_health" + | "odin.runtime_topology_correlation" + | "idunn.expected_incarnation" + | "idunn.runtime_activation" + | "idunn.process_write_lease" + | "gamecult.service_trust_anchor" + ), + "generic provider traffic cannot write an authority-owned Odin/Idunn document" + ); + let replacement = CultCacheEnvelope { + key: document.record_key.clone(), + r#type: document_type.clone(), + payload: document.payload.clone(), + stored_at: document.stored_at.clone(), + schema_id: Some(document.schema_id.clone()), + }; + let store = SingleFileMessagePackBackingStore::new(path); + for _ in 0..CAS_ATTEMPTS { + let entries = if path.is_file() { + store.pull_all_read_only_snapshot()? + } else { + Vec::new() + }; + let mut matches = entries + .iter() + .filter(|entry| entry.r#type == document_type && entry.key == document.record_key); + let current = matches.next().cloned(); + ensure!( + matches.next().is_none(), + "generic CultCache identity is ambiguous" + ); + if current.as_ref() == Some(&replacement) { + return Ok(()); + } + if store.compare_exchange( + &[CultCacheExpectedEnvelope { + key: document.record_key.clone(), + r#type: document_type.clone(), + current, + }], + &[replacement.clone()], + )? { + return Ok(()); + } + } + bail!("Odin catalog changed repeatedly while persisting a provider document") +} + +fn document_type_for_schema(schema_id: &str) -> Result { + ensure!( + !schema_id.is_empty() && schema_id.trim() == schema_id, + "provider schema id is empty or padded" + ); + let Some((prefix, version)) = schema_id.rsplit_once(".v") else { + bail!("provider schema id has no explicit version suffix"); + }; + ensure!( + !prefix.is_empty() + && !version.is_empty() + && version.chars().all(|value| value.is_ascii_digit()), + "provider schema id has an invalid version suffix" + ); + Ok(prefix.into()) +} + +fn validate_raw_document_shape(document: &CultNetRawDocumentRecord) -> Result<()> { + ensure!( + document.payload_encoding == CultNetRawPayloadEncoding::Messagepack + && !document.record_key.is_empty() + && document.record_key.trim() == document.record_key + && !document.payload.is_empty(), + "raw provider document has an invalid key, encoding, or payload" + ); + document_type_for_schema(&document.schema_id)?; + DateTime::parse_from_rfc3339(&document.stored_at) + .context("raw provider document has an invalid stored_at")?; + Ok(()) +} + +fn decode_presence(payload: &[u8]) -> Result { + let presence: GameCultRuntimePresenceHealthRecord = rmp_serde::from_slice(payload)?; + ensure!( + rmp_serde::to_vec(&presence)? == payload, + "runtime presence is not canonical MessagePack" + ); + presence.validate()?; + Ok(presence) +} + +fn projection_targets(path: &Path) -> Result> { + let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; + entries + .into_iter() + .filter(|entry| entry.r#type == IdunnExpectedIncarnationRecord::TYPE) + .map(|entry| { + ensure!( + entry.schema_id.as_deref() == Some(IDUNN_EXPECTED_INCARNATION_SCHEMA), + "Idunn projection contains an Expected record under the wrong schema" + ); + let expected = IdunnExpectedIncarnationRecord::decode_canonical(&entry.payload)?; + ensure!( + entry.key == expected.target, + "Idunn Expected key is substituted" + ); + Ok(expected.target) + }) + .collect() +} + +fn correlation_targets(path: &Path) -> Result> { + if !path.is_file() { + return Ok(BTreeSet::new()); + } + SingleFileMessagePackBackingStore::new(path) + .pull_all_read_only_snapshot()? + .into_iter() + .filter(|entry| entry.r#type == OdinRuntimeTopologyCorrelationRecord::TYPE) + .map(|entry| { + ensure!( + entry.schema_id.as_deref() == Some(ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA), + "Odin correlation is stored under the wrong schema" + ); + let (record, _) = + OdinRuntimeTopologyCorrelationRecord::decode_canonical_signed_payload( + &entry.payload, + )?; + ensure!( + entry.key == record.target, + "Odin correlation key is substituted" + ); + Ok(record.target) + }) + .collect() +} + +fn validate_snapshot_filters(query: &CultMeshRudpSnapshotQuery) -> Result<()> { + ensure!( + !query.message_id.is_empty() && query.message_id.trim() == query.message_id, + "snapshot message id is empty or padded" + ); + for (label, values) in [ + ("schema", query.schema_ids.as_ref()), + ("record key", query.record_keys.as_ref()), + ] { + if let Some(values) = values { + ensure!(!values.is_empty(), "snapshot {label} filter is empty"); + let unique = values.iter().collect::>(); + ensure!( + unique.len() == values.len() + && values + .iter() + .all(|value| !value.is_empty() && value.trim() == value), + "snapshot {label} filter is duplicated, empty, or padded" + ); + } + } + Ok(()) +} + +fn exact_self_presence_query(query: &CultMeshRudpSnapshotQuery) -> bool { + query.schema_ids.as_deref() == Some(&[GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.to_owned()]) + && query.record_keys.as_deref() == Some(&[TARGET.to_owned()]) +} + +fn query_allows(query: &CultMeshRudpSnapshotQuery, document: &CultNetRawDocumentRecord) -> bool { + query + .schema_ids + .as_ref() + .is_none_or(|values| values.contains(&document.schema_id)) + && query + .record_keys + .as_ref() + .is_none_or(|values| values.contains(&document.record_key)) +} + +fn sibling_lock_path(path: &Path) -> Result { + let mut name = path + .file_name() + .context("CultCache authority path has no filename")? + .to_os_string(); + name.push(".lock"); + Ok(path.with_file_name(name)) +} + +fn required_environment(name: &str) -> Result { + std::env::var(name) + .ok() + .filter(|value| !value.is_empty() && value.trim() == value) + .with_context(|| format!("{name} is required for an Idunn-managed Odin")) +} + +fn unix_millis() -> Result { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .context("system clock predates Unix epoch")? + .as_millis() + .try_into() + .context("Unix time exceeds u64") +} + +fn rfc3339_millis(value: u64) -> Result { + let millis: i64 = value + .try_into() + .context("timestamp exceeds RFC3339 range")?; + Ok(DateTime::::from_timestamp_millis(millis) + .context("timestamp exceeds RFC3339 range")? + .to_rfc3339()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn options_are_exact_and_absolute() { + let parsed = parse_options( + [ + "--store", + "/var/lib/gamecult/odin-v2/topology.cc", + "--idunn-projection", + "/var/lib/gamecult/idunn-projection/topology.cc", + "--idunn-anchor", + "/etc/gamecult/idunn/idunn-public-anchor.cc", + ] + .into_iter() + .map(str::to_owned), + ) + .unwrap(); + assert_eq!( + parsed.store, + Path::new("/var/lib/gamecult/odin-v2/topology.cc") + ); + assert!( + parse_options( + [ + "--store", + "relative", + "--idunn-projection", + "/p", + "--idunn-anchor", + "/a" + ] + .into_iter() + .map(str::to_owned) + ) + .is_err() + ); + } + + #[test] + fn schema_to_document_type_is_versioned_and_deterministic() { + assert_eq!( + document_type_for_schema("heimdall.command_boundary.v1").unwrap(), + "heimdall.command_boundary" + ); + for value in ["heimdall.command_boundary", "heimdall.v", ".v1", " x.v1"] { + assert!(document_type_for_schema(value).is_err()); + } + } + + #[test] + fn only_exact_singleton_self_presence_query_is_a_route_challenge() { + let exact = CultMeshRudpSnapshotQuery { + session: cultmesh_rs::CultMeshRudpSessionKey { + remote_addr: "127.0.0.1:1".parse().unwrap(), + connection_id: 7, + }, + message_id: "challenge".into(), + requested_at_unix_millis: 1, + schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), + record_keys: Some(vec![TARGET.into()]), + }; + assert!(exact_self_presence_query(&exact)); + let mut broad = exact.clone(); + broad.record_keys = None; + assert!(!exact_self_presence_query(&broad)); + } + + #[test] + fn first_odin_accepts_only_recent_provider_owned_warming_proofs() { + let mut proofs = VecDeque::new(); + remember_recent_warming_proof(&mut proofs, "warming-one".into(), 100); + assert!(recent_warming_proof( + &proofs, + "warming-one", + 100 + WARMING_PROOF_LIFETIME_MILLIS, + )); + assert!(!recent_warming_proof( + &proofs, + "warming-one", + 101 + WARMING_PROOF_LIFETIME_MILLIS, + )); + assert!(!recent_warming_proof(&proofs, "foreign-proof", 100)); + + for index in 0..=MAX_RECENT_WARMING_PROOFS { + remember_recent_warming_proof(&mut proofs, format!("warming-{index}"), 200); + } + assert_eq!(proofs.len(), MAX_RECENT_WARMING_PROOFS); + assert_eq!(proofs.back().unwrap().0, "warming-64"); + assert!(!proofs.iter().any(|(proof, _)| proof == "warming-one")); + } +} diff --git a/deployment/idunn/recipe.toml b/deployment/idunn/recipe.toml new file mode 100644 index 0000000..f4455df --- /dev/null +++ b/deployment/idunn/recipe.toml @@ -0,0 +1,72 @@ +schema = "gamecult.idunn.target_declaration.v1" +target = "odin" +source_stamp_environment = "ODIN_BUILD_COMMIT" +required_gitlinks = [] + +[[steps]] +id = "test-odin-daemon" +phase = "test" +runner = "rust-build" +argv = ["cargo", "test", "--locked", "-p", "odin-daemon"] + +[[steps]] +id = "build-odin-daemon" +phase = "build" +runner = "rust-build" +argv = [ + "cargo", + "build", + "--locked", + "--release", + "-p", + "odin-daemon", + "--bin", + "odin-daemon", +] + +[[artifacts]] +id = "odin-daemon" +source_kind = "runner-output" +runner = "rust-build" +source = "target/release/odin-daemon" +destination = "odin-daemon" +executable = true + +[service] +executable_artifact = "odin-daemon" +arguments = [ + { kind = "literal", value = "--store" }, + { kind = "binding", name = "store_path" }, + { kind = "literal", value = "--idunn-projection" }, + { kind = "binding", name = "idunn_projection_path" }, + { kind = "literal", value = "--idunn-anchor" }, + { kind = "binding", name = "idunn_anchor_path" }, +] +transport = "rudp" +route_required = true +required_environment = [ + "GAMECULT_IDUNN_CANDIDATE_BIND", + "GAMECULT_IDUNN_PROCESS_WRITE_LEASE", + "GAMECULT_IDUNN_RUNTIME_BUNDLE", + "ODIN_TOPOLOGY_IDENTITY", +] + +[service.health] +contract = "odin.runtime-health.v1" + +[state] +schema_generation = "odin-v2" + +[[state.slots]] +id = "topology" +relative_path = "topology.cc" +kind = "cultcache-file" +schema = "odin.runtime_topology_correlation.v2" +writer = "process-bound-single-writer" +recovery = "preserve" +startup = "create-or-open-after-write-lease" + +[[provides]] +capability = "odin.verse-rendezvous" +schema = "odin.verse-topology.v1" +compatibility = "v1"