diff --git a/src/control_plane.rs b/src/control_plane.rs index ac38edf..7943698 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -37,7 +37,7 @@ use crate::deployment_plan::{ CompiledDeploymentPlan, DependencyProviderAuthority, SealedRelease, compile_deployment_plan, }; use crate::drivers::{ - CultCacheTopologyDriver, CultCacheWriteLeaseDriver, DockerRunnerDriver, FrozenSourceReceipt, + ChallengeFailure, CultCacheTopologyDriver, CultCacheWriteLeaseDriver, DockerRunnerDriver, FrozenSourceReceipt, GitSourceDriver, InstalledReleaseObservation, IsolationEvidence, NginxRouteDriver, ProcessIdentity, RouteObservation, RoutePreflightReceipt, RunnerPort, SourcePort, SystemdTransientWorkloadDriver, TopologyPort, WorkloadObservation, WorkloadPort, @@ -55,7 +55,7 @@ const DEPLOYMENT_TRANSACTION_SCHEMA_V2: &str = "idunn.deployment_transaction.v2" const ADMITTED_GENERATION_SCHEMA: &str = "idunn.admitted_generation.v3"; const ADMITTED_GENERATION_SCHEMA_V2: &str = "idunn.admitted_generation.v2"; /// The capability whose declaration makes a target Odin-correlated. -const ODIN_RENDEZVOUS_CAPABILITY: &str = "odin.verse-rendezvous"; +pub(crate) const ODIN_RENDEZVOUS_CAPABILITY: &str = "odin.verse-rendezvous"; /// How many times continuity will restart one admitted release before it /// concludes the release itself is the problem. More than one because a start /// can fail for a passing reason -- a port still held, a peer not yet up -- @@ -262,20 +262,34 @@ impl RuntimePresenceEvidence { enum WarmingEvidence { OdinTopology { evidence: TopologyEvidence }, FirstOdinDirect { evidence: RuntimePresenceEvidence }, + /// A route-proof target's candidate answered Idunn's challenge itself. + RouteProofDirect { evidence: RuntimePresenceEvidence }, } impl WarmingEvidence { fn validate_shape(&self) -> Result<()> { match self { Self::OdinTopology { evidence } => evidence.validate_shape(), - Self::FirstOdinDirect { evidence } => evidence.validate_shape(), + Self::FirstOdinDirect { evidence } | Self::RouteProofDirect { evidence } => { + evidence.validate_shape() + } } } - fn canonical_sha256(&self) -> &str { + fn voucher(&self) -> Voucher { match self { - Self::OdinTopology { evidence } => &evidence.canonical_sha256, - Self::FirstOdinDirect { evidence } => &evidence.canonical_sha256, + Self::OdinTopology { .. } | Self::FirstOdinDirect { .. } => Voucher::Odin, + Self::RouteProofDirect { .. } => Voucher::Candidate, + } + } + + /// The presence Idunn challenged for itself, when no Odin carried it. + fn direct(&self) -> Option<&RuntimePresenceEvidence> { + match self { + Self::OdinTopology { .. } => None, + Self::FirstOdinDirect { evidence } | Self::RouteProofDirect { evidence } => { + Some(evidence) + } } } } @@ -287,7 +301,6 @@ impl WarmingEvidence { #[derive(Clone, Debug)] pub struct SequenceAdmittedWarming { transaction_id: String, - evidence: WarmingEvidence, signed_presence_sha256: String, runtime_instance_id: String, } @@ -295,7 +308,6 @@ pub struct SequenceAdmittedWarming { impl SequenceAdmittedWarming { fn from_topology( transaction_id: String, - evidence: TopologyEvidence, authenticated: AuthenticatedOdinRuntimeTopologyCorrelation, ) -> Result { let record = authenticated.record(); @@ -309,27 +321,28 @@ impl SequenceAdmittedWarming { .context("Warming topology has no runtime instance")?; Ok(Self { transaction_id, - evidence: WarmingEvidence::OdinTopology { evidence }, signed_presence_sha256, runtime_instance_id, }) } - fn from_first_odin_presence( + fn from_direct_presence( transaction_id: String, - evidence: RuntimePresenceEvidence, + warming: WarmingEvidence, present: cultnet_rs::VerifiedRuntimePresence, ) -> Result { + let evidence = warming + .direct() + .context("direct Warming evidence is an Odin topology receipt")?; ensure!( present.canonical_bytes() == evidence.canonical_bytes && present.signed_presence_sha256() == evidence.canonical_sha256, - "direct first-Odin Warming evidence differs from its authenticated presence" + "direct Warming evidence differs from its authenticated presence" ); let runtime_instance_id = present.record().runtime_instance_id.clone(); Ok(Self { transaction_id, signed_presence_sha256: evidence.canonical_sha256.clone(), - evidence: WarmingEvidence::FirstOdinDirect { evidence }, runtime_instance_id, }) } @@ -338,7 +351,6 @@ impl SequenceAdmittedWarming { pub(crate) fn for_test( transaction_id: impl Into, authenticated: AuthenticatedOdinRuntimeTopologyCorrelation, - admitted_at_unix_millis: u64, ) -> Result { let record = authenticated.record(); ensure!( @@ -351,8 +363,6 @@ impl SequenceAdmittedWarming { ); let transaction_id = transaction_id.into(); require_id(&transaction_id, "test transaction id")?; - let evidence = - TopologyEvidence::from_authenticated(&authenticated, admitted_at_unix_millis)?; let signed_presence_sha256 = record .signed_presence_sha256 .clone() @@ -361,7 +371,7 @@ impl SequenceAdmittedWarming { .runtime_instance_id .clone() .context("test Warming receipt has no runtime instance")?; - let token = Self::from_topology(transaction_id, evidence, authenticated)?; + let token = Self::from_topology(transaction_id, authenticated)?; ensure!( token.signed_presence_sha256 == signed_presence_sha256 && token.runtime_instance_id == runtime_instance_id, @@ -374,10 +384,6 @@ impl SequenceAdmittedWarming { &self.transaction_id } - pub(crate) fn evidence_sha256(&self) -> &str { - self.evidence.canonical_sha256() - } - pub(crate) fn signed_presence_sha256(&self) -> &str { &self.signed_presence_sha256 } @@ -526,34 +532,219 @@ impl RoutingEvidence { } } -/// Which kind of proof admits a target as Ready. Derived from the target's -/// Expected, never stored: Expected is digest-bound, so the class cannot drift -/// from the incarnation it describes. A target that declares a -/// `shared-infrastructure odin.verse-rendezvous` dependency advertises into the -/// Verse and is Odin-correlated; one that declares none is route-proof. +/// The states a route-proof candidate may answer its Warming challenge in. A +/// stateful one is warming until it holds a lease it cannot hold yet, and the +/// lease binds that warming presence. A stateless one may have finished +/// warming before the first challenge landed. +fn route_proof_warming_states(expected: &IdunnExpectedIncarnationRecord) -> &'static [&'static str] { + if expected.write_lease_required { + &["warming"] + } else { + &["warming", "active"] + } +} + +/// A challenged runtime answered as something that disagrees with current +/// authority. Each disagreement is named, so a capacity below the Expected +/// minimum reads as that and not as a generic refusal. It stays the error's +/// typed value; callers read `disagreements`, and the text is only its +/// rendering. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct PresenceDisagrees { + pub(crate) disagreements: Vec, +} + +impl std::fmt::Display for PresenceDisagrees { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let named = self + .disagreements + .iter() + .map(|disagreement| { + format!( + "{} (expected {}, observed {})", + disagreement.code, + disagreement.expected.as_deref().unwrap_or("none"), + disagreement.observed.as_deref().unwrap_or("none"), + ) + }) + .collect::>() + .join("; "); + write!( + formatter, + "route proof answered with a runtime that disagrees with current authority: {named}" + ) + } +} + +impl std::error::Error for PresenceDisagrees {} + +/// What a direct challenge to a candidate endpoint produced. +enum CandidateAnswer { + /// No answer: the reason it is still being waited on. + Silent(String), + Answered { + evidence: RuntimePresenceEvidence, + present: cultnet_rs::VerifiedRuntimePresence, + }, +} + +//// Which kind of proof admits a target as Ready, derived from what the target's +/// recipe declares and from nothing else. Idunn infers no way to prove +/// readiness: a target that provides `odin.verse-rendezvous` is Odin itself, one +/// that declares a `shared-infrastructure odin.verse-rendezvous` dependency +/// advertises into the Verse and is Odin-correlated, and a routed one that +/// declares neither answers Idunn's own challenge and is route-proof. A target +/// with no route and no Odin dependency declares no proof at all, and is +/// refused. Odin collects the same receipts an Odin-correlated target does, and +/// its admitted generation is what names the Odin authority, so it is never +/// route-proof. +/// +/// `of` is the only place a class is decided, and Expected is digest-bound, so +/// the class cannot drift from the incarnation it describes. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum ReadinessClass { +pub(crate) enum ReadinessClass { + OdinSelf, OdinCorrelated, RouteProof, } +/// A recipe that declares neither a route nor an Odin dependency, and is not +/// Odin: it has said nothing about how it proves readiness. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct UndeclaredReadiness { + pub(crate) target: String, +} + +impl std::fmt::Display for UndeclaredReadiness { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + formatter, + "target {} declares no way to prove readiness: its recipe must declare a route \ + (Idunn challenges it) or a shared-infrastructure {ODIN_RENDEZVOUS_CAPABILITY} \ + dependency (Odin correlates it)", + self.target + ) + } +} + +impl std::error::Error for UndeclaredReadiness {} + impl ReadinessClass { - fn of(expected: &IdunnExpectedIncarnationRecord) -> Self { - let declares_odin = expected.dependencies.iter().any(|dependency| { - dependency.kind == "shared-infrastructure" - && dependency.capability == ODIN_RENDEZVOUS_CAPABILITY - }); - if declares_odin { - Self::OdinCorrelated + /// The class of a recipe's declarations. Plan compilation and `of` both + /// call it, so admission refuses exactly what `of` would refuse. + pub(crate) fn declared( + target: &str, + provides_odin: bool, + declares_odin: bool, + routed: bool, + ) -> Result { + if provides_odin { + Ok(Self::OdinSelf) + } else if declares_odin { + Ok(Self::OdinCorrelated) + } else if routed { + Ok(Self::RouteProof) } else { - Self::RouteProof + Err(UndeclaredReadiness { + target: target.to_owned(), + }) + } + } + + pub(crate) fn of( + expected: &IdunnExpectedIncarnationRecord, + ) -> Result { + Self::declared( + &expected.target, + expected + .capabilities + .iter() + .any(|capability| capability.capability == ODIN_RENDEZVOUS_CAPABILITY), + expected.dependencies.iter().any(|dependency| { + dependency.kind == "shared-infrastructure" + && dependency.capability == ODIN_RENDEZVOUS_CAPABILITY + }), + expected.route.is_some(), + ) + } + + /// The class of `expected`, provided every receipt collected under it was + /// vouched for by that class's voucher. The one way a stored record is + /// asked what class it is. + fn confirmed( + expected: &IdunnExpectedIncarnationRecord, + collected: impl IntoIterator, + ) -> Result { + let required = Self::of(expected).map_err(ReadinessDisagreement::Undeclared)?; + match collected + .into_iter() + .find(|voucher| *voucher != required.voucher()) + { + None => Ok(required), + Some(collected) => Err(ReadinessDisagreement::WrongVoucher { + target: expected.target.clone(), + required, + collected, + }), + } + } + + /// Who vouches for a target of this class: Odin's correlation, or the + /// candidate answering Idunn. + fn voucher(self) -> Voucher { + match self { + Self::OdinSelf | Self::OdinCorrelated => Voucher::Odin, + Self::RouteProof => Voucher::Candidate, } } } -/// The receipt that admitted a candidate as Ready, tagged by the class of proof -/// that produced it. The tag records what was collected; the class a target -/// must collect is `ReadinessClass::of(expected)`. +/// Who vouched for a piece of stored readiness evidence. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Voucher { + Odin, + Candidate, +} + +/// A stored record whose readiness evidence is not what its own Expected +/// requires. A record admitted before the target's recipe declared how it +/// proves readiness looks like this. It is reported and held, never repaired: +/// re-deriving the evidence around it would make the evidence tag a second +/// authority for the class. +#[derive(Clone, Debug, PartialEq, Eq)] +enum ReadinessDisagreement { + Undeclared(UndeclaredReadiness), + WrongVoucher { + target: String, + required: ReadinessClass, + collected: Voucher, + }, +} + +impl std::fmt::Display for ReadinessDisagreement { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Undeclared(undeclared) => write!(formatter, "{undeclared}"), + Self::WrongVoucher { + target, + required, + collected, + } => write!( + formatter, + "target {target} is {required:?} but its stored readiness evidence was vouched \ + for by {collected:?}: it was collected under a different class, and is held \ + until the target is redeployed" + ), + } + } +} + +impl std::error::Error for ReadinessDisagreement {} + +/// The receipt that admitted a candidate as Ready, tagged by who vouched for +/// it. The tag records what was collected; what a target must collect is +/// `ReadinessClass::of(expected)`, and a tag that differs is a +/// `ReadinessDisagreement`, never a second opinion about the class. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "class", rename_all = "kebab-case", deny_unknown_fields)] enum ReadinessEvidence { @@ -562,10 +753,10 @@ enum ReadinessEvidence { } impl ReadinessEvidence { - fn class(&self) -> ReadinessClass { + fn voucher(&self) -> Voucher { match self { - Self::OdinCorrelated { .. } => ReadinessClass::OdinCorrelated, - Self::RouteProof { .. } => ReadinessClass::RouteProof, + Self::OdinCorrelated { .. } => Voucher::Odin, + Self::RouteProof { .. } => Voucher::Candidate, } } @@ -584,7 +775,7 @@ impl ReadinessEvidence { } } -/// Signed evidence that the candidate process picked up the lease Idunn +// Signed evidence that the candidate process picked up the lease Idunn /// granted: its presence carries `write_lease_sha256` equal to the grant. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -1233,6 +1424,32 @@ impl DeploymentTransaction { .is_some_and(|cleanup| !cleanup.is_complete()) } + /// Where what this transaction has collected disagrees with what its own + /// Expected requires. There is nothing to disagree with before Expected. + fn readiness_disagreement(&self) -> Option { + let expected = self.expected.as_ref()?; + ReadinessClass::confirmed( + expected, + self.warming + .iter() + .map(WarmingEvidence::voucher) + .chain(self.ready.iter().map(ReadinessEvidence::voucher)), + ) + .err() + } + + /// The disagreement that holds this transaction: evidence collected under + /// another class is held and reported, not advanced and not repaired, since + /// no step can make it the right kind and running one anyway only fails it + /// every tick. What is finished, or being finished (a completion, or an + /// abort under way), never reads the class, so it is left to run. + fn held_disagreement(&self) -> Option { + let finishing = self.completion.is_some() + || self.pre_fencing_abort.is_some() + || self.post_fencing_abort.is_some(); + self.readiness_disagreement().filter(|_| !finishing) + } + fn validate(&self) -> Result<()> { ensure!( self.schema_version == DEPLOYMENT_TRANSACTION_SCHEMA, @@ -1316,10 +1533,15 @@ impl DeploymentTransaction { } if let Some(ready) = &self.ready { ready.validate_shape()?; - if let (ReadinessClass::RouteProof, Some(expected)) = (ready.class(), &self.expected) { + // A candidate-vouched receipt is only ever written by this code, so + // one on a target that is not route-proof is forged. The other + // direction -- an Odin receipt on a route-proof target -- is what a + // pre-B3 record looks like, and is a ReadinessDisagreement to hold, + // not a record to refuse: refusing it would refuse to boot. + if let (Voucher::Candidate, Some(expected)) = (ready.voucher(), &self.expected) { ensure!( - ReadinessClass::of(expected) == ReadinessClass::RouteProof, - "route-proof Ready evidence for a target that declares Odin" + ReadinessClass::of(expected).is_ok_and(|class| class == ReadinessClass::RouteProof), + "route-proof Ready evidence for a target that is not route-proof" ); } } @@ -1331,9 +1553,17 @@ impl DeploymentTransaction { "Warming topology evidence exceeds the transaction replay cursor" ), WarmingEvidence::FirstOdinDirect { .. } => ensure!( - self.target == "odin", + self.expected.as_ref().is_some_and(|expected| { + ReadinessClass::of(expected).is_ok_and(|class| class == ReadinessClass::OdinSelf) + }), "direct Warming evidence is reserved for Odin observing itself" ), + WarmingEvidence::RouteProofDirect { .. } => ensure!( + self.expected.as_ref().is_none_or(|expected| { + ReadinessClass::of(expected).is_ok_and(|class| class == ReadinessClass::RouteProof) + }), + "direct route-proof Warming evidence for a target that is not route-proof" + ), } } if let Some(leasing) = &self.leasing { @@ -1768,6 +1998,12 @@ struct OdinReceipts<'a> { } impl AdmittedGeneration { + /// This generation's class, or where its evidence disagrees with it. The + /// class comes from its Expected, never from the tag on its Ready receipt. + fn readiness(&self) -> Result { + ReadinessClass::confirmed(&self.expected, [self.ready.voucher()]) + } + fn odin_receipts(&self) -> Result> { match ( &self.ready, @@ -1786,7 +2022,7 @@ impl AdmittedGeneration { fn from_transaction( transaction: &DeploymentTransaction, - odin_authority: AdmittedOdinAuthority, + odin_authority: Option, incumbent: Option<&AdmittedGeneration>, now: u64, ) -> Result { @@ -1795,15 +2031,15 @@ impl AdmittedGeneration { "only Committing can create an admitted generation" ); let ready = required(&transaction.ready, "Ready receipt")?.clone(); - let (latest_odin_observation, odin_authority) = match ready.class() { - ReadinessClass::OdinCorrelated => ( + let (latest_odin_observation, odin_authority) = match &ready { + ReadinessEvidence::OdinCorrelated { .. } => ( Some( required(&transaction.latest_odin_observation, "latest Odin receipt")? .clone(), ), - Some(odin_authority), + Some(required(&odin_authority, "Odin authority")?.clone()), ), - ReadinessClass::RouteProof => (None, None), + ReadinessEvidence::RouteProof { .. } => (None, None), }; let routing = required(&transaction.routing, "route disposition")?.clone(); let route_supervision = matches!(&routing, RoutingEvidence::Promoted { .. }).then(|| { @@ -1871,8 +2107,8 @@ impl AdmittedGeneration { ); } (ReadinessEvidence::RouteProof { .. }, None, None) => ensure!( - ReadinessClass::of(&self.expected) == ReadinessClass::RouteProof, - "route-proof Ready evidence for a target that declares Odin" + ReadinessClass::of(&self.expected).is_ok_and(|class| class == ReadinessClass::RouteProof), + "route-proof Ready evidence for a target that is not route-proof" ), _ => bail!("admitted readiness evidence and Odin receipts disagree"), } @@ -2211,8 +2447,8 @@ fn parse_cancel(mut args: impl Iterator) -> Result { }) } -/// Withdraw queued work, or ask Idunn to clean up a live stateless candidate. -/// Stateful candidates past fencing cannot be cancelled because their prior +/// Withdraw queued work, a held deployment that has not fenced, or ask Idunn to +/// clean up a live stateless candidate. Stateful candidates past fencing cannot be cancelled because their prior /// writer may already be stopped and its state boundary needs explicit repair. fn cancel(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> { let snapshot = ControlSnapshot::read(store_path)?; @@ -2235,12 +2471,17 @@ fn cancel(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> "deployment command has multiple live transactions" ); ensure!( - cancel_is_safe_for_live_stateless(¤t.value), - "live deployment can only be cancelled after fencing when its candidate is stateless" + cancel_is_safe_for_live_stateless(¤t.value) + || cancel_is_safe_for_held_prefence(¤t.value), + "live deployment can only be cancelled before fencing when Idunn holds it, or after fencing when its candidate is stateless" ); let error = format!("cancelled by {requested_by}"); let mut next = current.value.clone(); - next.post_fencing_abort = Some(post_fencing_abort_intent(&next, &error)); + if cancel_is_safe_for_held_prefence(&next) { + next.pre_fencing_abort = Some(pre_fencing_abort_intent(&next, &error)); + } else { + next.post_fencing_abort = Some(post_fencing_abort_intent(&next, &error)); + } next.last_error = Some(error); next.updated_at_unix_millis = now_millis()?; replace_transaction(store_path, current, &next)?; @@ -2277,6 +2518,14 @@ fn cancel(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> Ok(()) } +/// A deployment Idunn holds before it fenced anything has touched nothing of +/// the incumbent, so an operator may withdraw it. Idunn never does so itself. +fn cancel_is_safe_for_held_prefence(transaction: &DeploymentTransaction) -> bool { + transaction.command_kind == CommandKind::Deploy + && transaction.phase < DeploymentPhase::Fencing + && transaction.held_disagreement().is_some() +} + fn cancel_is_safe_for_live_stateless(transaction: &DeploymentTransaction) -> bool { transaction.command_kind == CommandKind::Deploy && transaction.phase >= DeploymentPhase::Fencing @@ -2481,6 +2730,16 @@ impl ControlSnapshot { .find(|stored| stored.value.target == target) } + /// The admitted generation of the target that provides + /// `odin.verse-rendezvous`: Odin is whichever target says so, not + /// whichever is named "odin". + fn admitted_odin(&self) -> Option<&Stored> { + self.admitted.iter().find(|stored| { + ReadinessClass::of(&stored.value.expected) + .is_ok_and(|class| class == ReadinessClass::OdinSelf) + }) + } + fn transaction_for_command(&self, command_id: &str) -> Vec<&DeploymentTransaction> { let mut values = self .transactions @@ -3596,7 +3855,7 @@ struct Engine { idunn_signer: ServiceIdentitySigner, idunn_anchor: ServiceIdentityTrustAnchor, bootstrap_odin_authority: AdmittedOdinAuthority, - source: GitSourceDriver, + source: Arc, docker_runner: DockerRunnerDriver, systemd_workload: Arc, host_actuators: Option, @@ -3641,11 +3900,11 @@ impl Engine { let odin_anchor = read_trust_anchor::(&options.odin_trust_anchor) .context("reading bootstrap Odin topology anchor")?; let bootstrap_odin_authority = AdmittedOdinAuthority::from_anchor(&odin_anchor)?; - let source = GitSourceDriver::new( + let source: Arc = Arc::new(GitSourceDriver::new( &options.source_root, options.staging_root.join("frozen-sources"), options.source_identity, - ); + )); let host_actuators = options .host_actuator_bind .map(|bind| HostActuatorHub::bind(bind).map(|hub| Arc::new(Mutex::new(hub)))) @@ -3784,7 +4043,7 @@ impl Engine { // is, its own generation names the authority; an admitted Odin that // carries none has no authority to name, and falling back to the // bootstrap key would let a stale key vouch for the Verse. - let authority = match snapshot.admitted_for("odin") { + let authority = match snapshot.admitted_odin() { None => self.bootstrap_odin_authority.clone(), Some(stored) => stored.value.odin_authority.clone().with_context(|| { "admitted Odin generation carries no Odin authority (route-proof readiness)" @@ -3890,8 +4149,29 @@ impl Engine { &evidence.canonical_bytes, )?; } + WarmingEvidence::RouteProofDirect { evidence } => { + self.reauthenticate_route_proof( + transaction, + evidence, + route_proof_warming_states(required( + &transaction.expected, + "Warming Expected projection", + )?), + None, + evidence.admitted_at_unix_millis, + )?; + } } } + if let Some(ReadinessEvidence::RouteProof { evidence }) = &transaction.ready { + self.reauthenticate_route_proof( + transaction, + evidence, + &["active"], + lease, + evidence.admitted_at_unix_millis, + )?; + } if let Some(evidence) = transaction.ready.as_ref().and_then(ReadinessEvidence::odin) { let authenticated = self.authenticate_topology_bytes( snapshot, @@ -3910,7 +4190,11 @@ impl Engine { let odin_authority = self.current_odin_authority(snapshot)?; for stored in &snapshot.admitted { let generation = &stored.value; - if generation.ready.class() == ReadinessClass::RouteProof { + // Only a generation whose own Expected says Odin vouches for it has + // Odin receipts to re-prove. One whose evidence disagrees with its + // class is held and reported by supervision, not re-proved here. + if !matches!(generation.readiness(), Ok(class) if class != ReadinessClass::RouteProof) + { continue; } let odin = generation.odin_receipts()?; @@ -4271,6 +4555,10 @@ impl Engine { { return Ok(false); } + if let Some(disagreement) = current.value.held_disagreement() { + self.note_fault("holds transaction", id, &anyhow::Error::new(disagreement)); + return Ok(false); + } let advanced = self.advance_transaction(current); if advanced.is_ok() { // The step ran and succeeded: the fault, if any, has recovered, so @@ -4502,6 +4790,13 @@ impl Engine { let mut admitted = snapshot.admitted.iter().collect::>(); admitted.sort_by_key(|stored| stored.value.target.as_str()); for current in admitted { + if let Err(disagreement) = current.value.readiness() { + self.note_fault( + "holds admitted generation", + &format!("generation:{}", current.value.target), + &anyhow::Error::new(disagreement), + ); + } let blocker = snapshot.transactions.iter().find(|stored| { stored.value.target == current.value.target && stored.value.blocks_new_target_mutation() @@ -4663,6 +4958,21 @@ impl Engine { } let workload_error = operational_error .context("admitted operational state has neither observation nor error")?; + // A held generation is a record the recipe never declared + // readiness for. Idunn reports it and does not decide for it: a + // continuity minted over it copies the undeclared Expected, is + // held at once, and owns the target forever, so the declaring + // redeploy that would clear the hold could never freeze. The dead + // held target stays down and free. Nothing yields to it either: + // there is no continuity to yield to. + if current.value.readiness().is_err() { + self.note_fault( + "holds a dead admitted generation down; declare readiness in its recipe and redeploy", + &format!("generation-down:{}", current.value.target), + &anyhow!("admitted generation {} is not running", current.value.generation_id), + ); + continue; + } // Continuity restarts a release; it cannot repair one. When the // admitted release will not start, rescheduling it forever keeps // the target permanently occupied -- and a target with a live @@ -4719,6 +5029,8 @@ impl Engine { && refused_restarts < CONTINUITY_RESTART_ATTEMPTS && blocker.value.phase < DeploymentPhase::Fencing && blocker.value.pre_fencing_abort.is_none() + // A held record is reported and never aborted by Idunn. + && blocker.value.held_disagreement().is_none() { self.begin_pre_fencing_abort( blocker, @@ -4962,7 +5274,7 @@ impl Engine { snapshot: &ControlSnapshot, current: &Stored, ) -> Result { - if current.value.ready.class() == ReadinessClass::RouteProof { + if !matches!(current.value.readiness(), Ok(class) if class != ReadinessClass::RouteProof) { return Ok(false); } let odin = current.value.odin_receipts()?; @@ -5072,6 +5384,9 @@ impl Engine { now, &providers, )?; + // Idunn infers no way to prove readiness. A recipe that declares + // none is refused here, before anything is frozen, built or installed. + plan.readiness_class()?; return self.persist_same_phase(current, |next| { next.plan = Some(plan); next.updated_at_unix_millis = now; @@ -5253,6 +5568,31 @@ impl Engine { let snapshot = ControlSnapshot::read(&self.options.state_store)?; let incumbent_lease_sha256 = self.incumbent_lease_sha256_for_warming(&snapshot, ¤t.value)?; + let class = ReadinessClass::of(expected)?; + if current.value.warming.is_none() && class == ReadinessClass::RouteProof { + // A route-proof target's warming is Idunn's own challenge to the + // candidate endpoint. No Odin is read: the target declared no + // reason to be aware of one. + return match self.challenge_candidate( + ¤t.value, + route_proof_warming_states(expected), + None, + )? { + CandidateAnswer::Silent(reason) => self.record_gate_wait(current, &reason), + CandidateAnswer::Answered { evidence, present } => { + let warming = WarmingEvidence::RouteProofDirect { evidence }; + let _token = SequenceAdmittedWarming::from_direct_presence( + current.value.transaction_id.clone(), + warming.clone(), + present, + )?; + self.persist_same_phase(current, |next| { + next.warming = Some(warming); + Ok(()) + }) + } + }; + } if current.value.warming.is_none() { // Odin's warming is observed directly when nothing else can observe // it. That is the first bootstrap, and it is also every continuity @@ -5271,29 +5611,30 @@ impl Engine { // Asked once. Admitting the latest topology persists the sequence // cursor, so asking twice with the same record fails the second // time as "transaction changed before topology admission". - let odin_observation = if current.value.target == "odin" - && snapshot.admitted_for("odin").is_some() + let is_odin = class == ReadinessClass::OdinSelf; + let odin_observation = if is_odin + && snapshot.admitted_odin().is_some() && current.value.command_kind != CommandKind::Continuity { self.admit_latest_topology(current, None)? } else { None }; - let odin_can_observe = current.value.target != "odin" || odin_observation.is_some(); - let odin_observes_itself = current.value.target == "odin" && !odin_can_observe; + let odin_observes_itself = is_odin && odin_observation.is_none(); if odin_observes_itself { ensure!( expected.write_lease_required, "direct Odin warming must be a stateful incarnation" ); let (evidence, present) = self.observe_first_odin_warming(¤t.value)?; - let _token = SequenceAdmittedWarming::from_first_odin_presence( + let warming = WarmingEvidence::FirstOdinDirect { evidence }; + let _token = SequenceAdmittedWarming::from_direct_presence( current.value.transaction_id.clone(), - evidence.clone(), + warming.clone(), present, )?; return self.persist_same_phase(current, |next| { - next.warming = Some(WarmingEvidence::FirstOdinDirect { evidence }); + next.warming = Some(warming); Ok(()) }); } @@ -5330,7 +5671,6 @@ impl Engine { let evidence = latest.clone(); let _token = SequenceAdmittedWarming::from_topology( admitted.value.transaction_id.clone(), - evidence.clone(), authenticated, )?; return self.persist_same_phase(&admitted, |next| { @@ -5680,6 +6020,34 @@ impl Engine { "candidate process write lease is no longer exact" ); } + if ReadinessClass::of(expected)? == ReadinessClass::RouteProof { + // Ready is the candidate answering Idunn's own challenge Active, + // holding the exact lease Idunn granted. Once recorded it is + // history; the stable route's challenge is what keeps proving it. + if matches!( + current.value.ready, + Some(ReadinessEvidence::RouteProof { .. }) + ) { + return self.transition(current, DeploymentPhase::Routing); + } + let lease = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256); + return match self.challenge_candidate(¤t.value, &["warming", "active"], lease)? { + CandidateAnswer::Silent(reason) => self.record_gate_wait(current, &reason), + CandidateAnswer::Answered { present, .. } if present.record().state == "warming" => { + self.record_gate_wait(current, "candidate is still warming") + } + CandidateAnswer::Answered { evidence, .. } => { + self.persist_same_phase(current, |next| { + next.ready = Some(ReadinessEvidence::RouteProof { evidence }); + Ok(()) + }) + } + }; + } { let current_lease = current .value @@ -5725,39 +6093,50 @@ impl Engine { let activation = required(¤t.value.activation, "activation")?; self.observe_candidate_before_waiting(¤t.value)?; if current.value.routing.is_none() { - let current_lease = current - .value - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256); - let Some((admitted, authenticated)) = - self.admit_latest_topology(current, current_lease)? - else { - return Ok(()); + let admitted = if ReadinessClass::of(expected)? == ReadinessClass::RouteProof { + // Ready is the candidate's own proof, recorded in AwaitingReady. + // Validation refuses Routing without a Ready receipt, and a + // receipt the other party vouched for is held before any step + // runs. No Odin correlation is read to admit a route. + current.clone() + } else { + let current_lease = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256); + let Some((admitted, authenticated)) = + self.admit_latest_topology(current, current_lease)? + else { + return Ok(()); + }; + if admitted.envelope != current.envelope { + return Ok(()); + } + ensure!( + is_semantic_ready(&authenticated), + "latest Odin observation is not Ready at route admission" + ); + let latest = required( + &admitted.value.latest_odin_observation, + "current route-admission topology evidence", + )?; + if admitted.value.ready.as_ref().and_then(ReadinessEvidence::odin) + != Some(latest) + { + let latest = latest.clone(); + return self.persist_same_phase(&admitted, |next| { + next.ready = Some(ReadinessEvidence::OdinCorrelated { evidence: latest }); + Ok(()) + }); + } + let ready = self.rehydrate_ready_token(&admitted.value, now_millis()?, true)?; + ensure!( + ready.transaction_id() == admitted.value.transaction_id, + "Ready token belongs to another transaction" + ); + admitted }; - if admitted.envelope != current.envelope { - return Ok(()); - } - ensure!( - is_semantic_ready(&authenticated), - "latest Odin observation is not Ready at route admission" - ); - let latest = required( - &admitted.value.latest_odin_observation, - "current route-admission topology evidence", - )?; - if admitted.value.ready.as_ref().and_then(ReadinessEvidence::odin) != Some(latest) { - let latest = latest.clone(); - return self.persist_same_phase(&admitted, |next| { - next.ready = Some(ReadinessEvidence::OdinCorrelated { evidence: latest }); - Ok(()) - }); - } - let ready = self.rehydrate_ready_token(&admitted.value, now_millis()?, true)?; - ensure!( - ready.transaction_id() == admitted.value.transaction_id, - "Ready token belongs to another transaction" - ); validate_live_providers_for_deploy(admitted.value.command_kind, || { self.validate_selected_providers_current(required( &admitted.value.plan, @@ -5834,32 +6213,43 @@ impl Engine { fn advance_committing(&self, current: &Stored) -> Result<()> { self.observe_candidate_before_waiting(¤t.value)?; - let current_lease_sha256 = current - .value - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256) - .map(str::to_owned); - let Some((ready_current, authenticated)) = - self.admit_latest_topology(current, current_lease_sha256.as_deref())? - else { - return Ok(()); + let route_proof = ReadinessClass::of(required( + ¤t.value.expected, + "Expected projection", + )?)? == ReadinessClass::RouteProof; + let ready_current = if route_proof { + // The candidate's Ready proof is durable, and the stable route's + // challenge below is its currency. No Odin is read to commit. + current.clone() + } else { + let current_lease_sha256 = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256) + .map(str::to_owned); + let Some((ready_current, authenticated)) = + self.admit_latest_topology(current, current_lease_sha256.as_deref())? + else { + return Ok(()); + }; + if ready_current.envelope != current.envelope { + return Ok(()); + } + ensure!( + is_semantic_ready(&authenticated), + "latest Odin observation is not Ready at admission commit" + ); + // Not require_current: the Ready receipt is durable evidence, so + // authenticating it against `now` asks a stored record to be live and + // refuses it once it ages past the 30s observation window. Route + // admission (:3906) refreshes the receipt to the latest observation and + // can therefore demand currency; by Committing the receipt is history. + // Currency here is answered by the latest observation, authenticated + // and checked for semantic readiness immediately above. + self.rehydrate_ready_token(&ready_current.value, now_millis()?, false)?; + ready_current }; - if ready_current.envelope != current.envelope { - return Ok(()); - } - ensure!( - is_semantic_ready(&authenticated), - "latest Odin observation is not Ready at admission commit" - ); - // Not require_current: the Ready receipt is durable evidence, so - // authenticating it against `now` asks a stored record to be live and - // refuses it once it ages past the 30s observation window. Route - // admission (:3906) refreshes the receipt to the latest observation and - // can therefore demand currency; by Committing the receipt is history. - // Currency here is answered by the latest observation, authenticated - // and checked for semantic readiness immediately above. - self.rehydrate_ready_token(&ready_current.value, now_millis()?, false)?; validate_live_providers_for_deploy(ready_current.value.command_kind, || { self.validate_selected_providers_current(required( &ready_current.value.plan, @@ -5909,26 +6299,31 @@ impl Engine { self.ensure_transaction_write_lease_current(&ready_current.value, now_millis()?)?; } - let current_lease_sha256 = ready_current - .value - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256) - .map(str::to_owned); - let Some((commit_current, authenticated)) = - self.admit_latest_topology(&ready_current, current_lease_sha256.as_deref())? - else { - return Ok(()); + let commit_current = if route_proof { + ready_current + } else { + let current_lease_sha256 = ready_current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256) + .map(str::to_owned); + let Some((commit_current, authenticated)) = + self.admit_latest_topology(&ready_current, current_lease_sha256.as_deref())? + else { + return Ok(()); + }; + if commit_current.envelope != ready_current.envelope { + return Ok(()); + } + ensure!( + is_semantic_ready(&authenticated), + "latest Odin observation is not Ready after the final admission challenge" + ); + self.rehydrate_ready_token(&commit_current.value, now_millis()?, false)?; + commit_current }; - if commit_current.envelope != ready_current.envelope { - return Ok(()); - } - ensure!( - is_semantic_ready(&authenticated), - "latest Odin observation is not Ready after the final admission challenge" - ); let now = now_millis()?; - self.rehydrate_ready_token(&commit_current.value, now, false)?; validate_live_providers_for_deploy(commit_current.value.command_kind, || { self.validate_selected_providers_current(required( &commit_current.value.plan, @@ -5945,7 +6340,9 @@ impl Engine { let snapshot = ControlSnapshot::read(&self.options.state_store)?; let incumbent = self.exact_incumbent(&snapshot, &commit_current.value)?; - let odin_authority = self.current_odin_authority(&snapshot)?; + let odin_authority = (!route_proof) + .then(|| self.current_odin_authority(&snapshot)) + .transpose()?; let generation = AdmittedGeneration::from_transaction( &commit_current.value, odin_authority, @@ -6064,25 +6461,41 @@ impl Engine { // with the narrower "no admitted Odin" test contradicted the // caller and failed the refresh. What the refresh owes is // freshness, and the replay check below is what provides it. - ensure!( - current.value.target == "odin", - "direct Warming refresh is reserved for Odin" - ); + // That the target is Odin was already required by the + // rehydration above, which authenticates the recorded answer. let (evidence, present) = self.observe_first_odin_warming(¤t.value)?; - let token = SequenceAdmittedWarming::from_first_odin_presence( + let warming = WarmingEvidence::FirstOdinDirect { evidence }; + let token = SequenceAdmittedWarming::from_direct_presence( current.value.transaction_id.clone(), - evidence.clone(), + warming.clone(), present, )?; ensure!( token.signed_presence_sha256() != prior.signed_presence_sha256(), "first Odin replayed its pre-fence Warming presence" ); - Ok(Some(( - current.clone(), - WarmingEvidence::FirstOdinDirect { evidence }, - token, - ))) + Ok(Some((current.clone(), warming, token))) + } + WarmingEvidence::RouteProofDirect { .. } => { + // A stateful route-proof candidate is warming until it holds + // the lease this refresh is about to bind, so the fresh + // presence must be warming again, and must be a new one. + let CandidateAnswer::Answered { evidence, present } = + self.challenge_candidate(¤t.value, &["warming"], None)? + else { + return Ok(None); + }; + let warming = WarmingEvidence::RouteProofDirect { evidence }; + let token = SequenceAdmittedWarming::from_direct_presence( + current.value.transaction_id.clone(), + warming.clone(), + present, + )?; + ensure!( + token.signed_presence_sha256() != prior.signed_presence_sha256(), + "route-proof candidate replayed its pre-fence Warming presence" + ); + Ok(Some((current.clone(), warming, token))) } WarmingEvidence::OdinTopology { evidence: prior_evidence, @@ -6109,7 +6522,6 @@ impl Engine { } let token = SequenceAdmittedWarming::from_topology( admitted.value.transaction_id.clone(), - fresh_evidence.clone(), authenticated, )?; if !provider_warming_advanced( @@ -6421,6 +6833,7 @@ impl Engine { ) } + /// The stable route's proof: a challenged presence that is Active. fn authenticate_routed_presence( &self, authority: &cultnet_rs::VerifiedRuntimeAuthority, @@ -6430,9 +6843,41 @@ impl Engine { received_at_unix_millis: u64, canonical_presence: &[u8], ) -> Result<(String, u64)> { + let present = self.authenticate_challenged_presence( + authority, + &["active"], + current_write_lease_sha256, + message_id, + challenged_at_unix_millis, + received_at_unix_millis, + canonical_presence, + )?; + Ok(( + present.signed_presence_sha256().to_owned(), + received_at_unix_millis, + )) + } + + /// A presence Idunn challenged for itself, from the stable endpoint or the + /// candidate's: signed by the provider and the launch's activation key, + /// bound to current authority, minted after the challenge, answering that + /// exact challenge, in one of `states`. A warming presence holds no write + /// lease; a presence in any other state holds exactly the current one. + /// Every disagreement with authority is named, so a capacity below the + /// Expected minimum reads as that and not as a generic refusal. + fn authenticate_challenged_presence( + &self, + authority: &cultnet_rs::VerifiedRuntimeAuthority, + states: &[&str], + current_write_lease_sha256: Option<&str>, + message_id: &str, + challenged_at_unix_millis: u64, + received_at_unix_millis: u64, + canonical_presence: &[u8], + ) -> Result { ensure!( received_at_unix_millis >= challenged_at_unix_millis, - "route observation predates its challenge" + "route proof predates its challenge" ); let authenticated = authenticate_runtime_presence_claim( canonical_presence, @@ -6443,31 +6888,39 @@ impl Engine { maximum_future_skew_millis: self.options.topology_maximum_future_skew_millis, }, )?; - let signed_presence_sha256 = authenticated.signed_presence_sha256().to_owned(); let correlation = correlate_runtime_presence_claim(authenticated, authority)?; - ensure!( - correlation.disagreements().is_empty(), - "stable route answered with a runtime that disagrees with current authority" - ); + if !correlation.disagreements().is_empty() { + return Err(PresenceDisagrees { + disagreements: correlation.disagreements().to_vec(), + } + .into()); + } let present = correlation.into_undisputed_present()?; let presence = present.record(); ensure!( presence.observed_at_unix_millis >= challenged_at_unix_millis, - "stable route returned a presence minted before the route challenge" + "route proof returned a presence minted before its challenge" ); ensure!( - presence.state == "active", - "stable route runtime is not Active" + states.contains(&presence.state.as_str()), + "route proof runtime is {}, not {}", + presence.state, + states.join(" or ") ); ensure!( presence.detail == format!("route-observation:{message_id}"), - "stable route response is not bound to the exact challenge" + "route proof response is not bound to the exact challenge" ); + // A warming presence is exempt from the lease comparison: it is + // Idunn's own lease that a warming candidate cannot hold yet, and the + // authenticator above already refuses one that claims a lease. + let lease_is_current = presence.state == "warming" + || presence.write_lease_sha256.as_deref() == current_write_lease_sha256; ensure!( - presence.write_lease_sha256.as_deref() == current_write_lease_sha256, - "stable route runtime does not hold the exact current process write lease" + lease_is_current, + "route proof runtime does not hold the exact current process write lease" ); - Ok((signed_presence_sha256, received_at_unix_millis)) + Ok(present) } fn authenticate_first_odin_warming_presence( @@ -6484,9 +6937,10 @@ impl Engine { // Odin ever been admitted" test only contradicted that caller. What is // still checked is what this function itself depends on -- the target // is Odin, and the incarnation is stateful. + let expected = required(&transaction.expected, "Odin Expected")?; ensure!( - transaction.target == "odin" - && required(&transaction.expected, "Odin Expected")?.write_lease_required, + ReadinessClass::of(expected)? == ReadinessClass::OdinSelf + && expected.write_lease_required, "direct Warming presence is reserved for a stateful Odin incarnation" ); ensure!( @@ -6784,7 +7238,6 @@ impl Engine { ); SequenceAdmittedWarming::from_topology( transaction.transaction_id.clone(), - evidence, authenticated, ) } @@ -6800,15 +7253,109 @@ impl Engine { }, &evidence.canonical_bytes, )?; - SequenceAdmittedWarming::from_first_odin_presence( + SequenceAdmittedWarming::from_direct_presence( transaction.transaction_id.clone(), - evidence, + WarmingEvidence::FirstOdinDirect { evidence }, + present, + ) + } + WarmingEvidence::RouteProofDirect { evidence } => { + let present = self.reauthenticate_route_proof( + transaction, + &evidence, + route_proof_warming_states(required( + &transaction.expected, + "Warming Expected projection", + )?), + None, + if require_current { + now + } else { + evidence.admitted_at_unix_millis + }, + )?; + SequenceAdmittedWarming::from_direct_presence( + transaction.transaction_id.clone(), + WarmingEvidence::RouteProofDirect { evidence }, present, ) } } } + /// A route-proof presence Idunn recorded, authenticated again as of `at`. + fn reauthenticate_route_proof( + &self, + transaction: &DeploymentTransaction, + evidence: &RuntimePresenceEvidence, + states: &[&str], + current_write_lease_sha256: Option<&str>, + at_unix_millis: u64, + ) -> Result { + let authority = self.runtime_authority(transaction)?; + self.authenticate_challenged_presence( + &authority, + states, + current_write_lease_sha256, + &evidence.message_id, + evidence.challenged_at_unix_millis, + at_unix_millis, + &evidence.canonical_bytes, + ) + } + + /// One direct challenge to a route-proof target's candidate endpoint. + /// Failing to reach or hear the candidate is only silence: a process that + /// is still starting does not answer, and that is waiting, not a fault. + /// What does answer is authenticated, and a bad answer is an error, as is + /// a challenge Idunn could not make: those stay a `ChallengeFailure` in the + /// error's chain. + fn challenge_candidate( + &self, + transaction: &DeploymentTransaction, + states: &[&str], + current_write_lease_sha256: Option<&str>, + ) -> Result { + let expected = required(&transaction.expected, "candidate Expected")?; + let binding = required(&transaction.plan, "candidate plan")? + .parsed_inputs()? + .1; + let driver = NginxRouteDriver::new( + binding + .route + .context("route-proof candidate has no route binding")?, + ); + let message_id = format!("candidate-{}", Uuid::new_v4().simple()); + let challenged_at_unix_millis = now_millis()?; + let response = match driver.request_candidate_runtime_presence(expected, &message_id) { + Ok(response) => response, + Err(ChallengeFailure::Silent(error)) => { + return Ok(CandidateAnswer::Silent(format!( + "candidate endpoint did not answer its challenge: {error:#}" + ))); + } + Err(bad @ ChallengeFailure::Refused(_)) => return Err(bad.into()), + }; + let admitted_at_unix_millis = now_millis()?; + let authority = self.runtime_authority(transaction)?; + let present = self.authenticate_challenged_presence( + &authority, + states, + current_write_lease_sha256, + &message_id, + challenged_at_unix_millis, + admitted_at_unix_millis, + &response.canonical_presence, + )?; + let evidence = RuntimePresenceEvidence::from_present( + &present, + message_id, + challenged_at_unix_millis, + admitted_at_unix_millis, + )?; + Ok(CandidateAnswer::Answered { evidence, present }) + } + fn rehydrate_ready_token( &self, transaction: &DeploymentTransaction, @@ -7048,19 +7595,7 @@ impl Engine { current.value.pre_fencing_abort.is_none(), "pre-fencing abort intent is already durable" ); - let abort = PreFencingAbort { - error: truncate(&format!("{error:#}"), 2048), - candidate_cleanup: candidate_cleanup_requirement( - current.value.activation.is_some(), - current.value.workload.is_some(), - ), - topology_reconciliation: current.value.abort_topology_reconciliation(), - source_cleanup: if current.value.command_kind == CommandKind::Deploy { - CleanupEvidence::Pending - } else { - CleanupEvidence::Skipped - }, - }; + let abort = pre_fencing_abort_intent(¤t.value, &format!("{error:#}")); self.persist_same_phase(current, |next| { next.pre_fencing_abort = Some(abort); Ok(()) @@ -7453,6 +7988,22 @@ fn incumbent_was_stopped_during_fencing(fencing: &FencingEvidence) -> bool { ) } +fn pre_fencing_abort_intent(transaction: &DeploymentTransaction, error: &str) -> PreFencingAbort { + PreFencingAbort { + error: truncate(error, 2048), + candidate_cleanup: candidate_cleanup_requirement( + transaction.activation.is_some(), + transaction.workload.is_some(), + ), + topology_reconciliation: transaction.abort_topology_reconciliation(), + source_cleanup: if transaction.command_kind == CommandKind::Deploy { + CleanupEvidence::Pending + } else { + CleanupEvidence::Skipped + }, + } +} + fn post_fencing_abort_intent( transaction: &DeploymentTransaction, error: &str, @@ -9671,7 +10222,7 @@ mod tests { lifted.odin_publisher_sequence_cursor, legacy.odin_publisher_sequence_cursor ); - assert_eq!(lifted.ready.class(), ReadinessClass::OdinCorrelated); + assert_eq!(lifted.ready.voucher(), Voucher::Odin); // The repair intent still decides whatever it decided. assert_eq!( lifted.route_repair_started_at_unix_millis, @@ -9838,18 +10389,92 @@ mod tests { dependency.kind == "shared-infrastructure" && dependency.capability == ODIN_RENDEZVOUS_CAPABILITY })); - assert_eq!(ReadinessClass::of(&expected), ReadinessClass::OdinCorrelated); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinCorrelated)); for dependency in &mut expected.dependencies { // Same capability, other kind: only the shared-infrastructure // declaration makes a target Odin-correlated. dependency.kind = "required".into(); } - assert_eq!(ReadinessClass::of(&expected), ReadinessClass::RouteProof); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::RouteProof)); expected.dependencies.clear(); - assert_eq!(ReadinessClass::of(&expected), ReadinessClass::RouteProof); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::RouteProof)); + // Nothing to challenge and no Odin dependency: the recipe has declared + // no way to prove readiness, and Idunn does not pick one. + expected.route = None; + assert_eq!( + ReadinessClass::of(&expected), + Err(UndeclaredReadiness { + target: expected.target.clone() + }) + ); + // Declaring the Odin dependency is the declaration, routed or not: a + // routed target that only publishes to Odin is Odin-correlated. + let odin_dependency = generation.expected.dependencies.clone(); + expected.dependencies = odin_dependency.clone(); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinCorrelated)); + expected.route = generation.expected.route.clone(); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinCorrelated)); + // Providing the rendezvous is being Odin, whatever else is declared. + expected.dependencies.clear(); + expected.route = None; + provide_odin(&mut expected); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinSelf)); Ok(()) } + #[test] + fn warming_evidence_is_shape_checked_whatever_its_source() { + let good = |evidence: &ReadinessEvidence| match evidence { + ReadinessEvidence::RouteProof { evidence } => evidence.clone(), + ReadinessEvidence::OdinCorrelated { .. } => unreachable!("a route-proof fixture"), + }; + let presence = good(&route_proof_evidence()); + for warming in [ + WarmingEvidence::FirstOdinDirect { evidence: presence.clone() }, + WarmingEvidence::RouteProofDirect { evidence: presence.clone() }, + WarmingEvidence::OdinTopology { evidence: topology(3, 1) }, + ] { + warming.validate_shape().expect("well-formed evidence"); + } + let mut damaged = presence; + damaged.canonical_bytes.push(1); + for warming in [ + WarmingEvidence::FirstOdinDirect { evidence: damaged.clone() }, + WarmingEvidence::RouteProofDirect { evidence: damaged }, + WarmingEvidence::OdinTopology { + evidence: TopologyEvidence { + canonical_sha256: "sha256:not-the-digest".into(), + ..topology(3, 1) + }, + }, + ] { + assert!(warming.validate_shape().is_err(), "{warming:?}"); + } + } + + #[test] + fn a_presence_disagreement_names_each_disagreement() { + let shortfall = PresenceDisagrees { + disagreements: vec![ + OdinTopologyDisagreement { + code: "capacity-below-minimum".into(), + expected: Some("2".into()), + observed: Some("1".into()), + }, + OdinTopologyDisagreement { + code: "state".into(), + expected: None, + observed: None, + }, + ], + }; + assert_eq!( + shortfall.to_string(), + "route proof answered with a runtime that disagrees with current authority: \ + capacity-below-minimum (expected 2, observed 1); state (expected none, observed none)" + ); + } + fn route_proof_evidence() -> ReadinessEvidence { let canonical_bytes = vec![7, 7, 7]; ReadinessEvidence::RouteProof { @@ -9931,7 +10556,7 @@ mod tests { .context("fixture generation has no Odin authority")?; let next = AdmittedGeneration::from_transaction( &committing, - authority.clone(), + Some(authority.clone()), Some(&incumbent), 1_700_000_200_000, )?; @@ -9944,7 +10569,7 @@ mod tests { let first = AdmittedGeneration::from_transaction( &committing, - authority, + Some(authority), None, 1_700_000_200_000, )?; @@ -10181,6 +10806,37 @@ mod tests { }) } + /// What makes a target Odin: it provides the rendezvous. + fn provide_odin(expected: &mut IdunnExpectedIncarnationRecord) { + expected.capabilities.push(cultnet_rs::IdunnExpectedCapability { + capability: ODIN_RENDEZVOUS_CAPABILITY.into(), + schema: "odin.verse-topology.v1".into(), + compatibility: "v1".into(), + minimum_capacity: 1, + }); + } + + /// The declaration that makes an unrouted fixture target Odin-correlated. + /// Idunn infers no way to prove readiness, and a target with no route has + /// nothing for Idunn to challenge, so the world says outright that Odin + /// reports on it. The plan selects no provider: these worlds never plan + /// against Odin, they only read its correlation, which names the provider + /// Expected does. + fn declare_odin(expected: &mut IdunnExpectedIncarnationRecord) { + expected.dependencies.push(cultnet_rs::IdunnExpectedDependency { + kind: "shared-infrastructure".into(), + capability: ODIN_RENDEZVOUS_CAPABILITY.into(), + schema: "odin.verse-topology.v1".into(), + compatibility: "v1".into(), + minimum_capacity: 1, + startup: "before-promotion".into(), + provider_id: Some("odin".into()), + provider_authority: Some("managed-incarnation".into()), + provider_expected_projection_sha256: Some(sha256_id(b"odin expected projection")), + provider_endpoint: None, + }); + } + /// A transaction for an unrouted, stateless target, sitting at `phase` /// (Warming or later) with every earlier phase's evidence in place. The /// plan, sealed release, Expected, activation and Warming evidence are the @@ -10266,8 +10922,9 @@ mod tests { (plan, release) } }; - let expected = release.expected_projection(&plan)?; + let mut expected = release.expected_projection(&plan)?; assert!(!expected.write_lease_required && expected.route.is_none()); + declare_odin(&mut expected); let now = now_millis()?; let command = DeploymentCommand { @@ -10414,7 +11071,25 @@ mod tests { runtime_instance_id: Some(activation.runtime_instance_id.clone()), present: true, ready, - dependencies: Vec::new(), + dependencies: expected + .dependencies + .iter() + .map(|dependency| cultnet_rs::OdinRuntimeDependencyEvidence { + kind: dependency.kind.clone(), + capability: dependency.capability.clone(), + schema: dependency.schema.clone(), + compatibility: dependency.compatibility.clone(), + provider_id: dependency.provider_id.clone(), + provider_authority: dependency.provider_authority.clone(), + provider_expected_projection_sha256: dependency + .provider_expected_projection_sha256 + .clone(), + provider_endpoint: dependency.provider_endpoint.clone(), + observed_capacity: Some(dependency.minimum_capacity), + provider_evidence_sha256: Some(sha256_id(b"odin provider evidence")), + ready: true, + }) + .collect(), disagreements: Vec::new(), signer_identity_id: world.odin_signer.entry().identity_id.clone(), publisher_sequence: sequence, @@ -10688,6 +11363,10 @@ mod tests { let (_, mut odin) = fixture_generation(FIXTURE_GENERATION)?; odin.target = "odin".into(); + // Odin is the target that provides the rendezvous, whatever it is named. + odin.expected.target = "odin".into(); + odin.expected.dependencies.clear(); + provide_odin(&mut odin.expected); odin.odin_authority = Some(AdmittedOdinAuthority::from_anchor( &world.odin_signer.trust_anchor()?, )?); @@ -10734,33 +11413,13 @@ mod tests { /// A route-proof generation from a real admission: the world's Expected /// declares no Odin dependency, so route proof is its class. fn route_proof_generation() -> Result { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; - odin_reports_ready(&world, &seeded, 5)?; - drive(&world, |transaction| { - transaction.phase == DeploymentPhase::Committing - })?; - let mut committing = resident(&world)?.value; - assert_eq!( - ReadinessClass::of(committing.expected.as_ref().unwrap()), - ReadinessClass::RouteProof - ); - committing.ready = Some(route_proof_evidence()); - committing.latest_odin_observation = None; - let generation = AdmittedGeneration::from_transaction( - &committing, - world.engine.bootstrap_odin_authority.clone(), - None, - now_millis()?, - )?; - generation.validate()?; - Ok(generation) + route_proof::committed_generation() } #[test] fn a_route_proof_generation_is_admitted_only_in_its_exact_shape() -> Result<()> { let generation = route_proof_generation()?; - assert_eq!(generation.ready.class(), ReadinessClass::RouteProof); + assert_eq!(generation.ready.voucher(), Voucher::Candidate); assert!(generation.odin_authority.is_none()); assert!(generation.latest_odin_observation.is_none()); assert!(generation.odin_receipts().is_err()); @@ -10926,7 +11585,7 @@ mod tests { assert_eq!(snapshot.admitted.len(), 1); let lifted = &snapshot.admitted[0].value; assert_eq!(lifted.target, envelope.key); - assert_eq!(lifted.ready.class(), ReadinessClass::OdinCorrelated); + assert_eq!(lifted.ready.voucher(), Voucher::Odin); assert!(snapshot.admitted_for(&envelope.key).is_some()); // The lifted record is held to the same key rule as a current one. @@ -11357,6 +12016,51 @@ mod tests { .clone()) } + #[test] + fn an_admitted_odin_correlated_generation_takes_a_newer_odin_reading_once() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; + let first = seeded_sequence(None) + 1; + odin_reports_ready(&world, &seeded, first)?; + drive(&world, |transaction| transaction.phase == DeploymentPhase::Complete)?; + assert!(world.engine.retire_one_terminal_transaction()?); + let admitted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("commit wrote no admitted generation")? + .value + .clone(); + assert_eq!(admitted.readiness(), Ok(ReadinessClass::OdinCorrelated)); + assert_eq!(admitted.odin_publisher_sequence_cursor, first); + + // Nothing newer than what is held: nothing is written. + let snapshot = ControlSnapshot::read(&world.state_store)?; + let current = snapshot.admitted_for("service").context("no generation")?; + let envelope = current.envelope.clone(); + assert!(!world.engine.refresh_admitted_topology(&snapshot, current)?); + assert_eq!(incumbent_envelope(&world)?, envelope); + + // A newer reading is adopted, once. + odin_republishes(&world, &seeded, first + 4)?; + let snapshot = ControlSnapshot::read(&world.state_store)?; + let current = snapshot.admitted_for("service").context("no generation")?; + assert!(world.engine.refresh_admitted_topology(&snapshot, current)?); + let refreshed = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("no generation")? + .value + .clone(); + assert_eq!(refreshed.odin_publisher_sequence_cursor, first + 4); + assert_eq!( + refreshed.latest_odin_observation.as_ref().map(|latest| latest.publisher_sequence), + Some(first + 4) + ); + assert_eq!(refreshed.ready.odin().map(|ready| ready.publisher_sequence), Some(first + 4)); + let snapshot = ControlSnapshot::read(&world.state_store)?; + let current = snapshot.admitted_for("service").context("no generation")?; + assert!(!world.engine.refresh_admitted_topology(&snapshot, current)?); + Ok(()) + } + /// Change the incumbent in the store, compare-and-swap. fn edit_incumbent( world: &EngineFixture, @@ -12922,4 +13626,2124 @@ mod tests { assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); Ok(()) } + + /// A source that resolves one fixed recipe and never freezes anything. + struct FixedSource { + recipe: String, + } + + impl SourcePort for FixedSource { + fn resolve( + &self, + _binding: &OperatorBinding, + _resolution_id: &str, + _selected_at_unix_millis: u64, + ) -> Result { + Ok(crate::drivers::ResolvedSource { + facts: crate::deployment_plan::tests::source(&self.recipe), + recipe_bytes: self.recipe.clone().into_bytes(), + }) + } + + fn freeze(&self, _: &str, _: &CompiledDeploymentPlan) -> Result { + bail!("this source freezes nothing") + } + + fn observe_frozen( + &self, + _: &CompiledDeploymentPlan, + _: &FrozenSourceReceipt, + ) -> Result { + bail!("this source freezes nothing") + } + + fn cleanup(&self, _: &str, _: Option<&FrozenSourceReceipt>) -> Result<()> { + Ok(()) + } + } + + /// A Deploy transaction for an unrouted target, still Sealing, whose + /// source resolves `recipe`. One scheduler tick admits its plan or refuses it. + fn sealing_world(recipe: &str) -> Result { + use crate::deployment_plan::tests::BINDING; + let mut world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (binding_head, binding_tail) = BINDING + .split_once("[route]") + .context("binding has no route table")?; + let binding = format!( + "{binding_head}[brakes]{}", + binding_tail + .split_once("[brakes]") + .context("binding has no brakes table")? + .1 + ); + std::fs::create_dir_all(world.root.join("bindings"))?; + std::fs::write(world.root.join("bindings/service.toml"), binding)?; + world.engine.source = Arc::new(FixedSource { + recipe: recipe.to_owned(), + }); + let now = now_millis()?; + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "up-service".into(), + kind: CommandKind::Deploy, + selector: "service".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + let transaction = DeploymentTransaction::new(&command, "service".into(), 0, None, now)?; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: transaction.transaction_id.clone(), + current: None, + }, + ], + &[ + command_envelope(&command, command.requested_at_unix_millis)?, + transaction_envelope(&transaction, now)?, + ], + )? + ); + Ok(world) + } + + fn unrouted_recipe() -> String { + crate::deployment_plan::tests::RECIPE + .split("[[dependencies]]") + .next() + .expect("the recipe has a body") + .replace("route_required = true", "route_required = false") + .replace( + r#"["GAMECULT_IDUNN_CANDIDATE_BIND", "GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, + r#"["GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, + ) + } + + #[test] + fn admission_refuses_a_recipe_that_declares_no_way_to_prove_readiness() -> Result<()> { + let recipe = unrouted_recipe(); + let world = sealing_world(&recipe)?; + world.engine.run_scheduler_tick()?; + let refused = latest(&world)?; + assert!(refused.plan.is_none(), "the refused recipe was planned"); + assert!(refused.frozen_source.is_none(), "the refused recipe was frozen"); + let abort = refused + .pre_fencing_abort + .context("a recipe with no declared proof of readiness must be refused")?; + assert!( + abort.error.contains("target service declares no way to prove readiness"), + "{}", + abort.error + ); + Ok(()) + } + + #[test] + fn admission_takes_a_recipe_that_declares_its_proof() -> Result<()> { + let recipe = unrouted_recipe().replace( + "capability = \"service.runtime\"", + &format!("capability = \"{ODIN_RENDEZVOUS_CAPABILITY}\""), + ); + let world = sealing_world(&recipe)?; + world.engine.run_scheduler_tick()?; + let planned = latest(&world)?; + assert!(planned.pre_fencing_abort.is_none(), "{:?}", planned.pre_fencing_abort); + assert_eq!( + planned.plan.as_ref().map(CompiledDeploymentPlan::readiness_class).transpose()?, + Some(ReadinessClass::OdinSelf) + ); + Ok(()) + } + + // --------------------------------------------------------------------- + // A held record is reported and never decided for. Supervision mints no + // continuity over a held generation, and nothing yields to a held record. + // --------------------------------------------------------------------- + + /// Rewrite the world's admitted generation into what a pre-B3 binary left + /// behind: no Odin declaration, admitted on Odin receipts. + fn undeclare_incumbent(world: &EngineFixture) -> Result { + let now = now_millis()?; + let mut failure = None; + let next = edit_incumbent(world, |generation| { + generation.expected.dependencies.clear(); + match IdunnRuntimeActivationLaunch::issue( + &generation.expected, + generation.activation.runtime_instance_id.clone(), + now, + &world.engine.idunn_signer, + ) { + Ok(launch) => generation.activation = launch.activation().clone(), + Err(error) => failure = Some(error), + } + })?; + if let Some(error) = failure { + return Err(error); + } + ControlSnapshot::read(&world.state_store).context("store with the undeclared generation")?; + assert!(matches!(next.readiness(), Err(ReadinessDisagreement::Undeclared(_)))); + Ok(next) + } + + /// Queue the operator's declaring redeploy of `service`. + fn queue_declaring_redeploy(world: &EngineFixture, command_id: &str) -> Result<()> { + use crate::deployment_plan::tests::BINDING; + let (binding_head, binding_tail) = BINDING.split_once("[route]").context("route")?; + let binding = format!( + "{binding_head}[brakes]{}", + binding_tail.split_once("[brakes]").context("brakes")?.1 + ); + std::fs::create_dir_all(world.root.join("bindings"))?; + std::fs::write(world.root.join("bindings/service.toml"), binding)?; + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: command_id.into(), + kind: CommandKind::Deploy, + selector: "service".into(), + requested_by: "operator".into(), + requested_at_unix_millis: now_millis()?, + }; + assert!(SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }], + &[command_envelope(&command, command.requested_at_unix_millis)?], + )?); + Ok(()) + } + + /// Every transaction the store knows: live, then archived. + fn every_transaction(world: &EngineFixture) -> Result> { + let mut all = ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .map(|stored| stored.value) + .collect::>(); + all.extend(read_history_transactions(&world.state_store)); + Ok(all) + } + + fn is_continuity_over(transaction: &DeploymentTransaction, held: &AdmittedGeneration) -> bool { + transaction.command_kind == CommandKind::Continuity + && transaction.incumbent_generation_id.as_deref() == Some(held.generation_id.as_str()) + } + + #[test] + fn a_held_generation_that_dies_mints_no_continuity_and_leaves_its_target_free() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + let held = undeclare_incumbent(&world)?; + world.engine.validate_durable_authority(&ControlSnapshot::read(&world.state_store)?)?; + + workload.kill(); + for _ in 0..4 { + assert!(!world.engine.supervise_one_admitted_generation()?); + } + assert!( + ControlSnapshot::read(&world.state_store)?.transactions.is_empty(), + "supervision minted a transaction over a held generation" + ); + // Reported, once per state change: the same text is not offered again. + let reports = world.engine.fault_reports.lock().unwrap(); + let down = reports.get("generation-down:service").context("the dead hold is not reported")?; + let last = down.last.lock().unwrap().clone(); + assert!(last.is_some()); + assert!(down.offer(last).is_none()); + drop(reports); + + // The target is free: the declaring redeploy freezes and is not + // displaced by a continuity. + queue_declaring_redeploy(&world, "up-service-declared")?; + for _ in 0..8 { + let _ = world.engine.run_scheduler_tick(); + } + let all = every_transaction(&world)?; + assert!( + all.iter().all(|transaction| !is_continuity_over(transaction, &held)), + "a continuity was minted over the held generation" + ); + assert!( + all.iter().any(|transaction| transaction.command_id == "up-service-declared"), + "the declaring redeploy never froze" + ); + Ok(()) + } + + #[test] + fn a_declaring_redeploy_is_not_yielded_when_the_held_incumbent_dies() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + let held = undeclare_incumbent(&world)?; + let redeploy = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&held))?; + assert_eq!(redeploy.held_disagreement(), None); + workload.kill(); + for _ in 0..4 { + let _ = world.engine.supervise_one_admitted_generation()?; + } + let all = every_transaction(&world)?; + assert!( + all.iter().all(|transaction| !is_continuity_over(transaction, &held)), + "a continuity was minted over the held incumbent" + ); + let after = resident(&world)?.value; + assert_eq!(after.transaction_id, redeploy.transaction_id); + assert!( + after.pre_fencing_abort.is_none(), + "the redeploy yielded: {:?}", + after.pre_fencing_abort + ); + Ok(()) + } + + /// The control for the test above: the same redeploy over a dead incumbent + /// that is not held is yielded to continuity. + #[test] + fn a_declaring_redeploy_still_yields_to_continuity_when_the_incumbent_is_not_held() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + assert!(incumbent.readiness().is_ok()); + let redeploy = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + workload.kill(); + world.engine.supervise_one_admitted_generation()?; + let after = resident(&world)?.value; + assert_eq!(after.transaction_id, redeploy.transaction_id); + let abort = after.pre_fencing_abort.context("the redeploy was not yielded")?; + assert!(abort.error.contains("yielded to continuity"), "{}", abort.error); + Ok(()) + } + + /// `seeded` with its own Expected undeclared, so Idunn holds it. + fn undeclared(world: &EngineFixture, seeded: &DeploymentTransaction) -> Result { + let mut bare = seeded.clone(); + // Undeclare the Expected and re-derive what is bound to its digest. + let expected = bare.expected.as_mut().context("no expected")?; + expected.dependencies.clear(); + bare.expected_publication_sha256 = Some(expected.canonical_sha256()?); + let activation = IdunnRuntimeActivationLaunch::issue( + expected, + seeded.activation.as_ref().context("no activation")?.runtime_instance_id.clone(), + now_millis()?, + &world.engine.idunn_signer, + )? + .activation() + .clone(); + bare.activation_publication_sha256 = Some(activation.canonical_sha256()?); + bare.activation = Some(activation); + bare.validate()?; + assert!(bare.held_disagreement().is_some()); + Ok(bare) + } + + /// A pre-fence Deploy whose own Expected is undeclared, stored in place. + fn held_predeploy( + world: &EngineFixture, + incumbent: &AdmittedGeneration, + ) -> Result { + let seeded = + seeded_transaction(world, DeploymentPhase::Warming, CommandKind::Deploy, Some(incumbent))?; + let bare = undeclared(world, &seeded)?; + replace_transaction(&world.state_store, &resident(world)?, &bare)?; + Ok(bare) + } + + #[test] + fn a_held_predeploy_transaction_is_not_aborted_to_yield_to_continuity() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + // The incumbent is declared; the Deploy in front of it is the held one. + let held = held_predeploy(&world, &incumbent)?; + + workload.kill(); + for _ in 0..4 { + let _ = world.engine.supervise_one_admitted_generation()?; + } + let after = resident(&world)?.value; + assert_eq!(after.transaction_id, held.transaction_id); + assert!(after.pre_fencing_abort.is_none(), "Idunn aborted a held record"); + assert!(after.completion.is_none()); + Ok(()) + } + + #[test] + fn cancel_withdraws_a_held_predeploy_and_leaves_the_incumbent_untouched() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + let held = held_predeploy(&world, &incumbent)?; + let before = incumbent_envelope(&world)?; + + cancel(&world.state_store, &held.command_id, "operator")?; + let cancelling = resident(&world)?.value; + assert_eq!( + cancelling.pre_fencing_abort.as_ref().map(|abort| abort.error.as_str()), + Some("cancelled by operator") + ); + assert!(cancelling.post_fencing_abort.is_none()); + + drive(&world, |transaction| transaction.completion.is_some())?; + // History also holds the incumbent's own admission: find ours by id. + let finished = every_transaction(&world)? + .into_iter() + .find(|transaction| transaction.transaction_id == held.transaction_id) + .context("the cancelled redeploy is nowhere")?; + assert!(matches!( + finished.completion, + Some(TransactionCompletion::FailedBeforeFencing { .. }) + )); + assert!(finished.is_terminal()); + // The target is released and nothing of the incumbent was touched. + assert!(!finished.blocks_new_target_mutation()); + assert_eq!(incumbent_envelope(&world)?, before); + assert!( + every_transaction(&world)? + .iter() + .all(|transaction| !is_continuity_over(transaction, &incumbent)), + "cancelling the redeploy minted a continuity" + ); + Ok(()) + } + + #[test] + fn cancel_refuses_a_predeploy_that_is_not_held_and_a_held_one_past_the_fence() -> Result<()> { + let world = EngineFixture::with_workload(SwitchWorkload::new())?; + let incumbent = admit_incumbent(&world)?; + let seeded = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + assert!(seeded.held_disagreement().is_none()); + assert!(cancel(&world.state_store, &seeded.command_id, "operator").is_err()); + assert!(resident(&world)?.value.pre_fencing_abort.is_none()); + + let mut past_the_fence = undeclared(&world, &seeded)?; + past_the_fence.enter_phase(DeploymentPhase::Fencing, now_millis()?); + assert!(past_the_fence.held_disagreement().is_some()); + assert!(!cancel_is_safe_for_held_prefence(&past_the_fence)); + Ok(()) + } + + // --------------------------------------------------------------------- + // Route-proof readiness: a routed target that declares no Odin dependency + // is admitted by Idunn's own challenges. Odin is never read. + // --------------------------------------------------------------------- + mod route_proof { + use super::*; + use std::io::{Read, Write}; + use std::net::{TcpListener, TcpStream}; + use std::sync::Mutex; + use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering}; + use std::time::Duration; + + /// The runtime behind both endpoints, answering Idunn's snapshot + /// challenge the way the TypeScript signer does: locally, in the + /// reported health, with `route-observation:` as the detail. + struct RuntimeStub { + expected: IdunnExpectedIncarnationRecord, + activation: IdunnRuntimeActivationRecord, + provider: cultnet_rs::ServiceIdentitySigner, + activation_signer: cultnet_rs::IdunnRuntimeActivationSigner, + state: Mutex<&'static str>, + capacity: AtomicU32, + sequence: AtomicU64, + candidate_hits: AtomicUsize, + stable_hits: AtomicUsize, + stop: AtomicBool, + /// Accept the connection and close it without a word: a process + /// that is up but not answering. Unlike a closed port, no other + /// test's listener can be handed the address. + hang_up: AtomicBool, + /// The write lease its presence claims to hold: what a candidate + /// that picked up Idunn's grant reports. + lease: Mutex>, + reply: Mutex, + } + + /// How the runtime answers a well-formed challenge. + #[derive(Clone, Copy, PartialEq, Eq)] + enum Reply { + Honest, + /// An answer to some other challenge. + ForeignChallengeId, + /// An HTTP error where the snapshot should be. + HttpFailure, + /// A first Odin candidate answering its Warming challenge: the + /// detail names the challenge as `idunn-warming:`. + FirstOdinWarming, + } + + impl RuntimeStub { + fn answer(&self, message_id: &str) -> Result> { + let expected = &self.expected; + let mut presence = cultnet_rs::GameCultRuntimePresenceHealthRecord { + schema_version: cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + target: expected.target.clone(), + expected_projection_sha256: self.activation.expected_projection_sha256.clone(), + plan_id: expected.plan_id.clone(), + incarnation_id: expected.incarnation_id.clone(), + sealed_release_id: expected.sealed_release_id.clone(), + activation_witness_sha256: self.activation.canonical_sha256()?, + state_schema_generation: expected.state_schema_generation.clone(), + state_contract_sha256: expected.state_contract_sha256.clone(), + runtime_id: expected.runtime_id.clone(), + runtime_instance_id: self.activation.runtime_instance_id.clone(), + bound_endpoint: expected + .route + .as_ref() + .map(|route| route.candidate_endpoint.clone()), + capabilities: expected + .capabilities + .iter() + .map(|capability| cultnet_rs::GameCultRuntimeCapability { + capability: capability.capability.clone(), + schema: capability.schema.clone(), + compatibility: capability.compatibility.clone(), + capacity: self.capacity.load(Ordering::SeqCst), + }) + .collect(), + health_contract: expected.health_contract.clone(), + state: (*self.state.lock().unwrap()).into(), + detail: match *self.reply.lock().unwrap() { + Reply::FirstOdinWarming => format!("idunn-warming:{message_id}"), + _ => format!("route-observation:{message_id}"), + }, + write_lease_sha256: self.lease.lock().unwrap().clone(), + signer_identity_id: self.provider.entry().identity_id.clone(), + publisher_sequence: self.sequence.fetch_add(1, Ordering::SeqCst) + 1, + observed_at_unix_millis: now_millis()?, + signature_algorithm: "ed25519".into(), + signature: Vec::new(), + activation_signer_identity_id: self.activation_signer.identity_id(), + activation_signature: Vec::new(), + }; + let proof = presence.canonical_proof_payload()?; + presence.signature = self + .provider + .sign::(&proof) + .signature; + presence.activation_signature = + self.activation_signer.sign_presence_proof(&presence)?; + Ok(rmp_serde::to_vec(&presence)?) + } + + fn set_state(&self, state: &'static str) { + *self.state.lock().unwrap() = state; + } + + fn serve_one(&self, mut stream: TcpStream) -> Result<()> { + stream.set_read_timeout(Some(Duration::from_secs(3)))?; + let mut request = Vec::new(); + let mut chunk = [0_u8; 4096]; + let (header_end, length) = loop { + let read = stream.read(&mut chunk)?; + ensure!(read > 0, "stub client closed early"); + request.extend_from_slice(&chunk[..read]); + if let Some(index) = request.windows(4).position(|w| w == b"\r\n\r\n") { + let head = String::from_utf8_lossy(&request[..index]).to_lowercase(); + let length = head + .lines() + .find_map(|line| line.strip_prefix("content-length:")) + .context("stub request has no Content-Length")? + .trim() + .parse::()?; + if request.len() >= index + 4 + length { + break (index + 4, length); + } + } + }; + let message = cultnet_rs::decode_cultnet_message_from_slice( + &request[header_end..header_end + length], + cultnet_rs::CultNetWireContract::CultNetSchemaV0, + )?; + let cultnet_rs::CultNetMessage::SnapshotRequest { message_id, .. } = message + else { + bail!("stub received something other than a snapshot request"); + }; + let reply = *self.reply.lock().unwrap(); + if reply == Reply::HttpFailure { + stream.write_all( + b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + )?; + return Ok(()); + } + let response = cultnet_rs::CultNetMessage::SnapshotResponseRaw { + message_id: if reply == Reply::ForeignChallengeId { + "candidate-foreign".to_owned() + } else { + message_id.clone() + }, + documents: vec![cultnet_rs::CultNetRawDocumentRecord { + schema_id: cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + record_key: self.expected.target.clone(), + stored_at: rfc3339_millis(now_millis()?)?, + payload_encoding: cultnet_rs::CultNetRawPayloadEncoding::Messagepack, + payload: self.answer(&message_id)?, + source_runtime_id: None, + source_agent_id: None, + source_role: None, + tags: None, + }], + }; + let body = cultnet_rs::encode_cultnet_message_to_vec( + &response, + cultnet_rs::CultNetWireContract::CultNetSchemaV0, + )?; + stream.write_all( + format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/msgpack\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .as_bytes(), + )?; + stream.write_all(&body)?; + Ok(()) + } + + fn listen(self: &Arc, listener: TcpListener, candidate: bool) { + listener.set_nonblocking(true).expect("nonblocking listener"); + let stub = Arc::clone(self); + std::thread::spawn(move || { + while !stub.stop.load(Ordering::SeqCst) { + match listener.accept() { + Ok((stream, _)) => { + stream.set_nonblocking(false).ok(); + let hits = if candidate { + &stub.candidate_hits + } else { + &stub.stable_hits + }; + hits.fetch_add(1, Ordering::SeqCst); + if !stub.hang_up.load(Ordering::SeqCst) { + let _ = stub.serve_one(stream); + } + } + Err(_) => std::thread::sleep(Duration::from_millis(5)), + } + } + }); + } + } + + impl Drop for RuntimeStub { + fn drop(&mut self) { + self.stop.store(true, Ordering::SeqCst); + } + } + + /// Odin, as the world it runs in sees it. + #[derive(Clone, Copy, PartialEq, Eq)] + enum Odin { + /// Every read of the correlation store fails. + Unreachable, + /// Odin publishes a correlation that says the target is not Ready. + NotReady, + } + + struct RoutedWorld { + world: EngineFixture, + stub: Arc, + odin: Odin, + poisoned: Vec, + } + + /// A stateless, routed target that declares no Odin dependency, over + /// `minimum_capacity`, seeded at `phase` (Warming or Fencing): its + /// Warming evidence, when the phase needs it, is the candidate's own + /// answer to a real challenge. Listeners stand at both endpoints. + fn routed_world( + odin: Odin, + minimum_capacity: u32, + phase: DeploymentPhase, + provides_odin: bool, + ) -> Result { + build_routed_world(odin, minimum_capacity, phase, provides_odin, false) + } + + /// The same world for a target with process-writable state: its + /// candidate is granted a write lease and Ready must be the candidate + /// reporting exactly that lease. + fn stateful_routed_world(odin: Odin, phase: DeploymentPhase) -> Result { + build_routed_world(odin, 1, phase, false, true) + } + + fn build_routed_world( + odin: Odin, + minimum_capacity: u32, + phase: DeploymentPhase, + provides_odin: bool, + stateful: bool, + ) -> Result { + use crate::deployment_plan::tests::{ + BINDING, RECIPE, artifact_receipt, external_input_receipt, source, + }; + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let candidate_listener = TcpListener::bind("127.0.0.1:0")?; + let stable_listener = TcpListener::bind("127.0.0.1:0")?; + let candidate_port = candidate_listener.local_addr()?.port(); + let stable_port = stable_listener.local_addr()?.port(); + + let provider_path = world.root.join("identities/provider.cc"); + let provider_anchor = world.root.join("identities/provider-anchor.cc"); + let provider = + enroll_service_identity_at::(&provider_path)?; + export_service_identity_trust_anchor(&provider, &provider_anchor)?; + let config = world.root.join("service.conf"); + let binding = BINDING + .replace( + "/etc/gamecult/trust/service.cc", + &provider_anchor.display().to_string(), + ) + .replace("service-runtime-signer", &provider.entry().identity_id) + .replace("127.0.0.1:17999", &format!("127.0.0.1:{stable_port}")) + .replace( + "private_port_start = 18000", + &format!("private_port_start = {candidate_port}"), + ) + .replace( + "private_port_end = 18009", + &format!("private_port_end = {}", candidate_port + 1), + ) + .replace( + "/etc/nginx/idunn-stream-routes/service.conf", + &config.display().to_string().replace('\\', "/"), + ); + let (binding, recipe_source) = if stateful { + std::fs::create_dir_all(world.root.join("lease"))?; + ( + binding.replace( + "[brakes]", + &format!( + "[process_write_lease]\nrecord_path = \"{}\"\n\n[brakes]", + world.root.join("lease/process-write-lease.cc").display() + ), + ), + RECIPE + .replace( + "writer = \"none\"\nrecovery = \"rebuildable\"\nstartup = \"open-at-start\"", + "writer = \"process-bound-single-writer\"\nrecovery = \"preserve\"\nstartup = \"create-or-open-after-write-lease\"", + ) + .replace( + "required_environment = [\"GAMECULT_IDUNN_CANDIDATE_BIND\", \"GAMECULT_IDUNN_RUNTIME_BUNDLE\"]", + "required_environment = [\"GAMECULT_IDUNN_CANDIDATE_BIND\", \"GAMECULT_IDUNN_PROCESS_WRITE_LEASE\", \"GAMECULT_IDUNN_RUNTIME_BUNDLE\"]", + ), + ) + } else { + (binding, RECIPE.to_owned()) + }; + let recipe = recipe_source + .split("[[dependencies]]") + .next() + .context("recipe is empty")? + .replace( + "capability = \"service.runtime\"", + &format!( + "capability = \"{}\"\ncapacity = {minimum_capacity}", + if provides_odin { + ODIN_RENDEZVOUS_CAPABILITY + } else { + "service.runtime" + } + ), + ); + let plan = compile_deployment_plan( + recipe.as_bytes(), + binding.as_bytes(), + source(&recipe), + "service-incarnation-1", + Some(candidate_port), + 110, + &[], + )?; + let release = SealedRelease::new( + &plan, + vec![artifact_receipt()], + vec![external_input_receipt()], + 120, + )?; + let expected = release.expected_projection(&plan)?; + assert!(expected.route.is_some() && expected.dependencies.is_empty()); + assert_eq!(expected.write_lease_required, stateful); + assert_eq!( + ReadinessClass::of(&expected), + Ok(if provides_odin { + ReadinessClass::OdinSelf + } else { + ReadinessClass::RouteProof + }) + ); + + let now = now_millis()?; + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "continuity-service".into(), + kind: CommandKind::Continuity, + selector: "service".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + let mut transaction = + DeploymentTransaction::new(&command, "service".into(), 0, None, now)?; + transaction.lifecycle_authorized_at_unix_millis = Some(now); + let launch = IdunnRuntimeActivationLaunch::issue( + &expected, + runtime_instance_id(&transaction.transaction_id)?, + now, + &world.engine.idunn_signer, + )?; + let mut seed = Vec::new(); + let activation = launch.write_credential(&mut seed)?; + let recorded = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)?.1; + let mut workload = recorded.workload.clone().context("no recorded workload")?; + match &mut workload { + WorkloadObservation::Systemd(observed) => { + observed.runtime_instance_id = activation.runtime_instance_id.clone(); + observed.executable_sha256 = expected.artifact_sha256.clone(); + } + WorkloadObservation::Host(_) => bail!("recorded workload is not a systemd unit"), + } + transaction.isolation = recorded.isolation; + transaction.installed_release = Some(crate::drivers::InstalledReleaseObservation { + sealed_release_id: release.sealed_release_id.clone(), + root: PathBuf::from("/srv/service/releases/test"), + }); + transaction.expected_publication_sha256 = Some(expected.canonical_sha256()?); + transaction.activation_publication_sha256 = Some(activation.canonical_sha256()?); + transaction.sealed_release = Some(release); + transaction.expected = Some(expected.clone()); + transaction.activation = Some(activation.clone()); + transaction.workload = Some(workload); + transaction.plan = Some(plan); + + let stub = Arc::new(RuntimeStub { + expected, + activation, + provider: open_service_identity_at::( + &provider_path, + )?, + activation_signer: cultnet_rs::IdunnRuntimeActivationSigner::from_credential_reader( + &seed[..], + )?, + state: Mutex::new("warming"), + capacity: AtomicU32::new(minimum_capacity), + sequence: AtomicU64::new(0), + candidate_hits: AtomicUsize::new(0), + stable_hits: AtomicUsize::new(0), + stop: AtomicBool::new(false), + hang_up: AtomicBool::new(false), + lease: Mutex::new(None), + reply: Mutex::new(Reply::Honest), + }); + stub.listen(candidate_listener, true); + stub.listen(stable_listener, false); + + if phase >= DeploymentPhase::Fencing && provides_odin { + // Odin's own first word about the candidate, as for any + // Odin-correlated target. + let warming = signed_correlation(&world, &transaction, 4, false)?; + let authenticated = world.engine.authenticate_topology_bytes( + &ControlSnapshot::read(&world.state_store)?, + &transaction, + &warming, + None, + now, + )?; + transaction.warming = Some(WarmingEvidence::OdinTopology { + evidence: TopologyEvidence::from_authenticated(&authenticated, now)?, + }); + transaction.odin_publisher_sequence_cursor = 4; + } else if phase >= DeploymentPhase::Fencing { + let CandidateAnswer::Answered { evidence, .. } = + world + .engine + .challenge_candidate(&transaction, &["warming"], None)? + else { + bail!("the stub candidate did not answer"); + }; + transaction.warming = Some(WarmingEvidence::RouteProofDirect { evidence }); + } + if phase >= DeploymentPhase::Fencing { + transaction.route_preflight = Some(candidate_preflight(&transaction)?); + } + if stateful { + // The lease driver reads the Expected and the observed + // activation from the topology store, as Starting left them. + let plan = transaction.plan.as_ref().unwrap(); + let anchor = world.engine.provider_anchor_for_plan(plan)?; + let topology = world.engine.topology(); + topology.publish_expected(transaction.expected.as_ref().unwrap(), &anchor)?; + topology.publish_observed_activation( + transaction.expected.as_ref().unwrap(), + transaction.activation.as_ref().unwrap(), + transaction.workload.as_ref().unwrap(), + )?; + } + transaction.enter_phase(phase, now); + transaction.validate()?; + let store = SingleFileMessagePackBackingStore::new(&world.state_store); + assert!(store.compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: transaction.transaction_id.clone(), + current: None, + }, + ], + &[ + command_envelope(&command, command.requested_at_unix_millis)?, + transaction_envelope(&transaction, now)?, + ], + )?); + + // Odin's world. Unreachable: the correlation store is bytes no + // reader can decode, so any read of it is an error. NotReady: a + // valid correlation that says the target is not Ready. + let store = &world.engine.options.odin_correlation_store; + let poisoned = match odin { + _ if provides_odin => { + odin_reports_ready(&world, &transaction, 5)?; + Vec::new() + } + Odin::Unreachable => { + let garbage = b"odin is not a cache".to_vec(); + std::fs::write(store, &garbage)?; + garbage + } + Odin::NotReady => { + let wrongly = signed_correlation(&world, &transaction, 9, false)?; + SingleFileMessagePackBackingStore::new(store).insert_entry_if_absent( + CultCacheEnvelope { + key: crate::drivers::incarnation_key_of( + &transaction.target, + &transaction.expected.as_ref().unwrap().canonical_sha256()?, + ), + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + payload: wrongly, + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some( + cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into(), + ), + }, + )?; + Vec::new() + } + }; + Ok(RoutedWorld { + world, + stub, + odin, + poisoned, + }) + } + + /// The preflight receipt of a candidate whose route was never + /// installed: what Warming records before Fencing, without running the + /// host's nginx. + fn candidate_preflight( + transaction: &DeploymentTransaction, + ) -> Result { + let rendered = NginxRouteDriver::new( + transaction + .plan + .as_ref() + .unwrap() + .parsed_inputs()? + .1 + .route + .context("no route binding")?, + ) + .render(transaction.expected.as_ref().unwrap())?; + Ok(crate::drivers::RoutePreflightReceipt { + route_id: "service".into(), + candidate_runtime_instance_id: transaction + .activation + .as_ref() + .unwrap() + .runtime_instance_id + .clone(), + candidate_membership_sha256: sha256_id(&rendered), + incumbent_runtime_instance_id: None, + incumbent_membership_sha256: None, + incumbent_configuration: None, + }) + } + + impl RoutedWorld { + fn step(&self) -> Result<()> { + self.world.engine.advance_transaction(&resident(&self.world)?) + } + + fn run_to_routing(&self) -> Result<()> { + for _ in 0..12 { + if self.transaction()?.phase == DeploymentPhase::Routing { + return Ok(()); + } + self.step()?; + } + bail!("the transaction never reached Routing") + } + + fn transaction(&self) -> Result { + Ok(resident(&self.world)?.value) + } + + /// Odin was never read: the poisoned store is byte for byte what + /// it was, and no step failed on it. + fn assert_odin_untouched(&self) -> Result<()> { + if self.odin == Odin::Unreachable { + assert_eq!( + std::fs::read(&self.world.engine.options.odin_correlation_store)?, + self.poisoned + ); + } + Ok(()) + } + + /// Routing installs an nginx fragment and needs the host's programs, + /// so the world writes the fragment itself and records the promotion + /// the stable challenge proves, then enters Committing. + fn promote_by_hand(&self) -> Result<()> { + let current = resident(&self.world)?; + let transaction = ¤t.value; + assert_eq!(transaction.phase, DeploymentPhase::Routing); + let expected = transaction.expected.as_ref().unwrap(); + let binding = transaction.plan.as_ref().unwrap().parsed_inputs()?.1; + let driver = NginxRouteDriver::new(binding.route.unwrap()); + let rendered = driver.render(expected)?; + std::fs::write(&driver.binding.config_path, &rendered)?; + let observation = self.world.engine.prove_stable_route( + transaction, + &driver, + sha256_id(&rendered), + )?; + let mut next = transaction.clone(); + next.routing = Some(RoutingEvidence::Promoted { + promoted_at_unix_millis: observation.observed_at_unix_millis, + observation, + }); + next.enter_phase(DeploymentPhase::Committing, now_millis()?); + replace_transaction(&self.world.state_store, ¤t, &next) + } + } + + /// A route-proof generation from a real admission. + pub(super) fn committed_generation() -> Result { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.promote_by_hand()?; + routed.step()?; + admitted(&routed) + } + + fn admitted(world: &RoutedWorld) -> Result { + Ok(ControlSnapshot::read(&world.world.state_store)? + .admitted + .into_iter() + .next() + .context("nothing was admitted")? + .value) + } + + #[test] + fn a_route_proof_target_admits_with_odin_unreachable_and_never_reads_it() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + // Fencing, Leasing, AwaitingReady: the candidate is still warming. + for _ in 0..12 { + routed.step()?; + if routed.transaction()?.last_error.is_some() { + break; + } + } + let waiting = routed.transaction()?; + assert_eq!(waiting.phase, DeploymentPhase::AwaitingReady); + assert!(waiting.ready.is_none()); + assert_eq!( + waiting.last_error.as_deref(), + Some("candidate is still warming") + ); + // The candidate finishes warming: Ready is its own Active answer. + routed.stub.set_state("active"); + routed.step()?; + assert!(matches!( + routed.transaction()?.ready, + Some(ReadinessEvidence::RouteProof { .. }) + )); + routed.step()?; + assert_eq!(routed.transaction()?.phase, DeploymentPhase::Routing); + routed.promote_by_hand()?; + routed.step()?; + + let generation = admitted(&routed)?; + generation.validate()?; + assert_eq!(generation.ready.voucher(), Voucher::Candidate); + assert!(generation.odin_authority.is_none()); + assert!(generation.latest_odin_observation.is_none()); + assert!(generation.route_supervision.is_some()); + // Idunn's own challenges carried every proof: the seed's warming, + // two Ready polls; the stable endpoint proved the route twice. + assert_eq!(routed.stub.candidate_hits.load(Ordering::SeqCst), 3); + assert!(routed.stub.stable_hits.load(Ordering::SeqCst) >= 2); + routed.assert_odin_untouched()?; + + // Boot and supervision read no Odin either: the topology refresh + // returns before it opens Odin's store, and durable authority + // skips a route-proof generation's receipts. + let stored = ControlSnapshot::read(&routed.world.state_store)?; + let current = stored.admitted.first().context("no admitted generation")?; + assert!(!routed.world.engine.refresh_admitted_topology(&stored, current)?); + routed.world.engine.validate_durable_authority(&stored)?; + routed.assert_odin_untouched()?; + Ok(()) + } + + #[test] + fn an_odin_correlation_about_a_route_proof_target_decides_nothing() -> Result<()> { + let routed = routed_world(Odin::NotReady, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.promote_by_hand()?; + routed.step()?; + let generation = admitted(&routed)?; + assert_eq!(generation.ready.voucher(), Voucher::Candidate); + // The valid, not-Ready correlation was neither admitted as a + // receipt nor consulted: the record carries no Odin cursor. + assert_eq!(generation.odin_publisher_sequence_cursor, 0); + Ok(()) + } + + #[test] + fn a_route_proof_candidate_that_has_not_answered_is_waited_on_not_aborted() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + routed.step()?; + let warmed = routed.transaction()?; + assert_eq!(warmed.phase, DeploymentPhase::Warming); + let Some(WarmingEvidence::RouteProofDirect { evidence }) = &warmed.warming else { + bail!("Warming evidence is not the candidate's own presence"); + }; + assert!(evidence.message_id.starts_with("candidate-")); + assert_eq!(routed.stub.candidate_hits.load(Ordering::SeqCst), 1); + routed.assert_odin_untouched()?; + + // Nothing answers: the same step waits, and says why. + let silent = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + silent.stub.hang_up.store(true, Ordering::SeqCst); + silent.step()?; + let waiting = silent.transaction()?; + assert!(waiting.warming.is_none() && waiting.pre_fencing_abort.is_none()); + assert!( + waiting + .last_error + .as_deref() + .is_some_and(|error| error.contains("candidate endpoint did not answer")), + "{:?}", + waiting.last_error + ); + Ok(()) + } + + #[test] + fn a_capacity_below_the_expected_minimum_is_named_on_the_route_path() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 2, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.promote_by_hand()?; + routed.stub.capacity.store(1, Ordering::SeqCst); + let failure = routed.step().expect_err("a shortfall must be refused"); + // The shortfall is a typed value, not a sentence to search. + let disagrees = failure + .downcast_ref::() + .with_context(|| format!("the refusal is not typed: {failure:#}"))?; + let shortfall = disagrees + .disagreements + .iter() + .find(|disagreement| disagreement.code == "expected-capability-000-capacity") + .context("the capacity shortfall is not named")?; + assert!( + shortfall.expected.as_deref().is_some_and(|text| text.ends_with("capacity>=2")), + "{shortfall:?}" + ); + assert!(shortfall.observed.is_some(), "{shortfall:?}"); + assert!(admitted(&routed).is_err()); + Ok(()) + } + + #[test] + fn odin_itself_is_never_route_proof() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let mut expected = routed.stub.expected.clone(); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::RouteProof)); + // Odin provides the rendezvous capability and declares no dependency. + expected.capabilities[0].capability = ODIN_RENDEZVOUS_CAPABILITY.into(); + assert!(expected.dependencies.is_empty()); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinSelf)); + + // Route-proof evidence for it is refused: its admitted generation + // is what names the Odin authority, so it must carry one. + let (_, mut odin) = fixture_generation(FIXTURE_GENERATION)?; + odin.target = "odin".into(); + odin.expected = expected; + odin.ready = route_proof_evidence(); + odin.latest_odin_observation = None; + odin.odin_authority = None; + assert!(error_text(odin.validate()).contains("not route-proof")); + Ok(()) + } + + #[test] + fn route_admission_gets_past_its_gate_without_reading_odin() -> Result<()> { + // The step below would run the host's real nginx, so it only runs + // where there is none. + if Path::new("/usr/sbin/nginx").exists() { + return Ok(()); + } + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + // The step fails at installing the route: past the admission gate, + // which would have failed on Odin. + let failure = format!("{:#}", routed.step().expect_err("no nginx in the test world")); + assert!( + failure.contains("starting route actuator /usr/sbin/nginx"), + "{failure}" + ); + routed.assert_odin_untouched()?; + Ok(()) + } + + #[test] + fn a_stateless_candidate_that_finished_warming_first_still_warms() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + routed.stub.set_state("active"); + routed.step()?; + assert!(matches!( + routed.transaction()?.warming, + Some(WarmingEvidence::RouteProofDirect { .. }) + )); + Ok(()) + } + + #[test] + fn the_lease_refresh_asks_the_candidate_for_a_new_warming_presence() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let current = resident(&routed.world)?; + let prior = routed + .world + .engine + .rehydrate_warming_token(¤t.value, now_millis()?, false)?; + let (_, evidence, token) = routed + .world + .engine + .fresh_warming_for_lease(¤t, now_millis()?)? + .context("a warming candidate must refresh")?; + assert!(matches!(evidence, WarmingEvidence::RouteProofDirect { .. })); + assert_ne!(token.signed_presence_sha256(), prior.signed_presence_sha256()); + // A candidate that went Active is not warming: nothing to bind. + routed.stub.set_state("active"); + let failure = routed + .world + .engine + .fresh_warming_for_lease(¤t, now_millis()?) + .map(|_| ()) + .expect_err("an active candidate is not a warming one"); + assert!(format!("{failure:#}").contains("not warming"), "{failure:#}"); + routed.assert_odin_untouched()?; + Ok(()) + } + + #[test] + fn boot_reauthenticates_a_live_route_proof_transactions_evidence() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + routed.world.engine.validate_durable_authority(&snapshot)?; + + // Warming evidence that does not answer the challenge it names. + let current = resident(&routed.world)?; + let mut forged = current.value.clone(); + let Some(WarmingEvidence::RouteProofDirect { evidence }) = &mut forged.warming else { + bail!("no route-proof Warming evidence"); + }; + evidence.message_id = "candidate-another".into(); + replace_transaction(&routed.world.state_store, ¤t, &forged)?; + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + let failure = routed + .world + .engine + .validate_durable_authority(&snapshot) + .expect_err("Warming evidence for another challenge"); + assert!(format!("{failure:#}").contains("exact challenge"), "{failure:#}"); + + // A Ready label carried by a presence that was only warming. + let current = resident(&routed.world)?; + let CandidateAnswer::Answered { evidence, .. } = routed + .world + .engine + .challenge_candidate(¤t.value, &["warming"], None)? + else { + bail!("the stub candidate did not answer"); + }; + let mut forged = current.value.clone(); + forged.warming = Some(WarmingEvidence::RouteProofDirect { + evidence: evidence.clone(), + }); + forged.ready = Some(ReadinessEvidence::RouteProof { evidence }); + replace_transaction(&routed.world.state_store, ¤t, &forged)?; + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + let failure = routed + .world + .engine + .validate_durable_authority(&snapshot) + .expect_err("Ready carried by a warming presence"); + assert!(format!("{failure:#}").contains("not active"), "{failure:#}"); + routed.assert_odin_untouched()?; + Ok(()) + } + + #[test] + fn a_target_providing_the_rendezvous_commits_on_odin_evidence_and_carries_the_authority() + -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, true)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + let ready = routed.transaction()?; + assert!(ready.ready.as_ref().and_then(ReadinessEvidence::odin).is_some()); + routed.promote_by_hand()?; + routed.step()?; + let generation = admitted(&routed)?; + generation.validate()?; + assert_eq!(generation.ready.voucher(), Voucher::Odin); + assert_eq!( + generation.odin_authority, + Some(routed.world.engine.bootstrap_odin_authority.clone()) + ); + assert!(generation.latest_odin_observation.is_some()); + // Boot re-authenticates every Odin receipt this generation carries. + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + routed.world.engine.validate_durable_authority(&snapshot)?; + Ok(()) + } + + #[test] + fn a_store_of_route_proof_generations_boots_and_resolves_the_bootstrap_authority() + -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.promote_by_hand()?; + routed.step()?; + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + routed.world.engine.validate_durable_authority(&snapshot)?; + assert_eq!( + routed.world.engine.current_odin_authority(&snapshot)?, + routed.world.engine.bootstrap_odin_authority + ); + Ok(()) + } + + impl RoutedWorld { + /// Boot would accept this store, and Odin was never read. + fn assert_boots(&self) -> Result<()> { + let snapshot = ControlSnapshot::read(&self.world.state_store)?; + self.world.engine.validate_durable_authority(&snapshot)?; + self.assert_odin_untouched() + } + + /// Step until `done`, checking after every step that boot would + /// still accept the store. + fn drive_until(&self, done: impl Fn(&DeploymentTransaction) -> bool) -> Result<()> { + for _ in 0..20 { + if done(&self.transaction()?) { + return Ok(()); + } + self.step()?; + self.assert_boots()?; + } + bail!("the transaction never reached the awaited state") + } + + fn answer_now(&self) -> Result<(RuntimePresenceEvidence, cultnet_rs::VerifiedRuntimePresence)> { + match self.world.engine.challenge_candidate( + &self.transaction()?, + &["warming", "active"], + None, + )? { + CandidateAnswer::Answered { evidence, present } => Ok((evidence, present)), + CandidateAnswer::Silent(reason) => bail!("the candidate was silent: {reason}"), + } + } + } + + // ----------------------------------------------------------------- + // The stateful path, end to end through the Engine. + // ----------------------------------------------------------------- + + #[test] + fn a_stateful_route_proof_target_holds_its_lease_from_fencing_to_commit() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.assert_boots()?; + + routed.drive_until(|t| matches!(t.leasing, Some(LeasingEvidence::Prepared { .. })))?; + assert_eq!(routed.transaction()?.phase, DeploymentPhase::Leasing); + routed.drive_until(|t| matches!(t.leasing, Some(LeasingEvidence::Granted { .. })))?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let lease_sha256 = routed + .transaction()? + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256) + .context("no granted lease")? + .to_owned(); + + // The candidate has not picked the lease up: still warming. + routed.step()?; + assert_eq!( + routed.transaction()?.last_error.as_deref(), + Some("candidate is still warming") + ); + assert!(routed.transaction()?.ready.is_none()); + // Active without the lease Idunn granted is not Ready. + routed.stub.set_state("active"); + let refusal = routed + .step() + .expect_err("an active candidate without the granted lease is refused"); + assert!( + format!("{refusal:#}").contains("exact current process write lease"), + "{refusal:#}" + ); + assert!(routed.transaction()?.ready.is_none()); + // Active holding exactly that lease is. + *routed.stub.lease.lock().unwrap() = Some(lease_sha256.clone()); + routed.step()?; + assert!(matches!( + routed.transaction()?.ready, + Some(ReadinessEvidence::RouteProof { .. }) + )); + routed.assert_boots()?; + + routed.drive_until(|t| t.phase == DeploymentPhase::Routing)?; + routed.promote_by_hand()?; + routed.assert_boots()?; + routed.step()?; + + let generation = admitted(&routed)?; + generation.validate()?; + assert_eq!(generation.ready.voucher(), Voucher::Candidate); + assert!(generation.expected.write_lease_required); + assert_eq!(generation.leasing.lease_sha256(), Some(lease_sha256.as_str())); + assert!(generation.odin_authority.is_none()); + routed.assert_boots()?; + Ok(()) + } + + // ----------------------------------------------------------------- + // Warming is one challenge, then the phase moves. + // ----------------------------------------------------------------- + + #[test] + fn warming_challenges_once_and_then_advances_on_every_tick() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + routed.step()?; + let recorded = routed + .transaction()? + .warming + .context("the first step recorded no Warming evidence")?; + assert_eq!(routed.stub.candidate_hits.load(Ordering::SeqCst), 1); + + // The route preflight needs the host's nginx; its receipt is recorded + // here so the phase can move on. + let current = resident(&routed.world)?; + let mut next = current.value.clone(); + next.route_preflight = Some(candidate_preflight(&next)?); + replace_transaction(&routed.world.state_store, ¤t, &next)?; + + routed.drive_until(|t| t.phase == DeploymentPhase::Fencing)?; + assert_eq!( + routed.stub.candidate_hits.load(Ordering::SeqCst), + 1, + "a recorded Warming answer is never asked for again" + ); + assert_eq!(routed.transaction()?.warming, Some(recorded)); + Ok(()) + } + + #[test] + fn a_stateful_candidate_may_not_answer_its_warming_challenge_active() { + let mut expected = fixture_generation(FIXTURE_GENERATION).unwrap().1.expected; + expected.write_lease_required = true; + assert_eq!(route_proof_warming_states(&expected), ["warming"]); + expected.write_lease_required = false; + assert_eq!(route_proof_warming_states(&expected), ["warming", "active"]); + } + + #[test] + fn a_stateful_candidate_that_answers_warming_active_is_refused() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Warming)?; + routed.stub.set_state("active"); + let refusal = routed + .step() + .expect_err("a stateful candidate cannot be active before it holds a lease"); + assert!(format!("{refusal:#}").contains("not warming"), "{refusal:#}"); + assert!(routed.transaction()?.warming.is_none()); + Ok(()) + } + + // ----------------------------------------------------------------- + // What a challenged presence must prove. + // ----------------------------------------------------------------- + + #[test] + fn a_presence_holds_the_write_lease_exactly_when_it_is_not_warming() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Warming)?; + let authority = routed + .world + .engine + .runtime_authority(&routed.transaction()?)?; + let lease = sha256_id(b"the granted lease"); + let other = sha256_id(b"some other lease"); + let judge = |state: &'static str, + held: Option<&str>, + current: Option<&str>, + states: &[&str]| + -> Result<()> { + let challenged = now_millis()?; + routed.stub.set_state(state); + *routed.stub.lease.lock().unwrap() = held.map(str::to_owned); + let bytes = routed.stub.answer("candidate-probe")?; + routed + .world + .engine + .authenticate_challenged_presence( + &authority, + states, + current, + "candidate-probe", + challenged, + now_millis()?, + &bytes, + ) + .map(|_| ()) + }; + let either = ["warming", "active"]; + judge("warming", None, None, &either)?; + judge("active", Some(&lease), Some(&lease), &either)?; + for (state, held, current, named) in [ + // Warming holds nothing, whatever Idunn has granted. + ("warming", Some(lease.as_str()), None, "write lease"), + ("warming", Some(lease.as_str()), Some(lease.as_str()), "write lease"), + // Anything else holds exactly the current grant. + ("active", None, Some(lease.as_str()), "exact current process write lease"), + ("active", Some(lease.as_str()), None, "exact current process write lease"), + ( + "active", + Some(other.as_str()), + Some(lease.as_str()), + "exact current process write lease", + ), + ] { + let error = judge(state, held, current, &either) + .expect_err("a presence with the wrong lease is refused"); + assert!( + format!("{error:#}").contains(named), + "{state} {held:?} {current:?}: {error:#}" + ); + } + Ok(()) + } + + #[test] + fn a_presence_minted_before_its_challenge_is_refused() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let authority = routed + .world + .engine + .runtime_authority(&routed.transaction()?)?; + let bytes = routed.stub.answer("candidate-probe")?; + let minted = now_millis()?; + // Answered before the challenge that claims it: an old presence + // replayed for a new question. + let replay = routed + .world + .engine + .authenticate_challenged_presence( + &authority, + &["warming"], + None, + "candidate-probe", + minted + 1_000, + minted + 1_000, + &bytes, + ) + .expect_err("a presence minted before its challenge"); + assert!( + format!("{replay:#}").contains("minted before its challenge"), + "{replay:#}" + ); + let early = routed + .world + .engine + .authenticate_challenged_presence( + &authority, + &["warming"], + None, + "candidate-probe", + minted + 1_000, + minted, + &bytes, + ) + .expect_err("a receipt before its challenge"); + assert!(format!("{early:#}").contains("predates its challenge"), "{early:#}"); + Ok(()) + } + + #[test] + fn direct_warming_evidence_is_bound_to_the_bytes_it_authenticated() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let id = routed.transaction()?.transaction_id; + let direct = |evidence: RuntimePresenceEvidence| WarmingEvidence::RouteProofDirect { evidence }; + + let (evidence, present) = routed.answer_now()?; + SequenceAdmittedWarming::from_direct_presence(id.clone(), direct(evidence), present)?; + + // The digest is the authenticated presence's, but the bytes are not. + let (mut evidence, present) = routed.answer_now()?; + evidence.canonical_bytes.push(0); + let error = SequenceAdmittedWarming::from_direct_presence(id.clone(), direct(evidence), present) + .expect_err("bytes that are not the authenticated presence"); + assert!(format!("{error:#}").contains("differs from its authenticated"), "{error:#}"); + + // The bytes are the authenticated presence's, but the digest is not. + let (mut evidence, present) = routed.answer_now()?; + evidence.canonical_sha256 = sha256_id(b"another presence"); + let error = SequenceAdmittedWarming::from_direct_presence(id.clone(), direct(evidence), present) + .expect_err("a digest that is not the authenticated presence's"); + assert!(format!("{error:#}").contains("differs from its authenticated"), "{error:#}"); + + // Evidence about another answer altogether. + let (other, _) = routed.answer_now()?; + let (_, present) = routed.answer_now()?; + SequenceAdmittedWarming::from_direct_presence(id, direct(other), present) + .expect_err("evidence about another answer"); + Ok(()) + } + + // ----------------------------------------------------------------- + // Validation: candidate-vouched evidence belongs to route-proof targets. + // ----------------------------------------------------------------- + + #[test] + fn candidate_vouched_evidence_is_refused_on_a_target_odin_reports_on() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let (evidence, _) = routed.answer_now()?; + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let odin_reported = transaction_at(&world, DeploymentPhase::Warming)?; + assert_eq!( + ReadinessClass::of(odin_reported.expected.as_ref().unwrap()), + Ok(ReadinessClass::OdinCorrelated) + ); + + let mut warming = odin_reported.clone(); + warming.warming = Some(WarmingEvidence::RouteProofDirect { + evidence: evidence.clone(), + }); + assert!(error_text(warming.validate()).contains("not route-proof")); + + let mut ready = odin_reported.clone(); + ready.ready = Some(ReadinessEvidence::RouteProof { + evidence: evidence.clone(), + }); + assert!(error_text(ready.validate()).contains("not route-proof")); + + // Direct Odin warming is Odin's alone, by class: a route-proof + // target may not claim it, and a target that provides the + // rendezvous may, whatever it is named. + let mut direct = routed.transaction()?; + direct.warming = Some(WarmingEvidence::FirstOdinDirect { + evidence: evidence.clone(), + }); + assert!(error_text(direct.validate()).contains("reserved for Odin")); + let odin = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, true)?; + let mut direct = odin.transaction()?; + assert_eq!(direct.target, "service"); + direct.warming = Some(WarmingEvidence::FirstOdinDirect { evidence }); + direct.validate()?; + Ok(()) + } + + #[test] + fn a_route_proof_transaction_past_awaiting_ready_needs_the_candidates_ready_proof() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + let routing = routed.transaction()?; + routing.validate()?; + let mut bare = routing.clone(); + bare.ready = None; + assert!(error_text(bare.validate()).contains("lacks Ready evidence")); + routed.promote_by_hand()?; + let mut bare = routed.transaction()?; + assert_eq!(bare.phase, DeploymentPhase::Committing); + bare.ready = None; + assert!(error_text(bare.validate()).contains("lacks Ready evidence")); + Ok(()) + } + + // ----------------------------------------------------------------- + // The class is the Expected's alone. Evidence collected under another + // class is held and reported, never stepped, never repaired. + // ----------------------------------------------------------------- + + /// Odin's Ready for the world's target, as pre-B3 code recorded it for + /// every routed target. + fn odin_receipt_for( + routed: &RoutedWorld, + transaction: &DeploymentTransaction, + sequence: u64, + ) -> Result { + odin_receipt_with(routed, transaction, sequence, true) + } + + fn odin_receipt_with( + routed: &RoutedWorld, + transaction: &DeploymentTransaction, + sequence: u64, + ready: bool, + ) -> Result { + let now = now_millis()?; + let bytes = signed_correlation(&routed.world, transaction, sequence, ready)?; + let authenticated = routed.world.engine.authenticate_topology_bytes( + &ControlSnapshot::read(&routed.world.state_store)?, + transaction, + &bytes, + transaction + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256), + now, + )?; + TopologyEvidence::from_authenticated(&authenticated, now) + } + + fn stored(generation: &AdmittedGeneration) -> Stored { + Stored { + value: generation.clone(), + envelope: CultCacheEnvelope { + key: generation.target.clone(), + r#type: AdmittedGeneration::TYPE.into(), + payload: Vec::new(), + stored_at: "1970-01-01T00:00:00.100Z".into(), + schema_id: Some(ADMITTED_GENERATION_SCHEMA.into()), + }, + } + } + + #[test] + fn a_pre_b3_transaction_over_a_route_proof_target_is_held_and_reported_once() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + + // Ready as pre-B3 Idunn wrote it: an Odin receipt. + let current = resident(&routed.world)?; + let mut legacy = current.value.clone(); + let receipt = odin_receipt_for(&routed, &legacy, 9)?; + legacy.latest_odin_observation = Some(receipt.clone()); + legacy.odin_publisher_sequence_cursor = 9; + legacy.ready = Some(ReadinessEvidence::OdinCorrelated { evidence: receipt }); + replace_transaction(&routed.world.state_store, ¤t, &legacy)?; + + let disagreement = ReadinessDisagreement::WrongVoucher { + target: "service".into(), + required: ReadinessClass::RouteProof, + collected: Voucher::Odin, + }; + assert_eq!(legacy.readiness_disagreement(), Some(disagreement.clone())); + // The record is readable and boot accepts it: held, not refused. + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + routed.world.engine.validate_durable_authority(&snapshot)?; + // Stepping it would fail, on every tick, for good. + assert!(routed.step().is_err()); + + let hits = ( + routed.stub.candidate_hits.load(Ordering::SeqCst), + routed.stub.stable_hits.load(Ordering::SeqCst), + ); + for _ in 0..4 { + assert!(!routed.world.engine.run_scheduler_tick()?); + } + let held = resident(&routed.world)?; + assert_eq!(held.value.phase, DeploymentPhase::Routing); + assert!(held.value.pre_fencing_abort.is_none()); + assert!(held.value.post_fencing_abort.is_none()); + assert_eq!(held.value.ready, legacy.ready, "nothing re-derived the evidence"); + assert_eq!( + held.value.last_error.as_deref(), + Some(disagreement.to_string().as_str()) + ); + // Reported once: later ticks leave the record alone. + routed.world.engine.run_scheduler_tick()?; + assert_eq!(resident(&routed.world)?.envelope, held.envelope); + assert_eq!( + hits, + ( + routed.stub.candidate_hits.load(Ordering::SeqCst), + routed.stub.stable_hits.load(Ordering::SeqCst) + ), + "a held transaction is asked nothing" + ); + routed.assert_odin_untouched()?; + + // An abort already under way is not held: it never reads the class. + let mut aborting = held.value.clone(); + aborting.post_fencing_abort = Some(post_fencing_abort_intent(&aborting, "operator abort")); + replace_transaction(&routed.world.state_store, &held, &aborting)?; + let after = resident(&routed.world)?; + let _ = routed.world.engine.resume_candidate(&after); + assert_ne!( + resident(&routed.world)?.value.last_error.as_deref(), + Some(disagreement.to_string().as_str()), + "an abort in progress runs instead of being held" + ); + Ok(()) + } + + #[test] + fn a_pre_b3_generation_over_a_route_proof_target_boots_and_is_reported_not_refreshed() + -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + let at_routing = routed.transaction()?; + routed.promote_by_hand()?; + routed.step()?; + + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + let current = snapshot.admitted.first().context("nothing was admitted")?; + // Receipts that would not survive being re-proved: authentic, but + // not Ready. Boot re-proves against the current Odin authority and + // requires exact semantic Ready, never what a generation stores, so + // this record is only bootable if boot does not re-prove it. + let receipt = odin_receipt_with(&routed, &at_routing, 9, false)?; + let mut legacy = current.value.clone(); + legacy.ready = ReadinessEvidence::OdinCorrelated { + evidence: receipt.clone(), + }; + legacy.latest_odin_observation = Some(receipt); + // The stored authority is present because an Odin-tagged record + // carries one; it is not what boot reads. + let stranger = cultnet_rs::enroll_service_identity_at::( + &routed.world.root.join("identities/stranger.cc"), + )?; + legacy.odin_authority = Some(AdmittedOdinAuthority::from_anchor(&stranger.trust_anchor()?)?); + legacy.odin_publisher_sequence_cursor = 9; + assert!( + SingleFileMessagePackBackingStore::new(&routed.world.state_store) + .compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: AdmittedGeneration::TYPE.into(), + key: legacy.target.clone(), + current: Some(current.envelope.clone()), + }], + &[admitted_envelope(&legacy, now_millis()?)?], + )? + ); + + assert_eq!( + legacy.readiness(), + Err(ReadinessDisagreement::WrongVoucher { + target: "service".into(), + required: ReadinessClass::RouteProof, + collected: Voucher::Odin, + }) + ); + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + let current = snapshot.admitted.first().context("the generation is gone")?; + // The class chooses the path: no Odin receipt is re-proved at boot + // and none is refreshed, though the evidence is an Odin receipt. + routed.world.engine.validate_durable_authority(&snapshot)?; + assert!(!routed.world.engine.refresh_admitted_topology(&snapshot, current)?); + routed.assert_odin_untouched()?; + + routed.world.engine.supervise_one_admitted_generation()?; + assert!( + routed + .world + .engine + .fault_reports + .lock() + .unwrap() + .contains_key("generation:service"), + "supervision reports the disagreement" + ); + routed.assert_odin_untouched()?; + Ok(()) + } + + #[test] + fn only_a_transaction_that_still_reads_its_class_is_held_for_its_evidence() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + let at_routing = routed.transaction()?; + // What pre-B3 Idunn would have collected: an Odin receipt. + let mut legacy = at_routing.clone(); + legacy.ready = Some(ReadinessEvidence::OdinCorrelated { + evidence: odin_receipt_for(&routed, &at_routing, 9)?, + }); + let disagreement = legacy.readiness_disagreement(); + assert!(disagreement.is_some()); + assert_eq!(legacy.held_disagreement(), disagreement); + + // Finished work, and an abort under way, never read the class. + let mut completed = legacy.clone(); + completed.completion = Some(TransactionCompletion::Admitted { + generation_id: "generation-1".into(), + }); + assert_eq!(completed.held_disagreement(), None); + let mut aborting = legacy.clone(); + aborting.pre_fencing_abort = Some(PreFencingAbort { + error: "operator abort".into(), + candidate_cleanup: CleanupEvidence::Pending, + topology_reconciliation: CleanupEvidence::Pending, + source_cleanup: CleanupEvidence::Pending, + }); + assert_eq!(aborting.held_disagreement(), None); + let mut aborting = legacy; + aborting.post_fencing_abort = Some(post_fencing_abort_intent(&aborting, "operator abort")); + assert_eq!(aborting.held_disagreement(), None); + Ok(()) + } + + #[test] + fn a_stored_record_that_declares_no_way_to_prove_readiness_is_held() -> Result<()> { + let world = EngineFixture::new()?; + // Odin's store is unreadable: any read of it is an error. + std::fs::write(&world.engine.options.odin_correlation_store, b"odin is not a cache")?; + + let (_, mut generation) = fixture_generation(FIXTURE_GENERATION)?; + let target = generation.expected.target.clone(); + assert_eq!(generation.readiness(), Ok(ReadinessClass::OdinCorrelated)); + generation.expected.dependencies.clear(); + generation.expected.route = None; + let undeclared = UndeclaredReadiness { target: target.clone() }; + assert_eq!( + generation.readiness(), + Err(ReadinessDisagreement::Undeclared(undeclared.clone())) + ); + assert!( + !world + .engine + .refresh_admitted_topology(&ControlSnapshot::default(), &stored(&generation))? + ); + + // The mirror: Odin declared, but the receipt is the candidate's. + let (_, mut odin_class) = fixture_generation(FIXTURE_GENERATION)?; + odin_class.ready = route_proof_evidence(); + assert_eq!( + odin_class.readiness(), + Err(ReadinessDisagreement::WrongVoucher { + target: target.clone(), + required: ReadinessClass::OdinCorrelated, + collected: Voucher::Candidate, + }) + ); + + // A transaction says the same about what it has collected. + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let (evidence, _) = routed.answer_now()?; + let odin_world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = transaction_at(&odin_world, DeploymentPhase::Warming)?; + assert_eq!(seeded.readiness_disagreement(), None); + let mut collected = seeded.clone(); + collected.warming = Some(WarmingEvidence::RouteProofDirect { evidence }); + assert_eq!( + collected.readiness_disagreement(), + Some(ReadinessDisagreement::WrongVoucher { + target: "service".into(), + required: ReadinessClass::OdinCorrelated, + collected: Voucher::Candidate, + }) + ); + let mut bare = seeded.clone(); + bare.expected.as_mut().unwrap().dependencies.clear(); + assert_eq!( + bare.readiness_disagreement(), + Some(ReadinessDisagreement::Undeclared(UndeclaredReadiness { + target: "service".into() + })) + ); + let mut no_expected = seeded; + no_expected.expected = None; + assert_eq!(no_expected.readiness_disagreement(), None); + Ok(()) + } + + // ----------------------------------------------------------------- + // Odin is whichever target provides the rendezvous. + // ----------------------------------------------------------------- + + #[test] + fn odin_is_the_target_that_provides_the_rendezvous_not_the_target_named_odin() -> Result<()> { + let world = EngineFixture::new()?; + let (_, fixture) = fixture_generation(FIXTURE_GENERATION)?; + + // A target named "odin" that merely depends on Odin is not Odin. + let mut named = fixture.clone(); + named.target = "odin".into(); + named.expected.target = "odin".into(); + let mut snapshot = ControlSnapshot::default(); + snapshot.admitted.push(stored(&named)); + assert!(snapshot.admitted_odin().is_none()); + assert_eq!( + world.engine.current_odin_authority(&snapshot)?, + world.engine.bootstrap_odin_authority + ); + + // The target that provides the rendezvous is, whatever it is called. + let mut provider = fixture.clone(); + provider.target = "verse".into(); + provider.expected.target = "verse".into(); + provider.expected.dependencies.clear(); + provide_odin(&mut provider.expected); + let authority = AdmittedOdinAuthority::from_anchor(&world.odin_signer.trust_anchor()?)?; + provider.odin_authority = Some(authority.clone()); + snapshot.admitted.push(stored(&provider)); + assert_eq!( + snapshot.admitted_odin().map(|found| found.value.target.as_str()), + Some("verse") + ); + assert_eq!( + snapshot.admitted_odin().and_then(|found| found.value.odin_authority.clone()), + Some(authority) + ); + Ok(()) + } + + #[test] + fn a_target_that_provides_the_rendezvous_warms_as_odin_whatever_it_is_named() -> Result<()> { + // Nothing else has been admitted that could observe it, so Odin + // observes itself directly -- and only a stateful incarnation may. + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, true)?; + assert_eq!(routed.transaction()?.target, "service"); + let refusal = routed + .step() + .expect_err("a stateless Odin cannot be observed directly"); + assert!( + format!("{refusal:#}").contains("direct Odin warming must be a stateful incarnation"), + "{refusal:#}" + ); + Ok(()) + } + + #[test] + fn direct_odin_warming_is_reserved_to_odin_by_class_not_by_name() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let mut named = routed.transaction()?; + named.target = "odin".into(); + let error = routed + .world + .engine + .authenticate_first_odin_warming_presence(&named, "warming-probe", 1, 2, &[1]) + .expect_err("a target named odin that is not Odin"); + assert!( + format!("{error:#}").contains("reserved for a stateful Odin incarnation"), + "{error:#}" + ); + Ok(()) + } + + #[test] + fn a_stateful_target_that_provides_the_rendezvous_is_observed_directly_whatever_it_is_named() + -> Result<()> { + let provider = build_routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, true, true)?; + let transaction = provider.transaction()?; + assert_eq!(transaction.target, "service"); + assert!(transaction.expected.as_ref().unwrap().write_lease_required); + provider.stub.set_state("warming"); + *provider.stub.reply.lock().unwrap() = Reply::FirstOdinWarming; + let challenged_at = now_millis()?; + let answer = provider.stub.answer("warming-probe")?; + provider.world.engine.authenticate_first_odin_warming_presence( + &transaction, + "warming-probe", + challenged_at, + now_millis()?, + &answer, + )?; + + // The reverse: named odin, stateful, but nothing provides the + // rendezvous. + let stateful = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Warming)?; + let mut named = stateful.transaction()?; + assert!(named.expected.as_ref().unwrap().write_lease_required); + named.target = "odin".into(); + let error = stateful + .world + .engine + .authenticate_first_odin_warming_presence(&named, "warming-probe", 1, 2, &[1]) + .expect_err("a stateful target named odin that does not provide the rendezvous"); + assert!( + format!("{error:#}").contains("reserved for a stateful Odin incarnation"), + "{error:#}" + ); + Ok(()) + } + + // ----------------------------------------------------------------- + // A bad answer is an error. Only silence waits. + // ----------------------------------------------------------------- + + #[test] + fn a_bad_answer_to_a_challenge_is_an_error_and_only_silence_waits() -> Result<()> { + for reply in [Reply::ForeignChallengeId, Reply::HttpFailure] { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + *routed.stub.reply.lock().unwrap() = reply; + let error = match routed.world.engine.challenge_candidate( + &routed.transaction()?, + &["warming"], + None, + ) { + Err(error) => error, + Ok(_) => bail!("a bad answer was taken for silence"), + }; + assert!( + matches!( + error.downcast_ref::(), + Some(ChallengeFailure::Refused(_)) + ), + "{error:#}" + ); + // Before the fence a candidate that answers wrongly is aborted, + // not waited on. + routed.world.engine.run_scheduler_tick()?; + let after = routed.transaction()?; + assert!(after.pre_fencing_abort.is_some(), "the bad answer aborted it"); + assert!(after.warming.is_none()); + } + + let silent = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + silent.stub.hang_up.store(true, Ordering::SeqCst); + assert!(matches!( + silent.world.engine.challenge_candidate(&silent.transaction()?, &["warming"], None)?, + CandidateAnswer::Silent(_) + )); + Ok(()) + } + } } diff --git a/src/deployment_plan.rs b/src/deployment_plan.rs index d4b1cc4..5210448 100644 --- a/src/deployment_plan.rs +++ b/src/deployment_plan.rs @@ -10,7 +10,7 @@ use cultnet_rs::{ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; -use crate::control_plane::SequenceAdmittedReady; +use crate::control_plane::{ODIN_RENDEZVOUS_CAPABILITY, ReadinessClass, SequenceAdmittedReady}; use crate::deployment::{ CapabilityDependency, DeadlineBinding, DependencyKind, ExternalCapabilityBinding, OperatorBinding, ServiceTransport, SourceSelectionPolicy, StartupOrder, StateDeclaration, TargetDeclaration, @@ -640,6 +640,25 @@ impl CompiledDeploymentPlan { Ok((declaration, binding)) } + /// How this plan's target proves readiness, from its recipe's declarations + /// and its binding's route, or the typed refusal that it declares none. + /// Admission asks this before anything is sealed or installed. + pub(crate) fn readiness_class(&self) -> Result { + let (declaration, binding) = self.parsed_inputs()?; + Ok(ReadinessClass::declared( + &declaration.target, + declaration + .provides + .iter() + .any(|provided| provided.capability == ODIN_RENDEZVOUS_CAPABILITY), + declaration.dependencies.iter().any(|dependency| { + dependency.kind == DependencyKind::SharedInfrastructure + && dependency.capability == ODIN_RENDEZVOUS_CAPABILITY + }), + binding.route.is_some(), + )?) + } + fn recomputed_plan_id(&self) -> Result { let mut unsigned = self.clone(); unsigned.plan_id.clear(); @@ -1547,6 +1566,71 @@ nodes = ["yggdrasil"] ); } + #[test] + fn a_plan_names_how_its_target_proves_readiness_and_agrees_with_its_expected() { + use crate::control_plane::UndeclaredReadiness; + let unrouted_binding = { + let (head, tail) = BINDING.split_once("[route]").unwrap(); + format!("{head}[brakes]{}", tail.split_once("[brakes]").unwrap().1) + }; + let unrouted = |recipe: &str| { + recipe + .replace("route_required = true", "route_required = false") + .replace( + r#"["GAMECULT_IDUNN_CANDIDATE_BIND", "GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, + r#"["GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, + ) + }; + let without_odin = RECIPE.split("[[dependencies]]").next().unwrap().to_owned(); + let provides_odin = |recipe: &str| { + recipe.replace("capability = \"service.runtime\"", "capability = \"odin.verse-rendezvous\"") + }; + let cases: [(&str, String, &str, Option, Result); 6] = [ + ("depends on Odin, routed", RECIPE.to_owned(), BINDING, Some(18001), Ok(ReadinessClass::OdinCorrelated)), + ("routed, no Odin", without_odin.clone(), BINDING, Some(18001), Ok(ReadinessClass::RouteProof)), + ("unrouted, no Odin", unrouted(&without_odin), &unrouted_binding, None, Err(())), + ("provides Odin, unrouted", provides_odin(&unrouted(&without_odin)), &unrouted_binding, None, Ok(ReadinessClass::OdinSelf)), + ("depends on Odin, unrouted", unrouted(RECIPE), &unrouted_binding, None, Ok(ReadinessClass::OdinCorrelated)), + // Only the shared-infrastructure declaration is the declaration. + ("requires the Odin capability, routed", RECIPE.replace("kind = \"shared-infrastructure\"", "kind = \"required\""), BINDING, Some(18001), Ok(ReadinessClass::RouteProof)), + ]; + for (name, recipe, binding, port, outcome) in cases { + let providers = [ready_odin_provider("odin", "odin-yggdrasil", 1)]; + let plan = compile_deployment_plan( + recipe.as_bytes(), + binding.as_bytes(), + source(&recipe), + "service-incarnation-1", + port, + 110, + if recipe.contains("[[dependencies]]") { &providers } else { &[] }, + ) + .unwrap_or_else(|error| panic!("{name}: {error:#}")); + let release = SealedRelease::new( + &plan, + vec![artifact_receipt()], + vec![external_input_receipt()], + 120, + ) + .unwrap(); + let expected = release.expected_projection(&plan).unwrap(); + match outcome { + Ok(class) => { + assert_eq!(plan.readiness_class().unwrap(), class, "{name}"); + assert_eq!(ReadinessClass::of(&expected), Ok(class), "{name}"); + } + Err(()) => { + let error = plan.readiness_class().expect_err(name); + let undeclared = UndeclaredReadiness { + target: "service".into(), + }; + assert_eq!(error.downcast_ref::(), Some(&undeclared), "{name}"); + assert_eq!(ReadinessClass::of(&expected), Err(undeclared), "{name}"); + } + } + } + } + #[test] fn expected_projection_is_sanitized_and_names_the_selected_graph() { let plan = plan(); diff --git a/src/drivers.rs b/src/drivers.rs index d56e3d9..400566b 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; use std::ffi::{OsStr, OsString}; use std::fs::{self, OpenOptions}; -use std::io::{BufRead, ErrorKind, Read, Write}; +use std::io::{BufRead, BufReader, ErrorKind, Read, Write}; use std::net::{SocketAddr, TcpStream}; use std::path::{Path, PathBuf}; use std::process::{Command, Output, Stdio}; @@ -700,6 +700,7 @@ const ROUTE_CONNECT_RETRY_WINDOW: Duration = Duration::from_secs(2); const ROUTE_CONNECT_RETRY_INTERVAL: Duration = Duration::from_millis(25); const ROUTE_SNAPSHOT_MAX_BYTES: usize = 1024 * 1024; const ROUTE_HTTP_MAX_HEADER_BYTES: usize = 32 * 1024; +const ROUTE_HTTP_CHUNK_LINE_BYTES: usize = 128; const ROUTE_HTTP_SNAPSHOT_PATH: &str = "/cultnet/snapshot"; impl RoutePreflightReceipt { @@ -5128,7 +5129,7 @@ impl NginxRouteDriver { Ok(output) } - fn render(&self, expected: &IdunnExpectedIncarnationRecord) -> Result> { + pub(crate) fn render(&self, expected: &IdunnExpectedIncarnationRecord) -> Result> { expected.validate()?; let expected_projection_sha256 = expected.canonical_sha256()?; let route = expected @@ -5453,7 +5454,7 @@ impl NginxRouteDriver { && route.stable_endpoint == self.binding.stable_endpoint, "route challenge binding differs from Expected" ); - self.request_runtime_presence_at(expected, message_id, &route.stable_endpoint) + Ok(self.request_runtime_presence_at(expected, message_id, &route.stable_endpoint)?) } /// The bootstrap exception for the first managed Odin observes Warming on @@ -5462,16 +5463,14 @@ impl NginxRouteDriver { &self, expected: &IdunnExpectedIncarnationRecord, message_id: &str, - ) -> Result { + ) -> Result { let route = expected .route .as_ref() - .context("candidate challenge has no Expected route")?; - ensure!( - route.route_id == self.binding.route_id, - "candidate challenge binding differs from Expected" - ); - self.render(expected)?; + .context("candidate challenge has no Expected route") + .map_err(refused)?; + // Rendering holds the binding to the Expected route, id and endpoints. + self.render(expected).map_err(refused)?; self.request_runtime_presence_at(expected, message_id, &route.candidate_endpoint) } @@ -5480,13 +5479,14 @@ impl NginxRouteDriver { expected: &IdunnExpectedIncarnationRecord, message_id: &str, endpoint: &str, - ) -> Result { - expected.validate()?; - require_driver_id(message_id, "route challenge message")?; + ) -> Result { + expected.validate().map_err(refused)?; + require_driver_id(message_id, "route challenge message").map_err(refused)?; let route = expected .route .as_ref() - .context("route challenge has no Expected route")?; + .context("route challenge has no Expected route") + .map_err(refused)?; let request = CultNetMessage::SnapshotRequest { message_id: message_id.to_owned(), schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), @@ -5496,27 +5496,34 @@ impl NginxRouteDriver { "http" => "http://", "tcp" => "tcp://", "rudp" => "rudp://", - _ => bail!("route challenge transport is unsupported"), + _ => return Err(refused(anyhow!("route challenge transport is unsupported"))), }; - let (host, port) = endpoint_host_port(endpoint, endpoint_prefix)?; + let (host, port) = endpoint_host_port(endpoint, endpoint_prefix).map_err(refused)?; let target = SocketAddr::new( host.parse() - .context("route challenge endpoint host is not an IP address")?, + .context("route challenge endpoint host is not an IP address") + .map_err(refused)?, port, ); let response = match route.transport.as_str() { "http" => { let payload = - encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0)?; + encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0) + .map_err(refused)?; let response = request_http_snapshot(target, &payload)?; - decode_cultnet_message_from_slice(&response, CultNetWireContract::CultNetSchemaV0)? + decode_cultnet_message_from_slice(&response, CultNetWireContract::CultNetSchemaV0) + .map_err(refused)? } "tcp" => { let payload = - encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0)?; + encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0) + .map_err(refused)?; let response = request_tcp_snapshot(target, &payload)?; - decode_cultnet_message_from_slice(&response, CultNetWireContract::CultNetSchemaV0)? + decode_cultnet_message_from_slice(&response, CultNetWireContract::CultNetSchemaV0) + .map_err(refused)? } + // The RUDP client reports every failure as prose, so a bad answer + // cannot be told from no answer here: all of it reads as silence. "rudp" => request_raw_snapshot_from_rudp_catalog(CultMeshRudpSnapshotOptions { target, runtime_id: "idunn-route-observer".into(), @@ -5524,10 +5531,11 @@ impl NginxRouteDriver { schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), record_keys: Some(vec![expected.target.clone()]), ..CultMeshRudpSnapshotOptions::default() - })?, - _ => bail!("route challenge transport is unsupported"), + }) + .map_err(silent)?, + _ => return Err(refused(anyhow!("route challenge transport is unsupported"))), }; - exact_route_snapshot_response(response, message_id, &expected.target) + exact_route_snapshot_response(response, message_id, &expected.target).map_err(refused) } /// Restore the exact preflight baseline. This may only replace the exact @@ -5555,103 +5563,298 @@ impl NginxRouteDriver { } } -fn request_tcp_snapshot(target: SocketAddr, payload: &[u8]) -> Result> { - let mut stream = connect_route_socket(target) - .with_context(|| format!("connecting stable CultNet TCP route {target}"))?; - stream.set_read_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT))?; - stream.set_write_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT))?; - stream.write_all(&encode_frame(payload)?)?; - stream.flush()?; +/// Why a direct presence challenge produced no document to authenticate. +#[derive(Debug)] +pub enum ChallengeFailure { + /// Nothing answered: the connection was refused, or no byte arrived before + /// the timeout, or the peer closed with zero bytes. A process still + /// starting does this, and it is waiting, not a fault. Once any byte has + /// arrived, a stall or a stop is `Refused`, not this. + Silent(anyhow::Error), + /// Something answered wrongly, or the challenge could not be made: a + /// malformed or foreign answer, an oversized one, or a local configuration + /// error. Waiting does not fix it. + Refused(anyhow::Error), +} +impl std::fmt::Display for ChallengeFailure { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Silent(error) | Self::Refused(error) => write!(formatter, "{error:#}"), + } + } +} + +impl std::error::Error for ChallengeFailure { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Silent(error) | Self::Refused(error) => Some(error.as_ref()), + } + } +} + +fn silent(error: impl Into) -> ChallengeFailure { + ChallengeFailure::Silent(error.into()) +} + +fn refused(error: impl Into) -> ChallengeFailure { + ChallengeFailure::Refused(error.into()) +} + +fn ensure_or_refuse(condition: bool, message: &'static str) -> Result<(), ChallengeFailure> { + if condition { + Ok(()) + } else { + Err(refused(anyhow!(message))) + } +} + +/// One answer as it arrives, over either transport. A peer that stops is judged +/// the same way on both: no byte at all is silence (a process still starting +/// does that), and anything short of a whole answer after some bytes is a bad +/// answer. Waiting does not finish an answer that has begun and stopped. +struct Answer { + source: R, + seen: usize, +} + +impl Answer { + fn new(source: R) -> Self { + Self { source, seen: 0 } + } + + fn stopped(&self, what: &str) -> ChallengeFailure { + let error = anyhow!("stable CultNet route stopped {what}"); + if self.seen == 0 { + silent(error) + } else { + refused(error) + } + } + + fn read_some(&mut self, buffer: &mut [u8], what: &str) -> Result { + loop { + match self.source.read(buffer) { + Ok(0) => return Err(self.stopped(what)), + Ok(read) => { + self.seen += read; + return Ok(read); + } + Err(error) if error.kind() == ErrorKind::Interrupted => {} + Err(_) => return Err(self.stopped(what)), + } + } + } + + fn read_all(&mut self, buffer: &mut [u8], what: &str) -> Result<(), ChallengeFailure> { + let mut filled = 0; + while filled < buffer.len() { + filled += self.read_some(&mut buffer[filled..], what)?; + } + Ok(()) + } + + /// One CRLF-terminated line, without its terminator. + fn read_line(&mut self, bound: usize, what: &str) -> Result, ChallengeFailure> { + let mut line = Vec::new(); + loop { + let mut byte = [0_u8]; + self.read_all(&mut byte, what)?; + line.push(byte[0]); + if line.ends_with(b"\r\n") { + line.truncate(line.len() - 2); + return Ok(line); + } + ensure_or_refuse( + line.len() <= bound, + "stable CultNet HTTP response line exceeds its bound", + )?; + } + } +} + +fn request_tcp_snapshot(target: SocketAddr, payload: &[u8]) -> Result, ChallengeFailure> { + let mut stream = connect_route_socket(target) + .with_context(|| format!("connecting stable CultNet TCP route {target}")) + .map_err(silent)?; + stream + .set_read_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT)) + .map_err(refused)?; + stream + .set_write_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT)) + .map_err(refused)?; + stream + .write_all(&encode_frame(payload).map_err(refused)?) + .and_then(|()| stream.flush()) + .map_err(silent)?; + + let mut answer = Answer::new(stream); let mut header = [0_u8; 4]; - stream - .read_exact(&mut header) - .context("reading stable CultNet TCP response frame")?; + answer.read_all(&mut header, "before its response frame header ended")?; let length = u32::from_be_bytes(header) as usize; - ensure!( + ensure_or_refuse( (1..=ROUTE_SNAPSHOT_MAX_BYTES).contains(&length), - "stable CultNet TCP response exceeds the route observation bound" - ); + "stable CultNet TCP response exceeds the route observation bound", + )?; let mut response = vec![0_u8; length]; - stream - .read_exact(&mut response) - .context("reading stable CultNet TCP response payload")?; + answer.read_all(&mut response, "before its response frame ended")?; Ok(response) } -fn request_http_snapshot(target: SocketAddr, payload: &[u8]) -> Result> { +fn request_http_snapshot(target: SocketAddr, payload: &[u8]) -> Result, ChallengeFailure> { let mut stream = connect_route_socket(target) - .with_context(|| format!("connecting stable CultNet HTTP route {target}"))?; - stream.set_read_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT))?; - stream.set_write_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT))?; + .with_context(|| format!("connecting stable CultNet HTTP route {target}")) + .map_err(silent)?; + stream + .set_read_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT)) + .map_err(refused)?; + stream + .set_write_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT)) + .map_err(refused)?; let head = format!( "POST {ROUTE_HTTP_SNAPSHOT_PATH} HTTP/1.1\r\nHost: {target}\r\nContent-Type: application/msgpack\r\nAccept: application/msgpack\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", payload.len() ); - stream.write_all(head.as_bytes())?; - stream.write_all(payload)?; - stream.flush()?; + stream + .write_all(head.as_bytes()) + .and_then(|()| stream.write_all(payload)) + .and_then(|()| stream.flush()) + .map_err(silent)?; + + // The response is read by its own framing, never to end of stream: a peer + // that ignores `Connection: close` and keeps the socket open has still + // answered in full. + let mut answer = Answer::new(BufReader::new(stream)); + let mut head_bytes = 0_usize; + // Headers and chunked trailers draw on the one budget. + let mut head_line = |answer: &mut Answer>, what: &str| { + let line = answer.read_line(ROUTE_HTTP_MAX_HEADER_BYTES, what)?; + head_bytes += line.len() + 2; + ensure_or_refuse( + head_bytes <= ROUTE_HTTP_MAX_HEADER_BYTES, + "stable CultNet HTTP response headers exceed their bound", + )?; + Ok::<_, ChallengeFailure>(line) + }; + let headers = "before its headers ended"; + let status = head_line(&mut answer, headers)?; + let status = std::str::from_utf8(&status) + .context("stable CultNet HTTP response status is not UTF-8") + .map_err(refused)?; + if !is_http_success(status) { + return Err(refused(anyhow!( + "stable CultNet HTTP route rejected the snapshot challenge: {status}" + ))); + } - let maximum = ROUTE_HTTP_MAX_HEADER_BYTES - .checked_add(ROUTE_SNAPSHOT_MAX_BYTES) - .context("route HTTP response bound overflow")?; - let mut response = Vec::new(); - (&mut stream) - .take((maximum + 1) as u64) - .read_to_end(&mut response) - .context("reading stable CultNet HTTP response")?; - ensure!( - response.len() <= maximum, - "stable CultNet HTTP response exceeds the route observation bound" - ); - let header_end = response - .windows(4) - .position(|window| window == b"\r\n\r\n") - .map(|index| index + 4) - .context("stable CultNet HTTP response has no header boundary")?; - ensure!( - header_end <= ROUTE_HTTP_MAX_HEADER_BYTES, - "stable CultNet HTTP response headers exceed their bound" - ); - let headers = std::str::from_utf8(&response[..header_end]) - .context("stable CultNet HTTP response headers are not UTF-8")?; - let mut lines = headers.split("\r\n"); - let status = lines - .next() - .context("stable CultNet HTTP response has no status")?; - ensure!( - matches!(status, "HTTP/1.1 200 OK" | "HTTP/1.0 200 OK"), - "stable CultNet HTTP route rejected the snapshot challenge: {status}" - ); let mut content_length = None; - for line in lines.filter(|line| !line.is_empty()) { + let mut chunked = false; + loop { + let line = head_line(&mut answer, headers)?; + if line.is_empty() { + break; + } + let line = std::str::from_utf8(&line) + .context("stable CultNet HTTP response headers are not UTF-8") + .map_err(refused)?; let (name, value) = line .split_once(':') - .context("stable CultNet HTTP response contains a malformed header")?; + .context("stable CultNet HTTP response contains a malformed header") + .map_err(refused)?; + // RFC 9112 5.1: a name with whitespace around it (or none) is not a + // field name, and a recipient must not guess which one was meant. + ensure_or_refuse( + !name.is_empty() && !name.bytes().any(|byte| byte.is_ascii_whitespace()), + "stable CultNet HTTP response has a malformed header name", + )?; if name.eq_ignore_ascii_case("transfer-encoding") { - bail!("stable CultNet HTTP response uses unsupported transfer encoding"); + ensure_or_refuse( + value.trim().eq_ignore_ascii_case("chunked"), + "stable CultNet HTTP response uses unsupported transfer encoding", + )?; + chunked = true; } if name.eq_ignore_ascii_case("content-length") { - ensure!( + ensure_or_refuse( content_length.is_none(), - "stable CultNet HTTP response repeats Content-Length" - ); + "stable CultNet HTTP response repeats Content-Length", + )?; content_length = Some( - value - .trim() - .parse::() - .context("stable CultNet HTTP Content-Length is invalid")?, + parse_digits(value.trim_matches([' ', '\t']), 10) + .context("stable CultNet HTTP Content-Length is invalid") + .map_err(refused)?, ); } } - let content_length = - content_length.context("stable CultNet HTTP response has no Content-Length")?; - ensure!( - content_length <= ROUTE_SNAPSHOT_MAX_BYTES - && response.len().saturating_sub(header_end) == content_length, - "stable CultNet HTTP response body differs from its bounded Content-Length" - ); - Ok(response[header_end..].to_vec()) + + let body = "before its body ended"; + match (content_length, chunked) { + (Some(length), false) => { + ensure_or_refuse( + length <= ROUTE_SNAPSHOT_MAX_BYTES, + "stable CultNet HTTP response body exceeds the route observation bound", + )?; + let mut response = vec![0_u8; length]; + answer.read_all(&mut response, body)?; + Ok(response) + } + (None, true) => { + let mut response = Vec::new(); + loop { + let size = answer.read_line(ROUTE_HTTP_CHUNK_LINE_BYTES, body)?; + let size = std::str::from_utf8(&size) + .ok() + .and_then(|size| size.split(';').next()) + .and_then(|size| parse_digits(size.trim_matches([' ', '\t']), 16)) + .context("stable CultNet HTTP chunk size is invalid") + .map_err(refused)?; + if size == 0 { + // Trailers, if any, end at the empty line. + while !head_line(&mut answer, body)?.is_empty() {} + return Ok(response); + } + ensure_or_refuse( + response.len().saturating_add(size) <= ROUTE_SNAPSHOT_MAX_BYTES, + "stable CultNet HTTP response body exceeds the route observation bound", + )?; + let start = response.len(); + response.resize(start + size, 0); + answer.read_all(&mut response[start..], body)?; + let mut end = [0_u8; 2]; + answer.read_all(&mut end, body)?; + ensure_or_refuse(&end == b"\r\n", "stable CultNet HTTP chunk is not terminated")?; + } + } + _ => Err(refused(anyhow!( + "stable CultNet HTTP response has no single body framing" + ))), + } +} + +/// A number written only in digits of `radix`: no sign, no whitespace, not +/// empty. `from_str_radix` alone accepts a leading `+`, which RFC 9112 does +/// not. +fn parse_digits(text: &str, radix: u32) -> Option { + if text.is_empty() || !text.chars().all(|digit| digit.is_digit(radix)) { + return None; + } + usize::from_str_radix(text, radix).ok() +} + +/// A 2xx status line of HTTP/1.0 or HTTP/1.1. +fn is_http_success(status: &str) -> bool { + let Some(rest) = status + .strip_prefix("HTTP/1.1 ") + .or_else(|| status.strip_prefix("HTTP/1.0 ")) + else { + return false; + }; + let code = rest.as_bytes(); + code.len() >= 3 + && code[0] == b'2' + && code[1].is_ascii_digit() + && code[2].is_ascii_digit() + && (code.len() == 3 || code[3] == b' ') } /// nginx reload is signal-driven: `systemctl reload` can return before its @@ -8034,6 +8237,944 @@ fn apply_identity(command: &mut Command, identity: Option) -> R mod tests { use super::*; + // --------------------------------------------------------------------- + // A direct challenge fails one of two ways. Silence waits; a bad answer, + // or a challenge that cannot be made, is an error. + // --------------------------------------------------------------------- + + /// A peer that reads one snapshot request and writes back whatever `reply` + /// builds from the challenge id, then closes. Returns the port it listens on. + fn scripted_peer(reply: impl FnOnce(&str) -> Vec + Send + 'static) -> u16 { + lingering_peer(reply, Duration::ZERO) + } + + /// `scripted_peer`, except the socket stays open for `linger` after the + /// answer: keep-alive, ignoring `Connection: close`. + fn lingering_peer( + reply: impl FnOnce(&str) -> Vec + Send + 'static, + linger: Duration, + ) -> u16 { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("a free port"); + let port = listener.local_addr().expect("a bound port").port(); + thread::spawn(move || { + let Ok((mut stream, _)) = listener.accept() else { + return; + }; + let mut request = Vec::new(); + let mut chunk = [0_u8; 4096]; + let message_id = loop { + let Ok(read) = stream.read(&mut chunk) else { + return; + }; + if read == 0 { + return; + } + request.extend_from_slice(&chunk[..read]); + let Some(end) = request.windows(4).position(|window| window == b"\r\n\r\n") else { + continue; + }; + if let Ok(CultNetMessage::SnapshotRequest { message_id, .. }) = + decode_cultnet_message_from_slice( + &request[end + 4..], + CultNetWireContract::CultNetSchemaV0, + ) + { + break message_id; + } + }; + let _ = stream.write_all(&reply(&message_id)); + let _ = stream.flush(); + thread::sleep(linger); + }); + port + } + + fn http_ok(body: &[u8]) -> Vec { + let mut response = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .into_bytes(); + response.extend_from_slice(body); + response + } + + fn raw_snapshot(message_id: &str, schema: &str, key: &str, payload: Vec) -> Vec { + http_ok( + &encode_cultnet_message_to_vec( + &CultNetMessage::SnapshotResponseRaw { + message_id: message_id.into(), + documents: vec![cultnet_rs::CultNetRawDocumentRecord { + schema_id: schema.into(), + record_key: key.into(), + stored_at: "1970-01-01T00:00:00.100Z".into(), + payload_encoding: cultnet_rs::CultNetRawPayloadEncoding::Messagepack, + payload, + source_runtime_id: None, + source_agent_id: None, + source_role: None, + tags: None, + }], + }, + CultNetWireContract::CultNetSchemaV0, + ) + .expect("an encodable response"), + ) + } + + fn honest(message_id: &str) -> Vec { + raw_snapshot( + message_id, + cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, + "service", + vec![1, 2, 3], + ) + } + + /// One candidate challenge to `port`, with the Expected adjusted by `adjust`. + fn challenge( + port: u16, + adjust: impl FnOnce(&mut IdunnExpectedIncarnationRecord), + ) -> Result { + challenge_over("http", RouteDriver::NginxStreamTcp, port, adjust) + } + + /// The same challenge over another transport: `transport` names both the + /// Expected's transport and the endpoint scheme. + fn challenge_over( + transport: &str, + driver: RouteDriver, + port: u16, + adjust: impl FnOnce(&mut IdunnExpectedIncarnationRecord), + ) -> Result { + let temp = tempfile::tempdir().expect("a temporary directory"); + let mut candidate = expected(); + candidate.route = Some(cultnet_rs::IdunnExpectedRoute { + route_id: "service".into(), + transport: transport.into(), + stable_endpoint: format!("{transport}://127.0.0.1:17999"), + candidate_endpoint: format!("{transport}://127.0.0.1:{port}"), + }); + adjust(&mut candidate); + NginxRouteDriver::new(RouteBinding { + driver, + route_id: "service".into(), + stable_endpoint: format!("{transport}://127.0.0.1:17999"), + private_host: "127.0.0.1".into(), + private_port_start: port, + private_port_end: port, + config_path: temp.path().join("service.conf"), + reload_unit: "nginx.service".into(), + }) + .request_candidate_runtime_presence(&candidate, "candidate-probe") + } + /// A TCP peer that reads one framed snapshot request and writes back the + /// bytes `reply` builds from the challenge id, verbatim. + fn framed_peer(reply: impl FnOnce(&str) -> Vec + Send + 'static) -> u16 { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("a free port"); + let port = listener.local_addr().expect("a bound port").port(); + thread::spawn(move || { + let Ok((mut stream, _)) = listener.accept() else { + return; + }; + let mut header = [0_u8; 4]; + if stream.read_exact(&mut header).is_err() { + return; + } + let mut request = vec![0_u8; u32::from_be_bytes(header) as usize]; + if stream.read_exact(&mut request).is_err() { + return; + } + let Ok(CultNetMessage::SnapshotRequest { message_id, .. }) = + decode_cultnet_message_from_slice(&request, CultNetWireContract::CultNetSchemaV0) + else { + return; + }; + let _ = stream.write_all(&reply(&message_id)); + }); + port + } + + /// The body of an `honest` HTTP answer, framed for TCP. + fn framed_honest(message_id: &str) -> Vec { + let response = honest(message_id); + let body = response + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map(|end| &response[end + 4..]) + .expect("an HTTP answer"); + encode_frame(body).expect("a frame") + } + + fn tcp_challenge(port: u16) -> Result { + challenge_over("tcp", RouteDriver::NginxStreamTcp, port, |_| {}) + } + + #[test] + fn a_tcp_challenge_takes_the_exact_framed_answer_and_waits_only_on_silence() { + let answer = tcp_challenge(framed_peer(framed_honest)).expect("the exact answer"); + assert_eq!(answer.message_id, "candidate-probe"); + assert_eq!(answer.canonical_presence, [1, 2, 3]); + + assert_silent(tcp_challenge(framed_peer(|_| Vec::new())), "a peer that hangs up"); + // Some bytes and then nothing is a bad answer, not a late one. + assert_refused( + tcp_challenge(framed_peer(|_| vec![0, 0, 0, 9, 1])), + "a frame that never finishes", + ); + + // A frame that claims more than the observation bound, or nothing, is + // an answer that is wrong, not one that is late. + assert_refused( + tcp_challenge(framed_peer(|_| { + ((ROUTE_SNAPSHOT_MAX_BYTES + 1) as u32).to_be_bytes().to_vec() + })), + "a frame over the bound", + ); + assert_refused( + tcp_challenge(framed_peer(|_| 0_u32.to_be_bytes().to_vec())), + "an empty frame", + ); + assert_refused( + tcp_challenge(framed_peer(|_| encode_frame(b"not cultnet").expect("a frame"))), + "a frame that is not CultNet", + ); + } + + #[test] + fn a_rudp_challenge_that_nothing_answers_is_silence() { + let socket = std::net::UdpSocket::bind("127.0.0.1:0").expect("a free port"); + let port = socket.local_addr().expect("a bound port").port(); + // Bound but never read: nothing answers, and nothing refuses either. + assert_silent( + challenge_over("rudp", RouteDriver::NginxStreamUdp, port, |_| {}), + "a rudp peer that never answers", + ); + drop(socket); + } + + /// A challenge that cannot be made says why, whatever the message. + #[test] + fn a_challenge_failure_shows_and_chains_its_cause() { + for failure in [ + silent(anyhow!("cause of silence")), + refused(anyhow!("cause of refusal")), + ] { + let shown = failure.to_string(); + assert!(shown.starts_with("cause of "), "{shown}"); + let source = std::error::Error::source(&failure).expect("the underlying error"); + assert_eq!(source.to_string(), shown); + } + } + + + // --------------------------------------------------------------------- + // One rule for a peer that stops, on either transport: no byte at all is + // silence; some bytes and then nothing is a bad answer. An answer is read + // by its own framing, so a peer that keeps the socket open has still + // answered. + // --------------------------------------------------------------------- + + fn http_chunked(body: &[u8], chunk: usize, trailer: bool) -> Vec { + let mut response = b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n".to_vec(); + for piece in body.chunks(chunk) { + response.extend_from_slice(format!("{:x};ext=1\r\n", piece.len()).as_bytes()); + response.extend_from_slice(piece); + response.extend_from_slice(b"\r\n"); + } + response.extend_from_slice(if trailer { + b"0\r\nTrailer: x\r\n\r\n" + } else { + b"0\r\n\r\n" + }); + response + } + + fn honest_body(message_id: &str) -> Vec { + let response = honest(message_id); + let end = response + .windows(4) + .position(|window| window == b"\r\n\r\n") + .expect("an HTTP answer"); + response[end + 4..].to_vec() + } + + const LINGER: Duration = Duration::from_secs(6); + + #[test] + fn an_http_answer_is_read_by_its_framing_not_to_the_end_of_the_stream() { + let started = Instant::now(); + let answer = challenge(lingering_peer(honest, LINGER), |_| {}).expect("a whole answer"); + assert_eq!(answer.canonical_presence, [1, 2, 3]); + assert!(started.elapsed() < LINGER / 2, "the challenge waited on the socket"); + + for trailer in [true, false] { + let started = Instant::now(); + let answer = challenge( + lingering_peer(move |id| http_chunked(&honest_body(id), 7, trailer), LINGER), + |_| {}, + ) + .expect("a chunked answer"); + assert_eq!(answer.canonical_presence, [1, 2, 3]); + assert!(started.elapsed() < LINGER / 2, "the challenge waited on the socket"); + } + } + + #[test] + fn an_http_status_that_is_not_success_is_refused_even_when_the_peer_lingers() { + for status in [ + "404 Not Found", + "301 Moved Permanently", + "503 Service Unavailable", + "199 Odd", + ] { + // The body is the honest answer: only the status refuses it. + let reply = move |id: &str| { + let body = honest_body(id); + let mut reply = + format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\n\r\n", body.len()) + .into_bytes(); + reply.extend_from_slice(&body); + reply + }; + let started = Instant::now(); + assert_refused(challenge(lingering_peer(reply, LINGER), |_| {}), status); + assert!(started.elapsed() < LINGER / 2, "{status} waited on the socket"); + } + } + + #[test] + fn only_a_two_hundred_series_http_1_status_line_is_success() { + for status in ["HTTP/1.1 200 OK", "HTTP/1.0 200 OK", "HTTP/1.1 204", "HTTP/1.1 299 x"] { + assert!(is_http_success(status), "{status}"); + } + for status in [ + "", + "HTTP/2 200 OK", + "HTTP/1.1 ", + "HTTP/1.1 20", + "HTTP/1.1 2x0 OK", + "HTTP/1.1 20x OK", + "HTTP/1.1 2000 OK", + "HTTP/1.1 300 OK", + "HTTP/1.1 404 Not Found", + "200 OK", + ] { + assert!(!is_http_success(status), "{status:?}"); + } + } + + #[test] + fn a_peer_that_sent_some_bytes_and_stopped_is_refused_on_either_transport() { + // No byte at all is silence, on both. + assert_silent(challenge(scripted_peer(|_| Vec::new()), |_| {}), "an HTTP peer with nothing to say"); + assert_silent(tcp_challenge(framed_peer(|_| Vec::new())), "a TCP peer with nothing to say"); + + // Some bytes and then nothing is a bad answer, wherever it stops. + let head = b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n"; + for (why, bytes) in [ + ("a status line that stops", b"HTTP/1.1 2".to_vec()), + ("headers that stop", b"HTTP/1.1 200 OK\r\nContent-Le".to_vec()), + ("a body that stops", [head.as_slice(), b"abc"].concat()), + ( + "a chunk that stops", + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nab".to_vec(), + ), + ( + "a chunked body that never ends", + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n1\r\na\r\n".to_vec(), + ), + ] { + assert_refused(challenge(scripted_peer(move |_| bytes), |_| {}), why); + } + for (why, bytes) in [ + ("a 3-byte banner", b"OK\n".to_vec()), + ("a frame header that stops", vec![0, 0]), + ("a frame that stops", vec![0, 0, 0, 9, 1]), + ] { + assert_refused(tcp_challenge(framed_peer(move |_| bytes)), why); + } + } + + #[test] + fn an_http_answer_that_frames_its_body_wrongly_is_refused() { + for (why, reply) in [ + ( + "an unsupported transfer encoding", + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: gzip\r\n\r\n".to_vec(), + ), + ( + "both framings", + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nContent-Length: 1\r\n\r\n0\r\n\r\n" + .to_vec(), + ), + ("no framing", b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nabc".to_vec()), + ( + "a repeated Content-Length", + b"HTTP/1.1 200 OK\r\nContent-Length: 1\r\nContent-Length: 1\r\n\r\na".to_vec(), + ), + ( + "an invalid Content-Length", + b"HTTP/1.1 200 OK\r\nContent-Length: many\r\n\r\n".to_vec(), + ), + ( + "a Content-Length over the bound", + format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n", + ROUTE_SNAPSHOT_MAX_BYTES + 1 + ) + .into_bytes(), + ), + ("a malformed header", b"HTTP/1.1 200 OK\r\nno colon here\r\n\r\n".to_vec()), + ( + "an invalid chunk size", + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\nzz\r\n".to_vec(), + ), + ( + "an unterminated chunk", + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n1\r\naXX0\r\n\r\n".to_vec(), + ), + ( + "chunks over the bound", + format!( + "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n{:x}\r\n", + ROUTE_SNAPSHOT_MAX_BYTES + 1 + ) + .into_bytes(), + ), + ( + "one header line over the bound", + [ + b"HTTP/1.1 200 OK\r\nX: ".to_vec(), + vec![b'a'; ROUTE_HTTP_MAX_HEADER_BYTES + 1], + ] + .concat(), + ), + ("headers that are not UTF-8", b"HTTP/1.1 200 OK\r\nX: \xff\r\n\r\n".to_vec()), + ("a status that is not UTF-8", b"HTTP/1.1 \xff\r\n\r\n".to_vec()), + ] { + assert_refused(challenge(scripted_peer(move |_| reply), |_| {}), why); + } + } + + /// Every header line is small and the answer is honest and whole; only the + /// headers' total size refuses it. + #[test] + fn headers_that_are_over_their_bound_together_refuse_an_otherwise_honest_answer() { + let filler = b"X: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\n"; + let answer_with = |lines: usize| { + move |id: &str| { + let body = honest_body(id); + let mut reply = + format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n", body.len()).into_bytes(); + reply.extend_from_slice(&filler.repeat(lines)); + reply.extend_from_slice(b"\r\n"); + reply.extend_from_slice(&body); + reply + } + }; + let lines = ROUTE_HTTP_MAX_HEADER_BYTES / filler.len(); + // Just inside the bound, then just outside it (a whole line over). + challenge(scripted_peer(answer_with(lines - 2)), |_| {}).expect("headers inside the bound"); + assert_refused( + challenge(scripted_peer(answer_with(lines + 1)), |_| {}), + "headers over the bound together", + ); + } + + // --------------------------------------------------------------------- + // The reader's limits are pinned at their edges, on answers that are + // honest in every other way, so only the limit under test can refuse them + // and a peer that simply stops is not what refuses. + // --------------------------------------------------------------------- + + const CHUNKED_HEAD: &[u8] = b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n"; + + fn snapshot_request() -> Vec { + encode_cultnet_message_to_vec( + &CultNetMessage::SnapshotRequest { + message_id: "probe".into(), + schema_ids: None, + record_keys: None, + }, + CultNetWireContract::CultNetSchemaV0, + ) + .expect("an encodable request") + } + + /// The raw body the HTTP reader hands back, with no document judgment. + fn http_body( + reply: impl FnOnce(&str) -> Vec + Send + 'static, + ) -> Result, ChallengeFailure> { + let port = scripted_peer(reply); + request_http_snapshot(SocketAddr::from(([127, 0, 0, 1], port)), &snapshot_request()) + } + + fn tcp_body( + reply: impl FnOnce(&str) -> Vec + Send + 'static, + ) -> Result, ChallengeFailure> { + let port = framed_peer(reply); + request_tcp_snapshot(SocketAddr::from(([127, 0, 0, 1], port)), &snapshot_request()) + } + + fn assert_body_refused(outcome: Result, ChallengeFailure>, why: &str) { + assert!( + matches!(outcome, Err(ChallengeFailure::Refused(_))), + "{why} must be refused: {:?}", + outcome.map(|body| body.len()) + ); + } + + fn chunked(pieces: &[usize]) -> Vec { + let mut reply = CHUNKED_HEAD.to_vec(); + for size in pieces { + reply.extend_from_slice(format!("{size:x}\r\n").as_bytes()); + reply.extend(std::iter::repeat_n(b'a', *size)); + reply.extend_from_slice(b"\r\n"); + } + reply.extend_from_slice(b"0\r\n\r\n"); + reply + } + + #[test] + fn every_framing_takes_exactly_the_bound_and_refuses_one_byte_more() { + let most = ROUTE_SNAPSHOT_MAX_BYTES; + // Whole answers are sent in every case, so only the bound refuses. + let http_ok_of = |length: usize| move |_: &str| http_ok(&vec![b'a'; length]); + assert_eq!(http_body(http_ok_of(most)).expect("Content-Length at the bound").len(), most); + assert_body_refused(http_body(http_ok_of(most + 1)), "Content-Length over the bound"); + + for (pieces, taken) in [ + (vec![most], true), + (vec![most + 1], false), + (vec![most - 1, 1], true), + (vec![most - 1, 2], false), + ] { + let outcome = http_body(move |_| chunked(&pieces)); + if taken { + assert_eq!(outcome.expect("chunks at the bound").len(), most); + } else { + assert_body_refused(outcome, "chunks over the bound"); + } + } + + let framed = |length: usize| move |_: &str| encode_frame(&vec![b'a'; length]).expect("a frame"); + assert_eq!(tcp_body(framed(most)).expect("a frame at the bound").len(), most); + assert_eq!(tcp_body(framed(1)).expect("the smallest frame").len(), 1); + assert_body_refused(tcp_body(framed(most + 1)), "a frame over the bound"); + assert_body_refused(tcp_body(|_| 0_u32.to_be_bytes().to_vec()), "an empty frame"); + } + + #[test] + fn a_peer_that_sends_one_byte_and_stops_is_refused_not_silent() { + assert_body_refused(http_body(|_| b"H".to_vec()), "one HTTP byte"); + assert_body_refused(tcp_body(|_| vec![0]), "one TCP byte"); + assert!(matches!(http_body(|_| Vec::new()), Err(ChallengeFailure::Silent(_)))); + assert!(matches!(tcp_body(|_| Vec::new()), Err(ChallengeFailure::Silent(_)))); + } + + /// An honest chunked answer with `after_body` in place of its end. + fn chunked_honest_then(id: &str, after_body: &[u8]) -> Vec { + let body = honest_body(id); + let mut reply = CHUNKED_HEAD.to_vec(); + reply.extend_from_slice(format!("{:x}\r\n", body.len()).as_bytes()); + reply.extend_from_slice(&body); + reply.extend_from_slice(b"\r\n"); + reply.extend_from_slice(after_body); + reply + } + + #[test] + fn a_chunked_answer_must_end_its_trailers_and_terminate_each_chunk() { + // The same answer, ended properly, is taken. + challenge(scripted_peer(|id| chunked_honest_then(id, b"0\r\n\r\n")), |_| {}) + .expect("an honest chunked answer"); + challenge(scripted_peer(|id| chunked_honest_then(id, b"0\r\nT: 1\r\nU: 2\r\n\r\n")), |_| {}) + .expect("an honest chunked answer with trailers"); + + // A trailer that begins and then the peer stops. + assert_refused( + challenge(scripted_peer(|id| chunked_honest_then(id, b"0\r\nT: 1\r\n")), |_| {}), + "a partial trailer section", + ); + + // Chunk data followed by something other than CRLF, from a peer that + // then carries on as though nothing were wrong. + assert_refused( + challenge( + lingering_peer( + |id| { + let body = honest_body(id); + let mut reply = CHUNKED_HEAD.to_vec(); + reply.extend_from_slice(format!("{:x}\r\n", body.len()).as_bytes()); + reply.extend_from_slice(&body); + reply.extend_from_slice(b"XX0\r\n\r\n"); + reply + }, + LINGER, + ), + |_| {}, + ), + "a chunk that is not terminated", + ); + } + + /// Headers and trailers draw on one budget. The head is fixed, so the + /// trailer line that lands the total exactly on the bound is computed. + #[test] + fn trailers_share_the_header_budget_and_are_bounded_at_its_edge() { + // The trailer line is `T: `, counted with its CRLF, plus the + // empty line ending the trailers (2): together they fill what the head + // left of the budget. + let padding = ROUTE_HTTP_MAX_HEADER_BYTES - CHUNKED_HEAD.len() - 2 - 2 - "T: ".len(); + let reply_with = move |padding: usize| { + move |id: &str| { + let mut trailers = format!("0\r\nT: {}\r\n", "a".repeat(padding)).into_bytes(); + trailers.extend_from_slice(b"\r\n"); + chunked_honest_then(id, &trailers) + } + }; + challenge(scripted_peer(reply_with(padding)), |_| {}).expect("trailers exactly at the bound"); + + let started = Instant::now(); + assert_refused( + challenge(lingering_peer(reply_with(padding + 1), LINGER), |_| {}), + "trailers one byte over the bound", + ); + assert!(started.elapsed() < ROUTE_SNAPSHOT_TIMEOUT / 2, "the refusal waited on the socket"); + + // Trailers that never end are cut off at the bound, not at the timeout. + let started = Instant::now(); + assert_refused( + challenge( + lingering_peer( + |id| { + let line = b"T: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\n"; + let mut endless = b"0\r\n".to_vec(); + endless.extend_from_slice( + &line.repeat(2 * ROUTE_HTTP_MAX_HEADER_BYTES / line.len()), + ); + chunked_honest_then(id, &endless) + }, + LINGER, + ), + |_| {}, + ), + "trailers that never end", + ); + assert!(started.elapsed() < ROUTE_SNAPSHOT_TIMEOUT / 2, "the refusal waited on the socket"); + } + + /// RFC 9112 strictness, on answers that are honest but for the one field. + #[test] + fn numbers_and_header_names_are_read_strictly() { + // `{field}:{value}` then the true length, so a value of " +" is a signed one. + let content_length = |field: &'static str, value: &'static str| { + move |id: &str| { + let body = honest_body(id); + let mut reply = + format!("HTTP/1.1 200 OK\r\n{field}:{value}{}\r\n\r\n", body.len()).into_bytes(); + reply.extend_from_slice(&body); + reply + } + }; + // Optional whitespace around a value is allowed. + challenge(scripted_peer(content_length("Content-Length", " \t ")), |_| {}) + .expect("Content-Length with optional whitespace"); + for (why, field, value) in [ + ("a signed Content-Length", "Content-Length", " +"), + ("whitespace before the colon", "Content-Length ", " "), + ("a header name with leading whitespace", " Content-Length", " "), + ] { + assert_refused(challenge(scripted_peer(content_length(field, value)), |_| {}), why); + } + assert_refused( + challenge( + scripted_peer(|id| { + let body = honest_body(id); + let mut reply = format!( + "HTTP/1.1 200 OK\r\nX-Other : y\r\nContent-Length: {}\r\n\r\n", + body.len() + ) + .into_bytes(); + reply.extend_from_slice(&body); + reply + }), + |_| {}, + ), + "whitespace before the colon of another header", + ); + assert_refused( + challenge( + scripted_peer(|id| { + let body = honest_body(id); + let mut reply = CHUNKED_HEAD.to_vec(); + reply.extend_from_slice(format!("+{:x}\r\n", body.len()).as_bytes()); + reply.extend_from_slice(&body); + reply.extend_from_slice(b"\r\n0\r\n\r\n"); + reply + }), + |_| {}, + ), + "a signed chunk size", + ); + } + + /// A reader that fails `interruptions` times with `kind`, then serves `bytes`. + struct Flaky { + kind: ErrorKind, + interruptions: usize, + bytes: &'static [u8], + } + + impl Read for Flaky { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + if self.interruptions > 0 { + self.interruptions -= 1; + return Err(self.kind.into()); + } + let served = self.bytes.len().min(buffer.len()); + buffer[..served].copy_from_slice(&self.bytes[..served]); + self.bytes = &self.bytes[served..]; + Ok(served) + } + } + + #[test] + fn an_interrupted_read_is_retried_and_any_other_failure_is_a_stopped_peer() { + let mut interrupted = Answer::new(Flaky { + kind: ErrorKind::Interrupted, + interruptions: 3, + bytes: b"abcd", + }); + let mut buffer = [0_u8; 4]; + interrupted.read_all(&mut buffer, "in the test").expect("the read is retried"); + assert_eq!(&buffer, b"abcd"); + + // Before any byte the peer is silent; after some, it is refused. + let mut failing = Answer::new(Flaky { + kind: ErrorKind::TimedOut, + interruptions: 1, + bytes: b"abcd", + }); + assert!(matches!( + failing.read_all(&mut buffer, "in the test"), + Err(ChallengeFailure::Silent(_)) + )); + let mut failing = Answer::new(Flaky { + kind: ErrorKind::TimedOut, + interruptions: 0, + bytes: b"ab", + }); + let mut wanted = [0_u8; 3]; + failing.read_all(&mut wanted[..2], "in the test").expect("two bytes"); + assert!(matches!( + failing.read_all(&mut wanted[2..], "in the test"), + Err(ChallengeFailure::Refused(_)) + )); + } + + fn assert_silent(outcome: Result, why: &str) { + assert!( + matches!(outcome, Err(ChallengeFailure::Silent(_))), + "{why} must read as silence: {outcome:?}" + ); + } + + fn assert_refused(outcome: Result, why: &str) { + assert!( + matches!(outcome, Err(ChallengeFailure::Refused(_))), + "{why} must be an error, not a wait: {outcome:?}" + ); + } + + #[test] + fn an_exact_answer_is_taken() { + let port = scripted_peer(honest); + let answer = challenge(port, |_| {}).expect("the exact answer"); + assert_eq!(answer.message_id, "candidate-probe"); + assert_eq!(answer.canonical_presence, [1, 2, 3]); + } + + #[test] + fn a_challenge_that_nothing_answers_is_silence() { + assert_silent(challenge(scripted_peer(|_| Vec::new()), |_| {}), "a peer that hangs up"); + // Half a response, then the peer is gone. + assert_refused( + challenge(scripted_peer(|_| b"HTTP/1.1 200 OK\r\nContent-Le".to_vec()), |_| {}), + "a response that never finished its headers", + ); + // Nothing listens. + let reservation = std::net::TcpListener::bind("127.0.0.1:0").expect("a free port"); + let port = reservation.local_addr().expect("a bound port").port(); + drop(reservation); + assert_silent(challenge(port, |_| {}), "a closed port"); + } + + #[test] + fn a_bad_answer_is_refused_not_waited_on() { + assert_refused( + challenge( + scripted_peer(|_| { + raw_snapshot( + "candidate-another", + cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, + "service", + vec![1], + ) + }), + |_| {}, + ), + "an answer to another challenge", + ); + assert_refused( + challenge( + scripted_peer(|id| raw_snapshot(id, "some.other.schema", "service", vec![1])), + |_| {}, + ), + "a substituted document schema", + ); + assert_refused( + challenge( + scripted_peer(|id| { + raw_snapshot( + id, + cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, + "another-target", + vec![1], + ) + }), + |_| {}, + ), + "a substituted record key", + ); + assert_refused( + challenge( + scripted_peer(|id| { + raw_snapshot( + id, + cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, + "service", + vec![0; ROUTE_SNAPSHOT_MAX_BYTES + 1], + ) + }), + |_| {}, + ), + "an oversized payload", + ); + assert_refused( + challenge( + scripted_peer(|id| { + http_ok( + &encode_cultnet_message_to_vec( + &CultNetMessage::SnapshotRequest { + message_id: id.into(), + schema_ids: None, + record_keys: None, + }, + CultNetWireContract::CultNetSchemaV0, + ) + .expect("an encodable message"), + ) + }), + |_| {}, + ), + "a response that is not a raw snapshot", + ); + assert_refused( + challenge( + scripted_peer(|_| { + b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + .to_vec() + }), + |_| {}, + ), + "an HTTP error", + ); + assert_refused( + challenge(scripted_peer(|_| http_ok(b"not a cultnet message")), |_| {}), + "a body that is not CultNet", + ); + assert_refused( + challenge(scripted_peer(|_| b"HTTP/1.1 200 OK +Content-Le".to_vec()), |_| {}), + "a response that never finished its headers", + ); + } + + #[test] + fn a_challenge_that_cannot_be_made_is_an_error_not_a_wait() { + let port = scripted_peer(honest); + assert_refused( + challenge(port, |candidate| { + candidate.route.as_mut().unwrap().route_id = "another-route".into(); + }), + "a binding that differs from Expected", + ); + assert_refused( + challenge(port, |candidate| { + candidate.route.as_mut().unwrap().transport = "smtp".into(); + }), + "an unsupported transport", + ); + assert_refused( + challenge(port, |candidate| { + candidate.route.as_mut().unwrap().candidate_endpoint = + format!("http://candidate.example:{port}"); + }), + "a host that is not an IP address", + ); + assert_refused( + challenge(port, |candidate| { + candidate.route.as_mut().unwrap().candidate_endpoint = + format!("http://127.0.0.1:{}", port.wrapping_add(1)); + }), + "a candidate endpoint outside the route binding", + ); + assert_refused( + challenge(port, |candidate| candidate.route = None), + "an Expected with no route", + ); + } + + /// The candidate path renders the Expected first, and rendering parses the + /// same endpoints, so it never reaches the parse in the challenge. The + /// stable path does not render: a bound stable endpoint that is not a plain + /// host and port is caught by the challenge's own parse, and it is a + /// configuration error, not a listener that has not answered yet. + #[test] + fn a_stable_endpoint_that_does_not_parse_is_an_error_not_a_wait() { + let temp = tempfile::tempdir().expect("a temporary directory"); + let driver = NginxRouteDriver::new(RouteBinding { + driver: RouteDriver::NginxStreamTcp, + route_id: "service".into(), + stable_endpoint: "http://127.0.0.1:17999".into(), + private_host: "127.0.0.1".into(), + private_port_start: 17998, + private_port_end: 17998, + config_path: temp.path().join("service.conf"), + reload_unit: "nginx.service".into(), + }); + let mut candidate = expected(); + candidate.route = Some(cultnet_rs::IdunnExpectedRoute { + route_id: "service".into(), + transport: "http".into(), + stable_endpoint: "http://127.0.0.1:17999".into(), + candidate_endpoint: "http://127.0.0.1:17998".into(), + }); + for endpoint in [ + "http://127.0.0.1", + "http://127.0.0.1:0", + "http://127.0.0.1:17999/path", + "tcp://127.0.0.1:17999", + ] { + assert_refused( + driver.request_runtime_presence_at(&candidate, "stable-probe", endpoint), + endpoint, + ); + } + } + #[test] fn route_connect_retries_connection_refused_until_listener_is_ready() -> Result<()> { let reservation = std::net::TcpListener::bind("127.0.0.1:0")?; @@ -10194,7 +11335,7 @@ mod tests { maximum_future_skew_millis: 5, }, )?; - let warming = SequenceAdmittedWarming::for_test("test-transaction", warming, 120)?; + let warming = SequenceAdmittedWarming::for_test("test-transaction", warming)?; Ok((expected, activation, warming, provider.trust_anchor()?)) }