From 87fa57dd8c8a089891393b7fbdfaf6cd9eb8bb62 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 01:30:09 +0200 Subject: [PATCH 001/286] Revert "Recover fenced rollback from current activation" This reverts commit a148802cb6c24338f1ed376d23de94f18dc8411f. Rollback after fencing adopted whatever activation was projected under the shared Expected key, so a continuity abort's leftover activation could be demoted by a later deploy. The drift is closed at its source by the next commit: a continuity abort demotes its own exact activation. Rollback demotes the admitted generation's exact activation again. The doc paragraph is dropped; the 9f00e7a paragraph above it stays until B4. Co-Authored-By: Claude Sonnet 5 --- docs/deployment-authority.md | 9 -- src/control_plane.rs | 5 +- src/drivers.rs | 186 ++--------------------------------- 3 files changed, 12 insertions(+), 188 deletions(-) diff --git a/docs/deployment-authority.md b/docs/deployment-authority.md index 3b23009..c24a7f5 100644 --- a/docs/deployment-authority.md +++ b/docs/deployment-authority.md @@ -312,15 +312,6 @@ observation own current workload health. Historical receipts do not expire merely because their provider remains admitted longer than the observation window. -An Expected digest stays stable across continuity restarts, so the current -activation at that projection key can be newer than the activation cached in an -older admitted-generation receipt. During post-fencing rollback, Idunn first -stops the candidate and withdraws the lease. It then verifies any current -projected activation against Idunn's own trust anchor and the exact Expected -record, refuses if a projected write lease remains, and demotes the projection -to Expected-only. The admitted receipt does not overwrite a newer signed -activation, and no operator edits the CultCache projection by hand. - Idunn starts and recovers from its own durable admitted state. Odin is the first managed semantic daemon, never an Idunn bootstrap dependency. Odin's initial admission is the sole graph-bootstrap exception: Idunn uses its root-admitted diff --git a/src/control_plane.rs b/src/control_plane.rs index 99126cc..754f55b 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -5969,10 +5969,11 @@ impl Engine { if let Some(incumbent) = self.exact_incumbent(&snapshot, ¤t.value)? { let admitted_provider_anchor = self.provider_anchor_for_plan(&incumbent.value.plan)?; - let expected_sha256 = topology.demote_current_activation_to_expected_only( + let expected_sha256 = topology.demote_to_expected_only( &incumbent.value.expected, &admitted_provider_anchor, - &self.idunn_anchor, + &incumbent.value.activation, + incumbent.value.leasing.lease(), )?; ensure!( expected_sha256 == incumbent.value.expected.canonical_sha256()?, diff --git a/src/drivers.rs b/src/drivers.rs index 06c16c8..fa45b8c 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -4420,36 +4420,20 @@ impl CultCacheTopologyDriver { provider_anchor: &ServiceIdentityTrustAnchor, activation: &IdunnRuntimeActivationRecord, lease: Option<&IdunnProcessWriteLeaseRecord>, - ) -> Result { - self.demote_projection(expected, provider_anchor, Some(activation), lease) - } - - fn demote_projection( - &self, - expected: &IdunnExpectedIncarnationRecord, - provider_anchor: &ServiceIdentityTrustAnchor, - activation: Option<&IdunnRuntimeActivationRecord>, - lease: Option<&IdunnProcessWriteLeaseRecord>, ) -> Result { expected.validate()?; - if let Some(activation) = activation { - activation.validate()?; - ensure!( - activation.expected_projection_sha256 == expected.canonical_sha256()?, - "admitted activation does not bind the admitted Expected projection" - ); - } + activation.validate()?; + ensure!( + activation.expected_projection_sha256 == expected.canonical_sha256()?, + "admitted activation does not bind the admitted Expected projection" + ); if let Some(lease) = lease { - validate_topology_lease( - expected, - activation.context("demotion lease has no exact activation")?, - lease, - )?; + validate_topology_lease(expected, activation, lease)?; } let anchor = runtime_presence_trust_anchor(expected, provider_anchor)?; let key = incarnation_key(expected)?; self.mutate(|entries| { - let projected = ProjectedIncarnation::read(entries, expected, activation, lease)?; + let projected = ProjectedIncarnation::read(entries, expected, Some(activation), lease)?; let current_anchor = projection_entry( entries, GameCultServiceTrustAnchorRecord::TYPE, @@ -4490,55 +4474,6 @@ impl CultCacheTopologyDriver { expected.canonical_sha256() } - /// Demote the current projected activation after the owning transaction - /// has fenced the target. A continuity restart can replace the activation - /// under the same Expected key while the admitted-generation receipt still - /// carries the prior runtime instance. In that one post-fence recovery - /// path, trust the current projection only after verifying Idunn's - /// signature and exact Expected binding, and refuse while any projected - /// write lease remains. - pub fn demote_current_activation_to_expected_only( - &self, - expected: &IdunnExpectedIncarnationRecord, - provider_anchor: &ServiceIdentityTrustAnchor, - idunn_anchor: &ServiceIdentityTrustAnchor, - ) -> Result { - expected.validate()?; - let entries = self.snapshot()?; - let key = incarnation_key(expected)?; - let projected_activation = projection_entry( - &entries, - IdunnRuntimeActivationRecord::TYPE, - &key, - )? - .map(|envelope| -> Result { - ensure!( - envelope.schema_id.as_deref() == Some(IDUNN_RUNTIME_ACTIVATION_SCHEMA), - "projected activation schema is substituted" - ); - let activation = IdunnRuntimeActivationRecord::decode_canonical(&envelope.payload)?; - activation.verify_for_expected(expected, idunn_anchor)?; - Ok(activation) - }) - .transpose()?; - ensure!( - projection_entry( - &entries, - IdunnProcessWriteLeaseRecord::TYPE, - &key, - )? - .is_none(), - "cannot demote an incarnation while its projected write lease remains" - ); - - self.demote_projection( - expected, - provider_anchor, - projected_activation.as_ref(), - None, - ) - } - /// Whether the projection currently names an activation for this /// incarnation. Continuity asks so it can tell a projection that still /// describes a dead incarnation from one already demoted to Expected-only. @@ -8465,7 +8400,7 @@ mod tests { IDUNN_PROCESS_WRITE_LEASE_SCHEMA, IdunnServiceIdentity, OdinRuntimeTopologyCorrelationPurpose, OdinTopologyAuthenticationContext, OdinTopologyIdentity, authenticate_odin_runtime_topology_correlation, - enroll_service_identity_at, open_service_identity_at, verify_runtime_authority, + enroll_service_identity_at, verify_runtime_authority, }; fn digest(byte: char) -> String { @@ -10838,12 +10773,7 @@ mod tests { ); assert!(!driver.projected_activation_is_present(&incumbent)?); let idempotent_bytes = fs::read(&driver.projection_store)?; - driver.demote_to_expected_only( - &incumbent, - &provider_anchor, - &activation, - Some(&lease), - )?; + driver.demote_to_expected_only(&incumbent, &provider_anchor, &activation, Some(&lease))?; assert_eq!(fs::read(&driver.projection_store)?, idempotent_bytes); // A record under the incumbent's key that is not the incumbent's is a @@ -10907,104 +10837,6 @@ mod tests { Ok(()) } - #[test] - fn fenced_demotion_uses_the_current_verified_activation() -> Result<()> { - let temp = tempfile::tempdir()?; - let (expected, admitted_activation, warming, provider_anchor) = - authenticated_warming(temp.path())?; - let idunn = open_service_identity_at::(&temp.path().join("idunn.cc"))?; - let replacement = IdunnRuntimeActivationLaunch::issue( - &expected, - digest('a'), - 300, - &idunn, - )? - .activation() - .clone(); - assert_ne!(admitted_activation, replacement); - - let driver = topology_driver(temp.path(), "fenced-demotion"); - driver.publish_expected(&expected, &provider_anchor)?; - project_activation(&driver, &expected, &admitted_activation)?; - let key = incarnation_key(&expected)?; - upsert_record( - &driver.projection_store, - activation_envelope(&key, &replacement)?, - )?; - - assert_eq!( - driver.demote_current_activation_to_expected_only( - &expected, - &provider_anchor, - &idunn.trust_anchor()?, - )?, - expected.canonical_sha256()? - ); - assert!(!driver.projected_activation_is_present(&expected)?); - let after = SingleFileMessagePackBackingStore::new(&driver.projection_store) - .pull_all_read_only_snapshot()?; - assert!(after.iter().any(|entry| { - entry.key == key && entry.r#type == IdunnExpectedIncarnationRecord::TYPE - })); - assert!(after.iter().all(|entry| { - !(entry.key == key - && (entry.r#type == IdunnRuntimeActivationRecord::TYPE - || entry.r#type == IdunnProcessWriteLeaseRecord::TYPE)) - })); - - // Expected-only demotion is idempotent when the activation is already - // absent. - let settled = fs::read(&driver.projection_store)?; - driver.demote_current_activation_to_expected_only( - &expected, - &provider_anchor, - &idunn.trust_anchor()?, - )?; - assert_eq!(fs::read(&driver.projection_store)?, settled); - - // A record with an invalid Idunn signature is not adopted as current - // authority, even after fencing. - let mut forged = replacement; - forged.runtime_instance_id = digest('b'); - upsert_record( - &driver.projection_store, - activation_envelope(&key, &forged)?, - )?; - let before_forgery_attempt = fs::read(&driver.projection_store)?; - assert!(driver - .demote_current_activation_to_expected_only( - &expected, - &provider_anchor, - &idunn.trust_anchor()?, - ) - .is_err()); - assert_eq!(fs::read(&driver.projection_store)?, before_forgery_attempt); - - // A remaining lease means the target has not reached the demotion - // boundary. The operation leaves every byte untouched. - let incumbent_lease = lease(&expected, &admitted_activation, &warming); - upsert_record( - &driver.projection_store, - CultCacheEnvelope { - key: key.clone(), - r#type: IdunnProcessWriteLeaseRecord::TYPE.into(), - payload: incumbent_lease.canonical_bytes()?, - stored_at: "2026-09-03T00:00:00Z".into(), - schema_id: Some(IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into()), - }, - )?; - let before_leased_attempt = fs::read(&driver.projection_store)?; - assert!(driver - .demote_current_activation_to_expected_only( - &expected, - &provider_anchor, - &idunn.trust_anchor()?, - ) - .is_err()); - assert_eq!(fs::read(&driver.projection_store)?, before_leased_attempt); - Ok(()) - } - /// A replaced incarnation nothing owns is withdrawn whole; the one still /// admitted, and the target's anchor, are untouched. #[test] From 44277698c3dd487d1aca84aca27168c3ea45a43e Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 01:30:19 +0200 Subject: [PATCH 002/286] A continuity abort resolves its own projection (route continuity B1) A continuity candidate shares the admitted Expected key. Both abort intents marked its topology reconciliation Skipped, so a failed continuity left its candidate activation projected under the admitted key while the admitted receipt named the older one, and a later deploy's post-fence demotion refused as substituted. DeploymentTransaction::abort_topology_reconciliation is now the one rule for both intents and pre-fence validation: a deployment owes the projection when it published its Expected, a continuity owes it when it issued an activation. reconcile_failed_candidate_projection is the one resolution for both abort paths: a deployment withdraws its own key, a continuity demotes the shared key to Expected-only with its own exact activation and never withdraws the Expected. The post-fence incumbent demotion stays deployment-only and exact. Tests read the persisted topology store after the production publish/demote sequence. Co-Authored-By: Claude Sonnet 5 --- docs/deployment-authority.md | 6 + src/control_plane.rs | 332 ++++++++++++++++++++++++++++++----- 2 files changed, 290 insertions(+), 48 deletions(-) diff --git a/docs/deployment-authority.md b/docs/deployment-authority.md index c24a7f5..0ee3483 100644 --- a/docs/deployment-authority.md +++ b/docs/deployment-authority.md @@ -312,6 +312,12 @@ observation own current workload health. Historical receipts do not expire merely because their provider remains admitted longer than the observation window. +A failed continuity restart leaves its own candidate activation projected under +the shared Expected key. Its abort demotes that key to Expected-only with the +exact activation the transaction issued, so the projection never drifts from the +admitted receipt. Rollback after fencing demotes the admitted generation's exact +activation. No path adopts an activation the transaction did not issue. + Idunn starts and recovers from its own durable admitted state. Odin is the first managed semantic daemon, never an Idunn bootstrap dependency. Odin's initial admission is the sole graph-bootstrap exception: Idunn uses its root-admitted diff --git a/src/control_plane.rs b/src/control_plane.rs index 754f55b..6cbbee3 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -795,6 +795,22 @@ impl DeploymentTransaction { Ok(transaction) } + /// Whether an abort owes the projection a reconciliation. A deployment + /// published its own Expected key. A continuity shares the admitted key, + /// whose Expected is never withdrawn, so it owes only the demotion of the + /// activation it issued. + fn abort_topology_reconciliation(&self) -> CleanupEvidence { + let owes = match self.command_kind { + CommandKind::Deploy => self.expected_publication_sha256.is_some(), + CommandKind::Continuity => self.activation.is_some(), + }; + if owes { + CleanupEvidence::Pending + } else { + CleanupEvidence::Skipped + } + } + fn rejected(command: &DeploymentCommand, error: anyhow::Error, now: u64) -> Result { let mut transaction = Self::new(command, command.selector.clone(), 0, None, now)?; let detail = truncate(&format!("{error:#}"), 2048); @@ -1013,15 +1029,18 @@ impl DeploymentTransaction { ), "abort candidate cleanup differs from its prepared activation" ); - let topology_cleanup_required = self.command_kind == CommandKind::Deploy - && self.expected_publication_sha256.is_some(); ensure!( matches!( - (topology_cleanup_required, abort.topology_reconciliation), - (true, CleanupEvidence::Pending | CleanupEvidence::Complete) - | (false, CleanupEvidence::Skipped) + ( + self.abort_topology_reconciliation(), + abort.topology_reconciliation + ), + ( + CleanupEvidence::Pending, + CleanupEvidence::Pending | CleanupEvidence::Complete + ) | (CleanupEvidence::Skipped, CleanupEvidence::Skipped) ), - "abort topology cleanup differs from published deployment Expected" + "abort topology cleanup differs from what the transaction projected" ); ensure!( matches!( @@ -5819,13 +5838,7 @@ impl Engine { current.value.activation.is_some(), current.value.workload.is_some(), ), - topology_reconciliation: if current.value.command_kind == CommandKind::Deploy - && current.value.expected_publication_sha256.is_some() - { - CleanupEvidence::Pending - } else { - CleanupEvidence::Skipped - }, + topology_reconciliation: current.value.abort_topology_reconciliation(), source_cleanup: if current.value.command_kind == CommandKind::Deploy { CleanupEvidence::Pending } else { @@ -5947,26 +5960,27 @@ impl Engine { }); } if abort.topology_reconciliation == CleanupEvidence::Pending { - let expected = required(¤t.value.expected, "failed Expected projection")?; let snapshot = ControlSnapshot::read(&self.options.state_store)?; - // The failed candidate's records go, under its own key. Fencing - // stopped the incumbent and revoked its lease, so the incumbent is - // demoted to Expected-only under its key, which is what lets - // continuity bring it back; deployment does not restart it here. + // The failed candidate's projection is resolved first. For a + // deployment, fencing also stopped the incumbent and revoked its + // lease, so the incumbent is demoted to Expected-only under its + // key with its own exact activation, which is what lets + // continuity bring it back. A continuity candidate is the + // incumbent's key: it has nothing further to demote. let topology = self.topology(); let failed_provider_anchor = self.provider_anchor_for_plan(required( ¤t.value.plan, "failed transaction plan", )?)?; - topology - .withdraw_incarnation( - expected, - &failed_provider_anchor, - current.value.activation.as_ref(), - None, - ) - .context("withdrawing the failed candidate projection")?; - if let Some(incumbent) = self.exact_incumbent(&snapshot, ¤t.value)? { + reconcile_failed_candidate_projection( + &topology, + ¤t.value, + &failed_provider_anchor, + )?; + if let Some(incumbent) = self + .exact_incumbent(&snapshot, ¤t.value)? + .filter(|_| current.value.command_kind == CommandKind::Deploy) + { let admitted_provider_anchor = self.provider_anchor_for_plan(&incumbent.value.plan)?; let expected_sha256 = topology.demote_to_expected_only( @@ -6042,22 +6056,17 @@ impl Engine { }); } if abort.topology_reconciliation == CleanupEvidence::Pending { - let expected = required(¤t.value.expected, "failed Expected projection")?; - // Before fencing the incumbent was never touched: it is still - // running under its own key with its activation and lease. Only the - // failed candidate's records are withdrawn. The incumbent's - // projection is not demoted -- doing so would tell a live process - // it has no activation. + // Before fencing a deployment never touched its incumbent: it is + // still running under its own key with its activation and lease, + // so only the candidate's records are withdrawn. A continuity + // candidate shares the admitted key and demotes what it issued. let plan = required(¤t.value.plan, "failed transaction plan")?; let provider_anchor = self.provider_anchor_for_plan(plan)?; - self.topology() - .withdraw_incarnation( - expected, - &provider_anchor, - current.value.activation.as_ref(), - None, - ) - .context("withdrawing exact failed Expected projection")?; + reconcile_failed_candidate_projection( + &self.topology(), + ¤t.value, + &provider_anchor, + )?; return self.persist_same_phase(current, |next| { next.pre_fencing_abort .as_mut() @@ -6184,6 +6193,37 @@ fn provider_warming_advanced( && candidate_signed_presence_sha256 != prior_signed_presence_sha256 } +/// Resolve the projection a failed candidate left. A deployment candidate owns +/// its own incarnation key and withdraws it whole. A continuity candidate +/// shares the admitted key: the admitted Expected is never withdrawn, and the +/// only thing demoted is the activation this transaction issued, exactly. +fn reconcile_failed_candidate_projection( + topology: &CultCacheTopologyDriver, + transaction: &DeploymentTransaction, + provider_anchor: &ServiceIdentityTrustAnchor, +) -> Result<()> { + let expected = required(&transaction.expected, "failed Expected projection")?; + match transaction.command_kind { + CommandKind::Deploy => topology + .withdraw_incarnation( + expected, + provider_anchor, + transaction.activation.as_ref(), + None, + ) + .context("withdrawing the failed candidate projection"), + CommandKind::Continuity => topology + .demote_to_expected_only( + expected, + provider_anchor, + required(&transaction.activation, "failed continuity activation")?, + None, + ) + .map(drop) + .context("demoting the failed continuity activation"), + } +} + fn may_rollback_route_after_failed_proof(fencing: &FencingEvidence) -> bool { matches!(fencing, FencingEvidence::SkippedStateless) } @@ -6223,13 +6263,7 @@ fn post_fencing_abort_intent( transaction.activation.is_some(), transaction.workload.is_some(), ), - topology_reconciliation: if transaction.command_kind == CommandKind::Deploy - && transaction.expected_publication_sha256.is_some() - { - CleanupEvidence::Pending - } else { - CleanupEvidence::Skipped - }, + topology_reconciliation: transaction.abort_topology_reconciliation(), source_cleanup: if transaction.command_kind == CommandKind::Deploy { CleanupEvidence::Pending } else { @@ -8295,4 +8329,206 @@ mod tests { }) ); } + /// The persisted projection a continuity restart leaves behind, read at + /// the file the daemon writes. + struct ContinuityProjection { + _temp: TempDir, + topology: CultCacheTopologyDriver, + provider_anchor: ServiceIdentityTrustAnchor, + expected: IdunnExpectedIncarnationRecord, + admitted_activation: IdunnRuntimeActivationRecord, + candidate_activation: IdunnRuntimeActivationRecord, + } + + impl ContinuityProjection { + /// The production sequence: the admitted incarnation dies and is + /// demoted to Expected-only, then the continuity candidate publishes + /// its own activation under the same key once it is observed. + fn after_candidate_observed() -> Result { + let temp = TempDir::new()?; + let root = temp.path(); + let idunn = enroll_service_identity_at::(&root.join("idunn.cc"))?; + let provider = enroll_service_identity_at::( + &root.join("provider.cc"), + )?; + let provider_anchor = provider.trust_anchor()?; + let mut expected = TopologyFixture::new("ghostlight")?.expected; + expected.expected_signer_identity_id = provider.entry().identity_id.clone(); + expected.artifact_sha256 = sha256_id(&[1]); + expected.validate()?; + let topology = CultCacheTopologyDriver { + projection_store: root.join("topology.cc"), + correlation_store: root.join("correlation.cc"), + }; + let admitted = workload(1, 40, 50); + let candidate = workload(2, 41, 51); + let issue = |observation: &WorkloadObservation| -> Result<_> { + Ok(IdunnRuntimeActivationLaunch::issue( + &expected, + observation.runtime_instance_id().to_owned(), + NOW, + &idunn, + )? + .activation() + .clone()) + }; + let admitted_activation = issue(&admitted)?; + let candidate_activation = issue(&candidate)?; + assert_ne!(admitted_activation, candidate_activation); + + topology.publish_expected(&expected, &provider_anchor)?; + topology.publish_observed_activation(&expected, &admitted_activation, &admitted)?; + topology.demote_to_expected_only( + &expected, + &provider_anchor, + &admitted_activation, + None, + )?; + topology.publish_expected(&expected, &provider_anchor)?; + topology.publish_observed_activation(&expected, &candidate_activation, &candidate)?; + Ok(Self { + _temp: temp, + topology, + provider_anchor, + expected, + admitted_activation, + candidate_activation, + }) + } + + fn continuity_transaction(&self) -> Result { + let mut transaction = DeploymentTransaction::new( + &command(CommandKind::Continuity), + "ghostlight".into(), + 0, + None, + 100, + )?; + transaction.expected = Some(self.expected.clone()); + transaction.activation = Some(self.candidate_activation.clone()); + Ok(transaction) + } + + /// Types of the records projected under the shared incarnation key. + fn projected_types(&self) -> Result> { + let key = incarnation_key(&self.expected)?; + Ok( + SingleFileMessagePackBackingStore::new(&self.topology.projection_store) + .pull_all_read_only_snapshot()? + .into_iter() + .filter(|entry| entry.key == key) + .map(|entry| entry.r#type) + .collect(), + ) + } + } + + #[test] + fn a_failed_continuity_demotes_its_own_activation_and_keeps_the_expected() -> Result<()> { + let world = ContinuityProjection::after_candidate_observed()?; + assert!( + world + .projected_types()? + .contains(&IdunnRuntimeActivationRecord::TYPE.to_owned()) + ); + let transaction = world.continuity_transaction()?; + assert_eq!( + transaction.abort_topology_reconciliation(), + CleanupEvidence::Pending + ); + assert_eq!( + post_fencing_abort_intent(&transaction, "candidate died").topology_reconciliation, + CleanupEvidence::Pending + ); + + reconcile_failed_candidate_projection( + &world.topology, + &transaction, + &world.provider_anchor, + )?; + + assert_eq!( + world.projected_types()?, + vec![IdunnExpectedIncarnationRecord::TYPE.to_owned()], + "the shared key must keep its Expected and nothing else" + ); + Ok(()) + } + + #[test] + fn a_continuity_that_issued_no_activation_owes_no_projection_change() -> Result<()> { + let world = ContinuityProjection::after_candidate_observed()?; + let mut transaction = world.continuity_transaction()?; + transaction.activation = None; + assert_eq!( + transaction.abort_topology_reconciliation(), + CleanupEvidence::Skipped + ); + assert_eq!( + post_fencing_abort_intent(&transaction, "candidate never started") + .topology_reconciliation, + CleanupEvidence::Skipped + ); + Ok(()) + } + + #[test] + fn a_deploy_fencing_after_a_failed_continuity_resolves_with_the_admitted_activation() + -> Result<()> { + let world = ContinuityProjection::after_candidate_observed()?; + reconcile_failed_candidate_projection( + &world.topology, + &world.continuity_transaction()?, + &world.provider_anchor, + )?; + let settled = std::fs::read(&world.topology.projection_store)?; + + // The deploy's post-fence rollback demotes the admitted generation's + // exact activation. The failed continuity left nothing standing, so it + // resolves instead of refusing as substituted. + world.topology.demote_to_expected_only( + &world.expected, + &world.provider_anchor, + &world.admitted_activation, + None, + )?; + assert_eq!(std::fs::read(&world.topology.projection_store)?, settled); + Ok(()) + } + + #[test] + fn no_path_adopts_an_activation_the_transaction_did_not_issue() -> Result<()> { + let world = ContinuityProjection::after_candidate_observed()?; + let before = std::fs::read(&world.topology.projection_store)?; + + // The candidate's activation is still projected. The admitted + // generation's exact activation is not what stands there, so its + // demotion refuses rather than taking the newer one. + assert!( + world + .topology + .demote_to_expected_only( + &world.expected, + &world.provider_anchor, + &world.admitted_activation, + None, + ) + .is_err() + ); + + // Nor may a continuity transaction demote an activation it did not + // issue. + let mut foreign = world.continuity_transaction()?; + foreign.activation = Some(world.admitted_activation.clone()); + assert!( + reconcile_failed_candidate_projection( + &world.topology, + &foreign, + &world.provider_anchor + ) + .is_err() + ); + assert_eq!(std::fs::read(&world.topology.projection_store)?, before); + Ok(()) + } } From 4a9fe87787b165fce09797b8b938c92fb7d8a106 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 01:32:59 +0200 Subject: [PATCH 003/286] Import incarnation_key into the continuity projection tests Co-Authored-By: Claude Sonnet 5 --- src/control_plane.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 6cbbee3..5a862b2 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -6574,7 +6574,7 @@ mod tests { use tempfile::TempDir; use super::*; - use crate::drivers::SystemdWorkloadObservation; + use crate::drivers::{SystemdWorkloadObservation, incarnation_key}; /// Fixed clock for the signing fixture; correlations must land inside /// DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS of it to authenticate. From 6fcc1ba6f3877e0dc9af7acd13d3d6ff171d4348 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 02:14:47 +0200 Subject: [PATCH 004/286] Freeze per-phase deadline durations into the compiled plan (F0) An operator binding may now declare a [deadlines] table naming any of the five post-fencing phases (fencing, leasing, awaiting_ready, routing, committing) in seconds, 1..=86400. compile_deployment_plan resolves the binding over Idunn's defaults and freezes the result in the plan, so a later change to a default cannot move an in-flight plan. The plan becomes schema v3, with the frozen durations as a trailing field. A v2 plan keeps its nine-field encoding (the field is skipped when absent), so every existing plan id still verifies; it runs on Idunn's defaults at read time. Nothing reads the durations yet. Co-Authored-By: Claude Sonnet 5 --- src/deployment.rs | 45 ++++++++++ src/deployment_plan.rs | 194 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 236 insertions(+), 3 deletions(-) diff --git a/src/deployment.rs b/src/deployment.rs index 23ecac3..a67b4bc 100644 --- a/src/deployment.rs +++ b/src/deployment.rs @@ -275,6 +275,10 @@ pub struct OperatorBinding { pub process_write_lease: Option, pub brakes: BrakeBinding, pub rollout: RolloutBinding, + /// Optional per-phase deadline overrides; absent fields take Idunn's defaults + /// when the plan is compiled. + #[serde(default)] + pub deadlines: Option, pub placement: PlacementBinding, #[serde(default)] pub external_capabilities: Vec, @@ -709,6 +713,44 @@ pub struct RolloutBinding { pub retain_releases: u32, } +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DeadlineBinding { + #[serde(default)] + pub fencing_seconds: Option, + #[serde(default)] + pub leasing_seconds: Option, + #[serde(default)] + pub awaiting_ready_seconds: Option, + #[serde(default)] + pub routing_seconds: Option, + #[serde(default)] + pub committing_seconds: Option, +} + +/// Longest deadline a binding may declare: one day. +pub const MAXIMUM_DEADLINE_SECONDS: u32 = 86_400; + +impl DeadlineBinding { + fn validate(&self) -> Result<()> { + for (name, seconds) in [ + ("fencing", self.fencing_seconds), + ("leasing", self.leasing_seconds), + ("awaiting-ready", self.awaiting_ready_seconds), + ("routing", self.routing_seconds), + ("committing", self.committing_seconds), + ] { + if let Some(seconds) = seconds { + ensure!( + (1..=MAXIMUM_DEADLINE_SECONDS).contains(&seconds), + "{name} deadline must be between 1 and {MAXIMUM_DEADLINE_SECONDS} seconds" + ); + } + } + Ok(()) + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub enum RolloutStrategy { @@ -1343,6 +1385,9 @@ impl OperatorBinding { /// The part of validation that does not depend on where the workload runs. fn validate_shared_tail(&self) -> Result<()> { + if let Some(deadlines) = &self.deadlines { + deadlines.validate()?; + } ensure!( self.rollout.retain_releases >= 2, "candidate rollout must retain at least current and prior releases" diff --git a/src/deployment_plan.rs b/src/deployment_plan.rs index c768657..e3d690b 100644 --- a/src/deployment_plan.rs +++ b/src/deployment_plan.rs @@ -12,13 +12,16 @@ use sha2::{Digest, Sha256}; use crate::control_plane::SequenceAdmittedReady; use crate::deployment::{ - CapabilityDependency, DependencyKind, ExternalCapabilityBinding, OperatorBinding, + CapabilityDependency, DeadlineBinding, DependencyKind, ExternalCapabilityBinding, OperatorBinding, ServiceTransport, SourceSelectionPolicy, StartupOrder, StateDeclaration, TargetDeclaration, capability_compatible, }; pub const SOURCE_SELECTION_FACTS_SCHEMA: &str = "idunn.source_selection_facts.v1"; -pub const COMPILED_DEPLOYMENT_PLAN_SCHEMA: &str = "idunn.compiled_deployment_plan.v2"; +pub const COMPILED_DEPLOYMENT_PLAN_SCHEMA: &str = "idunn.compiled_deployment_plan.v3"; +/// Plans sealed before phase deadlines existed. They keep their bytes, and so +/// their plan id; their deadlines are Idunn's defaults at read time. +pub const COMPILED_DEPLOYMENT_PLAN_SCHEMA_V2: &str = "idunn.compiled_deployment_plan.v2"; pub const SEALED_RELEASE_SCHEMA: &str = "idunn.sealed_release.v1"; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -460,6 +463,86 @@ pub fn select_dependencies( .collect() } +/// Idunn's default duration for each post-fencing phase, in seconds. A binding +/// overrides any of them; the resolved values are frozen into the plan. +pub const DEFAULT_FENCING_DEADLINE_SECONDS: u32 = 120; +pub const DEFAULT_LEASING_DEADLINE_SECONDS: u32 = 120; +pub const DEFAULT_AWAITING_READY_DEADLINE_SECONDS: u32 = 300; +pub const DEFAULT_ROUTING_DEADLINE_SECONDS: u32 = 120; +pub const DEFAULT_COMMITTING_DEADLINE_SECONDS: u32 = 120; + +/// The resolved per-phase deadline durations a plan carries. Nothing reads +/// them at the cut that introduced them. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PhaseDeadlines { + pub fencing_seconds: u32, + pub leasing_seconds: u32, + pub awaiting_ready_seconds: u32, + pub routing_seconds: u32, + pub committing_seconds: u32, +} + +impl PhaseDeadlines { + pub const IDUNN_DEFAULTS: Self = Self { + fencing_seconds: DEFAULT_FENCING_DEADLINE_SECONDS, + leasing_seconds: DEFAULT_LEASING_DEADLINE_SECONDS, + awaiting_ready_seconds: DEFAULT_AWAITING_READY_DEADLINE_SECONDS, + routing_seconds: DEFAULT_ROUTING_DEADLINE_SECONDS, + committing_seconds: DEFAULT_COMMITTING_DEADLINE_SECONDS, + }; + + /// Binding overrides over Idunn's defaults. + pub fn resolve(binding: Option<&DeadlineBinding>) -> Self { + let defaults = Self::IDUNN_DEFAULTS; + let Some(binding) = binding else { + return defaults; + }; + Self { + fencing_seconds: binding.fencing_seconds.unwrap_or(defaults.fencing_seconds), + leasing_seconds: binding.leasing_seconds.unwrap_or(defaults.leasing_seconds), + awaiting_ready_seconds: binding + .awaiting_ready_seconds + .unwrap_or(defaults.awaiting_ready_seconds), + routing_seconds: binding.routing_seconds.unwrap_or(defaults.routing_seconds), + committing_seconds: binding + .committing_seconds + .unwrap_or(defaults.committing_seconds), + } + } + + fn validate_against(&self, binding: &OperatorBinding) -> Result<()> { + let declared = binding.deadlines.clone().unwrap_or_default(); + for (name, frozen, declared) in [ + ("fencing", self.fencing_seconds, declared.fencing_seconds), + ("leasing", self.leasing_seconds, declared.leasing_seconds), + ( + "awaiting-ready", + self.awaiting_ready_seconds, + declared.awaiting_ready_seconds, + ), + ("routing", self.routing_seconds, declared.routing_seconds), + ( + "committing", + self.committing_seconds, + declared.committing_seconds, + ), + ] { + ensure!( + (1..=crate::deployment::MAXIMUM_DEADLINE_SECONDS).contains(&frozen), + "frozen {name} deadline is out of range" + ); + if let Some(declared) = declared { + ensure!( + frozen == declared, + "frozen {name} deadline differs from the binding" + ); + } + } + Ok(()) + } +} + /// Private Idunn control-plane state. This contains host binding details and is /// never an Odin/CultMesh projection. The shared CultNet Expected incarnation /// is the sanitized topology projection derived only after a release validates @@ -476,12 +559,17 @@ pub struct CompiledDeploymentPlan { pub binding_blob: Vec, pub dependencies: Vec, pub candidate_port: Option, + /// Present exactly on v3 plans. Skipped when absent so a v2 plan re-encodes + /// to its original bytes and keeps its plan id. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub deadlines: Option, } impl CompiledDeploymentPlan { pub fn validate(&self) -> Result<()> { ensure!( - self.schema == COMPILED_DEPLOYMENT_PLAN_SCHEMA, + self.schema == COMPILED_DEPLOYMENT_PLAN_SCHEMA + || self.schema == COMPILED_DEPLOYMENT_PLAN_SCHEMA_V2, "unsupported deployment-plan schema" ); require_sha256(&self.plan_id, "plan id")?; @@ -492,6 +580,11 @@ impl CompiledDeploymentPlan { ); let (declaration, binding) = self.parsed_inputs()?; self.source.validate_against(&binding)?; + match (&self.schema[..], &self.deadlines) { + (COMPILED_DEPLOYMENT_PLAN_SCHEMA, Some(deadlines)) => deadlines.validate_against(&binding)?, + (COMPILED_DEPLOYMENT_PLAN_SCHEMA_V2, None) => {} + _ => bail!("deployment plan deadlines differ from its schema"), + } ensure!( sha256_id(&self.recipe_blob) == self.source.recipe_blob_sha256, "selected recipe blob differs from the exact plan recipe bytes" @@ -527,6 +620,12 @@ impl CompiledDeploymentPlan { Ok(()) } + /// The deadline durations this plan runs under: frozen for v3, Idunn's + /// defaults for a v2 plan that predates them. + pub fn phase_deadlines(&self) -> PhaseDeadlines { + self.deadlines.unwrap_or(PhaseDeadlines::IDUNN_DEFAULTS) + } + pub(crate) fn parsed_inputs(&self) -> Result<(TargetDeclaration, OperatorBinding)> { let recipe_text = std::str::from_utf8(&self.recipe_blob) .context("stored deployment recipe is not UTF-8")?; @@ -589,6 +688,7 @@ pub(crate) fn compile_deployment_plan( binding_blob: binding_bytes.to_vec(), dependencies, candidate_port, + deadlines: Some(PhaseDeadlines::resolve(binding.deadlines.as_ref())), }; plan.plan_id = plan.recomputed_plan_id()?; plan.validate()?; @@ -1594,4 +1694,92 @@ nodes = ["yggdrasil"] changed.external_inputs[0].size_bytes = 18; assert!(changed.validate_against(&plan).is_err()); } + + fn binding_with_deadlines(table: &str) -> String { + BINDING.replace("[placement]", &format!("{table}\n[placement]")) + } + + fn plan_under(binding: &str) -> Result { + let providers = [ready_odin_provider("odin", "odin-yggdrasil", 1)]; + compile_deployment_plan( + RECIPE.as_bytes(), + binding.as_bytes(), + source(RECIPE), + "service-incarnation-1", + Some(18001), + 110, + &providers, + ) + } + + #[test] + fn a_compiled_plan_freezes_idunn_default_deadlines() { + let plan = plan(); + assert_eq!(plan.schema, COMPILED_DEPLOYMENT_PLAN_SCHEMA); + assert_eq!(plan.deadlines, Some(PhaseDeadlines::IDUNN_DEFAULTS)); + assert_eq!(plan.phase_deadlines(), PhaseDeadlines::IDUNN_DEFAULTS); + plan.validate().unwrap(); + } + + #[test] + fn a_binding_overrides_only_the_deadlines_it_names() { + let binding = binding_with_deadlines("[deadlines]\nawaiting_ready_seconds = 45\n"); + let plan = plan_under(&binding).unwrap(); + let frozen = plan.phase_deadlines(); + assert_eq!(frozen.awaiting_ready_seconds, 45); + assert_eq!( + PhaseDeadlines { + awaiting_ready_seconds: DEFAULT_AWAITING_READY_DEADLINE_SECONDS, + ..frozen + }, + PhaseDeadlines::IDUNN_DEFAULTS + ); + } + + #[test] + fn deadline_bounds_and_unknown_phases_are_refused() { + for table in [ + "[deadlines]\nrouting_seconds = 0\n", + "[deadlines]\nrouting_seconds = 86401\n", + "[deadlines]\nwarming_seconds = 5\n", + ] { + assert!( + OperatorBinding::parse(&binding_with_deadlines(table)).is_err(), + "{table}" + ); + } + } + + #[test] + fn frozen_deadlines_must_agree_with_the_binding_and_the_plan_schema() { + let binding = binding_with_deadlines("[deadlines]\nrouting_seconds = 45\n"); + let sealed = |mut plan: CompiledDeploymentPlan| { + plan.plan_id = plan.recomputed_plan_id().unwrap(); + plan + }; + let honest = plan_under(&binding).unwrap(); + + let mut drifted = honest.clone(); + drifted.deadlines.as_mut().unwrap().routing_seconds = 46; + assert!(sealed(drifted).validate().is_err()); + + let mut absent = honest.clone(); + absent.deadlines = None; + assert!(sealed(absent).validate().is_err()); + + let mut v2_with_deadlines = honest.clone(); + v2_with_deadlines.schema = COMPILED_DEPLOYMENT_PLAN_SCHEMA_V2.into(); + assert!(sealed(v2_with_deadlines).validate().is_err()); + + // A plan sealed before deadlines existed keeps its nine-field encoding, + // so its plan id still holds, and it runs on Idunn's defaults. + let mut v2 = plan(); + v2.schema = COMPILED_DEPLOYMENT_PLAN_SCHEMA_V2.into(); + v2.deadlines = None; + let v2 = sealed(v2); + v2.validate().unwrap(); + assert_eq!(v2.phase_deadlines(), PhaseDeadlines::IDUNN_DEFAULTS); + assert_eq!(rmp_serde::to_vec(&v2).unwrap()[0], 0x99); + assert_eq!(rmp_serde::to_vec(&honest).unwrap()[0], 0x9a); + } } From 09222ad8eb796dd1ecdd070fafe475f7053bb072 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 02:14:47 +0200 Subject: [PATCH 005/286] Bump admitted generation to v3 and transaction to v4 with typed readiness (F0) One migration for the route-continuity foundation. No decision reads the new state. - ReadinessEvidence (OdinCorrelated | RouteProof) replaces the bare Odin Ready receipt on both records. The class a target must collect is derived from its Expected (ReadinessClass::of): a shared-infrastructure odin.verse-rendezvous dependency means Odin-correlated, none means route-proof. It is not stored. RouteProof evidence is refused for a target that declares Odin. - Generation carries route supervision state (challenge times, failures, degraded marker, actuation window) and continuity backoff. A new generation inherits the actuation window and backoff and starts unobserved and undegraded. route_repair_started_at stays readable. - Transaction carries the current phase deadline (stamped by transition from the plan's frozen durations) and lease adoption evidence. - FailedAfterFencing names its recovery: RestartAdmitted, RestoreIncumbent (the default for older records) or OperatorRequired{reason}. - Every v2 generation lifts as Odin-correlated with its receipts unchanged; v2 and v3 transactions lift through LegacyDeploymentTransaction. The boot migration rewrites both, preserving stored_at, and history reads lift the same way. The Legacy structs die when no older record can remain. Fixtures under tests/fixtures/idunn-control-legacy are the bytes the 9001b58 encoder wrote for records built through its own constructors. Co-Authored-By: Claude Sonnet 5 --- src/control_plane.rs | 1382 +++++++++++++++-- tests/fixtures/idunn-control-legacy/README.md | 24 + .../generation-route-repair-started.hex | 2 + .../generation-with-receipts.hex | 2 + .../transaction-awaiting-ready.hex | 2 + .../transaction-committing-v2.hex | 2 + .../transaction-committing.hex | 2 + .../transaction-failed-after-fencing.hex | 2 + .../transaction-fencing.hex | 2 + .../transaction-leasing.hex | 2 + .../transaction-routing.hex | 2 + 11 files changed, 1309 insertions(+), 115 deletions(-) create mode 100644 tests/fixtures/idunn-control-legacy/README.md create mode 100644 tests/fixtures/idunn-control-legacy/generation-route-repair-started.hex create mode 100644 tests/fixtures/idunn-control-legacy/generation-with-receipts.hex create mode 100644 tests/fixtures/idunn-control-legacy/transaction-awaiting-ready.hex create mode 100644 tests/fixtures/idunn-control-legacy/transaction-committing-v2.hex create mode 100644 tests/fixtures/idunn-control-legacy/transaction-committing.hex create mode 100644 tests/fixtures/idunn-control-legacy/transaction-failed-after-fencing.hex create mode 100644 tests/fixtures/idunn-control-legacy/transaction-fencing.hex create mode 100644 tests/fixtures/idunn-control-legacy/transaction-leasing.hex create mode 100644 tests/fixtures/idunn-control-legacy/transaction-routing.hex diff --git a/src/control_plane.rs b/src/control_plane.rs index 99126cc..fc3e9aa 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -49,9 +49,13 @@ use crate::host_actuator::{ }; const DEPLOYMENT_COMMAND_SCHEMA: &str = "idunn.deployment_command.v2"; -const DEPLOYMENT_TRANSACTION_SCHEMA: &str = "idunn.deployment_transaction.v3"; +const DEPLOYMENT_TRANSACTION_SCHEMA: &str = "idunn.deployment_transaction.v4"; +const DEPLOYMENT_TRANSACTION_SCHEMA_V3: &str = "idunn.deployment_transaction.v3"; const DEPLOYMENT_TRANSACTION_SCHEMA_V2: &str = "idunn.deployment_transaction.v2"; -const ADMITTED_GENERATION_SCHEMA: &str = "idunn.admitted_generation.v2"; +const ADMITTED_GENERATION_SCHEMA: &str = "idunn.admitted_generation.v3"; +const ADMITTED_GENERATION_SCHEMA_V2: &str = "idunn.admitted_generation.v2"; +/// The capability whose declaration makes a target Odin-correlated. +const ODIN_RENDEZVOUS_CAPABILITY: &str = "odin.verse-rendezvous"; /// How many times continuity will restart one admitted release before it /// concludes the release itself is the problem. More than one because a start /// can fail for a passing reason -- a port still held, a peer not yet up -- @@ -518,12 +522,223 @@ impl RoutingEvidence { } } +/// Which kind of proof admits a target as Ready. Derived from the target's +/// Expected, never stored: Expected is digest-bound, so the class cannot drift +/// from the incarnation it describes. A target that declares a +/// `shared-infrastructure odin.verse-rendezvous` dependency advertises into the +/// Verse and is Odin-correlated; one that declares none is route-proof. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ReadinessClass { + OdinCorrelated, + RouteProof, +} + +impl ReadinessClass { + fn of(expected: &IdunnExpectedIncarnationRecord) -> Self { + let declares_odin = expected.dependencies.iter().any(|dependency| { + dependency.kind == "shared-infrastructure" + && dependency.capability == ODIN_RENDEZVOUS_CAPABILITY + }); + if declares_odin { + Self::OdinCorrelated + } else { + Self::RouteProof + } + } +} + +/// The receipt that admitted a candidate as Ready, tagged by the class of proof +/// that produced it. The tag records what was collected; the class a target +/// must collect is `ReadinessClass::of(expected)`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "class", rename_all = "kebab-case", deny_unknown_fields)] +enum ReadinessEvidence { + OdinCorrelated { evidence: TopologyEvidence }, + RouteProof { evidence: RuntimePresenceEvidence }, +} + +impl ReadinessEvidence { + fn class(&self) -> ReadinessClass { + match self { + Self::OdinCorrelated { .. } => ReadinessClass::OdinCorrelated, + Self::RouteProof { .. } => ReadinessClass::RouteProof, + } + } + + fn validate_shape(&self) -> Result<()> { + match self { + Self::OdinCorrelated { evidence } => evidence.validate_shape(), + Self::RouteProof { evidence } => evidence.validate_shape(), + } + } + + fn odin(&self) -> Option<&TopologyEvidence> { + match self { + Self::OdinCorrelated { evidence } => Some(evidence), + Self::RouteProof { .. } => None, + } + } +} + +/// Signed evidence that the candidate process picked up the lease Idunn +/// granted: its presence carries `write_lease_sha256` equal to the grant. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct LeaseAdoptionEvidence { + write_lease_sha256: String, + signed_presence_sha256: String, + source: AdoptionSource, + observed_at_unix_millis: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +enum AdoptionSource { + Direct, + OdinTopology, +} + +impl LeaseAdoptionEvidence { + /// Whether this evidence is about exactly the lease that was granted. + fn names(&self, leasing: &LeasingEvidence) -> bool { + leasing.lease_sha256() == Some(self.write_lease_sha256.as_str()) + } + + fn validate_shape(&self) -> Result<()> { + require_id(&self.write_lease_sha256, "adopted lease digest")?; + require_id(&self.signed_presence_sha256, "lease adoption presence digest")?; + ensure!( + self.observed_at_unix_millis > 0, + "lease adoption has no observation time" + ); + Ok(()) + } +} + +/// When the transaction entered its current post-fencing phase and when that +/// phase must end. Durations come from the plan's frozen `PhaseDeadlines`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct PhaseDeadline { + phase: DeploymentPhase, + entered_at_unix_millis: u64, + deadline_at_unix_millis: u64, +} + +impl PhaseDeadline { + /// `None` for a phase that has no deadline (pre-fencing and Complete). + fn entering( + phase: DeploymentPhase, + plan: &CompiledDeploymentPlan, + now: u64, + ) -> Option { + let deadlines = plan.phase_deadlines(); + let seconds = match phase { + DeploymentPhase::Fencing => deadlines.fencing_seconds, + DeploymentPhase::Leasing => deadlines.leasing_seconds, + DeploymentPhase::AwaitingReady => deadlines.awaiting_ready_seconds, + DeploymentPhase::Routing => deadlines.routing_seconds, + DeploymentPhase::Committing => deadlines.committing_seconds, + _ => return None, + }; + Some(Self { + phase, + entered_at_unix_millis: now, + deadline_at_unix_millis: now.saturating_add(u64::from(seconds) * 1000), + }) + } +} + +/// How a post-fencing failure recovers. `RestoreIncumbent` is what a +/// post-fencing abort has always done and is the default for records that +/// predate the field. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "action", rename_all = "kebab-case", deny_unknown_fields)] +enum TerminalRecovery { + RestartAdmitted, + #[default] + RestoreIncumbent, + OperatorRequired { + reason: String, + }, +} + +/// Route supervision's durable memory for one admitted routed target. Carried +/// across commits except for the per-incarnation parts. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct RouteSupervisionState { + last_challenge_at_unix_millis: Option, + consecutive_failures: u32, + next_challenge_at_unix_millis: Option, + /// Set while route proof keeps failing. Marks the route degraded so + /// dependents stop selecting it; it never authorizes a restart. + degraded_since_unix_millis: Option, + actuations: ActuationWindow, +} + +/// Route actuations (write, reload, firewall) inside a rolling window, so a +/// per-target ceiling survives an Idunn restart. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct ActuationWindow { + window_started_at_unix_millis: u64, + count: u32, +} + +impl RouteSupervisionState { + /// A new incarnation starts unobserved and undegraded, but inherits the + /// target's actuation window: a ceiling that reset on every commit would + /// not be a ceiling. + fn for_new_incarnation(incumbent: Option<&Self>) -> Self { + Self { + actuations: incumbent.map(|state| state.actuations).unwrap_or_default(), + ..Self::default() + } + } + + fn validate(&self) -> Result<()> { + if let Some(since) = self.degraded_since_unix_millis { + ensure!(since > 0, "route degradation has no start time"); + } + ensure!( + self.actuations.count == 0 || self.actuations.window_started_at_unix_millis > 0, + "route actuation window has a count but no start" + ); + Ok(()) + } +} + +/// Continuity restarts of one target inside a rolling window. Belongs to the +/// target, not the generation: a restart that succeeds must not reset it. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct ContinuityBackoff { + window_started_at_unix_millis: Option, + attempts: u32, + next_restart_at_unix_millis: Option, +} + +impl ContinuityBackoff { + fn validate(&self) -> Result<()> { + ensure!( + self.attempts == 0 || self.window_started_at_unix_millis.is_some(), + "continuity backoff has attempts but no window" + ); + Ok(()) + } +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "outcome", rename_all = "kebab-case", deny_unknown_fields)] enum TransactionCompletion { Admitted { generation_id: String }, FailedBeforeFencing { error: String }, - FailedAfterFencing { error: String }, + FailedAfterFencing { + error: String, + #[serde(default)] + recovery: TerminalRecovery, + }, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -642,7 +857,7 @@ impl PostCommitCleanup { #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] #[cultcache( type = "idunn.deployment_transaction", - schema = "idunn.deployment_transaction.v2" + schema = "idunn.deployment_transaction.v4" )] struct DeploymentTransaction { #[cultcache(key = 0)] @@ -700,7 +915,7 @@ struct DeploymentTransaction { #[cultcache(key = 26)] leasing: Option, #[cultcache(key = 27)] - ready: Option, + ready: Option, #[cultcache(key = 28)] routing: Option, #[cultcache(key = 29)] @@ -719,6 +934,11 @@ struct DeploymentTransaction { // control store and crashloops. #[cultcache(key = 34, default)] post_fencing_abort: Option, + /// Set by `transition` on entry to each post-fencing phase. + #[cultcache(key = 35)] + phase_deadline: Option, + #[cultcache(key = 36)] + lease_adoption: Option, } impl DeploymentTransaction { @@ -775,6 +995,8 @@ impl DeploymentTransaction { pre_fencing_abort: None, post_fencing_abort: None, post_commit_cleanup: None, + phase_deadline: None, + lease_adoption: None, }; transaction.validate()?; Ok(transaction) @@ -921,7 +1143,8 @@ impl DeploymentTransaction { self.workload.is_none() || self.activation.is_some(), "workload observation exists without its prepared activation" ); - for evidence in [&self.latest_odin_observation, &self.ready] + let ready_odin = self.ready.as_ref().and_then(ReadinessEvidence::odin); + for evidence in [self.latest_odin_observation.as_ref(), ready_odin] .into_iter() .flatten() { @@ -931,6 +1154,15 @@ impl DeploymentTransaction { "topology evidence exceeds the transaction replay cursor" ); } + if let Some(ready) = &self.ready { + ready.validate_shape()?; + if let (ReadinessClass::RouteProof, Some(expected)) = (ready.class(), &self.expected) { + ensure!( + ReadinessClass::of(expected) == ReadinessClass::RouteProof, + "route-proof Ready evidence for a target that declares Odin" + ); + } + } if let Some(warming) = &self.warming { warming.validate_shape()?; match warming { @@ -993,6 +1225,26 @@ impl DeploymentTransaction { if let Some(error) = &self.last_error { require_detail(error, "transaction error")?; } + if let Some(deadline) = &self.phase_deadline { + ensure!( + deadline.phase == self.phase + && (DeploymentPhase::Fencing..=DeploymentPhase::Committing) + .contains(&deadline.phase) + && deadline.entered_at_unix_millis > 0 + && deadline.entered_at_unix_millis <= self.updated_at_unix_millis + && deadline.deadline_at_unix_millis > deadline.entered_at_unix_millis, + "phase deadline does not describe the current post-fencing phase" + ); + } + if let Some(adoption) = &self.lease_adoption { + adoption.validate_shape()?; + ensure!( + self.leasing + .as_ref() + .is_some_and(|leasing| adoption.names(leasing)), + "lease adoption does not name the granted lease" + ); + } if let Some(authorization) = &self.deployment_authorization { authorization.validate_shape()?; } @@ -1069,11 +1321,14 @@ impl DeploymentTransaction { &self.post_fencing_abort, "terminal post-fence abort evidence", )?; - let TransactionCompletion::FailedAfterFencing { error } = + let TransactionCompletion::FailedAfterFencing { error, recovery } = self.completion.as_ref().unwrap() else { unreachable!() }; + if let TerminalRecovery::OperatorRequired { reason } = recovery { + require_detail(reason, "operator-required recovery reason")?; + } ensure!( abort.is_complete() && abort.error == *error, "terminal failure lacks complete matching abort evidence" @@ -1290,7 +1545,7 @@ impl AdmittedOdinAuthority { #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] #[cultcache( type = "idunn.admitted_generation", - schema = "idunn.admitted_generation.v2" + schema = "idunn.admitted_generation.v3" )] struct AdmittedGeneration { #[cultcache(key = 0)] @@ -1319,30 +1574,84 @@ struct AdmittedGeneration { workload: WorkloadObservation, #[cultcache(key = 12)] leasing: LeasingEvidence, + /// The receipt that admitted this incarnation, tagged by proof class. #[cultcache(key = 13)] - ready: TopologyEvidence, + ready: ReadinessEvidence, + /// Present exactly when `ready` is Odin-correlated. #[cultcache(key = 14)] - latest_odin_observation: TopologyEvidence, + latest_odin_observation: Option, #[cultcache(key = 15)] routing: RoutingEvidence, + /// Present exactly when `ready` is Odin-correlated. #[cultcache(key = 16)] - odin_authority: AdmittedOdinAuthority, + odin_authority: Option, + /// Meaningful only for an Odin-correlated generation. #[cultcache(key = 17)] odin_publisher_sequence_cursor: u64, + /// Still readable, decides nothing new: route supervision state replaces it. #[cultcache(key = 18)] route_repair_started_at_unix_millis: Option, + /// Present exactly when `routing` is Promoted. + #[cultcache(key = 19)] + route_supervision: Option, + #[cultcache(key = 20)] + continuity_backoff: ContinuityBackoff, +} + +/// The Odin-side receipts of an Odin-correlated generation. Decisions that +/// consume Odin evidence read it through here, so a route-proof generation +/// cannot be mistaken for one. +struct OdinReceipts<'a> { + ready: &'a TopologyEvidence, + latest: &'a TopologyEvidence, + authority: &'a AdmittedOdinAuthority, } impl AdmittedGeneration { + fn odin_receipts(&self) -> Result> { + match ( + &self.ready, + &self.latest_odin_observation, + &self.odin_authority, + ) { + (ReadinessEvidence::OdinCorrelated { evidence }, Some(latest), Some(authority)) => { + Ok(OdinReceipts { + ready: evidence, + latest, + authority, + }) + } + _ => bail!("admitted generation of {} is not Odin-correlated", self.target), + } + } + fn from_transaction( transaction: &DeploymentTransaction, odin_authority: AdmittedOdinAuthority, + incumbent: Option<&AdmittedGeneration>, now: u64, ) -> Result { ensure!( transaction.phase == DeploymentPhase::Committing, "only Committing can create an admitted generation" ); + let ready = required(&transaction.ready, "Ready receipt")?.clone(); + let (latest_odin_observation, odin_authority) = match ready.class() { + ReadinessClass::OdinCorrelated => ( + Some( + required(&transaction.latest_odin_observation, "latest Odin receipt")? + .clone(), + ), + Some(odin_authority), + ), + ReadinessClass::RouteProof => (None, None), + }; + let routing = required(&transaction.routing, "route disposition")?.clone(); + let route_supervision = matches!(&routing, RoutingEvidence::Promoted { .. }).then(|| { + RouteSupervisionState::for_new_incarnation( + incumbent.and_then(|value| value.route_supervision.as_ref()), + ) + }); let generation = Self { schema_version: ADMITTED_GENERATION_SCHEMA.into(), target: transaction.target.clone(), @@ -1358,16 +1667,16 @@ impl AdmittedGeneration { activation: required(&transaction.activation, "activation")?.clone(), workload: required(&transaction.workload, "workload")?.clone(), leasing: required(&transaction.leasing, "lease disposition")?.clone(), - ready: required(&transaction.ready, "Ready receipt")?.clone(), - latest_odin_observation: required( - &transaction.latest_odin_observation, - "latest Odin receipt", - )? - .clone(), - routing: required(&transaction.routing, "route disposition")?.clone(), + ready, + latest_odin_observation, + routing, odin_authority, odin_publisher_sequence_cursor: transaction.odin_publisher_sequence_cursor, route_repair_started_at_unix_millis: None, + route_supervision, + continuity_backoff: incumbent + .map(|value| value.continuity_backoff.clone()) + .unwrap_or_default(), }; generation.validate()?; Ok(generation) @@ -1388,18 +1697,33 @@ impl AdmittedGeneration { self.expected.validate()?; self.activation.validate()?; self.ready.validate_shape()?; - self.latest_odin_observation.validate_shape()?; - self.odin_authority.validate()?; + match ( + &self.ready, + &self.latest_odin_observation, + &self.odin_authority, + ) { + (ReadinessEvidence::OdinCorrelated { evidence }, Some(latest), Some(authority)) => { + latest.validate_shape()?; + authority.validate()?; + ensure!( + evidence.publisher_sequence <= self.odin_publisher_sequence_cursor + && latest.publisher_sequence == self.odin_publisher_sequence_cursor, + "admitted generation evidence does not describe one incarnation" + ); + } + (ReadinessEvidence::RouteProof { .. }, None, None) => ensure!( + ReadinessClass::of(&self.expected) == ReadinessClass::RouteProof, + "route-proof Ready evidence for a target that declares Odin" + ), + _ => bail!("admitted readiness evidence and Odin receipts disagree"), + } ensure!( self.target == self.expected.target && self.expected.plan_id == self.plan.plan_id && self.expected.sealed_release_id == self.sealed_release.sealed_release_id && self.activation.expected_projection_sha256 == self.expected.canonical_sha256()? - && self.activation.runtime_instance_id == self.workload.runtime_instance_id() - && self.ready.publisher_sequence <= self.odin_publisher_sequence_cursor - && self.latest_odin_observation.publisher_sequence - == self.odin_publisher_sequence_cursor, + && self.activation.runtime_instance_id == self.workload.runtime_instance_id(), "admitted generation evidence does not describe one incarnation" ); ensure!( @@ -1424,6 +1748,15 @@ impl AdmittedGeneration { "only a promoted routed generation can own route repair intent" ); } + ensure!( + self.route_supervision.is_some() + == matches!(&self.routing, RoutingEvidence::Promoted { .. }), + "route supervision state exists exactly for a promoted route" + ); + if let Some(state) = &self.route_supervision { + state.validate()?; + } + self.continuity_backoff.validate()?; Ok(()) } } @@ -1859,12 +2192,7 @@ impl ControlSnapshot { snapshot.transactions.push(Stored { envelope, value }); } AdmittedGeneration::TYPE => { - ensure!( - envelope.schema_id.as_deref() == Some(ADMITTED_GENERATION_SCHEMA), - "Idunn control store contains an unsupported admitted generation" - ); - let value: AdmittedGeneration = decode_record(&envelope)?; - value.validate()?; + let value = read_generation_record(&envelope)?; ensure!( envelope.key == value.target, "admitted generation key is not its target" @@ -2049,7 +2377,11 @@ impl ControlSnapshot { .iter() .filter(|stored| { stored.value.target == target - && stored.value.odin_authority.signer_identity_id == signer_identity_id + && stored + .value + .odin_authority + .as_ref() + .is_some_and(|authority| authority.signer_identity_id == signer_identity_id) }) .map(|stored| stored.value.odin_publisher_sequence_cursor) .max() @@ -2081,20 +2413,255 @@ fn command_envelope(value: &DeploymentCommand, now: u64) -> Result, + #[cultcache(key = 10)] + plan: Option, + #[cultcache(key = 11)] + frozen_source: Option, + #[cultcache(key = 12)] + sealed_release: Option, + #[cultcache(key = 13)] + installed_release: Option, + #[cultcache(key = 14)] + expected: Option, + #[cultcache(key = 15)] + expected_publication_sha256: Option, + #[cultcache(key = 16)] + deployment_authorization: Option, + #[cultcache(key = 17)] + lifecycle_authorized_at_unix_millis: Option, + #[cultcache(key = 18)] + activation: Option, + #[cultcache(key = 19)] + workload: Option, + #[cultcache(key = 20)] + activation_publication_sha256: Option, + #[cultcache(key = 21)] + latest_odin_observation: Option, + #[cultcache(key = 22)] + warming: Option, + #[cultcache(key = 23)] + route_preflight: Option, + #[cultcache(key = 24)] + isolation: Option, + #[cultcache(key = 25)] + fencing: Option, + #[cultcache(key = 26)] + leasing: Option, + #[cultcache(key = 27)] + ready: Option, + #[cultcache(key = 28)] + routing: Option, + #[cultcache(key = 29)] + odin_publisher_sequence_cursor: u64, + #[cultcache(key = 30)] + last_error: Option, + #[cultcache(key = 31)] + completion: Option, + #[cultcache(key = 32)] + pre_fencing_abort: Option, + #[cultcache(key = 33)] + post_commit_cleanup: Option, + // A v2 record has no key 34: it predates post-fencing aborts. + #[cultcache(key = 34, default)] + post_fencing_abort: Option, +} + +impl LegacyDeploymentTransaction { + /// Every legacy Ready receipt is an Odin receipt: no route-proof class + /// existed. The new slots start empty, which is what an older transaction + /// had. + fn into_current(self) -> DeploymentTransaction { + DeploymentTransaction { + schema_version: DEPLOYMENT_TRANSACTION_SCHEMA.into(), + transaction_id: self.transaction_id, + command_id: self.command_id, + command_kind: self.command_kind, + target: self.target, + ordinal: self.ordinal, + phase: self.phase, + created_at_unix_millis: self.created_at_unix_millis, + updated_at_unix_millis: self.updated_at_unix_millis, + incumbent_generation_id: self.incumbent_generation_id, + plan: self.plan, + frozen_source: self.frozen_source, + sealed_release: self.sealed_release, + installed_release: self.installed_release, + expected: self.expected, + expected_publication_sha256: self.expected_publication_sha256, + deployment_authorization: self.deployment_authorization, + lifecycle_authorized_at_unix_millis: self.lifecycle_authorized_at_unix_millis, + activation: self.activation, + workload: self.workload, + activation_publication_sha256: self.activation_publication_sha256, + latest_odin_observation: self.latest_odin_observation, + warming: self.warming, + route_preflight: self.route_preflight, + isolation: self.isolation, + fencing: self.fencing, + leasing: self.leasing, + ready: self + .ready + .map(|evidence| ReadinessEvidence::OdinCorrelated { evidence }), + routing: self.routing, + odin_publisher_sequence_cursor: self.odin_publisher_sequence_cursor, + last_error: self.last_error, + completion: self.completion, + pre_fencing_abort: self.pre_fencing_abort, + post_commit_cleanup: self.post_commit_cleanup, + post_fencing_abort: self.post_fencing_abort, + phase_deadline: None, + lease_adoption: None, + } + } +} + +/// The v2 admitted-generation layout. Same lifetime rule as +/// `LegacyDeploymentTransaction`. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.admitted_generation", + schema = "idunn.admitted_generation.v2" +)] +struct LegacyAdmittedGeneration { + #[cultcache(key = 0)] + schema_version: String, + #[cultcache(key = 1)] + target: String, + #[cultcache(key = 2)] + generation_id: String, + #[cultcache(key = 3)] + command_id: String, + #[cultcache(key = 4)] + transaction_id: String, + #[cultcache(key = 5)] + admitted_at_unix_millis: u64, + #[cultcache(key = 6)] + plan: CompiledDeploymentPlan, + #[cultcache(key = 7)] + sealed_release: SealedRelease, + #[cultcache(key = 8)] + installed_release: InstalledReleaseObservation, + #[cultcache(key = 9)] + expected: IdunnExpectedIncarnationRecord, + #[cultcache(key = 10)] + activation: IdunnRuntimeActivationRecord, + #[cultcache(key = 11)] + workload: WorkloadObservation, + #[cultcache(key = 12)] + leasing: LeasingEvidence, + #[cultcache(key = 13)] + ready: TopologyEvidence, + #[cultcache(key = 14)] + latest_odin_observation: TopologyEvidence, + #[cultcache(key = 15)] + routing: RoutingEvidence, + #[cultcache(key = 16)] + odin_authority: AdmittedOdinAuthority, + #[cultcache(key = 17)] + odin_publisher_sequence_cursor: u64, + #[cultcache(key = 18)] + route_repair_started_at_unix_millis: Option, +} + +impl LegacyAdmittedGeneration { + /// Every v2 generation was admitted on Odin receipts, so it migrates as + /// Odin-correlated with those receipts unchanged. Route supervision state + /// starts empty for a promoted route; backoff starts empty. + fn into_current(self) -> AdmittedGeneration { + let route_supervision = matches!(&self.routing, RoutingEvidence::Promoted { .. }) + .then(RouteSupervisionState::default); + AdmittedGeneration { + schema_version: ADMITTED_GENERATION_SCHEMA.into(), + target: self.target, + generation_id: self.generation_id, + command_id: self.command_id, + transaction_id: self.transaction_id, + admitted_at_unix_millis: self.admitted_at_unix_millis, + plan: self.plan, + sealed_release: self.sealed_release, + installed_release: self.installed_release, + expected: self.expected, + activation: self.activation, + workload: self.workload, + leasing: self.leasing, + ready: ReadinessEvidence::OdinCorrelated { + evidence: self.ready, + }, + latest_odin_observation: Some(self.latest_odin_observation), + routing: self.routing, + odin_authority: Some(self.odin_authority), + odin_publisher_sequence_cursor: self.odin_publisher_sequence_cursor, + route_repair_started_at_unix_millis: self.route_repair_started_at_unix_millis, + route_supervision, + continuity_backoff: ContinuityBackoff::default(), + } + } +} + +/// Decode a v2 or v3 transaction and lift it to the current shape, without +/// validating it. History reads stop here; the control store validates. +fn lift_legacy_transaction(envelope: &CultCacheEnvelope) -> Result { + let schema = match envelope.schema_id.as_deref() { + Some(schema @ (DEPLOYMENT_TRANSACTION_SCHEMA_V2 | DEPLOYMENT_TRANSACTION_SCHEMA_V3)) => { + schema + } + _ => bail!("Idunn control store contains an unsupported transaction"), + }; + let value: LegacyDeploymentTransaction = rmp_serde::from_slice(&envelope.payload) + .with_context(|| format!("decoding a {schema} deployment transaction"))?; + ensure!( + value.schema_version == schema, + "stored transaction schema differs from its envelope" + ); + ensure!( + schema != DEPLOYMENT_TRANSACTION_SCHEMA_V2 || value.post_fencing_abort.is_none(), + "a v2 transaction cannot carry post-fencing abort evidence" + ); + Ok(value.into_current()) +} + +/// Read one stored transaction, lifting a v2 or v3 record to the current shape. /// /// The control store refuses noncanonical bytes: a record must re-encode to /// exactly what is stored. That check is what makes tampering visible, and it -/// also means adding a field is a schema change -- every transaction written -/// before `post_fencing_abort` existed re-encodes with one extra key and would -/// be read as tampered. +/// also means adding a field is a schema change: an older record re-encodes +/// with extra keys and would be read as tampered. /// -/// So a v2 record is decoded by key, which tolerates the absent key, and lifted -/// with no post-fencing abort, because a transaction written before the field -/// existed cannot have had one. Byte-exactness cannot apply across a version -/// boundary -- the bytes are a different schema by definition -- so the lift -/// leans on the full semantic `validate()` instead, and everything downstream -/// sees only the current shape. +/// So an older record is decoded by its own layout and lifted. Byte-exactness +/// cannot apply across a version boundary -- the bytes are a different schema +/// by definition -- so the lift leans on the full semantic `validate()` +/// instead, and everything downstream sees only the current shape. fn read_transaction_record(envelope: &CultCacheEnvelope) -> Result { match envelope.schema_id.as_deref() { Some(DEPLOYMENT_TRANSACTION_SCHEMA) => { @@ -2102,33 +2669,46 @@ fn read_transaction_record(envelope: &CultCacheEnvelope) -> Result { - let mut value: DeploymentTransaction = rmp_serde::from_slice(&envelope.payload) - .context("decoding a v2 deployment transaction")?; - ensure!( - value.schema_version == DEPLOYMENT_TRANSACTION_SCHEMA_V2, - "stored transaction schema differs from its envelope" - ); - ensure!( - value.post_fencing_abort.is_none(), - "a v2 transaction cannot carry post-fencing abort evidence" - ); - value.schema_version = DEPLOYMENT_TRANSACTION_SCHEMA.into(); + _ => { + let value = lift_legacy_transaction(envelope)?; value.validate()?; Ok(value) } - _ => bail!("Idunn control store contains an unsupported transaction"), } } -/// Rewrite every v2 transaction as v3, once, before the engine runs. +/// Read one stored admitted generation, lifting a v2 record to the current +/// shape by the same rule as `read_transaction_record`. +fn read_generation_record(envelope: &CultCacheEnvelope) -> Result { + match envelope.schema_id.as_deref() { + Some(ADMITTED_GENERATION_SCHEMA) => { + let value: AdmittedGeneration = decode_record(envelope)?; + value.validate()?; + Ok(value) + } + Some(ADMITTED_GENERATION_SCHEMA_V2) => { + let legacy: LegacyAdmittedGeneration = rmp_serde::from_slice(&envelope.payload) + .context("decoding a v2 admitted generation")?; + ensure!( + legacy.schema_version == ADMITTED_GENERATION_SCHEMA_V2, + "stored generation schema differs from its envelope" + ); + let value = legacy.into_current(); + value.validate()?; + Ok(value) + } + _ => bail!("Idunn control store contains an unsupported admitted generation"), + } +} + +/// Rewrite every v2 and v3 record as current, once, before the engine runs. /// -/// The read path lifts v2 records on its own, so this is convergence rather -/// than correctness: without it the store keeps records in two shapes for as -/// long as the oldest terminal transaction survives retention. Each rewrite is -/// a compare-exchange against the exact stored envelope, so a record that -/// changed underneath is left alone rather than clobbered. -fn migrate_transactions_to_current_schema(store_path: &Path) -> Result { +/// The read path lifts older records on its own, so this is convergence rather +/// than correctness: without it the store keeps records in several shapes for +/// as long as the oldest one survives. Each rewrite is a compare-exchange +/// against the exact stored envelope, so a record that changed underneath is +/// left alone rather than clobbered. +fn migrate_control_store_to_current_schema(store_path: &Path) -> Result { if !store_path.exists() { return Ok(0); } @@ -2138,24 +2718,38 @@ fn migrate_transactions_to_current_schema(store_path: &Path) -> Result { .context("reading Idunn control snapshot for migration")? .into_iter() .filter(|envelope| { - envelope.r#type == DeploymentTransaction::TYPE - && envelope.schema_id.as_deref() == Some(DEPLOYMENT_TRANSACTION_SCHEMA_V2) + let schema = envelope.schema_id.as_deref(); + (envelope.r#type == DeploymentTransaction::TYPE + && matches!( + schema, + Some(DEPLOYMENT_TRANSACTION_SCHEMA_V2 | DEPLOYMENT_TRANSACTION_SCHEMA_V3) + )) + || (envelope.r#type == AdmittedGeneration::TYPE + && schema == Some(ADMITTED_GENERATION_SCHEMA_V2)) }) .collect::>(); let mut migrated = 0; for envelope in stale { - let value = read_transaction_record(&envelope)?; - let next = transaction_envelope(&value, value.updated_at_unix_millis)?; + let (record_type, key, mut next) = if envelope.r#type == DeploymentTransaction::TYPE { + let value = read_transaction_record(&envelope)?; + let next = transaction_envelope(&value, value.updated_at_unix_millis)?; + (DeploymentTransaction::TYPE, value.transaction_id, next) + } else { + let value = read_generation_record(&envelope)?; + let next = admitted_envelope(&value, value.admitted_at_unix_millis)?; + (AdmittedGeneration::TYPE, value.target, next) + }; + next.stored_at = envelope.stored_at.clone(); ensure!( store.compare_exchange( &[CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: value.transaction_id.clone(), + r#type: record_type.into(), + key, current: Some(envelope.clone()), }], &[next], )?, - "deployment transaction changed during schema migration" + "control record changed during schema migration" ); migrated += 1; } @@ -2268,8 +2862,11 @@ fn read_history_transactions(state_store: &Path) -> Vec { envelopes .into_iter() .filter(|envelope| envelope.r#type == DeploymentTransaction::TYPE) - .filter_map(|envelope| { - rmp_serde::from_slice::(&envelope.payload).ok() + .filter_map(|envelope| match envelope.schema_id.as_deref() { + Some(DEPLOYMENT_TRANSACTION_SCHEMA_V2 | DEPLOYMENT_TRANSACTION_SCHEMA_V3) => { + lift_legacy_transaction(&envelope).ok() + } + _ => rmp_serde::from_slice::(&envelope.payload).ok(), }) .collect() } @@ -2509,7 +3106,7 @@ fn submit( .iter() .find_map(|transaction| match &transaction.completion { Some(TransactionCompletion::FailedBeforeFencing { error }) - | Some(TransactionCompletion::FailedAfterFencing { error }) => { + | Some(TransactionCompletion::FailedAfterFencing { error, .. }) => { Some(error.as_str()) } _ => None, @@ -2635,7 +3232,7 @@ fn derived_command_status(transactions: &[&DeploymentTransaction]) -> (&'static .iter() .find_map(|transaction| match &transaction.completion { Some(TransactionCompletion::FailedBeforeFencing { error }) - | Some(TransactionCompletion::FailedAfterFencing { error }) => Some(error.clone()), + | Some(TransactionCompletion::FailedAfterFencing { error, .. }) => Some(error.clone()), _ => None, }) { @@ -2849,7 +3446,7 @@ impl Engine { fn current_odin_authority(&self, snapshot: &ControlSnapshot) -> Result { let authority = snapshot .admitted_for("odin") - .map(|stored| stored.value.odin_authority.clone()) + .and_then(|stored| stored.value.odin_authority.clone()) .unwrap_or_else(|| self.bootstrap_odin_authority.clone()); authority.validate()?; Ok(authority) @@ -2953,7 +3550,7 @@ impl Engine { } } } - if let Some(evidence) = &transaction.ready { + if let Some(evidence) = transaction.ready.as_ref().and_then(ReadinessEvidence::odin) { let authenticated = self.authenticate_topology_bytes( snapshot, transaction, @@ -2971,29 +3568,31 @@ impl Engine { let odin_authority = self.current_odin_authority(snapshot)?; for stored in &snapshot.admitted { let generation = &stored.value; + if generation.ready.class() == ReadinessClass::RouteProof { + continue; + } + let odin = generation.odin_receipts()?; let authority = self.runtime_authority_parts( &generation.plan, &generation.expected, &generation.activation, )?; let latest = authenticate_odin_runtime_topology_correlation( - &generation.latest_odin_observation.canonical_bytes, + &odin.latest.canonical_bytes, &authority, generation.leasing.lease_sha256(), &odin_authority.signer_public_key, - self.trusted_topology_context( - generation.latest_odin_observation.admitted_at_unix_millis, - ), + self.trusted_topology_context(odin.latest.admitted_at_unix_millis), )?; - validate_authenticated_evidence(&generation.latest_odin_observation, &latest)?; + validate_authenticated_evidence(odin.latest, &latest)?; let ready = authenticate_odin_runtime_topology_correlation( - &generation.ready.canonical_bytes, + &odin.ready.canonical_bytes, &authority, generation.leasing.lease_sha256(), &odin_authority.signer_public_key, - self.trusted_topology_context(generation.ready.admitted_at_unix_millis), + self.trusted_topology_context(odin.ready.admitted_at_unix_millis), )?; - validate_authenticated_evidence(&generation.ready, &ready)?; + validate_authenticated_evidence(odin.ready, &ready)?; ensure!( is_semantic_ready(&ready), "admitted generation Ready label is not exact semantic Ready" @@ -3019,10 +3618,12 @@ fn serve(options: RuntimeOptions) -> Result<()> { .with_context(|| format!("creating Idunn directory {}", directory.display()))?; } let _lock = ProcessLock::acquire(&options.state_store)?; - let migrated = migrate_transactions_to_current_schema(&options.state_store) - .context("migrating Idunn transactions to the current schema")?; + let migrated = migrate_control_store_to_current_schema(&options.state_store) + .context("migrating Idunn control records to the current schema")?; if migrated > 0 { - println!("migrated {migrated} transaction(s) to {DEPLOYMENT_TRANSACTION_SCHEMA}"); + println!( + "migrated {migrated} control record(s) to {DEPLOYMENT_TRANSACTION_SCHEMA} / {ADMITTED_GENERATION_SCHEMA}" + ); } ControlSnapshot::read(&options.state_store).context("validating all Idunn records")?; let engine = Engine::open(options)?; @@ -3728,6 +4329,10 @@ impl Engine { snapshot: &ControlSnapshot, current: &Stored, ) -> Result { + if current.value.ready.class() == ReadinessClass::RouteProof { + return Ok(false); + } + let odin = current.value.odin_receipts()?; let Some(received) = self.topology().receive( ¤t.value.target, ¤t.value.expected.canonical_sha256()?, @@ -3761,17 +4366,17 @@ impl Engine { }; let evidence = TopologyEvidence::from_authenticated(&authenticated, now)?; if !sequence_requires_admission( - Some(¤t.value.latest_odin_observation), + Some(odin.latest), snapshot.max_odin_sequence(¤t.value.target, &evidence.signer_identity_id), &evidence, )? { return Ok(false); } let mut next = current.value.clone(); - next.latest_odin_observation = evidence.clone(); + next.latest_odin_observation = Some(evidence.clone()); next.odin_publisher_sequence_cursor = evidence.publisher_sequence; if is_semantic_ready(&authenticated) { - next.ready = evidence; + next.ready = ReadinessEvidence::OdinCorrelated { evidence }; } next.validate()?; ensure!( @@ -4463,7 +5068,7 @@ impl Engine { &admitted.value.latest_odin_observation, "sequence-admitted Ready evidence", )?; - if current.value.ready.as_ref() == Some(latest) { + if current.value.ready.as_ref().and_then(ReadinessEvidence::odin) == Some(latest) { ensure!(semantic_ready, "stored Ready receipt changed meaning"); } else if semantic_ready { let evidence = latest.clone(); @@ -4474,7 +5079,7 @@ impl Engine { authenticated, }; return self.persist_same_phase(&admitted, |next| { - next.ready = Some(evidence); + next.ready = Some(ReadinessEvidence::OdinCorrelated { evidence }); Ok(()) }); } else { @@ -4510,10 +5115,10 @@ impl Engine { &admitted.value.latest_odin_observation, "current route-admission topology evidence", )?; - if admitted.value.ready.as_ref() != Some(latest) { + if admitted.value.ready.as_ref().and_then(ReadinessEvidence::odin) != Some(latest) { let latest = latest.clone(); return self.persist_same_phase(&admitted, |next| { - next.ready = Some(latest); + next.ready = Some(ReadinessEvidence::OdinCorrelated { evidence: latest }); Ok(()) }); } @@ -4710,8 +5315,12 @@ impl Engine { let snapshot = ControlSnapshot::read(&self.options.state_store)?; let incumbent = self.exact_incumbent(&snapshot, &commit_current.value)?; let odin_authority = self.current_odin_authority(&snapshot)?; - let generation = - AdmittedGeneration::from_transaction(&commit_current.value, odin_authority, now)?; + let generation = AdmittedGeneration::from_transaction( + &commit_current.value, + odin_authority, + incumbent.map(|stored| &stored.value), + now, + )?; let post_commit_cleanup = PostCommitCleanup { incumbent: match incumbent { Some(incumbent) @@ -5576,7 +6185,10 @@ impl Engine { now: u64, require_current: bool, ) -> Result { - let evidence = required(&transaction.ready, "durable Ready evidence")?.clone(); + let evidence = required(&transaction.ready, "durable Ready evidence")? + .odin() + .context("durable Ready evidence is not an Odin receipt")? + .clone(); let snapshot = ControlSnapshot::read(&self.options.state_store)?; let current_lease = transaction .leasing @@ -5631,6 +6243,7 @@ impl Engine { snapshot: &ControlSnapshot, generation: &AdmittedGeneration, ) -> Result { + let odin = generation.odin_receipts()?; let authority = self.runtime_authority_parts( &generation.plan, &generation.expected, @@ -5639,13 +6252,13 @@ impl Engine { let odin_authority = self.current_odin_authority(snapshot)?; let current_lease = generation.leasing.lease_sha256(); let authenticated = authenticate_odin_runtime_topology_correlation( - &generation.ready.canonical_bytes, + &odin.ready.canonical_bytes, &authority, current_lease, &odin_authority.signer_public_key, - self.trusted_topology_context(generation.ready.admitted_at_unix_millis), + self.trusted_topology_context(odin.ready.admitted_at_unix_millis), )?; - validate_authenticated_evidence(&generation.ready, &authenticated)?; + validate_authenticated_evidence(odin.ready, &authenticated)?; ensure!( is_semantic_ready(&authenticated), "admitted provider's Ready receipt is not Ready at its admission time" @@ -5660,27 +6273,20 @@ impl Engine { // Odin. Both receipts are historical proofs; current workload health // is owned by continuity and route observation, not receipt age. let authenticated_latest = authenticate_odin_runtime_topology_correlation( - &generation.latest_odin_observation.canonical_bytes, + &odin.latest.canonical_bytes, &authority, current_lease, &odin_authority.signer_public_key, - self.trusted_topology_context( - generation - .latest_odin_observation - .admitted_at_unix_millis, - ), - )?; - validate_authenticated_evidence( - &generation.latest_odin_observation, - &authenticated_latest, + self.trusted_topology_context(odin.latest.admitted_at_unix_millis), )?; + validate_authenticated_evidence(odin.latest, &authenticated_latest)?; ensure!( is_semantic_ready(&authenticated_latest), "admitted provider's latest receipt is not Ready at its admission time" ); Ok(SequenceAdmittedReady { transaction_id: generation.transaction_id.clone(), - evidence: generation.ready.clone(), + evidence: odin.ready.clone(), expected: generation.expected.clone(), authenticated, }) @@ -5781,6 +6387,10 @@ impl Engine { let mut next = current.value.clone(); next.phase = next_phase; next.updated_at_unix_millis = now_millis()?; + next.phase_deadline = next + .plan + .as_ref() + .and_then(|plan| PhaseDeadline::entering(next_phase, plan, next.updated_at_unix_millis)); next.last_error = None; replace_transaction(&self.options.state_store, current, &next) } @@ -6010,6 +6620,7 @@ impl Engine { next.last_error = Some(abort.error.clone()); next.completion = Some(TransactionCompletion::FailedAfterFencing { error: abort.error.clone(), + recovery: TerminalRecovery::RestoreIncumbent, }); replace_transaction(&self.options.state_store, current, &next) } @@ -6598,7 +7209,10 @@ mod tests { if value.completion.is_some() { continue; } - let (Some(ready), Some(latest)) = (&value.ready, &value.latest_odin_observation) else { + let (Some(ready), Some(latest)) = ( + value.ready.as_ref().and_then(ReadinessEvidence::odin), + &value.latest_odin_observation, + ) else { continue; }; println!( @@ -7454,6 +8068,7 @@ mod tests { value .ready .as_ref() + .and_then(ReadinessEvidence::odin) .map(|evidence| evidence.publisher_sequence), value .latest_odin_observation @@ -7465,15 +8080,21 @@ mod tests { for stored in &snapshot.admitted { let value = &stored.value; println!( - "ADMITTED target={} generation={} tx={} admitted_at={} ready_seq={} ready_admitted_at={} latest_seq={} latest_admitted_at={} instance={} {}", + "ADMITTED target={} generation={} tx={} admitted_at={} ready_seq={:?} ready_admitted_at={:?} latest_seq={:?} latest_admitted_at={:?} instance={} {}", value.target, value.generation_id, value.transaction_id, value.admitted_at_unix_millis, - value.ready.publisher_sequence, - value.ready.admitted_at_unix_millis, - value.latest_odin_observation.publisher_sequence, - value.latest_odin_observation.admitted_at_unix_millis, + value.ready.odin().map(|evidence| evidence.publisher_sequence), + value.ready.odin().map(|evidence| evidence.admitted_at_unix_millis), + value + .latest_odin_observation + .as_ref() + .map(|evidence| evidence.publisher_sequence), + value + .latest_odin_observation + .as_ref() + .map(|evidence| evidence.admitted_at_unix_millis), value.activation.runtime_instance_id, value.workload.describe(), ); @@ -7787,6 +8408,7 @@ mod tests { abandoned.transactions[1].value.completion = Some(TransactionCompletion::FailedAfterFencing { error: "candidate died after the fence".into(), + recovery: TerminalRecovery::RestoreIncumbent, }); assert_eq!(abandoned.max_odin_sequence("odin", "odin-signer"), 0); Ok(()) @@ -7880,6 +8502,7 @@ mod tests { transaction.last_error = Some("candidate died after the fence".into()); transaction.completion = Some(TransactionCompletion::FailedAfterFencing { error: "candidate died after the fence".into(), + recovery: TerminalRecovery::RestoreIncumbent, }); // The evidence exists so that a target is released only once the // candidate's route, lease, process and projection are actually gone. @@ -7895,6 +8518,7 @@ mod tests { // The recorded error and the completion must name the same failure. transaction.completion = Some(TransactionCompletion::FailedAfterFencing { error: "a different story".into(), + recovery: TerminalRecovery::RestoreIncumbent, }); assert!(transaction.validate().is_err()); Ok(()) @@ -8294,4 +8918,532 @@ mod tests { }) ); } + + // ---- v2/v3 control records lifted to the v3 generation / v4 transaction ---- + // + // The bytes under tests/fixtures/idunn-control-legacy were written by the + // encoder at Idunn 9001b58 (see the README there). Nothing below encodes a + // legacy record: the legacy layout is decoded from those bytes, and the + // lifted record is compared with what the decoded legacy record says. + + const FIXTURE_GENERATION: &str = + include_str!("../tests/fixtures/idunn-control-legacy/generation-with-receipts.hex"); + const FIXTURE_GENERATION_REPAIRING: &str = include_str!( + "../tests/fixtures/idunn-control-legacy/generation-route-repair-started.hex" + ); + const FIXTURE_TRANSACTIONS: [(&str, &str); 6] = [ + ( + "fencing", + include_str!("../tests/fixtures/idunn-control-legacy/transaction-fencing.hex"), + ), + ( + "leasing", + include_str!("../tests/fixtures/idunn-control-legacy/transaction-leasing.hex"), + ), + ( + "awaiting-ready", + include_str!("../tests/fixtures/idunn-control-legacy/transaction-awaiting-ready.hex"), + ), + ( + "routing", + include_str!("../tests/fixtures/idunn-control-legacy/transaction-routing.hex"), + ), + ( + "committing", + include_str!("../tests/fixtures/idunn-control-legacy/transaction-committing.hex"), + ), + ( + "failed-after-fencing", + include_str!( + "../tests/fixtures/idunn-control-legacy/transaction-failed-after-fencing.hex" + ), + ), + ]; + const FIXTURE_TRANSACTION_V2: &str = + include_str!("../tests/fixtures/idunn-control-legacy/transaction-committing-v2.hex"); + + fn fixture_envelope(text: &str, record_type: &str) -> Result { + let mut lines = text.lines(); + let schema = lines.next().context("fixture has no schema line")?; + let hex = lines.next().context("fixture has no payload line")?.trim(); + let payload = (0..hex.len()) + .step_by(2) + .map(|index| u8::from_str_radix(&hex[index..index + 2], 16)) + .collect::, _>>()?; + let key = if record_type == AdmittedGeneration::TYPE { + rmp_serde::from_slice::(&payload)?.target + } else { + rmp_serde::from_slice::(&payload)?.transaction_id + }; + Ok(CultCacheEnvelope { + key, + r#type: record_type.into(), + payload, + stored_at: rfc3339_millis(1_700_000_100_000)?, + schema_id: Some(schema.into()), + }) + } + + fn fixture_generation(text: &str) -> Result<(LegacyAdmittedGeneration, AdmittedGeneration)> { + let envelope = fixture_envelope(text, AdmittedGeneration::TYPE)?; + let legacy = rmp_serde::from_slice(&envelope.payload)?; + Ok((legacy, read_generation_record(&envelope)?)) + } + + fn fixture_transaction( + text: &str, + ) -> Result<(LegacyDeploymentTransaction, DeploymentTransaction)> { + let envelope = fixture_envelope(text, DeploymentTransaction::TYPE)?; + let legacy = rmp_serde::from_slice(&envelope.payload)?; + Ok((legacy, read_transaction_record(&envelope)?)) + } + + #[test] + fn a_v2_generation_lifts_to_the_decisions_it_encoded() -> Result<()> { + for text in [FIXTURE_GENERATION, FIXTURE_GENERATION_REPAIRING] { + let (legacy, lifted) = fixture_generation(text)?; + // Every receipt the old decisions read is the same receipt, now + // tagged Odin-correlated. + let odin = lifted.odin_receipts()?; + assert_eq!(odin.ready, &legacy.ready); + assert_eq!(odin.latest, &legacy.latest_odin_observation); + assert_eq!(odin.authority, &legacy.odin_authority); + assert_eq!( + lifted.odin_publisher_sequence_cursor, + legacy.odin_publisher_sequence_cursor + ); + assert_eq!(lifted.ready.class(), ReadinessClass::OdinCorrelated); + // The repair intent still decides whatever it decided. + assert_eq!( + lifted.route_repair_started_at_unix_millis, + legacy.route_repair_started_at_unix_millis + ); + assert_eq!(lifted.routing, legacy.routing); + assert_eq!(lifted.leasing, legacy.leasing); + assert_eq!(lifted.plan, legacy.plan); + assert_eq!(lifted.expected, legacy.expected); + assert_eq!(lifted.activation, legacy.activation); + assert_eq!(lifted.workload, legacy.workload); + assert_eq!(lifted.generation_id, legacy.generation_id); + // The new state starts empty and the plan is still a v2 plan. + assert_eq!(lifted.route_supervision, Some(RouteSupervisionState::default())); + assert_eq!(lifted.continuity_backoff, ContinuityBackoff::default()); + assert_eq!(lifted.plan.schema, crate::deployment_plan::COMPILED_DEPLOYMENT_PLAN_SCHEMA_V2); + assert_eq!( + lifted.plan.phase_deadlines(), + crate::deployment_plan::PhaseDeadlines::IDUNN_DEFAULTS + ); + } + let (_, repairing) = fixture_generation(FIXTURE_GENERATION_REPAIRING)?; + assert!(repairing.route_repair_started_at_unix_millis.is_some()); + let (_, plain) = fixture_generation(FIXTURE_GENERATION)?; + assert!(plain.route_repair_started_at_unix_millis.is_none()); + Ok(()) + } + + #[test] + fn v2_and_v3_transactions_lift_to_the_decisions_they_encoded() -> Result<()> { + let mut phases = Vec::new(); + for (name, text) in FIXTURE_TRANSACTIONS { + let (legacy, lifted) = fixture_transaction(text)?; + phases.push((name, lifted.phase)); + assert_eq!(lifted.schema_version, DEPLOYMENT_TRANSACTION_SCHEMA); + assert_eq!(lifted.phase, legacy.phase); + assert_eq!(lifted.completion, legacy.completion); + assert_eq!(lifted.plan, legacy.plan); + assert_eq!(lifted.expected, legacy.expected); + assert_eq!(lifted.leasing, legacy.leasing); + assert_eq!(lifted.fencing, legacy.fencing); + assert_eq!(lifted.routing, legacy.routing); + assert_eq!(lifted.warming, legacy.warming); + assert_eq!(lifted.latest_odin_observation, legacy.latest_odin_observation); + assert_eq!( + lifted.odin_publisher_sequence_cursor, + legacy.odin_publisher_sequence_cursor + ); + assert_eq!( + lifted.ready.as_ref().and_then(ReadinessEvidence::odin), + legacy.ready.as_ref() + ); + assert_eq!(lifted.phase_deadline, None); + assert_eq!(lifted.lease_adoption, None); + assert_eq!(lifted.owns_target_authority(), legacy.phase != DeploymentPhase::Complete); + } + assert_eq!( + phases.iter().map(|(_, phase)| *phase).collect::>(), + [ + DeploymentPhase::Fencing, + DeploymentPhase::Leasing, + DeploymentPhase::AwaitingReady, + DeploymentPhase::Routing, + DeploymentPhase::Committing, + DeploymentPhase::Complete, + ] + ); + + // The genuine v2 layout lifts to the same record as the v3 one. + let (_, from_v2) = fixture_transaction(FIXTURE_TRANSACTION_V2)?; + let (_, from_v3) = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?; + assert_eq!(from_v2, from_v3); + + // A terminal failure predating recovery names the recovery it always did. + let (_, failed) = fixture_transaction(FIXTURE_TRANSACTIONS[5].1)?; + assert!(matches!( + failed.completion, + Some(TransactionCompletion::FailedAfterFencing { + recovery: TerminalRecovery::RestoreIncumbent, + .. + }) + )); + assert!(failed.is_terminal()); + Ok(()) + } + + #[test] + fn the_boot_migration_rewrites_every_legacy_record_once() -> Result<()> { + // The fixtures were cut from one build, so their records share keys + // and each migrates in a store of its own. + let mut fixtures = vec![ + fixture_envelope(FIXTURE_GENERATION, AdmittedGeneration::TYPE)?, + fixture_envelope(FIXTURE_GENERATION_REPAIRING, AdmittedGeneration::TYPE)?, + fixture_envelope(FIXTURE_TRANSACTION_V2, DeploymentTransaction::TYPE)?, + ]; + for (_, text) in FIXTURE_TRANSACTIONS { + fixtures.push(fixture_envelope(text, DeploymentTransaction::TYPE)?); + } + for envelope in fixtures { + let temporary = tempfile::tempdir()?; + let path = temporary.path().join("control.cc"); + let store = SingleFileMessagePackBackingStore::new(&path); + assert!(store.compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: envelope.r#type.clone(), + key: envelope.key.clone(), + current: None, + }], + std::slice::from_ref(&envelope), + )?); + let debug_of = |envelope: &CultCacheEnvelope| -> Result { + Ok(if envelope.r#type == AdmittedGeneration::TYPE { + format!("{:?}", read_generation_record(envelope)?) + } else { + format!("{:?}", read_transaction_record(envelope)?) + }) + }; + let before = debug_of(&envelope)?; + + assert_eq!(migrate_control_store_to_current_schema(&path)?, 1); + assert_eq!(migrate_control_store_to_current_schema(&path)?, 0); + + let after = store.pull_all_read_only_snapshot()?; + assert_eq!(after.len(), 1); + let expected_schema = if envelope.r#type == AdmittedGeneration::TYPE { + ADMITTED_GENERATION_SCHEMA + } else { + DEPLOYMENT_TRANSACTION_SCHEMA + }; + assert_eq!(after[0].schema_id.as_deref(), Some(expected_schema)); + assert_eq!(after[0].stored_at, envelope.stored_at); + // The rewritten record is canonical and means what the legacy one meant. + assert_eq!(debug_of(&after[0])?, before); + } + Ok(()) + } + + #[test] + fn history_still_reads_legacy_transactions() -> Result<()> { + let temporary = tempfile::tempdir()?; + let state_store = temporary.path().join("control.cc"); + let envelope = fixture_envelope(FIXTURE_TRANSACTIONS[5].1, DeploymentTransaction::TYPE)?; + SingleFileMessagePackBackingStore::new(&history_store_path(&state_store)) + .compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: envelope.key.clone(), + current: None, + }], + std::slice::from_ref(&envelope), + )?; + let archived = read_history_transactions(&state_store); + assert_eq!(archived.len(), 1); + assert_eq!(archived[0].transaction_id, envelope.key); + assert_eq!(archived[0].schema_version, DEPLOYMENT_TRANSACTION_SCHEMA); + Ok(()) + } + + #[test] + fn readiness_class_follows_the_expected_odin_dependency() -> Result<()> { + let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; + let mut expected = generation.expected.clone(); + assert!(expected.dependencies.iter().any(|dependency| { + dependency.kind == "shared-infrastructure" + && dependency.capability == ODIN_RENDEZVOUS_CAPABILITY + })); + assert_eq!(ReadinessClass::of(&expected), ReadinessClass::OdinCorrelated); + for dependency in &mut expected.dependencies { + // Same capability, other kind: only the shared-infrastructure + // declaration makes a target Odin-correlated. + dependency.kind = "required".into(); + } + assert_eq!(ReadinessClass::of(&expected), ReadinessClass::RouteProof); + expected.dependencies.clear(); + assert_eq!(ReadinessClass::of(&expected), ReadinessClass::RouteProof); + Ok(()) + } + + fn route_proof_evidence() -> ReadinessEvidence { + let canonical_bytes = vec![7, 7, 7]; + ReadinessEvidence::RouteProof { + evidence: RuntimePresenceEvidence { + canonical_sha256: sha256_id(&canonical_bytes), + canonical_bytes, + message_id: "challenge-1".into(), + challenged_at_unix_millis: 10, + admitted_at_unix_millis: 11, + }, + } + } + + #[test] + fn route_proof_readiness_is_refused_for_a_target_that_declares_odin() -> Result<()> { + let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; + let mut proof = generation.clone(); + proof.ready = route_proof_evidence(); + proof.latest_odin_observation = None; + proof.odin_authority = None; + // The fixture's Expected declares Odin, so route proof is not its class. + assert!(proof.validate().is_err()); + + // Mixed shapes are refused whatever the class. + let mut mixed = generation.clone(); + mixed.latest_odin_observation = None; + assert!(mixed.validate().is_err()); + let mut mixed = generation; + mixed.odin_authority = None; + assert!(mixed.validate().is_err()); + + let mut transaction = fixture_transaction(FIXTURE_TRANSACTIONS[3].1)?.1; + transaction.validate()?; + transaction.ready = Some(route_proof_evidence()); + assert!(transaction.validate().is_err()); + Ok(()) + } + + #[test] + fn route_supervision_state_exists_exactly_for_a_promoted_route() -> Result<()> { + let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; + let mut missing = generation.clone(); + missing.route_supervision = None; + assert!(missing.validate().is_err()); + + let mut unrouted = generation; + unrouted.route_supervision = None; + unrouted.routing = RoutingEvidence::SkippedUnrouted; + // Unrouted while Expected still names a route: refused for that reason + // too, so the test above pins the supervision rule on its own. + assert!(unrouted.validate().is_err()); + Ok(()) + } + + #[test] + fn a_new_generation_inherits_actuation_and_backoff_but_not_observation() -> Result<()> { + let (_, mut incumbent) = fixture_generation(FIXTURE_GENERATION)?; + let mut state = RouteSupervisionState { + last_challenge_at_unix_millis: Some(5), + consecutive_failures: 4, + next_challenge_at_unix_millis: Some(9), + degraded_since_unix_millis: Some(6), + actuations: ActuationWindow { + window_started_at_unix_millis: 3, + count: 2, + }, + }; + incumbent.route_supervision = Some(state.clone()); + incumbent.continuity_backoff = ContinuityBackoff { + window_started_at_unix_millis: Some(1), + attempts: 2, + next_restart_at_unix_millis: Some(8), + }; + let committing = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?.1; + let authority = incumbent.odin_receipts()?.authority.clone(); + let next = AdmittedGeneration::from_transaction( + &committing, + authority.clone(), + Some(&incumbent), + 1_700_000_200_000, + )?; + state = RouteSupervisionState { + actuations: state.actuations, + ..RouteSupervisionState::default() + }; + assert_eq!(next.route_supervision, Some(state)); + assert_eq!(next.continuity_backoff, incumbent.continuity_backoff); + + let first = AdmittedGeneration::from_transaction( + &committing, + authority, + None, + 1_700_000_200_000, + )?; + assert_eq!(first.route_supervision, Some(RouteSupervisionState::default())); + assert_eq!(first.continuity_backoff, ContinuityBackoff::default()); + Ok(()) + } + + #[test] + fn phase_deadlines_follow_the_plan_and_only_cover_post_fencing_phases() -> Result<()> { + let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; + let plan = &generation.plan; + let defaults = plan.phase_deadlines(); + for (phase, seconds) in [ + (DeploymentPhase::Fencing, defaults.fencing_seconds), + (DeploymentPhase::Leasing, defaults.leasing_seconds), + (DeploymentPhase::AwaitingReady, defaults.awaiting_ready_seconds), + (DeploymentPhase::Routing, defaults.routing_seconds), + (DeploymentPhase::Committing, defaults.committing_seconds), + ] { + let deadline = PhaseDeadline::entering(phase, plan, 1_000).context("no deadline")?; + assert_eq!(deadline.phase, phase); + assert_eq!(deadline.entered_at_unix_millis, 1_000); + assert_eq!(deadline.deadline_at_unix_millis, 1_000 + u64::from(seconds) * 1000); + } + for phase in [ + DeploymentPhase::Sealing, + DeploymentPhase::Starting, + DeploymentPhase::Warming, + DeploymentPhase::Complete, + ] { + assert!(PhaseDeadline::entering(phase, plan, 1_000).is_none()); + } + Ok(()) + } + + #[test] + fn transition_stamps_the_deadline_of_the_phase_it_enters() -> Result<()> { + let world = EngineFixture::new()?; + // Routing, already holding the route receipt Committing requires. + let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[3].1)?; + assert_eq!(transaction.phase, DeploymentPhase::Routing); + transaction.routing = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?.1.routing; + let envelope = transaction_envelope(&transaction, transaction.updated_at_unix_millis)?; + let command = command_envelope(&command(CommandKind::Continuity), 100)?; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.key.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: transaction.transaction_id.clone(), + current: None, + }, + ], + &[command, envelope], + )? + ); + let stored = ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .next() + .context("stored transaction vanished")?; + world.engine.transition(&stored, DeploymentPhase::Committing)?; + let advanced = ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .next() + .context("advanced transaction vanished")?; + let deadline = advanced.value.phase_deadline.context("no deadline stamped")?; + assert_eq!(deadline.phase, DeploymentPhase::Committing); + assert_eq!( + deadline.deadline_at_unix_millis - deadline.entered_at_unix_millis, + u64::from(advanced.value.plan.as_ref().unwrap().phase_deadlines().committing_seconds) + * 1000 + ); + assert_eq!(deadline.entered_at_unix_millis, advanced.value.updated_at_unix_millis); + Ok(()) + } + + #[test] + fn a_deadline_must_describe_the_current_post_fencing_phase() -> Result<()> { + let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; + let plan = transaction.plan.clone().unwrap(); + let now = transaction.updated_at_unix_millis; + transaction.phase_deadline = + PhaseDeadline::entering(DeploymentPhase::AwaitingReady, &plan, now); + transaction.validate()?; + let mut wrong_phase = transaction.clone(); + wrong_phase.phase_deadline = + PhaseDeadline::entering(DeploymentPhase::Routing, &plan, now); + assert!(wrong_phase.validate().is_err()); + let mut backwards = transaction.clone(); + backwards.phase_deadline.as_mut().unwrap().deadline_at_unix_millis = now; + assert!(backwards.validate().is_err()); + let mut from_the_future = transaction; + from_the_future.phase_deadline = + PhaseDeadline::entering(DeploymentPhase::AwaitingReady, &plan, now + 1); + assert!(from_the_future.validate().is_err()); + Ok(()) + } + + #[test] + fn lease_adoption_names_only_the_granted_lease() -> Result<()> { + let lease = write_lease(); + let lease_sha256 = lease.canonical_sha256()?; + let adoption = LeaseAdoptionEvidence { + write_lease_sha256: lease_sha256.clone(), + signed_presence_sha256: sha256_id(b"adopting-presence"), + source: AdoptionSource::Direct, + observed_at_unix_millis: 100, + }; + adoption.validate_shape()?; + let granted = LeasingEvidence::Granted { + lease: lease.clone(), + lease_sha256: lease_sha256.clone(), + }; + assert!(adoption.names(&granted)); + // Prepared is not granted, stateless has nothing to adopt, and another + // lease's digest is not this lease. + assert!(!adoption.names(&LeasingEvidence::Prepared { + lease, + lease_sha256, + })); + assert!(!adoption.names(&LeasingEvidence::SkippedStateless)); + let mut other = adoption.clone(); + other.write_lease_sha256 = sha256_id(b"another-lease"); + assert!(!other.names(&granted)); + + // On a transaction, adoption without a granted lease is refused. + let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; + transaction.lease_adoption = Some(adoption); + assert!(transaction.validate().is_err()); + Ok(()) + } + + #[test] + fn an_operator_required_recovery_must_say_why() -> Result<()> { + let (_, mut failed) = fixture_transaction(FIXTURE_TRANSACTIONS[5].1)?; + let Some(TransactionCompletion::FailedAfterFencing { recovery, .. }) = + failed.completion.as_mut() + else { + panic!("fixture is not a post-fencing failure") + }; + *recovery = TerminalRecovery::OperatorRequired { + reason: "adopted-lease".into(), + }; + failed.validate()?; + assert!(failed.is_terminal()); + let Some(TransactionCompletion::FailedAfterFencing { recovery, .. }) = + failed.completion.as_mut() + else { + unreachable!() + }; + *recovery = TerminalRecovery::OperatorRequired { + reason: String::new(), + }; + assert!(failed.validate().is_err()); + Ok(()) + } } diff --git a/tests/fixtures/idunn-control-legacy/README.md b/tests/fixtures/idunn-control-legacy/README.md new file mode 100644 index 0000000..07086d7 --- /dev/null +++ b/tests/fixtures/idunn-control-legacy/README.md @@ -0,0 +1,24 @@ +# Legacy control-record wire fixtures + +Each `*.hex` file is two lines: the CultCache envelope `schema_id`, then the +record's MessagePack payload in hex. They are the exact bytes the encoder at +Idunn `9001b58` (the base of the F0 cut) wrote for records built through that +revision's own constructors: `DeploymentTransaction::new`, +`AdmittedGeneration::from_transaction`, `SealedRelease::new`, +`compile_deployment_plan`, and `rmp_serde::to_vec`. A throwaway test on a +scratch commit over `9001b58` printed them; it is not kept. + +Everything in them is synthetic: identities were enrolled fresh in a temporary +directory, the Odin receipts are placeholder bytes with correct digests, and +the plan is the `deployment_plan` test recipe. No byte comes from a live store. + +| File | Schema | What it is | +|---|---|---| +| `generation-with-receipts` | `idunn.admitted_generation.v2` | routed, stateless generation with Ready and latest Odin receipts | +| `generation-route-repair-started` | `idunn.admitted_generation.v2` | the same with `route_repair_started_at_unix_millis` set | +| `transaction-fencing` .. `transaction-committing` | `idunn.deployment_transaction.v3` | one continuity transaction in each post-fencing phase | +| `transaction-committing-v2` | `idunn.deployment_transaction.v2` | the Committing record in the v2 layout: the v3 tuple without key 34 (base code stops writing v2, so this one is the v3 encoding with its trailing nil slot dropped and the header count lowered) | +| `transaction-failed-after-fencing` | `idunn.deployment_transaction.v3` | terminal post-fencing failure with complete abort evidence | + +`control_plane.rs` decodes these through `LegacyDeploymentTransaction` and +`LegacyAdmittedGeneration`. The fixtures die with those structs. diff --git a/tests/fixtures/idunn-control-legacy/generation-route-repair-started.hex b/tests/fixtures/idunn-control-legacy/generation-route-repair-started.hex new file mode 100644 index 0000000..ac24691 --- /dev/null +++ b/tests/fixtures/idunn-control-legacy/generation-route-repair-started.hex @@ -0,0 +1,2 @@ +idunn.admitted_generation.v2 +dc0013bc6964756e6e2e61646d69747465645f67656e65726174696f6e2e7632a773657276696365d93267656e65726174696f6e2d74782d37636236616537392d323635302d346261322d623664392d643137333338643566393031af636f6e74696e756974792d74657374d92774782d37636236616537392d323635302d346261322d623664392d643137333338643566393031cf0000018bcfe6eea099d9216964756e6e2e636f6d70696c65645f6465706c6f796d656e745f706c616e2e7632d9477368613235362d64396364646465626366636362326366316261396164366239323566336666616237646561636332366661653631646138663236353861336139343039656232b5736572766963652d696e6361726e6174696f6e2d316e9bbf6964756e6e2e736f757263655f73656c656374696f6e5f66616374732e7631d92768747470733a2f2f6769746875622e636f6d2f47616d6543756c742f536572766963652e676974af726566732f68656164732f6d61696ed92832323232323232323232323232323232323232323232323232323232323232323232323232323232d92832323232323232323232323232323232323232323232323232323232323232323232323232323232d92833333333333333333333333333333333333333333333333333333333333333333333333333333333bc6465706c6f796d656e742f6964756e6e2f7265636970652e746f6d6cd9477368613235362d353962636334323766373939343035636134613531326232386138303637343435366661333134393732333631323830656130626332316363356532663635338091a87265662d6865616464dc058e0a736368656d61203d202267616d6563756c742e6964756e6e2e7461726765745f6465636c61726174696f6e2e7631220a746172676574203d202273657276696365220a736f757263655f7374616d705f656e7669726f6e6d656e74203d2022534552564943455f4255494c445f434f4d4d4954220a0a5b5b73746570735d5d0a6964203d20226275696c64220a7068617365203d20226275696c64220a72756e6e6572203d202272757374220a61726776203d205b22636172676f222c20226275696c64222c20222d2d6c6f636b6564225d0a0a5b5b65787465726e616c5f696e707574735d5d0a6964203d2022746f6f6c636861696e2d696e646578220a75726c203d202268747470733a2f2f6578616d706c652e696e76616c69642f746f6f6c636861696e2d696e646578220a736861323536203d202264646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464220a72756e6e6572203d202272757374220a64657374696e6174696f6e203d2022696e707574732f746f6f6c636861696e2d696e646578220a0a5b5b6172746966616374735d5d0a6964203d20226461656d6f6e220a736f757263655f6b696e64203d202272756e6e65722d6f7574707574220a72756e6e6572203d202272757374220a736f75726365203d20227461726765742f72656c656173652f73657276696365220a64657374696e6174696f6e203d202273657276696365220a65787065637465645f736861323536203d202263636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363220a65786563757461626c65203d20747275650a0a5b736572766963655d0a65786563757461626c655f6172746966616374203d20226461656d6f6e220a617267756d656e7473203d205b0a20207b206b696e64203d202262696e64696e67222c206e616d65203d202273746174655f726f6f7422207d2c0a5d0a7472616e73706f7274203d202268747470220a726f7574655f7265717569726564203d20747275650a72657175697265645f656e7669726f6e6d656e74203d205b2247414d4543554c545f4944554e4e5f43414e4449444154455f42494e44222c202247414d4543554c545f4944554e4e5f52554e54494d455f42554e444c45225d0a0a5b736572766963652e6865616c74685d0a636f6e7472616374203d2022736572766963652e6865616c7468220a0a5b73746174655d0a736368656d615f67656e65726174696f6e203d20227631220a0a5b5b73746174652e736c6f74735d5d0a6964203d202272756e74696d652d6361636865220a72656c61746976655f70617468203d202272756e74696d652d63616368652e6363220a6b696e64203d202263756c7463616368652d66696c65220a736368656d61203d2022736572766963652e72756e74696d652d63616368652e7631220a777269746572203d20226e6f6e65220a7265636f76657279203d202272656275696c6461626c65220a73746172747570203d20226f70656e2d61742d7374617274220a0a5b5b70726f76696465735d5d0a6361706162696c697479203d2022736572766963652e72756e74696d65220a736368656d61203d2022736572766963652e72756e74696d652e7631220a636f6d7061746962696c697479203d20227631220a0a5b5b646570656e64656e636965735d5d0a6b696e64203d20227368617265642d696e667261737472756374757265220a6361706162696c697479203d20226f64696e2e76657273652d72656e64657a766f7573220a736368656d61203d20226f64696e2e76657273652d746f706f6c6f67792e7631220a636f6d7061746962696c697479203d20227631220adc076e0a736368656d61203d202267616d6563756c742e6964756e6e2e6f70657261746f725f62696e64696e672e7632220a746172676574203d202273657276696365220a0a5b7265706f7369746f72795d0a6f726967696e203d202268747470733a2f2f6769746875622e636f6d2f47616d6543756c742f536572766963652e676974220a61646d69747465645f726566203d2022726566732f68656164732f6d61696e220a6d696e696d756d5f7265766973696f6e203d202231313131313131313131313131313131313131313131313131313131313131313131313131313131220a73656c656374696f6e203d20227265662d68656164220a636865636b6f7574203d20222f7372762f6275696c642f53657276696365220a7265636970655f70617468203d20226465706c6f796d656e742f6964756e6e2f7265636970652e746f6d6c220a0a5b72756e6e6572732e727573745d0a647269766572203d2022646f636b6572220a696d616765203d202272757374407368613235363a61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161220a75736572203d2022313030303a31303030220a6166666f7264616e636573203d205b22736f757263652d72656164222c202261727469666163742d7772697465225d0a616c6c6f7765645f70726f6772616d73203d205b22636172676f225d0a6e6574776f726b5f70726f66696c65203d20226275696c642d646570656e64656e63792d656772657373220a6d656d6f72795f6d6562696279746573203d20323034380a6370755f71756f74615f70657263656e74203d203230300a706964735f6c696d6974203d203531320a746d7066735f6d6562696279746573203d203531320a0a5b776f726b6c6f61645d0a647269766572203d202273797374656d642d7472616e7369656e74220a73746174655f67726f7570203d202273657276696365220a756e69745f707265666978203d20226964756e6e2d73657276696365220a72656c656173655f726f6f74203d20222f7372762f736572766963652f72656c6561736573220a73746174655f726f6f74203d20222f7661722f6c69622f67616d6563756c742f73657276696365220a72756e74696d655f726f6f74203d20222f72756e2f67616d6563756c742f6964756e6e2f73657276696365220a6e6574776f726b203d2022686f73742d70726976617465220a68617264656e696e67203d2022737472696374220a6d656d6f72795f6d6562696279746573203d20313032340a6370755f71756f74615f70657263656e74203d203130300a0a5b776f726b6c6f61642e617267756d656e745f62696e64696e67735d0a73746174655f726f6f74203d20222f7661722f6c69622f67616d6563756c742f73657276696365220a0a5b776f726b6c6f61642e7365637265745f66696c65735d0a47414d4543554c545f52554e54494d455f50524553454e43455f4944454e54495459203d20222f6574632f67616d6563756c742f736572766963652f72756e74696d652d70726573656e63652d6964656e746974792e6363220a0a5b72756e74696d655f6964656e746974795d0a72756e74696d655f6964203d2022736572766963652d79676764726173696c220a65787065637465645f7369676e65725f6964656e746974795f6964203d2022736572766963652d72756e74696d652d7369676e6572220a74727573745f616e63686f725f73746f7265203d20222f6574632f67616d6563756c742f74727573742f736572766963652e6363220a0a5b726f7574655d0a647269766572203d20226e67696e782d73747265616d2d746370220a726f7574655f6964203d202273657276696365220a737461626c655f656e64706f696e74203d2022687474703a2f2f3132372e302e302e313a3137393939220a707269766174655f686f7374203d20223132372e302e302e31220a707269766174655f706f72745f7374617274203d2031383030300a707269766174655f706f72745f656e64203d2031383030390a636f6e6669675f70617468203d20222f6574632f6e67696e782f6964756e6e2d73747265616d2d726f757465732f736572766963652e636f6e66220a72656c6f61645f756e6974203d20226e67696e782e73657276696365220a0a5b6272616b65735d0a6465706c6f796d656e745f73746f7265203d20222f7661722f6c69622f67616d6563756c742f6964756e6e2f736572766963652d6465706c6f796d656e742d6272616b652e6363220a6c6966656379636c655f73746f7265203d20222f7661722f6c69622f67616d6563756c742f6964756e6e2f736572766963652d6c6966656379636c652d6272616b652e6363220a0a5b726f6c6c6f75745d0a7374726174656779203d202263616e6469646174652d7468656e2d70726f6d6f7465220a647261696e5f7365636f6e6473203d2033300a72657461696e5f72656c6561736573203d20320a0a5b706c6163656d656e745d0a646573697265645f7265706c69636173203d20310a6e6f646573203d205b2279676764726173696c225d0a919296b57368617265642d696e667261737472756374757265b56f64696e2e76657273652d72656e64657a766f7573b66f64696e2e76657273652d746f706f6c6f67792e7631a2763101b06265666f72652d70726f6d6f74696f6e97a46f64696e98ad6d616e616765642d7265616479a46f64696eb26f64696e2d696e6361726e6174696f6e2d31d9477368613235362d61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161d9477368613235362d65656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565d9477368613235362d63343263333331373863366539666534633435313162623563663962626461376431633138356536646330373064323234653463653431303538626338633238d9477368613235362d6137313730323730313265383236396235666636363961313135386466636236313764346463636230663963333633656631373738656561633230353332633907b56f64696e2e76657273652d72656e64657a766f7573b66f64696e2e76657273652d746f706f6c6f67792e7631a2763101b57463703a2f2f31302e37372e302e313a3137383731cd465196b76964756e6e2e7365616c65645f72656c656173652e7631d9477368613235362d35663738353165373634333366383261376562666431626232646234323065366263333263646263346536666538373563356633326461363434643535656237d9477368613235362d643963646464656263666363623263663162613961643662393235663366666162376465616363323666616536316461386632363538613361393430396562329195a66461656d6f6ea773657276696365d9477368613235362d636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363632ac39196af746f6f6c636861696e2d696e646578d92768747470733a2f2f6578616d706c652e696e76616c69642f746f6f6c636861696e2d696e646578d9477368613235362d64646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464a472757374b6696e707574732f746f6f6c636861696e2d696e646578117892d9477368613235362d35663738353165373634333366383261376562666431626232646234323065366263333263646263346536666538373563356633326461363434643535656237bd2f7372762f736572766963652f72656c65617365732f63757272656e74dc0012bd6964756e6e2e65787065637465645f696e6361726e6174696f6e2e7632a773657276696365d9477368613235362d64396364646465626366636362326366316261396164366239323566336666616237646561636332366661653631646138663236353861336139343039656232b5736572766963652d696e6361726e6174696f6e2d31d9477368613235362d35663738353165373634333366383261376562666431626232646234323065366263333263646263346536666538373563356633326461363434643535656237bb6769746875622e636f6d2f47616d6543756c742f53657276696365d92832323232323232323232323232323232323232323232323232323232323232323232323232323232d9477368613235362d35396263633432376637393934303563613461353132623238613830363734343536666133313439373233363132383065613062633231636335653266363533b1736572766963652d79676764726173696cb6736572766963652d72756e74696d652d7369676e6572ae736572766963652e6865616c7468d9477368613235362d63636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363a27631d9477368613235362d32663837303134316366353638316566656332653234316364636435663731303831616539326162316535633534633839636438393666386335656236323731c294a773657276696365a468747470b6687474703a2f2f3132372e302e302e313a3137393939b6687474703a2f2f3132372e302e302e313a31383030319194af736572766963652e72756e74696d65b2736572766963652e72756e74696d652e7631a2763101919ab57368617265642d696e667261737472756374757265b56f64696e2e76657273652d72656e64657a766f7573b66f64696e2e76657273652d746f706f6c6f67792e7631a2763101b06265666f72652d70726f6d6f74696f6ea46f64696eb36d616e616765642d696e6361726e6174696f6ed9477368613235362d63343263333331373863366539666534633435313162623563663962626461376431633138356536646330373064323234653463653431303538626338633238b57463703a2f2f31302e37372e302e313a31373837319abb6964756e6e2e72756e74696d655f61637469766174696f6e2e7632d9477368613235362d39613662663530353466616430306232343363323331386330333436303931333230343665643436356231356534636561666532613736623966663231313039b1736572766963652d79676764726173696cd9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262d94032363535323862353235386435353563356434646533383931303830363164383965663439616336306631373337626464656337316535333335643163623935c420c06afef8f66e6ec88c96c81ad6d74dbd9933738032f4d8069a46412380e2272dcf0000018bcfe567ecd94066346536393735326334333432363965323262306130346137623430613065336235386364343561383238376364303936383263313864326565366539376239a765643235353139c44046048cf77cc9f524da966e16a131edb34180f4ea8420f5f24a64fd53178e44988bef994d6df103a5f117470283d533e9d7e8245c4f9965f9a4c0a0114ed1c201dc0033b26964756e6e2d343234322e73657276696365af4964756e6e20746573742034323432af696e766f636174696f6e2d3432343201a465786563a6616c77617973ad636f6e74726f6c2d67726f7570c3a57534323432a57534323432a0a0a0c3c3a9696e76697369626c65a3706964c3a430303737a0a0cd10920194cd1092cd1092cd1092cd109294cd1092cd1092cd1092cd109291cd10920000000000c391010807b52f6f70742f746573742f62696e2f736572766963650101d9477368613235362d34626635313232663334343535346335336264653265626238636432623765336431363030616436333163333835613564376363653233633737383534353961d9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262a92f6f70742f74657374a92f72756e2f74657374d9477368613235362d6462633162346339303066666534386435373562356461356336333830343031323566363564623066653365323434393462373665613938363435376439383690d9477368613235362d30383466656430386239373861663464376431393661373434366138366235383030396536333662363131646231363231316236356139616164666632396335d9202f73797374656d2e736c6963652f6964756e6e2d343234322e73657276696365c090aa61637469766174696f6edc002001010101010101010101010101010101010101010101010101010101010101019091b1736b69707065642d73746174656c657373959140d9477368613235362d63333634316638353434643763303266333538306230376330663938383766306336613237666635616231643461336532396361663139376366633239396165ab6f64696e2d7369676e65722864959141d9477368613235362d35353961656164303832363464353739356433393039373138636464303561626434393537326538346665353535393065656633316138386130386664666664ab6f64696e2d7369676e6572296493a870726f6d6f74656495a773657276696365d9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262d9477368613235362d62663563663539653335363635323235333236386336303463626638646638636664623033613461306433326232376164313538653538313730396338306534d9477368613235362d34313266303131626462396566646138616137323036636364336333613438633637613131386164646161363266616436303463326236346664396163643063cf0000018bcfe6ee96cf0000018bcfe6ee9692d94038663736666430326265623133353233373733373632306361666530383234636138663937326438623332643134373762346336333430656139666634643762dc002048ccd7cc85cc8c14ccb8384dccbccc8fcc83ccbacca50f25ccb072ccc126cc9fccee0cccd660316f5bcccf5e6f172529cf0000018bcfe79e68 diff --git a/tests/fixtures/idunn-control-legacy/generation-with-receipts.hex b/tests/fixtures/idunn-control-legacy/generation-with-receipts.hex new file mode 100644 index 0000000..86b80ea --- /dev/null +++ b/tests/fixtures/idunn-control-legacy/generation-with-receipts.hex @@ -0,0 +1,2 @@ +idunn.admitted_generation.v2 +dc0013bc6964756e6e2e61646d69747465645f67656e65726174696f6e2e7632a773657276696365d93267656e65726174696f6e2d74782d37636236616537392d323635302d346261322d623664392d643137333338643566393031af636f6e74696e756974792d74657374d92774782d37636236616537392d323635302d346261322d623664392d643137333338643566393031cf0000018bcfe6eea099d9216964756e6e2e636f6d70696c65645f6465706c6f796d656e745f706c616e2e7632d9477368613235362d64396364646465626366636362326366316261396164366239323566336666616237646561636332366661653631646138663236353861336139343039656232b5736572766963652d696e6361726e6174696f6e2d316e9bbf6964756e6e2e736f757263655f73656c656374696f6e5f66616374732e7631d92768747470733a2f2f6769746875622e636f6d2f47616d6543756c742f536572766963652e676974af726566732f68656164732f6d61696ed92832323232323232323232323232323232323232323232323232323232323232323232323232323232d92832323232323232323232323232323232323232323232323232323232323232323232323232323232d92833333333333333333333333333333333333333333333333333333333333333333333333333333333bc6465706c6f796d656e742f6964756e6e2f7265636970652e746f6d6cd9477368613235362d353962636334323766373939343035636134613531326232386138303637343435366661333134393732333631323830656130626332316363356532663635338091a87265662d6865616464dc058e0a736368656d61203d202267616d6563756c742e6964756e6e2e7461726765745f6465636c61726174696f6e2e7631220a746172676574203d202273657276696365220a736f757263655f7374616d705f656e7669726f6e6d656e74203d2022534552564943455f4255494c445f434f4d4d4954220a0a5b5b73746570735d5d0a6964203d20226275696c64220a7068617365203d20226275696c64220a72756e6e6572203d202272757374220a61726776203d205b22636172676f222c20226275696c64222c20222d2d6c6f636b6564225d0a0a5b5b65787465726e616c5f696e707574735d5d0a6964203d2022746f6f6c636861696e2d696e646578220a75726c203d202268747470733a2f2f6578616d706c652e696e76616c69642f746f6f6c636861696e2d696e646578220a736861323536203d202264646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464220a72756e6e6572203d202272757374220a64657374696e6174696f6e203d2022696e707574732f746f6f6c636861696e2d696e646578220a0a5b5b6172746966616374735d5d0a6964203d20226461656d6f6e220a736f757263655f6b696e64203d202272756e6e65722d6f7574707574220a72756e6e6572203d202272757374220a736f75726365203d20227461726765742f72656c656173652f73657276696365220a64657374696e6174696f6e203d202273657276696365220a65787065637465645f736861323536203d202263636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363220a65786563757461626c65203d20747275650a0a5b736572766963655d0a65786563757461626c655f6172746966616374203d20226461656d6f6e220a617267756d656e7473203d205b0a20207b206b696e64203d202262696e64696e67222c206e616d65203d202273746174655f726f6f7422207d2c0a5d0a7472616e73706f7274203d202268747470220a726f7574655f7265717569726564203d20747275650a72657175697265645f656e7669726f6e6d656e74203d205b2247414d4543554c545f4944554e4e5f43414e4449444154455f42494e44222c202247414d4543554c545f4944554e4e5f52554e54494d455f42554e444c45225d0a0a5b736572766963652e6865616c74685d0a636f6e7472616374203d2022736572766963652e6865616c7468220a0a5b73746174655d0a736368656d615f67656e65726174696f6e203d20227631220a0a5b5b73746174652e736c6f74735d5d0a6964203d202272756e74696d652d6361636865220a72656c61746976655f70617468203d202272756e74696d652d63616368652e6363220a6b696e64203d202263756c7463616368652d66696c65220a736368656d61203d2022736572766963652e72756e74696d652d63616368652e7631220a777269746572203d20226e6f6e65220a7265636f76657279203d202272656275696c6461626c65220a73746172747570203d20226f70656e2d61742d7374617274220a0a5b5b70726f76696465735d5d0a6361706162696c697479203d2022736572766963652e72756e74696d65220a736368656d61203d2022736572766963652e72756e74696d652e7631220a636f6d7061746962696c697479203d20227631220a0a5b5b646570656e64656e636965735d5d0a6b696e64203d20227368617265642d696e667261737472756374757265220a6361706162696c697479203d20226f64696e2e76657273652d72656e64657a766f7573220a736368656d61203d20226f64696e2e76657273652d746f706f6c6f67792e7631220a636f6d7061746962696c697479203d20227631220adc076e0a736368656d61203d202267616d6563756c742e6964756e6e2e6f70657261746f725f62696e64696e672e7632220a746172676574203d202273657276696365220a0a5b7265706f7369746f72795d0a6f726967696e203d202268747470733a2f2f6769746875622e636f6d2f47616d6543756c742f536572766963652e676974220a61646d69747465645f726566203d2022726566732f68656164732f6d61696e220a6d696e696d756d5f7265766973696f6e203d202231313131313131313131313131313131313131313131313131313131313131313131313131313131220a73656c656374696f6e203d20227265662d68656164220a636865636b6f7574203d20222f7372762f6275696c642f53657276696365220a7265636970655f70617468203d20226465706c6f796d656e742f6964756e6e2f7265636970652e746f6d6c220a0a5b72756e6e6572732e727573745d0a647269766572203d2022646f636b6572220a696d616765203d202272757374407368613235363a61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161220a75736572203d2022313030303a31303030220a6166666f7264616e636573203d205b22736f757263652d72656164222c202261727469666163742d7772697465225d0a616c6c6f7765645f70726f6772616d73203d205b22636172676f225d0a6e6574776f726b5f70726f66696c65203d20226275696c642d646570656e64656e63792d656772657373220a6d656d6f72795f6d6562696279746573203d20323034380a6370755f71756f74615f70657263656e74203d203230300a706964735f6c696d6974203d203531320a746d7066735f6d6562696279746573203d203531320a0a5b776f726b6c6f61645d0a647269766572203d202273797374656d642d7472616e7369656e74220a73746174655f67726f7570203d202273657276696365220a756e69745f707265666978203d20226964756e6e2d73657276696365220a72656c656173655f726f6f74203d20222f7372762f736572766963652f72656c6561736573220a73746174655f726f6f74203d20222f7661722f6c69622f67616d6563756c742f73657276696365220a72756e74696d655f726f6f74203d20222f72756e2f67616d6563756c742f6964756e6e2f73657276696365220a6e6574776f726b203d2022686f73742d70726976617465220a68617264656e696e67203d2022737472696374220a6d656d6f72795f6d6562696279746573203d20313032340a6370755f71756f74615f70657263656e74203d203130300a0a5b776f726b6c6f61642e617267756d656e745f62696e64696e67735d0a73746174655f726f6f74203d20222f7661722f6c69622f67616d6563756c742f73657276696365220a0a5b776f726b6c6f61642e7365637265745f66696c65735d0a47414d4543554c545f52554e54494d455f50524553454e43455f4944454e54495459203d20222f6574632f67616d6563756c742f736572766963652f72756e74696d652d70726573656e63652d6964656e746974792e6363220a0a5b72756e74696d655f6964656e746974795d0a72756e74696d655f6964203d2022736572766963652d79676764726173696c220a65787065637465645f7369676e65725f6964656e746974795f6964203d2022736572766963652d72756e74696d652d7369676e6572220a74727573745f616e63686f725f73746f7265203d20222f6574632f67616d6563756c742f74727573742f736572766963652e6363220a0a5b726f7574655d0a647269766572203d20226e67696e782d73747265616d2d746370220a726f7574655f6964203d202273657276696365220a737461626c655f656e64706f696e74203d2022687474703a2f2f3132372e302e302e313a3137393939220a707269766174655f686f7374203d20223132372e302e302e31220a707269766174655f706f72745f7374617274203d2031383030300a707269766174655f706f72745f656e64203d2031383030390a636f6e6669675f70617468203d20222f6574632f6e67696e782f6964756e6e2d73747265616d2d726f757465732f736572766963652e636f6e66220a72656c6f61645f756e6974203d20226e67696e782e73657276696365220a0a5b6272616b65735d0a6465706c6f796d656e745f73746f7265203d20222f7661722f6c69622f67616d6563756c742f6964756e6e2f736572766963652d6465706c6f796d656e742d6272616b652e6363220a6c6966656379636c655f73746f7265203d20222f7661722f6c69622f67616d6563756c742f6964756e6e2f736572766963652d6c6966656379636c652d6272616b652e6363220a0a5b726f6c6c6f75745d0a7374726174656779203d202263616e6469646174652d7468656e2d70726f6d6f7465220a647261696e5f7365636f6e6473203d2033300a72657461696e5f72656c6561736573203d20320a0a5b706c6163656d656e745d0a646573697265645f7265706c69636173203d20310a6e6f646573203d205b2279676764726173696c225d0a919296b57368617265642d696e667261737472756374757265b56f64696e2e76657273652d72656e64657a766f7573b66f64696e2e76657273652d746f706f6c6f67792e7631a2763101b06265666f72652d70726f6d6f74696f6e97a46f64696e98ad6d616e616765642d7265616479a46f64696eb26f64696e2d696e6361726e6174696f6e2d31d9477368613235362d61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161d9477368613235362d65656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565d9477368613235362d63343263333331373863366539666534633435313162623563663962626461376431633138356536646330373064323234653463653431303538626338633238d9477368613235362d6137313730323730313265383236396235666636363961313135386466636236313764346463636230663963333633656631373738656561633230353332633907b56f64696e2e76657273652d72656e64657a766f7573b66f64696e2e76657273652d746f706f6c6f67792e7631a2763101b57463703a2f2f31302e37372e302e313a3137383731cd465196b76964756e6e2e7365616c65645f72656c656173652e7631d9477368613235362d35663738353165373634333366383261376562666431626232646234323065366263333263646263346536666538373563356633326461363434643535656237d9477368613235362d643963646464656263666363623263663162613961643662393235663366666162376465616363323666616536316461386632363538613361393430396562329195a66461656d6f6ea773657276696365d9477368613235362d636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363632ac39196af746f6f6c636861696e2d696e646578d92768747470733a2f2f6578616d706c652e696e76616c69642f746f6f6c636861696e2d696e646578d9477368613235362d64646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464a472757374b6696e707574732f746f6f6c636861696e2d696e646578117892d9477368613235362d35663738353165373634333366383261376562666431626232646234323065366263333263646263346536666538373563356633326461363434643535656237bd2f7372762f736572766963652f72656c65617365732f63757272656e74dc0012bd6964756e6e2e65787065637465645f696e6361726e6174696f6e2e7632a773657276696365d9477368613235362d64396364646465626366636362326366316261396164366239323566336666616237646561636332366661653631646138663236353861336139343039656232b5736572766963652d696e6361726e6174696f6e2d31d9477368613235362d35663738353165373634333366383261376562666431626232646234323065366263333263646263346536666538373563356633326461363434643535656237bb6769746875622e636f6d2f47616d6543756c742f53657276696365d92832323232323232323232323232323232323232323232323232323232323232323232323232323232d9477368613235362d35396263633432376637393934303563613461353132623238613830363734343536666133313439373233363132383065613062633231636335653266363533b1736572766963652d79676764726173696cb6736572766963652d72756e74696d652d7369676e6572ae736572766963652e6865616c7468d9477368613235362d63636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363a27631d9477368613235362d32663837303134316366353638316566656332653234316364636435663731303831616539326162316535633534633839636438393666386335656236323731c294a773657276696365a468747470b6687474703a2f2f3132372e302e302e313a3137393939b6687474703a2f2f3132372e302e302e313a31383030319194af736572766963652e72756e74696d65b2736572766963652e72756e74696d652e7631a2763101919ab57368617265642d696e667261737472756374757265b56f64696e2e76657273652d72656e64657a766f7573b66f64696e2e76657273652d746f706f6c6f67792e7631a2763101b06265666f72652d70726f6d6f74696f6ea46f64696eb36d616e616765642d696e6361726e6174696f6ed9477368613235362d63343263333331373863366539666534633435313162623563663962626461376431633138356536646330373064323234653463653431303538626338633238b57463703a2f2f31302e37372e302e313a31373837319abb6964756e6e2e72756e74696d655f61637469766174696f6e2e7632d9477368613235362d39613662663530353466616430306232343363323331386330333436303931333230343665643436356231356534636561666532613736623966663231313039b1736572766963652d79676764726173696cd9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262d94032363535323862353235386435353563356434646533383931303830363164383965663439616336306631373337626464656337316535333335643163623935c420c06afef8f66e6ec88c96c81ad6d74dbd9933738032f4d8069a46412380e2272dcf0000018bcfe567ecd94066346536393735326334333432363965323262306130346137623430613065336235386364343561383238376364303936383263313864326565366539376239a765643235353139c44046048cf77cc9f524da966e16a131edb34180f4ea8420f5f24a64fd53178e44988bef994d6df103a5f117470283d533e9d7e8245c4f9965f9a4c0a0114ed1c201dc0033b26964756e6e2d343234322e73657276696365af4964756e6e20746573742034323432af696e766f636174696f6e2d3432343201a465786563a6616c77617973ad636f6e74726f6c2d67726f7570c3a57534323432a57534323432a0a0a0c3c3a9696e76697369626c65a3706964c3a430303737a0a0cd10920194cd1092cd1092cd1092cd109294cd1092cd1092cd1092cd109291cd10920000000000c391010807b52f6f70742f746573742f62696e2f736572766963650101d9477368613235362d34626635313232663334343535346335336264653265626238636432623765336431363030616436333163333835613564376363653233633737383534353961d9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262a92f6f70742f74657374a92f72756e2f74657374d9477368613235362d6462633162346339303066666534386435373562356461356336333830343031323566363564623066653365323434393462373665613938363435376439383690d9477368613235362d30383466656430386239373861663464376431393661373434366138366235383030396536333662363131646231363231316236356139616164666632396335d9202f73797374656d2e736c6963652f6964756e6e2d343234322e73657276696365c090aa61637469766174696f6edc002001010101010101010101010101010101010101010101010101010101010101019091b1736b69707065642d73746174656c657373959140d9477368613235362d63333634316638353434643763303266333538306230376330663938383766306336613237666635616231643461336532396361663139376366633239396165ab6f64696e2d7369676e65722864959141d9477368613235362d35353961656164303832363464353739356433393039373138636464303561626434393537326538346665353535393065656633316138386130386664666664ab6f64696e2d7369676e6572296493a870726f6d6f74656495a773657276696365d9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262d9477368613235362d62663563663539653335363635323235333236386336303463626638646638636664623033613461306433326232376164313538653538313730396338306534d9477368613235362d34313266303131626462396566646138616137323036636364336333613438633637613131386164646161363266616436303463326236346664396163643063cf0000018bcfe6ee96cf0000018bcfe6ee9692d94038663736666430326265623133353233373733373632306361666530383234636138663937326438623332643134373762346336333430656139666634643762dc002048ccd7cc85cc8c14ccb8384dccbccc8fcc83ccbacca50f25ccb072ccc126cc9fccee0cccd660316f5bcccf5e6f172529c0 diff --git a/tests/fixtures/idunn-control-legacy/transaction-awaiting-ready.hex b/tests/fixtures/idunn-control-legacy/transaction-awaiting-ready.hex new file mode 100644 index 0000000..760f6f4 --- /dev/null +++ b/tests/fixtures/idunn-control-legacy/transaction-awaiting-ready.hex @@ -0,0 +1,2 @@ +idunn.deployment_transaction.v3 +dc0023bf6964756e6e2e6465706c6f796d656e745f7472616e73616374696f6e2e7633d92774782d37636236616537392d323635302d346261322d623664392d643137333338643566393031af636f6e74696e756974792d74657374aa636f6e74696e75697479a77365727669636500ae6177616974696e672d7265616479cf0000018bcfe56800cf0000018bcfe6eea0c099d9216964756e6e2e636f6d70696c65645f6465706c6f796d656e745f706c616e2e7632d9477368613235362d64396364646465626366636362326366316261396164366239323566336666616237646561636332366661653631646138663236353861336139343039656232b5736572766963652d696e6361726e6174696f6e2d316e9bbf6964756e6e2e736f757263655f73656c656374696f6e5f66616374732e7631d92768747470733a2f2f6769746875622e636f6d2f47616d6543756c742f536572766963652e676974af726566732f68656164732f6d61696ed92832323232323232323232323232323232323232323232323232323232323232323232323232323232d92832323232323232323232323232323232323232323232323232323232323232323232323232323232d92833333333333333333333333333333333333333333333333333333333333333333333333333333333bc6465706c6f796d656e742f6964756e6e2f7265636970652e746f6d6cd9477368613235362d353962636334323766373939343035636134613531326232386138303637343435366661333134393732333631323830656130626332316363356532663635338091a87265662d6865616464dc058e0a736368656d61203d202267616d6563756c742e6964756e6e2e7461726765745f6465636c61726174696f6e2e7631220a746172676574203d202273657276696365220a736f757263655f7374616d705f656e7669726f6e6d656e74203d2022534552564943455f4255494c445f434f4d4d4954220a0a5b5b73746570735d5d0a6964203d20226275696c64220a7068617365203d20226275696c64220a72756e6e6572203d202272757374220a61726776203d205b22636172676f222c20226275696c64222c20222d2d6c6f636b6564225d0a0a5b5b65787465726e616c5f696e707574735d5d0a6964203d2022746f6f6c636861696e2d696e646578220a75726c203d202268747470733a2f2f6578616d706c652e696e76616c69642f746f6f6c636861696e2d696e646578220a736861323536203d202264646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464220a72756e6e6572203d202272757374220a64657374696e6174696f6e203d2022696e707574732f746f6f6c636861696e2d696e646578220a0a5b5b6172746966616374735d5d0a6964203d20226461656d6f6e220a736f757263655f6b696e64203d202272756e6e65722d6f7574707574220a72756e6e6572203d202272757374220a736f75726365203d20227461726765742f72656c656173652f73657276696365220a64657374696e6174696f6e203d202273657276696365220a65787065637465645f736861323536203d202263636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363220a65786563757461626c65203d20747275650a0a5b736572766963655d0a65786563757461626c655f6172746966616374203d20226461656d6f6e220a617267756d656e7473203d205b0a20207b206b696e64203d202262696e64696e67222c206e616d65203d202273746174655f726f6f7422207d2c0a5d0a7472616e73706f7274203d202268747470220a726f7574655f7265717569726564203d20747275650a72657175697265645f656e7669726f6e6d656e74203d205b2247414d4543554c545f4944554e4e5f43414e4449444154455f42494e44222c202247414d4543554c545f4944554e4e5f52554e54494d455f42554e444c45225d0a0a5b736572766963652e6865616c74685d0a636f6e7472616374203d2022736572766963652e6865616c7468220a0a5b73746174655d0a736368656d615f67656e65726174696f6e203d20227631220a0a5b5b73746174652e736c6f74735d5d0a6964203d202272756e74696d652d6361636865220a72656c61746976655f70617468203d202272756e74696d652d63616368652e6363220a6b696e64203d202263756c7463616368652d66696c65220a736368656d61203d2022736572766963652e72756e74696d652d63616368652e7631220a777269746572203d20226e6f6e65220a7265636f76657279203d202272656275696c6461626c65220a73746172747570203d20226f70656e2d61742d7374617274220a0a5b5b70726f76696465735d5d0a6361706162696c697479203d2022736572766963652e72756e74696d65220a736368656d61203d2022736572766963652e72756e74696d652e7631220a636f6d7061746962696c697479203d20227631220a0a5b5b646570656e64656e636965735d5d0a6b696e64203d20227368617265642d696e667261737472756374757265220a6361706162696c697479203d20226f64696e2e76657273652d72656e64657a766f7573220a736368656d61203d20226f64696e2e76657273652d746f706f6c6f67792e7631220a636f6d7061746962696c697479203d20227631220adc076e0a736368656d61203d202267616d6563756c742e6964756e6e2e6f70657261746f725f62696e64696e672e7632220a746172676574203d202273657276696365220a0a5b7265706f7369746f72795d0a6f726967696e203d202268747470733a2f2f6769746875622e636f6d2f47616d6543756c742f536572766963652e676974220a61646d69747465645f726566203d2022726566732f68656164732f6d61696e220a6d696e696d756d5f7265766973696f6e203d202231313131313131313131313131313131313131313131313131313131313131313131313131313131220a73656c656374696f6e203d20227265662d68656164220a636865636b6f7574203d20222f7372762f6275696c642f53657276696365220a7265636970655f70617468203d20226465706c6f796d656e742f6964756e6e2f7265636970652e746f6d6c220a0a5b72756e6e6572732e727573745d0a647269766572203d2022646f636b6572220a696d616765203d202272757374407368613235363a61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161220a75736572203d2022313030303a31303030220a6166666f7264616e636573203d205b22736f757263652d72656164222c202261727469666163742d7772697465225d0a616c6c6f7765645f70726f6772616d73203d205b22636172676f225d0a6e6574776f726b5f70726f66696c65203d20226275696c642d646570656e64656e63792d656772657373220a6d656d6f72795f6d6562696279746573203d20323034380a6370755f71756f74615f70657263656e74203d203230300a706964735f6c696d6974203d203531320a746d7066735f6d6562696279746573203d203531320a0a5b776f726b6c6f61645d0a647269766572203d202273797374656d642d7472616e7369656e74220a73746174655f67726f7570203d202273657276696365220a756e69745f707265666978203d20226964756e6e2d73657276696365220a72656c656173655f726f6f74203d20222f7372762f736572766963652f72656c6561736573220a73746174655f726f6f74203d20222f7661722f6c69622f67616d6563756c742f73657276696365220a72756e74696d655f726f6f74203d20222f72756e2f67616d6563756c742f6964756e6e2f73657276696365220a6e6574776f726b203d2022686f73742d70726976617465220a68617264656e696e67203d2022737472696374220a6d656d6f72795f6d6562696279746573203d20313032340a6370755f71756f74615f70657263656e74203d203130300a0a5b776f726b6c6f61642e617267756d656e745f62696e64696e67735d0a73746174655f726f6f74203d20222f7661722f6c69622f67616d6563756c742f73657276696365220a0a5b776f726b6c6f61642e7365637265745f66696c65735d0a47414d4543554c545f52554e54494d455f50524553454e43455f4944454e54495459203d20222f6574632f67616d6563756c742f736572766963652f72756e74696d652d70726573656e63652d6964656e746974792e6363220a0a5b72756e74696d655f6964656e746974795d0a72756e74696d655f6964203d2022736572766963652d79676764726173696c220a65787065637465645f7369676e65725f6964656e746974795f6964203d2022736572766963652d72756e74696d652d7369676e6572220a74727573745f616e63686f725f73746f7265203d20222f6574632f67616d6563756c742f74727573742f736572766963652e6363220a0a5b726f7574655d0a647269766572203d20226e67696e782d73747265616d2d746370220a726f7574655f6964203d202273657276696365220a737461626c655f656e64706f696e74203d2022687474703a2f2f3132372e302e302e313a3137393939220a707269766174655f686f7374203d20223132372e302e302e31220a707269766174655f706f72745f7374617274203d2031383030300a707269766174655f706f72745f656e64203d2031383030390a636f6e6669675f70617468203d20222f6574632f6e67696e782f6964756e6e2d73747265616d2d726f757465732f736572766963652e636f6e66220a72656c6f61645f756e6974203d20226e67696e782e73657276696365220a0a5b6272616b65735d0a6465706c6f796d656e745f73746f7265203d20222f7661722f6c69622f67616d6563756c742f6964756e6e2f736572766963652d6465706c6f796d656e742d6272616b652e6363220a6c6966656379636c655f73746f7265203d20222f7661722f6c69622f67616d6563756c742f6964756e6e2f736572766963652d6c6966656379636c652d6272616b652e6363220a0a5b726f6c6c6f75745d0a7374726174656779203d202263616e6469646174652d7468656e2d70726f6d6f7465220a647261696e5f7365636f6e6473203d2033300a72657461696e5f72656c6561736573203d20320a0a5b706c6163656d656e745d0a646573697265645f7265706c69636173203d20310a6e6f646573203d205b2279676764726173696c225d0a919296b57368617265642d696e667261737472756374757265b56f64696e2e76657273652d72656e64657a766f7573b66f64696e2e76657273652d746f706f6c6f67792e7631a2763101b06265666f72652d70726f6d6f74696f6e97a46f64696e98ad6d616e616765642d7265616479a46f64696eb26f64696e2d696e6361726e6174696f6e2d31d9477368613235362d61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161d9477368613235362d65656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565d9477368613235362d63343263333331373863366539666534633435313162623563663962626461376431633138356536646330373064323234653463653431303538626338633238d9477368613235362d6137313730323730313265383236396235666636363961313135386466636236313764346463636230663963333633656631373738656561633230353332633907b56f64696e2e76657273652d72656e64657a766f7573b66f64696e2e76657273652d746f706f6c6f67792e7631a2763101b57463703a2f2f31302e37372e302e313a3137383731cd4651c096b76964756e6e2e7365616c65645f72656c656173652e7631d9477368613235362d35663738353165373634333366383261376562666431626232646234323065366263333263646263346536666538373563356633326461363434643535656237d9477368613235362d643963646464656263666363623263663162613961643662393235663366666162376465616363323666616536316461386632363538613361393430396562329195a66461656d6f6ea773657276696365d9477368613235362d636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363632ac39196af746f6f6c636861696e2d696e646578d92768747470733a2f2f6578616d706c652e696e76616c69642f746f6f6c636861696e2d696e646578d9477368613235362d64646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464a472757374b6696e707574732f746f6f6c636861696e2d696e646578117892d9477368613235362d35663738353165373634333366383261376562666431626232646234323065366263333263646263346536666538373563356633326461363434643535656237bd2f7372762f736572766963652f72656c65617365732f63757272656e74dc0012bd6964756e6e2e65787065637465645f696e6361726e6174696f6e2e7632a773657276696365d9477368613235362d64396364646465626366636362326366316261396164366239323566336666616237646561636332366661653631646138663236353861336139343039656232b5736572766963652d696e6361726e6174696f6e2d31d9477368613235362d35663738353165373634333366383261376562666431626232646234323065366263333263646263346536666538373563356633326461363434643535656237bb6769746875622e636f6d2f47616d6543756c742f53657276696365d92832323232323232323232323232323232323232323232323232323232323232323232323232323232d9477368613235362d35396263633432376637393934303563613461353132623238613830363734343536666133313439373233363132383065613062633231636335653266363533b1736572766963652d79676764726173696cb6736572766963652d72756e74696d652d7369676e6572ae736572766963652e6865616c7468d9477368613235362d63636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363a27631d9477368613235362d32663837303134316366353638316566656332653234316364636435663731303831616539326162316535633534633839636438393666386335656236323731c294a773657276696365a468747470b6687474703a2f2f3132372e302e302e313a3137393939b6687474703a2f2f3132372e302e302e313a31383030319194af736572766963652e72756e74696d65b2736572766963652e72756e74696d652e7631a2763101919ab57368617265642d696e667261737472756374757265b56f64696e2e76657273652d72656e64657a766f7573b66f64696e2e76657273652d746f706f6c6f67792e7631a2763101b06265666f72652d70726f6d6f74696f6ea46f64696eb36d616e616765642d696e6361726e6174696f6ed9477368613235362d63343263333331373863366539666534633435313162623563663962626461376431633138356536646330373064323234653463653431303538626338633238b57463703a2f2f31302e37372e302e313a3137383731d9477368613235362d39613662663530353466616430306232343363323331386330333436303931333230343665643436356231356534636561666532613736623966663231313039c0cf0000018bcfe568019abb6964756e6e2e72756e74696d655f61637469766174696f6e2e7632d9477368613235362d39613662663530353466616430306232343363323331386330333436303931333230343665643436356231356534636561666532613736623966663231313039b1736572766963652d79676764726173696cd9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262d94032363535323862353235386435353563356434646533383931303830363164383965663439616336306631373337626464656337316535333335643163623935c420c06afef8f66e6ec88c96c81ad6d74dbd9933738032f4d8069a46412380e2272dcf0000018bcfe567ecd94066346536393735326334333432363965323262306130346137623430613065336235386364343561383238376364303936383263313864326565366539376239a765643235353139c44046048cf77cc9f524da966e16a131edb34180f4ea8420f5f24a64fd53178e44988bef994d6df103a5f117470283d533e9d7e8245c4f9965f9a4c0a0114ed1c201dc0033b26964756e6e2d343234322e73657276696365af4964756e6e20746573742034323432af696e766f636174696f6e2d3432343201a465786563a6616c77617973ad636f6e74726f6c2d67726f7570c3a57534323432a57534323432a0a0a0c3c3a9696e76697369626c65a3706964c3a430303737a0a0cd10920194cd1092cd1092cd1092cd109294cd1092cd1092cd1092cd109291cd10920000000000c391010807b52f6f70742f746573742f62696e2f736572766963650101d9477368613235362d34626635313232663334343535346335336264653265626238636432623765336431363030616436333163333835613564376363653233633737383534353961d9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262a92f6f70742f74657374a92f72756e2f74657374d9477368613235362d6462633162346339303066666534386435373562356461356336333830343031323566363564623066653365323434393462373665613938363435376439383690d9477368613235362d30383466656430386239373861663464376431393661373434366138366235383030396536333662363131646231363231316236356139616164666632396335d9202f73797374656d2e736c6963652f6964756e6e2d343234322e73657276696365c090aa61637469766174696f6edc0020010101010101010101010101010101010101010101010101010101010101010190d9477368613235362d39623734636534306561326663636337623632613533363838363563653533303438316130643838653931336161396330313333666662633838353763313838959141d9477368613235362d35353961656164303832363464353739356433393039373138636464303561626434393537326538346665353535393065656633316138386130386664666664ab6f64696e2d7369676e6572296492ad6f64696e2d746f706f6c6f6779959139d9477368613235362d31393538316532376465376365643030666631636535306232303437653761353637633736623163626165626162653565663033663763333031376262356237ab6f64696e2d7369676e6572276496a773657276696365d9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262d9477368613235362d35643838303630666232646530386139663366346338313936623062633863663737653934333732373363346135386330653430303737666630316461306534c0c0c096cd10920708c0c0c091b1736b69707065642d73746174656c65737391b1736b69707065642d73746174656c657373c0c029c0c0c0c0c0 diff --git a/tests/fixtures/idunn-control-legacy/transaction-committing-v2.hex b/tests/fixtures/idunn-control-legacy/transaction-committing-v2.hex new file mode 100644 index 0000000..6846f72 --- /dev/null +++ b/tests/fixtures/idunn-control-legacy/transaction-committing-v2.hex @@ -0,0 +1,2 @@ +idunn.deployment_transaction.v2 +dc0022bf6964756e6e2e6465706c6f796d656e745f7472616e73616374696f6e2e7632d92774782d37636236616537392d323635302d346261322d623664392d643137333338643566393031af636f6e74696e756974792d74657374aa636f6e74696e75697479a77365727669636500aa636f6d6d697474696e67cf0000018bcfe56800cf0000018bcfe6eea0c099d9216964756e6e2e636f6d70696c65645f6465706c6f796d656e745f706c616e2e7632d9477368613235362d64396364646465626366636362326366316261396164366239323566336666616237646561636332366661653631646138663236353861336139343039656232b5736572766963652d696e6361726e6174696f6e2d316e9bbf6964756e6e2e736f757263655f73656c656374696f6e5f66616374732e7631d92768747470733a2f2f6769746875622e636f6d2f47616d6543756c742f536572766963652e676974af726566732f68656164732f6d61696ed92832323232323232323232323232323232323232323232323232323232323232323232323232323232d92832323232323232323232323232323232323232323232323232323232323232323232323232323232d92833333333333333333333333333333333333333333333333333333333333333333333333333333333bc6465706c6f796d656e742f6964756e6e2f7265636970652e746f6d6cd9477368613235362d353962636334323766373939343035636134613531326232386138303637343435366661333134393732333631323830656130626332316363356532663635338091a87265662d6865616464dc058e0a736368656d61203d202267616d6563756c742e6964756e6e2e7461726765745f6465636c61726174696f6e2e7631220a746172676574203d202273657276696365220a736f757263655f7374616d705f656e7669726f6e6d656e74203d2022534552564943455f4255494c445f434f4d4d4954220a0a5b5b73746570735d5d0a6964203d20226275696c64220a7068617365203d20226275696c64220a72756e6e6572203d202272757374220a61726776203d205b22636172676f222c20226275696c64222c20222d2d6c6f636b6564225d0a0a5b5b65787465726e616c5f696e707574735d5d0a6964203d2022746f6f6c636861696e2d696e646578220a75726c203d202268747470733a2f2f6578616d706c652e696e76616c69642f746f6f6c636861696e2d696e646578220a736861323536203d202264646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464220a72756e6e6572203d202272757374220a64657374696e6174696f6e203d2022696e707574732f746f6f6c636861696e2d696e646578220a0a5b5b6172746966616374735d5d0a6964203d20226461656d6f6e220a736f757263655f6b696e64203d202272756e6e65722d6f7574707574220a72756e6e6572203d202272757374220a736f75726365203d20227461726765742f72656c656173652f73657276696365220a64657374696e6174696f6e203d202273657276696365220a65787065637465645f736861323536203d202263636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363220a65786563757461626c65203d20747275650a0a5b736572766963655d0a65786563757461626c655f6172746966616374203d20226461656d6f6e220a617267756d656e7473203d205b0a20207b206b696e64203d202262696e64696e67222c206e616d65203d202273746174655f726f6f7422207d2c0a5d0a7472616e73706f7274203d202268747470220a726f7574655f7265717569726564203d20747275650a72657175697265645f656e7669726f6e6d656e74203d205b2247414d4543554c545f4944554e4e5f43414e4449444154455f42494e44222c202247414d4543554c545f4944554e4e5f52554e54494d455f42554e444c45225d0a0a5b736572766963652e6865616c74685d0a636f6e7472616374203d2022736572766963652e6865616c7468220a0a5b73746174655d0a736368656d615f67656e65726174696f6e203d20227631220a0a5b5b73746174652e736c6f74735d5d0a6964203d202272756e74696d652d6361636865220a72656c61746976655f70617468203d202272756e74696d652d63616368652e6363220a6b696e64203d202263756c7463616368652d66696c65220a736368656d61203d2022736572766963652e72756e74696d652d63616368652e7631220a777269746572203d20226e6f6e65220a7265636f76657279203d202272656275696c6461626c65220a73746172747570203d20226f70656e2d61742d7374617274220a0a5b5b70726f76696465735d5d0a6361706162696c697479203d2022736572766963652e72756e74696d65220a736368656d61203d2022736572766963652e72756e74696d652e7631220a636f6d7061746962696c697479203d20227631220a0a5b5b646570656e64656e636965735d5d0a6b696e64203d20227368617265642d696e667261737472756374757265220a6361706162696c697479203d20226f64696e2e76657273652d72656e64657a766f7573220a736368656d61203d20226f64696e2e76657273652d746f706f6c6f67792e7631220a636f6d7061746962696c697479203d20227631220adc076e0a736368656d61203d202267616d6563756c742e6964756e6e2e6f70657261746f725f62696e64696e672e7632220a746172676574203d202273657276696365220a0a5b7265706f7369746f72795d0a6f726967696e203d202268747470733a2f2f6769746875622e636f6d2f47616d6543756c742f536572766963652e676974220a61646d69747465645f726566203d2022726566732f68656164732f6d61696e220a6d696e696d756d5f7265766973696f6e203d202231313131313131313131313131313131313131313131313131313131313131313131313131313131220a73656c656374696f6e203d20227265662d68656164220a636865636b6f7574203d20222f7372762f6275696c642f53657276696365220a7265636970655f70617468203d20226465706c6f796d656e742f6964756e6e2f7265636970652e746f6d6c220a0a5b72756e6e6572732e727573745d0a647269766572203d2022646f636b6572220a696d616765203d202272757374407368613235363a61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161220a75736572203d2022313030303a31303030220a6166666f7264616e636573203d205b22736f757263652d72656164222c202261727469666163742d7772697465225d0a616c6c6f7765645f70726f6772616d73203d205b22636172676f225d0a6e6574776f726b5f70726f66696c65203d20226275696c642d646570656e64656e63792d656772657373220a6d656d6f72795f6d6562696279746573203d20323034380a6370755f71756f74615f70657263656e74203d203230300a706964735f6c696d6974203d203531320a746d7066735f6d6562696279746573203d203531320a0a5b776f726b6c6f61645d0a647269766572203d202273797374656d642d7472616e7369656e74220a73746174655f67726f7570203d202273657276696365220a756e69745f707265666978203d20226964756e6e2d73657276696365220a72656c656173655f726f6f74203d20222f7372762f736572766963652f72656c6561736573220a73746174655f726f6f74203d20222f7661722f6c69622f67616d6563756c742f73657276696365220a72756e74696d655f726f6f74203d20222f72756e2f67616d6563756c742f6964756e6e2f73657276696365220a6e6574776f726b203d2022686f73742d70726976617465220a68617264656e696e67203d2022737472696374220a6d656d6f72795f6d6562696279746573203d20313032340a6370755f71756f74615f70657263656e74203d203130300a0a5b776f726b6c6f61642e617267756d656e745f62696e64696e67735d0a73746174655f726f6f74203d20222f7661722f6c69622f67616d6563756c742f73657276696365220a0a5b776f726b6c6f61642e7365637265745f66696c65735d0a47414d4543554c545f52554e54494d455f50524553454e43455f4944454e54495459203d20222f6574632f67616d6563756c742f736572766963652f72756e74696d652d70726573656e63652d6964656e746974792e6363220a0a5b72756e74696d655f6964656e746974795d0a72756e74696d655f6964203d2022736572766963652d79676764726173696c220a65787065637465645f7369676e65725f6964656e746974795f6964203d2022736572766963652d72756e74696d652d7369676e6572220a74727573745f616e63686f725f73746f7265203d20222f6574632f67616d6563756c742f74727573742f736572766963652e6363220a0a5b726f7574655d0a647269766572203d20226e67696e782d73747265616d2d746370220a726f7574655f6964203d202273657276696365220a737461626c655f656e64706f696e74203d2022687474703a2f2f3132372e302e302e313a3137393939220a707269766174655f686f7374203d20223132372e302e302e31220a707269766174655f706f72745f7374617274203d2031383030300a707269766174655f706f72745f656e64203d2031383030390a636f6e6669675f70617468203d20222f6574632f6e67696e782f6964756e6e2d73747265616d2d726f757465732f736572766963652e636f6e66220a72656c6f61645f756e6974203d20226e67696e782e73657276696365220a0a5b6272616b65735d0a6465706c6f796d656e745f73746f7265203d20222f7661722f6c69622f67616d6563756c742f6964756e6e2f736572766963652d6465706c6f796d656e742d6272616b652e6363220a6c6966656379636c655f73746f7265203d20222f7661722f6c69622f67616d6563756c742f6964756e6e2f736572766963652d6c6966656379636c652d6272616b652e6363220a0a5b726f6c6c6f75745d0a7374726174656779203d202263616e6469646174652d7468656e2d70726f6d6f7465220a647261696e5f7365636f6e6473203d2033300a72657461696e5f72656c6561736573203d20320a0a5b706c6163656d656e745d0a646573697265645f7265706c69636173203d20310a6e6f646573203d205b2279676764726173696c225d0a919296b57368617265642d696e667261737472756374757265b56f64696e2e76657273652d72656e64657a766f7573b66f64696e2e76657273652d746f706f6c6f67792e7631a2763101b06265666f72652d70726f6d6f74696f6e97a46f64696e98ad6d616e616765642d7265616479a46f64696eb26f64696e2d696e6361726e6174696f6e2d31d9477368613235362d61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161d9477368613235362d65656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565d9477368613235362d63343263333331373863366539666534633435313162623563663962626461376431633138356536646330373064323234653463653431303538626338633238d9477368613235362d6137313730323730313265383236396235666636363961313135386466636236313764346463636230663963333633656631373738656561633230353332633907b56f64696e2e76657273652d72656e64657a766f7573b66f64696e2e76657273652d746f706f6c6f67792e7631a2763101b57463703a2f2f31302e37372e302e313a3137383731cd4651c096b76964756e6e2e7365616c65645f72656c656173652e7631d9477368613235362d35663738353165373634333366383261376562666431626232646234323065366263333263646263346536666538373563356633326461363434643535656237d9477368613235362d643963646464656263666363623263663162613961643662393235663366666162376465616363323666616536316461386632363538613361393430396562329195a66461656d6f6ea773657276696365d9477368613235362d636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363632ac39196af746f6f6c636861696e2d696e646578d92768747470733a2f2f6578616d706c652e696e76616c69642f746f6f6c636861696e2d696e646578d9477368613235362d64646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464a472757374b6696e707574732f746f6f6c636861696e2d696e646578117892d9477368613235362d35663738353165373634333366383261376562666431626232646234323065366263333263646263346536666538373563356633326461363434643535656237bd2f7372762f736572766963652f72656c65617365732f63757272656e74dc0012bd6964756e6e2e65787065637465645f696e6361726e6174696f6e2e7632a773657276696365d9477368613235362d64396364646465626366636362326366316261396164366239323566336666616237646561636332366661653631646138663236353861336139343039656232b5736572766963652d696e6361726e6174696f6e2d31d9477368613235362d35663738353165373634333366383261376562666431626232646234323065366263333263646263346536666538373563356633326461363434643535656237bb6769746875622e636f6d2f47616d6543756c742f53657276696365d92832323232323232323232323232323232323232323232323232323232323232323232323232323232d9477368613235362d35396263633432376637393934303563613461353132623238613830363734343536666133313439373233363132383065613062633231636335653266363533b1736572766963652d79676764726173696cb6736572766963652d72756e74696d652d7369676e6572ae736572766963652e6865616c7468d9477368613235362d63636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363a27631d9477368613235362d32663837303134316366353638316566656332653234316364636435663731303831616539326162316535633534633839636438393666386335656236323731c294a773657276696365a468747470b6687474703a2f2f3132372e302e302e313a3137393939b6687474703a2f2f3132372e302e302e313a31383030319194af736572766963652e72756e74696d65b2736572766963652e72756e74696d652e7631a2763101919ab57368617265642d696e667261737472756374757265b56f64696e2e76657273652d72656e64657a766f7573b66f64696e2e76657273652d746f706f6c6f67792e7631a2763101b06265666f72652d70726f6d6f74696f6ea46f64696eb36d616e616765642d696e6361726e6174696f6ed9477368613235362d63343263333331373863366539666534633435313162623563663962626461376431633138356536646330373064323234653463653431303538626338633238b57463703a2f2f31302e37372e302e313a3137383731d9477368613235362d39613662663530353466616430306232343363323331386330333436303931333230343665643436356231356534636561666532613736623966663231313039c0cf0000018bcfe568019abb6964756e6e2e72756e74696d655f61637469766174696f6e2e7632d9477368613235362d39613662663530353466616430306232343363323331386330333436303931333230343665643436356231356534636561666532613736623966663231313039b1736572766963652d79676764726173696cd9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262d94032363535323862353235386435353563356434646533383931303830363164383965663439616336306631373337626464656337316535333335643163623935c420c06afef8f66e6ec88c96c81ad6d74dbd9933738032f4d8069a46412380e2272dcf0000018bcfe567ecd94066346536393735326334333432363965323262306130346137623430613065336235386364343561383238376364303936383263313864326565366539376239a765643235353139c44046048cf77cc9f524da966e16a131edb34180f4ea8420f5f24a64fd53178e44988bef994d6df103a5f117470283d533e9d7e8245c4f9965f9a4c0a0114ed1c201dc0033b26964756e6e2d343234322e73657276696365af4964756e6e20746573742034323432af696e766f636174696f6e2d3432343201a465786563a6616c77617973ad636f6e74726f6c2d67726f7570c3a57534323432a57534323432a0a0a0c3c3a9696e76697369626c65a3706964c3a430303737a0a0cd10920194cd1092cd1092cd1092cd109294cd1092cd1092cd1092cd109291cd10920000000000c391010807b52f6f70742f746573742f62696e2f736572766963650101d9477368613235362d34626635313232663334343535346335336264653265626238636432623765336431363030616436333163333835613564376363653233633737383534353961d9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262a92f6f70742f74657374a92f72756e2f74657374d9477368613235362d6462633162346339303066666534386435373562356461356336333830343031323566363564623066653365323434393462373665613938363435376439383690d9477368613235362d30383466656430386239373861663464376431393661373434366138366235383030396536333662363131646231363231316236356139616164666632396335d9202f73797374656d2e736c6963652f6964756e6e2d343234322e73657276696365c090aa61637469766174696f6edc0020010101010101010101010101010101010101010101010101010101010101010190d9477368613235362d39623734636534306561326663636337623632613533363838363563653533303438316130643838653931336161396330313333666662633838353763313838959141d9477368613235362d35353961656164303832363464353739356433393039373138636464303561626434393537326538346665353535393065656633316138386130386664666664ab6f64696e2d7369676e6572296492ad6f64696e2d746f706f6c6f6779959139d9477368613235362d31393538316532376465376365643030666631636535306232303437653761353637633736623163626165626162653565663033663763333031376262356237ab6f64696e2d7369676e6572276496a773657276696365d9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262d9477368613235362d35643838303630666232646530386139663366346338313936623062633863663737653934333732373363346135386330653430303737666630316461306534c0c0c096cd10920708c0c0c091b1736b69707065642d73746174656c65737391b1736b69707065642d73746174656c657373959140d9477368613235362d63333634316638353434643763303266333538306230376330663938383766306336613237666635616231643461336532396361663139376366633239396165ab6f64696e2d7369676e6572286493a870726f6d6f74656495a773657276696365d9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262d9477368613235362d62663563663539653335363635323235333236386336303463626638646638636664623033613461306433326232376164313538653538313730396338306534d9477368613235362d34313266303131626462396566646138616137323036636364336333613438633637613131386164646161363266616436303463326236346664396163643063cf0000018bcfe6ee96cf0000018bcfe6ee9629c0c0c0c0 diff --git a/tests/fixtures/idunn-control-legacy/transaction-committing.hex b/tests/fixtures/idunn-control-legacy/transaction-committing.hex new file mode 100644 index 0000000..36decd5 --- /dev/null +++ b/tests/fixtures/idunn-control-legacy/transaction-committing.hex @@ -0,0 +1,2 @@ +idunn.deployment_transaction.v3 +dc0023bf6964756e6e2e6465706c6f796d656e745f7472616e73616374696f6e2e7633d92774782d37636236616537392d323635302d346261322d623664392d643137333338643566393031af636f6e74696e756974792d74657374aa636f6e74696e75697479a77365727669636500aa636f6d6d697474696e67cf0000018bcfe56800cf0000018bcfe6eea0c099d9216964756e6e2e636f6d70696c65645f6465706c6f796d656e745f706c616e2e7632d9477368613235362d64396364646465626366636362326366316261396164366239323566336666616237646561636332366661653631646138663236353861336139343039656232b5736572766963652d696e6361726e6174696f6e2d316e9bbf6964756e6e2e736f757263655f73656c656374696f6e5f66616374732e7631d92768747470733a2f2f6769746875622e636f6d2f47616d6543756c742f536572766963652e676974af726566732f68656164732f6d61696ed92832323232323232323232323232323232323232323232323232323232323232323232323232323232d92832323232323232323232323232323232323232323232323232323232323232323232323232323232d92833333333333333333333333333333333333333333333333333333333333333333333333333333333bc6465706c6f796d656e742f6964756e6e2f7265636970652e746f6d6cd9477368613235362d353962636334323766373939343035636134613531326232386138303637343435366661333134393732333631323830656130626332316363356532663635338091a87265662d6865616464dc058e0a736368656d61203d202267616d6563756c742e6964756e6e2e7461726765745f6465636c61726174696f6e2e7631220a746172676574203d202273657276696365220a736f757263655f7374616d705f656e7669726f6e6d656e74203d2022534552564943455f4255494c445f434f4d4d4954220a0a5b5b73746570735d5d0a6964203d20226275696c64220a7068617365203d20226275696c64220a72756e6e6572203d202272757374220a61726776203d205b22636172676f222c20226275696c64222c20222d2d6c6f636b6564225d0a0a5b5b65787465726e616c5f696e707574735d5d0a6964203d2022746f6f6c636861696e2d696e646578220a75726c203d202268747470733a2f2f6578616d706c652e696e76616c69642f746f6f6c636861696e2d696e646578220a736861323536203d202264646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464220a72756e6e6572203d202272757374220a64657374696e6174696f6e203d2022696e707574732f746f6f6c636861696e2d696e646578220a0a5b5b6172746966616374735d5d0a6964203d20226461656d6f6e220a736f757263655f6b696e64203d202272756e6e65722d6f7574707574220a72756e6e6572203d202272757374220a736f75726365203d20227461726765742f72656c656173652f73657276696365220a64657374696e6174696f6e203d202273657276696365220a65787065637465645f736861323536203d202263636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363220a65786563757461626c65203d20747275650a0a5b736572766963655d0a65786563757461626c655f6172746966616374203d20226461656d6f6e220a617267756d656e7473203d205b0a20207b206b696e64203d202262696e64696e67222c206e616d65203d202273746174655f726f6f7422207d2c0a5d0a7472616e73706f7274203d202268747470220a726f7574655f7265717569726564203d20747275650a72657175697265645f656e7669726f6e6d656e74203d205b2247414d4543554c545f4944554e4e5f43414e4449444154455f42494e44222c202247414d4543554c545f4944554e4e5f52554e54494d455f42554e444c45225d0a0a5b736572766963652e6865616c74685d0a636f6e7472616374203d2022736572766963652e6865616c7468220a0a5b73746174655d0a736368656d615f67656e65726174696f6e203d20227631220a0a5b5b73746174652e736c6f74735d5d0a6964203d202272756e74696d652d6361636865220a72656c61746976655f70617468203d202272756e74696d652d63616368652e6363220a6b696e64203d202263756c7463616368652d66696c65220a736368656d61203d2022736572766963652e72756e74696d652d63616368652e7631220a777269746572203d20226e6f6e65220a7265636f76657279203d202272656275696c6461626c65220a73746172747570203d20226f70656e2d61742d7374617274220a0a5b5b70726f76696465735d5d0a6361706162696c697479203d2022736572766963652e72756e74696d65220a736368656d61203d2022736572766963652e72756e74696d652e7631220a636f6d7061746962696c697479203d20227631220a0a5b5b646570656e64656e636965735d5d0a6b696e64203d20227368617265642d696e667261737472756374757265220a6361706162696c697479203d20226f64696e2e76657273652d72656e64657a766f7573220a736368656d61203d20226f64696e2e76657273652d746f706f6c6f67792e7631220a636f6d7061746962696c697479203d20227631220adc076e0a736368656d61203d202267616d6563756c742e6964756e6e2e6f70657261746f725f62696e64696e672e7632220a746172676574203d202273657276696365220a0a5b7265706f7369746f72795d0a6f726967696e203d202268747470733a2f2f6769746875622e636f6d2f47616d6543756c742f536572766963652e676974220a61646d69747465645f726566203d2022726566732f68656164732f6d61696e220a6d696e696d756d5f7265766973696f6e203d202231313131313131313131313131313131313131313131313131313131313131313131313131313131220a73656c656374696f6e203d20227265662d68656164220a636865636b6f7574203d20222f7372762f6275696c642f53657276696365220a7265636970655f70617468203d20226465706c6f796d656e742f6964756e6e2f7265636970652e746f6d6c220a0a5b72756e6e6572732e727573745d0a647269766572203d2022646f636b6572220a696d616765203d202272757374407368613235363a61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161220a75736572203d2022313030303a31303030220a6166666f7264616e636573203d205b22736f757263652d72656164222c202261727469666163742d7772697465225d0a616c6c6f7765645f70726f6772616d73203d205b22636172676f225d0a6e6574776f726b5f70726f66696c65203d20226275696c642d646570656e64656e63792d656772657373220a6d656d6f72795f6d6562696279746573203d20323034380a6370755f71756f74615f70657263656e74203d203230300a706964735f6c696d6974203d203531320a746d7066735f6d6562696279746573203d203531320a0a5b776f726b6c6f61645d0a647269766572203d202273797374656d642d7472616e7369656e74220a73746174655f67726f7570203d202273657276696365220a756e69745f707265666978203d20226964756e6e2d73657276696365220a72656c656173655f726f6f74203d20222f7372762f736572766963652f72656c6561736573220a73746174655f726f6f74203d20222f7661722f6c69622f67616d6563756c742f73657276696365220a72756e74696d655f726f6f74203d20222f72756e2f67616d6563756c742f6964756e6e2f73657276696365220a6e6574776f726b203d2022686f73742d70726976617465220a68617264656e696e67203d2022737472696374220a6d656d6f72795f6d6562696279746573203d20313032340a6370755f71756f74615f70657263656e74203d203130300a0a5b776f726b6c6f61642e617267756d656e745f62696e64696e67735d0a73746174655f726f6f74203d20222f7661722f6c69622f67616d6563756c742f73657276696365220a0a5b776f726b6c6f61642e7365637265745f66696c65735d0a47414d4543554c545f52554e54494d455f50524553454e43455f4944454e54495459203d20222f6574632f67616d6563756c742f736572766963652f72756e74696d652d70726573656e63652d6964656e746974792e6363220a0a5b72756e74696d655f6964656e746974795d0a72756e74696d655f6964203d2022736572766963652d79676764726173696c220a65787065637465645f7369676e65725f6964656e746974795f6964203d2022736572766963652d72756e74696d652d7369676e6572220a74727573745f616e63686f725f73746f7265203d20222f6574632f67616d6563756c742f74727573742f736572766963652e6363220a0a5b726f7574655d0a647269766572203d20226e67696e782d73747265616d2d746370220a726f7574655f6964203d202273657276696365220a737461626c655f656e64706f696e74203d2022687474703a2f2f3132372e302e302e313a3137393939220a707269766174655f686f7374203d20223132372e302e302e31220a707269766174655f706f72745f7374617274203d2031383030300a707269766174655f706f72745f656e64203d2031383030390a636f6e6669675f70617468203d20222f6574632f6e67696e782f6964756e6e2d73747265616d2d726f757465732f736572766963652e636f6e66220a72656c6f61645f756e6974203d20226e67696e782e73657276696365220a0a5b6272616b65735d0a6465706c6f796d656e745f73746f7265203d20222f7661722f6c69622f67616d6563756c742f6964756e6e2f736572766963652d6465706c6f796d656e742d6272616b652e6363220a6c6966656379636c655f73746f7265203d20222f7661722f6c69622f67616d6563756c742f6964756e6e2f736572766963652d6c6966656379636c652d6272616b652e6363220a0a5b726f6c6c6f75745d0a7374726174656779203d202263616e6469646174652d7468656e2d70726f6d6f7465220a647261696e5f7365636f6e6473203d2033300a72657461696e5f72656c6561736573203d20320a0a5b706c6163656d656e745d0a646573697265645f7265706c69636173203d20310a6e6f646573203d205b2279676764726173696c225d0a919296b57368617265642d696e667261737472756374757265b56f64696e2e76657273652d72656e64657a766f7573b66f64696e2e76657273652d746f706f6c6f67792e7631a2763101b06265666f72652d70726f6d6f74696f6e97a46f64696e98ad6d616e616765642d7265616479a46f64696eb26f64696e2d696e6361726e6174696f6e2d31d9477368613235362d61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161d9477368613235362d65656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565d9477368613235362d63343263333331373863366539666534633435313162623563663962626461376431633138356536646330373064323234653463653431303538626338633238d9477368613235362d6137313730323730313265383236396235666636363961313135386466636236313764346463636230663963333633656631373738656561633230353332633907b56f64696e2e76657273652d72656e64657a766f7573b66f64696e2e76657273652d746f706f6c6f67792e7631a2763101b57463703a2f2f31302e37372e302e313a3137383731cd4651c096b76964756e6e2e7365616c65645f72656c656173652e7631d9477368613235362d35663738353165373634333366383261376562666431626232646234323065366263333263646263346536666538373563356633326461363434643535656237d9477368613235362d643963646464656263666363623263663162613961643662393235663366666162376465616363323666616536316461386632363538613361393430396562329195a66461656d6f6ea773657276696365d9477368613235362d636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363632ac39196af746f6f6c636861696e2d696e646578d92768747470733a2f2f6578616d706c652e696e76616c69642f746f6f6c636861696e2d696e646578d9477368613235362d64646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464a472757374b6696e707574732f746f6f6c636861696e2d696e646578117892d9477368613235362d35663738353165373634333366383261376562666431626232646234323065366263333263646263346536666538373563356633326461363434643535656237bd2f7372762f736572766963652f72656c65617365732f63757272656e74dc0012bd6964756e6e2e65787065637465645f696e6361726e6174696f6e2e7632a773657276696365d9477368613235362d64396364646465626366636362326366316261396164366239323566336666616237646561636332366661653631646138663236353861336139343039656232b5736572766963652d696e6361726e6174696f6e2d31d9477368613235362d35663738353165373634333366383261376562666431626232646234323065366263333263646263346536666538373563356633326461363434643535656237bb6769746875622e636f6d2f47616d6543756c742f53657276696365d92832323232323232323232323232323232323232323232323232323232323232323232323232323232d9477368613235362d35396263633432376637393934303563613461353132623238613830363734343536666133313439373233363132383065613062633231636335653266363533b1736572766963652d79676764726173696cb6736572766963652d72756e74696d652d7369676e6572ae736572766963652e6865616c7468d9477368613235362d63636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363a27631d9477368613235362d32663837303134316366353638316566656332653234316364636435663731303831616539326162316535633534633839636438393666386335656236323731c294a773657276696365a468747470b6687474703a2f2f3132372e302e302e313a3137393939b6687474703a2f2f3132372e302e302e313a31383030319194af736572766963652e72756e74696d65b2736572766963652e72756e74696d652e7631a2763101919ab57368617265642d696e667261737472756374757265b56f64696e2e76657273652d72656e64657a766f7573b66f64696e2e76657273652d746f706f6c6f67792e7631a2763101b06265666f72652d70726f6d6f74696f6ea46f64696eb36d616e616765642d696e6361726e6174696f6ed9477368613235362d63343263333331373863366539666534633435313162623563663962626461376431633138356536646330373064323234653463653431303538626338633238b57463703a2f2f31302e37372e302e313a3137383731d9477368613235362d39613662663530353466616430306232343363323331386330333436303931333230343665643436356231356534636561666532613736623966663231313039c0cf0000018bcfe568019abb6964756e6e2e72756e74696d655f61637469766174696f6e2e7632d9477368613235362d39613662663530353466616430306232343363323331386330333436303931333230343665643436356231356534636561666532613736623966663231313039b1736572766963652d79676764726173696cd9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262d94032363535323862353235386435353563356434646533383931303830363164383965663439616336306631373337626464656337316535333335643163623935c420c06afef8f66e6ec88c96c81ad6d74dbd9933738032f4d8069a46412380e2272dcf0000018bcfe567ecd94066346536393735326334333432363965323262306130346137623430613065336235386364343561383238376364303936383263313864326565366539376239a765643235353139c44046048cf77cc9f524da966e16a131edb34180f4ea8420f5f24a64fd53178e44988bef994d6df103a5f117470283d533e9d7e8245c4f9965f9a4c0a0114ed1c201dc0033b26964756e6e2d343234322e73657276696365af4964756e6e20746573742034323432af696e766f636174696f6e2d3432343201a465786563a6616c77617973ad636f6e74726f6c2d67726f7570c3a57534323432a57534323432a0a0a0c3c3a9696e76697369626c65a3706964c3a430303737a0a0cd10920194cd1092cd1092cd1092cd109294cd1092cd1092cd1092cd109291cd10920000000000c391010807b52f6f70742f746573742f62696e2f736572766963650101d9477368613235362d34626635313232663334343535346335336264653265626238636432623765336431363030616436333163333835613564376363653233633737383534353961d9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262a92f6f70742f74657374a92f72756e2f74657374d9477368613235362d6462633162346339303066666534386435373562356461356336333830343031323566363564623066653365323434393462373665613938363435376439383690d9477368613235362d30383466656430386239373861663464376431393661373434366138366235383030396536333662363131646231363231316236356139616164666632396335d9202f73797374656d2e736c6963652f6964756e6e2d343234322e73657276696365c090aa61637469766174696f6edc0020010101010101010101010101010101010101010101010101010101010101010190d9477368613235362d39623734636534306561326663636337623632613533363838363563653533303438316130643838653931336161396330313333666662633838353763313838959141d9477368613235362d35353961656164303832363464353739356433393039373138636464303561626434393537326538346665353535393065656633316138386130386664666664ab6f64696e2d7369676e6572296492ad6f64696e2d746f706f6c6f6779959139d9477368613235362d31393538316532376465376365643030666631636535306232303437653761353637633736623163626165626162653565663033663763333031376262356237ab6f64696e2d7369676e6572276496a773657276696365d9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262d9477368613235362d35643838303630666232646530386139663366346338313936623062633863663737653934333732373363346135386330653430303737666630316461306534c0c0c096cd10920708c0c0c091b1736b69707065642d73746174656c65737391b1736b69707065642d73746174656c657373959140d9477368613235362d63333634316638353434643763303266333538306230376330663938383766306336613237666635616231643461336532396361663139376366633239396165ab6f64696e2d7369676e6572286493a870726f6d6f74656495a773657276696365d9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262d9477368613235362d62663563663539653335363635323235333236386336303463626638646638636664623033613461306433326232376164313538653538313730396338306534d9477368613235362d34313266303131626462396566646138616137323036636364336333613438633637613131386164646161363266616436303463326236346664396163643063cf0000018bcfe6ee96cf0000018bcfe6ee9629c0c0c0c0c0 diff --git a/tests/fixtures/idunn-control-legacy/transaction-failed-after-fencing.hex b/tests/fixtures/idunn-control-legacy/transaction-failed-after-fencing.hex new file mode 100644 index 0000000..9f7e75c --- /dev/null +++ b/tests/fixtures/idunn-control-legacy/transaction-failed-after-fencing.hex @@ -0,0 +1,2 @@ +idunn.deployment_transaction.v3 +dc0023bf6964756e6e2e6465706c6f796d656e745f7472616e73616374696f6e2e7633d92774782d37636236616537392d323635302d346261322d623664392d643137333338643566393031af636f6e74696e756974792d74657374aa636f6e74696e75697479a77365727669636500a8636f6d706c657465cf0000018bcfe56800cf0000018bcfe6eea0c099d9216964756e6e2e636f6d70696c65645f6465706c6f796d656e745f706c616e2e7632d9477368613235362d64396364646465626366636362326366316261396164366239323566336666616237646561636332366661653631646138663236353861336139343039656232b5736572766963652d696e6361726e6174696f6e2d316e9bbf6964756e6e2e736f757263655f73656c656374696f6e5f66616374732e7631d92768747470733a2f2f6769746875622e636f6d2f47616d6543756c742f536572766963652e676974af726566732f68656164732f6d61696ed92832323232323232323232323232323232323232323232323232323232323232323232323232323232d92832323232323232323232323232323232323232323232323232323232323232323232323232323232d92833333333333333333333333333333333333333333333333333333333333333333333333333333333bc6465706c6f796d656e742f6964756e6e2f7265636970652e746f6d6cd9477368613235362d353962636334323766373939343035636134613531326232386138303637343435366661333134393732333631323830656130626332316363356532663635338091a87265662d6865616464dc058e0a736368656d61203d202267616d6563756c742e6964756e6e2e7461726765745f6465636c61726174696f6e2e7631220a746172676574203d202273657276696365220a736f757263655f7374616d705f656e7669726f6e6d656e74203d2022534552564943455f4255494c445f434f4d4d4954220a0a5b5b73746570735d5d0a6964203d20226275696c64220a7068617365203d20226275696c64220a72756e6e6572203d202272757374220a61726776203d205b22636172676f222c20226275696c64222c20222d2d6c6f636b6564225d0a0a5b5b65787465726e616c5f696e707574735d5d0a6964203d2022746f6f6c636861696e2d696e646578220a75726c203d202268747470733a2f2f6578616d706c652e696e76616c69642f746f6f6c636861696e2d696e646578220a736861323536203d202264646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464220a72756e6e6572203d202272757374220a64657374696e6174696f6e203d2022696e707574732f746f6f6c636861696e2d696e646578220a0a5b5b6172746966616374735d5d0a6964203d20226461656d6f6e220a736f757263655f6b696e64203d202272756e6e65722d6f7574707574220a72756e6e6572203d202272757374220a736f75726365203d20227461726765742f72656c656173652f73657276696365220a64657374696e6174696f6e203d202273657276696365220a65787065637465645f736861323536203d202263636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363220a65786563757461626c65203d20747275650a0a5b736572766963655d0a65786563757461626c655f6172746966616374203d20226461656d6f6e220a617267756d656e7473203d205b0a20207b206b696e64203d202262696e64696e67222c206e616d65203d202273746174655f726f6f7422207d2c0a5d0a7472616e73706f7274203d202268747470220a726f7574655f7265717569726564203d20747275650a72657175697265645f656e7669726f6e6d656e74203d205b2247414d4543554c545f4944554e4e5f43414e4449444154455f42494e44222c202247414d4543554c545f4944554e4e5f52554e54494d455f42554e444c45225d0a0a5b736572766963652e6865616c74685d0a636f6e7472616374203d2022736572766963652e6865616c7468220a0a5b73746174655d0a736368656d615f67656e65726174696f6e203d20227631220a0a5b5b73746174652e736c6f74735d5d0a6964203d202272756e74696d652d6361636865220a72656c61746976655f70617468203d202272756e74696d652d63616368652e6363220a6b696e64203d202263756c7463616368652d66696c65220a736368656d61203d2022736572766963652e72756e74696d652d63616368652e7631220a777269746572203d20226e6f6e65220a7265636f76657279203d202272656275696c6461626c65220a73746172747570203d20226f70656e2d61742d7374617274220a0a5b5b70726f76696465735d5d0a6361706162696c697479203d2022736572766963652e72756e74696d65220a736368656d61203d2022736572766963652e72756e74696d652e7631220a636f6d7061746962696c697479203d20227631220a0a5b5b646570656e64656e636965735d5d0a6b696e64203d20227368617265642d696e667261737472756374757265220a6361706162696c697479203d20226f64696e2e76657273652d72656e64657a766f7573220a736368656d61203d20226f64696e2e76657273652d746f706f6c6f67792e7631220a636f6d7061746962696c697479203d20227631220adc076e0a736368656d61203d202267616d6563756c742e6964756e6e2e6f70657261746f725f62696e64696e672e7632220a746172676574203d202273657276696365220a0a5b7265706f7369746f72795d0a6f726967696e203d202268747470733a2f2f6769746875622e636f6d2f47616d6543756c742f536572766963652e676974220a61646d69747465645f726566203d2022726566732f68656164732f6d61696e220a6d696e696d756d5f7265766973696f6e203d202231313131313131313131313131313131313131313131313131313131313131313131313131313131220a73656c656374696f6e203d20227265662d68656164220a636865636b6f7574203d20222f7372762f6275696c642f53657276696365220a7265636970655f70617468203d20226465706c6f796d656e742f6964756e6e2f7265636970652e746f6d6c220a0a5b72756e6e6572732e727573745d0a647269766572203d2022646f636b6572220a696d616765203d202272757374407368613235363a61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161220a75736572203d2022313030303a31303030220a6166666f7264616e636573203d205b22736f757263652d72656164222c202261727469666163742d7772697465225d0a616c6c6f7765645f70726f6772616d73203d205b22636172676f225d0a6e6574776f726b5f70726f66696c65203d20226275696c642d646570656e64656e63792d656772657373220a6d656d6f72795f6d6562696279746573203d20323034380a6370755f71756f74615f70657263656e74203d203230300a706964735f6c696d6974203d203531320a746d7066735f6d6562696279746573203d203531320a0a5b776f726b6c6f61645d0a647269766572203d202273797374656d642d7472616e7369656e74220a73746174655f67726f7570203d202273657276696365220a756e69745f707265666978203d20226964756e6e2d73657276696365220a72656c656173655f726f6f74203d20222f7372762f736572766963652f72656c6561736573220a73746174655f726f6f74203d20222f7661722f6c69622f67616d6563756c742f73657276696365220a72756e74696d655f726f6f74203d20222f72756e2f67616d6563756c742f6964756e6e2f73657276696365220a6e6574776f726b203d2022686f73742d70726976617465220a68617264656e696e67203d2022737472696374220a6d656d6f72795f6d6562696279746573203d20313032340a6370755f71756f74615f70657263656e74203d203130300a0a5b776f726b6c6f61642e617267756d656e745f62696e64696e67735d0a73746174655f726f6f74203d20222f7661722f6c69622f67616d6563756c742f73657276696365220a0a5b776f726b6c6f61642e7365637265745f66696c65735d0a47414d4543554c545f52554e54494d455f50524553454e43455f4944454e54495459203d20222f6574632f67616d6563756c742f736572766963652f72756e74696d652d70726573656e63652d6964656e746974792e6363220a0a5b72756e74696d655f6964656e746974795d0a72756e74696d655f6964203d2022736572766963652d79676764726173696c220a65787065637465645f7369676e65725f6964656e746974795f6964203d2022736572766963652d72756e74696d652d7369676e6572220a74727573745f616e63686f725f73746f7265203d20222f6574632f67616d6563756c742f74727573742f736572766963652e6363220a0a5b726f7574655d0a647269766572203d20226e67696e782d73747265616d2d746370220a726f7574655f6964203d202273657276696365220a737461626c655f656e64706f696e74203d2022687474703a2f2f3132372e302e302e313a3137393939220a707269766174655f686f7374203d20223132372e302e302e31220a707269766174655f706f72745f7374617274203d2031383030300a707269766174655f706f72745f656e64203d2031383030390a636f6e6669675f70617468203d20222f6574632f6e67696e782f6964756e6e2d73747265616d2d726f757465732f736572766963652e636f6e66220a72656c6f61645f756e6974203d20226e67696e782e73657276696365220a0a5b6272616b65735d0a6465706c6f796d656e745f73746f7265203d20222f7661722f6c69622f67616d6563756c742f6964756e6e2f736572766963652d6465706c6f796d656e742d6272616b652e6363220a6c6966656379636c655f73746f7265203d20222f7661722f6c69622f67616d6563756c742f6964756e6e2f736572766963652d6c6966656379636c652d6272616b652e6363220a0a5b726f6c6c6f75745d0a7374726174656779203d202263616e6469646174652d7468656e2d70726f6d6f7465220a647261696e5f7365636f6e6473203d2033300a72657461696e5f72656c6561736573203d20320a0a5b706c6163656d656e745d0a646573697265645f7265706c69636173203d20310a6e6f646573203d205b2279676764726173696c225d0a919296b57368617265642d696e667261737472756374757265b56f64696e2e76657273652d72656e64657a766f7573b66f64696e2e76657273652d746f706f6c6f67792e7631a2763101b06265666f72652d70726f6d6f74696f6e97a46f64696e98ad6d616e616765642d7265616479a46f64696eb26f64696e2d696e6361726e6174696f6e2d31d9477368613235362d61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161d9477368613235362d65656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565d9477368613235362d63343263333331373863366539666534633435313162623563663962626461376431633138356536646330373064323234653463653431303538626338633238d9477368613235362d6137313730323730313265383236396235666636363961313135386466636236313764346463636230663963333633656631373738656561633230353332633907b56f64696e2e76657273652d72656e64657a766f7573b66f64696e2e76657273652d746f706f6c6f67792e7631a2763101b57463703a2f2f31302e37372e302e313a3137383731cd4651c096b76964756e6e2e7365616c65645f72656c656173652e7631d9477368613235362d35663738353165373634333366383261376562666431626232646234323065366263333263646263346536666538373563356633326461363434643535656237d9477368613235362d643963646464656263666363623263663162613961643662393235663366666162376465616363323666616536316461386632363538613361393430396562329195a66461656d6f6ea773657276696365d9477368613235362d636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363632ac39196af746f6f6c636861696e2d696e646578d92768747470733a2f2f6578616d706c652e696e76616c69642f746f6f6c636861696e2d696e646578d9477368613235362d64646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464a472757374b6696e707574732f746f6f6c636861696e2d696e646578117892d9477368613235362d35663738353165373634333366383261376562666431626232646234323065366263333263646263346536666538373563356633326461363434643535656237bd2f7372762f736572766963652f72656c65617365732f63757272656e74dc0012bd6964756e6e2e65787065637465645f696e6361726e6174696f6e2e7632a773657276696365d9477368613235362d64396364646465626366636362326366316261396164366239323566336666616237646561636332366661653631646138663236353861336139343039656232b5736572766963652d696e6361726e6174696f6e2d31d9477368613235362d35663738353165373634333366383261376562666431626232646234323065366263333263646263346536666538373563356633326461363434643535656237bb6769746875622e636f6d2f47616d6543756c742f53657276696365d92832323232323232323232323232323232323232323232323232323232323232323232323232323232d9477368613235362d35396263633432376637393934303563613461353132623238613830363734343536666133313439373233363132383065613062633231636335653266363533b1736572766963652d79676764726173696cb6736572766963652d72756e74696d652d7369676e6572ae736572766963652e6865616c7468d9477368613235362d63636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363a27631d9477368613235362d32663837303134316366353638316566656332653234316364636435663731303831616539326162316535633534633839636438393666386335656236323731c294a773657276696365a468747470b6687474703a2f2f3132372e302e302e313a3137393939b6687474703a2f2f3132372e302e302e313a31383030319194af736572766963652e72756e74696d65b2736572766963652e72756e74696d652e7631a2763101919ab57368617265642d696e667261737472756374757265b56f64696e2e76657273652d72656e64657a766f7573b66f64696e2e76657273652d746f706f6c6f67792e7631a2763101b06265666f72652d70726f6d6f74696f6ea46f64696eb36d616e616765642d696e6361726e6174696f6ed9477368613235362d63343263333331373863366539666534633435313162623563663962626461376431633138356536646330373064323234653463653431303538626338633238b57463703a2f2f31302e37372e302e313a3137383731d9477368613235362d39613662663530353466616430306232343363323331386330333436303931333230343665643436356231356534636561666532613736623966663231313039c0cf0000018bcfe568019abb6964756e6e2e72756e74696d655f61637469766174696f6e2e7632d9477368613235362d39613662663530353466616430306232343363323331386330333436303931333230343665643436356231356534636561666532613736623966663231313039b1736572766963652d79676764726173696cd9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262d94032363535323862353235386435353563356434646533383931303830363164383965663439616336306631373337626464656337316535333335643163623935c420c06afef8f66e6ec88c96c81ad6d74dbd9933738032f4d8069a46412380e2272dcf0000018bcfe567ecd94066346536393735326334333432363965323262306130346137623430613065336235386364343561383238376364303936383263313864326565366539376239a765643235353139c44046048cf77cc9f524da966e16a131edb34180f4ea8420f5f24a64fd53178e44988bef994d6df103a5f117470283d533e9d7e8245c4f9965f9a4c0a0114ed1c201dc0033b26964756e6e2d343234322e73657276696365af4964756e6e20746573742034323432af696e766f636174696f6e2d3432343201a465786563a6616c77617973ad636f6e74726f6c2d67726f7570c3a57534323432a57534323432a0a0a0c3c3a9696e76697369626c65a3706964c3a430303737a0a0cd10920194cd1092cd1092cd1092cd109294cd1092cd1092cd1092cd109291cd10920000000000c391010807b52f6f70742f746573742f62696e2f736572766963650101d9477368613235362d34626635313232663334343535346335336264653265626238636432623765336431363030616436333163333835613564376363653233633737383534353961d9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262a92f6f70742f74657374a92f72756e2f74657374d9477368613235362d6462633162346339303066666534386435373562356461356336333830343031323566363564623066653365323434393462373665613938363435376439383690d9477368613235362d30383466656430386239373861663464376431393661373434366138366235383030396536333662363131646231363231316236356139616164666632396335d9202f73797374656d2e736c6963652f6964756e6e2d343234322e73657276696365c090aa61637469766174696f6edc0020010101010101010101010101010101010101010101010101010101010101010190d9477368613235362d39623734636534306561326663636337623632613533363838363563653533303438316130643838653931336161396330313333666662633838353763313838959141d9477368613235362d35353961656164303832363464353739356433393039373138636464303561626434393537326538346665353535393065656633316138386130386664666664ab6f64696e2d7369676e6572296492ad6f64696e2d746f706f6c6f6779959139d9477368613235362d31393538316532376465376365643030666631636535306232303437653761353637633736623163626165626162653565663033663763333031376262356237ab6f64696e2d7369676e6572276496a773657276696365d9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262d9477368613235362d35643838303630666232646530386139663366346338313936623062633863663737653934333732373363346135386330653430303737666630316461306534c0c0c096cd10920708c0c0c091b1736b69707065642d73746174656c657373c0c0c029c092b46661696c65642d61667465722d66656e63696e67bc63616e646964617465206e6576657220626563616d65207265616479c0c096bc63616e646964617465206e6576657220626563616d65207265616479a8636f6d706c657465a7736b6970706564a8636f6d706c657465a7736b6970706564a7736b6970706564 diff --git a/tests/fixtures/idunn-control-legacy/transaction-fencing.hex b/tests/fixtures/idunn-control-legacy/transaction-fencing.hex new file mode 100644 index 0000000..58d4765 --- /dev/null +++ b/tests/fixtures/idunn-control-legacy/transaction-fencing.hex @@ -0,0 +1,2 @@ +idunn.deployment_transaction.v3 +dc0023bf6964756e6e2e6465706c6f796d656e745f7472616e73616374696f6e2e7633d92774782d37636236616537392d323635302d346261322d623664392d643137333338643566393031af636f6e74696e756974792d74657374aa636f6e74696e75697479a77365727669636500a766656e63696e67cf0000018bcfe56800cf0000018bcfe6eea0c099d9216964756e6e2e636f6d70696c65645f6465706c6f796d656e745f706c616e2e7632d9477368613235362d64396364646465626366636362326366316261396164366239323566336666616237646561636332366661653631646138663236353861336139343039656232b5736572766963652d696e6361726e6174696f6e2d316e9bbf6964756e6e2e736f757263655f73656c656374696f6e5f66616374732e7631d92768747470733a2f2f6769746875622e636f6d2f47616d6543756c742f536572766963652e676974af726566732f68656164732f6d61696ed92832323232323232323232323232323232323232323232323232323232323232323232323232323232d92832323232323232323232323232323232323232323232323232323232323232323232323232323232d92833333333333333333333333333333333333333333333333333333333333333333333333333333333bc6465706c6f796d656e742f6964756e6e2f7265636970652e746f6d6cd9477368613235362d353962636334323766373939343035636134613531326232386138303637343435366661333134393732333631323830656130626332316363356532663635338091a87265662d6865616464dc058e0a736368656d61203d202267616d6563756c742e6964756e6e2e7461726765745f6465636c61726174696f6e2e7631220a746172676574203d202273657276696365220a736f757263655f7374616d705f656e7669726f6e6d656e74203d2022534552564943455f4255494c445f434f4d4d4954220a0a5b5b73746570735d5d0a6964203d20226275696c64220a7068617365203d20226275696c64220a72756e6e6572203d202272757374220a61726776203d205b22636172676f222c20226275696c64222c20222d2d6c6f636b6564225d0a0a5b5b65787465726e616c5f696e707574735d5d0a6964203d2022746f6f6c636861696e2d696e646578220a75726c203d202268747470733a2f2f6578616d706c652e696e76616c69642f746f6f6c636861696e2d696e646578220a736861323536203d202264646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464220a72756e6e6572203d202272757374220a64657374696e6174696f6e203d2022696e707574732f746f6f6c636861696e2d696e646578220a0a5b5b6172746966616374735d5d0a6964203d20226461656d6f6e220a736f757263655f6b696e64203d202272756e6e65722d6f7574707574220a72756e6e6572203d202272757374220a736f75726365203d20227461726765742f72656c656173652f73657276696365220a64657374696e6174696f6e203d202273657276696365220a65787065637465645f736861323536203d202263636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363220a65786563757461626c65203d20747275650a0a5b736572766963655d0a65786563757461626c655f6172746966616374203d20226461656d6f6e220a617267756d656e7473203d205b0a20207b206b696e64203d202262696e64696e67222c206e616d65203d202273746174655f726f6f7422207d2c0a5d0a7472616e73706f7274203d202268747470220a726f7574655f7265717569726564203d20747275650a72657175697265645f656e7669726f6e6d656e74203d205b2247414d4543554c545f4944554e4e5f43414e4449444154455f42494e44222c202247414d4543554c545f4944554e4e5f52554e54494d455f42554e444c45225d0a0a5b736572766963652e6865616c74685d0a636f6e7472616374203d2022736572766963652e6865616c7468220a0a5b73746174655d0a736368656d615f67656e65726174696f6e203d20227631220a0a5b5b73746174652e736c6f74735d5d0a6964203d202272756e74696d652d6361636865220a72656c61746976655f70617468203d202272756e74696d652d63616368652e6363220a6b696e64203d202263756c7463616368652d66696c65220a736368656d61203d2022736572766963652e72756e74696d652d63616368652e7631220a777269746572203d20226e6f6e65220a7265636f76657279203d202272656275696c6461626c65220a73746172747570203d20226f70656e2d61742d7374617274220a0a5b5b70726f76696465735d5d0a6361706162696c697479203d2022736572766963652e72756e74696d65220a736368656d61203d2022736572766963652e72756e74696d652e7631220a636f6d7061746962696c697479203d20227631220a0a5b5b646570656e64656e636965735d5d0a6b696e64203d20227368617265642d696e667261737472756374757265220a6361706162696c697479203d20226f64696e2e76657273652d72656e64657a766f7573220a736368656d61203d20226f64696e2e76657273652d746f706f6c6f67792e7631220a636f6d7061746962696c697479203d20227631220adc076e0a736368656d61203d202267616d6563756c742e6964756e6e2e6f70657261746f725f62696e64696e672e7632220a746172676574203d202273657276696365220a0a5b7265706f7369746f72795d0a6f726967696e203d202268747470733a2f2f6769746875622e636f6d2f47616d6543756c742f536572766963652e676974220a61646d69747465645f726566203d2022726566732f68656164732f6d61696e220a6d696e696d756d5f7265766973696f6e203d202231313131313131313131313131313131313131313131313131313131313131313131313131313131220a73656c656374696f6e203d20227265662d68656164220a636865636b6f7574203d20222f7372762f6275696c642f53657276696365220a7265636970655f70617468203d20226465706c6f796d656e742f6964756e6e2f7265636970652e746f6d6c220a0a5b72756e6e6572732e727573745d0a647269766572203d2022646f636b6572220a696d616765203d202272757374407368613235363a61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161220a75736572203d2022313030303a31303030220a6166666f7264616e636573203d205b22736f757263652d72656164222c202261727469666163742d7772697465225d0a616c6c6f7765645f70726f6772616d73203d205b22636172676f225d0a6e6574776f726b5f70726f66696c65203d20226275696c642d646570656e64656e63792d656772657373220a6d656d6f72795f6d6562696279746573203d20323034380a6370755f71756f74615f70657263656e74203d203230300a706964735f6c696d6974203d203531320a746d7066735f6d6562696279746573203d203531320a0a5b776f726b6c6f61645d0a647269766572203d202273797374656d642d7472616e7369656e74220a73746174655f67726f7570203d202273657276696365220a756e69745f707265666978203d20226964756e6e2d73657276696365220a72656c656173655f726f6f74203d20222f7372762f736572766963652f72656c6561736573220a73746174655f726f6f74203d20222f7661722f6c69622f67616d6563756c742f73657276696365220a72756e74696d655f726f6f74203d20222f72756e2f67616d6563756c742f6964756e6e2f73657276696365220a6e6574776f726b203d2022686f73742d70726976617465220a68617264656e696e67203d2022737472696374220a6d656d6f72795f6d6562696279746573203d20313032340a6370755f71756f74615f70657263656e74203d203130300a0a5b776f726b6c6f61642e617267756d656e745f62696e64696e67735d0a73746174655f726f6f74203d20222f7661722f6c69622f67616d6563756c742f73657276696365220a0a5b776f726b6c6f61642e7365637265745f66696c65735d0a47414d4543554c545f52554e54494d455f50524553454e43455f4944454e54495459203d20222f6574632f67616d6563756c742f736572766963652f72756e74696d652d70726573656e63652d6964656e746974792e6363220a0a5b72756e74696d655f6964656e746974795d0a72756e74696d655f6964203d2022736572766963652d79676764726173696c220a65787065637465645f7369676e65725f6964656e746974795f6964203d2022736572766963652d72756e74696d652d7369676e6572220a74727573745f616e63686f725f73746f7265203d20222f6574632f67616d6563756c742f74727573742f736572766963652e6363220a0a5b726f7574655d0a647269766572203d20226e67696e782d73747265616d2d746370220a726f7574655f6964203d202273657276696365220a737461626c655f656e64706f696e74203d2022687474703a2f2f3132372e302e302e313a3137393939220a707269766174655f686f7374203d20223132372e302e302e31220a707269766174655f706f72745f7374617274203d2031383030300a707269766174655f706f72745f656e64203d2031383030390a636f6e6669675f70617468203d20222f6574632f6e67696e782f6964756e6e2d73747265616d2d726f757465732f736572766963652e636f6e66220a72656c6f61645f756e6974203d20226e67696e782e73657276696365220a0a5b6272616b65735d0a6465706c6f796d656e745f73746f7265203d20222f7661722f6c69622f67616d6563756c742f6964756e6e2f736572766963652d6465706c6f796d656e742d6272616b652e6363220a6c6966656379636c655f73746f7265203d20222f7661722f6c69622f67616d6563756c742f6964756e6e2f736572766963652d6c6966656379636c652d6272616b652e6363220a0a5b726f6c6c6f75745d0a7374726174656779203d202263616e6469646174652d7468656e2d70726f6d6f7465220a647261696e5f7365636f6e6473203d2033300a72657461696e5f72656c6561736573203d20320a0a5b706c6163656d656e745d0a646573697265645f7265706c69636173203d20310a6e6f646573203d205b2279676764726173696c225d0a919296b57368617265642d696e667261737472756374757265b56f64696e2e76657273652d72656e64657a766f7573b66f64696e2e76657273652d746f706f6c6f67792e7631a2763101b06265666f72652d70726f6d6f74696f6e97a46f64696e98ad6d616e616765642d7265616479a46f64696eb26f64696e2d696e6361726e6174696f6e2d31d9477368613235362d61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161d9477368613235362d65656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565d9477368613235362d63343263333331373863366539666534633435313162623563663962626461376431633138356536646330373064323234653463653431303538626338633238d9477368613235362d6137313730323730313265383236396235666636363961313135386466636236313764346463636230663963333633656631373738656561633230353332633907b56f64696e2e76657273652d72656e64657a766f7573b66f64696e2e76657273652d746f706f6c6f67792e7631a2763101b57463703a2f2f31302e37372e302e313a3137383731cd4651c096b76964756e6e2e7365616c65645f72656c656173652e7631d9477368613235362d35663738353165373634333366383261376562666431626232646234323065366263333263646263346536666538373563356633326461363434643535656237d9477368613235362d643963646464656263666363623263663162613961643662393235663366666162376465616363323666616536316461386632363538613361393430396562329195a66461656d6f6ea773657276696365d9477368613235362d636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363632ac39196af746f6f6c636861696e2d696e646578d92768747470733a2f2f6578616d706c652e696e76616c69642f746f6f6c636861696e2d696e646578d9477368613235362d64646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464a472757374b6696e707574732f746f6f6c636861696e2d696e646578117892d9477368613235362d35663738353165373634333366383261376562666431626232646234323065366263333263646263346536666538373563356633326461363434643535656237bd2f7372762f736572766963652f72656c65617365732f63757272656e74dc0012bd6964756e6e2e65787065637465645f696e6361726e6174696f6e2e7632a773657276696365d9477368613235362d64396364646465626366636362326366316261396164366239323566336666616237646561636332366661653631646138663236353861336139343039656232b5736572766963652d696e6361726e6174696f6e2d31d9477368613235362d35663738353165373634333366383261376562666431626232646234323065366263333263646263346536666538373563356633326461363434643535656237bb6769746875622e636f6d2f47616d6543756c742f53657276696365d92832323232323232323232323232323232323232323232323232323232323232323232323232323232d9477368613235362d35396263633432376637393934303563613461353132623238613830363734343536666133313439373233363132383065613062633231636335653266363533b1736572766963652d79676764726173696cb6736572766963652d72756e74696d652d7369676e6572ae736572766963652e6865616c7468d9477368613235362d63636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363a27631d9477368613235362d32663837303134316366353638316566656332653234316364636435663731303831616539326162316535633534633839636438393666386335656236323731c294a773657276696365a468747470b6687474703a2f2f3132372e302e302e313a3137393939b6687474703a2f2f3132372e302e302e313a31383030319194af736572766963652e72756e74696d65b2736572766963652e72756e74696d652e7631a2763101919ab57368617265642d696e667261737472756374757265b56f64696e2e76657273652d72656e64657a766f7573b66f64696e2e76657273652d746f706f6c6f67792e7631a2763101b06265666f72652d70726f6d6f74696f6ea46f64696eb36d616e616765642d696e6361726e6174696f6ed9477368613235362d63343263333331373863366539666534633435313162623563663962626461376431633138356536646330373064323234653463653431303538626338633238b57463703a2f2f31302e37372e302e313a3137383731d9477368613235362d39613662663530353466616430306232343363323331386330333436303931333230343665643436356231356534636561666532613736623966663231313039c0cf0000018bcfe568019abb6964756e6e2e72756e74696d655f61637469766174696f6e2e7632d9477368613235362d39613662663530353466616430306232343363323331386330333436303931333230343665643436356231356534636561666532613736623966663231313039b1736572766963652d79676764726173696cd9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262d94032363535323862353235386435353563356434646533383931303830363164383965663439616336306631373337626464656337316535333335643163623935c420c06afef8f66e6ec88c96c81ad6d74dbd9933738032f4d8069a46412380e2272dcf0000018bcfe567ecd94066346536393735326334333432363965323262306130346137623430613065336235386364343561383238376364303936383263313864326565366539376239a765643235353139c44046048cf77cc9f524da966e16a131edb34180f4ea8420f5f24a64fd53178e44988bef994d6df103a5f117470283d533e9d7e8245c4f9965f9a4c0a0114ed1c201dc0033b26964756e6e2d343234322e73657276696365af4964756e6e20746573742034323432af696e766f636174696f6e2d3432343201a465786563a6616c77617973ad636f6e74726f6c2d67726f7570c3a57534323432a57534323432a0a0a0c3c3a9696e76697369626c65a3706964c3a430303737a0a0cd10920194cd1092cd1092cd1092cd109294cd1092cd1092cd1092cd109291cd10920000000000c391010807b52f6f70742f746573742f62696e2f736572766963650101d9477368613235362d34626635313232663334343535346335336264653265626238636432623765336431363030616436333163333835613564376363653233633737383534353961d9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262a92f6f70742f74657374a92f72756e2f74657374d9477368613235362d6462633162346339303066666534386435373562356461356336333830343031323566363564623066653365323434393462373665613938363435376439383690d9477368613235362d30383466656430386239373861663464376431393661373434366138366235383030396536333662363131646231363231316236356139616164666632396335d9202f73797374656d2e736c6963652f6964756e6e2d343234322e73657276696365c090aa61637469766174696f6edc0020010101010101010101010101010101010101010101010101010101010101010190d9477368613235362d39623734636534306561326663636337623632613533363838363563653533303438316130643838653931336161396330313333666662633838353763313838959141d9477368613235362d35353961656164303832363464353739356433393039373138636464303561626434393537326538346665353535393065656633316138386130386664666664ab6f64696e2d7369676e6572296492ad6f64696e2d746f706f6c6f6779959139d9477368613235362d31393538316532376465376365643030666631636535306232303437653761353637633736623163626165626162653565663033663763333031376262356237ab6f64696e2d7369676e6572276496a773657276696365d9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262d9477368613235362d35643838303630666232646530386139663366346338313936623062633863663737653934333732373363346135386330653430303737666630316461306534c0c0c096cd10920708c0c0c0c0c0c0c029c0c0c0c0c0 diff --git a/tests/fixtures/idunn-control-legacy/transaction-leasing.hex b/tests/fixtures/idunn-control-legacy/transaction-leasing.hex new file mode 100644 index 0000000..2c3bd54 --- /dev/null +++ b/tests/fixtures/idunn-control-legacy/transaction-leasing.hex @@ -0,0 +1,2 @@ +idunn.deployment_transaction.v3 +dc0023bf6964756e6e2e6465706c6f796d656e745f7472616e73616374696f6e2e7633d92774782d37636236616537392d323635302d346261322d623664392d643137333338643566393031af636f6e74696e756974792d74657374aa636f6e74696e75697479a77365727669636500a76c656173696e67cf0000018bcfe56800cf0000018bcfe6eea0c099d9216964756e6e2e636f6d70696c65645f6465706c6f796d656e745f706c616e2e7632d9477368613235362d64396364646465626366636362326366316261396164366239323566336666616237646561636332366661653631646138663236353861336139343039656232b5736572766963652d696e6361726e6174696f6e2d316e9bbf6964756e6e2e736f757263655f73656c656374696f6e5f66616374732e7631d92768747470733a2f2f6769746875622e636f6d2f47616d6543756c742f536572766963652e676974af726566732f68656164732f6d61696ed92832323232323232323232323232323232323232323232323232323232323232323232323232323232d92832323232323232323232323232323232323232323232323232323232323232323232323232323232d92833333333333333333333333333333333333333333333333333333333333333333333333333333333bc6465706c6f796d656e742f6964756e6e2f7265636970652e746f6d6cd9477368613235362d353962636334323766373939343035636134613531326232386138303637343435366661333134393732333631323830656130626332316363356532663635338091a87265662d6865616464dc058e0a736368656d61203d202267616d6563756c742e6964756e6e2e7461726765745f6465636c61726174696f6e2e7631220a746172676574203d202273657276696365220a736f757263655f7374616d705f656e7669726f6e6d656e74203d2022534552564943455f4255494c445f434f4d4d4954220a0a5b5b73746570735d5d0a6964203d20226275696c64220a7068617365203d20226275696c64220a72756e6e6572203d202272757374220a61726776203d205b22636172676f222c20226275696c64222c20222d2d6c6f636b6564225d0a0a5b5b65787465726e616c5f696e707574735d5d0a6964203d2022746f6f6c636861696e2d696e646578220a75726c203d202268747470733a2f2f6578616d706c652e696e76616c69642f746f6f6c636861696e2d696e646578220a736861323536203d202264646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464220a72756e6e6572203d202272757374220a64657374696e6174696f6e203d2022696e707574732f746f6f6c636861696e2d696e646578220a0a5b5b6172746966616374735d5d0a6964203d20226461656d6f6e220a736f757263655f6b696e64203d202272756e6e65722d6f7574707574220a72756e6e6572203d202272757374220a736f75726365203d20227461726765742f72656c656173652f73657276696365220a64657374696e6174696f6e203d202273657276696365220a65787065637465645f736861323536203d202263636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363220a65786563757461626c65203d20747275650a0a5b736572766963655d0a65786563757461626c655f6172746966616374203d20226461656d6f6e220a617267756d656e7473203d205b0a20207b206b696e64203d202262696e64696e67222c206e616d65203d202273746174655f726f6f7422207d2c0a5d0a7472616e73706f7274203d202268747470220a726f7574655f7265717569726564203d20747275650a72657175697265645f656e7669726f6e6d656e74203d205b2247414d4543554c545f4944554e4e5f43414e4449444154455f42494e44222c202247414d4543554c545f4944554e4e5f52554e54494d455f42554e444c45225d0a0a5b736572766963652e6865616c74685d0a636f6e7472616374203d2022736572766963652e6865616c7468220a0a5b73746174655d0a736368656d615f67656e65726174696f6e203d20227631220a0a5b5b73746174652e736c6f74735d5d0a6964203d202272756e74696d652d6361636865220a72656c61746976655f70617468203d202272756e74696d652d63616368652e6363220a6b696e64203d202263756c7463616368652d66696c65220a736368656d61203d2022736572766963652e72756e74696d652d63616368652e7631220a777269746572203d20226e6f6e65220a7265636f76657279203d202272656275696c6461626c65220a73746172747570203d20226f70656e2d61742d7374617274220a0a5b5b70726f76696465735d5d0a6361706162696c697479203d2022736572766963652e72756e74696d65220a736368656d61203d2022736572766963652e72756e74696d652e7631220a636f6d7061746962696c697479203d20227631220a0a5b5b646570656e64656e636965735d5d0a6b696e64203d20227368617265642d696e667261737472756374757265220a6361706162696c697479203d20226f64696e2e76657273652d72656e64657a766f7573220a736368656d61203d20226f64696e2e76657273652d746f706f6c6f67792e7631220a636f6d7061746962696c697479203d20227631220adc076e0a736368656d61203d202267616d6563756c742e6964756e6e2e6f70657261746f725f62696e64696e672e7632220a746172676574203d202273657276696365220a0a5b7265706f7369746f72795d0a6f726967696e203d202268747470733a2f2f6769746875622e636f6d2f47616d6543756c742f536572766963652e676974220a61646d69747465645f726566203d2022726566732f68656164732f6d61696e220a6d696e696d756d5f7265766973696f6e203d202231313131313131313131313131313131313131313131313131313131313131313131313131313131220a73656c656374696f6e203d20227265662d68656164220a636865636b6f7574203d20222f7372762f6275696c642f53657276696365220a7265636970655f70617468203d20226465706c6f796d656e742f6964756e6e2f7265636970652e746f6d6c220a0a5b72756e6e6572732e727573745d0a647269766572203d2022646f636b6572220a696d616765203d202272757374407368613235363a61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161220a75736572203d2022313030303a31303030220a6166666f7264616e636573203d205b22736f757263652d72656164222c202261727469666163742d7772697465225d0a616c6c6f7765645f70726f6772616d73203d205b22636172676f225d0a6e6574776f726b5f70726f66696c65203d20226275696c642d646570656e64656e63792d656772657373220a6d656d6f72795f6d6562696279746573203d20323034380a6370755f71756f74615f70657263656e74203d203230300a706964735f6c696d6974203d203531320a746d7066735f6d6562696279746573203d203531320a0a5b776f726b6c6f61645d0a647269766572203d202273797374656d642d7472616e7369656e74220a73746174655f67726f7570203d202273657276696365220a756e69745f707265666978203d20226964756e6e2d73657276696365220a72656c656173655f726f6f74203d20222f7372762f736572766963652f72656c6561736573220a73746174655f726f6f74203d20222f7661722f6c69622f67616d6563756c742f73657276696365220a72756e74696d655f726f6f74203d20222f72756e2f67616d6563756c742f6964756e6e2f73657276696365220a6e6574776f726b203d2022686f73742d70726976617465220a68617264656e696e67203d2022737472696374220a6d656d6f72795f6d6562696279746573203d20313032340a6370755f71756f74615f70657263656e74203d203130300a0a5b776f726b6c6f61642e617267756d656e745f62696e64696e67735d0a73746174655f726f6f74203d20222f7661722f6c69622f67616d6563756c742f73657276696365220a0a5b776f726b6c6f61642e7365637265745f66696c65735d0a47414d4543554c545f52554e54494d455f50524553454e43455f4944454e54495459203d20222f6574632f67616d6563756c742f736572766963652f72756e74696d652d70726573656e63652d6964656e746974792e6363220a0a5b72756e74696d655f6964656e746974795d0a72756e74696d655f6964203d2022736572766963652d79676764726173696c220a65787065637465645f7369676e65725f6964656e746974795f6964203d2022736572766963652d72756e74696d652d7369676e6572220a74727573745f616e63686f725f73746f7265203d20222f6574632f67616d6563756c742f74727573742f736572766963652e6363220a0a5b726f7574655d0a647269766572203d20226e67696e782d73747265616d2d746370220a726f7574655f6964203d202273657276696365220a737461626c655f656e64706f696e74203d2022687474703a2f2f3132372e302e302e313a3137393939220a707269766174655f686f7374203d20223132372e302e302e31220a707269766174655f706f72745f7374617274203d2031383030300a707269766174655f706f72745f656e64203d2031383030390a636f6e6669675f70617468203d20222f6574632f6e67696e782f6964756e6e2d73747265616d2d726f757465732f736572766963652e636f6e66220a72656c6f61645f756e6974203d20226e67696e782e73657276696365220a0a5b6272616b65735d0a6465706c6f796d656e745f73746f7265203d20222f7661722f6c69622f67616d6563756c742f6964756e6e2f736572766963652d6465706c6f796d656e742d6272616b652e6363220a6c6966656379636c655f73746f7265203d20222f7661722f6c69622f67616d6563756c742f6964756e6e2f736572766963652d6c6966656379636c652d6272616b652e6363220a0a5b726f6c6c6f75745d0a7374726174656779203d202263616e6469646174652d7468656e2d70726f6d6f7465220a647261696e5f7365636f6e6473203d2033300a72657461696e5f72656c6561736573203d20320a0a5b706c6163656d656e745d0a646573697265645f7265706c69636173203d20310a6e6f646573203d205b2279676764726173696c225d0a919296b57368617265642d696e667261737472756374757265b56f64696e2e76657273652d72656e64657a766f7573b66f64696e2e76657273652d746f706f6c6f67792e7631a2763101b06265666f72652d70726f6d6f74696f6e97a46f64696e98ad6d616e616765642d7265616479a46f64696eb26f64696e2d696e6361726e6174696f6e2d31d9477368613235362d61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161d9477368613235362d65656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565d9477368613235362d63343263333331373863366539666534633435313162623563663962626461376431633138356536646330373064323234653463653431303538626338633238d9477368613235362d6137313730323730313265383236396235666636363961313135386466636236313764346463636230663963333633656631373738656561633230353332633907b56f64696e2e76657273652d72656e64657a766f7573b66f64696e2e76657273652d746f706f6c6f67792e7631a2763101b57463703a2f2f31302e37372e302e313a3137383731cd4651c096b76964756e6e2e7365616c65645f72656c656173652e7631d9477368613235362d35663738353165373634333366383261376562666431626232646234323065366263333263646263346536666538373563356633326461363434643535656237d9477368613235362d643963646464656263666363623263663162613961643662393235663366666162376465616363323666616536316461386632363538613361393430396562329195a66461656d6f6ea773657276696365d9477368613235362d636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363632ac39196af746f6f6c636861696e2d696e646578d92768747470733a2f2f6578616d706c652e696e76616c69642f746f6f6c636861696e2d696e646578d9477368613235362d64646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464a472757374b6696e707574732f746f6f6c636861696e2d696e646578117892d9477368613235362d35663738353165373634333366383261376562666431626232646234323065366263333263646263346536666538373563356633326461363434643535656237bd2f7372762f736572766963652f72656c65617365732f63757272656e74dc0012bd6964756e6e2e65787065637465645f696e6361726e6174696f6e2e7632a773657276696365d9477368613235362d64396364646465626366636362326366316261396164366239323566336666616237646561636332366661653631646138663236353861336139343039656232b5736572766963652d696e6361726e6174696f6e2d31d9477368613235362d35663738353165373634333366383261376562666431626232646234323065366263333263646263346536666538373563356633326461363434643535656237bb6769746875622e636f6d2f47616d6543756c742f53657276696365d92832323232323232323232323232323232323232323232323232323232323232323232323232323232d9477368613235362d35396263633432376637393934303563613461353132623238613830363734343536666133313439373233363132383065613062633231636335653266363533b1736572766963652d79676764726173696cb6736572766963652d72756e74696d652d7369676e6572ae736572766963652e6865616c7468d9477368613235362d63636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363a27631d9477368613235362d32663837303134316366353638316566656332653234316364636435663731303831616539326162316535633534633839636438393666386335656236323731c294a773657276696365a468747470b6687474703a2f2f3132372e302e302e313a3137393939b6687474703a2f2f3132372e302e302e313a31383030319194af736572766963652e72756e74696d65b2736572766963652e72756e74696d652e7631a2763101919ab57368617265642d696e667261737472756374757265b56f64696e2e76657273652d72656e64657a766f7573b66f64696e2e76657273652d746f706f6c6f67792e7631a2763101b06265666f72652d70726f6d6f74696f6ea46f64696eb36d616e616765642d696e6361726e6174696f6ed9477368613235362d63343263333331373863366539666534633435313162623563663962626461376431633138356536646330373064323234653463653431303538626338633238b57463703a2f2f31302e37372e302e313a3137383731d9477368613235362d39613662663530353466616430306232343363323331386330333436303931333230343665643436356231356534636561666532613736623966663231313039c0cf0000018bcfe568019abb6964756e6e2e72756e74696d655f61637469766174696f6e2e7632d9477368613235362d39613662663530353466616430306232343363323331386330333436303931333230343665643436356231356534636561666532613736623966663231313039b1736572766963652d79676764726173696cd9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262d94032363535323862353235386435353563356434646533383931303830363164383965663439616336306631373337626464656337316535333335643163623935c420c06afef8f66e6ec88c96c81ad6d74dbd9933738032f4d8069a46412380e2272dcf0000018bcfe567ecd94066346536393735326334333432363965323262306130346137623430613065336235386364343561383238376364303936383263313864326565366539376239a765643235353139c44046048cf77cc9f524da966e16a131edb34180f4ea8420f5f24a64fd53178e44988bef994d6df103a5f117470283d533e9d7e8245c4f9965f9a4c0a0114ed1c201dc0033b26964756e6e2d343234322e73657276696365af4964756e6e20746573742034323432af696e766f636174696f6e2d3432343201a465786563a6616c77617973ad636f6e74726f6c2d67726f7570c3a57534323432a57534323432a0a0a0c3c3a9696e76697369626c65a3706964c3a430303737a0a0cd10920194cd1092cd1092cd1092cd109294cd1092cd1092cd1092cd109291cd10920000000000c391010807b52f6f70742f746573742f62696e2f736572766963650101d9477368613235362d34626635313232663334343535346335336264653265626238636432623765336431363030616436333163333835613564376363653233633737383534353961d9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262a92f6f70742f74657374a92f72756e2f74657374d9477368613235362d6462633162346339303066666534386435373562356461356336333830343031323566363564623066653365323434393462373665613938363435376439383690d9477368613235362d30383466656430386239373861663464376431393661373434366138366235383030396536333662363131646231363231316236356139616164666632396335d9202f73797374656d2e736c6963652f6964756e6e2d343234322e73657276696365c090aa61637469766174696f6edc0020010101010101010101010101010101010101010101010101010101010101010190d9477368613235362d39623734636534306561326663636337623632613533363838363563653533303438316130643838653931336161396330313333666662633838353763313838959141d9477368613235362d35353961656164303832363464353739356433393039373138636464303561626434393537326538346665353535393065656633316138386130386664666664ab6f64696e2d7369676e6572296492ad6f64696e2d746f706f6c6f6779959139d9477368613235362d31393538316532376465376365643030666631636535306232303437653761353637633736623163626165626162653565663033663763333031376262356237ab6f64696e2d7369676e6572276496a773657276696365d9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262d9477368613235362d35643838303630666232646530386139663366346338313936623062633863663737653934333732373363346135386330653430303737666630316461306534c0c0c096cd10920708c0c0c091b1736b69707065642d73746174656c657373c0c0c029c0c0c0c0c0 diff --git a/tests/fixtures/idunn-control-legacy/transaction-routing.hex b/tests/fixtures/idunn-control-legacy/transaction-routing.hex new file mode 100644 index 0000000..3245308 --- /dev/null +++ b/tests/fixtures/idunn-control-legacy/transaction-routing.hex @@ -0,0 +1,2 @@ +idunn.deployment_transaction.v3 +dc0023bf6964756e6e2e6465706c6f796d656e745f7472616e73616374696f6e2e7633d92774782d37636236616537392d323635302d346261322d623664392d643137333338643566393031af636f6e74696e756974792d74657374aa636f6e74696e75697479a77365727669636500a7726f7574696e67cf0000018bcfe56800cf0000018bcfe6eea0c099d9216964756e6e2e636f6d70696c65645f6465706c6f796d656e745f706c616e2e7632d9477368613235362d64396364646465626366636362326366316261396164366239323566336666616237646561636332366661653631646138663236353861336139343039656232b5736572766963652d696e6361726e6174696f6e2d316e9bbf6964756e6e2e736f757263655f73656c656374696f6e5f66616374732e7631d92768747470733a2f2f6769746875622e636f6d2f47616d6543756c742f536572766963652e676974af726566732f68656164732f6d61696ed92832323232323232323232323232323232323232323232323232323232323232323232323232323232d92832323232323232323232323232323232323232323232323232323232323232323232323232323232d92833333333333333333333333333333333333333333333333333333333333333333333333333333333bc6465706c6f796d656e742f6964756e6e2f7265636970652e746f6d6cd9477368613235362d353962636334323766373939343035636134613531326232386138303637343435366661333134393732333631323830656130626332316363356532663635338091a87265662d6865616464dc058e0a736368656d61203d202267616d6563756c742e6964756e6e2e7461726765745f6465636c61726174696f6e2e7631220a746172676574203d202273657276696365220a736f757263655f7374616d705f656e7669726f6e6d656e74203d2022534552564943455f4255494c445f434f4d4d4954220a0a5b5b73746570735d5d0a6964203d20226275696c64220a7068617365203d20226275696c64220a72756e6e6572203d202272757374220a61726776203d205b22636172676f222c20226275696c64222c20222d2d6c6f636b6564225d0a0a5b5b65787465726e616c5f696e707574735d5d0a6964203d2022746f6f6c636861696e2d696e646578220a75726c203d202268747470733a2f2f6578616d706c652e696e76616c69642f746f6f6c636861696e2d696e646578220a736861323536203d202264646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464220a72756e6e6572203d202272757374220a64657374696e6174696f6e203d2022696e707574732f746f6f6c636861696e2d696e646578220a0a5b5b6172746966616374735d5d0a6964203d20226461656d6f6e220a736f757263655f6b696e64203d202272756e6e65722d6f7574707574220a72756e6e6572203d202272757374220a736f75726365203d20227461726765742f72656c656173652f73657276696365220a64657374696e6174696f6e203d202273657276696365220a65787065637465645f736861323536203d202263636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363220a65786563757461626c65203d20747275650a0a5b736572766963655d0a65786563757461626c655f6172746966616374203d20226461656d6f6e220a617267756d656e7473203d205b0a20207b206b696e64203d202262696e64696e67222c206e616d65203d202273746174655f726f6f7422207d2c0a5d0a7472616e73706f7274203d202268747470220a726f7574655f7265717569726564203d20747275650a72657175697265645f656e7669726f6e6d656e74203d205b2247414d4543554c545f4944554e4e5f43414e4449444154455f42494e44222c202247414d4543554c545f4944554e4e5f52554e54494d455f42554e444c45225d0a0a5b736572766963652e6865616c74685d0a636f6e7472616374203d2022736572766963652e6865616c7468220a0a5b73746174655d0a736368656d615f67656e65726174696f6e203d20227631220a0a5b5b73746174652e736c6f74735d5d0a6964203d202272756e74696d652d6361636865220a72656c61746976655f70617468203d202272756e74696d652d63616368652e6363220a6b696e64203d202263756c7463616368652d66696c65220a736368656d61203d2022736572766963652e72756e74696d652d63616368652e7631220a777269746572203d20226e6f6e65220a7265636f76657279203d202272656275696c6461626c65220a73746172747570203d20226f70656e2d61742d7374617274220a0a5b5b70726f76696465735d5d0a6361706162696c697479203d2022736572766963652e72756e74696d65220a736368656d61203d2022736572766963652e72756e74696d652e7631220a636f6d7061746962696c697479203d20227631220a0a5b5b646570656e64656e636965735d5d0a6b696e64203d20227368617265642d696e667261737472756374757265220a6361706162696c697479203d20226f64696e2e76657273652d72656e64657a766f7573220a736368656d61203d20226f64696e2e76657273652d746f706f6c6f67792e7631220a636f6d7061746962696c697479203d20227631220adc076e0a736368656d61203d202267616d6563756c742e6964756e6e2e6f70657261746f725f62696e64696e672e7632220a746172676574203d202273657276696365220a0a5b7265706f7369746f72795d0a6f726967696e203d202268747470733a2f2f6769746875622e636f6d2f47616d6543756c742f536572766963652e676974220a61646d69747465645f726566203d2022726566732f68656164732f6d61696e220a6d696e696d756d5f7265766973696f6e203d202231313131313131313131313131313131313131313131313131313131313131313131313131313131220a73656c656374696f6e203d20227265662d68656164220a636865636b6f7574203d20222f7372762f6275696c642f53657276696365220a7265636970655f70617468203d20226465706c6f796d656e742f6964756e6e2f7265636970652e746f6d6c220a0a5b72756e6e6572732e727573745d0a647269766572203d2022646f636b6572220a696d616765203d202272757374407368613235363a61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161220a75736572203d2022313030303a31303030220a6166666f7264616e636573203d205b22736f757263652d72656164222c202261727469666163742d7772697465225d0a616c6c6f7765645f70726f6772616d73203d205b22636172676f225d0a6e6574776f726b5f70726f66696c65203d20226275696c642d646570656e64656e63792d656772657373220a6d656d6f72795f6d6562696279746573203d20323034380a6370755f71756f74615f70657263656e74203d203230300a706964735f6c696d6974203d203531320a746d7066735f6d6562696279746573203d203531320a0a5b776f726b6c6f61645d0a647269766572203d202273797374656d642d7472616e7369656e74220a73746174655f67726f7570203d202273657276696365220a756e69745f707265666978203d20226964756e6e2d73657276696365220a72656c656173655f726f6f74203d20222f7372762f736572766963652f72656c6561736573220a73746174655f726f6f74203d20222f7661722f6c69622f67616d6563756c742f73657276696365220a72756e74696d655f726f6f74203d20222f72756e2f67616d6563756c742f6964756e6e2f73657276696365220a6e6574776f726b203d2022686f73742d70726976617465220a68617264656e696e67203d2022737472696374220a6d656d6f72795f6d6562696279746573203d20313032340a6370755f71756f74615f70657263656e74203d203130300a0a5b776f726b6c6f61642e617267756d656e745f62696e64696e67735d0a73746174655f726f6f74203d20222f7661722f6c69622f67616d6563756c742f73657276696365220a0a5b776f726b6c6f61642e7365637265745f66696c65735d0a47414d4543554c545f52554e54494d455f50524553454e43455f4944454e54495459203d20222f6574632f67616d6563756c742f736572766963652f72756e74696d652d70726573656e63652d6964656e746974792e6363220a0a5b72756e74696d655f6964656e746974795d0a72756e74696d655f6964203d2022736572766963652d79676764726173696c220a65787065637465645f7369676e65725f6964656e746974795f6964203d2022736572766963652d72756e74696d652d7369676e6572220a74727573745f616e63686f725f73746f7265203d20222f6574632f67616d6563756c742f74727573742f736572766963652e6363220a0a5b726f7574655d0a647269766572203d20226e67696e782d73747265616d2d746370220a726f7574655f6964203d202273657276696365220a737461626c655f656e64706f696e74203d2022687474703a2f2f3132372e302e302e313a3137393939220a707269766174655f686f7374203d20223132372e302e302e31220a707269766174655f706f72745f7374617274203d2031383030300a707269766174655f706f72745f656e64203d2031383030390a636f6e6669675f70617468203d20222f6574632f6e67696e782f6964756e6e2d73747265616d2d726f757465732f736572766963652e636f6e66220a72656c6f61645f756e6974203d20226e67696e782e73657276696365220a0a5b6272616b65735d0a6465706c6f796d656e745f73746f7265203d20222f7661722f6c69622f67616d6563756c742f6964756e6e2f736572766963652d6465706c6f796d656e742d6272616b652e6363220a6c6966656379636c655f73746f7265203d20222f7661722f6c69622f67616d6563756c742f6964756e6e2f736572766963652d6c6966656379636c652d6272616b652e6363220a0a5b726f6c6c6f75745d0a7374726174656779203d202263616e6469646174652d7468656e2d70726f6d6f7465220a647261696e5f7365636f6e6473203d2033300a72657461696e5f72656c6561736573203d20320a0a5b706c6163656d656e745d0a646573697265645f7265706c69636173203d20310a6e6f646573203d205b2279676764726173696c225d0a919296b57368617265642d696e667261737472756374757265b56f64696e2e76657273652d72656e64657a766f7573b66f64696e2e76657273652d746f706f6c6f67792e7631a2763101b06265666f72652d70726f6d6f74696f6e97a46f64696e98ad6d616e616765642d7265616479a46f64696eb26f64696e2d696e6361726e6174696f6e2d31d9477368613235362d61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161d9477368613235362d65656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565d9477368613235362d63343263333331373863366539666534633435313162623563663962626461376431633138356536646330373064323234653463653431303538626338633238d9477368613235362d6137313730323730313265383236396235666636363961313135386466636236313764346463636230663963333633656631373738656561633230353332633907b56f64696e2e76657273652d72656e64657a766f7573b66f64696e2e76657273652d746f706f6c6f67792e7631a2763101b57463703a2f2f31302e37372e302e313a3137383731cd4651c096b76964756e6e2e7365616c65645f72656c656173652e7631d9477368613235362d35663738353165373634333366383261376562666431626232646234323065366263333263646263346536666538373563356633326461363434643535656237d9477368613235362d643963646464656263666363623263663162613961643662393235663366666162376465616363323666616536316461386632363538613361393430396562329195a66461656d6f6ea773657276696365d9477368613235362d636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363632ac39196af746f6f6c636861696e2d696e646578d92768747470733a2f2f6578616d706c652e696e76616c69642f746f6f6c636861696e2d696e646578d9477368613235362d64646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464a472757374b6696e707574732f746f6f6c636861696e2d696e646578117892d9477368613235362d35663738353165373634333366383261376562666431626232646234323065366263333263646263346536666538373563356633326461363434643535656237bd2f7372762f736572766963652f72656c65617365732f63757272656e74dc0012bd6964756e6e2e65787065637465645f696e6361726e6174696f6e2e7632a773657276696365d9477368613235362d64396364646465626366636362326366316261396164366239323566336666616237646561636332366661653631646138663236353861336139343039656232b5736572766963652d696e6361726e6174696f6e2d31d9477368613235362d35663738353165373634333366383261376562666431626232646234323065366263333263646263346536666538373563356633326461363434643535656237bb6769746875622e636f6d2f47616d6543756c742f53657276696365d92832323232323232323232323232323232323232323232323232323232323232323232323232323232d9477368613235362d35396263633432376637393934303563613461353132623238613830363734343536666133313439373233363132383065613062633231636335653266363533b1736572766963652d79676764726173696cb6736572766963652d72756e74696d652d7369676e6572ae736572766963652e6865616c7468d9477368613235362d63636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363a27631d9477368613235362d32663837303134316366353638316566656332653234316364636435663731303831616539326162316535633534633839636438393666386335656236323731c294a773657276696365a468747470b6687474703a2f2f3132372e302e302e313a3137393939b6687474703a2f2f3132372e302e302e313a31383030319194af736572766963652e72756e74696d65b2736572766963652e72756e74696d652e7631a2763101919ab57368617265642d696e667261737472756374757265b56f64696e2e76657273652d72656e64657a766f7573b66f64696e2e76657273652d746f706f6c6f67792e7631a2763101b06265666f72652d70726f6d6f74696f6ea46f64696eb36d616e616765642d696e6361726e6174696f6ed9477368613235362d63343263333331373863366539666534633435313162623563663962626461376431633138356536646330373064323234653463653431303538626338633238b57463703a2f2f31302e37372e302e313a3137383731d9477368613235362d39613662663530353466616430306232343363323331386330333436303931333230343665643436356231356534636561666532613736623966663231313039c0cf0000018bcfe568019abb6964756e6e2e72756e74696d655f61637469766174696f6e2e7632d9477368613235362d39613662663530353466616430306232343363323331386330333436303931333230343665643436356231356534636561666532613736623966663231313039b1736572766963652d79676764726173696cd9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262d94032363535323862353235386435353563356434646533383931303830363164383965663439616336306631373337626464656337316535333335643163623935c420c06afef8f66e6ec88c96c81ad6d74dbd9933738032f4d8069a46412380e2272dcf0000018bcfe567ecd94066346536393735326334333432363965323262306130346137623430613065336235386364343561383238376364303936383263313864326565366539376239a765643235353139c44046048cf77cc9f524da966e16a131edb34180f4ea8420f5f24a64fd53178e44988bef994d6df103a5f117470283d533e9d7e8245c4f9965f9a4c0a0114ed1c201dc0033b26964756e6e2d343234322e73657276696365af4964756e6e20746573742034323432af696e766f636174696f6e2d3432343201a465786563a6616c77617973ad636f6e74726f6c2d67726f7570c3a57534323432a57534323432a0a0a0c3c3a9696e76697369626c65a3706964c3a430303737a0a0cd10920194cd1092cd1092cd1092cd109294cd1092cd1092cd1092cd109291cd10920000000000c391010807b52f6f70742f746573742f62696e2f736572766963650101d9477368613235362d34626635313232663334343535346335336264653265626238636432623765336431363030616436333163333835613564376363653233633737383534353961d9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262a92f6f70742f74657374a92f72756e2f74657374d9477368613235362d6462633162346339303066666534386435373562356461356336333830343031323566363564623066653365323434393462373665613938363435376439383690d9477368613235362d30383466656430386239373861663464376431393661373434366138366235383030396536333662363131646231363231316236356139616164666632396335d9202f73797374656d2e736c6963652f6964756e6e2d343234322e73657276696365c090aa61637469766174696f6edc0020010101010101010101010101010101010101010101010101010101010101010190d9477368613235362d39623734636534306561326663636337623632613533363838363563653533303438316130643838653931336161396330313333666662633838353763313838959141d9477368613235362d35353961656164303832363464353739356433393039373138636464303561626434393537326538346665353535393065656633316138386130386664666664ab6f64696e2d7369676e6572296492ad6f64696e2d746f706f6c6f6779959139d9477368613235362d31393538316532376465376365643030666631636535306232303437653761353637633736623163626165626162653565663033663763333031376262356237ab6f64696e2d7369676e6572276496a773657276696365d9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262d9477368613235362d35643838303630666232646530386139663366346338313936623062633863663737653934333732373363346135386330653430303737666630316461306534c0c0c096cd10920708c0c0c091b1736b69707065642d73746174656c65737391b1736b69707065642d73746174656c657373959140d9477368613235362d63333634316638353434643763303266333538306230376330663938383766306336613237666635616231643461336532396361663139376366633239396165ab6f64696e2d7369676e65722864c029c0c0c0c0c0 From 5d8eb533ab1e3efdb7796e623bc9958bbddc06f7 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 02:28:14 +0200 Subject: [PATCH 006/286] Pin stored_at preservation in the legacy migration test The fixture envelope's stored_at now differs from the record's own update time, so a migration that re-stamps the envelope fails the test. Co-Authored-By: Claude Sonnet 5 --- src/control_plane.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index fc3e9aa..670ee96 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -8979,7 +8979,7 @@ mod tests { key, r#type: record_type.into(), payload, - stored_at: rfc3339_millis(1_700_000_100_000)?, + stored_at: rfc3339_millis(1_700_000_123_456)?, schema_id: Some(schema.into()), }) } From d795e09a9eed68727678eda12a4592892fd700e7 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 02:28:28 +0200 Subject: [PATCH 007/286] Widen B2 to the two storm paths Soul found beyond S1 Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index bb551bf..145639c 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -457,6 +457,29 @@ reverted (Idunn).** closed first. **B2 — Bounded, backed-off route and unit repair (Idunn).** + +*Widened 2026-09-29 by Self, from Soul's S1 pass.* S1 closed the healthy-target storm, and +Soul found two storm paths S1 does not claim. Both are B2's: +- **A failing reload deletes the fragment** (`restore_admitted_membership`, + `drivers.rs:5472-5478`). The next tick reads the file as missing, treats that as drift, and + restores again: one `systemd-run`, ufw, `nginx -t` and reload every 500 ms, unbounded. + This was probed over 4 ticks. A broken global nginx config does the same through the + private-mount unit. +- **`NginxRouteDriver::install` (`drivers.rs:5402`) runs ufw, `nginx -t` and reload + unconditionally.** The Routing phase re-runs it on every resume, and a post-fence proof + failure resumes forever (F17), so a candidate failing its proof reloads every tick. A + stateless candidate reloads twice (install, then rollback). The post-fence abort's + `withdraw_candidate_membership` then calls `restore`, which does the same. + +So B2's actuation ceiling covers **`install`, `restore` and `restore_admitted_membership`**, +not only supervision. B2 also adds the seam Soul specified: the route actuator program paths +and `preflight_root` on `RuntimeOptions`, defaulting to today's paths and used at every +`NginxRouteDriver::new` site, plus a routed `Stored` fixture beside +`EngineFixture`. Deleting `supervise_admitted_route`'s body currently survives every test +(cargo-mutants), and B2's reload-count timeline test needs the same seam. A failed proof +marks the route degraded (Q5 a). Today nothing is written and the next tick re-challenges +with no backoff. + - Challenges back off exponentially per target on consecutive failures, with a floor at max age and a cap. - Route actuations (write, reload, ufw) go through a per-target rolling From 35e4fad8ab83ca6d93d0f27d43a25d20b7785eb5 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 02:38:33 +0200 Subject: [PATCH 008/286] Record Soul on B1 and rule its fix batch Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index 145639c..ad15e5d 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -456,6 +456,35 @@ reverted (Idunn).** - F0 is not required. This cut could land before F0 if Self wants the drift closed first. +**B1 status, 2026-09-29 (Self).** Landed on `idunn/route-b1` (`87fa57d` revert of `a148802`, +`4427769`, `4a9fe87`); 177 tests pass. **Soul (Opus): do not close.** The rule held (one +rule, one resolution, exact activation, the Expected never withdrawn, foreign activations +refused, the revert exact). The transition failed: +- **High, CONFIRMED:** a continuity abort persisted under the old rule (`Skipped` although + an activation was issued) fails the new validation (`control_plane.rs:1031-1044`). + `ControlSnapshot::read` fails the whole store on one record, so a resident record stops + Idunn from booting, for every target. +- **Medium, CONFIRMED:** supervision's pre-restart demotion failure is only logged + (`:3557-3576`). A continuity failing between `prepare_activation` and + `publish_observed_activation` then wedges its abort on "substituted" every tick. Drift + left by a pre-B1 failed continuity wedges a later deploy abort the same way. +- Low: `docs/deployment-authority.md` says the projection "never drifts". + +**Self's ruling for the B1 fix batch, which runs on top of F0 because F0 owns the legacy lift:** +1. The legacy-transaction lift maps a pre-B1 continuity abort that issued an activation to + `Pending`. The new single resolution then demotes its own activation, which also cleans + the residue that record left behind. +2. Supervision owns the Expected-only precondition. It does not mint a continuity while its + pre-restart demotion fails; the failure is recorded, not merely logged. +3. A one-time boot reconciliation demotes a projected activation only when its issuing + transaction is a failed transaction in history (exact identity). It never adopts. +4. Engine-layer tests for both abort paths, using Soul's EngineFixture probe shape (candidate + cleanup already Complete, so no systemctl runs). cargo-mutants found 6 Engine-path mutants + missed. +5. Correct the doc's "never drifts". +Before any deploy, check the live host for resident continuity abort records. That is a read +of `control.cc`, and the operator's to authorise. + **B2 — Bounded, backed-off route and unit repair (Idunn).** *Widened 2026-09-29 by Self, from Soul's S1 pass.* S1 closed the healthy-target storm, and From b570114ae66190fb98797fa7b0f0fc9c0ac4a162 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 03:01:45 +0200 Subject: [PATCH 009/286] Record Soul on C1, and widen C2 to the StreamPixels service Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index ad15e5d..603c0dd 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -573,6 +573,19 @@ with no backoff. - The lease pickup contract is unchanged. - Tests are in the package. Verify under the node image on Yggdrasil. +**C1 status, 2026-09-29 (Self).** Landed on CultLib `idunn-ts/route-c1` (`795fd10`). Soul closed +it on conditions, and a fix batch is in Hands: +- stateful, lease-bound Rust vectors; +- the signer owns health (`reportHealth`), starting at `warming`; +- the signer records every warming it signs, so a route-proof stateful target can take its lease; +- one signer per authority, a frozen authority, and a subpath export without the Odin publisher. +**Two consequences for later cuts:** +- **C2 covers the StreamPixels service too.** `apps/service/src/app.ts:155` answers route + challenges through `publishRouteObservation`, which C1 removed, so the gitlink bump breaks + the service unless C2 moves it to `answerRouteObservation` and `reportHealth`. +- **B3 surfaces the typed capacity shortfall** on the route path. Today it reads as a + generic "disagrees with current authority" (`control_plane.rs:5190`). + **C2 — StreamPixels web binding proves invariant 3 end to end (StreamPixels, after C1, S3, B3).** - Point the gitlink at the C1 commit. From fd75014b6eb30b24b9a73159725d502d2a326f97 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 03:11:21 +0200 Subject: [PATCH 010/286] Route every phase write through one owner that sets the deadline with it (F0 fix) Every admission commit and every post-fencing abort failed validation: transition() stamped a phase deadline on entry to Fencing..Committing and nothing cleared it, so the record cloned into Complete still carried a Committing deadline and validate() refused it. The first deploy or continuity past Fencing wedged in Committing; a post-fence abort never completed. DeploymentTransaction::enter_phase is now the only writer of phase. It sets the phase, the update time and the deadline together, so a record outside Fencing..=Committing carries no deadline by construction. transition(), both abort completions, the commit, rejected() and Starting/Warming all use it. Wall-clock ordering rules are deleted: entered_at <= updated_at, created_at <= updated_at and promoted_at <= updated_at each wedged a transaction after an NTP step backwards, since same-phase writes stamp now_millis(). The deadline is entered_at + duration, fixed at entry. The Engine takes its systemd workload port as a trait object (Engine::open_with_systemd_workload), which lets three Engine-level tests drive a continuity transaction from Fencing through Leasing, AwaitingReady, Routing and Committing to Complete, and a post-fence abort to Complete, with real Odin-signed evidence and no systemd, nginx or network. deployment_plan's test fixtures become pub(crate) for them. Co-Authored-By: Claude Sonnet 5 --- src/control_plane.rs | 512 ++++++++++++++++++++++++++++++++++++++--- src/deployment_plan.rs | 12 +- 2 files changed, 483 insertions(+), 41 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 670ee96..aa8a3b8 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -942,6 +942,18 @@ struct DeploymentTransaction { } impl DeploymentTransaction { + /// The one writer of `phase`. It stamps the update time and the phase + /// deadline together, so a record outside Fencing..=Committing carries no + /// deadline by construction. No other code assigns `phase`. + fn enter_phase(&mut self, phase: DeploymentPhase, now: u64) { + self.phase = phase; + self.updated_at_unix_millis = now; + self.phase_deadline = self + .plan + .as_ref() + .and_then(|plan| PhaseDeadline::entering(phase, plan, now)); + } + /// Whether this transaction's binding declares stop-then-start. A /// transaction without a plan yet cannot, so it reads as false. fn rollout_stops_incumbent_first(&self) -> bool { @@ -1020,8 +1032,7 @@ impl DeploymentTransaction { fn rejected(command: &DeploymentCommand, error: anyhow::Error, now: u64) -> Result { let mut transaction = Self::new(command, command.selector.clone(), 0, None, now)?; let detail = truncate(&format!("{error:#}"), 2048); - transaction.phase = DeploymentPhase::Complete; - transaction.updated_at_unix_millis = now; + transaction.enter_phase(DeploymentPhase::Complete, now); transaction.last_error = Some(detail.clone()); transaction.pre_fencing_abort = Some(PreFencingAbort { error: detail.clone(), @@ -1081,8 +1092,7 @@ impl DeploymentTransaction { require_id(&self.command_id, "transaction command id")?; require_id(&self.target, "transaction target")?; ensure!( - self.created_at_unix_millis > 0 - && self.updated_at_unix_millis >= self.created_at_unix_millis, + self.created_at_unix_millis > 0 && self.updated_at_unix_millis > 0, "deployment transaction timestamps are invalid" ); if let Some(generation) = &self.incumbent_generation_id { @@ -1231,7 +1241,6 @@ impl DeploymentTransaction { && (DeploymentPhase::Fencing..=DeploymentPhase::Committing) .contains(&deadline.phase) && deadline.entered_at_unix_millis > 0 - && deadline.entered_at_unix_millis <= self.updated_at_unix_millis && deadline.deadline_at_unix_millis > deadline.entered_at_unix_millis, "phase deadline does not describe the current post-fencing phase" ); @@ -1440,11 +1449,7 @@ impl DeploymentTransaction { }) = &self.routing { observation.validate()?; - ensure!( - *promoted_at_unix_millis > 0 - && *promoted_at_unix_millis <= self.updated_at_unix_millis, - "route promotion time is outside the durable transaction timeline" - ); + ensure!(*promoted_at_unix_millis > 0, "route promotion has no time"); } if self.phase == DeploymentPhase::Complete { ensure!( @@ -3316,12 +3321,24 @@ struct Engine { bootstrap_odin_authority: AdmittedOdinAuthority, source: GitSourceDriver, docker_runner: DockerRunnerDriver, - systemd_workload: SystemdTransientWorkloadDriver, + systemd_workload: Arc, host_actuators: Option, } impl Engine { fn open(options: RuntimeOptions) -> Result { + Self::open_with_systemd_workload( + options, + Arc::new(SystemdTransientWorkloadDriver::default()), + ) + } + + /// `open` with the systemd workload port supplied. It is the seam that + /// lets the phase engine run past Fencing without a systemd to talk to. + fn open_with_systemd_workload( + options: RuntimeOptions, + systemd_workload: Arc, + ) -> Result { let idunn_signer = open_service_identity_at::(&options.idunn_identity_store) .context("opening Idunn activation identity")?; @@ -3358,7 +3375,7 @@ impl Engine { bootstrap_odin_authority, source, docker_runner: DockerRunnerDriver::default(), - systemd_workload: SystemdTransientWorkloadDriver::default(), + systemd_workload, host_actuators, }) } @@ -3426,11 +3443,11 @@ impl Engine { } /// The workload driver the plan's binding declares. - fn workload_for(&self, plan: &CompiledDeploymentPlan) -> Result> { + fn workload_for(&self, plan: &CompiledDeploymentPlan) -> Result> { let (_, binding) = plan.parsed_inputs()?; Ok(match &binding.workload { - WorkloadBinding::SystemdTransient(_) => Box::new(self.systemd_workload.clone()), - WorkloadBinding::HostActuator(_) => Box::new(HostActuatorWorkloadDriver { + WorkloadBinding::SystemdTransient(_) => Arc::clone(&self.systemd_workload), + WorkloadBinding::HostActuator(_) => Arc::new(HostActuatorWorkloadDriver { access: self.host_access()?, }), }) @@ -4520,8 +4537,7 @@ impl Engine { "transaction plan", )?) })?; - next.phase = DeploymentPhase::Starting; - next.updated_at_unix_millis = now; + next.enter_phase(DeploymentPhase::Starting, now); next.last_error = None; replace_transaction(&self.options.state_store, current, &next) } @@ -4607,8 +4623,7 @@ impl Engine { }); } let mut next = current.value.clone(); - next.phase = DeploymentPhase::Warming; - next.updated_at_unix_millis = now_millis()?; + next.enter_phase(DeploymentPhase::Warming, now_millis()?); next.last_error = None; replace_transaction(&self.options.state_store, current, &next) } @@ -5345,8 +5360,7 @@ impl Engine { }, }; let mut complete = commit_current.value.clone(); - complete.phase = DeploymentPhase::Complete; - complete.updated_at_unix_millis = now; + complete.enter_phase(DeploymentPhase::Complete, now); complete.last_error = None; complete.completion = Some(TransactionCompletion::Admitted { generation_id: generation.generation_id.clone(), @@ -6385,12 +6399,7 @@ impl Engine { "deployment phase transition is not adjacent" ); let mut next = current.value.clone(); - next.phase = next_phase; - next.updated_at_unix_millis = now_millis()?; - next.phase_deadline = next - .plan - .as_ref() - .and_then(|plan| PhaseDeadline::entering(next_phase, plan, next.updated_at_unix_millis)); + next.enter_phase(next_phase, now_millis()?); next.last_error = None; replace_transaction(&self.options.state_store, current, &next) } @@ -6615,8 +6624,7 @@ impl Engine { "post-fencing abort cleanup is incomplete" ); let mut next = current.value.clone(); - next.phase = DeploymentPhase::Complete; - next.updated_at_unix_millis = now_millis()?; + next.enter_phase(DeploymentPhase::Complete, now_millis()?); next.last_error = Some(abort.error.clone()); next.completion = Some(TransactionCompletion::FailedAfterFencing { error: abort.error.clone(), @@ -6693,8 +6701,7 @@ impl Engine { "pre-fencing abort cleanup is incomplete" ); let mut next = current.value.clone(); - next.phase = DeploymentPhase::Complete; - next.updated_at_unix_millis = now_millis()?; + next.enter_phase(DeploymentPhase::Complete, now_millis()?); next.last_error = Some(abort.error.clone()); next.completion = Some(TransactionCompletion::FailedBeforeFencing { error: abort.error.clone(), @@ -7263,10 +7270,25 @@ mod tests { _temp: TempDir, engine: Engine, state_store: PathBuf, + odin_signer: ServiceIdentitySigner, + root: PathBuf, } impl EngineFixture { fn new() -> Result { + Self::build(None) + } + + /// An Engine whose systemd workload port is `workload`, so the phase + /// machine can run without systemd. This is the whole seam: nginx, + /// systemctl and the network are never reached on an unrouted, + /// stateless target, and the Odin evidence is signed by the key this + /// fixture enrolled. + fn with_workload(workload: Arc) -> Result { + Self::build(Some(workload)) + } + + fn build(workload: Option>) -> Result { let temp = TempDir::new()?; let root = temp.path(); std::fs::create_dir_all(root.join("identities"))?; @@ -7290,11 +7312,16 @@ mod tests { deployment_brake_operator_anchor: root.join("brake-anchor.cc"), ..RuntimeOptions::default() }; - let engine = Engine::open(options)?; + let engine = match workload { + Some(workload) => Engine::open_with_systemd_workload(options, workload)?, + None => Engine::open(options)?, + }; Ok(Self { + root: root.to_path_buf(), _temp: temp, engine, state_store, + odin_signer, }) } } @@ -9381,10 +9408,425 @@ mod tests { let mut backwards = transaction.clone(); backwards.phase_deadline.as_mut().unwrap().deadline_at_unix_millis = now; assert!(backwards.validate().is_err()); - let mut from_the_future = transaction; - from_the_future.phase_deadline = + // Entry is not ordered against the wall-clock stamp of later writes. + let mut entered_after_the_stamp = transaction; + entered_after_the_stamp.phase_deadline = PhaseDeadline::entering(DeploymentPhase::AwaitingReady, &plan, now + 1); - assert!(from_the_future.validate().is_err()); + entered_after_the_stamp.validate()?; + Ok(()) + } + + // --------------------------------------------------------------------- + // The phase machine, driven by the real Engine past Fencing. + // --------------------------------------------------------------------- + + /// A workload port with no systemd behind it: observing a workload + /// returns what was observed, stopping and discarding succeed, and + /// anything that would launch a process refuses. + struct StillWorkload; + + impl WorkloadPort for StillWorkload { + fn install( + &self, + _: &CompiledDeploymentPlan, + _: &crate::drivers::MaterializedRelease, + ) -> Result { + bail!("StillWorkload launches nothing") + } + fn prepare_activation( + &self, + _: &CompiledDeploymentPlan, + _: &IdunnExpectedIncarnationRecord, + _: IdunnRuntimeActivationLaunch, + ) -> Result { + bail!("StillWorkload launches nothing") + } + fn start_prepared( + &self, + _: &CompiledDeploymentPlan, + _: &SealedRelease, + _: &crate::drivers::InstalledReleaseObservation, + _: &IdunnExpectedIncarnationRecord, + _: &IdunnRuntimeActivationRecord, + ) -> Result { + bail!("StillWorkload launches nothing") + } + fn discard_prepared( + &self, + _: &CompiledDeploymentPlan, + _: &IdunnExpectedIncarnationRecord, + _: &IdunnRuntimeActivationRecord, + ) -> Result<()> { + Ok(()) + } + fn observe( + &self, + _: &IdunnExpectedIncarnationRecord, + _: &IdunnRuntimeActivationRecord, + prior: &WorkloadObservation, + ) -> Result { + Ok(prior.clone()) + } + fn stop(&self, _: &WorkloadObservation) -> Result<()> { + Ok(()) + } + fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { + Ok(false) + } + } + + /// A Continuity transaction for an unrouted, stateless target, sitting at + /// Fencing with every earlier phase's evidence in place. The plan, sealed + /// release, Expected, activation and Warming evidence are the real + /// constructions; only the workload and isolation observations are + /// borrowed from a recorded transaction. Continuity is chosen because a + /// Deploy would need a frozen source and a signed brake record, which no + /// phase past Fencing reads. + fn transaction_at_fencing(world: &EngineFixture) -> Result { + use crate::deployment_plan::tests::{ + BINDING, RECIPE, artifact_receipt, external_input_receipt, source, + }; + let provider = enroll_service_identity_at::( + &world.root.join("identities/provider.cc"), + )?; + let provider_anchor = world.root.join("identities/provider-anchor.cc"); + export_service_identity_trust_anchor(&provider, &provider_anchor)?; + + let (binding_head, binding_tail) = BINDING + .split_once("[route]") + .context("binding has no route table")?; + let binding = format!( + "{binding_head}[brakes]{}", + binding_tail + .split_once("[brakes]") + .context("binding has no brakes table")? + .1 + ) + .replace( + "/etc/gamecult/trust/service.cc", + &provider_anchor.display().to_string(), + ) + .replace("service-runtime-signer", &provider.entry().identity_id); + let recipe = RECIPE + .split("[[dependencies]]") + .next() + .context("recipe is empty")? + .replace("route_required = true", "route_required = false") + .replace( + r#"["GAMECULT_IDUNN_CANDIDATE_BIND", "GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, + r#"["GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, + ); + let plan = compile_deployment_plan( + recipe.as_bytes(), + binding.as_bytes(), + source(&recipe), + "service-incarnation-1", + None, + 110, + &[], + )?; + let release = SealedRelease::new( + &plan, + vec![artifact_receipt()], + vec![external_input_receipt()], + 120, + )?; + let expected = release.expected_projection(&plan)?; + assert!(!expected.write_lease_required && expected.route.is_none()); + + let now = now_millis()?; + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "continuity-service".into(), + kind: CommandKind::Continuity, + selector: "service".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + let mut transaction = DeploymentTransaction::new(&command, "service".into(), 0, None, now)?; + let activation = IdunnRuntimeActivationLaunch::issue( + &expected, + runtime_instance_id(&transaction.transaction_id)?, + now, + &world.engine.idunn_signer, + )? + .activation() + .clone(); + let mut workload = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)? + .1 + .workload + .context("recorded transaction has no workload")?; + transaction.lifecycle_authorized_at_unix_millis = Some(now); + transaction.expected_publication_sha256 = Some(expected.canonical_sha256()?); + transaction.activation_publication_sha256 = Some(activation.canonical_sha256()?); + match &mut workload { + WorkloadObservation::Systemd(observed) => { + observed.runtime_instance_id = activation.runtime_instance_id.clone(); + } + WorkloadObservation::Host(_) => bail!("recorded workload is not a systemd unit"), + } + let recorded = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)?.1; + transaction.isolation = recorded.isolation; + transaction.installed_release = Some(crate::drivers::InstalledReleaseObservation { + sealed_release_id: release.sealed_release_id.clone(), + root: PathBuf::from("/srv/service/releases/test"), + }); + transaction.sealed_release = Some(release); + transaction.expected = Some(expected); + transaction.activation = Some(activation); + transaction.workload = Some(workload); + transaction.plan = Some(plan); + // Odin's first word about the candidate, before it was Ready. + let warming = signed_correlation(world, &transaction, 4, false)?; + let authenticated = world.engine.authenticate_topology_bytes( + &ControlSnapshot::read(&world.state_store)?, + &transaction, + &warming, + None, + now, + )?; + transaction.warming = Some(WarmingEvidence::OdinTopology { + evidence: TopologyEvidence::from_authenticated(&authenticated, now)?, + }); + transaction.odin_publisher_sequence_cursor = 4; + transaction.enter_phase(DeploymentPhase::Fencing, now); + transaction.validate()?; + + let store = SingleFileMessagePackBackingStore::new(&world.state_store); + assert!(store.compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: transaction.transaction_id.clone(), + current: None, + }, + ], + &[ + command_envelope(&command, command.requested_at_unix_millis)?, + transaction_envelope(&transaction, now)?, + ], + )?); + Ok(transaction) + } + + /// Odin's correlation for the transaction's incarnation, signed by the key + /// the Engine trusts and stamped now. + fn signed_correlation( + world: &EngineFixture, + transaction: &DeploymentTransaction, + sequence: u64, + ready: bool, + ) -> Result> { + let expected = transaction.expected.as_ref().context("no Expected")?; + let activation = transaction.activation.as_ref().context("no activation")?; + let mut record = OdinRuntimeTopologyCorrelationRecord { + schema_version: cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into(), + target: expected.target.clone(), + expected_projection_sha256: expected.canonical_sha256()?, + expected: true, + current_activation_sha256: Some(activation.canonical_sha256()?), + signed_presence_sha256: Some(sha256_id(b"presence")), + observed_presence_state: Some("active".into()), + observed_presence_publisher_sequence: Some(sequence), + observed_write_lease_sha256: None, + observed_capabilities: expected + .capabilities + .iter() + .map(|capability| cultnet_rs::GameCultRuntimeCapability { + capability: capability.capability.clone(), + schema: capability.schema.clone(), + compatibility: capability.compatibility.clone(), + capacity: 1, + }) + .collect(), + runtime_id: expected.runtime_id.clone(), + runtime_instance_id: Some(activation.runtime_instance_id.clone()), + present: true, + ready, + dependencies: Vec::new(), + disagreements: Vec::new(), + signer_identity_id: world.odin_signer.entry().identity_id.clone(), + publisher_sequence: sequence, + observed_at_unix_millis: now_millis()?, + signature_algorithm: "ed25519".into(), + signature: Vec::new(), + }; + record.signature = world + .odin_signer + .sign::(&record.unsigned_signature_payload()?) + .signature; + record.canonical_bytes() + } + + /// Odin publishes its Ready correlation for the transaction incarnation. + fn odin_reports_ready( + world: &EngineFixture, + transaction: &DeploymentTransaction, + sequence: u64, + ) -> Result<()> { + let expected = transaction.expected.as_ref().context("no Expected")?; + SingleFileMessagePackBackingStore::new(&world.engine.options.odin_correlation_store) + .insert_entry_if_absent(CultCacheEnvelope { + key: crate::drivers::incarnation_key_of( + &expected.target, + &expected.canonical_sha256()?, + ), + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + payload: signed_correlation(world, transaction, sequence, true)?, + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some(cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into()), + })?; + Ok(()) + } + + fn resident(world: &EngineFixture) -> Result> { + ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .next() + .context("the transaction left the live set") + } + + /// The transaction wherever it now lives: live, or archived once terminal. + fn latest(world: &EngineFixture) -> Result { + match resident(world) { + Ok(stored) => Ok(stored.value), + Err(_) => read_history_transactions(&world.state_store) + .into_iter() + .next() + .context("the transaction is in neither the live set nor history"), + } + } + + /// One engine step at a time until `stop` holds, recording each phase + /// entered and checking after every step that the deadline on the record + /// is exactly the current post-fencing phase's, and absent outside one. + fn drive( + world: &EngineFixture, + stop: impl Fn(&DeploymentTransaction) -> bool, + ) -> Result> { + let mut phases = Vec::new(); + for _ in 0..40 { + let Ok(current) = resident(world) else { + return Ok(phases); + }; + if stop(¤t.value) { + return Ok(phases); + } + world.engine.advance_transaction(¤t)?; + let after = latest(world)?; + match &after.phase_deadline { + Some(deadline) => assert_eq!(deadline.phase, after.phase), + None => assert!( + !(DeploymentPhase::Fencing..=DeploymentPhase::Committing) + .contains(&after.phase), + "a post-fencing record lost its deadline" + ), + } + if phases.last() != Some(&after.phase) { + phases.push(after.phase); + } + } + bail!("the transaction did not reach the stop condition in 40 steps") + } + + #[test] + fn an_admission_runs_from_fencing_to_complete_and_commits() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = transaction_at_fencing(&world)?; + odin_reports_ready(&world, &seeded, 5)?; + + let phases = drive(&world, |transaction| { + transaction.phase == DeploymentPhase::Complete + })?; + assert_eq!( + phases, + [ + DeploymentPhase::Fencing, + DeploymentPhase::Leasing, + DeploymentPhase::AwaitingReady, + DeploymentPhase::Routing, + DeploymentPhase::Committing, + DeploymentPhase::Complete, + ] + ); + let finished = latest(&world)?; + assert_eq!(finished.phase_deadline, None); + assert!(matches!( + finished.completion, + Some(TransactionCompletion::Admitted { .. }) + )); + let snapshot = ControlSnapshot::read(&world.state_store)?; + let admitted = snapshot + .admitted_for("service") + .context("commit wrote no admitted generation")?; + assert_eq!(admitted.value.transaction_id, finished.transaction_id); + assert_eq!(Some(&admitted.value.expected), seeded.expected.as_ref()); + Ok(()) + } + + #[test] + fn a_post_fence_abort_runs_to_complete() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = transaction_at_fencing(&world)?; + odin_reports_ready(&world, &seeded, 5)?; + drive(&world, |transaction| { + transaction.phase == DeploymentPhase::Routing + })?; + let routing = resident(&world)?; + assert!(routing.value.phase_deadline.is_some()); + + world + .engine + .begin_post_fencing_abort(&routing, anyhow!("candidate refused"))?; + drive(&world, |transaction| transaction.completion.is_some())?; + let finished = latest(&world)?; + assert_eq!(finished.phase, DeploymentPhase::Complete); + assert_eq!(finished.phase_deadline, None); + assert!(matches!( + finished.completion, + Some(TransactionCompletion::FailedAfterFencing { .. }) + )); + assert!( + ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .is_none(), + "an aborted candidate was admitted" + ); + Ok(()) + } + + #[test] + fn entering_a_phase_owns_the_deadline_and_a_stepped_clock_cannot_wedge_it() -> Result<()> { + let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?; + let plan = transaction.plan.clone().unwrap(); + let base = transaction.created_at_unix_millis; + transaction.enter_phase(DeploymentPhase::Routing, base + 1_000); + let deadline = transaction.phase_deadline.context("no deadline on entry")?; + assert_eq!(deadline.phase, DeploymentPhase::Routing); + assert_eq!( + deadline.deadline_at_unix_millis, + base + 1_000 + u64::from(plan.phase_deadlines().routing_seconds) * 1000 + ); + // A same-phase write after the wall clock stepped backwards is still a + // valid record: the deadline is fixed at entry, not ordered against + // later stamps, and neither is the creation time. + transaction.updated_at_unix_millis = deadline.entered_at_unix_millis - 1; + transaction.validate()?; + transaction.updated_at_unix_millis = base - 1; + transaction.validate()?; + transaction.enter_phase(DeploymentPhase::Committing, base + 2_000); + assert_eq!( + transaction.phase_deadline.unwrap().phase, + DeploymentPhase::Committing + ); + transaction.enter_phase(DeploymentPhase::Complete, base + 3_000); + assert_eq!(transaction.phase_deadline, None); Ok(()) } diff --git a/src/deployment_plan.rs b/src/deployment_plan.rs index e3d690b..1a966dc 100644 --- a/src/deployment_plan.rs +++ b/src/deployment_plan.rs @@ -1098,7 +1098,7 @@ fn sha256_id(bytes: &[u8]) -> String { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; use cultnet_rs::{ GameCultProviderHealthIdentity, GameCultRuntimeCapability, IdunnRuntimeActivationLaunch, @@ -1109,7 +1109,7 @@ mod tests { verify_runtime_authority, }; - const RECIPE: &str = r#" + pub(crate) const RECIPE: &str = r#" schema = "gamecult.idunn.target_declaration.v1" target = "service" source_stamp_environment = "SERVICE_BUILD_COMMIT" @@ -1172,7 +1172,7 @@ schema = "odin.verse-topology.v1" compatibility = "v1" "#; - const BINDING: &str = r#" + pub(crate) const BINDING: &str = r#" schema = "gamecult.idunn.operator_binding.v2" target = "service" @@ -1247,7 +1247,7 @@ nodes = ["yggdrasil"] format!("sha256-{}", char::from(byte).to_string().repeat(64)) } - fn source(recipe: &str) -> SourceSelectionFacts { + pub(crate) fn source(recipe: &str) -> SourceSelectionFacts { SourceSelectionFacts { schema: SOURCE_SELECTION_FACTS_SCHEMA.into(), origin: "https://github.com/GameCult/Service.git".into(), @@ -1387,7 +1387,7 @@ nodes = ["yggdrasil"] .unwrap() } - fn artifact_receipt() -> ArtifactReceipt { + pub(crate) fn artifact_receipt() -> ArtifactReceipt { ArtifactReceipt { artifact_id: "daemon".into(), destination: "service".into(), @@ -1397,7 +1397,7 @@ nodes = ["yggdrasil"] } } - fn external_input_receipt() -> ExternalInputMaterializationReceipt { + pub(crate) fn external_input_receipt() -> ExternalInputMaterializationReceipt { ExternalInputMaterializationReceipt { input_id: "toolchain-index".into(), url: "https://example.invalid/toolchain-index".into(), From 945cb0b3227f4ba73dedb370291f367e1219393c Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 03:25:20 +0200 Subject: [PATCH 011/286] Count undecodable history, refuse a route-proof Odin authority fallback, test the F0 rules (F0 fix) History: read_history_transactions dropped undecodable records with .ok(), which undercounts continuity restarts for backoff. decode_history_transactions now returns each failure with its key and reason, and the reader prints every one plus a count. An unreadable history.cc is reported instead of read as empty. The pre-c848459 IsolationEvidence shape needs no Legacy arm: that commit made the enum untagged with a Linux variant of the same six fields, and a test now decodes the old shape under both MessagePack encodings to prove it. The Legacy structs' stated death condition now says history.cc is never migrated or pruned, so they die only when history is. Odin authority: current_odin_authority fell back to the bootstrap key when the admitted Odin generation carried none (route-proof readiness). It now refuses. OdinReceipts.authority, read only by tests, is deleted. Tests through the production path for the rules cargo-mutants showed unexercised: ControlSnapshot::read of a lifted legacy generation; the route-proof arm of AdmittedGeneration::validate and route-proof shape checks; Odin readiness, lease adoption, route supervision and continuity backoff validation; the admitted half of max_odin_sequence; and the receipt-equality comparisons in AwaitingReady and Routing, driven through the Engine. The awaiting-ready default gets a comment that it is unsafe for stateful targets whose migrations run in AwaitingReady; B5 owns the fix. Co-Authored-By: Claude Sonnet 5 --- src/control_plane.rs | 475 +++++++++++++++++++++++++++++++++++++++-- src/deployment_plan.rs | 3 + 2 files changed, 457 insertions(+), 21 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index aa8a3b8..5632d12 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1609,7 +1609,6 @@ struct AdmittedGeneration { struct OdinReceipts<'a> { ready: &'a TopologyEvidence, latest: &'a TopologyEvidence, - authority: &'a AdmittedOdinAuthority, } impl AdmittedGeneration { @@ -1619,11 +1618,10 @@ impl AdmittedGeneration { &self.latest_odin_observation, &self.odin_authority, ) { - (ReadinessEvidence::OdinCorrelated { evidence }, Some(latest), Some(authority)) => { + (ReadinessEvidence::OdinCorrelated { evidence }, Some(latest), Some(_)) => { Ok(OdinReceipts { ready: evidence, latest, - authority, }) } _ => bail!("admitted generation of {} is not Odin-correlated", self.target), @@ -2420,9 +2418,10 @@ fn command_envelope(value: &DeploymentCommand, now: u64) -> Result Vec { if !path.exists() { return Vec::new(); } - let Ok(envelopes) = SingleFileMessagePackBackingStore::new(&path).pull_all_read_only_snapshot() - else { - return Vec::new(); + let envelopes = match SingleFileMessagePackBackingStore::new(&path).pull_all_read_only_snapshot() + { + Ok(envelopes) => envelopes, + Err(error) => { + eprintln!( + "Idunn cannot read {}: {error:#}; every archived transaction is invisible \ + to status and to continuity backoff", + path.display() + ); + return Vec::new(); + } }; - envelopes + let (transactions, undecodable) = decode_history_transactions(envelopes); + if !undecodable.is_empty() { + for (key, error) in &undecodable { + eprintln!("Idunn history holds an undecodable transaction {key}: {error}"); + } + eprintln!( + "Idunn history: {} archived transaction(s) could not be decoded and are not counted \ + by status or continuity backoff", + undecodable.len() + ); + } + transactions +} + +/// Decode every archived transaction. An entry that fails is returned with its +/// key and reason instead of vanishing, so the reader can say how many records +/// its answer is missing. +/// +/// `LegacyDeploymentTransaction` decodes the v2 and v3 layouts. `history.cc` is +/// never migrated or pruned, so it dies only when history is pruned or +/// migrated, not at boot. +fn decode_history_transactions( + envelopes: Vec, +) -> (Vec, Vec<(String, String)>) { + let mut transactions = Vec::new(); + let mut undecodable = Vec::new(); + for envelope in envelopes .into_iter() .filter(|envelope| envelope.r#type == DeploymentTransaction::TYPE) - .filter_map(|envelope| match envelope.schema_id.as_deref() { + { + let decoded = match envelope.schema_id.as_deref() { Some(DEPLOYMENT_TRANSACTION_SCHEMA_V2 | DEPLOYMENT_TRANSACTION_SCHEMA_V3) => { - lift_legacy_transaction(&envelope).ok() + lift_legacy_transaction(&envelope) } - _ => rmp_serde::from_slice::(&envelope.payload).ok(), - }) - .collect() + _ => rmp_serde::from_slice::(&envelope.payload) + .map_err(anyhow::Error::from), + }; + match decoded { + Ok(transaction) => transactions.push(transaction), + Err(error) => undecodable.push((envelope.key, format!("{error:#}"))), + } + } + (transactions, undecodable) } /// Consumed commands, read as leniently as their transactions. @@ -3461,10 +3501,16 @@ impl Engine { } fn current_odin_authority(&self, snapshot: &ControlSnapshot) -> Result { - let authority = snapshot - .admitted_for("odin") - .and_then(|stored| stored.value.odin_authority.clone()) - .unwrap_or_else(|| self.bootstrap_odin_authority.clone()); + // The bootstrap key stands only until Odin itself is admitted. Once it + // is, its own generation names the authority; an admitted Odin that + // carries none has no authority to name, and falling back to the + // bootstrap key would let a stale key vouch for the Verse. + let authority = match snapshot.admitted_for("odin") { + None => self.bootstrap_odin_authority.clone(), + Some(stored) => stored.value.odin_authority.clone().with_context(|| { + "admitted Odin generation carries no Odin authority (route-proof readiness)" + })?, + }; authority.validate()?; Ok(authority) } @@ -9034,7 +9080,7 @@ mod tests { let odin = lifted.odin_receipts()?; assert_eq!(odin.ready, &legacy.ready); assert_eq!(odin.latest, &legacy.latest_odin_observation); - assert_eq!(odin.authority, &legacy.odin_authority); + assert_eq!(lifted.odin_authority.as_ref(), Some(&legacy.odin_authority)); assert_eq!( lifted.odin_publisher_sequence_cursor, legacy.odin_publisher_sequence_cursor @@ -9292,7 +9338,11 @@ mod tests { next_restart_at_unix_millis: Some(8), }; let committing = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?.1; - let authority = incumbent.odin_receipts()?.authority.clone(); + incumbent.odin_receipts()?; + let authority = incumbent + .odin_authority + .clone() + .context("fixture generation has no Odin authority")?; let next = AdmittedGeneration::from_transaction( &committing, authority.clone(), @@ -9830,6 +9880,389 @@ mod tests { Ok(()) } + #[test] + fn an_undecodable_history_record_is_counted_and_never_dropped_silently() -> Result<()> { + let (finished, _) = terminal_transaction_with_command("ghostlight")?; + let good = transaction_envelope(&finished, finished.updated_at_unix_millis)?; + let mut torn = good.clone(); + torn.key = "tx-torn".into(); + // 0xc1 is msgpack's never-used byte: decodable by nothing. + torn.payload = vec![0xc1]; + let mut foreign = good.clone(); + foreign.key = "tx-foreign".into(); + foreign.schema_id = Some(DEPLOYMENT_TRANSACTION_SCHEMA_V3.into()); + foreign.payload = vec![0xc1]; + + let (decoded, undecodable) = + decode_history_transactions(vec![good, torn, foreign]); + assert_eq!(decoded, [finished]); + assert_eq!( + undecodable.iter().map(|(key, _)| key.as_str()).collect::>(), + ["tx-torn", "tx-foreign"] + ); + assert!(undecodable.iter().all(|(_, error)| !error.is_empty())); + Ok(()) + } + + /// The layout `IsolationEvidence` had before c848459 turned it into an + /// untagged Linux/Host enum. That commit claimed old records decode + /// unchanged; this is the check with the old shape itself. + #[test] + fn isolation_evidence_from_before_the_host_variant_still_decodes() -> Result<()> { + #[derive(Serialize)] + struct BeforeTheHostVariant { + candidate_uid: u32, + candidate_pid_namespace_id: u64, + candidate_mount_namespace_id: u64, + incumbent_uid: Option, + incumbent_pid_namespace_id: Option, + incumbent_mount_namespace_id: Option, + } + for incumbent in [None, Some(7)] { + let old = BeforeTheHostVariant { + candidate_uid: 61_000, + candidate_pid_namespace_id: 4_026_531_836, + candidate_mount_namespace_id: 4_026_531_841, + incumbent_uid: incumbent, + incumbent_pid_namespace_id: incumbent.map(|_| 8), + incumbent_mount_namespace_id: incumbent.map(|_| 9), + }; + for bytes in [rmp_serde::to_vec(&old)?, rmp_serde::to_vec_named(&old)?] { + let decoded: IsolationEvidence = rmp_serde::from_slice(&bytes)?; + let IsolationEvidence::Linux(linux) = decoded else { + bail!("an old Linux record decoded as a host record"); + }; + assert_eq!(linux.candidate_uid, 61_000); + assert_eq!(linux.incumbent_uid, incumbent); + } + } + Ok(()) + } + + #[test] + fn an_admitted_odin_without_odin_authority_is_refused_not_replaced_by_the_bootstrap_key() -> Result<()> { + let world = EngineFixture::new()?; + let empty = ControlSnapshot::read(&world.state_store)?; + assert_eq!( + world.engine.current_odin_authority(&empty)?, + world.engine.bootstrap_odin_authority + ); + + let (_, mut odin) = fixture_generation(FIXTURE_GENERATION)?; + odin.target = "odin".into(); + odin.odin_authority = Some(AdmittedOdinAuthority::from_anchor( + &world.odin_signer.trust_anchor()?, + )?); + let mut with_authority = ControlSnapshot::default(); + with_authority.admitted.push(Stored { + value: odin.clone(), + envelope: CultCacheEnvelope { + key: "odin".into(), + r#type: AdmittedGeneration::TYPE.into(), + payload: Vec::new(), + stored_at: "1970-01-01T00:00:00.100Z".into(), + schema_id: Some(ADMITTED_GENERATION_SCHEMA.into()), + }, + }); + assert_eq!( + world.engine.current_odin_authority(&with_authority)?, + odin.odin_authority.clone().unwrap() + ); + + odin.odin_authority = None; + let mut without = ControlSnapshot::default(); + without.admitted.push(Stored { + value: odin.clone(), + envelope: CultCacheEnvelope { + key: "odin".into(), + r#type: AdmittedGeneration::TYPE.into(), + payload: Vec::new(), + stored_at: "1970-01-01T00:00:00.100Z".into(), + schema_id: Some(ADMITTED_GENERATION_SCHEMA.into()), + }, + }); + let error = world + .engine + .current_odin_authority(&without) + .expect_err("a route-proof Odin must not fall back to the bootstrap key"); + assert!(format!("{error:#}").contains("no Odin authority"), "{error:#}"); + Ok(()) + } + + fn error_text(result: Result<()>) -> String { + format!("{:#}", result.expect_err("the record must be refused")) + } + + /// A route-proof generation from a real admission: the world's Expected + /// declares no Odin dependency, so route proof is its class. + fn route_proof_generation() -> Result { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = transaction_at_fencing(&world)?; + odin_reports_ready(&world, &seeded, 5)?; + drive(&world, |transaction| { + transaction.phase == DeploymentPhase::Committing + })?; + let mut committing = resident(&world)?.value; + assert_eq!( + ReadinessClass::of(committing.expected.as_ref().unwrap()), + ReadinessClass::RouteProof + ); + committing.ready = Some(route_proof_evidence()); + committing.latest_odin_observation = None; + let generation = AdmittedGeneration::from_transaction( + &committing, + world.engine.bootstrap_odin_authority.clone(), + None, + now_millis()?, + )?; + generation.validate()?; + Ok(generation) + } + + #[test] + fn a_route_proof_generation_is_admitted_only_in_its_exact_shape() -> Result<()> { + let generation = route_proof_generation()?; + assert_eq!(generation.ready.class(), ReadinessClass::RouteProof); + assert!(generation.odin_authority.is_none()); + assert!(generation.latest_odin_observation.is_none()); + assert!(generation.odin_receipts().is_err()); + + // Route proof carries no Odin receipt and no Odin authority. + let mut with_receipt = generation.clone(); + with_receipt.latest_odin_observation = Some(topology(1, 1)); + assert!(with_receipt.validate().is_err()); + let mut with_authority = generation.clone(); + with_authority.odin_authority = Some(AdmittedOdinAuthority { + signer_identity_id: "odin-signer".into(), + signer_public_key: vec![1; 32], + }); + assert!(with_authority.validate().is_err()); + + // Its readiness evidence is checked by shape. + let ReadinessEvidence::RouteProof { evidence } = &generation.ready else { + bail!("not route proof"); + }; + let mut tampered = generation.clone(); + tampered.ready = ReadinessEvidence::RouteProof { + evidence: RuntimePresenceEvidence { + canonical_sha256: sha256_id(b"another"), + ..evidence.clone() + }, + }; + assert!(error_text(tampered.validate()).contains("runtime presence evidence")); + let mut untimed = generation.clone(); + untimed.ready = ReadinessEvidence::RouteProof { + evidence: RuntimePresenceEvidence { + admitted_at_unix_millis: evidence.challenged_at_unix_millis - 1, + ..evidence.clone() + }, + }; + assert!(error_text(untimed.validate()).contains("timeline")); + Ok(()) + } + + #[test] + fn odin_correlated_readiness_evidence_is_checked_by_shape() -> Result<()> { + let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; + generation.validate()?; + let ReadinessEvidence::OdinCorrelated { evidence } = &generation.ready else { + bail!("fixture is not Odin-correlated"); + }; + for broken in [ + TopologyEvidence { + canonical_sha256: sha256_id(b"another"), + ..evidence.clone() + }, + TopologyEvidence { + publisher_sequence: 0, + ..evidence.clone() + }, + TopologyEvidence { + admitted_at_unix_millis: 0, + ..evidence.clone() + }, + ] { + let mut tampered = generation.clone(); + tampered.ready = ReadinessEvidence::OdinCorrelated { evidence: broken }; + assert!(tampered.validate().is_err()); + } + Ok(()) + } + + #[test] + fn lease_adoption_evidence_is_checked_by_shape_on_the_transaction() -> Result<()> { + let (_, transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; + let adoption = LeaseAdoptionEvidence { + write_lease_sha256: sha256_id(b"lease"), + signed_presence_sha256: sha256_id(b"presence"), + source: AdoptionSource::Direct, + observed_at_unix_millis: 100, + }; + let refused = |change: fn(&mut LeaseAdoptionEvidence)| { + let mut adopting = transaction.clone(); + let mut evidence = adoption.clone(); + change(&mut evidence); + adopting.lease_adoption = Some(evidence); + error_text(adopting.validate()) + }; + assert!(refused(|evidence| evidence.write_lease_sha256.clear()) + .contains("adopted lease digest")); + assert!(refused(|evidence| evidence.signed_presence_sha256 = "no spaces allowed".into()) + .contains("presence digest")); + assert!(refused(|evidence| evidence.observed_at_unix_millis = 0) + .contains("no observation time")); + // Well-formed but naming no granted lease: the record is refused for + // that reason and no other. + assert!(refused(|_| {}).contains("does not name the granted lease")); + Ok(()) + } + + #[test] + fn route_supervision_and_continuity_backoff_are_checked_on_the_generation() -> Result<()> { + let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; + generation.validate()?; + let supervised = |change: fn(&mut RouteSupervisionState)| { + let mut tampered = generation.clone(); + change(tampered.route_supervision.as_mut().unwrap()); + error_text(tampered.validate()) + }; + assert!(supervised(|state| state.degraded_since_unix_millis = Some(0)) + .contains("no start time")); + assert!(supervised(|state| state.actuations.count = 1) + .contains("count but no start")); + let mut degraded = generation.clone(); + let state = degraded.route_supervision.as_mut().unwrap(); + state.degraded_since_unix_millis = Some(5); + state.actuations = ActuationWindow { + window_started_at_unix_millis: 5, + count: 2, + }; + degraded.validate()?; + + let mut backoff = generation.clone(); + backoff.continuity_backoff.attempts = 2; + assert!(error_text(backoff.validate()).contains("attempts but no window")); + backoff.continuity_backoff.window_started_at_unix_millis = Some(5); + backoff.validate()?; + Ok(()) + } + + #[test] + fn max_odin_sequence_counts_admitted_generations_by_target_and_signer() -> Result<()> { + let (_, mut generation) = fixture_generation(FIXTURE_GENERATION)?; + let authority = generation.odin_authority.clone().context("no authority")?; + generation.odin_publisher_sequence_cursor = 88; + let target = generation.target.clone(); + let snapshot = ControlSnapshot { + commands: Vec::new(), + transactions: Vec::new(), + admitted: vec![Stored { + envelope: CultCacheEnvelope { + key: target.clone(), + r#type: AdmittedGeneration::TYPE.into(), + payload: Vec::new(), + stored_at: "1970-01-01T00:00:00.100Z".into(), + schema_id: Some(ADMITTED_GENERATION_SCHEMA.into()), + }, + value: generation, + }], + }; + assert_eq!(snapshot.max_odin_sequence(&target, &authority.signer_identity_id), 88); + assert_eq!(snapshot.max_odin_sequence(&target, "another-signer"), 0); + assert_eq!( + snapshot.max_odin_sequence("another-target", &authority.signer_identity_id), + 0 + ); + Ok(()) + } + + #[test] + fn a_store_holding_a_lifted_legacy_generation_reads_and_keeps_its_key_honest() -> Result<()> { + let temp = TempDir::new()?; + let store = temp.path().join("control.cc"); + let envelope = fixture_envelope(FIXTURE_GENERATION, AdmittedGeneration::TYPE)?; + let backing = SingleFileMessagePackBackingStore::new(&store); + backing.insert_entry_if_absent(envelope.clone())?; + + let snapshot = ControlSnapshot::read(&store)?; + assert_eq!(snapshot.admitted.len(), 1); + let lifted = &snapshot.admitted[0].value; + assert_eq!(lifted.target, envelope.key); + assert_eq!(lifted.ready.class(), ReadinessClass::OdinCorrelated); + assert!(snapshot.admitted_for(&envelope.key).is_some()); + + // The lifted record is held to the same key rule as a current one. + let other = temp.path().join("misfiled.cc"); + let mut misfiled = envelope; + misfiled.key = "not-its-target".into(); + SingleFileMessagePackBackingStore::new(&other).insert_entry_if_absent(misfiled)?; + assert!(error_text(ControlSnapshot::read(&other).map(|_| ())) + .contains("admitted generation key is not its target")); + Ok(()) + } + + /// Replace Odin's one correlation for the incarnation with a newer one. + fn odin_republishes( + world: &EngineFixture, + transaction: &DeploymentTransaction, + sequence: u64, + ) -> Result<()> { + let path = &world.engine.options.odin_correlation_store; + let store = SingleFileMessagePackBackingStore::new(path); + let existing = store.pull_all_read_only_snapshot()?; + assert!(store.delete_batch_if_unchanged(&existing)?); + odin_reports_ready(world, transaction, sequence) + } + + fn ready_sequence(transaction: &DeploymentTransaction) -> Option { + transaction + .ready + .as_ref() + .and_then(ReadinessEvidence::odin) + .map(|evidence| evidence.publisher_sequence) + } + + #[test] + fn a_newer_odin_reading_refreshes_the_receipt_before_awaiting_ready_or_routing_advance() + -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = transaction_at_fencing(&world)?; + odin_reports_ready(&world, &seeded, 5)?; + + // AwaitingReady: the receipt is written, then confirmed, then the + // phase advances only on a receipt equal to the latest reading. + drive(&world, |transaction| { + transaction.phase == DeploymentPhase::AwaitingReady + && ready_sequence(transaction) == Some(5) + })?; + odin_republishes(&world, &seeded, 6)?; + world.engine.advance_transaction(&resident(&world)?)?; // admits 6 + let admitted = resident(&world)?.value; + assert_eq!(admitted.phase, DeploymentPhase::AwaitingReady); + assert_eq!(admitted.latest_odin_observation.as_ref().unwrap().publisher_sequence, 6); + assert_eq!(ready_sequence(&admitted), Some(5)); + world.engine.advance_transaction(&resident(&world)?)?; // refreshes Ready to 6 + let refreshed = resident(&world)?.value; + assert_eq!(refreshed.phase, DeploymentPhase::AwaitingReady); + assert_eq!(ready_sequence(&refreshed), Some(6)); + world.engine.advance_transaction(&resident(&world)?)?; + assert_eq!(resident(&world)?.value.phase, DeploymentPhase::Routing); + + // Routing: route admission demands a receipt equal to the latest + // reading, so a newer one is adopted before anything is routed. + odin_republishes(&world, &seeded, 7)?; + world.engine.advance_transaction(&resident(&world)?)?; // admits 7 + assert_eq!(ready_sequence(&resident(&world)?.value), Some(6)); + world.engine.advance_transaction(&resident(&world)?)?; // refreshes Ready to 7 + let routing = resident(&world)?.value; + assert_eq!(routing.phase, DeploymentPhase::Routing); + assert_eq!(ready_sequence(&routing), Some(7)); + assert!(routing.routing.is_none()); + world.engine.advance_transaction(&resident(&world)?)?; + assert!(resident(&world)?.value.routing.is_some()); + Ok(()) + } + #[test] fn lease_adoption_names_only_the_granted_lease() -> Result<()> { let lease = write_lease(); diff --git a/src/deployment_plan.rs b/src/deployment_plan.rs index 1a966dc..d4b1cc4 100644 --- a/src/deployment_plan.rs +++ b/src/deployment_plan.rs @@ -467,6 +467,9 @@ pub fn select_dependencies( /// overrides any of them; the resolved values are frozen into the plan. pub const DEFAULT_FENCING_DEADLINE_SECONDS: u32 = 120; pub const DEFAULT_LEASING_DEADLINE_SECONDS: u32 = 120; +// Unsafe for streampixels-service (startup = "create-or-open-after-write-lease"): +// its Postgres migrations run inside AwaitingReady. B5 requires an explicit +// binding value for stateful targets instead of this default. pub const DEFAULT_AWAITING_READY_DEADLINE_SECONDS: u32 = 300; pub const DEFAULT_ROUTING_DEADLINE_SECONDS: u32 = 120; pub const DEFAULT_COMMITTING_DEADLINE_SECONDS: u32 = 120; From 4d8ac7a7e8f3e1a5fcdde40788a95f8b771bdb99 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 03:50:29 +0200 Subject: [PATCH 012/286] Drive the pre-fence abort through the Engine (F0 fix) advance_pre_fencing_abort survived cargo-mutants as Ok(()): the earlier pre-fence abort test ran with candidate cleanup already complete. The seeded transaction now takes its phase, so a Warming transaction is aborted and driven to Complete through the stub workload port, and the abort's terminal write is checked to carry no deadline. Co-Authored-By: Claude Sonnet 5 --- src/control_plane.rs | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 5632d12..fe98e0c 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -9526,13 +9526,16 @@ mod tests { } /// A Continuity transaction for an unrouted, stateless target, sitting at - /// Fencing with every earlier phase's evidence in place. The plan, sealed + /// `phase` (Warming or later) with every earlier phase's evidence in place. The plan, sealed /// release, Expected, activation and Warming evidence are the real /// constructions; only the workload and isolation observations are /// borrowed from a recorded transaction. Continuity is chosen because a /// Deploy would need a frozen source and a signed brake record, which no /// phase past Fencing reads. - fn transaction_at_fencing(world: &EngineFixture) -> Result { + fn transaction_at( + world: &EngineFixture, + phase: DeploymentPhase, + ) -> Result { use crate::deployment_plan::tests::{ BINDING, RECIPE, artifact_receipt, external_input_receipt, source, }; @@ -9639,7 +9642,7 @@ mod tests { evidence: TopologyEvidence::from_authenticated(&authenticated, now)?, }); transaction.odin_publisher_sequence_cursor = 4; - transaction.enter_phase(DeploymentPhase::Fencing, now); + transaction.enter_phase(phase, now); transaction.validate()?; let store = SingleFileMessagePackBackingStore::new(&world.state_store); @@ -9788,7 +9791,7 @@ mod tests { #[test] fn an_admission_runs_from_fencing_to_complete_and_commits() -> Result<()> { let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let seeded = transaction_at_fencing(&world)?; + let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; odin_reports_ready(&world, &seeded, 5)?; let phases = drive(&world, |transaction| { @@ -9820,10 +9823,32 @@ mod tests { Ok(()) } + #[test] + fn a_pre_fence_abort_runs_to_complete_without_a_deadline() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + transaction_at(&world, DeploymentPhase::Warming)?; + let warming = resident(&world)?; + assert_eq!(warming.value.phase_deadline, None); + + world + .engine + .begin_pre_fencing_abort(&warming, anyhow!("candidate refused"))?; + drive(&world, |transaction| transaction.completion.is_some())?; + let finished = latest(&world)?; + assert_eq!(finished.phase, DeploymentPhase::Complete); + assert_eq!(finished.phase_deadline, None); + assert!(matches!( + finished.completion, + Some(TransactionCompletion::FailedBeforeFencing { .. }) + )); + assert!(finished.is_terminal()); + Ok(()) + } + #[test] fn a_post_fence_abort_runs_to_complete() -> Result<()> { let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let seeded = transaction_at_fencing(&world)?; + let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; odin_reports_ready(&world, &seeded, 5)?; drive(&world, |transaction| { transaction.phase == DeploymentPhase::Routing @@ -9997,7 +10022,7 @@ mod tests { /// declares no Odin dependency, so route proof is its class. fn route_proof_generation() -> Result { let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let seeded = transaction_at_fencing(&world)?; + let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; odin_reports_ready(&world, &seeded, 5)?; drive(&world, |transaction| { transaction.phase == DeploymentPhase::Committing @@ -10226,7 +10251,7 @@ mod tests { fn a_newer_odin_reading_refreshes_the_receipt_before_awaiting_ready_or_routing_advance() -> Result<()> { let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let seeded = transaction_at_fencing(&world)?; + let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; odin_reports_ready(&world, &seeded, 5)?; // AwaitingReady: the receipt is written, then confirmed, then the From c03d646ae62585283307459c05907ffe275db822 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 04:15:58 +0200 Subject: [PATCH 013/286] Record S3 and C2 code status; hold the StreamPixels merge until B3 Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index 603c0dd..5d30194 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -586,6 +586,22 @@ it on conditions, and a fix batch is in Hands: - **B3 surfaces the typed capacity shortfall** on the route path. Today it reads as a generic "disagrees with current authority" (`control_plane.rs:5190`). +**S3 + C2 code status, 2026-09-29 (Self).** Landed on StreamPixels `route/s3-c2` (`8e6d891`), +after Soul and a fix batch: +- `vendor/CultLib` is at `30ee8b9`, and the web has no Odin dependency or env. +- Both apps answer through the signer, with one health owner, `driveRuntimeHealth`. +- CI now builds the vendored gitlink, not a sibling `main`. +- A production-named launcher test runs `runIdunnRuntime`. It caught a dropped `publish` + argument that would have broken the service's write lease. +- Checks: 172 tests, typecheck, the web build and the release assembly. +**Not merged to StreamPixels `main` until B3 lands.** Idunn cannot yet admit a web with no +Odin dependency, so a deploy of `main` would fail readiness. +Follow-up: the production fixtures are generated by applying +`test-data/idunn-runtime/fixture.patch` to CultLib's `idunn_runtime_fixture.rs`. The generator +should take the target, contract and dependencies as input, so no consumer patches it. +Recorded: web `/api/healthz` is constant `{ok:true}`, so the route proves "HTTP answers", not +store health. + **C2 — StreamPixels web binding proves invariant 3 end to end (StreamPixels, after C1, S3, B3).** - Point the gitlink at the C1 commit. From 9e5b7f84fd7fcd177dbb2d818f86afc603419de4 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 06:11:29 +0200 Subject: [PATCH 014/286] Close the B1 transition: lift, precondition, boot reconciliation, closed history Soul found B1's rule sound and its transition unsafe. This makes the rule's old inputs impossible to reintroduce a stuck state from. - The control store's legacy lift owns the transition. A pre-B1 continuity abort that issued an activation lifts with topology_reconciliation Pending, so the one resolution demotes its own activation. A record already terminal under the old rule is reopened at Starting, since a terminal record cannot owe work. History lifts records as written. Before this, one resident record of that shape failed validation and ControlSnapshot::read refused the whole store, so Idunn could not boot. - Supervision owns the Expected-only precondition. When its pre-restart demotion fails it mints no continuity and writes next_restart_at_unix_millis on the generation's ContinuityBackoff (attempts and window untouched), so it neither restarts over an undemotable projection nor retries every tick. - Boot reconciles once: a projected activation that exactly matches the activation a failed continuity in history (or a resident terminal record) issued is demoted. Nothing is adopted; an activation no admitted generation, live transaction or failed issuer accounts for is reported and left alone. New read-only CultCacheTopologyDriver::projected_activation. - Unreadable history.cc fails closed. The continuity restart ceiling and the consumed-command check no longer read an empty history; they stop, and the fault is reported once while it lasts (ReportOnce). The undecodable-record report is a pure function, pinned. - docs/deployment-authority.md no longer says the projection never drifts. Tests: legacy fixtures written by the 9001b58 encoder for the three abort shapes (a second throwaway generator, see the fixtures README), driven through the Engine; the seeded-transaction helper now takes a command kind and an incumbent, and a Deploy carries a frozen-source receipt and a signed brake release, so Deploy commit, Deploy pre- and post-fence aborts and a continuity abort over an incumbent run through the real phase machine. Co-Authored-By: Claude Sonnet 5 --- docs/deployment-authority.md | 16 +- src/control_plane.rs | 1444 ++++++++++++++++- src/drivers.rs | 16 + tests/fixtures/idunn-control-legacy/README.md | 20 + .../idunn-control-legacy/provider-anchor.hex | 1 + .../transaction-post-fence-abort.hex | 2 + .../transaction-pre-fence-abort-terminal.hex | 2 + .../transaction-pre-fence-abort.hex | 2 + 8 files changed, 1434 insertions(+), 69 deletions(-) create mode 100644 tests/fixtures/idunn-control-legacy/provider-anchor.hex create mode 100644 tests/fixtures/idunn-control-legacy/transaction-post-fence-abort.hex create mode 100644 tests/fixtures/idunn-control-legacy/transaction-pre-fence-abort-terminal.hex create mode 100644 tests/fixtures/idunn-control-legacy/transaction-pre-fence-abort.hex diff --git a/docs/deployment-authority.md b/docs/deployment-authority.md index 0ee3483..da7f028 100644 --- a/docs/deployment-authority.md +++ b/docs/deployment-authority.md @@ -314,9 +314,19 @@ window. A failed continuity restart leaves its own candidate activation projected under the shared Expected key. Its abort demotes that key to Expected-only with the -exact activation the transaction issued, so the projection never drifts from the -admitted receipt. Rollback after fencing demotes the admitted generation's exact -activation. No path adopts an activation the transaction did not issue. +exact activation the transaction issued. Rollback after fencing demotes the +admitted generation's exact activation. No path adopts an activation the +transaction did not issue. + +A projection can still hold an activation the abort never resolved: a failure +between preparing a candidate and publishing its observation, an abort written +by an earlier Idunn, or an operator edit. Continuity therefore restarts a +generation only from an Expected-only projection: when its own demotion is +refused it mints no restart, records the deferral in the generation's +continuity backoff, and tries again after it. At boot, Idunn demotes an +activation still projected for a failed continuity in history, by that +transaction's exact activation, and only reports one no failed transaction +issued. Idunn starts and recovers from its own durable admitted state. Odin is the first managed semantic daemon, never an Idunn bootstrap dependency. Odin's initial diff --git a/src/control_plane.rs b/src/control_plane.rs index cd82a0e..6c97384 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -62,6 +62,10 @@ const ODIN_RENDEZVOUS_CAPABILITY: &str = "odin.verse-rendezvous"; /// and few because each attempt holds the target against any deployment. const CONTINUITY_RESTART_ATTEMPTS: usize = 3; +/// How long continuity waits after it could not prepare a restart (the +/// projection would not demote) before trying again. +const CONTINUITY_DEFERRAL_MILLIS: u64 = 30_000; + const DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS: u64 = 30_000; const DEFAULT_TOPOLOGY_MAXIMUM_FUTURE_SKEW_MILLIS: u64 = 2_000; @@ -2693,13 +2697,46 @@ fn read_transaction_record(envelope: &CultCacheEnvelope) -> Result { - let value = lift_legacy_transaction(envelope)?; + let mut value = lift_legacy_transaction(envelope)?; + owe_legacy_continuity_projection(&mut value); value.validate()?; Ok(value) } } } +/// A continuity abort written before the single resolution rule recorded no +/// projection cleanup although its transaction had issued an activation, and +/// so left that activation standing. The current rule owes the demotion, so a +/// resident record of that shape lifts to `Pending`: the one resolution then +/// demotes the transaction's own activation and cleans the residue too. +/// +/// A record already terminal under the old rule is reopened at Starting (a +/// pre-fencing failure is before Fencing by definition) with its completion +/// dropped, because a terminal record cannot owe work. Only the control +/// store's read applies this: history describes what happened and is lifted +/// unchanged. +fn owe_legacy_continuity_projection(transaction: &mut DeploymentTransaction) { + if transaction.command_kind != CommandKind::Continuity || transaction.activation.is_none() { + return; + } + if let Some(abort) = transaction.pre_fencing_abort.as_mut() + && abort.topology_reconciliation == CleanupEvidence::Skipped + { + abort.topology_reconciliation = CleanupEvidence::Pending; + if transaction.completion.take().is_some() { + transaction.phase = DeploymentPhase::Starting; + transaction.phase_deadline = None; + } + } + if let Some(abort) = transaction.post_fencing_abort.as_mut() + && abort.topology_reconciliation == CleanupEvidence::Skipped + && transaction.completion.is_none() + { + abort.topology_reconciliation = CleanupEvidence::Pending; + } +} + /// Read one stored admitted generation, lifting a v2 record to the current /// shape by the same rule as `read_transaction_record`. fn read_generation_record(envelope: &CultCacheEnvelope) -> Result { @@ -2867,41 +2904,92 @@ fn archive_terminal_transaction(state_store: &Path, envelope: &CultCacheEnvelope Ok(()) } -/// Finished transactions, read leniently and for display only. -/// -/// `control.cc` keeps the byte-exact canonical check because its records still -/// gate decisions. History describes what already happened, so an unreadable -/// entry here is skipped rather than allowed to refuse the read -- that -/// asymmetry is the whole reason the two files are separate. -fn read_history_transactions(state_store: &Path) -> Vec { +/// What `history.cc` yielded: the transactions it decoded and the entries it +/// could not, each with its key and reason. +struct HistoryRead { + transactions: Vec, + undecodable: Vec<(String, String)>, +} + +impl HistoryRead { + /// What is missing from `transactions`, or `None` when nothing is. + fn report(&self) -> Option { + if self.undecodable.is_empty() { + return None; + } + let keys = self + .undecodable + .iter() + .map(|(key, error)| format!("{key} ({error})")) + .collect::>() + .join("; "); + Some(format!( + "Idunn history: {} archived transaction(s) could not be decoded and are not counted \ + by status or continuity backoff: {keys}", + self.undecodable.len() + )) + } +} + +/// Finished transactions. `control.cc` keeps the byte-exact canonical check +/// because its records still gate decisions. History describes what already +/// happened, so an undecodable entry is reported and skipped rather than +/// allowed to refuse the read -- that asymmetry is the whole reason the two +/// files are separate. A file that cannot be read at all is an error: the +/// caller decides whether an empty answer is safe. +fn read_history(state_store: &Path) -> Result { let path = history_store_path(state_store); if !path.exists() { - return Vec::new(); + return Ok(HistoryRead { + transactions: Vec::new(), + undecodable: Vec::new(), + }); } - let envelopes = match SingleFileMessagePackBackingStore::new(&path).pull_all_read_only_snapshot() - { - Ok(envelopes) => envelopes, - Err(error) => { - eprintln!( - "Idunn cannot read {}: {error:#}; every archived transaction is invisible \ - to status and to continuity backoff", - path.display() - ); - return Vec::new(); - } - }; + let envelopes = SingleFileMessagePackBackingStore::new(&path) + .pull_all_read_only_snapshot() + .with_context(|| format!("Idunn cannot read {}", path.display()))?; let (transactions, undecodable) = decode_history_transactions(envelopes); - if !undecodable.is_empty() { - for (key, error) in &undecodable { - eprintln!("Idunn history holds an undecodable transaction {key}: {error}"); + Ok(HistoryRead { + transactions, + undecodable, + }) +} + +/// History for display only (status). An unreadable file shows as empty and +/// says so; decisions use `Engine::history_for_decision`, which does not. +fn read_history_transactions(state_store: &Path) -> Vec { + match read_history(state_store) { + Ok(history) => { + if let Some(report) = history.report() { + eprintln!("{report}"); + } + history.transactions + } + Err(error) => { + eprintln!("{error:#}; every archived transaction is invisible to status"); + Vec::new() } - eprintln!( - "Idunn history: {} archived transaction(s) could not be decoded and are not counted \ - by status or continuity backoff", - undecodable.len() - ); } - transactions +} + +/// Says a fault once while it lasts. A scheduler tick that hits the same +/// unreadable file every half second must not print it every half second. +#[derive(Default)] +struct ReportOnce { + last: Mutex>, +} + +impl ReportOnce { + /// The text to print now: `report` when it differs from the last one + /// offered, nothing while the same fault stands or once it has cleared. + fn offer(&self, report: Option) -> Option { + let mut last = self.last.lock().expect("report mutex"); + if *last == report { + return None; + } + last.clone_from(&report); + report + } } /// Decode every archived transaction. An entry that fails is returned with its @@ -3382,6 +3470,7 @@ struct Engine { docker_runner: DockerRunnerDriver, systemd_workload: Arc, host_actuators: Option, + history_report: ReportOnce, } impl Engine { @@ -3436,6 +3525,7 @@ impl Engine { docker_runner: DockerRunnerDriver::default(), systemd_workload, host_actuators, + history_report: ReportOnce::default(), }) } @@ -3512,6 +3602,27 @@ impl Engine { }) } + /// History for a decision that would be wrong on an empty answer: the + /// continuity restart ceiling and the check that a command is consumed. + /// `None` means `history.cc` cannot be read, so the decision is not made; + /// a history that merely has no entries is `Some`. The fault is printed + /// once while it lasts, not on every tick. + fn history_for_decision(&self) -> Option> { + let (report, transactions) = match read_history(&self.options.state_store) { + Ok(history) => (history.report(), Some(history.transactions)), + Err(error) => ( + Some(format!( + "{error:#}; continuity and command retirement stop until it is readable" + )), + None, + ), + }; + if let Some(text) = self.history_report.offer(report) { + eprintln!("{text}"); + } + transactions + } + fn topology(&self) -> CultCacheTopologyDriver { CultCacheTopologyDriver { projection_store: self.options.topology_store.clone(), @@ -3684,6 +3795,96 @@ impl Engine { } } +/// What the boot reconciliation of failed continuity projections found. +#[derive(Debug, Default, PartialEq, Eq)] +struct ProjectionReconciliation { + /// Failed continuity transactions whose exact activation was demoted. + demoted: Vec, + /// Targets whose projection names an activation that neither the admitted + /// generation, a live transaction, nor a failed issuer accounts for. They + /// are reported and left alone. + unexplained: Vec, +} + +impl Engine { + /// Once at boot: demote the activation a failed continuity left standing, + /// where the projection still names exactly the activation that failed + /// transaction issued. A continuity abort before the single resolution + /// rule left such residue (and the lift owes the same demotion to any + /// abort still resident). The issuer is identified by exact activation, + /// from history and from resident terminal records not yet retired to it. + /// + /// Nothing is adopted: an activation with no failed issuer is never + /// demoted or kept on anyone's behalf, only reported. + fn reconcile_failed_continuity_projections(&self) -> Result { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let mut outcome = ProjectionReconciliation::default(); + let Some(history) = self.history_for_decision() else { + return Ok(outcome); + }; + let topology = self.topology(); + let issuers = history + .iter() + .chain(snapshot.transactions.iter().map(|stored| &stored.value)) + .filter(|transaction| { + transaction.command_kind == CommandKind::Continuity + && matches!( + transaction.completion, + Some(TransactionCompletion::FailedBeforeFencing { .. }) + | Some(TransactionCompletion::FailedAfterFencing { .. }) + ) + }); + for issuer in issuers { + let (Some(expected), Some(activation), Some(plan)) = + (&issuer.expected, &issuer.activation, &issuer.plan) + else { + continue; + }; + let demoted = (|| -> Result { + if topology.projected_activation(expected)?.as_ref() != Some(activation) { + return Ok(false); + } + let provider_anchor = self.provider_anchor_for_plan(plan)?; + topology.demote_to_expected_only(expected, &provider_anchor, activation, None)?; + Ok(true) + })(); + match demoted { + Ok(true) => outcome.demoted.push(issuer.transaction_id.clone()), + Ok(false) => {} + Err(error) => eprintln!( + "Idunn could not reconcile the projection of failed continuity {}: {error:#}", + issuer.transaction_id + ), + } + } + for generation in &snapshot.admitted { + let projected = match topology.projected_activation(&generation.value.expected) { + Ok(projected) => projected, + Err(error) => { + eprintln!( + "Idunn cannot read the projected activation of {}: {error:#}", + generation.value.target + ); + continue; + } + }; + let Some(projected) = projected else { + continue; + }; + let accounted = projected == generation.value.activation + || snapshot.transactions.iter().any(|stored| { + stored.value.target == generation.value.target + && !stored.value.is_terminal() + && stored.value.activation.as_ref() == Some(&projected) + }); + if !accounted { + outcome.unexplained.push(generation.value.target.clone()); + } + } + Ok(outcome) + } +} + fn serve(options: RuntimeOptions) -> Result<()> { for path in [ &options.state_store, @@ -3710,6 +3911,20 @@ fn serve(options: RuntimeOptions) -> Result<()> { ControlSnapshot::read(&options.state_store).context("validating all Idunn records")?; let engine = Engine::open(options)?; engine.validate_durable_authority(&ControlSnapshot::read(&engine.options.state_store)?)?; + match engine.reconcile_failed_continuity_projections() { + Ok(outcome) => { + for transaction_id in &outcome.demoted { + println!("demoted the activation failed continuity {transaction_id} left projected"); + } + for target in &outcome.unexplained { + eprintln!( + "Idunn found an activation projected for {target} that no admitted generation, \ + live transaction or failed continuity accounts for; left as it is" + ); + } + } + Err(error) => eprintln!("Idunn boot projection reconciliation failed: {error:#}"), + } loop { match engine.run_scheduler_tick() { @@ -3848,7 +4063,12 @@ impl Engine { // whose transactions were retired to history before commands travelled // with them are still resident; they are not queued, they are // history that never moved. Retire them here and never freeze them. - let historical_commands = read_history_transactions(&self.options.state_store) + // With history unreadable, whether a command was consumed is unknown: + // freezing it again could run a finished deployment twice. + let Some(history) = self.history_for_decision() else { + return Ok(false); + }; + let historical_commands = history .into_iter() .map(|transaction| transaction.command_id) .collect::>(); @@ -4165,12 +4385,17 @@ impl Engine { | Some(TransactionCompletion::FailedAfterFencing { .. }) ) }; + // An unreadable history counts as zero refusals only by lying: + // the ceiling would never trip. Without it, nothing is actuated. + let Some(history) = self.history_for_decision() else { + continue; + }; let refused_restarts = snapshot .transactions .iter() .filter(|stored| is_refused_restart(&stored.value)) .count() - + read_history_transactions(&self.options.state_store) + + history .iter() .filter(|transaction| is_refused_restart(transaction)) .count(); @@ -4218,6 +4443,17 @@ impl Engine { // Demote to Expected-only, the same shape an aborted deployment // leaves, and let the restart publish its own activation once it is // observed. + let now = now_millis()?; + // A restart whose projection could not be prepared is deferred in + // the generation's own backoff, not retried every tick. + if current + .value + .continuity_backoff + .next_restart_at_unix_millis + .is_some_and(|not_before| now < not_before) + { + continue; + } let demotion = (|| -> Result<()> { let topology = self.topology(); let provider_anchor = self.provider_anchor_for_plan(¤t.value.plan)?; @@ -4232,13 +4468,21 @@ impl Engine { )?; Ok(()) })(); + // Expected-only is the precondition of a restart: a continuity + // minted over a projection that still names an activation would + // fail between preparing and publishing its own, and its abort + // could not resolve. So a failed demotion mints nothing. The + // failure is written to the generation, and the next attempt is + // not before the deferral ends. if let Err(error) = demotion { eprintln!( - "Idunn preserved admitted {} after refusing to demote its projection: {error:#}", + "Idunn deferred continuity for admitted {} after refusing to demote its projection: {error:#}", current.value.target ); + self.defer_continuity_restart(current, now)?; + progressed = true; + continue; } - let now = now_millis()?; // An engaged lifecycle brake means no continuity transaction is // minted at all. A transaction that exists and waits on the brake // owns the target, and a target owned by a parked restart accepts @@ -4286,6 +4530,29 @@ impl Engine { Ok(progressed) } + /// Record that continuity may not restart this generation before + /// `now + CONTINUITY_DEFERRAL_MILLIS`. The generation's `ContinuityBackoff` + /// owns "not before": only its `next_restart_at_unix_millis` is written, + /// so the attempts and window that count real restarts are untouched. + fn defer_continuity_restart(&self, current: &Stored, now: u64) -> Result<()> { + let mut next = current.value.clone(); + next.continuity_backoff.next_restart_at_unix_millis = + Some(now + CONTINUITY_DEFERRAL_MILLIS); + next.validate()?; + ensure!( + SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: AdmittedGeneration::TYPE.into(), + key: current.value.target.clone(), + current: Some(current.envelope.clone()), + }], + &[admitted_envelope(&next, now)?], + )?, + "admitted generation changed before continuity deferral CAS" + ); + Ok(()) + } + fn supervise_admitted_route(&self, current: &Stored) -> Result { let Some(expected_route) = current.value.expected.route.as_ref() else { ensure!( @@ -9541,25 +9808,107 @@ mod tests { } } - /// A Continuity transaction for an unrouted, stateless target, sitting at - /// `phase` (Warming or later) with every earlier phase's evidence in place. The plan, sealed - /// release, Expected, activation and Warming evidence are the real - /// constructions; only the workload and isolation observations are - /// borrowed from a recorded transaction. Continuity is chosen because a - /// Deploy would need a frozen source and a signed brake record, which no - /// phase past Fencing reads. + /// A Continuity transaction for an unrouted, stateless target with no + /// incumbent: the shape most Engine tests need. fn transaction_at( world: &EngineFixture, phase: DeploymentPhase, + ) -> Result { + seeded_transaction(world, phase, CommandKind::Continuity, None) + } + + /// The Odin publisher sequence a transaction seeded over `incumbent` + /// warms at; its Ready is the next one. Above anything an earlier + /// generation of the world admitted. + fn seeded_sequence(incumbent: Option<&AdmittedGeneration>) -> u64 { + if incumbent.is_some() { 14 } else { 4 } + } + + /// A released deployment brake receipt, signed by an operator identity + /// enrolled in the world: what a Deploy's `deployment_authorization` holds. + fn deployment_authorization( + world: &EngineFixture, + expected: &IdunnExpectedIncarnationRecord, + transaction_id: &str, + now: u64, + ) -> Result { + use cultnet_rs::{ + IDUNN_DEPLOYMENT_BRAKE_AUTHORITY, IDUNN_DEPLOYMENT_BRAKE_ID, + IDUNN_DEPLOYMENT_BRAKE_SCOPE, IDUNN_DEPLOYMENT_RELEASE_PURPOSE, + IdunnDeploymentBrakeReleasePurpose, enroll_service_identity_at, + }; + let signer = enroll_service_identity_at::( + &world.root.join("identities/brake-operator.cc"), + )?; + export_service_identity_trust_anchor(&signer, &world.root.join("brake-anchor.cc"))?; + let mut record = IdunnDeploymentBrakeRecord { + schema_version: IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into(), + brake_id: IDUNN_DEPLOYMENT_BRAKE_ID.into(), + authority: IDUNN_DEPLOYMENT_BRAKE_AUTHORITY.into(), + runtime_id: expected.runtime_id.clone(), + status: "released".into(), + scope: IDUNN_DEPLOYMENT_BRAKE_SCOPE.into(), + reason: "test rollout".into(), + observed_at_unix_millis: now, + expires_at_unix_millis: Some(now + 600_000), + authorization_id: Some("authorization-test".into()), + authorization_purpose: Some(IDUNN_DEPLOYMENT_RELEASE_PURPOSE.into()), + authorized_release_id: Some(expected.sealed_release_id.clone()), + authorized_deployment_id: Some(transaction_id.to_owned()), + authorized_by: Some(signer.trust_anchor()?.identity_id), + authorization_issued_at_unix_millis: Some(now), + authorization_expires_at_unix_millis: Some(now + 600_000), + signature_algorithm: Some("ed25519".into()), + signature: None, + private_state_exposed: false, + updated_by: "operator/test".into(), + }; + record.signature = Some( + signer + .sign::(&rmp_serde::to_vec(&record)?) + .signature, + ); + record.validate()?; + let canonical_brake_bytes = rmp_serde::to_vec(&record)?; + Ok(DeploymentAuthorization { + authorization_id: "authorization-test".into(), + brake_sha256: sha256_id(&canonical_brake_bytes), + canonical_brake_bytes, + authorized_at_unix_millis: now, + }) + } + + /// A transaction for an unrouted, stateless target, sitting at `phase` + /// (Warming or later) with every earlier phase's evidence in place. The + /// plan, sealed release, Expected, activation and Warming evidence are the + /// real constructions; only the workload and isolation observations are + /// borrowed from a recorded transaction. + /// + /// `incumbent` is an admitted generation already in the world's store. A + /// Continuity over it restarts that generation's own release, so it shares + /// the incumbent's Expected key. A Deploy over it builds a second + /// incarnation. A Deploy carries what phases past Fencing never read but + /// validation requires: a three-field frozen-source receipt and a + /// well-formed release receipt from the brake. + fn seeded_transaction( + world: &EngineFixture, + phase: DeploymentPhase, + kind: CommandKind, + incumbent: Option<&AdmittedGeneration>, ) -> Result { use crate::deployment_plan::tests::{ BINDING, RECIPE, artifact_receipt, external_input_receipt, source, }; - let provider = enroll_service_identity_at::( - &world.root.join("identities/provider.cc"), - )?; + let provider_path = world.root.join("identities/provider.cc"); let provider_anchor = world.root.join("identities/provider-anchor.cc"); - export_service_identity_trust_anchor(&provider, &provider_anchor)?; + let provider = if provider_path.exists() { + open_service_identity_at::(&provider_path)? + } else { + let provider = + enroll_service_identity_at::(&provider_path)?; + export_service_identity_trust_anchor(&provider, &provider_anchor)?; + provider + }; let (binding_head, binding_tail) = BINDING .split_once("[route]") @@ -9585,34 +9934,60 @@ mod tests { r#"["GAMECULT_IDUNN_CANDIDATE_BIND", "GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, r#"["GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, ); - let plan = compile_deployment_plan( - recipe.as_bytes(), - binding.as_bytes(), - source(&recipe), - "service-incarnation-1", - None, - 110, - &[], - )?; - let release = SealedRelease::new( - &plan, - vec![artifact_receipt()], - vec![external_input_receipt()], - 120, - )?; + let (plan, release) = match (kind, incumbent) { + (CommandKind::Continuity, Some(incumbent)) => ( + incumbent.plan.clone(), + incumbent.sealed_release.clone(), + ), + _ => { + let incarnation = if incumbent.is_some() { + "service-incarnation-2" + } else { + "service-incarnation-1" + }; + let plan = compile_deployment_plan( + recipe.as_bytes(), + binding.as_bytes(), + source(&recipe), + incarnation, + None, + 110, + &[], + )?; + let release = SealedRelease::new( + &plan, + vec![artifact_receipt()], + vec![external_input_receipt()], + 120, + )?; + (plan, release) + } + }; let expected = release.expected_projection(&plan)?; assert!(!expected.write_lease_required && expected.route.is_none()); let now = now_millis()?; let command = DeploymentCommand { schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: "continuity-service".into(), - kind: CommandKind::Continuity, + command_id: format!( + "{}-service{}", + match kind { + CommandKind::Deploy => "up", + CommandKind::Continuity => "continuity", + }, + if incumbent.is_some() { "-again" } else { "" } + ), + kind, selector: "service".into(), requested_by: "test".into(), requested_at_unix_millis: 100, }; - let mut transaction = DeploymentTransaction::new(&command, "service".into(), 0, None, now)?; + let mut transaction = match (kind, incumbent) { + (CommandKind::Continuity, Some(incumbent)) => { + DeploymentTransaction::from_continuity(&command, incumbent, now)? + } + _ => DeploymentTransaction::new(&command, "service".into(), 0, incumbent, now)?, + }; let activation = IdunnRuntimeActivationLaunch::issue( &expected, runtime_instance_id(&transaction.transaction_id)?, @@ -9625,12 +10000,30 @@ mod tests { .1 .workload .context("recorded transaction has no workload")?; - transaction.lifecycle_authorized_at_unix_millis = Some(now); + match kind { + CommandKind::Continuity => { + transaction.lifecycle_authorized_at_unix_millis = Some(now); + } + CommandKind::Deploy => { + transaction.frozen_source = Some(FrozenSourceReceipt { + transaction_id: transaction.transaction_id.clone(), + plan_id: plan.plan_id.clone(), + snapshot_sha256: sha256_id(b"frozen source snapshot"), + }); + transaction.deployment_authorization = Some(deployment_authorization( + world, + &expected, + &transaction.transaction_id, + now, + )?); + } + } transaction.expected_publication_sha256 = Some(expected.canonical_sha256()?); transaction.activation_publication_sha256 = Some(activation.canonical_sha256()?); match &mut workload { WorkloadObservation::Systemd(observed) => { observed.runtime_instance_id = activation.runtime_instance_id.clone(); + observed.executable_sha256 = expected.artifact_sha256.clone(); } WorkloadObservation::Host(_) => bail!("recorded workload is not a systemd unit"), } @@ -9646,7 +10039,8 @@ mod tests { transaction.workload = Some(workload); transaction.plan = Some(plan); // Odin's first word about the candidate, before it was Ready. - let warming = signed_correlation(world, &transaction, 4, false)?; + let sequence = seeded_sequence(incumbent); + let warming = signed_correlation(world, &transaction, sequence, false)?; let authenticated = world.engine.authenticate_topology_bytes( &ControlSnapshot::read(&world.state_store)?, &transaction, @@ -9657,7 +10051,7 @@ mod tests { transaction.warming = Some(WarmingEvidence::OdinTopology { evidence: TopologyEvidence::from_authenticated(&authenticated, now)?, }); - transaction.odin_publisher_sequence_cursor = 4; + transaction.odin_publisher_sequence_cursor = sequence; transaction.enter_phase(phase, now); transaction.validate()?; @@ -10564,4 +10958,922 @@ mod tests { assert_eq!(std::fs::read(&world.topology.projection_store)?, before); Ok(()) } + + // --------------------------------------------------------------------- + // B1 fix batch: the transition, the precondition, the boot reconciliation, + // history that fails closed, and the abort paths through the Engine. + // --------------------------------------------------------------------- + + /// A workload port whose process can be killed: observing it succeeds + /// until `kill`, then fails the way a dead unit does. Everything else is + /// `StillWorkload`'s. + struct SwitchWorkload { + alive: std::sync::atomic::AtomicBool, + } + + impl SwitchWorkload { + fn new() -> Arc { + Arc::new(Self { + alive: std::sync::atomic::AtomicBool::new(true), + }) + } + + fn kill(&self) { + self.alive + .store(false, std::sync::atomic::Ordering::SeqCst); + } + } + + impl WorkloadPort for SwitchWorkload { + fn install( + &self, + plan: &CompiledDeploymentPlan, + release: &crate::drivers::MaterializedRelease, + ) -> Result { + StillWorkload.install(plan, release) + } + fn prepare_activation( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + launch: IdunnRuntimeActivationLaunch, + ) -> Result { + StillWorkload.prepare_activation(plan, expected, launch) + } + fn start_prepared( + &self, + plan: &CompiledDeploymentPlan, + release: &SealedRelease, + installed: &crate::drivers::InstalledReleaseObservation, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result { + StillWorkload.start_prepared(plan, release, installed, expected, activation) + } + fn discard_prepared( + &self, + _: &CompiledDeploymentPlan, + _: &IdunnExpectedIncarnationRecord, + _: &IdunnRuntimeActivationRecord, + ) -> Result<()> { + Ok(()) + } + fn observe( + &self, + _: &IdunnExpectedIncarnationRecord, + _: &IdunnRuntimeActivationRecord, + prior: &WorkloadObservation, + ) -> Result { + ensure!( + self.alive.load(std::sync::atomic::Ordering::SeqCst), + "the workload is gone" + ); + Ok(prior.clone()) + } + fn stop(&self, _: &WorkloadObservation) -> Result<()> { + Ok(()) + } + fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { + Ok(false) + } + } + + /// Admit a generation for "service" through the real phase machine and + /// retire its transaction, leaving the world with an incumbent. + fn admit_incumbent(world: &EngineFixture) -> Result { + let seeded = transaction_at(world, DeploymentPhase::Fencing)?; + odin_reports_ready(world, &seeded, seeded_sequence(None) + 1)?; + drive(world, |transaction| { + transaction.phase == DeploymentPhase::Complete + })?; + assert!(world.engine.retire_one_terminal_transaction()?); + Ok(ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("commit wrote no admitted generation")? + .value + .clone()) + } + + /// Change the incumbent in the store, compare-and-swap. + fn edit_incumbent( + world: &EngineFixture, + edit: impl FnOnce(&mut AdmittedGeneration), + ) -> Result { + let snapshot = ControlSnapshot::read(&world.state_store)?; + let stored = snapshot + .admitted_for("service") + .context("no admitted generation")?; + let mut next = stored.value.clone(); + edit(&mut next); + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: AdmittedGeneration::TYPE.into(), + key: "service".into(), + current: Some(stored.envelope.clone()), + }], + &[admitted_envelope(&next, now_millis()?)?], + )? + ); + Ok(next) + } + + fn incumbent_envelope(world: &EngineFixture) -> Result { + Ok(ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("no admitted generation")? + .envelope + .clone()) + } + + /// Publish what a transaction's candidate projects: its Expected and, + /// when asked, its observed activation. + fn project_candidate( + world: &EngineFixture, + transaction: &DeploymentTransaction, + with_activation: bool, + ) -> Result<()> { + let expected = transaction.expected.as_ref().context("no Expected")?; + let anchor = world + .engine + .provider_anchor_for_plan(transaction.plan.as_ref().context("no plan")?)?; + let topology = world.engine.topology(); + topology.publish_expected(expected, &anchor)?; + if with_activation { + // The observation must name the release the Expected declares; + // the borrowed workload is patched to. + let mut workload = transaction.workload.clone().context("no workload")?; + if let WorkloadObservation::Systemd(observed) = &mut workload { + observed.executable_sha256 = expected.artifact_sha256.clone(); + } + topology.publish_observed_activation( + expected, + transaction.activation.as_ref().context("no activation")?, + &workload, + )?; + } + Ok(()) + } + + /// Publish what an admitted generation projects while it runs. + fn project_admitted(world: &EngineFixture, generation: &AdmittedGeneration) -> Result<()> { + let anchor = world.engine.provider_anchor_for_plan(&generation.plan)?; + let topology = world.engine.topology(); + topology.publish_expected(&generation.expected, &anchor)?; + topology.publish_observed_activation( + &generation.expected, + &generation.activation, + &generation.workload, + )?; + Ok(()) + } + + /// The types of the records projected under one incarnation key, sorted. + fn projected_under( + world: &EngineFixture, + expected: &IdunnExpectedIncarnationRecord, + ) -> Result> { + let path = &world.engine.options.topology_store; + if !path.exists() { + return Ok(Vec::new()); + } + let key = incarnation_key(expected)?; + let mut types = SingleFileMessagePackBackingStore::new(path) + .pull_all_read_only_snapshot()? + .into_iter() + .filter(|entry| entry.key == key) + .map(|entry| entry.r#type) + .collect::>(); + types.sort(); + Ok(types) + } + + /// One transaction by id, wherever it now lives. A failed transaction is + /// archived the moment it finishes, so it is in history, not resident. + fn record_of(world: &EngineFixture, transaction_id: &str) -> Result { + if let Some(stored) = ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .find(|stored| stored.value.transaction_id == transaction_id) + { + return Ok(stored.value); + } + read_history_transactions(&world.state_store) + .into_iter() + .find(|transaction| transaction.transaction_id == transaction_id) + .context("the transaction is in neither the live set nor history") + } + + fn expected_only() -> Vec { + vec![IdunnExpectedIncarnationRecord::TYPE.to_owned()] + } + + // ---- the Engine's abort paths, Deploy and incumbent ---- + + #[test] + fn a_deploy_that_fences_commits_and_retires_the_incumbent_it_replaced() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + admit_incumbent(&world)?; + let backoff = ContinuityBackoff { + window_started_at_unix_millis: Some(1_000), + attempts: 2, + next_restart_at_unix_millis: Some(9_000), + }; + let incumbent = edit_incumbent(&world, |generation| { + generation.continuity_backoff = backoff.clone(); + })?; + + let candidate = seeded_transaction( + &world, + DeploymentPhase::Fencing, + CommandKind::Deploy, + Some(&incumbent), + )?; + odin_reports_ready(&world, &candidate, seeded_sequence(Some(&incumbent)) + 1)?; + drive(&world, |transaction| { + transaction.phase == DeploymentPhase::Complete + })?; + + let admitted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("the deploy admitted nothing")? + .value + .clone(); + assert_eq!(admitted.transaction_id, candidate.transaction_id); + assert_ne!(admitted.generation_id, incumbent.generation_id); + // The restart ceiling belongs to the target: a commit must not reset it. + assert_eq!(admitted.continuity_backoff, backoff); + assert_eq!(admitted.route_supervision, incumbent.route_supervision); + + // The commit did not stop the incumbent: it is retired after. + let committed = record_of(&world, &candidate.transaction_id)?; + let cleanup = committed.post_commit_cleanup.clone().context("no cleanup")?; + assert_eq!( + cleanup.incumbent, + IncumbentCleanupEvidence::Pending { + generation_id: incumbent.generation_id.clone(), + workload: incumbent.workload.clone(), + } + ); + assert_eq!(cleanup.source, SourceCleanupEvidence::Pending); + let frozen = world + .engine + .options + .staging_root + .join("frozen-sources") + .join(&candidate.transaction_id); + std::fs::create_dir_all(&frozen)?; + std::fs::write(frozen.join("tree"), b"frozen")?; + + drive(&world, |transaction| transaction.is_terminal())?; + let retired = record_of(&world, &candidate.transaction_id)? + .post_commit_cleanup + .context("no cleanup")?; + assert_eq!( + retired.incumbent, + IncumbentCleanupEvidence::Complete { + generation_id: incumbent.generation_id + } + ); + assert_eq!(retired.source, SourceCleanupEvidence::Complete); + assert!(!frozen.exists(), "the frozen source outlived its deploy"); + Ok(()) + } + + #[test] + fn a_deploy_post_fence_abort_cleans_up_and_restores_its_incumbent_exactly() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let incumbent = admit_incumbent(&world)?; + project_admitted(&world, &incumbent)?; + let candidate = seeded_transaction( + &world, + DeploymentPhase::Fencing, + CommandKind::Deploy, + Some(&incumbent), + )?; + project_candidate(&world, &candidate, true)?; + let candidate_expected = candidate.expected.clone().context("no Expected")?; + assert_eq!( + projected_under(&world, &candidate_expected)?.len(), + 2, + "the candidate did not project" + ); + let frozen = world + .engine + .options + .staging_root + .join("frozen-sources") + .join(&candidate.transaction_id); + std::fs::create_dir_all(&frozen)?; + std::fs::write(frozen.join("tree"), b"frozen")?; + let before = incumbent_envelope(&world)?; + + world + .engine + .begin_post_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + assert!(resident(&world)?.value.post_fencing_abort.is_some()); + drive(&world, |transaction| transaction.completion.is_some())?; + + let finished = record_of(&world, &candidate.transaction_id)?; + assert!(matches!( + finished.completion, + Some(TransactionCompletion::FailedAfterFencing { + recovery: TerminalRecovery::RestoreIncumbent, + .. + }) + )); + let abort = finished.post_fencing_abort.context("no abort evidence")?; + assert_eq!(abort.topology_reconciliation, CleanupEvidence::Complete); + assert_eq!(abort.source_cleanup, CleanupEvidence::Complete); + assert!(!frozen.exists(), "the abandoned source was not cleaned"); + assert!( + projected_under(&world, &candidate_expected)?.is_empty(), + "the abandoned candidate is still projected" + ); + // Fencing stopped the incumbent: it stands as Expected-only, with the + // exact activation it ran under withdrawn, ready for continuity. + assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); + assert_eq!(incumbent_envelope(&world)?, before); + Ok(()) + } + + #[test] + fn a_deploy_pre_fence_abort_cleans_up_and_leaves_its_incumbent_running() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let incumbent = admit_incumbent(&world)?; + project_admitted(&world, &incumbent)?; + let candidate = seeded_transaction( + &world, + DeploymentPhase::Warming, + CommandKind::Deploy, + Some(&incumbent), + )?; + project_candidate(&world, &candidate, true)?; + let candidate_expected = candidate.expected.clone().context("no Expected")?; + let frozen = world + .engine + .options + .staging_root + .join("frozen-sources") + .join(&candidate.transaction_id); + std::fs::create_dir_all(&frozen)?; + std::fs::write(frozen.join("tree"), b"frozen")?; + let before = incumbent_envelope(&world)?; + + world + .engine + .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + let intent = resident(&world)?.value.pre_fencing_abort.context("no intent")?; + assert_eq!(intent.topology_reconciliation, CleanupEvidence::Pending); + assert_eq!(intent.source_cleanup, CleanupEvidence::Pending); + drive(&world, |transaction| transaction.completion.is_some())?; + + let finished = record_of(&world, &candidate.transaction_id)?; + assert!(matches!( + finished.completion, + Some(TransactionCompletion::FailedBeforeFencing { .. }) + )); + assert!(finished.is_terminal()); + assert!(!frozen.exists(), "the abandoned source was not cleaned"); + assert!(projected_under(&world, &candidate_expected)?.is_empty()); + // Before fencing a deployment never touched its incumbent. + assert_eq!( + projected_under(&world, &incumbent.expected)?, + { + let mut types = vec![ + IdunnExpectedIncarnationRecord::TYPE.to_owned(), + IdunnRuntimeActivationRecord::TYPE.to_owned(), + ]; + types.sort(); + types + } + ); + assert_eq!(incumbent_envelope(&world)?, before); + Ok(()) + } + + #[test] + fn a_continuity_abort_over_its_incumbent_demotes_only_the_activation_it_issued() -> Result<()> { + for post_fence in [false, true] { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let incumbent = admit_incumbent(&world)?; + let candidate = seeded_transaction( + &world, + if post_fence { + DeploymentPhase::Fencing + } else { + DeploymentPhase::Warming + }, + CommandKind::Continuity, + Some(&incumbent), + )?; + // The candidate shares the incumbent's key: same Expected, its own + // activation. + assert_eq!(candidate.expected.as_ref(), Some(&incumbent.expected)); + assert_ne!(candidate.activation.as_ref(), Some(&incumbent.activation)); + project_candidate(&world, &candidate, true)?; + let before = incumbent_envelope(&world)?; + + let stored = resident(&world)?; + if post_fence { + world + .engine + .begin_post_fencing_abort(&stored, anyhow!("candidate died"))?; + assert_eq!( + resident(&world)? + .value + .post_fencing_abort + .context("no intent")? + .topology_reconciliation, + CleanupEvidence::Pending + ); + } else { + world + .engine + .begin_pre_fencing_abort(&stored, anyhow!("candidate died"))?; + assert_eq!( + resident(&world)? + .value + .pre_fencing_abort + .context("no intent")? + .topology_reconciliation, + CleanupEvidence::Pending + ); + } + drive(&world, |transaction| transaction.completion.is_some())?; + + assert_eq!( + projected_under(&world, &incumbent.expected)?, + expected_only(), + "post_fence={post_fence}: the shared key must keep its Expected and nothing else" + ); + let finished = record_of(&world, &candidate.transaction_id)?; + assert!(finished.is_terminal()); + assert!(matches!( + finished.completion, + Some( + TransactionCompletion::FailedBeforeFencing { .. } + | TransactionCompletion::FailedAfterFencing { .. } + ) + )); + assert_eq!(incumbent_envelope(&world)?, before); + } + Ok(()) + } + + // ---- fix 1: the legacy lift owns the transition ---- + + const FIXTURE_PRE_B1_ABORTS: [(&str, &str); 3] = [ + ( + "pre-fence abort in flight", + include_str!("../tests/fixtures/idunn-control-legacy/transaction-pre-fence-abort.hex"), + ), + ( + "pre-fence abort already terminal", + include_str!( + "../tests/fixtures/idunn-control-legacy/transaction-pre-fence-abort-terminal.hex" + ), + ), + ( + "post-fence abort in flight", + include_str!("../tests/fixtures/idunn-control-legacy/transaction-post-fence-abort.hex"), + ), + ]; + const FIXTURE_PROVIDER_ANCHOR: &str = + include_str!("../tests/fixtures/idunn-control-legacy/provider-anchor.hex"); + /// Where the fixtures' plan binding names the provider trust anchor. + const FIXTURE_ANCHOR_PATH: &str = "/tmp/idunn-b1-legacy/provider-anchor.cc"; + + fn hex_bytes(text: &str) -> Vec { + let hex = text.trim(); + (0..hex.len()) + .step_by(2) + .map(|index| u8::from_str_radix(&hex[index..index + 2], 16).expect("hex fixture")) + .collect() + } + + fn write_fixture_anchor() -> Result<()> { + let path = Path::new(FIXTURE_ANCHOR_PATH); + std::fs::create_dir_all(path.parent().context("anchor path has no parent")?)?; + std::fs::write(path, hex_bytes(FIXTURE_PROVIDER_ANCHOR))?; + Ok(()) + } + + #[test] + fn a_pre_b1_continuity_abort_lifts_owing_its_own_activation_and_resolves_it() -> Result<()> { + write_fixture_anchor()?; + for (name, text) in FIXTURE_PRE_B1_ABORTS { + let envelope = fixture_envelope(text, DeploymentTransaction::TYPE)?; + + // The record is what the old rule wrote: an activation was issued + // and the abort says it owes the projection nothing. + let legacy: LegacyDeploymentTransaction = rmp_serde::from_slice(&envelope.payload)?; + assert_eq!(legacy.command_kind, CommandKind::Continuity, "{name}"); + assert!(legacy.activation.is_some(), "{name}"); + let owed = legacy + .pre_fencing_abort + .as_ref() + .map(|abort| abort.topology_reconciliation) + .or_else(|| { + legacy + .post_fencing_abort + .as_ref() + .map(|abort| abort.topology_reconciliation) + }) + .context("no abort")?; + assert_eq!(owed, CleanupEvidence::Skipped, "{name}"); + + // History describes what happened, so it lifts the record as written. + let archived = lift_legacy_transaction(&envelope)?; + let archived_owed = archived + .pre_fencing_abort + .as_ref() + .map(|abort| abort.topology_reconciliation) + .or_else(|| { + archived + .post_fencing_abort + .as_ref() + .map(|abort| abort.topology_reconciliation) + }) + .context("no abort")?; + assert_eq!(archived_owed, CleanupEvidence::Skipped, "{name}"); + + // The control store lifts it owing the demotion, and validates. + let lifted = read_transaction_record(&envelope)?; + let lifted_owed = lifted + .pre_fencing_abort + .as_ref() + .map(|abort| abort.topology_reconciliation) + .or_else(|| { + lifted + .post_fencing_abort + .as_ref() + .map(|abort| abort.topology_reconciliation) + }) + .context("no abort")?; + assert_eq!(lifted_owed, CleanupEvidence::Pending, "{name}"); + assert!(!lifted.is_terminal(), "{name}: a record that owes work is terminal"); + assert_eq!(lifted.completion, None, "{name}"); + + // Boot: the store reads, and the abort resolves. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: lifted.command_id.clone(), + kind: CommandKind::Continuity, + selector: "service".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: envelope.key.clone(), + current: None, + }, + ], + &[command_envelope(&command, 100)?, envelope.clone()], + )?, + "{name}" + ); + assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); + let snapshot = ControlSnapshot::read(&world.state_store)?; + assert_eq!(snapshot.transactions.len(), 1, "{name}"); + project_candidate(&world, &lifted, true)?; + let expected = lifted.expected.clone().context("no Expected")?; + assert_eq!(projected_under(&world, &expected)?.len(), 2, "{name}"); + + for _ in 0..12 { + let Ok(current) = resident(&world) else { break }; + if current.value.completion.is_some() { + break; + } + world.engine.advance_transaction(¤t)?; + } + let finished = record_of(&world, &envelope.key)?; + assert!(finished.completion.is_some(), "{name}: the abort never resolved"); + assert_eq!( + projected_under(&world, &expected)?, + expected_only(), + "{name}: the projection was not demoted to Expected-only" + ); + } + Ok(()) + } + + // ---- fix 2: supervision owns the Expected-only precondition ---- + + #[test] + fn continuity_mints_no_restart_over_a_projection_it_cannot_demote() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + assert_eq!(incumbent.continuity_backoff, ContinuityBackoff::default()); + + // The projection names an activation that is not the incumbent's. + let anchor = world.engine.provider_anchor_for_plan(&incumbent.plan)?; + let topology = world.engine.topology(); + topology.publish_expected(&incumbent.expected, &anchor)?; + let stranger = seeded_transaction( + &world, + DeploymentPhase::Warming, + CommandKind::Continuity, + Some(&incumbent), + )?; + topology.publish_observed_activation( + &incumbent.expected, + stranger.activation.as_ref().context("no activation")?, + stranger.workload.as_ref().context("no workload")?, + )?; + // Remove the seeded stranger transaction: only the projection matters. + let seeded = resident(&world)?; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).delete_batch_if_unchanged( + &[ + seeded.envelope.clone(), + ControlSnapshot::read(&world.state_store)? + .commands + .into_iter() + .find(|stored| stored.value.command_id == seeded.value.command_id) + .context("seeded command")? + .envelope, + ] + )? + ); + workload.kill(); + + let before = now_millis()?; + assert!(world.engine.supervise_one_admitted_generation()?); + let snapshot = ControlSnapshot::read(&world.state_store)?; + assert!( + snapshot.transactions.is_empty(), + "continuity minted a restart over a projection it could not demote" + ); + let deferred = snapshot + .admitted_for("service") + .context("no generation")? + .value + .continuity_backoff + .next_restart_at_unix_millis + .context("the failed demotion was not recorded")?; + assert!(deferred >= before + CONTINUITY_DEFERRAL_MILLIS); + + // Deferred: the next tick neither retries nor writes. + let envelope = incumbent_envelope(&world)?; + assert!(!world.engine.supervise_one_admitted_generation()?); + assert_eq!(incumbent_envelope(&world)?, envelope); + assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); + + // The projection is repaired and the deferral has run out: the restart + // proceeds, from an Expected-only projection. + topology.withdraw_stale_incarnation(&incumbent.expected, &anchor)?; + project_admitted(&world, &incumbent)?; + edit_incumbent(&world, |generation| { + generation.continuity_backoff.next_restart_at_unix_millis = Some(1); + })?; + assert!(world.engine.supervise_one_admitted_generation()?); + let snapshot = ControlSnapshot::read(&world.state_store)?; + assert_eq!(snapshot.transactions.len(), 1); + assert_eq!(snapshot.transactions[0].value.command_kind, CommandKind::Continuity); + assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); + Ok(()) + } + + // ---- fix 3: one-time boot reconciliation, by exact issuer ---- + + #[test] + fn boot_demotes_the_activation_a_failed_continuity_left_and_only_that() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let incumbent = admit_incumbent(&world)?; + let failed = seeded_transaction( + &world, + DeploymentPhase::Warming, + CommandKind::Continuity, + Some(&incumbent), + )?; + project_candidate(&world, &failed, true)?; + world + .engine + .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate died"))?; + drive(&world, |transaction| transaction.completion.is_some())?; + // A failed transaction is archived the moment it finishes. + let aborted = record_of(&world, &failed.transaction_id)?; + assert!(aborted.is_terminal()); + assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); + assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); + + // Drift a pre-B1 abort left: the failed candidate's activation stands. + project_candidate(&world, &failed, true)?; + assert_eq!(projected_under(&world, &incumbent.expected)?.len(), 2); + + let outcome = world.engine.reconcile_failed_continuity_projections()?; + assert_eq!( + outcome, + ProjectionReconciliation { + demoted: vec![failed.transaction_id.clone()], + unexplained: Vec::new(), + } + ); + assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); + + // Once: nothing is left to demote. + assert_eq!( + world.engine.reconcile_failed_continuity_projections()?, + ProjectionReconciliation::default() + ); + Ok(()) + } + + #[test] + fn boot_never_adopts_an_activation_no_failed_transaction_issued() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let incumbent = admit_incumbent(&world)?; + + // The incumbent's own activation is accounted for. + project_admitted(&world, &incumbent)?; + assert_eq!( + world.engine.reconcile_failed_continuity_projections()?, + ProjectionReconciliation::default() + ); + + // Some other activation, issued by nothing in history, is reported and + // left exactly as it is. + let stranger = seeded_transaction( + &world, + DeploymentPhase::Warming, + CommandKind::Continuity, + Some(&incumbent), + )?; + let anchor = world.engine.provider_anchor_for_plan(&incumbent.plan)?; + let topology = world.engine.topology(); + topology.withdraw_stale_incarnation(&incumbent.expected, &anchor)?; + project_candidate(&world, &stranger, true)?; + // A live transaction accounts for its own activation... + assert_eq!( + world.engine.reconcile_failed_continuity_projections()?, + ProjectionReconciliation::default() + ); + // ...and once it is gone, nothing does. + let seeded = resident(&world)?; + let command = ControlSnapshot::read(&world.state_store)? + .commands + .into_iter() + .find(|stored| stored.value.command_id == seeded.value.command_id) + .context("seeded command")?; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store) + .delete_batch_if_unchanged(&[seeded.envelope.clone(), command.envelope])? + ); + let projection = std::fs::read(&world.engine.options.topology_store)?; + let outcome = world.engine.reconcile_failed_continuity_projections()?; + assert_eq!( + outcome, + ProjectionReconciliation { + demoted: Vec::new(), + unexplained: vec!["service".into()], + } + ); + assert_eq!(std::fs::read(&world.engine.options.topology_store)?, projection); + Ok(()) + } + + // ---- fix 4: history that cannot be read stops actuation ---- + + /// An operator binding for the target "service", so the command queue can + /// resolve selectors. + fn write_service_binding(world: &EngineFixture) -> Result<()> { + use crate::deployment_plan::tests::BINDING; + let (head, tail) = BINDING + .split_once("[route]") + .context("binding has no route table")?; + let binding = format!( + "{head}[brakes]{}", + tail.split_once("[brakes]") + .context("binding has no brakes table")? + .1 + ); + std::fs::create_dir_all(&world.engine.options.bindings_dir)?; + std::fs::write(world.engine.options.bindings_dir.join("service.toml"), binding)?; + Ok(()) + } + + fn corrupt_history(world: &EngineFixture) -> Result<()> { + std::fs::write(history_store_path(&world.state_store), b"\xc1\xc1 torn")?; + Ok(()) + } + + #[test] + fn history_that_cannot_be_read_is_an_error_and_missing_history_is_empty() -> Result<()> { + let world = EngineFixture::new()?; + let history = read_history(&world.state_store)?; + assert!(history.transactions.is_empty() && history.report().is_none()); + + corrupt_history(&world)?; + let error = read_history(&world.state_store).err().context("torn history read")?; + assert!(format!("{error:#}").contains("history"), "{error:#}"); + assert!(world.engine.history_for_decision().is_none()); + // The display path still answers, with nothing. + assert!(read_history_transactions(&world.state_store).is_empty()); + Ok(()) + } + + #[test] + fn undecodable_history_names_what_it_is_missing() -> Result<()> { + let (finished, _) = terminal_transaction_with_command("ghostlight")?; + let good = transaction_envelope(&finished, finished.updated_at_unix_millis)?; + let mut torn = good.clone(); + torn.key = "tx-torn".into(); + torn.payload = vec![0xc1]; + let (transactions, undecodable) = decode_history_transactions(vec![good, torn]); + let history = HistoryRead { + transactions, + undecodable, + }; + let report = history.report().context("nothing reported")?; + assert!(report.contains("1 archived transaction"), "{report}"); + assert!(report.contains("tx-torn"), "{report}"); + + let complete = HistoryRead { + transactions: history.transactions, + undecodable: Vec::new(), + }; + assert_eq!(complete.report(), None); + Ok(()) + } + + #[test] + fn a_fault_is_said_once_while_it_lasts() { + let once = ReportOnce::default(); + assert_eq!(once.offer(None), None); + assert_eq!(once.offer(Some("torn".into())), Some("torn".into())); + assert_eq!(once.offer(Some("torn".into())), None); + assert_eq!(once.offer(Some("torn".into())), None); + assert_eq!(once.offer(Some("worse".into())), Some("worse".into())); + assert_eq!(once.offer(None), None); + assert_eq!(once.offer(Some("worse".into())), Some("worse".into())); + } + + #[test] + fn unreadable_history_stops_continuity_instead_of_counting_no_failures() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + workload.kill(); + corrupt_history(&world)?; + let envelope = incumbent_envelope(&world)?; + + // A dead workload would be restarted at once, with the ceiling counting + // zero refusals. With history unreadable nothing is actuated. + assert!(!world.engine.supervise_one_admitted_generation()?); + assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); + assert_eq!(incumbent_envelope(&world)?, envelope); + + // Readable again, the same tick restarts it. + std::fs::remove_file(history_store_path(&world.state_store))?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(ControlSnapshot::read(&world.state_store)?.transactions.len(), 1); + Ok(()) + } + + #[test] + fn unreadable_history_stops_command_retirement_and_freezing() -> Result<()> { + let world = EngineFixture::new()?; + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "up-nowhere".into(), + kind: CommandKind::Deploy, + selector: "nowhere".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }], + &[command_envelope(&command, 100)?], + )? + ); + write_service_binding(&world)?; + corrupt_history(&world)?; + + // Whether the command was consumed is unknown, so it is not acted on. + assert!(!world.engine.freeze_one_queued_command()?); + assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); + + // Readable, it is acted on: the unknown selector is refused. + std::fs::remove_file(history_store_path(&world.state_store))?; + assert!(world.engine.freeze_one_queued_command()?); + assert_eq!(ControlSnapshot::read(&world.state_store)?.transactions.len(), 1); + Ok(()) + } } diff --git a/src/drivers.rs b/src/drivers.rs index 226906f..d56e3d9 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -4488,6 +4488,22 @@ impl CultCacheTopologyDriver { ) } + /// The activation the projection currently names under this incarnation's + /// key, if any. Read-only: it never judges whether the activation is the + /// right one, so a caller can tell an exact issuer from a substitute + /// before it demotes anything. + pub fn projected_activation( + &self, + expected: &IdunnExpectedIncarnationRecord, + ) -> Result> { + let key = incarnation_key(expected)?; + let entries = self.snapshot()?; + projection_entry(&entries, IdunnRuntimeActivationRecord::TYPE, &key)? + .map(|envelope| IdunnRuntimeActivationRecord::decode_canonical(&envelope.payload)) + .transpose() + .map_err(Into::into) + } + /// Every incarnation currently projected for one target, by its Expected. pub fn projected_incarnations( &self, diff --git a/tests/fixtures/idunn-control-legacy/README.md b/tests/fixtures/idunn-control-legacy/README.md index 07086d7..d1b00ec 100644 --- a/tests/fixtures/idunn-control-legacy/README.md +++ b/tests/fixtures/idunn-control-legacy/README.md @@ -22,3 +22,23 @@ the plan is the `deployment_plan` test recipe. No byte comes from a live store. `control_plane.rs` decodes these through `LegacyDeploymentTransaction` and `LegacyAdmittedGeneration`. The fixtures die with those structs. + +## Pre-B1 continuity aborts + +The `transaction-*-abort*` files and `provider-anchor` were printed by a second +throwaway test over `9001b58`. It built a continuity transaction that issued an +activation (the recipe and binding of the `deployment_plan` tests, a provider +identity enrolled fresh, the trust-anchor path in the binding fixed at +`/tmp/idunn-b1-legacy/provider-anchor.cc`) and gave it the aborts the base code +writes: the pre-fence abort literal of `begin_pre_fencing_abort`, and the base +`post_fencing_abort_intent`. In each, the transaction issued an activation and +its abort records `topology_reconciliation = Skipped`, because the base rule +owed a continuity no projection cleanup. The workload and Odin evidence are +borrowed from `transaction-fencing`. + +| File | What it is | +|---|---| +| `transaction-pre-fence-abort` | Warming continuity, pre-fencing abort in flight | +| `transaction-pre-fence-abort-terminal` | the same abort finished: Complete, `FailedBeforeFencing` | +| `transaction-post-fence-abort` | Fencing continuity, post-fencing abort in flight | +| `provider-anchor` | one line: the provider trust-anchor file bytes, hex. A test writes it to the fixed path so the Engine can read the plan's anchor | diff --git a/tests/fixtures/idunn-control-legacy/provider-anchor.hex b/tests/fixtures/idunn-control-legacy/provider-anchor.hex new file mode 100644 index 0000000..928d4cf --- /dev/null +++ b/tests/fixtures/idunn-control-legacy/provider-anchor.hex @@ -0,0 +1 @@ +93b263756c7463616368652e73746f72652e76319197d93167616d6563756c742e70726f76696465725f6865616c74685f6964656e746974792e74727573745f616e63686f722e7631d93167616d6563756c742e70726f76696465725f6865616c74685f6964656e746974792e74727573745f616e63686f722e7631d93467616d6563756c742e70726f76696465725f6865616c74685f6964656e746974792e74727573745f616e63686f722e76312e7631d93167616d6563756c742e70726f76696465725f6865616c74685f6964656e746974792e74727573745f616e63686f722e7631d9967b22736368656d614e616d65223a2267616d6563756c742e70726f76696465725f6865616c74685f6964656e746974792e74727573745f616e63686f722e7631222c22736368656d6156657273696f6e223a2267616d6563756c742e70726f76696465725f6865616c74685f6964656e746974792e74727573745f616e63686f722e76312e7631222c226d656d62657273223a5b5d7d91d93167616d6563756c742e70726f76696465725f6865616c74685f6964656e746974792e74727573745f616e63686f722e7631909194d92867616d6563756c742d70726f76696465722d6865616c74682d6964656e746974792d7075626c6963d93167616d6563756c742e70726f76696465725f6865616c74685f6964656e746974792e74727573745f616e63686f722e7631d923323032362d30392d32395430333a34353a31352e3031353232313330302b30303a3030c5014596d93167616d6563756c742e70726f76696465725f6865616c74685f6964656e746974792e74727573745f616e63686f722e7631d94031626661653932326138626630313132373562353536623236393135303363386366643538313234383931373134323365373736663133386463313235376337dc00205858cca9ccbb7dccb9cc8d01cccfccfccc9153ccdbcce23c224bcca757617dcce5cc8a163a6516cc9dccee2c2acce9d92d6f735f696e7374616c6c6174696f6e5f66696c655f626f756e645f636c6f6e6561626c655f626173656c696e65d923323032362d30392d32395430333a34353a31352e3031353232313330302b30303a3030d9477368613235362d63663933623766656534396235313133343736366538613937623432303065653636396338353164386464336634353366623537666664326233366630653430 diff --git a/tests/fixtures/idunn-control-legacy/transaction-post-fence-abort.hex b/tests/fixtures/idunn-control-legacy/transaction-post-fence-abort.hex new file mode 100644 index 0000000..37cd704 --- /dev/null +++ b/tests/fixtures/idunn-control-legacy/transaction-post-fence-abort.hex @@ -0,0 +1,2 @@ +idunn.deployment_transaction.v3 +dc0023bf6964756e6e2e6465706c6f796d656e745f7472616e73616374696f6e2e7633d92774782d39356138313166662d303533332d343562622d613031352d303038356132353835633061b2636f6e74696e756974792d73657276696365aa636f6e74696e75697479a77365727669636500a766656e63696e67cf0000018bcfe56800cf0000018bcfe56800c099d9216964756e6e2e636f6d70696c65645f6465706c6f796d656e745f706c616e2e7632d9477368613235362d38626532306364633335373930316463373365316162623262336365666633373835343661323962343161353535656131643633653530366562333762376330b5736572766963652d696e6361726e6174696f6e2d316e9bbf6964756e6e2e736f757263655f73656c656374696f6e5f66616374732e7631d92768747470733a2f2f6769746875622e636f6d2f47616d6543756c742f536572766963652e676974af726566732f68656164732f6d61696ed92832323232323232323232323232323232323232323232323232323232323232323232323232323232d92832323232323232323232323232323232323232323232323232323232323232323232323232323232d92833333333333333333333333333333333333333333333333333333333333333333333333333333333bc6465706c6f796d656e742f6964756e6e2f7265636970652e746f6d6cd9477368613235362d323739336466633034633331313961373962353037363965613637356538633030613938366665623331393939306631623031636364363833343563363732618091a87265662d6865616464dc04e20a736368656d61203d202267616d6563756c742e6964756e6e2e7461726765745f6465636c61726174696f6e2e7631220a746172676574203d202273657276696365220a736f757263655f7374616d705f656e7669726f6e6d656e74203d2022534552564943455f4255494c445f434f4d4d4954220a0a5b5b73746570735d5d0a6964203d20226275696c64220a7068617365203d20226275696c64220a72756e6e6572203d202272757374220a61726776203d205b22636172676f222c20226275696c64222c20222d2d6c6f636b6564225d0a0a5b5b65787465726e616c5f696e707574735d5d0a6964203d2022746f6f6c636861696e2d696e646578220a75726c203d202268747470733a2f2f6578616d706c652e696e76616c69642f746f6f6c636861696e2d696e646578220a736861323536203d202264646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464220a72756e6e6572203d202272757374220a64657374696e6174696f6e203d2022696e707574732f746f6f6c636861696e2d696e646578220a0a5b5b6172746966616374735d5d0a6964203d20226461656d6f6e220a736f757263655f6b696e64203d202272756e6e65722d6f7574707574220a72756e6e6572203d202272757374220a736f75726365203d20227461726765742f72656c656173652f73657276696365220a64657374696e6174696f6e203d202273657276696365220a65787065637465645f736861323536203d202263636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363220a65786563757461626c65203d20747275650a0a5b736572766963655d0a65786563757461626c655f6172746966616374203d20226461656d6f6e220a617267756d656e7473203d205b0a20207b206b696e64203d202262696e64696e67222c206e616d65203d202273746174655f726f6f7422207d2c0a5d0a7472616e73706f7274203d202268747470220a726f7574655f7265717569726564203d2066616c73650a72657175697265645f656e7669726f6e6d656e74203d205b2247414d4543554c545f4944554e4e5f52554e54494d455f42554e444c45225d0a0a5b736572766963652e6865616c74685d0a636f6e7472616374203d2022736572766963652e6865616c7468220a0a5b73746174655d0a736368656d615f67656e65726174696f6e203d20227631220a0a5b5b73746174652e736c6f74735d5d0a6964203d202272756e74696d652d6361636865220a72656c61746976655f70617468203d202272756e74696d652d63616368652e6363220a6b696e64203d202263756c7463616368652d66696c65220a736368656d61203d2022736572766963652e72756e74696d652d63616368652e7631220a777269746572203d20226e6f6e65220a7265636f76657279203d202272656275696c6461626c65220a73746172747570203d20226f70656e2d61742d7374617274220a0a5b5b70726f76696465735d5d0a6361706162696c697479203d2022736572766963652e72756e74696d65220a736368656d61203d2022736572766963652e72756e74696d652e7631220a636f6d7061746962696c697479203d20227631220a0adc06930a736368656d61203d202267616d6563756c742e6964756e6e2e6f70657261746f725f62696e64696e672e7632220a746172676574203d202273657276696365220a0a5b7265706f7369746f72795d0a6f726967696e203d202268747470733a2f2f6769746875622e636f6d2f47616d6543756c742f536572766963652e676974220a61646d69747465645f726566203d2022726566732f68656164732f6d61696e220a6d696e696d756d5f7265766973696f6e203d202231313131313131313131313131313131313131313131313131313131313131313131313131313131220a73656c656374696f6e203d20227265662d68656164220a636865636b6f7574203d20222f7372762f6275696c642f53657276696365220a7265636970655f70617468203d20226465706c6f796d656e742f6964756e6e2f7265636970652e746f6d6c220a0a5b72756e6e6572732e727573745d0a647269766572203d2022646f636b6572220a696d616765203d202272757374407368613235363a61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161220a75736572203d2022313030303a31303030220a6166666f7264616e636573203d205b22736f757263652d72656164222c202261727469666163742d7772697465225d0a616c6c6f7765645f70726f6772616d73203d205b22636172676f225d0a6e6574776f726b5f70726f66696c65203d20226275696c642d646570656e64656e63792d656772657373220a6d656d6f72795f6d6562696279746573203d20323034380a6370755f71756f74615f70657263656e74203d203230300a706964735f6c696d6974203d203531320a746d7066735f6d6562696279746573203d203531320a0a5b776f726b6c6f61645d0a647269766572203d202273797374656d642d7472616e7369656e74220a73746174655f67726f7570203d202273657276696365220a756e69745f707265666978203d20226964756e6e2d73657276696365220a72656c656173655f726f6f74203d20222f7372762f736572766963652f72656c6561736573220a73746174655f726f6f74203d20222f7661722f6c69622f67616d6563756c742f73657276696365220a72756e74696d655f726f6f74203d20222f72756e2f67616d6563756c742f6964756e6e2f73657276696365220a6e6574776f726b203d2022686f73742d70726976617465220a68617264656e696e67203d2022737472696374220a6d656d6f72795f6d6562696279746573203d20313032340a6370755f71756f74615f70657263656e74203d203130300a0a5b776f726b6c6f61642e617267756d656e745f62696e64696e67735d0a73746174655f726f6f74203d20222f7661722f6c69622f67616d6563756c742f73657276696365220a0a5b776f726b6c6f61642e7365637265745f66696c65735d0a47414d4543554c545f52554e54494d455f50524553454e43455f4944454e54495459203d20222f6574632f67616d6563756c742f736572766963652f72756e74696d652d70726573656e63652d6964656e746974792e6363220a0a5b72756e74696d655f6964656e746974795d0a72756e74696d655f6964203d2022736572766963652d79676764726173696c220a65787065637465645f7369676e65725f6964656e746974795f6964203d202231626661653932326138626630313132373562353536623236393135303363386366643538313234383931373134323365373736663133386463313235376337220a74727573745f616e63686f725f73746f7265203d20222f746d702f6964756e6e2d62312d6c65676163792f70726f76696465722d616e63686f722e6363220a0a5b6272616b65735d0a6465706c6f796d656e745f73746f7265203d20222f7661722f6c69622f67616d6563756c742f6964756e6e2f736572766963652d6465706c6f796d656e742d6272616b652e6363220a6c6966656379636c655f73746f7265203d20222f7661722f6c69622f67616d6563756c742f6964756e6e2f736572766963652d6c6966656379636c652d6272616b652e6363220a0a5b726f6c6c6f75745d0a7374726174656779203d202263616e6469646174652d7468656e2d70726f6d6f7465220a647261696e5f7365636f6e6473203d2033300a72657461696e5f72656c6561736573203d20320a0a5b706c6163656d656e745d0a646573697265645f7265706c69636173203d20310a6e6f646573203d205b2279676764726173696c225d0a90c0c096b76964756e6e2e7365616c65645f72656c656173652e7631d9477368613235362d39366633373231386438333663646364303037393631343233333032353334333362316666633766323538633135353037656536376163323961643665663662d9477368613235362d386265323063646333353739303164633733653161626232623363656666333738353436613239623431613535356561316436336535303665623337623763309195a66461656d6f6ea773657276696365d9477368613235362d636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363632ac39196af746f6f6c636861696e2d696e646578d92768747470733a2f2f6578616d706c652e696e76616c69642f746f6f6c636861696e2d696e646578d9477368613235362d64646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464a472757374b6696e707574732f746f6f6c636861696e2d696e646578117892d9477368613235362d39366633373231386438333663646364303037393631343233333032353334333362316666633766323538633135353037656536376163323961643665663662ba2f7372762f736572766963652f72656c65617365732f74657374dc0012bd6964756e6e2e65787065637465645f696e6361726e6174696f6e2e7632a773657276696365d9477368613235362d38626532306364633335373930316463373365316162623262336365666633373835343661323962343161353535656131643633653530366562333762376330b5736572766963652d696e6361726e6174696f6e2d31d9477368613235362d39366633373231386438333663646364303037393631343233333032353334333362316666633766323538633135353037656536376163323961643665663662bb6769746875622e636f6d2f47616d6543756c742f53657276696365d92832323232323232323232323232323232323232323232323232323232323232323232323232323232d9477368613235362d32373933646663303463333131396137396235303736396561363735653863303061393836666562333139393930663162303163636436383334356336373261b1736572766963652d79676764726173696cd94031626661653932326138626630313132373562353536623236393135303363386366643538313234383931373134323365373736663133386463313235376337ae736572766963652e6865616c7468d9477368613235362d63636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363a27631d9477368613235362d32663837303134316366353638316566656332653234316364636435663731303831616539326162316535633534633839636438393666386335656236323731c2c09194af736572766963652e72756e74696d65b2736572766963652e72756e74696d652e7631a276310190d9477368613235362d33336163613963353639636165373831643936396632313864633036613035313134613863666239333436313634633033613263336435343033613636323666c0cf0000018bcfe568009abb6964756e6e2e72756e74696d655f61637469766174696f6e2e7632d9477368613235362d33336163613963353639636165373831643936396632313864633036613035313134613863666239333436313634633033613263336435343033613636323666b1736572766963652d79676764726173696cd9477368613235362d66353962643235306336333830383661643330396235333539373636663232326266353033366333336230616633316239633935333130353831366636396137d94066313063623131353633346366613935383966356564616332313730343339363933353834343234643736653133366434396661613236373539323266376665c420f2977384801339144d13ebd2ab001bf29fbab79f7c3948334d440db8206cf974cf0000018bcfe56800d94063653461663030623963626662316362613963363136363965323337303232323465343763623163303761346233313431626634353237383236353965646637a765643235353139c4403a548c511ff3de6fcac116b903fa6badb14deef79dd81958d71947af575fae753b11692d0272c7cb5c437300cdd0d5315e28a1e7e12f9df0a0b817951bba4700dc0033b26964756e6e2d343234322e73657276696365af4964756e6e20746573742034323432af696e766f636174696f6e2d3432343201a465786563a6616c77617973ad636f6e74726f6c2d67726f7570c3a57534323432a57534323432a0a0a0c3c3a9696e76697369626c65a3706964c3a430303737a0a0cd10920194cd1092cd1092cd1092cd109294cd1092cd1092cd1092cd109291cd10920000000000c391010807b52f6f70742f746573742f62696e2f736572766963650101d9477368613235362d34626635313232663334343535346335336264653265626238636432623765336431363030616436333163333835613564376363653233633737383534353961d9477368613235362d66353962643235306336333830383661643330396235333539373636663232326266353033366333336230616633316239633935333130353831366636396137a92f6f70742f74657374a92f72756e2f74657374d9477368613235362d6462633162346339303066666534386435373562356461356336333830343031323566363564623066653365323434393462373665613938363435376439383690d9477368613235362d30383466656430386239373861663464376431393661373434366138366235383030396536333662363131646231363231316236356139616164666632396335d9202f73797374656d2e736c6963652f6964756e6e2d343234322e73657276696365c090aa61637469766174696f6edc0020010101010101010101010101010101010101010101010101010101010101010190d9477368613235362d32393364343764383932393938373139346131303664303236326236326563363766633534393165366339633366653961326663383132643935633237376661c092ad6f64696e2d746f706f6c6f6779959139d9477368613235362d31393538316532376465376365643030666631636535306232303437653761353637633736623163626165626162653565663033663763333031376262356237ab6f64696e2d7369676e65722764c096cd10920708c0c0c091b1736b69707065642d73746174656c657373c0c0c029c0c0c0c096b163616e6469646174652072656675736564a7736b6970706564a7736b6970706564a770656e64696e67a7736b6970706564a7736b6970706564 diff --git a/tests/fixtures/idunn-control-legacy/transaction-pre-fence-abort-terminal.hex b/tests/fixtures/idunn-control-legacy/transaction-pre-fence-abort-terminal.hex new file mode 100644 index 0000000..29011d9 --- /dev/null +++ b/tests/fixtures/idunn-control-legacy/transaction-pre-fence-abort-terminal.hex @@ -0,0 +1,2 @@ +idunn.deployment_transaction.v3 +dc0023bf6964756e6e2e6465706c6f796d656e745f7472616e73616374696f6e2e7633d92774782d39356138313166662d303533332d343562622d613031352d303038356132353835633061b2636f6e74696e756974792d73657276696365aa636f6e74696e75697479a77365727669636500a8636f6d706c657465cf0000018bcfe56800cf0000018bcfe56800c099d9216964756e6e2e636f6d70696c65645f6465706c6f796d656e745f706c616e2e7632d9477368613235362d38626532306364633335373930316463373365316162623262336365666633373835343661323962343161353535656131643633653530366562333762376330b5736572766963652d696e6361726e6174696f6e2d316e9bbf6964756e6e2e736f757263655f73656c656374696f6e5f66616374732e7631d92768747470733a2f2f6769746875622e636f6d2f47616d6543756c742f536572766963652e676974af726566732f68656164732f6d61696ed92832323232323232323232323232323232323232323232323232323232323232323232323232323232d92832323232323232323232323232323232323232323232323232323232323232323232323232323232d92833333333333333333333333333333333333333333333333333333333333333333333333333333333bc6465706c6f796d656e742f6964756e6e2f7265636970652e746f6d6cd9477368613235362d323739336466633034633331313961373962353037363965613637356538633030613938366665623331393939306631623031636364363833343563363732618091a87265662d6865616464dc04e20a736368656d61203d202267616d6563756c742e6964756e6e2e7461726765745f6465636c61726174696f6e2e7631220a746172676574203d202273657276696365220a736f757263655f7374616d705f656e7669726f6e6d656e74203d2022534552564943455f4255494c445f434f4d4d4954220a0a5b5b73746570735d5d0a6964203d20226275696c64220a7068617365203d20226275696c64220a72756e6e6572203d202272757374220a61726776203d205b22636172676f222c20226275696c64222c20222d2d6c6f636b6564225d0a0a5b5b65787465726e616c5f696e707574735d5d0a6964203d2022746f6f6c636861696e2d696e646578220a75726c203d202268747470733a2f2f6578616d706c652e696e76616c69642f746f6f6c636861696e2d696e646578220a736861323536203d202264646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464220a72756e6e6572203d202272757374220a64657374696e6174696f6e203d2022696e707574732f746f6f6c636861696e2d696e646578220a0a5b5b6172746966616374735d5d0a6964203d20226461656d6f6e220a736f757263655f6b696e64203d202272756e6e65722d6f7574707574220a72756e6e6572203d202272757374220a736f75726365203d20227461726765742f72656c656173652f73657276696365220a64657374696e6174696f6e203d202273657276696365220a65787065637465645f736861323536203d202263636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363220a65786563757461626c65203d20747275650a0a5b736572766963655d0a65786563757461626c655f6172746966616374203d20226461656d6f6e220a617267756d656e7473203d205b0a20207b206b696e64203d202262696e64696e67222c206e616d65203d202273746174655f726f6f7422207d2c0a5d0a7472616e73706f7274203d202268747470220a726f7574655f7265717569726564203d2066616c73650a72657175697265645f656e7669726f6e6d656e74203d205b2247414d4543554c545f4944554e4e5f52554e54494d455f42554e444c45225d0a0a5b736572766963652e6865616c74685d0a636f6e7472616374203d2022736572766963652e6865616c7468220a0a5b73746174655d0a736368656d615f67656e65726174696f6e203d20227631220a0a5b5b73746174652e736c6f74735d5d0a6964203d202272756e74696d652d6361636865220a72656c61746976655f70617468203d202272756e74696d652d63616368652e6363220a6b696e64203d202263756c7463616368652d66696c65220a736368656d61203d2022736572766963652e72756e74696d652d63616368652e7631220a777269746572203d20226e6f6e65220a7265636f76657279203d202272656275696c6461626c65220a73746172747570203d20226f70656e2d61742d7374617274220a0a5b5b70726f76696465735d5d0a6361706162696c697479203d2022736572766963652e72756e74696d65220a736368656d61203d2022736572766963652e72756e74696d652e7631220a636f6d7061746962696c697479203d20227631220a0adc06930a736368656d61203d202267616d6563756c742e6964756e6e2e6f70657261746f725f62696e64696e672e7632220a746172676574203d202273657276696365220a0a5b7265706f7369746f72795d0a6f726967696e203d202268747470733a2f2f6769746875622e636f6d2f47616d6543756c742f536572766963652e676974220a61646d69747465645f726566203d2022726566732f68656164732f6d61696e220a6d696e696d756d5f7265766973696f6e203d202231313131313131313131313131313131313131313131313131313131313131313131313131313131220a73656c656374696f6e203d20227265662d68656164220a636865636b6f7574203d20222f7372762f6275696c642f53657276696365220a7265636970655f70617468203d20226465706c6f796d656e742f6964756e6e2f7265636970652e746f6d6c220a0a5b72756e6e6572732e727573745d0a647269766572203d2022646f636b6572220a696d616765203d202272757374407368613235363a61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161220a75736572203d2022313030303a31303030220a6166666f7264616e636573203d205b22736f757263652d72656164222c202261727469666163742d7772697465225d0a616c6c6f7765645f70726f6772616d73203d205b22636172676f225d0a6e6574776f726b5f70726f66696c65203d20226275696c642d646570656e64656e63792d656772657373220a6d656d6f72795f6d6562696279746573203d20323034380a6370755f71756f74615f70657263656e74203d203230300a706964735f6c696d6974203d203531320a746d7066735f6d6562696279746573203d203531320a0a5b776f726b6c6f61645d0a647269766572203d202273797374656d642d7472616e7369656e74220a73746174655f67726f7570203d202273657276696365220a756e69745f707265666978203d20226964756e6e2d73657276696365220a72656c656173655f726f6f74203d20222f7372762f736572766963652f72656c6561736573220a73746174655f726f6f74203d20222f7661722f6c69622f67616d6563756c742f73657276696365220a72756e74696d655f726f6f74203d20222f72756e2f67616d6563756c742f6964756e6e2f73657276696365220a6e6574776f726b203d2022686f73742d70726976617465220a68617264656e696e67203d2022737472696374220a6d656d6f72795f6d6562696279746573203d20313032340a6370755f71756f74615f70657263656e74203d203130300a0a5b776f726b6c6f61642e617267756d656e745f62696e64696e67735d0a73746174655f726f6f74203d20222f7661722f6c69622f67616d6563756c742f73657276696365220a0a5b776f726b6c6f61642e7365637265745f66696c65735d0a47414d4543554c545f52554e54494d455f50524553454e43455f4944454e54495459203d20222f6574632f67616d6563756c742f736572766963652f72756e74696d652d70726573656e63652d6964656e746974792e6363220a0a5b72756e74696d655f6964656e746974795d0a72756e74696d655f6964203d2022736572766963652d79676764726173696c220a65787065637465645f7369676e65725f6964656e746974795f6964203d202231626661653932326138626630313132373562353536623236393135303363386366643538313234383931373134323365373736663133386463313235376337220a74727573745f616e63686f725f73746f7265203d20222f746d702f6964756e6e2d62312d6c65676163792f70726f76696465722d616e63686f722e6363220a0a5b6272616b65735d0a6465706c6f796d656e745f73746f7265203d20222f7661722f6c69622f67616d6563756c742f6964756e6e2f736572766963652d6465706c6f796d656e742d6272616b652e6363220a6c6966656379636c655f73746f7265203d20222f7661722f6c69622f67616d6563756c742f6964756e6e2f736572766963652d6c6966656379636c652d6272616b652e6363220a0a5b726f6c6c6f75745d0a7374726174656779203d202263616e6469646174652d7468656e2d70726f6d6f7465220a647261696e5f7365636f6e6473203d2033300a72657461696e5f72656c6561736573203d20320a0a5b706c6163656d656e745d0a646573697265645f7265706c69636173203d20310a6e6f646573203d205b2279676764726173696c225d0a90c0c096b76964756e6e2e7365616c65645f72656c656173652e7631d9477368613235362d39366633373231386438333663646364303037393631343233333032353334333362316666633766323538633135353037656536376163323961643665663662d9477368613235362d386265323063646333353739303164633733653161626232623363656666333738353436613239623431613535356561316436336535303665623337623763309195a66461656d6f6ea773657276696365d9477368613235362d636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363632ac39196af746f6f6c636861696e2d696e646578d92768747470733a2f2f6578616d706c652e696e76616c69642f746f6f6c636861696e2d696e646578d9477368613235362d64646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464a472757374b6696e707574732f746f6f6c636861696e2d696e646578117892d9477368613235362d39366633373231386438333663646364303037393631343233333032353334333362316666633766323538633135353037656536376163323961643665663662ba2f7372762f736572766963652f72656c65617365732f74657374dc0012bd6964756e6e2e65787065637465645f696e6361726e6174696f6e2e7632a773657276696365d9477368613235362d38626532306364633335373930316463373365316162623262336365666633373835343661323962343161353535656131643633653530366562333762376330b5736572766963652d696e6361726e6174696f6e2d31d9477368613235362d39366633373231386438333663646364303037393631343233333032353334333362316666633766323538633135353037656536376163323961643665663662bb6769746875622e636f6d2f47616d6543756c742f53657276696365d92832323232323232323232323232323232323232323232323232323232323232323232323232323232d9477368613235362d32373933646663303463333131396137396235303736396561363735653863303061393836666562333139393930663162303163636436383334356336373261b1736572766963652d79676764726173696cd94031626661653932326138626630313132373562353536623236393135303363386366643538313234383931373134323365373736663133386463313235376337ae736572766963652e6865616c7468d9477368613235362d63636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363a27631d9477368613235362d32663837303134316366353638316566656332653234316364636435663731303831616539326162316535633534633839636438393666386335656236323731c2c09194af736572766963652e72756e74696d65b2736572766963652e72756e74696d652e7631a276310190d9477368613235362d33336163613963353639636165373831643936396632313864633036613035313134613863666239333436313634633033613263336435343033613636323666c0cf0000018bcfe568009abb6964756e6e2e72756e74696d655f61637469766174696f6e2e7632d9477368613235362d33336163613963353639636165373831643936396632313864633036613035313134613863666239333436313634633033613263336435343033613636323666b1736572766963652d79676764726173696cd9477368613235362d66353962643235306336333830383661643330396235333539373636663232326266353033366333336230616633316239633935333130353831366636396137d94066313063623131353633346366613935383966356564616332313730343339363933353834343234643736653133366434396661613236373539323266376665c420f2977384801339144d13ebd2ab001bf29fbab79f7c3948334d440db8206cf974cf0000018bcfe56800d94063653461663030623963626662316362613963363136363965323337303232323465343763623163303761346233313431626634353237383236353965646637a765643235353139c4403a548c511ff3de6fcac116b903fa6badb14deef79dd81958d71947af575fae753b11692d0272c7cb5c437300cdd0d5315e28a1e7e12f9df0a0b817951bba4700dc0033b26964756e6e2d343234322e73657276696365af4964756e6e20746573742034323432af696e766f636174696f6e2d3432343201a465786563a6616c77617973ad636f6e74726f6c2d67726f7570c3a57534323432a57534323432a0a0a0c3c3a9696e76697369626c65a3706964c3a430303737a0a0cd10920194cd1092cd1092cd1092cd109294cd1092cd1092cd1092cd109291cd10920000000000c391010807b52f6f70742f746573742f62696e2f736572766963650101d9477368613235362d34626635313232663334343535346335336264653265626238636432623765336431363030616436333163333835613564376363653233633737383534353961d9477368613235362d66353962643235306336333830383661643330396235333539373636663232326266353033366333336230616633316239633935333130353831366636396137a92f6f70742f74657374a92f72756e2f74657374d9477368613235362d6462633162346339303066666534386435373562356461356336333830343031323566363564623066653365323434393462373665613938363435376439383690d9477368613235362d30383466656430386239373861663464376431393661373434366138366235383030396536333662363131646231363231316236356139616164666632396335d9202f73797374656d2e736c6963652f6964756e6e2d343234322e73657276696365c090aa61637469766174696f6edc0020010101010101010101010101010101010101010101010101010101010101010190d9477368613235362d32393364343764383932393938373139346131303664303236326236326563363766633534393165366339633366653961326663383132643935633237376661c092ad6f64696e2d746f706f6c6f6779959139d9477368613235362d31393538316532376465376365643030666631636535306232303437653761353637633736623163626165626162653565663033663763333031376262356237ab6f64696e2d7369676e65722764c0c0c0c0c0c029b163616e646964617465207265667573656492b56661696c65642d6265666f72652d66656e63696e67b163616e646964617465207265667573656494b163616e6469646174652072656675736564a8636f6d706c657465a7736b6970706564a7736b6970706564c0c0 diff --git a/tests/fixtures/idunn-control-legacy/transaction-pre-fence-abort.hex b/tests/fixtures/idunn-control-legacy/transaction-pre-fence-abort.hex new file mode 100644 index 0000000..ae345b1 --- /dev/null +++ b/tests/fixtures/idunn-control-legacy/transaction-pre-fence-abort.hex @@ -0,0 +1,2 @@ +idunn.deployment_transaction.v3 +dc0023bf6964756e6e2e6465706c6f796d656e745f7472616e73616374696f6e2e7633d92774782d39356138313166662d303533332d343562622d613031352d303038356132353835633061b2636f6e74696e756974792d73657276696365aa636f6e74696e75697479a77365727669636500a77761726d696e67cf0000018bcfe56800cf0000018bcfe56800c099d9216964756e6e2e636f6d70696c65645f6465706c6f796d656e745f706c616e2e7632d9477368613235362d38626532306364633335373930316463373365316162623262336365666633373835343661323962343161353535656131643633653530366562333762376330b5736572766963652d696e6361726e6174696f6e2d316e9bbf6964756e6e2e736f757263655f73656c656374696f6e5f66616374732e7631d92768747470733a2f2f6769746875622e636f6d2f47616d6543756c742f536572766963652e676974af726566732f68656164732f6d61696ed92832323232323232323232323232323232323232323232323232323232323232323232323232323232d92832323232323232323232323232323232323232323232323232323232323232323232323232323232d92833333333333333333333333333333333333333333333333333333333333333333333333333333333bc6465706c6f796d656e742f6964756e6e2f7265636970652e746f6d6cd9477368613235362d323739336466633034633331313961373962353037363965613637356538633030613938366665623331393939306631623031636364363833343563363732618091a87265662d6865616464dc04e20a736368656d61203d202267616d6563756c742e6964756e6e2e7461726765745f6465636c61726174696f6e2e7631220a746172676574203d202273657276696365220a736f757263655f7374616d705f656e7669726f6e6d656e74203d2022534552564943455f4255494c445f434f4d4d4954220a0a5b5b73746570735d5d0a6964203d20226275696c64220a7068617365203d20226275696c64220a72756e6e6572203d202272757374220a61726776203d205b22636172676f222c20226275696c64222c20222d2d6c6f636b6564225d0a0a5b5b65787465726e616c5f696e707574735d5d0a6964203d2022746f6f6c636861696e2d696e646578220a75726c203d202268747470733a2f2f6578616d706c652e696e76616c69642f746f6f6c636861696e2d696e646578220a736861323536203d202264646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464220a72756e6e6572203d202272757374220a64657374696e6174696f6e203d2022696e707574732f746f6f6c636861696e2d696e646578220a0a5b5b6172746966616374735d5d0a6964203d20226461656d6f6e220a736f757263655f6b696e64203d202272756e6e65722d6f7574707574220a72756e6e6572203d202272757374220a736f75726365203d20227461726765742f72656c656173652f73657276696365220a64657374696e6174696f6e203d202273657276696365220a65787065637465645f736861323536203d202263636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363220a65786563757461626c65203d20747275650a0a5b736572766963655d0a65786563757461626c655f6172746966616374203d20226461656d6f6e220a617267756d656e7473203d205b0a20207b206b696e64203d202262696e64696e67222c206e616d65203d202273746174655f726f6f7422207d2c0a5d0a7472616e73706f7274203d202268747470220a726f7574655f7265717569726564203d2066616c73650a72657175697265645f656e7669726f6e6d656e74203d205b2247414d4543554c545f4944554e4e5f52554e54494d455f42554e444c45225d0a0a5b736572766963652e6865616c74685d0a636f6e7472616374203d2022736572766963652e6865616c7468220a0a5b73746174655d0a736368656d615f67656e65726174696f6e203d20227631220a0a5b5b73746174652e736c6f74735d5d0a6964203d202272756e74696d652d6361636865220a72656c61746976655f70617468203d202272756e74696d652d63616368652e6363220a6b696e64203d202263756c7463616368652d66696c65220a736368656d61203d2022736572766963652e72756e74696d652d63616368652e7631220a777269746572203d20226e6f6e65220a7265636f76657279203d202272656275696c6461626c65220a73746172747570203d20226f70656e2d61742d7374617274220a0a5b5b70726f76696465735d5d0a6361706162696c697479203d2022736572766963652e72756e74696d65220a736368656d61203d2022736572766963652e72756e74696d652e7631220a636f6d7061746962696c697479203d20227631220a0adc06930a736368656d61203d202267616d6563756c742e6964756e6e2e6f70657261746f725f62696e64696e672e7632220a746172676574203d202273657276696365220a0a5b7265706f7369746f72795d0a6f726967696e203d202268747470733a2f2f6769746875622e636f6d2f47616d6543756c742f536572766963652e676974220a61646d69747465645f726566203d2022726566732f68656164732f6d61696e220a6d696e696d756d5f7265766973696f6e203d202231313131313131313131313131313131313131313131313131313131313131313131313131313131220a73656c656374696f6e203d20227265662d68656164220a636865636b6f7574203d20222f7372762f6275696c642f53657276696365220a7265636970655f70617468203d20226465706c6f796d656e742f6964756e6e2f7265636970652e746f6d6c220a0a5b72756e6e6572732e727573745d0a647269766572203d2022646f636b6572220a696d616765203d202272757374407368613235363a61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161220a75736572203d2022313030303a31303030220a6166666f7264616e636573203d205b22736f757263652d72656164222c202261727469666163742d7772697465225d0a616c6c6f7765645f70726f6772616d73203d205b22636172676f225d0a6e6574776f726b5f70726f66696c65203d20226275696c642d646570656e64656e63792d656772657373220a6d656d6f72795f6d6562696279746573203d20323034380a6370755f71756f74615f70657263656e74203d203230300a706964735f6c696d6974203d203531320a746d7066735f6d6562696279746573203d203531320a0a5b776f726b6c6f61645d0a647269766572203d202273797374656d642d7472616e7369656e74220a73746174655f67726f7570203d202273657276696365220a756e69745f707265666978203d20226964756e6e2d73657276696365220a72656c656173655f726f6f74203d20222f7372762f736572766963652f72656c6561736573220a73746174655f726f6f74203d20222f7661722f6c69622f67616d6563756c742f73657276696365220a72756e74696d655f726f6f74203d20222f72756e2f67616d6563756c742f6964756e6e2f73657276696365220a6e6574776f726b203d2022686f73742d70726976617465220a68617264656e696e67203d2022737472696374220a6d656d6f72795f6d6562696279746573203d20313032340a6370755f71756f74615f70657263656e74203d203130300a0a5b776f726b6c6f61642e617267756d656e745f62696e64696e67735d0a73746174655f726f6f74203d20222f7661722f6c69622f67616d6563756c742f73657276696365220a0a5b776f726b6c6f61642e7365637265745f66696c65735d0a47414d4543554c545f52554e54494d455f50524553454e43455f4944454e54495459203d20222f6574632f67616d6563756c742f736572766963652f72756e74696d652d70726573656e63652d6964656e746974792e6363220a0a5b72756e74696d655f6964656e746974795d0a72756e74696d655f6964203d2022736572766963652d79676764726173696c220a65787065637465645f7369676e65725f6964656e746974795f6964203d202231626661653932326138626630313132373562353536623236393135303363386366643538313234383931373134323365373736663133386463313235376337220a74727573745f616e63686f725f73746f7265203d20222f746d702f6964756e6e2d62312d6c65676163792f70726f76696465722d616e63686f722e6363220a0a5b6272616b65735d0a6465706c6f796d656e745f73746f7265203d20222f7661722f6c69622f67616d6563756c742f6964756e6e2f736572766963652d6465706c6f796d656e742d6272616b652e6363220a6c6966656379636c655f73746f7265203d20222f7661722f6c69622f67616d6563756c742f6964756e6e2f736572766963652d6c6966656379636c652d6272616b652e6363220a0a5b726f6c6c6f75745d0a7374726174656779203d202263616e6469646174652d7468656e2d70726f6d6f7465220a647261696e5f7365636f6e6473203d2033300a72657461696e5f72656c6561736573203d20320a0a5b706c6163656d656e745d0a646573697265645f7265706c69636173203d20310a6e6f646573203d205b2279676764726173696c225d0a90c0c096b76964756e6e2e7365616c65645f72656c656173652e7631d9477368613235362d39366633373231386438333663646364303037393631343233333032353334333362316666633766323538633135353037656536376163323961643665663662d9477368613235362d386265323063646333353739303164633733653161626232623363656666333738353436613239623431613535356561316436336535303665623337623763309195a66461656d6f6ea773657276696365d9477368613235362d636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363632ac39196af746f6f6c636861696e2d696e646578d92768747470733a2f2f6578616d706c652e696e76616c69642f746f6f6c636861696e2d696e646578d9477368613235362d64646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464a472757374b6696e707574732f746f6f6c636861696e2d696e646578117892d9477368613235362d39366633373231386438333663646364303037393631343233333032353334333362316666633766323538633135353037656536376163323961643665663662ba2f7372762f736572766963652f72656c65617365732f74657374dc0012bd6964756e6e2e65787065637465645f696e6361726e6174696f6e2e7632a773657276696365d9477368613235362d38626532306364633335373930316463373365316162623262336365666633373835343661323962343161353535656131643633653530366562333762376330b5736572766963652d696e6361726e6174696f6e2d31d9477368613235362d39366633373231386438333663646364303037393631343233333032353334333362316666633766323538633135353037656536376163323961643665663662bb6769746875622e636f6d2f47616d6543756c742f53657276696365d92832323232323232323232323232323232323232323232323232323232323232323232323232323232d9477368613235362d32373933646663303463333131396137396235303736396561363735653863303061393836666562333139393930663162303163636436383334356336373261b1736572766963652d79676764726173696cd94031626661653932326138626630313132373562353536623236393135303363386366643538313234383931373134323365373736663133386463313235376337ae736572766963652e6865616c7468d9477368613235362d63636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363a27631d9477368613235362d32663837303134316366353638316566656332653234316364636435663731303831616539326162316535633534633839636438393666386335656236323731c2c09194af736572766963652e72756e74696d65b2736572766963652e72756e74696d652e7631a276310190d9477368613235362d33336163613963353639636165373831643936396632313864633036613035313134613863666239333436313634633033613263336435343033613636323666c0cf0000018bcfe568009abb6964756e6e2e72756e74696d655f61637469766174696f6e2e7632d9477368613235362d33336163613963353639636165373831643936396632313864633036613035313134613863666239333436313634633033613263336435343033613636323666b1736572766963652d79676764726173696cd9477368613235362d66353962643235306336333830383661643330396235333539373636663232326266353033366333336230616633316239633935333130353831366636396137d94066313063623131353633346366613935383966356564616332313730343339363933353834343234643736653133366434396661613236373539323266376665c420f2977384801339144d13ebd2ab001bf29fbab79f7c3948334d440db8206cf974cf0000018bcfe56800d94063653461663030623963626662316362613963363136363965323337303232323465343763623163303761346233313431626634353237383236353965646637a765643235353139c4403a548c511ff3de6fcac116b903fa6badb14deef79dd81958d71947af575fae753b11692d0272c7cb5c437300cdd0d5315e28a1e7e12f9df0a0b817951bba4700dc0033b26964756e6e2d343234322e73657276696365af4964756e6e20746573742034323432af696e766f636174696f6e2d3432343201a465786563a6616c77617973ad636f6e74726f6c2d67726f7570c3a57534323432a57534323432a0a0a0c3c3a9696e76697369626c65a3706964c3a430303737a0a0cd10920194cd1092cd1092cd1092cd109294cd1092cd1092cd1092cd109291cd10920000000000c391010807b52f6f70742f746573742f62696e2f736572766963650101d9477368613235362d34626635313232663334343535346335336264653265626238636432623765336431363030616436333163333835613564376363653233633737383534353961d9477368613235362d66353962643235306336333830383661643330396235333539373636663232326266353033366333336230616633316239633935333130353831366636396137a92f6f70742f74657374a92f72756e2f74657374d9477368613235362d6462633162346339303066666534386435373562356461356336333830343031323566363564623066653365323434393462373665613938363435376439383690d9477368613235362d30383466656430386239373861663464376431393661373434366138366235383030396536333662363131646231363231316236356139616164666632396335d9202f73797374656d2e736c6963652f6964756e6e2d343234322e73657276696365c090aa61637469766174696f6edc0020010101010101010101010101010101010101010101010101010101010101010190d9477368613235362d32393364343764383932393938373139346131303664303236326236326563363766633534393165366339633366653961326663383132643935633237376661c092ad6f64696e2d746f706f6c6f6779959139d9477368613235362d31393538316532376465376365643030666631636535306232303437653761353637633736623163626165626162653565663033663763333031376262356237ab6f64696e2d7369676e65722764c0c0c0c0c0c029c0c094b163616e6469646174652072656675736564a770656e64696e67a7736b6970706564a7736b6970706564c0c0 From ff3cd519105ce0be6a17fd63b01a3a7e2ea432fd Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 06:45:37 +0200 Subject: [PATCH 015/286] Pin the B1 fix batch's survivors cargo-mutants over the batch's diff missed the lift's gate, the unexplained projection accounting, the refusal count and the deferral clock. Tests now cover each: the lift owes only a continuity that issued an activation; a live transaction accounts for its own activation and nobody else's; three refusals in history hold the restart ceiling; the deferral lands between now and now plus its length. Co-Authored-By: Claude Sonnet 5 --- src/control_plane.rs | 86 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 85 insertions(+), 1 deletion(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 6c97384..7edba23 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -11623,7 +11623,11 @@ mod tests { .continuity_backoff .next_restart_at_unix_millis .context("the failed demotion was not recorded")?; - assert!(deferred >= before + CONTINUITY_DEFERRAL_MILLIS); + let after = now_millis()?; + assert!( + (before + CONTINUITY_DEFERRAL_MILLIS..=after + CONTINUITY_DEFERRAL_MILLIS) + .contains(&deferred) + ); // Deferred: the next tick neither retries nor writes. let envelope = incumbent_envelope(&world)?; @@ -11720,6 +11724,28 @@ mod tests { world.engine.reconcile_failed_continuity_projections()?, ProjectionReconciliation::default() ); + // ...and for nobody else's. + topology.withdraw_stale_incarnation(&incumbent.expected, &anchor)?; + let mut other = stranger.clone(); + let other_activation = IdunnRuntimeActivationLaunch::issue( + &incumbent.expected, + runtime_instance_id("tx-someone-else")?, + now_millis()?, + &world.engine.idunn_signer, + )? + .activation() + .clone(); + if let Some(WorkloadObservation::Systemd(observed)) = other.workload.as_mut() { + observed.runtime_instance_id = other_activation.runtime_instance_id.clone(); + } + other.activation = Some(other_activation); + project_candidate(&world, &other, true)?; + assert_eq!( + world.engine.reconcile_failed_continuity_projections()?.unexplained, + vec!["service".to_owned()] + ); + topology.withdraw_stale_incarnation(&incumbent.expected, &anchor)?; + project_candidate(&world, &stranger, true)?; // ...and once it is gone, nothing does. let seeded = resident(&world)?; let command = ControlSnapshot::read(&world.state_store)? @@ -11876,4 +11902,62 @@ mod tests { assert_eq!(ControlSnapshot::read(&world.state_store)?.transactions.len(), 1); Ok(()) } + + #[test] + fn the_legacy_lift_owes_only_a_continuity_that_issued_an_activation() -> Result<()> { + let envelope = fixture_envelope( + FIXTURE_PRE_B1_ABORTS[0].1, + DeploymentTransaction::TYPE, + )?; + let written = lift_legacy_transaction(&envelope)?; + let owed = |transaction: &DeploymentTransaction| { + transaction + .pre_fencing_abort + .as_ref() + .map(|abort| abort.topology_reconciliation) + }; + assert_eq!(owed(&written), Some(CleanupEvidence::Skipped)); + + let mut lifted = written.clone(); + owe_legacy_continuity_projection(&mut lifted); + assert_eq!(owed(&lifted), Some(CleanupEvidence::Pending)); + + // A deployment published its own key and recorded what it owed. + let mut deploy = written.clone(); + deploy.command_kind = CommandKind::Deploy; + owe_legacy_continuity_projection(&mut deploy); + assert_eq!(owed(&deploy), Some(CleanupEvidence::Skipped)); + + // A continuity that issued nothing owes nothing. + let mut unissued = written; + unissued.activation = None; + owe_legacy_continuity_projection(&mut unissued); + assert_eq!(owed(&unissued), Some(CleanupEvidence::Skipped)); + Ok(()) + } + + #[test] + fn continuity_stops_after_three_refused_restarts_counted_from_history() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + for round in 0..CONTINUITY_RESTART_ATTEMPTS { + let failed = seeded_transaction( + &world, + DeploymentPhase::Warming, + CommandKind::Continuity, + Some(&incumbent), + )?; + world + .engine + .begin_pre_fencing_abort(&resident(&world)?, anyhow!("refused {round}"))?; + drive(&world, |transaction| transaction.completion.is_some())?; + assert!(record_of(&world, &failed.transaction_id)?.is_terminal()); + } + workload.kill(); + // Three refusals are in history and none is live: the ceiling holds. + assert!(!world.engine.supervise_one_admitted_generation()?); + assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); + Ok(()) + } } From 3d4c9d34f1b58805032f2db4b7edcd93a35895aa Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 07:10:31 +0200 Subject: [PATCH 016/286] Correct the B1 legacy ruling: no reopen, reconciliation owns the residue Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index 5d30194..512ac09 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -471,7 +471,7 @@ refused, the revert exact). The transition failed: - Low: `docs/deployment-authority.md` says the projection "never drifts". **Self's ruling for the B1 fix batch, which runs on top of F0 because F0 owns the legacy lift:** -1. The legacy-transaction lift maps a pre-B1 continuity abort that issued an activation to +1. ~~(superseded below)~~ The legacy-transaction lift maps a pre-B1 continuity abort that issued an activation to `Pending`. The new single resolution then demotes its own activation, which also cleans the residue that record left behind. 2. Supervision owns the Expected-only precondition. It does not mint a continuity while its @@ -485,6 +485,35 @@ refused, the revert exact). The transition failed: Before any deploy, check the live host for resident continuity abort records. That is a read of `control.cc`, and the operator's to authorise. +**B1 fix batch 1, Soul pass, 2026-09-29: do not close. Self's corrected ruling replaces item 1 above.** +Hands implemented item 1 by reopening terminal pre-B1 records at `Starting`. Soul confirmed the +cost: +- If the target also holds a live transaction, or a second resident pre-B1 abort, the reopened + record fails `ControlSnapshot::read` and Idunn does not boot. +- The migration has already written the record as v4, so the old binary cannot roll back. +- The reopen is a second owner for residue that boot reconciliation already cleans. +The defect was in Self's ruling: it set a state on a record without saying what happens to +terminal records. + +**Corrected ruling:** +- Nothing is reopened. +- A terminal pre-B1 abort keeps its phase. Validation accepts its shape under a **typed legacy + marker** that only the lift sets. +- **Boot reconciliation is the single owner of legacy residue.** +- In-flight pre-B1 aborts may lift to `Pending` only if they cannot collide with another live + transaction. + +**Also fixed in batch 2:** +- A failed pre-fence abort goes to `record_resumable_error`, never to + `begin_post_fencing_abort`. Before this, one wedged abort failed every scheduler tick for + every target. +- Errors are isolated per transaction, including `archive_terminal_transaction`. +- `serve`'s boot wiring is pinned by a test. + +**Moved to B2:** the continuity restart ceiling reads only `ContinuityBackoff`, never +`history.cc`. Otherwise an unreadable history file stops crash recovery for every target. +Deferral and backoff show in `idunn status`. + **B2 — Bounded, backed-off route and unit repair (Idunn).** *Widened 2026-09-29 by Self, from Soul's S1 pass.* S1 closed the healthy-target storm, and From b75fbdf96cc1b58f88fa112d26070caae58e06eb Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 07:17:11 +0200 Subject: [PATCH 017/286] Boot reconciliation owns legacy residue; the scheduler isolates per-record faults Soul's pass on the B1 fix batch found the lift's reopen of a terminal pre-B1 continuity abort could stop Idunn booting (the reopened record claims target authority beside a live transaction, and the store was already migrated to v4), and that one wedged abort stopped the whole scheduler. - The lift no longer reopens anything. A terminal pre-B1 continuity abort that issued an activation keeps its phase and completion and carries the typed marker CleanupEvidence::LegacySkippedBeforeB1 on its topology_reconciliation. Validation accepts that shape only for a terminal continuity abort. Boot reconciliation, already the exact-activation owner of that residue, is the only cleanup path. A pre-B1 abort still in flight lifts to Pending: it already holds its target's authority, so the lift adds no claimant. - A failed step of a pre-fence abort is recorded as a resumable error, never handed to the post-fence abort, which refuses a pre-fence phase and failed every tick. - One transaction's error no longer fails run_scheduler_tick: each candidate is resumed on its own and its fault logged, and retiring a terminal record tries the next one when an archive fails. An unreadable control store still fails the tick. - serve is boot plus the loop, so a test can run the boot sequence. Tests pin, through the Engine: the marker and no-reopen for each pre-B1 fixture; a resident terminal abort plus a live transaction plus a second resident abort for one target; a withdrawn incarnation never re-published and history keeping one copy after a crash between the archive steps; a wedged abort leaving freezing alive; an unarchivable terminal leaving resume alive; and boot demoting the residue. Co-Authored-By: Claude Sonnet 5 --- src/control_plane.rs | 611 +++++++++++++++++++++++++++++++++---------- 1 file changed, 471 insertions(+), 140 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 7edba23..48ecf07 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -751,11 +751,20 @@ enum CleanupEvidence { Pending, Skipped, Complete, + /// Set only by the legacy lift, on a terminal continuity abort written + /// before the single resolution rule: it issued an activation and recorded + /// `Skipped`. Boot reconciliation, not the record, owes that residue's + /// demotion. The marker is what lets validation accept the old shape + /// without reopening a finished transaction. + LegacySkippedBeforeB1, } impl CleanupEvidence { fn is_complete(self) -> bool { - matches!(self, Self::Skipped | Self::Complete) + matches!( + self, + Self::Skipped | Self::Complete | Self::LegacySkippedBeforeB1 + ) } } @@ -1304,7 +1313,10 @@ impl DeploymentTransaction { CleanupEvidence::Pending, CleanupEvidence::Pending | CleanupEvidence::Complete ) | (CleanupEvidence::Skipped, CleanupEvidence::Skipped) - ), + ) || (self.command_kind == CommandKind::Continuity + && self.phase == DeploymentPhase::Complete + && self.completion.is_some() + && abort.topology_reconciliation == CleanupEvidence::LegacySkippedBeforeB1), "abort topology cleanup differs from what the transaction projected" ); ensure!( @@ -2707,31 +2719,34 @@ fn read_transaction_record(envelope: &CultCacheEnvelope) -> Result Result<()> { +/// Everything the daemon does once, before its loop: lock, migrate, validate, +/// reconcile the projection. The lock is returned so the caller holds it for as +/// long as the engine runs. +fn boot(options: RuntimeOptions) -> Result<(ProcessLock, Engine)> { for path in [ &options.state_store, &options.topology_store, @@ -3900,7 +3918,7 @@ fn serve(options: RuntimeOptions) -> Result<()> { fs::create_dir_all(directory) .with_context(|| format!("creating Idunn directory {}", directory.display()))?; } - let _lock = ProcessLock::acquire(&options.state_store)?; + let lock = ProcessLock::acquire(&options.state_store)?; let migrated = migrate_control_store_to_current_schema(&options.state_store) .context("migrating Idunn control records to the current schema")?; if migrated > 0 { @@ -3925,7 +3943,11 @@ fn serve(options: RuntimeOptions) -> Result<()> { } Err(error) => eprintln!("Idunn boot projection reconciliation failed: {error:#}"), } + Ok((lock, engine)) +} +fn serve(options: RuntimeOptions) -> Result<()> { + let (_lock, engine) = boot(options)?; loop { match engine.run_scheduler_tick() { Ok(true) => continue, @@ -3944,9 +3966,6 @@ fn serve(options: RuntimeOptions) -> Result<()> { } impl Engine { - /// One pass of the scheduler, with the loop's ordering preserved exactly: - /// `Ok(true)` means "went round again immediately", which is what the - /// caller's `continue` did. /// One resident terminal transaction goes to history per tick. /// /// Transactions that finished before finished transactions travelled to @@ -3955,21 +3974,34 @@ impl Engine { /// reads. They leave the same way a transaction finishing today does. fn retire_one_terminal_transaction(&self) -> Result { let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let Some(stored) = snapshot + // A record that cannot be archived (history unreadable, a torn write) + // is that record's fault: the next terminal record is still tried, and + // the tick goes on to supervise and freeze. + for stored in snapshot .transactions .iter() - .find(|stored| stored.value.is_terminal()) - else { - return Ok(false); - }; - archive_terminal_transaction(&self.options.state_store, &stored.envelope)?; - eprintln!( - "Idunn retired resident terminal transaction {} to history", - stored.value.transaction_id - ); - Ok(true) + .filter(|stored| stored.value.is_terminal()) + { + match archive_terminal_transaction(&self.options.state_store, &stored.envelope) { + Ok(()) => { + eprintln!( + "Idunn retired resident terminal transaction {} to history", + stored.value.transaction_id + ); + return Ok(true); + } + Err(error) => eprintln!( + "Idunn could not retire terminal transaction {}: {error:#}", + stored.value.transaction_id + ), + } + } + Ok(false) } + /// One pass of the scheduler. An unreadable control store is the tick's + /// fault and is returned; anything one transaction or one admitted + /// generation does wrong is logged against it and never stops the others. fn run_scheduler_tick(&self) -> Result { if self.retire_one_terminal_transaction()? { return Ok(true); @@ -4002,45 +4034,61 @@ impl Engine { if snapshot.has_earlier_authority_sibling(¤t.value) { continue; } - if let Err(error) = self.advance_transaction(current) { - let latest_snapshot = ControlSnapshot::read(&self.options.state_store)?; - let latest = latest_snapshot - .transactions - .iter() - .find(|stored| stored.value.transaction_id == current.value.transaction_id) - .context("transaction disappeared while recording an execution error")?; - if latest.value.is_terminal() { - progressed = true; - } else if latest.value.phase < DeploymentPhase::Fencing - && latest.value.pre_fencing_abort.is_none() - { - self.begin_pre_fencing_abort(latest, error)?; - } else if latest.value.post_fencing_abort.is_none() - && self.candidate_is_permanently_stopped(&latest.value)? - { - // Past the fence an error is resumable while the candidate - // can still recover. This one cannot: its transient unit - // has failed and carries Restart=no, so retrying would hold - // the target forever behind a transaction that can never - // finish. - self.begin_post_fencing_abort(latest, error)?; - } else { - self.record_resumable_error(latest, &error)?; - } - } - let after = ControlSnapshot::read(&self.options.state_store)?; - let live = after - .transactions - .iter() - .find(|stored| stored.value.transaction_id == current.value.transaction_id) - .context("transaction disappeared while checking scheduler progress")?; - if live.envelope != current.envelope { - progressed = true; + match self.resume_candidate(current) { + Ok(moved) => progressed |= moved, + Err(error) => eprintln!( + "Idunn could not resume transaction {}: {error:#}", + current.value.transaction_id + ), } } Ok(progressed) } + /// Advance one transaction, turning its failure into the durable record + /// its phase calls for. Returns whether its record changed. + fn resume_candidate(&self, current: &Stored) -> Result { + if let Err(error) = self.advance_transaction(current) { + let latest_snapshot = ControlSnapshot::read(&self.options.state_store)?; + let latest = latest_snapshot + .transactions + .iter() + .find(|stored| stored.value.transaction_id == current.value.transaction_id) + .context("transaction disappeared while recording an execution error")?; + if latest.value.is_terminal() { + return Ok(true); + } + if latest.value.phase < DeploymentPhase::Fencing { + // Before the fence an abort that cannot finish a step is + // resumable, never a post-fence abort: that path refuses a + // pre-fence phase, so choosing it only wedged the tick. + if latest.value.pre_fencing_abort.is_none() { + self.begin_pre_fencing_abort(latest, error)?; + } else { + self.record_resumable_error(latest, &error)?; + } + } else if latest.value.post_fencing_abort.is_none() + && self.candidate_is_permanently_stopped(&latest.value)? + { + // Past the fence an error is resumable while the candidate + // can still recover. This one cannot: its transient unit + // has failed and carries Restart=no, so retrying would hold + // the target forever behind a transaction that can never + // finish. + self.begin_post_fencing_abort(latest, error)?; + } else { + self.record_resumable_error(latest, &error)?; + } + } + let after = ControlSnapshot::read(&self.options.state_store)?; + let live = after + .transactions + .iter() + .find(|stored| stored.value.transaction_id == current.value.transaction_id) + .context("transaction disappeared while checking scheduler progress")?; + Ok(live.envelope != current.envelope) + } + fn freeze_one_queued_command(&self) -> Result { let snapshot = ControlSnapshot::read(&self.options.state_store)?; let live_commands = snapshot @@ -11459,8 +11507,57 @@ mod tests { Ok(()) } + fn abort_owed(transaction: &DeploymentTransaction) -> Option { + transaction + .pre_fencing_abort + .as_ref() + .map(|abort| abort.topology_reconciliation) + .or_else(|| { + transaction + .post_fencing_abort + .as_ref() + .map(|abort| abort.topology_reconciliation) + }) + } + + /// Put a continuity's record and its command in the control store as they + /// stood: `envelope` is written as given, in whatever schema it is in. + fn make_resident( + world: &EngineFixture, + transaction: &DeploymentTransaction, + envelope: CultCacheEnvelope, + ) -> Result<()> { + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: transaction.command_id.clone(), + kind: CommandKind::Continuity, + selector: "service".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: envelope.key.clone(), + current: None, + }, + ], + &[command_envelope(&command, 100)?, envelope], + )? + ); + Ok(()) + } + #[test] - fn a_pre_b1_continuity_abort_lifts_owing_its_own_activation_and_resolves_it() -> Result<()> { + fn a_pre_b1_continuity_abort_lifts_without_reopening_and_boot_resolves_its_residue() + -> Result<()> { write_fixture_anchor()?; for (name, text) in FIXTURE_PRE_B1_ABORTS { let envelope = fixture_envelope(text, DeploymentTransaction::TYPE)?; @@ -11470,79 +11567,33 @@ mod tests { let legacy: LegacyDeploymentTransaction = rmp_serde::from_slice(&envelope.payload)?; assert_eq!(legacy.command_kind, CommandKind::Continuity, "{name}"); assert!(legacy.activation.is_some(), "{name}"); - let owed = legacy - .pre_fencing_abort - .as_ref() - .map(|abort| abort.topology_reconciliation) - .or_else(|| { - legacy - .post_fencing_abort - .as_ref() - .map(|abort| abort.topology_reconciliation) - }) - .context("no abort")?; - assert_eq!(owed, CleanupEvidence::Skipped, "{name}"); // History describes what happened, so it lifts the record as written. let archived = lift_legacy_transaction(&envelope)?; - let archived_owed = archived - .pre_fencing_abort - .as_ref() - .map(|abort| abort.topology_reconciliation) - .or_else(|| { - archived - .post_fencing_abort - .as_ref() - .map(|abort| abort.topology_reconciliation) - }) - .context("no abort")?; - assert_eq!(archived_owed, CleanupEvidence::Skipped, "{name}"); + assert_eq!(abort_owed(&archived), Some(CleanupEvidence::Skipped), "{name}"); - // The control store lifts it owing the demotion, and validates. + // The control store lifts it without reopening anything. let lifted = read_transaction_record(&envelope)?; - let lifted_owed = lifted - .pre_fencing_abort - .as_ref() - .map(|abort| abort.topology_reconciliation) - .or_else(|| { - lifted - .post_fencing_abort - .as_ref() - .map(|abort| abort.topology_reconciliation) - }) - .context("no abort")?; - assert_eq!(lifted_owed, CleanupEvidence::Pending, "{name}"); - assert!(!lifted.is_terminal(), "{name}: a record that owes work is terminal"); - assert_eq!(lifted.completion, None, "{name}"); + let terminal = archived.completion.is_some(); + if terminal { + assert_eq!( + abort_owed(&lifted), + Some(CleanupEvidence::LegacySkippedBeforeB1), + "{name}" + ); + assert!(lifted.is_terminal(), "{name}: the lift reopened a finished record"); + assert_eq!(lifted.phase, archived.phase, "{name}"); + assert_eq!(lifted.completion, archived.completion, "{name}"); + assert!(!lifted.owns_target_authority(), "{name}"); + } else { + assert_eq!(abort_owed(&lifted), Some(CleanupEvidence::Pending), "{name}"); + assert!(!lifted.is_terminal(), "{name}"); + assert_eq!(lifted.completion, None, "{name}"); + } - // Boot: the store reads, and the abort resolves. + // Boot: the store reads. let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let command = DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: lifted.command_id.clone(), - kind: CommandKind::Continuity, - selector: "service".into(), - requested_by: "test".into(), - requested_at_unix_millis: 100, - }; - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( - &[ - CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: command.command_id.clone(), - current: None, - }, - CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: envelope.key.clone(), - current: None, - }, - ], - &[command_envelope(&command, 100)?, envelope.clone()], - )?, - "{name}" - ); + make_resident(&world, &lifted, envelope.clone())?; assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); let snapshot = ControlSnapshot::read(&world.state_store)?; assert_eq!(snapshot.transactions.len(), 1, "{name}"); @@ -11550,15 +11601,27 @@ mod tests { let expected = lifted.expected.clone().context("no Expected")?; assert_eq!(projected_under(&world, &expected)?.len(), 2, "{name}"); - for _ in 0..12 { - let Ok(current) = resident(&world) else { break }; - if current.value.completion.is_some() { - break; + if terminal { + // A finished record owes nothing and is not touched: boot + // reconciliation demotes the residue by its exact activation. + let before = resident(&world)?.envelope; + let outcome = world.engine.reconcile_failed_continuity_projections()?; + assert_eq!(outcome.demoted, vec![lifted.transaction_id.clone()], "{name}"); + assert_eq!(resident(&world)?.envelope, before, "{name}"); + } else { + // A record in flight resolves through its own abort. + for _ in 0..12 { + let Ok(current) = resident(&world) else { break }; + if current.value.completion.is_some() { + break; + } + world.engine.advance_transaction(¤t)?; } - world.engine.advance_transaction(¤t)?; + assert!( + record_of(&world, &envelope.key)?.completion.is_some(), + "{name}: the abort never resolved" + ); } - let finished = record_of(&world, &envelope.key)?; - assert!(finished.completion.is_some(), "{name}: the abort never resolved"); assert_eq!( projected_under(&world, &expected)?, expected_only(), @@ -11568,6 +11631,274 @@ mod tests { Ok(()) } + #[test] + fn resident_terminal_pre_b1_aborts_never_claim_target_authority() -> Result<()> { + write_fixture_anchor()?; + let envelope = fixture_envelope(FIXTURE_PRE_B1_ABORTS[1].1, DeploymentTransaction::TYPE)?; + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let live = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, None)?; + let first = read_transaction_record(&envelope)?; + assert_eq!(first.target, live.target); + make_resident(&world, &first, envelope)?; + assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); + + // A second resident abort of the same shape for the same target, + // already written as the marker the migration persists. + let mut second = first.clone(); + second.transaction_id = "tx-second-legacy".into(); + second.command_id = "continuity-second-legacy".into(); + make_resident(&world, &second, transaction_envelope(&second, 200)?)?; + + // The store reads: three records, one of them live, none of the + // finished ones claiming the target. + let snapshot = ControlSnapshot::read(&world.state_store)?; + assert_eq!(snapshot.transactions.len(), 3); + let live_before = snapshot + .transactions + .iter() + .find(|stored| stored.value.transaction_id == live.transaction_id) + .context("live transaction")? + .envelope + .clone(); + + // Reconciliation cleans the residue once, by exact activation. + let expected = first.expected.clone().context("no Expected")?; + project_candidate(&world, &first, true)?; + assert_eq!(projected_under(&world, &expected)?.len(), 2); + let outcome = world.engine.reconcile_failed_continuity_projections()?; + assert_eq!(outcome.demoted.len(), 1); + assert_eq!(projected_under(&world, &expected)?, expected_only()); + let after = ControlSnapshot::read(&world.state_store)?; + assert_eq!( + after + .transactions + .iter() + .find(|stored| stored.value.transaction_id == live.transaction_id) + .context("live transaction")? + .envelope, + live_before + ); + Ok(()) + } + + #[test] + fn boot_never_republishes_a_withdrawn_incarnation_and_history_keeps_one_copy() -> Result<()> { + write_fixture_anchor()?; + let envelope = fixture_envelope(FIXTURE_PRE_B1_ABORTS[1].1, DeploymentTransaction::TYPE)?; + let lifted = read_transaction_record(&envelope)?; + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + make_resident(&world, &lifted, envelope.clone())?; + // A crash between the two archive steps: history already holds it. + assert!( + SingleFileMessagePackBackingStore::new(&history_store_path(&world.state_store)) + .insert_entry_if_absent(envelope)? + ); + assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); + + // The incarnation was withdrawn: nothing is projected for it. + let expected = lifted.expected.clone().context("no Expected")?; + assert!(projected_under(&world, &expected)?.is_empty()); + for _ in 0..3 { + world.engine.run_scheduler_tick()?; + } + assert!( + world + .engine + .reconcile_failed_continuity_projections()? + .demoted + .is_empty() + ); + assert!( + projected_under(&world, &expected)?.is_empty(), + "a withdrawn incarnation's Expected was published again" + ); + + // One copy in history, finished, and nothing resident. + let history = read_history(&world.state_store)?; + assert_eq!(history.transactions.len(), 1); + assert!(history.transactions[0].is_terminal()); + assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); + Ok(()) + } + + #[test] + fn boot_runs_the_projection_reconciliation_serve_starts_from() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let incumbent = admit_incumbent(&world)?; + let failed = seeded_transaction( + &world, + DeploymentPhase::Warming, + CommandKind::Continuity, + Some(&incumbent), + )?; + project_candidate(&world, &failed, true)?; + world + .engine + .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate died"))?; + drive(&world, |transaction| transaction.completion.is_some())?; + // A pre-B1 abort's residue: the failed candidate's activation stands. + project_candidate(&world, &failed, true)?; + assert_eq!(projected_under(&world, &incumbent.expected)?.len(), 2); + + let (_lock, _engine) = boot(world.engine.options.clone())?; + assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); + Ok(()) + } + + /// A workload whose candidate can neither be stopped nor recover: an abort + /// that cannot finish its first step. + struct WedgedWorkload; + + impl WorkloadPort for WedgedWorkload { + fn install( + &self, + plan: &CompiledDeploymentPlan, + release: &crate::drivers::MaterializedRelease, + ) -> Result { + StillWorkload.install(plan, release) + } + fn prepare_activation( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + launch: IdunnRuntimeActivationLaunch, + ) -> Result { + StillWorkload.prepare_activation(plan, expected, launch) + } + fn start_prepared( + &self, + plan: &CompiledDeploymentPlan, + release: &SealedRelease, + installed: &crate::drivers::InstalledReleaseObservation, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result { + StillWorkload.start_prepared(plan, release, installed, expected, activation) + } + fn discard_prepared( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result<()> { + StillWorkload.discard_prepared(plan, expected, activation) + } + fn observe( + &self, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + prior: &WorkloadObservation, + ) -> Result { + StillWorkload.observe(expected, activation, prior) + } + fn stop(&self, _: &WorkloadObservation) -> Result<()> { + bail!("the unit will not stop") + } + fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { + Ok(true) + } + } + + #[test] + fn one_wedged_pre_fence_abort_does_not_stop_the_scheduler() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(WedgedWorkload))?; + let wedged = transaction_at(&world, DeploymentPhase::Warming)?; + world + .engine + .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "up-nowhere".into(), + kind: CommandKind::Deploy, + selector: "nowhere".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }], + &[command_envelope(&command, 100)?], + )? + ); + + // Every tick answers: the wedged abort records its error and waits, + // and the rest of the scheduler goes on to freeze the queued command. + for _ in 0..6 { + world.engine.run_scheduler_tick()?; + } + let stuck = record_of(&world, &wedged.transaction_id)?; + assert!(!stuck.is_terminal()); + assert!(stuck.post_fencing_abort.is_none()); + assert!(stuck.last_error.is_some()); + assert!( + read_history_transactions(&world.state_store) + .iter() + .any(|transaction| transaction.command_id == command.command_id), + "the queued command was never frozen" + ); + Ok(()) + } + + #[test] + fn a_terminal_transaction_that_cannot_be_archived_does_not_stop_the_scheduler() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let live = transaction_at(&world, DeploymentPhase::Warming)?; + odin_reports_ready(&world, &live, seeded_sequence(None) + 1)?; + let (finished, command) = terminal_transaction_with_command("ghostlight")?; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: finished.transaction_id.clone(), + current: None, + }, + ], + &[ + command_envelope(&command, 100)?, + transaction_envelope(&finished, finished.updated_at_unix_millis)?, + ], + )? + ); + corrupt_history(&world)?; + let before = ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .find(|stored| stored.value.transaction_id == live.transaction_id) + .context("live transaction")? + .envelope; + + // The finished record cannot be archived, and the live one still moves. + world.engine.run_scheduler_tick()?; + let after = ControlSnapshot::read(&world.state_store)?; + assert!( + after + .transactions + .iter() + .any(|stored| stored.value.transaction_id == finished.transaction_id) + ); + assert_ne!( + after + .transactions + .iter() + .find(|stored| stored.value.transaction_id == live.transaction_id) + .context("live transaction")? + .envelope, + before, + "the live transaction was not resumed" + ); + Ok(()) + } + // ---- fix 2: supervision owns the Expected-only precondition ---- #[test] From 81ec1c7f40a4b496ac8fdffadb3c43d2dd2e9d9d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 07:21:21 +0200 Subject: [PATCH 018/286] Give the wedged-abort test the operator binding a freeze needs Co-Authored-By: Claude Sonnet 5 --- src/control_plane.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index 48ecf07..6904389 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -11827,6 +11827,7 @@ mod tests { // Every tick answers: the wedged abort records its error and waits, // and the rest of the scheduler goes on to freeze the queued command. + write_service_binding(&world)?; for _ in 0..6 { world.engine.run_scheduler_tick()?; } From 81478e6ddd6ab5cd8a3de52582c548e064e50b7a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 07:36:43 +0200 Subject: [PATCH 019/286] Pin the scheduler's progress report and the post-fence abort path Cargo-mutants on the previous commit missed the per-candidate progress flag and the post-fence branch selection in resume_candidate. A repeated error reports no progress; a dead candidate past the fence aborts; a wedged post-fence abort only waits. Co-Authored-By: Claude Sonnet 5 --- src/control_plane.rs | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 6904389..2efb9a2 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -11747,7 +11747,9 @@ mod tests { /// A workload whose candidate can neither be stopped nor recover: an abort /// that cannot finish its first step. - struct WedgedWorkload; + struct WedgedWorkload { + observe_fails: bool, + } impl WorkloadPort for WedgedWorkload { fn install( @@ -11789,6 +11791,7 @@ mod tests { activation: &IdunnRuntimeActivationRecord, prior: &WorkloadObservation, ) -> Result { + ensure!(!self.observe_fails, "the unit cannot be observed"); StillWorkload.observe(expected, activation, prior) } fn stop(&self, _: &WorkloadObservation) -> Result<()> { @@ -11801,7 +11804,9 @@ mod tests { #[test] fn one_wedged_pre_fence_abort_does_not_stop_the_scheduler() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(WedgedWorkload))?; + let world = EngineFixture::with_workload(Arc::new(WedgedWorkload { + observe_fails: false, + }))?; let wedged = transaction_at(&world, DeploymentPhase::Warming)?; world .engine @@ -11825,6 +11830,10 @@ mod tests { )? ); + // The first pass records the error and reports progress; the same + // error again changes nothing. + assert!(world.engine.resume_one_transaction()?); + assert!(!world.engine.resume_one_transaction()?); // Every tick answers: the wedged abort records its error and waits, // and the rest of the scheduler goes on to freeze the queued command. write_service_binding(&world)?; @@ -11844,6 +11853,29 @@ mod tests { Ok(()) } + #[test] + fn a_dead_candidate_past_the_fence_aborts_and_a_wedged_post_fence_abort_only_waits() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(WedgedWorkload { + observe_fails: true, + }))?; + let dead = transaction_at(&world, DeploymentPhase::AwaitingReady)?; + + // The candidate cannot be observed and can never run again, so the + // transaction abandons it after the fence. + world.engine.run_scheduler_tick()?; + assert!(record_of(&world, &dead.transaction_id)?.post_fencing_abort.is_some()); + + // Its abort cannot stop the unit: that is a resumable error, and the + // tick answers every time. + for _ in 0..3 { + world.engine.run_scheduler_tick()?; + } + let stuck = record_of(&world, &dead.transaction_id)?; + assert!(!stuck.is_terminal()); + assert!(stuck.last_error.is_some()); + Ok(()) + } + #[test] fn a_terminal_transaction_that_cannot_be_archived_does_not_stop_the_scheduler() -> Result<()> { let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; From 8998cfcf02894227c5a7f871f59457f5b8462806 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 07:56:38 +0200 Subject: [PATCH 020/286] Seed the wedged post-fence abort by hand A transaction seeded at AwaitingReady fails validation (no fencing evidence), and nothing at Fencing makes a workload error, so the post-fence abort is begun directly and the test pins that its failing step only waits. Co-Authored-By: Claude Sonnet 5 --- src/control_plane.rs | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 2efb9a2..9f3c452 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -11747,9 +11747,7 @@ mod tests { /// A workload whose candidate can neither be stopped nor recover: an abort /// that cannot finish its first step. - struct WedgedWorkload { - observe_fails: bool, - } + struct WedgedWorkload; impl WorkloadPort for WedgedWorkload { fn install( @@ -11791,7 +11789,6 @@ mod tests { activation: &IdunnRuntimeActivationRecord, prior: &WorkloadObservation, ) -> Result { - ensure!(!self.observe_fails, "the unit cannot be observed"); StillWorkload.observe(expected, activation, prior) } fn stop(&self, _: &WorkloadObservation) -> Result<()> { @@ -11804,9 +11801,7 @@ mod tests { #[test] fn one_wedged_pre_fence_abort_does_not_stop_the_scheduler() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(WedgedWorkload { - observe_fails: false, - }))?; + let world = EngineFixture::with_workload(Arc::new(WedgedWorkload))?; let wedged = transaction_at(&world, DeploymentPhase::Warming)?; world .engine @@ -11854,19 +11849,15 @@ mod tests { } #[test] - fn a_dead_candidate_past_the_fence_aborts_and_a_wedged_post_fence_abort_only_waits() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(WedgedWorkload { - observe_fails: true, - }))?; - let dead = transaction_at(&world, DeploymentPhase::AwaitingReady)?; + fn a_wedged_post_fence_abort_only_waits() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(WedgedWorkload))?; + let dead = transaction_at(&world, DeploymentPhase::Fencing)?; + world + .engine + .begin_post_fencing_abort(&resident(&world)?, anyhow!("candidate died"))?; - // The candidate cannot be observed and can never run again, so the - // transaction abandons it after the fence. - world.engine.run_scheduler_tick()?; - assert!(record_of(&world, &dead.transaction_id)?.post_fencing_abort.is_some()); - - // Its abort cannot stop the unit: that is a resumable error, and the - // tick answers every time. + // The abort cannot stop the unit. The abort is already durable, so + // that is a resumable error, and the tick answers every time. for _ in 0..3 { world.engine.run_scheduler_tick()?; } From 287aa92382ee6f7f290cee338720c7d7bc402aaf Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 08:41:25 +0200 Subject: [PATCH 021/286] B1 fix batch 3: progress means state advanced; one wedge starves no one - A tick's progress is a state change, never a last_error rewrite; freezing a queued command no longer waits on other targets' progress (still at most one freeze per tick, busy targets refused as before). - Failing transaction steps back off (process-local, doubling from the poll interval to 60s) instead of retrying every tick. - Boot reconciliation runs the resident-issuer pass with history unreadable. - Resume and retire faults are reported once per transaction and left in last_error, which status renders. - The legacy marker is a private-constructor type inside cleanup_evidence; transaction_envelope refuses a record carrying it, the schema migration writes through its own path. - Tests: changing-error wedge, backoff, dead-candidate post-fence branch, fault note, marker refusal, torn-history boot residue. Co-Authored-By: Claude Sonnet 5 --- src/control_plane.rs | 651 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 570 insertions(+), 81 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 9f3c452..fa51aa8 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -745,27 +745,133 @@ enum TransactionCompletion { }, } -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -enum CleanupEvidence { - Pending, - Skipped, - Complete, - /// Set only by the legacy lift, on a terminal continuity abort written - /// before the single resolution rule: it issued an activation and recorded - /// `Skipped`. Boot reconciliation, not the record, owes that residue's - /// demotion. The marker is what lets validation accept the old shape - /// without reopening a finished transaction. - LegacySkippedBeforeB1, +use cleanup_evidence::CleanupEvidence; + +/// Where the legacy marker lives, so that only the lift can make one. +/// +/// `LegacyLift` has a private field: outside this module no expression can +/// construct it, so no other code can write `CleanupEvidence::Legacy(..)`. +/// Decoding a stored record is the one other way in, and it stays here too. +/// `transaction_envelope` refuses a record carrying the marker; the schema +/// migration, which persists what the lift produced, writes through +/// `migrated_transaction_envelope`, defined here beside the only constructor. +mod cleanup_evidence { + use super::*; + + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub(super) struct LegacyLift(()); + + const LEGACY_MARKER: &str = "legacy-skipped-before-b1"; + + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub(super) enum CleanupEvidence { + Pending, + Skipped, + Complete, + /// Set only by the legacy lift, on a terminal continuity abort written + /// before the single resolution rule: it issued an activation and + /// recorded `Skipped`. Boot reconciliation, not the record, owes that + /// residue's demotion. The marker is what lets validation accept the + /// old shape without reopening a finished transaction. + Legacy(LegacyLift), + } + + impl CleanupEvidence { + pub(super) fn is_complete(self) -> bool { + !matches!(self, Self::Pending) + } + + pub(super) fn is_legacy_marker(self) -> bool { + matches!(self, Self::Legacy(_)) + } + } + + impl Serialize for CleanupEvidence { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(match self { + Self::Pending => "pending", + Self::Skipped => "skipped", + Self::Complete => "complete", + Self::Legacy(_) => LEGACY_MARKER, + }) + } + } + + impl<'de> Deserialize<'de> for CleanupEvidence { + fn deserialize>(deserializer: D) -> Result { + let text = String::deserialize(deserializer)?; + match text.as_str() { + "pending" => Ok(Self::Pending), + "skipped" => Ok(Self::Skipped), + "complete" => Ok(Self::Complete), + LEGACY_MARKER => Ok(Self::Legacy(LegacyLift(()))), + other => Err(serde::de::Error::unknown_variant( + other, + &["pending", "skipped", "complete", LEGACY_MARKER], + )), + } + } + } + + impl DeploymentTransaction { + pub(super) fn carries_legacy_marker(&self) -> bool { + self.pre_fencing_abort + .as_ref() + .map(|abort| abort.topology_reconciliation) + .into_iter() + .chain( + self.post_fencing_abort + .as_ref() + .map(|abort| abort.topology_reconciliation), + ) + .any(CleanupEvidence::is_legacy_marker) + } + } + + /// The envelope for a record the schema migration persists, the one write + /// path that may carry the marker. + pub(super) fn migrated_transaction_envelope( + value: &DeploymentTransaction, + now: u64, + ) -> Result { + encode_transaction(value, now) + } + +/// A continuity abort written before the single resolution rule recorded no +/// projection cleanup although its transaction had issued an activation, and +/// so left that activation standing. +/// +/// Nothing is reopened. A record already terminal keeps its terminal phase and +/// completion and is marked the legacy marker, the one shape validation +/// accepts for a terminal abort that owes nothing yet issued an activation; +/// boot reconciliation, the single owner of that residue, demotes it by exact +/// activation. A record still in flight lifts to `Pending`: it already holds +/// its target's authority (a live record claims it before and after the lift), +/// so the lift adds no claimant, and its own abort demotes the activation. +/// Only the control store's read applies this: history describes what happened +/// and is lifted unchanged. +pub(super) fn owe_legacy_continuity_projection(transaction: &mut DeploymentTransaction) { + if transaction.command_kind != CommandKind::Continuity || transaction.activation.is_none() { + return; + } + let terminal = transaction.completion.is_some(); + if let Some(abort) = transaction.pre_fencing_abort.as_mut() + && abort.topology_reconciliation == CleanupEvidence::Skipped + { + abort.topology_reconciliation = if terminal { + CleanupEvidence::Legacy(LegacyLift(())) + } else { + CleanupEvidence::Pending + }; + } + if let Some(abort) = transaction.post_fencing_abort.as_mut() + && abort.topology_reconciliation == CleanupEvidence::Skipped + && !terminal + { + abort.topology_reconciliation = CleanupEvidence::Pending; + } } -impl CleanupEvidence { - fn is_complete(self) -> bool { - matches!( - self, - Self::Skipped | Self::Complete | Self::LegacySkippedBeforeB1 - ) - } } /// Rollback of a transaction that has already fenced the incumbent. @@ -1316,7 +1422,7 @@ impl DeploymentTransaction { ) || (self.command_kind == CommandKind::Continuity && self.phase == DeploymentPhase::Complete && self.completion.is_some() - && abort.topology_reconciliation == CleanupEvidence::LegacySkippedBeforeB1), + && abort.topology_reconciliation.is_legacy_marker()), "abort topology cleanup differs from what the transaction projected" ); ensure!( @@ -2710,48 +2816,13 @@ fn read_transaction_record(envelope: &CultCacheEnvelope) -> Result { let mut value = lift_legacy_transaction(envelope)?; - owe_legacy_continuity_projection(&mut value); + cleanup_evidence::owe_legacy_continuity_projection(&mut value); value.validate()?; Ok(value) } } } -/// A continuity abort written before the single resolution rule recorded no -/// projection cleanup although its transaction had issued an activation, and -/// so left that activation standing. -/// -/// Nothing is reopened. A record already terminal keeps its terminal phase and -/// completion and is marked `LegacySkippedBeforeB1`, the one shape validation -/// accepts for a terminal abort that owes nothing yet issued an activation; -/// boot reconciliation, the single owner of that residue, demotes it by exact -/// activation. A record still in flight lifts to `Pending`: it already holds -/// its target's authority (a live record claims it before and after the lift), -/// so the lift adds no claimant, and its own abort demotes the activation. -/// Only the control store's read applies this: history describes what happened -/// and is lifted unchanged. -fn owe_legacy_continuity_projection(transaction: &mut DeploymentTransaction) { - if transaction.command_kind != CommandKind::Continuity || transaction.activation.is_none() { - return; - } - let terminal = transaction.completion.is_some(); - if let Some(abort) = transaction.pre_fencing_abort.as_mut() - && abort.topology_reconciliation == CleanupEvidence::Skipped - { - abort.topology_reconciliation = if terminal { - CleanupEvidence::LegacySkippedBeforeB1 - } else { - CleanupEvidence::Pending - }; - } - if let Some(abort) = transaction.post_fencing_abort.as_mut() - && abort.topology_reconciliation == CleanupEvidence::Skipped - && !terminal - { - abort.topology_reconciliation = CleanupEvidence::Pending; - } -} - /// Read one stored admitted generation, lifting a v2 record to the current /// shape by the same rule as `read_transaction_record`. fn read_generation_record(envelope: &CultCacheEnvelope) -> Result { @@ -2807,7 +2878,10 @@ fn migrate_control_store_to_current_schema(store_path: &Path) -> Result { for envelope in stale { let (record_type, key, mut next) = if envelope.r#type == DeploymentTransaction::TYPE { let value = read_transaction_record(&envelope)?; - let next = transaction_envelope(&value, value.updated_at_unix_millis)?; + let next = cleanup_evidence::migrated_transaction_envelope( + &value, + value.updated_at_unix_millis, + )?; (DeploymentTransaction::TYPE, value.transaction_id, next) } else { let value = read_generation_record(&envelope)?; @@ -2832,6 +2906,14 @@ fn migrate_control_store_to_current_schema(store_path: &Path) -> Result { } fn transaction_envelope(value: &DeploymentTransaction, now: u64) -> Result { + ensure!( + !value.carries_legacy_marker(), + "only the legacy lift may produce a legacy-marked record; it cannot be written as a new one" + ); + encode_transaction(value, now) +} + +fn encode_transaction(value: &DeploymentTransaction, now: u64) -> Result { value.validate()?; typed_envelope( &value.transaction_id, @@ -2987,6 +3069,15 @@ fn read_history_transactions(state_store: &Path) -> Vec { } } +/// Whether a transaction moved between two reads. An error note (`last_error`) +/// and its timestamp are commentary on a stuck step, not a step. +fn state_advanced(before: &DeploymentTransaction, after: &DeploymentTransaction) -> bool { + let mut after = after.clone(); + after.last_error.clone_from(&before.last_error); + after.updated_at_unix_millis = before.updated_at_unix_millis; + &after != before +} + /// Says a fault once while it lasts. A scheduler tick that hits the same /// unreadable file every half second must not print it every half second. #[derive(Default)] @@ -3486,8 +3577,25 @@ struct Engine { systemd_workload: Arc, host_actuators: Option, history_report: ReportOnce, + /// One `ReportOnce` per faulting record, so one wedged transaction says + /// its fault once and never drowns another's. + fault_reports: Mutex>, + /// Retry pacing for a transaction whose step keeps failing, by + /// transaction id: consecutive failures and the earliest next attempt. + /// Process-local on purpose: a restart retries at once, and no schema + /// carries a clock that only pacing reads. + resume_backoff: Mutex>, } +#[derive(Clone, Copy)] +struct ResumeBackoff { + failures: u32, + not_before_unix_millis: u64, +} + +/// Longest wait between attempts at a failing transaction step. +const RESUME_BACKOFF_CEILING_MILLIS: u64 = 60_000; + impl Engine { fn open(options: RuntimeOptions) -> Result { Self::open_with_systemd_workload( @@ -3541,6 +3649,8 @@ impl Engine { systemd_workload, host_actuators, history_report: ReportOnce::default(), + fault_reports: Mutex::default(), + resume_backoff: Mutex::default(), }) } @@ -3834,12 +3944,18 @@ impl Engine { fn reconcile_failed_continuity_projections(&self) -> Result { let snapshot = ControlSnapshot::read(&self.options.state_store)?; let mut outcome = ProjectionReconciliation::default(); - let Some(history) = self.history_for_decision() else { - return Ok(outcome); - }; + // Resident issuers need no history; only the history issuers wait on it. + let history = self.history_for_decision(); + if history.is_none() { + eprintln!( + "Idunn cannot read history at boot: failed continuities archived there are not \ + reconciled, resident ones still are" + ); + } let topology = self.topology(); let issuers = history .iter() + .flatten() .chain(snapshot.transactions.iter().map(|stored| &stored.value)) .filter(|transaction| { transaction.command_kind == CommandKind::Continuity @@ -3990,9 +4106,10 @@ impl Engine { ); return Ok(true); } - Err(error) => eprintln!( - "Idunn could not retire terminal transaction {}: {error:#}", - stored.value.transaction_id + Err(error) => self.note_fault( + "could not retire terminal transaction", + &stored.value.transaction_id, + &error, ), } } @@ -4008,7 +4125,68 @@ impl Engine { } let transaction_progress = self.resume_one_transaction()?; let continuity_progress = self.supervise_one_admitted_generation()?; - Ok(!transaction_progress && !continuity_progress && self.freeze_one_queued_command()?) + // Freezing is not gated on the others' progress: a queued command + // whose target is free must not wait behind an unrelated target's + // wedge. `freeze_command` already refuses a busy target, so this is + // still at most one freeze per tick and never a second claimant. + let froze = self.freeze_one_queued_command()?; + Ok(transaction_progress || continuity_progress || froze) + } + + /// Say a per-record fault once while it lasts and leave it in the + /// record's `last_error`, which status renders. Recording is best effort: + /// a store too broken to take the note is already in stderr. + fn note_fault(&self, what: &str, transaction_id: &str, error: &anyhow::Error) { + let detail = truncate(&format!("{error:#}"), 2048); + let offered = { + let mut reports = self.fault_reports.lock().expect("fault report mutex"); + reports + .entry(transaction_id.to_string()) + .or_default() + .offer(Some(detail.clone())) + }; + if offered.is_some() { + eprintln!("Idunn {what} {transaction_id}: {detail}"); + } + let _ = self.record_last_error(transaction_id, &detail); + } + + fn clear_fault(&self, transaction_id: &str) { + self.fault_reports + .lock() + .expect("fault report mutex") + .remove(transaction_id); + } + + /// Write `detail` as the record's `last_error`, touching nothing else and + /// not archiving: a terminal record whose archive is the fault stays put. + fn record_last_error(&self, transaction_id: &str, detail: &str) -> Result<()> { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let Some(stored) = snapshot + .transactions + .iter() + .find(|stored| stored.value.transaction_id == transaction_id) + else { + return Ok(()); + }; + if stored.value.last_error.as_deref() == Some(detail) { + return Ok(()); + } + let mut next = stored.value.clone(); + next.last_error = Some(detail.to_string()); + let envelope = transaction_envelope(&next, next.updated_at_unix_millis)?; + ensure!( + SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: transaction_id.to_string(), + current: Some(stored.envelope.clone()), + }], + &[envelope], + )?, + "transaction changed before its fault note" + ); + Ok(()) } /// Startup and every later loop use the same order: unfinished ownership @@ -4035,11 +4213,17 @@ impl Engine { continue; } match self.resume_candidate(current) { - Ok(moved) => progressed |= moved, - Err(error) => eprintln!( - "Idunn could not resume transaction {}: {error:#}", - current.value.transaction_id - ), + Ok(moved) => { + self.clear_fault(¤t.value.transaction_id); + progressed |= moved; + } + Err(error) => { + self.note_fault( + "could not resume transaction", + ¤t.value.transaction_id, + &error, + ); + } } } Ok(progressed) @@ -4047,8 +4231,32 @@ impl Engine { /// Advance one transaction, turning its failure into the durable record /// its phase calls for. Returns whether its record changed. + /// + /// "Moved" means the transaction's state advanced: a phase change, a + /// completion, or a durable abort step. Rewriting `last_error` to say the + /// same step failed again in new words is not movement, or a wedged + /// transaction would report progress every tick. fn resume_candidate(&self, current: &Stored) -> Result { - if let Err(error) = self.advance_transaction(current) { + let id = current.value.transaction_id.as_str(); + if self + .resume_backoff + .lock() + .expect("resume backoff mutex") + .get(id) + .is_some_and(|backoff| { + now_millis().is_ok_and(|now| now < backoff.not_before_unix_millis) + }) + { + return Ok(false); + } + let advanced = self.advance_transaction(current); + if advanced.is_ok() { + self.resume_backoff + .lock() + .expect("resume backoff mutex") + .remove(id); + } + if let Err(error) = advanced { let latest_snapshot = ControlSnapshot::read(&self.options.state_store)?; let latest = latest_snapshot .transactions @@ -4056,8 +4264,13 @@ impl Engine { .find(|stored| stored.value.transaction_id == current.value.transaction_id) .context("transaction disappeared while recording an execution error")?; if latest.value.is_terminal() { + self.resume_backoff + .lock() + .expect("resume backoff mutex") + .remove(id); return Ok(true); } + self.back_off(id)?; if latest.value.phase < DeploymentPhase::Fencing { // Before the fence an abort that cannot finish a step is // resumable, never a post-fence abort: that path refuses a @@ -4086,7 +4299,28 @@ impl Engine { .iter() .find(|stored| stored.value.transaction_id == current.value.transaction_id) .context("transaction disappeared while checking scheduler progress")?; - Ok(live.envelope != current.envelope) + Ok(state_advanced(¤t.value, &live.value)) + } + + /// Wait twice as long after each consecutive failure of a transaction's + /// step, from one poll interval up to a ceiling. + fn back_off(&self, transaction_id: &str) -> Result<()> { + let now = now_millis()?; + let mut all = self.resume_backoff.lock().expect("resume backoff mutex"); + let failures = all.get(transaction_id).map_or(0, |backoff| backoff.failures); + let wait = self + .options + .poll_millis + .saturating_mul(1u64 << failures.min(20)) + .min(RESUME_BACKOFF_CEILING_MILLIS); + all.insert( + transaction_id.to_string(), + ResumeBackoff { + failures: failures.saturating_add(1), + not_before_unix_millis: now.saturating_add(wait), + }, + ); + Ok(()) } fn freeze_one_queued_command(&self) -> Result { @@ -11576,9 +11810,8 @@ mod tests { let lifted = read_transaction_record(&envelope)?; let terminal = archived.completion.is_some(); if terminal { - assert_eq!( - abort_owed(&lifted), - Some(CleanupEvidence::LegacySkippedBeforeB1), + assert!( + abort_owed(&lifted).is_some_and(CleanupEvidence::is_legacy_marker), "{name}" ); assert!(lifted.is_terminal(), "{name}: the lift reopened a finished record"); @@ -11647,7 +11880,11 @@ mod tests { let mut second = first.clone(); second.transaction_id = "tx-second-legacy".into(); second.command_id = "continuity-second-legacy".into(); - make_resident(&world, &second, transaction_envelope(&second, 200)?)?; + make_resident( + &world, + &second, + cleanup_evidence::migrated_transaction_envelope(&second, 200)?, + )?; // The store reads: three records, one of them live, none of the // finished ones claiming the target. @@ -11825,9 +12062,8 @@ mod tests { )? ); - // The first pass records the error and reports progress; the same - // error again changes nothing. - assert!(world.engine.resume_one_transaction()?); + // Recording the error is not progress, and neither is repeating it. + assert!(!world.engine.resume_one_transaction()?); assert!(!world.engine.resume_one_transaction()?); // Every tick answers: the wedged abort records its error and waits, // and the rest of the scheduler goes on to freeze the queued command. @@ -11867,6 +12103,259 @@ mod tests { Ok(()) } + /// A wedged abort whose `stop()` error carries a changing counter, as + /// systemctl output with a PID or timestamp would. + struct ChangingErrorWorkload(std::sync::atomic::AtomicU32); + + impl WorkloadPort for ChangingErrorWorkload { + fn install( + &self, + plan: &CompiledDeploymentPlan, + release: &crate::drivers::MaterializedRelease, + ) -> Result { + WedgedWorkload.install(plan, release) + } + fn prepare_activation( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + launch: IdunnRuntimeActivationLaunch, + ) -> Result { + WedgedWorkload.prepare_activation(plan, expected, launch) + } + fn start_prepared( + &self, + plan: &CompiledDeploymentPlan, + release: &SealedRelease, + installed: &crate::drivers::InstalledReleaseObservation, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result { + WedgedWorkload.start_prepared(plan, release, installed, expected, activation) + } + fn discard_prepared( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result<()> { + WedgedWorkload.discard_prepared(plan, expected, activation) + } + fn observe( + &self, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + prior: &WorkloadObservation, + ) -> Result { + WedgedWorkload.observe(expected, activation, prior) + } + fn stop(&self, _: &WorkloadObservation) -> Result<()> { + let n = self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + bail!("the unit will not stop (main pid {n})") + } + fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { + Ok(true) + } + } + + fn queue_deploy_for_nowhere(world: &EngineFixture) -> Result { + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "up-nowhere".into(), + kind: CommandKind::Deploy, + selector: "nowhere".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }], + &[command_envelope(&command, 100)?], + )? + ); + Ok(command) + } + + #[test] + fn a_wedge_whose_error_text_changes_every_tick_does_not_starve_other_targets() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(ChangingErrorWorkload( + std::sync::atomic::AtomicU32::new(0), + )))?; + transaction_at(&world, DeploymentPhase::Warming)?; + world + .engine + .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + let command = queue_deploy_for_nowhere(&world)?; + write_service_binding(&world)?; + for _ in 0..20 { + world.engine.run_scheduler_tick()?; + } + assert!( + read_history_transactions(&world.state_store) + .iter() + .any(|transaction| transaction.command_id == command.command_id), + "the queued command was starved by another target's wedge" + ); + Ok(()) + } + + #[test] + fn rewriting_last_error_is_not_progress() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let before = transaction_at(&world, DeploymentPhase::Warming)?; + let mut noted = before.clone(); + noted.last_error = Some("the unit will not stop (main pid 7)".into()); + noted.updated_at_unix_millis += 1; + assert!(!state_advanced(&before, ¬ed)); + let mut moved = noted.clone(); + moved.phase = DeploymentPhase::AwaitingReady; + assert!(state_advanced(&before, &moved)); + Ok(()) + } + + #[test] + fn a_failing_step_is_retried_after_a_backoff_not_every_tick() -> Result<()> { + let workload = Arc::new(ChangingErrorWorkload(std::sync::atomic::AtomicU32::new(0))); + let world = EngineFixture::with_workload(workload.clone())?; + transaction_at(&world, DeploymentPhase::Warming)?; + world + .engine + .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + for _ in 0..10 { + world.engine.run_scheduler_tick()?; + } + assert_eq!( + workload.0.load(std::sync::atomic::Ordering::SeqCst), + 1, + "the wedged step was retried inside its backoff" + ); + Ok(()) + } + + /// A candidate that fails to advance and is dead for good. + struct DeadCandidateWorkload; + + impl WorkloadPort for DeadCandidateWorkload { + fn install( + &self, + plan: &CompiledDeploymentPlan, + release: &crate::drivers::MaterializedRelease, + ) -> Result { + StillWorkload.install(plan, release) + } + fn prepare_activation( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + launch: IdunnRuntimeActivationLaunch, + ) -> Result { + StillWorkload.prepare_activation(plan, expected, launch) + } + fn start_prepared( + &self, + plan: &CompiledDeploymentPlan, + release: &SealedRelease, + installed: &crate::drivers::InstalledReleaseObservation, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result { + StillWorkload.start_prepared(plan, release, installed, expected, activation) + } + fn discard_prepared( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result<()> { + StillWorkload.discard_prepared(plan, expected, activation) + } + fn observe( + &self, + _: &IdunnExpectedIncarnationRecord, + _: &IdunnRuntimeActivationRecord, + _: &WorkloadObservation, + ) -> Result { + bail!("the candidate is gone") + } + fn stop(&self, _: &WorkloadObservation) -> Result<()> { + Ok(()) + } + fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { + Ok(true) + } + } + + #[test] + fn a_permanently_dead_candidate_past_the_fence_begins_its_post_fence_abort() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(DeadCandidateWorkload))?; + let dead = transaction_at(&world, DeploymentPhase::Fencing)?; + assert!(record_of(&world, &dead.transaction_id)?.post_fencing_abort.is_none()); + for _ in 0..8 { + world.engine.resume_one_transaction()?; + } + assert!( + record_of(&world, &dead.transaction_id)? + .post_fencing_abort + .is_some(), + "a dead candidate past the fence was retried instead of aborted" + ); + Ok(()) + } + + #[test] + fn a_resume_fault_is_reported_once_and_left_in_the_record() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let live = transaction_at(&world, DeploymentPhase::Warming)?; + let error = anyhow!("store hiccup 1"); + world + .engine + .note_fault("could not resume transaction", &live.transaction_id, &error); + assert_eq!( + record_of(&world, &live.transaction_id)?.last_error.as_deref(), + Some("store hiccup 1") + ); + let reports = world.engine.fault_reports.lock().unwrap(); + assert_eq!(reports.len(), 1); + assert!(reports[&live.transaction_id].offer(Some("store hiccup 1".into())).is_none()); + Ok(()) + } + + #[test] + fn a_new_record_cannot_carry_the_legacy_marker() -> Result<()> { + write_fixture_anchor()?; + let envelope = fixture_envelope(FIXTURE_PRE_B1_ABORTS[1].1, DeploymentTransaction::TYPE)?; + let lifted = read_transaction_record(&envelope)?; + assert!(lifted.carries_legacy_marker()); + let error = error_text(transaction_envelope(&lifted, 1).map(|_| ())); + assert!(error.contains("legacy lift"), "{error}"); + // The migration's own path still writes it, and it reads back. + let migrated = cleanup_evidence::migrated_transaction_envelope(&lifted, 1)?; + assert!(read_transaction_record(&migrated)?.carries_legacy_marker()); + Ok(()) + } + + #[test] + fn unreadable_history_does_not_skip_resident_residue_at_boot() -> Result<()> { + write_fixture_anchor()?; + let envelope = fixture_envelope(FIXTURE_PRE_B1_ABORTS[1].1, DeploymentTransaction::TYPE)?; + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let first = read_transaction_record(&envelope)?; + make_resident(&world, &first, envelope)?; + assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); + let expected = first.expected.clone().context("no Expected")?; + project_candidate(&world, &first, true)?; + assert_eq!(projected_under(&world, &expected)?.len(), 2); + corrupt_history(&world)?; + let outcome = world.engine.reconcile_failed_continuity_projections()?; + assert_eq!(outcome.demoted.len(), 1); + assert_eq!(projected_under(&world, &expected)?, expected_only()); + Ok(()) + } + #[test] fn a_terminal_transaction_that_cannot_be_archived_does_not_stop_the_scheduler() -> Result<()> { let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; @@ -12274,19 +12763,19 @@ mod tests { assert_eq!(owed(&written), Some(CleanupEvidence::Skipped)); let mut lifted = written.clone(); - owe_legacy_continuity_projection(&mut lifted); + cleanup_evidence::owe_legacy_continuity_projection(&mut lifted); assert_eq!(owed(&lifted), Some(CleanupEvidence::Pending)); // A deployment published its own key and recorded what it owed. let mut deploy = written.clone(); deploy.command_kind = CommandKind::Deploy; - owe_legacy_continuity_projection(&mut deploy); + cleanup_evidence::owe_legacy_continuity_projection(&mut deploy); assert_eq!(owed(&deploy), Some(CleanupEvidence::Skipped)); // A continuity that issued nothing owes nothing. let mut unissued = written; unissued.activation = None; - owe_legacy_continuity_projection(&mut unissued); + cleanup_evidence::owe_legacy_continuity_projection(&mut unissued); assert_eq!(owed(&unissued), Some(CleanupEvidence::Skipped)); Ok(()) } From 5743a25389a0b4de14064c2e47b7bb4cc363236b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 10:28:34 +0200 Subject: [PATCH 022/286] B1 fix batch 4: faults re-arm only on recovery; notes reach marked records - Fault reports clear only when a step ran and succeeded, not on a backoff skip or a handled failure. - Fault notes are written through cleanup_evidence::last_error_envelope, so an unretirable pre-B1 terminal record shows its fault; a failed note is printed with the fault instead of swallowed. - backoff_wait and is_waiting are pure and unit-tested; tick and resume return values are pinned. - migrate_transaction_record is the migration's only entry; the encoder that can write a marker is private to cleanup_evidence. Co-Authored-By: Claude Sonnet 5 --- src/control_plane.rs | 178 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 148 insertions(+), 30 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index fa51aa8..ac38edf 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -754,7 +754,9 @@ use cleanup_evidence::CleanupEvidence; /// Decoding a stored record is the one other way in, and it stays here too. /// `transaction_envelope` refuses a record carrying the marker; the schema /// migration, which persists what the lift produced, writes through -/// `migrated_transaction_envelope`, defined here beside the only constructor. +/// `migrate_transaction_record`, and a fault note on an existing record through +/// `last_error_envelope`; both are defined here beside the only constructor, +/// and the encoder they share is private to this module. mod cleanup_evidence { use super::*; @@ -828,13 +830,26 @@ mod cleanup_evidence { } } - /// The envelope for a record the schema migration persists, the one write - /// path that may carry the marker. - pub(super) fn migrated_transaction_envelope( + /// The schema migration's one entry point: read a stored v2, v3 or v4 + /// transaction (lifting older ones) and return its key and current-shape + /// envelope. The record may carry the marker the lift just set. + pub(super) fn migrate_transaction_record( + envelope: &CultCacheEnvelope, + ) -> Result<(String, CultCacheEnvelope)> { + let value = read_transaction_record(envelope)?; + let next = encode_transaction(&value, value.updated_at_unix_millis)?; + Ok((value.transaction_id, next)) + } + + /// The envelope for an existing record with only its `last_error` set. The + /// record may already carry the marker; this cannot add one. + pub(super) fn last_error_envelope( value: &DeploymentTransaction, - now: u64, + detail: &str, ) -> Result { - encode_transaction(value, now) + let mut next = value.clone(); + next.last_error = Some(detail.to_string()); + encode_transaction(&next, next.updated_at_unix_millis) } /// A continuity abort written before the single resolution rule recorded no @@ -2877,12 +2892,8 @@ fn migrate_control_store_to_current_schema(store_path: &Path) -> Result { let mut migrated = 0; for envelope in stale { let (record_type, key, mut next) = if envelope.r#type == DeploymentTransaction::TYPE { - let value = read_transaction_record(&envelope)?; - let next = cleanup_evidence::migrated_transaction_envelope( - &value, - value.updated_at_unix_millis, - )?; - (DeploymentTransaction::TYPE, value.transaction_id, next) + let (key, next) = cleanup_evidence::migrate_transaction_record(&envelope)?; + (DeploymentTransaction::TYPE, key, next) } else { let value = read_generation_record(&envelope)?; let next = admitted_envelope(&value, value.admitted_at_unix_millis)?; @@ -3069,6 +3080,19 @@ fn read_history_transactions(state_store: &Path) -> Vec { } } +/// How long to wait after `failures` consecutive failures: one poll interval +/// after the first, doubling each time, never past the ceiling. +fn backoff_wait(poll_millis: u64, failures: u32) -> u64 { + poll_millis + .saturating_mul(1u64 << failures.min(20)) + .min(RESUME_BACKOFF_CEILING_MILLIS) +} + +/// Whether an attempt due at `not_before` must still wait at `now`. +fn is_waiting(now: u64, not_before: u64) -> bool { + now < not_before +} + /// Whether a transaction moved between two reads. An error note (`last_error`) /// and its timestamp are commentary on a stuck step, not a step. fn state_advanced(before: &DeploymentTransaction, after: &DeploymentTransaction) -> bool { @@ -4145,10 +4169,13 @@ impl Engine { .or_default() .offer(Some(detail.clone())) }; + let recorded = self.record_last_error(transaction_id, &detail); if offered.is_some() { eprintln!("Idunn {what} {transaction_id}: {detail}"); + if let Err(error) = recorded { + eprintln!("Idunn could not leave that in the record of {transaction_id}: {error:#}"); + } } - let _ = self.record_last_error(transaction_id, &detail); } fn clear_fault(&self, transaction_id: &str) { @@ -4172,9 +4199,7 @@ impl Engine { if stored.value.last_error.as_deref() == Some(detail) { return Ok(()); } - let mut next = stored.value.clone(); - next.last_error = Some(detail.to_string()); - let envelope = transaction_envelope(&next, next.updated_at_unix_millis)?; + let envelope = cleanup_evidence::last_error_envelope(&stored.value, detail)?; ensure!( SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange( &[CultCacheExpectedEnvelope { @@ -4213,10 +4238,7 @@ impl Engine { continue; } match self.resume_candidate(current) { - Ok(moved) => { - self.clear_fault(¤t.value.transaction_id); - progressed |= moved; - } + Ok(moved) => progressed |= moved, Err(error) => { self.note_fault( "could not resume transaction", @@ -4244,17 +4266,20 @@ impl Engine { .expect("resume backoff mutex") .get(id) .is_some_and(|backoff| { - now_millis().is_ok_and(|now| now < backoff.not_before_unix_millis) + now_millis().is_ok_and(|now| is_waiting(now, backoff.not_before_unix_millis)) }) { return Ok(false); } let advanced = self.advance_transaction(current); if advanced.is_ok() { + // The step ran and succeeded: the fault, if any, has recovered, so + // a later failure is news. A skipped or failed attempt clears nothing. self.resume_backoff .lock() .expect("resume backoff mutex") .remove(id); + self.clear_fault(id); } if let Err(error) = advanced { let latest_snapshot = ControlSnapshot::read(&self.options.state_store)?; @@ -4308,11 +4333,7 @@ impl Engine { let now = now_millis()?; let mut all = self.resume_backoff.lock().expect("resume backoff mutex"); let failures = all.get(transaction_id).map_or(0, |backoff| backoff.failures); - let wait = self - .options - .poll_millis - .saturating_mul(1u64 << failures.min(20)) - .min(RESUME_BACKOFF_CEILING_MILLIS); + let wait = backoff_wait(self.options.poll_millis, failures); all.insert( transaction_id.to_string(), ResumeBackoff { @@ -11883,7 +11904,7 @@ mod tests { make_resident( &world, &second, - cleanup_evidence::migrated_transaction_envelope(&second, 200)?, + cleanup_evidence::last_error_envelope(&second, "second")?, )?; // The store reads: three records, one of them live, none of the @@ -12105,7 +12126,7 @@ mod tests { /// A wedged abort whose `stop()` error carries a changing counter, as /// systemctl output with a PID or timestamp would. - struct ChangingErrorWorkload(std::sync::atomic::AtomicU32); + struct ChangingErrorWorkload(std::sync::atomic::AtomicU32, std::sync::atomic::AtomicBool); impl WorkloadPort for ChangingErrorWorkload { fn install( @@ -12151,6 +12172,9 @@ mod tests { } fn stop(&self, _: &WorkloadObservation) -> Result<()> { let n = self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + if self.1.load(std::sync::atomic::Ordering::SeqCst) { + return Ok(()); + } bail!("the unit will not stop (main pid {n})") } fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { @@ -12184,6 +12208,7 @@ mod tests { fn a_wedge_whose_error_text_changes_every_tick_does_not_starve_other_targets() -> Result<()> { let world = EngineFixture::with_workload(Arc::new(ChangingErrorWorkload( std::sync::atomic::AtomicU32::new(0), + std::sync::atomic::AtomicBool::new(false), )))?; transaction_at(&world, DeploymentPhase::Warming)?; world @@ -12219,7 +12244,10 @@ mod tests { #[test] fn a_failing_step_is_retried_after_a_backoff_not_every_tick() -> Result<()> { - let workload = Arc::new(ChangingErrorWorkload(std::sync::atomic::AtomicU32::new(0))); + let workload = Arc::new(ChangingErrorWorkload( + std::sync::atomic::AtomicU32::new(0), + std::sync::atomic::AtomicBool::new(false), + )); let world = EngineFixture::with_workload(workload.clone())?; transaction_at(&world, DeploymentPhase::Warming)?; world @@ -12333,7 +12361,7 @@ mod tests { let error = error_text(transaction_envelope(&lifted, 1).map(|_| ())); assert!(error.contains("legacy lift"), "{error}"); // The migration's own path still writes it, and it reads back. - let migrated = cleanup_evidence::migrated_transaction_envelope(&lifted, 1)?; + let (_, migrated) = cleanup_evidence::migrate_transaction_record(&envelope)?; assert!(read_transaction_record(&migrated)?.carries_legacy_marker()); Ok(()) } @@ -12356,6 +12384,96 @@ mod tests { Ok(()) } + #[test] + fn backoff_doubles_caps_and_waits_exactly() { + assert_eq!(backoff_wait(500, 0), 500); + assert_eq!(backoff_wait(500, 1), 1000); + assert_eq!(backoff_wait(500, 2), 2000); + assert_eq!(backoff_wait(500, 3), 4000); + assert_eq!(backoff_wait(500, 30), RESUME_BACKOFF_CEILING_MILLIS); + assert_eq!(backoff_wait(u64::MAX, 5), RESUME_BACKOFF_CEILING_MILLIS); + assert!(is_waiting(9, 10)); + assert!(!is_waiting(10, 10)); + assert!(!is_waiting(11, 10)); + } + + #[test] + fn a_persistent_fault_is_reported_once_and_again_only_after_recovery() -> Result<()> { + let workload = Arc::new(ChangingErrorWorkload( + std::sync::atomic::AtomicU32::new(0), + std::sync::atomic::AtomicBool::new(false), + )); + let world = EngineFixture::with_workload(workload.clone())?; + let wedged = transaction_at(&world, DeploymentPhase::Warming)?; + world + .engine + .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + let id = wedged.transaction_id; + let expire = || { + world.engine.resume_backoff.lock().unwrap().clear(); + }; + let reported = || world.engine.fault_reports.lock().unwrap().contains_key(&id); + // A fault already reported for this transaction (an Err from the + // resume itself); a handled, still-failing step must not re-arm it. + world + .engine + .note_fault("could not resume transaction", &id, &anyhow!("store hiccup")); + assert!(reported()); + world.engine.resume_one_transaction()?; + assert!(reported(), "a failing handled step re-armed the report"); + // Skipped inside the backoff: the fault is still the same fault. + world.engine.resume_one_transaction()?; + assert!(reported(), "a backoff skip re-armed the report"); + // Still failing after the backoff expires: still the same fault. + expire(); + world.engine.resume_one_transaction()?; + assert!(reported()); + // A real recovery clears it. + workload.1.store(true, std::sync::atomic::Ordering::SeqCst); + expire(); + world.engine.resume_one_transaction()?; + assert!(!reported(), "a step that ran and succeeded left the fault armed"); + Ok(()) + } + + #[test] + fn an_unretirable_legacy_marked_record_shows_its_fault_in_the_record() -> Result<()> { + write_fixture_anchor()?; + let envelope = fixture_envelope(FIXTURE_PRE_B1_ABORTS[1].1, DeploymentTransaction::TYPE)?; + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let first = read_transaction_record(&envelope)?; + make_resident(&world, &first, envelope)?; + assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); + assert!(record_of(&world, &first.transaction_id)?.carries_legacy_marker()); + corrupt_history(&world)?; + assert!(!world.engine.retire_one_terminal_transaction()?); + let noted = record_of(&world, &first.transaction_id)?; + assert!( + noted.last_error.is_some() && noted.last_error != first.last_error, + "the retire fault never reached the record" + ); + assert!(noted.carries_legacy_marker()); + Ok(()) + } + + #[test] + fn a_tick_that_only_freezes_or_only_advances_reports_progress() -> Result<()> { + // Only a freeze. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + queue_deploy_for_nowhere(&world)?; + write_service_binding(&world)?; + assert!(world.engine.run_scheduler_tick()?, "a freeze-only tick reported no progress"); + + // Only a phase advance. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + transaction_at(&world, DeploymentPhase::Fencing)?; + assert!(world.engine.resume_one_transaction()?, "a real advance reported none"); + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + transaction_at(&world, DeploymentPhase::Fencing)?; + assert!(world.engine.run_scheduler_tick()?); + Ok(()) + } + #[test] fn a_terminal_transaction_that_cannot_be_archived_does_not_stop_the_scheduler() -> Result<()> { let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; From a7b602ae6568ab255ea296d71f97877a2e7ea12e Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 10:48:33 +0200 Subject: [PATCH 023/286] WIP B3: route-proof readiness in the phase machine Co-Authored-By: Claude Sonnet 5 --- src/control_plane.rs | 571 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 457 insertions(+), 114 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index ac38edf..1977c9b 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -262,20 +262,36 @@ impl RuntimePresenceEvidence { enum WarmingEvidence { OdinTopology { evidence: TopologyEvidence }, FirstOdinDirect { evidence: RuntimePresenceEvidence }, + /// A route-proof target's candidate answered Idunn's challenge itself. + RouteProofDirect { evidence: RuntimePresenceEvidence }, } impl WarmingEvidence { fn validate_shape(&self) -> Result<()> { match self { Self::OdinTopology { evidence } => evidence.validate_shape(), - Self::FirstOdinDirect { evidence } => evidence.validate_shape(), + Self::FirstOdinDirect { evidence } | Self::RouteProofDirect { evidence } => { + evidence.validate_shape() + } } } fn canonical_sha256(&self) -> &str { match self { Self::OdinTopology { evidence } => &evidence.canonical_sha256, - Self::FirstOdinDirect { evidence } => &evidence.canonical_sha256, + Self::FirstOdinDirect { evidence } | Self::RouteProofDirect { evidence } => { + &evidence.canonical_sha256 + } + } + } + + /// The presence Idunn challenged for itself, when no Odin carried it. + fn direct(&self) -> Option<&RuntimePresenceEvidence> { + match self { + Self::OdinTopology { .. } => None, + Self::FirstOdinDirect { evidence } | Self::RouteProofDirect { evidence } => { + Some(evidence) + } } } } @@ -315,21 +331,24 @@ impl SequenceAdmittedWarming { }) } - fn from_first_odin_presence( + fn from_direct_presence( transaction_id: String, - evidence: RuntimePresenceEvidence, + warming: WarmingEvidence, present: cultnet_rs::VerifiedRuntimePresence, ) -> Result { + let evidence = warming + .direct() + .context("direct Warming evidence is an Odin topology receipt")?; ensure!( present.canonical_bytes() == evidence.canonical_bytes && present.signed_presence_sha256() == evidence.canonical_sha256, - "direct first-Odin Warming evidence differs from its authenticated presence" + "direct Warming evidence differs from its authenticated presence" ); let runtime_instance_id = present.record().runtime_instance_id.clone(); Ok(Self { transaction_id, signed_presence_sha256: evidence.canonical_sha256.clone(), - evidence: WarmingEvidence::FirstOdinDirect { evidence }, + evidence: warming, runtime_instance_id, }) } @@ -526,24 +545,58 @@ impl RoutingEvidence { } } +/// The states a route-proof candidate may answer its Warming challenge in. A +/// stateful one is warming until it holds a lease it cannot hold yet, and the +/// lease binds that warming presence. A stateless one may have finished +/// warming before the first challenge landed. +fn route_proof_warming_states(expected: &IdunnExpectedIncarnationRecord) -> &'static [&'static str] { + if expected.write_lease_required { + &["warming"] + } else { + &["warming", "active"] + } +} + +/// What a direct challenge to a candidate endpoint produced. +enum CandidateAnswer { + /// No answer: the reason it is still being waited on. + Silent(String), + Answered { + evidence: RuntimePresenceEvidence, + present: cultnet_rs::VerifiedRuntimePresence, + }, +} + /// Which kind of proof admits a target as Ready. Derived from the target's /// Expected, never stored: Expected is digest-bound, so the class cannot drift /// from the incarnation it describes. A target that declares a /// `shared-infrastructure odin.verse-rendezvous` dependency advertises into the -/// Verse and is Odin-correlated; one that declares none is route-proof. +/// Verse and is Odin-correlated; a routed one that declares none is +/// route-proof. Odin itself provides the rendezvous capability rather than +/// depending on it. It collects the same receipts an Odin-correlated target +/// does, and its admitted generation is what names the Odin authority, so it +/// is never route-proof. A target with no route has no endpoint to challenge, +/// so only Odin can report on it and it stays Odin-correlated. #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum ReadinessClass { + OdinSelf, OdinCorrelated, RouteProof, } impl ReadinessClass { fn of(expected: &IdunnExpectedIncarnationRecord) -> Self { + let provides_odin = expected + .capabilities + .iter() + .any(|capability| capability.capability == ODIN_RENDEZVOUS_CAPABILITY); let declares_odin = expected.dependencies.iter().any(|dependency| { dependency.kind == "shared-infrastructure" && dependency.capability == ODIN_RENDEZVOUS_CAPABILITY }); - if declares_odin { + if provides_odin { + Self::OdinSelf + } else if declares_odin || expected.route.is_none() { Self::OdinCorrelated } else { Self::RouteProof @@ -1334,6 +1387,12 @@ impl DeploymentTransaction { self.target == "odin", "direct Warming evidence is reserved for Odin observing itself" ), + WarmingEvidence::RouteProofDirect { .. } => ensure!( + self.expected.as_ref().is_none_or(|expected| { + ReadinessClass::of(expected) == ReadinessClass::RouteProof + }), + "direct route-proof Warming evidence for a target Odin observes" + ), } } if let Some(leasing) = &self.leasing { @@ -1786,7 +1845,7 @@ impl AdmittedGeneration { fn from_transaction( transaction: &DeploymentTransaction, - odin_authority: AdmittedOdinAuthority, + odin_authority: Option, incumbent: Option<&AdmittedGeneration>, now: u64, ) -> Result { @@ -1795,15 +1854,15 @@ impl AdmittedGeneration { "only Committing can create an admitted generation" ); let ready = required(&transaction.ready, "Ready receipt")?.clone(); - let (latest_odin_observation, odin_authority) = match ready.class() { - ReadinessClass::OdinCorrelated => ( + let (latest_odin_observation, odin_authority) = match &ready { + ReadinessEvidence::OdinCorrelated { .. } => ( Some( required(&transaction.latest_odin_observation, "latest Odin receipt")? .clone(), ), - Some(odin_authority), + Some(required(&odin_authority, "Odin authority")?.clone()), ), - ReadinessClass::RouteProof => (None, None), + ReadinessEvidence::RouteProof { .. } => (None, None), }; let routing = required(&transaction.routing, "route disposition")?.clone(); let route_supervision = matches!(&routing, RoutingEvidence::Promoted { .. }).then(|| { @@ -3890,8 +3949,29 @@ impl Engine { &evidence.canonical_bytes, )?; } + WarmingEvidence::RouteProofDirect { evidence } => { + self.reauthenticate_route_proof( + transaction, + evidence, + route_proof_warming_states(required( + &transaction.expected, + "Warming Expected projection", + )?), + None, + evidence.admitted_at_unix_millis, + )?; + } } } + if let Some(ReadinessEvidence::RouteProof { evidence }) = &transaction.ready { + self.reauthenticate_route_proof( + transaction, + evidence, + &["active"], + lease, + evidence.admitted_at_unix_millis, + )?; + } if let Some(evidence) = transaction.ready.as_ref().and_then(ReadinessEvidence::odin) { let authenticated = self.authenticate_topology_bytes( snapshot, @@ -5253,6 +5333,30 @@ impl Engine { let snapshot = ControlSnapshot::read(&self.options.state_store)?; let incumbent_lease_sha256 = self.incumbent_lease_sha256_for_warming(&snapshot, ¤t.value)?; + if current.value.warming.is_none() && ReadinessClass::of(expected) == ReadinessClass::RouteProof { + // A route-proof target's warming is Idunn's own challenge to the + // candidate endpoint. No Odin is read: the target declared no + // reason to be aware of one. + return match self.challenge_candidate( + ¤t.value, + route_proof_warming_states(expected), + None, + )? { + CandidateAnswer::Silent(reason) => self.record_gate_wait(current, &reason), + CandidateAnswer::Answered { evidence, present } => { + let warming = WarmingEvidence::RouteProofDirect { evidence }; + let _token = SequenceAdmittedWarming::from_direct_presence( + current.value.transaction_id.clone(), + warming.clone(), + present, + )?; + self.persist_same_phase(current, |next| { + next.warming = Some(warming); + Ok(()) + }) + } + }; + } if current.value.warming.is_none() { // Odin's warming is observed directly when nothing else can observe // it. That is the first bootstrap, and it is also every continuity @@ -5287,13 +5391,14 @@ impl Engine { "direct Odin warming must be a stateful incarnation" ); let (evidence, present) = self.observe_first_odin_warming(¤t.value)?; - let _token = SequenceAdmittedWarming::from_first_odin_presence( + let warming = WarmingEvidence::FirstOdinDirect { evidence }; + let _token = SequenceAdmittedWarming::from_direct_presence( current.value.transaction_id.clone(), - evidence.clone(), + warming.clone(), present, )?; return self.persist_same_phase(current, |next| { - next.warming = Some(WarmingEvidence::FirstOdinDirect { evidence }); + next.warming = Some(warming); Ok(()) }); } @@ -5680,6 +5785,34 @@ impl Engine { "candidate process write lease is no longer exact" ); } + if ReadinessClass::of(expected) == ReadinessClass::RouteProof { + // Ready is the candidate answering Idunn's own challenge Active, + // holding the exact lease Idunn granted. Once recorded it is + // history; the stable route's challenge is what keeps proving it. + if matches!( + current.value.ready, + Some(ReadinessEvidence::RouteProof { .. }) + ) { + return self.transition(current, DeploymentPhase::Routing); + } + let lease = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256); + return match self.challenge_candidate(¤t.value, &["warming", "active"], lease)? { + CandidateAnswer::Silent(reason) => self.record_gate_wait(current, &reason), + CandidateAnswer::Answered { present, .. } if present.record().state == "warming" => { + self.record_gate_wait(current, "candidate is still warming") + } + CandidateAnswer::Answered { evidence, .. } => { + self.persist_same_phase(current, |next| { + next.ready = Some(ReadinessEvidence::RouteProof { evidence }); + Ok(()) + }) + } + }; + } { let current_lease = current .value @@ -5725,39 +5858,55 @@ impl Engine { let activation = required(¤t.value.activation, "activation")?; self.observe_candidate_before_waiting(¤t.value)?; if current.value.routing.is_none() { - let current_lease = current - .value - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256); - let Some((admitted, authenticated)) = - self.admit_latest_topology(current, current_lease)? - else { - return Ok(()); + let admitted = if ReadinessClass::of(expected) == ReadinessClass::RouteProof { + // Ready is the candidate's own proof, recorded in AwaitingReady. + // No Odin correlation is read to admit a route. + ensure!( + matches!( + current.value.ready, + Some(ReadinessEvidence::RouteProof { .. }) + ), + "route-proof Routing without route-proof Ready evidence" + ); + current.clone() + } else { + let current_lease = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256); + let Some((admitted, authenticated)) = + self.admit_latest_topology(current, current_lease)? + else { + return Ok(()); + }; + if admitted.envelope != current.envelope { + return Ok(()); + } + ensure!( + is_semantic_ready(&authenticated), + "latest Odin observation is not Ready at route admission" + ); + let latest = required( + &admitted.value.latest_odin_observation, + "current route-admission topology evidence", + )?; + if admitted.value.ready.as_ref().and_then(ReadinessEvidence::odin) + != Some(latest) + { + let latest = latest.clone(); + return self.persist_same_phase(&admitted, |next| { + next.ready = Some(ReadinessEvidence::OdinCorrelated { evidence: latest }); + Ok(()) + }); + } + let ready = self.rehydrate_ready_token(&admitted.value, now_millis()?, true)?; + ensure!( + ready.transaction_id() == admitted.value.transaction_id, + "Ready token belongs to another transaction" + ); + admitted }; - if admitted.envelope != current.envelope { - return Ok(()); - } - ensure!( - is_semantic_ready(&authenticated), - "latest Odin observation is not Ready at route admission" - ); - let latest = required( - &admitted.value.latest_odin_observation, - "current route-admission topology evidence", - )?; - if admitted.value.ready.as_ref().and_then(ReadinessEvidence::odin) != Some(latest) { - let latest = latest.clone(); - return self.persist_same_phase(&admitted, |next| { - next.ready = Some(ReadinessEvidence::OdinCorrelated { evidence: latest }); - Ok(()) - }); - } - let ready = self.rehydrate_ready_token(&admitted.value, now_millis()?, true)?; - ensure!( - ready.transaction_id() == admitted.value.transaction_id, - "Ready token belongs to another transaction" - ); validate_live_providers_for_deploy(admitted.value.command_kind, || { self.validate_selected_providers_current(required( &admitted.value.plan, @@ -5834,32 +5983,50 @@ impl Engine { fn advance_committing(&self, current: &Stored) -> Result<()> { self.observe_candidate_before_waiting(¤t.value)?; - let current_lease_sha256 = current - .value - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256) - .map(str::to_owned); - let Some((ready_current, authenticated)) = - self.admit_latest_topology(current, current_lease_sha256.as_deref())? - else { - return Ok(()); + let route_proof = ReadinessClass::of(required( + ¤t.value.expected, + "Expected projection", + )?) == ReadinessClass::RouteProof; + let ready_current = if route_proof { + // The candidate's Ready proof is durable, and the stable route's + // challenge below is its currency. No Odin is read to commit. + ensure!( + matches!( + current.value.ready, + Some(ReadinessEvidence::RouteProof { .. }) + ), + "route-proof commit without route-proof Ready evidence" + ); + current.clone() + } else { + let current_lease_sha256 = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256) + .map(str::to_owned); + let Some((ready_current, authenticated)) = + self.admit_latest_topology(current, current_lease_sha256.as_deref())? + else { + return Ok(()); + }; + if ready_current.envelope != current.envelope { + return Ok(()); + } + ensure!( + is_semantic_ready(&authenticated), + "latest Odin observation is not Ready at admission commit" + ); + // Not require_current: the Ready receipt is durable evidence, so + // authenticating it against `now` asks a stored record to be live and + // refuses it once it ages past the 30s observation window. Route + // admission (:3906) refreshes the receipt to the latest observation and + // can therefore demand currency; by Committing the receipt is history. + // Currency here is answered by the latest observation, authenticated + // and checked for semantic readiness immediately above. + self.rehydrate_ready_token(&ready_current.value, now_millis()?, false)?; + ready_current }; - if ready_current.envelope != current.envelope { - return Ok(()); - } - ensure!( - is_semantic_ready(&authenticated), - "latest Odin observation is not Ready at admission commit" - ); - // Not require_current: the Ready receipt is durable evidence, so - // authenticating it against `now` asks a stored record to be live and - // refuses it once it ages past the 30s observation window. Route - // admission (:3906) refreshes the receipt to the latest observation and - // can therefore demand currency; by Committing the receipt is history. - // Currency here is answered by the latest observation, authenticated - // and checked for semantic readiness immediately above. - self.rehydrate_ready_token(&ready_current.value, now_millis()?, false)?; validate_live_providers_for_deploy(ready_current.value.command_kind, || { self.validate_selected_providers_current(required( &ready_current.value.plan, @@ -5909,26 +6076,31 @@ impl Engine { self.ensure_transaction_write_lease_current(&ready_current.value, now_millis()?)?; } - let current_lease_sha256 = ready_current - .value - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256) - .map(str::to_owned); - let Some((commit_current, authenticated)) = - self.admit_latest_topology(&ready_current, current_lease_sha256.as_deref())? - else { - return Ok(()); + let commit_current = if route_proof { + ready_current + } else { + let current_lease_sha256 = ready_current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256) + .map(str::to_owned); + let Some((commit_current, authenticated)) = + self.admit_latest_topology(&ready_current, current_lease_sha256.as_deref())? + else { + return Ok(()); + }; + if commit_current.envelope != ready_current.envelope { + return Ok(()); + } + ensure!( + is_semantic_ready(&authenticated), + "latest Odin observation is not Ready after the final admission challenge" + ); + self.rehydrate_ready_token(&commit_current.value, now_millis()?, false)?; + commit_current }; - if commit_current.envelope != ready_current.envelope { - return Ok(()); - } - ensure!( - is_semantic_ready(&authenticated), - "latest Odin observation is not Ready after the final admission challenge" - ); let now = now_millis()?; - self.rehydrate_ready_token(&commit_current.value, now, false)?; validate_live_providers_for_deploy(commit_current.value.command_kind, || { self.validate_selected_providers_current(required( &commit_current.value.plan, @@ -5945,7 +6117,9 @@ impl Engine { let snapshot = ControlSnapshot::read(&self.options.state_store)?; let incumbent = self.exact_incumbent(&snapshot, &commit_current.value)?; - let odin_authority = self.current_odin_authority(&snapshot)?; + let odin_authority = (!route_proof) + .then(|| self.current_odin_authority(&snapshot)) + .transpose()?; let generation = AdmittedGeneration::from_transaction( &commit_current.value, odin_authority, @@ -6069,20 +6243,38 @@ impl Engine { "direct Warming refresh is reserved for Odin" ); let (evidence, present) = self.observe_first_odin_warming(¤t.value)?; - let token = SequenceAdmittedWarming::from_first_odin_presence( + let warming = WarmingEvidence::FirstOdinDirect { evidence }; + let token = SequenceAdmittedWarming::from_direct_presence( current.value.transaction_id.clone(), - evidence.clone(), + warming.clone(), present, )?; ensure!( token.signed_presence_sha256() != prior.signed_presence_sha256(), "first Odin replayed its pre-fence Warming presence" ); - Ok(Some(( - current.clone(), - WarmingEvidence::FirstOdinDirect { evidence }, - token, - ))) + Ok(Some((current.clone(), warming, token))) + } + WarmingEvidence::RouteProofDirect { .. } => { + // A stateful route-proof candidate is warming until it holds + // the lease this refresh is about to bind, so the fresh + // presence must be warming again, and must be a new one. + let CandidateAnswer::Answered { evidence, present } = + self.challenge_candidate(¤t.value, &["warming"], None)? + else { + return Ok(None); + }; + let warming = WarmingEvidence::RouteProofDirect { evidence }; + let token = SequenceAdmittedWarming::from_direct_presence( + current.value.transaction_id.clone(), + warming.clone(), + present, + )?; + ensure!( + token.signed_presence_sha256() != prior.signed_presence_sha256(), + "route-proof candidate replayed its pre-fence Warming presence" + ); + Ok(Some((current.clone(), warming, token))) } WarmingEvidence::OdinTopology { evidence: prior_evidence, @@ -6421,6 +6613,7 @@ impl Engine { ) } + /// The stable route's proof: a challenged presence that is Active. fn authenticate_routed_presence( &self, authority: &cultnet_rs::VerifiedRuntimeAuthority, @@ -6430,9 +6623,41 @@ impl Engine { received_at_unix_millis: u64, canonical_presence: &[u8], ) -> Result<(String, u64)> { + let present = self.authenticate_challenged_presence( + authority, + &["active"], + current_write_lease_sha256, + message_id, + challenged_at_unix_millis, + received_at_unix_millis, + canonical_presence, + )?; + Ok(( + present.signed_presence_sha256().to_owned(), + received_at_unix_millis, + )) + } + + /// A presence Idunn challenged for itself, from the stable endpoint or the + /// candidate's: signed by the provider and the launch's activation key, + /// bound to current authority, minted after the challenge, answering that + /// exact challenge, in one of `states`. A warming presence holds no write + /// lease; a presence in any other state holds exactly the current one. + /// Every disagreement with authority is named, so a capacity below the + /// Expected minimum reads as that and not as a generic refusal. + fn authenticate_challenged_presence( + &self, + authority: &cultnet_rs::VerifiedRuntimeAuthority, + states: &[&str], + current_write_lease_sha256: Option<&str>, + message_id: &str, + challenged_at_unix_millis: u64, + received_at_unix_millis: u64, + canonical_presence: &[u8], + ) -> Result { ensure!( received_at_unix_millis >= challenged_at_unix_millis, - "route observation predates its challenge" + "route proof predates its challenge" ); let authenticated = authenticate_runtime_presence_claim( canonical_presence, @@ -6443,31 +6668,49 @@ impl Engine { maximum_future_skew_millis: self.options.topology_maximum_future_skew_millis, }, )?; - let signed_presence_sha256 = authenticated.signed_presence_sha256().to_owned(); let correlation = correlate_runtime_presence_claim(authenticated, authority)?; - ensure!( - correlation.disagreements().is_empty(), - "stable route answered with a runtime that disagrees with current authority" - ); + if !correlation.disagreements().is_empty() { + let named = correlation + .disagreements() + .iter() + .map(|disagreement| { + format!( + "{} (expected {}, observed {})", + disagreement.code, + disagreement.expected.as_deref().unwrap_or("none"), + disagreement.observed.as_deref().unwrap_or("none"), + ) + }) + .collect::>() + .join("; "); + bail!("route proof answered with a runtime that disagrees with current authority: {named}"); + } let present = correlation.into_undisputed_present()?; let presence = present.record(); ensure!( presence.observed_at_unix_millis >= challenged_at_unix_millis, - "stable route returned a presence minted before the route challenge" + "route proof returned a presence minted before its challenge" ); ensure!( - presence.state == "active", - "stable route runtime is not Active" + states.contains(&presence.state.as_str()), + "route proof runtime is {}, not {}", + presence.state, + states.join(" or ") ); ensure!( presence.detail == format!("route-observation:{message_id}"), - "stable route response is not bound to the exact challenge" + "route proof response is not bound to the exact challenge" ); + let lease_is_current = if presence.state == "warming" { + presence.write_lease_sha256.is_none() + } else { + presence.write_lease_sha256.as_deref() == current_write_lease_sha256 + }; ensure!( - presence.write_lease_sha256.as_deref() == current_write_lease_sha256, - "stable route runtime does not hold the exact current process write lease" + lease_is_current, + "route proof runtime does not hold the exact current process write lease" ); - Ok((signed_presence_sha256, received_at_unix_millis)) + Ok(present) } fn authenticate_first_odin_warming_presence( @@ -6800,15 +7043,110 @@ impl Engine { }, &evidence.canonical_bytes, )?; - SequenceAdmittedWarming::from_first_odin_presence( + SequenceAdmittedWarming::from_direct_presence( transaction.transaction_id.clone(), - evidence, + WarmingEvidence::FirstOdinDirect { evidence }, + present, + ) + } + WarmingEvidence::RouteProofDirect { evidence } => { + let present = self.reauthenticate_route_proof( + transaction, + &evidence, + route_proof_warming_states(required( + &transaction.expected, + "Warming Expected projection", + )?), + None, + if require_current { + now + } else { + evidence.admitted_at_unix_millis + }, + )?; + SequenceAdmittedWarming::from_direct_presence( + transaction.transaction_id.clone(), + WarmingEvidence::RouteProofDirect { evidence }, present, ) } } } + /// A route-proof presence Idunn recorded, authenticated again as of `at`. + fn reauthenticate_route_proof( + &self, + transaction: &DeploymentTransaction, + evidence: &RuntimePresenceEvidence, + states: &[&str], + current_write_lease_sha256: Option<&str>, + at_unix_millis: u64, + ) -> Result { + let authority = self.runtime_authority(transaction)?; + self.authenticate_challenged_presence( + &authority, + states, + current_write_lease_sha256, + &evidence.message_id, + evidence.challenged_at_unix_millis, + at_unix_millis, + &evidence.canonical_bytes, + ) + } + + /// One direct challenge to a route-proof target's candidate endpoint. + /// Failing to reach or hear the candidate is only silence: a process that + /// is still starting does not answer, and that is waiting, not a fault. + /// What does answer is authenticated, and a bad answer is an error. + fn challenge_candidate( + &self, + transaction: &DeploymentTransaction, + states: &[&str], + current_write_lease_sha256: Option<&str>, + ) -> Result { + let expected = required(&transaction.expected, "candidate Expected")?; + let binding = required(&transaction.plan, "candidate plan")? + .parsed_inputs()? + .1; + let driver = NginxRouteDriver::new( + binding + .route + .context("route-proof candidate has no route binding")?, + ); + let message_id = format!("candidate-{}", Uuid::new_v4().simple()); + let challenged_at_unix_millis = now_millis()?; + let response = match driver.request_candidate_runtime_presence(expected, &message_id) { + Ok(response) => response, + Err(error) => { + return Ok(CandidateAnswer::Silent(format!( + "candidate endpoint did not answer its challenge: {error:#}" + ))); + } + }; + ensure!( + response.message_id == message_id, + "candidate transport substituted its challenge identity" + ); + let admitted_at_unix_millis = now_millis()?; + let authority = self.runtime_authority(transaction)?; + let present = self.authenticate_challenged_presence( + &authority, + states, + current_write_lease_sha256, + &message_id, + challenged_at_unix_millis, + admitted_at_unix_millis, + &response.canonical_presence, + )?; + let evidence = RuntimePresenceEvidence::from_present( + &present, + message_id, + challenged_at_unix_millis, + admitted_at_unix_millis, + )?; + Ok(CandidateAnswer::Answered { evidence, present }) + } + fn rehydrate_ready_token( &self, transaction: &DeploymentTransaction, @@ -6854,6 +7192,11 @@ impl Engine { ) -> Result> { let mut providers = Vec::new(); for stored in &snapshot.admitted { + // A route-proof generation holds no Odin receipt to hand a + // dependent; its currency is the route observation, which is B4's. + if stored.value.ready.class() == ReadinessClass::RouteProof { + continue; + } // No pre-filter on ready == latest: a provider that published again // after going ready is still ready. rehydrate_admitted_ready owns // that judgment now, and reports why when it refuses. From cfe71f680caa6ec17f3eb54f8d571eacb3daef4f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 10:54:22 +0200 Subject: [PATCH 024/286] WIP B3: route-proof harness and tests Co-Authored-By: Claude Sonnet 5 --- src/control_plane.rs | 670 +++++++++++++++++++++++++++++++++++++++++-- src/drivers.rs | 2 +- 2 files changed, 648 insertions(+), 24 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 1977c9b..7f30f47 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -10190,6 +10190,9 @@ mod tests { assert_eq!(ReadinessClass::of(&expected), ReadinessClass::RouteProof); expected.dependencies.clear(); assert_eq!(ReadinessClass::of(&expected), ReadinessClass::RouteProof); + // Nothing to challenge: only Odin can report on an unrouted target. + expected.route = None; + assert_eq!(ReadinessClass::of(&expected), ReadinessClass::OdinCorrelated); Ok(()) } @@ -10274,7 +10277,7 @@ mod tests { .context("fixture generation has no Odin authority")?; let next = AdmittedGeneration::from_transaction( &committing, - authority.clone(), + Some(authority.clone()), Some(&incumbent), 1_700_000_200_000, )?; @@ -10287,7 +10290,7 @@ mod tests { let first = AdmittedGeneration::from_transaction( &committing, - authority, + Some(authority), None, 1_700_000_200_000, )?; @@ -11077,27 +11080,7 @@ mod tests { /// A route-proof generation from a real admission: the world's Expected /// declares no Odin dependency, so route proof is its class. fn route_proof_generation() -> Result { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; - odin_reports_ready(&world, &seeded, 5)?; - drive(&world, |transaction| { - transaction.phase == DeploymentPhase::Committing - })?; - let mut committing = resident(&world)?.value; - assert_eq!( - ReadinessClass::of(committing.expected.as_ref().unwrap()), - ReadinessClass::RouteProof - ); - committing.ready = Some(route_proof_evidence()); - committing.latest_odin_observation = None; - let generation = AdmittedGeneration::from_transaction( - &committing, - world.engine.bootstrap_odin_authority.clone(), - None, - now_millis()?, - )?; - generation.validate()?; - Ok(generation) + route_proof::committed_generation() } #[test] @@ -13265,4 +13248,645 @@ mod tests { assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); Ok(()) } + + // --------------------------------------------------------------------- + // Route-proof readiness: a routed target that declares no Odin dependency + // is admitted by Idunn's own challenges. Odin is never read. + // --------------------------------------------------------------------- + mod route_proof { + use super::*; + use std::io::{Read, Write}; + use std::net::{TcpListener, TcpStream}; + use std::sync::Mutex; + use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering}; + use std::time::Duration; + + /// The runtime behind both endpoints, answering Idunn's snapshot + /// challenge the way the TypeScript signer does: locally, in the + /// reported health, with `route-observation:` as the detail. + struct RuntimeStub { + expected: IdunnExpectedIncarnationRecord, + activation: IdunnRuntimeActivationRecord, + provider: cultnet_rs::ServiceIdentitySigner, + activation_signer: cultnet_rs::IdunnRuntimeActivationSigner, + state: Mutex<&'static str>, + capacity: AtomicU32, + sequence: AtomicU64, + candidate_hits: AtomicUsize, + stable_hits: AtomicUsize, + stop: AtomicBool, + } + + impl RuntimeStub { + fn answer(&self, message_id: &str) -> Result> { + let expected = &self.expected; + let mut presence = cultnet_rs::GameCultRuntimePresenceHealthRecord { + schema_version: cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + target: expected.target.clone(), + expected_projection_sha256: self.activation.expected_projection_sha256.clone(), + plan_id: expected.plan_id.clone(), + incarnation_id: expected.incarnation_id.clone(), + sealed_release_id: expected.sealed_release_id.clone(), + activation_witness_sha256: self.activation.canonical_sha256()?, + state_schema_generation: expected.state_schema_generation.clone(), + state_contract_sha256: expected.state_contract_sha256.clone(), + runtime_id: expected.runtime_id.clone(), + runtime_instance_id: self.activation.runtime_instance_id.clone(), + bound_endpoint: expected + .route + .as_ref() + .map(|route| route.candidate_endpoint.clone()), + capabilities: expected + .capabilities + .iter() + .map(|capability| cultnet_rs::GameCultRuntimeCapability { + capability: capability.capability.clone(), + schema: capability.schema.clone(), + compatibility: capability.compatibility.clone(), + capacity: self.capacity.load(Ordering::SeqCst), + }) + .collect(), + health_contract: expected.health_contract.clone(), + state: (*self.state.lock().unwrap()).into(), + detail: format!("route-observation:{message_id}"), + write_lease_sha256: None, + signer_identity_id: self.provider.entry().identity_id.clone(), + publisher_sequence: self.sequence.fetch_add(1, Ordering::SeqCst) + 1, + observed_at_unix_millis: now_millis()?, + signature_algorithm: "ed25519".into(), + signature: Vec::new(), + activation_signer_identity_id: self.activation_signer.identity_id(), + activation_signature: Vec::new(), + }; + let proof = presence.canonical_proof_payload()?; + presence.signature = self + .provider + .sign::(&proof) + .signature; + presence.activation_signature = + self.activation_signer.sign_presence_proof(&presence)?; + Ok(rmp_serde::to_vec(&presence)?) + } + + fn set_state(&self, state: &'static str) { + *self.state.lock().unwrap() = state; + } + + fn serve_one(&self, mut stream: TcpStream) -> Result<()> { + stream.set_read_timeout(Some(Duration::from_secs(3)))?; + let mut request = Vec::new(); + let mut chunk = [0_u8; 4096]; + let (header_end, length) = loop { + let read = stream.read(&mut chunk)?; + ensure!(read > 0, "stub client closed early"); + request.extend_from_slice(&chunk[..read]); + if let Some(index) = request.windows(4).position(|w| w == b"\r\n\r\n") { + let head = String::from_utf8_lossy(&request[..index]).to_lowercase(); + let length = head + .lines() + .find_map(|line| line.strip_prefix("content-length:")) + .context("stub request has no Content-Length")? + .trim() + .parse::()?; + if request.len() >= index + 4 + length { + break (index + 4, length); + } + } + }; + let message = cultnet_rs::decode_cultnet_message_from_slice( + &request[header_end..header_end + length], + cultnet_rs::CultNetWireContract::CultNetSchemaV0, + )?; + let cultnet_rs::CultNetMessage::SnapshotRequest { message_id, .. } = message + else { + bail!("stub received something other than a snapshot request"); + }; + let response = cultnet_rs::CultNetMessage::SnapshotResponseRaw { + message_id: message_id.clone(), + documents: vec![cultnet_rs::CultNetRawDocumentRecord { + schema_id: cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + record_key: self.expected.target.clone(), + stored_at: rfc3339_millis(now_millis()?)?, + payload_encoding: cultnet_rs::CultNetRawPayloadEncoding::Messagepack, + payload: self.answer(&message_id)?, + source_runtime_id: None, + source_agent_id: None, + source_role: None, + tags: None, + }], + }; + let body = cultnet_rs::encode_cultnet_message_to_vec( + &response, + cultnet_rs::CultNetWireContract::CultNetSchemaV0, + )?; + stream.write_all( + format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/msgpack\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .as_bytes(), + )?; + stream.write_all(&body)?; + Ok(()) + } + + fn listen(self: &Arc, listener: TcpListener, candidate: bool) { + listener.set_nonblocking(true).expect("nonblocking listener"); + let stub = Arc::clone(self); + std::thread::spawn(move || { + while !stub.stop.load(Ordering::SeqCst) { + match listener.accept() { + Ok((stream, _)) => { + stream.set_nonblocking(false).ok(); + let hits = if candidate { + &stub.candidate_hits + } else { + &stub.stable_hits + }; + hits.fetch_add(1, Ordering::SeqCst); + let _ = stub.serve_one(stream); + } + Err(_) => std::thread::sleep(Duration::from_millis(5)), + } + } + }); + } + } + + impl Drop for RuntimeStub { + fn drop(&mut self) { + self.stop.store(true, Ordering::SeqCst); + } + } + + /// Odin, as the world it runs in sees it. + #[derive(Clone, Copy, PartialEq, Eq)] + enum Odin { + /// Every read of the correlation store fails. + Unreachable, + /// Odin publishes a correlation that says the target is not Ready. + NotReady, + } + + struct RoutedWorld { + world: EngineFixture, + stub: Arc, + odin: Odin, + poisoned: Vec, + } + + /// A stateless, routed target that declares no Odin dependency, over + /// `minimum_capacity`, seeded at `phase` (Warming or Fencing): its + /// Warming evidence, when the phase needs it, is the candidate's own + /// answer to a real challenge. Listeners stand at both endpoints. + fn routed_world( + odin: Odin, + minimum_capacity: u32, + phase: DeploymentPhase, + ) -> Result { + use crate::deployment_plan::tests::{ + BINDING, RECIPE, artifact_receipt, external_input_receipt, source, + }; + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let candidate_listener = TcpListener::bind("127.0.0.1:0")?; + let stable_listener = TcpListener::bind("127.0.0.1:0")?; + let candidate_port = candidate_listener.local_addr()?.port(); + let stable_port = stable_listener.local_addr()?.port(); + + let provider_path = world.root.join("identities/provider.cc"); + let provider_anchor = world.root.join("identities/provider-anchor.cc"); + let provider = + enroll_service_identity_at::(&provider_path)?; + export_service_identity_trust_anchor(&provider, &provider_anchor)?; + let config = world.root.join("service.conf"); + let binding = BINDING + .replace( + "/etc/gamecult/trust/service.cc", + &provider_anchor.display().to_string(), + ) + .replace("service-runtime-signer", &provider.entry().identity_id) + .replace("127.0.0.1:17999", &format!("127.0.0.1:{stable_port}")) + .replace( + "private_port_start = 18000", + &format!("private_port_start = {candidate_port}"), + ) + .replace( + "private_port_end = 18009", + &format!("private_port_end = {candidate_port}"), + ) + .replace( + "/etc/nginx/idunn-stream-routes/service.conf", + &config.display().to_string().replace('\\', "/"), + ); + let recipe = RECIPE + .split("[[dependencies]]") + .next() + .context("recipe is empty")? + .replace( + "capability = \"service.runtime\"", + &format!("capability = \"service.runtime\"\ncapacity = {minimum_capacity}"), + ); + let plan = compile_deployment_plan( + recipe.as_bytes(), + binding.as_bytes(), + source(&recipe), + "service-incarnation-1", + Some(candidate_port), + 110, + &[], + )?; + let release = SealedRelease::new( + &plan, + vec![artifact_receipt()], + vec![external_input_receipt()], + 120, + )?; + let expected = release.expected_projection(&plan)?; + assert!(expected.route.is_some() && expected.dependencies.is_empty()); + assert_eq!(ReadinessClass::of(&expected), ReadinessClass::RouteProof); + + let now = now_millis()?; + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "continuity-service".into(), + kind: CommandKind::Continuity, + selector: "service".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + let mut transaction = + DeploymentTransaction::new(&command, "service".into(), 0, None, now)?; + transaction.lifecycle_authorized_at_unix_millis = Some(now); + let launch = IdunnRuntimeActivationLaunch::issue( + &expected, + runtime_instance_id(&transaction.transaction_id)?, + now, + &world.engine.idunn_signer, + )?; + let mut seed = Vec::new(); + let activation = launch.write_credential(&mut seed)?; + let recorded = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)?.1; + let mut workload = recorded.workload.clone().context("no recorded workload")?; + match &mut workload { + WorkloadObservation::Systemd(observed) => { + observed.runtime_instance_id = activation.runtime_instance_id.clone(); + observed.executable_sha256 = expected.artifact_sha256.clone(); + } + WorkloadObservation::Host(_) => bail!("recorded workload is not a systemd unit"), + } + transaction.isolation = recorded.isolation; + transaction.installed_release = Some(crate::drivers::InstalledReleaseObservation { + sealed_release_id: release.sealed_release_id.clone(), + root: PathBuf::from("/srv/service/releases/test"), + }); + transaction.expected_publication_sha256 = Some(expected.canonical_sha256()?); + transaction.activation_publication_sha256 = Some(activation.canonical_sha256()?); + transaction.sealed_release = Some(release); + transaction.expected = Some(expected.clone()); + transaction.activation = Some(activation.clone()); + transaction.workload = Some(workload); + transaction.plan = Some(plan); + + let stub = Arc::new(RuntimeStub { + expected, + activation, + provider: open_service_identity_at::( + &provider_path, + )?, + activation_signer: cultnet_rs::IdunnRuntimeActivationSigner::from_credential_reader( + &seed[..], + )?, + state: Mutex::new("warming"), + capacity: AtomicU32::new(minimum_capacity), + sequence: AtomicU64::new(0), + candidate_hits: AtomicUsize::new(0), + stable_hits: AtomicUsize::new(0), + stop: AtomicBool::new(false), + }); + stub.listen(candidate_listener, true); + stub.listen(stable_listener, false); + + if phase >= DeploymentPhase::Fencing { + let CandidateAnswer::Answered { evidence, .. } = + world + .engine + .challenge_candidate(&transaction, &["warming"], None)? + else { + bail!("the stub candidate did not answer"); + }; + transaction.warming = Some(WarmingEvidence::RouteProofDirect { evidence }); + let rendered = NginxRouteDriver::new( + transaction + .plan + .as_ref() + .unwrap() + .parsed_inputs()? + .1 + .route + .context("no route binding")?, + ) + .render(transaction.expected.as_ref().unwrap())?; + transaction.route_preflight = Some(crate::drivers::RoutePreflightReceipt { + route_id: "service".into(), + candidate_runtime_instance_id: transaction + .activation + .as_ref() + .unwrap() + .runtime_instance_id + .clone(), + candidate_membership_sha256: sha256_id(&rendered), + incumbent_runtime_instance_id: None, + incumbent_membership_sha256: None, + incumbent_configuration: None, + }); + } + transaction.enter_phase(phase, now); + transaction.validate()?; + let store = SingleFileMessagePackBackingStore::new(&world.state_store); + assert!(store.compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: transaction.transaction_id.clone(), + current: None, + }, + ], + &[ + command_envelope(&command, command.requested_at_unix_millis)?, + transaction_envelope(&transaction, now)?, + ], + )?); + + // Odin's world. Unreachable: the correlation store is bytes no + // reader can decode, so any read of it is an error. NotReady: a + // valid correlation that says the target is not Ready. + let store = &world.engine.options.odin_correlation_store; + let poisoned = match odin { + Odin::Unreachable => { + let garbage = b"odin is not a cache".to_vec(); + std::fs::write(store, &garbage)?; + garbage + } + Odin::NotReady => { + let wrongly = signed_correlation(&world, &transaction, 9, false)?; + SingleFileMessagePackBackingStore::new(store).insert_entry_if_absent( + CultCacheEnvelope { + key: crate::drivers::incarnation_key_of( + &transaction.target, + &transaction.expected.as_ref().unwrap().canonical_sha256()?, + ), + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + payload: wrongly, + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some( + cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into(), + ), + }, + )?; + Vec::new() + } + }; + Ok(RoutedWorld { + world, + stub, + odin, + poisoned, + }) + } + + impl RoutedWorld { + fn step(&self) -> Result<()> { + self.world.engine.advance_transaction(&resident(&self.world)?) + } + + fn run_to_routing(&self) -> Result<()> { + for _ in 0..12 { + if self.transaction()?.phase == DeploymentPhase::Routing { + return Ok(()); + } + self.step()?; + } + bail!("the transaction never reached Routing") + } + + fn transaction(&self) -> Result { + Ok(resident(&self.world)?.value) + } + + /// Odin was never read: the poisoned store is byte for byte what + /// it was, and no step failed on it. + fn assert_odin_untouched(&self) -> Result<()> { + if self.odin == Odin::Unreachable { + assert_eq!( + std::fs::read(&self.world.engine.options.odin_correlation_store)?, + self.poisoned + ); + } + Ok(()) + } + + /// Routing installs an nginx fragment and needs the host's programs, + /// so the world writes the fragment itself and records the promotion + /// the stable challenge proves, then enters Committing. + fn promote_by_hand(&self) -> Result<()> { + let current = resident(&self.world)?; + let transaction = ¤t.value; + assert_eq!(transaction.phase, DeploymentPhase::Routing); + let expected = transaction.expected.as_ref().unwrap(); + let binding = transaction.plan.as_ref().unwrap().parsed_inputs()?.1; + let driver = NginxRouteDriver::new(binding.route.unwrap()); + let rendered = driver.render(expected)?; + std::fs::write(&driver.binding.config_path, &rendered)?; + let observation = self.world.engine.prove_stable_route( + transaction, + &driver, + sha256_id(&rendered), + )?; + let mut next = transaction.clone(); + next.routing = Some(RoutingEvidence::Promoted { + promoted_at_unix_millis: observation.observed_at_unix_millis, + observation, + }); + next.enter_phase(DeploymentPhase::Committing, now_millis()?); + replace_transaction(&self.world.state_store, ¤t, &next) + } + } + + /// A route-proof generation from a real admission. + pub(super) fn committed_generation() -> Result { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.promote_by_hand()?; + routed.step()?; + admitted(&routed) + } + + fn admitted(world: &RoutedWorld) -> Result { + Ok(ControlSnapshot::read(&world.world.state_store)? + .admitted + .into_iter() + .next() + .context("nothing was admitted")? + .value) + } + + #[test] + fn a_route_proof_target_admits_with_odin_unreachable_and_never_reads_it() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; + // Fencing, Leasing, AwaitingReady: the candidate is still warming. + for _ in 0..12 { + routed.step()?; + if routed.transaction()?.last_error.is_some() { + break; + } + } + let waiting = routed.transaction()?; + assert_eq!(waiting.phase, DeploymentPhase::AwaitingReady); + assert!(waiting.ready.is_none()); + assert_eq!( + waiting.last_error.as_deref(), + Some("candidate is still warming") + ); + // The candidate finishes warming: Ready is its own Active answer. + routed.stub.set_state("active"); + routed.step()?; + assert!(matches!( + routed.transaction()?.ready, + Some(ReadinessEvidence::RouteProof { .. }) + )); + routed.step()?; + assert_eq!(routed.transaction()?.phase, DeploymentPhase::Routing); + routed.promote_by_hand()?; + routed.step()?; + + let generation = admitted(&routed)?; + generation.validate()?; + assert_eq!(generation.ready.class(), ReadinessClass::RouteProof); + assert!(generation.odin_authority.is_none()); + assert!(generation.latest_odin_observation.is_none()); + assert!(generation.route_supervision.is_some()); + // Idunn's own challenges carried every proof: the seed's warming, + // two Ready polls; the stable endpoint proved the route twice. + assert_eq!(routed.stub.candidate_hits.load(Ordering::SeqCst), 3); + assert!(routed.stub.stable_hits.load(Ordering::SeqCst) >= 2); + routed.assert_odin_untouched()?; + + // Boot and supervision read no Odin either: the topology refresh + // returns before it opens Odin's store, and durable authority + // skips a route-proof generation's receipts. + let stored = ControlSnapshot::read(&routed.world.state_store)?; + let current = stored.admitted.first().context("no admitted generation")?; + assert!(!routed.world.engine.refresh_admitted_topology(&stored, current)?); + routed.world.engine.validate_durable_authority(&stored)?; + routed.assert_odin_untouched()?; + Ok(()) + } + + #[test] + fn an_odin_correlation_about_a_route_proof_target_decides_nothing() -> Result<()> { + let routed = routed_world(Odin::NotReady, 1, DeploymentPhase::Fencing)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.promote_by_hand()?; + routed.step()?; + let generation = admitted(&routed)?; + assert_eq!(generation.ready.class(), ReadinessClass::RouteProof); + // The valid, not-Ready correlation was neither admitted as a + // receipt nor consulted: the record carries no Odin cursor. + assert_eq!(generation.odin_publisher_sequence_cursor, 0); + Ok(()) + } + + #[test] + fn a_route_proof_candidate_that_has_not_answered_is_waited_on_not_aborted() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming)?; + routed.step()?; + let warmed = routed.transaction()?; + assert_eq!(warmed.phase, DeploymentPhase::Warming); + let Some(WarmingEvidence::RouteProofDirect { evidence }) = &warmed.warming else { + bail!("Warming evidence is not the candidate's own presence"); + }; + assert!(evidence.message_id.starts_with("candidate-")); + assert_eq!(routed.stub.candidate_hits.load(Ordering::SeqCst), 1); + routed.assert_odin_untouched()?; + + // Nothing listens: the same step waits, and says why. + let silent = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming)?; + silent.stub.stop.store(true, Ordering::SeqCst); + std::thread::sleep(Duration::from_millis(50)); + silent.step()?; + let waiting = silent.transaction()?; + assert!(waiting.warming.is_none() && waiting.pre_fencing_abort.is_none()); + assert!( + waiting + .last_error + .as_deref() + .is_some_and(|error| error.contains("candidate endpoint did not answer")), + "{:?}", + waiting.last_error + ); + Ok(()) + } + + #[test] + fn a_capacity_below_the_expected_minimum_is_named_on_the_route_path() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 2, DeploymentPhase::Fencing)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.promote_by_hand()?; + routed.stub.capacity.store(1, Ordering::SeqCst); + let failure = format!("{:#}", routed.step().expect_err("a shortfall must be refused")); + assert!( + failure.contains("expected-capability-000-capacity"), + "{failure}" + ); + assert!(admitted(&routed).is_err()); + Ok(()) + } + + #[test] + fn odin_itself_is_never_route_proof() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming)?; + let mut expected = routed.stub.expected.clone(); + assert_eq!(ReadinessClass::of(&expected), ReadinessClass::RouteProof); + // Odin provides the rendezvous capability and declares no dependency. + expected.capabilities[0].capability = ODIN_RENDEZVOUS_CAPABILITY.into(); + assert!(expected.dependencies.is_empty()); + assert_eq!(ReadinessClass::of(&expected), ReadinessClass::OdinSelf); + + // Route-proof evidence for it is refused: its admitted generation + // is what names the Odin authority, so it must carry one. + let (_, mut odin) = fixture_generation(FIXTURE_GENERATION)?; + odin.target = "odin".into(); + odin.expected = expected; + odin.ready = route_proof_evidence(); + odin.latest_odin_observation = None; + odin.odin_authority = None; + assert!(error_text(odin.validate()).contains("declares Odin")); + Ok(()) + } + + #[test] + fn a_store_of_route_proof_generations_boots_and_resolves_the_bootstrap_authority() + -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.promote_by_hand()?; + routed.step()?; + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + routed.world.engine.validate_durable_authority(&snapshot)?; + assert_eq!( + routed.world.engine.current_odin_authority(&snapshot)?, + routed.world.engine.bootstrap_odin_authority + ); + Ok(()) + } + } } diff --git a/src/drivers.rs b/src/drivers.rs index d56e3d9..44e4223 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -5128,7 +5128,7 @@ impl NginxRouteDriver { Ok(output) } - fn render(&self, expected: &IdunnExpectedIncarnationRecord) -> Result> { + pub(crate) fn render(&self, expected: &IdunnExpectedIncarnationRecord) -> Result> { expected.validate()?; let expected_projection_sha256 = expected.canonical_sha256()?; let route = expected From 66fda86d8fe9a37ec615d48a2f222859cc905eb2 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 10:56:26 +0200 Subject: [PATCH 025/286] WIP B3: two private ports in the routed world --- src/control_plane.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 7f30f47..a962ec9 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -13472,7 +13472,7 @@ mod tests { ) .replace( "private_port_end = 18009", - &format!("private_port_end = {candidate_port}"), + &format!("private_port_end = {}", candidate_port + 1), ) .replace( "/etc/nginx/idunn-stream-routes/service.conf", From b9864b5041a4314b5f78f51f402f094bb498b515 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 11:00:47 +0200 Subject: [PATCH 026/286] WIP B3: Odin-providing target is driven through the engine --- src/control_plane.rs | 82 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 71 insertions(+), 11 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index a962ec9..712dc4a 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -13443,6 +13443,7 @@ mod tests { odin: Odin, minimum_capacity: u32, phase: DeploymentPhase, + provides_odin: bool, ) -> Result { use crate::deployment_plan::tests::{ BINDING, RECIPE, artifact_receipt, external_input_receipt, source, @@ -13484,7 +13485,14 @@ mod tests { .context("recipe is empty")? .replace( "capability = \"service.runtime\"", - &format!("capability = \"service.runtime\"\ncapacity = {minimum_capacity}"), + &format!( + "capability = \"{}\"\ncapacity = {minimum_capacity}", + if provides_odin { + ODIN_RENDEZVOUS_CAPABILITY + } else { + "service.runtime" + } + ), ); let plan = compile_deployment_plan( recipe.as_bytes(), @@ -13503,7 +13511,14 @@ mod tests { )?; let expected = release.expected_projection(&plan)?; assert!(expected.route.is_some() && expected.dependencies.is_empty()); - assert_eq!(ReadinessClass::of(&expected), ReadinessClass::RouteProof); + assert_eq!( + ReadinessClass::of(&expected), + if provides_odin { + ReadinessClass::OdinSelf + } else { + ReadinessClass::RouteProof + } + ); let now = now_millis()?; let command = DeploymentCommand { @@ -13566,7 +13581,22 @@ mod tests { stub.listen(candidate_listener, true); stub.listen(stable_listener, false); - if phase >= DeploymentPhase::Fencing { + if phase >= DeploymentPhase::Fencing && provides_odin { + // Odin's own first word about the candidate, as for any + // Odin-correlated target. + let warming = signed_correlation(&world, &transaction, 4, false)?; + let authenticated = world.engine.authenticate_topology_bytes( + &ControlSnapshot::read(&world.state_store)?, + &transaction, + &warming, + None, + now, + )?; + transaction.warming = Some(WarmingEvidence::OdinTopology { + evidence: TopologyEvidence::from_authenticated(&authenticated, now)?, + }); + transaction.odin_publisher_sequence_cursor = 4; + } else if phase >= DeploymentPhase::Fencing { let CandidateAnswer::Answered { evidence, .. } = world .engine @@ -13575,6 +13605,8 @@ mod tests { bail!("the stub candidate did not answer"); }; transaction.warming = Some(WarmingEvidence::RouteProofDirect { evidence }); + } + if phase >= DeploymentPhase::Fencing { let rendered = NginxRouteDriver::new( transaction .plan @@ -13627,6 +13659,10 @@ mod tests { // valid correlation that says the target is not Ready. let store = &world.engine.options.odin_correlation_store; let poisoned = match odin { + _ if provides_odin => { + odin_reports_ready(&world, &transaction, 5)?; + Vec::new() + } Odin::Unreachable => { let garbage = b"odin is not a cache".to_vec(); std::fs::write(store, &garbage)?; @@ -13719,7 +13755,7 @@ mod tests { /// A route-proof generation from a real admission. pub(super) fn committed_generation() -> Result { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; routed.stub.set_state("active"); routed.run_to_routing()?; routed.promote_by_hand()?; @@ -13738,7 +13774,7 @@ mod tests { #[test] fn a_route_proof_target_admits_with_odin_unreachable_and_never_reads_it() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; // Fencing, Leasing, AwaitingReady: the candidate is still warming. for _ in 0..12 { routed.step()?; @@ -13790,7 +13826,7 @@ mod tests { #[test] fn an_odin_correlation_about_a_route_proof_target_decides_nothing() -> Result<()> { - let routed = routed_world(Odin::NotReady, 1, DeploymentPhase::Fencing)?; + let routed = routed_world(Odin::NotReady, 1, DeploymentPhase::Fencing, false)?; routed.stub.set_state("active"); routed.run_to_routing()?; routed.promote_by_hand()?; @@ -13805,7 +13841,7 @@ mod tests { #[test] fn a_route_proof_candidate_that_has_not_answered_is_waited_on_not_aborted() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming)?; + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; routed.step()?; let warmed = routed.transaction()?; assert_eq!(warmed.phase, DeploymentPhase::Warming); @@ -13817,7 +13853,7 @@ mod tests { routed.assert_odin_untouched()?; // Nothing listens: the same step waits, and says why. - let silent = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming)?; + let silent = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; silent.stub.stop.store(true, Ordering::SeqCst); std::thread::sleep(Duration::from_millis(50)); silent.step()?; @@ -13836,7 +13872,7 @@ mod tests { #[test] fn a_capacity_below_the_expected_minimum_is_named_on_the_route_path() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 2, DeploymentPhase::Fencing)?; + let routed = routed_world(Odin::Unreachable, 2, DeploymentPhase::Fencing, false)?; routed.stub.set_state("active"); routed.run_to_routing()?; routed.promote_by_hand()?; @@ -13852,7 +13888,7 @@ mod tests { #[test] fn odin_itself_is_never_route_proof() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming)?; + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; let mut expected = routed.stub.expected.clone(); assert_eq!(ReadinessClass::of(&expected), ReadinessClass::RouteProof); // Odin provides the rendezvous capability and declares no dependency. @@ -13872,10 +13908,34 @@ mod tests { Ok(()) } + #[test] + fn a_target_providing_the_rendezvous_commits_on_odin_evidence_and_carries_the_authority() + -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, true)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + let ready = routed.transaction()?; + assert!(ready.ready.as_ref().and_then(ReadinessEvidence::odin).is_some()); + routed.promote_by_hand()?; + routed.step()?; + let generation = admitted(&routed)?; + generation.validate()?; + assert_eq!(generation.ready.class(), ReadinessClass::OdinCorrelated); + assert_eq!( + generation.odin_authority, + Some(routed.world.engine.bootstrap_odin_authority.clone()) + ); + assert!(generation.latest_odin_observation.is_some()); + // Boot re-authenticates every Odin receipt this generation carries. + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + routed.world.engine.validate_durable_authority(&snapshot)?; + Ok(()) + } + #[test] fn a_store_of_route_proof_generations_boots_and_resolves_the_bootstrap_authority() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; routed.stub.set_state("active"); routed.run_to_routing()?; routed.promote_by_hand()?; From 54507060a56c370d73545a538e36fcb7a8d1eb8f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 11:02:06 +0200 Subject: [PATCH 027/286] WIP B3: pin the state rules, the lease refresh and boot re-authentication --- src/control_plane.rs | 93 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 88 insertions(+), 5 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 712dc4a..49f8560 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -7192,11 +7192,6 @@ impl Engine { ) -> Result> { let mut providers = Vec::new(); for stored in &snapshot.admitted { - // A route-proof generation holds no Odin receipt to hand a - // dependent; its currency is the route observation, which is B4's. - if stored.value.ready.class() == ReadinessClass::RouteProof { - continue; - } // No pre-filter on ready == latest: a provider that published again // after going ready is still ready. rehydrate_admitted_ready owns // that judgment now, and reports why when it refuses. @@ -13908,6 +13903,94 @@ mod tests { Ok(()) } + #[test] + fn a_stateless_candidate_that_finished_warming_first_still_warms() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + routed.stub.set_state("active"); + routed.step()?; + assert!(matches!( + routed.transaction()?.warming, + Some(WarmingEvidence::RouteProofDirect { .. }) + )); + Ok(()) + } + + #[test] + fn the_lease_refresh_asks_the_candidate_for_a_new_warming_presence() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let current = resident(&routed.world)?; + let prior = routed + .world + .engine + .rehydrate_warming_token(¤t.value, now_millis()?, false)?; + let (_, evidence, token) = routed + .world + .engine + .fresh_warming_for_lease(¤t, now_millis()?)? + .context("a warming candidate must refresh")?; + assert!(matches!(evidence, WarmingEvidence::RouteProofDirect { .. })); + assert_ne!(token.signed_presence_sha256(), prior.signed_presence_sha256()); + // A candidate that went Active is not warming: nothing to bind. + routed.stub.set_state("active"); + let failure = routed + .world + .engine + .fresh_warming_for_lease(¤t, now_millis()?) + .map(|_| ()) + .expect_err("an active candidate is not a warming one"); + assert!(format!("{failure:#}").contains("not warming"), "{failure:#}"); + routed.assert_odin_untouched()?; + Ok(()) + } + + #[test] + fn boot_reauthenticates_a_live_route_proof_transactions_evidence() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + routed.world.engine.validate_durable_authority(&snapshot)?; + + // Warming evidence that does not answer the challenge it names. + let current = resident(&routed.world)?; + let mut forged = current.value.clone(); + let Some(WarmingEvidence::RouteProofDirect { evidence }) = &mut forged.warming else { + bail!("no route-proof Warming evidence"); + }; + evidence.message_id = "candidate-another".into(); + replace_transaction(&routed.world.state_store, ¤t, &forged)?; + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + let failure = routed + .world + .engine + .validate_durable_authority(&snapshot) + .expect_err("Warming evidence for another challenge"); + assert!(format!("{failure:#}").contains("exact challenge"), "{failure:#}"); + + // A Ready label carried by a presence that was only warming. + let current = resident(&routed.world)?; + let CandidateAnswer::Answered { evidence, .. } = routed + .world + .engine + .challenge_candidate(¤t.value, &["warming"], None)? + else { + bail!("the stub candidate did not answer"); + }; + let mut forged = current.value.clone(); + forged.warming = Some(WarmingEvidence::RouteProofDirect { + evidence: evidence.clone(), + }); + forged.ready = Some(ReadinessEvidence::RouteProof { evidence }); + replace_transaction(&routed.world.state_store, ¤t, &forged)?; + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + let failure = routed + .world + .engine + .validate_durable_authority(&snapshot) + .expect_err("Ready carried by a warming presence"); + assert!(format!("{failure:#}").contains("not active"), "{failure:#}"); + routed.assert_odin_untouched()?; + Ok(()) + } + #[test] fn a_target_providing_the_rendezvous_commits_on_odin_evidence_and_carries_the_authority() -> Result<()> { From 4dcdd3608c25c0ad8d822cbc037a00d07d31d642 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 11:03:47 +0200 Subject: [PATCH 028/286] WIP B3: route admission test --- src/control_plane.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index 49f8560..b1c2131 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -13903,6 +13903,19 @@ mod tests { Ok(()) } + #[test] + fn route_admission_gets_past_its_gate_without_reading_odin() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + // The host's nginx is not here, so the step fails at installing the + // route: past the admission gate, which would have failed on Odin. + let failure = format!("{:#}", routed.step().expect_err("no nginx in the test world")); + assert!(failure.contains("XXXX"), "{failure}"); + routed.assert_odin_untouched()?; + Ok(()) + } + #[test] fn a_stateless_candidate_that_finished_warming_first_still_warms() -> Result<()> { let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; From 5ca7760fdda8e6d910b77f00e9e4aa5ef8d17acf Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 11:06:25 +0200 Subject: [PATCH 029/286] WIP B3: pin route admission past the gate --- src/control_plane.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index b1c2131..5d52938 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -13905,13 +13905,21 @@ mod tests { #[test] fn route_admission_gets_past_its_gate_without_reading_odin() -> Result<()> { + // The step below would run the host's real nginx, so it only runs + // where there is none. + if Path::new("/usr/sbin/nginx").exists() { + return Ok(()); + } let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; routed.stub.set_state("active"); routed.run_to_routing()?; - // The host's nginx is not here, so the step fails at installing the - // route: past the admission gate, which would have failed on Odin. + // The step fails at installing the route: past the admission gate, + // which would have failed on Odin. let failure = format!("{:#}", routed.step().expect_err("no nginx in the test world")); - assert!(failure.contains("XXXX"), "{failure}"); + assert!( + failure.contains("starting route actuator /usr/sbin/nginx"), + "{failure}" + ); routed.assert_odin_untouched()?; Ok(()) } From 6501cafda3e23e0db0aef58311815d420ba7ee60 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 12:46:32 +0200 Subject: [PATCH 030/286] Record B3 Soul verdict and the recipe-declares-readiness ruling Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index 512ac09..0eb920b 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -11,6 +11,31 @@ on CultLib `codex/fix-node24-ajv-esm`. Behaviour cuts re-take their `file:line` anchors in their own briefs. The operator has asked for no live mitigation on Yggdrasil yet. +**B3 status, 2026-09-30 (Self).** Hands pushed B3 to `idunn/route-b3` at `5ca7760`; it is based on +B1's merge `a61540b`, not stacked on B2. The Opus Soul pass found: +- Full suite green: 252 passed, 2 ignored. +- `cargo-mutants --in-diff`: 31 caught, 4 missed. +- Soul's probe P1 drove a stateful route-proof target through the Engine, so the stateful path works; it + was only untested. + +Verdict: safe to merge for the StreamPixels web ship. The fix batch, in Hands, covers these findings: +- the evidence tag is a second class authority, and a pre-B3 transaction wedges Routing; +- the driver's rejections of a bad answer become gate waits instead of errors; +- the shortfall is prose; +- `OdinSelf` is decorative, and "is Odin" is still decided by name in five places; +- Warming is never driven past one step; +- seven unpinned checks. + +Before `idunn up`, one read-only decode of a *copy* of the live `control.cc` checks for non-terminal +transactions of routed targets with no Odin declaration (operator approved, 2026-09-30). + +**Ruling, 2026-09-30: the recipe declares readiness; Idunn infers nothing.** This settles the fork +Hands raised against a literal Q1(b). An unrouted target whose recipe declares neither a route nor an +Odin dependency is refused at admission with a typed error. It does not fall back to Odin correlation +because it has no route. `raven-muninn` and Heimdall both publish to Odin, so their recipes declare it. +The rejected alternative let Odin's availability gate Muninn's crash recovery through an undeclared, +inferred dependency. + Heads read for this map: | Repo | Ref | Notes | From 6ea824b5a9d605186150314afa50a4ce623d17d2 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 13:12:46 +0200 Subject: [PATCH 031/286] B3 fix: readiness class comes from the recipe alone ReadinessClass::of is the only class authority. A target whose recipe declares neither a route nor an Odin dependency, and does not provide the rendezvous, has declared no way to prove readiness: of() returns a typed UndeclaredReadiness naming the target, and admission refuses it after plan compile, before anything is frozen or installed. The OdinCorrelated fallback for an unrouted target is gone. The evidence tag no longer decides the path. Boot re-authentication and admitted-topology refresh choose by class; a stored transaction or admitted generation whose evidence was vouched for by the other party than its Expected requires is a typed ReadinessDisagreement, reported once and held, not a step that errors every tick. Nothing re-derives around it. "Is this Odin" is the class OdinSelf, not the name "odin": validation, warming, the Odin authority lookup and the direct-warming checks read the class. canonical_sha256 on WarmingEvidence and the token's unread evidence field are deleted. A bad answer to a direct challenge is an error. The driver splits its failures into ChallengeFailure::Silent (refused, dropped, hung up, timed out) and ::Refused (a malformed, foreign, oversized or non-raw answer, or a local configuration error); only Silent waits. The capacity/disagreement shortfall is returned as a typed PresenceDisagrees, not flattened to prose. Odin-driven fixture worlds declare the Odin dependency in Expected and the correlation they sign names its provider. Co-Authored-By: Claude Sonnet 5 --- src/control_plane.rs | 490 +++++++++++++++++++++++++++++++---------- src/deployment_plan.rs | 21 +- src/drivers.rs | 168 ++++++++++---- 3 files changed, 519 insertions(+), 160 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 5d52938..79e007f 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -37,7 +37,7 @@ use crate::deployment_plan::{ CompiledDeploymentPlan, DependencyProviderAuthority, SealedRelease, compile_deployment_plan, }; use crate::drivers::{ - CultCacheTopologyDriver, CultCacheWriteLeaseDriver, DockerRunnerDriver, FrozenSourceReceipt, + ChallengeFailure, CultCacheTopologyDriver, CultCacheWriteLeaseDriver, DockerRunnerDriver, FrozenSourceReceipt, GitSourceDriver, InstalledReleaseObservation, IsolationEvidence, NginxRouteDriver, ProcessIdentity, RouteObservation, RoutePreflightReceipt, RunnerPort, SourcePort, SystemdTransientWorkloadDriver, TopologyPort, WorkloadObservation, WorkloadPort, @@ -55,7 +55,7 @@ const DEPLOYMENT_TRANSACTION_SCHEMA_V2: &str = "idunn.deployment_transaction.v2" const ADMITTED_GENERATION_SCHEMA: &str = "idunn.admitted_generation.v3"; const ADMITTED_GENERATION_SCHEMA_V2: &str = "idunn.admitted_generation.v2"; /// The capability whose declaration makes a target Odin-correlated. -const ODIN_RENDEZVOUS_CAPABILITY: &str = "odin.verse-rendezvous"; +pub(crate) const ODIN_RENDEZVOUS_CAPABILITY: &str = "odin.verse-rendezvous"; /// How many times continuity will restart one admitted release before it /// concludes the release itself is the problem. More than one because a start /// can fail for a passing reason -- a port still held, a peer not yet up -- @@ -276,12 +276,10 @@ impl WarmingEvidence { } } - fn canonical_sha256(&self) -> &str { + fn voucher(&self) -> Voucher { match self { - Self::OdinTopology { evidence } => &evidence.canonical_sha256, - Self::FirstOdinDirect { evidence } | Self::RouteProofDirect { evidence } => { - &evidence.canonical_sha256 - } + Self::OdinTopology { .. } | Self::FirstOdinDirect { .. } => Voucher::Odin, + Self::RouteProofDirect { .. } => Voucher::Candidate, } } @@ -303,7 +301,6 @@ impl WarmingEvidence { #[derive(Clone, Debug)] pub struct SequenceAdmittedWarming { transaction_id: String, - evidence: WarmingEvidence, signed_presence_sha256: String, runtime_instance_id: String, } @@ -311,7 +308,6 @@ pub struct SequenceAdmittedWarming { impl SequenceAdmittedWarming { fn from_topology( transaction_id: String, - evidence: TopologyEvidence, authenticated: AuthenticatedOdinRuntimeTopologyCorrelation, ) -> Result { let record = authenticated.record(); @@ -325,7 +321,6 @@ impl SequenceAdmittedWarming { .context("Warming topology has no runtime instance")?; Ok(Self { transaction_id, - evidence: WarmingEvidence::OdinTopology { evidence }, signed_presence_sha256, runtime_instance_id, }) @@ -348,7 +343,6 @@ impl SequenceAdmittedWarming { Ok(Self { transaction_id, signed_presence_sha256: evidence.canonical_sha256.clone(), - evidence: warming, runtime_instance_id, }) } @@ -357,7 +351,6 @@ impl SequenceAdmittedWarming { pub(crate) fn for_test( transaction_id: impl Into, authenticated: AuthenticatedOdinRuntimeTopologyCorrelation, - admitted_at_unix_millis: u64, ) -> Result { let record = authenticated.record(); ensure!( @@ -370,8 +363,6 @@ impl SequenceAdmittedWarming { ); let transaction_id = transaction_id.into(); require_id(&transaction_id, "test transaction id")?; - let evidence = - TopologyEvidence::from_authenticated(&authenticated, admitted_at_unix_millis)?; let signed_presence_sha256 = record .signed_presence_sha256 .clone() @@ -380,7 +371,7 @@ impl SequenceAdmittedWarming { .runtime_instance_id .clone() .context("test Warming receipt has no runtime instance")?; - let token = Self::from_topology(transaction_id, evidence, authenticated)?; + let token = Self::from_topology(transaction_id, authenticated)?; ensure!( token.signed_presence_sha256 == signed_presence_sha256 && token.runtime_instance_id == runtime_instance_id, @@ -393,10 +384,6 @@ impl SequenceAdmittedWarming { &self.transaction_id } - pub(crate) fn evidence_sha256(&self) -> &str { - self.evidence.canonical_sha256() - } - pub(crate) fn signed_presence_sha256(&self) -> &str { &self.signed_presence_sha256 } @@ -557,6 +544,40 @@ fn route_proof_warming_states(expected: &IdunnExpectedIncarnationRecord) -> &'st } } +/// A challenged runtime answered as something that disagrees with current +/// authority. Each disagreement is named, so a capacity below the Expected +/// minimum reads as that and not as a generic refusal. It stays the error's +/// typed value; callers read `disagreements`, and the text is only its +/// rendering. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct PresenceDisagrees { + pub(crate) disagreements: Vec, +} + +impl std::fmt::Display for PresenceDisagrees { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let named = self + .disagreements + .iter() + .map(|disagreement| { + format!( + "{} (expected {}, observed {})", + disagreement.code, + disagreement.expected.as_deref().unwrap_or("none"), + disagreement.observed.as_deref().unwrap_or("none"), + ) + }) + .collect::>() + .join("; "); + write!( + formatter, + "route proof answered with a runtime that disagrees with current authority: {named}" + ) + } +} + +impl std::error::Error for PresenceDisagrees {} + /// What a direct challenge to a candidate endpoint produced. enum CandidateAnswer { /// No answer: the reason it is still being waited on. @@ -567,46 +588,161 @@ enum CandidateAnswer { }, } -/// Which kind of proof admits a target as Ready. Derived from the target's -/// Expected, never stored: Expected is digest-bound, so the class cannot drift -/// from the incarnation it describes. A target that declares a -/// `shared-infrastructure odin.verse-rendezvous` dependency advertises into the -/// Verse and is Odin-correlated; a routed one that declares none is -/// route-proof. Odin itself provides the rendezvous capability rather than -/// depending on it. It collects the same receipts an Odin-correlated target -/// does, and its admitted generation is what names the Odin authority, so it -/// is never route-proof. A target with no route has no endpoint to challenge, -/// so only Odin can report on it and it stays Odin-correlated. +//// Which kind of proof admits a target as Ready, derived from what the target's +/// recipe declares and from nothing else. Idunn infers no way to prove +/// readiness: a target that provides `odin.verse-rendezvous` is Odin itself, one +/// that declares a `shared-infrastructure odin.verse-rendezvous` dependency +/// advertises into the Verse and is Odin-correlated, and a routed one that +/// declares neither answers Idunn's own challenge and is route-proof. A target +/// with no route and no Odin dependency declares no proof at all, and is +/// refused. Odin collects the same receipts an Odin-correlated target does, and +/// its admitted generation is what names the Odin authority, so it is never +/// route-proof. +/// +/// `of` is the only place a class is decided, and Expected is digest-bound, so +/// the class cannot drift from the incarnation it describes. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum ReadinessClass { +pub(crate) enum ReadinessClass { OdinSelf, OdinCorrelated, RouteProof, } +/// A recipe that declares neither a route nor an Odin dependency, and is not +/// Odin: it has said nothing about how it proves readiness. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct UndeclaredReadiness { + pub(crate) target: String, +} + +impl std::fmt::Display for UndeclaredReadiness { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + formatter, + "target {} declares no way to prove readiness: its recipe must declare a route \ + (Idunn challenges it) or a shared-infrastructure {ODIN_RENDEZVOUS_CAPABILITY} \ + dependency (Odin correlates it)", + self.target + ) + } +} + +impl std::error::Error for UndeclaredReadiness {} + impl ReadinessClass { - fn of(expected: &IdunnExpectedIncarnationRecord) -> Self { - let provides_odin = expected - .capabilities - .iter() - .any(|capability| capability.capability == ODIN_RENDEZVOUS_CAPABILITY); - let declares_odin = expected.dependencies.iter().any(|dependency| { - dependency.kind == "shared-infrastructure" - && dependency.capability == ODIN_RENDEZVOUS_CAPABILITY - }); + /// The class of a recipe's declarations. Plan compilation and `of` both + /// call it, so admission refuses exactly what `of` would refuse. + pub(crate) fn declared( + target: &str, + provides_odin: bool, + declares_odin: bool, + routed: bool, + ) -> Result { if provides_odin { - Self::OdinSelf - } else if declares_odin || expected.route.is_none() { - Self::OdinCorrelated + Ok(Self::OdinSelf) + } else if declares_odin { + Ok(Self::OdinCorrelated) + } else if routed { + Ok(Self::RouteProof) } else { - Self::RouteProof + Err(UndeclaredReadiness { + target: target.to_owned(), + }) + } + } + + fn of(expected: &IdunnExpectedIncarnationRecord) -> Result { + Self::declared( + &expected.target, + expected + .capabilities + .iter() + .any(|capability| capability.capability == ODIN_RENDEZVOUS_CAPABILITY), + expected.dependencies.iter().any(|dependency| { + dependency.kind == "shared-infrastructure" + && dependency.capability == ODIN_RENDEZVOUS_CAPABILITY + }), + expected.route.is_some(), + ) + } + + /// The class of `expected`, provided every receipt collected under it was + /// vouched for by that class's voucher. The one way a stored record is + /// asked what class it is. + fn confirmed( + expected: &IdunnExpectedIncarnationRecord, + collected: impl IntoIterator, + ) -> Result { + let required = Self::of(expected).map_err(ReadinessDisagreement::Undeclared)?; + match collected + .into_iter() + .find(|voucher| *voucher != required.voucher()) + { + None => Ok(required), + Some(collected) => Err(ReadinessDisagreement::WrongVoucher { + target: expected.target.clone(), + required, + collected, + }), + } + } + + /// Who vouches for a target of this class: Odin's correlation, or the + /// candidate answering Idunn. + fn voucher(self) -> Voucher { + match self { + Self::OdinSelf | Self::OdinCorrelated => Voucher::Odin, + Self::RouteProof => Voucher::Candidate, } } } -/// The receipt that admitted a candidate as Ready, tagged by the class of proof -/// that produced it. The tag records what was collected; the class a target -/// must collect is `ReadinessClass::of(expected)`. +/// Who vouched for a piece of stored readiness evidence. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Voucher { + Odin, + Candidate, +} + +/// A stored record whose readiness evidence is not what its own Expected +/// requires. A record admitted before the target's recipe declared how it +/// proves readiness looks like this. It is reported and held, never repaired: +/// re-deriving the evidence around it would make the evidence tag a second +/// authority for the class. +#[derive(Clone, Debug, PartialEq, Eq)] +enum ReadinessDisagreement { + Undeclared(UndeclaredReadiness), + WrongVoucher { + target: String, + required: ReadinessClass, + collected: Voucher, + }, +} + +impl std::fmt::Display for ReadinessDisagreement { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Undeclared(undeclared) => write!(formatter, "{undeclared}"), + Self::WrongVoucher { + target, + required, + collected, + } => write!( + formatter, + "target {target} is {required:?} but its stored readiness evidence was vouched \ + for by {collected:?}: it was collected under a different class, and is held \ + until the target is redeployed" + ), + } + } +} + +impl std::error::Error for ReadinessDisagreement {} + +/// The receipt that admitted a candidate as Ready, tagged by who vouched for +/// it. The tag records what was collected; what a target must collect is +/// `ReadinessClass::of(expected)`, and a tag that differs is a +/// `ReadinessDisagreement`, never a second opinion about the class. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "class", rename_all = "kebab-case", deny_unknown_fields)] enum ReadinessEvidence { @@ -615,10 +751,10 @@ enum ReadinessEvidence { } impl ReadinessEvidence { - fn class(&self) -> ReadinessClass { + fn voucher(&self) -> Voucher { match self { - Self::OdinCorrelated { .. } => ReadinessClass::OdinCorrelated, - Self::RouteProof { .. } => ReadinessClass::RouteProof, + Self::OdinCorrelated { .. } => Voucher::Odin, + Self::RouteProof { .. } => Voucher::Candidate, } } @@ -637,7 +773,7 @@ impl ReadinessEvidence { } } -/// Signed evidence that the candidate process picked up the lease Idunn +// Signed evidence that the candidate process picked up the lease Idunn /// granted: its presence carries `write_lease_sha256` equal to the grant. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -1286,6 +1422,20 @@ impl DeploymentTransaction { .is_some_and(|cleanup| !cleanup.is_complete()) } + /// Where what this transaction has collected disagrees with what its own + /// Expected requires. There is nothing to disagree with before Expected. + fn readiness_disagreement(&self) -> Option { + let expected = self.expected.as_ref()?; + ReadinessClass::confirmed( + expected, + self.warming + .iter() + .map(WarmingEvidence::voucher) + .chain(self.ready.iter().map(ReadinessEvidence::voucher)), + ) + .err() + } + fn validate(&self) -> Result<()> { ensure!( self.schema_version == DEPLOYMENT_TRANSACTION_SCHEMA, @@ -1369,10 +1519,15 @@ impl DeploymentTransaction { } if let Some(ready) = &self.ready { ready.validate_shape()?; - if let (ReadinessClass::RouteProof, Some(expected)) = (ready.class(), &self.expected) { + // A candidate-vouched receipt is only ever written by this code, so + // one on a target that is not route-proof is forged. The other + // direction -- an Odin receipt on a route-proof target -- is what a + // pre-B3 record looks like, and is a ReadinessDisagreement to hold, + // not a record to refuse: refusing it would refuse to boot. + if let (Voucher::Candidate, Some(expected)) = (ready.voucher(), &self.expected) { ensure!( - ReadinessClass::of(expected) == ReadinessClass::RouteProof, - "route-proof Ready evidence for a target that declares Odin" + ReadinessClass::of(expected).is_ok_and(|class| class == ReadinessClass::RouteProof), + "route-proof Ready evidence for a target that is not route-proof" ); } } @@ -1384,14 +1539,16 @@ impl DeploymentTransaction { "Warming topology evidence exceeds the transaction replay cursor" ), WarmingEvidence::FirstOdinDirect { .. } => ensure!( - self.target == "odin", + self.expected.as_ref().is_some_and(|expected| { + ReadinessClass::of(expected).is_ok_and(|class| class == ReadinessClass::OdinSelf) + }), "direct Warming evidence is reserved for Odin observing itself" ), WarmingEvidence::RouteProofDirect { .. } => ensure!( self.expected.as_ref().is_none_or(|expected| { - ReadinessClass::of(expected) == ReadinessClass::RouteProof + ReadinessClass::of(expected).is_ok_and(|class| class == ReadinessClass::RouteProof) }), - "direct route-proof Warming evidence for a target Odin observes" + "direct route-proof Warming evidence for a target that is not route-proof" ), } } @@ -1827,6 +1984,12 @@ struct OdinReceipts<'a> { } impl AdmittedGeneration { + /// This generation's class, or where its evidence disagrees with it. The + /// class comes from its Expected, never from the tag on its Ready receipt. + fn readiness(&self) -> Result { + ReadinessClass::confirmed(&self.expected, [self.ready.voucher()]) + } + fn odin_receipts(&self) -> Result> { match ( &self.ready, @@ -1930,8 +2093,8 @@ impl AdmittedGeneration { ); } (ReadinessEvidence::RouteProof { .. }, None, None) => ensure!( - ReadinessClass::of(&self.expected) == ReadinessClass::RouteProof, - "route-proof Ready evidence for a target that declares Odin" + ReadinessClass::of(&self.expected).is_ok_and(|class| class == ReadinessClass::RouteProof), + "route-proof Ready evidence for a target that is not route-proof" ), _ => bail!("admitted readiness evidence and Odin receipts disagree"), } @@ -2540,6 +2703,16 @@ impl ControlSnapshot { .find(|stored| stored.value.target == target) } + /// The admitted generation of the target that provides + /// `odin.verse-rendezvous`: Odin is whichever target says so, not + /// whichever is named "odin". + fn admitted_odin(&self) -> Option<&Stored> { + self.admitted.iter().find(|stored| { + ReadinessClass::of(&stored.value.expected) + .is_ok_and(|class| class == ReadinessClass::OdinSelf) + }) + } + fn transaction_for_command(&self, command_id: &str) -> Vec<&DeploymentTransaction> { let mut values = self .transactions @@ -3843,7 +4016,7 @@ impl Engine { // is, its own generation names the authority; an admitted Odin that // carries none has no authority to name, and falling back to the // bootstrap key would let a stale key vouch for the Verse. - let authority = match snapshot.admitted_for("odin") { + let authority = match snapshot.admitted_odin() { None => self.bootstrap_odin_authority.clone(), Some(stored) => stored.value.odin_authority.clone().with_context(|| { "admitted Odin generation carries no Odin authority (route-proof readiness)" @@ -3990,7 +4163,11 @@ impl Engine { let odin_authority = self.current_odin_authority(snapshot)?; for stored in &snapshot.admitted { let generation = &stored.value; - if generation.ready.class() == ReadinessClass::RouteProof { + // Only a generation whose own Expected says Odin vouches for it has + // Odin receipts to re-prove. One whose evidence disagrees with its + // class is held and reported by supervision, not re-proved here. + if !matches!(generation.readiness(), Ok(class) if class != ReadinessClass::RouteProof) + { continue; } let odin = generation.odin_receipts()?; @@ -4351,6 +4528,17 @@ impl Engine { { return Ok(false); } + // Evidence collected under another class is held and reported, not + // advanced and not repaired: no step can make it the right kind, and + // running one anyway only fails it every tick. An abort already under + // way does not read the class, so it is left to finish. + let aborting = current.value.completion.is_some() + || current.value.pre_fencing_abort.is_some() + || current.value.post_fencing_abort.is_some(); + if let Some(disagreement) = current.value.readiness_disagreement().filter(|_| !aborting) { + self.note_fault("holds transaction", id, &anyhow::Error::new(disagreement)); + return Ok(false); + } let advanced = self.advance_transaction(current); if advanced.is_ok() { // The step ran and succeeded: the fault, if any, has recovered, so @@ -4582,6 +4770,13 @@ impl Engine { let mut admitted = snapshot.admitted.iter().collect::>(); admitted.sort_by_key(|stored| stored.value.target.as_str()); for current in admitted { + if let Err(disagreement) = current.value.readiness() { + self.note_fault( + "holds admitted generation", + &format!("generation:{}", current.value.target), + &anyhow::Error::new(disagreement), + ); + } let blocker = snapshot.transactions.iter().find(|stored| { stored.value.target == current.value.target && stored.value.blocks_new_target_mutation() @@ -5042,7 +5237,7 @@ impl Engine { snapshot: &ControlSnapshot, current: &Stored, ) -> Result { - if current.value.ready.class() == ReadinessClass::RouteProof { + if !matches!(current.value.readiness(), Ok(class) if class != ReadinessClass::RouteProof) { return Ok(false); } let odin = current.value.odin_receipts()?; @@ -5152,6 +5347,9 @@ impl Engine { now, &providers, )?; + // Idunn infers no way to prove readiness. A recipe that declares + // none is refused here, before anything is frozen, built or installed. + plan.readiness_class()?; return self.persist_same_phase(current, |next| { next.plan = Some(plan); next.updated_at_unix_millis = now; @@ -5333,7 +5531,8 @@ impl Engine { let snapshot = ControlSnapshot::read(&self.options.state_store)?; let incumbent_lease_sha256 = self.incumbent_lease_sha256_for_warming(&snapshot, ¤t.value)?; - if current.value.warming.is_none() && ReadinessClass::of(expected) == ReadinessClass::RouteProof { + let class = ReadinessClass::of(expected)?; + if current.value.warming.is_none() && class == ReadinessClass::RouteProof { // A route-proof target's warming is Idunn's own challenge to the // candidate endpoint. No Odin is read: the target declared no // reason to be aware of one. @@ -5375,16 +5574,17 @@ impl Engine { // Asked once. Admitting the latest topology persists the sequence // cursor, so asking twice with the same record fails the second // time as "transaction changed before topology admission". - let odin_observation = if current.value.target == "odin" - && snapshot.admitted_for("odin").is_some() + let is_odin = class == ReadinessClass::OdinSelf; + let odin_observation = if is_odin + && snapshot.admitted_odin().is_some() && current.value.command_kind != CommandKind::Continuity { self.admit_latest_topology(current, None)? } else { None }; - let odin_can_observe = current.value.target != "odin" || odin_observation.is_some(); - let odin_observes_itself = current.value.target == "odin" && !odin_can_observe; + let odin_can_observe = !is_odin || odin_observation.is_some(); + let odin_observes_itself = is_odin && !odin_can_observe; if odin_observes_itself { ensure!( expected.write_lease_required, @@ -5435,7 +5635,6 @@ impl Engine { let evidence = latest.clone(); let _token = SequenceAdmittedWarming::from_topology( admitted.value.transaction_id.clone(), - evidence.clone(), authenticated, )?; return self.persist_same_phase(&admitted, |next| { @@ -5785,7 +5984,7 @@ impl Engine { "candidate process write lease is no longer exact" ); } - if ReadinessClass::of(expected) == ReadinessClass::RouteProof { + if ReadinessClass::of(expected)? == ReadinessClass::RouteProof { // Ready is the candidate answering Idunn's own challenge Active, // holding the exact lease Idunn granted. Once recorded it is // history; the stable route's challenge is what keeps proving it. @@ -5858,7 +6057,7 @@ impl Engine { let activation = required(¤t.value.activation, "activation")?; self.observe_candidate_before_waiting(¤t.value)?; if current.value.routing.is_none() { - let admitted = if ReadinessClass::of(expected) == ReadinessClass::RouteProof { + let admitted = if ReadinessClass::of(expected)? == ReadinessClass::RouteProof { // Ready is the candidate's own proof, recorded in AwaitingReady. // No Odin correlation is read to admit a route. ensure!( @@ -5986,7 +6185,7 @@ impl Engine { let route_proof = ReadinessClass::of(required( ¤t.value.expected, "Expected projection", - )?) == ReadinessClass::RouteProof; + )?)? == ReadinessClass::RouteProof; let ready_current = if route_proof { // The candidate's Ready proof is durable, and the stable route's // challenge below is its currency. No Odin is read to commit. @@ -6239,7 +6438,8 @@ impl Engine { // caller and failed the refresh. What the refresh owes is // freshness, and the replay check below is what provides it. ensure!( - current.value.target == "odin", + ReadinessClass::of(required(¤t.value.expected, "Warming Expected")?)? + == ReadinessClass::OdinSelf, "direct Warming refresh is reserved for Odin" ); let (evidence, present) = self.observe_first_odin_warming(¤t.value)?; @@ -6301,7 +6501,6 @@ impl Engine { } let token = SequenceAdmittedWarming::from_topology( admitted.value.transaction_id.clone(), - fresh_evidence.clone(), authenticated, )?; if !provider_warming_advanced( @@ -6670,20 +6869,10 @@ impl Engine { )?; let correlation = correlate_runtime_presence_claim(authenticated, authority)?; if !correlation.disagreements().is_empty() { - let named = correlation - .disagreements() - .iter() - .map(|disagreement| { - format!( - "{} (expected {}, observed {})", - disagreement.code, - disagreement.expected.as_deref().unwrap_or("none"), - disagreement.observed.as_deref().unwrap_or("none"), - ) - }) - .collect::>() - .join("; "); - bail!("route proof answered with a runtime that disagrees with current authority: {named}"); + return Err(PresenceDisagrees { + disagreements: correlation.disagreements().to_vec(), + } + .into()); } let present = correlation.into_undisputed_present()?; let presence = present.record(); @@ -6727,9 +6916,10 @@ impl Engine { // Odin ever been admitted" test only contradicted that caller. What is // still checked is what this function itself depends on -- the target // is Odin, and the incarnation is stateful. + let expected = required(&transaction.expected, "Odin Expected")?; ensure!( - transaction.target == "odin" - && required(&transaction.expected, "Odin Expected")?.write_lease_required, + ReadinessClass::of(expected)? == ReadinessClass::OdinSelf + && expected.write_lease_required, "direct Warming presence is reserved for a stateful Odin incarnation" ); ensure!( @@ -7027,7 +7217,6 @@ impl Engine { ); SequenceAdmittedWarming::from_topology( transaction.transaction_id.clone(), - evidence, authenticated, ) } @@ -7097,7 +7286,9 @@ impl Engine { /// One direct challenge to a route-proof target's candidate endpoint. /// Failing to reach or hear the candidate is only silence: a process that /// is still starting does not answer, and that is waiting, not a fault. - /// What does answer is authenticated, and a bad answer is an error. + /// What does answer is authenticated, and a bad answer is an error, as is + /// a challenge Idunn could not make: those stay a `ChallengeFailure` in the + /// error's chain. fn challenge_candidate( &self, transaction: &DeploymentTransaction, @@ -7117,16 +7308,13 @@ impl Engine { let challenged_at_unix_millis = now_millis()?; let response = match driver.request_candidate_runtime_presence(expected, &message_id) { Ok(response) => response, - Err(error) => { + Err(ChallengeFailure::Silent(error)) => { return Ok(CandidateAnswer::Silent(format!( "candidate endpoint did not answer its challenge: {error:#}" ))); } + Err(bad @ ChallengeFailure::Refused(_)) => return Err(bad.into()), }; - ensure!( - response.message_id == message_id, - "candidate transport substituted its challenge identity" - ); let admitted_at_unix_millis = now_millis()?; let authority = self.runtime_authority(transaction)?; let present = self.authenticate_challenged_presence( @@ -10009,7 +10197,7 @@ mod tests { lifted.odin_publisher_sequence_cursor, legacy.odin_publisher_sequence_cursor ); - assert_eq!(lifted.ready.class(), ReadinessClass::OdinCorrelated); + assert_eq!(lifted.ready.voucher(), Voucher::Odin); // The repair intent still decides whatever it decided. assert_eq!( lifted.route_repair_started_at_unix_millis, @@ -10176,18 +10364,36 @@ mod tests { dependency.kind == "shared-infrastructure" && dependency.capability == ODIN_RENDEZVOUS_CAPABILITY })); - assert_eq!(ReadinessClass::of(&expected), ReadinessClass::OdinCorrelated); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinCorrelated)); for dependency in &mut expected.dependencies { // Same capability, other kind: only the shared-infrastructure // declaration makes a target Odin-correlated. dependency.kind = "required".into(); } - assert_eq!(ReadinessClass::of(&expected), ReadinessClass::RouteProof); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::RouteProof)); expected.dependencies.clear(); - assert_eq!(ReadinessClass::of(&expected), ReadinessClass::RouteProof); - // Nothing to challenge: only Odin can report on an unrouted target. + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::RouteProof)); + // Nothing to challenge and no Odin dependency: the recipe has declared + // no way to prove readiness, and Idunn does not pick one. expected.route = None; - assert_eq!(ReadinessClass::of(&expected), ReadinessClass::OdinCorrelated); + assert_eq!( + ReadinessClass::of(&expected), + Err(UndeclaredReadiness { + target: expected.target.clone() + }) + ); + // Declaring the Odin dependency is the declaration, routed or not: a + // routed target that only publishes to Odin is Odin-correlated. + let odin_dependency = generation.expected.dependencies.clone(); + expected.dependencies = odin_dependency.clone(); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinCorrelated)); + expected.route = generation.expected.route.clone(); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinCorrelated)); + // Providing the rendezvous is being Odin, whatever else is declared. + expected.dependencies.clear(); + expected.route = None; + expected.capabilities[0].capability = ODIN_RENDEZVOUS_CAPABILITY.into(); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinSelf)); Ok(()) } @@ -10522,6 +10728,27 @@ mod tests { }) } + /// The declaration that makes an unrouted fixture target Odin-correlated. + /// Idunn infers no way to prove readiness, and a target with no route has + /// nothing for Idunn to challenge, so the world says outright that Odin + /// reports on it. The plan selects no provider: these worlds never plan + /// against Odin, they only read its correlation, which names the provider + /// Expected does. + fn declare_odin(expected: &mut IdunnExpectedIncarnationRecord) { + expected.dependencies.push(cultnet_rs::IdunnExpectedDependency { + kind: "shared-infrastructure".into(), + capability: ODIN_RENDEZVOUS_CAPABILITY.into(), + schema: "odin.verse-topology.v1".into(), + compatibility: "v1".into(), + minimum_capacity: 1, + startup: "before-promotion".into(), + provider_id: Some("odin".into()), + provider_authority: Some("managed-incarnation".into()), + provider_expected_projection_sha256: Some(sha256_id(b"odin expected projection")), + provider_endpoint: None, + }); + } + /// A transaction for an unrouted, stateless target, sitting at `phase` /// (Warming or later) with every earlier phase's evidence in place. The /// plan, sealed release, Expected, activation and Warming evidence are the @@ -10607,8 +10834,9 @@ mod tests { (plan, release) } }; - let expected = release.expected_projection(&plan)?; + let mut expected = release.expected_projection(&plan)?; assert!(!expected.write_lease_required && expected.route.is_none()); + declare_odin(&mut expected); let now = now_millis()?; let command = DeploymentCommand { @@ -10755,7 +10983,25 @@ mod tests { runtime_instance_id: Some(activation.runtime_instance_id.clone()), present: true, ready, - dependencies: Vec::new(), + dependencies: expected + .dependencies + .iter() + .map(|dependency| cultnet_rs::OdinRuntimeDependencyEvidence { + kind: dependency.kind.clone(), + capability: dependency.capability.clone(), + schema: dependency.schema.clone(), + compatibility: dependency.compatibility.clone(), + provider_id: dependency.provider_id.clone(), + provider_authority: dependency.provider_authority.clone(), + provider_expected_projection_sha256: dependency + .provider_expected_projection_sha256 + .clone(), + provider_endpoint: dependency.provider_endpoint.clone(), + observed_capacity: Some(dependency.minimum_capacity), + provider_evidence_sha256: Some(sha256_id(b"odin provider evidence")), + ready: true, + }) + .collect(), disagreements: Vec::new(), signer_identity_id: world.odin_signer.entry().identity_id.clone(), publisher_sequence: sequence, @@ -11029,6 +11275,10 @@ mod tests { let (_, mut odin) = fixture_generation(FIXTURE_GENERATION)?; odin.target = "odin".into(); + // Odin is the target that provides the rendezvous, whatever it is named. + odin.expected.target = "odin".into(); + odin.expected.dependencies.clear(); + odin.expected.capabilities[0].capability = ODIN_RENDEZVOUS_CAPABILITY.into(); odin.odin_authority = Some(AdmittedOdinAuthority::from_anchor( &world.odin_signer.trust_anchor()?, )?); @@ -11081,7 +11331,7 @@ mod tests { #[test] fn a_route_proof_generation_is_admitted_only_in_its_exact_shape() -> Result<()> { let generation = route_proof_generation()?; - assert_eq!(generation.ready.class(), ReadinessClass::RouteProof); + assert_eq!(generation.ready.voucher(), Voucher::Candidate); assert!(generation.odin_authority.is_none()); assert!(generation.latest_odin_observation.is_none()); assert!(generation.odin_receipts().is_err()); @@ -11247,7 +11497,7 @@ mod tests { assert_eq!(snapshot.admitted.len(), 1); let lifted = &snapshot.admitted[0].value; assert_eq!(lifted.target, envelope.key); - assert_eq!(lifted.ready.class(), ReadinessClass::OdinCorrelated); + assert_eq!(lifted.ready.voucher(), Voucher::Odin); assert!(snapshot.admitted_for(&envelope.key).is_some()); // The lifted record is held to the same key rule as a current one. @@ -13270,6 +13520,10 @@ mod tests { candidate_hits: AtomicUsize, stable_hits: AtomicUsize, stop: AtomicBool, + /// Accept the connection and close it without a word: a process + /// that is up but not answering. Unlike a closed port, no other + /// test's listener can be handed the address. + hang_up: AtomicBool, } impl RuntimeStub { @@ -13399,7 +13653,9 @@ mod tests { &stub.stable_hits }; hits.fetch_add(1, Ordering::SeqCst); - let _ = stub.serve_one(stream); + if !stub.hang_up.load(Ordering::SeqCst) { + let _ = stub.serve_one(stream); + } } Err(_) => std::thread::sleep(Duration::from_millis(5)), } @@ -13508,11 +13764,11 @@ mod tests { assert!(expected.route.is_some() && expected.dependencies.is_empty()); assert_eq!( ReadinessClass::of(&expected), - if provides_odin { + Ok(if provides_odin { ReadinessClass::OdinSelf } else { ReadinessClass::RouteProof - } + }) ); let now = now_millis()?; @@ -13572,6 +13828,7 @@ mod tests { candidate_hits: AtomicUsize::new(0), stable_hits: AtomicUsize::new(0), stop: AtomicBool::new(false), + hang_up: AtomicBool::new(false), }); stub.listen(candidate_listener, true); stub.listen(stable_listener, false); @@ -13798,7 +14055,7 @@ mod tests { let generation = admitted(&routed)?; generation.validate()?; - assert_eq!(generation.ready.class(), ReadinessClass::RouteProof); + assert_eq!(generation.ready.voucher(), Voucher::Candidate); assert!(generation.odin_authority.is_none()); assert!(generation.latest_odin_observation.is_none()); assert!(generation.route_supervision.is_some()); @@ -13827,7 +14084,7 @@ mod tests { routed.promote_by_hand()?; routed.step()?; let generation = admitted(&routed)?; - assert_eq!(generation.ready.class(), ReadinessClass::RouteProof); + assert_eq!(generation.ready.voucher(), Voucher::Candidate); // The valid, not-Ready correlation was neither admitted as a // receipt nor consulted: the record carries no Odin cursor. assert_eq!(generation.odin_publisher_sequence_cursor, 0); @@ -13847,10 +14104,9 @@ mod tests { assert_eq!(routed.stub.candidate_hits.load(Ordering::SeqCst), 1); routed.assert_odin_untouched()?; - // Nothing listens: the same step waits, and says why. + // Nothing answers: the same step waits, and says why. let silent = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; - silent.stub.stop.store(true, Ordering::SeqCst); - std::thread::sleep(Duration::from_millis(50)); + silent.stub.hang_up.store(true, Ordering::SeqCst); silent.step()?; let waiting = silent.transaction()?; assert!(waiting.warming.is_none() && waiting.pre_fencing_abort.is_none()); @@ -13885,11 +14141,11 @@ mod tests { fn odin_itself_is_never_route_proof() -> Result<()> { let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; let mut expected = routed.stub.expected.clone(); - assert_eq!(ReadinessClass::of(&expected), ReadinessClass::RouteProof); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::RouteProof)); // Odin provides the rendezvous capability and declares no dependency. expected.capabilities[0].capability = ODIN_RENDEZVOUS_CAPABILITY.into(); assert!(expected.dependencies.is_empty()); - assert_eq!(ReadinessClass::of(&expected), ReadinessClass::OdinSelf); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinSelf)); // Route-proof evidence for it is refused: its admitted generation // is what names the Odin authority, so it must carry one. @@ -13899,7 +14155,7 @@ mod tests { odin.ready = route_proof_evidence(); odin.latest_odin_observation = None; odin.odin_authority = None; - assert!(error_text(odin.validate()).contains("declares Odin")); + assert!(error_text(odin.validate()).contains("not route-proof")); Ok(()) } @@ -14024,7 +14280,7 @@ mod tests { routed.step()?; let generation = admitted(&routed)?; generation.validate()?; - assert_eq!(generation.ready.class(), ReadinessClass::OdinCorrelated); + assert_eq!(generation.ready.voucher(), Voucher::Odin); assert_eq!( generation.odin_authority, Some(routed.world.engine.bootstrap_odin_authority.clone()) diff --git a/src/deployment_plan.rs b/src/deployment_plan.rs index d4b1cc4..fc65c4d 100644 --- a/src/deployment_plan.rs +++ b/src/deployment_plan.rs @@ -10,7 +10,7 @@ use cultnet_rs::{ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; -use crate::control_plane::SequenceAdmittedReady; +use crate::control_plane::{ODIN_RENDEZVOUS_CAPABILITY, ReadinessClass, SequenceAdmittedReady}; use crate::deployment::{ CapabilityDependency, DeadlineBinding, DependencyKind, ExternalCapabilityBinding, OperatorBinding, ServiceTransport, SourceSelectionPolicy, StartupOrder, StateDeclaration, TargetDeclaration, @@ -640,6 +640,25 @@ impl CompiledDeploymentPlan { Ok((declaration, binding)) } + /// How this plan's target proves readiness, from its recipe's declarations + /// and its binding's route, or the typed refusal that it declares none. + /// Admission asks this before anything is sealed or installed. + pub(crate) fn readiness_class(&self) -> Result { + let (declaration, binding) = self.parsed_inputs()?; + Ok(ReadinessClass::declared( + &declaration.target, + declaration + .provides + .iter() + .any(|provided| provided.capability == ODIN_RENDEZVOUS_CAPABILITY), + declaration.dependencies.iter().any(|dependency| { + dependency.kind == DependencyKind::SharedInfrastructure + && dependency.capability == ODIN_RENDEZVOUS_CAPABILITY + }), + binding.route.is_some(), + )?) + } + fn recomputed_plan_id(&self) -> Result { let mut unsigned = self.clone(); unsigned.plan_id.clear(); diff --git a/src/drivers.rs b/src/drivers.rs index 44e4223..b1db416 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -5453,7 +5453,7 @@ impl NginxRouteDriver { && route.stable_endpoint == self.binding.stable_endpoint, "route challenge binding differs from Expected" ); - self.request_runtime_presence_at(expected, message_id, &route.stable_endpoint) + Ok(self.request_runtime_presence_at(expected, message_id, &route.stable_endpoint)?) } /// The bootstrap exception for the first managed Odin observes Warming on @@ -5462,16 +5462,17 @@ impl NginxRouteDriver { &self, expected: &IdunnExpectedIncarnationRecord, message_id: &str, - ) -> Result { + ) -> Result { let route = expected .route .as_ref() - .context("candidate challenge has no Expected route")?; - ensure!( + .context("candidate challenge has no Expected route") + .map_err(refused)?; + ensure_or_refuse( route.route_id == self.binding.route_id, - "candidate challenge binding differs from Expected" - ); - self.render(expected)?; + "candidate challenge binding differs from Expected", + )?; + self.render(expected).map_err(refused)?; self.request_runtime_presence_at(expected, message_id, &route.candidate_endpoint) } @@ -5480,13 +5481,14 @@ impl NginxRouteDriver { expected: &IdunnExpectedIncarnationRecord, message_id: &str, endpoint: &str, - ) -> Result { - expected.validate()?; - require_driver_id(message_id, "route challenge message")?; + ) -> Result { + expected.validate().map_err(refused)?; + require_driver_id(message_id, "route challenge message").map_err(refused)?; let route = expected .route .as_ref() - .context("route challenge has no Expected route")?; + .context("route challenge has no Expected route") + .map_err(refused)?; let request = CultNetMessage::SnapshotRequest { message_id: message_id.to_owned(), schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), @@ -5496,27 +5498,34 @@ impl NginxRouteDriver { "http" => "http://", "tcp" => "tcp://", "rudp" => "rudp://", - _ => bail!("route challenge transport is unsupported"), + _ => return Err(refused(anyhow!("route challenge transport is unsupported"))), }; - let (host, port) = endpoint_host_port(endpoint, endpoint_prefix)?; + let (host, port) = endpoint_host_port(endpoint, endpoint_prefix).map_err(refused)?; let target = SocketAddr::new( host.parse() - .context("route challenge endpoint host is not an IP address")?, + .context("route challenge endpoint host is not an IP address") + .map_err(refused)?, port, ); let response = match route.transport.as_str() { "http" => { let payload = - encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0)?; + encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0) + .map_err(refused)?; let response = request_http_snapshot(target, &payload)?; - decode_cultnet_message_from_slice(&response, CultNetWireContract::CultNetSchemaV0)? + decode_cultnet_message_from_slice(&response, CultNetWireContract::CultNetSchemaV0) + .map_err(refused)? } "tcp" => { let payload = - encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0)?; + encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0) + .map_err(refused)?; let response = request_tcp_snapshot(target, &payload)?; - decode_cultnet_message_from_slice(&response, CultNetWireContract::CultNetSchemaV0)? + decode_cultnet_message_from_slice(&response, CultNetWireContract::CultNetSchemaV0) + .map_err(refused)? } + // The RUDP client reports every failure as prose, so a bad answer + // cannot be told from no answer here: all of it reads as silence. "rudp" => request_raw_snapshot_from_rudp_catalog(CultMeshRudpSnapshotOptions { target, runtime_id: "idunn-route-observer".into(), @@ -5524,10 +5533,11 @@ impl NginxRouteDriver { schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), record_keys: Some(vec![expected.target.clone()]), ..CultMeshRudpSnapshotOptions::default() - })?, - _ => bail!("route challenge transport is unsupported"), + }) + .map_err(silent)?, + _ => return Err(refused(anyhow!("route challenge transport is unsupported"))), }; - exact_route_snapshot_response(response, message_id, &expected.target) + exact_route_snapshot_response(response, message_id, &expected.target).map_err(refused) } /// Restore the exact preflight baseline. This may only replace the exact @@ -5555,51 +5565,125 @@ impl NginxRouteDriver { } } -fn request_tcp_snapshot(target: SocketAddr, payload: &[u8]) -> Result> { +/// Why a direct presence challenge produced no document to authenticate. +#[derive(Debug)] +pub enum ChallengeFailure { + /// Nothing answered: the connection was refused, timed out, or dropped, or + /// the peer closed it without a byte. A process still starting does this, + /// and it is waiting, not a fault. + Silent(anyhow::Error), + /// Something answered wrongly, or the challenge could not be made: a + /// malformed or foreign answer, an oversized one, or a local configuration + /// error. Waiting does not fix it. + Refused(anyhow::Error), +} + +impl std::fmt::Display for ChallengeFailure { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Silent(error) | Self::Refused(error) => write!(formatter, "{error:#}"), + } + } +} + +impl std::error::Error for ChallengeFailure { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Silent(error) | Self::Refused(error) => Some(error.as_ref()), + } + } +} + +fn silent(error: impl Into) -> ChallengeFailure { + ChallengeFailure::Silent(error.into()) +} + +fn refused(error: impl Into) -> ChallengeFailure { + ChallengeFailure::Refused(error.into()) +} + +fn ensure_or_refuse(condition: bool, message: &'static str) -> Result<(), ChallengeFailure> { + if condition { + Ok(()) + } else { + Err(refused(anyhow!(message))) + } +} + +fn request_tcp_snapshot(target: SocketAddr, payload: &[u8]) -> Result, ChallengeFailure> { let mut stream = connect_route_socket(target) - .with_context(|| format!("connecting stable CultNet TCP route {target}"))?; - stream.set_read_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT))?; - stream.set_write_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT))?; - stream.write_all(&encode_frame(payload)?)?; - stream.flush()?; + .with_context(|| format!("connecting stable CultNet TCP route {target}")) + .map_err(silent)?; + stream + .set_read_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT)) + .map_err(refused)?; + stream + .set_write_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT)) + .map_err(refused)?; + stream + .write_all(&encode_frame(payload).map_err(refused)?) + .and_then(|()| stream.flush()) + .map_err(silent)?; let mut header = [0_u8; 4]; stream .read_exact(&mut header) - .context("reading stable CultNet TCP response frame")?; + .context("reading stable CultNet TCP response frame") + .map_err(silent)?; let length = u32::from_be_bytes(header) as usize; - ensure!( + ensure_or_refuse( (1..=ROUTE_SNAPSHOT_MAX_BYTES).contains(&length), - "stable CultNet TCP response exceeds the route observation bound" - ); + "stable CultNet TCP response exceeds the route observation bound", + )?; let mut response = vec![0_u8; length]; stream .read_exact(&mut response) - .context("reading stable CultNet TCP response payload")?; + .context("reading stable CultNet TCP response payload") + .map_err(silent)?; Ok(response) } -fn request_http_snapshot(target: SocketAddr, payload: &[u8]) -> Result> { +fn request_http_snapshot(target: SocketAddr, payload: &[u8]) -> Result, ChallengeFailure> { let mut stream = connect_route_socket(target) - .with_context(|| format!("connecting stable CultNet HTTP route {target}"))?; - stream.set_read_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT))?; - stream.set_write_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT))?; + .with_context(|| format!("connecting stable CultNet HTTP route {target}")) + .map_err(silent)?; + stream + .set_read_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT)) + .map_err(refused)?; + stream + .set_write_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT)) + .map_err(refused)?; let head = format!( "POST {ROUTE_HTTP_SNAPSHOT_PATH} HTTP/1.1\r\nHost: {target}\r\nContent-Type: application/msgpack\r\nAccept: application/msgpack\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", payload.len() ); - stream.write_all(head.as_bytes())?; - stream.write_all(payload)?; - stream.flush()?; + stream + .write_all(head.as_bytes()) + .and_then(|()| stream.write_all(payload)) + .and_then(|()| stream.flush()) + .map_err(silent)?; let maximum = ROUTE_HTTP_MAX_HEADER_BYTES .checked_add(ROUTE_SNAPSHOT_MAX_BYTES) - .context("route HTTP response bound overflow")?; + .context("route HTTP response bound overflow") + .map_err(refused)?; let mut response = Vec::new(); (&mut stream) .take((maximum + 1) as u64) .read_to_end(&mut response) - .context("reading stable CultNet HTTP response")?; + .context("reading stable CultNet HTTP response") + .map_err(silent)?; + if response.is_empty() { + return Err(silent(anyhow!( + "stable CultNet HTTP route closed the connection without answering" + ))); + } + parse_http_snapshot(&response, maximum).map_err(refused) +} + +/// The body of an HTTP snapshot response that arrived, or why what arrived is +/// not one. +fn parse_http_snapshot(response: &[u8], maximum: usize) -> Result> { ensure!( response.len() <= maximum, "stable CultNet HTTP response exceeds the route observation bound" @@ -10194,7 +10278,7 @@ mod tests { maximum_future_skew_millis: 5, }, )?; - let warming = SequenceAdmittedWarming::for_test("test-transaction", warming, 120)?; + let warming = SequenceAdmittedWarming::for_test("test-transaction", warming)?; Ok((expected, activation, warming, provider.trust_anchor()?)) } From e11b457458924685698d93a490477bedcb4f7634 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 13:32:42 +0200 Subject: [PATCH 032/286] B3 fix: pin the route-proof rules, and delete guards another owner already holds Tests for each rule the Soul pass found unpinned, each written to fail under its own mutation: - a stateful route-proof target through the Engine, Fencing to commit: the lease is prepared and granted, Ready is refused without the exact lease and recorded with it, boot re-authenticates the store at every stage and Odin is never read; - Warming challenges once and then advances on every tick; - a stateful candidate may not answer its Warming challenge active; - a presence holds the write lease exactly when it is not warming, and is minted after its challenge; - direct Warming evidence is bound to the bytes and digest it authenticated; - candidate-vouched evidence is refused on a target Odin reports on; - a pre-B3 transaction and a pre-B3 generation over a route-proof target are held and reported once, not stepped every tick, and not re-derived; - a stored record with no declared readiness is held; - Odin is the class OdinSelf, not the name "odin"; - admission refuses an undeclared recipe with a typed error, before the source is frozen; a recipe that declares its proof is planned; - the driver's challenge failures: silence waits, every bad answer and every local configuration error is an error; - the capacity shortfall is read as a typed PresenceDisagrees. Deleted because another owner already holds them, so no test can reach them: the Routing and commit "route-proof Ready evidence" ensures (validation refuses a missing receipt, the hold refuses the other party's), the write lease comparison's warming arm (the authenticator refuses a warming presence that claims a lease), and the direct-Warming refresh check (rehydration authenticates the recorded answer as Odin's first). Engine.source is a SourcePort, the seam the admission test needs. Co-Authored-By: Claude Sonnet 5 --- src/control_plane.rs | 1059 +++++++++++++++++++++++++++++++++++++--- src/deployment_plan.rs | 63 +++ src/drivers.rs | 273 +++++++++++ 3 files changed, 1334 insertions(+), 61 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 79e007f..8d33260 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -651,7 +651,9 @@ impl ReadinessClass { } } - fn of(expected: &IdunnExpectedIncarnationRecord) -> Result { + pub(crate) fn of( + expected: &IdunnExpectedIncarnationRecord, + ) -> Result { Self::declared( &expected.target, expected @@ -3828,7 +3830,7 @@ struct Engine { idunn_signer: ServiceIdentitySigner, idunn_anchor: ServiceIdentityTrustAnchor, bootstrap_odin_authority: AdmittedOdinAuthority, - source: GitSourceDriver, + source: Arc, docker_runner: DockerRunnerDriver, systemd_workload: Arc, host_actuators: Option, @@ -3873,11 +3875,11 @@ impl Engine { let odin_anchor = read_trust_anchor::(&options.odin_trust_anchor) .context("reading bootstrap Odin topology anchor")?; let bootstrap_odin_authority = AdmittedOdinAuthority::from_anchor(&odin_anchor)?; - let source = GitSourceDriver::new( + let source: Arc = Arc::new(GitSourceDriver::new( &options.source_root, options.staging_root.join("frozen-sources"), options.source_identity, - ); + )); let host_actuators = options .host_actuator_bind .map(|bind| HostActuatorHub::bind(bind).map(|hub| Arc::new(Mutex::new(hub)))) @@ -6059,14 +6061,9 @@ impl Engine { if current.value.routing.is_none() { let admitted = if ReadinessClass::of(expected)? == ReadinessClass::RouteProof { // Ready is the candidate's own proof, recorded in AwaitingReady. - // No Odin correlation is read to admit a route. - ensure!( - matches!( - current.value.ready, - Some(ReadinessEvidence::RouteProof { .. }) - ), - "route-proof Routing without route-proof Ready evidence" - ); + // Validation refuses Routing without a Ready receipt, and a + // receipt the other party vouched for is held before any step + // runs. No Odin correlation is read to admit a route. current.clone() } else { let current_lease = current @@ -6189,13 +6186,6 @@ impl Engine { let ready_current = if route_proof { // The candidate's Ready proof is durable, and the stable route's // challenge below is its currency. No Odin is read to commit. - ensure!( - matches!( - current.value.ready, - Some(ReadinessEvidence::RouteProof { .. }) - ), - "route-proof commit without route-proof Ready evidence" - ); current.clone() } else { let current_lease_sha256 = current @@ -6437,11 +6427,8 @@ impl Engine { // with the narrower "no admitted Odin" test contradicted the // caller and failed the refresh. What the refresh owes is // freshness, and the replay check below is what provides it. - ensure!( - ReadinessClass::of(required(¤t.value.expected, "Warming Expected")?)? - == ReadinessClass::OdinSelf, - "direct Warming refresh is reserved for Odin" - ); + // That the target is Odin was already required by the + // rehydration above, which authenticates the recorded answer. let (evidence, present) = self.observe_first_odin_warming(¤t.value)?; let warming = WarmingEvidence::FirstOdinDirect { evidence }; let token = SequenceAdmittedWarming::from_direct_presence( @@ -6890,11 +6877,11 @@ impl Engine { presence.detail == format!("route-observation:{message_id}"), "route proof response is not bound to the exact challenge" ); - let lease_is_current = if presence.state == "warming" { - presence.write_lease_sha256.is_none() - } else { - presence.write_lease_sha256.as_deref() == current_write_lease_sha256 - }; + // A warming presence is exempt from the lease comparison: it is + // Idunn's own lease that a warming candidate cannot hold yet, and the + // authenticator above already refuses one that claims a lease. + let lease_is_current = presence.state == "warming" + || presence.write_lease_sha256.as_deref() == current_write_lease_sha256; ensure!( lease_is_current, "route proof runtime does not hold the exact current process write lease" @@ -10392,7 +10379,7 @@ mod tests { // Providing the rendezvous is being Odin, whatever else is declared. expected.dependencies.clear(); expected.route = None; - expected.capabilities[0].capability = ODIN_RENDEZVOUS_CAPABILITY.into(); + provide_odin(&mut expected); assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinSelf)); Ok(()) } @@ -10728,6 +10715,16 @@ mod tests { }) } + /// What makes a target Odin: it provides the rendezvous. + fn provide_odin(expected: &mut IdunnExpectedIncarnationRecord) { + expected.capabilities.push(cultnet_rs::IdunnExpectedCapability { + capability: ODIN_RENDEZVOUS_CAPABILITY.into(), + schema: "odin.verse-topology.v1".into(), + compatibility: "v1".into(), + minimum_capacity: 1, + }); + } + /// The declaration that makes an unrouted fixture target Odin-correlated. /// Idunn infers no way to prove readiness, and a target with no route has /// nothing for Idunn to challenge, so the world says outright that Odin @@ -11278,7 +11275,7 @@ mod tests { // Odin is the target that provides the rendezvous, whatever it is named. odin.expected.target = "odin".into(); odin.expected.dependencies.clear(); - odin.expected.capabilities[0].capability = ODIN_RENDEZVOUS_CAPABILITY.into(); + provide_odin(&mut odin.expected); odin.odin_authority = Some(AdmittedOdinAuthority::from_anchor( &world.odin_signer.trust_anchor()?, )?); @@ -13494,6 +13491,142 @@ mod tests { Ok(()) } + /// A source that resolves one fixed recipe and never freezes anything. + struct FixedSource { + recipe: String, + } + + impl SourcePort for FixedSource { + fn resolve( + &self, + _binding: &OperatorBinding, + _resolution_id: &str, + _selected_at_unix_millis: u64, + ) -> Result { + Ok(crate::drivers::ResolvedSource { + facts: crate::deployment_plan::tests::source(&self.recipe), + recipe_bytes: self.recipe.clone().into_bytes(), + }) + } + + fn freeze(&self, _: &str, _: &CompiledDeploymentPlan) -> Result { + bail!("this source freezes nothing") + } + + fn observe_frozen( + &self, + _: &CompiledDeploymentPlan, + _: &FrozenSourceReceipt, + ) -> Result { + bail!("this source freezes nothing") + } + + fn cleanup(&self, _: &str, _: Option<&FrozenSourceReceipt>) -> Result<()> { + Ok(()) + } + } + + /// A Deploy transaction for an unrouted target, still Sealing, whose + /// source resolves `recipe`. One scheduler tick admits its plan or refuses it. + fn sealing_world(recipe: &str) -> Result { + use crate::deployment_plan::tests::BINDING; + let mut world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (binding_head, binding_tail) = BINDING + .split_once("[route]") + .context("binding has no route table")?; + let binding = format!( + "{binding_head}[brakes]{}", + binding_tail + .split_once("[brakes]") + .context("binding has no brakes table")? + .1 + ); + std::fs::create_dir_all(world.root.join("bindings"))?; + std::fs::write(world.root.join("bindings/service.toml"), binding)?; + world.engine.source = Arc::new(FixedSource { + recipe: recipe.to_owned(), + }); + let now = now_millis()?; + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "up-service".into(), + kind: CommandKind::Deploy, + selector: "service".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + let transaction = DeploymentTransaction::new(&command, "service".into(), 0, None, now)?; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: transaction.transaction_id.clone(), + current: None, + }, + ], + &[ + command_envelope(&command, command.requested_at_unix_millis)?, + transaction_envelope(&transaction, now)?, + ], + )? + ); + Ok(world) + } + + fn unrouted_recipe() -> String { + crate::deployment_plan::tests::RECIPE + .split("[[dependencies]]") + .next() + .expect("the recipe has a body") + .replace("route_required = true", "route_required = false") + .replace( + r#"["GAMECULT_IDUNN_CANDIDATE_BIND", "GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, + r#"["GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, + ) + } + + #[test] + fn admission_refuses_a_recipe_that_declares_no_way_to_prove_readiness() -> Result<()> { + let recipe = unrouted_recipe(); + let world = sealing_world(&recipe)?; + world.engine.run_scheduler_tick()?; + let refused = latest(&world)?; + assert!(refused.plan.is_none(), "the refused recipe was planned"); + assert!(refused.frozen_source.is_none(), "the refused recipe was frozen"); + let abort = refused + .pre_fencing_abort + .context("a recipe with no declared proof of readiness must be refused")?; + assert!( + abort.error.contains("target service declares no way to prove readiness"), + "{}", + abort.error + ); + Ok(()) + } + + #[test] + fn admission_takes_a_recipe_that_declares_its_proof() -> Result<()> { + let recipe = unrouted_recipe().replace( + "capability = \"service.runtime\"", + &format!("capability = \"{ODIN_RENDEZVOUS_CAPABILITY}\""), + ); + let world = sealing_world(&recipe)?; + world.engine.run_scheduler_tick()?; + let planned = latest(&world)?; + assert!(planned.pre_fencing_abort.is_none(), "{:?}", planned.pre_fencing_abort); + assert_eq!( + planned.plan.as_ref().map(CompiledDeploymentPlan::readiness_class).transpose()?, + Some(ReadinessClass::OdinSelf) + ); + Ok(()) + } + // --------------------------------------------------------------------- // Route-proof readiness: a routed target that declares no Odin dependency // is admitted by Idunn's own challenges. Odin is never read. @@ -13524,6 +13657,20 @@ mod tests { /// that is up but not answering. Unlike a closed port, no other /// test's listener can be handed the address. hang_up: AtomicBool, + /// The write lease its presence claims to hold: what a candidate + /// that picked up Idunn's grant reports. + lease: Mutex>, + reply: Mutex, + } + + /// How the runtime answers a well-formed challenge. + #[derive(Clone, Copy, PartialEq, Eq)] + enum Reply { + Honest, + /// An answer to some other challenge. + ForeignChallengeId, + /// An HTTP error where the snapshot should be. + HttpFailure, } impl RuntimeStub { @@ -13558,7 +13705,7 @@ mod tests { health_contract: expected.health_contract.clone(), state: (*self.state.lock().unwrap()).into(), detail: format!("route-observation:{message_id}"), - write_lease_sha256: None, + write_lease_sha256: self.lease.lock().unwrap().clone(), signer_identity_id: self.provider.entry().identity_id.clone(), publisher_sequence: self.sequence.fetch_add(1, Ordering::SeqCst) + 1, observed_at_unix_millis: now_millis()?, @@ -13610,8 +13757,19 @@ mod tests { else { bail!("stub received something other than a snapshot request"); }; + let reply = *self.reply.lock().unwrap(); + if reply == Reply::HttpFailure { + stream.write_all( + b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + )?; + return Ok(()); + } let response = cultnet_rs::CultNetMessage::SnapshotResponseRaw { - message_id: message_id.clone(), + message_id: if reply == Reply::ForeignChallengeId { + "candidate-foreign".to_owned() + } else { + message_id.clone() + }, documents: vec![cultnet_rs::CultNetRawDocumentRecord { schema_id: cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), record_key: self.expected.target.clone(), @@ -13695,6 +13853,23 @@ mod tests { minimum_capacity: u32, phase: DeploymentPhase, provides_odin: bool, + ) -> Result { + build_routed_world(odin, minimum_capacity, phase, provides_odin, false) + } + + /// The same world for a target with process-writable state: its + /// candidate is granted a write lease and Ready must be the candidate + /// reporting exactly that lease. + fn stateful_routed_world(odin: Odin, phase: DeploymentPhase) -> Result { + build_routed_world(odin, 1, phase, false, true) + } + + fn build_routed_world( + odin: Odin, + minimum_capacity: u32, + phase: DeploymentPhase, + provides_odin: bool, + stateful: bool, ) -> Result { use crate::deployment_plan::tests::{ BINDING, RECIPE, artifact_receipt, external_input_receipt, source, @@ -13730,7 +13905,30 @@ mod tests { "/etc/nginx/idunn-stream-routes/service.conf", &config.display().to_string().replace('\\', "/"), ); - let recipe = RECIPE + let (binding, recipe_source) = if stateful { + std::fs::create_dir_all(world.root.join("lease"))?; + ( + binding.replace( + "[brakes]", + &format!( + "[process_write_lease]\nrecord_path = \"{}\"\n\n[brakes]", + world.root.join("lease/process-write-lease.cc").display() + ), + ), + RECIPE + .replace( + "writer = \"none\"\nrecovery = \"rebuildable\"\nstartup = \"open-at-start\"", + "writer = \"process-bound-single-writer\"\nrecovery = \"preserve\"\nstartup = \"create-or-open-after-write-lease\"", + ) + .replace( + "required_environment = [\"GAMECULT_IDUNN_CANDIDATE_BIND\", \"GAMECULT_IDUNN_RUNTIME_BUNDLE\"]", + "required_environment = [\"GAMECULT_IDUNN_CANDIDATE_BIND\", \"GAMECULT_IDUNN_PROCESS_WRITE_LEASE\", \"GAMECULT_IDUNN_RUNTIME_BUNDLE\"]", + ), + ) + } else { + (binding, RECIPE.to_owned()) + }; + let recipe = recipe_source .split("[[dependencies]]") .next() .context("recipe is empty")? @@ -13762,6 +13960,7 @@ mod tests { )?; let expected = release.expected_projection(&plan)?; assert!(expected.route.is_some() && expected.dependencies.is_empty()); + assert_eq!(expected.write_lease_required, stateful); assert_eq!( ReadinessClass::of(&expected), Ok(if provides_odin { @@ -13829,6 +14028,8 @@ mod tests { stable_hits: AtomicUsize::new(0), stop: AtomicBool::new(false), hang_up: AtomicBool::new(false), + lease: Mutex::new(None), + reply: Mutex::new(Reply::Honest), }); stub.listen(candidate_listener, true); stub.listen(stable_listener, false); @@ -13859,30 +14060,20 @@ mod tests { transaction.warming = Some(WarmingEvidence::RouteProofDirect { evidence }); } if phase >= DeploymentPhase::Fencing { - let rendered = NginxRouteDriver::new( - transaction - .plan - .as_ref() - .unwrap() - .parsed_inputs()? - .1 - .route - .context("no route binding")?, - ) - .render(transaction.expected.as_ref().unwrap())?; - transaction.route_preflight = Some(crate::drivers::RoutePreflightReceipt { - route_id: "service".into(), - candidate_runtime_instance_id: transaction - .activation - .as_ref() - .unwrap() - .runtime_instance_id - .clone(), - candidate_membership_sha256: sha256_id(&rendered), - incumbent_runtime_instance_id: None, - incumbent_membership_sha256: None, - incumbent_configuration: None, - }); + transaction.route_preflight = Some(candidate_preflight(&transaction)?); + } + if stateful { + // The lease driver reads the Expected and the observed + // activation from the topology store, as Starting left them. + let plan = transaction.plan.as_ref().unwrap(); + let anchor = world.engine.provider_anchor_for_plan(plan)?; + let topology = world.engine.topology(); + topology.publish_expected(transaction.expected.as_ref().unwrap(), &anchor)?; + topology.publish_observed_activation( + transaction.expected.as_ref().unwrap(), + transaction.activation.as_ref().unwrap(), + transaction.workload.as_ref().unwrap(), + )?; } transaction.enter_phase(phase, now); transaction.validate()?; @@ -13947,6 +14138,38 @@ mod tests { }) } + /// The preflight receipt of a candidate whose route was never + /// installed: what Warming records before Fencing, without running the + /// host's nginx. + fn candidate_preflight( + transaction: &DeploymentTransaction, + ) -> Result { + let rendered = NginxRouteDriver::new( + transaction + .plan + .as_ref() + .unwrap() + .parsed_inputs()? + .1 + .route + .context("no route binding")?, + ) + .render(transaction.expected.as_ref().unwrap())?; + Ok(crate::drivers::RoutePreflightReceipt { + route_id: "service".into(), + candidate_runtime_instance_id: transaction + .activation + .as_ref() + .unwrap() + .runtime_instance_id + .clone(), + candidate_membership_sha256: sha256_id(&rendered), + incumbent_runtime_instance_id: None, + incumbent_membership_sha256: None, + incumbent_configuration: None, + }) + } + impl RoutedWorld { fn step(&self) -> Result<()> { self.world.engine.advance_transaction(&resident(&self.world)?) @@ -14128,11 +14351,21 @@ mod tests { routed.run_to_routing()?; routed.promote_by_hand()?; routed.stub.capacity.store(1, Ordering::SeqCst); - let failure = format!("{:#}", routed.step().expect_err("a shortfall must be refused")); + let failure = routed.step().expect_err("a shortfall must be refused"); + // The shortfall is a typed value, not a sentence to search. + let disagrees = failure + .downcast_ref::() + .with_context(|| format!("the refusal is not typed: {failure:#}"))?; + let shortfall = disagrees + .disagreements + .iter() + .find(|disagreement| disagreement.code == "expected-capability-000-capacity") + .context("the capacity shortfall is not named")?; assert!( - failure.contains("expected-capability-000-capacity"), - "{failure}" + shortfall.expected.as_deref().is_some_and(|text| text.ends_with("capacity>=2")), + "{shortfall:?}" ); + assert!(shortfall.observed.is_some(), "{shortfall:?}"); assert!(admitted(&routed).is_err()); Ok(()) } @@ -14308,5 +14541,709 @@ mod tests { ); Ok(()) } + + impl RoutedWorld { + /// Boot would accept this store, and Odin was never read. + fn assert_boots(&self) -> Result<()> { + let snapshot = ControlSnapshot::read(&self.world.state_store)?; + self.world.engine.validate_durable_authority(&snapshot)?; + self.assert_odin_untouched() + } + + /// Step until `done`, checking after every step that boot would + /// still accept the store. + fn drive_until(&self, done: impl Fn(&DeploymentTransaction) -> bool) -> Result<()> { + for _ in 0..20 { + if done(&self.transaction()?) { + return Ok(()); + } + self.step()?; + self.assert_boots()?; + } + bail!("the transaction never reached the awaited state") + } + + fn answer_now(&self) -> Result<(RuntimePresenceEvidence, cultnet_rs::VerifiedRuntimePresence)> { + match self.world.engine.challenge_candidate( + &self.transaction()?, + &["warming", "active"], + None, + )? { + CandidateAnswer::Answered { evidence, present } => Ok((evidence, present)), + CandidateAnswer::Silent(reason) => bail!("the candidate was silent: {reason}"), + } + } + } + + // ----------------------------------------------------------------- + // The stateful path, end to end through the Engine. + // ----------------------------------------------------------------- + + #[test] + fn a_stateful_route_proof_target_holds_its_lease_from_fencing_to_commit() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.assert_boots()?; + + routed.drive_until(|t| matches!(t.leasing, Some(LeasingEvidence::Prepared { .. })))?; + assert_eq!(routed.transaction()?.phase, DeploymentPhase::Leasing); + routed.drive_until(|t| matches!(t.leasing, Some(LeasingEvidence::Granted { .. })))?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let lease_sha256 = routed + .transaction()? + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256) + .context("no granted lease")? + .to_owned(); + + // The candidate has not picked the lease up: still warming. + routed.step()?; + assert_eq!( + routed.transaction()?.last_error.as_deref(), + Some("candidate is still warming") + ); + assert!(routed.transaction()?.ready.is_none()); + // Active without the lease Idunn granted is not Ready. + routed.stub.set_state("active"); + let refusal = routed + .step() + .expect_err("an active candidate without the granted lease is refused"); + assert!( + format!("{refusal:#}").contains("exact current process write lease"), + "{refusal:#}" + ); + assert!(routed.transaction()?.ready.is_none()); + // Active holding exactly that lease is. + *routed.stub.lease.lock().unwrap() = Some(lease_sha256.clone()); + routed.step()?; + assert!(matches!( + routed.transaction()?.ready, + Some(ReadinessEvidence::RouteProof { .. }) + )); + routed.assert_boots()?; + + routed.drive_until(|t| t.phase == DeploymentPhase::Routing)?; + routed.promote_by_hand()?; + routed.assert_boots()?; + routed.step()?; + + let generation = admitted(&routed)?; + generation.validate()?; + assert_eq!(generation.ready.voucher(), Voucher::Candidate); + assert!(generation.expected.write_lease_required); + assert_eq!(generation.leasing.lease_sha256(), Some(lease_sha256.as_str())); + assert!(generation.odin_authority.is_none()); + routed.assert_boots()?; + Ok(()) + } + + // ----------------------------------------------------------------- + // Warming is one challenge, then the phase moves. + // ----------------------------------------------------------------- + + #[test] + fn warming_challenges_once_and_then_advances_on_every_tick() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + routed.step()?; + let recorded = routed + .transaction()? + .warming + .context("the first step recorded no Warming evidence")?; + assert_eq!(routed.stub.candidate_hits.load(Ordering::SeqCst), 1); + + // The route preflight needs the host's nginx; its receipt is recorded + // here so the phase can move on. + let current = resident(&routed.world)?; + let mut next = current.value.clone(); + next.route_preflight = Some(candidate_preflight(&next)?); + replace_transaction(&routed.world.state_store, ¤t, &next)?; + + routed.drive_until(|t| t.phase == DeploymentPhase::Fencing)?; + assert_eq!( + routed.stub.candidate_hits.load(Ordering::SeqCst), + 1, + "a recorded Warming answer is never asked for again" + ); + assert_eq!(routed.transaction()?.warming, Some(recorded)); + Ok(()) + } + + #[test] + fn a_stateful_candidate_may_not_answer_its_warming_challenge_active() { + let mut expected = fixture_generation(FIXTURE_GENERATION).unwrap().1.expected; + expected.write_lease_required = true; + assert_eq!(route_proof_warming_states(&expected), ["warming"]); + expected.write_lease_required = false; + assert_eq!(route_proof_warming_states(&expected), ["warming", "active"]); + } + + #[test] + fn a_stateful_candidate_that_answers_warming_active_is_refused() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Warming)?; + routed.stub.set_state("active"); + let refusal = routed + .step() + .expect_err("a stateful candidate cannot be active before it holds a lease"); + assert!(format!("{refusal:#}").contains("not warming"), "{refusal:#}"); + assert!(routed.transaction()?.warming.is_none()); + Ok(()) + } + + // ----------------------------------------------------------------- + // What a challenged presence must prove. + // ----------------------------------------------------------------- + + #[test] + fn a_presence_holds_the_write_lease_exactly_when_it_is_not_warming() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Warming)?; + let authority = routed + .world + .engine + .runtime_authority(&routed.transaction()?)?; + let lease = sha256_id(b"the granted lease"); + let other = sha256_id(b"some other lease"); + let judge = |state: &'static str, + held: Option<&str>, + current: Option<&str>, + states: &[&str]| + -> Result<()> { + let challenged = now_millis()?; + routed.stub.set_state(state); + *routed.stub.lease.lock().unwrap() = held.map(str::to_owned); + let bytes = routed.stub.answer("candidate-probe")?; + routed + .world + .engine + .authenticate_challenged_presence( + &authority, + states, + current, + "candidate-probe", + challenged, + now_millis()?, + &bytes, + ) + .map(|_| ()) + }; + let either = ["warming", "active"]; + judge("warming", None, None, &either)?; + judge("active", Some(&lease), Some(&lease), &either)?; + for (state, held, current, named) in [ + // Warming holds nothing, whatever Idunn has granted. + ("warming", Some(lease.as_str()), None, "write lease"), + ("warming", Some(lease.as_str()), Some(lease.as_str()), "write lease"), + // Anything else holds exactly the current grant. + ("active", None, Some(lease.as_str()), "exact current process write lease"), + ("active", Some(lease.as_str()), None, "exact current process write lease"), + ( + "active", + Some(other.as_str()), + Some(lease.as_str()), + "exact current process write lease", + ), + ] { + let error = judge(state, held, current, &either) + .expect_err("a presence with the wrong lease is refused"); + assert!( + format!("{error:#}").contains(named), + "{state} {held:?} {current:?}: {error:#}" + ); + } + Ok(()) + } + + #[test] + fn a_presence_minted_before_its_challenge_is_refused() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let authority = routed + .world + .engine + .runtime_authority(&routed.transaction()?)?; + let bytes = routed.stub.answer("candidate-probe")?; + let minted = now_millis()?; + // Answered before the challenge that claims it: an old presence + // replayed for a new question. + let replay = routed + .world + .engine + .authenticate_challenged_presence( + &authority, + &["warming"], + None, + "candidate-probe", + minted + 1_000, + minted + 1_000, + &bytes, + ) + .expect_err("a presence minted before its challenge"); + assert!( + format!("{replay:#}").contains("minted before its challenge"), + "{replay:#}" + ); + let early = routed + .world + .engine + .authenticate_challenged_presence( + &authority, + &["warming"], + None, + "candidate-probe", + minted + 1_000, + minted, + &bytes, + ) + .expect_err("a receipt before its challenge"); + assert!(format!("{early:#}").contains("predates its challenge"), "{early:#}"); + Ok(()) + } + + #[test] + fn direct_warming_evidence_is_bound_to_the_bytes_it_authenticated() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let id = routed.transaction()?.transaction_id; + let direct = |evidence: RuntimePresenceEvidence| WarmingEvidence::RouteProofDirect { evidence }; + + let (evidence, present) = routed.answer_now()?; + SequenceAdmittedWarming::from_direct_presence(id.clone(), direct(evidence), present)?; + + // The digest is the authenticated presence's, but the bytes are not. + let (mut evidence, present) = routed.answer_now()?; + evidence.canonical_bytes.push(0); + let error = SequenceAdmittedWarming::from_direct_presence(id.clone(), direct(evidence), present) + .expect_err("bytes that are not the authenticated presence"); + assert!(format!("{error:#}").contains("differs from its authenticated"), "{error:#}"); + + // The bytes are the authenticated presence's, but the digest is not. + let (mut evidence, present) = routed.answer_now()?; + evidence.canonical_sha256 = sha256_id(b"another presence"); + let error = SequenceAdmittedWarming::from_direct_presence(id.clone(), direct(evidence), present) + .expect_err("a digest that is not the authenticated presence's"); + assert!(format!("{error:#}").contains("differs from its authenticated"), "{error:#}"); + + // Evidence about another answer altogether. + let (other, _) = routed.answer_now()?; + let (_, present) = routed.answer_now()?; + SequenceAdmittedWarming::from_direct_presence(id, direct(other), present) + .expect_err("evidence about another answer"); + Ok(()) + } + + // ----------------------------------------------------------------- + // Validation: candidate-vouched evidence belongs to route-proof targets. + // ----------------------------------------------------------------- + + #[test] + fn candidate_vouched_evidence_is_refused_on_a_target_odin_reports_on() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let (evidence, _) = routed.answer_now()?; + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let odin_reported = transaction_at(&world, DeploymentPhase::Warming)?; + assert_eq!( + ReadinessClass::of(odin_reported.expected.as_ref().unwrap()), + Ok(ReadinessClass::OdinCorrelated) + ); + + let mut warming = odin_reported.clone(); + warming.warming = Some(WarmingEvidence::RouteProofDirect { + evidence: evidence.clone(), + }); + assert!(error_text(warming.validate()).contains("not route-proof")); + + let mut ready = odin_reported.clone(); + ready.ready = Some(ReadinessEvidence::RouteProof { + evidence: evidence.clone(), + }); + assert!(error_text(ready.validate()).contains("not route-proof")); + + // Direct Odin warming is Odin's alone, by class: a route-proof + // target may not claim it, and a target that provides the + // rendezvous may, whatever it is named. + let mut direct = routed.transaction()?; + direct.warming = Some(WarmingEvidence::FirstOdinDirect { + evidence: evidence.clone(), + }); + assert!(error_text(direct.validate()).contains("reserved for Odin")); + let odin = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, true)?; + let mut direct = odin.transaction()?; + assert_eq!(direct.target, "service"); + direct.warming = Some(WarmingEvidence::FirstOdinDirect { evidence }); + direct.validate()?; + Ok(()) + } + + #[test] + fn a_route_proof_transaction_past_awaiting_ready_needs_the_candidates_ready_proof() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + let routing = routed.transaction()?; + routing.validate()?; + let mut bare = routing.clone(); + bare.ready = None; + assert!(error_text(bare.validate()).contains("lacks Ready evidence")); + routed.promote_by_hand()?; + let mut bare = routed.transaction()?; + assert_eq!(bare.phase, DeploymentPhase::Committing); + bare.ready = None; + assert!(error_text(bare.validate()).contains("lacks Ready evidence")); + Ok(()) + } + + // ----------------------------------------------------------------- + // The class is the Expected's alone. Evidence collected under another + // class is held and reported, never stepped, never repaired. + // ----------------------------------------------------------------- + + /// Odin's Ready for the world's target, as pre-B3 code recorded it for + /// every routed target. + fn odin_receipt_for( + routed: &RoutedWorld, + transaction: &DeploymentTransaction, + sequence: u64, + ) -> Result { + let now = now_millis()?; + let bytes = signed_correlation(&routed.world, transaction, sequence, true)?; + let authenticated = routed.world.engine.authenticate_topology_bytes( + &ControlSnapshot::read(&routed.world.state_store)?, + transaction, + &bytes, + transaction + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256), + now, + )?; + TopologyEvidence::from_authenticated(&authenticated, now) + } + + fn stored(generation: &AdmittedGeneration) -> Stored { + Stored { + value: generation.clone(), + envelope: CultCacheEnvelope { + key: generation.target.clone(), + r#type: AdmittedGeneration::TYPE.into(), + payload: Vec::new(), + stored_at: "1970-01-01T00:00:00.100Z".into(), + schema_id: Some(ADMITTED_GENERATION_SCHEMA.into()), + }, + } + } + + #[test] + fn a_pre_b3_transaction_over_a_route_proof_target_is_held_and_reported_once() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + + // Ready as pre-B3 Idunn wrote it: an Odin receipt. + let current = resident(&routed.world)?; + let mut legacy = current.value.clone(); + let receipt = odin_receipt_for(&routed, &legacy, 9)?; + legacy.latest_odin_observation = Some(receipt.clone()); + legacy.odin_publisher_sequence_cursor = 9; + legacy.ready = Some(ReadinessEvidence::OdinCorrelated { evidence: receipt }); + replace_transaction(&routed.world.state_store, ¤t, &legacy)?; + + let disagreement = ReadinessDisagreement::WrongVoucher { + target: "service".into(), + required: ReadinessClass::RouteProof, + collected: Voucher::Odin, + }; + assert_eq!(legacy.readiness_disagreement(), Some(disagreement.clone())); + // The record is readable and boot accepts it: held, not refused. + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + routed.world.engine.validate_durable_authority(&snapshot)?; + // Stepping it would fail, on every tick, for good. + assert!(routed.step().is_err()); + + let hits = ( + routed.stub.candidate_hits.load(Ordering::SeqCst), + routed.stub.stable_hits.load(Ordering::SeqCst), + ); + for _ in 0..4 { + assert!(!routed.world.engine.run_scheduler_tick()?); + } + let held = resident(&routed.world)?; + assert_eq!(held.value.phase, DeploymentPhase::Routing); + assert!(held.value.pre_fencing_abort.is_none()); + assert!(held.value.post_fencing_abort.is_none()); + assert_eq!(held.value.ready, legacy.ready, "nothing re-derived the evidence"); + assert_eq!( + held.value.last_error.as_deref(), + Some(disagreement.to_string().as_str()) + ); + // Reported once: later ticks leave the record alone. + routed.world.engine.run_scheduler_tick()?; + assert_eq!(resident(&routed.world)?.envelope, held.envelope); + assert_eq!( + hits, + ( + routed.stub.candidate_hits.load(Ordering::SeqCst), + routed.stub.stable_hits.load(Ordering::SeqCst) + ), + "a held transaction is asked nothing" + ); + routed.assert_odin_untouched()?; + + // An abort already under way is not held: it never reads the class. + let mut aborting = held.value.clone(); + aborting.post_fencing_abort = Some(post_fencing_abort_intent(&aborting, "operator abort")); + replace_transaction(&routed.world.state_store, &held, &aborting)?; + let after = resident(&routed.world)?; + let _ = routed.world.engine.resume_candidate(&after); + assert_ne!( + resident(&routed.world)?.value.last_error.as_deref(), + Some(disagreement.to_string().as_str()), + "an abort in progress runs instead of being held" + ); + Ok(()) + } + + #[test] + fn a_pre_b3_generation_over_a_route_proof_target_boots_and_is_reported_not_refreshed() + -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + let at_routing = routed.transaction()?; + routed.promote_by_hand()?; + routed.step()?; + + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + let current = snapshot.admitted.first().context("nothing was admitted")?; + let receipt = odin_receipt_for(&routed, &at_routing, 9)?; + let mut legacy = current.value.clone(); + legacy.ready = ReadinessEvidence::OdinCorrelated { + evidence: receipt.clone(), + }; + legacy.latest_odin_observation = Some(receipt); + // Receipts that would not authenticate if they were re-proved: the + // authority is another key's. Held generations are not re-proved. + let stranger = cultnet_rs::enroll_service_identity_at::( + &routed.world.root.join("identities/stranger.cc"), + )?; + legacy.odin_authority = Some(AdmittedOdinAuthority::from_anchor(&stranger.trust_anchor()?)?); + legacy.odin_publisher_sequence_cursor = 9; + assert!( + SingleFileMessagePackBackingStore::new(&routed.world.state_store) + .compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: AdmittedGeneration::TYPE.into(), + key: legacy.target.clone(), + current: Some(current.envelope.clone()), + }], + &[admitted_envelope(&legacy, now_millis()?)?], + )? + ); + + assert_eq!( + legacy.readiness(), + Err(ReadinessDisagreement::WrongVoucher { + target: "service".into(), + required: ReadinessClass::RouteProof, + collected: Voucher::Odin, + }) + ); + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + let current = snapshot.admitted.first().context("the generation is gone")?; + // The class chooses the path: no Odin receipt is re-proved at boot + // and none is refreshed, though the evidence is an Odin receipt. + routed.world.engine.validate_durable_authority(&snapshot)?; + assert!(!routed.world.engine.refresh_admitted_topology(&snapshot, current)?); + routed.assert_odin_untouched()?; + + routed.world.engine.supervise_one_admitted_generation()?; + assert!( + routed + .world + .engine + .fault_reports + .lock() + .unwrap() + .contains_key("generation:service"), + "supervision reports the disagreement" + ); + routed.assert_odin_untouched()?; + Ok(()) + } + + #[test] + fn a_stored_record_that_declares_no_way_to_prove_readiness_is_held() -> Result<()> { + let world = EngineFixture::new()?; + // Odin's store is unreadable: any read of it is an error. + std::fs::write(&world.engine.options.odin_correlation_store, b"odin is not a cache")?; + + let (_, mut generation) = fixture_generation(FIXTURE_GENERATION)?; + let target = generation.expected.target.clone(); + assert_eq!(generation.readiness(), Ok(ReadinessClass::OdinCorrelated)); + generation.expected.dependencies.clear(); + generation.expected.route = None; + let undeclared = UndeclaredReadiness { target: target.clone() }; + assert_eq!( + generation.readiness(), + Err(ReadinessDisagreement::Undeclared(undeclared.clone())) + ); + assert!( + !world + .engine + .refresh_admitted_topology(&ControlSnapshot::default(), &stored(&generation))? + ); + + // The mirror: Odin declared, but the receipt is the candidate's. + let (_, mut odin_class) = fixture_generation(FIXTURE_GENERATION)?; + odin_class.ready = route_proof_evidence(); + assert_eq!( + odin_class.readiness(), + Err(ReadinessDisagreement::WrongVoucher { + target: target.clone(), + required: ReadinessClass::OdinCorrelated, + collected: Voucher::Candidate, + }) + ); + + // A transaction says the same about what it has collected. + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let (evidence, _) = routed.answer_now()?; + let odin_world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = transaction_at(&odin_world, DeploymentPhase::Warming)?; + assert_eq!(seeded.readiness_disagreement(), None); + let mut collected = seeded.clone(); + collected.warming = Some(WarmingEvidence::RouteProofDirect { evidence }); + assert_eq!( + collected.readiness_disagreement(), + Some(ReadinessDisagreement::WrongVoucher { + target: "service".into(), + required: ReadinessClass::OdinCorrelated, + collected: Voucher::Candidate, + }) + ); + let mut bare = seeded.clone(); + bare.expected.as_mut().unwrap().dependencies.clear(); + assert_eq!( + bare.readiness_disagreement(), + Some(ReadinessDisagreement::Undeclared(UndeclaredReadiness { + target: "service".into() + })) + ); + let mut no_expected = seeded; + no_expected.expected = None; + assert_eq!(no_expected.readiness_disagreement(), None); + Ok(()) + } + + // ----------------------------------------------------------------- + // Odin is whichever target provides the rendezvous. + // ----------------------------------------------------------------- + + #[test] + fn odin_is_the_target_that_provides_the_rendezvous_not_the_target_named_odin() -> Result<()> { + let world = EngineFixture::new()?; + let (_, fixture) = fixture_generation(FIXTURE_GENERATION)?; + + // A target named "odin" that merely depends on Odin is not Odin. + let mut named = fixture.clone(); + named.target = "odin".into(); + named.expected.target = "odin".into(); + let mut snapshot = ControlSnapshot::default(); + snapshot.admitted.push(stored(&named)); + assert!(snapshot.admitted_odin().is_none()); + assert_eq!( + world.engine.current_odin_authority(&snapshot)?, + world.engine.bootstrap_odin_authority + ); + + // The target that provides the rendezvous is, whatever it is called. + let mut provider = fixture.clone(); + provider.target = "verse".into(); + provider.expected.target = "verse".into(); + provider.expected.dependencies.clear(); + provide_odin(&mut provider.expected); + let authority = AdmittedOdinAuthority::from_anchor(&world.odin_signer.trust_anchor()?)?; + provider.odin_authority = Some(authority.clone()); + snapshot.admitted.push(stored(&provider)); + assert_eq!( + snapshot.admitted_odin().map(|found| found.value.target.as_str()), + Some("verse") + ); + assert_eq!( + snapshot.admitted_odin().and_then(|found| found.value.odin_authority.clone()), + Some(authority) + ); + Ok(()) + } + + #[test] + fn a_target_that_provides_the_rendezvous_warms_as_odin_whatever_it_is_named() -> Result<()> { + // Nothing else has been admitted that could observe it, so Odin + // observes itself directly -- and only a stateful incarnation may. + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, true)?; + assert_eq!(routed.transaction()?.target, "service"); + let refusal = routed + .step() + .expect_err("a stateless Odin cannot be observed directly"); + assert!( + format!("{refusal:#}").contains("direct Odin warming must be a stateful incarnation"), + "{refusal:#}" + ); + Ok(()) + } + + #[test] + fn direct_odin_warming_is_reserved_to_odin_by_class_not_by_name() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let mut named = routed.transaction()?; + named.target = "odin".into(); + let error = routed + .world + .engine + .authenticate_first_odin_warming_presence(&named, "warming-probe", 1, 2, &[1]) + .expect_err("a target named odin that is not Odin"); + assert!( + format!("{error:#}").contains("reserved for a stateful Odin incarnation"), + "{error:#}" + ); + Ok(()) + } + + // ----------------------------------------------------------------- + // A bad answer is an error. Only silence waits. + // ----------------------------------------------------------------- + + #[test] + fn a_bad_answer_to_a_challenge_is_an_error_and_only_silence_waits() -> Result<()> { + for reply in [Reply::ForeignChallengeId, Reply::HttpFailure] { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + *routed.stub.reply.lock().unwrap() = reply; + let error = match routed.world.engine.challenge_candidate( + &routed.transaction()?, + &["warming"], + None, + ) { + Err(error) => error, + Ok(_) => bail!("a bad answer was taken for silence"), + }; + assert!( + matches!( + error.downcast_ref::(), + Some(ChallengeFailure::Refused(_)) + ), + "{error:#}" + ); + // Before the fence a candidate that answers wrongly is aborted, + // not waited on. + routed.world.engine.run_scheduler_tick()?; + let after = routed.transaction()?; + assert!(after.pre_fencing_abort.is_some(), "the bad answer aborted it"); + assert!(after.warming.is_none()); + } + + let silent = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + silent.stub.hang_up.store(true, Ordering::SeqCst); + assert!(matches!( + silent.world.engine.challenge_candidate(&silent.transaction()?, &["warming"], None)?, + CandidateAnswer::Silent(_) + )); + Ok(()) + } } } diff --git a/src/deployment_plan.rs b/src/deployment_plan.rs index fc65c4d..7169601 100644 --- a/src/deployment_plan.rs +++ b/src/deployment_plan.rs @@ -1566,6 +1566,69 @@ nodes = ["yggdrasil"] ); } + #[test] + fn a_plan_names_how_its_target_proves_readiness_and_agrees_with_its_expected() { + use crate::control_plane::UndeclaredReadiness; + let unrouted_binding = { + let (head, tail) = BINDING.split_once("[route]").unwrap(); + format!("{head}[brakes]{}", tail.split_once("[brakes]").unwrap().1) + }; + let unrouted = |recipe: &str| { + recipe + .replace("route_required = true", "route_required = false") + .replace( + r#"["GAMECULT_IDUNN_CANDIDATE_BIND", "GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, + r#"["GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, + ) + }; + let without_odin = RECIPE.split("[[dependencies]]").next().unwrap().to_owned(); + let provides_odin = |recipe: &str| { + recipe.replace("capability = \"service.runtime\"", "capability = \"odin.verse-rendezvous\"") + }; + let cases: [(&str, String, &str, Option, Result); 5] = [ + ("depends on Odin, routed", RECIPE.to_owned(), BINDING, Some(18001), Ok(ReadinessClass::OdinCorrelated)), + ("routed, no Odin", without_odin.clone(), BINDING, Some(18001), Ok(ReadinessClass::RouteProof)), + ("unrouted, no Odin", unrouted(&without_odin), &unrouted_binding, None, Err(())), + ("provides Odin, unrouted", provides_odin(&unrouted(&without_odin)), &unrouted_binding, None, Ok(ReadinessClass::OdinSelf)), + ("depends on Odin, unrouted", unrouted(RECIPE), &unrouted_binding, None, Ok(ReadinessClass::OdinCorrelated)), + ]; + for (name, recipe, binding, port, outcome) in cases { + let providers = [ready_odin_provider("odin", "odin-yggdrasil", 1)]; + let plan = compile_deployment_plan( + recipe.as_bytes(), + binding.as_bytes(), + source(&recipe), + "service-incarnation-1", + port, + 110, + if recipe.contains("[[dependencies]]") { &providers } else { &[] }, + ) + .unwrap_or_else(|error| panic!("{name}: {error:#}")); + let release = SealedRelease::new( + &plan, + vec![artifact_receipt()], + vec![external_input_receipt()], + 120, + ) + .unwrap(); + let expected = release.expected_projection(&plan).unwrap(); + match outcome { + Ok(class) => { + assert_eq!(plan.readiness_class().unwrap(), class, "{name}"); + assert_eq!(ReadinessClass::of(&expected), Ok(class), "{name}"); + } + Err(()) => { + let error = plan.readiness_class().expect_err(name); + let undeclared = UndeclaredReadiness { + target: "service".into(), + }; + assert_eq!(error.downcast_ref::(), Some(&undeclared), "{name}"); + assert_eq!(ReadinessClass::of(&expected), Err(undeclared), "{name}"); + } + } + } + } + #[test] fn expected_projection_is_sanitized_and_names_the_selected_graph() { let plan = plan(); diff --git a/src/drivers.rs b/src/drivers.rs index b1db416..749ac22 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -8118,6 +8118,279 @@ fn apply_identity(command: &mut Command, identity: Option) -> R mod tests { use super::*; + // --------------------------------------------------------------------- + // A direct challenge fails one of two ways. Silence waits; a bad answer, + // or a challenge that cannot be made, is an error. + // --------------------------------------------------------------------- + + /// A peer that reads one snapshot request and writes back whatever `reply` + /// builds from the challenge id. Returns the port it listens on. + fn scripted_peer(reply: impl FnOnce(&str) -> Vec + Send + 'static) -> u16 { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("a free port"); + let port = listener.local_addr().expect("a bound port").port(); + thread::spawn(move || { + let Ok((mut stream, _)) = listener.accept() else { + return; + }; + let mut request = Vec::new(); + let mut chunk = [0_u8; 4096]; + let message_id = loop { + let Ok(read) = stream.read(&mut chunk) else { + return; + }; + if read == 0 { + return; + } + request.extend_from_slice(&chunk[..read]); + let Some(end) = request.windows(4).position(|window| window == b"\r\n\r\n") else { + continue; + }; + if let Ok(CultNetMessage::SnapshotRequest { message_id, .. }) = + decode_cultnet_message_from_slice( + &request[end + 4..], + CultNetWireContract::CultNetSchemaV0, + ) + { + break message_id; + } + }; + let _ = stream.write_all(&reply(&message_id)); + }); + port + } + + fn http_ok(body: &[u8]) -> Vec { + let mut response = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .into_bytes(); + response.extend_from_slice(body); + response + } + + fn raw_snapshot(message_id: &str, schema: &str, key: &str, payload: Vec) -> Vec { + http_ok( + &encode_cultnet_message_to_vec( + &CultNetMessage::SnapshotResponseRaw { + message_id: message_id.into(), + documents: vec![cultnet_rs::CultNetRawDocumentRecord { + schema_id: schema.into(), + record_key: key.into(), + stored_at: "1970-01-01T00:00:00.100Z".into(), + payload_encoding: cultnet_rs::CultNetRawPayloadEncoding::Messagepack, + payload, + source_runtime_id: None, + source_agent_id: None, + source_role: None, + tags: None, + }], + }, + CultNetWireContract::CultNetSchemaV0, + ) + .expect("an encodable response"), + ) + } + + fn honest(message_id: &str) -> Vec { + raw_snapshot( + message_id, + cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, + "service", + vec![1, 2, 3], + ) + } + + /// One candidate challenge to `port`, with the Expected adjusted by `adjust`. + fn challenge( + port: u16, + adjust: impl FnOnce(&mut IdunnExpectedIncarnationRecord), + ) -> Result { + let temp = tempfile::tempdir().expect("a temporary directory"); + let mut candidate = expected(); + candidate.route = Some(cultnet_rs::IdunnExpectedRoute { + route_id: "service".into(), + transport: "http".into(), + stable_endpoint: "http://127.0.0.1:17999".into(), + candidate_endpoint: format!("http://127.0.0.1:{port}"), + }); + adjust(&mut candidate); + NginxRouteDriver::new(RouteBinding { + driver: RouteDriver::NginxStreamTcp, + route_id: "service".into(), + stable_endpoint: "http://127.0.0.1:17999".into(), + private_host: "127.0.0.1".into(), + private_port_start: port, + private_port_end: port, + config_path: temp.path().join("service.conf"), + reload_unit: "nginx.service".into(), + }) + .request_candidate_runtime_presence(&candidate, "candidate-probe") + } + + fn assert_silent(outcome: Result, why: &str) { + assert!( + matches!(outcome, Err(ChallengeFailure::Silent(_))), + "{why} must read as silence: {outcome:?}" + ); + } + + fn assert_refused(outcome: Result, why: &str) { + assert!( + matches!(outcome, Err(ChallengeFailure::Refused(_))), + "{why} must be an error, not a wait: {outcome:?}" + ); + } + + #[test] + fn an_exact_answer_is_taken() { + let port = scripted_peer(honest); + let answer = challenge(port, |_| {}).expect("the exact answer"); + assert_eq!(answer.message_id, "candidate-probe"); + assert_eq!(answer.canonical_presence, [1, 2, 3]); + } + + #[test] + fn a_challenge_that_nothing_answers_is_silence() { + assert_silent(challenge(scripted_peer(|_| Vec::new()), |_| {}), "a peer that hangs up"); + // Half a response, then the peer is gone. + assert_refused( + challenge(scripted_peer(|_| b"HTTP/1.1 200 OK\r\nContent-Le".to_vec()), |_| {}), + "a response that never finished its headers", + ); + // Nothing listens. + let reservation = std::net::TcpListener::bind("127.0.0.1:0").expect("a free port"); + let port = reservation.local_addr().expect("a bound port").port(); + drop(reservation); + assert_silent(challenge(port, |_| {}), "a closed port"); + } + + #[test] + fn a_bad_answer_is_refused_not_waited_on() { + assert_refused( + challenge( + scripted_peer(|_| { + raw_snapshot( + "candidate-another", + cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, + "service", + vec![1], + ) + }), + |_| {}, + ), + "an answer to another challenge", + ); + assert_refused( + challenge( + scripted_peer(|id| raw_snapshot(id, "some.other.schema", "service", vec![1])), + |_| {}, + ), + "a substituted document schema", + ); + assert_refused( + challenge( + scripted_peer(|id| { + raw_snapshot( + id, + cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, + "another-target", + vec![1], + ) + }), + |_| {}, + ), + "a substituted record key", + ); + assert_refused( + challenge( + scripted_peer(|id| { + raw_snapshot( + id, + cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, + "service", + vec![0; ROUTE_SNAPSHOT_MAX_BYTES + 1], + ) + }), + |_| {}, + ), + "an oversized payload", + ); + assert_refused( + challenge( + scripted_peer(|id| { + http_ok( + &encode_cultnet_message_to_vec( + &CultNetMessage::SnapshotRequest { + message_id: id.into(), + schema_ids: None, + record_keys: None, + }, + CultNetWireContract::CultNetSchemaV0, + ) + .expect("an encodable message"), + ) + }), + |_| {}, + ), + "a response that is not a raw snapshot", + ); + assert_refused( + challenge( + scripted_peer(|_| { + b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + .to_vec() + }), + |_| {}, + ), + "an HTTP error", + ); + assert_refused( + challenge(scripted_peer(|_| http_ok(b"not a cultnet message")), |_| {}), + "a body that is not CultNet", + ); + assert_refused( + challenge(scripted_peer(|_| b"HTTP/1.1 200 OK +Content-Le".to_vec()), |_| {}), + "a response that never finished its headers", + ); + } + + #[test] + fn a_challenge_that_cannot_be_made_is_an_error_not_a_wait() { + let port = scripted_peer(honest); + assert_refused( + challenge(port, |candidate| { + candidate.route.as_mut().unwrap().route_id = "another-route".into(); + }), + "a binding that differs from Expected", + ); + assert_refused( + challenge(port, |candidate| { + candidate.route.as_mut().unwrap().transport = "smtp".into(); + }), + "an unsupported transport", + ); + assert_refused( + challenge(port, |candidate| { + candidate.route.as_mut().unwrap().candidate_endpoint = + format!("http://candidate.example:{port}"); + }), + "a host that is not an IP address", + ); + assert_refused( + challenge(port, |candidate| { + candidate.route.as_mut().unwrap().candidate_endpoint = + format!("http://127.0.0.1:{}", port.wrapping_add(1)); + }), + "a candidate endpoint outside the route binding", + ); + assert_refused( + challenge(port, |candidate| candidate.route = None), + "an Expected with no route", + ); + } + #[test] fn route_connect_retries_connection_refused_until_listener_is_ready() -> Result<()> { let reservation = std::net::TcpListener::bind("127.0.0.1:0")?; From 771138a164d6f2a137f315a85752a0bb3dec970d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 13:41:17 +0200 Subject: [PATCH 033/286] Record B2 Soul rejection and the rollback / first-deploy rulings Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index 0eb920b..794bc30 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -36,6 +36,30 @@ because it has no route. `raven-muninn` and Heimdall both publish to Odin, so th The rejected alternative let Odin's availability gate Muninn's crash recovery through an undeclared, inferred dependency. +**B2 status, 2026-09-30 (Self).** Hands pushed `idunn/route-b2` at `1786ddf`. The Opus Soul pass rejected +it for merge. +- **F1 (critical):** `ContinuityBackoff.deferral_reason` was added without a schema bump. B2 fails the + canonical re-encode of every three-field v3 generation that F0, B1 or B3 wrote, so Idunn cannot read its + own store. +- **F2:** the ceiling refuses a rollback, which leaves an unproven candidate on the stable route. +- **F3:** a failed challenge writes, and continuity's CAS then loses against a stale envelope. +- **F4:** neither window rolls; both are anchored at the first charge. +- **F5:** a clock stepped backwards stalls everything for the length of the step. +- **F6:** `record_proved_challenge` is unpinned. +- **F7:** the mount preflight is not gated. +- B2 also does not compile on top of B3, and after a one-line fix two tests fail, because the fixture target + is now route-proof. + +The rework runs after B3 merges and is rebased onto it. + +**Rulings, 2026-09-30.** +- **The ceiling never refuses a rollback.** Restoring the admitted or incumbent route is survival, not + deployment. It is always allowed and always counted. Only installs and other forward changes can be + refused. A rollback that keeps failing is spaced by backoff, never refused. +- **First routed deployments are metered now.** The actuation window lives on the target, not on the + admitted generation, so a target with no admitted generation is charged too. That schema change is folded + into the store-version bump that F1 needs: a v4 generation, with a typed lift from v3. + Heads read for this map: | Repo | Ref | Notes | From 8d1dc9a87e8a0b54f2862b306381e1a6774125ad Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 14:09:04 +0200 Subject: [PATCH 034/286] Record the B3 fix batch and the hold / rollout rulings Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index 794bc30..1406316 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -29,6 +29,20 @@ Verdict: safe to merge for the StreamPixels web ship. The fix batch, in Hands, c Before `idunn up`, one read-only decode of a *copy* of the live `control.cc` checks for non-terminal transactions of routed targets with no Odin declaration (operator approved, 2026-09-30). +**B3 fix batch, 2026-09-30.** Pushed `6ea824b` and `e11b457`; 275 tests pass. +- `ReadinessClass::of` is the only class authority. The evidence tag is demoted to `Voucher`. +- `ChallengeFailure::{Silent, Refused}` separates silence from a refused answer. +- `PresenceDisagrees` makes the capacity shortfall a typed value. +- The recipes are on `route/b3-declare`: Heimdall `d5acc94`, Muninn `2260853`. +- Still open, for a fresh Hands: three survivors (boot re-proof by tag, config error as silence, first-Odin + check by name) and `cargo mutants --in-diff`. + +**Rulings, 2026-09-30:** +- A stored record whose evidence disagrees with its declared class is **held and reported, never aborted** + by Idunn. B5's deadlines or the operator resolve it. +- **Heimdall and raven-muninn are redeployed with their declaring recipes in the same window as Idunn B3 + and the StreamPixels ship.** + **Ruling, 2026-09-30: the recipe declares readiness; Idunn infers nothing.** This settles the fork Hands raised against a literal Q1(b). An unrouted target whose recipe declares neither a route nor an Odin dependency is refused at admission with a typed error. It does not fall back to Odin correlation From aa1889b2884697cc3c8833e8b834b5fddbea3888 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 14:41:25 +0200 Subject: [PATCH 035/286] B3 fix: kill the three surviving mutants Boot re-proof by tag: the held-generation test used receipts that still re-proved cleanly (boot reads the current Odin authority, not the stored one), so re-proving them passed. The receipts are now authentic but not Ready, so only skipping them lets the store boot. Config error as silence: the candidate path renders Expected first, which parses the same endpoints, so the challenge's own parse is reached only from the stable path. A new test drives that parse directly. First-Odin gate by name: the stub can now answer a first-Odin Warming challenge, and a stateful provider not named odin is accepted while a stateful target named odin that provides nothing is refused. Co-Authored-By: Claude Sonnet 5 --- src/control_plane.rs | 66 ++++++++++++++++++++++++++++++++++++++++---- src/drivers.rs | 40 ++++++++++++++++++++++++++- 2 files changed, 100 insertions(+), 6 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 8d33260..84dd4d2 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -13671,6 +13671,9 @@ mod tests { ForeignChallengeId, /// An HTTP error where the snapshot should be. HttpFailure, + /// A first Odin candidate answering its Warming challenge: the + /// detail names the challenge as `idunn-warming:`. + FirstOdinWarming, } impl RuntimeStub { @@ -13704,7 +13707,10 @@ mod tests { .collect(), health_contract: expected.health_contract.clone(), state: (*self.state.lock().unwrap()).into(), - detail: format!("route-observation:{message_id}"), + detail: match *self.reply.lock().unwrap() { + Reply::FirstOdinWarming => format!("idunn-warming:{message_id}"), + _ => format!("route-observation:{message_id}"), + }, write_lease_sha256: self.lease.lock().unwrap().clone(), signer_identity_id: self.provider.entry().identity_id.clone(), publisher_sequence: self.sequence.fetch_add(1, Ordering::SeqCst) + 1, @@ -14900,9 +14906,18 @@ mod tests { routed: &RoutedWorld, transaction: &DeploymentTransaction, sequence: u64, + ) -> Result { + odin_receipt_with(routed, transaction, sequence, true) + } + + fn odin_receipt_with( + routed: &RoutedWorld, + transaction: &DeploymentTransaction, + sequence: u64, + ready: bool, ) -> Result { let now = now_millis()?; - let bytes = signed_correlation(&routed.world, transaction, sequence, true)?; + let bytes = signed_correlation(&routed.world, transaction, sequence, ready)?; let authenticated = routed.world.engine.authenticate_topology_bytes( &ControlSnapshot::read(&routed.world.state_store)?, transaction, @@ -15011,14 +15026,18 @@ mod tests { let snapshot = ControlSnapshot::read(&routed.world.state_store)?; let current = snapshot.admitted.first().context("nothing was admitted")?; - let receipt = odin_receipt_for(&routed, &at_routing, 9)?; + // Receipts that would not survive being re-proved: authentic, but + // not Ready. Boot re-proves against the current Odin authority and + // requires exact semantic Ready, never what a generation stores, so + // this record is only bootable if boot does not re-prove it. + let receipt = odin_receipt_with(&routed, &at_routing, 9, false)?; let mut legacy = current.value.clone(); legacy.ready = ReadinessEvidence::OdinCorrelated { evidence: receipt.clone(), }; legacy.latest_odin_observation = Some(receipt); - // Receipts that would not authenticate if they were re-proved: the - // authority is another key's. Held generations are not re-proved. + // The stored authority is present because an Odin-tagged record + // carries one; it is not what boot reads. let stranger = cultnet_rs::enroll_service_identity_at::( &routed.world.root.join("identities/stranger.cc"), )?; @@ -15205,6 +15224,43 @@ mod tests { Ok(()) } + #[test] + fn a_stateful_target_that_provides_the_rendezvous_is_observed_directly_whatever_it_is_named() + -> Result<()> { + let provider = build_routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, true, true)?; + let transaction = provider.transaction()?; + assert_eq!(transaction.target, "service"); + assert!(transaction.expected.as_ref().unwrap().write_lease_required); + provider.stub.set_state("warming"); + *provider.stub.reply.lock().unwrap() = Reply::FirstOdinWarming; + let challenged_at = now_millis()?; + let answer = provider.stub.answer("warming-probe")?; + provider.world.engine.authenticate_first_odin_warming_presence( + &transaction, + "warming-probe", + challenged_at, + now_millis()?, + &answer, + )?; + + // The reverse: named odin, stateful, but nothing provides the + // rendezvous. + let stateful = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Warming)?; + let mut named = stateful.transaction()?; + assert!(named.expected.as_ref().unwrap().write_lease_required); + named.target = "odin".into(); + let error = stateful + .world + .engine + .authenticate_first_odin_warming_presence(&named, "warming-probe", 1, 2, &[1]) + .expect_err("a stateful target named odin that does not provide the rendezvous"); + assert!( + format!("{error:#}").contains("reserved for a stateful Odin incarnation"), + "{error:#}" + ); + Ok(()) + } + // ----------------------------------------------------------------- // A bad answer is an error. Only silence waits. // ----------------------------------------------------------------- diff --git a/src/drivers.rs b/src/drivers.rs index 749ac22..edb4699 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -8350,7 +8350,7 @@ mod tests { "a body that is not CultNet", ); assert_refused( - challenge(scripted_peer(|_| b"HTTP/1.1 200 OK + challenge(scripted_peer(|_| b"HTTP/1.1 200 OK Content-Le".to_vec()), |_| {}), "a response that never finished its headers", ); @@ -8391,6 +8391,44 @@ Content-Le".to_vec()), |_| {}), ); } + /// The candidate path renders the Expected first, and rendering parses the + /// same endpoints, so it never reaches the parse in the challenge. The + /// stable path does not render: a bound stable endpoint that is not a plain + /// host and port is caught by the challenge's own parse, and it is a + /// configuration error, not a listener that has not answered yet. + #[test] + fn a_stable_endpoint_that_does_not_parse_is_an_error_not_a_wait() { + let temp = tempfile::tempdir().expect("a temporary directory"); + let driver = NginxRouteDriver::new(RouteBinding { + driver: RouteDriver::NginxStreamTcp, + route_id: "service".into(), + stable_endpoint: "http://127.0.0.1:17999".into(), + private_host: "127.0.0.1".into(), + private_port_start: 17998, + private_port_end: 17998, + config_path: temp.path().join("service.conf"), + reload_unit: "nginx.service".into(), + }); + let mut candidate = expected(); + candidate.route = Some(cultnet_rs::IdunnExpectedRoute { + route_id: "service".into(), + transport: "http".into(), + stable_endpoint: "http://127.0.0.1:17999".into(), + candidate_endpoint: "http://127.0.0.1:17998".into(), + }); + for endpoint in [ + "http://127.0.0.1", + "http://127.0.0.1:0", + "http://127.0.0.1:17999/path", + "tcp://127.0.0.1:17999", + ] { + assert_refused( + driver.request_runtime_presence_at(&candidate, "stable-probe", endpoint), + endpoint, + ); + } + } + #[test] fn route_connect_retries_connection_refused_until_listener_is_ready() -> Result<()> { let reservation = std::net::TcpListener::bind("127.0.0.1:0")?; From 86ee21dac5f65f2653356717080d60aa0c147a21 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 15:24:45 +0200 Subject: [PATCH 036/286] B3 fix: pin what cargo-mutants found unpinned, delete what it found redundant First cargo-mutants --in-diff pass over a61540b..aa1889b left 16 misses. Deleted: - the candidate challenge's own route-id check: rendering the Expected next holds the same binding, so it could never decide anything. - odin_can_observe in advance_warming: is_odin && !(!is_odin || observed) is is_odin && !observed; two of the misses were operators in that detour. Moved: which transactions are held for their evidence is now DeploymentTransaction::held_disagreement, so "finished or aborting work is never held" is a value with a test, not a local in resume_candidate. Pinned: the admitted-generation topology refresh (taken once, then not again), WarmingEvidence shape checks per source, the PresenceDisagrees text, the plan's readiness declaration needing the shared-infrastructure kind, TCP framed challenges (exact answer, bound, empty, non-CultNet, silence), RUDP silence, and ChallengeFailure's text and source. Co-Authored-By: Claude Sonnet 5 --- src/control_plane.rs | 157 ++++++++++++++++++++++++++++++++++++++--- src/deployment_plan.rs | 4 +- src/drivers.rs | 124 +++++++++++++++++++++++++++++--- 3 files changed, 265 insertions(+), 20 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 84dd4d2..2152d5c 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1438,6 +1438,18 @@ impl DeploymentTransaction { .err() } + /// The disagreement that holds this transaction: evidence collected under + /// another class is held and reported, not advanced and not repaired, since + /// no step can make it the right kind and running one anyway only fails it + /// every tick. What is finished, or being finished (a completion, or an + /// abort under way), never reads the class, so it is left to run. + fn held_disagreement(&self) -> Option { + let finishing = self.completion.is_some() + || self.pre_fencing_abort.is_some() + || self.post_fencing_abort.is_some(); + self.readiness_disagreement().filter(|_| !finishing) + } + fn validate(&self) -> Result<()> { ensure!( self.schema_version == DEPLOYMENT_TRANSACTION_SCHEMA, @@ -4530,14 +4542,7 @@ impl Engine { { return Ok(false); } - // Evidence collected under another class is held and reported, not - // advanced and not repaired: no step can make it the right kind, and - // running one anyway only fails it every tick. An abort already under - // way does not read the class, so it is left to finish. - let aborting = current.value.completion.is_some() - || current.value.pre_fencing_abort.is_some() - || current.value.post_fencing_abort.is_some(); - if let Some(disagreement) = current.value.readiness_disagreement().filter(|_| !aborting) { + if let Some(disagreement) = current.value.held_disagreement() { self.note_fault("holds transaction", id, &anyhow::Error::new(disagreement)); return Ok(false); } @@ -5585,8 +5590,7 @@ impl Engine { } else { None }; - let odin_can_observe = !is_odin || odin_observation.is_some(); - let odin_observes_itself = is_odin && !odin_can_observe; + let odin_observes_itself = is_odin && odin_observation.is_none(); if odin_observes_itself { ensure!( expected.write_lease_required, @@ -10384,6 +10388,59 @@ mod tests { Ok(()) } + #[test] + fn warming_evidence_is_shape_checked_whatever_its_source() { + let good = |evidence: &ReadinessEvidence| match evidence { + ReadinessEvidence::RouteProof { evidence } => evidence.clone(), + ReadinessEvidence::OdinCorrelated { .. } => unreachable!("a route-proof fixture"), + }; + let presence = good(&route_proof_evidence()); + for warming in [ + WarmingEvidence::FirstOdinDirect { evidence: presence.clone() }, + WarmingEvidence::RouteProofDirect { evidence: presence.clone() }, + WarmingEvidence::OdinTopology { evidence: topology(3, 1) }, + ] { + warming.validate_shape().expect("well-formed evidence"); + } + let mut damaged = presence; + damaged.canonical_bytes.push(1); + for warming in [ + WarmingEvidence::FirstOdinDirect { evidence: damaged.clone() }, + WarmingEvidence::RouteProofDirect { evidence: damaged }, + WarmingEvidence::OdinTopology { + evidence: TopologyEvidence { + canonical_sha256: "sha256:not-the-digest".into(), + ..topology(3, 1) + }, + }, + ] { + assert!(warming.validate_shape().is_err(), "{warming:?}"); + } + } + + #[test] + fn a_presence_disagreement_names_each_disagreement() { + let shortfall = PresenceDisagrees { + disagreements: vec![ + OdinTopologyDisagreement { + code: "capacity-below-minimum".into(), + expected: Some("2".into()), + observed: Some("1".into()), + }, + OdinTopologyDisagreement { + code: "state".into(), + expected: None, + observed: None, + }, + ], + }; + assert_eq!( + shortfall.to_string(), + "route proof answered with a runtime that disagrees with current authority: \ + capacity-below-minimum (expected 2, observed 1); state (expected none, observed none)" + ); + } + fn route_proof_evidence() -> ReadinessEvidence { let canonical_bytes = vec![7, 7, 7]; ReadinessEvidence::RouteProof { @@ -11925,6 +11982,51 @@ mod tests { .clone()) } + #[test] + fn an_admitted_odin_correlated_generation_takes_a_newer_odin_reading_once() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; + let first = seeded_sequence(None) + 1; + odin_reports_ready(&world, &seeded, first)?; + drive(&world, |transaction| transaction.phase == DeploymentPhase::Complete)?; + assert!(world.engine.retire_one_terminal_transaction()?); + let admitted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("commit wrote no admitted generation")? + .value + .clone(); + assert_eq!(admitted.readiness(), Ok(ReadinessClass::OdinCorrelated)); + assert_eq!(admitted.odin_publisher_sequence_cursor, first); + + // Nothing newer than what is held: nothing is written. + let snapshot = ControlSnapshot::read(&world.state_store)?; + let current = snapshot.admitted_for("service").context("no generation")?; + let envelope = current.envelope.clone(); + assert!(!world.engine.refresh_admitted_topology(&snapshot, current)?); + assert_eq!(incumbent_envelope(&world)?, envelope); + + // A newer reading is adopted, once. + odin_republishes(&world, &seeded, first + 4)?; + let snapshot = ControlSnapshot::read(&world.state_store)?; + let current = snapshot.admitted_for("service").context("no generation")?; + assert!(world.engine.refresh_admitted_topology(&snapshot, current)?); + let refreshed = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("no generation")? + .value + .clone(); + assert_eq!(refreshed.odin_publisher_sequence_cursor, first + 4); + assert_eq!( + refreshed.latest_odin_observation.as_ref().map(|latest| latest.publisher_sequence), + Some(first + 4) + ); + assert_eq!(refreshed.ready.odin().map(|ready| ready.publisher_sequence), Some(first + 4)); + let snapshot = ControlSnapshot::read(&world.state_store)?; + let current = snapshot.admitted_for("service").context("no generation")?; + assert!(!world.engine.refresh_admitted_topology(&snapshot, current)?); + Ok(()) + } + /// Change the incumbent in the store, compare-and-swap. fn edit_incumbent( world: &EngineFixture, @@ -15086,6 +15188,41 @@ mod tests { Ok(()) } + #[test] + fn only_a_transaction_that_still_reads_its_class_is_held_for_its_evidence() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + let at_routing = routed.transaction()?; + // What pre-B3 Idunn would have collected: an Odin receipt. + let mut legacy = at_routing.clone(); + legacy.ready = Some(ReadinessEvidence::OdinCorrelated { + evidence: odin_receipt_for(&routed, &at_routing, 9)?, + }); + let disagreement = legacy.readiness_disagreement(); + assert!(disagreement.is_some()); + assert_eq!(legacy.held_disagreement(), disagreement); + + // Finished work, and an abort under way, never read the class. + let mut completed = legacy.clone(); + completed.completion = Some(TransactionCompletion::Admitted { + generation_id: "generation-1".into(), + }); + assert_eq!(completed.held_disagreement(), None); + let mut aborting = legacy.clone(); + aborting.pre_fencing_abort = Some(PreFencingAbort { + error: "operator abort".into(), + candidate_cleanup: CleanupEvidence::Pending, + topology_reconciliation: CleanupEvidence::Pending, + source_cleanup: CleanupEvidence::Pending, + }); + assert_eq!(aborting.held_disagreement(), None); + let mut aborting = legacy; + aborting.post_fencing_abort = Some(post_fencing_abort_intent(&aborting, "operator abort")); + assert_eq!(aborting.held_disagreement(), None); + Ok(()) + } + #[test] fn a_stored_record_that_declares_no_way_to_prove_readiness_is_held() -> Result<()> { let world = EngineFixture::new()?; diff --git a/src/deployment_plan.rs b/src/deployment_plan.rs index 7169601..5210448 100644 --- a/src/deployment_plan.rs +++ b/src/deployment_plan.rs @@ -1585,12 +1585,14 @@ nodes = ["yggdrasil"] let provides_odin = |recipe: &str| { recipe.replace("capability = \"service.runtime\"", "capability = \"odin.verse-rendezvous\"") }; - let cases: [(&str, String, &str, Option, Result); 5] = [ + let cases: [(&str, String, &str, Option, Result); 6] = [ ("depends on Odin, routed", RECIPE.to_owned(), BINDING, Some(18001), Ok(ReadinessClass::OdinCorrelated)), ("routed, no Odin", without_odin.clone(), BINDING, Some(18001), Ok(ReadinessClass::RouteProof)), ("unrouted, no Odin", unrouted(&without_odin), &unrouted_binding, None, Err(())), ("provides Odin, unrouted", provides_odin(&unrouted(&without_odin)), &unrouted_binding, None, Ok(ReadinessClass::OdinSelf)), ("depends on Odin, unrouted", unrouted(RECIPE), &unrouted_binding, None, Ok(ReadinessClass::OdinCorrelated)), + // Only the shared-infrastructure declaration is the declaration. + ("requires the Odin capability, routed", RECIPE.replace("kind = \"shared-infrastructure\"", "kind = \"required\""), BINDING, Some(18001), Ok(ReadinessClass::RouteProof)), ]; for (name, recipe, binding, port, outcome) in cases { let providers = [ready_odin_provider("odin", "odin-yggdrasil", 1)]; diff --git a/src/drivers.rs b/src/drivers.rs index edb4699..495fc86 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -5468,10 +5468,7 @@ impl NginxRouteDriver { .as_ref() .context("candidate challenge has no Expected route") .map_err(refused)?; - ensure_or_refuse( - route.route_id == self.binding.route_id, - "candidate challenge binding differs from Expected", - )?; + // Rendering holds the binding to the Expected route, id and endpoints. self.render(expected).map_err(refused)?; self.request_runtime_presence_at(expected, message_id, &route.candidate_endpoint) } @@ -8205,20 +8202,31 @@ mod tests { fn challenge( port: u16, adjust: impl FnOnce(&mut IdunnExpectedIncarnationRecord), + ) -> Result { + challenge_over("http", RouteDriver::NginxStreamTcp, port, adjust) + } + + /// The same challenge over another transport: `transport` names both the + /// Expected's transport and the endpoint scheme. + fn challenge_over( + transport: &str, + driver: RouteDriver, + port: u16, + adjust: impl FnOnce(&mut IdunnExpectedIncarnationRecord), ) -> Result { let temp = tempfile::tempdir().expect("a temporary directory"); let mut candidate = expected(); candidate.route = Some(cultnet_rs::IdunnExpectedRoute { route_id: "service".into(), - transport: "http".into(), - stable_endpoint: "http://127.0.0.1:17999".into(), - candidate_endpoint: format!("http://127.0.0.1:{port}"), + transport: transport.into(), + stable_endpoint: format!("{transport}://127.0.0.1:17999"), + candidate_endpoint: format!("{transport}://127.0.0.1:{port}"), }); adjust(&mut candidate); NginxRouteDriver::new(RouteBinding { - driver: RouteDriver::NginxStreamTcp, + driver, route_id: "service".into(), - stable_endpoint: "http://127.0.0.1:17999".into(), + stable_endpoint: format!("{transport}://127.0.0.1:17999"), private_host: "127.0.0.1".into(), private_port_start: port, private_port_end: port, @@ -8227,6 +8235,104 @@ mod tests { }) .request_candidate_runtime_presence(&candidate, "candidate-probe") } + /// A TCP peer that reads one framed snapshot request and writes back the + /// bytes `reply` builds from the challenge id, verbatim. + fn framed_peer(reply: impl FnOnce(&str) -> Vec + Send + 'static) -> u16 { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("a free port"); + let port = listener.local_addr().expect("a bound port").port(); + thread::spawn(move || { + let Ok((mut stream, _)) = listener.accept() else { + return; + }; + let mut header = [0_u8; 4]; + if stream.read_exact(&mut header).is_err() { + return; + } + let mut request = vec![0_u8; u32::from_be_bytes(header) as usize]; + if stream.read_exact(&mut request).is_err() { + return; + } + let Ok(CultNetMessage::SnapshotRequest { message_id, .. }) = + decode_cultnet_message_from_slice(&request, CultNetWireContract::CultNetSchemaV0) + else { + return; + }; + let _ = stream.write_all(&reply(&message_id)); + }); + port + } + + /// The body of an `honest` HTTP answer, framed for TCP. + fn framed_honest(message_id: &str) -> Vec { + let response = honest(message_id); + let body = response + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map(|end| &response[end + 4..]) + .expect("an HTTP answer"); + encode_frame(body).expect("a frame") + } + + fn tcp_challenge(port: u16) -> Result { + challenge_over("tcp", RouteDriver::NginxStreamTcp, port, |_| {}) + } + + #[test] + fn a_tcp_challenge_takes_the_exact_framed_answer_and_waits_only_on_silence() { + let answer = tcp_challenge(framed_peer(framed_honest)).expect("the exact answer"); + assert_eq!(answer.message_id, "candidate-probe"); + assert_eq!(answer.canonical_presence, [1, 2, 3]); + + assert_silent(tcp_challenge(framed_peer(|_| Vec::new())), "a peer that hangs up"); + assert_silent( + tcp_challenge(framed_peer(|_| vec![0, 0, 0, 9, 1])), + "a frame that never finishes", + ); + + // A frame that claims more than the observation bound, or nothing, is + // an answer that is wrong, not one that is late. + assert_refused( + tcp_challenge(framed_peer(|_| { + ((ROUTE_SNAPSHOT_MAX_BYTES + 1) as u32).to_be_bytes().to_vec() + })), + "a frame over the bound", + ); + assert_refused( + tcp_challenge(framed_peer(|_| 0_u32.to_be_bytes().to_vec())), + "an empty frame", + ); + assert_refused( + tcp_challenge(framed_peer(|_| encode_frame(b"not cultnet").expect("a frame"))), + "a frame that is not CultNet", + ); + } + + #[test] + fn a_rudp_challenge_that_nothing_answers_is_silence() { + let socket = std::net::UdpSocket::bind("127.0.0.1:0").expect("a free port"); + let port = socket.local_addr().expect("a bound port").port(); + // Bound but never read: nothing answers, and nothing refuses either. + assert_silent( + challenge_over("rudp", RouteDriver::NginxStreamUdp, port, |_| {}), + "a rudp peer that never answers", + ); + drop(socket); + } + + /// A challenge that cannot be made says why, whatever the message. + #[test] + fn a_challenge_failure_shows_and_chains_its_cause() { + for failure in [ + silent(anyhow!("cause of silence")), + refused(anyhow!("cause of refusal")), + ] { + let shown = failure.to_string(); + assert!(shown.starts_with("cause of "), "{shown}"); + let source = std::error::Error::source(&failure).expect("the underlying error"); + assert_eq!(source.to_string(), shown); + } + } + fn assert_silent(outcome: Result, why: &str) { assert!( From 7dc3a4bb8c438afd9820843d36e5c76b13a6ba42 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 17:00:39 +0200 Subject: [PATCH 037/286] Record the live store decode, B3 pass 2, and the ship sequence Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index 1406316..b1b09aa 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -37,6 +37,33 @@ transactions of routed targets with no Odin declaration (operator approved, 2026 - Still open, for a fresh Hands: three survivors (boot re-proof by tag, config error as silence, first-Odin check by name) and `cargo mutants --in-diff`. +**Live store, read-only decode of a copy, 2026-09-30 14:56 (operator-approved).** +- `control.cc` holds 5 envelopes, all `admitted_generation.v2` (pre-F0), and **no transactions**. +- `history.cc` holds 957 transaction v3 envelopes and 641 command v2 envelopes. +- Admitted generations, all on Odin receipts: + - ghostlight: routed, declares Odin, stateful. + - odin: routed rudp, provides the rendezvous. + - raven-muninn: unrouted, no declaration, stateless. B3 class `Undeclared`, so **held**. + - streampixels-service: routed, declares Odin, stateful. + - streampixels-web: routed, declares Odin, needs `streampixels.service.api`. +- **Heimdall is not admitted.** It drops out of the ship window. Its declaring recipe still merges. Its + first deploy will hit Soul pass 2's F2: a routed, Odin-correlated target must pass the stable-route + proof, and Heimdall answers no challenge. Record that for its own deploy. + +**B3 pass 2 (Soul, 86ee21d):** safe for the web ship. One same-window blocker: +- **F1:** supervision mints a held continuity over a dead held generation, and that continuity owns the + target, so the declaring redeploy can never freeze. The fix is in Hands. +- F5, bad answers read as Silent over HTTP keep-alive and TCP, is in the same batch. + +**Ship sequence:** +1. B3 fix, then narrow Soul, then merge B3. +2. Merge StreamPixels `route/s3-c2` and the Muninn and Heimdall `route/b3-declare` recipes. +3. Build the Idunn release on its deploy path. +4. Back up `control.cc` and `history.cc`. F0's lift is irreversible. +5. Install Idunn. +6. `idunn up` raven-muninn with the declaring recipe. +7. `idunn up` streampixels (service, then web). + **Rulings, 2026-09-30:** - A stored record whose evidence disagrees with its declared class is **held and reported, never aborted** by Idunn. B5's deadlines or the operator resolve it. From 477927509ac8cd65f741cf069e35b48296322bc3 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 17:13:36 +0200 Subject: [PATCH 038/286] B3 fix: a held generation mints no continuity, and nothing yields to a held record A held, undeclared generation that died reached the continuity mint, which copied the undeclared Expected, was held at once, and owned the target for good: cancel refuses continuity, so the declaring redeploy could never freeze. Supervision now reports a dead held generation once and leaves it down and its target free. With no continuity minted over it, a declaring redeploy in Warming no longer yields to one either. The yield path also aborted a pre-fence Deploy blocker without reading its hold, which is Idunn aborting a held record. It now leaves a held blocker alone. Co-Authored-By: Claude Sonnet 5 --- src/control_plane.rs | 208 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 208 insertions(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index 2152d5c..4bd6b59 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -4945,6 +4945,21 @@ impl Engine { } let workload_error = operational_error .context("admitted operational state has neither observation nor error")?; + // A held generation is a record the recipe never declared + // readiness for. Idunn reports it and does not decide for it: a + // continuity minted over it copies the undeclared Expected, is + // held at once, and owns the target forever, so the declaring + // redeploy that would clear the hold could never freeze. The dead + // held target stays down and free. Nothing yields to it either: + // there is no continuity to yield to. + if current.value.readiness().is_err() { + self.note_fault( + "holds a dead admitted generation down; declare readiness in its recipe and redeploy", + &format!("generation-down:{}", current.value.target), + &anyhow!("admitted generation {} is not running", current.value.generation_id), + ); + continue; + } // Continuity restarts a release; it cannot repair one. When the // admitted release will not start, rescheduling it forever keeps // the target permanently occupied -- and a target with a live @@ -5001,6 +5016,8 @@ impl Engine { && refused_restarts < CONTINUITY_RESTART_ATTEMPTS && blocker.value.phase < DeploymentPhase::Fencing && blocker.value.pre_fencing_abort.is_none() + // A held record is reported and never aborted by Idunn. + && blocker.value.held_disagreement().is_none() { self.begin_pre_fencing_abort( blocker, @@ -13729,6 +13746,197 @@ mod tests { Ok(()) } + // --------------------------------------------------------------------- + // A held record is reported and never decided for. Supervision mints no + // continuity over a held generation, and nothing yields to a held record. + // --------------------------------------------------------------------- + + /// Rewrite the world's admitted generation into what a pre-B3 binary left + /// behind: no Odin declaration, admitted on Odin receipts. + fn undeclare_incumbent(world: &EngineFixture) -> Result { + let now = now_millis()?; + let mut failure = None; + let next = edit_incumbent(world, |generation| { + generation.expected.dependencies.clear(); + match IdunnRuntimeActivationLaunch::issue( + &generation.expected, + generation.activation.runtime_instance_id.clone(), + now, + &world.engine.idunn_signer, + ) { + Ok(launch) => generation.activation = launch.activation().clone(), + Err(error) => failure = Some(error), + } + })?; + if let Some(error) = failure { + return Err(error); + } + ControlSnapshot::read(&world.state_store).context("store with the undeclared generation")?; + assert!(matches!(next.readiness(), Err(ReadinessDisagreement::Undeclared(_)))); + Ok(next) + } + + /// Queue the operator's declaring redeploy of `service`. + fn queue_declaring_redeploy(world: &EngineFixture, command_id: &str) -> Result<()> { + use crate::deployment_plan::tests::BINDING; + let (binding_head, binding_tail) = BINDING.split_once("[route]").context("route")?; + let binding = format!( + "{binding_head}[brakes]{}", + binding_tail.split_once("[brakes]").context("brakes")?.1 + ); + std::fs::create_dir_all(world.root.join("bindings"))?; + std::fs::write(world.root.join("bindings/service.toml"), binding)?; + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: command_id.into(), + kind: CommandKind::Deploy, + selector: "service".into(), + requested_by: "operator".into(), + requested_at_unix_millis: now_millis()?, + }; + assert!(SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }], + &[command_envelope(&command, command.requested_at_unix_millis)?], + )?); + Ok(()) + } + + /// Every transaction the store knows: live, then archived. + fn every_transaction(world: &EngineFixture) -> Result> { + let mut all = ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .map(|stored| stored.value) + .collect::>(); + all.extend(read_history_transactions(&world.state_store)); + Ok(all) + } + + fn is_continuity_over(transaction: &DeploymentTransaction, held: &AdmittedGeneration) -> bool { + transaction.command_kind == CommandKind::Continuity + && transaction.incumbent_generation_id.as_deref() == Some(held.generation_id.as_str()) + } + + #[test] + fn a_held_generation_that_dies_mints_no_continuity_and_leaves_its_target_free() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + let held = undeclare_incumbent(&world)?; + world.engine.validate_durable_authority(&ControlSnapshot::read(&world.state_store)?)?; + + workload.kill(); + for _ in 0..4 { + assert!(!world.engine.supervise_one_admitted_generation()?); + } + assert!( + ControlSnapshot::read(&world.state_store)?.transactions.is_empty(), + "supervision minted a transaction over a held generation" + ); + // Reported, once per state change: the same text is not offered again. + let reports = world.engine.fault_reports.lock().unwrap(); + let down = reports.get("generation-down:service").context("the dead hold is not reported")?; + let last = down.last.lock().unwrap().clone(); + assert!(last.is_some()); + assert!(down.offer(last).is_none()); + drop(reports); + + // The target is free: the declaring redeploy freezes and is not + // displaced by a continuity. + queue_declaring_redeploy(&world, "up-service-declared")?; + for _ in 0..8 { + let _ = world.engine.run_scheduler_tick(); + } + let all = every_transaction(&world)?; + assert!( + all.iter().all(|transaction| !is_continuity_over(transaction, &held)), + "a continuity was minted over the held generation" + ); + assert!( + all.iter().any(|transaction| transaction.command_id == "up-service-declared"), + "the declaring redeploy never froze" + ); + Ok(()) + } + + #[test] + fn a_declaring_redeploy_is_not_yielded_to_a_continuity_when_the_held_incumbent_dies() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + let held = undeclare_incumbent(&world)?; + let redeploy = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&held))?; + assert_eq!(redeploy.held_disagreement(), None); + workload.kill(); + for _ in 0..40 { + let _ = world.engine.run_scheduler_tick(); + } + let all = every_transaction(&world)?; + assert!( + all.iter().all(|transaction| !is_continuity_over(transaction, &held)), + "a continuity was minted over the held incumbent" + ); + let redeploy = all + .iter() + .find(|transaction| transaction.transaction_id == redeploy.transaction_id) + .context("the redeploy is nowhere")?; + // The fixture's one workload is the candidate too, so the redeploy may + // fail for its own reasons; it is never yielded away. + assert!( + redeploy + .pre_fencing_abort + .as_ref() + .is_none_or(|abort| !abort.error.contains("yielded to continuity")), + "the redeploy yielded: {:?}", + redeploy.pre_fencing_abort + ); + Ok(()) + } + + #[test] + fn a_held_predeploy_transaction_is_not_aborted_to_yield_to_continuity() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + // The incumbent is declared; the Deploy in front of it is the held one. + let seeded = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + let mut bare = seeded.clone(); + // Undeclare the Expected and re-derive what is bound to its digest. + let expected = bare.expected.as_mut().context("no expected")?; + expected.dependencies.clear(); + bare.expected_publication_sha256 = Some(expected.canonical_sha256()?); + let activation = IdunnRuntimeActivationLaunch::issue( + expected, + seeded.activation.as_ref().context("no activation")?.runtime_instance_id.clone(), + now_millis()?, + &world.engine.idunn_signer, + )? + .activation() + .clone(); + bare.activation_publication_sha256 = Some(activation.canonical_sha256()?); + bare.activation = Some(activation); + bare.validate()?; + let stored = resident(&world)?; + replace_transaction(&world.state_store, &stored, &bare)?; + assert!(bare.held_disagreement().is_some()); + + workload.kill(); + for _ in 0..4 { + let _ = world.engine.supervise_one_admitted_generation()?; + } + let after = resident(&world)?.value; + assert_eq!(after.transaction_id, seeded.transaction_id); + assert!(after.pre_fencing_abort.is_none(), "Idunn aborted a held record"); + assert!(after.completion.is_none()); + Ok(()) + } + // --------------------------------------------------------------------- // Route-proof readiness: a routed target that declares no Odin dependency // is admitted by Idunn's own challenges. Odin is never read. From 2f22dcac96c6c14d66c6f781a19d6ef1b3ca81cd Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 17:13:37 +0200 Subject: [PATCH 039/286] B3 fix: read a route answer by its framing, and judge a stopped peer one way The HTTP challenge read to end of stream, so a peer that answered in full and kept the socket open (or answered 404 and kept it open) read as silence. It now reads by Content-Length or chunked framing, refuses any non-2xx status, and refuses bytes that are not a CultNet frame. One rule for both transports: a peer that sent no byte is silent; a peer that sent some bytes and stopped is refused. A truncated TCP frame used to read as silence; that pinned test now expects a refusal. Co-Authored-By: Claude Sonnet 5 --- src/drivers.rs | 440 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 374 insertions(+), 66 deletions(-) diff --git a/src/drivers.rs b/src/drivers.rs index 495fc86..2429c9a 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; use std::ffi::{OsStr, OsString}; use std::fs::{self, OpenOptions}; -use std::io::{BufRead, ErrorKind, Read, Write}; +use std::io::{BufRead, BufReader, ErrorKind, Read, Write}; use std::net::{SocketAddr, TcpStream}; use std::path::{Path, PathBuf}; use std::process::{Command, Output, Stdio}; @@ -700,6 +700,7 @@ const ROUTE_CONNECT_RETRY_WINDOW: Duration = Duration::from_secs(2); const ROUTE_CONNECT_RETRY_INTERVAL: Duration = Duration::from_millis(25); const ROUTE_SNAPSHOT_MAX_BYTES: usize = 1024 * 1024; const ROUTE_HTTP_MAX_HEADER_BYTES: usize = 32 * 1024; +const ROUTE_HTTP_CHUNK_LINE_BYTES: usize = 128; const ROUTE_HTTP_SNAPSHOT_PATH: &str = "/cultnet/snapshot"; impl RoutePreflightReceipt { @@ -5607,6 +5608,70 @@ fn ensure_or_refuse(condition: bool, message: &'static str) -> Result<(), Challe } } +/// One answer as it arrives, over either transport. A peer that stops is judged +/// the same way on both: no byte at all is silence (a process still starting +/// does that), and anything short of a whole answer after some bytes is a bad +/// answer. Waiting does not finish an answer that has begun and stopped. +struct Answer { + source: R, + seen: usize, +} + +impl Answer { + fn new(source: R) -> Self { + Self { source, seen: 0 } + } + + fn stopped(&self, what: &str) -> ChallengeFailure { + let error = anyhow!("stable CultNet route stopped {what}"); + if self.seen == 0 { + silent(error) + } else { + refused(error) + } + } + + fn read_some(&mut self, buffer: &mut [u8], what: &str) -> Result { + loop { + match self.source.read(buffer) { + Ok(0) => return Err(self.stopped(what)), + Ok(read) => { + self.seen += read; + return Ok(read); + } + Err(error) if error.kind() == ErrorKind::Interrupted => {} + Err(_) => return Err(self.stopped(what)), + } + } + } + + fn read_all(&mut self, buffer: &mut [u8], what: &str) -> Result<(), ChallengeFailure> { + let mut filled = 0; + while filled < buffer.len() { + filled += self.read_some(&mut buffer[filled..], what)?; + } + Ok(()) + } + + /// One CRLF-terminated line, without its terminator. + fn read_line(&mut self, bound: usize, what: &str) -> Result, ChallengeFailure> { + let mut line = Vec::new(); + loop { + let mut byte = [0_u8]; + self.read_all(&mut byte, what)?; + line.push(byte[0]); + if line.ends_with(b"\r\n") { + line.truncate(line.len() - 2); + return Ok(line); + } + ensure_or_refuse( + line.len() <= bound, + "stable CultNet HTTP response line exceeds its bound", + )?; + } + } +} + fn request_tcp_snapshot(target: SocketAddr, payload: &[u8]) -> Result, ChallengeFailure> { let mut stream = connect_route_socket(target) .with_context(|| format!("connecting stable CultNet TCP route {target}")) @@ -5622,21 +5687,16 @@ fn request_tcp_snapshot(target: SocketAddr, payload: &[u8]) -> Result, C .and_then(|()| stream.flush()) .map_err(silent)?; + let mut answer = Answer::new(stream); let mut header = [0_u8; 4]; - stream - .read_exact(&mut header) - .context("reading stable CultNet TCP response frame") - .map_err(silent)?; + answer.read_all(&mut header, "before its response frame header ended")?; let length = u32::from_be_bytes(header) as usize; ensure_or_refuse( (1..=ROUTE_SNAPSHOT_MAX_BYTES).contains(&length), "stable CultNet TCP response exceeds the route observation bound", )?; let mut response = vec![0_u8; length]; - stream - .read_exact(&mut response) - .context("reading stable CultNet TCP response payload") - .map_err(silent)?; + answer.read_all(&mut response, "before its response frame ended")?; Ok(response) } @@ -5660,79 +5720,124 @@ fn request_http_snapshot(target: SocketAddr, payload: &[u8]) -> Result, .and_then(|()| stream.flush()) .map_err(silent)?; - let maximum = ROUTE_HTTP_MAX_HEADER_BYTES - .checked_add(ROUTE_SNAPSHOT_MAX_BYTES) - .context("route HTTP response bound overflow") + // The response is read by its own framing, never to end of stream: a peer + // that ignores `Connection: close` and keeps the socket open has still + // answered in full. + let mut answer = Answer::new(BufReader::new(stream)); + let mut head_bytes = 0_usize; + let mut head_line = |answer: &mut Answer>| { + let line = answer.read_line(ROUTE_HTTP_MAX_HEADER_BYTES, "before its headers ended")?; + head_bytes += line.len() + 2; + ensure_or_refuse( + head_bytes <= ROUTE_HTTP_MAX_HEADER_BYTES, + "stable CultNet HTTP response headers exceed their bound", + )?; + Ok::<_, ChallengeFailure>(line) + }; + let status = head_line(&mut answer)?; + let status = std::str::from_utf8(&status) + .context("stable CultNet HTTP response status is not UTF-8") .map_err(refused)?; - let mut response = Vec::new(); - (&mut stream) - .take((maximum + 1) as u64) - .read_to_end(&mut response) - .context("reading stable CultNet HTTP response") - .map_err(silent)?; - if response.is_empty() { - return Err(silent(anyhow!( - "stable CultNet HTTP route closed the connection without answering" + if !is_http_success(status) { + return Err(refused(anyhow!( + "stable CultNet HTTP route rejected the snapshot challenge: {status}" ))); } - parse_http_snapshot(&response, maximum).map_err(refused) -} -/// The body of an HTTP snapshot response that arrived, or why what arrived is -/// not one. -fn parse_http_snapshot(response: &[u8], maximum: usize) -> Result> { - ensure!( - response.len() <= maximum, - "stable CultNet HTTP response exceeds the route observation bound" - ); - let header_end = response - .windows(4) - .position(|window| window == b"\r\n\r\n") - .map(|index| index + 4) - .context("stable CultNet HTTP response has no header boundary")?; - ensure!( - header_end <= ROUTE_HTTP_MAX_HEADER_BYTES, - "stable CultNet HTTP response headers exceed their bound" - ); - let headers = std::str::from_utf8(&response[..header_end]) - .context("stable CultNet HTTP response headers are not UTF-8")?; - let mut lines = headers.split("\r\n"); - let status = lines - .next() - .context("stable CultNet HTTP response has no status")?; - ensure!( - matches!(status, "HTTP/1.1 200 OK" | "HTTP/1.0 200 OK"), - "stable CultNet HTTP route rejected the snapshot challenge: {status}" - ); let mut content_length = None; - for line in lines.filter(|line| !line.is_empty()) { + let mut chunked = false; + loop { + let line = head_line(&mut answer)?; + if line.is_empty() { + break; + } + let line = std::str::from_utf8(&line) + .context("stable CultNet HTTP response headers are not UTF-8") + .map_err(refused)?; let (name, value) = line .split_once(':') - .context("stable CultNet HTTP response contains a malformed header")?; + .context("stable CultNet HTTP response contains a malformed header") + .map_err(refused)?; if name.eq_ignore_ascii_case("transfer-encoding") { - bail!("stable CultNet HTTP response uses unsupported transfer encoding"); + ensure_or_refuse( + value.trim().eq_ignore_ascii_case("chunked"), + "stable CultNet HTTP response uses unsupported transfer encoding", + )?; + chunked = true; } if name.eq_ignore_ascii_case("content-length") { - ensure!( + ensure_or_refuse( content_length.is_none(), - "stable CultNet HTTP response repeats Content-Length" - ); + "stable CultNet HTTP response repeats Content-Length", + )?; content_length = Some( value .trim() .parse::() - .context("stable CultNet HTTP Content-Length is invalid")?, + .context("stable CultNet HTTP Content-Length is invalid") + .map_err(refused)?, ); } } - let content_length = - content_length.context("stable CultNet HTTP response has no Content-Length")?; - ensure!( - content_length <= ROUTE_SNAPSHOT_MAX_BYTES - && response.len().saturating_sub(header_end) == content_length, - "stable CultNet HTTP response body differs from its bounded Content-Length" - ); - Ok(response[header_end..].to_vec()) + + let body = "before its body ended"; + match (content_length, chunked) { + (Some(length), false) => { + ensure_or_refuse( + length <= ROUTE_SNAPSHOT_MAX_BYTES, + "stable CultNet HTTP response body exceeds the route observation bound", + )?; + let mut response = vec![0_u8; length]; + answer.read_all(&mut response, body)?; + Ok(response) + } + (None, true) => { + let mut response = Vec::new(); + loop { + let size = answer.read_line(ROUTE_HTTP_CHUNK_LINE_BYTES, body)?; + let size = std::str::from_utf8(&size) + .ok() + .and_then(|size| size.split(';').next()) + .and_then(|size| usize::from_str_radix(size.trim(), 16).ok()) + .context("stable CultNet HTTP chunk size is invalid") + .map_err(refused)?; + if size == 0 { + // Trailers, if any, end at the empty line. + while !answer.read_line(ROUTE_HTTP_MAX_HEADER_BYTES, body)?.is_empty() {} + return Ok(response); + } + ensure_or_refuse( + response.len().saturating_add(size) <= ROUTE_SNAPSHOT_MAX_BYTES, + "stable CultNet HTTP response body exceeds the route observation bound", + )?; + let start = response.len(); + response.resize(start + size, 0); + answer.read_all(&mut response[start..], body)?; + let mut end = [0_u8; 2]; + answer.read_all(&mut end, body)?; + ensure_or_refuse(&end == b"\r\n", "stable CultNet HTTP chunk is not terminated")?; + } + } + _ => Err(refused(anyhow!( + "stable CultNet HTTP response has no single body framing" + ))), + } +} + +/// A 2xx status line of HTTP/1.0 or HTTP/1.1. +fn is_http_success(status: &str) -> bool { + let Some(rest) = status + .strip_prefix("HTTP/1.1 ") + .or_else(|| status.strip_prefix("HTTP/1.0 ")) + else { + return false; + }; + let code = rest.as_bytes(); + code.len() >= 3 + && code[0] == b'2' + && code[1].is_ascii_digit() + && code[2].is_ascii_digit() + && (code.len() == 3 || code[3] == b' ') } /// nginx reload is signal-driven: `systemctl reload` can return before its @@ -8121,8 +8226,17 @@ mod tests { // --------------------------------------------------------------------- /// A peer that reads one snapshot request and writes back whatever `reply` - /// builds from the challenge id. Returns the port it listens on. + /// builds from the challenge id, then closes. Returns the port it listens on. fn scripted_peer(reply: impl FnOnce(&str) -> Vec + Send + 'static) -> u16 { + lingering_peer(reply, Duration::ZERO) + } + + /// `scripted_peer`, except the socket stays open for `linger` after the + /// answer: keep-alive, ignoring `Connection: close`. + fn lingering_peer( + reply: impl FnOnce(&str) -> Vec + Send + 'static, + linger: Duration, + ) -> u16 { let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("a free port"); let port = listener.local_addr().expect("a bound port").port(); thread::spawn(move || { @@ -8152,6 +8266,8 @@ mod tests { } }; let _ = stream.write_all(&reply(&message_id)); + let _ = stream.flush(); + thread::sleep(linger); }); port } @@ -8284,7 +8400,8 @@ mod tests { assert_eq!(answer.canonical_presence, [1, 2, 3]); assert_silent(tcp_challenge(framed_peer(|_| Vec::new())), "a peer that hangs up"); - assert_silent( + // Some bytes and then nothing is a bad answer, not a late one. + assert_refused( tcp_challenge(framed_peer(|_| vec![0, 0, 0, 9, 1])), "a frame that never finishes", ); @@ -8334,6 +8451,197 @@ mod tests { } + // --------------------------------------------------------------------- + // One rule for a peer that stops, on either transport: no byte at all is + // silence; some bytes and then nothing is a bad answer. An answer is read + // by its own framing, so a peer that keeps the socket open has still + // answered. + // --------------------------------------------------------------------- + + fn http_chunked(body: &[u8], chunk: usize) -> Vec { + let mut response = b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n".to_vec(); + for piece in body.chunks(chunk) { + response.extend_from_slice(format!("{:x};ext=1\r\n", piece.len()).as_bytes()); + response.extend_from_slice(piece); + response.extend_from_slice(b"\r\n"); + } + response.extend_from_slice(b"0\r\nTrailer: x\r\n\r\n"); + response + } + + fn honest_body(message_id: &str) -> Vec { + let response = honest(message_id); + let end = response + .windows(4) + .position(|window| window == b"\r\n\r\n") + .expect("an HTTP answer"); + response[end + 4..].to_vec() + } + + const LINGER: Duration = Duration::from_secs(6); + + #[test] + fn an_http_answer_is_read_by_its_framing_not_to_the_end_of_the_stream() { + let started = Instant::now(); + let answer = challenge(lingering_peer(honest, LINGER), |_| {}).expect("a whole answer"); + assert_eq!(answer.canonical_presence, [1, 2, 3]); + assert!(started.elapsed() < LINGER / 2, "the challenge waited on the socket"); + + let started = Instant::now(); + let answer = challenge( + lingering_peer(|id| http_chunked(&honest_body(id), 7), LINGER), + |_| {}, + ) + .expect("a chunked answer"); + assert_eq!(answer.canonical_presence, [1, 2, 3]); + assert!(started.elapsed() < LINGER / 2, "the challenge waited on the socket"); + } + + #[test] + fn an_http_status_that_is_not_success_is_refused_even_when_the_peer_lingers() { + for status in [ + "404 Not Found", + "301 Moved Permanently", + "503 Service Unavailable", + "199 Odd", + ] { + // The body is the honest answer: only the status refuses it. + let reply = move |id: &str| { + let body = honest_body(id); + let mut reply = + format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\n\r\n", body.len()) + .into_bytes(); + reply.extend_from_slice(&body); + reply + }; + let started = Instant::now(); + assert_refused(challenge(lingering_peer(reply, LINGER), |_| {}), status); + assert!(started.elapsed() < LINGER / 2, "{status} waited on the socket"); + } + } + + #[test] + fn only_a_two_hundred_series_http_1_status_line_is_success() { + for status in ["HTTP/1.1 200 OK", "HTTP/1.0 200 OK", "HTTP/1.1 204", "HTTP/1.1 299 x"] { + assert!(is_http_success(status), "{status}"); + } + for status in [ + "", + "HTTP/2 200 OK", + "HTTP/1.1 ", + "HTTP/1.1 20", + "HTTP/1.1 2x0 OK", + "HTTP/1.1 20x OK", + "HTTP/1.1 2000 OK", + "HTTP/1.1 300 OK", + "HTTP/1.1 404 Not Found", + "200 OK", + ] { + assert!(!is_http_success(status), "{status:?}"); + } + } + + #[test] + fn a_peer_that_sent_some_bytes_and_stopped_is_refused_on_either_transport() { + // No byte at all is silence, on both. + assert_silent(challenge(scripted_peer(|_| Vec::new()), |_| {}), "an HTTP peer with nothing to say"); + assert_silent(tcp_challenge(framed_peer(|_| Vec::new())), "a TCP peer with nothing to say"); + + // Some bytes and then nothing is a bad answer, wherever it stops. + let head = b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n"; + for (why, bytes) in [ + ("a status line that stops", b"HTTP/1.1 2".to_vec()), + ("headers that stop", b"HTTP/1.1 200 OK\r\nContent-Le".to_vec()), + ("a body that stops", [head.as_slice(), b"abc"].concat()), + ( + "a chunk that stops", + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nab".to_vec(), + ), + ( + "a chunked body that never ends", + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n1\r\na\r\n".to_vec(), + ), + ] { + assert_refused(challenge(scripted_peer(move |_| bytes), |_| {}), why); + } + for (why, bytes) in [ + ("a 3-byte banner", b"OK\n".to_vec()), + ("a frame header that stops", vec![0, 0]), + ("a frame that stops", vec![0, 0, 0, 9, 1]), + ] { + assert_refused(tcp_challenge(framed_peer(move |_| bytes)), why); + } + } + + #[test] + fn an_http_answer_that_frames_its_body_wrongly_is_refused() { + for (why, reply) in [ + ( + "an unsupported transfer encoding", + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: gzip\r\n\r\n".to_vec(), + ), + ( + "both framings", + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nContent-Length: 1\r\n\r\n0\r\n\r\n" + .to_vec(), + ), + ("no framing", b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nabc".to_vec()), + ( + "a repeated Content-Length", + b"HTTP/1.1 200 OK\r\nContent-Length: 1\r\nContent-Length: 1\r\n\r\na".to_vec(), + ), + ( + "an invalid Content-Length", + b"HTTP/1.1 200 OK\r\nContent-Length: many\r\n\r\n".to_vec(), + ), + ( + "a Content-Length over the bound", + format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n", + ROUTE_SNAPSHOT_MAX_BYTES + 1 + ) + .into_bytes(), + ), + ("a malformed header", b"HTTP/1.1 200 OK\r\nno colon here\r\n\r\n".to_vec()), + ( + "an invalid chunk size", + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\nzz\r\n".to_vec(), + ), + ( + "an unterminated chunk", + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n1\r\naXX0\r\n\r\n".to_vec(), + ), + ( + "chunks over the bound", + format!( + "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n{:x}\r\n", + ROUTE_SNAPSHOT_MAX_BYTES + 1 + ) + .into_bytes(), + ), + ( + "one header line over the bound", + [ + b"HTTP/1.1 200 OK\r\nX: ".to_vec(), + vec![b'a'; ROUTE_HTTP_MAX_HEADER_BYTES + 1], + ] + .concat(), + ), + ( + "headers over the bound together", + [ + b"HTTP/1.1 200 OK\r\n".to_vec(), + b"X: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\n".repeat(600), + ] + .concat(), + ), + ("headers that are not UTF-8", b"HTTP/1.1 200 OK\r\nX: \xff\r\n\r\n".to_vec()), + ("a status that is not UTF-8", b"HTTP/1.1 \xff\r\n\r\n".to_vec()), + ] { + assert_refused(challenge(scripted_peer(move |_| reply), |_| {}), why); + } + } + fn assert_silent(outcome: Result, why: &str) { assert!( matches!(outcome, Err(ChallengeFailure::Silent(_))), From 3803725bf2a10311e99d3059d4007dcc2ce303ca Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 17:21:25 +0200 Subject: [PATCH 040/286] Record the Odin timeout diagnosis and the Idunn-first ship order Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index b1b09aa..bf77513 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -55,6 +55,20 @@ transactions of routed targets with no Odin declaration (operator approved, 2026 target, so the declaring redeploy can never freeze. The fix is in Hands. - F5, bad answers read as Silent over HTTP keep-alive and TCP, is in the same batch. +**Odin diagnosis (Eyes, read-only, 2026-09-30).** +- The 109 Odin route timeouts are self-inflicted. Live Idunn (`8ae00a1`, pre-S1) reloads nginx inside + `restore_admitted_membership` before every Odin challenge. The reload moves the reuseport UDP flow, so + the challenge misses. All 109 have a reload 3.05-3.12 s before them. +- 2,977 reloads in 6 h. 479 shutting-down nginx workers, because the stream has no `proxy_timeout`. +- The route timeouts do not block Ready correlation. +- Odin has crashed twice right after a reload. A stray RUDP packet with a foreign connection id escapes + `cultnet-rs` `rudp.rs:1217` (`require_connection`) and ends the process. Each crash costs about 75 s + without a lease. The CultLib fix is in Hands on `hands/rudp-stray-packets`. Odin takes it at its next pin + bump, after the ship. + +**Operator ruling: Idunn first.** Install the new Idunn (B3 carries S1), then watch for about 15 minutes that +reloads stop and Odin stays up. Only then redeploy raven-muninn and StreamPixels. + **Ship sequence:** 1. B3 fix, then narrow Soul, then merge B3. 2. Merge StreamPixels `route/s3-c2` and the Muninn and Heimdall `route/b3-declare` recipes. From d5669af3522da309f6d1da58facba5a5a2773b00 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 17:49:56 +0200 Subject: [PATCH 041/286] B3 fix: pin what cargo-mutants found unpinned in the framed route reader The header-size bound was tested only against answers that also stopped short, so it never had to hold on its own; the interrupted-read retry was exercised by no reader that could be interrupted. The header test now sends an honest, whole answer whose headers alone exceed the bound, and a scripted reader pins the retry and the silent/refused split at the Answer boundary. Co-Authored-By: Claude Sonnet 5 --- src/drivers.rs | 86 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 78 insertions(+), 8 deletions(-) diff --git a/src/drivers.rs b/src/drivers.rs index 2429c9a..f06e019 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -8627,14 +8627,6 @@ mod tests { ] .concat(), ), - ( - "headers over the bound together", - [ - b"HTTP/1.1 200 OK\r\n".to_vec(), - b"X: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\n".repeat(600), - ] - .concat(), - ), ("headers that are not UTF-8", b"HTTP/1.1 200 OK\r\nX: \xff\r\n\r\n".to_vec()), ("a status that is not UTF-8", b"HTTP/1.1 \xff\r\n\r\n".to_vec()), ] { @@ -8642,6 +8634,84 @@ mod tests { } } + /// Every header line is small and the answer is honest and whole; only the + /// headers' total size refuses it. + #[test] + fn headers_that_are_over_their_bound_together_refuse_an_otherwise_honest_answer() { + let filler = b"X: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\n"; + let answer_with = |lines: usize| { + move |id: &str| { + let body = honest_body(id); + let mut reply = + format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n", body.len()).into_bytes(); + reply.extend_from_slice(&filler.repeat(lines)); + reply.extend_from_slice(b"\r\n"); + reply.extend_from_slice(&body); + reply + } + }; + let lines = ROUTE_HTTP_MAX_HEADER_BYTES / filler.len(); + challenge(scripted_peer(answer_with(lines / 2)), |_| {}).expect("headers inside the bound"); + assert_refused( + challenge(scripted_peer(answer_with(lines * 2)), |_| {}), + "headers over the bound together", + ); + } + + /// A reader that fails `interruptions` times with `kind`, then serves `bytes`. + struct Flaky { + kind: ErrorKind, + interruptions: usize, + bytes: &'static [u8], + } + + impl Read for Flaky { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + if self.interruptions > 0 { + self.interruptions -= 1; + return Err(self.kind.into()); + } + let served = self.bytes.len().min(buffer.len()); + buffer[..served].copy_from_slice(&self.bytes[..served]); + self.bytes = &self.bytes[served..]; + Ok(served) + } + } + + #[test] + fn an_interrupted_read_is_retried_and_any_other_failure_is_a_stopped_peer() { + let mut interrupted = Answer::new(Flaky { + kind: ErrorKind::Interrupted, + interruptions: 3, + bytes: b"abcd", + }); + let mut buffer = [0_u8; 4]; + interrupted.read_all(&mut buffer, "in the test").expect("the read is retried"); + assert_eq!(&buffer, b"abcd"); + + // Before any byte the peer is silent; after some, it is refused. + let mut failing = Answer::new(Flaky { + kind: ErrorKind::TimedOut, + interruptions: 1, + bytes: b"abcd", + }); + assert!(matches!( + failing.read_all(&mut buffer, "in the test"), + Err(ChallengeFailure::Silent(_)) + )); + let mut failing = Answer::new(Flaky { + kind: ErrorKind::TimedOut, + interruptions: 0, + bytes: b"ab", + }); + let mut wanted = [0_u8; 3]; + failing.read_all(&mut wanted[..2], "in the test").expect("two bytes"); + assert!(matches!( + failing.read_all(&mut wanted[2..], "in the test"), + Err(ChallengeFailure::Refused(_)) + )); + } + fn assert_silent(outcome: Result, why: &str) { assert!( matches!(outcome, Err(ChallengeFailure::Silent(_))), From 4b3b86d8633ff95db92461229f23a64720d2f463 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 18:19:07 +0200 Subject: [PATCH 042/286] B3 fix: pin the header bound at its edge and the chunked trailer loop Two more mutants survived: the header-size bound was only tested far from its edge, so doubling each line's cost went unseen; and the chunked trailer loop was only exercised with a trailer present, so inverting it went unseen. The bound is now tested one line inside and at its edge, and a chunked answer with no trailer is read from a peer that keeps the socket open. Co-Authored-By: Claude Sonnet 5 --- src/drivers.rs | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/drivers.rs b/src/drivers.rs index f06e019..78eb028 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -8458,14 +8458,18 @@ mod tests { // answered. // --------------------------------------------------------------------- - fn http_chunked(body: &[u8], chunk: usize) -> Vec { + fn http_chunked(body: &[u8], chunk: usize, trailer: bool) -> Vec { let mut response = b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n".to_vec(); for piece in body.chunks(chunk) { response.extend_from_slice(format!("{:x};ext=1\r\n", piece.len()).as_bytes()); response.extend_from_slice(piece); response.extend_from_slice(b"\r\n"); } - response.extend_from_slice(b"0\r\nTrailer: x\r\n\r\n"); + response.extend_from_slice(if trailer { + b"0\r\nTrailer: x\r\n\r\n" + } else { + b"0\r\n\r\n" + }); response } @@ -8487,14 +8491,16 @@ mod tests { assert_eq!(answer.canonical_presence, [1, 2, 3]); assert!(started.elapsed() < LINGER / 2, "the challenge waited on the socket"); - let started = Instant::now(); - let answer = challenge( - lingering_peer(|id| http_chunked(&honest_body(id), 7), LINGER), - |_| {}, - ) - .expect("a chunked answer"); - assert_eq!(answer.canonical_presence, [1, 2, 3]); - assert!(started.elapsed() < LINGER / 2, "the challenge waited on the socket"); + for trailer in [true, false] { + let started = Instant::now(); + let answer = challenge( + lingering_peer(move |id| http_chunked(&honest_body(id), 7, trailer), LINGER), + |_| {}, + ) + .expect("a chunked answer"); + assert_eq!(answer.canonical_presence, [1, 2, 3]); + assert!(started.elapsed() < LINGER / 2, "the challenge waited on the socket"); + } } #[test] @@ -8651,9 +8657,10 @@ mod tests { } }; let lines = ROUTE_HTTP_MAX_HEADER_BYTES / filler.len(); - challenge(scripted_peer(answer_with(lines / 2)), |_| {}).expect("headers inside the bound"); + // Just inside the bound, then just outside it. + challenge(scripted_peer(answer_with(lines - 2)), |_| {}).expect("headers inside the bound"); assert_refused( - challenge(scripted_peer(answer_with(lines * 2)), |_| {}), + challenge(scripted_peer(answer_with(lines)), |_| {}), "headers over the bound together", ); } From e8b44878b76cf88ca01728d157e84e1f4eb07bfa Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 18:23:11 +0200 Subject: [PATCH 043/286] B3 fix: the over-bound header test must be over the bound The previous commit's over-bound case used floor(bound / line) lines, which can still fit under the bound once the status and Content-Length lines are counted; the answer was accepted and the test failed. It now sends one whole line more. Co-Authored-By: Claude Sonnet 5 --- src/drivers.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/drivers.rs b/src/drivers.rs index 78eb028..89d5a3a 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -8657,10 +8657,10 @@ mod tests { } }; let lines = ROUTE_HTTP_MAX_HEADER_BYTES / filler.len(); - // Just inside the bound, then just outside it. + // Just inside the bound, then just outside it (a whole line over). challenge(scripted_peer(answer_with(lines - 2)), |_| {}).expect("headers inside the bound"); assert_refused( - challenge(scripted_peer(answer_with(lines)), |_| {}), + challenge(scripted_peer(answer_with(lines + 1)), |_| {}), "headers over the bound together", ); } From bf2c645a60b632c7999f9dde6c96041da462c392 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 19:35:16 +0200 Subject: [PATCH 044/286] Map: B3 fix batch Soul pass 3 and the held pre-fence cancel ruling Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index bf77513..54502d5 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -55,6 +55,24 @@ transactions of routed targets with no Odin declaration (operator approved, 2026 target, so the declaring redeploy can never freeze. The fix is in Hands. - F5, bad answers read as Silent over HTTP keep-alive and TCP, is in the same batch. +**B3 fix batch (`86ee21d..e8b4487`), Soul pass 3, 2026-09-30: merge with follow-ups.** Nothing here hurts a +live host on install. 294 passed, 2 ignored. `cargo mutants --in-diff`: 53 caught, 0 missed. +- **F1 holds.** There is one continuity mint site, and the hold check comes before it. A declaring redeploy + replaces a dead held incumbent end to end. +- **F5 holds.** Zero bytes is Silent. Any byte then a stall or stop is Refused. Non-2xx responses and + non-chunked Transfer-Encoding are Refused. +- **Follow-ups, in Hands on the same branch:** + - the P1b test never reaches the yield branch (mutant M7 survives); + - the chunked trailer loop is unbounded; + - `+N` lengths and whitespace before the colon are accepted; + - exact 1 MiB and one-byte-then-stop are unpinned; + - the `Silent` doc is stale; + - the dead-hold report goes to stderr only and never clears. + +**Ruling, 2026-09-30 (operator): cancel accepts a held pre-fence Deploy.** Idunn still never aborts a held +record on its own. An operator can discard a held Deploy that has not reached the fence with `idunn cancel`. +Nothing is fenced, so the incumbent is untouched. + **Odin diagnosis (Eyes, read-only, 2026-09-30).** - The 109 Odin route timeouts are self-inflicted. Live Idunn (`8ae00a1`, pre-S1) reloads nginx inside `restore_admitted_membership` before every Odin challenge. The reload moves the reuseport UDP flow, so From 137e120ce95ed3c214179f7e94a9b6d1867da892 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 19:36:54 +0200 Subject: [PATCH 045/286] B3 follow-up: cancel accepts a held pre-fence Deploy; test yield by direct supervision Operator ruling 2026-09-30 (F4): an operator may cancel a held Deploy that has not fenced. Nothing was fenced, so the incumbent is untouched; the pre-fencing abort releases the target. Idunn itself still never aborts a held record: the held_disagreement guard on the yield path is unchanged and pinned. The P1b test now calls supervise_one_admitted_generation directly with the redeploy in Warming over a dead held incumbent, with a positive control (the same redeploy over a dead non-held incumbent yields). Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 171 +++++++++++++++++++++++++++++++++---------- 1 file changed, 133 insertions(+), 38 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 4bd6b59..bba9e4a 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -2447,8 +2447,8 @@ fn parse_cancel(mut args: impl Iterator) -> Result { }) } -/// Withdraw queued work, or ask Idunn to clean up a live stateless candidate. -/// Stateful candidates past fencing cannot be cancelled because their prior +/// Withdraw queued work, a held deployment that has not fenced, or ask Idunn to +/// clean up a live stateless candidate. Stateful candidates past fencing cannot be cancelled because their prior /// writer may already be stopped and its state boundary needs explicit repair. fn cancel(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> { let snapshot = ControlSnapshot::read(store_path)?; @@ -2471,12 +2471,17 @@ fn cancel(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> "deployment command has multiple live transactions" ); ensure!( - cancel_is_safe_for_live_stateless(¤t.value), - "live deployment can only be cancelled after fencing when its candidate is stateless" + cancel_is_safe_for_live_stateless(¤t.value) + || cancel_is_safe_for_held_prefence(¤t.value), + "live deployment can only be cancelled before fencing when Idunn holds it, or after fencing when its candidate is stateless" ); let error = format!("cancelled by {requested_by}"); let mut next = current.value.clone(); - next.post_fencing_abort = Some(post_fencing_abort_intent(&next, &error)); + if cancel_is_safe_for_held_prefence(&next) { + next.pre_fencing_abort = Some(pre_fencing_abort_intent(&next, &error)); + } else { + next.post_fencing_abort = Some(post_fencing_abort_intent(&next, &error)); + } next.last_error = Some(error); next.updated_at_unix_millis = now_millis()?; replace_transaction(store_path, current, &next)?; @@ -2513,6 +2518,14 @@ fn cancel(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> Ok(()) } +/// A deployment Idunn holds before it fenced anything has touched nothing of +/// the incumbent, so an operator may withdraw it. Idunn never does so itself. +fn cancel_is_safe_for_held_prefence(transaction: &DeploymentTransaction) -> bool { + transaction.command_kind == CommandKind::Deploy + && transaction.phase < DeploymentPhase::Fencing + && transaction.held_disagreement().is_some() +} + fn cancel_is_safe_for_live_stateless(transaction: &DeploymentTransaction) -> bool { transaction.command_kind == CommandKind::Deploy && transaction.phase >= DeploymentPhase::Fencing @@ -7582,19 +7595,7 @@ impl Engine { current.value.pre_fencing_abort.is_none(), "pre-fencing abort intent is already durable" ); - let abort = PreFencingAbort { - error: truncate(&format!("{error:#}"), 2048), - candidate_cleanup: candidate_cleanup_requirement( - current.value.activation.is_some(), - current.value.workload.is_some(), - ), - topology_reconciliation: current.value.abort_topology_reconciliation(), - source_cleanup: if current.value.command_kind == CommandKind::Deploy { - CleanupEvidence::Pending - } else { - CleanupEvidence::Skipped - }, - }; + let abort = pre_fencing_abort_intent(¤t.value, &format!("{error:#}")); self.persist_same_phase(current, |next| { next.pre_fencing_abort = Some(abort); Ok(()) @@ -7987,6 +7988,22 @@ fn incumbent_was_stopped_during_fencing(fencing: &FencingEvidence) -> bool { ) } +fn pre_fencing_abort_intent(transaction: &DeploymentTransaction, error: &str) -> PreFencingAbort { + PreFencingAbort { + error: truncate(error, 2048), + candidate_cleanup: candidate_cleanup_requirement( + transaction.activation.is_some(), + transaction.workload.is_some(), + ), + topology_reconciliation: transaction.abort_topology_reconciliation(), + source_cleanup: if transaction.command_kind == CommandKind::Deploy { + CleanupEvidence::Pending + } else { + CleanupEvidence::Skipped + }, + } +} + fn post_fencing_abort_intent( transaction: &DeploymentTransaction, error: &str, @@ -13864,7 +13881,7 @@ mod tests { } #[test] - fn a_declaring_redeploy_is_not_yielded_to_a_continuity_when_the_held_incumbent_dies() -> Result<()> { + fn a_declaring_redeploy_is_not_yielded_when_the_held_incumbent_dies() -> Result<()> { let workload = SwitchWorkload::new(); let world = EngineFixture::with_workload(workload.clone())?; admit_incumbent(&world)?; @@ -13873,39 +13890,45 @@ mod tests { seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&held))?; assert_eq!(redeploy.held_disagreement(), None); workload.kill(); - for _ in 0..40 { - let _ = world.engine.run_scheduler_tick(); + for _ in 0..4 { + let _ = world.engine.supervise_one_admitted_generation()?; } let all = every_transaction(&world)?; assert!( all.iter().all(|transaction| !is_continuity_over(transaction, &held)), "a continuity was minted over the held incumbent" ); - let redeploy = all - .iter() - .find(|transaction| transaction.transaction_id == redeploy.transaction_id) - .context("the redeploy is nowhere")?; - // The fixture's one workload is the candidate too, so the redeploy may - // fail for its own reasons; it is never yielded away. + let after = resident(&world)?.value; + assert_eq!(after.transaction_id, redeploy.transaction_id); assert!( - redeploy - .pre_fencing_abort - .as_ref() - .is_none_or(|abort| !abort.error.contains("yielded to continuity")), + after.pre_fencing_abort.is_none(), "the redeploy yielded: {:?}", - redeploy.pre_fencing_abort + after.pre_fencing_abort ); Ok(()) } + /// The control for the test above: the same redeploy over a dead incumbent + /// that is not held is yielded to continuity. #[test] - fn a_held_predeploy_transaction_is_not_aborted_to_yield_to_continuity() -> Result<()> { + fn a_declaring_redeploy_still_yields_to_continuity_when_the_incumbent_is_not_held() -> Result<()> { let workload = SwitchWorkload::new(); let world = EngineFixture::with_workload(workload.clone())?; let incumbent = admit_incumbent(&world)?; - // The incumbent is declared; the Deploy in front of it is the held one. - let seeded = + assert!(incumbent.readiness().is_ok()); + let redeploy = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + workload.kill(); + world.engine.supervise_one_admitted_generation()?; + let after = resident(&world)?.value; + assert_eq!(after.transaction_id, redeploy.transaction_id); + let abort = after.pre_fencing_abort.context("the redeploy was not yielded")?; + assert!(abort.error.contains("yielded to continuity"), "{}", abort.error); + Ok(()) + } + + /// `seeded` with its own Expected undeclared, so Idunn holds it. + fn undeclared(world: &EngineFixture, seeded: &DeploymentTransaction) -> Result { let mut bare = seeded.clone(); // Undeclare the Expected and re-derive what is bound to its digest. let expected = bare.expected.as_mut().context("no expected")?; @@ -13922,21 +13945,93 @@ mod tests { bare.activation_publication_sha256 = Some(activation.canonical_sha256()?); bare.activation = Some(activation); bare.validate()?; - let stored = resident(&world)?; - replace_transaction(&world.state_store, &stored, &bare)?; assert!(bare.held_disagreement().is_some()); + Ok(bare) + } + + /// A pre-fence Deploy whose own Expected is undeclared, stored in place. + fn held_predeploy( + world: &EngineFixture, + incumbent: &AdmittedGeneration, + ) -> Result { + let seeded = + seeded_transaction(world, DeploymentPhase::Warming, CommandKind::Deploy, Some(incumbent))?; + let bare = undeclared(world, &seeded)?; + replace_transaction(&world.state_store, &resident(world)?, &bare)?; + Ok(bare) + } + + #[test] + fn a_held_predeploy_transaction_is_not_aborted_to_yield_to_continuity() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + // The incumbent is declared; the Deploy in front of it is the held one. + let held = held_predeploy(&world, &incumbent)?; workload.kill(); for _ in 0..4 { let _ = world.engine.supervise_one_admitted_generation()?; } let after = resident(&world)?.value; - assert_eq!(after.transaction_id, seeded.transaction_id); + assert_eq!(after.transaction_id, held.transaction_id); assert!(after.pre_fencing_abort.is_none(), "Idunn aborted a held record"); assert!(after.completion.is_none()); Ok(()) } + #[test] + fn cancel_withdraws_a_held_predeploy_and_leaves_the_incumbent_untouched() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + let held = held_predeploy(&world, &incumbent)?; + let before = incumbent_envelope(&world)?; + + cancel(&world.state_store, &held.command_id, "operator")?; + let cancelling = resident(&world)?.value; + assert_eq!( + cancelling.pre_fencing_abort.as_ref().map(|abort| abort.error.as_str()), + Some("cancelled by operator") + ); + assert!(cancelling.post_fencing_abort.is_none()); + + drive(&world, |transaction| transaction.completion.is_some())?; + let finished = latest(&world)?; + assert!(matches!( + finished.completion, + Some(TransactionCompletion::FailedBeforeFencing { .. }) + )); + assert!(finished.is_terminal()); + // The target is released and nothing of the incumbent was touched. + assert!(!finished.blocks_new_target_mutation()); + assert_eq!(incumbent_envelope(&world)?, before); + assert!( + every_transaction(&world)? + .iter() + .all(|transaction| !is_continuity_over(transaction, &incumbent)), + "cancelling the redeploy minted a continuity" + ); + Ok(()) + } + + #[test] + fn cancel_refuses_a_predeploy_that_is_not_held_and_a_held_one_past_the_fence() -> Result<()> { + let world = EngineFixture::with_workload(SwitchWorkload::new())?; + let incumbent = admit_incumbent(&world)?; + let seeded = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + assert!(seeded.held_disagreement().is_none()); + assert!(cancel(&world.state_store, &seeded.command_id, "operator").is_err()); + assert!(resident(&world)?.value.pre_fencing_abort.is_none()); + + let mut past_the_fence = undeclared(&world, &seeded)?; + past_the_fence.enter_phase(DeploymentPhase::Fencing, now_millis()?); + assert!(past_the_fence.held_disagreement().is_some()); + assert!(!cancel_is_safe_for_held_prefence(&past_the_fence)); + Ok(()) + } + // --------------------------------------------------------------------- // Route-proof readiness: a routed target that declares no Odin dependency // is admitted by Idunn's own challenges. Odin is never read. From 8432d45bbeda127b8f1423fbf262b37334e2188a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 19:36:54 +0200 Subject: [PATCH 046/286] B3 follow-up: bound the chunked trailer loop, refuse signed numbers and spaced header names Trailers draw on the header budget instead of an unbounded loop. Content-Length and chunk sizes are digits only (RFC 9112), and a header name with whitespace is refused. Pins found by cargo-mutants: partial trailer, exact bounds on all three framings, one byte then stop, unterminated chunk from a peer that keeps going. ChallengeFailure::Silent documents what it actually is. Co-Authored-By: Claude Opus 5.5 --- src/drivers.rs | 279 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 267 insertions(+), 12 deletions(-) diff --git a/src/drivers.rs b/src/drivers.rs index 89d5a3a..400566b 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -5566,9 +5566,10 @@ impl NginxRouteDriver { /// Why a direct presence challenge produced no document to authenticate. #[derive(Debug)] pub enum ChallengeFailure { - /// Nothing answered: the connection was refused, timed out, or dropped, or - /// the peer closed it without a byte. A process still starting does this, - /// and it is waiting, not a fault. + /// Nothing answered: the connection was refused, or no byte arrived before + /// the timeout, or the peer closed with zero bytes. A process still + /// starting does this, and it is waiting, not a fault. Once any byte has + /// arrived, a stall or a stop is `Refused`, not this. Silent(anyhow::Error), /// Something answered wrongly, or the challenge could not be made: a /// malformed or foreign answer, an oversized one, or a local configuration @@ -5725,8 +5726,9 @@ fn request_http_snapshot(target: SocketAddr, payload: &[u8]) -> Result, // answered in full. let mut answer = Answer::new(BufReader::new(stream)); let mut head_bytes = 0_usize; - let mut head_line = |answer: &mut Answer>| { - let line = answer.read_line(ROUTE_HTTP_MAX_HEADER_BYTES, "before its headers ended")?; + // Headers and chunked trailers draw on the one budget. + let mut head_line = |answer: &mut Answer>, what: &str| { + let line = answer.read_line(ROUTE_HTTP_MAX_HEADER_BYTES, what)?; head_bytes += line.len() + 2; ensure_or_refuse( head_bytes <= ROUTE_HTTP_MAX_HEADER_BYTES, @@ -5734,7 +5736,8 @@ fn request_http_snapshot(target: SocketAddr, payload: &[u8]) -> Result, )?; Ok::<_, ChallengeFailure>(line) }; - let status = head_line(&mut answer)?; + let headers = "before its headers ended"; + let status = head_line(&mut answer, headers)?; let status = std::str::from_utf8(&status) .context("stable CultNet HTTP response status is not UTF-8") .map_err(refused)?; @@ -5747,7 +5750,7 @@ fn request_http_snapshot(target: SocketAddr, payload: &[u8]) -> Result, let mut content_length = None; let mut chunked = false; loop { - let line = head_line(&mut answer)?; + let line = head_line(&mut answer, headers)?; if line.is_empty() { break; } @@ -5758,6 +5761,12 @@ fn request_http_snapshot(target: SocketAddr, payload: &[u8]) -> Result, .split_once(':') .context("stable CultNet HTTP response contains a malformed header") .map_err(refused)?; + // RFC 9112 5.1: a name with whitespace around it (or none) is not a + // field name, and a recipient must not guess which one was meant. + ensure_or_refuse( + !name.is_empty() && !name.bytes().any(|byte| byte.is_ascii_whitespace()), + "stable CultNet HTTP response has a malformed header name", + )?; if name.eq_ignore_ascii_case("transfer-encoding") { ensure_or_refuse( value.trim().eq_ignore_ascii_case("chunked"), @@ -5771,9 +5780,7 @@ fn request_http_snapshot(target: SocketAddr, payload: &[u8]) -> Result, "stable CultNet HTTP response repeats Content-Length", )?; content_length = Some( - value - .trim() - .parse::() + parse_digits(value.trim_matches([' ', '\t']), 10) .context("stable CultNet HTTP Content-Length is invalid") .map_err(refused)?, ); @@ -5798,12 +5805,12 @@ fn request_http_snapshot(target: SocketAddr, payload: &[u8]) -> Result, let size = std::str::from_utf8(&size) .ok() .and_then(|size| size.split(';').next()) - .and_then(|size| usize::from_str_radix(size.trim(), 16).ok()) + .and_then(|size| parse_digits(size.trim_matches([' ', '\t']), 16)) .context("stable CultNet HTTP chunk size is invalid") .map_err(refused)?; if size == 0 { // Trailers, if any, end at the empty line. - while !answer.read_line(ROUTE_HTTP_MAX_HEADER_BYTES, body)?.is_empty() {} + while !head_line(&mut answer, body)?.is_empty() {} return Ok(response); } ensure_or_refuse( @@ -5824,6 +5831,16 @@ fn request_http_snapshot(target: SocketAddr, payload: &[u8]) -> Result, } } +/// A number written only in digits of `radix`: no sign, no whitespace, not +/// empty. `from_str_radix` alone accepts a leading `+`, which RFC 9112 does +/// not. +fn parse_digits(text: &str, radix: u32) -> Option { + if text.is_empty() || !text.chars().all(|digit| digit.is_digit(radix)) { + return None; + } + usize::from_str_radix(text, radix).ok() +} + /// A 2xx status line of HTTP/1.0 or HTTP/1.1. fn is_http_success(status: &str) -> bool { let Some(rest) = status @@ -8665,6 +8682,244 @@ mod tests { ); } + // --------------------------------------------------------------------- + // The reader's limits are pinned at their edges, on answers that are + // honest in every other way, so only the limit under test can refuse them + // and a peer that simply stops is not what refuses. + // --------------------------------------------------------------------- + + const CHUNKED_HEAD: &[u8] = b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n"; + + fn snapshot_request() -> Vec { + encode_cultnet_message_to_vec( + &CultNetMessage::SnapshotRequest { + message_id: "probe".into(), + schema_ids: None, + record_keys: None, + }, + CultNetWireContract::CultNetSchemaV0, + ) + .expect("an encodable request") + } + + /// The raw body the HTTP reader hands back, with no document judgment. + fn http_body( + reply: impl FnOnce(&str) -> Vec + Send + 'static, + ) -> Result, ChallengeFailure> { + let port = scripted_peer(reply); + request_http_snapshot(SocketAddr::from(([127, 0, 0, 1], port)), &snapshot_request()) + } + + fn tcp_body( + reply: impl FnOnce(&str) -> Vec + Send + 'static, + ) -> Result, ChallengeFailure> { + let port = framed_peer(reply); + request_tcp_snapshot(SocketAddr::from(([127, 0, 0, 1], port)), &snapshot_request()) + } + + fn assert_body_refused(outcome: Result, ChallengeFailure>, why: &str) { + assert!( + matches!(outcome, Err(ChallengeFailure::Refused(_))), + "{why} must be refused: {:?}", + outcome.map(|body| body.len()) + ); + } + + fn chunked(pieces: &[usize]) -> Vec { + let mut reply = CHUNKED_HEAD.to_vec(); + for size in pieces { + reply.extend_from_slice(format!("{size:x}\r\n").as_bytes()); + reply.extend(std::iter::repeat_n(b'a', *size)); + reply.extend_from_slice(b"\r\n"); + } + reply.extend_from_slice(b"0\r\n\r\n"); + reply + } + + #[test] + fn every_framing_takes_exactly_the_bound_and_refuses_one_byte_more() { + let most = ROUTE_SNAPSHOT_MAX_BYTES; + // Whole answers are sent in every case, so only the bound refuses. + let http_ok_of = |length: usize| move |_: &str| http_ok(&vec![b'a'; length]); + assert_eq!(http_body(http_ok_of(most)).expect("Content-Length at the bound").len(), most); + assert_body_refused(http_body(http_ok_of(most + 1)), "Content-Length over the bound"); + + for (pieces, taken) in [ + (vec![most], true), + (vec![most + 1], false), + (vec![most - 1, 1], true), + (vec![most - 1, 2], false), + ] { + let outcome = http_body(move |_| chunked(&pieces)); + if taken { + assert_eq!(outcome.expect("chunks at the bound").len(), most); + } else { + assert_body_refused(outcome, "chunks over the bound"); + } + } + + let framed = |length: usize| move |_: &str| encode_frame(&vec![b'a'; length]).expect("a frame"); + assert_eq!(tcp_body(framed(most)).expect("a frame at the bound").len(), most); + assert_eq!(tcp_body(framed(1)).expect("the smallest frame").len(), 1); + assert_body_refused(tcp_body(framed(most + 1)), "a frame over the bound"); + assert_body_refused(tcp_body(|_| 0_u32.to_be_bytes().to_vec()), "an empty frame"); + } + + #[test] + fn a_peer_that_sends_one_byte_and_stops_is_refused_not_silent() { + assert_body_refused(http_body(|_| b"H".to_vec()), "one HTTP byte"); + assert_body_refused(tcp_body(|_| vec![0]), "one TCP byte"); + assert!(matches!(http_body(|_| Vec::new()), Err(ChallengeFailure::Silent(_)))); + assert!(matches!(tcp_body(|_| Vec::new()), Err(ChallengeFailure::Silent(_)))); + } + + /// An honest chunked answer with `after_body` in place of its end. + fn chunked_honest_then(id: &str, after_body: &[u8]) -> Vec { + let body = honest_body(id); + let mut reply = CHUNKED_HEAD.to_vec(); + reply.extend_from_slice(format!("{:x}\r\n", body.len()).as_bytes()); + reply.extend_from_slice(&body); + reply.extend_from_slice(b"\r\n"); + reply.extend_from_slice(after_body); + reply + } + + #[test] + fn a_chunked_answer_must_end_its_trailers_and_terminate_each_chunk() { + // The same answer, ended properly, is taken. + challenge(scripted_peer(|id| chunked_honest_then(id, b"0\r\n\r\n")), |_| {}) + .expect("an honest chunked answer"); + challenge(scripted_peer(|id| chunked_honest_then(id, b"0\r\nT: 1\r\nU: 2\r\n\r\n")), |_| {}) + .expect("an honest chunked answer with trailers"); + + // A trailer that begins and then the peer stops. + assert_refused( + challenge(scripted_peer(|id| chunked_honest_then(id, b"0\r\nT: 1\r\n")), |_| {}), + "a partial trailer section", + ); + + // Chunk data followed by something other than CRLF, from a peer that + // then carries on as though nothing were wrong. + assert_refused( + challenge( + lingering_peer( + |id| { + let body = honest_body(id); + let mut reply = CHUNKED_HEAD.to_vec(); + reply.extend_from_slice(format!("{:x}\r\n", body.len()).as_bytes()); + reply.extend_from_slice(&body); + reply.extend_from_slice(b"XX0\r\n\r\n"); + reply + }, + LINGER, + ), + |_| {}, + ), + "a chunk that is not terminated", + ); + } + + /// Headers and trailers draw on one budget. The head is fixed, so the + /// trailer line that lands the total exactly on the bound is computed. + #[test] + fn trailers_share_the_header_budget_and_are_bounded_at_its_edge() { + // The trailer line is `T: `, counted with its CRLF, plus the + // empty line ending the trailers (2): together they fill what the head + // left of the budget. + let padding = ROUTE_HTTP_MAX_HEADER_BYTES - CHUNKED_HEAD.len() - 2 - 2 - "T: ".len(); + let reply_with = move |padding: usize| { + move |id: &str| { + let mut trailers = format!("0\r\nT: {}\r\n", "a".repeat(padding)).into_bytes(); + trailers.extend_from_slice(b"\r\n"); + chunked_honest_then(id, &trailers) + } + }; + challenge(scripted_peer(reply_with(padding)), |_| {}).expect("trailers exactly at the bound"); + + let started = Instant::now(); + assert_refused( + challenge(lingering_peer(reply_with(padding + 1), LINGER), |_| {}), + "trailers one byte over the bound", + ); + assert!(started.elapsed() < ROUTE_SNAPSHOT_TIMEOUT / 2, "the refusal waited on the socket"); + + // Trailers that never end are cut off at the bound, not at the timeout. + let started = Instant::now(); + assert_refused( + challenge( + lingering_peer( + |id| { + let line = b"T: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\n"; + let mut endless = b"0\r\n".to_vec(); + endless.extend_from_slice( + &line.repeat(2 * ROUTE_HTTP_MAX_HEADER_BYTES / line.len()), + ); + chunked_honest_then(id, &endless) + }, + LINGER, + ), + |_| {}, + ), + "trailers that never end", + ); + assert!(started.elapsed() < ROUTE_SNAPSHOT_TIMEOUT / 2, "the refusal waited on the socket"); + } + + /// RFC 9112 strictness, on answers that are honest but for the one field. + #[test] + fn numbers_and_header_names_are_read_strictly() { + // `{field}:{value}` then the true length, so a value of " +" is a signed one. + let content_length = |field: &'static str, value: &'static str| { + move |id: &str| { + let body = honest_body(id); + let mut reply = + format!("HTTP/1.1 200 OK\r\n{field}:{value}{}\r\n\r\n", body.len()).into_bytes(); + reply.extend_from_slice(&body); + reply + } + }; + // Optional whitespace around a value is allowed. + challenge(scripted_peer(content_length("Content-Length", " \t ")), |_| {}) + .expect("Content-Length with optional whitespace"); + for (why, field, value) in [ + ("a signed Content-Length", "Content-Length", " +"), + ("whitespace before the colon", "Content-Length ", " "), + ("a header name with leading whitespace", " Content-Length", " "), + ] { + assert_refused(challenge(scripted_peer(content_length(field, value)), |_| {}), why); + } + assert_refused( + challenge( + scripted_peer(|id| { + let body = honest_body(id); + let mut reply = format!( + "HTTP/1.1 200 OK\r\nX-Other : y\r\nContent-Length: {}\r\n\r\n", + body.len() + ) + .into_bytes(); + reply.extend_from_slice(&body); + reply + }), + |_| {}, + ), + "whitespace before the colon of another header", + ); + assert_refused( + challenge( + scripted_peer(|id| { + let body = honest_body(id); + let mut reply = CHUNKED_HEAD.to_vec(); + reply.extend_from_slice(format!("+{:x}\r\n", body.len()).as_bytes()); + reply.extend_from_slice(&body); + reply.extend_from_slice(b"\r\n0\r\n\r\n"); + reply + }), + |_| {}, + ), + "a signed chunk size", + ); + } + /// A reader that fails `interruptions` times with `kind`, then serves `bytes`. struct Flaky { kind: ErrorKind, From d34f5dc3f9a4fa74c2ec570c4892229ce776491f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 19:38:28 +0200 Subject: [PATCH 047/286] wip: show the completion in the cancel test Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index bba9e4a..26934fd 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -13998,10 +13998,12 @@ mod tests { drive(&world, |transaction| transaction.completion.is_some())?; let finished = latest(&world)?; - assert!(matches!( + assert!( + matches!(finished.completion, Some(TransactionCompletion::FailedBeforeFencing { .. })), + "{:?} / {:?}", finished.completion, - Some(TransactionCompletion::FailedBeforeFencing { .. }) - )); + finished.pre_fencing_abort + ); assert!(finished.is_terminal()); // The target is released and nothing of the incumbent was touched. assert!(!finished.blocks_new_target_mutation()); From 5c49646b81e563a5242f7697704f71b3ffc7aa75 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 19:39:55 +0200 Subject: [PATCH 048/286] B3 follow-up: find the cancelled redeploy by id, not by history order Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 26934fd..7943698 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -13997,13 +13997,15 @@ mod tests { assert!(cancelling.post_fencing_abort.is_none()); drive(&world, |transaction| transaction.completion.is_some())?; - let finished = latest(&world)?; - assert!( - matches!(finished.completion, Some(TransactionCompletion::FailedBeforeFencing { .. })), - "{:?} / {:?}", + // History also holds the incumbent's own admission: find ours by id. + let finished = every_transaction(&world)? + .into_iter() + .find(|transaction| transaction.transaction_id == held.transaction_id) + .context("the cancelled redeploy is nowhere")?; + assert!(matches!( finished.completion, - finished.pre_fencing_abort - ); + Some(TransactionCompletion::FailedBeforeFencing { .. }) + )); assert!(finished.is_terminal()); // The target is released and nothing of the incumbent was touched. assert!(!finished.blocks_new_target_mutation()); From ae3788909ad745aa4ad96483abc507beaacba3eb Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 20:15:08 +0200 Subject: [PATCH 049/286] Map: B3 follow-ups and the dead-hold report ruling Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index 54502d5..241115c 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -73,6 +73,23 @@ live host on install. 294 passed, 2 ignored. `cargo mutants --in-diff`: 53 caugh record on its own. An operator can discard a held Deploy that has not reached the fence with `idunn cancel`. Nothing is fenced, so the incumbent is untouched. +**B3 follow-ups landed on `idunn/route-b3` at `5c49646`, 2026-09-30.** +- Cancel accepts a held pre-fence Deploy. It shares `pre_fencing_abort_intent` with the scheduler. +- The P1b test is now in direct-supervision form, with a positive control. +- The reader is strict: trailers share the 32 KiB header budget; digits only; header names without whitespace. +- The `Silent` doc is corrected. +- Tests: 301 pass. `a_failing_step_is_retried_after_a_backoff_not_every_tick` is flaky under a full parallel + run, on the base too. +- `cargo mutants --in-diff`: 24 caught, 0 missed. +- A narrow Soul pass gates the install. + +**Ruling, 2026-09-30 (operator): the dead-hold report is stored on the generation.** +- Add an optional `last_error` to `AdmittedGeneration` (new key 21), and make `status` render admitted + generations. +- It is cleared when the generation is replaced or its hold is released. +- The `generation:` readiness report gets the same home. +- It is queued after the ship, with the B2 rework. + **Odin diagnosis (Eyes, read-only, 2026-09-30).** - The 109 Odin route timeouts are self-inflicted. Live Idunn (`8ae00a1`, pre-S1) reloads nginx inside `restore_admitted_membership` before every Odin challenge. The reload moves the reuseport UDP flow, so From d3db58227a56361811f3a78933e575de3a826ebf Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 20:57:45 +0200 Subject: [PATCH 050/286] Define the frozen-source symlink limit on Windows so idunn-host builds The constant was unix-only since 9475dfa while its uses are not, so the Windows idunn-host has not compiled since S1. Found installing B3: Raven's actuator (built 2026-09-11) cannot decode the new plan types. Co-Authored-By: Claude Opus 5.5 --- src/drivers.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/drivers.rs b/src/drivers.rs index 400566b..fbf9558 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -68,6 +68,9 @@ struct GitTreeEntry { /// finally closed it. #[cfg(unix)] const FROZEN_SOURCE_SYMLINK_TARGET_LIMIT: usize = libc::PATH_MAX as usize; +/// Windows' long-path limit, in UTF-16 units; the same bound on a symlink blob's size. +#[cfg(not(unix))] +const FROZEN_SOURCE_SYMLINK_TARGET_LIMIT: usize = 32_767; /// F2/F3 layer (b): the only way `materialize_tree_raw` is allowed to obtain /// a directory to write into. `path` must already be a directory this From 28662cb807c3dd7cb2af5b66b64a3e3c95c6617f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 21:05:35 +0200 Subject: [PATCH 051/286] Map: ship log for the B3 install, the Raven actuator and the bulk fetch bug Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index 241115c..49696b1 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -83,6 +83,22 @@ Nothing is fenced, so the incumbent is untouched. - `cargo mutants --in-diff`: 24 caught, 0 missed. - A narrow Soul pass gates the install. +**Ship log, 2026-09-29 (host clock).** +- B3 merged at `d32395a` and installed on Yggdrasil at 18:36 UTC, with sha `854aa351`. It migrated 5 control + records. The store backup is `/root/idunn-store-backup-20260929T183604Z-pre-route-b3`. +- Nginx reloads fell from 82 per 10 minutes to 0. Odin continuity timeouts fell from 92 in 6 hours to 0. +- **Raven's actuator (built 2026-09-11) could not decode the new plan types.** A rebuild then found that the + Windows `idunn-host` has not compiled since S1: `FROZEN_SOURCE_SYMLINK_TARGET_LIMIT` was defined for unix + only. That is fixed at `d3db582`. + - The new actuator (sha `f36bf1fc`) is swapped in on Raven. The old one is kept as + `idunn-host.exe.prev-20260929T190006Z`. + - Follow-up: nothing builds the Windows actuator in verification. +- raven-muninn was admitted at 19:02 by Odin correlation, replacing the held generation. +- **The streampixels-service deploy failed before fencing.** The S1 `bulk_fetch_objects` asks for bare blob + ids with `git fetch --stdin`. In a `blob:none` clone, the connectivity check then fails with "bad revision + ". The fix, with a test, is in Hands on `fix/bulk-blob-fetch`: a promisor-style fetch with + `--filter=blob:none` and `fetch.negotiationAlgorithm=noop`. + **Ruling, 2026-09-30 (operator): the dead-hold report is stored on the generation.** - Add an optional `last_error` to `AdmittedGeneration` (new key 21), and make `status` render admitted generations. From 7f776ec3a6fe1c384e304f539037de1d187faa5c Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 21:06:03 +0200 Subject: [PATCH 052/286] Pin bulk_fetch_objects against a real blob:none partial clone Every existing source-driver fixture uses an origin without uploadpack.allowFilter, so --filter=blob:none is ignored, the clone is full, and bulk_fetch_objects never has a missing blob to fetch. This fixture advertises the filter, so the checkout really lacks the blob, and checks delivery with GIT_NO_LAZY_FETCH=1 so only the bulk fetch can supply it. It fails on the current fetch arguments. Co-Authored-By: Claude Opus 5.5 --- src/drivers.rs | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/src/drivers.rs b/src/drivers.rs index fbf9558..83a76c6 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -10557,6 +10557,72 @@ Content-Le".to_vec()), |_| {}), Ok(()) } + /// A real partial clone: the origin advertises `uploadpack.allowFilter`, + /// so `--filter=blob:none` is honoured and the checkout holds no blobs. + /// Every other fixture's origin ignores the filter and clones in full, + /// so `bulk_fetch_objects` never had a missing blob to fetch and never + /// met Git's post-fetch connectivity check on a bare blob id. The blob + /// is checked with `GIT_NO_LAZY_FETCH=1`, so only the bulk fetch itself + /// can have delivered it. + #[cfg(unix)] + #[test] + fn bulk_fetch_objects_delivers_a_blob_into_a_blob_none_partial_clone() -> Result<()> { + let temp = tempfile::tempdir()?; + let origin_repo = temp.path().join("origin"); + fs::create_dir(&origin_repo)?; + git_at(&origin_repo, &["init", "--initial-branch=main"])?; + git_at(&origin_repo, &["config", "user.name", "Idunn Test"])?; + git_at( + &origin_repo, + &["config", "user.email", "idunn-test@example.invalid"], + )?; + git_at(&origin_repo, &["config", "uploadpack.allowFilter", "true"])?; + git_at(&origin_repo, &["config", "uploadpack.allowAnySHA1InWant", "true"])?; + fs::write(origin_repo.join("payload.txt"), b"partial clone payload\n")?; + git_at(&origin_repo, &["add", "--all"])?; + git_at(&origin_repo, &["commit", "-m", "fixture"])?; + let blob = git_at(&origin_repo, &["rev-parse", "HEAD:payload.txt"])?; + + let source_cache_root = temp.path().join("source-cache"); + fs::create_dir(&source_cache_root)?; + let driver = GitSourceDriver::new( + source_cache_root.clone(), + temp.path().join("frozen"), + None, + ); + let checkout = source_cache_root.join("checkout"); + driver.git([ + OsString::from("clone"), + OsString::from("--filter=blob:none"), + OsString::from("--no-checkout"), + OsString::from(format!("file://{}", origin_repo.display())), + checkout.as_os_str().to_owned(), + ])?; + + let blob_present = || -> Result { + Ok(Command::new("/usr/bin/git") + .arg("-C") + .arg(&checkout) + .args(["cat-file", "-t", &blob]) + .env_clear() + .env("PATH", "/usr/bin:/bin") + .env("GIT_NO_LAZY_FETCH", "1") + .output()? + .status + .success()) + }; + ensure!( + !blob_present()?, + "fixture is not a partial clone: the blob is already local" + ); + driver.bulk_fetch_objects(&checkout, std::slice::from_ref(&blob))?; + assert!( + blob_present()?, + "bulk fetch succeeded but did not deliver the blob" + ); + Ok(()) + } + /// F2/F3 (Self's rulings, second Cut 1 fix batch): a tree with two /// entries literally named `a` at the same level -- one a symlink /// pointing outside the frozen root, one a subtree holding `pwn` -- must From f9cebb33d0ce12514c31ff7b0eb45819d66937e2 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 21:12:06 +0200 Subject: [PATCH 053/286] Pin the partial-clone bulk fetch arguments in the spy test A file:// origin delivers an explicitly wanted blob with or without the promisor-style flags, so the real-shape fixture cannot fail on the bug that GitHub triggers. The spy test already records the bulk fetch line; assert it carries fetch.negotiationAlgorithm=noop, --recurse-submodules=no and --filter=blob:none. Fails on the current arguments. Co-Authored-By: Claude Opus 5.5 --- src/drivers.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/drivers.rs b/src/drivers.rs index 83a76c6..c39f0bd 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -10554,6 +10554,24 @@ Content-Le".to_vec()), |_| {}), bulk_fetches, 1, "expected exactly one bulk --stdin fetch; invocations:\n{log_text}" ); + // Git's own lazy fetch for a promisor remote sends exactly these; a + // hosted origin (GitHub) omits an explicitly wanted blob, and Git's + // connectivity check then fails, without them. A file:// origin + // delivers the blob either way, so only the arguments can pin this. + let bulk_line = log_text + .lines() + .find(|line| line.contains("fetch") && line.contains("--stdin")) + .unwrap_or_default(); + for required in [ + "fetch.negotiationAlgorithm=noop", + "--recurse-submodules=no", + "--filter=blob:none", + ] { + assert!( + bulk_line.contains(required), + "bulk fetch lacks {required}: {bulk_line}" + ); + } Ok(()) } From 08d94ee61be50cab10b70a8590b3fa8308c42141 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 21:13:22 +0200 Subject: [PATCH 054/286] Fetch blobs in bulk the way Git's lazy fetch does for a promisor remote In a blob:none partial clone, fetching bare blob ids with --stdin made Git's post-fetch connectivity check treat the id as a revision and fail with "bad revision" / "did not send all necessary objects" against GitHub, which broke a live deploy fetching a vendor gitlink's blob. Add fetch.negotiationAlgorithm=noop, --recurse-submodules=no and --filter=blob:none, the arguments Git's own lazy fetch uses. Co-Authored-By: Claude Opus 5.5 --- src/drivers.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/drivers.rs b/src/drivers.rs index c39f0bd..6af3559 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -1133,6 +1133,10 @@ impl GitSourceDriver { /// in depth: Git itself refuses a fetched tree with duplicate names /// before any of it reaches disk, independent of the explicit `fsck` /// call S1 added around the revision this bulk fetch serves. + /// The noop negotiation, `--recurse-submodules=no` and explicit + /// `--filter=blob:none` are the arguments Git's own lazy fetch uses for a + /// promisor remote; without them a hosted origin omits the wanted blob and + /// Git's connectivity check fails on the bare blob id. fn bulk_fetch_objects(&self, repository: &Path, objects: &[String]) -> Result<()> { if objects.is_empty() { return Ok(()); @@ -1155,11 +1159,15 @@ impl GitSourceDriver { let mut command = self.git_command([ OsString::from("-c"), OsString::from("transfer.fsckObjects=true"), + OsString::from("-c"), + OsString::from("fetch.negotiationAlgorithm=noop"), OsString::from("-C"), repository.as_os_str().to_owned(), OsString::from("fetch"), OsString::from("--no-tags"), OsString::from("--no-write-fetch-head"), + OsString::from("--recurse-submodules=no"), + OsString::from("--filter=blob:none"), OsString::from("--stdin"), OsString::from("origin"), ])?; From 7480ee9d1c7288a27ed63859ee7d62bb9dab28d1 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 21:55:34 +0200 Subject: [PATCH 055/286] Map: StreamPixels ship completed through Idunn Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index 49696b1..caff977 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -99,6 +99,33 @@ Nothing is fenced, so the incumbent is untouched. ". The fix, with a test, is in Hands on `fix/bulk-blob-fetch`: a promisor-style fetch with `--filter=blob:none` and `fetch.negotiationAlgorithm=noop`. +**Ship completed, 2026-09-29 (host clock).** +- **Bulk-fetch fix.** Merged at `583b2a7` and installed at 19:17. The Idunn sha is `01bde676`. +- **The first streampixels-service redeploy failed after fencing.** The incumbent was restored, and every restart + was refused by Odin with "runtime presence publisher sequence was reordered". Cause: Odin compared publisher + sequences across incarnations, and the TS publisher counts from 1 per process. + - The operator ruled: fix Odin's rule. + - Odin `db041ec` compares sequences only while the stored presence still authenticates under the current + activation. Soul said deploy, with follow-ups; Odin was deployed at 19:4x. +- **Admissions.** + - streampixels-service was admitted at 19:50 on s3-c2. + - The web binding dropped `STREAMPIXELS_ODIN_CULTMESH_RUDP`, and gamecult-ops `37b5ed9` does the same. + - streampixels-web was admitted with Odin-free readiness. +- **Public cutover at 19:54.** The vhost upstreams moved to `:8833` (web) and `:8832` (service). The backup is + `/root/streampixels.gamecult.org.conf.bak-20260929-195415-pre-idunn-upstreams`. + - `https://streampixels.gamecult.org/api/catalog` returns 200 (120,893 B), after being 502 since migration 013. + - The overlay SSE stream answers `text/event-stream`, and the ops checker passes. +- **Legacy units.** `streampixels-web.service` is disabled and stopped. `streampixels-service.service` is + disabled, and was already stopped. +- **Follow-ups.** + - The C2 acceptance hour: sample the nginx reload counter at T0 and T0+60, with no rejected challenges. + - Nothing builds the Windows `idunn-host` in verification. + - Odin presence: pin the stored-at re-authentication time (Soul's M3 survived), and consider checking activation + identity directly rather than any failure to authenticate. + - The dead-hold report field (ruled). + - B2 rework on B3. + - Odin's CultLib pin bump, after the stray-packet merge. + **Ruling, 2026-09-30 (operator): the dead-hold report is stored on the generation.** - Add an optional `last_error` to `AdmittedGeneration` (new key 21), and make `status` render admitted generations. From 94c7691cb610aea8b43f00bedae1ea7178fa5599 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 22:56:10 +0200 Subject: [PATCH 056/286] Map: the C2 acceptance hour passed; Odin 5c37860 deployed Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index caff977..f8c8306 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -126,6 +126,16 @@ Nothing is fenced, so the incumbent is untouched. - B2 rework on B3. - Odin's CultLib pin bump, after the stray-packet merge. +**C2 acceptance hour passed, 2026-09-29 19:55-20:55 UTC (21:55-22:55 CEST).** +- 1 nginx reload, which was Odin's own route promotion during its `5c37860` deploy. +- 0 rejected StreamPixels challenges, 0 Odin continuity timeouts, and Idunn `NRestarts=0`. +- One running unit each for service, web and Odin. Public `/` and `/api/catalog` return 200. +- Also deployed in the window: Odin `5c37860`, carrying CultLib `3bf1c0c` (RUDP stray-packet hardening in every + runtime), Sleipnir removed, and presence ordering decided by activation identity. +- Still open for Odin: it exits when its 64-slot RUDP session table fills, because its own heartbeat publish + fails. Publishers that never Disconnect hold slots for 30 s. The fix is in Hands. +- The second run, with Odin broken on purpose, remains the operator's to schedule. + **Ruling, 2026-09-30 (operator): the dead-hold report is stored on the generation.** - Add an optional `last_error` to `AdmittedGeneration` (new key 21), and make `status` render admitted generations. From 5f8417589d22f84a306d0329444a41d01884eb4f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 23:38:26 +0200 Subject: [PATCH 057/286] Candidate cleanup owed is one method both abort intents share The two-flag helper let either call site swap or fold an argument unnoticed. DeploymentTransaction::candidate_cleanup_owed reads the transaction itself, and a four-row table drives the method and both intents. Co-Authored-By: Claude Sonnet 5.5 --- src/control_plane.rs | 77 ++++++++++++++++++++++++++------------------ 1 file changed, 45 insertions(+), 32 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 7943698..c068360 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1370,6 +1370,17 @@ impl DeploymentTransaction { } } + /// Whether an abort owes the candidate's durable cleanup: it does once an + /// activation was issued or a workload was observed, either of which leaves + /// something behind. + fn candidate_cleanup_owed(&self) -> CleanupEvidence { + if self.activation.is_some() || self.workload.is_some() { + CleanupEvidence::Pending + } else { + CleanupEvidence::Skipped + } + } + fn rejected(command: &DeploymentCommand, error: anyhow::Error, now: u64) -> Result { let mut transaction = Self::new(command, command.selector.clone(), 0, None, now)?; let detail = truncate(&format!("{error:#}"), 2048); @@ -7991,10 +8002,7 @@ fn incumbent_was_stopped_during_fencing(fencing: &FencingEvidence) -> bool { fn pre_fencing_abort_intent(transaction: &DeploymentTransaction, error: &str) -> PreFencingAbort { PreFencingAbort { error: truncate(error, 2048), - candidate_cleanup: candidate_cleanup_requirement( - transaction.activation.is_some(), - transaction.workload.is_some(), - ), + candidate_cleanup: transaction.candidate_cleanup_owed(), topology_reconciliation: transaction.abort_topology_reconciliation(), source_cleanup: if transaction.command_kind == CommandKind::Deploy { CleanupEvidence::Pending @@ -8025,10 +8033,7 @@ fn post_fencing_abort_intent( } else { CleanupEvidence::Skipped }, - candidate_cleanup: candidate_cleanup_requirement( - transaction.activation.is_some(), - transaction.workload.is_some(), - ), + candidate_cleanup: transaction.candidate_cleanup_owed(), topology_reconciliation: transaction.abort_topology_reconciliation(), source_cleanup: if transaction.command_kind == CommandKind::Deploy { CleanupEvidence::Pending @@ -8038,17 +8043,6 @@ fn post_fencing_abort_intent( } } -fn candidate_cleanup_requirement( - has_prepared_activation: bool, - has_workload_observation: bool, -) -> CleanupEvidence { - if has_prepared_activation || has_workload_observation { - CleanupEvidence::Pending - } else { - CleanupEvidence::Skipped - } -} - fn sequence_requires_admission( latest: Option<&TopologyEvidence>, maximum_admitted_sequence: u64, @@ -9803,19 +9797,38 @@ mod tests { } #[test] - fn activation_without_workload_still_requires_durable_candidate_cleanup() { - assert_eq!( - candidate_cleanup_requirement(true, false), - CleanupEvidence::Pending - ); - assert_eq!( - candidate_cleanup_requirement(true, true), - CleanupEvidence::Pending - ); - assert_eq!( - candidate_cleanup_requirement(false, false), - CleanupEvidence::Skipped - ); + fn candidate_cleanup_is_owed_when_an_activation_or_a_workload_exists() -> Result<()> { + // (activation issued, workload observed, expected cleanup) + let rows = [ + (false, false, CleanupEvidence::Skipped), + (true, false, CleanupEvidence::Pending), + (false, true, CleanupEvidence::Pending), + (true, true, CleanupEvidence::Pending), + ]; + for (has_activation, has_workload, expected) in rows { + let world = ContinuityProjection::after_candidate_observed()?; + let mut transaction = world.continuity_transaction()?; + if !has_activation { + transaction.activation = None; + } + transaction.workload = has_workload.then(|| workload(1000, 1, 2)); + assert_eq!( + transaction.candidate_cleanup_owed(), + expected, + "activation {has_activation}, workload {has_workload}" + ); + assert_eq!( + pre_fencing_abort_intent(&transaction, "x").candidate_cleanup, + expected, + "pre-fencing intent, activation {has_activation}, workload {has_workload}" + ); + assert_eq!( + post_fencing_abort_intent(&transaction, "x").candidate_cleanup, + expected, + "post-fencing intent, activation {has_activation}, workload {has_workload}" + ); + } + Ok(()) } #[test] From bf5347c3704d2f7bb5626c5c284a4d97ba0eeb2b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 23:40:11 +0200 Subject: [PATCH 058/286] The chunk-size line admits whitespace only before an extension RFC 9112 7.1 allows whitespace before a chunk extension and nowhere else. The reader trimmed both ends of the size, so " 3" and "3 " were taken. It now splits at the first ';', trims trailing SP and HTAB only when an extension follows, and leaves parse_digits as the digit authority. Rows use an honest body so only the size line differs. Co-Authored-By: Claude Sonnet 5.5 --- src/drivers.rs | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/src/drivers.rs b/src/drivers.rs index 6af3559..dd9f6c0 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -5813,10 +5813,16 @@ fn request_http_snapshot(target: SocketAddr, payload: &[u8]) -> Result, let mut response = Vec::new(); loop { let size = answer.read_line(ROUTE_HTTP_CHUNK_LINE_BYTES, body)?; + // RFC 9112 7.1: `chunk-size [ chunk-ext ] CRLF`, where an + // extension may follow whitespace. Nothing else is trimmed. let size = std::str::from_utf8(&size) .ok() - .and_then(|size| size.split(';').next()) - .and_then(|size| parse_digits(size.trim_matches([' ', '\t']), 16)) + .and_then(|line| match line.split_once(';') { + Some((digits, _extension)) => { + parse_digits(digits.trim_end_matches([' ', '\t']), 16) + } + None => parse_digits(line, 16), + }) .context("stable CultNet HTTP chunk size is invalid") .map_err(refused)?; if size == 0 { @@ -8786,9 +8792,15 @@ mod tests { /// An honest chunked answer with `after_body` in place of its end. fn chunked_honest_then(id: &str, after_body: &[u8]) -> Vec { + chunked_honest_sized(id, "", "", after_body) + } + + /// An honest chunked answer whose chunk-size line is the hex length between + /// `before` and `after`. + fn chunked_honest_sized(id: &str, before: &str, after: &str, after_body: &[u8]) -> Vec { let body = honest_body(id); let mut reply = CHUNKED_HEAD.to_vec(); - reply.extend_from_slice(format!("{:x}\r\n", body.len()).as_bytes()); + reply.extend_from_slice(format!("{before}{:x}{after}\r\n", body.len()).as_bytes()); reply.extend_from_slice(&body); reply.extend_from_slice(b"\r\n"); reply.extend_from_slice(after_body); @@ -8803,6 +8815,30 @@ mod tests { challenge(scripted_peer(|id| chunked_honest_then(id, b"0\r\nT: 1\r\nU: 2\r\n\r\n")), |_| {}) .expect("an honest chunked answer with trailers"); + // RFC 9112 7.1: whitespace may precede a chunk extension and nothing + // else. The answer is otherwise honest, so only the size line differs. + for (before, after) in [("", ";x"), ("", " ;x"), ("", "\t;x"), ("", ";x=1;y")] { + challenge( + scripted_peer(move |id| chunked_honest_sized(id, before, after, b"0\r\n\r\n")), + |_| {}, + ) + .unwrap_or_else(|error| panic!("size line {before:?}+size+{after:?}: {error:?}")); + } + for (why, before, after) in [ + ("whitespace before the chunk size", " ", ""), + ("a tab before the chunk size", "\t", ""), + ("whitespace after the chunk size and no extension", "", " "), + ("whitespace before the size, ahead of an extension", " ", ";x"), + ] { + assert_refused( + challenge( + scripted_peer(move |id| chunked_honest_sized(id, before, after, b"0\r\n\r\n")), + |_| {}, + ), + why, + ); + } + // A trailer that begins and then the peer stops. assert_refused( challenge(scripted_peer(|id| chunked_honest_then(id, b"0\r\nT: 1\r\n")), |_| {}), From 97fb2b88719077d279d1b55618672441e677dcf1 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 23:44:25 +0200 Subject: [PATCH 059/286] Map: B2 rework plan, D1/T1/T2/W1/R1 cuts, and the 2026-09-30 rulings Splice the B2 rework map (owner map, per-file cut, verification, live relevance, subtraction estimate) under B2; add cuts D1, T1, T2, W1 and R1; record Q-B2-1 (a), Q-B2-2 (fail at once), Q-R1 (a) and Q-B4 (routed by Idunn's route proof, unrouted by admission plus workload; the Odin non-Ready veto clause held open). B4 is blocked on the Odin-lifecycle authority audit. B2, D1, R1 ready for Hands; W1, T1, T2 in progress on hands/idunn-w1-t1-t2. Co-Authored-By: Claude Sonnet 5.5 --- docs/route-continuity-cut.md | 745 ++++++++++++++++++++++++++++++++++- 1 file changed, 729 insertions(+), 16 deletions(-) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index f8c8306..bba4d5f 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -193,7 +193,7 @@ it for merge. - B2 also does not compile on top of B3, and after a one-line fix two tests fail, because the fixture target is now route-proof. -The rework runs after B3 merges and is rebased onto it. +The rework is a fresh branch from `583b2a7`, not a rebase of `1786ddf`. Its map is under B2 in section 4. **Rulings, 2026-09-30.** - **The ceiling never refuses a rollback.** Restoring the admitted or incumbent route is survival, not @@ -203,6 +203,27 @@ The rework runs after B3 merges and is rebased onto it. admitted generation, so a target with no admitted generation is charged too. That schema change is folded into the store-version bump that F1 needs: a v4 generation, with a typed lift from v3. +**Rulings, 2026-09-30 (operator, on the B2 rework map).** +- **Q-B2-1 (a):** the continuity restart log moves into `TargetSupervision`; the restart ceiling reads the + target's own log; the carry compensator is deleted. +- **Q-B2-2: fail at once.** A deploy refused by the route/actuation ceiling before the fence fails immediately, + with the reopen time in the error. +- **Q-R1 (a):** the route declaration is the challenge declaration. Every routed target answers the + stable-route challenge regardless of readiness class. R1 checks it before the fence; H1 gives Heimdall a + responder. Operator note: the question's first framing was confusing; the cause is that Heimdall has no + responder, not its class. +- **Q-B4: Idunn's own proof.** Routed providers by current route proof and not degraded; unrouted providers by + admission plus a live workload observation. One clause is **held open**: "a current Odin non-Ready word + still excludes a provider" is not ruled, pending the Odin-lifecycle authority audit (a separate Imagination + pass). Operator: "the last few rulings on Odin are making my authority sense tingle, it feels like Odin is + doing lifecycle work when that's very much Idunn's wheelhouse." B4 is blocked on that audit. + +**Cut status, 2026-09-30.** +- **Ready for Hands:** B2 (rework map in section 4); D1 (after B2); R1 (after B2). +- **In progress** on `hands/idunn-w1-t1-t2`: W1, T1, T2. +- **Blocked:** B4, on the Odin-lifecycle authority audit; H1 (Heimdall repo), on CultLib C1. + + Heads read for this map: | Repo | Ref | Notes | @@ -571,6 +592,97 @@ it. This is a CultLib foundation merge, so the operator should rule on it. Depends on it: C1 and the StreamPixels gitlink in C2. +**Q-B2-1. Does the continuity restart log move into `TargetSupervision` too?** + +**RULED (a) by the operator, 2026-09-30.** + +Options: +- (a) Move it. `ContinuityBackoff`'s own doc (`:907-908`) already says it "belongs to the target, not the + generation". On the generation it survives only through the copy in `from_transaction` (`:2074-2076`). + Moving it deletes that carry and key 20. The mint CAS pins the target record instead of racing the + generation, which is the F3 shape. One record holds both of a target's meters. +- (b) Keep it on the generation, reshaped to the sliding log in place. The route meter still needs + `TargetSupervision` for first deploys, so the meters split across two records and the carry stays. + +Outcome: the continuity restart log lives in `TargetSupervision`; the restart ceiling reads the target's own +log; the carry compensator in `from_transaction` is deleted. + +Depends on it: B2 (see the B2 rework map in section 4). + +**Q-B2-2. When the route ceiling refuses a first or forward deploy before the fence, does `idunn up` fail or +wait?** + +**RULED: fail at once, by the operator, 2026-09-30.** + +Options: +- (a) Fail. The deploy is aborted pre-fence with "route actuation ceiling reached for ; reopens at +