Merge B5: every post-fencing phase ends; reload skip confined to supervision

Phase deadlines are resolved before every post-fence step; `idunn expire` requests an
expiry through the same resolver; lease adoption is recorded from Odin's Ready; an abort
withdraws any issued lease (Prepared or Granted); install/rollback/withdrawal inside a
transaction always write and reload; a first-deploy withdrawal closes the firewall rule
before reloading. The operator-required arm for an adopted-lease deploy stays until the
Q3-ii snapshot cuts (Q-S5: B5 first).

Soul passes 1-3. Merged tree verified on Yggdrasil: 360 passed, 2 ignored.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/deployment_plan.rs

@@ -467,15 +467,17 @@ pub fn select_dependencies(
 /// overrides any of them; the resolved values are frozen into the plan.
 pub const DEFAULT_FENCING_DEADLINE_SECONDS: u32 = 120;
 pub const DEFAULT_LEASING_DEADLINE_SECONDS: u32 = 120;
-// Unsafe for streampixels-service (startup = "create-or-open-after-write-lease"):
-// its Postgres migrations run inside AwaitingReady. B5 requires an explicit
-// binding value for stateful targets instead of this default.
+// A stateful candidate may run migrations inside AwaitingReady (streampixels-
+// service runs Postgres's there). When this passes, a deployment whose
+// candidate was issued a write lease is left to the operator, not aborted; a
+// continuity is aborted and its release restarted.
 pub const DEFAULT_AWAITING_READY_DEADLINE_SECONDS: u32 = 300;
 pub const DEFAULT_ROUTING_DEADLINE_SECONDS: u32 = 120;
 pub const DEFAULT_COMMITTING_DEADLINE_SECONDS: u32 = 120;
 
-/// The resolved per-phase deadline durations a plan carries. Nothing reads
-/// them at the cut that introduced them.
+/// The resolved per-phase deadline durations a plan carries. Entering a
+/// post-fencing phase stamps its deadline from them, and the deadline
+/// resolver ends the phase once it passes.
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
 pub struct PhaseDeadlines {

src/drivers.rs

@@ -707,6 +707,14 @@ const ROUTE_HTTP_CHUNK_LINE_BYTES: usize = 128;
 const ROUTE_HTTP_SNAPSHOT_PATH: &str = "/cultnet/snapshot";
 
 impl RoutePreflightReceipt {
+    /// Whether the candidate's membership differs from the baseline the
+    /// preflight found. A continuity over its admitted route does not: its
+    /// transaction has no route change to make or to undo.
+    pub fn changes_route(&self) -> bool {
+        self.incumbent_membership_sha256.as_deref()
+            != Some(self.candidate_membership_sha256.as_str())
+    }
+
     pub fn validate(&self) -> Result<()> {
         require_driver_id(&self.route_id, "route preflight")?;
         require_sha256_id(
@@ -5332,31 +5340,31 @@ impl NginxRouteDriver {
         }
     }
 
+    /// Put `prior` back as the route's fragment, and always write and reload:
+    /// on a transaction path, bytes on disk that already equal `prior` may be
+    /// a write whose reload a crash cut off. A reload that fails puts the
+    /// previous fragment back, so the disk shows what nginx still serves.
+    ///
+    /// With no prior membership the stable endpoint routes to nothing, and
+    /// its firewall allow is withdrawn first: a failing `ufw delete` then
+    /// retries without touching nginx, and a crash after it is safe, because
+    /// a rule already gone reads as withdrawn.
     fn restore(&self, prior: Option<&[u8]>, gate: &dyn RouteActuationGate) -> Result<()> {
         gate.admit(RouteActuation::Survival)?;
-        self.write_fragment(prior)?;
-        self.reload()?;
-        // No prior membership means the stable endpoint no longer routes to
-        // anything; its firewall allow goes with the fragment.
         if prior.is_none() {
             self.withdraw_endpoint()?;
         }
-        Ok(())
-    }
-
-    fn fail_after_rollback<T>(
-        &self,
-        prior: Option<&[u8]>,
-        failure: anyhow::Error,
-        context: &str,
-        gate: &dyn RouteActuationGate,
-    ) -> Result<T> {
-        match self.restore(prior, gate) {
-            Ok(()) => Err(failure).context(context.to_owned()),
-            Err(rollback) => Err(failure).context(format!(
-                "{context}; route rollback also failed: {rollback:#}"
-            )),
+        let current = self.current_configuration()?;
+        self.write_fragment(prior)?;
+        if let Err(reload) = self.reload() {
+            return match self.write_fragment(current.as_deref()) {
+                Ok(()) => Err(reload).context("reloading the restored route"),
+                Err(undo) => Err(reload).context(format!(
+                    "reloading the restored route; putting the previous fragment back also failed: {undo:#}"
+                )),
+            };
         }
+        Ok(())
     }
 
     pub fn preflight(
@@ -5431,22 +5439,42 @@ impl NginxRouteDriver {
                     && prior_sha256 == preflight.incumbent_membership_sha256),
             "route baseline changed after preflight"
         );
-        gate.admit(RouteActuation::Forward)?;
-        if !candidate_already_written {
-            atomic_replace(&self.binding.config_path, &rendered)?;
+        // A candidate whose membership is the baseline's (a continuity over
+        // its admitted route) changes nothing, so nothing is written, and no
+        // crash can have cut a write of this transaction short.
+        if !preflight.changes_route() {
+            return Ok(sha256_id(&rendered));
         }
+        // Otherwise bytes already on disk prove nothing: a crash may have cut
+        // off the reload after an earlier attempt's write. So it always
+        // writes and reloads.
+        gate.admit(RouteActuation::Forward)?;
+        atomic_replace(&self.binding.config_path, &rendered)?;
         if let Err(error) = self.admit_endpoint().and_then(|()| self.reload()) {
-            if rollback_allowed {
-                return self.fail_after_rollback(
-                    preflight.incumbent_configuration.as_deref(),
-                    error,
-                    "candidate route validation or reload failed",
-                    gate,
-                );
-            }
-            return Err(error).context(
-                "candidate route reload failed after incumbent route authority was fenced; candidate fragment retained for retry",
-            );
+            // nginx did not load the candidate, so it still serves the
+            // baseline. The disk is put back to match it, which is the whole
+            // rollback, and a retry installs and reloads again rather than
+            // mistaking unloaded bytes for a live route. Past the fence the
+            // firewall allow stays for that retry; before it, an allow the
+            // baseline never had goes too.
+            let context = if rollback_allowed {
+                "candidate route validation or reload failed"
+            } else {
+                "candidate route reload failed after incumbent route authority was fenced; the fragment was put back for retry"
+            };
+            let undo = self.write_fragment(prior.as_deref()).and_then(|()| {
+                if rollback_allowed && prior.is_none() {
+                    self.withdraw_endpoint()
+                } else {
+                    Ok(())
+                }
+            });
+            return match undo {
+                Ok(()) => Err(error).context(context),
+                Err(undo) => Err(error).context(format!(
+                    "{context}; putting the route back also failed: {undo:#}"
+                )),
+            };
         }
         ensure!(
             self.current_configuration()?.as_deref() == Some(rendered.as_slice()),
@@ -5468,12 +5496,6 @@ impl NginxRouteDriver {
         Ok(self.current_configuration()?.as_deref() == Some(rendered.as_slice()))
     }
 
-    /// Restore the exact membership owned by an admitted generation. Any
-    /// different fragment is drift, not a rollback baseline, so it is never
-    /// restored after a failed reload. A failed reload removes the newly
-    /// written fragment so the next continuity pass cannot mistake disk bytes
-    /// for an adopted route. The caller must still challenge the stable
-    /// listener to prove that nginx workers adopted this membership.
     /// Put the route back exactly as the candidate found it.
     ///
     /// The preflight receipt captured the incumbent's configuration before this
@@ -5490,11 +5512,21 @@ impl NginxRouteDriver {
             preflight.route_id == self.binding.route_id,
             "route preflight receipt describes another route"
         );
+        if !preflight.changes_route() {
+            return Ok(());
+        }
         self.restore(preflight.incumbent_configuration.as_deref(), gate)
     }
 
+    /// Restore the exact membership owned by an admitted generation. A failed
+    /// reload puts back the bytes it replaced, so the disk still shows what
+    /// nginx serves and the next pass sees the drift again rather than taking
+    /// unloaded bytes for an adopted route. The caller must still challenge
+    /// the stable listener to prove that nginx workers adopted this membership.
+    ///
     /// A fragment already equal to the admitted membership is left alone: no
-    /// write, no `nginx -t`, no firewall call, no reload.
+    /// write, no `nginx -t`, no firewall call, no reload. Supervision is the
+    /// only path that skips on equal bytes.
     pub fn restore_admitted_membership(
         &self,
         expected: &IdunnExpectedIncarnationRecord,
@@ -5511,12 +5543,13 @@ impl NginxRouteDriver {
         }
         gate.admit(RouteActuation::Survival)?;
         self.validate_candidate_in_private_mount(&rendered)?;
+        let replaced = self.current_configuration()?;
         atomic_replace(&self.binding.config_path, &rendered)?;
         if let Err(reload) = self.admit_endpoint().and_then(|()| self.reload()) {
-            return match self.write_fragment(None) {
+            return match self.write_fragment(replaced.as_deref()) {
                 Ok(()) => Err(reload).context("reloading the exact admitted route membership"),
-                Err(cleanup) => Err(reload).context(format!(
-                    "reloading the exact admitted route membership; removing its unproved fragment also failed: {cleanup:#}"
+                Err(undo) => Err(reload).context(format!(
+                    "reloading the exact admitted route membership; putting the replaced fragment back also failed: {undo:#}"
                 )),
             };
         }
@@ -5651,6 +5684,9 @@ impl NginxRouteDriver {
             self.current_configuration()?.as_deref() == Some(rendered.as_slice()),
             "candidate route changed before rollback"
         );
+        if !preflight.changes_route() {
+            return Ok(());
+        }
         self.restore(preflight.incumbent_configuration.as_deref(), gate)
     }
 }
@@ -11817,6 +11853,226 @@ Content-Le".to_vec()), |_| {}),
         Ok(())
     }
 
+    /// Stub host programs that log every call. `systemctl reload` fails while
+    /// `<dir>/systemctl.fail` exists, `ufw delete` while `<dir>/ufw.fail` does.
+    #[cfg(unix)]
+    fn failable_route_driver(
+        temp: &Path,
+    ) -> Result<(NginxRouteDriver, IdunnExpectedIncarnationRecord, PathBuf)> {
+        use std::os::unix::fs::PermissionsExt;
+
+        let (mut driver, candidate, calls) = stubbed_route_driver(temp)?;
+        let failable = |name: &str, verb: &str| -> Result<PathBuf> {
+            let path = temp.join(name);
+            fs::write(
+                &path,
+                format!(
+                    "#!/bin/sh\necho \"{name} $*\" >> '{}'\nif [ \"$1\" = {verb} ] && [ -e \"$0.fail\" ]; then exit 1; fi\nexit 0\n",
+                    calls.display()
+                ),
+            )?;
+            fs::set_permissions(&path, fs::Permissions::from_mode(0o755))?;
+            Ok(path)
+        };
+        driver.systemctl_program = failable("systemctl", "reload")?;
+        driver.ufw_program = failable("ufw", "delete")?;
+        // `<dir>/ufw.absent`: the rule is already gone, as ufw reports it.
+        let ufw = fs::read_to_string(&driver.ufw_program)?.replace(
+            "exit 0\n",
+            "if [ \"$1\" = delete ] && [ -e \"$0.absent\" ]; then echo 'Could not delete non-existent rule' >&2; exit 1; fi\nexit 0\n",
+        );
+        fs::write(&driver.ufw_program, ufw)?;
+        Ok((driver, candidate, calls))
+    }
+
+    #[cfg(unix)]
+    fn logged(calls: &Path, prefix: &str) -> usize {
+        fs::read_to_string(calls)
+            .unwrap_or_default()
+            .lines()
+            .filter(|line| line.starts_with(prefix))
+            .count()
+    }
+
+    /// Admits everything and records what it was asked.
+    struct Recording(std::cell::RefCell<Vec<RouteActuation>>);
+
+    impl RouteActuationGate for Recording {
+        fn admit(&self, kind: RouteActuation) -> Result<()> {
+            self.0.borrow_mut().push(kind);
+            Ok(())
+        }
+    }
+
+    /// On a transaction path, bytes already on disk prove nothing: a crash may
+    /// have cut off the reload after the write. The install past the fence and
+    /// the withdrawal both write and reload whatever the disk shows, and the
+    /// install admits its firewall allow again.
+    #[cfg(unix)]
+    #[test]
+    fn a_crash_between_a_write_and_its_reload_is_reloaded_on_retry() -> Result<()> {
+        let temp = tempfile::tempdir()?;
+        let (driver, candidate, calls) = failable_route_driver(temp.path())?;
+        let receipt = preflight_receipt(&driver, &candidate)?;
+        let rendered = driver.render(&candidate)?;
+
+        // The install wrote its fragment and Idunn died before the reload.
+        fs::write(&driver.binding.config_path, &rendered)?;
+        driver.install(&candidate, &digest('a'), &receipt, false, &Unmetered)?;
+        assert_eq!(logged(&calls, "systemctl reload"), 1, "the unloaded install was not reloaded");
+        assert_eq!(logged(&calls, "ufw allow"), 1, "the first deploy's allow was skipped");
+
+        // The withdrawal removed it and Idunn died before the reload.
+        fs::remove_file(&driver.binding.config_path)?;
+        driver.withdraw_candidate_membership(&receipt, &Unmetered)?;
+        assert_eq!(logged(&calls, "systemctl reload"), 2, "the unloaded withdrawal was not reloaded");
+        assert_eq!(logged(&calls, "ufw delete"), 1);
+        Ok(())
+    }
+
+    /// A redeploy over an existing route changes it: the candidate differs
+    /// from the incumbent's fragment, so it installs and reloads, and its
+    /// rollback and withdrawal put the incumbent back and reload.
+    #[cfg(unix)]
+    #[test]
+    fn a_deploy_over_an_incumbent_route_installs_and_reloads() -> Result<()> {
+        let temp = tempfile::tempdir()?;
+        let (driver, candidate, calls) = failable_route_driver(temp.path())?;
+        let incumbent = b"# incumbent membership\n".to_vec();
+        let mut receipt = preflight_receipt(&driver, &candidate)?;
+        receipt.incumbent_runtime_instance_id = Some(digest('b'));
+        receipt.incumbent_membership_sha256 = Some(sha256_id(&incumbent));
+        receipt.incumbent_configuration = Some(incumbent.clone());
+        assert!(receipt.changes_route());
+        fs::write(&driver.binding.config_path, &incumbent)?;
+
+        driver.install(&candidate, &digest('a'), &receipt, true, &Unmetered)?;
+        assert_eq!(logged(&calls, "systemctl reload"), 1, "the redeploy did not install");
+        assert_eq!(fs::read(&driver.binding.config_path)?, driver.render(&candidate)?);
+        driver.rollback(&candidate, &digest('a'), &receipt, &Unmetered)?;
+        assert_eq!(logged(&calls, "systemctl reload"), 2, "the rollback did not reload");
+        assert_eq!(fs::read(&driver.binding.config_path)?, incumbent);
+        driver.install(&candidate, &digest('a'), &receipt, true, &Unmetered)?;
+        driver.withdraw_candidate_membership(&receipt, &Unmetered)?;
+        assert_eq!(logged(&calls, "systemctl reload"), 4, "the withdrawal did not reload");
+        assert_eq!(fs::read(&driver.binding.config_path)?, incumbent);
+        Ok(())
+    }
+
+    /// With nothing left to route, the withdrawal takes the firewall allow
+    /// first. A `ufw delete` that keeps failing retries without reloading
+    /// nginx; once it goes through, the fragment is removed and reloaded.
+    #[cfg(unix)]
+    #[test]
+    fn a_failing_firewall_withdrawal_retries_without_reloading() -> Result<()> {
+        let temp = tempfile::tempdir()?;
+        let (driver, candidate, calls) = failable_route_driver(temp.path())?;
+        let receipt = preflight_receipt(&driver, &candidate)?;
+        driver.install(&candidate, &digest('a'), &receipt, false, &Unmetered)?;
+        let installed = logged(&calls, "systemctl reload");
+
+        fs::write(temp.path().join("ufw.fail"), b"x")?;
+        for _ in 0..5 {
+            assert!(driver.withdraw_candidate_membership(&receipt, &Unmetered).is_err());
+        }
+        assert_eq!(logged(&calls, "systemctl reload"), installed, "a failing ufw delete reloaded nginx");
+        assert_eq!(logged(&calls, "ufw delete"), 5);
+        fs::remove_file(temp.path().join("ufw.fail"))?;
+        driver.withdraw_candidate_membership(&receipt, &Unmetered)?;
+        assert_eq!(logged(&calls, "systemctl reload"), installed + 1);
+        assert!(!driver.binding.config_path.exists());
+        Ok(())
+    }
+
+    /// A crash after the firewall rule went and before the fragment did: the
+    /// retry reads the gone rule as withdrawn and finishes the withdrawal.
+    #[cfg(unix)]
+    #[test]
+    fn a_withdrawal_resumed_after_its_firewall_rule_went_finishes() -> Result<()> {
+        let temp = tempfile::tempdir()?;
+        let (driver, candidate, calls) = failable_route_driver(temp.path())?;
+        let receipt = preflight_receipt(&driver, &candidate)?;
+        driver.install(&candidate, &digest('a'), &receipt, false, &Unmetered)?;
+        let installed = logged(&calls, "systemctl reload");
+
+        fs::write(temp.path().join("ufw.absent"), b"x")?;
+        driver.withdraw_candidate_membership(&receipt, &Unmetered)?;
+        assert!(!driver.binding.config_path.exists(), "the fragment outlived the withdrawal");
+        assert_eq!(logged(&calls, "systemctl reload"), installed + 1, "nginx was not reloaded");
+        Ok(())
+    }
+
+    /// A continuity over its admitted route has the baseline's membership: its
+    /// install, rollback and withdrawal change nothing and run nothing.
+    #[cfg(unix)]
+    #[test]
+    fn a_transaction_that_does_not_change_the_route_runs_nothing() -> Result<()> {
+        let temp = tempfile::tempdir()?;
+        let (driver, candidate, calls) = failable_route_driver(temp.path())?;
+        let rendered = driver.render(&candidate)?;
+        let mut receipt = preflight_receipt(&driver, &candidate)?;
+        receipt.incumbent_runtime_instance_id = Some(digest('b'));
+        receipt.incumbent_membership_sha256 = Some(sha256_id(&rendered));
+        receipt.incumbent_configuration = Some(rendered.clone());
+        assert!(!receipt.changes_route());
+        fs::write(&driver.binding.config_path, &rendered)?;
+        let gate = Recording(Default::default());
+
+        driver.install(&candidate, &digest('a'), &receipt, false, &gate)?;
+        driver.rollback(&candidate, &digest('a'), &receipt, &gate)?;
+        driver.withdraw_candidate_membership(&receipt, &gate)?;
+        assert_eq!(logged(&calls, ""), 0, "an unchanged route ran a program");
+        assert!(gate.0.borrow().is_empty(), "an unchanged route was charged");
+        assert_eq!(fs::read(&driver.binding.config_path)?, rendered);
+
+        // The first-route receipt does change it.
+        assert!(preflight_receipt(&driver, &candidate)?.changes_route());
+        Ok(())
+    }
+
+    /// Bytes on disk are what nginx loaded: a write whose reload fails is
+    /// undone, so the retry writes and reloads again instead of taking the
+    /// unloaded bytes for a live route.
+    #[cfg(unix)]
+    #[test]
+    fn a_failed_reload_puts_the_fragment_back_so_the_retry_reloads() -> Result<()> {
+        let temp = tempfile::tempdir()?;
+        let (driver, candidate, calls) = failable_route_driver(temp.path())?;
+        let receipt = preflight_receipt(&driver, &candidate)?;
+        let rendered = driver.render(&candidate)?;
+        let fail = temp.path().join("systemctl.fail");
+
+        // Install, fenced (no rollback) and not.
+        for rollback_allowed in [false, true] {
+            fs::write(&fail, b"x")?;
+            assert!(
+                driver
+                    .install(&candidate, &digest('a'), &receipt, rollback_allowed, &Unmetered)
+                    .is_err()
+            );
+            assert!(!driver.binding.config_path.exists(), "an unloaded candidate stayed on disk");
+            fs::remove_file(&fail)?;
+            let before = logged(&calls, "systemctl reload");
+            driver.install(&candidate, &digest('a'), &receipt, rollback_allowed, &Unmetered)?;
+            assert_eq!(logged(&calls, "systemctl reload"), before + 1, "the retry did not reload");
+            assert_eq!(fs::read(&driver.binding.config_path)?, rendered);
+            if !rollback_allowed {
+                fs::remove_file(&driver.binding.config_path)?;
+            }
+        }
+
+        // Restore: the candidate is live; withdrawing it fails at the reload.
+        fs::write(&fail, b"x")?;
+        assert!(driver.withdraw_candidate_membership(&receipt, &Unmetered).is_err());
+        assert_eq!(fs::read(&driver.binding.config_path)?, rendered, "the unloaded removal stayed");
+        fs::remove_file(&fail)?;
+        let before = logged(&calls, "systemctl reload");
+        driver.withdraw_candidate_membership(&receipt, &Unmetered)?;
+        assert_eq!(logged(&calls, "systemctl reload"), before + 1, "the retry did not reload");
+        assert!(!driver.binding.config_path.exists());
+        Ok(())
+    }
+
     /// Observation observes: restoring a membership whose fragment is already
     /// exact touches nothing, and drifted bytes are restored with one reload.
     #[cfg(unix)]
@@ -12014,8 +12270,6 @@ Content-Le".to_vec()), |_| {}),
         assert!(!config.exists());
         assert_eq!(fs::read_dir(&driver.preflight_root)?.count(), 0);
 
-        let candidate_bytes = driver.render(&candidate)?;
-        fs::write(&config, &candidate_bytes)?;
         let membership_sha256 = driver.install(&candidate, &digest('a'), &preflight, true, &Unmetered)?;
         assert_eq!(
             fs::read_to_string(systemctl.with_extension("calls"))?,
@@ -12040,15 +12294,16 @@ Content-Le".to_vec()), |_| {}),
 
         let next_preflight = driver.preflight(&candidate, &digest('b'), Some(&admitted), &Unmetered)?;
         assert_eq!(fs::read(&config)?, admitted_bytes);
+        // The next candidate renders the bytes already live: installing it
+        // runs nothing, even while a reload would fail.
         fs::write(systemctl.with_extension("fail"), b"fail\n")?;
-        assert!(
-            driver
-                .install(&candidate, &digest('b'), &next_preflight, false, &Unmetered)
-                .is_err()
-        );
-        assert_eq!(fs::read(&config)?, admitted_bytes);
-        fs::remove_file(systemctl.with_extension("fail"))?;
         driver.install(&candidate, &digest('b'), &next_preflight, false, &Unmetered)?;
+        assert_eq!(fs::read(&config)?, admitted_bytes);
+        assert_eq!(
+            fs::read_to_string(systemctl.with_extension("calls"))?,
+            "reload\n"
+        );
+        fs::remove_file(systemctl.with_extension("fail"))?;
         fs::write(&config, b"foreign route\n")?;
         assert!(
             driver
@@ -12066,7 +12321,9 @@ Content-Le".to_vec()), |_| {}),
                 .restore_admitted_membership(&candidate, &admitted.membership_sha256, &Unmetered)
                 .is_err()
         );
-        assert!(!config.exists());
+        // The failed reload puts back what it replaced: the disk shows what
+        // nginx still serves.
+        assert_eq!(fs::read(&config)?, b"foreign route after admission\n");
         fs::remove_file(systemctl.with_extension("fail"))?;
         driver.restore_admitted_membership(&candidate, &admitted.membership_sha256, &Unmetered)?;
         assert_eq!(fs::read(&config)?, admitted_bytes);