diff --git a/README.md b/README.md index 297a9f1..503409e 100644 --- a/README.md +++ b/README.md @@ -204,7 +204,9 @@ purpose; Idunn discovers no authority implicitly. Idunn targets Linux and does not compile on Windows (`drivers.rs` uses `std::os::unix` unconditionally). A local `cargo check` on a Windows -workstation proves nothing. Test and build where it runs, in the same image +workstation proves nothing. `scripts/verify.sh` is the verification command: +the library tests plus a Windows-target check of `idunn-host`. Run it on Linux +with the `x86_64-pc-windows-gnu` std installed. Test and build where it runs, in the same image Odin uses, with the working tree shipped to the host's build root: ```bash diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index f3e6fdd..5d132ef 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -1239,7 +1239,7 @@ Engine-level on `RoutedWorld` with `RouteStubs`, `#[cfg(unix)]`, except where ma | Restart ceiling reads no history | rename `unreadable_history_stops_continuity…` (`:13516`) as B2 did: a corrupt history still restarts, and the target's own log exhausts it | re-adding `history_for_decision` | | Backoff and window | port `continuity_restarts_are_backed_off_and_bounded_per_target` on `SwitchWorkload` | halving the wait; resetting the log on a new generation | | Storm (a) | port `a_failing_reload_that_deletes_the_fragment_backs_the_route_off`: 200 ticks give one reload | the `is_waiting` gate deleted | -| Storm (b) | port `a_candidate_whose_route_proof_fails_reloads_at_most_the_ceiling`: Forward installs are capped at 12 | the install gate deleted | +| Storm (b) | port `a_candidate_whose_route_proof_fails_reloads_at_most_the_ceiling`: installs and their Survival rollbacks share the 12-per-hour route ledger, so a stateless candidate whose proof keeps failing gets 6 install+rollback pairs per hour; only Forward is refused at the ceiling, and Survival is always admitted (counted when the ledger can record it; ruled 2026-09-30 after B2 Soul) | the install gate deleted | | Driver gates | split B2's refusing-gate test in two: Forward refused runs no program; Survival under a gate that "refuses" Forward still runs | a gate call moved after the write | - Run `cargo mutants --in-diff` on the cut. The target is 0 missed in the meters, the diff --git a/scripts/verify.sh b/scripts/verify.sh new file mode 100755 index 0000000..365694e --- /dev/null +++ b/scripts/verify.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# The one verification command. Run it on Linux with the Windows GNU std installed +# (the Eureka verify image has both); a Windows workstation proves nothing. +# The Windows check runs first: it is fast, and a hung test cannot mask it. +# --lib because the tests under tests/ assume the host layout. +set -euo pipefail +cargo check --locked --target x86_64-pc-windows-gnu --bin idunn-host +cargo test --locked --lib diff --git a/src/control_plane.rs b/src/control_plane.rs index 5f64770..2b1e225 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1552,6 +1552,30 @@ impl DeploymentTransaction { } } + /// Whether an abort owes the candidate's durable cleanup: it does once an + /// activation was issued or a workload was observed, either of which leaves + /// something behind. + fn candidate_cleanup_owed(&self) -> CleanupEvidence { + if self.activation.is_some() || self.workload.is_some() { + CleanupEvidence::Pending + } else { + CleanupEvidence::Skipped + } + } + + /// Whether an abort's recorded candidate cleanup is what + /// `candidate_cleanup_owed` says it must be: owed cleanup is pending or + /// done, unowed cleanup is skipped. + fn candidate_cleanup_recorded(&self, recorded: CleanupEvidence) -> bool { + matches!( + (self.candidate_cleanup_owed(), recorded), + ( + CleanupEvidence::Pending, + CleanupEvidence::Pending | CleanupEvidence::Complete + ) | (CleanupEvidence::Skipped, CleanupEvidence::Skipped) + ) + } + fn rejected(command: &DeploymentCommand, error: anyhow::Error, now: u64) -> Result { let mut transaction = Self::new(command, command.selector.clone(), 0, None, now)?; let detail = truncate(&format!("{error:#}"), 2048); @@ -1826,14 +1850,7 @@ impl DeploymentTransaction { "aborted transaction also carries post-commit cleanup" ); ensure!( - matches!( - ( - self.workload.is_some() || self.activation.is_some(), - abort.candidate_cleanup - ), - (true, CleanupEvidence::Pending | CleanupEvidence::Complete) - | (false, CleanupEvidence::Skipped) - ), + self.candidate_cleanup_recorded(abort.candidate_cleanup), "abort candidate cleanup differs from its prepared activation" ); ensure!( @@ -1883,6 +1900,10 @@ impl DeploymentTransaction { self.phase >= DeploymentPhase::Fencing, "post-fencing abort exists before the fence" ); + ensure!( + self.candidate_cleanup_recorded(abort.candidate_cleanup), + "abort candidate cleanup differs from its prepared activation" + ); } let failed_after_fencing = matches!( @@ -8645,10 +8666,7 @@ fn incumbent_was_stopped_during_fencing(fencing: &FencingEvidence) -> bool { fn pre_fencing_abort_intent(transaction: &DeploymentTransaction, error: &str) -> PreFencingAbort { PreFencingAbort { error: truncate(error, 2048), - candidate_cleanup: candidate_cleanup_requirement( - transaction.activation.is_some(), - transaction.workload.is_some(), - ), + candidate_cleanup: transaction.candidate_cleanup_owed(), topology_reconciliation: transaction.abort_topology_reconciliation(), source_cleanup: if transaction.command_kind == CommandKind::Deploy { CleanupEvidence::Pending @@ -8679,10 +8697,7 @@ fn post_fencing_abort_intent( } else { CleanupEvidence::Skipped }, - candidate_cleanup: candidate_cleanup_requirement( - transaction.activation.is_some(), - transaction.workload.is_some(), - ), + candidate_cleanup: transaction.candidate_cleanup_owed(), topology_reconciliation: transaction.abort_topology_reconciliation(), source_cleanup: if transaction.command_kind == CommandKind::Deploy { CleanupEvidence::Pending @@ -8692,17 +8707,6 @@ fn post_fencing_abort_intent( } } -fn candidate_cleanup_requirement( - has_prepared_activation: bool, - has_workload_observation: bool, -) -> CleanupEvidence { - if has_prepared_activation || has_workload_observation { - CleanupEvidence::Pending - } else { - CleanupEvidence::Skipped - } -} - fn sequence_requires_admission( latest: Option<&TopologyEvidence>, maximum_admitted_sequence: u64, @@ -10478,19 +10482,149 @@ mod tests { } #[test] - fn activation_without_workload_still_requires_durable_candidate_cleanup() { - assert_eq!( - candidate_cleanup_requirement(true, false), - CleanupEvidence::Pending - ); - assert_eq!( - candidate_cleanup_requirement(true, true), - CleanupEvidence::Pending - ); - assert_eq!( - candidate_cleanup_requirement(false, false), - CleanupEvidence::Skipped - ); + fn continuity_gives_up_on_a_release_that_will_not_start() -> Result<()> { + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "continuity-1".into(), + kind: CommandKind::Continuity, + selector: "odin".into(), + requested_by: "idunn-continuity".into(), + requested_at_unix_millis: 100, + }; + let failed_restart = |generation: &str, ordinal: u32| -> Result { + let mut transaction = + DeploymentTransaction::new(&command, "odin".into(), ordinal, None, 100)?; + transaction.incumbent_generation_id = Some(generation.into()); + transaction.completion = Some(TransactionCompletion::FailedBeforeFencing { + error: "systemd unit is not running".into(), + }); + Ok(transaction) + }; + + let refused = |transactions: &[DeploymentTransaction], generation: &str| -> usize { + transactions + .iter() + .filter(|value| { + value.target == "odin" + && value.command_kind == CommandKind::Continuity + && value.incumbent_generation_id.as_deref() == Some(generation) + && matches!( + value.completion, + Some(TransactionCompletion::FailedBeforeFencing { .. }) + | Some(TransactionCompletion::FailedAfterFencing { .. }) + ) + }) + .count() + }; + + let attempts = vec![ + failed_restart("generation-1", 0)?, + failed_restart("generation-1", 1)?, + failed_restart("generation-1", 2)?, + ]; + assert!(refused(&attempts, "generation-1") >= CONTINUITY_RESTART_ATTEMPTS); + + // A restart that succeeds admits a new generation, and the count is + // kept against the generation id, so the next release is not condemned + // by the failures of the one it replaced. + assert_eq!(refused(&attempts, "generation-2"), 0); + Ok(()) + } + + #[test] + fn candidate_cleanup_is_owed_when_an_activation_or_a_workload_exists() -> Result<()> { + // (activation issued, workload observed, expected cleanup) + let rows = [ + (false, false, CleanupEvidence::Skipped), + (true, false, CleanupEvidence::Pending), + (false, true, CleanupEvidence::Pending), + (true, true, CleanupEvidence::Pending), + ]; + for (has_activation, has_workload, expected) in rows { + let world = ContinuityProjection::after_candidate_observed()?; + let mut transaction = world.continuity_transaction()?; + if !has_activation { + transaction.activation = None; + } + transaction.workload = has_workload.then(|| workload(1000, 1, 2)); + assert_eq!( + transaction.candidate_cleanup_owed(), + expected, + "activation {has_activation}, workload {has_workload}" + ); + assert_eq!( + pre_fencing_abort_intent(&transaction, "x").candidate_cleanup, + expected, + "pre-fencing intent, activation {has_activation}, workload {has_workload}" + ); + assert_eq!( + post_fencing_abort_intent(&transaction, "x").candidate_cleanup, + expected, + "post-fencing intent, activation {has_activation}, workload {has_workload}" + ); + } + Ok(()) + } + + #[test] + fn a_stored_abort_whose_candidate_cleanup_disagrees_with_the_owed_rule_is_refused() -> Result<()> { + let world = ContinuityProjection::after_candidate_observed()?; + // (activation issued, recorded cleanup, accepted). + // Rows with a workload are omitted: a transaction holding one is refused on + // other grounds, so they could not speak for this rule. The workload operand + // of the owed rule is pinned by the method-level test above. + let rows = [ + (false, CleanupEvidence::Skipped, true), + (false, CleanupEvidence::Pending, false), + (false, CleanupEvidence::Complete, false), + (true, CleanupEvidence::Skipped, false), + (true, CleanupEvidence::Pending, true), + (true, CleanupEvidence::Complete, true), + ]; + for (has_activation, recorded, accepted) in rows { + let label = format!("activation {has_activation}, {recorded:?}"); + let shape = |mut transaction: DeploymentTransaction| -> DeploymentTransaction { + transaction.activation = has_activation.then(|| world.candidate_activation.clone()); + transaction + }; + + let mut pre = shape(world.continuity_transaction()?); + pre.pre_fencing_abort = Some(PreFencingAbort { + error: "candidate refused".into(), + candidate_cleanup: recorded, + topology_reconciliation: pre.abort_topology_reconciliation(), + source_cleanup: CleanupEvidence::Skipped, + }); + assert_eq!(pre.validate().is_ok(), accepted, "pre-fencing, {label}"); + + let mut post = shape(DeploymentTransaction::new( + &command(CommandKind::Deploy), + "ghostlight".into(), + 0, + None, + 100, + )?); + post.phase = DeploymentPhase::Complete; + post.post_fencing_abort = Some(PostFencingAbort { + error: "candidate died after the fence".into(), + route_restoration: CleanupEvidence::Skipped, + lease_withdrawal: CleanupEvidence::Skipped, + candidate_cleanup: recorded, + topology_reconciliation: post.abort_topology_reconciliation(), + source_cleanup: CleanupEvidence::Complete, + }); + post.last_error = Some("candidate died after the fence".into()); + post.completion = Some(TransactionCompletion::FailedAfterFencing { + error: "candidate died after the fence".into(), + recovery: TerminalRecovery::RestoreIncumbent, + }); + // A Pending cleanup is not terminal, so this completed shape is + // only judged for the recorded values a terminal abort can carry. + if recorded != CleanupEvidence::Pending { + assert_eq!(post.validate().is_ok(), accepted, "post-fencing, {label}"); + } + } + Ok(()) } #[test] diff --git a/src/drivers.rs b/src/drivers.rs index 49cb22f..af18a80 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -5894,10 +5894,16 @@ fn request_http_snapshot(target: SocketAddr, payload: &[u8]) -> Result, let mut response = Vec::new(); loop { let size = answer.read_line(ROUTE_HTTP_CHUNK_LINE_BYTES, body)?; + // RFC 9112 7.1: `chunk-size [ chunk-ext ] CRLF`, where an + // extension may follow whitespace. Nothing else is trimmed. let size = std::str::from_utf8(&size) .ok() - .and_then(|size| size.split(';').next()) - .and_then(|size| parse_digits(size.trim_matches([' ', '\t']), 16)) + .and_then(|line| match line.split_once(';') { + Some((digits, _extension)) => { + parse_digits(digits.trim_end_matches([' ', '\t']), 16) + } + None => parse_digits(line, 16), + }) .context("stable CultNet HTTP chunk size is invalid") .map_err(refused)?; if size == 0 { @@ -8876,9 +8882,15 @@ mod tests { /// An honest chunked answer with `after_body` in place of its end. fn chunked_honest_then(id: &str, after_body: &[u8]) -> Vec { + chunked_honest_sized(id, "", "", after_body) + } + + /// An honest chunked answer whose chunk-size line is the hex length between + /// `before` and `after`. + fn chunked_honest_sized(id: &str, before: &str, after: &str, after_body: &[u8]) -> Vec { let body = honest_body(id); let mut reply = CHUNKED_HEAD.to_vec(); - reply.extend_from_slice(format!("{:x}\r\n", body.len()).as_bytes()); + reply.extend_from_slice(format!("{before}{:x}{after}\r\n", body.len()).as_bytes()); reply.extend_from_slice(&body); reply.extend_from_slice(b"\r\n"); reply.extend_from_slice(after_body); @@ -8893,6 +8905,34 @@ mod tests { challenge(scripted_peer(|id| chunked_honest_then(id, b"0\r\nT: 1\r\nU: 2\r\n\r\n")), |_| {}) .expect("an honest chunked answer with trailers"); + // RFC 9112 7.1: whitespace may precede a chunk extension and nothing + // else. The answer is otherwise honest, so only the size line differs. + for (before, after) in [("", ";x"), ("", " ;x"), ("", "\t;x"), ("", ";x=1;y")] { + challenge( + scripted_peer(move |id| chunked_honest_sized(id, before, after, b"0\r\n\r\n")), + |_| {}, + ) + .unwrap_or_else(|error| panic!("size line {before:?}+size+{after:?}: {error:?}")); + } + for (why, before, after) in [ + ("whitespace before the chunk size", " ", ""), + ("a tab before the chunk size", "\t", ""), + ("whitespace after the chunk size and no extension", "", " "), + ("whitespace before the size, ahead of an extension", " ", ";x"), + ("a vertical tab before an extension", "", "\x0b;x"), + ("a carriage return before an extension", "", "\r;x"), + ("a bare carriage return ends the size line", "", "\r"), + ("a doubled carriage return ends the size line", "", "\r\r"), + ] { + assert_refused( + challenge( + scripted_peer(move |id| chunked_honest_sized(id, before, after, b"0\r\n\r\n")), + |_| {}, + ), + why, + ); + } + // A trailer that begins and then the peer stops. assert_refused( challenge(scripted_peer(|id| chunked_honest_then(id, b"0\r\nT: 1\r\n")), |_| {}),