x
# Conflicts: # src/control_plane.rs
This commit is contained in:
commit
cccfe8e174
5 changed files with 229 additions and 45 deletions
|
|
@ -204,7 +204,9 @@ purpose; Idunn discovers no authority implicitly.
|
|||
|
||||
Idunn targets Linux and does not compile on Windows (`drivers.rs` uses
|
||||
`std::os::unix` unconditionally). A local `cargo check` on a Windows
|
||||
workstation proves nothing. Test and build where it runs, in the same image
|
||||
workstation proves nothing. `scripts/verify.sh` is the verification command:
|
||||
the library tests plus a Windows-target check of `idunn-host`. Run it on Linux
|
||||
with the `x86_64-pc-windows-gnu` std installed. Test and build where it runs, in the same image
|
||||
Odin uses, with the working tree shipped to the host's build root:
|
||||
|
||||
```bash
|
||||
|
|
|
|||
|
|
@ -1239,7 +1239,7 @@ Engine-level on `RoutedWorld` with `RouteStubs`, `#[cfg(unix)]`, except where ma
|
|||
| Restart ceiling reads no history | rename `unreadable_history_stops_continuity…` (`:13516`) as B2 did: a corrupt history still restarts, and the target's own log exhausts it | re-adding `history_for_decision` |
|
||||
| Backoff and window | port `continuity_restarts_are_backed_off_and_bounded_per_target` on `SwitchWorkload` | halving the wait; resetting the log on a new generation |
|
||||
| Storm (a) | port `a_failing_reload_that_deletes_the_fragment_backs_the_route_off`: 200 ticks give one reload | the `is_waiting` gate deleted |
|
||||
| Storm (b) | port `a_candidate_whose_route_proof_fails_reloads_at_most_the_ceiling`: Forward installs are capped at 12 | the install gate deleted |
|
||||
| Storm (b) | port `a_candidate_whose_route_proof_fails_reloads_at_most_the_ceiling`: installs and their Survival rollbacks share the 12-per-hour route ledger, so a stateless candidate whose proof keeps failing gets 6 install+rollback pairs per hour; only Forward is refused at the ceiling, and Survival is always admitted (counted when the ledger can record it; ruled 2026-09-30 after B2 Soul) | the install gate deleted |
|
||||
| Driver gates | split B2's refusing-gate test in two: Forward refused runs no program; Survival under a gate that "refuses" Forward still runs | a gate call moved after the write |
|
||||
|
||||
- Run `cargo mutants --in-diff` on the cut. The target is 0 missed in the meters, the
|
||||
|
|
|
|||
8
scripts/verify.sh
Executable file
8
scripts/verify.sh
Executable file
|
|
@ -0,0 +1,8 @@
|
|||
#!/usr/bin/env bash
|
||||
# The one verification command. Run it on Linux with the Windows GNU std installed
|
||||
# (the Eureka verify image has both); a Windows workstation proves nothing.
|
||||
# The Windows check runs first: it is fast, and a hung test cannot mask it.
|
||||
# --lib because the tests under tests/ assume the host layout.
|
||||
set -euo pipefail
|
||||
cargo check --locked --target x86_64-pc-windows-gnu --bin idunn-host
|
||||
cargo test --locked --lib
|
||||
|
|
@ -1552,6 +1552,30 @@ impl DeploymentTransaction {
|
|||
}
|
||||
}
|
||||
|
||||
/// Whether an abort owes the candidate's durable cleanup: it does once an
|
||||
/// activation was issued or a workload was observed, either of which leaves
|
||||
/// something behind.
|
||||
fn candidate_cleanup_owed(&self) -> CleanupEvidence {
|
||||
if self.activation.is_some() || self.workload.is_some() {
|
||||
CleanupEvidence::Pending
|
||||
} else {
|
||||
CleanupEvidence::Skipped
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether an abort's recorded candidate cleanup is what
|
||||
/// `candidate_cleanup_owed` says it must be: owed cleanup is pending or
|
||||
/// done, unowed cleanup is skipped.
|
||||
fn candidate_cleanup_recorded(&self, recorded: CleanupEvidence) -> bool {
|
||||
matches!(
|
||||
(self.candidate_cleanup_owed(), recorded),
|
||||
(
|
||||
CleanupEvidence::Pending,
|
||||
CleanupEvidence::Pending | CleanupEvidence::Complete
|
||||
) | (CleanupEvidence::Skipped, CleanupEvidence::Skipped)
|
||||
)
|
||||
}
|
||||
|
||||
fn rejected(command: &DeploymentCommand, error: anyhow::Error, now: u64) -> Result<Self> {
|
||||
let mut transaction = Self::new(command, command.selector.clone(), 0, None, now)?;
|
||||
let detail = truncate(&format!("{error:#}"), 2048);
|
||||
|
|
@ -1826,14 +1850,7 @@ impl DeploymentTransaction {
|
|||
"aborted transaction also carries post-commit cleanup"
|
||||
);
|
||||
ensure!(
|
||||
matches!(
|
||||
(
|
||||
self.workload.is_some() || self.activation.is_some(),
|
||||
abort.candidate_cleanup
|
||||
),
|
||||
(true, CleanupEvidence::Pending | CleanupEvidence::Complete)
|
||||
| (false, CleanupEvidence::Skipped)
|
||||
),
|
||||
self.candidate_cleanup_recorded(abort.candidate_cleanup),
|
||||
"abort candidate cleanup differs from its prepared activation"
|
||||
);
|
||||
ensure!(
|
||||
|
|
@ -1883,6 +1900,10 @@ impl DeploymentTransaction {
|
|||
self.phase >= DeploymentPhase::Fencing,
|
||||
"post-fencing abort exists before the fence"
|
||||
);
|
||||
ensure!(
|
||||
self.candidate_cleanup_recorded(abort.candidate_cleanup),
|
||||
"abort candidate cleanup differs from its prepared activation"
|
||||
);
|
||||
}
|
||||
|
||||
let failed_after_fencing = matches!(
|
||||
|
|
@ -8645,10 +8666,7 @@ fn incumbent_was_stopped_during_fencing(fencing: &FencingEvidence) -> bool {
|
|||
fn pre_fencing_abort_intent(transaction: &DeploymentTransaction, error: &str) -> PreFencingAbort {
|
||||
PreFencingAbort {
|
||||
error: truncate(error, 2048),
|
||||
candidate_cleanup: candidate_cleanup_requirement(
|
||||
transaction.activation.is_some(),
|
||||
transaction.workload.is_some(),
|
||||
),
|
||||
candidate_cleanup: transaction.candidate_cleanup_owed(),
|
||||
topology_reconciliation: transaction.abort_topology_reconciliation(),
|
||||
source_cleanup: if transaction.command_kind == CommandKind::Deploy {
|
||||
CleanupEvidence::Pending
|
||||
|
|
@ -8679,10 +8697,7 @@ fn post_fencing_abort_intent(
|
|||
} else {
|
||||
CleanupEvidence::Skipped
|
||||
},
|
||||
candidate_cleanup: candidate_cleanup_requirement(
|
||||
transaction.activation.is_some(),
|
||||
transaction.workload.is_some(),
|
||||
),
|
||||
candidate_cleanup: transaction.candidate_cleanup_owed(),
|
||||
topology_reconciliation: transaction.abort_topology_reconciliation(),
|
||||
source_cleanup: if transaction.command_kind == CommandKind::Deploy {
|
||||
CleanupEvidence::Pending
|
||||
|
|
@ -8692,17 +8707,6 @@ fn post_fencing_abort_intent(
|
|||
}
|
||||
}
|
||||
|
||||
fn candidate_cleanup_requirement(
|
||||
has_prepared_activation: bool,
|
||||
has_workload_observation: bool,
|
||||
) -> CleanupEvidence {
|
||||
if has_prepared_activation || has_workload_observation {
|
||||
CleanupEvidence::Pending
|
||||
} else {
|
||||
CleanupEvidence::Skipped
|
||||
}
|
||||
}
|
||||
|
||||
fn sequence_requires_admission(
|
||||
latest: Option<&TopologyEvidence>,
|
||||
maximum_admitted_sequence: u64,
|
||||
|
|
@ -10478,19 +10482,149 @@ mod tests {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn activation_without_workload_still_requires_durable_candidate_cleanup() {
|
||||
assert_eq!(
|
||||
candidate_cleanup_requirement(true, false),
|
||||
CleanupEvidence::Pending
|
||||
);
|
||||
assert_eq!(
|
||||
candidate_cleanup_requirement(true, true),
|
||||
CleanupEvidence::Pending
|
||||
);
|
||||
assert_eq!(
|
||||
candidate_cleanup_requirement(false, false),
|
||||
CleanupEvidence::Skipped
|
||||
);
|
||||
fn continuity_gives_up_on_a_release_that_will_not_start() -> Result<()> {
|
||||
let command = DeploymentCommand {
|
||||
schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(),
|
||||
command_id: "continuity-1".into(),
|
||||
kind: CommandKind::Continuity,
|
||||
selector: "odin".into(),
|
||||
requested_by: "idunn-continuity".into(),
|
||||
requested_at_unix_millis: 100,
|
||||
};
|
||||
let failed_restart = |generation: &str, ordinal: u32| -> Result<DeploymentTransaction> {
|
||||
let mut transaction =
|
||||
DeploymentTransaction::new(&command, "odin".into(), ordinal, None, 100)?;
|
||||
transaction.incumbent_generation_id = Some(generation.into());
|
||||
transaction.completion = Some(TransactionCompletion::FailedBeforeFencing {
|
||||
error: "systemd unit is not running".into(),
|
||||
});
|
||||
Ok(transaction)
|
||||
};
|
||||
|
||||
let refused = |transactions: &[DeploymentTransaction], generation: &str| -> usize {
|
||||
transactions
|
||||
.iter()
|
||||
.filter(|value| {
|
||||
value.target == "odin"
|
||||
&& value.command_kind == CommandKind::Continuity
|
||||
&& value.incumbent_generation_id.as_deref() == Some(generation)
|
||||
&& matches!(
|
||||
value.completion,
|
||||
Some(TransactionCompletion::FailedBeforeFencing { .. })
|
||||
| Some(TransactionCompletion::FailedAfterFencing { .. })
|
||||
)
|
||||
})
|
||||
.count()
|
||||
};
|
||||
|
||||
let attempts = vec![
|
||||
failed_restart("generation-1", 0)?,
|
||||
failed_restart("generation-1", 1)?,
|
||||
failed_restart("generation-1", 2)?,
|
||||
];
|
||||
assert!(refused(&attempts, "generation-1") >= CONTINUITY_RESTART_ATTEMPTS);
|
||||
|
||||
// A restart that succeeds admits a new generation, and the count is
|
||||
// kept against the generation id, so the next release is not condemned
|
||||
// by the failures of the one it replaced.
|
||||
assert_eq!(refused(&attempts, "generation-2"), 0);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn candidate_cleanup_is_owed_when_an_activation_or_a_workload_exists() -> Result<()> {
|
||||
// (activation issued, workload observed, expected cleanup)
|
||||
let rows = [
|
||||
(false, false, CleanupEvidence::Skipped),
|
||||
(true, false, CleanupEvidence::Pending),
|
||||
(false, true, CleanupEvidence::Pending),
|
||||
(true, true, CleanupEvidence::Pending),
|
||||
];
|
||||
for (has_activation, has_workload, expected) in rows {
|
||||
let world = ContinuityProjection::after_candidate_observed()?;
|
||||
let mut transaction = world.continuity_transaction()?;
|
||||
if !has_activation {
|
||||
transaction.activation = None;
|
||||
}
|
||||
transaction.workload = has_workload.then(|| workload(1000, 1, 2));
|
||||
assert_eq!(
|
||||
transaction.candidate_cleanup_owed(),
|
||||
expected,
|
||||
"activation {has_activation}, workload {has_workload}"
|
||||
);
|
||||
assert_eq!(
|
||||
pre_fencing_abort_intent(&transaction, "x").candidate_cleanup,
|
||||
expected,
|
||||
"pre-fencing intent, activation {has_activation}, workload {has_workload}"
|
||||
);
|
||||
assert_eq!(
|
||||
post_fencing_abort_intent(&transaction, "x").candidate_cleanup,
|
||||
expected,
|
||||
"post-fencing intent, activation {has_activation}, workload {has_workload}"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stored_abort_whose_candidate_cleanup_disagrees_with_the_owed_rule_is_refused() -> Result<()> {
|
||||
let world = ContinuityProjection::after_candidate_observed()?;
|
||||
// (activation issued, recorded cleanup, accepted).
|
||||
// Rows with a workload are omitted: a transaction holding one is refused on
|
||||
// other grounds, so they could not speak for this rule. The workload operand
|
||||
// of the owed rule is pinned by the method-level test above.
|
||||
let rows = [
|
||||
(false, CleanupEvidence::Skipped, true),
|
||||
(false, CleanupEvidence::Pending, false),
|
||||
(false, CleanupEvidence::Complete, false),
|
||||
(true, CleanupEvidence::Skipped, false),
|
||||
(true, CleanupEvidence::Pending, true),
|
||||
(true, CleanupEvidence::Complete, true),
|
||||
];
|
||||
for (has_activation, recorded, accepted) in rows {
|
||||
let label = format!("activation {has_activation}, {recorded:?}");
|
||||
let shape = |mut transaction: DeploymentTransaction| -> DeploymentTransaction {
|
||||
transaction.activation = has_activation.then(|| world.candidate_activation.clone());
|
||||
transaction
|
||||
};
|
||||
|
||||
let mut pre = shape(world.continuity_transaction()?);
|
||||
pre.pre_fencing_abort = Some(PreFencingAbort {
|
||||
error: "candidate refused".into(),
|
||||
candidate_cleanup: recorded,
|
||||
topology_reconciliation: pre.abort_topology_reconciliation(),
|
||||
source_cleanup: CleanupEvidence::Skipped,
|
||||
});
|
||||
assert_eq!(pre.validate().is_ok(), accepted, "pre-fencing, {label}");
|
||||
|
||||
let mut post = shape(DeploymentTransaction::new(
|
||||
&command(CommandKind::Deploy),
|
||||
"ghostlight".into(),
|
||||
0,
|
||||
None,
|
||||
100,
|
||||
)?);
|
||||
post.phase = DeploymentPhase::Complete;
|
||||
post.post_fencing_abort = Some(PostFencingAbort {
|
||||
error: "candidate died after the fence".into(),
|
||||
route_restoration: CleanupEvidence::Skipped,
|
||||
lease_withdrawal: CleanupEvidence::Skipped,
|
||||
candidate_cleanup: recorded,
|
||||
topology_reconciliation: post.abort_topology_reconciliation(),
|
||||
source_cleanup: CleanupEvidence::Complete,
|
||||
});
|
||||
post.last_error = Some("candidate died after the fence".into());
|
||||
post.completion = Some(TransactionCompletion::FailedAfterFencing {
|
||||
error: "candidate died after the fence".into(),
|
||||
recovery: TerminalRecovery::RestoreIncumbent,
|
||||
});
|
||||
// A Pending cleanup is not terminal, so this completed shape is
|
||||
// only judged for the recorded values a terminal abort can carry.
|
||||
if recorded != CleanupEvidence::Pending {
|
||||
assert_eq!(post.validate().is_ok(), accepted, "post-fencing, {label}");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -5894,10 +5894,16 @@ fn request_http_snapshot(target: SocketAddr, payload: &[u8]) -> Result<Vec<u8>,
|
|||
let mut response = Vec::new();
|
||||
loop {
|
||||
let size = answer.read_line(ROUTE_HTTP_CHUNK_LINE_BYTES, body)?;
|
||||
// RFC 9112 7.1: `chunk-size [ chunk-ext ] CRLF`, where an
|
||||
// extension may follow whitespace. Nothing else is trimmed.
|
||||
let size = std::str::from_utf8(&size)
|
||||
.ok()
|
||||
.and_then(|size| size.split(';').next())
|
||||
.and_then(|size| parse_digits(size.trim_matches([' ', '\t']), 16))
|
||||
.and_then(|line| match line.split_once(';') {
|
||||
Some((digits, _extension)) => {
|
||||
parse_digits(digits.trim_end_matches([' ', '\t']), 16)
|
||||
}
|
||||
None => parse_digits(line, 16),
|
||||
})
|
||||
.context("stable CultNet HTTP chunk size is invalid")
|
||||
.map_err(refused)?;
|
||||
if size == 0 {
|
||||
|
|
@ -8876,9 +8882,15 @@ mod tests {
|
|||
|
||||
/// An honest chunked answer with `after_body` in place of its end.
|
||||
fn chunked_honest_then(id: &str, after_body: &[u8]) -> Vec<u8> {
|
||||
chunked_honest_sized(id, "", "", after_body)
|
||||
}
|
||||
|
||||
/// An honest chunked answer whose chunk-size line is the hex length between
|
||||
/// `before` and `after`.
|
||||
fn chunked_honest_sized(id: &str, before: &str, after: &str, after_body: &[u8]) -> Vec<u8> {
|
||||
let body = honest_body(id);
|
||||
let mut reply = CHUNKED_HEAD.to_vec();
|
||||
reply.extend_from_slice(format!("{:x}\r\n", body.len()).as_bytes());
|
||||
reply.extend_from_slice(format!("{before}{:x}{after}\r\n", body.len()).as_bytes());
|
||||
reply.extend_from_slice(&body);
|
||||
reply.extend_from_slice(b"\r\n");
|
||||
reply.extend_from_slice(after_body);
|
||||
|
|
@ -8893,6 +8905,34 @@ mod tests {
|
|||
challenge(scripted_peer(|id| chunked_honest_then(id, b"0\r\nT: 1\r\nU: 2\r\n\r\n")), |_| {})
|
||||
.expect("an honest chunked answer with trailers");
|
||||
|
||||
// RFC 9112 7.1: whitespace may precede a chunk extension and nothing
|
||||
// else. The answer is otherwise honest, so only the size line differs.
|
||||
for (before, after) in [("", ";x"), ("", " ;x"), ("", "\t;x"), ("", ";x=1;y")] {
|
||||
challenge(
|
||||
scripted_peer(move |id| chunked_honest_sized(id, before, after, b"0\r\n\r\n")),
|
||||
|_| {},
|
||||
)
|
||||
.unwrap_or_else(|error| panic!("size line {before:?}+size+{after:?}: {error:?}"));
|
||||
}
|
||||
for (why, before, after) in [
|
||||
("whitespace before the chunk size", " ", ""),
|
||||
("a tab before the chunk size", "\t", ""),
|
||||
("whitespace after the chunk size and no extension", "", " "),
|
||||
("whitespace before the size, ahead of an extension", " ", ";x"),
|
||||
("a vertical tab before an extension", "", "\x0b;x"),
|
||||
("a carriage return before an extension", "", "\r;x"),
|
||||
("a bare carriage return ends the size line", "", "\r"),
|
||||
("a doubled carriage return ends the size line", "", "\r\r"),
|
||||
] {
|
||||
assert_refused(
|
||||
challenge(
|
||||
scripted_peer(move |id| chunked_honest_sized(id, before, after, b"0\r\n\r\n")),
|
||||
|_| {},
|
||||
),
|
||||
why,
|
||||
);
|
||||
}
|
||||
|
||||
// A trailer that begins and then the peer stops.
|
||||
assert_refused(
|
||||
challenge(scripted_peer(|id| chunked_honest_then(id, b"0\r\nT: 1\r\n")), |_| {}),
|
||||
|
|
|
|||
Loading…
Reference in a new issue