GameCult/Idunn
Watch
9
0
Fork
You've already forked Idunn
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Runner failure tests: named step, bounded tail, never echoed, private store

Browse source 
Drive the real DockerRunnerDriver over a fake docker program, through the
engine's Sealing path: the second step is named with its exit status, both
tails are kept in a 0600 store capped at 32, no error, control.cc or journal
line carries the output, status escapes control bytes, an unwritable store
does not stop the abort, and control.cc's record bytes equal those captured at
base 804ed75.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Meta Vens 2026-10-25 02:16:03 +01:00
commit 22bd904ed9
3 changed files with 647 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

313
src/control_plane.rs
View file

@ -5397,10 +5397,12 @@ impl Engine {
/// store and says so on the journal, naming no output. The store write is
/// best effort: its failure is named in the line and never stops the
/// abort that follows.
fn keep_runner_failure(&self, transaction: &DeploymentTransaction, error: &anyhow::Error) {
let Some(failure) = error.downcast_ref::<RunnerStepFailure>() else {
return;
};
fn keep_runner_failure(
&self,
transaction: &DeploymentTransaction,
error: &anyhow::Error,
) -> Option<String> {
let failure = error.downcast_ref::<RunnerStepFailure>()?;
let path = runner_evidence_store_path(&self.options.state_store);
let recorded = now_millis().and_then(|now| {
runner_evidence::record(
@ -5414,10 +5416,7 @@ impl Engine {
),
)
});
eprintln!(
"{}",
runner_failure_journal_line(transaction, failure, &path, &recorded)
);
Some(runner_failure_journal_line(transaction, failure, &path, &recorded))
}
/// Settle the outcome of one incident operation. A failure never leaves
@ -6361,7 +6360,9 @@ impl Engine {
if latest.value.pre_fencing_abort.is_none()
&& error.downcast_ref::<LockContended>().is_none()
{
self.keep_runner_failure(&latest.value, &error);
if let Some(line) = self.keep_runner_failure(&latest.value, &error) {
eprintln!("{line}");
}
self.begin_pre_fencing_abort(latest, error)?;
} else {
self.record_resumable_error(latest, &error)?;
@ -24726,4 +24727,298 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide
Ok(())
}
}
// BEGIN runner-failure-evidence tests
// A failed runner step is named, its output tail is kept in
// runner-evidence.cc, and no output reaches an error, control.cc or the
// journal line. The runner is the real DockerRunnerDriver driving a fake
// docker program, through the engine's Sealing path.
const RUNNER_CANARY: &str = "CANARY-7c41e0d9";
/// A source that freezes a fixed tree, so a deployment reaches the runner.
struct TreeSource {
recipe: String,
root: PathBuf,
}
impl SourcePort for TreeSource {
fn resolve(
&self,
_: &OperatorBinding,
_: &str,
_: u64,
) -> Result<crate::drivers::ResolvedSource> {
Ok(crate::drivers::ResolvedSource {
facts: crate::deployment_plan::tests::source(&self.recipe),
recipe_bytes: self.recipe.clone().into_bytes(),
})
}
fn freeze(&self, transaction_id: &str, plan: &CompiledDeploymentPlan) -> Result<FrozenSourceReceipt> {
Ok(FrozenSourceReceipt {
transaction_id: transaction_id.to_owned(),
plan_id: plan.plan_id.clone(),
snapshot_sha256: format!("sha256-{}", "a".repeat(64)),
})
}
fn observe_frozen(
&self,
plan: &CompiledDeploymentPlan,
receipt: &FrozenSourceReceipt,
) -> Result<crate::drivers::FrozenSource> {
Ok(crate::drivers::FrozenSource::fixture(
receipt.clone(),
plan.source.clone(),
plan.recipe_blob.clone(),
self.root.clone(),
))
}
fn cleanup(&self, _: &str, _: Option<&FrozenSourceReceipt>) -> Result<()> {
Ok(())
}
}
/// The unrouted recipe with two steps ahead of its build, and no external
/// input: the second step, `test-daemon`, is the one the fake docker fails.
fn two_step_recipe() -> String {
let recipe = unrouted_recipe();
let (head, rest) = recipe
.split_once("[[external_inputs]]")
.expect("the recipe declares an external input");
let (_, tail) = rest.split_once("[[artifacts]]").expect("the recipe declares artifacts");
let head = head.replacen(
"[[steps]]\nid = \"build\"",
"[[steps]]\nid = \"install\"\nphase = \"prepare\"\nrunner = \"rust\"\nargv = [\"cargo\", \"fetch\"]\n\n\
[[steps]]\nid = \"test-daemon\"\nphase = \"test\"\nrunner = \"rust\"\nargv = [\"cargo\", \"test\"]\n\n\
[[steps]]\nid = \"build\"",
1,
);
format!("{head}[[artifacts]]{tail}")
}
/// A Sealing world whose runner is the real driver over a fake docker: the
/// first call succeeds, the second prints the canary and an escape
/// sequence on both streams and exits 3.
#[cfg(unix)]
fn failing_runner_world() -> Result<(EngineFixture, TempDir)> {
use std::os::unix::fs::PermissionsExt;
let tools = TempDir::new()?;
let docker = tools.path().join("docker");
std::fs::write(
&docker,
format!(
"#!/bin/sh\nn=$(cat \"$0.count\" 2>/dev/null || echo 0)\nn=$((n+1))\necho $n > \"$0.count\"\n\
[ \"$n\" -lt 2 ] && exit 0\n\
printf 'out {RUNNER_CANARY} \\033[2J\\n'\nprintf 'err {RUNNER_CANARY}\\n' >&2\nexit 3\n"
),
)?;
std::fs::set_permissions(&docker, std::fs::Permissions::from_mode(0o755))?;
let recipe = two_step_recipe();
let mut world = sealing_world(&recipe)?;
let tree = tools.path().join("tree");
std::fs::create_dir_all(&tree)?;
std::fs::write(tree.join("README"), b"source")?;
world.engine.source = Arc::new(TreeSource { recipe, root: tree });
world.engine.docker_runner = DockerRunnerDriver { docker_program: docker };
Ok((world, tools))
}
/// Scheduler ticks until the deployment ends (or the budget is spent).
fn tick_until_ended(world: &EngineFixture) -> Result<DeploymentTransaction> {
for _ in 0..24 {
world.engine.resume_backoff.lock().unwrap().clear();
let _ = world.engine.run_scheduler_tick();
if latest(world)?.completion.is_some() {
break;
}
}
latest(world)
}
fn printed_status(world: &EngineFixture) -> Result<String> {
let mut printed = Vec::new();
status(&mut printed, &world.state_store, &world.engine.options.incident_store, Some("up-service"))?;
Ok(String::from_utf8(printed)?)
}
fn canary_failure(stdout: &[u8]) -> RunnerStepFailure {
RunnerStepFailure {
subject: crate::drivers::RunnerSubject::Step("test-daemon".into()),
runner: "rust".into(),
exit_code: Some(3),
signal: None,
stdout_tail: stdout.to_vec(),
stdout_total_bytes: stdout.len() as u64,
stderr_tail: stdout.to_vec(),
stderr_total_bytes: stdout.len() as u64,
}
}
#[cfg(unix)]
#[test]
fn a_failed_runner_step_is_named_kept_and_never_echoed() -> Result<()> {
use std::os::unix::fs::PermissionsExt;
let (world, _tools) = failing_runner_world()?;
let ended = tick_until_ended(&world)?;
let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else {
bail!("the deployment did not fail before fencing: {:?} {:?}", ended.phase, ended.last_error);
};
// Named: the second step, on its runner, with the exit status.
assert!(
error.contains("runner step test-daemon on runner rust exited with exit status 3"),
"{error}"
);
// Never echoed: no error text, no control.cc byte, carries the output.
let abort = ended.pre_fencing_abort.as_ref().context("no pre-fencing abort")?;
for text in [error.as_str(), abort.error.as_str(), &format!("{ended:?}")] {
assert!(!text.contains(RUNNER_CANARY), "runner output reached a record: {text}");
}
let control = std::fs::read(&world.state_store)?;
assert!(
!control.windows(RUNNER_CANARY.len()).any(|window| window == RUNNER_CANARY.as_bytes()),
"runner output reached control.cc"
);
// Kept: in the private store, tails of both streams, and on status.
let evidence = runner_evidence_store_path(&world.state_store);
assert_eq!(std::fs::metadata(&evidence)?.permissions().mode() & 0o777, 0o600);
let record = runner_evidence::for_transaction(&evidence, &ended.transaction_id)?
.context("no evidence for the failed transaction")?;
assert_eq!(record.subject_id, "test-daemon");
assert_eq!(record.exit_code, Some(3));
assert!(String::from_utf8_lossy(&record.stdout_tail).contains(RUNNER_CANARY));
assert!(String::from_utf8_lossy(&record.stderr_tail).contains(RUNNER_CANARY));
let printed = printed_status(&world)?;
assert!(printed.contains("failed step test-daemon"), "{printed}");
assert!(printed.contains(RUNNER_CANARY), "{printed}");
assert!(!printed.contains('\u{1b}'), "status printed a raw escape byte");
Ok(())
}
#[test]
fn the_journal_line_names_the_store_and_never_the_output() -> Result<()> {
let world = EngineFixture::new()?;
let (transaction, _) = terminal_transaction_with_command("ghostlight")?;
let failure = canary_failure(RUNNER_CANARY.as_bytes());
let line = world
.engine
.keep_runner_failure(&transaction, &anyhow::Error::new(failure.clone()).context("sealing"))
.context("a runner failure produced no journal line")?;
assert!(!line.contains(RUNNER_CANARY), "{line}");
for needle in [
transaction.transaction_id.as_str(),
"up-ghostlight",
"ghostlight",
"runner step test-daemon on runner rust exited with exit status 3",
"runner-evidence.cc",
"idunn status --command up-ghostlight",
] {
assert!(line.contains(needle), "{needle} missing from {line}");
}
// Not a runner failure: nothing is kept and nothing is said.
assert!(
world
.engine
.keep_runner_failure(&transaction, &anyhow::anyhow!("a different failure"))
.is_none()
);
// The line a failed write produces says so, with no output.
let unwritable = runner_failure_journal_line(
&transaction,
&failure,
Path::new("/var/lib/gamecult/idunn/runner-evidence.cc"),
&Err(anyhow::anyhow!("disk full")),
);
assert!(unwritable.contains("evidence was not kept: disk full"), "{unwritable}");
assert!(!unwritable.contains(RUNNER_CANARY));
Ok(())
}
#[cfg(unix)]
#[test]
fn a_failed_evidence_write_does_not_stop_the_abort() -> Result<()> {
let (world, _tools) = failing_runner_world()?;
// A directory where the store should be: every write to it fails.
std::fs::create_dir(runner_evidence_store_path(&world.state_store))?;
let ended = tick_until_ended(&world)?;
let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else {
bail!("the unwritable store stopped the abort: {:?} {:?}", ended.phase, ended.last_error);
};
assert!(error.contains("runner step test-daemon on runner rust"), "{error}");
let line = world
.engine
.keep_runner_failure(&ended, &anyhow::Error::new(canary_failure(b"")))
.context("no journal line")?;
assert!(line.contains("evidence was not kept"), "{line}");
Ok(())
}
#[cfg(unix)]
#[test]
fn an_unreadable_evidence_store_costs_only_its_lines() -> Result<()> {
let (world, _tools) = failing_runner_world()?;
let ended = tick_until_ended(&world)?;
std::fs::write(runner_evidence_store_path(&world.state_store), b"\xc1not a store")?;
let printed = printed_status(&world)?;
assert!(printed.contains(&format!("transaction {}", ended.transaction_id)), "{printed}");
assert!(printed.contains("target service phase"), "{printed}");
assert!(printed.contains("pre-fencing abort"), "{printed}");
assert_eq!(printed.matches("runner evidence unreadable").count(), 1, "{printed}");
Ok(())
}
#[cfg(unix)]
#[test]
fn control_store_types_are_unchanged_by_a_runner_failure() -> Result<()> {
let (world, _tools) = failing_runner_world()?;
tick_until_ended(&world)?;
// The store a rolled-back Idunn reads: it refuses any foreign type.
ControlSnapshot::read(&world.state_store)?;
let envelopes =
SingleFileMessagePackBackingStore::new(&world.state_store).pull_all_read_only_snapshot()?;
assert!(!envelopes.is_empty());
for envelope in &envelopes {
assert!(!envelope.r#type.contains("runner_failure"), "{}", envelope.r#type);
if envelope.r#type == DeploymentTransaction::TYPE {
assert_eq!(envelope.schema_id.as_deref(), Some("idunn.deployment_transaction.v4"));
}
}
Ok(())
}
/// The records control.cc holds re-encode byte for byte. The hex was
/// captured at base 804ed75, before this cut touched anything: a runner
/// failure adds no field, no type and no schema to the control store.
#[test]
fn control_records_encode_exactly_as_they_did_at_base() -> Result<()> {
let command = DeploymentCommand {
schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(),
command_id: "up-service".into(),
kind: CommandKind::Deploy,
selector: "service".into(),
requested_by: "test".into(),
requested_at_unix_millis: 100,
};
let mut transaction = DeploymentTransaction::new(&command, "service".into(), 0, None, 100)?;
transaction.transaction_id = "tx-pinned".into();
transaction.last_error = Some("runner step test-daemon on runner rust exited with exit status 3".into());
transaction.pre_fencing_abort = Some(PreFencingAbort {
error: "runner step test-daemon on runner rust exited with exit status 3".into(),
candidate_cleanup: CleanupEvidence::Skipped,
topology_reconciliation: CleanupEvidence::Skipped,
source_cleanup: CleanupEvidence::Complete,
});
let hex = |bytes: &[u8]| bytes.iter().map(|byte| format!("{byte:02x}")).collect::<String>();
let command_hex = hex(&command_envelope(&command, 100)?.payload);
let transaction_hex = hex(&transaction_envelope(&transaction, 100)?.payload);
assert_eq!(command_hex, PINNED_COMMAND_HEX, "command bytes changed");
assert_eq!(transaction_hex, PINNED_TRANSACTION_HEX, "transaction bytes changed");
Ok(())
}
const PINNED_COMMAND_HEX: &str = "96bb6964756e6e2e6465706c6f796d656e745f636f6d6d616e642e7632aa75702d73657276696365a66465706c6f79a773657276696365a47465737464";
const PINNED_TRANSACTION_HEX: &str = "dc0025bf6964756e6e2e6465706c6f796d656e745f7472616e73616374696f6e2e7634a974782d70696e6e6564aa75702d73657276696365a66465706c6f79a77365727669636500a77365616c696e676464c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c000d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033c094d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033a7736b6970706564a7736b6970706564a8636f6d706c657465c0c0c0c0";
// END runner-failure-evidence tests
}

197
src/drivers.rs
View file

@ -157,6 +157,24 @@ pub struct FrozenSourceReceipt {
pub snapshot_sha256: String,
}
#[cfg(test)]
impl FrozenSource {
/// A frozen source over an existing tree, for tests that drive a runner.
pub(crate) fn fixture(
receipt: FrozenSourceReceipt,
facts: SourceSelectionFacts,
recipe_bytes: Vec<u8>,
root: PathBuf,
) -> Self {
Self {
receipt,
facts,
recipe_bytes,
root,
}
}
}
impl FrozenSourceReceipt {
pub fn validate_against(&self, plan: &CompiledDeploymentPlan) -> Result<()> {
plan.validate()?;
@ -16372,4 +16390,183 @@ processes 9
);
Ok(())
}
// BEGIN runner-failure-evidence tests
// The runner decides what failed and what it kept: these drive the real
// DockerRunnerDriver over a fake docker program, never the capture alone.
const RUNNER_CANARY: &str = "CANARY-5d2e91b7";
#[cfg(unix)]
fn fake_docker(dir: &Path, body: &str) -> Result<DockerRunnerDriver> {
use std::os::unix::fs::PermissionsExt;
let program = dir.join("docker");
fs::write(&program, format!("#!/bin/sh\n{body}\n"))?;
fs::set_permissions(&program, fs::Permissions::from_mode(0o755))?;
Ok(DockerRunnerDriver {
docker_program: program,
})
}
/// One recipe step through the real `run_in_workspace` path.
#[cfg(unix)]
fn run_step(driver: &DockerRunnerDriver, dir: &Path, step_id: &str) -> Result<()> {
let workspace = dir.join("workspace");
fs::create_dir_all(&workspace)?;
driver.run_in_workspace(
step_id,
"rust",
&fixture_docker_runner_binding(),
&workspace,
Path::new("."),
&["cargo".to_owned(), "test".to_owned()],
&std::collections::BTreeSet::new(),
("IDUNN_STAMP", "rev"),
)
}
#[cfg(unix)]
fn failure_of(result: Result<()>) -> Result<RunnerStepFailure> {
let error = result.expect_err("the step did not fail");
error
.downcast_ref::<RunnerStepFailure>()
.cloned()
.with_context(|| format!("the failure is not typed: {error:#}"))
}
#[cfg(unix)]
#[test]
fn a_failed_step_is_named_with_both_tails() -> Result<()> {
let temp = tempfile::tempdir()?;
// The first step passes, the second fails: the count is the fake's.
let driver = fake_docker(
temp.path(),
&format!(
"n=$(cat \"$0.count\" 2>/dev/null || echo 0); n=$((n+1)); echo $n > \"$0.count\"\n\
[ \"$n\" -lt 2 ] && exit 0\n\
echo 'stdout line one'; echo 'stdout report {RUNNER_CANARY}'\n\
echo 'stderr report' >&2\nexit 3"
),
)?;
run_step(&driver, temp.path(), "first")?;
let failure = failure_of(run_step(&driver, temp.path(), "second"))?;
assert_eq!(failure.subject, RunnerSubject::Step("second".into()));
assert_eq!(failure.runner, "rust");
assert_eq!(failure.exit_code, Some(3));
assert_eq!(failure.signal, None);
assert!(
failure
.stdout_tail
.ends_with(format!("stdout report {RUNNER_CANARY}\n").as_bytes()),
"stdout was dropped: {:?}",
String::from_utf8_lossy(&failure.stdout_tail)
);
assert_eq!(failure.stderr_tail, b"stderr report\n");
assert_eq!(failure.stdout_total_bytes, failure.stdout_tail.len() as u64);
Ok(())
}
#[cfg(unix)]
#[test]
fn a_step_killed_by_a_signal_is_named_by_its_signal() -> Result<()> {
let temp = tempfile::tempdir()?;
let driver = fake_docker(temp.path(), "kill -9 $$")?;
let failure = failure_of(run_step(&driver, temp.path(), "build"))?;
assert_eq!((failure.exit_code, failure.signal), (None, Some(9)));
assert!(failure.to_string().contains("exited with signal 9"), "{failure}");
Ok(())
}
#[cfg(unix)]
#[test]
fn a_failed_external_input_is_named_as_one() -> Result<()> {
let temp = tempfile::tempdir()?;
let driver = fake_docker(temp.path(), "echo curl failed >&2\nexit 22")?;
let declaration = TargetDeclaration::parse(crate::deployment_plan::tests::RECIPE)?;
let binding = OperatorBinding::parse(crate::deployment_plan::tests::BINDING)?;
let input_id = declaration.external_inputs[0].id.clone();
let workspaces = BTreeMap::from([("rust".to_owned(), temp.path().join("workspace"))]);
fs::create_dir_all(&workspaces["rust"])?;
let error = driver
.materialize_external_input(&declaration, &binding, &workspaces, &input_id)
.expect_err("the download did not fail");
let failure = error
.downcast_ref::<RunnerStepFailure>()
.with_context(|| format!("the failure is not typed: {error:#}"))?;
assert_eq!(failure.subject, RunnerSubject::ExternalInput(input_id.clone()));
assert_eq!(failure.exit_code, Some(22));
assert!(
failure.to_string().contains(&format!("runner external input {input_id} on runner rust")),
"{failure}"
);
Ok(())
}
#[cfg(unix)]
#[test]
fn runner_output_never_reaches_the_error_text() -> Result<()> {
let temp = tempfile::tempdir()?;
let driver = fake_docker(
temp.path(),
&format!("echo 'out {RUNNER_CANARY}'\necho 'err {RUNNER_CANARY}' >&2\nexit 1"),
)?;
let failure = failure_of(run_step(&driver, temp.path(), "build"))?;
assert!(String::from_utf8_lossy(&failure.stdout_tail).contains(RUNNER_CANARY));
assert!(String::from_utf8_lossy(&failure.stderr_tail).contains(RUNNER_CANARY));
let error = anyhow::Error::new(failure.clone()).context("sealing");
for text in [
format!("{error:#}"),
format!("{error:?}"),
format!("{failure}"),
format!("{failure:?}"),
] {
assert!(!text.contains(RUNNER_CANARY), "runner output reached text: {text}");
}
Ok(())
}
#[cfg(unix)]
#[test]
fn each_tail_is_the_last_bound_bytes_with_the_total() -> Result<()> {
let temp = tempfile::tempdir()?;
let data = temp.path().join("data");
for size in [
0,
1,
RUNNER_OUTPUT_TAIL_BYTES - 1,
RUNNER_OUTPUT_TAIL_BYTES,
RUNNER_OUTPUT_TAIL_BYTES + 1,
1024 * 1024,
] {
// Distinct neighbours, so a shifted or head-keeping tail differs.
let bytes: Vec<u8> = (0..size).map(|index| (index % 251) as u8).collect();
fs::write(&data, &bytes)?;
let driver = fake_docker(
temp.path(),
&format!("cat '{}'\ncat '{}' >&2\nexit 1", data.display(), data.display()),
)?;
let failure = failure_of(run_step(&driver, temp.path(), "build"))?;
let expected = &bytes[size.saturating_sub(RUNNER_OUTPUT_TAIL_BYTES)..];
assert_eq!(failure.stdout_tail, expected, "stdout tail at {size} bytes");
assert_eq!(failure.stderr_tail, expected, "stderr tail at {size} bytes");
assert_eq!(failure.stdout_total_bytes, size as u64);
assert_eq!(failure.stderr_total_bytes, size as u64);
}
Ok(())
}
#[cfg(unix)]
#[test]
fn a_successful_step_and_a_missing_program_keep_their_outcomes() -> Result<()> {
let temp = tempfile::tempdir()?;
run_step(&fake_docker(temp.path(), "echo fine")?, temp.path(), "build")?;
let missing = DockerRunnerDriver {
docker_program: temp.path().join("no-such-docker"),
};
let error = run_step(&missing, temp.path(), "build").expect_err("a missing program ran");
assert!(error.downcast_ref::<RunnerStepFailure>().is_none());
assert!(format!("{error:#}").contains("starting Docker runner"), "{error:#}");
Ok(())
}
// END runner-failure-evidence tests
}

146
src/runner_evidence.rs
View file

@ -281,3 +281,149 @@ fn escape(text: &str) -> String {
}
escaped
}
#[cfg(all(test, unix))]
mod tests {
use super::*;
use std::os::unix::fs::PermissionsExt;
fn failure(subject: RunnerSubject, stdout: &[u8], stderr: &[u8]) -> RunnerStepFailure {
RunnerStepFailure {
subject,
runner: "rust".into(),
exit_code: Some(3),
signal: None,
stdout_tail: stdout.to_vec(),
stdout_total_bytes: stdout.len() as u64,
stderr_tail: stderr.to_vec(),
stderr_total_bytes: stderr.len() as u64,
}
}
fn record_at(transaction: &str, now: u64) -> RunnerFailureRecord {
RunnerFailureRecord::from_failure(
transaction,
"up-service",
"service",
&failure(RunnerSubject::Step("build".into()), b"out\n", b"err\n"),
now,
)
}
#[test]
fn the_evidence_store_keeps_the_newest_32() -> Result<()> {
let temp = tempfile::tempdir()?;
let path = runner_evidence_store_path(&temp.path().join("control.cc"));
for index in 0..=RUNNER_EVIDENCE_RECORDS {
record(&path, &record_at(&format!("tx-{index:02}"), 1000 + index as u64))?;
}
let kept: Vec<_> = snapshot(&path)?.1.into_iter().map(|stored| stored.transaction_id).collect();
assert_eq!(kept.len(), RUNNER_EVIDENCE_RECORDS);
assert!(!kept.contains(&"tx-00".to_owned()), "the oldest survived");
assert!(kept.contains(&"tx-32".to_owned()), "the newest was evicted");
assert!(kept.contains(&"tx-01".to_owned()), "the second oldest was evicted early");
// Rewriting a resident transaction replaces it and drops nothing.
let mut again = record_at("tx-05", 5000);
again.stdout_tail = b"rewritten\n".to_vec();
again.stdout_total_bytes = 10;
record(&path, &again)?;
let kept = snapshot(&path)?.1;
assert_eq!(kept.len(), RUNNER_EVIDENCE_RECORDS);
assert!(kept.iter().any(|stored| stored.transaction_id == "tx-01"));
let stored = for_transaction(&path, "tx-05")?.context("rewritten record is gone")?;
assert_eq!(stored.stdout_tail, b"rewritten\n");
assert_eq!(for_transaction(&path, "tx-00")?, None);
Ok(())
}
#[test]
fn a_record_round_trips_byte_for_byte() -> Result<()> {
let temp = tempfile::tempdir()?;
let path = runner_evidence_store_path(&temp.path().join("control.cc"));
let written = RunnerFailureRecord::from_failure(
"tx-a",
"up-service",
"service",
&failure(RunnerSubject::ExternalInput("toolchain".into()), &[0, 1, 255, 10], b""),
77,
);
record(&path, &written)?;
assert_eq!(for_transaction(&path, "tx-a")?, Some(written));
assert_eq!(for_transaction(&path, "tx-b")?, None);
Ok(())
}
#[test]
fn the_evidence_store_is_private() -> Result<()> {
let temp = tempfile::tempdir()?;
let path = runner_evidence_store_path(&temp.path().join("control.cc"));
// Whatever the umask, the mode is the store's own rule, on every write.
for index in 0..2 {
record(&path, &record_at(&format!("tx-{index}"), 10 + index))?;
assert_eq!(std::fs::metadata(&path)?.permissions().mode() & 0o777, 0o600);
}
Ok(())
}
#[test]
fn a_tail_over_its_bound_or_its_stream_is_refused() {
let mut oversize = record_at("tx-a", 1);
oversize.stdout_tail = vec![b'x'; RUNNER_OUTPUT_TAIL_BYTES + 1];
oversize.stdout_total_bytes = u64::MAX;
assert!(oversize.validate().is_err());
let mut impossible = record_at("tx-a", 1);
impossible.stderr_total_bytes = 1;
assert!(impossible.validate().is_err());
let mut exact = record_at("tx-a", 1);
exact.stdout_tail = vec![b'x'; RUNNER_OUTPUT_TAIL_BYTES];
exact.stdout_total_bytes = RUNNER_OUTPUT_TAIL_BYTES as u64;
assert!(exact.validate().is_ok());
}
#[test]
fn status_renders_the_tail_with_control_bytes_escaped() {
let mut stored = record_at("tx-a", 1);
stored.stdout_tail = b"first\n\x1b[2Jsecond\x00end\tTab\nlast\n".to_vec();
stored.stdout_total_bytes = 5000;
stored.stderr_tail = b"bad\r\x07".to_vec();
stored.stderr_total_bytes = 6;
let lines = render(&stored);
let text = lines.join("\n");
assert!(!text.contains('\u{1b}') && !text.contains('\0') && !text.contains('\r'));
assert!(!text.contains('\u{7}'));
assert_eq!(
lines,
vec![
" failed step build",
" runner rust",
" status exit status 3",
" stdout (last 30 of 5000 bytes)",
" | first",
" | \\u{1b}[2Jsecond\\u{0}end\tTab",
" | last",
" stderr (last 5 of 6 bytes)",
" | bad\\u{d}\\u{7}",
]
);
}
#[test]
fn an_empty_tail_renders_only_its_header() {
let mut stored = record_at("tx-a", 1);
stored.stdout_tail = Vec::new();
stored.stdout_total_bytes = 0;
let lines = render(&stored);
assert!(lines.contains(&" stdout (last 0 of 0 bytes)".to_owned()));
assert!(!lines.iter().any(|line| line == " | "));
}
#[test]
fn a_foreign_or_corrupt_store_is_an_error_not_an_empty_store() -> Result<()> {
let temp = tempfile::tempdir()?;
let path = runner_evidence_store_path(&temp.path().join("control.cc"));
std::fs::write(&path, b"\xc1 not a store")?;
assert!(for_transaction(&path, "tx-a").is_err());
assert!(record(&path, &record_at("tx-a", 1)).is_err());
Ok(())
}
}