If the boot lift returned Err, boot logged and served with nothing held, so every pending continuity abort ran and stopped a running candidate. Boot now sets lift_unsettled; retry_held_lift runs the lift for each continuity abort until one succeeds, and no abort step runs before. Boot's call moves into Engine::boot_lift so the failure is testable. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
24659 lines
1 MiB
24659 lines
1 MiB
use std::collections::{BTreeMap, BTreeSet};
|
||
use std::env;
|
||
use std::fs::{self, File, OpenOptions};
|
||
use std::io::Write;
|
||
use std::net::SocketAddr;
|
||
use std::path::{Path, PathBuf};
|
||
use std::sync::{Arc, Mutex};
|
||
use std::thread;
|
||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||
|
||
use anyhow::{Context, Result, anyhow, bail, ensure};
|
||
use cultcache_rs::{
|
||
CultCacheEnvelope, CultCacheExpectedEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore,
|
||
};
|
||
use cultnet_rs::{
|
||
AuthenticatedOdinRuntimeTopologyCorrelation, GameCultProviderHealthIdentity,
|
||
IDUNN_DEPLOYMENT_BRAKE_SCHEMA, IDUNN_LIFECYCLE_BRAKE_SCHEMA, IDUNN_PROCESS_WRITE_LEASE_SCHEMA,
|
||
IdunnDeploymentBrakeObservation, IdunnDeploymentBrakeOperatorIdentity,
|
||
IdunnDeploymentBrakeRecord, IdunnExpectedIncarnationRecord, IdunnLifecycleBrakeObservation,
|
||
IdunnLifecycleBrakeRecord, IdunnProcessWriteLeaseRecord, IdunnRuntimeActivationLaunch,
|
||
IdunnRuntimeActivationRecord, IdunnServiceIdentity, OdinTopologyAuthenticationContext,
|
||
OdinTopologyDisagreement, OdinTopologyIdentity, RuntimePresenceAuthenticationContext,
|
||
ServiceIdentityProfile, ServiceIdentitySigner, ServiceIdentityTrustAnchor,
|
||
authenticate_odin_runtime_topology_correlation, authenticate_runtime_presence_claim,
|
||
correlate_runtime_presence_claim, derive_service_identity_id,
|
||
evaluate_idunn_continuity_restart, evaluate_idunn_deployment_brake, open_service_identity_at,
|
||
verify_idunn_deployment_brake_authorization, verify_runtime_authority,
|
||
};
|
||
use serde::{Deserialize, Serialize};
|
||
use sha2::{Digest, Sha256};
|
||
use uuid::Uuid;
|
||
|
||
use crate::deployment::{
|
||
DependencyKind, OperatorBinding, RolloutStrategy, RouteBinding, WorkloadBinding,
|
||
capability_compatible,
|
||
};
|
||
use crate::deployment_plan::{
|
||
CompiledDeploymentPlan, DependencyProviderAuthority, PRE_FENCING_ABORT_DEADLINE_SECONDS,
|
||
SealedRelease, WARMING_DEADLINE_SECONDS, compile_deployment_plan,
|
||
};
|
||
use crate::drivers::{
|
||
ChallengeFailure, CultCacheTopologyDriver, CultCacheWriteLeaseDriver, DockerRunnerDriver, FrozenSourceReceipt,
|
||
GitSourceDriver, InstalledReleaseObservation, IsolationEvidence, NginxRouteDriver,
|
||
LockContended, ProcessIdentity, RouteActuation, RouteActuationGate, RouteActuationRefused, RouteActuators,
|
||
RouteObservation, RoutePreflightReceipt, RunnerPort, SourcePort,
|
||
SystemdTransientWorkloadDriver, TopologyPort, WorkloadObservation, WorkloadPort,
|
||
WriteLeasePort,
|
||
};
|
||
use crate::incident::{CloseReason, IncidentCondition, IncidentStore, render_incidents};
|
||
use crate::host_actuator::{
|
||
HostActuatorAccess, HostActuatorHub, HostActuatorRunnerDriver, HostActuatorWorkloadDriver,
|
||
HostUnobservable, IdunnHostActuatorIdentity, SharedHostActuatorHub, spawn_hub_service,
|
||
};
|
||
|
||
const DEPLOYMENT_COMMAND_SCHEMA: &str = "idunn.deployment_command.v2";
|
||
const DEPLOYMENT_TRANSACTION_SCHEMA: &str = "idunn.deployment_transaction.v4";
|
||
const DEPLOYMENT_TRANSACTION_SCHEMA_V3: &str = "idunn.deployment_transaction.v3";
|
||
const DEPLOYMENT_TRANSACTION_SCHEMA_V2: &str = "idunn.deployment_transaction.v2";
|
||
const ADMITTED_GENERATION_SCHEMA: &str = "idunn.admitted_generation.v4";
|
||
const ADMITTED_GENERATION_SCHEMA_V3: &str = "idunn.admitted_generation.v3";
|
||
const ADMITTED_GENERATION_SCHEMA_V2: &str = "idunn.admitted_generation.v2";
|
||
/// The capability whose declaration makes a target Odin-correlated.
|
||
pub(crate) const ODIN_RENDEZVOUS_CAPABILITY: &str = "odin.verse-rendezvous";
|
||
const TARGET_SUPERVISION_SCHEMA: &str = "idunn.target_supervision.v2";
|
||
/// The seven-key layout written before continuity's streak, anchor and
|
||
/// re-materialization bound. Read and migrated by `read_supervision_record`
|
||
/// and `migrate_control_store_to_current_schema`; delete it, `LegacyTargetSupervisionV1`
|
||
/// and those two arms once every store has opened under a build that
|
||
/// migrates (the key-34 precedent's deletion line).
|
||
const TARGET_SUPERVISION_SCHEMA_V1: &str = "idunn.target_supervision.v1";
|
||
const EXPIRY_REQUEST_SCHEMA: &str = "idunn.expiry_request.v1";
|
||
/// The number of charged continuity attempts inside
|
||
/// `CONTINUITY_RESTART_WINDOW_MILLIS` at which the ContinuityExhausted incident
|
||
/// opens and is held. It gates no actuation: continuity never stops
|
||
/// (ruling budget-never-stop).
|
||
const CONTINUITY_PAGING_RESTARTS: usize = 6;
|
||
const CONTINUITY_RESTART_WINDOW_MILLIS: u64 = 3_600_000;
|
||
/// The base of the streak backoff: the wait after the first charged attempt;
|
||
/// it doubles with each further one in the streak.
|
||
const CONTINUITY_RESTART_BACKOFF_MILLIS: u64 = 5_000;
|
||
/// The longest streak backoff between charged continuity attempts (ruling
|
||
/// continuity-backoff-defaults).
|
||
const CONTINUITY_BACKOFF_CEILING_MILLIS: u64 = 300_000;
|
||
/// How long an admitted generation must have run, with no attempt failing on
|
||
/// it, before the attempt streak resets (ruling continuity-backoff-defaults).
|
||
const CONTINUITY_SUSTAINED_RUN_MILLIS: u64 = 600_000;
|
||
|
||
/// Forward route actuations (fragment write, firewall, `nginx -t`, reload,
|
||
/// private-mount validation) one target may perform per sliding window. A
|
||
/// healthy target performs none; a legitimate deployment performs one or two.
|
||
/// Only deployments count: a Survival actuation is recorded beside this
|
||
/// ceiling, never counted against it and never refused by it, so a target
|
||
/// that is being restarted can still be rescued by a deploy.
|
||
/// Survival is bounded elsewhere: a continuity's actuations by the streak
|
||
/// backoff (at most one attempt per `CONTINUITY_BACKOFF_CEILING_MILLIS` once
|
||
/// the streak is long) times what one transaction can do before its phase
|
||
/// deadline ends it (none, over its admitted route), and an admitted-route
|
||
/// repair by the challenge backoff. A withdrawal that keeps failing retries,
|
||
/// reloading each time, under the resume backoff alone: the abort has no
|
||
/// deadline. Nothing bounds
|
||
/// reloads across the host.
|
||
const ROUTE_ACTUATION_CEILING: usize = 12;
|
||
const ROUTE_ACTUATION_WINDOW_MILLIS: u64 = 3_600_000;
|
||
/// How many Survival route actuation times a target keeps, for `status`
|
||
/// only: they are a record, not a meter, and no decision reads them.
|
||
const ROUTE_SURVIVAL_RECORD: usize = 12;
|
||
/// The longest wait between route challenges while proofs keep failing. The
|
||
/// shortest is the observation max age.
|
||
const ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS: u64 = 600_000;
|
||
|
||
/// How long continuity waits after it could not prepare a restart (the
|
||
/// projection would not demote) before trying again.
|
||
const CONTINUITY_DEFERRAL_MILLIS: u64 = 30_000;
|
||
|
||
const DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS: u64 = 30_000;
|
||
const DEFAULT_TOPOLOGY_MAXIMUM_FUTURE_SKEW_MILLIS: u64 = 2_000;
|
||
|
||
/// An immutable request. Execution state deliberately does not fit in this
|
||
/// record; status is derived from the transactions created for it.
|
||
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||
#[cultcache(
|
||
type = "idunn.deployment_command",
|
||
schema = "idunn.deployment_command.v2"
|
||
)]
|
||
struct DeploymentCommand {
|
||
#[cultcache(key = 0)]
|
||
schema_version: String,
|
||
#[cultcache(key = 1)]
|
||
command_id: String,
|
||
#[cultcache(key = 2)]
|
||
kind: CommandKind,
|
||
#[cultcache(key = 3)]
|
||
selector: String,
|
||
#[cultcache(key = 4)]
|
||
requested_by: String,
|
||
#[cultcache(key = 5)]
|
||
requested_at_unix_millis: u64,
|
||
}
|
||
|
||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(rename_all = "kebab-case")]
|
||
enum CommandKind {
|
||
Deploy,
|
||
Continuity,
|
||
}
|
||
|
||
impl DeploymentCommand {
|
||
fn validate(&self) -> Result<()> {
|
||
ensure!(
|
||
self.schema_version == DEPLOYMENT_COMMAND_SCHEMA,
|
||
"deployment command schema is unsupported"
|
||
);
|
||
require_id(&self.command_id, "deployment command id")?;
|
||
require_selector(&self.selector)?;
|
||
require_value(&self.requested_by, "deployment requester")?;
|
||
ensure!(
|
||
self.requested_at_unix_millis > 0,
|
||
"deployment command has no request time"
|
||
);
|
||
match self.kind {
|
||
CommandKind::Deploy => ensure!(
|
||
!self.command_id.starts_with("continuity-"),
|
||
"operator deployment uses the continuity command namespace"
|
||
),
|
||
CommandKind::Continuity => ensure!(
|
||
self.command_id.starts_with("continuity-")
|
||
&& !self.selector.starts_with("profile:"),
|
||
"continuity command identity or selector is invalid"
|
||
),
|
||
}
|
||
Ok(())
|
||
}
|
||
}
|
||
|
||
/// An operator's request that a live transaction's post-fencing phase end now
|
||
/// (`idunn expire`). The CLI only writes this record. The daemon's deadline
|
||
/// resolver reads it as a deadline that has passed, and the record is deleted
|
||
/// once its transaction no longer has a phase to end. No CLI writes a
|
||
/// transaction field.
|
||
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||
#[cultcache(type = "idunn.expiry_request", schema = "idunn.expiry_request.v1")]
|
||
struct ExpiryRequest {
|
||
#[cultcache(key = 0)]
|
||
schema_version: String,
|
||
#[cultcache(key = 1)]
|
||
command_id: String,
|
||
#[cultcache(key = 2)]
|
||
requested_by: String,
|
||
#[cultcache(key = 3)]
|
||
requested_at_unix_millis: u64,
|
||
}
|
||
|
||
impl ExpiryRequest {
|
||
fn validate(&self) -> Result<()> {
|
||
ensure!(
|
||
self.schema_version == EXPIRY_REQUEST_SCHEMA,
|
||
"expiry request schema is unsupported"
|
||
);
|
||
require_id(&self.command_id, "expiry request command id")?;
|
||
require_value(&self.requested_by, "expiry requester")?;
|
||
ensure!(
|
||
self.requested_at_unix_millis > 0,
|
||
"expiry request has no request time"
|
||
);
|
||
Ok(())
|
||
}
|
||
}
|
||
|
||
fn expiry_request_envelope(value: &ExpiryRequest, now: u64) -> Result<CultCacheEnvelope> {
|
||
value.validate()?;
|
||
typed_envelope(
|
||
&value.command_id,
|
||
ExpiryRequest::TYPE,
|
||
EXPIRY_REQUEST_SCHEMA,
|
||
value,
|
||
now,
|
||
)
|
||
}
|
||
|
||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||
#[serde(rename_all = "kebab-case")]
|
||
#[repr(u8)]
|
||
enum DeploymentPhase {
|
||
Sealing,
|
||
Starting,
|
||
Warming,
|
||
Fencing,
|
||
Leasing,
|
||
AwaitingReady,
|
||
Routing,
|
||
Committing,
|
||
Complete,
|
||
}
|
||
|
||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(deny_unknown_fields)]
|
||
struct DeploymentAuthorization {
|
||
authorization_id: String,
|
||
brake_sha256: String,
|
||
canonical_brake_bytes: Vec<u8>,
|
||
authorized_at_unix_millis: u64,
|
||
}
|
||
|
||
impl DeploymentAuthorization {
|
||
fn validate_shape(&self) -> Result<()> {
|
||
require_id(&self.authorization_id, "deployment authorization id")?;
|
||
ensure!(
|
||
!self.canonical_brake_bytes.is_empty()
|
||
&& self.brake_sha256 == sha256_id(&self.canonical_brake_bytes)
|
||
&& self.authorized_at_unix_millis > 0,
|
||
"deployment authorization receipt is incomplete"
|
||
);
|
||
let record: IdunnDeploymentBrakeRecord =
|
||
rmp_serde::from_slice(&self.canonical_brake_bytes)?;
|
||
record.validate()?;
|
||
ensure!(
|
||
rmp_serde::to_vec(&record)? == self.canonical_brake_bytes
|
||
&& record.authorization_id.as_deref() == Some(&self.authorization_id),
|
||
"deployment authorization receipt is noncanonical or mismatched"
|
||
);
|
||
Ok(())
|
||
}
|
||
}
|
||
|
||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(deny_unknown_fields)]
|
||
struct TopologyEvidence {
|
||
canonical_bytes: Vec<u8>,
|
||
canonical_sha256: String,
|
||
signer_identity_id: String,
|
||
publisher_sequence: u64,
|
||
admitted_at_unix_millis: u64,
|
||
}
|
||
|
||
impl TopologyEvidence {
|
||
fn from_authenticated(
|
||
topology: &AuthenticatedOdinRuntimeTopologyCorrelation,
|
||
admitted_at_unix_millis: u64,
|
||
) -> Result<Self> {
|
||
let record = topology.record();
|
||
let evidence = Self {
|
||
canonical_bytes: topology.canonical_bytes().to_vec(),
|
||
canonical_sha256: sha256_id(topology.canonical_bytes()),
|
||
signer_identity_id: record.signer_identity_id.clone(),
|
||
publisher_sequence: record.publisher_sequence,
|
||
admitted_at_unix_millis,
|
||
};
|
||
evidence.validate_shape()?;
|
||
Ok(evidence)
|
||
}
|
||
|
||
fn validate_shape(&self) -> Result<()> {
|
||
ensure!(
|
||
!self.canonical_bytes.is_empty()
|
||
&& self.canonical_sha256 == sha256_id(&self.canonical_bytes),
|
||
"topology evidence bytes or digest are invalid"
|
||
);
|
||
require_id(&self.signer_identity_id, "Odin topology signer")?;
|
||
ensure!(
|
||
self.publisher_sequence > 0 && self.admitted_at_unix_millis > 0,
|
||
"topology evidence sequence or admission time is invalid"
|
||
);
|
||
Ok(())
|
||
}
|
||
}
|
||
|
||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(deny_unknown_fields)]
|
||
struct RuntimePresenceEvidence {
|
||
canonical_bytes: Vec<u8>,
|
||
canonical_sha256: String,
|
||
message_id: String,
|
||
challenged_at_unix_millis: u64,
|
||
admitted_at_unix_millis: u64,
|
||
}
|
||
|
||
impl RuntimePresenceEvidence {
|
||
fn from_present(
|
||
present: &cultnet_rs::VerifiedRuntimePresence,
|
||
message_id: String,
|
||
challenged_at_unix_millis: u64,
|
||
admitted_at_unix_millis: u64,
|
||
) -> Result<Self> {
|
||
let evidence = Self {
|
||
canonical_bytes: present.canonical_bytes().to_vec(),
|
||
canonical_sha256: present.signed_presence_sha256().to_owned(),
|
||
message_id,
|
||
challenged_at_unix_millis,
|
||
admitted_at_unix_millis,
|
||
};
|
||
evidence.validate_shape()?;
|
||
Ok(evidence)
|
||
}
|
||
|
||
fn validate_shape(&self) -> Result<()> {
|
||
ensure!(
|
||
!self.canonical_bytes.is_empty()
|
||
&& self.canonical_sha256 == sha256_id(&self.canonical_bytes),
|
||
"runtime presence evidence bytes or digest are invalid"
|
||
);
|
||
require_id(&self.message_id, "runtime presence challenge")?;
|
||
ensure!(
|
||
self.challenged_at_unix_millis > 0
|
||
&& self.admitted_at_unix_millis >= self.challenged_at_unix_millis,
|
||
"runtime presence evidence timeline is invalid"
|
||
);
|
||
Ok(())
|
||
}
|
||
}
|
||
|
||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(tag = "source", rename_all = "kebab-case", deny_unknown_fields)]
|
||
enum WarmingEvidence {
|
||
OdinTopology { evidence: TopologyEvidence },
|
||
FirstOdinDirect { evidence: RuntimePresenceEvidence },
|
||
/// A route-proof target's candidate answered Idunn's challenge itself.
|
||
RouteProofDirect { evidence: RuntimePresenceEvidence },
|
||
}
|
||
|
||
impl WarmingEvidence {
|
||
fn validate_shape(&self) -> Result<()> {
|
||
match self {
|
||
Self::OdinTopology { evidence } => evidence.validate_shape(),
|
||
Self::FirstOdinDirect { evidence } | Self::RouteProofDirect { evidence } => {
|
||
evidence.validate_shape()
|
||
}
|
||
}
|
||
}
|
||
|
||
fn voucher(&self) -> Voucher {
|
||
match self {
|
||
Self::OdinTopology { .. } | Self::FirstOdinDirect { .. } => Voucher::Odin,
|
||
Self::RouteProofDirect { .. } => Voucher::Candidate,
|
||
}
|
||
}
|
||
|
||
/// The presence Idunn challenged for itself, when no Odin carried it.
|
||
fn direct(&self) -> Option<&RuntimePresenceEvidence> {
|
||
match self {
|
||
Self::OdinTopology { .. } => None,
|
||
Self::FirstOdinDirect { evidence } | Self::RouteProofDirect { evidence } => {
|
||
Some(evidence)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Opaque capability minted only after the transaction CAS stores exact
|
||
/// Warming evidence. Adjacent drivers receive only the candidate identity and
|
||
/// signed-presence digest needed to bind the write lease; they cannot inspect
|
||
/// or construct semantic admission.
|
||
#[derive(Clone, Debug)]
|
||
pub struct SequenceAdmittedWarming {
|
||
transaction_id: String,
|
||
signed_presence_sha256: String,
|
||
runtime_instance_id: String,
|
||
}
|
||
|
||
impl SequenceAdmittedWarming {
|
||
fn from_topology(
|
||
transaction_id: String,
|
||
authenticated: AuthenticatedOdinRuntimeTopologyCorrelation,
|
||
) -> Result<Self> {
|
||
let record = authenticated.record();
|
||
let signed_presence_sha256 = record
|
||
.signed_presence_sha256
|
||
.clone()
|
||
.context("Warming topology has no signed presence")?;
|
||
let runtime_instance_id = record
|
||
.runtime_instance_id
|
||
.clone()
|
||
.context("Warming topology has no runtime instance")?;
|
||
Ok(Self {
|
||
transaction_id,
|
||
signed_presence_sha256,
|
||
runtime_instance_id,
|
||
})
|
||
}
|
||
|
||
fn from_direct_presence(
|
||
transaction_id: String,
|
||
warming: WarmingEvidence,
|
||
present: cultnet_rs::VerifiedRuntimePresence,
|
||
) -> Result<Self> {
|
||
let evidence = warming
|
||
.direct()
|
||
.context("direct Warming evidence is an Odin topology receipt")?;
|
||
ensure!(
|
||
present.canonical_bytes() == evidence.canonical_bytes
|
||
&& present.signed_presence_sha256() == evidence.canonical_sha256,
|
||
"direct Warming evidence differs from its authenticated presence"
|
||
);
|
||
let runtime_instance_id = present.record().runtime_instance_id.clone();
|
||
Ok(Self {
|
||
transaction_id,
|
||
signed_presence_sha256: evidence.canonical_sha256.clone(),
|
||
runtime_instance_id,
|
||
})
|
||
}
|
||
|
||
#[cfg(test)]
|
||
pub(crate) fn for_test(
|
||
transaction_id: impl Into<String>,
|
||
authenticated: AuthenticatedOdinRuntimeTopologyCorrelation,
|
||
) -> Result<Self> {
|
||
let record = authenticated.record();
|
||
ensure!(
|
||
record.present
|
||
&& !record.ready
|
||
&& record.observed_presence_state.as_deref() == Some("warming")
|
||
&& record.observed_write_lease_sha256.is_none()
|
||
&& record.disagreements.is_empty(),
|
||
"test receipt is not exact Warming evidence"
|
||
);
|
||
let transaction_id = transaction_id.into();
|
||
require_id(&transaction_id, "test transaction id")?;
|
||
let signed_presence_sha256 = record
|
||
.signed_presence_sha256
|
||
.clone()
|
||
.context("test Warming receipt has no signed presence")?;
|
||
let runtime_instance_id = record
|
||
.runtime_instance_id
|
||
.clone()
|
||
.context("test Warming receipt has no runtime instance")?;
|
||
let token = Self::from_topology(transaction_id, authenticated)?;
|
||
ensure!(
|
||
token.signed_presence_sha256 == signed_presence_sha256
|
||
&& token.runtime_instance_id == runtime_instance_id,
|
||
"test Warming token extraction changed"
|
||
);
|
||
Ok(token)
|
||
}
|
||
|
||
pub(crate) fn transaction_id(&self) -> &str {
|
||
&self.transaction_id
|
||
}
|
||
|
||
pub(crate) fn signed_presence_sha256(&self) -> &str {
|
||
&self.signed_presence_sha256
|
||
}
|
||
|
||
pub(crate) fn runtime_instance_id(&self) -> &str {
|
||
&self.runtime_instance_id
|
||
}
|
||
}
|
||
|
||
/// Ready proof with the same construction boundary. Dependency planning and
|
||
/// promotion receive this token, never caller-supplied digest/sequence tuples.
|
||
#[derive(Clone, Debug)]
|
||
pub(crate) struct SequenceAdmittedReady {
|
||
transaction_id: String,
|
||
evidence: TopologyEvidence,
|
||
expected: IdunnExpectedIncarnationRecord,
|
||
authenticated: AuthenticatedOdinRuntimeTopologyCorrelation,
|
||
}
|
||
|
||
impl SequenceAdmittedReady {
|
||
#[cfg(test)]
|
||
pub(crate) fn for_test(
|
||
expected: &IdunnExpectedIncarnationRecord,
|
||
authenticated: AuthenticatedOdinRuntimeTopologyCorrelation,
|
||
admitted_at_unix_millis: u64,
|
||
) -> Result<Self> {
|
||
expected.validate()?;
|
||
let record = authenticated.record();
|
||
ensure!(
|
||
record.expected_projection_sha256 == expected.canonical_sha256()?
|
||
&& record.target == expected.target
|
||
&& record.runtime_id == expected.runtime_id
|
||
&& record.present
|
||
&& record.ready
|
||
&& record.observed_presence_state.as_deref() == Some("active")
|
||
&& record.disagreements.is_empty(),
|
||
"test receipt is not exact Ready evidence"
|
||
);
|
||
let evidence =
|
||
TopologyEvidence::from_authenticated(&authenticated, admitted_at_unix_millis)?;
|
||
Ok(Self {
|
||
transaction_id: "test-sequence-admission".into(),
|
||
evidence,
|
||
expected: expected.clone(),
|
||
authenticated,
|
||
})
|
||
}
|
||
|
||
pub(crate) fn authenticated(&self) -> &AuthenticatedOdinRuntimeTopologyCorrelation {
|
||
&self.authenticated
|
||
}
|
||
|
||
pub(crate) fn expected(&self) -> &IdunnExpectedIncarnationRecord {
|
||
&self.expected
|
||
}
|
||
|
||
pub(crate) fn transaction_id(&self) -> &str {
|
||
&self.transaction_id
|
||
}
|
||
|
||
pub(crate) fn evidence_sha256(&self) -> &str {
|
||
&self.evidence.canonical_sha256
|
||
}
|
||
|
||
pub(crate) fn publisher_sequence(&self) -> u64 {
|
||
self.evidence.publisher_sequence
|
||
}
|
||
}
|
||
|
||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(tag = "result", rename_all = "kebab-case", deny_unknown_fields)]
|
||
enum FencingEvidence {
|
||
SkippedStateless,
|
||
Revoked {
|
||
incumbent_lease_sha256: Option<String>,
|
||
candidate_lease_path_verified_empty: bool,
|
||
},
|
||
}
|
||
|
||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(tag = "result", rename_all = "kebab-case", deny_unknown_fields)]
|
||
enum LeasingEvidence {
|
||
SkippedStateless,
|
||
Prepared {
|
||
lease: IdunnProcessWriteLeaseRecord,
|
||
lease_sha256: String,
|
||
},
|
||
Granted {
|
||
lease: IdunnProcessWriteLeaseRecord,
|
||
lease_sha256: String,
|
||
},
|
||
}
|
||
|
||
impl LeasingEvidence {
|
||
fn lease(&self) -> Option<&IdunnProcessWriteLeaseRecord> {
|
||
match self {
|
||
Self::SkippedStateless | Self::Prepared { .. } => None,
|
||
Self::Granted { lease, .. } => Some(lease),
|
||
}
|
||
}
|
||
|
||
/// The lease Idunn issued, granted or only prepared. A prepared lease may
|
||
/// already be on disk: the physical grant runs before `Granted` is durable.
|
||
fn issued_lease(&self) -> Option<&IdunnProcessWriteLeaseRecord> {
|
||
match self {
|
||
Self::SkippedStateless => None,
|
||
Self::Prepared { lease, .. } | Self::Granted { lease, .. } => Some(lease),
|
||
}
|
||
}
|
||
|
||
fn prepared_lease(&self) -> Option<(&IdunnProcessWriteLeaseRecord, &str)> {
|
||
match self {
|
||
Self::Prepared {
|
||
lease,
|
||
lease_sha256,
|
||
} => Some((lease, lease_sha256)),
|
||
Self::SkippedStateless | Self::Granted { .. } => None,
|
||
}
|
||
}
|
||
|
||
fn lease_sha256(&self) -> Option<&str> {
|
||
match self {
|
||
Self::SkippedStateless | Self::Prepared { .. } => None,
|
||
Self::Granted { lease_sha256, .. } => Some(lease_sha256),
|
||
}
|
||
}
|
||
}
|
||
|
||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(tag = "result", rename_all = "kebab-case", deny_unknown_fields)]
|
||
enum RoutingEvidence {
|
||
SkippedUnrouted,
|
||
Promoted {
|
||
observation: RouteObservation,
|
||
promoted_at_unix_millis: u64,
|
||
},
|
||
}
|
||
|
||
impl RoutingEvidence {
|
||
fn observation(&self) -> Option<&RouteObservation> {
|
||
match self {
|
||
Self::SkippedUnrouted => None,
|
||
Self::Promoted { observation, .. } => Some(observation),
|
||
}
|
||
}
|
||
|
||
fn promoted_at_unix_millis(&self) -> Option<u64> {
|
||
match self {
|
||
Self::SkippedUnrouted => None,
|
||
Self::Promoted {
|
||
promoted_at_unix_millis,
|
||
..
|
||
} => Some(*promoted_at_unix_millis),
|
||
}
|
||
}
|
||
}
|
||
|
||
/// The states a route-proof candidate may answer its Warming challenge in. A
|
||
/// stateful one is warming until it holds a lease it cannot hold yet, and the
|
||
/// lease binds that warming presence. A stateless one may have finished
|
||
/// warming before the first challenge landed.
|
||
fn route_proof_warming_states(expected: &IdunnExpectedIncarnationRecord) -> &'static [&'static str] {
|
||
if expected.write_lease_required {
|
||
&["warming"]
|
||
} else {
|
||
&["warming", "active"]
|
||
}
|
||
}
|
||
|
||
/// A challenged runtime answered as something that disagrees with current
|
||
/// authority. Each disagreement is named, so a capacity below the Expected
|
||
/// minimum reads as that and not as a generic refusal. It stays the error's
|
||
/// typed value; callers read `disagreements`, and the text is only its
|
||
/// rendering.
|
||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||
pub(crate) struct PresenceDisagrees {
|
||
pub(crate) disagreements: Vec<OdinTopologyDisagreement>,
|
||
}
|
||
|
||
impl std::fmt::Display for PresenceDisagrees {
|
||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||
let named = self
|
||
.disagreements
|
||
.iter()
|
||
.map(|disagreement| {
|
||
format!(
|
||
"{} (expected {}, observed {})",
|
||
disagreement.code,
|
||
disagreement.expected.as_deref().unwrap_or("none"),
|
||
disagreement.observed.as_deref().unwrap_or("none"),
|
||
)
|
||
})
|
||
.collect::<Vec<_>>()
|
||
.join("; ");
|
||
write!(
|
||
formatter,
|
||
"route proof answered with a runtime that disagrees with current authority: {named}"
|
||
)
|
||
}
|
||
}
|
||
|
||
impl std::error::Error for PresenceDisagrees {}
|
||
|
||
/// What a direct challenge to a candidate endpoint produced.
|
||
enum CandidateAnswer {
|
||
/// No answer: the reason it is still being waited on.
|
||
Silent(String),
|
||
Answered {
|
||
evidence: RuntimePresenceEvidence,
|
||
present: cultnet_rs::VerifiedRuntimePresence,
|
||
},
|
||
}
|
||
|
||
//// Which kind of proof admits a target as Ready, derived from what the target's
|
||
/// recipe declares and from nothing else. Idunn infers no way to prove
|
||
/// readiness: a target that provides `odin.verse-rendezvous` is Odin itself, one
|
||
/// that declares a `shared-infrastructure odin.verse-rendezvous` dependency
|
||
/// advertises into the Verse and is Odin-correlated, and a routed one that
|
||
/// declares neither answers Idunn's own challenge and is route-proof. A target
|
||
/// with no route and no Odin dependency declares no proof at all, and is
|
||
/// refused. Odin collects the same receipts an Odin-correlated target does, and
|
||
/// its admitted generation is what names the Odin authority, so it is never
|
||
/// route-proof.
|
||
///
|
||
/// `of` is the only place a class is decided, and Expected is digest-bound, so
|
||
/// the class cannot drift from the incarnation it describes.
|
||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||
pub(crate) enum ReadinessClass {
|
||
OdinSelf,
|
||
OdinCorrelated,
|
||
RouteProof,
|
||
}
|
||
|
||
/// A recipe that declares neither a route nor an Odin dependency, and is not
|
||
/// Odin: it has said nothing about how it proves readiness.
|
||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||
pub(crate) struct UndeclaredReadiness {
|
||
pub(crate) target: String,
|
||
}
|
||
|
||
impl std::fmt::Display for UndeclaredReadiness {
|
||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||
write!(
|
||
formatter,
|
||
"target {} declares no way to prove readiness: its recipe must declare a route \
|
||
(Idunn challenges it) or a shared-infrastructure {ODIN_RENDEZVOUS_CAPABILITY} \
|
||
dependency (Odin correlates it)",
|
||
self.target
|
||
)
|
||
}
|
||
}
|
||
|
||
impl std::error::Error for UndeclaredReadiness {}
|
||
|
||
impl ReadinessClass {
|
||
/// The class of a recipe's declarations. Plan compilation and `of` both
|
||
/// call it, so admission refuses exactly what `of` would refuse.
|
||
pub(crate) fn declared(
|
||
target: &str,
|
||
provides_odin: bool,
|
||
declares_odin: bool,
|
||
routed: bool,
|
||
) -> Result<Self, UndeclaredReadiness> {
|
||
if provides_odin {
|
||
Ok(Self::OdinSelf)
|
||
} else if declares_odin {
|
||
Ok(Self::OdinCorrelated)
|
||
} else if routed {
|
||
Ok(Self::RouteProof)
|
||
} else {
|
||
Err(UndeclaredReadiness {
|
||
target: target.to_owned(),
|
||
})
|
||
}
|
||
}
|
||
|
||
pub(crate) fn of(
|
||
expected: &IdunnExpectedIncarnationRecord,
|
||
) -> Result<Self, UndeclaredReadiness> {
|
||
Self::declared(
|
||
&expected.target,
|
||
expected
|
||
.capabilities
|
||
.iter()
|
||
.any(|capability| capability.capability == ODIN_RENDEZVOUS_CAPABILITY),
|
||
expected.dependencies.iter().any(|dependency| {
|
||
dependency.kind == "shared-infrastructure"
|
||
&& dependency.capability == ODIN_RENDEZVOUS_CAPABILITY
|
||
}),
|
||
expected.route.is_some(),
|
||
)
|
||
}
|
||
|
||
/// The class of `expected`, provided every receipt collected under it was
|
||
/// vouched for by that class's voucher. The one way a stored record is
|
||
/// asked what class it is.
|
||
fn confirmed(
|
||
expected: &IdunnExpectedIncarnationRecord,
|
||
collected: impl IntoIterator<Item = Voucher>,
|
||
) -> Result<Self, ReadinessDisagreement> {
|
||
let required = Self::of(expected).map_err(ReadinessDisagreement::Undeclared)?;
|
||
match collected
|
||
.into_iter()
|
||
.find(|voucher| *voucher != required.voucher())
|
||
{
|
||
None => Ok(required),
|
||
Some(collected) => Err(ReadinessDisagreement::WrongVoucher {
|
||
target: expected.target.clone(),
|
||
required,
|
||
collected,
|
||
}),
|
||
}
|
||
}
|
||
|
||
/// Who vouches for a target of this class: Odin's correlation, or the
|
||
/// candidate answering Idunn.
|
||
fn voucher(self) -> Voucher {
|
||
match self {
|
||
Self::OdinSelf | Self::OdinCorrelated => Voucher::Odin,
|
||
Self::RouteProof => Voucher::Candidate,
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Who vouched for a piece of stored readiness evidence.
|
||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||
enum Voucher {
|
||
Odin,
|
||
Candidate,
|
||
}
|
||
|
||
/// A stored record whose readiness evidence is not what its own Expected
|
||
/// requires. A record admitted before the target's recipe declared how it
|
||
/// proves readiness looks like this. It is reported and held, never repaired:
|
||
/// re-deriving the evidence around it would make the evidence tag a second
|
||
/// authority for the class.
|
||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||
enum ReadinessDisagreement {
|
||
Undeclared(UndeclaredReadiness),
|
||
WrongVoucher {
|
||
target: String,
|
||
required: ReadinessClass,
|
||
collected: Voucher,
|
||
},
|
||
}
|
||
|
||
impl std::fmt::Display for ReadinessDisagreement {
|
||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||
match self {
|
||
Self::Undeclared(undeclared) => write!(formatter, "{undeclared}"),
|
||
Self::WrongVoucher {
|
||
target,
|
||
required,
|
||
collected,
|
||
} => write!(
|
||
formatter,
|
||
"target {target} is {required:?} but its stored readiness evidence was vouched \
|
||
for by {collected:?}: it was collected under a different class, and is held \
|
||
until the target is redeployed"
|
||
),
|
||
}
|
||
}
|
||
}
|
||
|
||
impl std::error::Error for ReadinessDisagreement {}
|
||
|
||
/// The receipt that admitted a candidate as Ready, tagged by who vouched for
|
||
/// it. The tag records what was collected; what a target must collect is
|
||
/// `ReadinessClass::of(expected)`, and a tag that differs is a
|
||
/// `ReadinessDisagreement`, never a second opinion about the class.
|
||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(tag = "class", rename_all = "kebab-case", deny_unknown_fields)]
|
||
enum ReadinessEvidence {
|
||
OdinCorrelated { evidence: TopologyEvidence },
|
||
RouteProof { evidence: RuntimePresenceEvidence },
|
||
}
|
||
|
||
impl ReadinessEvidence {
|
||
fn voucher(&self) -> Voucher {
|
||
match self {
|
||
Self::OdinCorrelated { .. } => Voucher::Odin,
|
||
Self::RouteProof { .. } => Voucher::Candidate,
|
||
}
|
||
}
|
||
|
||
fn validate_shape(&self) -> Result<()> {
|
||
match self {
|
||
Self::OdinCorrelated { evidence } => evidence.validate_shape(),
|
||
Self::RouteProof { evidence } => evidence.validate_shape(),
|
||
}
|
||
}
|
||
|
||
fn odin(&self) -> Option<&TopologyEvidence> {
|
||
match self {
|
||
Self::OdinCorrelated { evidence } => Some(evidence),
|
||
Self::RouteProof { .. } => None,
|
||
}
|
||
}
|
||
}
|
||
|
||
// Signed evidence that the candidate process picked up the lease Idunn
|
||
/// granted: its presence carries `write_lease_sha256` equal to the grant.
|
||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(deny_unknown_fields)]
|
||
struct LeaseAdoptionEvidence {
|
||
write_lease_sha256: String,
|
||
signed_presence_sha256: String,
|
||
source: AdoptionSource,
|
||
observed_at_unix_millis: u64,
|
||
}
|
||
|
||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(rename_all = "kebab-case")]
|
||
enum AdoptionSource {
|
||
Direct,
|
||
OdinTopology,
|
||
}
|
||
|
||
impl LeaseAdoptionEvidence {
|
||
/// The adoption a stateful candidate's Ready proves: its signed presence
|
||
/// held exactly the granted lease. `None` for a stateless candidate.
|
||
fn with_ready(
|
||
granted_lease_sha256: Option<&str>,
|
||
signed_presence_sha256: &str,
|
||
source: AdoptionSource,
|
||
observed_at_unix_millis: u64,
|
||
) -> Option<Self> {
|
||
granted_lease_sha256.map(|lease| Self {
|
||
write_lease_sha256: lease.to_owned(),
|
||
signed_presence_sha256: signed_presence_sha256.to_owned(),
|
||
source,
|
||
observed_at_unix_millis,
|
||
})
|
||
}
|
||
|
||
/// Whether this evidence is about exactly the lease that was granted.
|
||
fn names(&self, leasing: &LeasingEvidence) -> bool {
|
||
leasing.lease_sha256() == Some(self.write_lease_sha256.as_str())
|
||
}
|
||
|
||
fn validate_shape(&self) -> Result<()> {
|
||
require_id(&self.write_lease_sha256, "adopted lease digest")?;
|
||
require_id(&self.signed_presence_sha256, "lease adoption presence digest")?;
|
||
ensure!(
|
||
self.observed_at_unix_millis > 0,
|
||
"lease adoption has no observation time"
|
||
);
|
||
Ok(())
|
||
}
|
||
}
|
||
|
||
/// The phases whose entry starts a deadline: the one spelling of it.
|
||
const DEADLINE_PHASES: std::ops::RangeInclusive<DeploymentPhase> =
|
||
DeploymentPhase::Warming..=DeploymentPhase::Committing;
|
||
|
||
/// What a live record is doing now and when that must end: a deadline phase
|
||
/// from its entry, an abort from its beginning, a drain to its retire. The
|
||
/// post-fencing durations come from the plan's frozen `PhaseDeadlines`; the
|
||
/// rest are Idunn's. `DeploymentTransaction::deadline_for_current_activity`
|
||
/// is the one derivation.
|
||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(deny_unknown_fields)]
|
||
struct PhaseDeadline {
|
||
phase: DeploymentPhase,
|
||
entered_at_unix_millis: u64,
|
||
deadline_at_unix_millis: u64,
|
||
}
|
||
|
||
impl PhaseDeadline {
|
||
/// `None` for a phase that has no deadline of its own (Sealing, Starting
|
||
/// and Complete).
|
||
fn entering(
|
||
phase: DeploymentPhase,
|
||
plan: &CompiledDeploymentPlan,
|
||
now: u64,
|
||
) -> Option<Self> {
|
||
let deadlines = plan.phase_deadlines();
|
||
let seconds = match phase {
|
||
DeploymentPhase::Warming => WARMING_DEADLINE_SECONDS,
|
||
DeploymentPhase::Fencing => deadlines.fencing_seconds,
|
||
DeploymentPhase::Leasing => deadlines.leasing_seconds,
|
||
DeploymentPhase::AwaitingReady => deadlines.awaiting_ready_seconds,
|
||
DeploymentPhase::Routing => deadlines.routing_seconds,
|
||
DeploymentPhase::Committing => deadlines.committing_seconds,
|
||
_ => return None,
|
||
};
|
||
Some(Self {
|
||
phase,
|
||
entered_at_unix_millis: now,
|
||
deadline_at_unix_millis: now.saturating_add(u64::from(seconds) * 1000),
|
||
})
|
||
}
|
||
}
|
||
|
||
/// How the deadline resolver ends a deadline phase.
|
||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||
enum PhaseEnd {
|
||
/// Before the fence the transaction owns only its candidate: it ends
|
||
/// through its own pre-fencing abort.
|
||
AbortBeforeFencing,
|
||
/// Abort through the post-fencing path. `lease_not_adopted` names a
|
||
/// granted lease with no adoption evidence.
|
||
Abort { lease_not_adopted: bool },
|
||
/// A stateful deployment whose candidate holds an issued lease: the phase
|
||
/// is reported and not resolved.
|
||
OperatorRequired,
|
||
/// A continuity at or past Fencing: its candidate is the only running copy
|
||
/// of the admitted body, so aborting it restores what it already runs. The
|
||
/// resolver leaves it to the operator and its phase step keeps running,
|
||
/// until a deployment awaits the handover, when the continuity yields
|
||
/// through its own post-fencing abort (ruling continuity-yields-to-deploy).
|
||
RetryForward,
|
||
}
|
||
|
||
/// How a post-fencing failure recovers. `RestoreIncumbent` is what a
|
||
/// post-fencing abort has always done and is the default for records that
|
||
/// predate the field.
|
||
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(tag = "action", rename_all = "kebab-case", deny_unknown_fields)]
|
||
enum TerminalRecovery {
|
||
RestartAdmitted,
|
||
#[default]
|
||
RestoreIncumbent,
|
||
OperatorRequired {
|
||
reason: String,
|
||
},
|
||
}
|
||
|
||
/// Route supervision's durable memory for one admitted routed incarnation:
|
||
/// challenge pacing and the degradation mark. It belongs to the incarnation, so
|
||
/// a new generation starts from `default()`. What a target may actuate outlives
|
||
/// every generation and lives in `TargetSupervision`.
|
||
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(deny_unknown_fields)]
|
||
struct RouteSupervisionState {
|
||
last_challenge_at_unix_millis: Option<u64>,
|
||
consecutive_failures: u32,
|
||
next_challenge_at_unix_millis: Option<u64>,
|
||
/// Set while route proof keeps failing. Marks the route degraded so
|
||
/// dependents stop selecting it; it never authorizes a restart.
|
||
degraded_since_unix_millis: Option<u64>,
|
||
}
|
||
|
||
impl RouteSupervisionState {
|
||
/// The next challenge is not due yet.
|
||
fn is_waiting(&self, now: u64, maximum_age_millis: u64) -> bool {
|
||
self.next_challenge_at_unix_millis.is_some_and(|next| {
|
||
is_waiting(
|
||
now,
|
||
next,
|
||
ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS.max(maximum_age_millis),
|
||
)
|
||
})
|
||
}
|
||
|
||
/// A failed proof, or a failed repair of the fragment it needs. Observation
|
||
/// state only: the route is marked degraded and the next challenge waits,
|
||
/// twice as long each time, from the observation max age up to a cap.
|
||
fn record_failed_challenge(&mut self, now: u64, maximum_age_millis: u64) {
|
||
self.consecutive_failures = self.consecutive_failures.saturating_add(1);
|
||
let doublings = self.consecutive_failures.saturating_sub(1).min(16);
|
||
let wait = maximum_age_millis
|
||
.saturating_mul(1 << doublings)
|
||
.min(ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS)
|
||
.max(maximum_age_millis);
|
||
self.last_challenge_at_unix_millis = Some(now);
|
||
self.next_challenge_at_unix_millis = Some(now.saturating_add(wait));
|
||
self.degraded_since_unix_millis.get_or_insert(now);
|
||
}
|
||
|
||
fn record_proved_challenge(&mut self, now: u64) {
|
||
self.consecutive_failures = 0;
|
||
self.last_challenge_at_unix_millis = Some(now);
|
||
self.next_challenge_at_unix_millis = None;
|
||
self.degraded_since_unix_millis = None;
|
||
}
|
||
|
||
fn validate(&self) -> Result<()> {
|
||
if let Some(since) = self.degraded_since_unix_millis {
|
||
ensure!(since > 0, "route degradation has no start time");
|
||
}
|
||
Ok(())
|
||
}
|
||
}
|
||
|
||
/// What one target has been allowed to do, kept across every generation and
|
||
/// present before the first: Forward route actuations and continuity restarts,
|
||
/// each a sliding log of the times they happened, and a record of Survival
|
||
/// route actuations. Owned by the meter primitives below;
|
||
/// the route driver only asks, and no generation write carries it.
|
||
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||
#[cultcache(
|
||
type = "idunn.target_supervision",
|
||
schema = "idunn.target_supervision.v2"
|
||
)]
|
||
struct TargetSupervision {
|
||
#[cultcache(key = 0)]
|
||
schema_version: String,
|
||
#[cultcache(key = 1)]
|
||
target: String,
|
||
/// The newest `ROUTE_ACTUATION_CEILING` Forward actuation times,
|
||
/// ascending: the only entries the ceiling counts.
|
||
#[cultcache(key = 2)]
|
||
route_actuations: Vec<u64>,
|
||
/// The newest `CONTINUITY_PAGING_RESTARTS` restart times, ascending: the
|
||
/// paging threshold's log.
|
||
#[cultcache(key = 3)]
|
||
continuity_restarts: Vec<u64>,
|
||
/// A restart whose projection could not be prepared waits until this time.
|
||
#[cultcache(key = 4)]
|
||
continuity_deferred_until: Option<u64>,
|
||
#[cultcache(key = 5)]
|
||
continuity_deferral_reason: Option<String>,
|
||
/// The newest `ROUTE_SURVIVAL_RECORD` Survival actuation times, ascending.
|
||
/// Visibility only: never counted, never refused.
|
||
#[cultcache(key = 6)]
|
||
route_survivals: Vec<u64>,
|
||
// Keys 7-9 are why this is v2: a v1 record is lifted by
|
||
// `LegacyTargetSupervisionV1` with no streak, no anchor and no
|
||
// re-materialization.
|
||
/// Charged attempts since the last sustained run. Owned by continuity
|
||
/// supervision; it decides the next attempt's backoff step.
|
||
#[cultcache(key = 7)]
|
||
continuity_attempt_streak: u32,
|
||
/// When a dead-and-free pass began waiting out the next attempt. Owned by
|
||
/// continuity supervision; it decides when the wait is over. Older than
|
||
/// the admitted generation, it decides nothing.
|
||
#[cultcache(key = 8)]
|
||
continuity_backoff_from_unix_millis: Option<u64>,
|
||
/// When the admitted generation's one uncharged re-materialization was
|
||
/// minted. Owned by continuity supervision; it can only withhold a free
|
||
/// re-materialization, never grant one.
|
||
#[cultcache(key = 9)]
|
||
continuity_rematerialized_at_unix_millis: Option<u64>,
|
||
}
|
||
|
||
/// What continuity supervision does about the next attempt for a dead target.
|
||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||
enum ContinuityBackoff {
|
||
/// Mint now, carrying this streak into the attempt.
|
||
Due { streak: u32 },
|
||
/// Start the wait: write the anchor and mint nothing this pass.
|
||
Begin,
|
||
/// The wait has not run out.
|
||
Waiting,
|
||
}
|
||
|
||
/// The entries of a sliding log still inside `window` at `now`. An entry from
|
||
/// the future means the clock stepped back; it counts as `now`, so a backwards
|
||
/// step holds a meter for at most one window however far the clock moved.
|
||
fn live_entries(log: &[u64], now: u64, window: u64) -> Vec<u64> {
|
||
log.iter()
|
||
.map(|&at| at.min(now))
|
||
.filter(|&at| now - at < window)
|
||
.collect()
|
||
}
|
||
|
||
/// Record `now`, keeping the newest `limit` entries: all a meter needs to
|
||
/// decide "at most `limit` in any window".
|
||
fn record_entry(log: &mut Vec<u64>, now: u64, limit: usize) {
|
||
clamp_to(log, now);
|
||
log.push(now);
|
||
if log.len() > limit {
|
||
log.drain(..log.len() - limit);
|
||
}
|
||
}
|
||
|
||
/// Make a stepped-back clock's future entries the present. `live_entries`
|
||
/// only reads them that way, so a log that is never written would keep
|
||
/// looking recent until the clock caught up; writing the clamp is what makes
|
||
/// a backwards step cost one window and no more.
|
||
fn clamp_to(log: &mut [u64], now: u64) {
|
||
for at in log.iter_mut() {
|
||
*at = (*at).min(now);
|
||
}
|
||
}
|
||
|
||
impl TargetSupervision {
|
||
fn new(target: &str) -> Self {
|
||
Self {
|
||
schema_version: TARGET_SUPERVISION_SCHEMA.into(),
|
||
target: target.into(),
|
||
route_actuations: Vec::new(),
|
||
continuity_restarts: Vec::new(),
|
||
continuity_deferred_until: None,
|
||
continuity_deferral_reason: None,
|
||
route_survivals: Vec::new(),
|
||
continuity_attempt_streak: 0,
|
||
continuity_backoff_from_unix_millis: None,
|
||
continuity_rematerialized_at_unix_millis: None,
|
||
}
|
||
}
|
||
|
||
/// Clamp every log to `now`. Callers that decide from a log write the
|
||
/// result back when it changed.
|
||
fn settle(&mut self, now: u64) {
|
||
clamp_to(&mut self.route_actuations, now);
|
||
clamp_to(&mut self.continuity_restarts, now);
|
||
clamp_to(&mut self.route_survivals, now);
|
||
}
|
||
|
||
fn route_used(&self, now: u64) -> usize {
|
||
live_entries(&self.route_actuations, now, ROUTE_ACTUATION_WINDOW_MILLIS).len()
|
||
}
|
||
|
||
/// When the oldest counted actuation leaves the window, if the ceiling is reached.
|
||
fn route_reopens_at(&self, now: u64) -> Option<u64> {
|
||
let live = live_entries(&self.route_actuations, now, ROUTE_ACTUATION_WINDOW_MILLIS);
|
||
(live.len() >= ROUTE_ACTUATION_CEILING)
|
||
.then(|| live[live.len() - ROUTE_ACTUATION_CEILING] + ROUTE_ACTUATION_WINDOW_MILLIS)
|
||
}
|
||
|
||
/// Charge one route actuation. A `Forward` change is counted against the
|
||
/// ceiling and refused at it; a `Survival` is only recorded, never
|
||
/// counted and never refused.
|
||
fn charge_route(&mut self, now: u64, kind: RouteActuation) -> Result<(), RouteActuationRefused> {
|
||
self.settle(now);
|
||
match kind {
|
||
RouteActuation::Forward => {
|
||
if let Some(reopens_at) = self.route_reopens_at(now) {
|
||
return Err(RouteActuationRefused {
|
||
target: self.target.clone(),
|
||
used: self.route_used(now),
|
||
reopens_at_unix_millis: reopens_at,
|
||
});
|
||
}
|
||
record_entry(&mut self.route_actuations, now, ROUTE_ACTUATION_CEILING);
|
||
}
|
||
RouteActuation::Survival => {
|
||
record_entry(&mut self.route_survivals, now, ROUTE_SURVIVAL_RECORD);
|
||
}
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
fn survivals_in_window(&self, now: u64) -> usize {
|
||
live_entries(&self.route_survivals, now, ROUTE_ACTUATION_WINDOW_MILLIS).len()
|
||
}
|
||
|
||
fn restarts_used(&self, now: u64) -> usize {
|
||
live_entries(&self.continuity_restarts, now, CONTINUITY_RESTART_WINDOW_MILLIS).len()
|
||
}
|
||
|
||
/// The paging threshold of the ContinuityExhausted incident; it gates no actuation.
|
||
fn restarts_page(&self, now: u64) -> bool {
|
||
self.restarts_used(now) >= CONTINUITY_PAGING_RESTARTS
|
||
}
|
||
|
||
/// Whether continuity must still wait out the deferral after a projection
|
||
/// that would not demote.
|
||
fn continuity_deferred(&self, now: u64) -> bool {
|
||
self.continuity_deferred_until
|
||
.is_some_and(|due| is_waiting(now, due, CONTINUITY_DEFERRAL_MILLIS))
|
||
}
|
||
|
||
/// The doubling steps the next attempt waits, if it waits. A streak of `n`
|
||
/// waits `n - 1` steps. The generation's one re-materialization is an
|
||
/// attempt that failed on it though it charged no streak, so the attempt
|
||
/// after it waits the base step: every attempt leaves a gap.
|
||
fn owed_steps(&self, admitted_at: u64) -> Option<u32> {
|
||
match self.continuity_attempt_streak.checked_sub(1) {
|
||
None if self.rematerialization_spent(admitted_at) => Some(0),
|
||
steps => steps,
|
||
}
|
||
}
|
||
|
||
/// When the wait for the next charged attempt ends, from the anchor and
|
||
/// the streak; the one spelling `continuity_backoff` and `status` share.
|
||
/// An anchor older than the admitted generation decides nothing.
|
||
fn backoff_due(&self, admitted_at: u64) -> Option<u64> {
|
||
let from = self
|
||
.continuity_backoff_from_unix_millis
|
||
.filter(|&from| from >= admitted_at)?;
|
||
let wait = self.owed_steps(admitted_at).map_or(0, |steps| {
|
||
backoff_wait(
|
||
CONTINUITY_RESTART_BACKOFF_MILLIS,
|
||
steps,
|
||
CONTINUITY_BACKOFF_CEILING_MILLIS,
|
||
)
|
||
});
|
||
Some(from.saturating_add(wait))
|
||
}
|
||
|
||
/// What to do about the next attempt for the dead admitted generation
|
||
/// admitted at `admitted_at`. The streak resets only after a sustained
|
||
/// run: the generation has lived the sustained time and no attempt, charged
|
||
/// or the one re-materialization, has been minted since it was admitted.
|
||
/// That rule decides first: a sustained death never began a wait, so a
|
||
/// standing anchor, however it came to stand, has no say in it. A wait
|
||
/// already running when the generation reaches the sustained time ends
|
||
/// there instead: an accepted shortening, always under the cap.
|
||
fn continuity_backoff(&self, now: u64, admitted_at: u64) -> ContinuityBackoff {
|
||
let streak = self.continuity_attempt_streak;
|
||
let sustained = self
|
||
.continuity_restarts
|
||
.last()
|
||
.into_iter()
|
||
.chain(&self.continuity_rematerialized_at_unix_millis)
|
||
.all(|&last| admitted_at >= last)
|
||
&& now.saturating_sub(admitted_at) >= CONTINUITY_SUSTAINED_RUN_MILLIS;
|
||
if sustained {
|
||
return ContinuityBackoff::Due { streak: 0 };
|
||
}
|
||
if let Some(due) = self.backoff_due(admitted_at) {
|
||
return if self.owed_steps(admitted_at).is_some()
|
||
&& is_waiting(now, due, CONTINUITY_BACKOFF_CEILING_MILLIS)
|
||
{
|
||
ContinuityBackoff::Waiting
|
||
} else {
|
||
ContinuityBackoff::Due { streak }
|
||
};
|
||
}
|
||
if self.owed_steps(admitted_at).is_none() {
|
||
ContinuityBackoff::Due { streak: 0 }
|
||
} else {
|
||
ContinuityBackoff::Begin
|
||
}
|
||
}
|
||
|
||
/// One charged attempt scheduled at `streak`. It ends the wait and any
|
||
/// deferral: the projection was prepared.
|
||
fn record_restart(&mut self, now: u64, streak: u32) {
|
||
record_entry(&mut self.continuity_restarts, now, CONTINUITY_PAGING_RESTARTS);
|
||
self.continuity_attempt_streak = streak.saturating_add(1);
|
||
self.continuity_backoff_from_unix_millis = None;
|
||
self.continuity_deferred_until = None;
|
||
self.continuity_deferral_reason = None;
|
||
}
|
||
|
||
/// The target was found dead and free: the wait for its next attempt runs from now.
|
||
fn begin_backoff(&mut self, now: u64) {
|
||
self.continuity_backoff_from_unix_millis = Some(now);
|
||
}
|
||
|
||
/// The admitted workload was seen running: the death the anchor waited
|
||
/// out is over, and the next death waits from its own discovery.
|
||
fn end_backoff(&mut self) {
|
||
self.continuity_backoff_from_unix_millis = None;
|
||
}
|
||
|
||
/// The one uncharged re-materialization scheduled. The log and the streak
|
||
/// are left alone.
|
||
fn record_rematerialization(&mut self, now: u64) {
|
||
self.continuity_rematerialized_at_unix_millis = Some(now);
|
||
self.continuity_backoff_from_unix_millis = None;
|
||
self.continuity_deferred_until = None;
|
||
self.continuity_deferral_reason = None;
|
||
}
|
||
|
||
/// Whether the generation admitted at `admitted_at` has had its free re-materialization.
|
||
fn rematerialization_spent(&self, admitted_at: u64) -> bool {
|
||
self.continuity_rematerialized_at_unix_millis
|
||
.is_some_and(|at| at >= admitted_at)
|
||
}
|
||
|
||
fn defer_continuity(&mut self, now: u64, reason: &str) {
|
||
self.continuity_deferred_until = Some(now.saturating_add(CONTINUITY_DEFERRAL_MILLIS));
|
||
self.continuity_deferral_reason = Some(bounded_detail(reason));
|
||
}
|
||
|
||
fn validate(&self) -> Result<()> {
|
||
ensure!(
|
||
self.schema_version == TARGET_SUPERVISION_SCHEMA,
|
||
"target supervision schema is unsupported"
|
||
);
|
||
require_id(&self.target, "supervised target")?;
|
||
for (log, limit, what) in [
|
||
(&self.route_actuations, ROUTE_ACTUATION_CEILING, "route actuation"),
|
||
(&self.continuity_restarts, CONTINUITY_PAGING_RESTARTS, "continuity restart"),
|
||
(&self.route_survivals, ROUTE_SURVIVAL_RECORD, "survival route actuation"),
|
||
] {
|
||
ensure!(log.len() <= limit, "{what} log exceeds its bound");
|
||
ensure!(
|
||
log.iter().all(|&at| at > 0) && log.windows(2).all(|pair| pair[0] <= pair[1]),
|
||
"{what} log is not an ascending list of times"
|
||
);
|
||
}
|
||
ensure!(
|
||
self.continuity_backoff_from_unix_millis != Some(0)
|
||
&& self.continuity_rematerialized_at_unix_millis != Some(0),
|
||
"continuity backoff anchor or re-materialization has no time"
|
||
);
|
||
ensure!(
|
||
self.continuity_deferred_until.is_some() == self.continuity_deferral_reason.is_some(),
|
||
"continuity deferral has a time or a reason but not both"
|
||
);
|
||
Ok(())
|
||
}
|
||
}
|
||
|
||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(tag = "outcome", rename_all = "kebab-case", deny_unknown_fields)]
|
||
enum TransactionCompletion {
|
||
Admitted { generation_id: String },
|
||
FailedBeforeFencing { error: String },
|
||
FailedAfterFencing {
|
||
error: String,
|
||
#[serde(default)]
|
||
recovery: TerminalRecovery,
|
||
},
|
||
}
|
||
|
||
use cleanup_evidence::CleanupEvidence;
|
||
|
||
/// Where the legacy marker lives, so that only the lift can make one.
|
||
///
|
||
/// `LegacyLift` has a private field: outside this module no expression can
|
||
/// construct it, so no other code can write `CleanupEvidence::Legacy(..)`.
|
||
/// Decoding a stored record is the one other way in, and it stays here too.
|
||
/// `transaction_envelope` refuses a record carrying the marker; the schema
|
||
/// migration, which persists what the lift produced, writes through
|
||
/// `migrate_transaction_record`, and a fault note on an existing record through
|
||
/// `last_error_envelope`; both are defined here beside the only constructor,
|
||
/// and the encoder they share is private to this module.
|
||
mod cleanup_evidence {
|
||
use super::*;
|
||
|
||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||
pub(super) struct LegacyLift(());
|
||
|
||
const LEGACY_MARKER: &str = "legacy-skipped-before-b1";
|
||
|
||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||
pub(super) enum CleanupEvidence {
|
||
Pending,
|
||
Skipped,
|
||
Complete,
|
||
/// Set only by the legacy lift, on a terminal continuity abort written
|
||
/// before the single resolution rule: it issued an activation and
|
||
/// recorded `Skipped`. Boot reconciliation, not the record, owes that
|
||
/// residue's demotion. The marker is what lets validation accept the
|
||
/// old shape without reopening a finished transaction.
|
||
Legacy(LegacyLift),
|
||
}
|
||
|
||
impl CleanupEvidence {
|
||
pub(super) fn is_complete(self) -> bool {
|
||
!matches!(self, Self::Pending)
|
||
}
|
||
|
||
pub(super) fn is_legacy_marker(self) -> bool {
|
||
matches!(self, Self::Legacy(_))
|
||
}
|
||
}
|
||
|
||
impl Serialize for CleanupEvidence {
|
||
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||
serializer.serialize_str(match self {
|
||
Self::Pending => "pending",
|
||
Self::Skipped => "skipped",
|
||
Self::Complete => "complete",
|
||
Self::Legacy(_) => LEGACY_MARKER,
|
||
})
|
||
}
|
||
}
|
||
|
||
impl<'de> Deserialize<'de> for CleanupEvidence {
|
||
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||
let text = String::deserialize(deserializer)?;
|
||
match text.as_str() {
|
||
"pending" => Ok(Self::Pending),
|
||
"skipped" => Ok(Self::Skipped),
|
||
"complete" => Ok(Self::Complete),
|
||
LEGACY_MARKER => Ok(Self::Legacy(LegacyLift(()))),
|
||
other => Err(serde::de::Error::unknown_variant(
|
||
other,
|
||
&["pending", "skipped", "complete", LEGACY_MARKER],
|
||
)),
|
||
}
|
||
}
|
||
}
|
||
|
||
impl DeploymentTransaction {
|
||
pub(super) fn carries_legacy_marker(&self) -> bool {
|
||
self.pre_fencing_abort
|
||
.as_ref()
|
||
.map(|abort| abort.topology_reconciliation)
|
||
.into_iter()
|
||
.chain(
|
||
self.post_fencing_abort
|
||
.as_ref()
|
||
.map(|abort| abort.topology_reconciliation),
|
||
)
|
||
.any(CleanupEvidence::is_legacy_marker)
|
||
}
|
||
}
|
||
|
||
/// The schema migration's one entry point: read a stored v2, v3 or v4
|
||
/// transaction (lifting older ones) and return its key and current-shape
|
||
/// envelope. The record may carry the marker the lift just set.
|
||
pub(super) fn migrate_transaction_record(
|
||
envelope: &CultCacheEnvelope,
|
||
) -> Result<(String, CultCacheEnvelope)> {
|
||
let value = read_transaction_record(envelope)?;
|
||
let next = encode_transaction(&value, value.updated_at_unix_millis)?;
|
||
Ok((value.transaction_id, next))
|
||
}
|
||
|
||
/// The envelope for an existing record with only its `last_error` set. The
|
||
/// record may already carry the marker; this cannot add one.
|
||
pub(super) fn last_error_envelope(
|
||
value: &DeploymentTransaction,
|
||
detail: &str,
|
||
) -> Result<CultCacheEnvelope> {
|
||
let mut next = value.clone();
|
||
next.last_error = Some(detail.to_string());
|
||
encode_transaction(&next, next.updated_at_unix_millis)
|
||
}
|
||
|
||
/// A continuity abort written before the single resolution rule recorded no
|
||
/// projection cleanup although its transaction had issued an activation, and
|
||
/// so left that activation standing.
|
||
///
|
||
/// Nothing is reopened. A record already terminal keeps its terminal phase and
|
||
/// completion and is marked the legacy marker, the one shape validation
|
||
/// accepts for a terminal abort that owes nothing yet issued an activation;
|
||
/// boot reconciliation, the single owner of that residue, demotes it by exact
|
||
/// activation. A record still in flight lifts to `Pending`: it already holds
|
||
/// its target's authority (a live record claims it before and after the lift),
|
||
/// so the lift adds no claimant, and its own abort demotes the activation.
|
||
/// Only the control store's read applies this: history describes what happened
|
||
/// and is lifted unchanged.
|
||
pub(super) fn owe_legacy_continuity_projection(transaction: &mut DeploymentTransaction) {
|
||
if transaction.command_kind != CommandKind::Continuity || transaction.activation.is_none() {
|
||
return;
|
||
}
|
||
let terminal = transaction.completion.is_some();
|
||
if let Some(abort) = transaction.pre_fencing_abort.as_mut()
|
||
&& abort.topology_reconciliation == CleanupEvidence::Skipped
|
||
{
|
||
abort.topology_reconciliation = if terminal {
|
||
CleanupEvidence::Legacy(LegacyLift(()))
|
||
} else {
|
||
CleanupEvidence::Pending
|
||
};
|
||
}
|
||
if let Some(abort) = transaction.post_fencing_abort.as_mut()
|
||
&& abort.topology_reconciliation == CleanupEvidence::Skipped
|
||
&& !terminal
|
||
{
|
||
abort.topology_reconciliation = CleanupEvidence::Pending;
|
||
}
|
||
}
|
||
|
||
}
|
||
|
||
/// Rollback of a transaction that has already fenced the incumbent.
|
||
///
|
||
/// Fencing stops the admitted incarnation and revokes its write lease, so a
|
||
/// candidate that then dies for good leaves the target with nothing serving it.
|
||
/// Retrying forever cannot fix that -- a transient unit with `Restart=no` will
|
||
/// not come back -- and it holds the target, leaving every later command queued
|
||
/// behind a transaction that can never finish.
|
||
///
|
||
/// So the candidate's own artifacts are withdrawn in the order that never
|
||
/// leaves two writers: route first, then the write lease, then the process,
|
||
/// then the projection. Reconciling the projection restores the incumbent's
|
||
/// admitted Expected where there is one, which is what lets continuity bring
|
||
/// the incumbent back; where there is none, the failed Expected is withdrawn.
|
||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(deny_unknown_fields)]
|
||
struct PostFencingAbort {
|
||
error: String,
|
||
route_restoration: CleanupEvidence,
|
||
lease_withdrawal: CleanupEvidence,
|
||
candidate_cleanup: CleanupEvidence,
|
||
topology_reconciliation: CleanupEvidence,
|
||
source_cleanup: CleanupEvidence,
|
||
}
|
||
|
||
impl PostFencingAbort {
|
||
fn is_complete(&self) -> bool {
|
||
self.route_restoration.is_complete()
|
||
&& self.lease_withdrawal.is_complete()
|
||
&& self.candidate_cleanup.is_complete()
|
||
&& self.topology_reconciliation.is_complete()
|
||
&& self.source_cleanup.is_complete()
|
||
}
|
||
}
|
||
|
||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(deny_unknown_fields)]
|
||
struct PreFencingAbort {
|
||
error: String,
|
||
candidate_cleanup: CleanupEvidence,
|
||
topology_reconciliation: CleanupEvidence,
|
||
source_cleanup: CleanupEvidence,
|
||
}
|
||
|
||
impl PreFencingAbort {
|
||
fn is_complete(&self) -> bool {
|
||
self.candidate_cleanup.is_complete()
|
||
&& self.topology_reconciliation.is_complete()
|
||
&& self.source_cleanup.is_complete()
|
||
}
|
||
}
|
||
|
||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(tag = "result", rename_all = "kebab-case", deny_unknown_fields)]
|
||
enum IncumbentCleanupEvidence {
|
||
SkippedNoIncumbent,
|
||
Pending {
|
||
generation_id: String,
|
||
workload: WorkloadObservation,
|
||
},
|
||
Complete {
|
||
generation_id: String,
|
||
},
|
||
}
|
||
|
||
impl IncumbentCleanupEvidence {
|
||
fn is_complete(&self) -> bool {
|
||
matches!(self, Self::SkippedNoIncumbent | Self::Complete { .. })
|
||
}
|
||
}
|
||
|
||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(rename_all = "kebab-case")]
|
||
enum SourceCleanupEvidence {
|
||
SkippedContinuity,
|
||
Pending,
|
||
Complete,
|
||
}
|
||
|
||
impl SourceCleanupEvidence {
|
||
fn is_complete(self) -> bool {
|
||
!matches!(self, Self::Pending)
|
||
}
|
||
}
|
||
|
||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(deny_unknown_fields)]
|
||
struct PostCommitCleanup {
|
||
incumbent: IncumbentCleanupEvidence,
|
||
source: SourceCleanupEvidence,
|
||
}
|
||
|
||
impl PostCommitCleanup {
|
||
fn is_complete(&self) -> bool {
|
||
self.incumbent.is_complete() && self.source.is_complete()
|
||
}
|
||
}
|
||
|
||
/// The sole owner of an in-flight deployment decision. Every actuator result
|
||
/// is durable here before a later phase is entered.
|
||
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||
#[cultcache(
|
||
type = "idunn.deployment_transaction",
|
||
schema = "idunn.deployment_transaction.v4"
|
||
)]
|
||
struct DeploymentTransaction {
|
||
#[cultcache(key = 0)]
|
||
schema_version: String,
|
||
#[cultcache(key = 1)]
|
||
transaction_id: String,
|
||
#[cultcache(key = 2)]
|
||
command_id: String,
|
||
#[cultcache(key = 3)]
|
||
command_kind: CommandKind,
|
||
#[cultcache(key = 4)]
|
||
target: String,
|
||
#[cultcache(key = 5)]
|
||
ordinal: u32,
|
||
#[cultcache(key = 6)]
|
||
phase: DeploymentPhase,
|
||
#[cultcache(key = 7)]
|
||
created_at_unix_millis: u64,
|
||
#[cultcache(key = 8)]
|
||
updated_at_unix_millis: u64,
|
||
#[cultcache(key = 9)]
|
||
incumbent_generation_id: Option<String>,
|
||
#[cultcache(key = 10)]
|
||
plan: Option<CompiledDeploymentPlan>,
|
||
#[cultcache(key = 11)]
|
||
frozen_source: Option<FrozenSourceReceipt>,
|
||
#[cultcache(key = 12)]
|
||
sealed_release: Option<SealedRelease>,
|
||
#[cultcache(key = 13)]
|
||
installed_release: Option<InstalledReleaseObservation>,
|
||
#[cultcache(key = 14)]
|
||
expected: Option<IdunnExpectedIncarnationRecord>,
|
||
#[cultcache(key = 15)]
|
||
expected_publication_sha256: Option<String>,
|
||
#[cultcache(key = 16)]
|
||
deployment_authorization: Option<DeploymentAuthorization>,
|
||
#[cultcache(key = 17)]
|
||
lifecycle_authorized_at_unix_millis: Option<u64>,
|
||
#[cultcache(key = 18)]
|
||
activation: Option<IdunnRuntimeActivationRecord>,
|
||
#[cultcache(key = 19)]
|
||
workload: Option<WorkloadObservation>,
|
||
#[cultcache(key = 20)]
|
||
activation_publication_sha256: Option<String>,
|
||
#[cultcache(key = 21)]
|
||
latest_odin_observation: Option<TopologyEvidence>,
|
||
#[cultcache(key = 22)]
|
||
warming: Option<WarmingEvidence>,
|
||
#[cultcache(key = 23)]
|
||
route_preflight: Option<RoutePreflightReceipt>,
|
||
#[cultcache(key = 24)]
|
||
isolation: Option<IsolationEvidence>,
|
||
#[cultcache(key = 25)]
|
||
fencing: Option<FencingEvidence>,
|
||
#[cultcache(key = 26)]
|
||
leasing: Option<LeasingEvidence>,
|
||
#[cultcache(key = 27)]
|
||
ready: Option<ReadinessEvidence>,
|
||
#[cultcache(key = 28)]
|
||
routing: Option<RoutingEvidence>,
|
||
#[cultcache(key = 29)]
|
||
odin_publisher_sequence_cursor: u64,
|
||
#[cultcache(key = 30)]
|
||
last_error: Option<String>,
|
||
#[cultcache(key = 31)]
|
||
completion: Option<TransactionCompletion>,
|
||
#[cultcache(key = 32)]
|
||
pre_fencing_abort: Option<PreFencingAbort>,
|
||
#[cultcache(key = 33)]
|
||
post_commit_cleanup: Option<PostCommitCleanup>,
|
||
// because every transaction already durable predates this field:
|
||
// an absent key 34 decodes as None, which is exactly "this transaction has
|
||
// no post-fencing abort". Without it the daemon refuses to read its own
|
||
// control store and crashloops.
|
||
#[cultcache(key = 34, default)]
|
||
post_fencing_abort: Option<PostFencingAbort>,
|
||
/// Set by `transition` on entry to each post-fencing phase.
|
||
#[cultcache(key = 35)]
|
||
phase_deadline: Option<PhaseDeadline>,
|
||
#[cultcache(key = 36)]
|
||
lease_adoption: Option<LeaseAdoptionEvidence>,
|
||
}
|
||
|
||
/// The note a deployment keeps while another transaction owns its target.
|
||
fn waiting_for_owner(target: &str, owner: &str) -> String {
|
||
format!("target {target} is owned by {owner}; this deployment fences what is admitted when it completes")
|
||
}
|
||
|
||
/// What `Engine::take_target` decided about one transaction's entry into the
|
||
/// phase where it takes its target.
|
||
#[derive(Debug, PartialEq, Eq)]
|
||
enum Handover {
|
||
/// The CAS wrote the entry: the transaction owns the target.
|
||
Took,
|
||
/// The CAS lost to a concurrent write; nothing was written.
|
||
Lost,
|
||
/// Another transaction owns or drains the target; nothing was written.
|
||
OwnedBy(String),
|
||
/// The admitted generation is no longer the one the transaction names.
|
||
Moved,
|
||
}
|
||
|
||
/// The phase whose entry is the handover of a target: Starting when the rollout
|
||
/// stops the incumbent first (it is stopped there by design), else Fencing. The
|
||
/// one spelling of the rule; a free function so the stop-then-start arm is
|
||
/// testable without a host-actuator fixture.
|
||
fn handover_phase(stops_incumbent_first: bool) -> DeploymentPhase {
|
||
if stops_incumbent_first {
|
||
DeploymentPhase::Starting
|
||
} else {
|
||
DeploymentPhase::Fencing
|
||
}
|
||
}
|
||
|
||
impl DeploymentTransaction {
|
||
/// The one writer of `phase`. It stamps the update time and, through
|
||
/// `retime`, the deadline of what the record now does. No other code
|
||
/// assigns `phase`.
|
||
fn enter_phase(&mut self, phase: DeploymentPhase, now: u64) {
|
||
self.phase = phase;
|
||
self.updated_at_unix_millis = now;
|
||
self.retime(now);
|
||
}
|
||
|
||
/// Whether the abort of the pre-fence path still has cleanup to do.
|
||
fn pre_fencing_abort_pending(&self) -> bool {
|
||
self.completion.is_none()
|
||
&& self
|
||
.pre_fencing_abort
|
||
.as_ref()
|
||
.is_some_and(|abort| !abort.is_complete())
|
||
}
|
||
|
||
/// The one derivation of every duration the record is held to. (1) A
|
||
/// pending pre-fencing abort is timed from now by Idunn's constant. (2) A
|
||
/// draining Complete is timed from its incumbent's retire time (the route
|
||
/// promotion plus the drain, or now when nothing was promoted) for the
|
||
/// plan's committing duration. (3) Any other Complete has none. (4) The
|
||
/// rest take their phase's duration from now, so a post-fencing abort gets
|
||
/// the frozen duration of the phase it aborts, counted from its beginning.
|
||
fn deadline_for_current_activity(&self, now: u64) -> Option<PhaseDeadline> {
|
||
if self.pre_fencing_abort_pending() {
|
||
return Some(PhaseDeadline {
|
||
phase: self.phase,
|
||
entered_at_unix_millis: now,
|
||
deadline_at_unix_millis: now.saturating_add(
|
||
u64::from(PRE_FENCING_ABORT_DEADLINE_SECONDS) * 1000,
|
||
),
|
||
});
|
||
}
|
||
if self.phase == DeploymentPhase::Complete {
|
||
if !matches!(self.completion, Some(TransactionCompletion::Admitted { .. }))
|
||
|| !self.drains_incumbent()
|
||
{
|
||
return None;
|
||
}
|
||
let plan = self.plan.as_ref()?;
|
||
// The retire time needs the binding's drain, which the running
|
||
// binary re-parses and re-admits: a plan frozen by an older binary
|
||
// may not parse now. That must not leave a drain with no deadline,
|
||
// so a retire time that cannot be derived is now, and the
|
||
// committing duration from it bounds the drain.
|
||
let from = self
|
||
.routing
|
||
.as_ref()
|
||
.and_then(RoutingEvidence::promoted_at_unix_millis)
|
||
.and_then(|promoted_at| {
|
||
let (_, binding) = plan.parsed_inputs().ok()?;
|
||
route_drain_deadline(promoted_at, binding.rollout.drain_seconds).ok()
|
||
})
|
||
.unwrap_or(now);
|
||
return Some(PhaseDeadline {
|
||
phase: self.phase,
|
||
entered_at_unix_millis: now,
|
||
deadline_at_unix_millis: from
|
||
.max(now)
|
||
.saturating_add(u64::from(plan.phase_deadlines().committing_seconds) * 1000),
|
||
});
|
||
}
|
||
PhaseDeadline::entering(self.phase, self.plan.as_ref()?, now)
|
||
}
|
||
|
||
/// Stamp the deadline of what the record does now. The only non-test
|
||
/// writer that sets one: phase entry, an abort's beginning, the commit to a
|
||
/// draining Complete and the resolver's lift call it.
|
||
fn retime(&mut self, now: u64) {
|
||
self.phase_deadline = self.deadline_for_current_activity(now);
|
||
}
|
||
|
||
/// Whether a deadline can describe the record's current activity: a
|
||
/// deadline phase, a pre-fencing abort not yet ended, or an admitted Complete.
|
||
fn carries_a_deadline(&self) -> bool {
|
||
// An abort carries its deadline until the record ends, including the
|
||
// write that completes its last cleanup: a Sealing or Starting record
|
||
// has no phase duration to fall back on, and that write is validated.
|
||
DEADLINE_PHASES.contains(&self.phase)
|
||
|| (self.completion.is_none() && self.pre_fencing_abort.is_some())
|
||
|| (self.phase == DeploymentPhase::Complete
|
||
&& matches!(self.completion, Some(TransactionCompletion::Admitted { .. })))
|
||
}
|
||
|
||
/// A live transaction in a deadline phase that is not already being
|
||
/// aborted: the only kind whose phase a deadline, or an operator's expiry,
|
||
/// ends.
|
||
fn resolves_by_deadline(&self) -> bool {
|
||
self.completion.is_none()
|
||
&& self.pre_fencing_abort.is_none()
|
||
&& self.post_fencing_abort.is_none()
|
||
&& DEADLINE_PHASES.contains(&self.phase)
|
||
}
|
||
|
||
/// Whether Idunn has run out of ways to end this record and leaves it to
|
||
/// the operator: its deadline has passed in an abort that cannot finish, a
|
||
/// drain whose stop keeps failing, or a stateful deploy. Derived, never
|
||
/// stored. The step goes on retrying and the target stays owned.
|
||
fn leaves_to_operator(&self, now: u64) -> bool {
|
||
if self.is_terminal() {
|
||
return false;
|
||
}
|
||
let Some(deadline) = &self.phase_deadline else {
|
||
return false;
|
||
};
|
||
now >= deadline.deadline_at_unix_millis
|
||
&& (self.pre_fencing_abort.is_some()
|
||
|| self.post_fencing_abort.is_some()
|
||
|| self.drains_incumbent()
|
||
|| (self.resolves_by_deadline()
|
||
&& matches!(self.phase_end(), PhaseEnd::OperatorRequired | PhaseEnd::RetryForward)))
|
||
}
|
||
|
||
/// Whether an operator's expiry can end this record's phase: it resolves by
|
||
/// deadline, and is not a continuity past Fencing, whose expiry abort would
|
||
/// stop the only running copy of the admitted body (ruling
|
||
/// c2c-expire-refuses-continuity-past-fencing).
|
||
fn expirable(&self) -> bool {
|
||
self.resolves_by_deadline()
|
||
&& !(self.command_kind == CommandKind::Continuity && self.phase >= DeploymentPhase::Fencing)
|
||
}
|
||
|
||
/// The operator's only way to restart a forward-retrying continuity, worded
|
||
/// from the driver its plan binds: a systemd target has a unit to stop (a
|
||
/// stopped transient unit is forgotten, which is permanently stopped); a
|
||
/// host-actuator target has no unit, only a process on the managed host
|
||
/// that the actuator then reports not running.
|
||
fn manual_exit(&self) -> &'static str {
|
||
match self
|
||
.plan
|
||
.as_ref()
|
||
.and_then(|plan| plan.parsed_inputs().ok())
|
||
.map(|(_, binding)| binding.workload)
|
||
{
|
||
Some(WorkloadBinding::HostActuator(_)) => "kill its process on the managed host by hand",
|
||
_ => "stop its unit by hand",
|
||
}
|
||
}
|
||
|
||
/// The candidate's route proof has succeeded: it is Committing, or its
|
||
/// routing evidence is recorded and only the transition remains. It serves,
|
||
/// so no waiting deployment stops it (ruling c2d-yield-only-unproven-to-
|
||
/// movable). Derived, never stored.
|
||
fn serving_proven(&self) -> bool {
|
||
self.phase >= DeploymentPhase::Committing || self.routing.is_some()
|
||
}
|
||
|
||
/// A continuity at or past Fencing that its deadline leaves to the operator
|
||
/// while its step runs, and whose candidate is still unproven: it yields to
|
||
/// a deployment that can take the handover (ruling continuity-yields-to-
|
||
/// deploy), so a waiter never ends on it. A proven continuity keeps serving
|
||
/// and paging and does not yield. Derived, never stored.
|
||
fn yields_to_a_deploy(&self) -> bool {
|
||
self.resolves_by_deadline()
|
||
&& self.phase_end() == PhaseEnd::RetryForward
|
||
&& !self.serving_proven()
|
||
}
|
||
|
||
/// What an ended post-fencing phase resolves to (Q3). A continuity at or
|
||
/// past Fencing retries forward: its incumbent is the admitted generation,
|
||
/// which supervision mints a continuity only after seeing dead, and from
|
||
/// Fencing the continuity owns the target, so aborting restores the body
|
||
/// its candidate already runs (ruling c2c-retry-decided-by-record). A
|
||
/// deployment whose candidate was never issued a write lease cannot have
|
||
/// written, so it aborts to the incumbent through the post-fencing path,
|
||
/// stateless or not. One whose candidate was issued a lease may have
|
||
/// written under a new state contract, and restoring the incumbent over
|
||
/// that is not Idunn's call: it is left for the operator.
|
||
fn phase_end(&self) -> PhaseEnd {
|
||
if self.phase < DeploymentPhase::Fencing {
|
||
return PhaseEnd::AbortBeforeFencing;
|
||
}
|
||
let leasing = self.leasing.as_ref();
|
||
let granted = leasing.and_then(LeasingEvidence::lease).is_some();
|
||
let issued = leasing.and_then(LeasingEvidence::issued_lease).is_some();
|
||
match self.command_kind {
|
||
CommandKind::Continuity => PhaseEnd::RetryForward,
|
||
CommandKind::Deploy if issued => PhaseEnd::OperatorRequired,
|
||
CommandKind::Deploy => PhaseEnd::Abort {
|
||
lease_not_adopted: granted && self.lease_adoption.is_none(),
|
||
},
|
||
}
|
||
}
|
||
|
||
/// The recovery a completed post-fencing abort names. A continuity whose
|
||
/// candidate adopted its lease restarts the admitted release; everything
|
||
/// else restores the incumbent.
|
||
fn terminal_recovery(&self) -> TerminalRecovery {
|
||
if self.command_kind == CommandKind::Continuity && self.lease_adoption.is_some() {
|
||
TerminalRecovery::RestartAdmitted
|
||
} else {
|
||
TerminalRecovery::RestoreIncumbent
|
||
}
|
||
}
|
||
|
||
/// Whether this transaction's binding declares stop-then-start. A
|
||
/// transaction without a plan yet cannot, so it reads as false.
|
||
fn rollout_stops_incumbent_first(&self) -> bool {
|
||
self.plan
|
||
.as_ref()
|
||
.and_then(|plan| plan.parsed_inputs().ok())
|
||
.is_some_and(|(_, binding)| binding.rollout.strategy == RolloutStrategy::StopThenStart)
|
||
}
|
||
|
||
fn new(
|
||
command: &DeploymentCommand,
|
||
target: String,
|
||
ordinal: u32,
|
||
incumbent: Option<&AdmittedGeneration>,
|
||
now: u64,
|
||
) -> Result<Self> {
|
||
let transaction = Self {
|
||
schema_version: DEPLOYMENT_TRANSACTION_SCHEMA.into(),
|
||
transaction_id: format!("tx-{}", Uuid::new_v4()),
|
||
command_id: command.command_id.clone(),
|
||
command_kind: command.kind,
|
||
target,
|
||
ordinal,
|
||
phase: DeploymentPhase::Sealing,
|
||
created_at_unix_millis: now,
|
||
updated_at_unix_millis: now,
|
||
incumbent_generation_id: incumbent.map(|value| value.generation_id.clone()),
|
||
plan: None,
|
||
frozen_source: None,
|
||
sealed_release: None,
|
||
installed_release: None,
|
||
expected: None,
|
||
expected_publication_sha256: None,
|
||
deployment_authorization: None,
|
||
lifecycle_authorized_at_unix_millis: None,
|
||
activation: None,
|
||
workload: None,
|
||
activation_publication_sha256: None,
|
||
latest_odin_observation: None,
|
||
warming: None,
|
||
route_preflight: None,
|
||
isolation: None,
|
||
fencing: None,
|
||
leasing: None,
|
||
ready: None,
|
||
routing: None,
|
||
odin_publisher_sequence_cursor: incumbent
|
||
.map_or(0, |value| value.odin_publisher_sequence_cursor),
|
||
last_error: None,
|
||
completion: None,
|
||
pre_fencing_abort: None,
|
||
post_fencing_abort: None,
|
||
post_commit_cleanup: None,
|
||
phase_deadline: None,
|
||
lease_adoption: None,
|
||
};
|
||
transaction.validate()?;
|
||
Ok(transaction)
|
||
}
|
||
|
||
fn from_continuity(
|
||
command: &DeploymentCommand,
|
||
incumbent: &AdmittedGeneration,
|
||
now: u64,
|
||
) -> Result<Self> {
|
||
let mut transaction =
|
||
Self::new(command, incumbent.target.clone(), 0, Some(incumbent), now)?;
|
||
transaction.plan = Some(incumbent.plan.clone());
|
||
transaction.sealed_release = Some(incumbent.sealed_release.clone());
|
||
transaction.installed_release = Some(incumbent.installed_release.clone());
|
||
transaction.expected = Some(incumbent.expected.clone());
|
||
transaction.validate()?;
|
||
Ok(transaction)
|
||
}
|
||
|
||
/// Whether an abort owes the projection a reconciliation. A deployment
|
||
/// published its own Expected key. A continuity shares the admitted key,
|
||
/// whose Expected is never withdrawn, so it owes only the demotion of the
|
||
/// activation it issued.
|
||
fn abort_topology_reconciliation(&self) -> CleanupEvidence {
|
||
let owes = match self.command_kind {
|
||
CommandKind::Deploy => self.expected_publication_sha256.is_some(),
|
||
CommandKind::Continuity => self.activation.is_some(),
|
||
};
|
||
if owes {
|
||
CleanupEvidence::Pending
|
||
} else {
|
||
CleanupEvidence::Skipped
|
||
}
|
||
}
|
||
|
||
/// Whether an abort owes the candidate's durable cleanup: it does once an
|
||
/// activation was issued or a workload was observed, either of which leaves
|
||
/// something behind.
|
||
fn candidate_cleanup_owed(&self) -> CleanupEvidence {
|
||
if self.activation.is_some() || self.workload.is_some() {
|
||
CleanupEvidence::Pending
|
||
} else {
|
||
CleanupEvidence::Skipped
|
||
}
|
||
}
|
||
|
||
/// Whether an abort's recorded candidate cleanup is what
|
||
/// `candidate_cleanup_owed` says it must be: owed cleanup is pending or
|
||
/// done, unowed cleanup is skipped.
|
||
fn candidate_cleanup_recorded(&self, recorded: CleanupEvidence) -> bool {
|
||
matches!(
|
||
(self.candidate_cleanup_owed(), recorded),
|
||
(
|
||
CleanupEvidence::Pending,
|
||
CleanupEvidence::Pending | CleanupEvidence::Complete
|
||
) | (CleanupEvidence::Skipped, CleanupEvidence::Skipped)
|
||
)
|
||
}
|
||
|
||
fn rejected(command: &DeploymentCommand, error: anyhow::Error, now: u64) -> Result<Self> {
|
||
let mut transaction = Self::new(command, command.selector.clone(), 0, None, now)?;
|
||
let detail = bounded_detail(&format!("{error:#}"));
|
||
transaction.enter_phase(DeploymentPhase::Complete, now);
|
||
transaction.last_error = Some(detail.clone());
|
||
transaction.pre_fencing_abort = Some(PreFencingAbort {
|
||
error: detail.clone(),
|
||
candidate_cleanup: CleanupEvidence::Skipped,
|
||
topology_reconciliation: CleanupEvidence::Skipped,
|
||
source_cleanup: CleanupEvidence::Skipped,
|
||
});
|
||
transaction.completion = Some(TransactionCompletion::FailedBeforeFencing { error: detail });
|
||
transaction.validate()?;
|
||
Ok(transaction)
|
||
}
|
||
|
||
fn is_terminal(&self) -> bool {
|
||
if self.phase != DeploymentPhase::Complete {
|
||
return false;
|
||
}
|
||
match &self.completion {
|
||
Some(TransactionCompletion::FailedBeforeFencing { .. }) => self
|
||
.pre_fencing_abort
|
||
.as_ref()
|
||
.is_some_and(PreFencingAbort::is_complete),
|
||
Some(TransactionCompletion::FailedAfterFencing { .. }) => self
|
||
.post_fencing_abort
|
||
.as_ref()
|
||
.is_some_and(PostFencingAbort::is_complete),
|
||
Some(TransactionCompletion::Admitted { .. }) => self
|
||
.post_commit_cleanup
|
||
.as_ref()
|
||
.is_some_and(PostCommitCleanup::is_complete),
|
||
None => false,
|
||
}
|
||
}
|
||
|
||
/// The phase whose entry is the handover: the one transaction that makes
|
||
/// it takes the target's incumbent slot from continuity supervision.
|
||
fn takes_target_at(&self) -> DeploymentPhase {
|
||
handover_phase(self.rollout_stops_incumbent_first())
|
||
}
|
||
|
||
/// Not Complete: the transaction still holds its own candidate slot.
|
||
/// Read by sibling order inside one command, freeze (a live Deploy
|
||
/// refuses a second), the mint gate (a live Continuity refuses a second),
|
||
/// candidate-port reservation and validation.
|
||
fn is_live(&self) -> bool {
|
||
self.phase != DeploymentPhase::Complete
|
||
}
|
||
|
||
/// A live deployment whose next move is the handover: not ended, not
|
||
/// aborting, one phase short of `takes_target_at`. Derived, never stored.
|
||
/// Read by `deploy_the_continuity_yields_to`: the continuity's yield and the
|
||
/// boot lift.
|
||
fn awaits_handover(&self) -> bool {
|
||
self.command_kind == CommandKind::Deploy
|
||
&& self.completion.is_none()
|
||
&& self.pre_fencing_abort.is_none()
|
||
&& self.phase as u8 + 1 == self.takes_target_at() as u8
|
||
}
|
||
|
||
/// From the handover until Complete the transaction owns the target's
|
||
/// incumbent slot: it fences, leases, routes and commits. Derived from the
|
||
/// phase, never stored. Read by validation (one owner), supervision (which
|
||
/// defers to an owner), the handover CAS, a waiting deployment and status.
|
||
fn owns_target(&self) -> bool {
|
||
(self.takes_target_at()..DeploymentPhase::Complete).contains(&self.phase)
|
||
}
|
||
|
||
/// Admission has moved to the new generation at Complete, but an exact
|
||
/// draining incumbent still reserves its process and candidate endpoint.
|
||
/// Read by the mint gate, the handover, candidate-port reservation and
|
||
/// status: no later mutation of the target may overlap that cleanup.
|
||
fn drains_incumbent(&self) -> bool {
|
||
self.post_commit_cleanup
|
||
.as_ref()
|
||
.is_some_and(|cleanup| !cleanup.is_complete())
|
||
}
|
||
|
||
/// Where what this transaction has collected disagrees with what its own
|
||
/// Expected requires. There is nothing to disagree with before Expected.
|
||
fn readiness_disagreement(&self) -> Option<ReadinessDisagreement> {
|
||
let expected = self.expected.as_ref()?;
|
||
ReadinessClass::confirmed(
|
||
expected,
|
||
self.warming
|
||
.iter()
|
||
.map(WarmingEvidence::voucher)
|
||
.chain(self.ready.iter().map(ReadinessEvidence::voucher)),
|
||
)
|
||
.err()
|
||
}
|
||
|
||
/// The disagreement that holds this transaction: evidence collected under
|
||
/// another class is held and reported, not advanced and not repaired, since
|
||
/// no step can make it the right kind and running one anyway only fails it
|
||
/// every tick. What is finished, or being finished (a completion, or an
|
||
/// abort under way), never reads the class, so it is left to run.
|
||
fn held_disagreement(&self) -> Option<ReadinessDisagreement> {
|
||
let finishing = self.completion.is_some()
|
||
|| self.pre_fencing_abort.is_some()
|
||
|| self.post_fencing_abort.is_some();
|
||
self.readiness_disagreement().filter(|_| !finishing)
|
||
}
|
||
|
||
fn validate(&self) -> Result<()> {
|
||
ensure!(
|
||
self.schema_version == DEPLOYMENT_TRANSACTION_SCHEMA,
|
||
"deployment transaction schema is unsupported"
|
||
);
|
||
require_id(&self.transaction_id, "deployment transaction id")?;
|
||
require_id(&self.command_id, "transaction command id")?;
|
||
require_id(&self.target, "transaction target")?;
|
||
ensure!(
|
||
self.created_at_unix_millis > 0 && self.updated_at_unix_millis > 0,
|
||
"deployment transaction timestamps are invalid"
|
||
);
|
||
if let Some(generation) = &self.incumbent_generation_id {
|
||
require_id(generation, "incumbent generation id")?;
|
||
}
|
||
if let Some(plan) = &self.plan {
|
||
plan.validate()?;
|
||
ensure!(
|
||
plan.parsed_inputs()?.1.target == self.target,
|
||
"transaction plan belongs to another target"
|
||
);
|
||
}
|
||
if let (Some(plan), Some(frozen)) = (&self.plan, &self.frozen_source) {
|
||
frozen.validate_against(plan)?;
|
||
}
|
||
if let (Some(plan), Some(release)) = (&self.plan, &self.sealed_release) {
|
||
release.validate_against(plan)?;
|
||
}
|
||
if let Some(expected) = &self.expected {
|
||
expected.validate()?;
|
||
ensure!(expected.target == self.target, "Expected target differs");
|
||
if let Some(plan) = &self.plan {
|
||
ensure!(expected.plan_id == plan.plan_id, "Expected plan differs");
|
||
}
|
||
if let Some(release) = &self.sealed_release {
|
||
ensure!(
|
||
expected.sealed_release_id == release.sealed_release_id,
|
||
"Expected release differs"
|
||
);
|
||
}
|
||
if let Some(digest) = &self.expected_publication_sha256 {
|
||
ensure!(
|
||
digest == &expected.canonical_sha256()?,
|
||
"Expected publication receipt differs"
|
||
);
|
||
}
|
||
}
|
||
if let (Some(expected), Some(activation)) = (&self.expected, &self.activation) {
|
||
activation.validate()?;
|
||
ensure!(
|
||
activation.expected_projection_sha256 == expected.canonical_sha256()?,
|
||
"activation belongs to another Expected projection"
|
||
);
|
||
if let Some(workload) = &self.workload {
|
||
ensure!(
|
||
workload.runtime_instance_id() == activation.runtime_instance_id,
|
||
"workload belongs to another activation"
|
||
);
|
||
}
|
||
if let Some(digest) = &self.activation_publication_sha256 {
|
||
ensure!(
|
||
digest == &activation.canonical_sha256()?,
|
||
"activation publication receipt differs"
|
||
);
|
||
}
|
||
}
|
||
ensure!(
|
||
self.workload.is_none() || self.activation.is_some(),
|
||
"workload observation exists without its prepared activation"
|
||
);
|
||
let ready_odin = self.ready.as_ref().and_then(ReadinessEvidence::odin);
|
||
for evidence in [self.latest_odin_observation.as_ref(), ready_odin]
|
||
.into_iter()
|
||
.flatten()
|
||
{
|
||
evidence.validate_shape()?;
|
||
ensure!(
|
||
evidence.publisher_sequence <= self.odin_publisher_sequence_cursor,
|
||
"topology evidence exceeds the transaction replay cursor"
|
||
);
|
||
}
|
||
if let Some(ready) = &self.ready {
|
||
ready.validate_shape()?;
|
||
// A candidate-vouched receipt is only ever written by this code, so
|
||
// one on a target that is not route-proof is forged. The other
|
||
// direction -- an Odin receipt on a route-proof target -- is what a
|
||
// pre-B3 record looks like, and is a ReadinessDisagreement to hold,
|
||
// not a record to refuse: refusing it would refuse to boot.
|
||
if let (Voucher::Candidate, Some(expected)) = (ready.voucher(), &self.expected) {
|
||
ensure!(
|
||
ReadinessClass::of(expected).is_ok_and(|class| class == ReadinessClass::RouteProof),
|
||
"route-proof Ready evidence for a target that is not route-proof"
|
||
);
|
||
}
|
||
}
|
||
if let Some(warming) = &self.warming {
|
||
warming.validate_shape()?;
|
||
match warming {
|
||
WarmingEvidence::OdinTopology { evidence } => ensure!(
|
||
evidence.publisher_sequence <= self.odin_publisher_sequence_cursor,
|
||
"Warming topology evidence exceeds the transaction replay cursor"
|
||
),
|
||
WarmingEvidence::FirstOdinDirect { .. } => ensure!(
|
||
self.expected.as_ref().is_some_and(|expected| {
|
||
ReadinessClass::of(expected).is_ok_and(|class| class == ReadinessClass::OdinSelf)
|
||
}),
|
||
"direct Warming evidence is reserved for Odin observing itself"
|
||
),
|
||
WarmingEvidence::RouteProofDirect { .. } => ensure!(
|
||
self.expected.as_ref().is_none_or(|expected| {
|
||
ReadinessClass::of(expected).is_ok_and(|class| class == ReadinessClass::RouteProof)
|
||
}),
|
||
"direct route-proof Warming evidence for a target that is not route-proof"
|
||
),
|
||
}
|
||
}
|
||
if let Some(leasing) = &self.leasing {
|
||
if let Some(expected) = &self.expected {
|
||
ensure!(
|
||
matches!(
|
||
(expected.write_lease_required, leasing),
|
||
(true, LeasingEvidence::Prepared { .. })
|
||
| (true, LeasingEvidence::Granted { .. })
|
||
| (false, LeasingEvidence::SkippedStateless)
|
||
),
|
||
"lease evidence differs from the Expected state contract"
|
||
);
|
||
}
|
||
let lease_and_sha256 = match leasing {
|
||
LeasingEvidence::SkippedStateless => None,
|
||
LeasingEvidence::Prepared {
|
||
lease,
|
||
lease_sha256,
|
||
}
|
||
| LeasingEvidence::Granted {
|
||
lease,
|
||
lease_sha256,
|
||
} => Some((lease, lease_sha256)),
|
||
};
|
||
if let Some((lease, lease_sha256)) = lease_and_sha256 {
|
||
lease.validate()?;
|
||
ensure!(
|
||
lease.canonical_sha256()? == *lease_sha256,
|
||
"lease evidence digest differs"
|
||
);
|
||
}
|
||
}
|
||
if let (Some(expected), Some(fencing)) = (&self.expected, &self.fencing) {
|
||
match fencing {
|
||
FencingEvidence::SkippedStateless => ensure!(
|
||
!expected.write_lease_required,
|
||
"stateful candidate claims stateless fencing"
|
||
),
|
||
FencingEvidence::Revoked {
|
||
candidate_lease_path_verified_empty,
|
||
..
|
||
} => ensure!(
|
||
*candidate_lease_path_verified_empty == expected.write_lease_required,
|
||
"candidate lease-path fencing differs from Expected"
|
||
),
|
||
}
|
||
}
|
||
if let Some(error) = &self.last_error {
|
||
require_detail(error, "transaction error")?;
|
||
}
|
||
if let Some(deadline) = &self.phase_deadline {
|
||
ensure!(
|
||
deadline.phase == self.phase
|
||
&& self.carries_a_deadline()
|
||
&& deadline.entered_at_unix_millis > 0
|
||
&& deadline.deadline_at_unix_millis > deadline.entered_at_unix_millis,
|
||
"phase deadline does not describe the record's current activity"
|
||
);
|
||
}
|
||
if let Some(adoption) = &self.lease_adoption {
|
||
adoption.validate_shape()?;
|
||
ensure!(
|
||
self.leasing
|
||
.as_ref()
|
||
.is_some_and(|leasing| adoption.names(leasing)),
|
||
"lease adoption does not name the granted lease"
|
||
);
|
||
}
|
||
if let Some(authorization) = &self.deployment_authorization {
|
||
authorization.validate_shape()?;
|
||
}
|
||
if let Some(abort) = &self.pre_fencing_abort {
|
||
require_detail(&abort.error, "pre-fencing abort error")?;
|
||
ensure!(
|
||
self.post_commit_cleanup.is_none(),
|
||
"aborted transaction also carries post-commit cleanup"
|
||
);
|
||
ensure!(
|
||
self.candidate_cleanup_recorded(abort.candidate_cleanup),
|
||
"abort candidate cleanup differs from its prepared activation"
|
||
);
|
||
ensure!(
|
||
matches!(
|
||
(
|
||
self.abort_topology_reconciliation(),
|
||
abort.topology_reconciliation
|
||
),
|
||
(
|
||
CleanupEvidence::Pending,
|
||
CleanupEvidence::Pending | CleanupEvidence::Complete
|
||
) | (CleanupEvidence::Skipped, CleanupEvidence::Skipped)
|
||
) || (self.command_kind == CommandKind::Continuity
|
||
&& self.phase == DeploymentPhase::Complete
|
||
&& self.completion.is_some()
|
||
&& abort.topology_reconciliation.is_legacy_marker()),
|
||
"abort topology cleanup differs from what the transaction projected"
|
||
);
|
||
ensure!(
|
||
matches!(
|
||
(self.command_kind, abort.source_cleanup),
|
||
(
|
||
CommandKind::Deploy,
|
||
CleanupEvidence::Pending | CleanupEvidence::Complete
|
||
) | (CommandKind::Continuity, CleanupEvidence::Skipped)
|
||
) || (self.command_kind == CommandKind::Deploy
|
||
&& abort.source_cleanup == CleanupEvidence::Skipped
|
||
&& self.plan.is_none()
|
||
&& self.frozen_source.is_none()
|
||
&& self.expected.is_none()
|
||
&& self.phase == DeploymentPhase::Complete),
|
||
"abort source cleanup differs from command work"
|
||
);
|
||
ensure!(
|
||
self.phase < DeploymentPhase::Fencing || self.phase == DeploymentPhase::Complete,
|
||
"pre-fencing abort crossed the fencing boundary"
|
||
);
|
||
}
|
||
|
||
if let Some(abort) = &self.post_fencing_abort {
|
||
require_detail(&abort.error, "post-fencing abort error")?;
|
||
ensure!(
|
||
self.pre_fencing_abort.is_none() && self.post_commit_cleanup.is_none(),
|
||
"post-fencing abort collides with another terminal path"
|
||
);
|
||
ensure!(
|
||
self.phase >= DeploymentPhase::Fencing,
|
||
"post-fencing abort exists before the fence"
|
||
);
|
||
ensure!(
|
||
self.candidate_cleanup_recorded(abort.candidate_cleanup),
|
||
"abort candidate cleanup differs from its prepared activation"
|
||
);
|
||
}
|
||
|
||
let failed_after_fencing = matches!(
|
||
self.completion,
|
||
Some(TransactionCompletion::FailedAfterFencing { .. })
|
||
);
|
||
if failed_after_fencing {
|
||
ensure!(
|
||
self.phase == DeploymentPhase::Complete,
|
||
"terminal failure is not Complete"
|
||
);
|
||
let abort = required(
|
||
&self.post_fencing_abort,
|
||
"terminal post-fence abort evidence",
|
||
)?;
|
||
let TransactionCompletion::FailedAfterFencing { error, recovery } =
|
||
self.completion.as_ref().unwrap()
|
||
else {
|
||
unreachable!()
|
||
};
|
||
if let TerminalRecovery::OperatorRequired { reason } = recovery {
|
||
require_detail(reason, "operator-required recovery reason")?;
|
||
}
|
||
ensure!(
|
||
abort.is_complete() && abort.error == *error,
|
||
"terminal failure lacks complete matching abort evidence"
|
||
);
|
||
return Ok(());
|
||
}
|
||
|
||
let failed = matches!(
|
||
self.completion,
|
||
Some(TransactionCompletion::FailedBeforeFencing { .. })
|
||
);
|
||
if failed {
|
||
ensure!(
|
||
self.phase == DeploymentPhase::Complete,
|
||
"terminal failure is not Complete"
|
||
);
|
||
let abort = required(&self.pre_fencing_abort, "terminal abort evidence")?;
|
||
let TransactionCompletion::FailedBeforeFencing { error } =
|
||
self.completion.as_ref().unwrap()
|
||
else {
|
||
unreachable!()
|
||
};
|
||
ensure!(
|
||
abort.is_complete() && abort.error == *error,
|
||
"terminal failure lacks complete matching abort evidence"
|
||
);
|
||
} else {
|
||
ensure!(
|
||
!matches!(
|
||
self.completion,
|
||
Some(TransactionCompletion::FailedBeforeFencing { .. })
|
||
),
|
||
"non-failure branch carries failure completion"
|
||
);
|
||
if self.phase >= DeploymentPhase::Starting {
|
||
ensure!(
|
||
self.plan.is_some()
|
||
&& self.sealed_release.is_some()
|
||
&& self.installed_release.is_some()
|
||
&& self.expected.is_some(),
|
||
"Starting transaction lacks sealed release evidence"
|
||
);
|
||
// Expected is published as Starting's first step, after the
|
||
// brake; from the activation onward it must be there.
|
||
ensure!(
|
||
self.expected_publication_sha256.is_some()
|
||
|| (self.activation.is_none() && self.phase == DeploymentPhase::Starting),
|
||
"started transaction has not published its Expected"
|
||
);
|
||
match self.command_kind {
|
||
CommandKind::Deploy => ensure!(
|
||
self.frozen_source.is_some() && self.deployment_authorization.is_some(),
|
||
"deployment entered Starting without frozen source and brake consumption"
|
||
),
|
||
CommandKind::Continuity => ensure!(
|
||
self.lifecycle_authorized_at_unix_millis.is_some()
|
||
&& self.deployment_authorization.is_none(),
|
||
"continuity entered Starting without its lifecycle gate"
|
||
),
|
||
}
|
||
}
|
||
if self.phase >= DeploymentPhase::Warming {
|
||
ensure!(
|
||
self.activation.is_some()
|
||
&& self.workload.is_some()
|
||
&& self.activation_publication_sha256.is_some(),
|
||
"Warming transaction lacks observed activation"
|
||
);
|
||
}
|
||
if self.phase >= DeploymentPhase::Fencing {
|
||
ensure!(
|
||
self.warming.is_some() && self.isolation.is_some(),
|
||
"Fencing transaction lacks warming or isolation evidence"
|
||
);
|
||
ensure!(
|
||
self.expected
|
||
.as_ref()
|
||
.and_then(|expected| expected.route.as_ref())
|
||
.is_some()
|
||
== self.route_preflight.is_some(),
|
||
"route preflight does not match routed Expected"
|
||
);
|
||
if let Some(preflight) = &self.route_preflight {
|
||
preflight.validate()?;
|
||
}
|
||
}
|
||
if self.phase >= DeploymentPhase::Leasing {
|
||
ensure!(self.fencing.is_some(), "Leasing lacks fencing evidence");
|
||
}
|
||
if self.phase >= DeploymentPhase::AwaitingReady {
|
||
ensure!(
|
||
matches!(
|
||
self.leasing.as_ref(),
|
||
Some(LeasingEvidence::SkippedStateless)
|
||
| Some(LeasingEvidence::Granted { .. })
|
||
),
|
||
"AwaitingReady lacks finalized lease evidence"
|
||
);
|
||
}
|
||
if self.phase >= DeploymentPhase::Routing {
|
||
ensure!(self.ready.is_some(), "Routing lacks Ready evidence");
|
||
}
|
||
if self.phase >= DeploymentPhase::Committing {
|
||
ensure!(self.routing.is_some(), "Committing lacks routing evidence");
|
||
}
|
||
if let Some(RoutingEvidence::Promoted {
|
||
observation,
|
||
promoted_at_unix_millis,
|
||
}) = &self.routing
|
||
{
|
||
observation.validate()?;
|
||
ensure!(*promoted_at_unix_millis > 0, "route promotion has no time");
|
||
}
|
||
if self.phase == DeploymentPhase::Complete {
|
||
ensure!(
|
||
matches!(
|
||
self.completion,
|
||
Some(TransactionCompletion::Admitted { .. })
|
||
),
|
||
"successful Complete transaction lacks admission receipt"
|
||
);
|
||
ensure!(
|
||
self.pre_fencing_abort.is_none() && self.post_commit_cleanup.is_some(),
|
||
"admitted transaction lacks exclusive post-commit cleanup evidence"
|
||
);
|
||
let TransactionCompletion::Admitted { generation_id } =
|
||
self.completion.as_ref().unwrap()
|
||
else {
|
||
unreachable!()
|
||
};
|
||
ensure!(
|
||
generation_id == &format!("generation-{}", self.transaction_id),
|
||
"admission completion names another transaction generation"
|
||
);
|
||
} else {
|
||
ensure!(
|
||
self.completion.is_none() && self.post_commit_cleanup.is_none(),
|
||
"non-Complete transaction has completion or post-commit state"
|
||
);
|
||
}
|
||
}
|
||
if let Some(cleanup) = &self.post_commit_cleanup {
|
||
ensure!(
|
||
self.phase == DeploymentPhase::Complete
|
||
&& matches!(
|
||
self.completion,
|
||
Some(TransactionCompletion::Admitted { .. })
|
||
),
|
||
"post-commit cleanup exists without an admitted Complete transaction"
|
||
);
|
||
match (&self.incumbent_generation_id, &cleanup.incumbent) {
|
||
(None, IncumbentCleanupEvidence::SkippedNoIncumbent) => {}
|
||
(
|
||
Some(expected),
|
||
IncumbentCleanupEvidence::Pending { generation_id, .. }
|
||
| IncumbentCleanupEvidence::Complete { generation_id },
|
||
) if expected == generation_id => {}
|
||
_ => bail!("post-commit incumbent cleanup names another generation"),
|
||
}
|
||
ensure!(
|
||
matches!(
|
||
(self.command_kind, cleanup.source),
|
||
(CommandKind::Deploy, SourceCleanupEvidence::Pending)
|
||
| (CommandKind::Deploy, SourceCleanupEvidence::Complete)
|
||
| (
|
||
CommandKind::Continuity,
|
||
SourceCleanupEvidence::SkippedContinuity
|
||
)
|
||
),
|
||
"post-commit source cleanup differs from command kind"
|
||
);
|
||
}
|
||
Ok(())
|
||
}
|
||
}
|
||
|
||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(deny_unknown_fields)]
|
||
struct AdmittedOdinAuthority {
|
||
signer_identity_id: String,
|
||
signer_public_key: Vec<u8>,
|
||
}
|
||
|
||
impl AdmittedOdinAuthority {
|
||
fn from_anchor(anchor: &ServiceIdentityTrustAnchor) -> Result<Self> {
|
||
ensure!(
|
||
derive_service_identity_id::<OdinTopologyIdentity>(&anchor.public_key)?
|
||
== anchor.identity_id,
|
||
"Odin topology anchor identity differs from its key"
|
||
);
|
||
Ok(Self {
|
||
signer_identity_id: anchor.identity_id.clone(),
|
||
signer_public_key: anchor.public_key.clone(),
|
||
})
|
||
}
|
||
|
||
fn validate(&self) -> Result<()> {
|
||
require_id(&self.signer_identity_id, "admitted Odin signer")?;
|
||
ensure!(
|
||
derive_service_identity_id::<OdinTopologyIdentity>(&self.signer_public_key)?
|
||
== self.signer_identity_id,
|
||
"admitted Odin signer identity differs from its key"
|
||
);
|
||
Ok(())
|
||
}
|
||
}
|
||
|
||
/// The only current-incarnation owner. Unit state, route state, topology
|
||
/// projections, and CLI output are observations of this record, never peers.
|
||
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||
#[cultcache(
|
||
type = "idunn.admitted_generation",
|
||
schema = "idunn.admitted_generation.v4"
|
||
)]
|
||
struct AdmittedGeneration {
|
||
#[cultcache(key = 0)]
|
||
schema_version: String,
|
||
#[cultcache(key = 1)]
|
||
target: String,
|
||
#[cultcache(key = 2)]
|
||
generation_id: String,
|
||
#[cultcache(key = 3)]
|
||
command_id: String,
|
||
#[cultcache(key = 4)]
|
||
transaction_id: String,
|
||
#[cultcache(key = 5)]
|
||
admitted_at_unix_millis: u64,
|
||
#[cultcache(key = 6)]
|
||
plan: CompiledDeploymentPlan,
|
||
#[cultcache(key = 7)]
|
||
sealed_release: SealedRelease,
|
||
#[cultcache(key = 8)]
|
||
installed_release: InstalledReleaseObservation,
|
||
#[cultcache(key = 9)]
|
||
expected: IdunnExpectedIncarnationRecord,
|
||
#[cultcache(key = 10)]
|
||
activation: IdunnRuntimeActivationRecord,
|
||
#[cultcache(key = 11)]
|
||
workload: WorkloadObservation,
|
||
#[cultcache(key = 12)]
|
||
leasing: LeasingEvidence,
|
||
/// The receipt that admitted this incarnation, tagged by proof class.
|
||
#[cultcache(key = 13)]
|
||
ready: ReadinessEvidence,
|
||
/// Present exactly when `ready` is Odin-correlated.
|
||
#[cultcache(key = 14)]
|
||
latest_odin_observation: Option<TopologyEvidence>,
|
||
#[cultcache(key = 15)]
|
||
routing: RoutingEvidence,
|
||
/// Present exactly when `ready` is Odin-correlated.
|
||
#[cultcache(key = 16)]
|
||
odin_authority: Option<AdmittedOdinAuthority>,
|
||
/// Meaningful only for an Odin-correlated generation.
|
||
#[cultcache(key = 17)]
|
||
odin_publisher_sequence_cursor: u64,
|
||
// Keys 18 (route repair start) and 20 (continuity backoff) are retired.
|
||
// The slots stay as gaps: the layout is positional, so a slot cannot be
|
||
// reused without a bump.
|
||
/// Present exactly when `routing` is Promoted.
|
||
#[cultcache(key = 19)]
|
||
route_supervision: Option<RouteSupervisionState>,
|
||
/// Why this generation is held or down, for `status`. Cleared when the
|
||
/// generation is replaced.
|
||
#[cultcache(key = 21)]
|
||
last_error: Option<String>,
|
||
}
|
||
|
||
/// The Odin-side receipts of an Odin-correlated generation. Decisions that
|
||
/// consume Odin evidence read it through here, so a route-proof generation
|
||
/// cannot be mistaken for one.
|
||
struct OdinReceipts<'a> {
|
||
ready: &'a TopologyEvidence,
|
||
latest: &'a TopologyEvidence,
|
||
}
|
||
|
||
impl AdmittedGeneration {
|
||
/// This generation's class, or where its evidence disagrees with it. The
|
||
/// class comes from its Expected, never from the tag on its Ready receipt.
|
||
fn readiness(&self) -> Result<ReadinessClass, ReadinessDisagreement> {
|
||
ReadinessClass::confirmed(&self.expected, [self.ready.voucher()])
|
||
}
|
||
|
||
fn odin_receipts(&self) -> Result<OdinReceipts<'_>> {
|
||
match (
|
||
&self.ready,
|
||
&self.latest_odin_observation,
|
||
&self.odin_authority,
|
||
) {
|
||
(ReadinessEvidence::OdinCorrelated { evidence }, Some(latest), Some(_)) => {
|
||
Ok(OdinReceipts {
|
||
ready: evidence,
|
||
latest,
|
||
})
|
||
}
|
||
_ => bail!("admitted generation of {} is not Odin-correlated", self.target),
|
||
}
|
||
}
|
||
|
||
fn from_transaction(
|
||
transaction: &DeploymentTransaction,
|
||
odin_authority: Option<AdmittedOdinAuthority>,
|
||
now: u64,
|
||
) -> Result<Self> {
|
||
ensure!(
|
||
transaction.phase == DeploymentPhase::Committing,
|
||
"only Committing can create an admitted generation"
|
||
);
|
||
let ready = required(&transaction.ready, "Ready receipt")?.clone();
|
||
let (latest_odin_observation, odin_authority) = match &ready {
|
||
ReadinessEvidence::OdinCorrelated { .. } => (
|
||
Some(
|
||
required(&transaction.latest_odin_observation, "latest Odin receipt")?
|
||
.clone(),
|
||
),
|
||
Some(required(&odin_authority, "Odin authority")?.clone()),
|
||
),
|
||
ReadinessEvidence::RouteProof { .. } => (None, None),
|
||
};
|
||
let routing = required(&transaction.routing, "route disposition")?.clone();
|
||
let route_supervision = matches!(&routing, RoutingEvidence::Promoted { .. })
|
||
.then(RouteSupervisionState::default);
|
||
let generation = Self {
|
||
schema_version: ADMITTED_GENERATION_SCHEMA.into(),
|
||
target: transaction.target.clone(),
|
||
generation_id: format!("generation-{}", transaction.transaction_id),
|
||
command_id: transaction.command_id.clone(),
|
||
transaction_id: transaction.transaction_id.clone(),
|
||
admitted_at_unix_millis: now,
|
||
plan: required(&transaction.plan, "transaction plan")?.clone(),
|
||
sealed_release: required(&transaction.sealed_release, "sealed release")?.clone(),
|
||
installed_release: required(&transaction.installed_release, "installed release")?
|
||
.clone(),
|
||
expected: required(&transaction.expected, "Expected projection")?.clone(),
|
||
activation: required(&transaction.activation, "activation")?.clone(),
|
||
workload: required(&transaction.workload, "workload")?.clone(),
|
||
leasing: required(&transaction.leasing, "lease disposition")?.clone(),
|
||
ready,
|
||
latest_odin_observation,
|
||
routing,
|
||
odin_authority,
|
||
odin_publisher_sequence_cursor: transaction.odin_publisher_sequence_cursor,
|
||
route_supervision,
|
||
last_error: None,
|
||
};
|
||
generation.validate()?;
|
||
Ok(generation)
|
||
}
|
||
|
||
fn validate(&self) -> Result<()> {
|
||
ensure!(
|
||
self.schema_version == ADMITTED_GENERATION_SCHEMA,
|
||
"admitted generation schema is unsupported"
|
||
);
|
||
require_id(&self.target, "admitted target")?;
|
||
require_id(&self.generation_id, "admitted generation id")?;
|
||
require_id(&self.command_id, "admitted command id")?;
|
||
require_id(&self.transaction_id, "admitted transaction id")?;
|
||
ensure!(self.admitted_at_unix_millis > 0, "admission has no time");
|
||
self.plan.validate()?;
|
||
self.sealed_release.validate_against(&self.plan)?;
|
||
self.expected.validate()?;
|
||
self.activation.validate()?;
|
||
self.ready.validate_shape()?;
|
||
match (
|
||
&self.ready,
|
||
&self.latest_odin_observation,
|
||
&self.odin_authority,
|
||
) {
|
||
(ReadinessEvidence::OdinCorrelated { evidence }, Some(latest), Some(authority)) => {
|
||
latest.validate_shape()?;
|
||
authority.validate()?;
|
||
ensure!(
|
||
evidence.publisher_sequence <= self.odin_publisher_sequence_cursor
|
||
&& latest.publisher_sequence == self.odin_publisher_sequence_cursor,
|
||
"admitted generation evidence does not describe one incarnation"
|
||
);
|
||
}
|
||
(ReadinessEvidence::RouteProof { .. }, None, None) => ensure!(
|
||
ReadinessClass::of(&self.expected).is_ok_and(|class| class == ReadinessClass::RouteProof),
|
||
"route-proof Ready evidence for a target that is not route-proof"
|
||
),
|
||
_ => bail!("admitted readiness evidence and Odin receipts disagree"),
|
||
}
|
||
ensure!(
|
||
self.target == self.expected.target
|
||
&& self.expected.plan_id == self.plan.plan_id
|
||
&& self.expected.sealed_release_id == self.sealed_release.sealed_release_id
|
||
&& self.activation.expected_projection_sha256
|
||
== self.expected.canonical_sha256()?
|
||
&& self.activation.runtime_instance_id == self.workload.runtime_instance_id(),
|
||
"admitted generation evidence does not describe one incarnation"
|
||
);
|
||
ensure!(
|
||
self.expected.write_lease_required == self.leasing.lease().is_some(),
|
||
"admitted write-lease disposition differs from Expected"
|
||
);
|
||
ensure!(
|
||
self.expected.route.is_some() == self.routing.observation().is_some(),
|
||
"admitted route disposition differs from Expected"
|
||
);
|
||
if let Some(route) = self.routing.observation() {
|
||
route.validate()?;
|
||
ensure!(
|
||
route.runtime_instance_id == self.activation.runtime_instance_id,
|
||
"admitted route observation belongs to another runtime instance"
|
||
);
|
||
}
|
||
ensure!(
|
||
self.route_supervision.is_some()
|
||
== matches!(&self.routing, RoutingEvidence::Promoted { .. }),
|
||
"route supervision state exists exactly for a promoted route"
|
||
);
|
||
if let Some(state) = &self.route_supervision {
|
||
state.validate()?;
|
||
}
|
||
Ok(())
|
||
}
|
||
}
|
||
|
||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||
struct RuntimeOptions {
|
||
state_store: PathBuf,
|
||
bindings_dir: PathBuf,
|
||
source_root: PathBuf,
|
||
staging_root: PathBuf,
|
||
topology_store: PathBuf,
|
||
/// Operator incidents, published world-readable beside the topology
|
||
/// projection. Not part of the control store: see `incident`.
|
||
incident_store: PathBuf,
|
||
odin_correlation_store: PathBuf,
|
||
odin_trust_anchor: PathBuf,
|
||
idunn_identity_store: PathBuf,
|
||
deployment_brake_operator_anchor: PathBuf,
|
||
source_identity: Option<ProcessIdentity>,
|
||
topology_maximum_age_millis: u64,
|
||
topology_maximum_future_skew_millis: u64,
|
||
poll_millis: u64,
|
||
/// Where host actuators dial in. `None` means no host-actuator workload
|
||
/// can be served; a binding that names one then fails at its first
|
||
/// driver call, not at startup, so an Idunn without hosts is unchanged.
|
||
host_actuator_bind: Option<SocketAddr>,
|
||
/// Where route drivers find nginx, systemctl, ufw and systemd-run, and
|
||
/// their scratch directory. Every route driver is built from this.
|
||
route_actuators: RouteActuators,
|
||
}
|
||
|
||
impl Default for RuntimeOptions {
|
||
fn default() -> Self {
|
||
Self {
|
||
state_store: PathBuf::from("/var/lib/gamecult/idunn/control.cc"),
|
||
bindings_dir: PathBuf::from("/etc/gamecult/idunn/bindings"),
|
||
source_root: PathBuf::from("/var/lib/gamecult/idunn/sources"),
|
||
staging_root: PathBuf::from("/var/lib/gamecult/idunn/staging"),
|
||
topology_store: PathBuf::from("/var/lib/gamecult/idunn/topology.cc"),
|
||
incident_store: PathBuf::from("/var/lib/gamecult/idunn-projection/incidents.cc"),
|
||
odin_correlation_store: PathBuf::from(
|
||
"/var/lib/gamecult/odin/idunn-runtime-topology.cc",
|
||
),
|
||
odin_trust_anchor: PathBuf::from("/etc/gamecult/idunn/odin-topology-anchor.cc"),
|
||
idunn_identity_store: PathBuf::from(
|
||
"/var/lib/gamecult/idunn/idunn-service-identity.cc",
|
||
),
|
||
deployment_brake_operator_anchor: PathBuf::from(
|
||
"/etc/gamecult/idunn/deployment-brake-operator-anchor.cc",
|
||
),
|
||
source_identity: None,
|
||
topology_maximum_age_millis: DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS,
|
||
topology_maximum_future_skew_millis: DEFAULT_TOPOLOGY_MAXIMUM_FUTURE_SKEW_MILLIS,
|
||
poll_millis: 500,
|
||
host_actuator_bind: None,
|
||
route_actuators: RouteActuators::default(),
|
||
}
|
||
}
|
||
}
|
||
|
||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||
enum Command {
|
||
Serve(RuntimeOptions),
|
||
Up {
|
||
selector: String,
|
||
requested_by: String,
|
||
state_store: PathBuf,
|
||
wait: bool,
|
||
timeout_seconds: u64,
|
||
},
|
||
Status {
|
||
state_store: PathBuf,
|
||
incident_store: PathBuf,
|
||
command_id: Option<String>,
|
||
},
|
||
Cancel {
|
||
state_store: PathBuf,
|
||
command_id: String,
|
||
requested_by: String,
|
||
},
|
||
Expire {
|
||
state_store: PathBuf,
|
||
command_id: String,
|
||
requested_by: String,
|
||
},
|
||
Validate {
|
||
recipe: PathBuf,
|
||
binding: Option<PathBuf>,
|
||
},
|
||
}
|
||
|
||
pub fn run(args: impl Iterator<Item = String>) -> Result<()> {
|
||
match parse(args)? {
|
||
Command::Serve(options) => serve(options),
|
||
Command::Up {
|
||
selector,
|
||
requested_by,
|
||
state_store,
|
||
wait,
|
||
timeout_seconds,
|
||
} => submit(
|
||
&state_store,
|
||
&selector,
|
||
&requested_by,
|
||
wait,
|
||
timeout_seconds,
|
||
),
|
||
Command::Status {
|
||
state_store,
|
||
incident_store,
|
||
command_id,
|
||
} => status(
|
||
&mut std::io::stdout().lock(),
|
||
&state_store,
|
||
&incident_store,
|
||
command_id.as_deref(),
|
||
),
|
||
Command::Cancel {
|
||
state_store,
|
||
command_id,
|
||
requested_by,
|
||
} => cancel(&state_store, &command_id, &requested_by),
|
||
Command::Expire {
|
||
state_store,
|
||
command_id,
|
||
requested_by,
|
||
} => expire(&state_store, &command_id, &requested_by),
|
||
Command::Validate { recipe, binding } => validate(&recipe, binding.as_deref()),
|
||
}
|
||
}
|
||
|
||
/// Offline admission check for authoring. Reads a recipe, optionally reads a
|
||
/// binding, and runs the same `admit` the daemon runs at load. It opens no
|
||
/// store, contacts no host, and actuates nothing, so it is safe to run against
|
||
/// a candidate binding before installing it.
|
||
fn validate(recipe_path: &Path, binding_path: Option<&Path>) -> Result<()> {
|
||
let recipe_text = fs::read_to_string(recipe_path)
|
||
.with_context(|| format!("reading recipe {}", recipe_path.display()))?;
|
||
let recipe = crate::deployment::TargetDeclaration::parse(&recipe_text)
|
||
.with_context(|| format!("validating recipe {}", recipe_path.display()))?;
|
||
println!(
|
||
"recipe ok: {} declares target {}",
|
||
recipe_path.display(),
|
||
recipe.target
|
||
);
|
||
|
||
let Some(binding_path) = binding_path else {
|
||
return Ok(());
|
||
};
|
||
let binding_text = fs::read_to_string(binding_path)
|
||
.with_context(|| format!("reading operator binding {}", binding_path.display()))?;
|
||
let binding = OperatorBinding::parse(&binding_text)
|
||
.with_context(|| format!("validating operator binding {}", binding_path.display()))?;
|
||
ensure!(
|
||
binding.target == recipe.target,
|
||
"operator binding targets {} but the recipe declares {}",
|
||
binding.target,
|
||
recipe.target
|
||
);
|
||
binding
|
||
.admit(&recipe)
|
||
.with_context(|| format!("admitting {} against its recipe", binding.target))?;
|
||
println!(
|
||
"binding ok: {} admits target {}",
|
||
binding_path.display(),
|
||
binding.target
|
||
);
|
||
Ok(())
|
||
}
|
||
|
||
fn parse(args: impl Iterator<Item = String>) -> Result<Command> {
|
||
let mut args = args.peekable();
|
||
let command = args.next().ok_or_else(|| anyhow!(usage()))?;
|
||
match command.as_str() {
|
||
"serve" => parse_serve(args),
|
||
"up" => parse_up(args),
|
||
"status" => parse_status(args),
|
||
"cancel" => parse_cancel(args),
|
||
"expire" => parse_expire(args),
|
||
"validate" => parse_validate(args),
|
||
"--help" | "-h" | "help" => bail!(usage()),
|
||
_ => bail!("unknown Idunn command {command:?}\n\n{}", usage()),
|
||
}
|
||
}
|
||
|
||
fn parse_serve(mut args: impl Iterator<Item = String>) -> Result<Command> {
|
||
let mut options = RuntimeOptions::default();
|
||
let mut source_uid = None;
|
||
let mut source_gid = None;
|
||
while let Some(argument) = args.next() {
|
||
match argument.as_str() {
|
||
"--state-store" => options.state_store = path_value(&mut args, &argument)?,
|
||
"--bindings-dir" => options.bindings_dir = path_value(&mut args, &argument)?,
|
||
"--source-root" => options.source_root = path_value(&mut args, &argument)?,
|
||
"--staging-root" => options.staging_root = path_value(&mut args, &argument)?,
|
||
"--topology-store" => options.topology_store = path_value(&mut args, &argument)?,
|
||
"--incident-store" => options.incident_store = path_value(&mut args, &argument)?,
|
||
"--odin-correlation-store" => {
|
||
options.odin_correlation_store = path_value(&mut args, &argument)?
|
||
}
|
||
"--odin-trust-anchor" => options.odin_trust_anchor = path_value(&mut args, &argument)?,
|
||
"--idunn-identity-store" => {
|
||
options.idunn_identity_store = path_value(&mut args, &argument)?
|
||
}
|
||
"--deployment-brake-operator-anchor" => {
|
||
options.deployment_brake_operator_anchor = path_value(&mut args, &argument)?
|
||
}
|
||
"--topology-maximum-age-millis" => {
|
||
options.topology_maximum_age_millis = u64_value(&mut args, &argument)?
|
||
}
|
||
"--topology-maximum-future-skew-millis" => {
|
||
options.topology_maximum_future_skew_millis = u64_value(&mut args, &argument)?
|
||
}
|
||
"--source-uid" => source_uid = Some(u32_value(&mut args, &argument)?),
|
||
"--source-gid" => source_gid = Some(u32_value(&mut args, &argument)?),
|
||
"--poll-millis" => options.poll_millis = u64_value(&mut args, &argument)?,
|
||
"--host-actuator-bind" => {
|
||
let value = string_value(&mut args, &argument)?;
|
||
options.host_actuator_bind = Some(
|
||
value
|
||
.parse()
|
||
.with_context(|| format!("{argument} is not a socket address"))?,
|
||
);
|
||
}
|
||
"--help" | "-h" => bail!(usage()),
|
||
_ => bail!("unknown Idunn serve option {argument:?}"),
|
||
}
|
||
}
|
||
ensure!(
|
||
options.poll_millis > 0 && options.topology_maximum_age_millis > 0,
|
||
"poll and topology maximum age must be positive"
|
||
);
|
||
options.source_identity = match (source_uid, source_gid) {
|
||
(Some(uid), Some(gid)) => {
|
||
ensure!(
|
||
uid > 0 && gid > 0,
|
||
"source UID and GID must be unprivileged"
|
||
);
|
||
Some(ProcessIdentity { uid, gid })
|
||
}
|
||
(None, None) => None,
|
||
_ => bail!("--source-uid and --source-gid must be supplied together"),
|
||
};
|
||
Ok(Command::Serve(options))
|
||
}
|
||
|
||
fn parse_up(mut args: impl Iterator<Item = String>) -> Result<Command> {
|
||
let selector = args
|
||
.next()
|
||
.ok_or_else(|| anyhow!("idunn up requires a service or profile selector"))?;
|
||
require_selector(&selector)?;
|
||
let mut state_store = RuntimeOptions::default().state_store;
|
||
let mut requested_by = env::var("SUDO_USER")
|
||
.or_else(|_| env::var("USER"))
|
||
.or_else(|_| env::var("USERNAME"))
|
||
.unwrap_or_else(|_| "operator".into());
|
||
let mut wait = true;
|
||
let mut timeout_seconds = 1800;
|
||
while let Some(argument) = args.next() {
|
||
match argument.as_str() {
|
||
"--state-store" => state_store = path_value(&mut args, &argument)?,
|
||
"--requested-by" => requested_by = string_value(&mut args, &argument)?,
|
||
"--no-wait" => wait = false,
|
||
"--timeout-seconds" => timeout_seconds = u64_value(&mut args, &argument)?,
|
||
"--help" | "-h" => bail!(usage()),
|
||
_ => bail!("unknown Idunn up option {argument:?}"),
|
||
}
|
||
}
|
||
ensure!(timeout_seconds > 0, "--timeout-seconds must be positive");
|
||
require_value(&requested_by, "deployment requester")?;
|
||
Ok(Command::Up {
|
||
selector,
|
||
requested_by,
|
||
state_store,
|
||
wait,
|
||
timeout_seconds,
|
||
})
|
||
}
|
||
|
||
fn parse_status(mut args: impl Iterator<Item = String>) -> Result<Command> {
|
||
let defaults = RuntimeOptions::default();
|
||
let mut state_store = defaults.state_store;
|
||
let mut incident_store = defaults.incident_store;
|
||
let mut command_id = None;
|
||
while let Some(argument) = args.next() {
|
||
match argument.as_str() {
|
||
"--state-store" => state_store = path_value(&mut args, &argument)?,
|
||
"--incident-store" => incident_store = path_value(&mut args, &argument)?,
|
||
"--command" => command_id = Some(string_value(&mut args, &argument)?),
|
||
"--help" | "-h" => bail!(usage()),
|
||
_ => bail!("unknown Idunn status option {argument:?}"),
|
||
}
|
||
}
|
||
Ok(Command::Status {
|
||
state_store,
|
||
incident_store,
|
||
command_id,
|
||
})
|
||
}
|
||
|
||
fn parse_expire(mut args: impl Iterator<Item = String>) -> Result<Command> {
|
||
let command_id = args
|
||
.next()
|
||
.ok_or_else(|| anyhow!("idunn expire requires a command id"))?;
|
||
require_id(&command_id, "command id")?;
|
||
let mut state_store = RuntimeOptions::default().state_store;
|
||
let mut requested_by = env::var("SUDO_USER")
|
||
.or_else(|_| env::var("USER"))
|
||
.or_else(|_| env::var("USERNAME"))
|
||
.unwrap_or_else(|_| "operator".into());
|
||
while let Some(argument) = args.next() {
|
||
match argument.as_str() {
|
||
"--state-store" => state_store = path_value(&mut args, &argument)?,
|
||
"--requested-by" => requested_by = string_value(&mut args, &argument)?,
|
||
"--help" | "-h" => bail!(usage()),
|
||
_ => bail!("unknown Idunn expire option {argument:?}"),
|
||
}
|
||
}
|
||
require_value(&requested_by, "expiry requester")?;
|
||
Ok(Command::Expire {
|
||
state_store,
|
||
command_id,
|
||
requested_by,
|
||
})
|
||
}
|
||
|
||
/// Ask the daemon to end a live transaction's post-fencing phase now. This
|
||
/// writes a request and nothing else: the daemon's deadline resolver ends the
|
||
/// phase exactly as it ends one whose deadline has passed.
|
||
fn expire(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> {
|
||
let snapshot = ControlSnapshot::read(store_path)?;
|
||
let mut live = snapshot
|
||
.transactions
|
||
.iter()
|
||
.filter(|stored| stored.value.command_id == command_id && !stored.value.is_terminal());
|
||
let current = live
|
||
.next()
|
||
.context("command has no live transaction to expire")?;
|
||
ensure!(
|
||
live.next().is_none(),
|
||
"command has multiple live transactions"
|
||
);
|
||
ensure!(
|
||
current.value.expirable(),
|
||
"{}",
|
||
if current.value.command_kind == CommandKind::Continuity && current.value.phase >= DeploymentPhase::Fencing && current.value.post_fencing_abort.is_none() {
|
||
format!(
|
||
"a continuity past Fencing retries forward and cannot be expired; to restart it, {} and Idunn aborts it as dead",
|
||
current.value.manual_exit()
|
||
)
|
||
} else {
|
||
"only a live transaction in Warming through Committing, and not already aborting, can be expired; before Warming, use idunn cancel".to_owned()
|
||
}
|
||
);
|
||
let now = now_millis()?;
|
||
let request = ExpiryRequest {
|
||
schema_version: EXPIRY_REQUEST_SCHEMA.into(),
|
||
command_id: command_id.to_owned(),
|
||
requested_by: requested_by.to_owned(),
|
||
requested_at_unix_millis: now,
|
||
};
|
||
ensure!(
|
||
SingleFileMessagePackBackingStore::new(store_path)
|
||
.insert_entry_if_absent(expiry_request_envelope(&request, now)?)?,
|
||
"an expiry request for this command is already waiting"
|
||
);
|
||
println!("{command_id} expiry requested; Idunn ends its phase as an expired deadline");
|
||
Ok(())
|
||
}
|
||
|
||
fn parse_cancel(mut args: impl Iterator<Item = String>) -> Result<Command> {
|
||
let command_id = args
|
||
.next()
|
||
.ok_or_else(|| anyhow!("idunn cancel requires a deployment command id"))?;
|
||
require_id(&command_id, "deployment command id")?;
|
||
let mut state_store = RuntimeOptions::default().state_store;
|
||
let mut requested_by = env::var("SUDO_USER")
|
||
.or_else(|_| env::var("USER"))
|
||
.or_else(|_| env::var("USERNAME"))
|
||
.unwrap_or_else(|_| "operator".into());
|
||
while let Some(argument) = args.next() {
|
||
match argument.as_str() {
|
||
"--state-store" => state_store = path_value(&mut args, &argument)?,
|
||
"--requested-by" => requested_by = string_value(&mut args, &argument)?,
|
||
"--help" | "-h" => bail!(usage()),
|
||
_ => bail!("unknown Idunn cancel option {argument:?}"),
|
||
}
|
||
}
|
||
require_value(&requested_by, "cancellation requester")?;
|
||
Ok(Command::Cancel {
|
||
state_store,
|
||
command_id,
|
||
requested_by,
|
||
})
|
||
}
|
||
|
||
/// Withdraw queued work, a held deployment that has not fenced, or ask Idunn to
|
||
/// clean up a live stateless candidate. Stateful candidates past fencing cannot be cancelled because their prior
|
||
/// writer may already be stopped and its state boundary needs explicit repair.
|
||
fn cancel(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> {
|
||
let snapshot = ControlSnapshot::read(store_path)?;
|
||
let command = snapshot
|
||
.commands
|
||
.iter()
|
||
.find(|stored| stored.value.command_id == command_id)
|
||
.context("deployment command is unknown or already retired")?;
|
||
ensure!(
|
||
command.value.kind == CommandKind::Deploy,
|
||
"only deployment commands can be cancelled; continuity is Idunn's own"
|
||
);
|
||
let mut live = snapshot
|
||
.transactions
|
||
.iter()
|
||
.filter(|stored| stored.value.command_id == command_id && !stored.value.is_terminal());
|
||
if let Some(current) = live.next() {
|
||
ensure!(
|
||
live.next().is_none(),
|
||
"deployment command has multiple live transactions"
|
||
);
|
||
ensure!(
|
||
cancel_is_safe_for_live_stateless(¤t.value)
|
||
|| cancel_is_safe_for_held_prefence(¤t.value),
|
||
"live deployment can only be cancelled before fencing when Idunn holds it, or after fencing when its candidate is stateless"
|
||
);
|
||
let error = format!("cancelled by {requested_by}");
|
||
let mut next = current.value.clone();
|
||
if cancel_is_safe_for_held_prefence(&next) {
|
||
next.pre_fencing_abort = Some(pre_fencing_abort_intent(&next, &error));
|
||
} else {
|
||
next.post_fencing_abort = Some(post_fencing_abort_intent(&next, &error));
|
||
}
|
||
next.last_error = Some(bounded_detail(&error));
|
||
next.updated_at_unix_millis = now_millis()?;
|
||
replace_transaction(store_path, current, &next)?;
|
||
println!("{command_id} cancellation requested; Idunn is performing exact candidate cleanup");
|
||
return Ok(());
|
||
}
|
||
let now = now_millis()?;
|
||
let refusal = DeploymentTransaction::rejected(
|
||
&command.value,
|
||
anyhow!("cancelled by {requested_by} before freezing"),
|
||
now,
|
||
)?;
|
||
let envelope = transaction_envelope(&refusal, now)?;
|
||
ensure!(
|
||
SingleFileMessagePackBackingStore::new(store_path).compare_exchange(
|
||
&[
|
||
CultCacheExpectedEnvelope {
|
||
r#type: DeploymentCommand::TYPE.into(),
|
||
key: command_id.to_owned(),
|
||
current: Some(command.envelope.clone()),
|
||
},
|
||
CultCacheExpectedEnvelope {
|
||
r#type: DeploymentTransaction::TYPE.into(),
|
||
key: refusal.transaction_id.clone(),
|
||
current: None,
|
||
},
|
||
],
|
||
std::slice::from_ref(&envelope),
|
||
)?,
|
||
"deployment command changed before it could be cancelled"
|
||
);
|
||
archive_terminal_transaction(store_path, &envelope)?;
|
||
println!("{command_id} cancelled");
|
||
Ok(())
|
||
}
|
||
|
||
/// A deployment Idunn holds before it fenced anything has touched nothing of
|
||
/// the incumbent, so an operator may withdraw it. Idunn never does so itself.
|
||
fn cancel_is_safe_for_held_prefence(transaction: &DeploymentTransaction) -> bool {
|
||
transaction.command_kind == CommandKind::Deploy
|
||
&& transaction.phase < DeploymentPhase::Fencing
|
||
&& transaction.held_disagreement().is_some()
|
||
}
|
||
|
||
fn cancel_is_safe_for_live_stateless(transaction: &DeploymentTransaction) -> bool {
|
||
transaction.command_kind == CommandKind::Deploy
|
||
&& transaction.phase >= DeploymentPhase::Fencing
|
||
&& matches!(
|
||
transaction.fencing.as_ref(),
|
||
Some(FencingEvidence::SkippedStateless)
|
||
)
|
||
&& transaction.completion.is_none()
|
||
&& transaction.pre_fencing_abort.is_none()
|
||
&& transaction.post_fencing_abort.is_none()
|
||
&& transaction.post_commit_cleanup.is_none()
|
||
}
|
||
|
||
fn parse_validate(mut args: impl Iterator<Item = String>) -> Result<Command> {
|
||
let mut recipe = None;
|
||
let mut binding = None;
|
||
while let Some(argument) = args.next() {
|
||
match argument.as_str() {
|
||
"--recipe" => recipe = Some(path_value(&mut args, &argument)?),
|
||
"--binding" => binding = Some(path_value(&mut args, &argument)?),
|
||
"--help" | "-h" => bail!(usage()),
|
||
_ => bail!("unknown Idunn validate option {argument:?}"),
|
||
}
|
||
}
|
||
let recipe = recipe.ok_or_else(|| anyhow!("validate requires --recipe"))?;
|
||
Ok(Command::Validate { recipe, binding })
|
||
}
|
||
|
||
#[derive(Clone)]
|
||
struct Stored<T> {
|
||
envelope: CultCacheEnvelope,
|
||
value: T,
|
||
}
|
||
|
||
#[derive(Default)]
|
||
struct ControlSnapshot {
|
||
commands: Vec<Stored<DeploymentCommand>>,
|
||
transactions: Vec<Stored<DeploymentTransaction>>,
|
||
admitted: Vec<Stored<AdmittedGeneration>>,
|
||
targets: Vec<Stored<TargetSupervision>>,
|
||
expiry_requests: Vec<Stored<ExpiryRequest>>,
|
||
}
|
||
|
||
impl ControlSnapshot {
|
||
fn read(path: &Path) -> Result<Self> {
|
||
if !path.exists() {
|
||
return Ok(Self::default());
|
||
}
|
||
let mut snapshot = Self::default();
|
||
for envelope in SingleFileMessagePackBackingStore::new(path)
|
||
.pull_all_read_only_snapshot()
|
||
.context("reading Idunn control snapshot")?
|
||
{
|
||
match envelope.r#type.as_str() {
|
||
DeploymentCommand::TYPE => {
|
||
ensure!(
|
||
envelope.schema_id.as_deref() == Some(DEPLOYMENT_COMMAND_SCHEMA),
|
||
"Idunn control store contains an unsupported command"
|
||
);
|
||
let value: DeploymentCommand = decode_record(&envelope)?;
|
||
value.validate()?;
|
||
ensure!(
|
||
envelope.key == value.command_id,
|
||
"deployment command key differs from its identity"
|
||
);
|
||
snapshot.commands.push(Stored { envelope, value });
|
||
}
|
||
DeploymentTransaction::TYPE => {
|
||
let value = read_transaction_record(&envelope)?;
|
||
ensure!(
|
||
envelope.key == value.transaction_id,
|
||
"deployment transaction key differs from its identity"
|
||
);
|
||
snapshot.transactions.push(Stored { envelope, value });
|
||
}
|
||
AdmittedGeneration::TYPE => {
|
||
let value = read_generation_record(&envelope)?;
|
||
ensure!(
|
||
envelope.key == value.target,
|
||
"admitted generation key is not its target"
|
||
);
|
||
snapshot.admitted.push(Stored { envelope, value });
|
||
}
|
||
TargetSupervision::TYPE => {
|
||
let value = read_supervision_record(&envelope)?;
|
||
snapshot.targets.push(Stored { envelope, value });
|
||
}
|
||
ExpiryRequest::TYPE => {
|
||
ensure!(
|
||
envelope.schema_id.as_deref() == Some(EXPIRY_REQUEST_SCHEMA),
|
||
"Idunn control store contains an unsupported expiry request"
|
||
);
|
||
let value: ExpiryRequest = decode_record(&envelope)?;
|
||
value.validate()?;
|
||
ensure!(
|
||
envelope.key == value.command_id,
|
||
"expiry request key is not its command"
|
||
);
|
||
snapshot.expiry_requests.push(Stored { envelope, value });
|
||
}
|
||
_ => bail!("Idunn control store contains a foreign document"),
|
||
}
|
||
}
|
||
snapshot.validate_relations()?;
|
||
Ok(snapshot)
|
||
}
|
||
|
||
fn validate_relations(&self) -> Result<()> {
|
||
let command_ids = self
|
||
.commands
|
||
.iter()
|
||
.map(|stored| stored.value.command_id.as_str())
|
||
.collect::<BTreeSet<_>>();
|
||
ensure!(
|
||
command_ids.len() == self.commands.len(),
|
||
"Idunn control store contains duplicate commands"
|
||
);
|
||
let transaction_ids = self
|
||
.transactions
|
||
.iter()
|
||
.map(|stored| stored.value.transaction_id.as_str())
|
||
.collect::<BTreeSet<_>>();
|
||
ensure!(
|
||
transaction_ids.len() == self.transactions.len(),
|
||
"Idunn control store contains duplicate transactions"
|
||
);
|
||
let admitted_targets = self
|
||
.admitted
|
||
.iter()
|
||
.map(|stored| stored.value.target.as_str())
|
||
.collect::<BTreeSet<_>>();
|
||
ensure!(
|
||
admitted_targets.len() == self.admitted.len(),
|
||
"Idunn control store contains duplicate current generations"
|
||
);
|
||
ensure!(
|
||
self.targets
|
||
.iter()
|
||
.map(|stored| stored.value.target.as_str())
|
||
.collect::<BTreeSet<_>>()
|
||
.len()
|
||
== self.targets.len(),
|
||
"Idunn control store contains duplicate target supervision records"
|
||
);
|
||
for transaction in &self.transactions {
|
||
let command = self
|
||
.commands
|
||
.iter()
|
||
.find(|candidate| candidate.value.command_id == transaction.value.command_id)
|
||
.context("deployment transaction has no immutable command")?;
|
||
ensure!(
|
||
command.value.kind == transaction.value.command_kind,
|
||
"transaction kind differs from its immutable command"
|
||
);
|
||
}
|
||
// Per target: one owner of the incumbent slot, one live Deploy, one
|
||
// live Continuity. A continuity may sit beside a deployment that has
|
||
// not yet taken the target (handover-phase-split).
|
||
let mut owners = BTreeSet::new();
|
||
let mut live_deploys = BTreeSet::new();
|
||
let mut live_continuities = BTreeSet::new();
|
||
for transaction in self
|
||
.transactions
|
||
.iter()
|
||
.filter(|stored| stored.value.is_live())
|
||
{
|
||
let target = transaction.value.target.as_str();
|
||
ensure!(
|
||
!transaction.value.owns_target() || owners.insert(target),
|
||
"multiple transactions own the incumbent slot of one target"
|
||
);
|
||
let (seen, kind) = match transaction.value.command_kind {
|
||
CommandKind::Deploy => (&mut live_deploys, "deployments"),
|
||
CommandKind::Continuity => (&mut live_continuities, "continuity transactions"),
|
||
};
|
||
ensure!(seen.insert(target), "multiple live {kind} for one target");
|
||
if let Some(fencing) = transaction
|
||
.value
|
||
.fencing
|
||
.as_ref()
|
||
.filter(|_| transaction.value.phase < DeploymentPhase::Complete)
|
||
{
|
||
let incumbent = self.admitted_for(&transaction.value.target);
|
||
let incumbent_matches =
|
||
match (&transaction.value.incumbent_generation_id, incumbent) {
|
||
(None, None) => true,
|
||
(Some(expected), Some(current)) => current.value.generation_id == *expected,
|
||
_ => false,
|
||
};
|
||
ensure!(
|
||
incumbent_matches,
|
||
"fenced transaction incumbent differs from current admission"
|
||
);
|
||
let incumbent_lease_sha256 = incumbent
|
||
.and_then(|stored| stored.value.leasing.lease())
|
||
.map(IdunnProcessWriteLeaseRecord::canonical_sha256)
|
||
.transpose()?;
|
||
match fencing {
|
||
FencingEvidence::SkippedStateless => ensure!(
|
||
incumbent_lease_sha256.is_none(),
|
||
"stateless fencing omitted the admitted incumbent lease"
|
||
),
|
||
FencingEvidence::Revoked {
|
||
incumbent_lease_sha256: recorded,
|
||
..
|
||
} => {
|
||
ensure!(
|
||
recorded == &incumbent_lease_sha256,
|
||
"fencing evidence names another incumbent lease"
|
||
);
|
||
ensure!(
|
||
incumbent_lease_sha256.is_some()
|
||
|| transaction
|
||
.value
|
||
.expected
|
||
.as_ref()
|
||
.is_some_and(|expected| expected.write_lease_required),
|
||
"stateless transaction did not use its explicit fencing skip"
|
||
);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
let mut authorizations = BTreeSet::new();
|
||
for authorization in self
|
||
.transactions
|
||
.iter()
|
||
.filter_map(|stored| stored.value.deployment_authorization.as_ref())
|
||
{
|
||
ensure!(
|
||
authorizations.insert(authorization.authorization_id.as_str()),
|
||
"deployment authorization was consumed more than once"
|
||
);
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
/// Every stored record that concerns `target`, as read: the generation, the
|
||
/// meters and its transactions. Two reads with equal stamps saw no write
|
||
/// for the target between them.
|
||
fn target_stamp(&self, target: &str) -> Vec<CultCacheEnvelope> {
|
||
let mut stamp = self
|
||
.admitted
|
||
.iter()
|
||
.filter(|stored| stored.value.target == target)
|
||
.map(|stored| stored.envelope.clone())
|
||
.chain(
|
||
self.targets
|
||
.iter()
|
||
.filter(|stored| stored.value.target == target)
|
||
.map(|stored| stored.envelope.clone()),
|
||
)
|
||
.chain(
|
||
self.transactions
|
||
.iter()
|
||
.filter(|stored| stored.value.target == target)
|
||
.map(|stored| stored.envelope.clone()),
|
||
)
|
||
.collect::<Vec<_>>();
|
||
stamp.sort_by(|a, b| (&a.r#type, &a.key).cmp(&(&b.r#type, &b.key)));
|
||
stamp
|
||
}
|
||
|
||
fn admitted_for(&self, target: &str) -> Option<&Stored<AdmittedGeneration>> {
|
||
self.admitted
|
||
.iter()
|
||
.find(|stored| stored.value.target == target)
|
||
}
|
||
|
||
/// The transaction, other than `except`, that holds the target's
|
||
/// incumbent slot or is still draining it. The one lookup supervision, a
|
||
/// waiting deployment and the handover share.
|
||
fn target_owner(&self, target: &str, except: Option<&str>) -> Option<&Stored<DeploymentTransaction>> {
|
||
self.on_target(target).find(|stored| {
|
||
Some(stored.value.transaction_id.as_str()) != except
|
||
&& (stored.value.owns_target() || stored.value.drains_incumbent())
|
||
})
|
||
}
|
||
|
||
/// Every transaction of the target, in whatever state.
|
||
fn on_target<'a>(
|
||
&'a self,
|
||
target: &str,
|
||
) -> impl Iterator<Item = &'a Stored<DeploymentTransaction>> + use<'a> {
|
||
let target = target.to_owned();
|
||
self.transactions
|
||
.iter()
|
||
.filter(move |stored| stored.value.target == target)
|
||
}
|
||
|
||
/// The candidate ports taken by admitted generations and by transactions
|
||
/// that are live or still draining; a finished one has released its port.
|
||
fn candidate_ports_in_use(&self) -> BTreeSet<u16> {
|
||
self.admitted
|
||
.iter()
|
||
.map(|stored| &stored.value.plan)
|
||
.chain(
|
||
self.transactions
|
||
.iter()
|
||
.filter(|stored| stored.value.is_live() || stored.value.drains_incumbent())
|
||
.filter_map(|stored| stored.value.plan.as_ref()),
|
||
)
|
||
.filter_map(|plan| plan.candidate_port)
|
||
.collect()
|
||
}
|
||
|
||
fn supervision_for(&self, target: &str) -> Option<&Stored<TargetSupervision>> {
|
||
self.targets
|
||
.iter()
|
||
.find(|stored| stored.value.target == target)
|
||
}
|
||
|
||
/// The target's meters, or empty ones: a target that was never metered has
|
||
/// used nothing.
|
||
fn supervision_or_new(&self, target: &str) -> TargetSupervision {
|
||
self.supervision_for(target)
|
||
.map_or_else(|| TargetSupervision::new(target), |stored| stored.value.clone())
|
||
}
|
||
|
||
/// The admitted generation of the target that provides
|
||
/// `odin.verse-rendezvous`: Odin is whichever target says so, not
|
||
/// whichever is named "odin".
|
||
fn admitted_odin(&self) -> Option<&Stored<AdmittedGeneration>> {
|
||
self.admitted.iter().find(|stored| {
|
||
ReadinessClass::of(&stored.value.expected)
|
||
.is_ok_and(|class| class == ReadinessClass::OdinSelf)
|
||
})
|
||
}
|
||
|
||
fn transaction_for_command(&self, command_id: &str) -> Vec<&DeploymentTransaction> {
|
||
let mut values = self
|
||
.transactions
|
||
.iter()
|
||
.filter(|stored| stored.value.command_id == command_id)
|
||
.map(|stored| &stored.value)
|
||
.collect::<Vec<_>>();
|
||
values.sort_by_key(|value| value.ordinal);
|
||
values
|
||
}
|
||
|
||
fn has_earlier_authority_sibling(&self, transaction: &DeploymentTransaction) -> bool {
|
||
self.transactions.iter().any(|stored| {
|
||
stored.value.is_live()
|
||
&& stored.value.command_id == transaction.command_id
|
||
&& stored.value.ordinal < transaction.ordinal
|
||
})
|
||
}
|
||
|
||
/// The highest Odin publisher sequence this target has ever *acted on*.
|
||
///
|
||
/// A transaction that failed contributes nothing. Its observations were
|
||
/// never built upon -- the deployment was abandoned and its projection
|
||
/// withdrawn -- so treating them as admitted history only raises the bar
|
||
/// for the next attempt. Odin does not reuse a sequence, so nothing is
|
||
/// weakened by forgetting the readings of an attempt that came to nothing;
|
||
/// what would be weakened is refusing every retry after a failure, which is
|
||
/// how a target ends up unable to deploy at all.
|
||
fn max_odin_sequence(&self, target: &str, signer_identity_id: &str) -> u64 {
|
||
let transaction_max = self
|
||
.transactions
|
||
.iter()
|
||
.filter_map(|stored| {
|
||
stored
|
||
.value
|
||
.latest_odin_observation
|
||
.as_ref()
|
||
.filter(|evidence| {
|
||
stored.value.target == target
|
||
&& evidence.signer_identity_id == signer_identity_id
|
||
&& !matches!(
|
||
stored.value.completion,
|
||
Some(TransactionCompletion::FailedBeforeFencing { .. })
|
||
| Some(TransactionCompletion::FailedAfterFencing { .. })
|
||
)
|
||
})
|
||
.map(|evidence| evidence.publisher_sequence)
|
||
})
|
||
.max()
|
||
.unwrap_or(0);
|
||
let admitted_max = self
|
||
.admitted
|
||
.iter()
|
||
.filter(|stored| {
|
||
stored.value.target == target
|
||
&& stored
|
||
.value
|
||
.odin_authority
|
||
.as_ref()
|
||
.is_some_and(|authority| authority.signer_identity_id == signer_identity_id)
|
||
})
|
||
.map(|stored| stored.value.odin_publisher_sequence_cursor)
|
||
.max()
|
||
.unwrap_or(0);
|
||
transaction_max.max(admitted_max)
|
||
}
|
||
}
|
||
|
||
pub(crate) fn decode_record<T>(envelope: &CultCacheEnvelope) -> Result<T>
|
||
where
|
||
T: for<'de> Deserialize<'de> + Serialize,
|
||
{
|
||
let value: T = rmp_serde::from_slice(&envelope.payload)?;
|
||
ensure!(
|
||
rmp_serde::to_vec(&value)? == envelope.payload,
|
||
"Idunn control store contains a noncanonical record"
|
||
);
|
||
Ok(value)
|
||
}
|
||
|
||
fn command_envelope(value: &DeploymentCommand, now: u64) -> Result<CultCacheEnvelope> {
|
||
value.validate()?;
|
||
typed_envelope(
|
||
&value.command_id,
|
||
DeploymentCommand::TYPE,
|
||
DEPLOYMENT_COMMAND_SCHEMA,
|
||
value,
|
||
now,
|
||
)
|
||
}
|
||
|
||
/// The transaction layout of v2 and v3 records: `ready` is a bare Odin receipt,
|
||
/// and there are no phase deadline or lease adoption slots. It exists only to
|
||
/// decode records written before v4. `migrate_control_store_to_current_schema`
|
||
/// rewrites every one in `control.cc` at boot, but `history.cc` is never
|
||
/// migrated or pruned, so this struct dies only when history is pruned or
|
||
/// migrated.
|
||
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||
#[cultcache(
|
||
type = "idunn.deployment_transaction",
|
||
schema = "idunn.deployment_transaction.v3"
|
||
)]
|
||
struct LegacyDeploymentTransaction {
|
||
#[cultcache(key = 0)]
|
||
schema_version: String,
|
||
#[cultcache(key = 1)]
|
||
transaction_id: String,
|
||
#[cultcache(key = 2)]
|
||
command_id: String,
|
||
#[cultcache(key = 3)]
|
||
command_kind: CommandKind,
|
||
#[cultcache(key = 4)]
|
||
target: String,
|
||
#[cultcache(key = 5)]
|
||
ordinal: u32,
|
||
#[cultcache(key = 6)]
|
||
phase: DeploymentPhase,
|
||
#[cultcache(key = 7)]
|
||
created_at_unix_millis: u64,
|
||
#[cultcache(key = 8)]
|
||
updated_at_unix_millis: u64,
|
||
#[cultcache(key = 9)]
|
||
incumbent_generation_id: Option<String>,
|
||
#[cultcache(key = 10)]
|
||
plan: Option<CompiledDeploymentPlan>,
|
||
#[cultcache(key = 11)]
|
||
frozen_source: Option<FrozenSourceReceipt>,
|
||
#[cultcache(key = 12)]
|
||
sealed_release: Option<SealedRelease>,
|
||
#[cultcache(key = 13)]
|
||
installed_release: Option<InstalledReleaseObservation>,
|
||
#[cultcache(key = 14)]
|
||
expected: Option<IdunnExpectedIncarnationRecord>,
|
||
#[cultcache(key = 15)]
|
||
expected_publication_sha256: Option<String>,
|
||
#[cultcache(key = 16)]
|
||
deployment_authorization: Option<DeploymentAuthorization>,
|
||
#[cultcache(key = 17)]
|
||
lifecycle_authorized_at_unix_millis: Option<u64>,
|
||
#[cultcache(key = 18)]
|
||
activation: Option<IdunnRuntimeActivationRecord>,
|
||
#[cultcache(key = 19)]
|
||
workload: Option<WorkloadObservation>,
|
||
#[cultcache(key = 20)]
|
||
activation_publication_sha256: Option<String>,
|
||
#[cultcache(key = 21)]
|
||
latest_odin_observation: Option<TopologyEvidence>,
|
||
#[cultcache(key = 22)]
|
||
warming: Option<WarmingEvidence>,
|
||
#[cultcache(key = 23)]
|
||
route_preflight: Option<RoutePreflightReceipt>,
|
||
#[cultcache(key = 24)]
|
||
isolation: Option<IsolationEvidence>,
|
||
#[cultcache(key = 25)]
|
||
fencing: Option<FencingEvidence>,
|
||
#[cultcache(key = 26)]
|
||
leasing: Option<LeasingEvidence>,
|
||
#[cultcache(key = 27)]
|
||
ready: Option<TopologyEvidence>,
|
||
#[cultcache(key = 28)]
|
||
routing: Option<RoutingEvidence>,
|
||
#[cultcache(key = 29)]
|
||
odin_publisher_sequence_cursor: u64,
|
||
#[cultcache(key = 30)]
|
||
last_error: Option<String>,
|
||
#[cultcache(key = 31)]
|
||
completion: Option<TransactionCompletion>,
|
||
#[cultcache(key = 32)]
|
||
pre_fencing_abort: Option<PreFencingAbort>,
|
||
#[cultcache(key = 33)]
|
||
post_commit_cleanup: Option<PostCommitCleanup>,
|
||
// A v2 record has no key 34: it predates post-fencing aborts.
|
||
#[cultcache(key = 34, default)]
|
||
post_fencing_abort: Option<PostFencingAbort>,
|
||
}
|
||
|
||
impl LegacyDeploymentTransaction {
|
||
/// Every legacy Ready receipt is an Odin receipt: no route-proof class
|
||
/// existed. The new slots start empty, which is what an older transaction
|
||
/// had.
|
||
fn into_current(self) -> DeploymentTransaction {
|
||
DeploymentTransaction {
|
||
schema_version: DEPLOYMENT_TRANSACTION_SCHEMA.into(),
|
||
transaction_id: self.transaction_id,
|
||
command_id: self.command_id,
|
||
command_kind: self.command_kind,
|
||
target: self.target,
|
||
ordinal: self.ordinal,
|
||
phase: self.phase,
|
||
created_at_unix_millis: self.created_at_unix_millis,
|
||
updated_at_unix_millis: self.updated_at_unix_millis,
|
||
incumbent_generation_id: self.incumbent_generation_id,
|
||
plan: self.plan,
|
||
frozen_source: self.frozen_source,
|
||
sealed_release: self.sealed_release,
|
||
installed_release: self.installed_release,
|
||
expected: self.expected,
|
||
expected_publication_sha256: self.expected_publication_sha256,
|
||
deployment_authorization: self.deployment_authorization,
|
||
lifecycle_authorized_at_unix_millis: self.lifecycle_authorized_at_unix_millis,
|
||
activation: self.activation,
|
||
workload: self.workload,
|
||
activation_publication_sha256: self.activation_publication_sha256,
|
||
latest_odin_observation: self.latest_odin_observation,
|
||
warming: self.warming,
|
||
route_preflight: self.route_preflight,
|
||
isolation: self.isolation,
|
||
fencing: self.fencing,
|
||
leasing: self.leasing,
|
||
ready: self
|
||
.ready
|
||
.map(|evidence| ReadinessEvidence::OdinCorrelated { evidence }),
|
||
routing: self.routing,
|
||
odin_publisher_sequence_cursor: self.odin_publisher_sequence_cursor,
|
||
last_error: self.last_error,
|
||
completion: self.completion,
|
||
pre_fencing_abort: self.pre_fencing_abort,
|
||
post_commit_cleanup: self.post_commit_cleanup,
|
||
post_fencing_abort: self.post_fencing_abort,
|
||
phase_deadline: None,
|
||
lease_adoption: None,
|
||
}
|
||
}
|
||
}
|
||
|
||
/// The v2 admitted-generation layout. Same lifetime rule as
|
||
/// `LegacyDeploymentTransaction`.
|
||
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||
#[cultcache(
|
||
type = "idunn.admitted_generation",
|
||
schema = "idunn.admitted_generation.v2"
|
||
)]
|
||
struct LegacyAdmittedGeneration {
|
||
#[cultcache(key = 0)]
|
||
schema_version: String,
|
||
#[cultcache(key = 1)]
|
||
target: String,
|
||
#[cultcache(key = 2)]
|
||
generation_id: String,
|
||
#[cultcache(key = 3)]
|
||
command_id: String,
|
||
#[cultcache(key = 4)]
|
||
transaction_id: String,
|
||
#[cultcache(key = 5)]
|
||
admitted_at_unix_millis: u64,
|
||
#[cultcache(key = 6)]
|
||
plan: CompiledDeploymentPlan,
|
||
#[cultcache(key = 7)]
|
||
sealed_release: SealedRelease,
|
||
#[cultcache(key = 8)]
|
||
installed_release: InstalledReleaseObservation,
|
||
#[cultcache(key = 9)]
|
||
expected: IdunnExpectedIncarnationRecord,
|
||
#[cultcache(key = 10)]
|
||
activation: IdunnRuntimeActivationRecord,
|
||
#[cultcache(key = 11)]
|
||
workload: WorkloadObservation,
|
||
#[cultcache(key = 12)]
|
||
leasing: LeasingEvidence,
|
||
#[cultcache(key = 13)]
|
||
ready: TopologyEvidence,
|
||
#[cultcache(key = 14)]
|
||
latest_odin_observation: TopologyEvidence,
|
||
#[cultcache(key = 15)]
|
||
routing: RoutingEvidence,
|
||
#[cultcache(key = 16)]
|
||
odin_authority: AdmittedOdinAuthority,
|
||
#[cultcache(key = 17)]
|
||
odin_publisher_sequence_cursor: u64,
|
||
#[cultcache(key = 18)]
|
||
route_repair_started_at_unix_millis: Option<u64>,
|
||
}
|
||
|
||
impl LegacyAdmittedGeneration {
|
||
/// Every v2 generation was admitted on Odin receipts, so it migrates as
|
||
/// Odin-correlated with those receipts unchanged. Route supervision state
|
||
/// starts empty for a promoted route. The v2 repair start decided nothing
|
||
/// after S1 and is not carried.
|
||
fn into_current(self) -> AdmittedGeneration {
|
||
let route_supervision = matches!(&self.routing, RoutingEvidence::Promoted { .. })
|
||
.then(RouteSupervisionState::default);
|
||
AdmittedGeneration {
|
||
schema_version: ADMITTED_GENERATION_SCHEMA.into(),
|
||
target: self.target,
|
||
generation_id: self.generation_id,
|
||
command_id: self.command_id,
|
||
transaction_id: self.transaction_id,
|
||
admitted_at_unix_millis: self.admitted_at_unix_millis,
|
||
plan: self.plan,
|
||
sealed_release: self.sealed_release,
|
||
installed_release: self.installed_release,
|
||
expected: self.expected,
|
||
activation: self.activation,
|
||
workload: self.workload,
|
||
leasing: self.leasing,
|
||
ready: ReadinessEvidence::OdinCorrelated {
|
||
evidence: self.ready,
|
||
},
|
||
latest_odin_observation: Some(self.latest_odin_observation),
|
||
routing: self.routing,
|
||
odin_authority: Some(self.odin_authority),
|
||
odin_publisher_sequence_cursor: self.odin_publisher_sequence_cursor,
|
||
route_supervision,
|
||
last_error: None,
|
||
}
|
||
}
|
||
}
|
||
|
||
/// The v3 admitted-generation layout. Its meters (`actuations`, the continuity
|
||
/// backoff, the repair start) are retired: no released Idunn ever wrote a
|
||
/// nonzero one, and a record that shows one is refused instead of lifted.
|
||
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||
#[cultcache(
|
||
type = "idunn.admitted_generation",
|
||
schema = "idunn.admitted_generation.v3"
|
||
)]
|
||
struct LegacyAdmittedGenerationV3 {
|
||
#[cultcache(key = 0)]
|
||
schema_version: String,
|
||
#[cultcache(key = 1)]
|
||
target: String,
|
||
#[cultcache(key = 2)]
|
||
generation_id: String,
|
||
#[cultcache(key = 3)]
|
||
command_id: String,
|
||
#[cultcache(key = 4)]
|
||
transaction_id: String,
|
||
#[cultcache(key = 5)]
|
||
admitted_at_unix_millis: u64,
|
||
#[cultcache(key = 6)]
|
||
plan: CompiledDeploymentPlan,
|
||
#[cultcache(key = 7)]
|
||
sealed_release: SealedRelease,
|
||
#[cultcache(key = 8)]
|
||
installed_release: InstalledReleaseObservation,
|
||
#[cultcache(key = 9)]
|
||
expected: IdunnExpectedIncarnationRecord,
|
||
#[cultcache(key = 10)]
|
||
activation: IdunnRuntimeActivationRecord,
|
||
#[cultcache(key = 11)]
|
||
workload: WorkloadObservation,
|
||
#[cultcache(key = 12)]
|
||
leasing: LeasingEvidence,
|
||
#[cultcache(key = 13)]
|
||
ready: ReadinessEvidence,
|
||
#[cultcache(key = 14)]
|
||
latest_odin_observation: Option<TopologyEvidence>,
|
||
#[cultcache(key = 15)]
|
||
routing: RoutingEvidence,
|
||
#[cultcache(key = 16)]
|
||
odin_authority: Option<AdmittedOdinAuthority>,
|
||
#[cultcache(key = 17)]
|
||
odin_publisher_sequence_cursor: u64,
|
||
#[cultcache(key = 18)]
|
||
route_repair_started_at_unix_millis: Option<u64>,
|
||
#[cultcache(key = 19)]
|
||
route_supervision: Option<LegacyRouteSupervisionStateV3>,
|
||
#[cultcache(key = 20)]
|
||
continuity_backoff: LegacyContinuityBackoffV3,
|
||
}
|
||
|
||
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(deny_unknown_fields)]
|
||
struct LegacyRouteSupervisionStateV3 {
|
||
last_challenge_at_unix_millis: Option<u64>,
|
||
consecutive_failures: u32,
|
||
next_challenge_at_unix_millis: Option<u64>,
|
||
degraded_since_unix_millis: Option<u64>,
|
||
actuations: LegacyActuationWindowV3,
|
||
}
|
||
|
||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(deny_unknown_fields)]
|
||
struct LegacyActuationWindowV3 {
|
||
window_started_at_unix_millis: u64,
|
||
count: u32,
|
||
}
|
||
|
||
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(deny_unknown_fields)]
|
||
struct LegacyContinuityBackoffV3 {
|
||
window_started_at_unix_millis: Option<u64>,
|
||
attempts: u32,
|
||
next_restart_at_unix_millis: Option<u64>,
|
||
}
|
||
|
||
impl LegacyAdmittedGenerationV3 {
|
||
/// Drops the retired meters. The lift creates no `TargetSupervision`: every
|
||
/// target starts unmetered, which is what no released binary ever changed.
|
||
fn into_current(self) -> Result<AdmittedGeneration> {
|
||
ensure!(
|
||
self.continuity_backoff.attempts == 0
|
||
&& self
|
||
.route_supervision
|
||
.as_ref()
|
||
.is_none_or(|state| state.actuations.count == 0),
|
||
"admitted generation of {} records route actuations or continuity restarts that no released Idunn wrote; refusing to drop them",
|
||
self.target
|
||
);
|
||
Ok(AdmittedGeneration {
|
||
schema_version: ADMITTED_GENERATION_SCHEMA.into(),
|
||
target: self.target,
|
||
generation_id: self.generation_id,
|
||
command_id: self.command_id,
|
||
transaction_id: self.transaction_id,
|
||
admitted_at_unix_millis: self.admitted_at_unix_millis,
|
||
plan: self.plan,
|
||
sealed_release: self.sealed_release,
|
||
installed_release: self.installed_release,
|
||
expected: self.expected,
|
||
activation: self.activation,
|
||
workload: self.workload,
|
||
leasing: self.leasing,
|
||
ready: self.ready,
|
||
latest_odin_observation: self.latest_odin_observation,
|
||
routing: self.routing,
|
||
odin_authority: self.odin_authority,
|
||
odin_publisher_sequence_cursor: self.odin_publisher_sequence_cursor,
|
||
route_supervision: self.route_supervision.map(|state| RouteSupervisionState {
|
||
last_challenge_at_unix_millis: state.last_challenge_at_unix_millis,
|
||
consecutive_failures: state.consecutive_failures,
|
||
next_challenge_at_unix_millis: state.next_challenge_at_unix_millis,
|
||
degraded_since_unix_millis: state.degraded_since_unix_millis,
|
||
}),
|
||
last_error: None,
|
||
})
|
||
}
|
||
}
|
||
|
||
/// Decode a v2 or v3 transaction and lift it to the current shape, without
|
||
/// validating it. History reads stop here; the control store validates.
|
||
fn lift_legacy_transaction(envelope: &CultCacheEnvelope) -> Result<DeploymentTransaction> {
|
||
let schema = match envelope.schema_id.as_deref() {
|
||
Some(schema @ (DEPLOYMENT_TRANSACTION_SCHEMA_V2 | DEPLOYMENT_TRANSACTION_SCHEMA_V3)) => {
|
||
schema
|
||
}
|
||
_ => bail!("Idunn control store contains an unsupported transaction"),
|
||
};
|
||
let value: LegacyDeploymentTransaction = rmp_serde::from_slice(&envelope.payload)
|
||
.with_context(|| format!("decoding a {schema} deployment transaction"))?;
|
||
ensure!(
|
||
value.schema_version == schema,
|
||
"stored transaction schema differs from its envelope"
|
||
);
|
||
ensure!(
|
||
schema != DEPLOYMENT_TRANSACTION_SCHEMA_V2 || value.post_fencing_abort.is_none(),
|
||
"a v2 transaction cannot carry post-fencing abort evidence"
|
||
);
|
||
Ok(value.into_current())
|
||
}
|
||
|
||
/// Read one stored transaction, lifting a v2 or v3 record to the current shape.
|
||
///
|
||
/// The control store refuses noncanonical bytes: a record must re-encode to
|
||
/// exactly what is stored. That check is what makes tampering visible, and it
|
||
/// also means adding a field is a schema change: an older record re-encodes
|
||
/// with extra keys and would be read as tampered.
|
||
///
|
||
/// So an older record is decoded by its own layout and lifted. Byte-exactness
|
||
/// cannot apply across a version boundary -- the bytes are a different schema
|
||
/// by definition -- so the lift leans on the full semantic `validate()`
|
||
/// instead, and everything downstream sees only the current shape.
|
||
fn read_transaction_record(envelope: &CultCacheEnvelope) -> Result<DeploymentTransaction> {
|
||
match envelope.schema_id.as_deref() {
|
||
Some(DEPLOYMENT_TRANSACTION_SCHEMA) => {
|
||
let value: DeploymentTransaction = decode_record(envelope)?;
|
||
value.validate()?;
|
||
Ok(value)
|
||
}
|
||
_ => {
|
||
let mut value = lift_legacy_transaction(envelope)?;
|
||
cleanup_evidence::owe_legacy_continuity_projection(&mut value);
|
||
value.validate()?;
|
||
Ok(value)
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Read one stored admitted generation, lifting a v2 or v3 record to the
|
||
/// current shape by the same rule as `read_transaction_record`.
|
||
fn read_generation_record(envelope: &CultCacheEnvelope) -> Result<AdmittedGeneration> {
|
||
match envelope.schema_id.as_deref() {
|
||
Some(ADMITTED_GENERATION_SCHEMA) => {
|
||
let value: AdmittedGeneration = decode_record(envelope)?;
|
||
value.validate()?;
|
||
Ok(value)
|
||
}
|
||
Some(ADMITTED_GENERATION_SCHEMA_V3) => {
|
||
let legacy: LegacyAdmittedGenerationV3 = rmp_serde::from_slice(&envelope.payload)
|
||
.context("decoding a v3 admitted generation")?;
|
||
ensure!(
|
||
legacy.schema_version == ADMITTED_GENERATION_SCHEMA_V3,
|
||
"stored generation schema differs from its envelope"
|
||
);
|
||
let value = legacy.into_current()?;
|
||
value.validate()?;
|
||
Ok(value)
|
||
}
|
||
Some(ADMITTED_GENERATION_SCHEMA_V2) => {
|
||
let legacy: LegacyAdmittedGeneration = rmp_serde::from_slice(&envelope.payload)
|
||
.context("decoding a v2 admitted generation")?;
|
||
ensure!(
|
||
legacy.schema_version == ADMITTED_GENERATION_SCHEMA_V2,
|
||
"stored generation schema differs from its envelope"
|
||
);
|
||
let value = legacy.into_current();
|
||
value.validate()?;
|
||
Ok(value)
|
||
}
|
||
_ => bail!("Idunn control store contains an unsupported admitted generation"),
|
||
}
|
||
}
|
||
|
||
/// The v1 target-supervision layout: the first seven keys of the current one.
|
||
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||
#[cultcache(
|
||
type = "idunn.target_supervision",
|
||
schema = "idunn.target_supervision.v1"
|
||
)]
|
||
struct LegacyTargetSupervisionV1 {
|
||
#[cultcache(key = 0)]
|
||
schema_version: String,
|
||
#[cultcache(key = 1)]
|
||
target: String,
|
||
#[cultcache(key = 2)]
|
||
route_actuations: Vec<u64>,
|
||
#[cultcache(key = 3)]
|
||
continuity_restarts: Vec<u64>,
|
||
#[cultcache(key = 4)]
|
||
continuity_deferred_until: Option<u64>,
|
||
#[cultcache(key = 5)]
|
||
continuity_deferral_reason: Option<String>,
|
||
#[cultcache(key = 6)]
|
||
route_survivals: Vec<u64>,
|
||
}
|
||
|
||
impl LegacyTargetSupervisionV1 {
|
||
/// No streak, no anchor, no re-materialization: the same defaults a fresh
|
||
/// record starts with.
|
||
fn into_current(self) -> TargetSupervision {
|
||
TargetSupervision {
|
||
schema_version: TARGET_SUPERVISION_SCHEMA.into(),
|
||
target: self.target,
|
||
route_actuations: self.route_actuations,
|
||
continuity_restarts: self.continuity_restarts,
|
||
continuity_deferred_until: self.continuity_deferred_until,
|
||
continuity_deferral_reason: self.continuity_deferral_reason,
|
||
route_survivals: self.route_survivals,
|
||
continuity_attempt_streak: 0,
|
||
continuity_backoff_from_unix_millis: None,
|
||
continuity_rematerialized_at_unix_millis: None,
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Read one stored target supervision record, lifting a v1 record to the
|
||
/// current shape by the same rule as `read_generation_record`. One schema id
|
||
/// names one shape: a v1 record that carries ten keys, or a v2 record that
|
||
/// carries seven, is refused.
|
||
fn read_supervision_record(envelope: &CultCacheEnvelope) -> Result<TargetSupervision> {
|
||
let value = match envelope.schema_id.as_deref() {
|
||
Some(TARGET_SUPERVISION_SCHEMA) => decode_record::<TargetSupervision>(envelope)?,
|
||
Some(TARGET_SUPERVISION_SCHEMA_V1) => {
|
||
let legacy: LegacyTargetSupervisionV1 = decode_record(envelope)
|
||
.context("decoding a v1 target supervision record")?;
|
||
ensure!(
|
||
legacy.schema_version == TARGET_SUPERVISION_SCHEMA_V1,
|
||
"stored supervision schema differs from its envelope"
|
||
);
|
||
legacy.into_current()
|
||
}
|
||
_ => bail!("Idunn control store contains an unsupported target supervision record"),
|
||
};
|
||
value.validate()?;
|
||
ensure!(
|
||
envelope.key == value.target,
|
||
"target supervision key is not its target"
|
||
);
|
||
Ok(value)
|
||
}
|
||
|
||
/// Rewrite every older record as current, once, before the engine runs.
|
||
///
|
||
/// The read path lifts older records on its own, so this is convergence rather
|
||
/// than correctness: without it the store keeps records in several shapes for
|
||
/// as long as the oldest one survives. Each rewrite is a compare-exchange
|
||
/// against the exact stored envelope, so a record that changed underneath is
|
||
/// left alone rather than clobbered.
|
||
fn migrate_control_store_to_current_schema(store_path: &Path) -> Result<usize> {
|
||
if !store_path.exists() {
|
||
return Ok(0);
|
||
}
|
||
let store = SingleFileMessagePackBackingStore::new(store_path);
|
||
let stale = store
|
||
.pull_all_read_only_snapshot()
|
||
.context("reading Idunn control snapshot for migration")?
|
||
.into_iter()
|
||
.filter(|envelope| {
|
||
let schema = envelope.schema_id.as_deref();
|
||
(envelope.r#type == DeploymentTransaction::TYPE
|
||
&& matches!(
|
||
schema,
|
||
Some(DEPLOYMENT_TRANSACTION_SCHEMA_V2 | DEPLOYMENT_TRANSACTION_SCHEMA_V3)
|
||
))
|
||
|| (envelope.r#type == AdmittedGeneration::TYPE
|
||
&& matches!(
|
||
schema,
|
||
Some(ADMITTED_GENERATION_SCHEMA_V2 | ADMITTED_GENERATION_SCHEMA_V3)
|
||
))
|
||
|| (envelope.r#type == TargetSupervision::TYPE
|
||
&& schema == Some(TARGET_SUPERVISION_SCHEMA_V1))
|
||
})
|
||
.collect::<Vec<_>>();
|
||
let mut migrated = 0;
|
||
for envelope in stale {
|
||
let (record_type, key, mut next) = if envelope.r#type == DeploymentTransaction::TYPE {
|
||
let (key, next) = cleanup_evidence::migrate_transaction_record(&envelope)?;
|
||
(DeploymentTransaction::TYPE, key, next)
|
||
} else if envelope.r#type == TargetSupervision::TYPE {
|
||
let value = read_supervision_record(&envelope)?;
|
||
let next = target_supervision_envelope(&value, now_millis()?)?;
|
||
(TargetSupervision::TYPE, value.target, next)
|
||
} else {
|
||
let value = read_generation_record(&envelope)?;
|
||
let next = admitted_envelope(&value, value.admitted_at_unix_millis)?;
|
||
(AdmittedGeneration::TYPE, value.target, next)
|
||
};
|
||
next.stored_at = envelope.stored_at.clone();
|
||
ensure!(
|
||
store.compare_exchange(
|
||
&[CultCacheExpectedEnvelope {
|
||
r#type: record_type.into(),
|
||
key,
|
||
current: Some(envelope.clone()),
|
||
}],
|
||
&[next],
|
||
)?,
|
||
"control record changed during schema migration"
|
||
);
|
||
migrated += 1;
|
||
}
|
||
Ok(migrated)
|
||
}
|
||
|
||
fn transaction_envelope(value: &DeploymentTransaction, now: u64) -> Result<CultCacheEnvelope> {
|
||
ensure!(
|
||
!value.carries_legacy_marker(),
|
||
"only the legacy lift may produce a legacy-marked record; it cannot be written as a new one"
|
||
);
|
||
encode_transaction(value, now)
|
||
}
|
||
|
||
fn encode_transaction(value: &DeploymentTransaction, now: u64) -> Result<CultCacheEnvelope> {
|
||
value.validate()?;
|
||
typed_envelope(
|
||
&value.transaction_id,
|
||
DeploymentTransaction::TYPE,
|
||
DEPLOYMENT_TRANSACTION_SCHEMA,
|
||
value,
|
||
now,
|
||
)
|
||
}
|
||
|
||
fn admitted_envelope(value: &AdmittedGeneration, now: u64) -> Result<CultCacheEnvelope> {
|
||
value.validate()?;
|
||
typed_envelope(
|
||
&value.target,
|
||
AdmittedGeneration::TYPE,
|
||
ADMITTED_GENERATION_SCHEMA,
|
||
value,
|
||
now,
|
||
)
|
||
}
|
||
|
||
fn target_supervision_envelope(value: &TargetSupervision, now: u64) -> Result<CultCacheEnvelope> {
|
||
value.validate()?;
|
||
typed_envelope(
|
||
&value.target,
|
||
TargetSupervision::TYPE,
|
||
TARGET_SUPERVISION_SCHEMA,
|
||
value,
|
||
now,
|
||
)
|
||
}
|
||
|
||
pub(crate) fn typed_envelope<T: Serialize>(
|
||
key: &str,
|
||
record_type: &str,
|
||
schema: &str,
|
||
value: &T,
|
||
now: u64,
|
||
) -> Result<CultCacheEnvelope> {
|
||
Ok(CultCacheEnvelope {
|
||
key: key.into(),
|
||
r#type: record_type.into(),
|
||
payload: rmp_serde::to_vec(value)?,
|
||
stored_at: rfc3339_millis(now)?,
|
||
schema_id: Some(schema.into()),
|
||
})
|
||
}
|
||
|
||
/// `history.cc`, beside the control store. Terminal transactions live here so
|
||
/// the control store holds only what still gates a decision.
|
||
fn history_store_path(state_store: &Path) -> PathBuf {
|
||
state_store.with_file_name("history.cc")
|
||
}
|
||
|
||
/// Move one finished transaction out of the live set.
|
||
///
|
||
/// History first, then the live copy: a crash between the two leaves the
|
||
/// record in both, which reads as still-resident and is archived again on the
|
||
/// next attempt. The other order can lose it, and a completion nobody can
|
||
/// observe is the failure that actually costs an operator something.
|
||
///
|
||
/// `insert_entry_if_absent` makes the repeat a no-op, and the delete is a
|
||
/// compare-and-swap on the exact envelope archived, so a transaction that
|
||
/// changed underneath us is left alone rather than dropped.
|
||
fn archive_terminal_transaction(state_store: &Path, envelope: &CultCacheEnvelope) -> Result<()> {
|
||
SingleFileMessagePackBackingStore::new(&history_store_path(state_store))
|
||
.insert_entry_if_absent(envelope.clone())
|
||
.context("archiving a finished transaction to history")?;
|
||
SingleFileMessagePackBackingStore::new(state_store)
|
||
.delete_batch_if_unchanged(std::slice::from_ref(envelope))
|
||
.context("retiring a finished transaction from the control store")?;
|
||
// The command goes with its last transaction. A command is consumed by
|
||
// being frozen once; left resident after its transaction retired, it read
|
||
// as queued again and was frozen again on the next tick, which is how one
|
||
// `idunn up` became an unbounded series of deployment attempts.
|
||
let transaction = read_transaction_record(envelope)?;
|
||
let snapshot = ControlSnapshot::read(state_store)?;
|
||
if !snapshot
|
||
.transaction_for_command(&transaction.command_id)
|
||
.is_empty()
|
||
{
|
||
return Ok(());
|
||
}
|
||
if let Some(command) = snapshot
|
||
.commands
|
||
.iter()
|
||
.find(|stored| stored.value.command_id == transaction.command_id)
|
||
{
|
||
SingleFileMessagePackBackingStore::new(&history_store_path(state_store))
|
||
.insert_entry_if_absent(command.envelope.clone())
|
||
.context("archiving a consumed command to history")?;
|
||
SingleFileMessagePackBackingStore::new(state_store)
|
||
.delete_batch_if_unchanged(std::slice::from_ref(&command.envelope))
|
||
.context("retiring a consumed command from the control store")?;
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
/// What `history.cc` yielded: the transactions it decoded and the entries it
|
||
/// could not, each with its key and reason.
|
||
struct HistoryRead {
|
||
transactions: Vec<DeploymentTransaction>,
|
||
undecodable: Vec<(String, String)>,
|
||
}
|
||
|
||
impl HistoryRead {
|
||
/// What is missing from `transactions`, or `None` when nothing is.
|
||
fn report(&self) -> Option<String> {
|
||
if self.undecodable.is_empty() {
|
||
return None;
|
||
}
|
||
let keys = self
|
||
.undecodable
|
||
.iter()
|
||
.map(|(key, error)| format!("{key} ({error})"))
|
||
.collect::<Vec<_>>()
|
||
.join("; ");
|
||
Some(format!(
|
||
"Idunn history: {} archived transaction(s) could not be decoded and are not counted \
|
||
by status or continuity backoff: {keys}",
|
||
self.undecodable.len()
|
||
))
|
||
}
|
||
}
|
||
|
||
/// Finished transactions. `control.cc` keeps the byte-exact canonical check
|
||
/// because its records still gate decisions. History describes what already
|
||
/// happened, so an undecodable entry is reported and skipped rather than
|
||
/// allowed to refuse the read -- that asymmetry is the whole reason the two
|
||
/// files are separate. A file that cannot be read at all is an error: the
|
||
/// caller decides whether an empty answer is safe.
|
||
fn read_history(state_store: &Path) -> Result<HistoryRead> {
|
||
let path = history_store_path(state_store);
|
||
if !path.exists() {
|
||
return Ok(HistoryRead {
|
||
transactions: Vec::new(),
|
||
undecodable: Vec::new(),
|
||
});
|
||
}
|
||
let envelopes = SingleFileMessagePackBackingStore::new(&path)
|
||
.pull_all_read_only_snapshot()
|
||
.with_context(|| format!("Idunn cannot read {}", path.display()))?;
|
||
let (transactions, undecodable) = decode_history_transactions(envelopes);
|
||
Ok(HistoryRead {
|
||
transactions,
|
||
undecodable,
|
||
})
|
||
}
|
||
|
||
/// History for display only (status). An unreadable file shows as empty and
|
||
/// says so; decisions use `Engine::history_for_decision`, which does not.
|
||
fn read_history_transactions(state_store: &Path) -> Vec<DeploymentTransaction> {
|
||
match read_history(state_store) {
|
||
Ok(history) => {
|
||
if let Some(report) = history.report() {
|
||
eprintln!("{report}");
|
||
}
|
||
history.transactions
|
||
}
|
||
Err(error) => {
|
||
eprintln!("{error:#}; every archived transaction is invisible to status");
|
||
Vec::new()
|
||
}
|
||
}
|
||
}
|
||
|
||
/// How long to wait after `failures` consecutive failures: one poll interval
|
||
/// after the first, doubling each time, never past the ceiling.
|
||
fn backoff_wait(base_millis: u64, failures: u32, ceiling_millis: u64) -> u64 {
|
||
base_millis
|
||
.saturating_mul(1u64 << failures.min(20))
|
||
.min(ceiling_millis)
|
||
}
|
||
|
||
/// Whether an attempt due at `not_before` must still wait at `now`. A due time
|
||
/// further ahead than any wait its owner can set means the clock stepped back,
|
||
/// so the attempt is due: a step stalls nothing longer than the wait itself.
|
||
fn is_waiting(now: u64, not_before: u64, longest_wait: u64) -> bool {
|
||
now < not_before && not_before - now <= longest_wait
|
||
}
|
||
|
||
/// Whether a transaction moved between two reads. An error note (`last_error`)
|
||
/// and its timestamp are commentary on a stuck step, not a step.
|
||
fn state_advanced(before: &DeploymentTransaction, after: &DeploymentTransaction) -> bool {
|
||
let mut after = after.clone();
|
||
after.last_error.clone_from(&before.last_error);
|
||
after.updated_at_unix_millis = before.updated_at_unix_millis;
|
||
&after != before
|
||
}
|
||
|
||
/// Says a fault once on stderr while it lasts. A scheduler tick that hits the same
|
||
/// unreadable file every half second must not print it every half second.
|
||
///
|
||
/// It is a trace and owns nothing else: it is in memory, keyed by free-form
|
||
/// strings, and an Idunn restart forgets it. Whether the operator is told
|
||
/// belongs to the incident record (`incident`), never to this map.
|
||
#[derive(Default)]
|
||
struct ReportOnce {
|
||
last: Mutex<Option<String>>,
|
||
}
|
||
|
||
impl ReportOnce {
|
||
/// The text to print now: `report` when it differs from the last one
|
||
/// offered, nothing while the same fault stands or once it has cleared.
|
||
fn offer(&self, report: Option<String>) -> Option<String> {
|
||
let mut last = self.last.lock().expect("report mutex");
|
||
if *last == report {
|
||
return None;
|
||
}
|
||
last.clone_from(&report);
|
||
report
|
||
}
|
||
}
|
||
|
||
/// Decode every archived transaction. An entry that fails is returned with its
|
||
/// key and reason instead of vanishing, so the reader can say how many records
|
||
/// its answer is missing.
|
||
///
|
||
/// `LegacyDeploymentTransaction` decodes the v2 and v3 layouts. `history.cc` is
|
||
/// never migrated or pruned, so it dies only when history is pruned or
|
||
/// migrated, not at boot.
|
||
fn decode_history_transactions(
|
||
envelopes: Vec<CultCacheEnvelope>,
|
||
) -> (Vec<DeploymentTransaction>, Vec<(String, String)>) {
|
||
let mut transactions = Vec::new();
|
||
let mut undecodable = Vec::new();
|
||
for envelope in envelopes
|
||
.into_iter()
|
||
.filter(|envelope| envelope.r#type == DeploymentTransaction::TYPE)
|
||
{
|
||
let decoded = match envelope.schema_id.as_deref() {
|
||
Some(DEPLOYMENT_TRANSACTION_SCHEMA_V2 | DEPLOYMENT_TRANSACTION_SCHEMA_V3) => {
|
||
lift_legacy_transaction(&envelope)
|
||
}
|
||
_ => rmp_serde::from_slice::<DeploymentTransaction>(&envelope.payload)
|
||
.map_err(anyhow::Error::from),
|
||
};
|
||
match decoded {
|
||
Ok(transaction) => transactions.push(transaction),
|
||
Err(error) => undecodable.push((envelope.key, format!("{error:#}"))),
|
||
}
|
||
}
|
||
(transactions, undecodable)
|
||
}
|
||
|
||
/// Consumed commands, read as leniently as their transactions.
|
||
fn read_history_commands(state_store: &Path) -> Vec<DeploymentCommand> {
|
||
let path = history_store_path(state_store);
|
||
if !path.exists() {
|
||
return Vec::new();
|
||
}
|
||
let Ok(envelopes) = SingleFileMessagePackBackingStore::new(&path).pull_all_read_only_snapshot()
|
||
else {
|
||
return Vec::new();
|
||
};
|
||
envelopes
|
||
.into_iter()
|
||
.filter(|envelope| envelope.r#type == DeploymentCommand::TYPE)
|
||
.filter_map(|envelope| rmp_serde::from_slice::<DeploymentCommand>(&envelope.payload).ok())
|
||
.collect()
|
||
}
|
||
|
||
fn replace_transaction(
|
||
store_path: &Path,
|
||
current: &Stored<DeploymentTransaction>,
|
||
next: &DeploymentTransaction,
|
||
) -> Result<()> {
|
||
next.validate()?;
|
||
ensure!(
|
||
current.value.transaction_id == next.transaction_id,
|
||
"transaction replacement changes identity"
|
||
);
|
||
let envelope = transaction_envelope(next, next.updated_at_unix_millis)?;
|
||
ensure!(
|
||
SingleFileMessagePackBackingStore::new(store_path).compare_exchange(
|
||
&[CultCacheExpectedEnvelope {
|
||
r#type: DeploymentTransaction::TYPE.into(),
|
||
key: current.value.transaction_id.clone(),
|
||
current: Some(current.envelope.clone()),
|
||
}],
|
||
std::slice::from_ref(&envelope),
|
||
)?,
|
||
"deployment transaction changed before its compare-exchange"
|
||
);
|
||
if next.is_terminal() {
|
||
archive_terminal_transaction(store_path, &envelope)?;
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
#[derive(Clone)]
|
||
struct LoadedBinding {
|
||
binding: OperatorBinding,
|
||
bytes: Vec<u8>,
|
||
}
|
||
|
||
fn load_bindings(directory: &Path) -> Result<BTreeMap<String, LoadedBinding>> {
|
||
let mut paths = fs::read_dir(directory)
|
||
.with_context(|| format!("reading Idunn binding directory {}", directory.display()))?
|
||
.map(|entry| entry.map(|entry| entry.path()))
|
||
.collect::<std::io::Result<Vec<_>>>()?;
|
||
paths.retain(|path| path.extension() == Some(std::ffi::OsStr::new("toml")));
|
||
paths.sort();
|
||
let mut bindings = BTreeMap::new();
|
||
for path in paths {
|
||
let bytes = fs::read(&path)
|
||
.with_context(|| format!("reading operator binding {}", path.display()))?;
|
||
let text = std::str::from_utf8(&bytes)
|
||
.with_context(|| format!("operator binding {} is not UTF-8", path.display()))?;
|
||
let binding = OperatorBinding::parse(text)
|
||
.with_context(|| format!("validating operator binding {}", path.display()))?;
|
||
let target = binding.target.clone();
|
||
ensure!(
|
||
bindings
|
||
.insert(target.clone(), LoadedBinding { binding, bytes })
|
||
.is_none(),
|
||
"operator binding target {target} is duplicated"
|
||
);
|
||
}
|
||
ensure!(!bindings.is_empty(), "Idunn binding directory is empty");
|
||
validate_route_bindings(&bindings)?;
|
||
Ok(bindings)
|
||
}
|
||
|
||
fn validate_route_bindings(bindings: &BTreeMap<String, LoadedBinding>) -> Result<()> {
|
||
let routes = bindings
|
||
.iter()
|
||
.filter_map(|(target, loaded)| {
|
||
loaded
|
||
.binding
|
||
.route
|
||
.as_ref()
|
||
.map(|route| (target.as_str(), route))
|
||
})
|
||
.collect::<Vec<_>>();
|
||
validate_route_binding_set(&routes)
|
||
}
|
||
|
||
fn validate_route_binding_set(routes: &[(&str, &RouteBinding)]) -> Result<()> {
|
||
for (index, (target, route)) in routes.iter().enumerate() {
|
||
let (stable_host, stable_port) = route.stable_socket()?;
|
||
let private_host = route
|
||
.private_host
|
||
.parse::<std::net::IpAddr>()
|
||
.context("validated route private host stopped being an IP address")?;
|
||
ensure!(
|
||
stable_host != private_host
|
||
|| !(route.private_port_start..=route.private_port_end).contains(&stable_port),
|
||
"route {target} stable socket overlaps its candidate port range"
|
||
);
|
||
|
||
for (other_target, other) in &routes[index + 1..] {
|
||
ensure!(
|
||
route.route_id != other.route_id,
|
||
"route id {} is shared by targets {target} and {other_target}",
|
||
route.route_id
|
||
);
|
||
ensure!(
|
||
route.config_path != other.config_path,
|
||
"route fragment {} is shared by targets {target} and {other_target}",
|
||
route.config_path.display()
|
||
);
|
||
let other_stable = other.stable_socket()?;
|
||
ensure!(
|
||
route.driver != other.driver || (stable_host, stable_port) != other_stable,
|
||
"stable route socket {}:{} is shared by targets {target} and {other_target}",
|
||
stable_host,
|
||
stable_port
|
||
);
|
||
if route.driver != other.driver {
|
||
continue;
|
||
}
|
||
let other_private_host = other
|
||
.private_host
|
||
.parse::<std::net::IpAddr>()
|
||
.context("validated route private host stopped being an IP address")?;
|
||
let ranges_overlap = route.private_host == other.private_host
|
||
&& route.private_port_start <= other.private_port_end
|
||
&& other.private_port_start <= route.private_port_end;
|
||
ensure!(
|
||
!ranges_overlap,
|
||
"candidate port ranges overlap for targets {target} and {other_target}"
|
||
);
|
||
ensure!(
|
||
stable_host != other_private_host
|
||
|| !(other.private_port_start..=other.private_port_end).contains(&stable_port),
|
||
"stable route socket for {target} overlaps {other_target}'s candidate range"
|
||
);
|
||
ensure!(
|
||
other_stable.0 != private_host
|
||
|| !(route.private_port_start..=route.private_port_end)
|
||
.contains(&other_stable.1),
|
||
"stable route socket for {other_target} overlaps {target}'s candidate range"
|
||
);
|
||
}
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
fn resolve_selector(
|
||
bindings: &BTreeMap<String, LoadedBinding>,
|
||
selector: &str,
|
||
) -> Result<Vec<String>> {
|
||
let mut targets = if let Some(profile) = selector.strip_prefix("profile:") {
|
||
bindings
|
||
.values()
|
||
.filter(|binding| binding.binding.profiles.contains(profile))
|
||
.map(|binding| binding.binding.target.clone())
|
||
.collect::<Vec<_>>()
|
||
} else {
|
||
ensure!(
|
||
bindings.contains_key(selector),
|
||
"unknown deployment target {selector}"
|
||
);
|
||
vec![selector.to_owned()]
|
||
};
|
||
ensure!(
|
||
!targets.is_empty(),
|
||
"unknown or empty deployment profile {selector}"
|
||
);
|
||
targets.sort_by(|left, right| {
|
||
(left != "odin")
|
||
.cmp(&(right != "odin"))
|
||
.then_with(|| left.cmp(right))
|
||
});
|
||
Ok(targets)
|
||
}
|
||
|
||
fn submit(
|
||
store_path: &Path,
|
||
selector: &str,
|
||
requested_by: &str,
|
||
wait: bool,
|
||
timeout_seconds: u64,
|
||
) -> Result<()> {
|
||
if let Some(parent) = store_path.parent() {
|
||
fs::create_dir_all(parent)?;
|
||
}
|
||
let now = now_millis()?;
|
||
let command = DeploymentCommand {
|
||
schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(),
|
||
command_id: format!("up-{}", Uuid::new_v4()),
|
||
kind: CommandKind::Deploy,
|
||
selector: selector.into(),
|
||
requested_by: requested_by.into(),
|
||
requested_at_unix_millis: now,
|
||
};
|
||
command.validate()?;
|
||
ensure!(
|
||
SingleFileMessagePackBackingStore::new(store_path).compare_exchange(
|
||
&[CultCacheExpectedEnvelope {
|
||
r#type: DeploymentCommand::TYPE.into(),
|
||
key: command.command_id.clone(),
|
||
current: None,
|
||
}],
|
||
&[command_envelope(&command, now)?],
|
||
)?,
|
||
"deployment command id collided"
|
||
);
|
||
println!("{}", command.command_id);
|
||
if !wait {
|
||
return Ok(());
|
||
}
|
||
let deadline = now.saturating_add(timeout_seconds.saturating_mul(1000));
|
||
loop {
|
||
let snapshot = ControlSnapshot::read(store_path)?;
|
||
ensure!(
|
||
snapshot
|
||
.commands
|
||
.iter()
|
||
.any(|stored| stored.value.command_id == command.command_id),
|
||
"submitted deployment command disappeared"
|
||
);
|
||
let transactions = snapshot.transaction_for_command(&command.command_id);
|
||
if !transactions.is_empty() && transactions.iter().all(|value| value.is_terminal()) {
|
||
if let Some(error) =
|
||
transactions
|
||
.iter()
|
||
.find_map(|transaction| match &transaction.completion {
|
||
Some(TransactionCompletion::FailedBeforeFencing { error })
|
||
| Some(TransactionCompletion::FailedAfterFencing { error, .. }) => {
|
||
Some(error.as_str())
|
||
}
|
||
_ => None,
|
||
})
|
||
{
|
||
bail!("deployment failed: {error}")
|
||
}
|
||
println!("succeeded {} target(s)", transactions.len());
|
||
return Ok(());
|
||
}
|
||
if now_millis()? >= deadline {
|
||
bail!("deployment command timed out")
|
||
}
|
||
thread::sleep(Duration::from_millis(250));
|
||
}
|
||
}
|
||
|
||
fn status(
|
||
out: &mut impl Write,
|
||
store_path: &Path,
|
||
incident_store: &Path,
|
||
command_id: Option<&str>,
|
||
) -> Result<()> {
|
||
let snapshot = ControlSnapshot::read(store_path)?;
|
||
// Commands stay resident; their finished transactions do not. Without
|
||
// history a completed command would report as though it had never run,
|
||
// which is the operator surface R11 refused to trade away for a smaller
|
||
// live set.
|
||
let archived = read_history_transactions(store_path);
|
||
let archived_commands = read_history_commands(store_path);
|
||
let mut commands = snapshot
|
||
.commands
|
||
.iter()
|
||
.map(|stored| &stored.value)
|
||
.chain(archived_commands.iter())
|
||
.collect::<Vec<_>>();
|
||
commands.sort_by_key(|command| command.requested_at_unix_millis);
|
||
if let Some(command_id) = command_id {
|
||
commands.retain(|command| command.command_id == command_id);
|
||
ensure!(!commands.is_empty(), "deployment command is unknown");
|
||
}
|
||
for command in commands {
|
||
let mut transactions = snapshot.transaction_for_command(&command.command_id);
|
||
transactions.extend(
|
||
archived
|
||
.iter()
|
||
.filter(|value| value.command_id == command.command_id),
|
||
);
|
||
transactions.sort_by_key(|value| value.ordinal);
|
||
let (state, detail) = derived_command_status(&transactions);
|
||
writeln!(
|
||
out,
|
||
"{} {} {} {}",
|
||
command.command_id, command.selector, state, detail
|
||
)?;
|
||
// Naming one command asks about that command, so print what a stuck
|
||
// transaction is actually waiting on. A gate reason lives in
|
||
// `last_error` and was never rendered anywhere, which left "Sealing"
|
||
// looking identical whether the brake had not authorized the
|
||
// transaction or the phase was simply slow. The release and deployment
|
||
// ids are here because they are exactly what a brake release must name.
|
||
if command_id.is_some() {
|
||
for transaction in transactions {
|
||
writeln!(out, " transaction {}", transaction.transaction_id)?;
|
||
writeln!(
|
||
out,
|
||
" target {} phase {:?}",
|
||
transaction.target, transaction.phase
|
||
)?;
|
||
if let Some(expected) = &transaction.expected {
|
||
writeln!(out, " runtime {}", expected.runtime_id)?;
|
||
writeln!(out, " release {}", expected.sealed_release_id)?;
|
||
}
|
||
writeln!(
|
||
out,
|
||
" odin publisher cursor {}",
|
||
transaction.odin_publisher_sequence_cursor
|
||
)?;
|
||
if let Some(evidence) = &transaction.latest_odin_observation {
|
||
writeln!(
|
||
out,
|
||
" latest odin observation sequence {}",
|
||
evidence.publisher_sequence
|
||
)?;
|
||
}
|
||
if let Some(reason) = &transaction.last_error {
|
||
writeln!(out, " waiting on {reason}")?;
|
||
}
|
||
// A Complete transaction that is not terminal still owns its
|
||
// target through unfinished cleanup. Without this an operator
|
||
// sees "Complete" and a stale reason and cannot tell what is
|
||
// holding the target.
|
||
if let Some(completion) = &transaction.completion {
|
||
writeln!(out, " completion {completion:?}")?;
|
||
}
|
||
if let Some(abort) = &transaction.pre_fencing_abort {
|
||
writeln!(out, " pre-fencing abort {abort:?}")?;
|
||
}
|
||
if let Some(abort) = &transaction.post_fencing_abort {
|
||
writeln!(out, " post-fencing abort {abort:?}")?;
|
||
}
|
||
if let Some(cleanup) = &transaction.post_commit_cleanup {
|
||
writeln!(out, " post-commit cleanup {cleanup:?}")?;
|
||
}
|
||
writeln!(
|
||
out,
|
||
" terminal {} owns-target {} drains {}",
|
||
transaction.is_terminal(),
|
||
transaction.owns_target(),
|
||
transaction.drains_incumbent()
|
||
)?;
|
||
}
|
||
}
|
||
}
|
||
// What supervision holds for each target: the union of the admitted
|
||
// generations and the metered targets, so a target that has only been
|
||
// deployed to (no generation yet) still shows what it has used.
|
||
for line in render_targets(&snapshot, incident_store, now_millis()?) {
|
||
writeln!(out, "{line}")?;
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
/// Every supervised target with its meters and its incidents. An incident
|
||
/// store that cannot be read costs the incident lines and nothing else.
|
||
fn render_targets(snapshot: &ControlSnapshot, incident_store: &Path, now: u64) -> Vec<String> {
|
||
let incidents = IncidentStore::new(incident_store).read();
|
||
let mut lines = Vec::new();
|
||
for target in supervised_targets(snapshot) {
|
||
lines.extend(render_supervision(
|
||
target,
|
||
snapshot.admitted_for(target).map(|stored| &stored.value),
|
||
snapshot.supervision_for(target).map(|stored| &stored.value),
|
||
now,
|
||
));
|
||
if let Ok(records) = &incidents {
|
||
lines.extend(render_incidents(records, target, now));
|
||
}
|
||
}
|
||
if let Err(error) = &incidents {
|
||
lines.push(format!("incidents unreadable: {}", error.root_cause()));
|
||
}
|
||
lines
|
||
}
|
||
|
||
/// Every target supervision holds something for: the admitted generations
|
||
/// and the metered targets.
|
||
fn supervised_targets(snapshot: &ControlSnapshot) -> BTreeSet<&str> {
|
||
snapshot
|
||
.admitted
|
||
.iter()
|
||
.map(|stored| stored.value.target.as_str())
|
||
.chain(snapshot.targets.iter().map(|stored| stored.value.target.as_str()))
|
||
.collect()
|
||
}
|
||
|
||
/// One target's meters and route pacing, one line each, times in unix
|
||
/// milliseconds. The counts are what is still inside the window at `now`.
|
||
fn render_supervision(
|
||
target: &str,
|
||
generation: Option<&AdmittedGeneration>,
|
||
supervision: Option<&TargetSupervision>,
|
||
now: u64,
|
||
) -> Vec<String> {
|
||
let at = |time: Option<u64>| time.map_or("none".to_owned(), |millis| millis.to_string());
|
||
let meters = supervision.cloned().unwrap_or_else(|| TargetSupervision::new(target));
|
||
let mut lines = vec![format!(
|
||
"target {target} {}",
|
||
generation.map_or("no-admitted-generation", |value| value.generation_id.as_str())
|
||
)];
|
||
if let Some(reason) = generation.and_then(|value| value.last_error.as_deref()) {
|
||
lines.push(format!(" held: {reason}"));
|
||
}
|
||
lines.push(format!(
|
||
" continuity attempts-in-hour {} pages-at {} streak {} next-attempt-at {}",
|
||
meters.restarts_used(now),
|
||
CONTINUITY_PAGING_RESTARTS,
|
||
meters.continuity_attempt_streak,
|
||
at(generation.and_then(|value| meters.backoff_due(value.admitted_at_unix_millis)))
|
||
));
|
||
if let Some(rematerialized) = meters.continuity_rematerialized_at_unix_millis {
|
||
lines.push(format!(" continuity rematerialized-at {rematerialized}"));
|
||
}
|
||
if let (Some(until), Some(reason)) = (
|
||
meters.continuity_deferred_until,
|
||
&meters.continuity_deferral_reason,
|
||
) {
|
||
lines.push(format!(" continuity deferred until {until}: {reason}"));
|
||
}
|
||
if let Some(route) = generation.and_then(|value| value.route_supervision.as_ref()) {
|
||
lines.push(format!(
|
||
" route {} consecutive-failures {} last-challenge-at {} next-challenge-at {}",
|
||
match route.degraded_since_unix_millis {
|
||
Some(since) => format!("degraded-since {since}"),
|
||
None => "healthy".to_owned(),
|
||
},
|
||
route.consecutive_failures,
|
||
at(route.last_challenge_at_unix_millis),
|
||
at(route.next_challenge_at_unix_millis)
|
||
));
|
||
}
|
||
if generation.is_none_or(|value| value.route_supervision.is_some())
|
||
|| !meters.route_actuations.is_empty()
|
||
|| !meters.route_survivals.is_empty()
|
||
{
|
||
lines.push(format!(
|
||
" route actuations {}/{} in window, reopens-at {}",
|
||
meters.route_used(now),
|
||
ROUTE_ACTUATION_CEILING,
|
||
at(meters.route_reopens_at(now))
|
||
));
|
||
// A record, not a meter: only the newest are kept, so a saturated
|
||
// count says "at least".
|
||
let survivals = meters.survivals_in_window(now);
|
||
lines.push(format!(
|
||
" route survival actuations {}{} in window (uncounted), last-at {}",
|
||
if survivals == ROUTE_SURVIVAL_RECORD { "at least " } else { "" },
|
||
survivals,
|
||
at(meters.route_survivals.last().copied())
|
||
));
|
||
}
|
||
lines
|
||
}
|
||
|
||
fn derived_command_status(transactions: &[&DeploymentTransaction]) -> (&'static str, String) {
|
||
if transactions.is_empty() {
|
||
return ("queued", String::new());
|
||
}
|
||
// A live transaction is the command's present tense. Reporting an older
|
||
// attempt's failure while a new one is sealing hid a running deployment
|
||
// behind the word "failed".
|
||
let current = transactions
|
||
.iter()
|
||
.filter(|transaction| !transaction.is_terminal())
|
||
.map(|transaction| format!("{}:{:?}", transaction.target, transaction.phase))
|
||
.collect::<Vec<_>>();
|
||
if !current.is_empty() {
|
||
return ("running", current.join(","));
|
||
}
|
||
if let Some(error) = transactions
|
||
.iter()
|
||
.find_map(|transaction| match &transaction.completion {
|
||
Some(TransactionCompletion::FailedBeforeFencing { error })
|
||
| Some(TransactionCompletion::FailedAfterFencing { error, .. }) => Some(error.clone()),
|
||
_ => None,
|
||
})
|
||
{
|
||
return ("failed", error);
|
||
}
|
||
if transactions.iter().all(|transaction| {
|
||
transaction.is_terminal()
|
||
&& matches!(
|
||
transaction.completion,
|
||
Some(TransactionCompletion::Admitted { .. })
|
||
)
|
||
}) {
|
||
return (
|
||
"succeeded",
|
||
transactions
|
||
.iter()
|
||
.map(|transaction| transaction.target.as_str())
|
||
.collect::<Vec<_>>()
|
||
.join(","),
|
||
);
|
||
}
|
||
("complete", String::new())
|
||
}
|
||
|
||
struct ProcessLock {
|
||
file: File,
|
||
#[cfg(not(unix))]
|
||
path: PathBuf,
|
||
}
|
||
|
||
impl ProcessLock {
|
||
fn acquire(store_path: &Path) -> Result<Self> {
|
||
let path = sibling_path(store_path, ".daemon.lock");
|
||
if let Some(parent) = path.parent() {
|
||
fs::create_dir_all(parent)?;
|
||
}
|
||
#[cfg(unix)]
|
||
{
|
||
use std::os::fd::AsRawFd;
|
||
let file = OpenOptions::new()
|
||
.create(true)
|
||
.read(true)
|
||
.write(true)
|
||
.open(&path)
|
||
.with_context(|| format!("opening Idunn daemon lock {}", path.display()))?;
|
||
let result = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) };
|
||
ensure!(result == 0, "another Idunn daemon owns this control store");
|
||
Ok(Self { file })
|
||
}
|
||
#[cfg(not(unix))]
|
||
{
|
||
let file = OpenOptions::new()
|
||
.create_new(true)
|
||
.read(true)
|
||
.write(true)
|
||
.open(&path)
|
||
.with_context(|| format!("another Idunn daemon owns {}", path.display()))?;
|
||
Ok(Self { file, path })
|
||
}
|
||
}
|
||
}
|
||
|
||
impl Drop for ProcessLock {
|
||
fn drop(&mut self) {
|
||
#[cfg(unix)]
|
||
{
|
||
use std::os::fd::AsRawFd;
|
||
let _ = unsafe { libc::flock(self.file.as_raw_fd(), libc::LOCK_UN) };
|
||
}
|
||
#[cfg(not(unix))]
|
||
{
|
||
let _ = fs::remove_file(&self.path);
|
||
}
|
||
}
|
||
}
|
||
|
||
/// One target's in-memory challenge wait: when it is due, and the stored
|
||
/// records of the target as they were when the record failed to land.
|
||
struct ChallengeWait {
|
||
due_unix_millis: u64,
|
||
stamp: Vec<CultCacheEnvelope>,
|
||
}
|
||
|
||
struct Engine {
|
||
options: RuntimeOptions,
|
||
idunn_signer: ServiceIdentitySigner<IdunnServiceIdentity>,
|
||
idunn_anchor: ServiceIdentityTrustAnchor,
|
||
bootstrap_odin_authority: AdmittedOdinAuthority,
|
||
source: Arc<dyn SourcePort>,
|
||
docker_runner: DockerRunnerDriver,
|
||
systemd_workload: Arc<dyn WorkloadPort>,
|
||
host_actuators: Option<SharedHostActuatorHub>,
|
||
/// A test world that serves no host actuator runs a host-actuator
|
||
/// binding's workload through this port instead.
|
||
#[cfg(test)]
|
||
host_workload: Mutex<Option<Arc<dyn WorkloadPort>>>,
|
||
history_report: ReportOnce,
|
||
/// One `ReportOnce` per faulting record, so one wedged transaction says
|
||
/// its fault once and never drowns another's.
|
||
fault_reports: Mutex<BTreeMap<String, ReportOnce>>,
|
||
/// Retry pacing for a transaction whose step keeps failing, by
|
||
/// transaction id: consecutive failures and the earliest next attempt.
|
||
/// Process-local on purpose: a restart retries at once, and no schema
|
||
/// carries a clock that only pacing reads.
|
||
resume_backoff: Mutex<BTreeMap<String, ResumeBackoff>>,
|
||
/// Continuity aborts the boot lift could not decide (their candidate could
|
||
/// not be observed): no step of them runs until `retry_held_lift` decides.
|
||
/// Process-local: a restart asks again at boot.
|
||
lift_held: Mutex<BTreeSet<String>>,
|
||
/// Set when the boot lift itself failed, so no record's decision exists:
|
||
/// every continuity with a post-fencing abort is held until a later lift
|
||
/// succeeds. Process-local, like `lift_held`.
|
||
lift_unsettled: std::sync::atomic::AtomicBool,
|
||
/// When each routed target's next challenge is due, held only when the
|
||
/// stored `RouteSupervisionState` could not record the last one. The stored
|
||
/// value owns the schedule and survives restarts; this spaces attempts
|
||
/// while the record did not land. An entry ends the moment any write for
|
||
/// its target lands (its stamp no longer matches the store) or the target
|
||
/// is retired or withdrawn: it never outlives the episode that made it.
|
||
challenge_wait: Mutex<BTreeMap<String, ChallengeWait>>,
|
||
}
|
||
|
||
/// The route driver's door to the actuation ceiling: every group of host
|
||
/// mutations a driver makes for `target` is charged through here. The gate
|
||
/// owns which charge a request is: a driver asks for the change it is making,
|
||
/// and the command that owns the transaction decides whether that change is
|
||
/// deployment (`Forward`, refusable) or the continuity of the admitted
|
||
/// generation (`Survival`, never refused).
|
||
struct EngineRouteGate<'a> {
|
||
engine: &'a Engine,
|
||
target: &'a str,
|
||
command: CommandKind,
|
||
}
|
||
|
||
impl RouteActuationGate for EngineRouteGate<'_> {
|
||
fn admit(&self, requested: RouteActuation) -> Result<()> {
|
||
let kind = match self.command {
|
||
CommandKind::Deploy => requested,
|
||
CommandKind::Continuity => RouteActuation::Survival,
|
||
};
|
||
self.engine.charge_route_actuation(self.target, kind)
|
||
}
|
||
}
|
||
|
||
#[derive(Clone, Copy)]
|
||
struct ResumeBackoff {
|
||
failures: u32,
|
||
not_before_unix_millis: u64,
|
||
/// The longest wait this backoff was armed with: the skip check reads it,
|
||
/// so the check and the wait cannot disagree.
|
||
ceiling_millis: u64,
|
||
}
|
||
|
||
/// Longest wait between attempts at a failing transaction step.
|
||
const RESUME_BACKOFF_CEILING_MILLIS: u64 = 60_000;
|
||
|
||
/// The longest wait between attempts at one transaction's step: the continuity
|
||
/// cap for a record left to the operator, which retries on the slow cadence its
|
||
/// page promises (ruling c2c-forward-retry-cadence), and the shorter one for
|
||
/// every other.
|
||
fn resume_ceiling(transaction: &DeploymentTransaction, now: u64) -> u64 {
|
||
if transaction.leaves_to_operator(now) {
|
||
CONTINUITY_BACKOFF_CEILING_MILLIS
|
||
} else {
|
||
RESUME_BACKOFF_CEILING_MILLIS
|
||
}
|
||
}
|
||
|
||
impl Engine {
|
||
fn open(options: RuntimeOptions) -> Result<Self> {
|
||
Self::open_with_systemd_workload(
|
||
options,
|
||
Arc::new(SystemdTransientWorkloadDriver::default()),
|
||
)
|
||
}
|
||
|
||
/// `open` with the systemd workload port supplied. It is the seam that
|
||
/// lets the phase engine run past Fencing without a systemd to talk to.
|
||
fn open_with_systemd_workload(
|
||
options: RuntimeOptions,
|
||
systemd_workload: Arc<dyn WorkloadPort>,
|
||
) -> Result<Self> {
|
||
let idunn_signer =
|
||
open_service_identity_at::<IdunnServiceIdentity>(&options.idunn_identity_store)
|
||
.context("opening Idunn activation identity")?;
|
||
let idunn_anchor = idunn_signer.trust_anchor()?;
|
||
let odin_anchor = read_trust_anchor::<OdinTopologyIdentity>(&options.odin_trust_anchor)
|
||
.context("reading bootstrap Odin topology anchor")?;
|
||
let bootstrap_odin_authority = AdmittedOdinAuthority::from_anchor(&odin_anchor)?;
|
||
let source: Arc<dyn SourcePort> = Arc::new(GitSourceDriver::new(
|
||
&options.source_root,
|
||
options.staging_root.join("frozen-sources"),
|
||
options.source_identity,
|
||
));
|
||
let host_actuators = options
|
||
.host_actuator_bind
|
||
.map(|bind| HostActuatorHub::bind(bind).map(|hub| Arc::new(Mutex::new(hub))))
|
||
.transpose()?;
|
||
if let Some(hub) = &host_actuators {
|
||
eprintln!(
|
||
"Idunn host actuator hub listening on {}",
|
||
hub.lock().expect("hub mutex").local_addr()?
|
||
);
|
||
let service_signer =
|
||
open_service_identity_at::<IdunnServiceIdentity>(&options.idunn_identity_store)
|
||
.context("opening Idunn identity for the host actuator hub thread")?;
|
||
let bindings_dir = options.bindings_dir.clone();
|
||
spawn_hub_service(Arc::clone(hub), service_signer, move || {
|
||
host_anchors_from(&bindings_dir)
|
||
});
|
||
}
|
||
Ok(Self {
|
||
options,
|
||
idunn_signer,
|
||
idunn_anchor,
|
||
bootstrap_odin_authority,
|
||
source,
|
||
docker_runner: DockerRunnerDriver::default(),
|
||
systemd_workload,
|
||
host_actuators,
|
||
#[cfg(test)]
|
||
host_workload: Mutex::default(),
|
||
history_report: ReportOnce::default(),
|
||
fault_reports: Mutex::default(),
|
||
resume_backoff: Mutex::default(),
|
||
lift_held: Mutex::default(),
|
||
lift_unsettled: std::sync::atomic::AtomicBool::new(false),
|
||
challenge_wait: Mutex::default(),
|
||
})
|
||
}
|
||
|
||
fn host_anchors(&self) -> BTreeMap<String, ServiceIdentityTrustAnchor> {
|
||
host_anchors_from(&self.options.bindings_dir)
|
||
}
|
||
|
||
/// Every route driver comes from here, so the actuator paths are the
|
||
/// options' and nothing else's.
|
||
fn route_driver(&self, binding: RouteBinding) -> NginxRouteDriver {
|
||
NginxRouteDriver::with_actuators(binding, &self.options.route_actuators)
|
||
}
|
||
|
||
fn route_gate<'a>(&'a self, target: &'a str, command: CommandKind) -> EngineRouteGate<'a> {
|
||
EngineRouteGate {
|
||
engine: self,
|
||
target,
|
||
command,
|
||
}
|
||
}
|
||
|
||
/// Charge one route actuation to the target's `TargetSupervision` before
|
||
/// the driver acts. The record is created by the first charge, so a first
|
||
/// deployment is metered like any other. A `Forward` is counted against
|
||
/// the sliding ceiling; a refused one is the typed `RouteActuationRefused`,
|
||
/// and one that cannot be recorded is refused too: no charge, no
|
||
/// actuation. A `Survival` is never counted or refused, only recorded when
|
||
/// it can be, and runs regardless when it cannot: an unwritable store must
|
||
/// not stop a rollback, a withdrawal or a repair.
|
||
fn charge_route_actuation(&self, target: &str, kind: RouteActuation) -> Result<()> {
|
||
match self.record_route_actuation(target, kind) {
|
||
Ok(charged) => charged.map_err(anyhow::Error::new),
|
||
Err(error) if kind == RouteActuation::Survival => {
|
||
eprintln!(
|
||
"Idunn could not record a survival route actuation of {target}; it proceeds unrecorded: {error:#}"
|
||
);
|
||
Ok(())
|
||
}
|
||
Err(error) => Err(error),
|
||
}
|
||
}
|
||
|
||
/// The outer error is the ledger failing to read or write; the inner one
|
||
/// is the ceiling refusing.
|
||
fn record_route_actuation(
|
||
&self,
|
||
target: &str,
|
||
kind: RouteActuation,
|
||
) -> Result<Result<(), RouteActuationRefused>> {
|
||
let snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
let seen = snapshot.supervision_for(target);
|
||
let mut next = snapshot.supervision_or_new(target);
|
||
let now = now_millis()?;
|
||
let charged = next.charge_route(now, kind);
|
||
// A refusal writes nothing unless the clock had stepped back and the
|
||
// log needed settling.
|
||
if charged.is_ok() || seen.is_some_and(|stored| stored.value != next) {
|
||
self.write_target_supervision(seen, &next, now)?;
|
||
}
|
||
Ok(charged)
|
||
}
|
||
|
||
/// Replace a target's meters, or create them when `seen` is `None`, by
|
||
/// compare-exchange against exactly what was read.
|
||
fn write_target_supervision(
|
||
&self,
|
||
seen: Option<&Stored<TargetSupervision>>,
|
||
next: &TargetSupervision,
|
||
now: u64,
|
||
) -> Result<()> {
|
||
ensure!(
|
||
SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange(
|
||
&[CultCacheExpectedEnvelope {
|
||
r#type: TargetSupervision::TYPE.into(),
|
||
key: next.target.clone(),
|
||
current: seen.map(|stored| stored.envelope.clone()),
|
||
}],
|
||
&[target_supervision_envelope(next, now)?],
|
||
)?,
|
||
"target supervision changed before its meters were written"
|
||
);
|
||
Ok(())
|
||
}
|
||
|
||
/// Replace an admitted generation by compare-exchange against the envelope
|
||
/// that was read. `Ok(false)` is a lost race: another writer moved the
|
||
/// record, so the caller's snapshot is stale. `Err` is a write the store
|
||
/// refused, which changed nothing.
|
||
fn replace_generation(
|
||
&self,
|
||
seen: &Stored<AdmittedGeneration>,
|
||
next: &AdmittedGeneration,
|
||
now: u64,
|
||
) -> Result<bool> {
|
||
Ok(SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange(
|
||
&[CultCacheExpectedEnvelope {
|
||
r#type: AdmittedGeneration::TYPE.into(),
|
||
key: seen.value.target.clone(),
|
||
current: Some(seen.envelope.clone()),
|
||
}],
|
||
&[admitted_envelope(next, now)?],
|
||
)?)
|
||
}
|
||
|
||
/// Say a fault once while it lasts, under `key`.
|
||
fn report_once(&self, key: &str, message: String) {
|
||
let offered = self
|
||
.fault_reports
|
||
.lock()
|
||
.expect("fault report mutex")
|
||
.entry(key.to_string())
|
||
.or_default()
|
||
.offer(Some(message));
|
||
if let Some(message) = offered {
|
||
eprintln!("{message}");
|
||
}
|
||
}
|
||
|
||
fn incidents(&self) -> IncidentStore {
|
||
IncidentStore::new(&self.options.incident_store)
|
||
}
|
||
|
||
/// Settle the outcome of one incident operation. A failure never leaves
|
||
/// this function: it is traced once per episode, under the operation and
|
||
/// the subject it was decided for (`*` for a pass over the whole store),
|
||
/// and continuity decides exactly as it would have with a healthy store.
|
||
/// Each decision site has its own key, so one site's success never ends
|
||
/// another site's episode. The message names the operation and the root
|
||
/// cause and never a path or a value. Incident writes are never progress.
|
||
fn record_incident<T>(&self, operation: &str, subject: &str, outcome: Result<T>) {
|
||
let key = format!("incident-store:{operation}:{subject}");
|
||
match outcome {
|
||
Ok(_) => self.clear_fault(&key),
|
||
Err(error) => self.report_once(
|
||
&key,
|
||
format!(
|
||
"Idunn could not {operation} an operator incident: {}",
|
||
error.root_cause()
|
||
),
|
||
),
|
||
}
|
||
}
|
||
|
||
/// Close every open incident whose subject is a target that is no longer
|
||
/// admitted, then retire what has been closed past retention.
|
||
fn reconcile_incidents(&self, snapshot: &ControlSnapshot) {
|
||
let store = self.incidents();
|
||
let close_unadmitted = || -> Result<()> {
|
||
let now = now_millis()?;
|
||
for record in store.read()? {
|
||
if record.is_open()
|
||
&& record.condition.ends_with_admission()
|
||
&& snapshot.admitted_for(&record.subject).is_none()
|
||
{
|
||
store.close(
|
||
record.condition,
|
||
&record.subject,
|
||
CloseReason::NoLongerAdmitted,
|
||
now,
|
||
)?;
|
||
}
|
||
}
|
||
Ok(())
|
||
};
|
||
self.record_incident("close", "*", close_unadmitted());
|
||
self.record_incident("retire", "*", now_millis().and_then(|now| store.retire_closed(now)));
|
||
}
|
||
|
||
fn host_access(&self) -> Result<HostActuatorAccess<'_>> {
|
||
let hub = self
|
||
.host_actuators
|
||
.as_ref()
|
||
.context("this Idunn serves no host actuators (no --host-actuator-bind)")?;
|
||
Ok(HostActuatorAccess {
|
||
hub: Arc::clone(hub),
|
||
anchors: self.host_anchors(),
|
||
signer: &self.idunn_signer,
|
||
})
|
||
}
|
||
}
|
||
|
||
/// The trust anchor of every host a binding names. A binding whose anchor
|
||
/// cannot be read leaves its host unattachable and is logged; the other
|
||
/// hosts are unaffected.
|
||
fn host_anchors_from(bindings_dir: &Path) -> BTreeMap<String, ServiceIdentityTrustAnchor> {
|
||
{
|
||
let mut anchors = BTreeMap::new();
|
||
let bindings = match load_bindings(bindings_dir) {
|
||
Ok(bindings) => bindings,
|
||
Err(error) => {
|
||
eprintln!("Idunn cannot read bindings for host anchors: {error:#}");
|
||
return anchors;
|
||
}
|
||
};
|
||
for loaded in bindings.values() {
|
||
let WorkloadBinding::HostActuator(workload) = &loaded.binding.workload else {
|
||
continue;
|
||
};
|
||
match read_trust_anchor::<IdunnHostActuatorIdentity>(&workload.host_trust_anchor_store)
|
||
{
|
||
Ok(anchor) => {
|
||
anchors.insert(workload.host.clone(), anchor);
|
||
}
|
||
Err(error) => eprintln!(
|
||
"Idunn cannot read the actuator anchor for host {}: {error:#}",
|
||
workload.host
|
||
),
|
||
}
|
||
}
|
||
anchors
|
||
}
|
||
}
|
||
|
||
impl Engine {
|
||
/// The runner the plan's binding declares. Every runner in one binding
|
||
/// is of one kind; the binding validated that.
|
||
fn runner_for(&self, plan: &CompiledDeploymentPlan) -> Result<Box<dyn RunnerPort + '_>> {
|
||
let (_, binding) = plan.parsed_inputs()?;
|
||
Ok(match &binding.workload {
|
||
WorkloadBinding::SystemdTransient(_) => Box::new(self.docker_runner.clone()),
|
||
WorkloadBinding::HostActuator(_) => Box::new(HostActuatorRunnerDriver {
|
||
access: self.host_access()?,
|
||
}),
|
||
})
|
||
}
|
||
|
||
/// The workload driver the plan's binding declares.
|
||
fn workload_for(&self, plan: &CompiledDeploymentPlan) -> Result<Arc<dyn WorkloadPort + '_>> {
|
||
let (_, binding) = plan.parsed_inputs()?;
|
||
Ok(match &binding.workload {
|
||
WorkloadBinding::SystemdTransient(_) => Arc::clone(&self.systemd_workload),
|
||
WorkloadBinding::HostActuator(_) => {
|
||
#[cfg(test)]
|
||
if let Some(port) = self.host_workload.lock().expect("host workload mutex").clone() {
|
||
return Ok(port);
|
||
}
|
||
Arc::new(HostActuatorWorkloadDriver {
|
||
access: self.host_access()?,
|
||
})
|
||
}
|
||
})
|
||
}
|
||
|
||
/// History for a decision that would be wrong on an empty answer: the
|
||
/// continuity restart ceiling and the check that a command is consumed.
|
||
/// `None` means `history.cc` cannot be read, so the decision is not made;
|
||
/// a history that merely has no entries is `Some`. The fault is printed
|
||
/// once while it lasts, not on every tick.
|
||
fn history_for_decision(&self) -> Option<Vec<DeploymentTransaction>> {
|
||
let (report, transactions) = match read_history(&self.options.state_store) {
|
||
Ok(history) => (history.report(), Some(history.transactions)),
|
||
Err(error) => (
|
||
Some(format!(
|
||
"{error:#}; continuity and command retirement stop until it is readable"
|
||
)),
|
||
None,
|
||
),
|
||
};
|
||
if let Some(text) = self.history_report.offer(report) {
|
||
eprintln!("{text}");
|
||
}
|
||
transactions
|
||
}
|
||
|
||
fn topology(&self) -> CultCacheTopologyDriver {
|
||
CultCacheTopologyDriver {
|
||
projection_store: self.options.topology_store.clone(),
|
||
correlation_store: self.options.odin_correlation_store.clone(),
|
||
}
|
||
}
|
||
|
||
fn current_odin_authority(&self, snapshot: &ControlSnapshot) -> Result<AdmittedOdinAuthority> {
|
||
// The bootstrap key stands only until Odin itself is admitted. Once it
|
||
// is, its own generation names the authority; an admitted Odin that
|
||
// carries none has no authority to name, and falling back to the
|
||
// bootstrap key would let a stale key vouch for the Verse.
|
||
let authority = match snapshot.admitted_odin() {
|
||
None => self.bootstrap_odin_authority.clone(),
|
||
Some(stored) => stored.value.odin_authority.clone().with_context(|| {
|
||
"admitted Odin generation carries no Odin authority (route-proof readiness)"
|
||
})?,
|
||
};
|
||
authority.validate()?;
|
||
Ok(authority)
|
||
}
|
||
|
||
fn trusted_topology_context(&self, now: u64) -> OdinTopologyAuthenticationContext {
|
||
OdinTopologyAuthenticationContext {
|
||
trusted_received_at_unix_millis: now,
|
||
maximum_age_millis: self.options.topology_maximum_age_millis,
|
||
maximum_future_skew_millis: self.options.topology_maximum_future_skew_millis,
|
||
}
|
||
}
|
||
|
||
/// Re-prove the evidence that still gates a decision.
|
||
///
|
||
/// Terminal transactions are skipped: they describe what already happened
|
||
/// and authorize nothing further, so re-proving them only creates ways for
|
||
/// history to refuse a boot. On yggdrasil that is 259 of 260 records.
|
||
///
|
||
/// The operator anchor is read on demand rather than up front. Reading it
|
||
/// unconditionally meant an absent brake artifact refused startup against
|
||
/// an *empty* store -- a deployment brake gating Idunn itself, which
|
||
/// `F:\Projects\CLAUDE.md` forbids outright: a target brake "may never gate
|
||
/// Idunn itself or unrelated service lifecycle".
|
||
fn validate_durable_authority(&self, snapshot: &ControlSnapshot) -> Result<()> {
|
||
let mut operator_anchor = None;
|
||
for stored in &snapshot.transactions {
|
||
let transaction = &stored.value;
|
||
if transaction.is_terminal() {
|
||
continue;
|
||
}
|
||
if let Some(authorization) = &transaction.deployment_authorization {
|
||
authorization.validate_shape()?;
|
||
let record: IdunnDeploymentBrakeRecord =
|
||
rmp_serde::from_slice(&authorization.canonical_brake_bytes)?;
|
||
if operator_anchor.is_none() {
|
||
operator_anchor =
|
||
Some(read_trust_anchor::<IdunnDeploymentBrakeOperatorIdentity>(
|
||
&self.options.deployment_brake_operator_anchor,
|
||
)?);
|
||
}
|
||
verify_idunn_deployment_brake_authorization(
|
||
&record,
|
||
operator_anchor.as_ref().expect("read directly above"),
|
||
)?;
|
||
let expected = required(&transaction.expected, "authorized Expected projection")?;
|
||
ensure!(
|
||
record.authorized_release_id.as_deref()
|
||
== Some(expected.sealed_release_id.as_str())
|
||
&& record.authorized_deployment_id.as_deref()
|
||
== Some(transaction.transaction_id.as_str())
|
||
&& record.runtime_id == expected.runtime_id,
|
||
"durable deployment authorization names another release or transaction"
|
||
);
|
||
}
|
||
let lease = transaction
|
||
.leasing
|
||
.as_ref()
|
||
.and_then(LeasingEvidence::lease_sha256);
|
||
if let Some(evidence) = &transaction.latest_odin_observation {
|
||
let authenticated = self.authenticate_topology_bytes(
|
||
snapshot,
|
||
transaction,
|
||
&evidence.canonical_bytes,
|
||
lease,
|
||
evidence.admitted_at_unix_millis,
|
||
)?;
|
||
validate_authenticated_evidence(evidence, &authenticated)?;
|
||
}
|
||
if let Some(evidence) = &transaction.warming {
|
||
match evidence {
|
||
WarmingEvidence::OdinTopology { evidence } => {
|
||
let authenticated = self.authenticate_topology_bytes(
|
||
snapshot,
|
||
transaction,
|
||
&evidence.canonical_bytes,
|
||
None,
|
||
evidence.admitted_at_unix_millis,
|
||
)?;
|
||
validate_authenticated_evidence(evidence, &authenticated)?;
|
||
let incumbent_lease_sha256 =
|
||
self.incumbent_lease_sha256_for_warming(snapshot, transaction)?;
|
||
ensure!(
|
||
is_semantic_warming(
|
||
required(&transaction.expected, "Warming Expected projection",)?,
|
||
required(&transaction.activation, "Warming activation")?,
|
||
incumbent_lease_sha256.as_deref(),
|
||
&authenticated,
|
||
)?,
|
||
"durable Warming gate is not supported by current runtime evidence"
|
||
);
|
||
}
|
||
WarmingEvidence::FirstOdinDirect { evidence } => {
|
||
self.authenticate_first_odin_warming_presence(
|
||
transaction,
|
||
&evidence.message_id,
|
||
evidence.challenged_at_unix_millis,
|
||
evidence.admitted_at_unix_millis,
|
||
&evidence.canonical_bytes,
|
||
)?;
|
||
}
|
||
WarmingEvidence::RouteProofDirect { evidence } => {
|
||
self.reauthenticate_route_proof(
|
||
transaction,
|
||
evidence,
|
||
route_proof_warming_states(required(
|
||
&transaction.expected,
|
||
"Warming Expected projection",
|
||
)?),
|
||
None,
|
||
evidence.admitted_at_unix_millis,
|
||
)?;
|
||
}
|
||
}
|
||
}
|
||
if let Some(ReadinessEvidence::RouteProof { evidence }) = &transaction.ready {
|
||
self.reauthenticate_route_proof(
|
||
transaction,
|
||
evidence,
|
||
&["active"],
|
||
lease,
|
||
evidence.admitted_at_unix_millis,
|
||
)?;
|
||
}
|
||
if let Some(evidence) = transaction.ready.as_ref().and_then(ReadinessEvidence::odin) {
|
||
let authenticated = self.authenticate_topology_bytes(
|
||
snapshot,
|
||
transaction,
|
||
&evidence.canonical_bytes,
|
||
lease,
|
||
evidence.admitted_at_unix_millis,
|
||
)?;
|
||
validate_authenticated_evidence(evidence, &authenticated)?;
|
||
ensure!(
|
||
is_semantic_ready(&authenticated),
|
||
"durable Ready label is not exact semantic Ready"
|
||
);
|
||
}
|
||
}
|
||
let odin_authority = self.current_odin_authority(snapshot)?;
|
||
for stored in &snapshot.admitted {
|
||
let generation = &stored.value;
|
||
// Only a generation whose own Expected says Odin vouches for it has
|
||
// Odin receipts to re-prove. One whose evidence disagrees with its
|
||
// class is held and reported by supervision, not re-proved here.
|
||
if !matches!(generation.readiness(), Ok(class) if class != ReadinessClass::RouteProof)
|
||
{
|
||
continue;
|
||
}
|
||
let odin = generation.odin_receipts()?;
|
||
let authority = self.runtime_authority_parts(
|
||
&generation.plan,
|
||
&generation.expected,
|
||
&generation.activation,
|
||
)?;
|
||
let latest = authenticate_odin_runtime_topology_correlation(
|
||
&odin.latest.canonical_bytes,
|
||
&authority,
|
||
generation.leasing.lease_sha256(),
|
||
&odin_authority.signer_public_key,
|
||
self.trusted_topology_context(odin.latest.admitted_at_unix_millis),
|
||
)?;
|
||
validate_authenticated_evidence(odin.latest, &latest)?;
|
||
let ready = authenticate_odin_runtime_topology_correlation(
|
||
&odin.ready.canonical_bytes,
|
||
&authority,
|
||
generation.leasing.lease_sha256(),
|
||
&odin_authority.signer_public_key,
|
||
self.trusted_topology_context(odin.ready.admitted_at_unix_millis),
|
||
)?;
|
||
validate_authenticated_evidence(odin.ready, &ready)?;
|
||
ensure!(
|
||
is_semantic_ready(&ready),
|
||
"admitted generation Ready label is not exact semantic Ready"
|
||
);
|
||
}
|
||
Ok(())
|
||
}
|
||
}
|
||
|
||
/// What the boot reconciliation of failed continuity projections found.
|
||
#[derive(Debug, Default, PartialEq, Eq)]
|
||
struct ProjectionReconciliation {
|
||
/// Failed continuity transactions whose exact activation was demoted.
|
||
demoted: Vec<String>,
|
||
/// Targets whose projection names an activation that neither the admitted
|
||
/// generation, a live transaction, nor a failed issuer accounts for. They
|
||
/// are reported and left alone.
|
||
unexplained: Vec<String>,
|
||
}
|
||
|
||
/// What the lift decided about one continuity abort.
|
||
enum LiftDecision {
|
||
/// The abort was withdrawn and the record retimed.
|
||
Lifted,
|
||
/// The abort stands and runs: not a lift candidate, or its candidate is
|
||
/// provably dead.
|
||
KeepAborting,
|
||
/// The candidate could not be observed and is not provably dead.
|
||
Undecided(anyhow::Error),
|
||
}
|
||
|
||
impl Engine {
|
||
/// Boot's call of the lift. A lift that errs decided nothing, so boot fails
|
||
/// closed: every continuity abort is held, none of its steps run, until
|
||
/// `retry_held_lift` completes a lift.
|
||
fn boot_lift(&self) {
|
||
match self.return_wedged_continuity_aborts_forward() {
|
||
Ok(returned) => {
|
||
for transaction_id in &returned {
|
||
println!(
|
||
"returned continuity {transaction_id} to forward retry: its abort had not stopped its running candidate"
|
||
);
|
||
}
|
||
}
|
||
Err(error) => {
|
||
self.lift_unsettled
|
||
.store(true, std::sync::atomic::Ordering::SeqCst);
|
||
eprintln!(
|
||
"Idunn boot could not lift wedged continuity aborts; every continuity abort is held until a lift succeeds: {error:#}"
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Once at boot: withdraw the abort of a continuity that has not stopped
|
||
/// its running candidate, and retime the record so its phase retries
|
||
/// forward. A pre-c2c Idunn aborted a continuity at its deadline; that
|
||
/// abort stops the only running copy of the admitted body (ruling
|
||
/// c2c-boot-returns-wedged-aborts-forward). The result is the exact inverse
|
||
/// of `begin_post_fencing_abort`.
|
||
///
|
||
/// A continuity abort now begins only over a permanently stopped candidate
|
||
/// or to yield to a deployment awaiting the handover, so this predicate
|
||
/// matches only aborts over a running candidate with nothing awaiting the
|
||
/// target, which Idunn no longer creates: it is a migration with a deletion
|
||
/// line (follow-up c2c-delete-boot-lift). A record whose candidate cannot
|
||
/// be observed is held, not aborted: `resume_candidate` runs no step of it
|
||
/// and asks `retry_held_lift` each pass, until observation answers (ruling
|
||
/// c2c-lift-defers-on-observe-error).
|
||
fn return_wedged_continuity_aborts_forward(&self) -> Result<Vec<String>> {
|
||
let snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
let mut returned = Vec::new();
|
||
for stored in &snapshot.transactions {
|
||
let id = &stored.value.transaction_id;
|
||
match self.consider_lifting_continuity_abort(&snapshot, stored) {
|
||
LiftDecision::Lifted => returned.push(id.clone()),
|
||
LiftDecision::KeepAborting => {}
|
||
LiftDecision::Undecided(error) => {
|
||
eprintln!(
|
||
"Idunn could not decide whether continuity {id} may retry forward; its abort is held, not run: {error:#}"
|
||
);
|
||
self.lift_held.lock().expect("lift held mutex").insert(id.clone());
|
||
}
|
||
}
|
||
}
|
||
Ok(returned)
|
||
}
|
||
|
||
/// A held record's later pass: lift it, release it to its abort over
|
||
/// positive evidence of death, or keep holding. `Some(moved)` means the
|
||
/// pass is spent on the record; `None` lets the pass go on.
|
||
fn retry_held_lift(&self, current: &Stored<DeploymentTransaction>) -> Result<Option<bool>> {
|
||
let id = current.value.transaction_id.as_str();
|
||
if current.value.command_kind == CommandKind::Continuity
|
||
&& current.value.post_fencing_abort.is_some()
|
||
&& self.lift_unsettled.load(std::sync::atomic::Ordering::SeqCst)
|
||
{
|
||
match self.return_wedged_continuity_aborts_forward() {
|
||
Err(error) => {
|
||
self.note_fault("holds continuity abort", id, &error);
|
||
return Ok(Some(false));
|
||
}
|
||
Ok(returned) => {
|
||
self.lift_unsettled
|
||
.store(false, std::sync::atomic::Ordering::SeqCst);
|
||
for transaction_id in &returned {
|
||
println!(
|
||
"returned continuity {transaction_id} to forward retry: its abort had not stopped its running candidate"
|
||
);
|
||
}
|
||
if returned.iter().any(|lifted| lifted == id) {
|
||
return Ok(Some(true));
|
||
}
|
||
}
|
||
}
|
||
}
|
||
if !self.lift_held.lock().expect("lift held mutex").contains(id) {
|
||
return Ok(None);
|
||
}
|
||
let snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
match self.consider_lifting_continuity_abort(&snapshot, current) {
|
||
LiftDecision::Undecided(error) => {
|
||
self.note_fault("holds continuity abort", id, &error);
|
||
Ok(Some(false))
|
||
}
|
||
LiftDecision::Lifted => {
|
||
self.lift_held.lock().expect("lift held mutex").remove(id);
|
||
self.clear_fault(id);
|
||
println!(
|
||
"returned continuity {id} to forward retry: its abort had not stopped its running candidate"
|
||
);
|
||
Ok(Some(true))
|
||
}
|
||
LiftDecision::KeepAborting => {
|
||
self.lift_held.lock().expect("lift held mutex").remove(id);
|
||
self.clear_fault(id);
|
||
Ok(None)
|
||
}
|
||
}
|
||
}
|
||
|
||
/// The lift's predicate and write for one record. An abort proceeds only
|
||
/// over positive evidence that its candidate is dead; an observation that
|
||
/// merely errs is undecided.
|
||
fn consider_lifting_continuity_abort(
|
||
&self,
|
||
snapshot: &ControlSnapshot,
|
||
stored: &Stored<DeploymentTransaction>,
|
||
) -> LiftDecision {
|
||
let transaction = &stored.value;
|
||
let Some(abort) = &transaction.post_fencing_abort else {
|
||
return LiftDecision::KeepAborting;
|
||
};
|
||
if transaction.command_kind != CommandKind::Continuity
|
||
|| transaction.completion.is_some()
|
||
|| abort.candidate_cleanup != CleanupEvidence::Pending
|
||
|| abort.lease_withdrawal == CleanupEvidence::Complete
|
||
{
|
||
return LiftDecision::KeepAborting;
|
||
}
|
||
match now_millis() {
|
||
Ok(now) => {
|
||
if self
|
||
.deploy_the_continuity_yields_to(snapshot, transaction, now)
|
||
.is_some()
|
||
{
|
||
return LiftDecision::KeepAborting;
|
||
}
|
||
}
|
||
Err(error) => return LiftDecision::Undecided(error),
|
||
}
|
||
let lifted = (|| -> Result<bool> {
|
||
let plan = required(&transaction.plan, "transaction plan")?;
|
||
let expected = required(&transaction.expected, "Expected projection")?;
|
||
let activation = required(&transaction.activation, "activation")?;
|
||
let Some(workload) = &transaction.workload else {
|
||
return Ok(false);
|
||
};
|
||
self.workload_for(plan)?
|
||
.observe(expected, activation, workload)?;
|
||
if let Some(lease) = transaction
|
||
.leasing
|
||
.as_ref()
|
||
.and_then(LeasingEvidence::issued_lease)
|
||
{
|
||
let path = plan
|
||
.parsed_inputs()?
|
||
.1
|
||
.process_write_lease
|
||
.context("leased target has no write-lease binding")?
|
||
.record_path;
|
||
if !CultCacheWriteLeaseDriver::new(&transaction.target, path)
|
||
.observe_exact(lease)?
|
||
{
|
||
return Ok(false);
|
||
}
|
||
}
|
||
let now = now_millis()?;
|
||
let mut next = transaction.clone();
|
||
next.post_fencing_abort = None;
|
||
next.last_error = Some(abort.error.clone());
|
||
next.updated_at_unix_millis = now;
|
||
next.retime(now);
|
||
replace_transaction(&self.options.state_store, stored, &next)?;
|
||
Ok(true)
|
||
})();
|
||
match lifted {
|
||
Ok(true) => LiftDecision::Lifted,
|
||
Ok(false) => LiftDecision::KeepAborting,
|
||
Err(error) => match self.candidate_is_permanently_stopped(transaction) {
|
||
Ok(true) => LiftDecision::KeepAborting,
|
||
_ => LiftDecision::Undecided(error),
|
||
},
|
||
}
|
||
}
|
||
|
||
/// Once at boot: demote the activation a failed continuity left standing,
|
||
/// where the projection still names exactly the activation that failed
|
||
/// transaction issued. A continuity abort before the single resolution
|
||
/// rule left such residue (and the lift owes the same demotion to any
|
||
/// abort still resident). The issuer is identified by exact activation,
|
||
/// from history and from resident terminal records not yet retired to it.
|
||
///
|
||
/// Nothing is adopted: an activation with no failed issuer is never
|
||
/// demoted or kept on anyone's behalf, only reported.
|
||
fn reconcile_failed_continuity_projections(&self) -> Result<ProjectionReconciliation> {
|
||
let snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
let mut outcome = ProjectionReconciliation::default();
|
||
// Resident issuers need no history; only the history issuers wait on it.
|
||
let history = self.history_for_decision();
|
||
if history.is_none() {
|
||
eprintln!(
|
||
"Idunn cannot read history at boot: failed continuities archived there are not \
|
||
reconciled, resident ones still are"
|
||
);
|
||
}
|
||
let topology = self.topology();
|
||
let issuers = history
|
||
.iter()
|
||
.flatten()
|
||
.chain(snapshot.transactions.iter().map(|stored| &stored.value))
|
||
.filter(|transaction| {
|
||
transaction.command_kind == CommandKind::Continuity
|
||
&& matches!(
|
||
transaction.completion,
|
||
Some(TransactionCompletion::FailedBeforeFencing { .. })
|
||
| Some(TransactionCompletion::FailedAfterFencing { .. })
|
||
)
|
||
});
|
||
for issuer in issuers {
|
||
let (Some(expected), Some(activation), Some(plan)) =
|
||
(&issuer.expected, &issuer.activation, &issuer.plan)
|
||
else {
|
||
continue;
|
||
};
|
||
let demoted = (|| -> Result<bool> {
|
||
if topology.projected_activation(expected)?.as_ref() != Some(activation) {
|
||
return Ok(false);
|
||
}
|
||
let provider_anchor = self.provider_anchor_for_plan(plan)?;
|
||
topology.demote_to_expected_only(expected, &provider_anchor, activation, None)?;
|
||
Ok(true)
|
||
})();
|
||
match demoted {
|
||
Ok(true) => outcome.demoted.push(issuer.transaction_id.clone()),
|
||
Ok(false) => {}
|
||
Err(error) => eprintln!(
|
||
"Idunn could not reconcile the projection of failed continuity {}: {error:#}",
|
||
issuer.transaction_id
|
||
),
|
||
}
|
||
}
|
||
for generation in &snapshot.admitted {
|
||
let projected = match topology.projected_activation(&generation.value.expected) {
|
||
Ok(projected) => projected,
|
||
Err(error) => {
|
||
eprintln!(
|
||
"Idunn cannot read the projected activation of {}: {error:#}",
|
||
generation.value.target
|
||
);
|
||
continue;
|
||
}
|
||
};
|
||
let Some(projected) = projected else {
|
||
continue;
|
||
};
|
||
let accounted = projected == generation.value.activation
|
||
|| snapshot.transactions.iter().any(|stored| {
|
||
stored.value.target == generation.value.target
|
||
&& !stored.value.is_terminal()
|
||
&& stored.value.activation.as_ref() == Some(&projected)
|
||
});
|
||
if !accounted {
|
||
outcome.unexplained.push(generation.value.target.clone());
|
||
}
|
||
}
|
||
Ok(outcome)
|
||
}
|
||
}
|
||
|
||
/// Everything the daemon does once, before its loop: lock, migrate, validate,
|
||
/// reconcile the projection. The lock is returned so the caller holds it for as
|
||
/// long as the engine runs.
|
||
fn boot(options: RuntimeOptions) -> Result<(ProcessLock, Engine)> {
|
||
for path in [
|
||
&options.state_store,
|
||
&options.topology_store,
|
||
&options.staging_root,
|
||
] {
|
||
let directory = if path.extension().is_some() {
|
||
path.parent()
|
||
.context("configured Idunn path has no parent")?
|
||
} else {
|
||
path.as_path()
|
||
};
|
||
fs::create_dir_all(directory)
|
||
.with_context(|| format!("creating Idunn directory {}", directory.display()))?;
|
||
}
|
||
let lock = ProcessLock::acquire(&options.state_store)?;
|
||
let migrated = migrate_control_store_to_current_schema(&options.state_store)
|
||
.context("migrating Idunn control records to the current schema")?;
|
||
if migrated > 0 {
|
||
println!(
|
||
"migrated {migrated} control record(s) to {DEPLOYMENT_TRANSACTION_SCHEMA} / {ADMITTED_GENERATION_SCHEMA} / {TARGET_SUPERVISION_SCHEMA}"
|
||
);
|
||
}
|
||
ControlSnapshot::read(&options.state_store).context("validating all Idunn records")?;
|
||
let engine = Engine::open(options)?;
|
||
engine.validate_durable_authority(&ControlSnapshot::read(&engine.options.state_store)?)?;
|
||
engine.boot_lift();
|
||
match engine.reconcile_failed_continuity_projections() {
|
||
Ok(outcome) => {
|
||
for transaction_id in &outcome.demoted {
|
||
println!("demoted the activation failed continuity {transaction_id} left projected");
|
||
}
|
||
for target in &outcome.unexplained {
|
||
eprintln!(
|
||
"Idunn found an activation projected for {target} that no admitted generation, \
|
||
live transaction or failed continuity accounts for; left as it is"
|
||
);
|
||
}
|
||
}
|
||
Err(error) => eprintln!("Idunn boot projection reconciliation failed: {error:#}"),
|
||
}
|
||
Ok((lock, engine))
|
||
}
|
||
|
||
fn serve(options: RuntimeOptions) -> Result<()> {
|
||
let (_lock, engine) = boot(options)?;
|
||
serve_while(&engine, || true)
|
||
}
|
||
|
||
/// The daemon's loop: tick, sleep, and go again while `again` says so. `serve`
|
||
/// says so forever; a test says so a few times.
|
||
fn serve_while(engine: &Engine, mut again: impl FnMut() -> bool) -> Result<()> {
|
||
while again() {
|
||
match engine.run_scheduler_tick() {
|
||
Ok(true) => continue,
|
||
Ok(false) => {}
|
||
// A fault in one transaction is that transaction's problem. Idunn
|
||
// runs under Restart=always, so propagating it here turns a single
|
||
// unreadable record into a crashloop that takes every unrelated
|
||
// target's continuity down with it -- the daemon-survival organ
|
||
// killed by the thing it exists to survive. Log against the tick
|
||
// and keep going; the fault recurs every poll until it is fixed,
|
||
// which is louder than a restart loop and cheaper than an outage.
|
||
Err(error) => eprintln!("Idunn scheduler tick failed: {error:#}"),
|
||
}
|
||
thread::sleep(Duration::from_millis(engine.options.poll_millis));
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
impl Engine {
|
||
/// One resident terminal transaction goes to history per tick.
|
||
///
|
||
/// Transactions that finished before finished transactions travelled to
|
||
/// history are still resident with their commands, gating nothing and
|
||
/// consuming their commands correctly, but growing the store every decision
|
||
/// reads. They leave the same way a transaction finishing today does.
|
||
fn retire_one_terminal_transaction(&self) -> Result<bool> {
|
||
let snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
// A record that cannot be archived (history unreadable, a torn write)
|
||
// is that record's fault: the next terminal record is still tried, and
|
||
// the tick goes on to supervise and freeze.
|
||
for stored in snapshot
|
||
.transactions
|
||
.iter()
|
||
.filter(|stored| stored.value.is_terminal())
|
||
{
|
||
match archive_terminal_transaction(&self.options.state_store, &stored.envelope) {
|
||
Ok(()) => {
|
||
eprintln!(
|
||
"Idunn retired resident terminal transaction {} to history",
|
||
stored.value.transaction_id
|
||
);
|
||
return Ok(true);
|
||
}
|
||
Err(error) => self.note_fault(
|
||
"could not retire terminal transaction",
|
||
&stored.value.transaction_id,
|
||
&error,
|
||
),
|
||
}
|
||
}
|
||
Ok(false)
|
||
}
|
||
|
||
/// One pass of the scheduler. An unreadable control store is the tick's
|
||
/// fault and is returned; anything one transaction or one admitted
|
||
/// generation does wrong is logged against it and never stops the others.
|
||
fn run_scheduler_tick(&self) -> Result<bool> {
|
||
if self.retire_one_terminal_transaction()? {
|
||
return Ok(true);
|
||
}
|
||
let transaction_progress = self.resume_one_transaction()?;
|
||
let continuity_progress = self.supervise_one_admitted_generation()?;
|
||
// Freezing is not gated on the others' progress: a queued command
|
||
// whose target is free must not wait behind an unrelated target's
|
||
// wedge. `freeze_command` refuses only a second live deployment of a
|
||
// target, and this freezes at most one command per tick.
|
||
let froze = self.freeze_one_queued_command()?;
|
||
Ok(transaction_progress || continuity_progress || froze)
|
||
}
|
||
|
||
/// Say a per-record fault once while it lasts and leave it in the
|
||
/// record's `last_error`, which status renders. Recording is best effort:
|
||
/// a store too broken to take the note is already in stderr.
|
||
fn note_fault(&self, what: &str, transaction_id: &str, error: &anyhow::Error) {
|
||
let detail = bounded_detail(&format!("{error:#}"));
|
||
let offered = {
|
||
let mut reports = self.fault_reports.lock().expect("fault report mutex");
|
||
reports
|
||
.entry(transaction_id.to_string())
|
||
.or_default()
|
||
.offer(Some(detail.clone()))
|
||
};
|
||
let recorded = self.record_last_error(transaction_id, &detail);
|
||
if offered.is_some() {
|
||
eprintln!("Idunn {what} {transaction_id}: {detail}");
|
||
if let Err(error) = recorded {
|
||
eprintln!("Idunn could not leave that in the record of {transaction_id}: {error:#}");
|
||
}
|
||
}
|
||
}
|
||
|
||
fn clear_fault(&self, transaction_id: &str) {
|
||
self.fault_reports
|
||
.lock()
|
||
.expect("fault report mutex")
|
||
.remove(transaction_id);
|
||
}
|
||
|
||
/// Write `detail` as the record's `last_error`, touching nothing else and
|
||
/// not archiving: a terminal record whose archive is the fault stays put.
|
||
fn record_last_error(&self, transaction_id: &str, detail: &str) -> Result<()> {
|
||
let detail = bounded_detail(detail);
|
||
let detail = detail.as_str();
|
||
let snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
let Some(stored) = snapshot
|
||
.transactions
|
||
.iter()
|
||
.find(|stored| stored.value.transaction_id == transaction_id)
|
||
else {
|
||
return Ok(());
|
||
};
|
||
if stored.value.last_error.as_deref() == Some(detail) {
|
||
return Ok(());
|
||
}
|
||
let envelope = cleanup_evidence::last_error_envelope(&stored.value, detail)?;
|
||
ensure!(
|
||
SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange(
|
||
&[CultCacheExpectedEnvelope {
|
||
r#type: DeploymentTransaction::TYPE.into(),
|
||
key: transaction_id.to_string(),
|
||
current: Some(stored.envelope.clone()),
|
||
}],
|
||
&[envelope],
|
||
)?,
|
||
"transaction changed before its fault note"
|
||
);
|
||
Ok(())
|
||
}
|
||
|
||
/// Startup and every later loop use the same order: unfinished ownership
|
||
/// work first, admitted-body continuity second, new commands last. A
|
||
/// waiting transaction yields without relaxing ordinal order inside its
|
||
/// own command, so one target brake cannot suspend unrelated continuity.
|
||
fn resume_one_transaction(&self) -> Result<bool> {
|
||
let snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
let mut progressed = self.retire_spent_expiry_requests(&snapshot)?;
|
||
// Incident writes are never progress.
|
||
self.close_spent_operator_incidents(&snapshot, now_millis()?);
|
||
let mut candidates = snapshot
|
||
.transactions
|
||
.iter()
|
||
.filter(|stored| !stored.value.is_terminal())
|
||
.collect::<Vec<_>>();
|
||
candidates.sort_by_key(|stored| {
|
||
(
|
||
stored.value.created_at_unix_millis,
|
||
stored.value.ordinal,
|
||
stored.value.transaction_id.as_str(),
|
||
)
|
||
});
|
||
for current in candidates {
|
||
if snapshot.has_earlier_authority_sibling(¤t.value) {
|
||
continue;
|
||
}
|
||
match self.resume_candidate(current) {
|
||
Ok(moved) => progressed |= moved,
|
||
Err(error) => {
|
||
self.note_fault(
|
||
"could not resume transaction",
|
||
¤t.value.transaction_id,
|
||
&error,
|
||
);
|
||
}
|
||
}
|
||
}
|
||
Ok(progressed)
|
||
}
|
||
|
||
/// Advance one transaction, turning its failure into the durable record
|
||
/// its phase calls for. Returns whether its record changed.
|
||
///
|
||
/// "Moved" means the transaction's state advanced: a phase change, a
|
||
/// completion, or a durable abort step. Rewriting `last_error` to say the
|
||
/// same step failed again in new words is not movement, or a wedged
|
||
/// transaction would report progress every tick.
|
||
fn resume_candidate(&self, current: &Stored<DeploymentTransaction>) -> Result<bool> {
|
||
let id = current.value.transaction_id.as_str();
|
||
if let Some(moved) = self.retry_held_lift(current)? {
|
||
return Ok(moved);
|
||
}
|
||
if let Some(moved) = self.resolve_phase_deadline(current)? {
|
||
if moved {
|
||
// A phase-ending write is movement: the next step is not held
|
||
// behind the backoff a forward retry armed.
|
||
self.resume_backoff.lock().expect("resume backoff mutex").remove(id);
|
||
}
|
||
return Ok(moved);
|
||
}
|
||
if self
|
||
.resume_backoff
|
||
.lock()
|
||
.expect("resume backoff mutex")
|
||
.get(id)
|
||
.is_some_and(|backoff| {
|
||
now_millis().is_ok_and(|now| {
|
||
is_waiting(now, backoff.not_before_unix_millis, backoff.ceiling_millis)
|
||
})
|
||
})
|
||
{
|
||
return Ok(false);
|
||
}
|
||
if let Some(disagreement) = current.value.held_disagreement() {
|
||
self.note_fault("holds transaction", id, &anyhow::Error::new(disagreement));
|
||
return Ok(false);
|
||
}
|
||
let left = now_millis().is_ok_and(|now| current.value.leaves_to_operator(now));
|
||
let advanced = self.advance_transaction(current);
|
||
if advanced.is_ok() {
|
||
// A step that returns Ok without moving a record left to the
|
||
// operator (a gate-wait on a silent candidate) is an attempt like
|
||
// a failed one: it backs off, and the fault it recorded stands.
|
||
let unmoved = left
|
||
&& ControlSnapshot::read(&self.options.state_store)?
|
||
.transactions
|
||
.iter()
|
||
.find(|stored| stored.value.transaction_id == id)
|
||
.is_some_and(|live| !state_advanced(¤t.value, &live.value));
|
||
if unmoved {
|
||
self.back_off(¤t.value)?;
|
||
} else {
|
||
// The step ran and succeeded: the fault, if any, has recovered, so
|
||
// a later failure is news. A skipped or failed attempt clears nothing.
|
||
self.resume_backoff
|
||
.lock()
|
||
.expect("resume backoff mutex")
|
||
.remove(id);
|
||
self.clear_fault(id);
|
||
}
|
||
}
|
||
if let Err(error) = advanced {
|
||
let latest_snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
let latest = latest_snapshot
|
||
.transactions
|
||
.iter()
|
||
.find(|stored| stored.value.transaction_id == current.value.transaction_id)
|
||
.context("transaction disappeared while recording an execution error")?;
|
||
if latest.value.is_terminal() {
|
||
self.resume_backoff
|
||
.lock()
|
||
.expect("resume backoff mutex")
|
||
.remove(id);
|
||
return Ok(true);
|
||
}
|
||
self.back_off(&latest.value)?;
|
||
if latest.value.phase < DeploymentPhase::Fencing {
|
||
// Before the fence an abort that cannot finish a step is
|
||
// resumable, never a post-fence abort: that path refuses a
|
||
// pre-fence phase, so choosing it only wedged the tick.
|
||
//
|
||
// A publish that lost to a held lock is not a failed step: the
|
||
// holder lets go, so the transaction stays in its phase with
|
||
// the error recorded and the backoff above retries it. The
|
||
// phase deadline, not an abort, bounds the wait.
|
||
if latest.value.pre_fencing_abort.is_none()
|
||
&& error.downcast_ref::<LockContended>().is_none()
|
||
{
|
||
self.begin_pre_fencing_abort(latest, error)?;
|
||
} else {
|
||
self.record_resumable_error(latest, &error)?;
|
||
}
|
||
} else if latest.value.post_fencing_abort.is_none()
|
||
&& self.candidate_is_permanently_stopped(&latest.value)?
|
||
{
|
||
// Past the fence an error is resumable while the candidate
|
||
// can still recover. This one cannot: its transient unit
|
||
// has failed and carries Restart=no, so retrying would hold
|
||
// the target forever behind a transaction that can never
|
||
// finish.
|
||
self.begin_post_fencing_abort(latest, error)?;
|
||
} else {
|
||
// Resumable only until the phase deadline: the resolver ends
|
||
// the phase before the next step once it has passed.
|
||
self.record_resumable_error(latest, &error)?;
|
||
}
|
||
}
|
||
let after = ControlSnapshot::read(&self.options.state_store)?;
|
||
let Some(live) = after
|
||
.transactions
|
||
.iter()
|
||
.find(|stored| stored.value.transaction_id == current.value.transaction_id)
|
||
else {
|
||
// The step that ends a transaction archives it inside its own
|
||
// write, so it is gone from the live set because it finished: that
|
||
// is the step's progress. Gone from both is a fault.
|
||
ensure!(
|
||
read_history_transactions(&self.options.state_store)
|
||
.iter()
|
||
.any(|archived| archived.transaction_id == id && archived.is_terminal()),
|
||
"transaction disappeared while checking scheduler progress"
|
||
);
|
||
return Ok(true);
|
||
};
|
||
Ok(state_advanced(¤t.value, &live.value))
|
||
}
|
||
|
||
/// Wait twice as long after each consecutive failure of a transaction's
|
||
/// step, from one poll interval up to a ceiling.
|
||
fn back_off(&self, transaction: &DeploymentTransaction) -> Result<()> {
|
||
let now = now_millis()?;
|
||
let ceiling_millis = resume_ceiling(transaction, now);
|
||
let mut all = self.resume_backoff.lock().expect("resume backoff mutex");
|
||
let failures = all
|
||
.get(&transaction.transaction_id)
|
||
.map_or(0, |backoff| backoff.failures);
|
||
let wait = backoff_wait(self.options.poll_millis, failures, ceiling_millis);
|
||
all.insert(
|
||
transaction.transaction_id.clone(),
|
||
ResumeBackoff {
|
||
failures: failures.saturating_add(1),
|
||
not_before_unix_millis: now.saturating_add(wait),
|
||
ceiling_millis,
|
||
},
|
||
);
|
||
Ok(())
|
||
}
|
||
|
||
fn freeze_one_queued_command(&self) -> Result<bool> {
|
||
let snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
let live_commands = snapshot
|
||
.transactions
|
||
.iter()
|
||
.map(|stored| stored.value.command_id.clone())
|
||
.collect::<BTreeSet<_>>();
|
||
let mut candidates = snapshot
|
||
.commands
|
||
.iter()
|
||
.filter(|stored| {
|
||
stored.value.kind == CommandKind::Deploy
|
||
&& !live_commands.contains(&stored.value.command_id)
|
||
})
|
||
.collect::<Vec<_>>();
|
||
if candidates.is_empty() {
|
||
return Ok(false);
|
||
}
|
||
// A command is consumed by any transaction, live or retired. Commands
|
||
// whose transactions were retired to history before commands travelled
|
||
// with them are still resident; they are not queued, they are
|
||
// history that never moved. Retire them here and never freeze them.
|
||
// With history unreadable, whether a command was consumed is unknown:
|
||
// freezing it again could run a finished deployment twice.
|
||
let Some(history) = self.history_for_decision() else {
|
||
return Ok(false);
|
||
};
|
||
let historical_commands = history
|
||
.into_iter()
|
||
.map(|transaction| transaction.command_id)
|
||
.collect::<BTreeSet<_>>();
|
||
let mut retired = false;
|
||
candidates.retain(|stored| {
|
||
if !historical_commands.contains(&stored.value.command_id) {
|
||
return true;
|
||
}
|
||
match SingleFileMessagePackBackingStore::new(&history_store_path(
|
||
&self.options.state_store,
|
||
))
|
||
.insert_entry_if_absent(stored.envelope.clone())
|
||
.and_then(|_| {
|
||
SingleFileMessagePackBackingStore::new(&self.options.state_store)
|
||
.delete_batch_if_unchanged(std::slice::from_ref(&stored.envelope))
|
||
}) {
|
||
Ok(_) => {
|
||
eprintln!(
|
||
"Idunn retired consumed command {}: its transactions are history",
|
||
stored.value.command_id
|
||
);
|
||
retired = true;
|
||
}
|
||
Err(error) => eprintln!(
|
||
"Idunn left consumed command {} resident: {error:#}",
|
||
stored.value.command_id
|
||
),
|
||
}
|
||
false
|
||
});
|
||
if retired {
|
||
return Ok(true);
|
||
}
|
||
candidates.sort_by_key(|stored| stored.value.requested_at_unix_millis);
|
||
// Oldest first, but a command whose target is busy does not block the
|
||
// ones behind it for other targets.
|
||
for command in candidates {
|
||
if self.freeze_command(&snapshot, command)? {
|
||
return Ok(true);
|
||
}
|
||
}
|
||
Ok(false)
|
||
}
|
||
|
||
/// Freeze one queued command into its transactions. `Ok(false)` means a
|
||
/// live deployment already exists for one of its targets and nothing was
|
||
/// written. Continuity, owners and drains never refuse a freeze: a
|
||
/// deployment that has not taken its target owns nothing.
|
||
fn freeze_command(
|
||
&self,
|
||
snapshot: &ControlSnapshot,
|
||
command: &Stored<DeploymentCommand>,
|
||
) -> Result<bool> {
|
||
let bindings = match load_bindings(&self.options.bindings_dir) {
|
||
Ok(bindings) => bindings,
|
||
Err(error) => {
|
||
eprintln!(
|
||
"Idunn left command {} queued because operator bindings are invalid: {error:#}",
|
||
command.value.command_id
|
||
);
|
||
return Ok(false);
|
||
}
|
||
};
|
||
let targets = match resolve_selector(&bindings, &command.value.selector) {
|
||
Ok(targets) => targets,
|
||
Err(error) => {
|
||
let now = now_millis()?;
|
||
let rejected = DeploymentTransaction::rejected(&command.value, error, now)?;
|
||
ensure!(
|
||
SingleFileMessagePackBackingStore::new(&self.options.state_store)
|
||
.compare_exchange(
|
||
&[CultCacheExpectedEnvelope {
|
||
r#type: DeploymentTransaction::TYPE.into(),
|
||
key: rejected.transaction_id.clone(),
|
||
current: None,
|
||
}],
|
||
&[transaction_envelope(&rejected, now)?],
|
||
)?,
|
||
"bad selector changed before refusal was recorded"
|
||
);
|
||
return Ok(true);
|
||
}
|
||
};
|
||
let busy = snapshot
|
||
.transactions
|
||
.iter()
|
||
.filter(|stored| {
|
||
stored.value.command_kind == CommandKind::Deploy && stored.value.is_live()
|
||
})
|
||
.map(|stored| stored.value.target.as_str())
|
||
.collect::<BTreeSet<_>>();
|
||
if targets.iter().any(|target| busy.contains(target.as_str())) {
|
||
return Ok(false);
|
||
}
|
||
let now = now_millis()?;
|
||
let transactions = targets
|
||
.into_iter()
|
||
.enumerate()
|
||
.map(|(ordinal, target)| {
|
||
DeploymentTransaction::new(
|
||
&command.value,
|
||
target.clone(),
|
||
u32::try_from(ordinal)?,
|
||
snapshot.admitted_for(&target).map(|stored| &stored.value),
|
||
now,
|
||
)
|
||
})
|
||
.collect::<Result<Vec<_>>>()?;
|
||
let expected = transactions
|
||
.iter()
|
||
.map(|transaction| CultCacheExpectedEnvelope {
|
||
r#type: DeploymentTransaction::TYPE.into(),
|
||
key: transaction.transaction_id.clone(),
|
||
current: None,
|
||
})
|
||
.collect::<Vec<_>>();
|
||
let next = transactions
|
||
.iter()
|
||
.map(|transaction| transaction_envelope(transaction, now))
|
||
.collect::<Result<Vec<_>>>()?;
|
||
ensure!(
|
||
SingleFileMessagePackBackingStore::new(&self.options.state_store)
|
||
.compare_exchange(&expected, &next)?,
|
||
"queued command lost its ordered transaction creation CAS"
|
||
);
|
||
Ok(true)
|
||
}
|
||
|
||
fn supervise_one_admitted_generation(&self) -> Result<bool> {
|
||
let snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
// A wait belongs to the failed record it spaces. Once any write for its
|
||
// target has landed, or the target is no longer admitted, the stored
|
||
// schedule owns the target again.
|
||
self.challenge_wait
|
||
.lock()
|
||
.expect("challenge wait mutex")
|
||
.retain(|target, wait| snapshot.target_stamp(target) == wait.stamp);
|
||
self.reconcile_incidents(&snapshot);
|
||
let mut progressed = false;
|
||
let mut admitted = snapshot.admitted.iter().collect::<Vec<_>>();
|
||
admitted.sort_by_key(|stored| stored.value.target.as_str());
|
||
for current in admitted {
|
||
if let Err(disagreement) = current.value.readiness() {
|
||
self.note_fault(
|
||
"holds admitted generation",
|
||
&format!("generation:{}", current.value.target),
|
||
&anyhow::Error::new(disagreement),
|
||
);
|
||
}
|
||
// A transaction that owns the incumbent slot fences, stops or
|
||
// replaces this generation: nothing here touches it meanwhile.
|
||
if snapshot
|
||
.on_target(¤t.value.target)
|
||
.any(|stored| stored.value.owns_target())
|
||
{
|
||
continue;
|
||
}
|
||
// Continuity defers to an owner, a drain or its own live attempt,
|
||
// never to a deployment that has not taken the target, and aborts
|
||
// nothing (handover-phase-split).
|
||
let mint_blocked = snapshot.target_owner(¤t.value.target, None).is_some()
|
||
|| snapshot.on_target(¤t.value.target).any(|stored| {
|
||
stored.value.is_live() && stored.value.command_kind == CommandKind::Continuity
|
||
});
|
||
match self.supervise_admitted_route(&snapshot, current) {
|
||
Ok(true) => {
|
||
progressed = true;
|
||
continue;
|
||
}
|
||
Ok(false) => {}
|
||
Err(error) => eprintln!(
|
||
"Idunn rejected admitted {} route continuity: {error:#}",
|
||
current.value.target
|
||
),
|
||
}
|
||
match self.refresh_admitted_topology(&snapshot, current) {
|
||
Ok(true) => {
|
||
progressed = true;
|
||
continue;
|
||
}
|
||
Ok(false) => {}
|
||
Err(error) => eprintln!(
|
||
"Idunn preserved admitted {} after rejecting topology observation: {error:#}",
|
||
current.value.target
|
||
),
|
||
}
|
||
match self.retire_stale_incarnations(&snapshot, current) {
|
||
Ok(true) => {
|
||
progressed = true;
|
||
continue;
|
||
}
|
||
Ok(false) => {}
|
||
Err(error) => eprintln!(
|
||
"Idunn preserved a stale {} incarnation projection: {error:#}",
|
||
current.value.target
|
||
),
|
||
}
|
||
// Restore the admitted Expected before anything is asked of the
|
||
// workload, and only the Expected.
|
||
//
|
||
// A target that cannot see its own Expected cannot start, and an
|
||
// aborted deployment leaves exactly that gap, so this cannot wait
|
||
// for an observation the dead process will never provide. The
|
||
// activation is deliberately not restored: it names one running
|
||
// incarnation, every restart issues a fresh one, and republishing
|
||
// the admitted one leaves the projection naming an incarnation that
|
||
// no longer exists -- which the workload rightly refuses to adopt.
|
||
// That record is published from observation once the process is up.
|
||
let repair = (|| -> Result<()> {
|
||
let topology = self.topology();
|
||
let provider_anchor = self.provider_anchor_for_plan(¤t.value.plan)?;
|
||
if !topology.admitted_expected_projection_is_exact(
|
||
¤t.value.expected,
|
||
&provider_anchor,
|
||
)? {
|
||
ensure!(
|
||
topology.publish_expected(¤t.value.expected, &provider_anchor)?
|
||
== current.value.expected.canonical_sha256()?,
|
||
"admitted Expected projection repair differs"
|
||
);
|
||
}
|
||
Ok(())
|
||
})();
|
||
if let Err(error) = repair {
|
||
eprintln!(
|
||
"Idunn preserved admitted {} after refusing topology projection repair: {error:#}",
|
||
current.value.target
|
||
);
|
||
}
|
||
let mut operational_error = None;
|
||
let observation = match self.workload_for(¤t.value.plan).and_then(|workload| {
|
||
workload.observe(
|
||
¤t.value.expected,
|
||
¤t.value.activation,
|
||
¤t.value.workload,
|
||
)
|
||
}) {
|
||
Ok(observation) => {
|
||
let lease_is_missing = if let Some(lease) = current.value.leasing.lease() {
|
||
let lease_health = (|| -> Result<bool> {
|
||
let lease_path = current
|
||
.value
|
||
.plan
|
||
.parsed_inputs()?
|
||
.1
|
||
.process_write_lease
|
||
.context("admitted lease has no operator binding")?
|
||
.record_path;
|
||
let driver =
|
||
CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path);
|
||
if driver.observe_exact(lease)? {
|
||
return Ok(true);
|
||
}
|
||
if driver.observe_empty()? {
|
||
return Ok(false);
|
||
}
|
||
bail!("write-lease store contains unexpected authority")
|
||
})();
|
||
match lease_health {
|
||
Ok(true) => false,
|
||
Ok(false) => {
|
||
operational_error =
|
||
Some(anyhow!("admitted physical write lease is missing"));
|
||
true
|
||
}
|
||
Err(error) => {
|
||
eprintln!(
|
||
"Idunn preserved admitted {} after refusing an unsafe write-lease mutation: {error:#}",
|
||
current.value.target
|
||
);
|
||
continue;
|
||
}
|
||
}
|
||
} else {
|
||
false
|
||
};
|
||
if lease_is_missing {
|
||
None
|
||
} else {
|
||
Some(observation)
|
||
}
|
||
}
|
||
Err(error) if error.downcast_ref::<HostUnobservable>().is_some() => {
|
||
// The host cannot be asked right now (its actuator is
|
||
// between sessions, or Idunn just restarted). That is
|
||
// silence, not a death; counting it as one burned every
|
||
// continuity attempt in the reattach window on
|
||
// 2026-09-11 and left a healthy Muninn demoted.
|
||
eprintln!(
|
||
"Idunn cannot observe admitted {} this tick: {error:#}",
|
||
current.value.target
|
||
);
|
||
continue;
|
||
}
|
||
Err(error) => {
|
||
operational_error = Some(error);
|
||
None
|
||
}
|
||
};
|
||
if observation.is_some() {
|
||
let seen = snapshot.supervision_for(¤t.value.target);
|
||
if let Some(stored) = seen.filter(|stored| {
|
||
stored.value.continuity_backoff_from_unix_millis.is_some()
|
||
}) {
|
||
let mut next = stored.value.clone();
|
||
next.end_backoff();
|
||
self.write_target_supervision(Some(stored), &next, now_millis()?)?;
|
||
}
|
||
// The admitted workload was seen running: the only evidence
|
||
// that ends an exhaustion incident, once the window no longer holds it.
|
||
// The window sliding alone is not health and never closes one.
|
||
self.record_incident(
|
||
"close",
|
||
¤t.value.target,
|
||
now_millis().and_then(|now| {
|
||
// Running is not enough while the window still holds
|
||
// the restarts that exhausted it: a release that
|
||
// survives one poll after each restart is still the
|
||
// same incident.
|
||
if snapshot
|
||
.supervision_or_new(¤t.value.target)
|
||
.restarts_page(now)
|
||
{
|
||
return Ok(false);
|
||
}
|
||
self.incidents().close(
|
||
IncidentCondition::ContinuityExhausted,
|
||
¤t.value.target,
|
||
CloseReason::Recovered,
|
||
now,
|
||
)
|
||
}),
|
||
);
|
||
continue;
|
||
}
|
||
let workload_error = operational_error
|
||
.context("admitted operational state has neither observation nor error")?;
|
||
// A held generation is a record the recipe never declared
|
||
// readiness for. Idunn reports it and does not decide for it: a
|
||
// continuity minted over it copies the undeclared Expected, is
|
||
// held at once, and owns the target forever, so the declaring
|
||
// redeploy that would clear the hold could never freeze. The dead
|
||
// held target stays down and free.
|
||
if current.value.readiness().is_err() {
|
||
self.note_fault(
|
||
"holds a dead admitted generation down; declare readiness in its recipe and redeploy",
|
||
&format!("generation-down:{}", current.value.target),
|
||
&anyhow!("admitted generation {} is not running", current.value.generation_id),
|
||
);
|
||
continue;
|
||
}
|
||
// Continuity restarts a release; it cannot repair one, and it
|
||
// never stops (budget-never-stop). Every charged attempt is spaced
|
||
// by a doubling, capped wait, so a queued deployment always gets
|
||
// a free gap (map B10). The log outlives every generation and only
|
||
// pages. Nothing here reads history: an unreadable history file
|
||
// must not stop crash recovery.
|
||
let now = now_millis()?;
|
||
let supervision = snapshot.supervision_or_new(¤t.value.target);
|
||
// A stepped-back clock leaves entries in the future. Settle them
|
||
// before deciding from them; the write ends this target's pass.
|
||
let mut settled = supervision.clone();
|
||
settled.settle(now);
|
||
if settled != supervision {
|
||
self.write_target_supervision(
|
||
snapshot.supervision_for(¤t.value.target),
|
||
&settled,
|
||
now,
|
||
)?;
|
||
progressed = true;
|
||
continue;
|
||
}
|
||
let continuity_key = format!("continuity:{}", current.value.target);
|
||
|
||
if mint_blocked {
|
||
continue;
|
||
}
|
||
if supervision.restarts_page(now) {
|
||
// The record comes first and the trace after it: the trace is
|
||
// this record's shadow, never its substitute.
|
||
self.record_incident(
|
||
"open",
|
||
¤t.value.target,
|
||
self.incidents().open(
|
||
IncidentCondition::ContinuityExhausted,
|
||
¤t.value.target,
|
||
now,
|
||
),
|
||
);
|
||
self.report_once(
|
||
&continuity_key,
|
||
format!(
|
||
"Idunn has restarted admitted {} {} times inside the hour and keeps restarting it on a capped backoff (streak {}): {workload_error:#}",
|
||
current.value.target,
|
||
supervision.restarts_used(now),
|
||
supervision.continuity_attempt_streak,
|
||
),
|
||
);
|
||
} else {
|
||
self.clear_fault(&continuity_key);
|
||
}
|
||
if supervision.continuity_deferred(now) {
|
||
continue;
|
||
}
|
||
// A unit the host lost to a reboot is re-materialized once per
|
||
// generation, uncharged and with no wait; the driver owns the
|
||
// evidence and supervision owns the budget. An unanswerable
|
||
// driver is not evidence of a reboot.
|
||
let admitted_at = current.value.admitted_at_unix_millis;
|
||
let rematerialize = !supervision.rematerialization_spent(admitted_at)
|
||
&& match self.workload_for(¤t.value.plan).and_then(|workload| {
|
||
workload.lost_to_host_reboot(¤t.value.workload, admitted_at)
|
||
}) {
|
||
Ok(lost) => lost,
|
||
Err(error) => {
|
||
self.note_fault(
|
||
"could not tell whether the host rebooted under",
|
||
&format!("reboot:{}", current.value.target),
|
||
&error,
|
||
);
|
||
false
|
||
}
|
||
};
|
||
let mut streak = 0;
|
||
if !rematerialize {
|
||
match supervision.continuity_backoff(now, admitted_at) {
|
||
ContinuityBackoff::Waiting => continue,
|
||
ContinuityBackoff::Begin => {
|
||
let mut began = supervision.clone();
|
||
began.begin_backoff(now);
|
||
self.write_target_supervision(
|
||
snapshot.supervision_for(¤t.value.target),
|
||
&began,
|
||
now,
|
||
)?;
|
||
progressed = true;
|
||
continue;
|
||
}
|
||
ContinuityBackoff::Due { streak: due } => streak = due,
|
||
}
|
||
}
|
||
|
||
// The projected activation names an incarnation that is gone. Every
|
||
// restart is issued a fresh one, so leaving the old record standing
|
||
// makes the projection describe a process that no longer exists --
|
||
// and a workload that reads it refuses to adopt an activation that
|
||
// is not its own, which is a restart loop rather than a recovery.
|
||
// Demote to Expected-only, the same shape an aborted deployment
|
||
// leaves, and let the restart publish its own activation once it is
|
||
// observed.
|
||
let demotion = (|| -> Result<()> {
|
||
let topology = self.topology();
|
||
let provider_anchor = self.provider_anchor_for_plan(¤t.value.plan)?;
|
||
if !topology.projected_activation_is_present(¤t.value.expected)? {
|
||
return Ok(());
|
||
}
|
||
topology.demote_to_expected_only(
|
||
¤t.value.expected,
|
||
&provider_anchor,
|
||
¤t.value.activation,
|
||
current.value.leasing.lease(),
|
||
)?;
|
||
Ok(())
|
||
})();
|
||
// Expected-only is the precondition of a restart: a continuity
|
||
// minted over a projection that still names an activation would
|
||
// fail between preparing and publishing its own, and its abort
|
||
// could not resolve. So a failed demotion mints nothing. The
|
||
// failure is written to the generation, and the next attempt is
|
||
// not before the deferral ends.
|
||
if let Err(error) = demotion {
|
||
eprintln!(
|
||
"Idunn deferred continuity for admitted {} after refusing to demote its projection: {error:#}",
|
||
current.value.target
|
||
);
|
||
let mut deferred = supervision.clone();
|
||
deferred.defer_continuity(now, &bounded_detail(&format!("{error:#}")));
|
||
self.write_target_supervision(
|
||
snapshot.supervision_for(¤t.value.target),
|
||
&deferred,
|
||
now,
|
||
)?;
|
||
progressed = true;
|
||
continue;
|
||
}
|
||
// An engaged lifecycle brake means no continuity transaction is
|
||
// minted at all. A transaction that exists and waits on the brake
|
||
// owns the target, and a target owned by a parked restart accepts
|
||
// no deployment -- which is the one thing a lifecycle brake must
|
||
// never gate. The Sealing-phase check remains as the guard for a
|
||
// transaction minted just before the brake was engaged.
|
||
if !self.lifecycle_allows_generation(current, now)? {
|
||
continue;
|
||
}
|
||
let command = DeploymentCommand {
|
||
schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(),
|
||
command_id: format!("continuity-{}", Uuid::new_v4()),
|
||
kind: CommandKind::Continuity,
|
||
selector: current.value.target.clone(),
|
||
requested_by: "idunn-continuity".into(),
|
||
requested_at_unix_millis: now,
|
||
};
|
||
command.validate()?;
|
||
let transaction =
|
||
DeploymentTransaction::from_continuity(&command, ¤t.value, now)?;
|
||
// The attempt is charged, or recorded as the one re-materialization,
|
||
// in the same CAS that mints it, so a crash cannot schedule one the
|
||
// meters never saw.
|
||
let mut counted = supervision;
|
||
if rematerialize {
|
||
counted.record_rematerialization(now);
|
||
} else {
|
||
counted.record_restart(now, streak);
|
||
}
|
||
ensure!(
|
||
SingleFileMessagePackBackingStore::new(&self.options.state_store)
|
||
.compare_exchange(
|
||
&[
|
||
CultCacheExpectedEnvelope {
|
||
r#type: DeploymentCommand::TYPE.into(),
|
||
key: command.command_id.clone(),
|
||
current: None,
|
||
},
|
||
CultCacheExpectedEnvelope {
|
||
r#type: DeploymentTransaction::TYPE.into(),
|
||
key: transaction.transaction_id.clone(),
|
||
current: None,
|
||
},
|
||
CultCacheExpectedEnvelope {
|
||
r#type: TargetSupervision::TYPE.into(),
|
||
key: counted.target.clone(),
|
||
current: snapshot
|
||
.supervision_for(&counted.target)
|
||
.map(|stored| stored.envelope.clone()),
|
||
},
|
||
],
|
||
&[
|
||
command_envelope(&command, now)?,
|
||
transaction_envelope(&transaction, now)?,
|
||
target_supervision_envelope(&counted, now)?,
|
||
],
|
||
)?,
|
||
"continuity scheduling lost its command/transaction/meter CAS"
|
||
);
|
||
progressed = true;
|
||
}
|
||
Ok(progressed)
|
||
}
|
||
|
||
fn supervise_admitted_route(
|
||
&self,
|
||
snapshot: &ControlSnapshot,
|
||
current: &Stored<AdmittedGeneration>,
|
||
) -> Result<bool> {
|
||
let Some(expected_route) = current.value.expected.route.as_ref() else {
|
||
ensure!(
|
||
matches!(¤t.value.routing, RoutingEvidence::SkippedUnrouted),
|
||
"unrouted admitted generation carries route authority"
|
||
);
|
||
return Ok(false);
|
||
};
|
||
let RoutingEvidence::Promoted {
|
||
observation,
|
||
promoted_at_unix_millis,
|
||
} = ¤t.value.routing
|
||
else {
|
||
bail!("routed admitted generation has no promoted route receipt")
|
||
};
|
||
let now = now_millis()?;
|
||
// A route whose proofs keep failing is challenged on a widening
|
||
// schedule, not every tick.
|
||
if current
|
||
.value
|
||
.route_supervision
|
||
.as_ref()
|
||
.is_some_and(|state| state.is_waiting(now, self.options.topology_maximum_age_millis))
|
||
{
|
||
return Ok(false);
|
||
}
|
||
if self
|
||
.challenge_wait
|
||
.lock()
|
||
.expect("challenge wait mutex")
|
||
.get(¤t.value.target)
|
||
.is_some_and(|wait| {
|
||
is_waiting(
|
||
now,
|
||
wait.due_unix_millis,
|
||
ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS.max(self.options.topology_maximum_age_millis),
|
||
)
|
||
})
|
||
{
|
||
return Ok(false);
|
||
}
|
||
ensure!(
|
||
observation.route_id == expected_route.route_id
|
||
&& observation.runtime_instance_id == current.value.activation.runtime_instance_id,
|
||
"admitted route receipt names another incarnation"
|
||
);
|
||
let binding = current.value.plan.parsed_inputs()?.1;
|
||
let lease_driver = if let Some(lease) = current.value.leasing.lease() {
|
||
let lease_path = &binding
|
||
.process_write_lease
|
||
.as_ref()
|
||
.context("stateful admitted generation has no write-lease binding")?
|
||
.record_path;
|
||
let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path);
|
||
ensure!(
|
||
driver.observe_exact(lease)?,
|
||
"admitted process write lease is no longer exact"
|
||
);
|
||
Some(driver)
|
||
} else {
|
||
None
|
||
};
|
||
let route_binding = binding
|
||
.route
|
||
.context("routed admitted generation has no operator route binding")?;
|
||
let driver = self.route_driver(route_binding);
|
||
let gate = self.route_gate(¤t.value.target, CommandKind::Continuity);
|
||
|
||
if driver.observe_membership(¤t.value.expected, &observation.membership_sha256)?
|
||
&& route_observation_is_current(
|
||
observation.observed_at_unix_millis,
|
||
now,
|
||
self.options.topology_maximum_age_millis,
|
||
self.options.topology_maximum_future_skew_millis,
|
||
)
|
||
{
|
||
return Ok(false);
|
||
}
|
||
let route_key = format!("route:{}", current.value.target);
|
||
// A failed repair or a failed proof changes observation state only: it
|
||
// marks the route degraded and widens the wait. That record is written
|
||
// like a proved one: when it lands the pass ends (`Ok(true)`), because
|
||
// the caller's snapshot no longer matches the generation; when the store
|
||
// refuses it the pass reports no progress and the in-memory wait below
|
||
// spaces the next attempt.
|
||
let challenge = (|| -> Result<RouteObservation> {
|
||
// Actuates only when the fragment on disk differs from the
|
||
// admitted membership; an exact fragment makes this a no-op. It is
|
||
// survival: restoring the admitted route is never refused.
|
||
driver.restore_admitted_membership(
|
||
¤t.value.expected,
|
||
&observation.membership_sha256,
|
||
&gate,
|
||
)?;
|
||
let authority = self.runtime_authority_parts(
|
||
¤t.value.plan,
|
||
¤t.value.expected,
|
||
¤t.value.activation,
|
||
)?;
|
||
let refreshed = self.prove_stable_route_against(
|
||
¤t.value.expected,
|
||
¤t.value.activation,
|
||
&authority,
|
||
current.value.leasing.lease_sha256(),
|
||
&driver,
|
||
observation.membership_sha256.clone(),
|
||
)?;
|
||
ensure!(
|
||
driver.observe_membership(¤t.value.expected, &refreshed.membership_sha256)?,
|
||
"admitted route membership changed during its continuity challenge"
|
||
);
|
||
if let (Some(lease_driver), Some(lease)) = (&lease_driver, current.value.leasing.lease())
|
||
{
|
||
ensure!(
|
||
lease_driver.observe_exact(lease)?,
|
||
"admitted process write lease changed during its continuity challenge"
|
||
);
|
||
}
|
||
Ok(refreshed)
|
||
})();
|
||
let mut next = current.value.clone();
|
||
// The wait the record carries, also held in memory until a record lands.
|
||
let (recorded, wait_until) = match challenge {
|
||
Ok(refreshed) => {
|
||
next.routing = RoutingEvidence::Promoted {
|
||
observation: refreshed,
|
||
promoted_at_unix_millis: *promoted_at_unix_millis,
|
||
};
|
||
if let Some(state) = next.route_supervision.as_mut() {
|
||
state.record_proved_challenge(now);
|
||
}
|
||
self.clear_fault(&route_key);
|
||
(
|
||
"proved",
|
||
now.saturating_add(self.options.topology_maximum_age_millis),
|
||
)
|
||
}
|
||
Err(error) => {
|
||
if let Some(state) = next.route_supervision.as_mut() {
|
||
state.record_failed_challenge(now, self.options.topology_maximum_age_millis);
|
||
}
|
||
self.report_once(
|
||
&route_key,
|
||
format!(
|
||
"Idunn could not prove admitted {} on its stable route: {error:#}",
|
||
current.value.target
|
||
),
|
||
);
|
||
let due = next
|
||
.route_supervision
|
||
.as_ref()
|
||
.and_then(|state| state.next_challenge_at_unix_millis)
|
||
.unwrap_or(now);
|
||
("failed", due)
|
||
}
|
||
};
|
||
// Progress is a store that changed: our record landed, or another
|
||
// writer beat it (the snapshot is stale either way, so the pass ends).
|
||
// A write the store refused changed nothing: no progress, the scheduler
|
||
// sleeps if nothing else moved, and the in-memory wait spaces the next
|
||
// challenge.
|
||
let record_key = format!("route-record:{}", current.value.target);
|
||
match self.replace_generation(current, &next, now) {
|
||
Ok(_) => {
|
||
self.challenge_wait
|
||
.lock()
|
||
.expect("challenge wait mutex")
|
||
.remove(¤t.value.target);
|
||
self.clear_fault(&record_key);
|
||
Ok(true)
|
||
}
|
||
Err(record) => {
|
||
self.challenge_wait
|
||
.lock()
|
||
.expect("challenge wait mutex")
|
||
.insert(
|
||
current.value.target.clone(),
|
||
ChallengeWait {
|
||
due_unix_millis: wait_until,
|
||
stamp: snapshot.target_stamp(¤t.value.target),
|
||
},
|
||
);
|
||
self.report_once(
|
||
&record_key,
|
||
format!(
|
||
"Idunn could not record the {recorded} route challenge of {}: {}",
|
||
current.value.target,
|
||
// The root cause only: the error chain names a fresh temp file each
|
||
// attempt, which would defeat once-per-fault.
|
||
record.root_cause()
|
||
),
|
||
);
|
||
Ok(false)
|
||
}
|
||
}
|
||
}
|
||
|
||
fn refresh_admitted_topology(
|
||
&self,
|
||
snapshot: &ControlSnapshot,
|
||
current: &Stored<AdmittedGeneration>,
|
||
) -> Result<bool> {
|
||
if !matches!(current.value.readiness(), Ok(class) if class != ReadinessClass::RouteProof) {
|
||
return Ok(false);
|
||
}
|
||
let odin = current.value.odin_receipts()?;
|
||
let Some(received) = self.topology().receive(
|
||
¤t.value.target,
|
||
¤t.value.expected.canonical_sha256()?,
|
||
)?
|
||
else {
|
||
return Ok(false);
|
||
};
|
||
ensure!(
|
||
received.target == current.value.target,
|
||
"topology transport substituted admitted target"
|
||
);
|
||
let now = now_millis()?;
|
||
let authority = self.runtime_authority_parts(
|
||
¤t.value.plan,
|
||
¤t.value.expected,
|
||
¤t.value.activation,
|
||
)?;
|
||
let odin_authority = self.current_odin_authority(snapshot)?;
|
||
let authenticated = match authenticate_odin_runtime_topology_correlation(
|
||
&received.canonical_bytes,
|
||
&authority,
|
||
current.value.leasing.lease_sha256(),
|
||
&odin_authority.signer_public_key,
|
||
self.trusted_topology_context(now),
|
||
) {
|
||
Ok(authenticated) => authenticated,
|
||
// An aged-out correlation is nothing new about the admitted
|
||
// generation, not a rejected observation.
|
||
Err(error) if is_stale_observation(&error) => return Ok(false),
|
||
Err(error) => return Err(error),
|
||
};
|
||
let evidence = TopologyEvidence::from_authenticated(&authenticated, now)?;
|
||
if !sequence_requires_admission(
|
||
Some(odin.latest),
|
||
snapshot.max_odin_sequence(¤t.value.target, &evidence.signer_identity_id),
|
||
&evidence,
|
||
)? {
|
||
return Ok(false);
|
||
}
|
||
let mut next = current.value.clone();
|
||
next.latest_odin_observation = Some(evidence.clone());
|
||
next.odin_publisher_sequence_cursor = evidence.publisher_sequence;
|
||
if is_semantic_ready(&authenticated) {
|
||
next.ready = ReadinessEvidence::OdinCorrelated { evidence };
|
||
}
|
||
next.validate()?;
|
||
ensure!(
|
||
SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange(
|
||
&[CultCacheExpectedEnvelope {
|
||
r#type: AdmittedGeneration::TYPE.into(),
|
||
key: current.value.target.clone(),
|
||
current: Some(current.envelope.clone()),
|
||
}],
|
||
&[admitted_envelope(&next, now)?],
|
||
)?,
|
||
"admitted generation changed before topology sequence CAS"
|
||
);
|
||
Ok(true)
|
||
}
|
||
|
||
fn advance_transaction(&self, current: &Stored<DeploymentTransaction>) -> Result<()> {
|
||
if current.value.post_fencing_abort.is_some() && current.value.completion.is_none() {
|
||
return self.advance_post_fencing_abort(current);
|
||
}
|
||
if current.value.pre_fencing_abort.is_some() && current.value.completion.is_none() {
|
||
return self.advance_pre_fencing_abort(current);
|
||
}
|
||
if current.value.phase < current.value.takes_target_at()
|
||
&& self.settle_before_handover(current)?
|
||
{
|
||
return Ok(());
|
||
}
|
||
match current.value.phase {
|
||
DeploymentPhase::Sealing => self.advance_sealing(current),
|
||
DeploymentPhase::Starting => self.advance_starting(current),
|
||
DeploymentPhase::Warming => self.advance_warming(current),
|
||
DeploymentPhase::Fencing => self.advance_fencing(current),
|
||
DeploymentPhase::Leasing => self.advance_leasing(current),
|
||
DeploymentPhase::AwaitingReady => self.advance_awaiting_ready(current),
|
||
DeploymentPhase::Routing => self.advance_routing(current),
|
||
DeploymentPhase::Committing => self.advance_committing(current),
|
||
DeploymentPhase::Complete => self.advance_post_commit_cleanup(current),
|
||
}
|
||
}
|
||
|
||
fn advance_sealing(&self, current: &Stored<DeploymentTransaction>) -> Result<()> {
|
||
if current.value.plan.is_none() {
|
||
ensure!(
|
||
current.value.command_kind == CommandKind::Deploy,
|
||
"continuity transaction lost its admitted plan"
|
||
);
|
||
let bindings = load_bindings(&self.options.bindings_dir)?;
|
||
let loaded = bindings
|
||
.get(¤t.value.target)
|
||
.context("operator binding disappeared before sealing")?;
|
||
let now = now_millis()?;
|
||
let resolved =
|
||
self.source
|
||
.resolve(&loaded.binding, ¤t.value.transaction_id, now)?;
|
||
let provider_snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
let providers = self.current_ready_provider_tokens(&provider_snapshot)?;
|
||
let candidate_port = self.select_candidate_port(&loaded.binding)?;
|
||
let plan = compile_deployment_plan(
|
||
&resolved.recipe_bytes,
|
||
&loaded.bytes,
|
||
resolved.facts,
|
||
format!("incarnation-{}", current.value.transaction_id),
|
||
candidate_port,
|
||
now,
|
||
&providers,
|
||
)?;
|
||
// Idunn infers no way to prove readiness. A recipe that declares
|
||
// none is refused here, before anything is frozen, built or installed.
|
||
plan.readiness_class()?;
|
||
return self.persist_same_phase(current, |next| {
|
||
next.plan = Some(plan);
|
||
next.updated_at_unix_millis = now;
|
||
next.last_error = None;
|
||
Ok(())
|
||
});
|
||
}
|
||
|
||
if current.value.command_kind == CommandKind::Deploy
|
||
&& current.value.frozen_source.is_none()
|
||
{
|
||
let now = now_millis()?;
|
||
let plan = required(¤t.value.plan, "transaction plan")?;
|
||
let receipt = self.source.freeze(¤t.value.transaction_id, plan)?;
|
||
return self.persist_same_phase(current, |next| {
|
||
next.frozen_source = Some(receipt);
|
||
next.updated_at_unix_millis = now;
|
||
next.last_error = None;
|
||
Ok(())
|
||
});
|
||
}
|
||
|
||
if current.value.sealed_release.is_none()
|
||
|| current.value.installed_release.is_none()
|
||
|| current.value.expected.is_none()
|
||
{
|
||
ensure!(
|
||
current.value.command_kind == CommandKind::Deploy,
|
||
"continuity transaction lost its admitted release evidence"
|
||
);
|
||
let now = now_millis()?;
|
||
let plan = required(¤t.value.plan, "transaction plan")?;
|
||
let frozen_receipt = required(¤t.value.frozen_source, "frozen source")?;
|
||
let frozen = self.source.observe_frozen(plan, frozen_receipt)?;
|
||
let materialized = self.runner_for(plan)?.materialize(
|
||
&frozen,
|
||
plan,
|
||
&self.options.staging_root,
|
||
now,
|
||
)?;
|
||
let installed = self.workload_for(plan)?.install(plan, &materialized)?;
|
||
let expected = materialized.release.expected_projection(plan)?;
|
||
return self.persist_same_phase(current, |next| {
|
||
next.sealed_release = Some(materialized.release);
|
||
next.installed_release = Some(installed);
|
||
next.expected = Some(expected);
|
||
next.updated_at_unix_millis = now;
|
||
next.last_error = None;
|
||
Ok(())
|
||
});
|
||
}
|
||
|
||
// The candidate's Expected is published in Starting, after the brake
|
||
// has admitted this exact transaction. Publishing it is the first
|
||
// Verse-visible change a deployment makes, and the brake gates changes.
|
||
let now = now_millis()?;
|
||
let mut next = current.value.clone();
|
||
match current.value.command_kind {
|
||
CommandKind::Deploy => {
|
||
let Some(authorization) = self.deployment_authorization(current, now)? else {
|
||
return self.record_gate_wait(
|
||
current,
|
||
"deployment brake has not released this exact transaction",
|
||
);
|
||
};
|
||
next.deployment_authorization = Some(authorization);
|
||
}
|
||
CommandKind::Continuity => {
|
||
if !self.lifecycle_allows(current, now)? {
|
||
return self
|
||
.record_gate_wait(current, "lifecycle brake denies continuity restart");
|
||
}
|
||
next.lifecycle_authorized_at_unix_millis = Some(now);
|
||
}
|
||
}
|
||
validate_live_providers_for_deploy(current.value.command_kind, || {
|
||
self.validate_selected_providers_current(required(
|
||
¤t.value.plan,
|
||
"transaction plan",
|
||
)?)
|
||
})?;
|
||
next.enter_phase(DeploymentPhase::Starting, now);
|
||
next.last_error = None;
|
||
// Starting is the handover for stop-then-start, so this write goes
|
||
// through the seam that decides it.
|
||
self.enter(current, next)
|
||
}
|
||
|
||
fn advance_starting(&self, current: &Stored<DeploymentTransaction>) -> Result<()> {
|
||
let expected = required(¤t.value.expected, "Expected projection")?;
|
||
let plan = required(¤t.value.plan, "transaction plan")?;
|
||
let release = required(¤t.value.sealed_release, "sealed release")?;
|
||
let installed = required(¤t.value.installed_release, "installed release")?;
|
||
if current.value.expected_publication_sha256.is_none() {
|
||
// Published under the candidate's own incarnation key, beside
|
||
// whatever the target's admitted incarnation currently projects.
|
||
// Nothing the incumbent reads about itself changes here.
|
||
let now = now_millis()?;
|
||
let provider_anchor = self.provider_anchor_for_plan(plan)?;
|
||
let digest = self
|
||
.topology()
|
||
.publish_expected(expected, &provider_anchor)?;
|
||
return self.persist_same_phase(current, |next| {
|
||
next.expected_publication_sha256 = Some(digest);
|
||
next.updated_at_unix_millis = now;
|
||
next.last_error = None;
|
||
Ok(())
|
||
});
|
||
}
|
||
if current.value.activation.is_none() {
|
||
validate_live_providers_for_deploy(current.value.command_kind, || {
|
||
self.validate_selected_providers_current(plan)
|
||
})?;
|
||
let now = now_millis()?;
|
||
let runtime_instance_id = runtime_instance_id(¤t.value.transaction_id)?;
|
||
let launch = IdunnRuntimeActivationLaunch::issue(
|
||
expected,
|
||
runtime_instance_id,
|
||
now,
|
||
&self.idunn_signer,
|
||
)?;
|
||
let activation = self
|
||
.workload_for(plan)?
|
||
.prepare_activation(plan, expected, launch)?;
|
||
return self.persist_same_phase(current, |next| {
|
||
next.activation = Some(activation);
|
||
next.updated_at_unix_millis = now;
|
||
next.last_error = None;
|
||
Ok(())
|
||
});
|
||
}
|
||
if current.value.workload.is_none() {
|
||
let now = now_millis()?;
|
||
let activation = required(¤t.value.activation, "activation")?;
|
||
if current.value.rollout_stops_incumbent_first() {
|
||
let snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
if let Some(incumbent) = self.exact_incumbent(&snapshot, ¤t.value)? {
|
||
self.workload_for(&incumbent.value.plan)?
|
||
.stop(&incumbent.value.workload)
|
||
.context("stopping the incumbent before the candidate starts")?;
|
||
}
|
||
}
|
||
let observation = self
|
||
.workload_for(plan)?
|
||
.start_prepared(plan, release, installed, expected, activation)?;
|
||
return self.persist_same_phase(current, |next| {
|
||
next.workload = Some(observation);
|
||
next.updated_at_unix_millis = now;
|
||
next.last_error = None;
|
||
Ok(())
|
||
});
|
||
}
|
||
if current.value.activation_publication_sha256.is_none() {
|
||
let now = now_millis()?;
|
||
let activation = required(¤t.value.activation, "activation")?;
|
||
let workload = required(¤t.value.workload, "workload")?;
|
||
self.workload_for(required(¤t.value.plan, "transaction plan")?)?
|
||
.observe(expected, activation, workload)?;
|
||
let digest = self
|
||
.topology()
|
||
.publish_observed_activation(expected, activation, workload)?;
|
||
return self.persist_same_phase(current, |next| {
|
||
next.activation_publication_sha256 = Some(digest);
|
||
next.updated_at_unix_millis = now;
|
||
next.last_error = None;
|
||
Ok(())
|
||
});
|
||
}
|
||
let mut next = current.value.clone();
|
||
next.enter_phase(DeploymentPhase::Warming, now_millis()?);
|
||
next.last_error = None;
|
||
replace_transaction(&self.options.state_store, current, &next)
|
||
}
|
||
|
||
fn advance_warming(&self, current: &Stored<DeploymentTransaction>) -> Result<()> {
|
||
let expected = required(¤t.value.expected, "Expected projection")?;
|
||
let activation = required(¤t.value.activation, "activation")?;
|
||
let workload = required(¤t.value.workload, "workload")?;
|
||
self.workload_for(required(¤t.value.plan, "transaction plan")?)?
|
||
.observe(expected, activation, workload)?;
|
||
let snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
let incumbent_lease_sha256 =
|
||
self.incumbent_lease_sha256_for_warming(&snapshot, ¤t.value)?;
|
||
let class = ReadinessClass::of(expected)?;
|
||
if current.value.warming.is_none() && class == ReadinessClass::RouteProof {
|
||
// A route-proof target's warming is Idunn's own challenge to the
|
||
// candidate endpoint. No Odin is read: the target declared no
|
||
// reason to be aware of one.
|
||
return match self.challenge_candidate(
|
||
¤t.value,
|
||
route_proof_warming_states(expected),
|
||
None,
|
||
)? {
|
||
CandidateAnswer::Silent(reason) => self.record_gate_wait(current, &reason),
|
||
CandidateAnswer::Answered { evidence, present } => {
|
||
let warming = WarmingEvidence::RouteProofDirect { evidence };
|
||
let _token = SequenceAdmittedWarming::from_direct_presence(
|
||
current.value.transaction_id.clone(),
|
||
warming.clone(),
|
||
present,
|
||
)?;
|
||
self.persist_same_phase(current, |next| {
|
||
next.warming = Some(warming);
|
||
Ok(())
|
||
})
|
||
}
|
||
};
|
||
}
|
||
if current.value.warming.is_none() {
|
||
// Odin's warming is observed directly when nothing else can observe
|
||
// it. That is the first bootstrap, and it is also every continuity
|
||
// restart of Odin: the incumbent that would do the observing is the
|
||
// very process being restarted, so it cannot report on its own
|
||
// return. A deployment of Odin is not included -- there the healthy
|
||
// incumbent observes the candidate, which is the point.
|
||
// Odin's warming is observed directly exactly when no Odin can
|
||
// observe it: the first bootstrap, a continuity restart of Odin
|
||
// itself, or any moment the admitted Odin has no usable correlation
|
||
// to report through. The rule is the same one that justified the
|
||
// bootstrap case -- "observed through Odin" is not available -- and
|
||
// it stays scoped to this one target. Every other target keeps
|
||
// waiting for Odin, which is what makes Odin the root of the chain.
|
||
//
|
||
// Asked once. Admitting the latest topology persists the sequence
|
||
// cursor, so asking twice with the same record fails the second
|
||
// time as "transaction changed before topology admission".
|
||
let is_odin = class == ReadinessClass::OdinSelf;
|
||
let odin_observation = if is_odin
|
||
&& snapshot.admitted_odin().is_some()
|
||
&& current.value.command_kind != CommandKind::Continuity
|
||
{
|
||
self.admit_latest_topology(current, None)?
|
||
} else {
|
||
None
|
||
};
|
||
let odin_observes_itself = is_odin && odin_observation.is_none();
|
||
if odin_observes_itself {
|
||
ensure!(
|
||
expected.write_lease_required,
|
||
"direct Odin warming must be a stateful incarnation"
|
||
);
|
||
let (evidence, present) = self.observe_first_odin_warming(¤t.value)?;
|
||
let warming = WarmingEvidence::FirstOdinDirect { evidence };
|
||
let _token = SequenceAdmittedWarming::from_direct_presence(
|
||
current.value.transaction_id.clone(),
|
||
warming.clone(),
|
||
present,
|
||
)?;
|
||
return self.persist_same_phase(current, |next| {
|
||
next.warming = Some(warming);
|
||
Ok(())
|
||
});
|
||
}
|
||
let observation = match odin_observation {
|
||
Some(observation) => Some(observation),
|
||
None => self.admit_latest_topology(current, None)?,
|
||
};
|
||
let Some((admitted, authenticated)) = observation else {
|
||
// Only the first-Odin bootstrap above observes presence
|
||
// directly. Every other target's warming presence arrives as
|
||
// Odin's authenticated runtime topology correlation, so until
|
||
// Odin is admitted and publishing, a candidate warms forever
|
||
// with nothing to say why.
|
||
return self.record_gate_wait(
|
||
current,
|
||
"no authenticated Odin topology correlation yet: warming presence for a \
|
||
non-Odin target is observed through Odin, which must be admitted first",
|
||
);
|
||
};
|
||
if admitted.envelope != current.envelope {
|
||
return Ok(());
|
||
}
|
||
let semantic_warming = is_semantic_warming(
|
||
expected,
|
||
activation,
|
||
incumbent_lease_sha256.as_deref(),
|
||
&authenticated,
|
||
)?;
|
||
let latest = required(
|
||
&admitted.value.latest_odin_observation,
|
||
"sequence-admitted warming evidence",
|
||
)?;
|
||
if semantic_warming {
|
||
let evidence = latest.clone();
|
||
let _token = SequenceAdmittedWarming::from_topology(
|
||
admitted.value.transaction_id.clone(),
|
||
authenticated,
|
||
)?;
|
||
return self.persist_same_phase(&admitted, |next| {
|
||
next.warming = Some(WarmingEvidence::OdinTopology { evidence });
|
||
Ok(())
|
||
});
|
||
} else {
|
||
return Ok(());
|
||
}
|
||
}
|
||
let warming = self.rehydrate_warming_token(¤t.value, now_millis()?, false)?;
|
||
ensure!(
|
||
warming.transaction_id() == current.value.transaction_id
|
||
&& warming.runtime_instance_id() == activation.runtime_instance_id.as_str(),
|
||
"durable Warming evidence belongs to another candidate"
|
||
);
|
||
|
||
if expected.route.is_some() && current.value.route_preflight.is_none() {
|
||
let now = now_millis()?;
|
||
let snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
let incumbent = self.exact_incumbent(&snapshot, ¤t.value)?;
|
||
let incumbent_route =
|
||
incumbent.and_then(|generation| generation.value.routing.observation().cloned());
|
||
let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1;
|
||
let route_binding = binding
|
||
.route
|
||
.context("routed Expected lost route binding")?;
|
||
let driver = self.route_driver(route_binding);
|
||
let receipt = driver.preflight(
|
||
expected,
|
||
&activation.runtime_instance_id,
|
||
incumbent_route.as_ref(),
|
||
&self.route_gate(¤t.value.target, current.value.command_kind),
|
||
)?;
|
||
return self.persist_same_phase(current, |next| {
|
||
next.route_preflight = Some(receipt);
|
||
next.updated_at_unix_millis = now;
|
||
Ok(())
|
||
});
|
||
}
|
||
|
||
if current.value.isolation.is_none() {
|
||
let now = now_millis()?;
|
||
let snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
let incumbent = self.exact_incumbent(&snapshot, ¤t.value)?;
|
||
// Isolation is a property of two processes running at once. A
|
||
// stopped incumbent has already released its DynamicUser UID, and
|
||
// systemd is free to hand that same UID to the candidate -- so
|
||
// comparing the candidate against the incumbent's *recorded*
|
||
// identity turns ordinary UID reuse into a permanent refusal to
|
||
// fence. There is nothing left to be isolated from.
|
||
let incumbent_workload = match incumbent {
|
||
Some(value)
|
||
if self
|
||
.workload_for(&value.value.plan)?
|
||
.is_permanently_stopped(&value.value.workload)? =>
|
||
{
|
||
None
|
||
}
|
||
other => other.map(|value| &value.value.workload),
|
||
};
|
||
let isolation = WorkloadObservation::prove_isolation(workload, incumbent_workload)?;
|
||
return self.persist_same_phase(current, |next| {
|
||
next.isolation = Some(isolation);
|
||
next.updated_at_unix_millis = now;
|
||
Ok(())
|
||
});
|
||
}
|
||
|
||
self.transition(current, DeploymentPhase::Fencing)
|
||
}
|
||
|
||
fn advance_fencing(&self, current: &Stored<DeploymentTransaction>) -> Result<()> {
|
||
if current.value.fencing.is_none() {
|
||
let now = now_millis()?;
|
||
let snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
let incumbent = self.exact_incumbent(&snapshot, ¤t.value)?;
|
||
let incumbent_lease = incumbent.and_then(|generation| generation.value.leasing.lease());
|
||
let expected = required(¤t.value.expected, "Expected projection")?;
|
||
let evidence = if expected.write_lease_required || incumbent_lease.is_some() {
|
||
let incumbent_lease_path = incumbent_lease
|
||
.map(|_| -> Result<PathBuf> {
|
||
Ok(incumbent
|
||
.context("incumbent lease has no admitted generation")?
|
||
.value
|
||
.plan
|
||
.parsed_inputs()?
|
||
.1
|
||
.process_write_lease
|
||
.context("incumbent lease has no admitted binding")?
|
||
.record_path)
|
||
})
|
||
.transpose()?;
|
||
if let (Some(lease), Some(path)) = (incumbent_lease, &incumbent_lease_path) {
|
||
let incumbent = incumbent.context("incumbent lease lost its generation")?;
|
||
self.workload_for(&incumbent.value.plan)?
|
||
.stop(&incumbent.value.workload)
|
||
.context("stopping the exact incumbent before revoking its lifetime-held write lease")?;
|
||
let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, path);
|
||
driver.revoke_exact(Some(lease))?;
|
||
ensure!(
|
||
driver.observe_empty()?,
|
||
"incumbent write lease remained after exact fencing"
|
||
);
|
||
self.topology().withdraw_process_write_lease(
|
||
&incumbent.value.expected,
|
||
&incumbent.value.activation,
|
||
Some(lease),
|
||
)?;
|
||
}
|
||
let candidate_lease_path = if expected.write_lease_required {
|
||
Some(
|
||
current
|
||
.value
|
||
.plan
|
||
.as_ref()
|
||
.unwrap()
|
||
.parsed_inputs()?
|
||
.1
|
||
.process_write_lease
|
||
.context("stateful candidate has no write-lease binding")?
|
||
.record_path,
|
||
)
|
||
} else {
|
||
None
|
||
};
|
||
if let Some(path) = &candidate_lease_path {
|
||
if incumbent_lease_path.as_ref() != Some(path) {
|
||
let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, path);
|
||
driver.revoke_exact(None)?;
|
||
ensure!(
|
||
driver.observe_empty()?,
|
||
"candidate write-lease path was not empty before grant"
|
||
);
|
||
}
|
||
}
|
||
self.topology().withdraw_process_write_lease(
|
||
expected,
|
||
required(¤t.value.activation, "candidate activation")?,
|
||
None,
|
||
)?;
|
||
FencingEvidence::Revoked {
|
||
incumbent_lease_sha256: incumbent_lease
|
||
.map(IdunnProcessWriteLeaseRecord::canonical_sha256)
|
||
.transpose()?,
|
||
candidate_lease_path_verified_empty: candidate_lease_path.is_some(),
|
||
}
|
||
} else {
|
||
FencingEvidence::SkippedStateless
|
||
};
|
||
return self.persist_same_phase(current, |next| {
|
||
next.fencing = Some(evidence);
|
||
next.updated_at_unix_millis = now;
|
||
Ok(())
|
||
});
|
||
}
|
||
self.transition(current, DeploymentPhase::Leasing)
|
||
}
|
||
|
||
fn advance_leasing(&self, current: &Stored<DeploymentTransaction>) -> Result<()> {
|
||
let expected = required(¤t.value.expected, "Expected projection")?;
|
||
if !expected.write_lease_required {
|
||
if current.value.leasing.is_none() {
|
||
let now = now_millis()?;
|
||
return self.persist_same_phase(current, |next| {
|
||
next.leasing = Some(LeasingEvidence::SkippedStateless);
|
||
next.updated_at_unix_millis = now;
|
||
Ok(())
|
||
});
|
||
}
|
||
return self.transition(current, DeploymentPhase::AwaitingReady);
|
||
}
|
||
|
||
let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1;
|
||
let lease_path = binding
|
||
.process_write_lease
|
||
.context("stateful target has no write-lease binding")?
|
||
.record_path;
|
||
let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path);
|
||
|
||
if matches!(
|
||
current.value.leasing.as_ref(),
|
||
Some(LeasingEvidence::Granted { .. })
|
||
) {
|
||
let activation = required(¤t.value.activation, "activation")?;
|
||
let warming = self.rehydrate_warming_token(¤t.value, now_millis()?, false)?;
|
||
let lease = current
|
||
.value
|
||
.leasing
|
||
.as_ref()
|
||
.and_then(LeasingEvidence::lease)
|
||
.context("granted Leasing evidence has no write lease")?;
|
||
let recorded_sha256 = current
|
||
.value
|
||
.leasing
|
||
.as_ref()
|
||
.and_then(LeasingEvidence::lease_sha256)
|
||
.context("granted Leasing evidence has no lease digest")?;
|
||
ensure!(
|
||
driver.observe_exact(lease)?,
|
||
"physical write lease disappeared after Granted became durable"
|
||
);
|
||
self.workload_for(required(¤t.value.plan, "transaction plan")?)?
|
||
.observe(
|
||
expected,
|
||
activation,
|
||
required(¤t.value.workload, "candidate workload")?,
|
||
)?;
|
||
ensure!(
|
||
driver.grant(expected, activation, &warming, lease)? == recorded_sha256,
|
||
"replayed physical write lease differs from Granted evidence"
|
||
);
|
||
ensure!(
|
||
self.topology()
|
||
.publish_process_write_lease(expected, activation, lease)?
|
||
== recorded_sha256,
|
||
"replayed write-lease projection differs from Granted evidence"
|
||
);
|
||
return self.transition(current, DeploymentPhase::AwaitingReady);
|
||
}
|
||
|
||
if let Some((lease, prepared_sha256)) = current
|
||
.value
|
||
.leasing
|
||
.as_ref()
|
||
.and_then(LeasingEvidence::prepared_lease)
|
||
{
|
||
let now = now_millis()?;
|
||
let activation = required(¤t.value.activation, "activation")?;
|
||
let historical_warming = self.rehydrate_warming_token(¤t.value, now, false)?;
|
||
let physical_is_exact = driver.observe_exact(lease)?;
|
||
let warming = if physical_is_exact {
|
||
historical_warming
|
||
} else {
|
||
ensure!(
|
||
driver.observe_empty()?,
|
||
"candidate write-lease store contains authority other than its durable prepared lease"
|
||
);
|
||
match self.rehydrate_warming_token(¤t.value, now, true) {
|
||
Ok(warming) => warming,
|
||
Err(_) => {
|
||
let Some((admitted, fresh_evidence, fresh_warming)) =
|
||
self.fresh_warming_for_lease(current, now)?
|
||
else {
|
||
return Ok(());
|
||
};
|
||
let replacement = self.prepare_candidate_write_lease(
|
||
&admitted.value,
|
||
&fresh_warming,
|
||
now,
|
||
)?;
|
||
let replacement_sha256 = replacement.canonical_sha256()?;
|
||
return self.persist_same_phase(&admitted, |next| {
|
||
next.warming = Some(fresh_evidence);
|
||
next.leasing = Some(LeasingEvidence::Prepared {
|
||
lease: replacement,
|
||
lease_sha256: replacement_sha256,
|
||
});
|
||
next.updated_at_unix_millis = now;
|
||
Ok(())
|
||
});
|
||
}
|
||
}
|
||
};
|
||
self.workload_for(required(¤t.value.plan, "transaction plan")?)?
|
||
.observe(
|
||
expected,
|
||
activation,
|
||
required(¤t.value.workload, "candidate workload")?,
|
||
)?;
|
||
if !physical_is_exact {
|
||
ensure!(
|
||
driver.observe_empty()?,
|
||
"candidate write-lease store changed before physical grant"
|
||
);
|
||
}
|
||
let granted_sha256 = driver.grant(expected, activation, &warming, lease)?;
|
||
ensure!(
|
||
granted_sha256 == prepared_sha256,
|
||
"granted write lease differs from the durable prepared lease"
|
||
);
|
||
let projected_sha256 = self
|
||
.topology()
|
||
.publish_process_write_lease(expected, activation, lease)?;
|
||
ensure!(
|
||
projected_sha256 == granted_sha256,
|
||
"projected write lease differs from the granted process authority"
|
||
);
|
||
let lease = lease.clone();
|
||
return self.persist_same_phase(current, |next| {
|
||
next.leasing = Some(LeasingEvidence::Granted {
|
||
lease,
|
||
lease_sha256: projected_sha256,
|
||
});
|
||
next.updated_at_unix_millis = now;
|
||
Ok(())
|
||
});
|
||
}
|
||
|
||
ensure!(
|
||
current.value.leasing.is_none(),
|
||
"stateful Leasing phase carries invalid lease evidence"
|
||
);
|
||
ensure!(
|
||
driver.observe_empty()?,
|
||
"candidate write-lease store contains authority before lease preparation"
|
||
);
|
||
let now = now_millis()?;
|
||
let Some((admitted, fresh_evidence, fresh_warming)) =
|
||
self.fresh_warming_for_lease(current, now)?
|
||
else {
|
||
return Ok(());
|
||
};
|
||
let lease = self.prepare_candidate_write_lease(&admitted.value, &fresh_warming, now)?;
|
||
let lease_sha256 = lease.canonical_sha256()?;
|
||
self.persist_same_phase(&admitted, |next| {
|
||
next.warming = Some(fresh_evidence);
|
||
next.leasing = Some(LeasingEvidence::Prepared {
|
||
lease,
|
||
lease_sha256,
|
||
});
|
||
next.updated_at_unix_millis = now;
|
||
Ok(())
|
||
})
|
||
}
|
||
|
||
fn advance_awaiting_ready(&self, current: &Stored<DeploymentTransaction>) -> Result<()> {
|
||
let now = now_millis()?;
|
||
let expected = required(¤t.value.expected, "Expected projection")?;
|
||
self.observe_candidate_before_waiting(¤t.value)?;
|
||
if expected.write_lease_required {
|
||
let activation = required(¤t.value.activation, "activation")?;
|
||
let warming = self.rehydrate_warming_token(¤t.value, now, false)?;
|
||
let lease = current
|
||
.value
|
||
.leasing
|
||
.as_ref()
|
||
.and_then(LeasingEvidence::lease)
|
||
.context("stateful transaction has no process write lease")?;
|
||
let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1;
|
||
let path = binding
|
||
.process_write_lease
|
||
.context("stateful target has no write-lease binding")?
|
||
.record_path;
|
||
ensure!(
|
||
CultCacheWriteLeaseDriver::new(¤t.value.target, path)
|
||
.observe(expected, activation, &warming, lease)?,
|
||
"candidate process write lease is no longer exact"
|
||
);
|
||
}
|
||
if ReadinessClass::of(expected)? == ReadinessClass::RouteProof {
|
||
// Ready is the candidate answering Idunn's own challenge Active,
|
||
// holding the exact lease Idunn granted. Once recorded it is
|
||
// history; the stable route's challenge is what keeps proving it.
|
||
if matches!(
|
||
current.value.ready,
|
||
Some(ReadinessEvidence::RouteProof { .. })
|
||
) {
|
||
return self.transition(current, DeploymentPhase::Routing);
|
||
}
|
||
let lease = current
|
||
.value
|
||
.leasing
|
||
.as_ref()
|
||
.and_then(LeasingEvidence::lease_sha256);
|
||
return match self.challenge_candidate(¤t.value, &["warming", "active"], lease)? {
|
||
CandidateAnswer::Silent(reason) => self.record_gate_wait(current, &reason),
|
||
CandidateAnswer::Answered { present, .. } if present.record().state == "warming" => {
|
||
self.record_gate_wait(current, "candidate is still warming")
|
||
}
|
||
CandidateAnswer::Answered { evidence, .. } => {
|
||
let adoption = LeaseAdoptionEvidence::with_ready(
|
||
lease,
|
||
&evidence.canonical_sha256,
|
||
AdoptionSource::Direct,
|
||
evidence.admitted_at_unix_millis,
|
||
);
|
||
self.persist_same_phase(current, |next| {
|
||
next.ready = Some(ReadinessEvidence::RouteProof { evidence });
|
||
next.lease_adoption = adoption;
|
||
Ok(())
|
||
})
|
||
}
|
||
};
|
||
}
|
||
{
|
||
let current_lease = current
|
||
.value
|
||
.leasing
|
||
.as_ref()
|
||
.and_then(LeasingEvidence::lease_sha256);
|
||
let Some((admitted, authenticated)) =
|
||
self.admit_latest_topology(current, current_lease)?
|
||
else {
|
||
return Ok(());
|
||
};
|
||
if admitted.envelope != current.envelope {
|
||
return Ok(());
|
||
}
|
||
let semantic_ready = is_semantic_ready(&authenticated);
|
||
let latest = required(
|
||
&admitted.value.latest_odin_observation,
|
||
"sequence-admitted Ready evidence",
|
||
)?;
|
||
if current.value.ready.as_ref().and_then(ReadinessEvidence::odin) == Some(latest) {
|
||
ensure!(semantic_ready, "stored Ready receipt changed meaning");
|
||
} else if semantic_ready {
|
||
let evidence = latest.clone();
|
||
// A Ready stateful correlation carries exactly the current
|
||
// lease (the authenticator refuses one that does not).
|
||
let adoption = authenticated
|
||
.record()
|
||
.signed_presence_sha256
|
||
.as_deref()
|
||
.and_then(|presence| {
|
||
LeaseAdoptionEvidence::with_ready(
|
||
current_lease,
|
||
presence,
|
||
AdoptionSource::OdinTopology,
|
||
evidence.admitted_at_unix_millis,
|
||
)
|
||
});
|
||
let _token = SequenceAdmittedReady {
|
||
transaction_id: admitted.value.transaction_id.clone(),
|
||
evidence: evidence.clone(),
|
||
expected: expected.clone(),
|
||
authenticated,
|
||
};
|
||
return self.persist_same_phase(&admitted, |next| {
|
||
next.ready = Some(ReadinessEvidence::OdinCorrelated { evidence });
|
||
next.lease_adoption = adoption;
|
||
Ok(())
|
||
});
|
||
} else {
|
||
return Ok(());
|
||
}
|
||
}
|
||
self.transition(current, DeploymentPhase::Routing)
|
||
}
|
||
|
||
fn advance_routing(&self, current: &Stored<DeploymentTransaction>) -> Result<()> {
|
||
let expected = required(¤t.value.expected, "Expected projection")?;
|
||
let activation = required(¤t.value.activation, "activation")?;
|
||
self.observe_candidate_before_waiting(¤t.value)?;
|
||
if current.value.routing.is_none() {
|
||
let admitted = if ReadinessClass::of(expected)? == ReadinessClass::RouteProof {
|
||
// Ready is the candidate's own proof, recorded in AwaitingReady.
|
||
// Validation refuses Routing without a Ready receipt, and a
|
||
// receipt the other party vouched for is held before any step
|
||
// runs. No Odin correlation is read to admit a route.
|
||
current.clone()
|
||
} else {
|
||
let current_lease = current
|
||
.value
|
||
.leasing
|
||
.as_ref()
|
||
.and_then(LeasingEvidence::lease_sha256);
|
||
let Some((admitted, authenticated)) =
|
||
self.admit_latest_topology(current, current_lease)?
|
||
else {
|
||
return Ok(());
|
||
};
|
||
if admitted.envelope != current.envelope {
|
||
return Ok(());
|
||
}
|
||
ensure!(
|
||
is_semantic_ready(&authenticated),
|
||
"latest Odin observation is not Ready at route admission"
|
||
);
|
||
let latest = required(
|
||
&admitted.value.latest_odin_observation,
|
||
"current route-admission topology evidence",
|
||
)?;
|
||
if admitted.value.ready.as_ref().and_then(ReadinessEvidence::odin)
|
||
!= Some(latest)
|
||
{
|
||
let latest = latest.clone();
|
||
return self.persist_same_phase(&admitted, |next| {
|
||
next.ready = Some(ReadinessEvidence::OdinCorrelated { evidence: latest });
|
||
Ok(())
|
||
});
|
||
}
|
||
let ready = self.rehydrate_ready_token(&admitted.value, now_millis()?, true)?;
|
||
ensure!(
|
||
ready.transaction_id() == admitted.value.transaction_id,
|
||
"Ready token belongs to another transaction"
|
||
);
|
||
admitted
|
||
};
|
||
validate_live_providers_for_deploy(admitted.value.command_kind, || {
|
||
self.validate_selected_providers_current(required(
|
||
&admitted.value.plan,
|
||
"transaction plan",
|
||
)?)
|
||
})?;
|
||
self.ensure_transaction_write_lease_current(&admitted.value, now_millis()?)?;
|
||
let evidence = if expected.route.is_some() {
|
||
let binding = admitted.value.plan.as_ref().unwrap().parsed_inputs()?.1;
|
||
let route_binding = binding.route.context("routed plan has no route binding")?;
|
||
let preflight = required(&admitted.value.route_preflight, "route preflight")?;
|
||
let driver = self.route_driver(route_binding);
|
||
let gate = self.route_gate(&admitted.value.target, admitted.value.command_kind);
|
||
ensure!(
|
||
preflight.candidate_runtime_instance_id == activation.runtime_instance_id,
|
||
"route preflight belongs to another runtime instance"
|
||
);
|
||
let rollback_allowed = may_rollback_route_after_failed_proof(required(
|
||
&admitted.value.fencing,
|
||
"route fencing evidence",
|
||
)?);
|
||
let membership_sha256 = driver.install(
|
||
expected,
|
||
&activation.runtime_instance_id,
|
||
preflight,
|
||
rollback_allowed,
|
||
&gate,
|
||
)?;
|
||
let observation = match self.prove_stable_route(
|
||
&admitted.value,
|
||
&driver,
|
||
membership_sha256,
|
||
) {
|
||
Ok(observation) => observation,
|
||
Err(proof_error) if !rollback_allowed => {
|
||
return Err(proof_error).context(
|
||
"incumbent authority was fenced; candidate route remains installed for fail-closed retry",
|
||
);
|
||
}
|
||
Err(proof_error) => {
|
||
return match driver.rollback(
|
||
expected,
|
||
&activation.runtime_instance_id,
|
||
preflight,
|
||
&gate,
|
||
) {
|
||
Ok(()) => Err(proof_error)
|
||
.context("candidate did not answer its stable route challenge"),
|
||
Err(rollback_error) => Err(proof_error).context(format!(
|
||
"candidate did not answer its stable route challenge; exact route rollback also failed: {rollback_error:#}"
|
||
)),
|
||
};
|
||
}
|
||
};
|
||
ensure!(
|
||
driver.observe_membership(expected, &observation.membership_sha256)?,
|
||
"route membership changed during its signed stable-listener observation"
|
||
);
|
||
self.ensure_transaction_write_lease_current(&admitted.value, now_millis()?)?;
|
||
let promoted_at_unix_millis = observation.observed_at_unix_millis;
|
||
RoutingEvidence::Promoted {
|
||
observation,
|
||
promoted_at_unix_millis,
|
||
}
|
||
} else {
|
||
RoutingEvidence::SkippedUnrouted
|
||
};
|
||
let now = now_millis()?;
|
||
return self.persist_same_phase(&admitted, |next| {
|
||
next.routing = Some(evidence);
|
||
next.updated_at_unix_millis = now;
|
||
Ok(())
|
||
});
|
||
}
|
||
self.transition(current, DeploymentPhase::Committing)
|
||
}
|
||
|
||
fn advance_committing(&self, current: &Stored<DeploymentTransaction>) -> Result<()> {
|
||
self.observe_candidate_before_waiting(¤t.value)?;
|
||
let route_proof = ReadinessClass::of(required(
|
||
¤t.value.expected,
|
||
"Expected projection",
|
||
)?)? == ReadinessClass::RouteProof;
|
||
let ready_current = if route_proof {
|
||
// The candidate's Ready proof is durable, and the stable route's
|
||
// challenge below is its currency. No Odin is read to commit.
|
||
current.clone()
|
||
} else {
|
||
let current_lease_sha256 = current
|
||
.value
|
||
.leasing
|
||
.as_ref()
|
||
.and_then(LeasingEvidence::lease_sha256)
|
||
.map(str::to_owned);
|
||
// A newer sequence admitted here changed the record, and commit
|
||
// goes on from that record rather than returning: returning let
|
||
// Odin's next re-stamp abort every attempt (F16).
|
||
let Some((ready_current, authenticated)) =
|
||
self.admit_latest_topology(current, current_lease_sha256.as_deref())?
|
||
else {
|
||
return Ok(());
|
||
};
|
||
ensure!(
|
||
is_semantic_ready(&authenticated),
|
||
"latest Odin observation is not Ready at admission commit"
|
||
);
|
||
// Not require_current: the Ready receipt is durable evidence, so
|
||
// authenticating it against `now` asks a stored record to be live and
|
||
// refuses it once it ages past the 30s observation window. Route
|
||
// admission (:3906) refreshes the receipt to the latest observation and
|
||
// can therefore demand currency; by Committing the receipt is history.
|
||
// Currency here is answered by the latest observation, authenticated
|
||
// and checked for semantic readiness immediately above.
|
||
self.rehydrate_ready_token(&ready_current.value, now_millis()?, false)?;
|
||
ready_current
|
||
};
|
||
validate_live_providers_for_deploy(ready_current.value.command_kind, || {
|
||
self.validate_selected_providers_current(required(
|
||
&ready_current.value.plan,
|
||
"transaction plan",
|
||
)?)
|
||
})?;
|
||
|
||
let expected = required(&ready_current.value.expected, "Expected projection")?;
|
||
let activation = required(&ready_current.value.activation, "activation")?;
|
||
let workload = required(&ready_current.value.workload, "workload")?;
|
||
self.workload_for(required(&ready_current.value.plan, "transaction plan")?)?
|
||
.observe(expected, activation, workload)?;
|
||
self.ensure_transaction_write_lease_current(&ready_current.value, now_millis()?)?;
|
||
if let Some(route) = ready_current
|
||
.value
|
||
.routing
|
||
.as_ref()
|
||
.and_then(RoutingEvidence::observation)
|
||
{
|
||
let binding = ready_current
|
||
.value
|
||
.plan
|
||
.as_ref()
|
||
.unwrap()
|
||
.parsed_inputs()?
|
||
.1;
|
||
let driver =
|
||
self.route_driver(binding.route.context("routed plan has no route binding")?);
|
||
ensure!(
|
||
driver.observe_membership(expected, &route.membership_sha256)?,
|
||
"route membership changed before admission commit"
|
||
);
|
||
let current_route = self.prove_stable_route(
|
||
&ready_current.value,
|
||
&driver,
|
||
route.membership_sha256.clone(),
|
||
)?;
|
||
ensure!(
|
||
current_route.route_id == route.route_id
|
||
&& current_route.runtime_instance_id == route.runtime_instance_id,
|
||
"stable route changed incarnation before admission commit"
|
||
);
|
||
ensure!(
|
||
driver.observe_membership(expected, &route.membership_sha256)?,
|
||
"route membership changed during the final signed admission challenge"
|
||
);
|
||
self.ensure_transaction_write_lease_current(&ready_current.value, now_millis()?)?;
|
||
}
|
||
|
||
let commit_current = if route_proof {
|
||
ready_current
|
||
} else {
|
||
let current_lease_sha256 = ready_current
|
||
.value
|
||
.leasing
|
||
.as_ref()
|
||
.and_then(LeasingEvidence::lease_sha256)
|
||
.map(str::to_owned);
|
||
// The final proof's own presence makes Odin re-stamp, so this
|
||
// admission usually writes. Commit is from the record it wrote.
|
||
let Some((commit_current, authenticated)) =
|
||
self.admit_latest_topology(&ready_current, current_lease_sha256.as_deref())?
|
||
else {
|
||
return Ok(());
|
||
};
|
||
ensure!(
|
||
is_semantic_ready(&authenticated),
|
||
"latest Odin observation is not Ready after the final admission challenge"
|
||
);
|
||
self.rehydrate_ready_token(&commit_current.value, now_millis()?, false)?;
|
||
commit_current
|
||
};
|
||
let now = now_millis()?;
|
||
validate_live_providers_for_deploy(commit_current.value.command_kind, || {
|
||
self.validate_selected_providers_current(required(
|
||
&commit_current.value.plan,
|
||
"transaction plan",
|
||
)?)
|
||
})?;
|
||
self.workload_for(required(&commit_current.value.plan, "transaction plan")?)?
|
||
.observe(
|
||
required(&commit_current.value.expected, "Expected projection")?,
|
||
required(&commit_current.value.activation, "activation")?,
|
||
required(&commit_current.value.workload, "workload")?,
|
||
)?;
|
||
self.ensure_transaction_write_lease_current(&commit_current.value, now)?;
|
||
|
||
let snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
let incumbent = self.exact_incumbent(&snapshot, &commit_current.value)?;
|
||
let odin_authority = (!route_proof)
|
||
.then(|| self.current_odin_authority(&snapshot))
|
||
.transpose()?;
|
||
let generation = AdmittedGeneration::from_transaction(
|
||
&commit_current.value,
|
||
odin_authority,
|
||
now,
|
||
)?;
|
||
let post_commit_cleanup = PostCommitCleanup {
|
||
incumbent: match incumbent {
|
||
Some(incumbent)
|
||
if incumbent_was_stopped_during_fencing(required(
|
||
&commit_current.value.fencing,
|
||
"commit fencing evidence",
|
||
)?) =>
|
||
{
|
||
IncumbentCleanupEvidence::Complete {
|
||
generation_id: incumbent.value.generation_id.clone(),
|
||
}
|
||
}
|
||
Some(incumbent) => IncumbentCleanupEvidence::Pending {
|
||
generation_id: incumbent.value.generation_id.clone(),
|
||
workload: incumbent.value.workload.clone(),
|
||
},
|
||
None => IncumbentCleanupEvidence::SkippedNoIncumbent,
|
||
},
|
||
source: match commit_current.value.command_kind {
|
||
CommandKind::Deploy => SourceCleanupEvidence::Pending,
|
||
CommandKind::Continuity => SourceCleanupEvidence::SkippedContinuity,
|
||
},
|
||
};
|
||
let mut complete = commit_current.value.clone();
|
||
complete.enter_phase(DeploymentPhase::Complete, now);
|
||
complete.last_error = None;
|
||
complete.completion = Some(TransactionCompletion::Admitted {
|
||
generation_id: generation.generation_id.clone(),
|
||
});
|
||
complete.post_commit_cleanup = Some(post_commit_cleanup);
|
||
// enter_phase ran before the cleanup was set: a draining record
|
||
// carries its drain's deadline.
|
||
complete.retime(now);
|
||
complete.validate()?;
|
||
|
||
let admitted_expected = CultCacheExpectedEnvelope {
|
||
r#type: AdmittedGeneration::TYPE.into(),
|
||
key: generation.target.clone(),
|
||
current: incumbent.map(|stored| stored.envelope.clone()),
|
||
};
|
||
ensure!(
|
||
SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange(
|
||
&[
|
||
CultCacheExpectedEnvelope {
|
||
r#type: DeploymentTransaction::TYPE.into(),
|
||
key: commit_current.value.transaction_id.clone(),
|
||
current: Some(commit_current.envelope.clone()),
|
||
},
|
||
admitted_expected,
|
||
],
|
||
&[
|
||
transaction_envelope(&complete, now)?,
|
||
admitted_envelope(&generation, now)?,
|
||
],
|
||
)?,
|
||
"incumbent or transaction changed before atomic admission commit"
|
||
);
|
||
Ok(())
|
||
}
|
||
|
||
/// The admitted generation this transaction was proven against. A
|
||
/// pre-handover deployment is rebased onto the admitted generation before
|
||
/// any step reads its incumbent, and the handover pins it after, so a
|
||
/// mismatch here is a fault, not a race.
|
||
fn exact_incumbent<'a>(
|
||
&self,
|
||
snapshot: &'a ControlSnapshot,
|
||
transaction: &DeploymentTransaction,
|
||
) -> Result<Option<&'a Stored<AdmittedGeneration>>> {
|
||
let current = snapshot.admitted_for(&transaction.target);
|
||
match (&transaction.incumbent_generation_id, current) {
|
||
(None, None) => Ok(None),
|
||
(Some(expected), Some(current)) if current.value.generation_id == *expected => {
|
||
Ok(Some(current))
|
||
}
|
||
_ => bail!(
|
||
"admitted generation changed under a transaction that owns the target or names it as its source"
|
||
),
|
||
}
|
||
}
|
||
|
||
fn incumbent_lease_sha256_for_warming(
|
||
&self,
|
||
snapshot: &ControlSnapshot,
|
||
transaction: &DeploymentTransaction,
|
||
) -> Result<Option<String>> {
|
||
if let Some(fencing) = &transaction.fencing {
|
||
return match fencing {
|
||
FencingEvidence::SkippedStateless => Ok(None),
|
||
FencingEvidence::Revoked {
|
||
incumbent_lease_sha256,
|
||
..
|
||
} => Ok(incumbent_lease_sha256.clone()),
|
||
};
|
||
}
|
||
self.exact_incumbent(snapshot, transaction)?
|
||
.and_then(|incumbent| incumbent.value.leasing.lease())
|
||
.map(IdunnProcessWriteLeaseRecord::canonical_sha256)
|
||
.transpose()
|
||
}
|
||
|
||
fn fresh_warming_for_lease(
|
||
&self,
|
||
current: &Stored<DeploymentTransaction>,
|
||
now: u64,
|
||
) -> Result<
|
||
Option<(
|
||
Stored<DeploymentTransaction>,
|
||
WarmingEvidence,
|
||
SequenceAdmittedWarming,
|
||
)>,
|
||
> {
|
||
let prior = self.rehydrate_warming_token(¤t.value, now, false)?;
|
||
match required(¤t.value.warming, "pre-fence Warming evidence")?.clone() {
|
||
WarmingEvidence::FirstOdinDirect { .. } => {
|
||
// Whether this transaction may observe Odin directly was
|
||
// settled when the Warming evidence was recorded, and that
|
||
// decision is durable in the transaction. Re-deciding it here
|
||
// with the narrower "no admitted Odin" test contradicted the
|
||
// caller and failed the refresh. What the refresh owes is
|
||
// freshness, and the replay check below is what provides it.
|
||
// That the target is Odin was already required by the
|
||
// rehydration above, which authenticates the recorded answer.
|
||
let (evidence, present) = self.observe_first_odin_warming(¤t.value)?;
|
||
let warming = WarmingEvidence::FirstOdinDirect { evidence };
|
||
let token = SequenceAdmittedWarming::from_direct_presence(
|
||
current.value.transaction_id.clone(),
|
||
warming.clone(),
|
||
present,
|
||
)?;
|
||
ensure!(
|
||
token.signed_presence_sha256() != prior.signed_presence_sha256(),
|
||
"first Odin replayed its pre-fence Warming presence"
|
||
);
|
||
Ok(Some((current.clone(), warming, token)))
|
||
}
|
||
WarmingEvidence::RouteProofDirect { .. } => {
|
||
// A stateful route-proof candidate is warming until it holds
|
||
// the lease this refresh is about to bind, so the fresh
|
||
// presence must be warming again, and must be a new one.
|
||
let CandidateAnswer::Answered { evidence, present } =
|
||
self.challenge_candidate(¤t.value, &["warming"], None)?
|
||
else {
|
||
return Ok(None);
|
||
};
|
||
let warming = WarmingEvidence::RouteProofDirect { evidence };
|
||
let token = SequenceAdmittedWarming::from_direct_presence(
|
||
current.value.transaction_id.clone(),
|
||
warming.clone(),
|
||
present,
|
||
)?;
|
||
ensure!(
|
||
token.signed_presence_sha256() != prior.signed_presence_sha256(),
|
||
"route-proof candidate replayed its pre-fence Warming presence"
|
||
);
|
||
Ok(Some((current.clone(), warming, token)))
|
||
}
|
||
WarmingEvidence::OdinTopology {
|
||
evidence: prior_evidence,
|
||
} => {
|
||
let Some((admitted, authenticated)) = self.admit_latest_topology(current, None)?
|
||
else {
|
||
return Ok(None);
|
||
};
|
||
let fresh_evidence = required(
|
||
&admitted.value.latest_odin_observation,
|
||
"post-fence Odin Warming observation",
|
||
)?
|
||
.clone();
|
||
let snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
let incumbent_lease_sha256 =
|
||
self.incumbent_lease_sha256_for_warming(&snapshot, &admitted.value)?;
|
||
if !is_semantic_warming(
|
||
required(&admitted.value.expected, "Warming Expected projection")?,
|
||
required(&admitted.value.activation, "Warming activation")?,
|
||
incumbent_lease_sha256.as_deref(),
|
||
&authenticated,
|
||
)? {
|
||
return Ok(None);
|
||
}
|
||
let token = SequenceAdmittedWarming::from_topology(
|
||
admitted.value.transaction_id.clone(),
|
||
authenticated,
|
||
)?;
|
||
if !provider_warming_advanced(
|
||
prior_evidence.publisher_sequence,
|
||
prior.signed_presence_sha256(),
|
||
fresh_evidence.publisher_sequence,
|
||
token.signed_presence_sha256(),
|
||
) {
|
||
return Ok(None);
|
||
}
|
||
Ok(Some((
|
||
admitted,
|
||
WarmingEvidence::OdinTopology {
|
||
evidence: fresh_evidence,
|
||
},
|
||
token,
|
||
)))
|
||
}
|
||
}
|
||
}
|
||
|
||
fn prepare_candidate_write_lease(
|
||
&self,
|
||
transaction: &DeploymentTransaction,
|
||
warming: &SequenceAdmittedWarming,
|
||
now: u64,
|
||
) -> Result<IdunnProcessWriteLeaseRecord> {
|
||
let expected = required(&transaction.expected, "Expected projection")?;
|
||
let activation = required(&transaction.activation, "activation")?;
|
||
ensure!(
|
||
expected.write_lease_required
|
||
&& warming.transaction_id() == transaction.transaction_id
|
||
&& warming.runtime_instance_id() == activation.runtime_instance_id,
|
||
"fresh Warming token does not own this stateful candidate"
|
||
);
|
||
let snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
let epoch = self
|
||
.exact_incumbent(&snapshot, transaction)?
|
||
.and_then(|generation| generation.value.leasing.lease())
|
||
.map_or(1, |lease| lease.lease_epoch.saturating_add(1));
|
||
let lease = IdunnProcessWriteLeaseRecord {
|
||
schema_version: IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into(),
|
||
target: expected.target.clone(),
|
||
expected_projection_sha256: expected.canonical_sha256()?,
|
||
plan_id: expected.plan_id.clone(),
|
||
incarnation_id: expected.incarnation_id.clone(),
|
||
sealed_release_id: expected.sealed_release_id.clone(),
|
||
activation_witness_sha256: activation.canonical_sha256()?,
|
||
state_schema_generation: expected
|
||
.state_schema_generation
|
||
.clone()
|
||
.context("stateful Expected has no schema generation")?,
|
||
state_contract_sha256: expected
|
||
.state_contract_sha256
|
||
.clone()
|
||
.context("stateful Expected has no state contract")?,
|
||
runtime_id: expected.runtime_id.clone(),
|
||
runtime_instance_id: activation.runtime_instance_id.clone(),
|
||
warming_presence_sha256: warming.signed_presence_sha256().to_owned(),
|
||
lease_epoch: epoch,
|
||
issued_at_unix_millis: now,
|
||
};
|
||
lease.validate()?;
|
||
Ok(lease)
|
||
}
|
||
|
||
fn ensure_transaction_write_lease_current(
|
||
&self,
|
||
transaction: &DeploymentTransaction,
|
||
now: u64,
|
||
) -> Result<()> {
|
||
let expected = required(&transaction.expected, "Expected projection")?;
|
||
let lease = transaction
|
||
.leasing
|
||
.as_ref()
|
||
.and_then(LeasingEvidence::lease);
|
||
ensure!(
|
||
expected.write_lease_required == lease.is_some(),
|
||
"transaction write-lease disposition differs from Expected"
|
||
);
|
||
let Some(lease) = lease else {
|
||
return Ok(());
|
||
};
|
||
let activation = required(&transaction.activation, "activation")?;
|
||
let warming = self.rehydrate_warming_token(transaction, now, false)?;
|
||
let lease_path = required(&transaction.plan, "transaction plan")?
|
||
.parsed_inputs()?
|
||
.1
|
||
.process_write_lease
|
||
.context("stateful target has no write-lease binding")?
|
||
.record_path;
|
||
ensure!(
|
||
CultCacheWriteLeaseDriver::new(&transaction.target, lease_path)
|
||
.observe(expected, activation, &warming, lease,)?,
|
||
"process write lease changed across the authority boundary"
|
||
);
|
||
Ok(())
|
||
}
|
||
|
||
fn select_candidate_port(&self, binding: &OperatorBinding) -> Result<Option<u16>> {
|
||
let Some(route) = &binding.route else {
|
||
return Ok(None);
|
||
};
|
||
let used = ControlSnapshot::read(&self.options.state_store)?.candidate_ports_in_use();
|
||
(route.private_port_start..=route.private_port_end)
|
||
.find(|port| !used.contains(port))
|
||
.map(Some)
|
||
.context("no private candidate port remains in the operator range")
|
||
}
|
||
|
||
fn deployment_authorization(
|
||
&self,
|
||
current: &Stored<DeploymentTransaction>,
|
||
now: u64,
|
||
) -> Result<Option<DeploymentAuthorization>> {
|
||
let expected = required(¤t.value.expected, "Expected projection")?;
|
||
let plan = required(¤t.value.plan, "transaction plan")?;
|
||
let binding = plan.parsed_inputs()?.1;
|
||
let Some((record, canonical_bytes)) =
|
||
read_deployment_brake(&binding.brakes.deployment_store)?
|
||
else {
|
||
return Ok(None);
|
||
};
|
||
let anchor = read_trust_anchor::<IdunnDeploymentBrakeOperatorIdentity>(
|
||
&self.options.deployment_brake_operator_anchor,
|
||
)?;
|
||
if evaluate_idunn_deployment_brake(
|
||
IdunnDeploymentBrakeObservation::Present(&record),
|
||
&anchor,
|
||
&expected.runtime_id,
|
||
&expected.sealed_release_id,
|
||
¤t.value.transaction_id,
|
||
now,
|
||
)
|
||
.is_err()
|
||
{
|
||
return Ok(None);
|
||
}
|
||
let authorization_id = record
|
||
.authorization_id
|
||
.clone()
|
||
.context("released brake has no authorization id")?;
|
||
let snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
ensure!(
|
||
!snapshot.transactions.iter().any(|stored| {
|
||
stored
|
||
.value
|
||
.deployment_authorization
|
||
.as_ref()
|
||
.is_some_and(|used| used.authorization_id == authorization_id)
|
||
}),
|
||
"deployment authorization was already consumed"
|
||
);
|
||
Ok(Some(DeploymentAuthorization {
|
||
authorization_id,
|
||
brake_sha256: sha256_id(&canonical_bytes),
|
||
canonical_brake_bytes: canonical_bytes,
|
||
authorized_at_unix_millis: now,
|
||
}))
|
||
}
|
||
|
||
/// Whether a deployment awaiting the handover is free to take it: the
|
||
/// deployment brake has released its exact transaction, so the stop a yield
|
||
/// makes is followed by the deployment's own landing. A Deploy at its
|
||
/// Sealing tail has not passed the brake yet and is read live; one that is
|
||
/// still building, or whose brake cannot be read, is held. An operator's
|
||
/// hold is a reason to leave a serving or retrying body alone.
|
||
fn deploy_can_move(&self, deploy: &Stored<DeploymentTransaction>, now: u64) -> bool {
|
||
deploy.value.deployment_authorization.is_some()
|
||
|| matches!(self.deployment_authorization(deploy, now), Ok(Some(_)))
|
||
}
|
||
|
||
/// The deployment a continuity gives way to: one on its target that awaits
|
||
/// the handover and can take it, while the continuity's own candidate is
|
||
/// unproven (ruling c2d-yield-only-unproven-to-movable). The one test the
|
||
/// deadline's yield and the boot lift's abort both ask, so neither stops a
|
||
/// serving candidate, or one a braked Deploy cannot replace.
|
||
fn deploy_the_continuity_yields_to<'a>(
|
||
&self,
|
||
snapshot: &'a ControlSnapshot,
|
||
continuity: &DeploymentTransaction,
|
||
now: u64,
|
||
) -> Option<&'a Stored<DeploymentTransaction>> {
|
||
if continuity.serving_proven() {
|
||
return None;
|
||
}
|
||
snapshot.on_target(&continuity.target).find(|other| {
|
||
other.value.awaits_handover() && self.deploy_can_move(other, now)
|
||
})
|
||
}
|
||
|
||
fn lifecycle_allows(&self, current: &Stored<DeploymentTransaction>, now: u64) -> Result<bool> {
|
||
let plan = required(¤t.value.plan, "continuity plan")?;
|
||
let binding = plan.parsed_inputs()?.1;
|
||
let expected = required(¤t.value.expected, "continuity Expected")?;
|
||
match read_lifecycle_brake(&binding.brakes.lifecycle_store) {
|
||
Ok(Some(record)) => Ok(evaluate_idunn_continuity_restart(
|
||
IdunnLifecycleBrakeObservation::Present(&record),
|
||
&expected.runtime_id,
|
||
¤t.value.target,
|
||
now,
|
||
)
|
||
.is_ok()),
|
||
Ok(None) => Ok(evaluate_idunn_continuity_restart(
|
||
IdunnLifecycleBrakeObservation::Missing,
|
||
&expected.runtime_id,
|
||
¤t.value.target,
|
||
now,
|
||
)
|
||
.is_ok()),
|
||
Err(_) => Ok(false),
|
||
}
|
||
}
|
||
|
||
/// A phase that waits on Odin's evidence about the candidate must first
|
||
/// confirm the candidate still exists. Otherwise a candidate that died
|
||
/// while its correlation was pending is never observed again: the phase
|
||
/// returns early on "no evidence yet" every tick, the error that would have
|
||
/// triggered the post-fencing abort never happens, and the transaction
|
||
/// owns the target forever behind a process that is gone.
|
||
fn observe_candidate_before_waiting(&self, transaction: &DeploymentTransaction) -> Result<()> {
|
||
self.workload_for(required(&transaction.plan, "transaction plan")?)?
|
||
.observe(
|
||
required(&transaction.expected, "Expected projection")?,
|
||
required(&transaction.activation, "activation")?,
|
||
required(&transaction.workload, "candidate workload")?,
|
||
)
|
||
.map(|_| ())
|
||
}
|
||
|
||
/// Withdraw one projected incarnation of this target that nothing owns:
|
||
/// not the admitted generation, not any live transaction. A replaced
|
||
/// incumbent, an aborted candidate whose reconciliation was skipped, or a
|
||
/// record left by the projection migration all end here. One per tick;
|
||
/// the projection is a shared file and each withdrawal is one CAS.
|
||
fn retire_stale_incarnations(
|
||
&self,
|
||
snapshot: &ControlSnapshot,
|
||
current: &Stored<AdmittedGeneration>,
|
||
) -> Result<bool> {
|
||
let topology = self.topology();
|
||
let target = current.value.target.as_str();
|
||
let owned = snapshot
|
||
.transactions
|
||
.iter()
|
||
.filter(|stored| stored.value.target == target && !stored.value.is_terminal())
|
||
.filter_map(|stored| stored.value.expected.as_ref())
|
||
.map(IdunnExpectedIncarnationRecord::canonical_sha256)
|
||
.collect::<Result<BTreeSet<_>>>()?;
|
||
let admitted_sha256 = current.value.expected.canonical_sha256()?;
|
||
let Some(stale) = topology
|
||
.projected_incarnations(target)?
|
||
.into_iter()
|
||
.find(|expected| {
|
||
expected
|
||
.canonical_sha256()
|
||
.is_ok_and(|sha256| sha256 != admitted_sha256 && !owned.contains(&sha256))
|
||
})
|
||
else {
|
||
return Ok(false);
|
||
};
|
||
let provider_anchor = self.provider_anchor_for_plan(¤t.value.plan)?;
|
||
topology.withdraw_stale_incarnation(&stale, &provider_anchor)?;
|
||
eprintln!(
|
||
"Idunn withdrew stale {} incarnation {} from the projection",
|
||
target, stale.incarnation_id
|
||
);
|
||
Ok(true)
|
||
}
|
||
|
||
/// The lifecycle brake as it applies to restarting one admitted
|
||
/// generation: the same evaluation `lifecycle_allows` makes for a
|
||
/// continuity transaction, asked before one is minted.
|
||
fn lifecycle_allows_generation(
|
||
&self,
|
||
current: &Stored<AdmittedGeneration>,
|
||
now: u64,
|
||
) -> Result<bool> {
|
||
let binding = current.value.plan.parsed_inputs()?.1;
|
||
let expected = ¤t.value.expected;
|
||
match read_lifecycle_brake(&binding.brakes.lifecycle_store) {
|
||
Ok(Some(record)) => Ok(evaluate_idunn_continuity_restart(
|
||
IdunnLifecycleBrakeObservation::Present(&record),
|
||
&expected.runtime_id,
|
||
¤t.value.target,
|
||
now,
|
||
)
|
||
.is_ok()),
|
||
Ok(None) => Ok(evaluate_idunn_continuity_restart(
|
||
IdunnLifecycleBrakeObservation::Missing,
|
||
&expected.runtime_id,
|
||
¤t.value.target,
|
||
now,
|
||
)
|
||
.is_ok()),
|
||
Err(_) => Ok(false),
|
||
}
|
||
}
|
||
|
||
/// A deployment waits on the transaction that owns or drains its target.
|
||
/// The owner bounds the wait, so the waiter's own Warming deadline is
|
||
/// disarmed (the resolver re-arms it when the target is free). When the
|
||
/// owner's deadline has passed in a state Idunn will not end by itself and
|
||
/// will not yield (a wedged abort or drain), the waiter ends through its own
|
||
/// pre-fencing abort (the caller's Err path): it never aborts the owner. A
|
||
/// waiter keeps waiting on an owner that retries forward, because that
|
||
/// owner yields to it on its next pass.
|
||
fn wait_for_owner(
|
||
&self,
|
||
current: &Stored<DeploymentTransaction>,
|
||
owner_id: &str,
|
||
) -> Result<()> {
|
||
let held = ¤t.value;
|
||
let now = now_millis()?;
|
||
let snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
if snapshot.transactions.iter().any(|stored| {
|
||
stored.value.transaction_id == owner_id
|
||
&& stored.value.leaves_to_operator(now)
|
||
&& !stored.value.yields_to_a_deploy()
|
||
}) {
|
||
bail!(
|
||
"owner-left-to-operator: target {} is owned by {owner_id}, which Idunn has left to the operator; this deployment ends before fencing",
|
||
held.target
|
||
);
|
||
}
|
||
let detail = bounded_detail(&waiting_for_owner(&held.target, owner_id));
|
||
if held.last_error.as_deref() == Some(detail.as_str()) && held.phase_deadline.is_none() {
|
||
return Ok(());
|
||
}
|
||
let mut next = held.clone();
|
||
next.last_error = Some(detail);
|
||
next.phase_deadline = None;
|
||
next.updated_at_unix_millis = now;
|
||
replace_transaction(&self.options.state_store, current, &next)
|
||
}
|
||
|
||
/// Idunn has run out of ways to end this record. It stays owned and its
|
||
/// step keeps retrying; the operator is told. The incident comes first and
|
||
/// the trace second, and only for a record that is left to the operator
|
||
/// (an operator's expiry of a stateful deploy is reported, not paged).
|
||
fn leave_to_operator(&self, transaction: &DeploymentTransaction, cause: &str) {
|
||
let id = &transaction.transaction_id;
|
||
let now = now_millis();
|
||
if now.as_ref().is_ok_and(|now| transaction.leaves_to_operator(*now)) {
|
||
self.record_incident(
|
||
"open",
|
||
id,
|
||
now.and_then(|now| self.incidents().open(IncidentCondition::OperatorRequired, id, now)),
|
||
);
|
||
}
|
||
self.report_once(
|
||
&format!("deadline:{id}"),
|
||
format!("Idunn leaves {id} to the operator: {cause}; it keeps retrying the step"),
|
||
);
|
||
}
|
||
|
||
fn record_gate_wait(
|
||
&self,
|
||
current: &Stored<DeploymentTransaction>,
|
||
detail: &str,
|
||
) -> Result<()> {
|
||
let detail = bounded_detail(detail);
|
||
if current.value.last_error.as_deref() == Some(detail.as_str()) {
|
||
return Ok(());
|
||
}
|
||
let mut next = current.value.clone();
|
||
next.last_error = Some(detail);
|
||
next.updated_at_unix_millis = now_millis()?;
|
||
replace_transaction(&self.options.state_store, current, &next)
|
||
}
|
||
|
||
fn runtime_authority(
|
||
&self,
|
||
transaction: &DeploymentTransaction,
|
||
) -> Result<cultnet_rs::VerifiedRuntimeAuthority> {
|
||
self.runtime_authority_parts(
|
||
required(&transaction.plan, "transaction plan")?,
|
||
required(&transaction.expected, "Expected projection")?,
|
||
required(&transaction.activation, "activation")?,
|
||
)
|
||
}
|
||
|
||
/// The stable route's proof: a challenged presence that is Active.
|
||
fn authenticate_routed_presence(
|
||
&self,
|
||
authority: &cultnet_rs::VerifiedRuntimeAuthority,
|
||
current_write_lease_sha256: Option<&str>,
|
||
message_id: &str,
|
||
challenged_at_unix_millis: u64,
|
||
received_at_unix_millis: u64,
|
||
canonical_presence: &[u8],
|
||
) -> Result<(String, u64)> {
|
||
let present = self.authenticate_challenged_presence(
|
||
authority,
|
||
&["active"],
|
||
current_write_lease_sha256,
|
||
message_id,
|
||
challenged_at_unix_millis,
|
||
received_at_unix_millis,
|
||
canonical_presence,
|
||
)?;
|
||
Ok((
|
||
present.signed_presence_sha256().to_owned(),
|
||
received_at_unix_millis,
|
||
))
|
||
}
|
||
|
||
/// A presence Idunn challenged for itself, from the stable endpoint or the
|
||
/// candidate's: signed by the provider and the launch's activation key,
|
||
/// bound to current authority, minted after the challenge, answering that
|
||
/// exact challenge, in one of `states`. A warming presence holds no write
|
||
/// lease; a presence in any other state holds exactly the current one.
|
||
/// Every disagreement with authority is named, so a capacity below the
|
||
/// Expected minimum reads as that and not as a generic refusal.
|
||
fn authenticate_challenged_presence(
|
||
&self,
|
||
authority: &cultnet_rs::VerifiedRuntimeAuthority,
|
||
states: &[&str],
|
||
current_write_lease_sha256: Option<&str>,
|
||
message_id: &str,
|
||
challenged_at_unix_millis: u64,
|
||
received_at_unix_millis: u64,
|
||
canonical_presence: &[u8],
|
||
) -> Result<cultnet_rs::VerifiedRuntimePresence> {
|
||
ensure!(
|
||
received_at_unix_millis >= challenged_at_unix_millis,
|
||
"route proof predates its challenge"
|
||
);
|
||
let authenticated = authenticate_runtime_presence_claim(
|
||
canonical_presence,
|
||
authority,
|
||
RuntimePresenceAuthenticationContext {
|
||
trusted_received_at_unix_millis: received_at_unix_millis,
|
||
maximum_age_millis: self.options.topology_maximum_age_millis,
|
||
maximum_future_skew_millis: self.options.topology_maximum_future_skew_millis,
|
||
},
|
||
)?;
|
||
let correlation = correlate_runtime_presence_claim(authenticated, authority)?;
|
||
if !correlation.disagreements().is_empty() {
|
||
return Err(PresenceDisagrees {
|
||
disagreements: correlation.disagreements().to_vec(),
|
||
}
|
||
.into());
|
||
}
|
||
let present = correlation.into_undisputed_present()?;
|
||
let presence = present.record();
|
||
ensure!(
|
||
presence.observed_at_unix_millis >= challenged_at_unix_millis,
|
||
"route proof returned a presence minted before its challenge"
|
||
);
|
||
ensure!(
|
||
states.contains(&presence.state.as_str()),
|
||
"route proof runtime is {}, not {}",
|
||
presence.state,
|
||
states.join(" or ")
|
||
);
|
||
ensure!(
|
||
presence.detail == format!("route-observation:{message_id}"),
|
||
"route proof response is not bound to the exact challenge"
|
||
);
|
||
// A warming presence is exempt from the lease comparison: it is
|
||
// Idunn's own lease that a warming candidate cannot hold yet, and the
|
||
// authenticator above already refuses one that claims a lease.
|
||
let lease_is_current = presence.state == "warming"
|
||
|| presence.write_lease_sha256.as_deref() == current_write_lease_sha256;
|
||
ensure!(
|
||
lease_is_current,
|
||
"route proof runtime does not hold the exact current process write lease"
|
||
);
|
||
Ok(present)
|
||
}
|
||
|
||
fn authenticate_first_odin_warming_presence(
|
||
&self,
|
||
transaction: &DeploymentTransaction,
|
||
message_id: &str,
|
||
challenged_at_unix_millis: u64,
|
||
received_at_unix_millis: u64,
|
||
canonical_presence: &[u8],
|
||
) -> Result<cultnet_rs::VerifiedRuntimePresence> {
|
||
// Whether direct observation is warranted at all is decided in
|
||
// advance_warming, which asks the question that matters: can any Odin
|
||
// report on this one. Re-deciding it here with the narrower "has an
|
||
// Odin ever been admitted" test only contradicted that caller. What is
|
||
// still checked is what this function itself depends on -- the target
|
||
// is Odin, and the incarnation is stateful.
|
||
let expected = required(&transaction.expected, "Odin Expected")?;
|
||
ensure!(
|
||
ReadinessClass::of(expected)? == ReadinessClass::OdinSelf
|
||
&& expected.write_lease_required,
|
||
"direct Warming presence is reserved for a stateful Odin incarnation"
|
||
);
|
||
ensure!(
|
||
received_at_unix_millis >= challenged_at_unix_millis,
|
||
"first Odin Warming observation predates its challenge"
|
||
);
|
||
let authority = self.runtime_authority(transaction)?;
|
||
let authenticated = authenticate_runtime_presence_claim(
|
||
canonical_presence,
|
||
&authority,
|
||
RuntimePresenceAuthenticationContext {
|
||
trusted_received_at_unix_millis: received_at_unix_millis,
|
||
maximum_age_millis: self.options.topology_maximum_age_millis,
|
||
maximum_future_skew_millis: self.options.topology_maximum_future_skew_millis,
|
||
},
|
||
)?;
|
||
let correlation = correlate_runtime_presence_claim(authenticated, &authority)?;
|
||
ensure!(
|
||
correlation.disagreements().is_empty(),
|
||
"first Odin Warming presence disagrees with its Expected activation"
|
||
);
|
||
let present = correlation.into_undisputed_present()?;
|
||
let presence = present.record();
|
||
ensure!(
|
||
presence.observed_at_unix_millis >= challenged_at_unix_millis
|
||
&& presence.state == "warming"
|
||
&& presence.write_lease_sha256.is_none()
|
||
&& presence.detail == format!("idunn-warming:{message_id}"),
|
||
"first Odin candidate did not return exact fresh pre-lease Warming evidence"
|
||
);
|
||
Ok(present)
|
||
}
|
||
|
||
fn observe_first_odin_warming(
|
||
&self,
|
||
transaction: &DeploymentTransaction,
|
||
) -> Result<(RuntimePresenceEvidence, cultnet_rs::VerifiedRuntimePresence)> {
|
||
let expected = required(&transaction.expected, "first Odin Expected")?;
|
||
let binding = required(&transaction.plan, "first Odin plan")?
|
||
.parsed_inputs()?
|
||
.1;
|
||
let driver = self.route_driver(
|
||
binding
|
||
.route
|
||
.context("first Odin bootstrap has no candidate route binding")?,
|
||
);
|
||
let message_id = format!("warming-{}", Uuid::new_v4().simple());
|
||
let challenged_at_unix_millis = now_millis()?;
|
||
let response = driver.request_candidate_runtime_presence(expected, &message_id)?;
|
||
ensure!(
|
||
response.message_id == message_id,
|
||
"first Odin candidate transport substituted its challenge identity"
|
||
);
|
||
let admitted_at_unix_millis = now_millis()?;
|
||
let present = self.authenticate_first_odin_warming_presence(
|
||
transaction,
|
||
&message_id,
|
||
challenged_at_unix_millis,
|
||
admitted_at_unix_millis,
|
||
&response.canonical_presence,
|
||
)?;
|
||
let evidence = RuntimePresenceEvidence::from_present(
|
||
&present,
|
||
message_id,
|
||
challenged_at_unix_millis,
|
||
admitted_at_unix_millis,
|
||
)?;
|
||
Ok((evidence, present))
|
||
}
|
||
|
||
fn prove_stable_route(
|
||
&self,
|
||
transaction: &DeploymentTransaction,
|
||
driver: &NginxRouteDriver,
|
||
membership_sha256: String,
|
||
) -> Result<RouteObservation> {
|
||
let expected = required(&transaction.expected, "route Expected projection")?;
|
||
let activation = required(&transaction.activation, "route activation")?;
|
||
let authority = self.runtime_authority(transaction)?;
|
||
let current_write_lease_sha256 = transaction
|
||
.leasing
|
||
.as_ref()
|
||
.and_then(LeasingEvidence::lease_sha256);
|
||
self.prove_stable_route_against(
|
||
expected,
|
||
activation,
|
||
&authority,
|
||
current_write_lease_sha256,
|
||
driver,
|
||
membership_sha256,
|
||
)
|
||
}
|
||
|
||
fn prove_stable_route_against(
|
||
&self,
|
||
expected: &IdunnExpectedIncarnationRecord,
|
||
activation: &IdunnRuntimeActivationRecord,
|
||
authority: &cultnet_rs::VerifiedRuntimeAuthority,
|
||
current_write_lease_sha256: Option<&str>,
|
||
driver: &NginxRouteDriver,
|
||
membership_sha256: String,
|
||
) -> Result<RouteObservation> {
|
||
let message_id = format!("route-{}", Uuid::new_v4().simple());
|
||
let challenged_at_unix_millis = now_millis()?;
|
||
let response = driver.request_runtime_presence(expected, &message_id)?;
|
||
ensure!(
|
||
response.message_id == message_id,
|
||
"route transport substituted its challenge identity"
|
||
);
|
||
let received_at_unix_millis = now_millis()?;
|
||
let (signed_presence_sha256, observed_at_unix_millis) = self.authenticate_routed_presence(
|
||
authority,
|
||
current_write_lease_sha256,
|
||
&message_id,
|
||
challenged_at_unix_millis,
|
||
received_at_unix_millis,
|
||
&response.canonical_presence,
|
||
)?;
|
||
let observation = RouteObservation {
|
||
route_id: driver.binding.route_id.clone(),
|
||
runtime_instance_id: activation.runtime_instance_id.clone(),
|
||
membership_sha256,
|
||
signed_presence_sha256,
|
||
observed_at_unix_millis,
|
||
};
|
||
observation.validate()?;
|
||
Ok(observation)
|
||
}
|
||
|
||
fn runtime_authority_parts(
|
||
&self,
|
||
plan: &CompiledDeploymentPlan,
|
||
expected: &IdunnExpectedIncarnationRecord,
|
||
activation: &IdunnRuntimeActivationRecord,
|
||
) -> Result<cultnet_rs::VerifiedRuntimeAuthority> {
|
||
let provider_anchor = self.provider_anchor_for_plan(plan)?;
|
||
verify_runtime_authority(
|
||
expected,
|
||
activation,
|
||
&self.idunn_anchor,
|
||
&provider_anchor.public_key,
|
||
)
|
||
}
|
||
|
||
fn provider_anchor_for_plan(
|
||
&self,
|
||
plan: &CompiledDeploymentPlan,
|
||
) -> Result<ServiceIdentityTrustAnchor> {
|
||
let binding = plan.parsed_inputs()?.1;
|
||
let provider_anchor = read_trust_anchor::<GameCultProviderHealthIdentity>(
|
||
&binding.runtime_identity.trust_anchor_store,
|
||
)?;
|
||
ensure!(
|
||
provider_anchor.schema_version
|
||
== <GameCultProviderHealthIdentity as ServiceIdentityProfile>::TRUST_ANCHOR_SCHEMA,
|
||
"provider runtime presence trust anchor schema is unsupported"
|
||
);
|
||
Ok(provider_anchor)
|
||
}
|
||
|
||
fn authenticate_topology_bytes(
|
||
&self,
|
||
snapshot: &ControlSnapshot,
|
||
transaction: &DeploymentTransaction,
|
||
canonical_bytes: &[u8],
|
||
current_write_lease_sha256: Option<&str>,
|
||
trusted_received_at: u64,
|
||
) -> Result<AuthenticatedOdinRuntimeTopologyCorrelation> {
|
||
let authority = self.runtime_authority(transaction)?;
|
||
let odin_authority = self.current_odin_authority(snapshot)?;
|
||
authenticate_odin_runtime_topology_correlation(
|
||
canonical_bytes,
|
||
&authority,
|
||
current_write_lease_sha256,
|
||
&odin_authority.signer_public_key,
|
||
self.trusted_topology_context(trusted_received_at),
|
||
)
|
||
}
|
||
|
||
/// Persist every newly authenticated publisher sequence before deciding
|
||
/// whether it means Warming, Ready, degraded, or disagreement.
|
||
fn admit_latest_topology(
|
||
&self,
|
||
current: &Stored<DeploymentTransaction>,
|
||
current_write_lease_sha256: Option<&str>,
|
||
) -> Result<
|
||
Option<(
|
||
Stored<DeploymentTransaction>,
|
||
AuthenticatedOdinRuntimeTopologyCorrelation,
|
||
)>,
|
||
> {
|
||
// Odin keeps one correlation per incarnation, so the transaction's own
|
||
// Expected selects exactly the evidence about it. A correlation about
|
||
// the incarnation this one is replacing lives under another key and is
|
||
// never handed back here.
|
||
let expected_sha256 =
|
||
required(¤t.value.expected, "Expected projection")?.canonical_sha256()?;
|
||
let Some(received) = self
|
||
.topology()
|
||
.receive(¤t.value.target, &expected_sha256)?
|
||
else {
|
||
return Ok(None);
|
||
};
|
||
ensure!(
|
||
received.target == current.value.target,
|
||
"topology transport substituted target"
|
||
);
|
||
let now = now_millis()?;
|
||
let snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
let live = snapshot
|
||
.transactions
|
||
.iter()
|
||
.find(|stored| stored.value.transaction_id == current.value.transaction_id)
|
||
.context("transaction disappeared before topology admission")?;
|
||
ensure!(
|
||
live.envelope == current.envelope,
|
||
"transaction changed before topology admission"
|
||
);
|
||
let authenticated = match self.authenticate_topology_bytes(
|
||
&snapshot,
|
||
&live.value,
|
||
&received.canonical_bytes,
|
||
current_write_lease_sha256,
|
||
now,
|
||
) {
|
||
Ok(authenticated) => authenticated,
|
||
// Odin re-stamps a correlation only when its facts change, and a
|
||
// live presence changes them every heartbeat. A correlation that
|
||
// has aged out therefore says one thing: nothing about this
|
||
// incarnation has arrived lately. That is absence to wait on, not
|
||
// evidence to refuse; it failed every Heimdall deployment and the
|
||
// first observed Odin deployment as a fault.
|
||
Err(error) if is_stale_observation(&error) => return Ok(None),
|
||
Err(error) => return Err(error),
|
||
};
|
||
let evidence = TopologyEvidence::from_authenticated(&authenticated, now)?;
|
||
if !sequence_requires_admission(
|
||
live.value.latest_odin_observation.as_ref(),
|
||
snapshot.max_odin_sequence(&live.value.target, &evidence.signer_identity_id),
|
||
&evidence,
|
||
)? {
|
||
return Ok(Some((live.clone(), authenticated)));
|
||
}
|
||
let mut next = live.value.clone();
|
||
next.latest_odin_observation = Some(evidence.clone());
|
||
next.odin_publisher_sequence_cursor = evidence.publisher_sequence;
|
||
next.updated_at_unix_millis = now;
|
||
next.last_error = None;
|
||
replace_transaction(&self.options.state_store, live, &next)?;
|
||
let admitted = ControlSnapshot::read(&self.options.state_store)?
|
||
.transactions
|
||
.into_iter()
|
||
.find(|stored| stored.value.transaction_id == next.transaction_id)
|
||
.context("sequence-admitted transaction disappeared")?;
|
||
ensure!(
|
||
admitted.value.latest_odin_observation.as_ref() == Some(&evidence)
|
||
&& admitted.value.odin_publisher_sequence_cursor == evidence.publisher_sequence,
|
||
"topology sequence was not durably admitted"
|
||
);
|
||
Ok(Some((admitted, authenticated)))
|
||
}
|
||
|
||
fn rehydrate_warming_token(
|
||
&self,
|
||
transaction: &DeploymentTransaction,
|
||
now: u64,
|
||
require_current: bool,
|
||
) -> Result<SequenceAdmittedWarming> {
|
||
let evidence = required(&transaction.warming, "durable warming evidence")?.clone();
|
||
match evidence {
|
||
WarmingEvidence::OdinTopology { evidence } => {
|
||
let snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
let authenticated = self.authenticate_topology_bytes(
|
||
&snapshot,
|
||
transaction,
|
||
&evidence.canonical_bytes,
|
||
None,
|
||
if require_current {
|
||
now
|
||
} else {
|
||
evidence.admitted_at_unix_millis
|
||
},
|
||
)?;
|
||
validate_authenticated_evidence(&evidence, &authenticated)?;
|
||
let incumbent_lease_sha256 =
|
||
self.incumbent_lease_sha256_for_warming(&snapshot, transaction)?;
|
||
ensure!(
|
||
is_semantic_warming(
|
||
required(&transaction.expected, "Warming Expected projection")?,
|
||
required(&transaction.activation, "Warming activation")?,
|
||
incumbent_lease_sha256.as_deref(),
|
||
&authenticated,
|
||
)?,
|
||
"durable warming evidence no longer satisfies the Warming gate"
|
||
);
|
||
SequenceAdmittedWarming::from_topology(
|
||
transaction.transaction_id.clone(),
|
||
authenticated,
|
||
)
|
||
}
|
||
WarmingEvidence::FirstOdinDirect { evidence } => {
|
||
let present = self.authenticate_first_odin_warming_presence(
|
||
transaction,
|
||
&evidence.message_id,
|
||
evidence.challenged_at_unix_millis,
|
||
if require_current {
|
||
now
|
||
} else {
|
||
evidence.admitted_at_unix_millis
|
||
},
|
||
&evidence.canonical_bytes,
|
||
)?;
|
||
SequenceAdmittedWarming::from_direct_presence(
|
||
transaction.transaction_id.clone(),
|
||
WarmingEvidence::FirstOdinDirect { evidence },
|
||
present,
|
||
)
|
||
}
|
||
WarmingEvidence::RouteProofDirect { evidence } => {
|
||
let present = self.reauthenticate_route_proof(
|
||
transaction,
|
||
&evidence,
|
||
route_proof_warming_states(required(
|
||
&transaction.expected,
|
||
"Warming Expected projection",
|
||
)?),
|
||
None,
|
||
if require_current {
|
||
now
|
||
} else {
|
||
evidence.admitted_at_unix_millis
|
||
},
|
||
)?;
|
||
SequenceAdmittedWarming::from_direct_presence(
|
||
transaction.transaction_id.clone(),
|
||
WarmingEvidence::RouteProofDirect { evidence },
|
||
present,
|
||
)
|
||
}
|
||
}
|
||
}
|
||
|
||
/// A route-proof presence Idunn recorded, authenticated again as of `at`.
|
||
fn reauthenticate_route_proof(
|
||
&self,
|
||
transaction: &DeploymentTransaction,
|
||
evidence: &RuntimePresenceEvidence,
|
||
states: &[&str],
|
||
current_write_lease_sha256: Option<&str>,
|
||
at_unix_millis: u64,
|
||
) -> Result<cultnet_rs::VerifiedRuntimePresence> {
|
||
let authority = self.runtime_authority(transaction)?;
|
||
self.authenticate_challenged_presence(
|
||
&authority,
|
||
states,
|
||
current_write_lease_sha256,
|
||
&evidence.message_id,
|
||
evidence.challenged_at_unix_millis,
|
||
at_unix_millis,
|
||
&evidence.canonical_bytes,
|
||
)
|
||
}
|
||
|
||
/// One direct challenge to a route-proof target's candidate endpoint.
|
||
/// Failing to reach or hear the candidate is only silence: a process that
|
||
/// is still starting does not answer, and that is waiting, not a fault.
|
||
/// What does answer is authenticated, and a bad answer is an error, as is
|
||
/// a challenge Idunn could not make: those stay a `ChallengeFailure` in the
|
||
/// error's chain.
|
||
fn challenge_candidate(
|
||
&self,
|
||
transaction: &DeploymentTransaction,
|
||
states: &[&str],
|
||
current_write_lease_sha256: Option<&str>,
|
||
) -> Result<CandidateAnswer> {
|
||
let expected = required(&transaction.expected, "candidate Expected")?;
|
||
let binding = required(&transaction.plan, "candidate plan")?
|
||
.parsed_inputs()?
|
||
.1;
|
||
let driver = self.route_driver(
|
||
binding
|
||
.route
|
||
.context("route-proof candidate has no route binding")?,
|
||
);
|
||
let message_id = format!("candidate-{}", Uuid::new_v4().simple());
|
||
let challenged_at_unix_millis = now_millis()?;
|
||
let response = match driver.request_candidate_runtime_presence(expected, &message_id) {
|
||
Ok(response) => response,
|
||
Err(ChallengeFailure::Silent(error)) => {
|
||
return Ok(CandidateAnswer::Silent(format!(
|
||
"candidate endpoint did not answer its challenge: {error:#}"
|
||
)));
|
||
}
|
||
Err(bad @ ChallengeFailure::Refused(_)) => return Err(bad.into()),
|
||
};
|
||
let admitted_at_unix_millis = now_millis()?;
|
||
let authority = self.runtime_authority(transaction)?;
|
||
let present = self.authenticate_challenged_presence(
|
||
&authority,
|
||
states,
|
||
current_write_lease_sha256,
|
||
&message_id,
|
||
challenged_at_unix_millis,
|
||
admitted_at_unix_millis,
|
||
&response.canonical_presence,
|
||
)?;
|
||
let evidence = RuntimePresenceEvidence::from_present(
|
||
&present,
|
||
message_id,
|
||
challenged_at_unix_millis,
|
||
admitted_at_unix_millis,
|
||
)?;
|
||
Ok(CandidateAnswer::Answered { evidence, present })
|
||
}
|
||
|
||
fn rehydrate_ready_token(
|
||
&self,
|
||
transaction: &DeploymentTransaction,
|
||
now: u64,
|
||
require_current: bool,
|
||
) -> Result<SequenceAdmittedReady> {
|
||
let evidence = required(&transaction.ready, "durable Ready evidence")?
|
||
.odin()
|
||
.context("durable Ready evidence is not an Odin receipt")?
|
||
.clone();
|
||
let snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
let current_lease = transaction
|
||
.leasing
|
||
.as_ref()
|
||
.and_then(LeasingEvidence::lease_sha256);
|
||
let authenticated = self.authenticate_topology_bytes(
|
||
&snapshot,
|
||
transaction,
|
||
&evidence.canonical_bytes,
|
||
current_lease,
|
||
if require_current {
|
||
now
|
||
} else {
|
||
evidence.admitted_at_unix_millis
|
||
},
|
||
)?;
|
||
validate_authenticated_evidence(&evidence, &authenticated)?;
|
||
ensure!(
|
||
is_semantic_ready(&authenticated),
|
||
"durable Ready evidence no longer authenticates as Ready"
|
||
);
|
||
Ok(SequenceAdmittedReady {
|
||
transaction_id: transaction.transaction_id.clone(),
|
||
evidence,
|
||
expected: required(&transaction.expected, "Expected projection")?.clone(),
|
||
authenticated,
|
||
})
|
||
}
|
||
|
||
fn current_ready_provider_tokens(
|
||
&self,
|
||
snapshot: &ControlSnapshot,
|
||
) -> Result<Vec<SequenceAdmittedReady>> {
|
||
let mut providers = Vec::new();
|
||
for stored in &snapshot.admitted {
|
||
// No pre-filter on ready == latest: a provider that published again
|
||
// after going ready is still ready. rehydrate_admitted_ready owns
|
||
// that judgment now, and reports why when it refuses.
|
||
match self.rehydrate_admitted_ready(snapshot, &stored.value) {
|
||
Ok(provider) => providers.push(provider),
|
||
Err(error) => eprintln!(
|
||
"Idunn excluded non-current provider {}: {error:#}",
|
||
stored.value.target
|
||
),
|
||
}
|
||
}
|
||
Ok(providers)
|
||
}
|
||
|
||
fn rehydrate_admitted_ready(
|
||
&self,
|
||
snapshot: &ControlSnapshot,
|
||
generation: &AdmittedGeneration,
|
||
) -> Result<SequenceAdmittedReady> {
|
||
let odin = generation.odin_receipts()?;
|
||
let authority = self.runtime_authority_parts(
|
||
&generation.plan,
|
||
&generation.expected,
|
||
&generation.activation,
|
||
)?;
|
||
let odin_authority = self.current_odin_authority(snapshot)?;
|
||
let current_lease = generation.leasing.lease_sha256();
|
||
let authenticated = authenticate_odin_runtime_topology_correlation(
|
||
&odin.ready.canonical_bytes,
|
||
&authority,
|
||
current_lease,
|
||
&odin_authority.signer_public_key,
|
||
self.trusted_topology_context(odin.ready.admitted_at_unix_millis),
|
||
)?;
|
||
validate_authenticated_evidence(odin.ready, &authenticated)?;
|
||
ensure!(
|
||
is_semantic_ready(&authenticated),
|
||
"admitted provider's Ready receipt is not Ready at its admission time"
|
||
);
|
||
// A provider that published after going ready is the ordinary case, not
|
||
// a stale one, so the cursor is authenticated on its own terms rather
|
||
// than compared to the receipt. Authentication is what binds it to this
|
||
// incarnation -- it requires the record's current_activation_sha256 and
|
||
// runtime_instance_id to match this generation's activation -- so a
|
||
// separate identity check here would only restate it more weakly.
|
||
// Note this is the cursor as frozen at admission, not a live read of
|
||
// Odin. Both receipts are historical proofs; current workload health
|
||
// is owned by continuity and route observation, not receipt age.
|
||
let authenticated_latest = authenticate_odin_runtime_topology_correlation(
|
||
&odin.latest.canonical_bytes,
|
||
&authority,
|
||
current_lease,
|
||
&odin_authority.signer_public_key,
|
||
self.trusted_topology_context(odin.latest.admitted_at_unix_millis),
|
||
)?;
|
||
validate_authenticated_evidence(odin.latest, &authenticated_latest)?;
|
||
ensure!(
|
||
is_semantic_ready(&authenticated_latest),
|
||
"admitted provider's latest receipt is not Ready at its admission time"
|
||
);
|
||
Ok(SequenceAdmittedReady {
|
||
transaction_id: generation.transaction_id.clone(),
|
||
evidence: odin.ready.clone(),
|
||
expected: generation.expected.clone(),
|
||
authenticated,
|
||
})
|
||
}
|
||
|
||
fn validate_selected_providers_current(
|
||
&self,
|
||
plan: &CompiledDeploymentPlan,
|
||
) -> Result<()> {
|
||
let snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
for selection in &plan.dependencies {
|
||
let Some(provider) = &selection.provider else {
|
||
ensure!(
|
||
selection.requirement.kind == DependencyKind::Optional,
|
||
"required dependency has no provider"
|
||
);
|
||
continue;
|
||
};
|
||
let DependencyProviderAuthority::ManagedReady {
|
||
target,
|
||
incarnation_id,
|
||
plan_id,
|
||
sealed_release_id,
|
||
expected_projection_sha256,
|
||
odin_topology_correlation_sha256,
|
||
odin_topology_publisher_sequence,
|
||
} = &provider.authority
|
||
else {
|
||
continue;
|
||
};
|
||
let admitted = snapshot
|
||
.admitted_for(target)
|
||
.context("selected managed dependency is no longer admitted")?;
|
||
let token = self.rehydrate_admitted_ready(&snapshot, &admitted.value)?;
|
||
ensure!(
|
||
admitted.value.expected.incarnation_id == *incarnation_id
|
||
&& admitted.value.plan.plan_id == *plan_id
|
||
&& admitted.value.sealed_release.sealed_release_id == *sealed_release_id
|
||
&& admitted.value.expected.canonical_sha256()? == *expected_projection_sha256
|
||
&& token.publisher_sequence() >= *odin_topology_publisher_sequence,
|
||
"selected dependency authority changed before actuation"
|
||
);
|
||
ensure!(
|
||
token.publisher_sequence() != *odin_topology_publisher_sequence
|
||
|| token.evidence_sha256() == odin_topology_correlation_sha256,
|
||
"selected dependency Odin sequence changed evidence"
|
||
);
|
||
ensure!(
|
||
token
|
||
.authenticated()
|
||
.record()
|
||
.observed_capabilities
|
||
.iter()
|
||
.any(|capability| {
|
||
capability_compatible(
|
||
&selection.requirement.capability,
|
||
&selection.requirement.schema,
|
||
&selection.requirement.compatibility,
|
||
&capability.capability,
|
||
&capability.schema,
|
||
&capability.compatibility,
|
||
) && capability.capacity >= selection.requirement.minimum_capacity
|
||
}),
|
||
"selected dependency no longer provides its required capability"
|
||
);
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
fn persist_same_phase<F>(
|
||
&self,
|
||
current: &Stored<DeploymentTransaction>,
|
||
mutation: F,
|
||
) -> Result<()>
|
||
where
|
||
F: FnOnce(&mut DeploymentTransaction) -> Result<()>,
|
||
{
|
||
let mut next = current.value.clone();
|
||
mutation(&mut next)?;
|
||
ensure!(
|
||
next.phase == current.value.phase,
|
||
"same-phase evidence update changed phase"
|
||
);
|
||
next.updated_at_unix_millis = now_millis()?;
|
||
next.last_error = None;
|
||
replace_transaction(&self.options.state_store, current, &next)
|
||
}
|
||
|
||
fn transition(
|
||
&self,
|
||
current: &Stored<DeploymentTransaction>,
|
||
next_phase: DeploymentPhase,
|
||
) -> Result<()> {
|
||
let mut next = current.value.clone();
|
||
next.enter_phase(next_phase, now_millis()?);
|
||
next.last_error = None;
|
||
self.enter(current, next)
|
||
}
|
||
|
||
/// Before its handover a Deploy is proven against the generation that was
|
||
/// admitted when the evidence was gathered. When another generation has
|
||
/// been admitted since (continuity restarted the incumbent), rewrite the
|
||
/// incumbent and drop exactly the evidence that names the old one: the
|
||
/// warming (its lease), the route preflight (its route) and the isolation
|
||
/// (its workload). A continuity is never rebased. Returns whether it wrote.
|
||
fn rebase_onto_admitted(&self, current: &Stored<DeploymentTransaction>) -> Result<bool> {
|
||
let held = ¤t.value;
|
||
if held.command_kind != CommandKind::Deploy
|
||
|| held.pre_fencing_abort.is_some()
|
||
|| held.phase >= held.takes_target_at()
|
||
{
|
||
return Ok(false);
|
||
}
|
||
let snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
let admitted = snapshot.admitted_for(&held.target);
|
||
let admitted_id = admitted.map(|stored| stored.value.generation_id.as_str());
|
||
if admitted_id == held.incumbent_generation_id.as_deref() {
|
||
return Ok(false);
|
||
}
|
||
self.persist_same_phase(current, |next| {
|
||
next.incumbent_generation_id = admitted_id.map(str::to_owned);
|
||
next.warming = None;
|
||
next.route_preflight = None;
|
||
next.isolation = None;
|
||
next.odin_publisher_sequence_cursor = next
|
||
.odin_publisher_sequence_cursor
|
||
.max(admitted.map_or(0, |stored| stored.value.odin_publisher_sequence_cursor));
|
||
Ok(())
|
||
})?;
|
||
Ok(true)
|
||
}
|
||
|
||
/// What a transaction that has not yet taken its target does first, when
|
||
/// the world moved under it. `true` means this decided the step.
|
||
///
|
||
/// A Deploy rebases onto the generation now admitted; in Warming, whose
|
||
/// steps read the incumbent an owner is fencing, it waits while another
|
||
/// transaction owns or drains the target. Both return Ok: a pre-fencing
|
||
/// error is an abort, and no deployment is aborted for continuity. A
|
||
/// Continuity restarts one generation and no other, so it ends itself
|
||
/// (through the ordinary pre-fencing abort) when its target is taken or
|
||
/// its generation replaced.
|
||
fn settle_before_handover(&self, current: &Stored<DeploymentTransaction>) -> Result<bool> {
|
||
let held = ¤t.value;
|
||
match held.command_kind {
|
||
CommandKind::Deploy => {
|
||
if self.rebase_onto_admitted(current)? {
|
||
return Ok(true);
|
||
}
|
||
if held.phase != DeploymentPhase::Warming {
|
||
return Ok(false);
|
||
}
|
||
let snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
let Some(owner) = snapshot.target_owner(&held.target, Some(&held.transaction_id))
|
||
else {
|
||
return Ok(false);
|
||
};
|
||
self.wait_for_owner(current, &owner.value.transaction_id)
|
||
.map(|()| true)
|
||
}
|
||
CommandKind::Continuity => {
|
||
let snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
if let Some(owner) = snapshot.target_owner(&held.target, Some(&held.transaction_id))
|
||
{
|
||
bail!(
|
||
"continuity superseded: target {} is owned by {}",
|
||
held.target,
|
||
owner.value.transaction_id
|
||
);
|
||
}
|
||
ensure!(
|
||
snapshot
|
||
.admitted_for(&held.target)
|
||
.map(|stored| stored.value.generation_id.as_str())
|
||
== held.incumbent_generation_id.as_deref(),
|
||
"continuity superseded: the generation it restarts is no longer admitted"
|
||
);
|
||
Ok(false)
|
||
}
|
||
}
|
||
}
|
||
|
||
/// The seam for the forward moves that can be the handover (`transition`
|
||
/// and the end of Sealing). Entering the phase where the transaction takes
|
||
/// its target (`takes_target_at`) is decided by `take_target`: that CAS is
|
||
/// the handover, and nothing else writes that entry. The other writes
|
||
/// (Starting to Warming, completions, aborts, same-phase evidence) go to
|
||
/// `replace_transaction` directly because none of them enters that phase.
|
||
fn enter(
|
||
&self,
|
||
current: &Stored<DeploymentTransaction>,
|
||
next: DeploymentTransaction,
|
||
) -> Result<()> {
|
||
ensure!(
|
||
next.phase as u8 == current.value.phase as u8 + 1,
|
||
"deployment phase transition is not adjacent"
|
||
);
|
||
if next.phase != next.takes_target_at() {
|
||
return replace_transaction(&self.options.state_store, current, &next);
|
||
}
|
||
let snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
let target = ¤t.value.target;
|
||
match (self.take_target(&snapshot, current, &next)?, current.value.command_kind) {
|
||
// Lost to a concurrent writer: the next step reads the new world.
|
||
(Handover::Took | Handover::Lost, _) => Ok(()),
|
||
// Wait, never abort: a pre-fencing error would end the deployment.
|
||
(Handover::OwnedBy(owner), CommandKind::Deploy) => {
|
||
self.wait_for_owner(current, &owner)
|
||
}
|
||
// The next step rebases onto the generation now admitted.
|
||
(Handover::Moved, CommandKind::Deploy) => Ok(()),
|
||
(Handover::OwnedBy(owner), CommandKind::Continuity) => {
|
||
bail!("continuity superseded: target {target} is owned by {owner}")
|
||
}
|
||
(Handover::Moved, CommandKind::Continuity) => {
|
||
bail!("continuity superseded: the generation it restarts is no longer admitted")
|
||
}
|
||
}
|
||
}
|
||
|
||
/// The handover: one CAS that writes `next` (the entry into
|
||
/// `takes_target_at`) only if nothing else moved. It expects the
|
||
/// transaction's own envelope, the target's admitted envelope as read
|
||
/// (or its absence), and every other non-terminal transaction of the
|
||
/// target as read, so of two transactions entering from one view exactly
|
||
/// one succeeds.
|
||
fn take_target(
|
||
&self,
|
||
snapshot: &ControlSnapshot,
|
||
current: &Stored<DeploymentTransaction>,
|
||
next: &DeploymentTransaction,
|
||
) -> Result<Handover> {
|
||
let target = current.value.target.as_str();
|
||
let id = current.value.transaction_id.as_str();
|
||
if let Some(owner) = snapshot.target_owner(target, Some(id)) {
|
||
return Ok(Handover::OwnedBy(owner.value.transaction_id.clone()));
|
||
}
|
||
let admitted = snapshot.admitted_for(target);
|
||
if admitted.map(|stored| stored.value.generation_id.as_str())
|
||
!= current.value.incumbent_generation_id.as_deref()
|
||
{
|
||
return Ok(Handover::Moved);
|
||
}
|
||
next.validate()?;
|
||
let mut expected = vec![
|
||
CultCacheExpectedEnvelope {
|
||
r#type: DeploymentTransaction::TYPE.into(),
|
||
key: id.into(),
|
||
current: Some(current.envelope.clone()),
|
||
},
|
||
CultCacheExpectedEnvelope {
|
||
r#type: AdmittedGeneration::TYPE.into(),
|
||
key: target.into(),
|
||
current: admitted.map(|stored| stored.envelope.clone()),
|
||
},
|
||
];
|
||
expected.extend(
|
||
snapshot
|
||
.on_target(target)
|
||
.filter(|stored| stored.value.transaction_id != id && !stored.value.is_terminal())
|
||
.map(|stored| CultCacheExpectedEnvelope {
|
||
r#type: DeploymentTransaction::TYPE.into(),
|
||
key: stored.value.transaction_id.clone(),
|
||
current: Some(stored.envelope.clone()),
|
||
}),
|
||
);
|
||
let took = SingleFileMessagePackBackingStore::new(&self.options.state_store)
|
||
.compare_exchange(
|
||
&expected,
|
||
&[transaction_envelope(next, next.updated_at_unix_millis)?],
|
||
)?;
|
||
Ok(if took { Handover::Took } else { Handover::Lost })
|
||
}
|
||
|
||
/// Every live record is held to a deadline. Before a transaction runs its
|
||
/// step, a deadline phase whose deadline has passed, or whose command an
|
||
/// operator asked to expire, is resolved as `phase_end` says: aborted before
|
||
/// the fence, aborted after it, or left to the operator with its step
|
||
/// retrying (a stateful deploy, or a continuity, whose candidate is the only
|
||
/// running copy of its body). An abort or a drain past its deadline is left
|
||
/// to the operator the same way. A continuity left to the operator yields,
|
||
/// through its own post-fencing abort, to a deployment awaiting the
|
||
/// handover; it never writes the deployment.
|
||
/// Returns `Some(moved)` when it decided the tick, `None` to run the step.
|
||
///
|
||
/// Only `retime` sets a deadline (at phase entry, an abort's beginning, the
|
||
/// commit and this lift), so nothing in a step or a resume path extends
|
||
/// one. A record with none whose activity needs one is timed once, from
|
||
/// the first time this sees it: that is also how a waiter is re-armed once
|
||
/// the target is free.
|
||
fn resolve_phase_deadline(
|
||
&self,
|
||
current: &Stored<DeploymentTransaction>,
|
||
) -> Result<Option<bool>> {
|
||
let transaction = ¤t.value;
|
||
if transaction.is_terminal() {
|
||
return Ok(None);
|
||
}
|
||
// An owned waiter is timed by its owner.
|
||
if transaction.phase < DeploymentPhase::Fencing
|
||
&& transaction.pre_fencing_abort.is_none()
|
||
&& ControlSnapshot::read(&self.options.state_store)?
|
||
.target_owner(&transaction.target, Some(&transaction.transaction_id))
|
||
.is_some()
|
||
{
|
||
return Ok(None);
|
||
}
|
||
let now = now_millis()?;
|
||
let Some(deadline) = transaction.phase_deadline else {
|
||
let mut next = transaction.clone();
|
||
next.retime(now);
|
||
if next.phase_deadline.is_none() {
|
||
return Ok(None);
|
||
}
|
||
replace_transaction(&self.options.state_store, current, &next)?;
|
||
return Ok(Some(true));
|
||
};
|
||
if !transaction.resolves_by_deadline() {
|
||
if transaction.leaves_to_operator(now) {
|
||
let what = if transaction.drains_incumbent() {
|
||
"draining its incumbent"
|
||
} else {
|
||
"its abort"
|
||
};
|
||
self.leave_to_operator(
|
||
transaction,
|
||
&format!(
|
||
"{what} in the {:?} phase passed its deadline at unix ms {}",
|
||
transaction.phase, deadline.deadline_at_unix_millis
|
||
),
|
||
);
|
||
}
|
||
return Ok(None);
|
||
}
|
||
let requested = transaction.expirable()
|
||
&& ControlSnapshot::read(&self.options.state_store)?
|
||
.expiry_requests
|
||
.iter()
|
||
.any(|request| request.value.command_id == transaction.command_id);
|
||
let cause = if now >= deadline.deadline_at_unix_millis {
|
||
format!(
|
||
"the {:?} phase passed its deadline at unix ms {}",
|
||
transaction.phase, deadline.deadline_at_unix_millis
|
||
)
|
||
} else if requested {
|
||
format!("the {:?} phase was expired by the operator", transaction.phase)
|
||
} else {
|
||
return Ok(None);
|
||
};
|
||
match transaction.phase_end() {
|
||
PhaseEnd::AbortBeforeFencing => {
|
||
self.begin_pre_fencing_abort(current, anyhow!("{cause}"))?;
|
||
Ok(Some(true))
|
||
}
|
||
PhaseEnd::Abort { lease_not_adopted } => {
|
||
let error = if lease_not_adopted {
|
||
anyhow!("lease-not-adopted: {cause}, and the granted write lease has no adoption evidence")
|
||
} else {
|
||
anyhow!("{cause}")
|
||
};
|
||
self.begin_post_fencing_abort(current, error)?;
|
||
Ok(Some(true))
|
||
}
|
||
PhaseEnd::OperatorRequired => {
|
||
self.leave_to_operator(
|
||
transaction,
|
||
&format!("{cause}, and its stateful candidate was issued a write lease"),
|
||
);
|
||
Ok(None)
|
||
}
|
||
PhaseEnd::RetryForward => {
|
||
let snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
let now = now_millis()?;
|
||
if let Some(deploy) = transaction
|
||
.yields_to_a_deploy()
|
||
.then(|| self.deploy_the_continuity_yields_to(&snapshot, transaction, now))
|
||
.flatten()
|
||
{
|
||
self.begin_post_fencing_abort(
|
||
current,
|
||
anyhow!(
|
||
"continuity-yields: {cause}; deployment {} awaits the handover of {}",
|
||
deploy.value.transaction_id,
|
||
transaction.target
|
||
),
|
||
)?;
|
||
return Ok(Some(true));
|
||
}
|
||
self.leave_to_operator(
|
||
transaction,
|
||
&format!(
|
||
"{cause}; its candidate is the only running copy of the admitted body, so Idunn retries the phase forward and never stops it; to restart it, {}",
|
||
transaction.manual_exit()
|
||
),
|
||
);
|
||
Ok(None)
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Close the operator-required incident of every transaction that is no
|
||
/// longer left to the operator: it ended, or its step went on. A
|
||
/// transaction that is still left keeps its incident open.
|
||
fn close_spent_operator_incidents(&self, snapshot: &ControlSnapshot, now: u64) {
|
||
let store = self.incidents();
|
||
let close = || -> Result<()> {
|
||
for subject in store.open_subjects(IncidentCondition::OperatorRequired)? {
|
||
if !snapshot.transactions.iter().any(|stored| {
|
||
stored.value.transaction_id == subject && stored.value.leaves_to_operator(now)
|
||
}) {
|
||
store.close(IncidentCondition::OperatorRequired, &subject, CloseReason::Recovered, now)?;
|
||
}
|
||
}
|
||
Ok(())
|
||
};
|
||
self.record_incident("close", "*", close());
|
||
}
|
||
|
||
/// Delete expiry requests whose command has no phase left to end: it was
|
||
/// resolved, it finished, or its command is gone.
|
||
fn retire_spent_expiry_requests(&self, snapshot: &ControlSnapshot) -> Result<bool> {
|
||
let spent = snapshot
|
||
.expiry_requests
|
||
.iter()
|
||
.filter(|request| {
|
||
!snapshot.transactions.iter().any(|stored| {
|
||
stored.value.command_id == request.value.command_id
|
||
&& stored.value.expirable()
|
||
})
|
||
})
|
||
.map(|request| request.envelope.clone())
|
||
.collect::<Vec<_>>();
|
||
if spent.is_empty() {
|
||
return Ok(false);
|
||
}
|
||
SingleFileMessagePackBackingStore::new(&self.options.state_store)
|
||
.delete_batch_if_unchanged(&spent)
|
||
}
|
||
|
||
fn record_resumable_error(
|
||
&self,
|
||
current: &Stored<DeploymentTransaction>,
|
||
error: &anyhow::Error,
|
||
) -> Result<()> {
|
||
let detail = bounded_detail(&format!("{error:#}"));
|
||
if current.value.last_error.as_deref() == Some(detail.as_str()) {
|
||
return Ok(());
|
||
}
|
||
let mut next = current.value.clone();
|
||
next.last_error = Some(detail);
|
||
next.updated_at_unix_millis = now_millis()?;
|
||
replace_transaction(&self.options.state_store, current, &next)
|
||
}
|
||
|
||
fn begin_pre_fencing_abort(
|
||
&self,
|
||
current: &Stored<DeploymentTransaction>,
|
||
error: anyhow::Error,
|
||
) -> Result<()> {
|
||
ensure!(
|
||
current.value.phase < DeploymentPhase::Fencing,
|
||
"post-fence transaction cannot terminal-fail"
|
||
);
|
||
ensure!(
|
||
current.value.pre_fencing_abort.is_none(),
|
||
"pre-fencing abort intent is already durable"
|
||
);
|
||
let abort = pre_fencing_abort_intent(¤t.value, &format!("{error:#}"));
|
||
let now = now_millis()?;
|
||
self.persist_same_phase(current, |next| {
|
||
next.pre_fencing_abort = Some(abort);
|
||
next.retime(now);
|
||
Ok(())
|
||
})
|
||
}
|
||
|
||
/// Whether this transaction's candidate can no longer run at all.
|
||
///
|
||
/// A transaction that has never started one has no candidate to be dead, so
|
||
/// it is not permanently stopped -- it is simply not there yet.
|
||
fn candidate_is_permanently_stopped(
|
||
&self,
|
||
transaction: &DeploymentTransaction,
|
||
) -> Result<bool> {
|
||
let Some(workload) = &transaction.workload else {
|
||
return Ok(false);
|
||
};
|
||
self.workload_for(required(&transaction.plan, "transaction plan")?)?
|
||
.is_permanently_stopped(workload)
|
||
}
|
||
|
||
fn begin_post_fencing_abort(
|
||
&self,
|
||
current: &Stored<DeploymentTransaction>,
|
||
error: anyhow::Error,
|
||
) -> Result<()> {
|
||
ensure!(
|
||
current.value.phase >= DeploymentPhase::Fencing,
|
||
"pre-fence transaction must abort through the pre-fencing path"
|
||
);
|
||
ensure!(
|
||
current.value.post_fencing_abort.is_none(),
|
||
"post-fencing abort intent is already durable"
|
||
);
|
||
let now = now_millis()?;
|
||
let abort = post_fencing_abort_intent(¤t.value, &bounded_detail(&format!("{error:#}")));
|
||
self.persist_same_phase(current, |next| {
|
||
next.post_fencing_abort = Some(abort);
|
||
next.retime(now);
|
||
Ok(())
|
||
})
|
||
}
|
||
|
||
/// Withdraw the candidate in the one order that never leaves two writers:
|
||
/// take away its route, then stop the process and withdraw its write lease
|
||
/// (the lease is lifetime-held, so the holder stops first), then clean up,
|
||
/// and only then reconcile the projection.
|
||
fn advance_post_fencing_abort(&self, current: &Stored<DeploymentTransaction>) -> Result<()> {
|
||
let abort = required(
|
||
¤t.value.post_fencing_abort,
|
||
"post-fencing abort intent",
|
||
)?;
|
||
if abort.route_restoration == CleanupEvidence::Pending {
|
||
let preflight = required(¤t.value.route_preflight, "route preflight receipt")?;
|
||
let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1;
|
||
let driver =
|
||
self.route_driver(binding.route.context("routed plan has no route binding")?);
|
||
driver
|
||
.withdraw_candidate_membership(
|
||
preflight,
|
||
&self.route_gate(¤t.value.target, current.value.command_kind),
|
||
)
|
||
.context("restoring the route the candidate found")?;
|
||
return self.persist_same_phase(current, |next| {
|
||
next.post_fencing_abort.as_mut().unwrap().route_restoration =
|
||
CleanupEvidence::Complete;
|
||
Ok(())
|
||
});
|
||
}
|
||
if abort.lease_withdrawal == CleanupEvidence::Pending {
|
||
let expected = required(¤t.value.expected, "Expected projection")?;
|
||
let activation = required(¤t.value.activation, "activation")?;
|
||
// A prepared lease is withdrawn too: its physical grant may have
|
||
// landed before `Granted` was durable. Revoking an absent lease,
|
||
// and withdrawing an absent projection, are no-ops.
|
||
let lease = current
|
||
.value
|
||
.leasing
|
||
.as_ref()
|
||
.and_then(LeasingEvidence::issued_lease)
|
||
.context("post-fencing abort lost the lease it must withdraw")?;
|
||
let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1;
|
||
let lease_path = binding
|
||
.process_write_lease
|
||
.context("leased target has no write-lease binding")?
|
||
.record_path;
|
||
// The candidate holds its lifetime write lease; revoking it under
|
||
// a live holder fails on the shared lock forever. A stopped
|
||
// process writes nothing, so stop first (as advance_fencing does).
|
||
if let Some(workload) = ¤t.value.workload {
|
||
let plan = required(¤t.value.plan, "abandoned candidate plan")?;
|
||
self.workload_for(plan)?.stop(workload).context(
|
||
"stopping the abandoned candidate before revoking its lifetime-held write lease",
|
||
)?;
|
||
}
|
||
let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path);
|
||
driver
|
||
.revoke_exact(Some(lease))
|
||
.context("revoking the abandoned candidate write lease")?;
|
||
ensure!(
|
||
driver.observe_empty()?,
|
||
"candidate write lease remained after post-fencing withdrawal"
|
||
);
|
||
self.topology()
|
||
.withdraw_process_write_lease(expected, activation, Some(lease))
|
||
.context("withdrawing the abandoned candidate write-lease projection")?;
|
||
return self.persist_same_phase(current, |next| {
|
||
next.post_fencing_abort.as_mut().unwrap().lease_withdrawal =
|
||
CleanupEvidence::Complete;
|
||
Ok(())
|
||
});
|
||
}
|
||
if abort.candidate_cleanup == CleanupEvidence::Pending {
|
||
let plan = required(¤t.value.plan, "abandoned candidate plan")?;
|
||
if let Some(workload) = ¤t.value.workload {
|
||
self.workload_for(plan)?
|
||
.stop(workload)
|
||
.context("stopping the abandoned candidate")?;
|
||
}
|
||
self.workload_for(plan)?
|
||
.discard_prepared(
|
||
required(¤t.value.plan, "abandoned candidate plan")?,
|
||
required(¤t.value.expected, "abandoned Expected projection")?,
|
||
required(¤t.value.activation, "abandoned activation")?,
|
||
)
|
||
.context("discarding the abandoned activation material")?;
|
||
return self.persist_same_phase(current, |next| {
|
||
next.post_fencing_abort.as_mut().unwrap().candidate_cleanup =
|
||
CleanupEvidence::Complete;
|
||
Ok(())
|
||
});
|
||
}
|
||
if abort.topology_reconciliation == CleanupEvidence::Pending {
|
||
let snapshot = ControlSnapshot::read(&self.options.state_store)?;
|
||
// The failed candidate's projection is resolved first. For a
|
||
// deployment, fencing also stopped the incumbent and revoked its
|
||
// lease, so the incumbent is demoted to Expected-only under its
|
||
// key with its own exact activation, which is what lets
|
||
// continuity bring it back. A continuity candidate is the
|
||
// incumbent's key: it has nothing further to demote.
|
||
let topology = self.topology();
|
||
let failed_provider_anchor = self.provider_anchor_for_plan(required(
|
||
¤t.value.plan,
|
||
"failed transaction plan",
|
||
)?)?;
|
||
reconcile_failed_candidate_projection(
|
||
&topology,
|
||
¤t.value,
|
||
&failed_provider_anchor,
|
||
)?;
|
||
if let Some(incumbent) = self
|
||
.exact_incumbent(&snapshot, ¤t.value)?
|
||
.filter(|_| current.value.command_kind == CommandKind::Deploy)
|
||
{
|
||
let admitted_provider_anchor =
|
||
self.provider_anchor_for_plan(&incumbent.value.plan)?;
|
||
let expected_sha256 = topology.demote_to_expected_only(
|
||
&incumbent.value.expected,
|
||
&admitted_provider_anchor,
|
||
&incumbent.value.activation,
|
||
incumbent.value.leasing.lease(),
|
||
)?;
|
||
ensure!(
|
||
expected_sha256 == incumbent.value.expected.canonical_sha256()?,
|
||
"restored incumbent Expected receipt differs"
|
||
);
|
||
}
|
||
return self.persist_same_phase(current, |next| {
|
||
next.post_fencing_abort
|
||
.as_mut()
|
||
.unwrap()
|
||
.topology_reconciliation = CleanupEvidence::Complete;
|
||
Ok(())
|
||
});
|
||
}
|
||
if abort.source_cleanup == CleanupEvidence::Pending {
|
||
self.source
|
||
.cleanup(
|
||
¤t.value.transaction_id,
|
||
current.value.frozen_source.as_ref(),
|
||
)
|
||
.context("cleaning the abandoned source")?;
|
||
return self.persist_same_phase(current, |next| {
|
||
next.post_fencing_abort.as_mut().unwrap().source_cleanup =
|
||
CleanupEvidence::Complete;
|
||
Ok(())
|
||
});
|
||
}
|
||
ensure!(
|
||
abort.is_complete(),
|
||
"post-fencing abort cleanup is incomplete"
|
||
);
|
||
let mut next = current.value.clone();
|
||
next.enter_phase(DeploymentPhase::Complete, now_millis()?);
|
||
next.last_error = Some(abort.error.clone());
|
||
next.completion = Some(TransactionCompletion::FailedAfterFencing {
|
||
error: abort.error.clone(),
|
||
recovery: current.value.terminal_recovery(),
|
||
});
|
||
replace_transaction(&self.options.state_store, current, &next)
|
||
}
|
||
|
||
fn advance_pre_fencing_abort(&self, current: &Stored<DeploymentTransaction>) -> Result<()> {
|
||
ensure!(
|
||
current.value.phase < DeploymentPhase::Fencing,
|
||
"pre-fencing abort crossed the fencing boundary"
|
||
);
|
||
let abort = required(¤t.value.pre_fencing_abort, "pre-fencing abort intent")?;
|
||
if abort.candidate_cleanup == CleanupEvidence::Pending {
|
||
let plan = required(¤t.value.plan, "transaction plan")?;
|
||
if let Some(workload) = ¤t.value.workload {
|
||
self.workload_for(plan)?
|
||
.stop(workload)
|
||
.context("stopping exact pre-fence candidate")?;
|
||
}
|
||
self.workload_for(plan)?
|
||
.discard_prepared(
|
||
required(¤t.value.plan, "pre-fencing candidate plan")?,
|
||
required(¤t.value.expected, "pre-fencing Expected projection")?,
|
||
required(¤t.value.activation, "pre-fencing activation")?,
|
||
)
|
||
.context("discarding exact pre-fence activation material")?;
|
||
return self.persist_same_phase(current, |next| {
|
||
next.pre_fencing_abort.as_mut().unwrap().candidate_cleanup =
|
||
CleanupEvidence::Complete;
|
||
Ok(())
|
||
});
|
||
}
|
||
if abort.topology_reconciliation == CleanupEvidence::Pending {
|
||
// Before fencing a deployment never touched its incumbent: it is
|
||
// still running under its own key with its activation and lease,
|
||
// so only the candidate's records are withdrawn. A continuity
|
||
// candidate shares the admitted key and demotes what it issued.
|
||
let plan = required(¤t.value.plan, "failed transaction plan")?;
|
||
let provider_anchor = self.provider_anchor_for_plan(plan)?;
|
||
reconcile_failed_candidate_projection(
|
||
&self.topology(),
|
||
¤t.value,
|
||
&provider_anchor,
|
||
)?;
|
||
return self.persist_same_phase(current, |next| {
|
||
next.pre_fencing_abort
|
||
.as_mut()
|
||
.unwrap()
|
||
.topology_reconciliation = CleanupEvidence::Complete;
|
||
Ok(())
|
||
});
|
||
}
|
||
if abort.source_cleanup == CleanupEvidence::Pending {
|
||
self.source
|
||
.cleanup(
|
||
¤t.value.transaction_id,
|
||
current.value.frozen_source.as_ref(),
|
||
)
|
||
.context("cleaning failed pre-fence source")?;
|
||
return self.persist_same_phase(current, |next| {
|
||
next.pre_fencing_abort.as_mut().unwrap().source_cleanup = CleanupEvidence::Complete;
|
||
Ok(())
|
||
});
|
||
}
|
||
ensure!(
|
||
abort.is_complete(),
|
||
"pre-fencing abort cleanup is incomplete"
|
||
);
|
||
let mut next = current.value.clone();
|
||
next.enter_phase(DeploymentPhase::Complete, now_millis()?);
|
||
next.last_error = Some(abort.error.clone());
|
||
next.completion = Some(TransactionCompletion::FailedBeforeFencing {
|
||
error: abort.error.clone(),
|
||
});
|
||
replace_transaction(&self.options.state_store, current, &next)
|
||
}
|
||
|
||
fn advance_post_commit_cleanup(&self, current: &Stored<DeploymentTransaction>) -> Result<()> {
|
||
let cleanup = required(
|
||
¤t.value.post_commit_cleanup,
|
||
"post-commit cleanup evidence",
|
||
)?;
|
||
if let IncumbentCleanupEvidence::Pending {
|
||
generation_id,
|
||
workload,
|
||
} = &cleanup.incumbent
|
||
{
|
||
let binding = required(¤t.value.plan, "committed transaction plan")?
|
||
.parsed_inputs()?
|
||
.1;
|
||
if let Some(promoted_at_unix_millis) = required(
|
||
¤t.value.routing,
|
||
"committed transaction routing evidence",
|
||
)?
|
||
.promoted_at_unix_millis()
|
||
{
|
||
let retire_not_before =
|
||
route_drain_deadline(promoted_at_unix_millis, binding.rollout.drain_seconds)?;
|
||
if now_millis()? < retire_not_before {
|
||
return Ok(());
|
||
}
|
||
}
|
||
// The incumbent is the same target under the same binding kind;
|
||
// an incumbent of another kind is refused by the driver, not
|
||
// guessed at.
|
||
self.workload_for(required(¤t.value.plan, "committed transaction plan")?)?
|
||
.stop(workload)
|
||
.with_context(|| format!("retiring admitted incumbent {generation_id}"))?;
|
||
let generation_id = generation_id.clone();
|
||
return self.persist_same_phase(current, |next| {
|
||
next.post_commit_cleanup.as_mut().unwrap().incumbent =
|
||
IncumbentCleanupEvidence::Complete { generation_id };
|
||
Ok(())
|
||
});
|
||
}
|
||
if cleanup.source == SourceCleanupEvidence::Pending {
|
||
self.source
|
||
.cleanup(
|
||
¤t.value.transaction_id,
|
||
current.value.frozen_source.as_ref(),
|
||
)
|
||
.context("cleaning committed deployment source")?;
|
||
return self.persist_same_phase(current, |next| {
|
||
next.post_commit_cleanup.as_mut().unwrap().source = SourceCleanupEvidence::Complete;
|
||
Ok(())
|
||
});
|
||
}
|
||
ensure!(cleanup.is_complete(), "post-commit cleanup is incomplete");
|
||
Ok(())
|
||
}
|
||
}
|
||
|
||
fn route_drain_deadline(promoted_at_unix_millis: u64, drain_seconds: u32) -> Result<u64> {
|
||
promoted_at_unix_millis
|
||
.checked_add(
|
||
u64::from(drain_seconds)
|
||
.checked_mul(1_000)
|
||
.context("route drain duration overflows milliseconds")?,
|
||
)
|
||
.context("route drain deadline overflows Unix milliseconds")
|
||
}
|
||
|
||
/// CultLib refuses a correlation older than the trusted window with this
|
||
/// text. Idunn reads that one refusal as "nothing new", everywhere it waits
|
||
/// on Odin's evidence; every other refusal stays a refusal.
|
||
fn is_stale_observation(error: &anyhow::Error) -> bool {
|
||
format!("{error:#}").contains("outside the trusted observation window")
|
||
}
|
||
|
||
fn route_observation_is_current(
|
||
observed_at_unix_millis: u64,
|
||
now_unix_millis: u64,
|
||
maximum_age_millis: u64,
|
||
maximum_future_skew_millis: u64,
|
||
) -> bool {
|
||
observed_at_unix_millis <= now_unix_millis.saturating_add(maximum_future_skew_millis)
|
||
&& now_unix_millis.saturating_sub(observed_at_unix_millis) <= maximum_age_millis
|
||
}
|
||
|
||
fn provider_warming_advanced(
|
||
prior_odin_sequence: u64,
|
||
prior_signed_presence_sha256: &str,
|
||
candidate_odin_sequence: u64,
|
||
candidate_signed_presence_sha256: &str,
|
||
) -> bool {
|
||
candidate_odin_sequence > prior_odin_sequence
|
||
&& candidate_signed_presence_sha256 != prior_signed_presence_sha256
|
||
}
|
||
|
||
/// Resolve the projection a failed candidate left. A deployment candidate owns
|
||
/// its own incarnation key and withdraws it whole. A continuity candidate
|
||
/// shares the admitted key: the admitted Expected is never withdrawn, and the
|
||
/// only thing demoted is the activation this transaction issued, exactly.
|
||
fn reconcile_failed_candidate_projection(
|
||
topology: &CultCacheTopologyDriver,
|
||
transaction: &DeploymentTransaction,
|
||
provider_anchor: &ServiceIdentityTrustAnchor,
|
||
) -> Result<()> {
|
||
let expected = required(&transaction.expected, "failed Expected projection")?;
|
||
match transaction.command_kind {
|
||
CommandKind::Deploy => topology
|
||
.withdraw_incarnation(
|
||
expected,
|
||
provider_anchor,
|
||
transaction.activation.as_ref(),
|
||
None,
|
||
)
|
||
.context("withdrawing the failed candidate projection"),
|
||
CommandKind::Continuity => topology
|
||
.demote_to_expected_only(
|
||
expected,
|
||
provider_anchor,
|
||
required(&transaction.activation, "failed continuity activation")?,
|
||
None,
|
||
)
|
||
.map(drop)
|
||
.context("demoting the failed continuity activation"),
|
||
}
|
||
}
|
||
|
||
fn may_rollback_route_after_failed_proof(fencing: &FencingEvidence) -> bool {
|
||
matches!(fencing, FencingEvidence::SkippedStateless)
|
||
}
|
||
|
||
fn incumbent_was_stopped_during_fencing(fencing: &FencingEvidence) -> bool {
|
||
matches!(
|
||
fencing,
|
||
FencingEvidence::Revoked {
|
||
incumbent_lease_sha256: Some(_),
|
||
..
|
||
}
|
||
)
|
||
}
|
||
|
||
fn pre_fencing_abort_intent(transaction: &DeploymentTransaction, error: &str) -> PreFencingAbort {
|
||
PreFencingAbort {
|
||
error: bounded_detail(error),
|
||
candidate_cleanup: transaction.candidate_cleanup_owed(),
|
||
topology_reconciliation: transaction.abort_topology_reconciliation(),
|
||
source_cleanup: if transaction.command_kind == CommandKind::Deploy {
|
||
CleanupEvidence::Pending
|
||
} else {
|
||
CleanupEvidence::Skipped
|
||
},
|
||
}
|
||
}
|
||
|
||
fn post_fencing_abort_intent(
|
||
transaction: &DeploymentTransaction,
|
||
error: &str,
|
||
) -> PostFencingAbort {
|
||
PostFencingAbort {
|
||
error: bounded_detail(error),
|
||
route_restoration: if transaction.route_preflight.is_some() {
|
||
CleanupEvidence::Pending
|
||
} else {
|
||
CleanupEvidence::Skipped
|
||
},
|
||
lease_withdrawal: if transaction
|
||
.leasing
|
||
.as_ref()
|
||
.and_then(LeasingEvidence::issued_lease)
|
||
.is_some()
|
||
{
|
||
CleanupEvidence::Pending
|
||
} else {
|
||
CleanupEvidence::Skipped
|
||
},
|
||
candidate_cleanup: transaction.candidate_cleanup_owed(),
|
||
topology_reconciliation: transaction.abort_topology_reconciliation(),
|
||
source_cleanup: if transaction.command_kind == CommandKind::Deploy {
|
||
CleanupEvidence::Pending
|
||
} else {
|
||
CleanupEvidence::Skipped
|
||
},
|
||
}
|
||
}
|
||
|
||
fn sequence_requires_admission(
|
||
latest: Option<&TopologyEvidence>,
|
||
maximum_admitted_sequence: u64,
|
||
candidate: &TopologyEvidence,
|
||
) -> Result<bool> {
|
||
candidate.validate_shape()?;
|
||
if latest.is_some_and(|existing| {
|
||
existing.canonical_bytes == candidate.canonical_bytes
|
||
&& existing.canonical_sha256 == candidate.canonical_sha256
|
||
&& existing.signer_identity_id == candidate.signer_identity_id
|
||
&& existing.publisher_sequence == candidate.publisher_sequence
|
||
}) {
|
||
return Ok(false);
|
||
}
|
||
ensure!(
|
||
candidate.publisher_sequence > maximum_admitted_sequence,
|
||
"Odin topology publisher sequence was replayed or reordered"
|
||
);
|
||
Ok(true)
|
||
}
|
||
|
||
fn validate_live_providers_for_deploy<F>(command_kind: CommandKind, validation: F) -> Result<()>
|
||
where
|
||
F: FnOnce() -> Result<()>,
|
||
{
|
||
match command_kind {
|
||
CommandKind::Deploy => validation(),
|
||
CommandKind::Continuity => Ok(()),
|
||
}
|
||
}
|
||
|
||
fn validate_authenticated_evidence(
|
||
evidence: &TopologyEvidence,
|
||
authenticated: &AuthenticatedOdinRuntimeTopologyCorrelation,
|
||
) -> Result<()> {
|
||
let record = authenticated.record();
|
||
ensure!(
|
||
authenticated.canonical_bytes() == evidence.canonical_bytes.as_slice()
|
||
&& sha256_id(authenticated.canonical_bytes()) == evidence.canonical_sha256
|
||
&& record.signer_identity_id == evidence.signer_identity_id
|
||
&& record.publisher_sequence == evidence.publisher_sequence,
|
||
"durable topology evidence differs from its authenticated receipt"
|
||
);
|
||
Ok(())
|
||
}
|
||
|
||
fn is_semantic_warming(
|
||
expected: &IdunnExpectedIncarnationRecord,
|
||
activation: &IdunnRuntimeActivationRecord,
|
||
incumbent_lease_sha256: Option<&str>,
|
||
authenticated: &AuthenticatedOdinRuntimeTopologyCorrelation,
|
||
) -> Result<bool> {
|
||
let record = authenticated.record();
|
||
if !record.present || record.observed_write_lease_sha256.is_some() {
|
||
return Ok(false);
|
||
}
|
||
let expected_projection_detail = format!(
|
||
"expected:{};activation:{}",
|
||
expected.canonical_sha256()?,
|
||
activation.canonical_sha256()?
|
||
);
|
||
if !warming_disagreements_match_incumbent(
|
||
&expected_projection_detail,
|
||
incumbent_lease_sha256,
|
||
&record.disagreements,
|
||
) {
|
||
return Ok(false);
|
||
}
|
||
Ok(if expected.write_lease_required {
|
||
!record.ready && record.observed_presence_state.as_deref() == Some("warming")
|
||
} else {
|
||
matches!(
|
||
record.observed_presence_state.as_deref(),
|
||
Some("warming" | "active")
|
||
)
|
||
})
|
||
}
|
||
|
||
fn warming_disagreements_match_incumbent(
|
||
expected_projection_detail: &str,
|
||
incumbent_lease_sha256: Option<&str>,
|
||
disagreements: &[OdinTopologyDisagreement],
|
||
) -> bool {
|
||
if disagreements.is_empty() {
|
||
true
|
||
} else if let (Some(incumbent_lease_sha256), [disagreement]) =
|
||
(incumbent_lease_sha256, disagreements)
|
||
{
|
||
disagreement.code == "projected-write-lease"
|
||
&& disagreement.expected.as_deref() == Some(expected_projection_detail)
|
||
&& disagreement.observed.as_deref() == Some(incumbent_lease_sha256)
|
||
} else {
|
||
false
|
||
}
|
||
}
|
||
|
||
fn is_semantic_ready(authenticated: &AuthenticatedOdinRuntimeTopologyCorrelation) -> bool {
|
||
let record = authenticated.record();
|
||
record.present
|
||
&& record.ready
|
||
&& record.observed_presence_state.as_deref() == Some("active")
|
||
&& record.disagreements.is_empty()
|
||
}
|
||
|
||
fn read_deployment_brake(path: &Path) -> Result<Option<(IdunnDeploymentBrakeRecord, Vec<u8>)>> {
|
||
let Some(envelope) = read_single_envelope(path)? else {
|
||
return Ok(None);
|
||
};
|
||
ensure!(
|
||
envelope.r#type == IdunnDeploymentBrakeRecord::TYPE
|
||
&& envelope.schema_id.as_deref() == Some(IDUNN_DEPLOYMENT_BRAKE_SCHEMA),
|
||
"deployment brake store contains a foreign record"
|
||
);
|
||
let record: IdunnDeploymentBrakeRecord = rmp_serde::from_slice(&envelope.payload)?;
|
||
record.validate()?;
|
||
ensure!(
|
||
rmp_serde::to_vec(&record)? == envelope.payload && envelope.key == record.brake_id,
|
||
"deployment brake is noncanonical or keyed by another authority"
|
||
);
|
||
Ok(Some((record, envelope.payload)))
|
||
}
|
||
|
||
fn read_lifecycle_brake(path: &Path) -> Result<Option<IdunnLifecycleBrakeRecord>> {
|
||
let Some(envelope) = read_single_envelope(path)? else {
|
||
return Ok(None);
|
||
};
|
||
ensure!(
|
||
envelope.r#type == IdunnLifecycleBrakeRecord::TYPE
|
||
&& envelope.schema_id.as_deref() == Some(IDUNN_LIFECYCLE_BRAKE_SCHEMA),
|
||
"lifecycle brake store contains a foreign record"
|
||
);
|
||
IdunnLifecycleBrakeRecord::decode_canonical(&envelope.payload).map(Some)
|
||
}
|
||
|
||
fn read_single_envelope(path: &Path) -> Result<Option<CultCacheEnvelope>> {
|
||
if !path.exists() {
|
||
return Ok(None);
|
||
}
|
||
let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?;
|
||
match entries.as_slice() {
|
||
[] => Ok(None),
|
||
[entry] => Ok(Some(entry.clone())),
|
||
_ => bail!("single-record authority store is ambiguous"),
|
||
}
|
||
}
|
||
|
||
pub(crate) fn read_trust_anchor<P: ServiceIdentityProfile>(
|
||
path: &Path,
|
||
) -> Result<ServiceIdentityTrustAnchor> {
|
||
let envelope = read_single_envelope(path)?
|
||
.with_context(|| format!("service identity trust anchor {} is absent", path.display()))?;
|
||
ensure!(
|
||
envelope.r#type == P::TRUST_ANCHOR_TYPE
|
||
&& envelope.key == P::TRUST_ANCHOR_KEY
|
||
&& envelope.schema_id.as_deref() == Some(P::TRUST_ANCHOR_SCHEMA),
|
||
"service identity trust anchor belongs to another profile"
|
||
);
|
||
let anchor: ServiceIdentityTrustAnchor = rmp_serde::from_slice(&envelope.payload)?;
|
||
ensure!(
|
||
rmp_serde::to_vec(&anchor)? == envelope.payload
|
||
&& derive_service_identity_id::<P>(&anchor.public_key)? == anchor.identity_id,
|
||
"service identity trust anchor is noncanonical or self-inconsistent"
|
||
);
|
||
Ok(anchor)
|
||
}
|
||
|
||
fn required<'a, T>(value: &'a Option<T>, label: &str) -> Result<&'a T> {
|
||
value.as_ref().with_context(|| format!("{label} is absent"))
|
||
}
|
||
|
||
fn now_millis() -> Result<u64> {
|
||
Ok(SystemTime::now()
|
||
.duration_since(UNIX_EPOCH)
|
||
.context("system clock predates Unix epoch")?
|
||
.as_millis()
|
||
.try_into()?)
|
||
}
|
||
|
||
fn rfc3339_millis(millis: u64) -> Result<String> {
|
||
chrono::DateTime::from_timestamp_millis(i64::try_from(millis)?)
|
||
.context("timestamp is out of range")
|
||
.map(|value| value.to_rfc3339())
|
||
}
|
||
|
||
fn sibling_path(path: &Path, suffix: &str) -> PathBuf {
|
||
let mut value = path.as_os_str().to_os_string();
|
||
value.push(suffix);
|
||
PathBuf::from(value)
|
||
}
|
||
|
||
fn string_value(args: &mut impl Iterator<Item = String>, name: &str) -> Result<String> {
|
||
args.next()
|
||
.ok_or_else(|| anyhow!("{name} requires a value"))
|
||
}
|
||
|
||
fn path_value(args: &mut impl Iterator<Item = String>, name: &str) -> Result<PathBuf> {
|
||
Ok(PathBuf::from(string_value(args, name)?))
|
||
}
|
||
|
||
fn u32_value(args: &mut impl Iterator<Item = String>, name: &str) -> Result<u32> {
|
||
string_value(args, name)?
|
||
.parse()
|
||
.with_context(|| format!("{name} requires a u32"))
|
||
}
|
||
|
||
fn u64_value(args: &mut impl Iterator<Item = String>, name: &str) -> Result<u64> {
|
||
string_value(args, name)?
|
||
.parse()
|
||
.with_context(|| format!("{name} requires a u64"))
|
||
}
|
||
|
||
fn require_selector(value: &str) -> Result<()> {
|
||
let value = value.strip_prefix("profile:").unwrap_or(value);
|
||
require_id(value, "deployment selector")
|
||
}
|
||
|
||
pub(crate) fn require_id(value: &str, label: &str) -> Result<()> {
|
||
ensure!(
|
||
!value.is_empty()
|
||
&& value.len() <= 256
|
||
&& value.bytes().all(|byte| {
|
||
byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b':' | b'/')
|
||
}),
|
||
"{label} is invalid"
|
||
);
|
||
Ok(())
|
||
}
|
||
|
||
fn require_value(value: &str, label: &str) -> Result<()> {
|
||
ensure!(
|
||
!value.trim().is_empty()
|
||
&& value == value.trim()
|
||
&& value.len() <= 1024
|
||
&& !value.contains('\0'),
|
||
"{label} is invalid"
|
||
);
|
||
Ok(())
|
||
}
|
||
|
||
fn require_detail(value: &str, label: &str) -> Result<()> {
|
||
ensure!(
|
||
!value.is_empty() && value.len() <= 2048 && !value.contains('\0'),
|
||
"{label} is invalid"
|
||
);
|
||
Ok(())
|
||
}
|
||
|
||
const DETAIL_LIMIT: usize = 2048;
|
||
const DETAIL_HEAD: usize = 512;
|
||
const DETAIL_ELISION: &str = "
|
||
...[elided]...
|
||
";
|
||
const DETAIL_EMPTY: &str = "(no detail)";
|
||
|
||
/// The only producer of persisted free-text details; `require_detail` is the
|
||
/// only judge, and this output always satisfies it. NUL is dropped, empty
|
||
/// input gets a fixed placeholder, and an over-long value keeps its head and
|
||
/// its tail (where a build log's error is) cut on char boundaries.
|
||
fn bounded_detail(value: &str) -> String {
|
||
let value = value.replace(' |