diff --git a/crates/ghostlight-dungeon/src/app_session.rs b/crates/ghostlight-dungeon/src/app_session.rs index 1fd62fc..f35a449 100644 --- a/crates/ghostlight-dungeon/src/app_session.rs +++ b/crates/ghostlight-dungeon/src/app_session.rs @@ -46,6 +46,21 @@ struct AppSession { revoked_at: Option>, } +/// The clause by which the local commit refuses a refresh Heimdall executed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub(crate) enum RefreshRefusal { + #[error("local session was revoked during refresh")] + Revoked, + #[error("local session moved to another revision since the refresh candidate was taken")] + MovedSinceCandidate, + #[error("refresh receipt revision went backwards")] + RevisionWentBackwards, + #[error("refresh receipt names a different Heimdall session")] + SessionId, + #[error("refresh receipt names a different account")] + Account, +} + pub(crate) struct RefreshCandidate { pub(crate) cookie_hash: String, pub(crate) account_subject_hash: String, @@ -290,14 +305,25 @@ impl AppSessionOwner { .sessions .get_mut(cookie_hash) .context("local session vanished during refresh")?; - if session.revoked_at.is_some() - || session.access_revision != expected_access_revision - || input.access_revision() <= expected_access_revision - || session.heimdall_session_id != input.heimdall_session_id() - || session.account_subject_hash - != secret_hash(&format!("heimdall-account:{}", input.account_id())) + if session.revoked_at.is_some() { + return Err(RefreshRefusal::Revoked.into()); + } + if session.access_revision != expected_access_revision { + return Err(RefreshRefusal::MovedSinceCandidate.into()); + } + // Heimdall's access_revision is a revocation epoch: refresh rotates + // the token at the same revision, so equal is the normal receipt and + // only a lower one is a refusal. + if input.access_revision() < expected_access_revision { + return Err(RefreshRefusal::RevisionWentBackwards.into()); + } + if session.heimdall_session_id != input.heimdall_session_id() { + return Err(RefreshRefusal::SessionId.into()); + } + if session.account_subject_hash + != secret_hash(&format!("heimdall-account:{}", input.account_id())) { - bail!("local session changed during refresh"); + return Err(RefreshRefusal::Account.into()); } session.access_revision = input.access_revision(); session.verified_capabilities = input.capabilities().to_vec(); @@ -367,6 +393,16 @@ impl AppSessionOwner { } } +/// The idempotency key of the refresh that spends `refresh_claim`: a retry of +/// that refresh derives the same key, the next refresh (a new claim) a new one, +/// and the claim itself never leaves the process in it. +pub(crate) fn refresh_idempotency_key(refresh_claim: &str) -> String { + format!( + "refresh:{}", + secret_hash(&format!("heimdall-refresh-key:{refresh_claim}")) + ) +} + pub(crate) fn secret_hash(value: &str) -> String { format!("sha256:{:x}", Sha256::digest(value.as_bytes())) } @@ -787,6 +823,40 @@ mod tests { assert!(AppSessionOwner::open(&store_path, &key).is_err()); } + #[test] + fn a_refresh_for_a_session_that_moved_is_refused_by_name() { + let directory = tempfile::tempdir().unwrap(); + let key = directory.path().join("session.key"); + std::fs::write(&key, [5_u8; 32]).unwrap(); + let mut owner = + AppSessionOwner::open(directory.path().join("sessions.cc"), &key).unwrap(); + let cookie = owner + .create_session(VerifiedSessionAdmission::fixture( + "account", + "session", + 1, + Utc::now() + chrono::Duration::minutes(1), + Utc::now() + chrono::Duration::days(1), + "claim-0", + )) + .unwrap(); + let receipt = VerifiedSessionRefresh::fixture( + "account", + "session", + 1, + Utc::now() + chrono::Duration::minutes(1), + Utc::now() + chrono::Duration::days(1), + "claim-1", + ); + let error = owner + .apply_refresh(&secret_hash(&cookie), 0, receipt) + .unwrap_err(); + assert_eq!( + error.downcast_ref::(), + Some(&RefreshRefusal::MovedSinceCandidate) + ); + } + #[test] fn refresh_claim_is_encrypted_and_exactly_bound() { let directory = tempfile::tempdir().unwrap(); diff --git a/crates/ghostlight-dungeon/src/heimdall.rs b/crates/ghostlight-dungeon/src/heimdall.rs index 01345f6..8816536 100644 --- a/crates/ghostlight-dungeon/src/heimdall.rs +++ b/crates/ghostlight-dungeon/src/heimdall.rs @@ -30,6 +30,48 @@ const APP_SLUG: &str = "ghostlight"; const PRIVATE_SERVICE: &str = "heimdall.private.commands"; const PRIVATE_ENVELOPE_SCHEMA: &str = "heimdall.private_command_envelope.v1"; +/// Heimdall answered a private command with status denied. Distinct from a +/// transport failure so the refresh loop never reads a message string. +#[derive(Debug, thiserror::Error)] +#[error("Heimdall denied the private command: {diagnostics}")] +pub(crate) struct HeimdallDenied { + pub(crate) diagnostics: String, +} + +/// The two Heimdall calls the refresh loop makes, as a port so a scripted +/// Heimdall can stand in for the real one. +#[async_trait::async_trait] +pub(crate) trait SessionRefresher { + async fn refresh( + &self, + refresh_token: &str, + idempotency_key: &str, + ) -> anyhow::Result; + + async fn verify_refresh( + &self, + completion: AuthCompletionReceipt, + ) -> anyhow::Result; +} + +#[async_trait::async_trait] +impl SessionRefresher for HeimdallClient { + async fn refresh( + &self, + refresh_token: &str, + idempotency_key: &str, + ) -> anyhow::Result { + HeimdallClient::refresh(self, refresh_token, idempotency_key).await + } + + async fn verify_refresh( + &self, + completion: AuthCompletionReceipt, + ) -> anyhow::Result { + HeimdallClient::verify_refresh(self, completion).await + } +} + #[derive(Clone)] pub struct HeimdallClient { http: Client, @@ -325,6 +367,28 @@ impl VerifiedSessionAdmission { } } +#[cfg(test)] +impl VerifiedSessionRefresh { + pub(crate) fn fixture( + account_id: impl Into, + heimdall_session_id: impl Into, + access_revision: u64, + access_expires_at: DateTime, + refresh_expires_at: DateTime, + refresh_claim: impl Into, + ) -> Self { + Self(VerifiedSessionMaterial { + account_id: account_id.into(), + heimdall_session_id: heimdall_session_id.into(), + access_revision, + capabilities: vec!["app_access".into()], + access_expires_at, + refresh_expires_at, + refresh_claim: refresh_claim.into(), + }) + } +} + impl HeimdallClient { pub fn from_env( runtime_id: &str, @@ -581,10 +645,10 @@ impl HeimdallClient { bail!("Heimdall returned a non-operation response"); }; if status == "denied" { - bail!( - "Heimdall denied the private command: {}", - diagnostics.join("; ") - ); + return Err(HeimdallDenied { + diagnostics: diagnostics.join("; "), + } + .into()); } if payload_schema != PRIVATE_ENVELOPE_SCHEMA || payload_encoding != "messagepack-base64" { bail!("Heimdall returned the wrong private response contract"); diff --git a/crates/ghostlight-dungeon/src/runtime.rs b/crates/ghostlight-dungeon/src/runtime.rs index 6f83252..6a2b84c 100644 --- a/crates/ghostlight-dungeon/src/runtime.rs +++ b/crates/ghostlight-dungeon/src/runtime.rs @@ -1,9 +1,9 @@ //! One-process runtime for the sealed world owner. use crate::{ - app_session::AppSessionOwner, + app_session::{AppSessionOwner, RefreshRefusal}, eve::{self, EveCommandInvocation}, - heimdall::{self, HeimdallClient}, + heimdall::{self, HeimdallClient, HeimdallDenied, SessionRefresher}, idunn_health::{ IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, ProcessWriteLeaseGuard, RuntimePresencePublisher, TARGET as GHOSTLIGHT_TARGET, @@ -1867,66 +1867,86 @@ async fn maintain_session_refresh(state: AppState) -> anyhow::Result<()> { interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); loop { interval.tick().await; - let candidates = state - .sessions - .lock() - .await - .sessions_due_for_refresh(Utc::now(), chrono::Duration::minutes(2)) - .context("app-session refresh scan lost custody")?; - for candidate in candidates { - let idempotency = format!( - "refresh:{}:{}", - candidate.heimdall_session_id, candidate.access_revision - ); - let completion = match state - .heimdall - .refresh(&candidate.refresh_claim, &idempotency) - .await - { - Ok(value) => value, - Err(error) => { - tracing::warn!(%error, "Heimdall refresh transport unavailable"); - continue; - } - }; - let verified = match state.heimdall.verify_refresh(completion).await { - Ok(value) => value, - Err(error) => { - tracing::warn!(%error, "Heimdall refresh receipt was invalid"); - let mut sessions = state.sessions.lock().await; - if let Err(revoke_error) = sessions.revoke_cookie_hash(&candidate.cookie_hash) { - return Err(revoke_error) - .context("invalid Heimdall refresh could not revoke local custody"); - } - continue; - } - }; - if verified.heimdall_session_id() != candidate.heimdall_session_id - || crate::app_session::secret_hash(&format!( - "heimdall-account:{}", - verified.account_id() - )) != candidate.account_subject_hash - { - tracing::warn!("Heimdall refresh changed local session custody"); - let mut sessions = state.sessions.lock().await; + refresh_due_sessions(&state.sessions, state.heimdall.as_ref()).await?; + } +} + +/// One pass of the refresh loop over every session due for refresh. +async fn refresh_due_sessions( + sessions: &Mutex, + heimdall: &impl SessionRefresher, +) -> anyhow::Result<()> { + let candidates = sessions + .lock() + .await + .sessions_due_for_refresh(Utc::now(), chrono::Duration::minutes(2)) + .context("app-session refresh scan lost custody")?; + for candidate in candidates { + let idempotency = crate::app_session::refresh_idempotency_key(&candidate.refresh_claim); + let completion = match heimdall.refresh(&candidate.refresh_claim, &idempotency).await { + Ok(value) => value, + Err(error) if error.downcast_ref::().is_some() => { + // Heimdall refused this refresh claim: it can never succeed. + tracing::warn!(%error, "Heimdall denied the refresh; revoking the local session"); + let mut sessions = sessions.lock().await; if let Err(revoke_error) = sessions.revoke_cookie_hash(&candidate.cookie_hash) { return Err(revoke_error) - .context("changed Heimdall custody could not revoke local session"); + .context("denied Heimdall refresh could not revoke local custody"); } continue; } - let mut sessions = state.sessions.lock().await; - if let Err(error) = - sessions.apply_refresh(&candidate.cookie_hash, candidate.access_revision, verified) - { - if sessions.is_healthy() { - tracing::warn!(%error, "local app session rejected Heimdall refresh"); - } else { - return Err(error).context("app-session refresh commit lost custody"); + Err(error) => { + tracing::warn!(%error, "Heimdall refresh transport unavailable"); + continue; + } + }; + let verified = match heimdall.verify_refresh(completion).await { + Ok(value) => value, + Err(error) => { + tracing::warn!(%error, "Heimdall refresh receipt was invalid"); + let mut sessions = sessions.lock().await; + if let Err(revoke_error) = sessions.revoke_cookie_hash(&candidate.cookie_hash) { + return Err(revoke_error) + .context("invalid Heimdall refresh could not revoke local custody"); } + continue; + } + }; + if verified.heimdall_session_id() != candidate.heimdall_session_id + || crate::app_session::secret_hash(&format!( + "heimdall-account:{}", + verified.account_id() + )) != candidate.account_subject_hash + { + tracing::warn!("Heimdall refresh changed local session custody"); + let mut sessions = sessions.lock().await; + if let Err(revoke_error) = sessions.revoke_cookie_hash(&candidate.cookie_hash) { + return Err(revoke_error) + .context("changed Heimdall custody could not revoke local session"); + } + continue; + } + let mut sessions = sessions.lock().await; + if let Err(error) = + sessions.apply_refresh(&candidate.cookie_hash, candidate.access_revision, verified) + { + if !sessions.is_healthy() { + return Err(error).context("app-session refresh commit lost custody"); + } + tracing::warn!(%error, "local app session refused Heimdall refresh"); + // The refresh claim is spent upstream, so a receipt that went + // backwards leaves nothing to retry. A session that merely moved + // is newer state: the next scan decides. + if error.downcast_ref::() + == Some(&RefreshRefusal::RevisionWentBackwards) + && let Err(revoke_error) = sessions.revoke_cookie_hash(&candidate.cookie_hash) + { + return Err(revoke_error) + .context("backwards Heimdall refresh could not revoke local session"); } } } + Ok(()) } /// Idunn publishes Expected and starts the unit before it observes the @@ -6370,3 +6390,223 @@ mod tests { assert_eq!(after.revision, before.revision + 1); } } + +/// The refresh loop against a scripted Heimdall: it answers each call from a +/// queue and records the claim and idempotency key it was sent. +#[cfg(test)] +mod refresh_loop_tests { + use super::*; + use crate::heimdall::{AuthCompletionReceipt, VerifiedSessionRefresh}; + use std::{collections::VecDeque, sync::Mutex as StdMutex}; + + enum Reply { + Deny, + Timeout, + Receipt { revision: u64, claim: &'static str }, + } + + #[derive(Default)] + struct ScriptedHeimdall { + replies: StdMutex>, + pending: StdMutex>, + calls: StdMutex>, + } + + impl ScriptedHeimdall { + fn answering(replies: Vec) -> Self { + Self { + replies: StdMutex::new(replies.into()), + ..Self::default() + } + } + + fn calls(&self) -> Vec<(String, String)> { + self.calls.lock().unwrap().clone() + } + } + + #[async_trait::async_trait] + impl SessionRefresher for ScriptedHeimdall { + async fn refresh( + &self, + refresh_token: &str, + idempotency_key: &str, + ) -> anyhow::Result { + self.calls + .lock() + .unwrap() + .push((refresh_token.into(), idempotency_key.into())); + match self.replies.lock().unwrap().pop_front().expect("scripted reply") { + Reply::Deny => Err(HeimdallDenied { + diagnostics: "Idempotency key was reused".into(), + } + .into()), + Reply::Timeout => bail!("Heimdall private command timed out"), + Reply::Receipt { revision, claim } => { + *self.pending.lock().unwrap() = Some(VerifiedSessionRefresh::fixture( + "operator-account", + "heimdall-session", + revision, + Utc::now() + chrono::Duration::seconds(90), + Utc::now() + chrono::Duration::days(1), + claim, + )); + Ok(AuthCompletionReceipt { + status: "authenticated".into(), + handle: None, + error: None, + account: None, + session: None, + access_token: None, + refresh_token: None, + refresh: None, + shared_capabilities: Vec::new(), + }) + } + } + } + + async fn verify_refresh( + &self, + _completion: AuthCompletionReceipt, + ) -> anyhow::Result { + Ok(self.pending.lock().unwrap().take().expect("scripted receipt")) + } + } + + /// A store holding one live session at `revision` whose access expires in + /// a minute, so every pass finds it due. + fn due_session(revision: u64) -> (tempfile::TempDir, Mutex, String) { + let directory = tempfile::tempdir().unwrap(); + let key = directory.path().join("session.key"); + std::fs::write(&key, [23_u8; 32]).unwrap(); + let mut owner = AppSessionOwner::open(directory.path().join("sessions.cc"), &key).unwrap(); + let cookie = owner + .create_session(heimdall::VerifiedSessionAdmission::fixture( + "operator-account", + "heimdall-session", + revision, + Utc::now() + chrono::Duration::seconds(60), + Utc::now() + chrono::Duration::days(1), + "claim-0", + )) + .unwrap(); + (directory, Mutex::new(owner), cookie) + } + + async fn is_live(sessions: &Mutex, cookie: &str) -> bool { + sessions + .lock() + .await + .account_for_cookie(cookie, Utc::now()) + .unwrap() + .is_some() + } + + #[derive(Clone, Default)] + struct Captured(std::sync::Arc>>); + + impl std::io::Write for Captured { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for Captured { + type Writer = Captured; + + fn make_writer(&'a self) -> Captured { + self.clone() + } + } + + #[tokio::test] + async fn a_real_refresh_at_an_equal_revision_is_applied_and_the_session_stays_live() { + let (_directory, sessions, cookie) = due_session(1); + let heimdall = ScriptedHeimdall::answering(vec![ + Reply::Receipt { revision: 1, claim: "claim-1" }, + Reply::Timeout, + ]); + refresh_due_sessions(&sessions, &heimdall).await.unwrap(); + assert!(is_live(&sessions, &cookie).await); + // The wrapped claim is the receipt's: the next refresh spends it. + refresh_due_sessions(&sessions, &heimdall).await.unwrap(); + assert_eq!(heimdall.calls()[1].0, "claim-1"); + assert!(is_live(&sessions, &cookie).await); + } + + #[tokio::test] + async fn a_retry_of_the_same_refresh_sends_the_same_key() { + let (_directory, sessions, cookie) = due_session(1); + let heimdall = ScriptedHeimdall::answering(vec![Reply::Timeout, Reply::Timeout]); + refresh_due_sessions(&sessions, &heimdall).await.unwrap(); + refresh_due_sessions(&sessions, &heimdall).await.unwrap(); + let calls = heimdall.calls(); + assert_eq!(calls.len(), 2, "a transport failure leaves the session offered"); + assert_eq!(calls[0], calls[1]); + assert!(is_live(&sessions, &cookie).await); + } + + #[tokio::test] + async fn the_refresh_after_a_successful_one_sends_a_new_key() { + let (_directory, sessions, _cookie) = due_session(1); + let heimdall = ScriptedHeimdall::answering(vec![ + Reply::Receipt { revision: 1, claim: "claim-1" }, + Reply::Timeout, + ]); + refresh_due_sessions(&sessions, &heimdall).await.unwrap(); + refresh_due_sessions(&sessions, &heimdall).await.unwrap(); + let calls = heimdall.calls(); + assert_ne!(calls[0].1, calls[1].1); + assert!(calls.iter().all(|(claim, key)| !key.contains(claim))); + } + + #[tokio::test] + async fn a_receipt_with_a_lower_revision_is_refused_and_the_session_revoked() { + let (_directory, sessions, cookie) = due_session(2); + let heimdall = ScriptedHeimdall::answering(vec![ + Reply::Receipt { revision: 1, claim: "claim-1" }, + Reply::Timeout, + ]); + refresh_due_sessions(&sessions, &heimdall).await.unwrap(); + assert!(!is_live(&sessions, &cookie).await); + refresh_due_sessions(&sessions, &heimdall).await.unwrap(); + assert_eq!(heimdall.calls().len(), 1, "a revoked session is not offered again"); + } + + #[tokio::test] + async fn a_heimdall_denial_revokes_the_session_and_it_is_not_offered_again() { + let (_directory, sessions, cookie) = due_session(1); + let heimdall = ScriptedHeimdall::answering(vec![Reply::Deny, Reply::Timeout]); + refresh_due_sessions(&sessions, &heimdall).await.unwrap(); + assert!(!is_live(&sessions, &cookie).await); + refresh_due_sessions(&sessions, &heimdall).await.unwrap(); + assert_eq!(heimdall.calls().len(), 1); + } + + #[tokio::test] + async fn a_local_refusal_logs_the_clause_that_fired() { + let (_directory, sessions, _cookie) = due_session(2); + let heimdall = ScriptedHeimdall::answering(vec![Reply::Receipt { + revision: 1, + claim: "claim-1", + }]); + let captured = Captured::default(); + let subscriber = tracing_subscriber::fmt() + .with_writer(captured.clone()) + .with_ansi(false) + .finish(); + let _guard = tracing::subscriber::set_default(subscriber); + refresh_due_sessions(&sessions, &heimdall).await.unwrap(); + let logged = String::from_utf8(captured.0.lock().unwrap().clone()).unwrap(); + assert!( + logged.contains("refresh receipt revision went backwards"), + "{logged}" + ); + } +}