From 72a98d8468d20dcd6cd3d8b0baea11978b458966 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 9 Oct 2026 17:21:23 +0200 Subject: [PATCH] Refresh: keep a receipt whose keys were unavailable, and pin the loop and the branch rules The refresh pass keeps a spent-upstream receipt when verification could not fetch the signing keys and verifies it on the next pass instead of refreshing again; only a positive verification failure revokes. The loop body is run_refresh_loop, tested for calling the pass; tests pin that a moved session is not revoked and a higher revision is applied. Co-Authored-By: Claude Opus 5.5 --- crates/ghostlight-dungeon/src/runtime.rs | 272 ++++++++++++++++++++--- 1 file changed, 236 insertions(+), 36 deletions(-) diff --git a/crates/ghostlight-dungeon/src/runtime.rs b/crates/ghostlight-dungeon/src/runtime.rs index 6de4527..9d1e6c7 100644 --- a/crates/ghostlight-dungeon/src/runtime.rs +++ b/crates/ghostlight-dungeon/src/runtime.rs @@ -3,7 +3,7 @@ use crate::{ app_session::{AppSessionOwner, RefreshRefusal}, eve::{self, EveCommandInvocation}, - heimdall::{self, HeimdallClient, HeimdallDenied, SessionRefresher}, + heimdall::{self, HeimdallClient, HeimdallDenied, SessionRefresher, VerificationUnavailable}, idunn_health::{ IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, ProcessWriteLeaseGuard, RuntimePresencePublisher, TARGET as GHOSTLIGHT_TARGET, @@ -41,7 +41,7 @@ use cultnet_rs::{ use serde::Deserialize; use serde_json::{Value, json}; use std::{ - collections::BTreeMap, + collections::{BTreeMap, HashMap}, convert::Infallible, fs, net::SocketAddr, @@ -1863,45 +1863,86 @@ fn session_cookie(raw: &str, expires_at: DateTime, now: DateTime) -> H } async fn maintain_session_refresh(state: AppState) -> anyhow::Result<()> { - let mut interval = tokio::time::interval(Duration::from_secs(60)); + run_refresh_loop( + &state.sessions, + state.heimdall.as_ref(), + Duration::from_secs(60), + ) + .await +} + +/// Runs `refresh_due_sessions` once per `period` for as long as the daemon +/// serves. The receipts it holds are Heimdall refreshes already spent +/// upstream whose verification could not yet be judged. +async fn run_refresh_loop( + sessions: &Mutex, + heimdall: &impl SessionRefresher, + period: Duration, +) -> anyhow::Result<()> { + let mut interval = tokio::time::interval(period); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut unverified = HashMap::new(); loop { interval.tick().await; - refresh_due_sessions(&state.sessions, state.heimdall.as_ref()).await?; + refresh_due_sessions(sessions, heimdall, &mut unverified).await?; } } /// One pass of the refresh loop over every session due for refresh. +/// `unverified` holds, per session, a receipt Heimdall issued for a refresh +/// that is already spent upstream but whose signing keys could not be +/// fetched: the next pass verifies it instead of refreshing again. async fn refresh_due_sessions( sessions: &Mutex, heimdall: &impl SessionRefresher, + unverified: &mut HashMap, ) -> anyhow::Result<()> { let candidates = sessions .lock() .await .sessions_due_for_refresh(Utc::now(), chrono::Duration::minutes(2)) .context("app-session refresh scan lost custody")?; + unverified.retain(|cookie_hash, _| { + candidates + .iter() + .any(|candidate| &candidate.cookie_hash == cookie_hash) + }); for candidate in candidates { - let idempotency = crate::app_session::refresh_idempotency_key(&candidate.refresh_claim); - let completion = match heimdall.refresh(&candidate.refresh_claim, &idempotency).await { - Ok(value) => value, - Err(error) if error.downcast_ref::().is_some() => { - // Heimdall refused this refresh claim: it can never succeed. - tracing::warn!(%error, "Heimdall denied the refresh; revoking the local session"); - let mut sessions = sessions.lock().await; - if let Err(revoke_error) = sessions.revoke_cookie_hash(&candidate.cookie_hash) { - return Err(revoke_error) - .context("denied Heimdall refresh could not revoke local custody"); + let completion = match unverified.remove(&candidate.cookie_hash) { + Some(completion) => completion, + None => { + let idempotency = + crate::app_session::refresh_idempotency_key(&candidate.refresh_claim); + match heimdall.refresh(&candidate.refresh_claim, &idempotency).await { + Ok(value) => value, + Err(error) if error.downcast_ref::().is_some() => { + // Heimdall refused this refresh claim: it can never succeed. + tracing::warn!(%error, "Heimdall denied the refresh; revoking the local session"); + let mut sessions = sessions.lock().await; + if let Err(revoke_error) = + sessions.revoke_cookie_hash(&candidate.cookie_hash) + { + return Err(revoke_error) + .context("denied Heimdall refresh could not revoke local custody"); + } + continue; + } + Err(error) => { + tracing::warn!(%error, "Heimdall refresh transport unavailable"); + continue; + } } - continue; - } - Err(error) => { - tracing::warn!(%error, "Heimdall refresh transport unavailable"); - continue; } }; - let verified = match heimdall.verify_refresh(completion).await { + let verified = match heimdall.verify_refresh(completion.clone()).await { Ok(value) => value, + Err(error) if error.downcast_ref::().is_some() => { + // The claim is spent upstream and the receipt may be genuine: + // keep it and verify again on the next pass. + tracing::warn!(%error, "Heimdall refresh receipt could not be verified yet"); + unverified.insert(candidate.cookie_hash.clone(), completion); + continue; + } Err(error) => { tracing::warn!(%error, "Heimdall refresh receipt was invalid"); let mut sessions = sessions.lock().await; @@ -6396,7 +6437,7 @@ mod tests { #[cfg(test)] mod refresh_loop_tests { use super::*; - use crate::heimdall::{AuthCompletionReceipt, VerifiedSessionRefresh}; + use crate::heimdall::{AuthCompletionReceipt, VerificationUnavailable, VerifiedSessionRefresh}; use std::{collections::VecDeque, sync::Mutex as StdMutex}; enum Reply { @@ -6409,9 +6450,22 @@ mod refresh_loop_tests { OtherAccount, } + /// What `verify_refresh` answers; an empty queue accepts the receipt. + enum Verdict { + /// The signing keys could not be fetched. + Unavailable, + /// The receipt positively fails verification. + Invalid, + } + #[derive(Default)] struct ScriptedHeimdall { replies: StdMutex>, + verdicts: StdMutex>, + verifications: StdMutex, + /// While answering a refresh, apply a newer revision to this store, + /// as a concurrent refresh of the same session would. + moving: Option>>, pending: StdMutex>, calls: StdMutex>, } @@ -6424,6 +6478,15 @@ mod refresh_loop_tests { } } + fn verifying(mut self, verdicts: Vec) -> Self { + self.verdicts = StdMutex::new(verdicts.into()); + self + } + + fn verifications(&self) -> usize { + *self.verifications.lock().unwrap() + } + fn calls(&self) -> Vec<(String, String)> { self.calls.lock().unwrap().clone() } @@ -6440,6 +6503,27 @@ mod refresh_loop_tests { .lock() .unwrap() .push((refresh_token.into(), idempotency_key.into())); + if let Some(sessions) = &self.moving { + let mut sessions = sessions.lock().await; + let candidate = sessions + .sessions_due_for_refresh(Utc::now(), chrono::Duration::minutes(2)) + .unwrap() + .remove(0); + sessions + .apply_refresh( + &candidate.cookie_hash, + candidate.access_revision, + VerifiedSessionRefresh::fixture( + "operator-account", + "heimdall-session", + candidate.access_revision + 1, + Utc::now() + chrono::Duration::seconds(90), + Utc::now() + chrono::Duration::days(1), + "claim-moved", + ), + ) + .unwrap(); + } match self.replies.lock().unwrap().pop_front().expect("scripted reply") { Reply::Deny => Err(HeimdallDenied { diagnostics: "Idempotency key was reused".into(), @@ -6482,13 +6566,22 @@ mod refresh_loop_tests { &self, _completion: AuthCompletionReceipt, ) -> anyhow::Result { - Ok(self.pending.lock().unwrap().take().expect("scripted receipt")) + *self.verifications.lock().unwrap() += 1; + match self.verdicts.lock().unwrap().pop_front() { + Some(Verdict::Unavailable) => { + Err(VerificationUnavailable("jwks fetch timed out".into()).into()) + } + Some(Verdict::Invalid) => bail!("Heimdall access token signature is invalid"), + None => Ok(self.pending.lock().unwrap().take().expect("scripted receipt")), + } } } /// A store holding one live session at `revision` whose access expires in /// a minute, so every pass finds it due. - fn due_session(revision: u64) -> (tempfile::TempDir, Mutex, String) { + fn due_session( + revision: u64, + ) -> (tempfile::TempDir, std::sync::Arc>, String) { let directory = tempfile::tempdir().unwrap(); let key = directory.path().join("session.key"); std::fs::write(&key, [23_u8; 32]).unwrap(); @@ -6503,7 +6596,14 @@ mod refresh_loop_tests { "claim-0", )) .unwrap(); - (directory, Mutex::new(owner), cookie) + (directory, std::sync::Arc::new(Mutex::new(owner)), cookie) + } + + /// One refresh pass with no receipt carried over from an earlier one. + async fn pass(sessions: &Mutex, heimdall: &ScriptedHeimdall) { + refresh_due_sessions(sessions, heimdall, &mut HashMap::new()) + .await + .unwrap(); } async fn is_live(sessions: &Mutex, cookie: &str) -> bool { @@ -6544,10 +6644,10 @@ mod refresh_loop_tests { Reply::Receipt { revision: 1, claim: "claim-1" }, Reply::Timeout, ]); - refresh_due_sessions(&sessions, &heimdall).await.unwrap(); + pass(&sessions, &heimdall).await; assert!(is_live(&sessions, &cookie).await); // The wrapped claim is the receipt's: the next refresh spends it. - refresh_due_sessions(&sessions, &heimdall).await.unwrap(); + pass(&sessions, &heimdall).await; assert_eq!(heimdall.calls()[1].0, "claim-1"); assert!(is_live(&sessions, &cookie).await); } @@ -6556,8 +6656,8 @@ mod refresh_loop_tests { async fn a_retry_of_the_same_refresh_sends_the_same_key() { let (_directory, sessions, cookie) = due_session(1); let heimdall = ScriptedHeimdall::answering(vec![Reply::Timeout, Reply::Timeout]); - refresh_due_sessions(&sessions, &heimdall).await.unwrap(); - refresh_due_sessions(&sessions, &heimdall).await.unwrap(); + pass(&sessions, &heimdall).await; + pass(&sessions, &heimdall).await; let calls = heimdall.calls(); assert_eq!(calls.len(), 2, "a transport failure leaves the session offered"); assert_eq!(calls[0], calls[1]); @@ -6571,8 +6671,8 @@ mod refresh_loop_tests { Reply::Receipt { revision: 1, claim: "claim-1" }, Reply::Timeout, ]); - refresh_due_sessions(&sessions, &heimdall).await.unwrap(); - refresh_due_sessions(&sessions, &heimdall).await.unwrap(); + pass(&sessions, &heimdall).await; + pass(&sessions, &heimdall).await; let calls = heimdall.calls(); assert_ne!(calls[0].1, calls[1].1); assert!(calls.iter().all(|(claim, key)| !key.contains(claim))); @@ -6585,9 +6685,9 @@ mod refresh_loop_tests { Reply::Receipt { revision: 1, claim: "claim-1" }, Reply::Timeout, ]); - refresh_due_sessions(&sessions, &heimdall).await.unwrap(); + pass(&sessions, &heimdall).await; assert!(!is_live(&sessions, &cookie).await); - refresh_due_sessions(&sessions, &heimdall).await.unwrap(); + pass(&sessions, &heimdall).await; assert_eq!(heimdall.calls().len(), 1, "a revoked session is not offered again"); } @@ -6596,7 +6696,7 @@ mod refresh_loop_tests { for reply in [Reply::OtherSession, Reply::OtherAccount] { let (_directory, sessions, cookie) = due_session(1); let heimdall = ScriptedHeimdall::answering(vec![reply]); - refresh_due_sessions(&sessions, &heimdall).await.unwrap(); + pass(&sessions, &heimdall).await; assert!(!is_live(&sessions, &cookie).await); } } @@ -6605,9 +6705,9 @@ mod refresh_loop_tests { async fn a_heimdall_denial_revokes_the_session_and_it_is_not_offered_again() { let (_directory, sessions, cookie) = due_session(1); let heimdall = ScriptedHeimdall::answering(vec![Reply::Deny, Reply::Timeout]); - refresh_due_sessions(&sessions, &heimdall).await.unwrap(); + pass(&sessions, &heimdall).await; assert!(!is_live(&sessions, &cookie).await); - refresh_due_sessions(&sessions, &heimdall).await.unwrap(); + pass(&sessions, &heimdall).await; assert_eq!(heimdall.calls().len(), 1); } @@ -6624,11 +6724,111 @@ mod refresh_loop_tests { .with_ansi(false) .finish(); let _guard = tracing::subscriber::set_default(subscriber); - refresh_due_sessions(&sessions, &heimdall).await.unwrap(); + pass(&sessions, &heimdall).await; let logged = String::from_utf8(captured.0.lock().unwrap().clone()).unwrap(); assert!( logged.contains("refresh receipt revision went backwards"), "{logged}" ); } + + #[tokio::test] + async fn a_receipt_whose_keys_were_unavailable_is_kept_and_applied_on_the_next_pass() { + let (_directory, sessions, cookie) = due_session(1); + let heimdall = ScriptedHeimdall::answering(vec![ + Reply::Receipt { revision: 1, claim: "claim-1" }, + Reply::Timeout, + ]) + .verifying(vec![Verdict::Unavailable]); + let mut unverified = HashMap::new(); + refresh_due_sessions(&sessions, &heimdall, &mut unverified).await.unwrap(); + assert!(is_live(&sessions, &cookie).await, "unavailable keys do not revoke"); + assert_eq!(unverified.len(), 1); + refresh_due_sessions(&sessions, &heimdall, &mut unverified).await.unwrap(); + assert_eq!(heimdall.calls().len(), 1, "the spent claim is not sent again"); + assert_eq!(heimdall.verifications(), 2); + assert!(unverified.is_empty(), "an applied receipt is not kept"); + assert!(is_live(&sessions, &cookie).await); + // The applied receipt's claim is the one the next refresh spends. + refresh_due_sessions(&sessions, &heimdall, &mut unverified).await.unwrap(); + assert_eq!(heimdall.calls()[1].0, "claim-1"); + assert_eq!(heimdall.verifications(), 2, "the receipt was applied once"); + } + + #[tokio::test] + async fn a_receipt_that_positively_fails_verification_revokes_and_is_not_kept() { + let (_directory, sessions, cookie) = due_session(1); + let heimdall = ScriptedHeimdall::answering(vec![ + Reply::Receipt { revision: 1, claim: "claim-1" }, + Reply::Timeout, + ]) + .verifying(vec![Verdict::Invalid]); + let mut unverified = HashMap::new(); + refresh_due_sessions(&sessions, &heimdall, &mut unverified).await.unwrap(); + assert!(!is_live(&sessions, &cookie).await); + assert!(unverified.is_empty()); + refresh_due_sessions(&sessions, &heimdall, &mut unverified).await.unwrap(); + assert_eq!(heimdall.calls().len(), 1); + assert_eq!(heimdall.verifications(), 1); + } + + #[tokio::test] + async fn a_kept_receipt_for_a_session_that_is_gone_is_dropped() { + let (_directory, sessions, cookie) = due_session(1); + let heimdall = ScriptedHeimdall::answering(vec![Reply::Receipt { + revision: 1, + claim: "claim-1", + }]) + .verifying(vec![Verdict::Unavailable]); + let mut unverified = HashMap::new(); + refresh_due_sessions(&sessions, &heimdall, &mut unverified).await.unwrap(); + assert_eq!(unverified.len(), 1); + let cookie_hash = unverified.keys().next().unwrap().clone(); + sessions.lock().await.revoke_cookie_hash(&cookie_hash).unwrap(); + refresh_due_sessions(&sessions, &heimdall, &mut unverified).await.unwrap(); + assert!(unverified.is_empty()); + assert!(!is_live(&sessions, &cookie).await); + assert_eq!(heimdall.verifications(), 1); + } + + #[tokio::test] + async fn a_session_that_moved_while_refreshing_is_not_revoked() { + let (_directory, sessions, cookie) = due_session(1); + let mut heimdall = ScriptedHeimdall::answering(vec![Reply::Receipt { + revision: 1, + claim: "claim-1", + }]); + heimdall.moving = Some(sessions.clone()); + pass(&sessions, &heimdall).await; + assert!(is_live(&sessions, &cookie).await); + } + + #[tokio::test] + async fn a_receipt_with_a_higher_revision_is_applied() { + let (_directory, sessions, cookie) = due_session(1); + let heimdall = ScriptedHeimdall::answering(vec![ + Reply::Receipt { revision: 2, claim: "claim-1" }, + Reply::Receipt { revision: 1, claim: "claim-2" }, + ]); + pass(&sessions, &heimdall).await; + assert!(is_live(&sessions, &cookie).await); + // The session now sits at revision 2, so a revision-1 receipt is + // backwards. + pass(&sessions, &heimdall).await; + assert_eq!(heimdall.calls()[1].0, "claim-1"); + assert!(!is_live(&sessions, &cookie).await); + } + + #[tokio::test] + async fn the_refresh_loop_runs_a_pass_on_every_tick() { + let (_directory, sessions, _cookie) = due_session(1); + let heimdall = ScriptedHeimdall::answering((0..200).map(|_| Reply::Timeout).collect()); + let ran = tokio::time::timeout( + Duration::from_millis(400), + run_refresh_loop(&sessions, &heimdall, Duration::from_millis(10)), + ) + .await; + assert!(ran.is_err(), "the loop never returns"); + assert!(heimdall.calls().len() >= 2, "{:?}", heimdall.calls()); + } }