Refresh: keep a receipt whose keys were unavailable, and pin the loop and the branch rules

The refresh pass keeps a spent-upstream receipt when verification could not
fetch the signing keys and verifies it on the next pass instead of refreshing
again; only a positive verification failure revokes. The loop body is
run_refresh_loop, tested for calling the pass; tests pin that a moved session
is not revoked and a higher revision is applied.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Meta Vens 2026-10-09 17:21:23 +02:00
commit 72a98d8468

View file

@ -3,7 +3,7 @@
use crate::{
app_session::{AppSessionOwner, RefreshRefusal},
eve::{self, EveCommandInvocation},
heimdall::{self, HeimdallClient, HeimdallDenied, SessionRefresher},
heimdall::{self, HeimdallClient, HeimdallDenied, SessionRefresher, VerificationUnavailable},
idunn_health::{
IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, ProcessWriteLeaseGuard, RuntimePresencePublisher,
TARGET as GHOSTLIGHT_TARGET,
@ -41,7 +41,7 @@ use cultnet_rs::{
use serde::Deserialize;
use serde_json::{Value, json};
use std::{
collections::BTreeMap,
collections::{BTreeMap, HashMap},
convert::Infallible,
fs,
net::SocketAddr,
@ -1863,45 +1863,86 @@ fn session_cookie(raw: &str, expires_at: DateTime<Utc>, now: DateTime<Utc>) -> H
}
async fn maintain_session_refresh(state: AppState) -> anyhow::Result<()> {
let mut interval = tokio::time::interval(Duration::from_secs(60));
run_refresh_loop(
&state.sessions,
state.heimdall.as_ref(),
Duration::from_secs(60),
)
.await
}
/// Runs `refresh_due_sessions` once per `period` for as long as the daemon
/// serves. The receipts it holds are Heimdall refreshes already spent
/// upstream whose verification could not yet be judged.
async fn run_refresh_loop(
sessions: &Mutex<AppSessionOwner>,
heimdall: &impl SessionRefresher,
period: Duration,
) -> anyhow::Result<()> {
let mut interval = tokio::time::interval(period);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
let mut unverified = HashMap::new();
loop {
interval.tick().await;
refresh_due_sessions(&state.sessions, state.heimdall.as_ref()).await?;
refresh_due_sessions(sessions, heimdall, &mut unverified).await?;
}
}
/// One pass of the refresh loop over every session due for refresh.
/// `unverified` holds, per session, a receipt Heimdall issued for a refresh
/// that is already spent upstream but whose signing keys could not be
/// fetched: the next pass verifies it instead of refreshing again.
async fn refresh_due_sessions(
sessions: &Mutex<AppSessionOwner>,
heimdall: &impl SessionRefresher,
unverified: &mut HashMap<String, heimdall::AuthCompletionReceipt>,
) -> anyhow::Result<()> {
let candidates = sessions
.lock()
.await
.sessions_due_for_refresh(Utc::now(), chrono::Duration::minutes(2))
.context("app-session refresh scan lost custody")?;
unverified.retain(|cookie_hash, _| {
candidates
.iter()
.any(|candidate| &candidate.cookie_hash == cookie_hash)
});
for candidate in candidates {
let idempotency = crate::app_session::refresh_idempotency_key(&candidate.refresh_claim);
let completion = match heimdall.refresh(&candidate.refresh_claim, &idempotency).await {
Ok(value) => value,
Err(error) if error.downcast_ref::<HeimdallDenied>().is_some() => {
// Heimdall refused this refresh claim: it can never succeed.
tracing::warn!(%error, "Heimdall denied the refresh; revoking the local session");
let mut sessions = sessions.lock().await;
if let Err(revoke_error) = sessions.revoke_cookie_hash(&candidate.cookie_hash) {
return Err(revoke_error)
.context("denied Heimdall refresh could not revoke local custody");
let completion = match unverified.remove(&candidate.cookie_hash) {
Some(completion) => completion,
None => {
let idempotency =
crate::app_session::refresh_idempotency_key(&candidate.refresh_claim);
match heimdall.refresh(&candidate.refresh_claim, &idempotency).await {
Ok(value) => value,
Err(error) if error.downcast_ref::<HeimdallDenied>().is_some() => {
// Heimdall refused this refresh claim: it can never succeed.
tracing::warn!(%error, "Heimdall denied the refresh; revoking the local session");
let mut sessions = sessions.lock().await;
if let Err(revoke_error) =
sessions.revoke_cookie_hash(&candidate.cookie_hash)
{
return Err(revoke_error)
.context("denied Heimdall refresh could not revoke local custody");
}
continue;
}
Err(error) => {
tracing::warn!(%error, "Heimdall refresh transport unavailable");
continue;
}
}
continue;
}
Err(error) => {
tracing::warn!(%error, "Heimdall refresh transport unavailable");
continue;
}
};
let verified = match heimdall.verify_refresh(completion).await {
let verified = match heimdall.verify_refresh(completion.clone()).await {
Ok(value) => value,
Err(error) if error.downcast_ref::<VerificationUnavailable>().is_some() => {
// The claim is spent upstream and the receipt may be genuine:
// keep it and verify again on the next pass.
tracing::warn!(%error, "Heimdall refresh receipt could not be verified yet");
unverified.insert(candidate.cookie_hash.clone(), completion);
continue;
}
Err(error) => {
tracing::warn!(%error, "Heimdall refresh receipt was invalid");
let mut sessions = sessions.lock().await;
@ -6396,7 +6437,7 @@ mod tests {
#[cfg(test)]
mod refresh_loop_tests {
use super::*;
use crate::heimdall::{AuthCompletionReceipt, VerifiedSessionRefresh};
use crate::heimdall::{AuthCompletionReceipt, VerificationUnavailable, VerifiedSessionRefresh};
use std::{collections::VecDeque, sync::Mutex as StdMutex};
enum Reply {
@ -6409,9 +6450,22 @@ mod refresh_loop_tests {
OtherAccount,
}
/// What `verify_refresh` answers; an empty queue accepts the receipt.
enum Verdict {
/// The signing keys could not be fetched.
Unavailable,
/// The receipt positively fails verification.
Invalid,
}
#[derive(Default)]
struct ScriptedHeimdall {
replies: StdMutex<VecDeque<Reply>>,
verdicts: StdMutex<VecDeque<Verdict>>,
verifications: StdMutex<usize>,
/// While answering a refresh, apply a newer revision to this store,
/// as a concurrent refresh of the same session would.
moving: Option<std::sync::Arc<Mutex<AppSessionOwner>>>,
pending: StdMutex<Option<VerifiedSessionRefresh>>,
calls: StdMutex<Vec<(String, String)>>,
}
@ -6424,6 +6478,15 @@ mod refresh_loop_tests {
}
}
fn verifying(mut self, verdicts: Vec<Verdict>) -> Self {
self.verdicts = StdMutex::new(verdicts.into());
self
}
fn verifications(&self) -> usize {
*self.verifications.lock().unwrap()
}
fn calls(&self) -> Vec<(String, String)> {
self.calls.lock().unwrap().clone()
}
@ -6440,6 +6503,27 @@ mod refresh_loop_tests {
.lock()
.unwrap()
.push((refresh_token.into(), idempotency_key.into()));
if let Some(sessions) = &self.moving {
let mut sessions = sessions.lock().await;
let candidate = sessions
.sessions_due_for_refresh(Utc::now(), chrono::Duration::minutes(2))
.unwrap()
.remove(0);
sessions
.apply_refresh(
&candidate.cookie_hash,
candidate.access_revision,
VerifiedSessionRefresh::fixture(
"operator-account",
"heimdall-session",
candidate.access_revision + 1,
Utc::now() + chrono::Duration::seconds(90),
Utc::now() + chrono::Duration::days(1),
"claim-moved",
),
)
.unwrap();
}
match self.replies.lock().unwrap().pop_front().expect("scripted reply") {
Reply::Deny => Err(HeimdallDenied {
diagnostics: "Idempotency key was reused".into(),
@ -6482,13 +6566,22 @@ mod refresh_loop_tests {
&self,
_completion: AuthCompletionReceipt,
) -> anyhow::Result<VerifiedSessionRefresh> {
Ok(self.pending.lock().unwrap().take().expect("scripted receipt"))
*self.verifications.lock().unwrap() += 1;
match self.verdicts.lock().unwrap().pop_front() {
Some(Verdict::Unavailable) => {
Err(VerificationUnavailable("jwks fetch timed out".into()).into())
}
Some(Verdict::Invalid) => bail!("Heimdall access token signature is invalid"),
None => Ok(self.pending.lock().unwrap().take().expect("scripted receipt")),
}
}
}
/// A store holding one live session at `revision` whose access expires in
/// a minute, so every pass finds it due.
fn due_session(revision: u64) -> (tempfile::TempDir, Mutex<AppSessionOwner>, String) {
fn due_session(
revision: u64,
) -> (tempfile::TempDir, std::sync::Arc<Mutex<AppSessionOwner>>, String) {
let directory = tempfile::tempdir().unwrap();
let key = directory.path().join("session.key");
std::fs::write(&key, [23_u8; 32]).unwrap();
@ -6503,7 +6596,14 @@ mod refresh_loop_tests {
"claim-0",
))
.unwrap();
(directory, Mutex::new(owner), cookie)
(directory, std::sync::Arc::new(Mutex::new(owner)), cookie)
}
/// One refresh pass with no receipt carried over from an earlier one.
async fn pass(sessions: &Mutex<AppSessionOwner>, heimdall: &ScriptedHeimdall) {
refresh_due_sessions(sessions, heimdall, &mut HashMap::new())
.await
.unwrap();
}
async fn is_live(sessions: &Mutex<AppSessionOwner>, cookie: &str) -> bool {
@ -6544,10 +6644,10 @@ mod refresh_loop_tests {
Reply::Receipt { revision: 1, claim: "claim-1" },
Reply::Timeout,
]);
refresh_due_sessions(&sessions, &heimdall).await.unwrap();
pass(&sessions, &heimdall).await;
assert!(is_live(&sessions, &cookie).await);
// The wrapped claim is the receipt's: the next refresh spends it.
refresh_due_sessions(&sessions, &heimdall).await.unwrap();
pass(&sessions, &heimdall).await;
assert_eq!(heimdall.calls()[1].0, "claim-1");
assert!(is_live(&sessions, &cookie).await);
}
@ -6556,8 +6656,8 @@ mod refresh_loop_tests {
async fn a_retry_of_the_same_refresh_sends_the_same_key() {
let (_directory, sessions, cookie) = due_session(1);
let heimdall = ScriptedHeimdall::answering(vec![Reply::Timeout, Reply::Timeout]);
refresh_due_sessions(&sessions, &heimdall).await.unwrap();
refresh_due_sessions(&sessions, &heimdall).await.unwrap();
pass(&sessions, &heimdall).await;
pass(&sessions, &heimdall).await;
let calls = heimdall.calls();
assert_eq!(calls.len(), 2, "a transport failure leaves the session offered");
assert_eq!(calls[0], calls[1]);
@ -6571,8 +6671,8 @@ mod refresh_loop_tests {
Reply::Receipt { revision: 1, claim: "claim-1" },
Reply::Timeout,
]);
refresh_due_sessions(&sessions, &heimdall).await.unwrap();
refresh_due_sessions(&sessions, &heimdall).await.unwrap();
pass(&sessions, &heimdall).await;
pass(&sessions, &heimdall).await;
let calls = heimdall.calls();
assert_ne!(calls[0].1, calls[1].1);
assert!(calls.iter().all(|(claim, key)| !key.contains(claim)));
@ -6585,9 +6685,9 @@ mod refresh_loop_tests {
Reply::Receipt { revision: 1, claim: "claim-1" },
Reply::Timeout,
]);
refresh_due_sessions(&sessions, &heimdall).await.unwrap();
pass(&sessions, &heimdall).await;
assert!(!is_live(&sessions, &cookie).await);
refresh_due_sessions(&sessions, &heimdall).await.unwrap();
pass(&sessions, &heimdall).await;
assert_eq!(heimdall.calls().len(), 1, "a revoked session is not offered again");
}
@ -6596,7 +6696,7 @@ mod refresh_loop_tests {
for reply in [Reply::OtherSession, Reply::OtherAccount] {
let (_directory, sessions, cookie) = due_session(1);
let heimdall = ScriptedHeimdall::answering(vec![reply]);
refresh_due_sessions(&sessions, &heimdall).await.unwrap();
pass(&sessions, &heimdall).await;
assert!(!is_live(&sessions, &cookie).await);
}
}
@ -6605,9 +6705,9 @@ mod refresh_loop_tests {
async fn a_heimdall_denial_revokes_the_session_and_it_is_not_offered_again() {
let (_directory, sessions, cookie) = due_session(1);
let heimdall = ScriptedHeimdall::answering(vec![Reply::Deny, Reply::Timeout]);
refresh_due_sessions(&sessions, &heimdall).await.unwrap();
pass(&sessions, &heimdall).await;
assert!(!is_live(&sessions, &cookie).await);
refresh_due_sessions(&sessions, &heimdall).await.unwrap();
pass(&sessions, &heimdall).await;
assert_eq!(heimdall.calls().len(), 1);
}
@ -6624,11 +6724,111 @@ mod refresh_loop_tests {
.with_ansi(false)
.finish();
let _guard = tracing::subscriber::set_default(subscriber);
refresh_due_sessions(&sessions, &heimdall).await.unwrap();
pass(&sessions, &heimdall).await;
let logged = String::from_utf8(captured.0.lock().unwrap().clone()).unwrap();
assert!(
logged.contains("refresh receipt revision went backwards"),
"{logged}"
);
}
#[tokio::test]
async fn a_receipt_whose_keys_were_unavailable_is_kept_and_applied_on_the_next_pass() {
let (_directory, sessions, cookie) = due_session(1);
let heimdall = ScriptedHeimdall::answering(vec![
Reply::Receipt { revision: 1, claim: "claim-1" },
Reply::Timeout,
])
.verifying(vec![Verdict::Unavailable]);
let mut unverified = HashMap::new();
refresh_due_sessions(&sessions, &heimdall, &mut unverified).await.unwrap();
assert!(is_live(&sessions, &cookie).await, "unavailable keys do not revoke");
assert_eq!(unverified.len(), 1);
refresh_due_sessions(&sessions, &heimdall, &mut unverified).await.unwrap();
assert_eq!(heimdall.calls().len(), 1, "the spent claim is not sent again");
assert_eq!(heimdall.verifications(), 2);
assert!(unverified.is_empty(), "an applied receipt is not kept");
assert!(is_live(&sessions, &cookie).await);
// The applied receipt's claim is the one the next refresh spends.
refresh_due_sessions(&sessions, &heimdall, &mut unverified).await.unwrap();
assert_eq!(heimdall.calls()[1].0, "claim-1");
assert_eq!(heimdall.verifications(), 2, "the receipt was applied once");
}
#[tokio::test]
async fn a_receipt_that_positively_fails_verification_revokes_and_is_not_kept() {
let (_directory, sessions, cookie) = due_session(1);
let heimdall = ScriptedHeimdall::answering(vec![
Reply::Receipt { revision: 1, claim: "claim-1" },
Reply::Timeout,
])
.verifying(vec![Verdict::Invalid]);
let mut unverified = HashMap::new();
refresh_due_sessions(&sessions, &heimdall, &mut unverified).await.unwrap();
assert!(!is_live(&sessions, &cookie).await);
assert!(unverified.is_empty());
refresh_due_sessions(&sessions, &heimdall, &mut unverified).await.unwrap();
assert_eq!(heimdall.calls().len(), 1);
assert_eq!(heimdall.verifications(), 1);
}
#[tokio::test]
async fn a_kept_receipt_for_a_session_that_is_gone_is_dropped() {
let (_directory, sessions, cookie) = due_session(1);
let heimdall = ScriptedHeimdall::answering(vec![Reply::Receipt {
revision: 1,
claim: "claim-1",
}])
.verifying(vec![Verdict::Unavailable]);
let mut unverified = HashMap::new();
refresh_due_sessions(&sessions, &heimdall, &mut unverified).await.unwrap();
assert_eq!(unverified.len(), 1);
let cookie_hash = unverified.keys().next().unwrap().clone();
sessions.lock().await.revoke_cookie_hash(&cookie_hash).unwrap();
refresh_due_sessions(&sessions, &heimdall, &mut unverified).await.unwrap();
assert!(unverified.is_empty());
assert!(!is_live(&sessions, &cookie).await);
assert_eq!(heimdall.verifications(), 1);
}
#[tokio::test]
async fn a_session_that_moved_while_refreshing_is_not_revoked() {
let (_directory, sessions, cookie) = due_session(1);
let mut heimdall = ScriptedHeimdall::answering(vec![Reply::Receipt {
revision: 1,
claim: "claim-1",
}]);
heimdall.moving = Some(sessions.clone());
pass(&sessions, &heimdall).await;
assert!(is_live(&sessions, &cookie).await);
}
#[tokio::test]
async fn a_receipt_with_a_higher_revision_is_applied() {
let (_directory, sessions, cookie) = due_session(1);
let heimdall = ScriptedHeimdall::answering(vec![
Reply::Receipt { revision: 2, claim: "claim-1" },
Reply::Receipt { revision: 1, claim: "claim-2" },
]);
pass(&sessions, &heimdall).await;
assert!(is_live(&sessions, &cookie).await);
// The session now sits at revision 2, so a revision-1 receipt is
// backwards.
pass(&sessions, &heimdall).await;
assert_eq!(heimdall.calls()[1].0, "claim-1");
assert!(!is_live(&sessions, &cookie).await);
}
#[tokio::test]
async fn the_refresh_loop_runs_a_pass_on_every_tick() {
let (_directory, sessions, _cookie) = due_session(1);
let heimdall = ScriptedHeimdall::answering((0..200).map(|_| Reply::Timeout).collect());
let ran = tokio::time::timeout(
Duration::from_millis(400),
run_refresh_loop(&sessions, &heimdall, Duration::from_millis(10)),
)
.await;
assert!(ran.is_err(), "the loop never returns");
assert!(heimdall.calls().len() >= 2, "{:?}", heimdall.calls());
}
}