Refresh: keep a receipt whose keys were unavailable, and pin the loop and the branch rules
The refresh pass keeps a spent-upstream receipt when verification could not fetch the signing keys and verifies it on the next pass instead of refreshing again; only a positive verification failure revokes. The loop body is run_refresh_loop, tested for calling the pass; tests pin that a moved session is not revoked and a higher revision is applied. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
2b6e8c7826
commit
72a98d8468
1 changed files with 236 additions and 36 deletions
|
|
@ -3,7 +3,7 @@
|
|||
use crate::{
|
||||
app_session::{AppSessionOwner, RefreshRefusal},
|
||||
eve::{self, EveCommandInvocation},
|
||||
heimdall::{self, HeimdallClient, HeimdallDenied, SessionRefresher},
|
||||
heimdall::{self, HeimdallClient, HeimdallDenied, SessionRefresher, VerificationUnavailable},
|
||||
idunn_health::{
|
||||
IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, ProcessWriteLeaseGuard, RuntimePresencePublisher,
|
||||
TARGET as GHOSTLIGHT_TARGET,
|
||||
|
|
@ -41,7 +41,7 @@ use cultnet_rs::{
|
|||
use serde::Deserialize;
|
||||
use serde_json::{Value, json};
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
collections::{BTreeMap, HashMap},
|
||||
convert::Infallible,
|
||||
fs,
|
||||
net::SocketAddr,
|
||||
|
|
@ -1863,45 +1863,86 @@ fn session_cookie(raw: &str, expires_at: DateTime<Utc>, now: DateTime<Utc>) -> H
|
|||
}
|
||||
|
||||
async fn maintain_session_refresh(state: AppState) -> anyhow::Result<()> {
|
||||
let mut interval = tokio::time::interval(Duration::from_secs(60));
|
||||
run_refresh_loop(
|
||||
&state.sessions,
|
||||
state.heimdall.as_ref(),
|
||||
Duration::from_secs(60),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Runs `refresh_due_sessions` once per `period` for as long as the daemon
|
||||
/// serves. The receipts it holds are Heimdall refreshes already spent
|
||||
/// upstream whose verification could not yet be judged.
|
||||
async fn run_refresh_loop(
|
||||
sessions: &Mutex<AppSessionOwner>,
|
||||
heimdall: &impl SessionRefresher,
|
||||
period: Duration,
|
||||
) -> anyhow::Result<()> {
|
||||
let mut interval = tokio::time::interval(period);
|
||||
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
let mut unverified = HashMap::new();
|
||||
loop {
|
||||
interval.tick().await;
|
||||
refresh_due_sessions(&state.sessions, state.heimdall.as_ref()).await?;
|
||||
refresh_due_sessions(sessions, heimdall, &mut unverified).await?;
|
||||
}
|
||||
}
|
||||
|
||||
/// One pass of the refresh loop over every session due for refresh.
|
||||
/// `unverified` holds, per session, a receipt Heimdall issued for a refresh
|
||||
/// that is already spent upstream but whose signing keys could not be
|
||||
/// fetched: the next pass verifies it instead of refreshing again.
|
||||
async fn refresh_due_sessions(
|
||||
sessions: &Mutex<AppSessionOwner>,
|
||||
heimdall: &impl SessionRefresher,
|
||||
unverified: &mut HashMap<String, heimdall::AuthCompletionReceipt>,
|
||||
) -> anyhow::Result<()> {
|
||||
let candidates = sessions
|
||||
.lock()
|
||||
.await
|
||||
.sessions_due_for_refresh(Utc::now(), chrono::Duration::minutes(2))
|
||||
.context("app-session refresh scan lost custody")?;
|
||||
unverified.retain(|cookie_hash, _| {
|
||||
candidates
|
||||
.iter()
|
||||
.any(|candidate| &candidate.cookie_hash == cookie_hash)
|
||||
});
|
||||
for candidate in candidates {
|
||||
let idempotency = crate::app_session::refresh_idempotency_key(&candidate.refresh_claim);
|
||||
let completion = match heimdall.refresh(&candidate.refresh_claim, &idempotency).await {
|
||||
Ok(value) => value,
|
||||
Err(error) if error.downcast_ref::<HeimdallDenied>().is_some() => {
|
||||
// Heimdall refused this refresh claim: it can never succeed.
|
||||
tracing::warn!(%error, "Heimdall denied the refresh; revoking the local session");
|
||||
let mut sessions = sessions.lock().await;
|
||||
if let Err(revoke_error) = sessions.revoke_cookie_hash(&candidate.cookie_hash) {
|
||||
return Err(revoke_error)
|
||||
.context("denied Heimdall refresh could not revoke local custody");
|
||||
let completion = match unverified.remove(&candidate.cookie_hash) {
|
||||
Some(completion) => completion,
|
||||
None => {
|
||||
let idempotency =
|
||||
crate::app_session::refresh_idempotency_key(&candidate.refresh_claim);
|
||||
match heimdall.refresh(&candidate.refresh_claim, &idempotency).await {
|
||||
Ok(value) => value,
|
||||
Err(error) if error.downcast_ref::<HeimdallDenied>().is_some() => {
|
||||
// Heimdall refused this refresh claim: it can never succeed.
|
||||
tracing::warn!(%error, "Heimdall denied the refresh; revoking the local session");
|
||||
let mut sessions = sessions.lock().await;
|
||||
if let Err(revoke_error) =
|
||||
sessions.revoke_cookie_hash(&candidate.cookie_hash)
|
||||
{
|
||||
return Err(revoke_error)
|
||||
.context("denied Heimdall refresh could not revoke local custody");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
Err(error) => {
|
||||
tracing::warn!(%error, "Heimdall refresh transport unavailable");
|
||||
continue;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
Err(error) => {
|
||||
tracing::warn!(%error, "Heimdall refresh transport unavailable");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let verified = match heimdall.verify_refresh(completion).await {
|
||||
let verified = match heimdall.verify_refresh(completion.clone()).await {
|
||||
Ok(value) => value,
|
||||
Err(error) if error.downcast_ref::<VerificationUnavailable>().is_some() => {
|
||||
// The claim is spent upstream and the receipt may be genuine:
|
||||
// keep it and verify again on the next pass.
|
||||
tracing::warn!(%error, "Heimdall refresh receipt could not be verified yet");
|
||||
unverified.insert(candidate.cookie_hash.clone(), completion);
|
||||
continue;
|
||||
}
|
||||
Err(error) => {
|
||||
tracing::warn!(%error, "Heimdall refresh receipt was invalid");
|
||||
let mut sessions = sessions.lock().await;
|
||||
|
|
@ -6396,7 +6437,7 @@ mod tests {
|
|||
#[cfg(test)]
|
||||
mod refresh_loop_tests {
|
||||
use super::*;
|
||||
use crate::heimdall::{AuthCompletionReceipt, VerifiedSessionRefresh};
|
||||
use crate::heimdall::{AuthCompletionReceipt, VerificationUnavailable, VerifiedSessionRefresh};
|
||||
use std::{collections::VecDeque, sync::Mutex as StdMutex};
|
||||
|
||||
enum Reply {
|
||||
|
|
@ -6409,9 +6450,22 @@ mod refresh_loop_tests {
|
|||
OtherAccount,
|
||||
}
|
||||
|
||||
/// What `verify_refresh` answers; an empty queue accepts the receipt.
|
||||
enum Verdict {
|
||||
/// The signing keys could not be fetched.
|
||||
Unavailable,
|
||||
/// The receipt positively fails verification.
|
||||
Invalid,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ScriptedHeimdall {
|
||||
replies: StdMutex<VecDeque<Reply>>,
|
||||
verdicts: StdMutex<VecDeque<Verdict>>,
|
||||
verifications: StdMutex<usize>,
|
||||
/// While answering a refresh, apply a newer revision to this store,
|
||||
/// as a concurrent refresh of the same session would.
|
||||
moving: Option<std::sync::Arc<Mutex<AppSessionOwner>>>,
|
||||
pending: StdMutex<Option<VerifiedSessionRefresh>>,
|
||||
calls: StdMutex<Vec<(String, String)>>,
|
||||
}
|
||||
|
|
@ -6424,6 +6478,15 @@ mod refresh_loop_tests {
|
|||
}
|
||||
}
|
||||
|
||||
fn verifying(mut self, verdicts: Vec<Verdict>) -> Self {
|
||||
self.verdicts = StdMutex::new(verdicts.into());
|
||||
self
|
||||
}
|
||||
|
||||
fn verifications(&self) -> usize {
|
||||
*self.verifications.lock().unwrap()
|
||||
}
|
||||
|
||||
fn calls(&self) -> Vec<(String, String)> {
|
||||
self.calls.lock().unwrap().clone()
|
||||
}
|
||||
|
|
@ -6440,6 +6503,27 @@ mod refresh_loop_tests {
|
|||
.lock()
|
||||
.unwrap()
|
||||
.push((refresh_token.into(), idempotency_key.into()));
|
||||
if let Some(sessions) = &self.moving {
|
||||
let mut sessions = sessions.lock().await;
|
||||
let candidate = sessions
|
||||
.sessions_due_for_refresh(Utc::now(), chrono::Duration::minutes(2))
|
||||
.unwrap()
|
||||
.remove(0);
|
||||
sessions
|
||||
.apply_refresh(
|
||||
&candidate.cookie_hash,
|
||||
candidate.access_revision,
|
||||
VerifiedSessionRefresh::fixture(
|
||||
"operator-account",
|
||||
"heimdall-session",
|
||||
candidate.access_revision + 1,
|
||||
Utc::now() + chrono::Duration::seconds(90),
|
||||
Utc::now() + chrono::Duration::days(1),
|
||||
"claim-moved",
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
match self.replies.lock().unwrap().pop_front().expect("scripted reply") {
|
||||
Reply::Deny => Err(HeimdallDenied {
|
||||
diagnostics: "Idempotency key was reused".into(),
|
||||
|
|
@ -6482,13 +6566,22 @@ mod refresh_loop_tests {
|
|||
&self,
|
||||
_completion: AuthCompletionReceipt,
|
||||
) -> anyhow::Result<VerifiedSessionRefresh> {
|
||||
Ok(self.pending.lock().unwrap().take().expect("scripted receipt"))
|
||||
*self.verifications.lock().unwrap() += 1;
|
||||
match self.verdicts.lock().unwrap().pop_front() {
|
||||
Some(Verdict::Unavailable) => {
|
||||
Err(VerificationUnavailable("jwks fetch timed out".into()).into())
|
||||
}
|
||||
Some(Verdict::Invalid) => bail!("Heimdall access token signature is invalid"),
|
||||
None => Ok(self.pending.lock().unwrap().take().expect("scripted receipt")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A store holding one live session at `revision` whose access expires in
|
||||
/// a minute, so every pass finds it due.
|
||||
fn due_session(revision: u64) -> (tempfile::TempDir, Mutex<AppSessionOwner>, String) {
|
||||
fn due_session(
|
||||
revision: u64,
|
||||
) -> (tempfile::TempDir, std::sync::Arc<Mutex<AppSessionOwner>>, String) {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let key = directory.path().join("session.key");
|
||||
std::fs::write(&key, [23_u8; 32]).unwrap();
|
||||
|
|
@ -6503,7 +6596,14 @@ mod refresh_loop_tests {
|
|||
"claim-0",
|
||||
))
|
||||
.unwrap();
|
||||
(directory, Mutex::new(owner), cookie)
|
||||
(directory, std::sync::Arc::new(Mutex::new(owner)), cookie)
|
||||
}
|
||||
|
||||
/// One refresh pass with no receipt carried over from an earlier one.
|
||||
async fn pass(sessions: &Mutex<AppSessionOwner>, heimdall: &ScriptedHeimdall) {
|
||||
refresh_due_sessions(sessions, heimdall, &mut HashMap::new())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn is_live(sessions: &Mutex<AppSessionOwner>, cookie: &str) -> bool {
|
||||
|
|
@ -6544,10 +6644,10 @@ mod refresh_loop_tests {
|
|||
Reply::Receipt { revision: 1, claim: "claim-1" },
|
||||
Reply::Timeout,
|
||||
]);
|
||||
refresh_due_sessions(&sessions, &heimdall).await.unwrap();
|
||||
pass(&sessions, &heimdall).await;
|
||||
assert!(is_live(&sessions, &cookie).await);
|
||||
// The wrapped claim is the receipt's: the next refresh spends it.
|
||||
refresh_due_sessions(&sessions, &heimdall).await.unwrap();
|
||||
pass(&sessions, &heimdall).await;
|
||||
assert_eq!(heimdall.calls()[1].0, "claim-1");
|
||||
assert!(is_live(&sessions, &cookie).await);
|
||||
}
|
||||
|
|
@ -6556,8 +6656,8 @@ mod refresh_loop_tests {
|
|||
async fn a_retry_of_the_same_refresh_sends_the_same_key() {
|
||||
let (_directory, sessions, cookie) = due_session(1);
|
||||
let heimdall = ScriptedHeimdall::answering(vec![Reply::Timeout, Reply::Timeout]);
|
||||
refresh_due_sessions(&sessions, &heimdall).await.unwrap();
|
||||
refresh_due_sessions(&sessions, &heimdall).await.unwrap();
|
||||
pass(&sessions, &heimdall).await;
|
||||
pass(&sessions, &heimdall).await;
|
||||
let calls = heimdall.calls();
|
||||
assert_eq!(calls.len(), 2, "a transport failure leaves the session offered");
|
||||
assert_eq!(calls[0], calls[1]);
|
||||
|
|
@ -6571,8 +6671,8 @@ mod refresh_loop_tests {
|
|||
Reply::Receipt { revision: 1, claim: "claim-1" },
|
||||
Reply::Timeout,
|
||||
]);
|
||||
refresh_due_sessions(&sessions, &heimdall).await.unwrap();
|
||||
refresh_due_sessions(&sessions, &heimdall).await.unwrap();
|
||||
pass(&sessions, &heimdall).await;
|
||||
pass(&sessions, &heimdall).await;
|
||||
let calls = heimdall.calls();
|
||||
assert_ne!(calls[0].1, calls[1].1);
|
||||
assert!(calls.iter().all(|(claim, key)| !key.contains(claim)));
|
||||
|
|
@ -6585,9 +6685,9 @@ mod refresh_loop_tests {
|
|||
Reply::Receipt { revision: 1, claim: "claim-1" },
|
||||
Reply::Timeout,
|
||||
]);
|
||||
refresh_due_sessions(&sessions, &heimdall).await.unwrap();
|
||||
pass(&sessions, &heimdall).await;
|
||||
assert!(!is_live(&sessions, &cookie).await);
|
||||
refresh_due_sessions(&sessions, &heimdall).await.unwrap();
|
||||
pass(&sessions, &heimdall).await;
|
||||
assert_eq!(heimdall.calls().len(), 1, "a revoked session is not offered again");
|
||||
}
|
||||
|
||||
|
|
@ -6596,7 +6696,7 @@ mod refresh_loop_tests {
|
|||
for reply in [Reply::OtherSession, Reply::OtherAccount] {
|
||||
let (_directory, sessions, cookie) = due_session(1);
|
||||
let heimdall = ScriptedHeimdall::answering(vec![reply]);
|
||||
refresh_due_sessions(&sessions, &heimdall).await.unwrap();
|
||||
pass(&sessions, &heimdall).await;
|
||||
assert!(!is_live(&sessions, &cookie).await);
|
||||
}
|
||||
}
|
||||
|
|
@ -6605,9 +6705,9 @@ mod refresh_loop_tests {
|
|||
async fn a_heimdall_denial_revokes_the_session_and_it_is_not_offered_again() {
|
||||
let (_directory, sessions, cookie) = due_session(1);
|
||||
let heimdall = ScriptedHeimdall::answering(vec![Reply::Deny, Reply::Timeout]);
|
||||
refresh_due_sessions(&sessions, &heimdall).await.unwrap();
|
||||
pass(&sessions, &heimdall).await;
|
||||
assert!(!is_live(&sessions, &cookie).await);
|
||||
refresh_due_sessions(&sessions, &heimdall).await.unwrap();
|
||||
pass(&sessions, &heimdall).await;
|
||||
assert_eq!(heimdall.calls().len(), 1);
|
||||
}
|
||||
|
||||
|
|
@ -6624,11 +6724,111 @@ mod refresh_loop_tests {
|
|||
.with_ansi(false)
|
||||
.finish();
|
||||
let _guard = tracing::subscriber::set_default(subscriber);
|
||||
refresh_due_sessions(&sessions, &heimdall).await.unwrap();
|
||||
pass(&sessions, &heimdall).await;
|
||||
let logged = String::from_utf8(captured.0.lock().unwrap().clone()).unwrap();
|
||||
assert!(
|
||||
logged.contains("refresh receipt revision went backwards"),
|
||||
"{logged}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_receipt_whose_keys_were_unavailable_is_kept_and_applied_on_the_next_pass() {
|
||||
let (_directory, sessions, cookie) = due_session(1);
|
||||
let heimdall = ScriptedHeimdall::answering(vec![
|
||||
Reply::Receipt { revision: 1, claim: "claim-1" },
|
||||
Reply::Timeout,
|
||||
])
|
||||
.verifying(vec![Verdict::Unavailable]);
|
||||
let mut unverified = HashMap::new();
|
||||
refresh_due_sessions(&sessions, &heimdall, &mut unverified).await.unwrap();
|
||||
assert!(is_live(&sessions, &cookie).await, "unavailable keys do not revoke");
|
||||
assert_eq!(unverified.len(), 1);
|
||||
refresh_due_sessions(&sessions, &heimdall, &mut unverified).await.unwrap();
|
||||
assert_eq!(heimdall.calls().len(), 1, "the spent claim is not sent again");
|
||||
assert_eq!(heimdall.verifications(), 2);
|
||||
assert!(unverified.is_empty(), "an applied receipt is not kept");
|
||||
assert!(is_live(&sessions, &cookie).await);
|
||||
// The applied receipt's claim is the one the next refresh spends.
|
||||
refresh_due_sessions(&sessions, &heimdall, &mut unverified).await.unwrap();
|
||||
assert_eq!(heimdall.calls()[1].0, "claim-1");
|
||||
assert_eq!(heimdall.verifications(), 2, "the receipt was applied once");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_receipt_that_positively_fails_verification_revokes_and_is_not_kept() {
|
||||
let (_directory, sessions, cookie) = due_session(1);
|
||||
let heimdall = ScriptedHeimdall::answering(vec![
|
||||
Reply::Receipt { revision: 1, claim: "claim-1" },
|
||||
Reply::Timeout,
|
||||
])
|
||||
.verifying(vec![Verdict::Invalid]);
|
||||
let mut unverified = HashMap::new();
|
||||
refresh_due_sessions(&sessions, &heimdall, &mut unverified).await.unwrap();
|
||||
assert!(!is_live(&sessions, &cookie).await);
|
||||
assert!(unverified.is_empty());
|
||||
refresh_due_sessions(&sessions, &heimdall, &mut unverified).await.unwrap();
|
||||
assert_eq!(heimdall.calls().len(), 1);
|
||||
assert_eq!(heimdall.verifications(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_kept_receipt_for_a_session_that_is_gone_is_dropped() {
|
||||
let (_directory, sessions, cookie) = due_session(1);
|
||||
let heimdall = ScriptedHeimdall::answering(vec![Reply::Receipt {
|
||||
revision: 1,
|
||||
claim: "claim-1",
|
||||
}])
|
||||
.verifying(vec![Verdict::Unavailable]);
|
||||
let mut unverified = HashMap::new();
|
||||
refresh_due_sessions(&sessions, &heimdall, &mut unverified).await.unwrap();
|
||||
assert_eq!(unverified.len(), 1);
|
||||
let cookie_hash = unverified.keys().next().unwrap().clone();
|
||||
sessions.lock().await.revoke_cookie_hash(&cookie_hash).unwrap();
|
||||
refresh_due_sessions(&sessions, &heimdall, &mut unverified).await.unwrap();
|
||||
assert!(unverified.is_empty());
|
||||
assert!(!is_live(&sessions, &cookie).await);
|
||||
assert_eq!(heimdall.verifications(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_session_that_moved_while_refreshing_is_not_revoked() {
|
||||
let (_directory, sessions, cookie) = due_session(1);
|
||||
let mut heimdall = ScriptedHeimdall::answering(vec![Reply::Receipt {
|
||||
revision: 1,
|
||||
claim: "claim-1",
|
||||
}]);
|
||||
heimdall.moving = Some(sessions.clone());
|
||||
pass(&sessions, &heimdall).await;
|
||||
assert!(is_live(&sessions, &cookie).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_receipt_with_a_higher_revision_is_applied() {
|
||||
let (_directory, sessions, cookie) = due_session(1);
|
||||
let heimdall = ScriptedHeimdall::answering(vec![
|
||||
Reply::Receipt { revision: 2, claim: "claim-1" },
|
||||
Reply::Receipt { revision: 1, claim: "claim-2" },
|
||||
]);
|
||||
pass(&sessions, &heimdall).await;
|
||||
assert!(is_live(&sessions, &cookie).await);
|
||||
// The session now sits at revision 2, so a revision-1 receipt is
|
||||
// backwards.
|
||||
pass(&sessions, &heimdall).await;
|
||||
assert_eq!(heimdall.calls()[1].0, "claim-1");
|
||||
assert!(!is_live(&sessions, &cookie).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_refresh_loop_runs_a_pass_on_every_tick() {
|
||||
let (_directory, sessions, _cookie) = due_session(1);
|
||||
let heimdall = ScriptedHeimdall::answering((0..200).map(|_| Reply::Timeout).collect());
|
||||
let ran = tokio::time::timeout(
|
||||
Duration::from_millis(400),
|
||||
run_refresh_loop(&sessions, &heimdall, Duration::from_millis(10)),
|
||||
)
|
||||
.await;
|
||||
assert!(ran.is_err(), "the loop never returns");
|
||||
assert!(heimdall.calls().len() >= 2, "{:?}", heimdall.calls());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue