use std::collections::VecDeque;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;
use tokio::sync::Semaphore;
use tokio::time::Instant;
use super::{TokenCache, is_fresh, replicate};
use crate::error::{ClientError, ClientResult};
type Answer = ClientResult<(String, Duration)>;
struct Script {
calls: AtomicUsize,
planned: usize,
gate: Semaphore,
answers: Mutex<VecDeque<Answer>>,
}
impl Script {
fn new(answers: Vec<Answer>) -> Arc<Self> {
Arc::new(Self {
calls: AtomicUsize::new(0),
planned: answers.len(),
gate: Semaphore::new(0),
answers: Mutex::new(answers.into()),
})
}
fn release(&self, n: usize) {
self.gate.add_permits(n);
}
fn calls(&self) -> usize {
self.calls.load(Ordering::SeqCst)
}
async fn refresh(&self) -> Answer {
let n = self.calls.fetch_add(1, Ordering::SeqCst);
assert!(n < self.planned, "unplanned refresh #{}", n + 1);
self.gate.acquire().await.expect("gate open").forget();
self.answers
.lock()
.expect("script lock")
.pop_front()
.expect("a scripted answer for every refresh")
}
}
#[allow(clippy::unnecessary_wraps)] fn ok(token: &str, ttl_secs: u64) -> Answer {
Ok((token.to_owned(), Duration::from_secs(ttl_secs)))
}
fn down() -> Answer {
Err(ClientError::Timeout(
"token endpoint request timed out".into(),
))
}
async fn get(cache: &TokenCache, script: &Script, backoff: Duration) -> ClientResult<String> {
cache.get(backoff, || script.refresh()).await
}
async fn until_waiting(cache: &TokenCache, n: usize) {
for _ in 0..10_000 {
if cache.waiters() == n {
return;
}
tokio::task::yield_now().await;
}
panic!("expected {n} waiters, have {}", cache.waiters());
}
fn spawn_get(
cache: &Arc<TokenCache>,
script: &Arc<Script>,
backoff: Duration,
) -> tokio::task::JoinHandle<ClientResult<String>> {
let (cache, script) = (Arc::clone(cache), Arc::clone(script));
tokio::spawn(async move { get(&cache, &script, backoff).await })
}
#[test]
fn is_fresh_is_exclusive_at_the_deadline() {
let t = Instant::now();
assert!(
!is_fresh(t, t),
"at the deadline a token is due for refresh"
);
assert!(is_fresh(t, t + Duration::from_secs(1)), "before it, usable");
assert!(!is_fresh(t + Duration::from_secs(1), t), "after it, stale");
}
#[tokio::test(start_paused = true)]
async fn waiters_share_the_leaders_failure() {
let cache = Arc::new(TokenCache::default());
let script = Script::new(vec![down()]);
let backoff = Duration::ZERO;
let leader = spawn_get(&cache, &script, backoff);
let waiters: Vec<_> = (0..4)
.map(|_| spawn_get(&cache, &script, backoff))
.collect();
until_waiting(&cache, 4).await;
script.release(1);
assert!(matches!(leader.await, Ok(Err(ClientError::Timeout(_)))));
for w in waiters {
let err = w.await.expect("join").expect_err("shared failure");
assert!(matches!(err, ClientError::Timeout(_)), "{err:?}");
}
assert_eq!(script.calls(), 1);
}
#[tokio::test(start_paused = true)]
async fn waiters_share_the_leaders_success() {
let cache = Arc::new(TokenCache::default());
let script = Script::new(vec![ok("tok", 60)]);
let leader = spawn_get(&cache, &script, Duration::ZERO);
let waiters: Vec<_> = (0..3)
.map(|_| spawn_get(&cache, &script, Duration::ZERO))
.collect();
until_waiting(&cache, 3).await;
script.release(1);
assert_eq!(leader.await.expect("join").expect("token"), "tok");
for w in waiters {
assert_eq!(w.await.expect("join").expect("token"), "tok");
}
assert_eq!(script.calls(), 1);
assert_eq!(cache.waiters(), 0, "the in-flight slot is cleared");
}
#[tokio::test(start_paused = true)]
async fn a_failure_is_served_from_cache_until_the_backoff_ends() {
let cache = TokenCache::default();
let script = Script::new(vec![down(), ok("tok", 60)]);
script.release(2);
let backoff = Duration::from_secs(1);
assert!(get(&cache, &script, backoff).await.is_err());
tokio::time::advance(Duration::from_millis(999)).await;
let err = get(&cache, &script, backoff).await.expect_err("cached");
assert!(matches!(err, ClientError::Timeout(_)), "{err:?}");
assert_eq!(script.calls(), 1, "inside the backoff: no new attempt");
tokio::time::advance(Duration::from_millis(1)).await;
assert_eq!(get(&cache, &script, backoff).await.expect("tok"), "tok");
assert_eq!(script.calls(), 2, "at the backoff's end: a new attempt");
}
#[tokio::test(start_paused = true)]
async fn a_zero_backoff_retries_at_once() {
let cache = TokenCache::default();
let script = Script::new(vec![down(), ok("tok", 60)]);
script.release(2);
assert!(get(&cache, &script, Duration::ZERO).await.is_err());
assert_eq!(
get(&cache, &script, Duration::ZERO).await.expect("tok"),
"tok"
);
assert_eq!(script.calls(), 2);
}
#[tokio::test(start_paused = true)]
async fn a_token_is_served_until_its_ttl_then_refreshed() {
let cache = TokenCache::default();
let script = Script::new(vec![ok("a", 10), ok("b", 10)]);
script.release(2);
assert_eq!(get(&cache, &script, Duration::ZERO).await.expect("a"), "a");
tokio::time::advance(Duration::from_millis(9_999)).await;
assert_eq!(get(&cache, &script, Duration::ZERO).await.expect("a"), "a");
tokio::time::advance(Duration::from_millis(1)).await;
assert_eq!(get(&cache, &script, Duration::ZERO).await.expect("b"), "b");
assert_eq!(script.calls(), 2);
}
#[tokio::test(start_paused = true)]
async fn a_cancelled_leader_hands_over_to_a_waiter() {
let cache = Arc::new(TokenCache::default());
let script = Script::new(vec![ok("from-the-waiter", 60), ok("unused", 60)]);
let backoff = Duration::from_secs(60);
let leader = spawn_get(&cache, &script, backoff);
let waiter = spawn_get(&cache, &script, backoff);
until_waiting(&cache, 1).await;
leader.abort();
assert!(leader.await.expect_err("aborted").is_cancelled());
for _ in 0..10_000 {
if script.calls() == 2 {
break;
}
tokio::task::yield_now().await;
}
assert_eq!(script.calls(), 2, "the waiter started a new attempt");
script.release(1);
assert_eq!(
waiter.await.expect("join").expect("token"),
"from-the-waiter"
);
}
#[test]
fn replicas_keep_their_variant_or_retry_class() {
let serde_err = serde_json::from_str::<u8>("x").expect_err("not a u8");
let all = vec![
ClientError::HttpClient("h".into()),
ClientError::Serialization(serde_err),
ClientError::Protocol(a2a_protocol_types::A2aError::internal("p")),
ClientError::Transport("t".into()),
ClientError::InvalidEndpoint("e".into()),
ClientError::UnexpectedStatus {
status: 503,
body: "b".into(),
retry_after: Some(Duration::from_secs(2)),
},
ClientError::AuthRequired {
task_id: "t-1".into(),
},
ClientError::Timeout("to".into()),
ClientError::TooManyPendingRequests { limit: 3 },
ClientError::ProtocolBindingMismatch("m".into()),
ClientError::IncompleteStream {
last_event_id: Some("7".into()),
detail: "d".into(),
},
];
for e in all {
let copy = replicate(&e);
assert_eq!(copy.is_retryable(), e.is_retryable(), "{e:?}");
assert_eq!(copy.retry_after(), e.retry_after(), "{e:?}");
if !matches!(e, ClientError::Serialization(_)) {
assert_eq!(copy.to_string(), e.to_string(), "{e:?}");
}
}
}
#[tokio::test(start_paused = true)]
async fn invalidating_the_cached_token_forces_a_refresh() {
let cache = TokenCache::default();
let script = Script::new(vec![ok("a", 60), ok("b", 60)]);
script.release(2);
assert_eq!(get(&cache, &script, Duration::ZERO).await.expect("a"), "a");
cache.invalidate("a");
assert_eq!(get(&cache, &script, Duration::ZERO).await.expect("b"), "b");
assert_eq!(script.calls(), 2);
}
#[tokio::test(start_paused = true)]
async fn invalidating_a_stale_token_keeps_the_current_one() {
let cache = TokenCache::default();
let script = Script::new(vec![ok("new", 60)]);
script.release(1);
assert_eq!(
get(&cache, &script, Duration::ZERO).await.expect("new"),
"new"
);
cache.invalidate("old");
assert_eq!(
get(&cache, &script, Duration::ZERO).await.expect("new"),
"new"
);
assert_eq!(script.calls(), 1);
}