use std::collections::HashMap;
use std::net::{IpAddr, SocketAddr};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant, SystemTime};
use rand::RngExt;
use url::Url;
use crate::push::claim::{
A2aPushDeliveryStore, AbandonedReason, DeliveryClaim, DeliveryErrorClass, DeliveryOutcome,
GaveUpReason,
};
use crate::push::secret::Secret;
use crate::push::ssrf::{
OutboundUrlValidator, SsrfBlockReason, SsrfDecision, check_url, pick_destination,
};
use crate::storage::{A2aPushNotificationStorage, A2aStorageError, A2aTaskStorage};
pub const MAX_FAILED_DELIVERY_RETENTION: Duration = Duration::from_secs(1000 * 365 * 24 * 60 * 60);
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct PushDeliveryConfig {
pub max_attempts: u32,
pub backoff_base: Duration,
pub backoff_cap: Duration,
pub backoff_jitter: f32,
pub request_timeout: Duration,
pub connect_timeout: Duration,
pub read_timeout: Duration,
pub claim_expiry: Duration,
pub max_payload_bytes: usize,
pub allow_insecure_urls: bool,
pub config_cache_ttl: Duration,
pub failed_delivery_retention: Duration,
}
impl Default for PushDeliveryConfig {
fn default() -> Self {
Self {
max_attempts: 8,
backoff_base: Duration::from_secs(2),
backoff_cap: Duration::from_secs(60),
backoff_jitter: 0.25,
request_timeout: Duration::from_secs(30),
connect_timeout: Duration::from_secs(5),
read_timeout: Duration::from_secs(30),
claim_expiry: Duration::from_secs(600),
max_payload_bytes: 1024 * 1024,
allow_insecure_urls: false,
config_cache_ttl: Duration::ZERO,
failed_delivery_retention: Duration::from_secs(7 * 24 * 60 * 60),
}
}
}
impl PushDeliveryConfig {
pub fn from_runtime_config(runtime: &crate::server::RuntimeConfig) -> Self {
Self {
max_attempts: runtime.push_max_attempts as u32,
backoff_base: runtime.push_backoff_base,
backoff_cap: runtime.push_backoff_cap,
backoff_jitter: runtime.push_backoff_jitter,
request_timeout: runtime.push_request_timeout,
connect_timeout: runtime.push_connect_timeout,
read_timeout: runtime.push_read_timeout,
claim_expiry: runtime.push_claim_expiry,
max_payload_bytes: runtime.push_max_payload_bytes,
allow_insecure_urls: runtime.allow_insecure_push_urls,
config_cache_ttl: runtime.push_config_cache_ttl,
failed_delivery_retention: runtime.push_failed_delivery_retention,
}
}
}
#[derive(Clone)]
pub struct PushTarget {
pub tenant: String,
pub owner: String,
pub task_id: String,
pub event_sequence: u64,
pub config_id: String,
pub url: Url,
pub auth_scheme: String,
pub auth_credentials: Secret,
pub token: Option<Secret>,
}
pub trait PushDnsResolver: Send + Sync {
fn resolve(
&self,
host: &str,
port: u16,
) -> futures::future::BoxFuture<'_, Result<Vec<IpAddr>, String>>;
}
pub struct TokioDnsResolver;
impl PushDnsResolver for TokioDnsResolver {
fn resolve(
&self,
host: &str,
port: u16,
) -> futures::future::BoxFuture<'_, Result<Vec<IpAddr>, String>> {
let host = host.to_string();
Box::pin(async move {
let ips: Vec<IpAddr> = tokio::net::lookup_host((host.as_str(), port))
.await
.map_err(|e| e.to_string())?
.map(|sa| sa.ip())
.collect();
Ok(ips)
})
}
}
#[derive(Clone)]
#[non_exhaustive]
pub struct DeliveryTargetLookup {
pub push_storage: Arc<dyn A2aPushNotificationStorage>,
pub task_storage: Arc<dyn A2aTaskStorage>,
}
impl DeliveryTargetLookup {
pub fn new(
push_storage: Arc<dyn A2aPushNotificationStorage>,
task_storage: Arc<dyn A2aTaskStorage>,
) -> Self {
Self {
push_storage,
task_storage,
}
}
async fn gone(
&self,
target: &PushTarget,
cache: &PushConfigCache,
ttl: Duration,
) -> Option<AbandonedReason> {
if !cache.is_fresh(target, ttl) {
let generation = cache.generation();
match self
.push_storage
.get_config(&target.tenant, &target.task_id, &target.config_id)
.await
{
Ok(None) => return Some(AbandonedReason::ConfigDeleted),
Ok(Some(_)) => cache.record_present(target, ttl, generation),
Err(_) => {}
}
}
if let Ok(None) = self
.task_storage
.get_task(&target.tenant, &target.task_id, &target.owner, Some(0))
.await
{
return Some(AbandonedReason::TaskDeleted);
}
None
}
}
#[derive(Default)]
struct PushConfigCache {
state: Mutex<PushConfigCacheState>,
}
#[derive(Default)]
struct PushConfigCacheState {
generation: u64,
by_task: HashMap<(String, String), HashMap<String, Instant>>,
}
impl PushConfigCache {
fn lock(&self) -> std::sync::MutexGuard<'_, PushConfigCacheState> {
self.state.lock().unwrap_or_else(|e| e.into_inner())
}
fn is_fresh(&self, target: &PushTarget, ttl: Duration) -> bool {
self.lock()
.by_task
.get(&(target.tenant.clone(), target.task_id.clone()))
.and_then(|configs| configs.get(&target.config_id))
.is_some_and(|confirmed| confirmed.elapsed() < ttl)
}
fn generation(&self) -> u64 {
self.lock().generation
}
fn record_present(&self, target: &PushTarget, ttl: Duration, read_generation: u64) {
if ttl.is_zero() {
return;
}
let mut state = self.lock();
if state.generation != read_generation {
return;
}
state.by_task.retain(|_, configs| {
configs.retain(|_, confirmed| confirmed.elapsed() < ttl);
!configs.is_empty()
});
state
.by_task
.entry((target.tenant.clone(), target.task_id.clone()))
.or_default()
.insert(target.config_id.clone(), Instant::now());
}
fn invalidate(&self, tenant: &str, task_id: &str) {
let mut state = self.lock();
state.generation = state.generation.wrapping_add(1);
state
.by_task
.remove(&(tenant.to_string(), task_id.to_string()));
}
}
#[derive(Clone)]
pub struct PushDeliveryWorker {
pub push_delivery_store: Arc<dyn A2aPushDeliveryStore>,
pub dns_resolver: Arc<dyn PushDnsResolver>,
pub config: PushDeliveryConfig,
pub outbound_validator: Option<OutboundUrlValidator>,
pub instance_id: String,
pub(crate) target_lookup: Option<DeliveryTargetLookup>,
config_cache: Arc<PushConfigCache>,
}
impl PushDeliveryWorker {
pub fn new(
push_delivery_store: Arc<dyn A2aPushDeliveryStore>,
config: PushDeliveryConfig,
outbound_validator: Option<OutboundUrlValidator>,
instance_id: String,
) -> Result<Self, String> {
Ok(Self {
push_delivery_store,
dns_resolver: Arc::new(TokioDnsResolver),
config,
outbound_validator,
instance_id,
target_lookup: None,
config_cache: Arc::default(),
})
}
pub fn invalidate_push_configs(&self, tenant: &str, task_id: &str) {
self.config_cache.invalidate(tenant, task_id);
}
pub fn with_target_lookup(mut self, lookup: DeliveryTargetLookup) -> Self {
self.target_lookup = Some(lookup);
self
}
pub fn with_dns_resolver(mut self, resolver: Arc<dyn PushDnsResolver>) -> Self {
self.dns_resolver = resolver;
self
}
pub async fn deliver(&self, target: &PushTarget, payload: &[u8]) -> DeliveryReport {
if payload.len() > self.config.max_payload_bytes {
let claim = match self.claim(target).await {
Ok(c) => c,
Err(_e) => return DeliveryReport::UnclaimedSkip,
};
return self
.persist_terminal(
target,
&claim,
self.gave_up(
GaveUpReason::PayloadTooLarge,
DeliveryErrorClass::PayloadTooLarge,
None,
),
)
.await;
}
let claim = match self.claim(target).await {
Ok(c) => c,
Err(ClaimFailure::AlreadyHeld) => return DeliveryReport::ClaimLostOrFinal,
Err(ClaimFailure::Other(_)) => return DeliveryReport::UnclaimedSkip,
};
let mut current_count = claim.delivery_attempt_count;
loop {
if current_count >= self.config.max_attempts {
return self
.persist_terminal(
target,
&claim,
self.gave_up(
GaveUpReason::MaxAttemptsExhausted,
DeliveryErrorClass::Timeout,
None,
),
)
.await;
}
if let Some(lookup) = &self.target_lookup
&& let Some(reason) = lookup
.gone(target, &self.config_cache, self.config.config_cache_ttl)
.await
{
return self
.persist_terminal(target, &claim, DeliveryOutcome::Abandoned { reason })
.await;
}
let decision = match check_url(
&target.url,
self.config.allow_insecure_urls,
self.outbound_validator.as_ref(),
) {
Err(reason) => SsrfDecision::Block(reason),
Ok(()) => {
let ips = self.resolve(&target.url).await.unwrap_or_default();
pick_destination(&ips, self.config.allow_insecure_urls)
}
};
let resolved_ip = match decision {
SsrfDecision::Allow { resolved_ip } => resolved_ip,
SsrfDecision::Block(SsrfBlockReason::InsecureScheme) => {
tracing::warn!(
target: "turul_a2a::push_non_https_url",
tenant = %target.tenant,
task_id = %target.task_id,
config_id = %target.config_id,
event_sequence = target.event_sequence,
"push delivery abandoned: only https:// webhook URLs are \
delivered unless allow_insecure_push_urls is set"
);
return self
.persist_terminal(
target,
&claim,
DeliveryOutcome::Abandoned {
reason: AbandonedReason::NonHttpsUrlInProduction,
},
)
.await;
}
SsrfDecision::Block(_) => {
return self
.persist_terminal(
target,
&claim,
self.gave_up(
GaveUpReason::SsrfBlocked,
DeliveryErrorClass::SSRFBlocked,
None,
),
)
.await;
}
};
let new_count = match self
.push_delivery_store
.record_attempt_started(
&target.tenant,
&target.task_id,
target.event_sequence,
&target.config_id,
&claim.claimant,
claim.generation,
)
.await
{
Ok(n) => n,
Err(A2aStorageError::StaleDeliveryClaim { .. }) => {
return DeliveryReport::ClaimLostOrFinal;
}
Err(_) => return DeliveryReport::TransientStoreError,
};
current_count = new_count;
let result = self.post(target, payload, resolved_ip).await;
let outcome = match &result {
Ok(status) if (200..400).contains(&status.as_u16()) => DeliveryOutcome::Succeeded {
http_status: status.as_u16(),
},
Ok(status) if status.as_u16() == 429 => DeliveryOutcome::Retry {
next_attempt_at: SystemTime::now() + self.backoff_for(new_count),
http_status: Some(status.as_u16()),
error_class: DeliveryErrorClass::HttpError429,
},
Ok(status) if status.as_u16() == 408 => DeliveryOutcome::Retry {
next_attempt_at: SystemTime::now() + self.backoff_for(new_count),
http_status: Some(status.as_u16()),
error_class: DeliveryErrorClass::Timeout,
},
Ok(status) if (500..600).contains(&status.as_u16()) => DeliveryOutcome::Retry {
next_attempt_at: SystemTime::now() + self.backoff_for(new_count),
http_status: Some(status.as_u16()),
error_class: DeliveryErrorClass::HttpError5xx {
status: status.as_u16(),
},
},
Ok(status) => {
self.gave_up(
GaveUpReason::NonRetryableHttpStatus,
DeliveryErrorClass::HttpError4xx {
status: status.as_u16(),
},
Some(status.as_u16()),
)
}
Err(DeliveryErrorClass::TlsRejected) => self.gave_up(
GaveUpReason::TlsRejected,
DeliveryErrorClass::TlsRejected,
None,
),
Err(err_class) => DeliveryOutcome::Retry {
next_attempt_at: SystemTime::now() + self.backoff_for(new_count),
http_status: None,
error_class: *err_class,
},
};
match outcome {
DeliveryOutcome::Succeeded { .. }
| DeliveryOutcome::GaveUp { .. }
| DeliveryOutcome::Abandoned { .. } => {
return self.persist_terminal(target, &claim, outcome).await;
}
DeliveryOutcome::Retry {
http_status,
error_class,
..
} => {
let _ = self
.push_delivery_store
.record_delivery_outcome(
&target.tenant,
&target.task_id,
target.event_sequence,
&target.config_id,
&claim.claimant,
claim.generation,
DeliveryOutcome::Retry {
next_attempt_at: SystemTime::now() + self.backoff_for(new_count),
http_status,
error_class,
},
)
.await;
if current_count >= self.config.max_attempts {
return self
.persist_terminal(
target,
&claim,
self.gave_up(
GaveUpReason::MaxAttemptsExhausted,
error_class,
http_status,
),
)
.await;
}
tokio::time::sleep(self.backoff_for(new_count)).await;
continue;
}
}
}
}
fn gave_up(
&self,
reason: GaveUpReason,
last_error_class: DeliveryErrorClass,
last_http_status: Option<u16>,
) -> DeliveryOutcome {
DeliveryOutcome::GaveUp {
reason,
last_error_class,
last_http_status,
retained_until: SystemTime::now()
+ self
.config
.failed_delivery_retention
.min(MAX_FAILED_DELIVERY_RETENTION),
}
}
async fn persist_terminal(
&self,
target: &PushTarget,
claim: &DeliveryClaim,
outcome: DeliveryOutcome,
) -> DeliveryReport {
let success_report = match &outcome {
DeliveryOutcome::Succeeded { http_status } => DeliveryReport::Succeeded(*http_status),
DeliveryOutcome::GaveUp { reason, .. } => DeliveryReport::GaveUp(*reason),
DeliveryOutcome::Abandoned { reason } => DeliveryReport::Abandoned(*reason),
DeliveryOutcome::Retry { .. } => return DeliveryReport::TransientStoreError,
};
const TERMINAL_PERSIST_MAX_ATTEMPTS: u32 = 3;
let backoffs = [
Duration::from_millis(50),
Duration::from_millis(150),
Duration::from_millis(500),
];
let mut last_error: Option<A2aStorageError> = None;
for attempt in 0..TERMINAL_PERSIST_MAX_ATTEMPTS {
if attempt > 0 {
tokio::time::sleep(backoffs[(attempt as usize).min(backoffs.len() - 1)]).await;
}
match self
.push_delivery_store
.record_delivery_outcome(
&target.tenant,
&target.task_id,
target.event_sequence,
&target.config_id,
&claim.claimant,
claim.generation,
outcome.clone(),
)
.await
{
Ok(()) => return success_report,
Err(A2aStorageError::StaleDeliveryClaim { .. }) => {
return DeliveryReport::ClaimLostOrFinal;
}
Err(e) => {
last_error = Some(e);
}
}
}
let _ = last_error; DeliveryReport::TransientStoreError
}
pub async fn abandon_reclaimed(
&self,
target: &PushTarget,
reason: AbandonedReason,
) -> DeliveryReport {
let claim = match self.claim(target).await {
Ok(c) => c,
Err(ClaimFailure::AlreadyHeld) => return DeliveryReport::ClaimLostOrFinal,
Err(ClaimFailure::Other(_)) => return DeliveryReport::UnclaimedSkip,
};
self.persist_terminal(target, &claim, DeliveryOutcome::Abandoned { reason })
.await
}
async fn claim(&self, target: &PushTarget) -> Result<DeliveryClaim, ClaimFailure> {
match self
.push_delivery_store
.claim_delivery(
&target.tenant,
&target.task_id,
target.event_sequence,
&target.config_id,
&self.instance_id,
&target.owner,
self.config.claim_expiry,
)
.await
{
Ok(c) => Ok(c),
Err(A2aStorageError::ClaimAlreadyHeld { .. }) => Err(ClaimFailure::AlreadyHeld),
Err(e) => Err(ClaimFailure::Other(e.to_string())),
}
}
async fn resolve(&self, url: &Url) -> Result<Vec<IpAddr>, String> {
let host = url
.host_str()
.ok_or_else(|| "url has no host".to_string())?;
let port = url.port_or_known_default().unwrap_or(443);
self.dns_resolver.resolve(host, port).await
}
async fn post(
&self,
target: &PushTarget,
payload: &[u8],
resolved_ip: IpAddr,
) -> Result<reqwest::StatusCode, DeliveryErrorClass> {
let host = target
.url
.host_str()
.ok_or(DeliveryErrorClass::NetworkError)?
.to_string();
let pinned = SocketAddr::new(resolved_ip, 0);
let client = reqwest::Client::builder()
.connect_timeout(self.config.connect_timeout)
.read_timeout(self.config.read_timeout)
.timeout(self.config.request_timeout)
.redirect(reqwest::redirect::Policy::none())
.resolve(&host, pinned)
.build()
.map_err(|_| DeliveryErrorClass::NetworkError)?;
let mut req = client
.post(target.url.clone())
.header(
"Content-Type",
turul_a2a_types::wire::protocol::MEDIA_TYPE_A2A_JSON,
)
.header(
"User-Agent",
format!("turul-a2a/{}", env!("CARGO_PKG_VERSION")),
)
.header("X-Turul-Event-Sequence", target.event_sequence.to_string());
if !target.auth_scheme.is_empty() {
req = req.header(
"Authorization",
format!(
"{} {}",
target.auth_scheme,
target.auth_credentials.expose()
),
);
}
if let Some(tok) = &target.token {
req = req.header("X-Turul-Push-Token", tok.expose());
}
let resp = req.body(payload.to_vec()).send().await;
match resp {
Ok(r) => Ok(r.status()),
Err(e) if is_tls_failure(&e) => Err(DeliveryErrorClass::TlsRejected),
Err(e) if e.is_timeout() => Err(DeliveryErrorClass::Timeout),
Err(_) => Err(DeliveryErrorClass::NetworkError),
}
}
fn backoff_for(&self, attempt: u32) -> Duration {
let base = self.config.backoff_base.as_millis() as u64;
let cap = self.config.backoff_cap.as_millis() as u64;
let raw = base.saturating_mul(1u64 << attempt.min(31).saturating_sub(1));
let target = raw.min(cap);
let jitter = self.config.backoff_jitter.max(0.0);
let delta = (target as f64 * jitter as f64) as i64;
let offset = if delta == 0 {
0i64
} else {
rand::rng().random_range(-delta..=delta)
};
let final_ms = (target as i64).saturating_add(offset).max(0) as u64;
Duration::from_millis(final_ms)
}
}
fn is_tls_failure(err: &(dyn std::error::Error + 'static)) -> bool {
let mut current = Some(err);
while let Some(e) = current {
if e.is::<rustls::Error>() {
return true;
}
current = match e.downcast_ref::<std::io::Error>() {
Some(io) => io
.get_ref()
.map(|inner| inner as &(dyn std::error::Error + 'static)),
None => e.source(),
};
}
false
}
enum ClaimFailure {
AlreadyHeld,
Other(#[allow(dead_code)] String),
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum DeliveryReport {
Succeeded(u16),
GaveUp(GaveUpReason),
Abandoned(AbandonedReason),
ClaimLostOrFinal,
UnclaimedSkip,
TransientStoreError,
}
#[cfg(test)]
mod tests {
use super::*;
use crate::storage::InMemoryA2aStorage;
fn worker_with_store(
store: Arc<InMemoryA2aStorage>,
cfg: PushDeliveryConfig,
) -> PushDeliveryWorker {
PushDeliveryWorker::new(store, cfg, None, format!("worker-{}", uuid::Uuid::now_v7()))
.expect("build")
}
fn target(url: &str) -> PushTarget {
PushTarget {
tenant: "t".into(),
owner: "anonymous".into(),
task_id: format!("task-{}", uuid::Uuid::now_v7()),
event_sequence: 1,
config_id: "cfg-A".into(),
url: Url::parse(url).unwrap(),
auth_scheme: "Bearer".into(),
auth_credentials: Secret::new("cred".into()),
token: None,
}
}
#[test]
fn backoff_doubles_until_cap() {
let store = Arc::new(InMemoryA2aStorage::new());
let cfg = PushDeliveryConfig {
backoff_base: Duration::from_secs(1),
backoff_cap: Duration::from_secs(8),
backoff_jitter: 0.0,
..Default::default()
};
let w = worker_with_store(store, cfg);
assert_eq!(w.backoff_for(1), Duration::from_secs(1));
assert_eq!(w.backoff_for(2), Duration::from_secs(2));
assert_eq!(w.backoff_for(3), Duration::from_secs(4));
assert_eq!(w.backoff_for(4), Duration::from_secs(8));
assert_eq!(w.backoff_for(5), Duration::from_secs(8));
assert_eq!(w.backoff_for(10), Duration::from_secs(8));
}
#[tokio::test]
async fn payload_too_large_short_circuits_with_gaveup() {
let store = Arc::new(InMemoryA2aStorage::new());
let cfg = PushDeliveryConfig {
max_payload_bytes: 10,
..Default::default()
};
let w = worker_with_store(store.clone(), cfg);
let t = target("https://example.com/");
let payload = vec![0u8; 1024];
let report = w.deliver(&t, &payload).await;
assert_eq!(
report,
DeliveryReport::GaveUp(GaveUpReason::PayloadTooLarge)
);
let failed = store
.list_failed_deliveries(&t.tenant, SystemTime::UNIX_EPOCH, 10)
.await
.unwrap();
assert_eq!(failed.len(), 1);
assert!(matches!(
failed[0].last_error_class,
DeliveryErrorClass::PayloadTooLarge
));
}
#[test]
fn runtime_config_maps_onto_delivery_config() {
let runtime = crate::server::RuntimeConfig {
push_max_attempts: 3,
push_read_timeout: Duration::from_millis(250),
push_config_cache_ttl: Duration::from_secs(9),
push_failed_delivery_retention: Duration::from_secs(60),
allow_insecure_push_urls: true,
..Default::default()
};
let cfg = PushDeliveryConfig::from_runtime_config(&runtime);
assert_eq!(cfg.max_attempts, 3);
assert_eq!(cfg.read_timeout, Duration::from_millis(250));
assert_eq!(cfg.config_cache_ttl, Duration::from_secs(9));
assert_eq!(cfg.failed_delivery_retention, Duration::from_secs(60));
assert!(cfg.allow_insecure_urls);
}
#[test]
fn runtime_config_push_defaults_are_delivery_defaults() {
let from_runtime =
PushDeliveryConfig::from_runtime_config(&crate::server::RuntimeConfig::default());
assert_eq!(
format!("{from_runtime:?}"),
format!("{:?}", PushDeliveryConfig::default())
);
}
#[test]
fn certificate_rejection_is_a_tls_failure() {
let cert = std::io::Error::other(std::io::Error::new(
std::io::ErrorKind::InvalidData,
rustls::Error::InvalidCertificate(rustls::CertificateError::UnknownIssuer),
));
assert!(is_tls_failure(&cert));
assert!(is_tls_failure(&rustls::Error::InvalidCertificate(
rustls::CertificateError::Expired
)));
let refused = std::io::Error::from(std::io::ErrorKind::ConnectionRefused);
assert!(!is_tls_failure(&refused));
}
#[tokio::test]
async fn non_https_url_in_production_is_abandoned_without_post() {
let store = Arc::new(InMemoryA2aStorage::new());
let cfg = PushDeliveryConfig {
allow_insecure_urls: false,
..Default::default()
};
let w = worker_with_store(store.clone(), cfg);
let t = target("http://webhook.invalid/");
let report = w.deliver(&t, b"{}").await;
assert_eq!(
report,
DeliveryReport::Abandoned(AbandonedReason::NonHttpsUrlInProduction)
);
let failed = store
.list_failed_deliveries(&t.tenant, SystemTime::UNIX_EPOCH, 10)
.await
.unwrap();
assert!(failed.is_empty(), "abandoned is not a failed delivery");
assert!(matches!(w.claim(&t).await, Err(ClaimFailure::AlreadyHeld)));
}
#[test]
fn default_config_cache_is_off() {
assert_eq!(
PushDeliveryConfig::default().config_cache_ttl,
Duration::ZERO
);
}
#[test]
fn presence_read_before_invalidation_is_not_cached() {
let cache = PushConfigCache::default();
let t = target("https://example.com/hook");
let ttl = Duration::from_secs(3600);
let read_generation = cache.generation();
cache.invalidate(&t.tenant, &t.task_id);
cache.record_present(&t, ttl, read_generation);
assert!(!cache.is_fresh(&t, ttl), "stale read was cached");
let read_generation = cache.generation();
cache.record_present(&t, ttl, read_generation);
assert!(cache.is_fresh(&t, ttl), "current read is cached");
cache.invalidate(&t.tenant, &t.task_id);
assert!(!cache.is_fresh(&t, ttl), "invalidation drops the entry");
}
#[test]
fn huge_failed_delivery_retention_is_clamped() {
let cfg = PushDeliveryConfig {
failed_delivery_retention: Duration::MAX,
..Default::default()
};
let w = worker_with_store(Arc::new(InMemoryA2aStorage::new()), cfg);
let before = SystemTime::now();
let DeliveryOutcome::GaveUp { retained_until, .. } = w.gave_up(
GaveUpReason::MaxAttemptsExhausted,
DeliveryErrorClass::Timeout,
None,
) else {
panic!("gave_up returns GaveUp");
};
assert!(retained_until >= before + MAX_FAILED_DELIVERY_RETENTION);
let micros = retained_until
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_micros();
assert!(micros <= i64::MAX as u128, "fits an i64 microsecond column");
}
}