use std::future::Future;
use std::sync::Mutex;
use std::time::{Duration, Instant};
const WARM_THRESHOLD: u64 = 10;
const SEED_FLOOR_US: f64 = 1_000_000.0;
const MEAN_EWMA_ALPHA: f64 = 1.0 / 64.0;
const VALUE_EWMA_ALPHA: f64 = 0.01;
const EXPECTED_OFFSET_US: f64 = 700_000.0;
const ESCAPE_US: f64 = 5_000_000.0;
const HEDGE_COST: f64 = 1.0;
const HEDGE_REWARD: f64 = 0.05;
const HEDGE_CAPACITY: f64 = 10.0;
pub(crate) enum GuardError<E> {
DeadlineExceeded(Duration),
Inner(E),
}
fn percentile(samples: &mut [f64], q: f64) -> f64 {
assert!(!samples.is_empty(), "percentile of no samples");
samples.sort_by(|a, b| a.total_cmp(b));
let n = samples.len();
let rank = (q.clamp(0.0, 1.0) * n as f64).ceil() as usize;
let idx = rank.saturating_sub(1).min(n - 1);
samples[idx]
}
struct Ewma {
alpha: f64,
value: Option<f64>,
}
impl Ewma {
const fn new(alpha: f64) -> Self {
Self { alpha, value: None }
}
fn observe(&mut self, sample: f64) -> f64 {
let next = match self.value {
Some(v) => (1.0 - self.alpha) * v + self.alpha * sample,
None => sample,
};
self.value = Some(next);
next
}
}
struct DeadlineController {
mean_ttfb: Ewma,
successes: u64,
warmup_samples: Vec<f64>,
stop_timeout: bool,
deadline_us: Option<f64>,
lifetime_timeouts: u64,
hedge_tokens: f64,
}
impl DeadlineController {
fn new() -> Self {
Self {
mean_ttfb: Ewma::new(MEAN_EWMA_ALPHA),
successes: 0,
warmup_samples: Vec::new(),
stop_timeout: false,
deadline_us: None,
lifetime_timeouts: 0,
hedge_tokens: HEDGE_CAPACITY,
}
}
fn deadline(&self) -> Option<Duration> {
self.deadline_us.map(|us| Duration::from_micros(us as u64))
}
fn can_hedge(&self) -> bool {
self.hedge_tokens >= HEDGE_COST
}
fn charge_hedge(&mut self) {
self.hedge_tokens = (self.hedge_tokens - HEDGE_COST).max(0.0);
}
fn record_success(&mut self, latency: Duration) {
let sample_us = latency.as_micros() as f64;
let mean = self.mean_ttfb.observe(sample_us);
self.successes += 1;
self.hedge_tokens = (self.hedge_tokens + HEDGE_REWARD).min(HEDGE_CAPACITY);
if self.successes < WARM_THRESHOLD {
self.warmup_samples.push(sample_us);
return;
}
if self.successes == WARM_THRESHOLD {
self.warmup_samples.push(sample_us);
let warmup_mean =
self.warmup_samples.iter().sum::<f64>() / self.warmup_samples.len() as f64;
if warmup_mean >= ESCAPE_US {
self.stop_timeout = true;
self.warmup_samples = Vec::new();
return;
}
let seed = percentile(&mut self.warmup_samples, 0.90).max(SEED_FLOOR_US);
self.warmup_samples = Vec::new();
self.deadline_us = Some(seed);
return;
}
if self.stop_timeout {
return;
}
let target = mean + EXPECTED_OFFSET_US;
let current = self.deadline_us.unwrap_or(target);
let next = (1.0 - VALUE_EWMA_ALPHA) * current + VALUE_EWMA_ALPHA * target;
self.deadline_us = Some(next);
}
fn record_timeout(&mut self) {
self.lifetime_timeouts += 1;
}
}
pub(crate) struct LatencyTracker {
controller: Mutex<DeadlineController>,
}
impl LatencyTracker {
pub(crate) fn new() -> Self {
Self {
controller: Mutex::new(DeadlineController::new()),
}
}
pub(crate) fn record(&self, latency: Duration) {
self.controller.lock().unwrap().record_success(latency);
}
#[cfg(test)]
pub(crate) fn record_timeout(&self) {
self.controller.lock().unwrap().record_timeout();
}
fn hedge_deadline(&self, allow_hedge: bool) -> Option<Duration> {
if !allow_hedge {
return None;
}
let c = self.controller.lock().unwrap();
c.can_hedge().then(|| c.deadline()).flatten()
}
#[cfg(test)]
pub(crate) fn timeout_count(&self) -> u64 {
self.controller.lock().unwrap().lifetime_timeouts
}
#[cfg(test)]
fn deadline(&self) -> Option<Duration> {
self.controller.lock().unwrap().deadline()
}
pub(crate) async fn guarded<T, E, Fut>(
&self,
allow_hedge: bool,
fut: Fut,
) -> Result<T, GuardError<E>>
where
Fut: Future<Output = Result<T, E>>,
{
let start = Instant::now();
match self.hedge_deadline(allow_hedge) {
None => match fut.await {
Ok(val) => {
self.record(start.elapsed());
Ok(val)
}
Err(e) => Err(GuardError::Inner(e)),
},
Some(dl) => match tokio::time::timeout(dl, fut).await {
Ok(Ok(val)) => {
self.record(start.elapsed());
Ok(val)
}
Ok(Err(e)) => Err(GuardError::Inner(e)),
Err(_timeout) => {
{
let mut c = self.controller.lock().unwrap();
c.charge_hedge();
c.record_timeout();
}
tracing::debug!(
target: crate::telemetry::TARGET_TRANSFER,
deadline_ms = dl.as_millis() as u64,
"request exceeded latency deadline"
);
Err(GuardError::DeadlineExceeded(dl))
}
},
}
}
}
impl std::fmt::Debug for LatencyTracker {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let c = self.controller.lock().unwrap();
f.debug_struct("LatencyTracker")
.field("successes", &c.successes)
.field("timeouts", &c.lifetime_timeouts)
.field("deadline", &c.deadline())
.finish()
}
}
#[cfg(test)]
mod tests {
use super::*;
fn dl_us(c: &DeadlineController) -> f64 {
c.deadline_us.expect("expected a warm deadline")
}
fn warm(c: &mut DeadlineController, latency: Duration, n: u64) {
for _ in 0..n {
c.record_success(latency);
}
}
#[test]
fn cold_until_warm_threshold() {
let mut c = DeadlineController::new();
warm(&mut c, Duration::from_millis(50), WARM_THRESHOLD - 1);
assert_eq!(c.deadline(), None, "must stay cold below WARM_THRESHOLD");
c.record_success(Duration::from_millis(50));
let seeded = dl_us(&c);
assert!(
(seeded - SEED_FLOOR_US).abs() < 1.0,
"a fast warmup must seed at the floor = {SEED_FLOOR_US}, got {seeded}"
);
}
#[test]
fn seed_tracks_the_warmup_tail_not_the_mean() {
let mut c = DeadlineController::new();
for _ in 0..WARM_THRESHOLD - 2 {
c.record_success(Duration::from_millis(100));
}
c.record_success(Duration::from_millis(3000));
c.record_success(Duration::from_millis(3000));
let seeded = dl_us(&c);
assert!(
(seeded - 3_000_000.0).abs() < 1.0,
"seed must be the p90 tail (3s), got {seeded}"
);
assert!(
seeded > 1_380_000.0,
"seed must exceed the mean-based value it replaces"
);
}
#[test]
fn slow_warmup_never_arms_a_deadline() {
let mut c = DeadlineController::new();
warm(&mut c, Duration::from_millis(5500), WARM_THRESHOLD);
assert_eq!(
c.deadline(),
None,
"a warmup mean past the escape must never arm a deadline"
);
assert!(c.stop_timeout, "the slow-link gate must latch");
warm(&mut c, Duration::from_millis(50), 500);
assert_eq!(c.deadline(), None, "stop_timeout is a permanent latch");
}
#[test]
fn cold_first_sample_does_not_latch_a_healthy_link() {
let mut c = DeadlineController::new();
c.record_success(Duration::from_millis(8000));
for _ in 0..WARM_THRESHOLD - 1 {
c.record_success(Duration::from_millis(50));
}
assert!(
!c.stop_timeout,
"a single cold-start sample must not latch a healthy link off"
);
assert!(
c.deadline().is_some(),
"a link whose arithmetic warmup mean is under the escape must be timed"
);
}
#[test]
fn ewma_seeds_on_first_sample_not_against_zero() {
let mut e = Ewma::new(0.01);
assert_eq!(e.observe(1234.0), 1234.0);
}
#[test]
fn success_relaxes_deadline_toward_target_slowly() {
let mut c = DeadlineController::new();
for _ in 0..WARM_THRESHOLD - 2 {
c.record_success(Duration::from_millis(100));
}
c.record_success(Duration::from_millis(3000));
c.record_success(Duration::from_millis(3000));
let seeded = dl_us(&c); assert!(
(seeded - 3_000_000.0).abs() < 1.0,
"precondition: p90 seed = 3s"
);
c.record_success(Duration::from_millis(100));
let after_one = dl_us(&c);
assert!(
after_one < seeded,
"a target below the seed must relax the deadline down"
);
let moved_fraction = (seeded - after_one) / seeded;
assert!(
moved_fraction < 0.02,
"one success should move the deadline <2%, moved {moved_fraction}"
);
warm(&mut c, Duration::from_millis(100), 2000);
let converged = dl_us(&c);
assert!(
(converged - (100_000.0 + EXPECTED_OFFSET_US)).abs() < 50_000.0,
"should converge near 100ms+700ms, got {converged}"
);
}
#[cfg_attr(miri, ignore)]
#[test]
fn shared_tracker_survives_concurrent_access() {
use std::sync::Arc;
use std::thread;
let tracker = Arc::new(LatencyTracker::new());
let threads = 8;
let timeouts_per_thread = 500;
let successes_per_thread = 500;
let handles: Vec<_> = (0..threads)
.map(|_| {
let t = tracker.clone();
thread::spawn(move || {
for _ in 0..successes_per_thread {
t.record(Duration::from_millis(100));
}
for _ in 0..timeouts_per_thread {
t.record_timeout();
}
})
})
.collect();
for h in handles {
h.join().expect("worker thread must not panic");
}
assert_eq!(
tracker.timeout_count(),
threads * timeouts_per_thread,
"every timeout must be counted exactly once under contention"
);
}
#[test]
fn timeout_never_moves_the_mean() {
let mut c = DeadlineController::new();
warm(&mut c, Duration::from_millis(100), WARM_THRESHOLD);
let mean_before = c.mean_ttfb.value.unwrap();
c.record_timeout();
assert_eq!(
c.mean_ttfb.value.unwrap(),
mean_before,
"a timeout must not feed the latency mean"
);
}
#[test]
fn hedge_budget_starts_full_and_caps_at_capacity() {
let mut c = DeadlineController::new();
assert_eq!(c.hedge_tokens, HEDGE_CAPACITY, "starts full");
warm(&mut c, Duration::from_millis(100), WARM_THRESHOLD + 50);
assert!(
c.hedge_tokens <= HEDGE_CAPACITY,
"reward must not exceed capacity, got {}",
c.hedge_tokens
);
}
#[test]
fn sustained_hedging_drains_the_budget_despite_successes() {
let mut c = DeadlineController::new();
warm(&mut c, Duration::from_millis(100), WARM_THRESHOLD);
for _ in 0..5 {
c.charge_hedge();
}
let mut rounds = 0;
while c.can_hedge() && rounds < 100 {
for _ in 0..5 {
c.record_success(Duration::from_millis(100));
}
c.charge_hedge();
rounds += 1;
}
assert!(
!c.can_hedge(),
"a candidate rate above the setpoint must drain the budget despite \
interleaved successes (drained in {rounds} rounds)"
);
}
#[test]
fn drained_budget_cannot_afford_a_hedge_until_successes_refill() {
let mut c = DeadlineController::new();
warm(&mut c, Duration::from_millis(100), WARM_THRESHOLD);
for _ in 0..(HEDGE_CAPACITY as u32 + 1) {
c.charge_hedge();
}
assert!(!c.can_hedge(), "a drained budget cannot afford a hedge");
let needed = (HEDGE_COST / HEDGE_REWARD).ceil() as u32;
for _ in 0..needed - 1 {
c.record_success(Duration::from_millis(100));
}
assert!(!c.can_hedge(), "one token needs ~cost/reward successes");
c.record_success(Duration::from_millis(100));
assert!(c.can_hedge(), "enough successes re-afford a hedge");
}
#[cfg_attr(miri, ignore)]
#[tokio::test(start_paused = true)]
async fn guarded_cold_applies_no_timeout() {
let tracker = LatencyTracker::new();
assert_eq!(tracker.deadline(), None);
let result = tracker
.guarded(true, async {
tokio::time::sleep(Duration::from_millis(100)).await;
Ok::<_, crate::error::Error>(42)
})
.await;
assert!(matches!(result, Ok(42)));
}
#[cfg_attr(miri, ignore)]
#[tokio::test(start_paused = true)]
async fn guarded_success_records_and_returns() {
let tracker = LatencyTracker::new();
for _ in 0..WARM_THRESHOLD - 1 {
tracker.record(Duration::from_millis(100));
}
assert_eq!(tracker.deadline(), None, "precondition: one sample short");
let result = tracker
.guarded(true, async { Ok::<_, crate::error::Error>(42) })
.await;
assert!(matches!(result, Ok(42)));
assert!(
tracker.deadline().is_some(),
"a guarded success must record its sample (warming the tracker)"
);
}
#[cfg_attr(miri, ignore)]
#[tokio::test(start_paused = true)]
async fn guarded_deadline_exceeded_records_timeout() {
let tracker = LatencyTracker::new();
for _ in 0..WARM_THRESHOLD {
tracker.record(Duration::from_millis(100));
}
let result: Result<(), _> = tracker
.guarded(true, async {
tokio::time::sleep(Duration::from_secs(30)).await;
Ok::<_, crate::error::Error>(())
})
.await;
assert!(matches!(result, Err(GuardError::DeadlineExceeded(_))));
assert_eq!(tracker.timeout_count(), 1);
}
#[cfg_attr(miri, ignore)]
#[tokio::test(start_paused = true)]
async fn guarded_disallowed_hedge_runs_untimed() {
let tracker = LatencyTracker::new();
for _ in 0..WARM_THRESHOLD {
tracker.record(Duration::from_millis(100));
}
let result = tracker
.guarded(false, async {
tokio::time::sleep(Duration::from_secs(30)).await;
Ok::<_, crate::error::Error>(7)
})
.await;
assert!(matches!(result, Ok(7)), "disallowed hedge must run untimed");
assert_eq!(tracker.timeout_count(), 0, "no hedge, no timeout");
}
#[cfg_attr(miri, ignore)]
#[tokio::test(start_paused = true)]
async fn guarded_drained_budget_runs_untimed() {
let tracker = LatencyTracker::new();
for _ in 0..WARM_THRESHOLD {
tracker.record(Duration::from_millis(100));
}
{
let mut c = tracker.controller.lock().unwrap();
for _ in 0..(HEDGE_CAPACITY as u32 + 1) {
c.charge_hedge();
}
assert!(!c.can_hedge(), "precondition: drained");
}
let result = tracker
.guarded(true, async {
tokio::time::sleep(Duration::from_secs(30)).await;
Ok::<_, crate::error::Error>(9)
})
.await;
assert!(matches!(result, Ok(9)), "drained budget must run untimed");
assert_eq!(tracker.timeout_count(), 0, "no hedge, no timeout");
}
#[cfg_attr(miri, ignore)]
#[tokio::test(start_paused = true)]
async fn guarded_inner_error_passed_through() {
let tracker = LatencyTracker::new();
let result: Result<(), _> = tracker
.guarded(true, async {
Err::<(), _>(crate::error::Error::new(
crate::error::ErrorKind::RuntimeError,
"simulated SDK error",
))
})
.await;
assert!(matches!(result, Err(GuardError::Inner(_))));
}
#[cfg_attr(miri, ignore)]
#[tokio::test(start_paused = true)]
async fn guarded_warm_inner_error_is_not_a_timeout() {
let tracker = LatencyTracker::new();
for _ in 0..WARM_THRESHOLD {
tracker.record(Duration::from_millis(100));
}
assert!(tracker.deadline().is_some(), "precondition: warm");
let result: Result<(), _> = tracker
.guarded(true, async {
Err::<(), _>(crate::error::Error::new(
crate::error::ErrorKind::RuntimeError,
"simulated SDK error",
))
})
.await;
assert!(matches!(result, Err(GuardError::Inner(_))));
assert_eq!(
tracker.timeout_count(),
0,
"a warm inner error must not be recorded as a timeout"
);
}
}