use std::time::{Duration, Instant};
use dashmap::DashMap;
use rand::RngExt;
pub const DEFAULT_MIN_INTERVAL: Duration = Duration::from_secs(1);
pub const DEFAULT_MAX_INTERVAL: Duration = Duration::from_secs(60);
pub const DEFAULT_COOLDOWN: Duration = Duration::from_secs(60);
pub const DEFAULT_BARE403_THRESHOLD: u32 = 3;
pub const DEFAULT_TARGET_CONCURRENCY: u32 = 8;
pub const DEFAULT_BACKOFF_BASE: Duration = Duration::from_secs(2);
pub const DEFAULT_BACKOFF_MAX: Duration = Duration::from_secs(60);
pub const DEFAULT_STALLED_THRESHOLD: u32 = 3;
const MAX_GATE_WAIT: Duration = Duration::from_secs(5);
const SWEEP_INTERVAL: Duration = Duration::from_secs(60);
#[derive(Debug, Clone)]
pub struct PolitenessOptions {
pub min_interval: Duration,
pub max_interval: Duration,
pub cooldown_default: Duration,
pub bare403_threshold: u32,
pub autothrottle_enabled: bool,
pub target_concurrency: u32,
pub backoff_base: Duration,
pub backoff_max: Duration,
pub stalled_threshold: u32,
pub jitter_ratio: f64,
}
impl Default for PolitenessOptions {
fn default() -> Self {
Self {
min_interval: DEFAULT_MIN_INTERVAL,
max_interval: DEFAULT_MAX_INTERVAL,
cooldown_default: DEFAULT_COOLDOWN,
bare403_threshold: DEFAULT_BARE403_THRESHOLD,
autothrottle_enabled: false,
target_concurrency: DEFAULT_TARGET_CONCURRENCY,
backoff_base: DEFAULT_BACKOFF_BASE,
backoff_max: DEFAULT_BACKOFF_MAX,
stalled_threshold: DEFAULT_STALLED_THRESHOLD,
jitter_ratio: 0.0,
}
}
}
#[derive(Debug, Clone)]
struct HostState {
next_allowed: Instant,
cooldown_until: Option<Instant>,
bare403_streak: u32,
tls_suspect: bool,
current_interval: Duration,
backoff_exponent: u32,
rate_limited_streak: u32,
stalled: bool,
crawl_delay: Option<Duration>,
}
impl HostState {
fn fresh(next_allowed: Instant, min_interval: Duration) -> Self {
Self {
next_allowed,
cooldown_until: None,
bare403_streak: 0,
tls_suspect: false,
current_interval: min_interval,
backoff_exponent: 0,
rate_limited_streak: 0,
stalled: false,
crawl_delay: None,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ObserveOutcome {
Success,
RateLimited {
retry_after: Option<Duration>,
},
Forbidden {
challenge: bool,
},
Failure,
}
#[derive(Debug)]
pub struct HostDiscipline {
state: DashMap<String, HostState>,
min_interval: Duration,
max_interval: Duration,
cooldown_default: Duration,
bare403_threshold: u32,
autothrottle_enabled: bool,
target_concurrency: u32,
backoff_base: Duration,
backoff_max: Duration,
stalled_threshold: u32,
jitter_ratio: f64,
last_sweep: parking_lot::Mutex<Instant>,
}
impl HostDiscipline {
pub fn new(min_interval: Duration, cooldown_default: Duration, bare403_threshold: u32) -> Self {
Self::with_options(PolitenessOptions {
min_interval,
max_interval: min_interval,
cooldown_default,
bare403_threshold,
autothrottle_enabled: false,
target_concurrency: DEFAULT_TARGET_CONCURRENCY,
backoff_base: Duration::ZERO,
backoff_max: DEFAULT_BACKOFF_MAX,
stalled_threshold: 0,
jitter_ratio: 0.0,
})
}
pub fn with_options(options: PolitenessOptions) -> Self {
assert!(
options.bare403_threshold >= 1,
"bare403_threshold must be >= 1, got {}",
options.bare403_threshold
);
assert!(
options.target_concurrency >= 1,
"target_concurrency must be >= 1, got {}",
options.target_concurrency
);
assert!(
options.max_interval >= options.min_interval,
"max_interval {:?} must be >= min_interval {:?}",
options.max_interval,
options.min_interval
);
Self {
state: DashMap::new(),
min_interval: options.min_interval,
max_interval: options.max_interval,
cooldown_default: options.cooldown_default,
bare403_threshold: options.bare403_threshold,
autothrottle_enabled: options.autothrottle_enabled,
target_concurrency: options.target_concurrency,
backoff_base: options.backoff_base,
backoff_max: options.backoff_max,
stalled_threshold: options.stalled_threshold,
jitter_ratio: options.jitter_ratio.clamp(0.0, 1.0),
last_sweep: parking_lot::Mutex::new(Instant::now()),
}
}
pub fn with_defaults() -> Self {
Self::with_options(PolitenessOptions::default())
}
pub fn from_politeness(settings: &crate::config::settings::PolitenessSettings) -> Self {
Self::with_options(PolitenessOptions {
min_interval: Duration::from_millis(settings.min_interval_ms),
max_interval: Duration::from_millis(settings.max_interval_ms),
cooldown_default: Duration::from_secs(settings.cooldown_default_secs),
bare403_threshold: settings.bare403_threshold,
autothrottle_enabled: settings.autothrottle_enabled,
target_concurrency: settings.target_concurrency,
backoff_base: Duration::from_secs(settings.backoff_base_secs),
backoff_max: Duration::from_secs(settings.backoff_max_secs),
stalled_threshold: settings.stalled_threshold,
jitter_ratio: settings.jitter_ratio,
})
}
pub async fn gate(&self, host: &str) {
self.maybe_sweep();
let now = Instant::now();
let release_at = {
let mut st = self
.state
.entry(host.to_string())
.or_insert_with(|| HostState::fresh(now, self.min_interval));
let step = st.current_interval.max(st.crawl_delay.unwrap_or_default());
let slot = st
.cooldown_until
.unwrap_or(now)
.max(st.next_allowed)
.max(now);
st.next_allowed = slot + step;
self.jitter_release(slot, st.current_interval)
};
let wait = release_at
.checked_duration_since(Instant::now())
.unwrap_or_default()
.min(MAX_GATE_WAIT);
if !wait.is_zero() {
tokio::time::sleep(wait).await;
}
}
fn jitter_release(&self, slot: Instant, interval: Duration) -> Instant {
if self.jitter_ratio <= 0.0 {
return slot;
}
let span = interval.mul_f64(self.jitter_ratio);
if span.is_zero() {
return slot;
}
let factor: f64 = rand::rng().random_range(-1.0..1.0);
let magnitude = span.mul_f64(factor.abs());
if factor >= 0.0 {
slot.checked_add(magnitude).unwrap_or(slot)
} else {
slot.checked_sub(magnitude).unwrap_or(slot)
}
}
pub fn observe(&self, host: &str, outcome: ObserveOutcome) {
self.maybe_sweep();
match outcome {
ObserveOutcome::Success => self.observe_success_inner(host, None),
ObserveOutcome::RateLimited { retry_after } => {
let now = Instant::now();
let retry = retry_after.unwrap_or_default();
let until_fresh = now + retry.max(self.cooldown_floor(0));
self.state
.entry(host.to_string())
.and_modify(|st| {
st.rate_limited_streak = st.rate_limited_streak.saturating_add(1);
if self.stalled_threshold > 0
&& st.rate_limited_streak >= self.stalled_threshold
{
st.stalled = true;
}
let in_window = st.cooldown_until.is_some_and(|until| now < until);
let n = st.backoff_exponent;
if !in_window {
st.backoff_exponent = n.saturating_add(1);
}
let until = now + retry.max(self.cooldown_floor(n));
st.cooldown_until = Some(if in_window {
st.cooldown_until.map_or(until, |old| old.max(until))
} else {
until
});
})
.or_insert_with(|| {
let mut st = HostState::fresh(now + self.min_interval, self.min_interval);
st.cooldown_until = Some(until_fresh);
st.backoff_exponent = 1;
st.rate_limited_streak = 1;
st.stalled = self.stalled_threshold == 1;
st
});
}
ObserveOutcome::Forbidden { challenge: false } => {
let threshold = self.bare403_threshold;
self.state
.entry(host.to_string())
.and_modify(|st| {
st.bare403_streak = st.bare403_streak.saturating_add(1);
if st.bare403_streak >= threshold {
st.tls_suspect = true;
}
})
.or_insert_with(|| {
let mut st =
HostState::fresh(Instant::now() + self.min_interval, self.min_interval);
st.bare403_streak = 1;
st.tls_suspect = threshold <= 1;
st
});
}
ObserveOutcome::Forbidden { challenge: true } | ObserveOutcome::Failure => {}
}
}
fn cooldown_floor(&self, exponent: u32) -> Duration {
if self.backoff_base.is_zero() {
return self.cooldown_default;
}
self.backoff_base
.checked_mul(1u32 << exponent.min(31))
.unwrap_or(self.backoff_max)
.min(self.backoff_max)
}
pub fn set_crawl_delay(&self, host: &str, delay: Duration) {
self.state
.entry(host.to_string())
.and_modify(|st| {
if st.crawl_delay.is_none() {
st.crawl_delay = Some(delay);
}
})
.or_insert_with(|| {
let mut st = HostState::fresh(Instant::now(), self.min_interval);
st.crawl_delay = Some(delay);
st
});
}
pub fn is_stalled(&self, host: &str) -> bool {
self.state.get(host).map(|st| st.stalled).unwrap_or(false)
}
pub fn observe_success_with_latency(&self, host: &str, latency: Duration) {
self.observe_success_inner(host, Some(latency));
}
fn observe_success_inner(&self, host: &str, latency: Option<Duration>) {
self.maybe_sweep();
let now = Instant::now();
if let Some(mut st) = self.state.get_mut(host) {
st.cooldown_until = None;
st.next_allowed = st.next_allowed.min(now);
st.bare403_streak = 0;
st.tls_suspect = false;
st.backoff_exponent = 0;
st.rate_limited_streak = 0;
st.stalled = false;
if let Some(latency) = latency {
if self.autothrottle_enabled {
let target = latency / self.target_concurrency;
let next = ((st.current_interval + target) / 2).max(target);
st.current_interval = next.clamp(self.min_interval, self.max_interval);
}
}
}
}
pub fn tls_suspect(&self, host: &str) -> bool {
self.state
.get(host)
.map(|st| st.tls_suspect)
.unwrap_or(false)
}
fn maybe_sweep(&self) {
let mut last = self.last_sweep.lock();
if last.elapsed() < SWEEP_INTERVAL {
return;
}
*last = Instant::now();
drop(last);
self.state
.retain(|_, st| !Self::is_stale(st, Instant::now()));
}
fn is_stale(st: &HostState, now: Instant) -> bool {
st.bare403_streak == 0
&& !st.tls_suspect
&& st.rate_limited_streak == 0
&& !st.stalled
&& st.crawl_delay.is_none()
&& st.cooldown_until.is_none_or(|until| until <= now)
&& st.next_allowed <= now
}
#[cfg(test)]
fn sweep_now(&self) {
self.state
.retain(|_, st| !Self::is_stale(st, Instant::now()));
}
}
impl Default for HostDiscipline {
fn default() -> Self {
Self::with_defaults()
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Arc;
fn discipline() -> HostDiscipline {
HostDiscipline::new(Duration::from_millis(100), Duration::from_secs(60), 3)
}
#[test]
#[should_panic(expected = "bare403_threshold must be >= 1")]
fn new_zero_threshold_panics() {
let _ = HostDiscipline::new(Duration::from_secs(1), Duration::from_secs(60), 0);
}
#[test]
fn tls_suspect_false_for_unknown_host() {
let d = discipline();
assert!(!d.tls_suspect("never-seen.example.com"));
}
#[tokio::test]
async fn gate_first_call_does_not_wait() {
let d = discipline();
let start = Instant::now();
d.gate("fresh.example.com").await;
let elapsed = start.elapsed();
assert!(
elapsed < Duration::from_millis(50),
"first gate must not wait, took {elapsed:?}"
);
}
#[tokio::test]
async fn gate_waits_min_interval_between_requests() {
let d = discipline();
d.gate("hot.example.com").await;
let start = Instant::now();
d.gate("hot.example.com").await;
let elapsed = start.elapsed();
assert!(
elapsed >= Duration::from_millis(90),
"second gate should wait ~min_interval, took {elapsed:?}"
);
assert!(
elapsed < Duration::from_millis(500),
"second gate wait should be bounded by min_interval, took {elapsed:?}"
);
}
#[tokio::test]
async fn gate_waits_until_cooldown_expiry() {
let d = HostDiscipline::new(Duration::from_millis(1), Duration::from_millis(100), 3);
d.gate("rl.example.com").await;
d.observe(
"rl.example.com",
ObserveOutcome::RateLimited {
retry_after: Some(Duration::from_millis(200)),
},
);
let start = Instant::now();
d.gate("rl.example.com").await;
let elapsed = start.elapsed();
assert!(
elapsed >= Duration::from_millis(180),
"gate should wait until cooldown expiry, took {elapsed:?}"
);
assert!(
elapsed < Duration::from_millis(600),
"gate wait should be bounded by cooldown, took {elapsed:?}"
);
}
fn cooldown_remaining(d: &HostDiscipline, host: &str) -> Duration {
let now = Instant::now();
d.state
.get(host)
.and_then(|st| st.cooldown_until)
.and_then(|until| until.checked_duration_since(now))
.unwrap_or_default()
}
#[test]
fn state_absent_for_unknown_host() {
let d = discipline();
assert!(
d.state.get("x.example.com").is_none(),
"未知 host 不应被预先建档"
);
}
#[test]
fn next_allowed_scheduled_min_interval_after_forbidden_creation() {
let d = HostDiscipline::new(Duration::from_millis(300), Duration::from_secs(60), 3);
d.observe(
"i.example.com",
ObserveOutcome::Forbidden { challenge: false },
);
let now = Instant::now();
let next_allowed = d.state.get("i.example.com").unwrap().next_allowed;
let remaining = next_allowed.checked_duration_since(now).unwrap_or_default();
assert!(
(Duration::from_millis(200)..=Duration::from_millis(300)).contains(&remaining),
"next_allowed should be ~min_interval ahead, got {remaining:?}"
);
}
#[test]
fn rate_limited_writes_cooldown_with_retry_after() {
let d = HostDiscipline::new(Duration::from_millis(1), Duration::from_secs(60), 3);
d.observe(
"rl.example.com",
ObserveOutcome::RateLimited {
retry_after: Some(Duration::from_secs(30)),
},
);
let wait = cooldown_remaining(&d, "rl.example.com");
assert!(
wait >= Duration::from_secs(29),
"RateLimited should write ~30s cooldown, got {wait:?}"
);
}
#[test]
fn rate_limited_without_retry_after_uses_default() {
let d = HostDiscipline::new(Duration::from_millis(1), Duration::from_secs(20), 3);
d.observe(
"rl.example.com",
ObserveOutcome::RateLimited { retry_after: None },
);
let wait = cooldown_remaining(&d, "rl.example.com");
assert!(
wait >= Duration::from_secs(19),
"RateLimited without header should fall back to cooldown_default, got {wait:?}"
);
}
#[test]
fn rate_limited_retry_after_below_default_takes_max() {
let d = HostDiscipline::new(Duration::from_millis(1), Duration::from_secs(20), 3);
d.observe(
"rl.example.com",
ObserveOutcome::RateLimited {
retry_after: Some(Duration::from_secs(2)),
},
);
let wait = cooldown_remaining(&d, "rl.example.com");
assert!(
wait >= Duration::from_secs(19),
"cooldown should be max(retry_after, default)=20s, got {wait:?}"
);
}
#[test]
fn gate_wait_clamped_to_five_seconds() {
let d = HostDiscipline::new(Duration::from_millis(1), Duration::from_secs(60), 3);
d.observe(
"clamped.example.com",
ObserveOutcome::RateLimited {
retry_after: Some(Duration::from_secs(600)),
},
);
let raw = cooldown_remaining(&d, "clamped.example.com");
assert!(
raw > MAX_GATE_WAIT,
"raw cooldown should exceed the clamp for this scenario"
);
assert_eq!(
raw.min(MAX_GATE_WAIT),
MAX_GATE_WAIT,
"gate wait must be clamped to 5s"
);
}
#[test]
fn bare403_streak_triggers_tls_suspect_then_success_clears() {
let d = discipline();
d.observe(
"waf.example.com",
ObserveOutcome::Forbidden { challenge: false },
);
d.observe(
"waf.example.com",
ObserveOutcome::Forbidden { challenge: false },
);
assert!(
!d.tls_suspect("waf.example.com"),
"2 strikes below threshold=3 must not set tls_suspect"
);
d.observe(
"waf.example.com",
ObserveOutcome::Forbidden { challenge: false },
);
assert!(
d.tls_suspect("waf.example.com"),
"3 consecutive bare 403s must set tls_suspect"
);
d.observe("waf.example.com", ObserveOutcome::Success);
assert!(
!d.tls_suspect("waf.example.com"),
"Success must clear tls_suspect"
);
}
#[test]
fn challenge403_and_failure_do_not_count_streak() {
let d = discipline();
for _ in 0..5 {
d.observe(
"c.example.com",
ObserveOutcome::Forbidden { challenge: true },
);
d.observe("c.example.com", ObserveOutcome::Failure);
}
assert!(
!d.tls_suspect("c.example.com"),
"challenge 403 / Failure must not advance the streak"
);
}
#[test]
fn success_clears_cooldown() {
let d = HostDiscipline::new(Duration::from_millis(1), Duration::from_secs(60), 3);
d.observe(
"ok.example.com",
ObserveOutcome::RateLimited {
retry_after: Some(Duration::from_secs(30)),
},
);
assert!(
cooldown_remaining(&d, "ok.example.com") >= Duration::from_secs(29),
"precondition: host should be cooling down"
);
d.observe("ok.example.com", ObserveOutcome::Success);
let st = d.state.get("ok.example.com").unwrap();
assert!(st.cooldown_until.is_none(), "Success must clear cooldown");
assert!(
st.next_allowed <= Instant::now(),
"Success must collapse the schedule so later gates are not held back"
);
}
#[tokio::test]
async fn concurrent_gate_release_times_spaced_by_min_interval() {
let interval = Duration::from_millis(40);
let d = Arc::new(HostDiscipline::new(interval, Duration::from_secs(60), 3));
let n = 12u32;
let host = "burst.example.com";
d.gate(host).await;
let pre = d.state.get(host).unwrap().next_allowed;
let mut handles = Vec::new();
for _ in 0..n {
let dd = Arc::clone(&d);
handles.push(tokio::spawn(async move {
dd.gate(host).await;
Instant::now()
}));
}
let mut released: Vec<Instant> = Vec::new();
for handle in handles {
released.push(handle.await.expect("task join failed"));
}
let post = d.state.get(host).unwrap().next_allowed;
assert_eq!(
post - pre,
interval * n,
"{n} concurrent gates must each reserve exactly one slot (timetable advanced {:+?})",
post - pre
);
for (i, released_at) in released.iter().enumerate() {
assert!(
released_at >= &pre,
"release #{i} at {released_at:?} must not precede timetable base {pre:?}"
);
}
}
#[tokio::test]
async fn concurrent_gate_and_observe_no_deadlock() {
let d = Arc::new(HostDiscipline::new(
Duration::from_millis(1),
Duration::from_millis(1),
3,
));
d.gate("hot.example.com").await;
let mut handles = Vec::new();
for _ in 0..8 {
let dd = Arc::clone(&d);
handles.push(tokio::spawn(async move {
for _ in 0..25 {
dd.gate("hot.example.com").await;
dd.observe(
"hot.example.com",
ObserveOutcome::Forbidden { challenge: false },
);
let _ = dd.tls_suspect("hot.example.com");
}
}));
}
tokio::time::timeout(Duration::from_secs(10), futures::future::join_all(handles))
.await
.expect("concurrent gate/observe/tls_suspect must not deadlock");
}
#[test]
fn test_sweep_removes_stale_keeps_disciplined_entries() {
let d = HostDiscipline::with_defaults();
let now = Instant::now();
d.state.insert(
"stale.example".to_string(),
HostState::fresh(now - Duration::from_secs(1), Duration::from_secs(1)),
);
d.state.insert(
"suspect.example".to_string(),
HostState {
tls_suspect: true,
..HostState::fresh(now - Duration::from_secs(1), Duration::from_secs(1))
},
);
d.state.insert(
"cooldown.example".to_string(),
HostState {
cooldown_until: Some(now + Duration::from_secs(30)),
..HostState::fresh(now - Duration::from_secs(1), Duration::from_secs(1))
},
);
d.sweep_now();
assert!(
!d.state.contains_key("stale.example"),
"stale entry must be evicted"
);
assert!(
d.state.contains_key("suspect.example"),
"tls_suspect entry must persist"
);
assert!(
d.state.contains_key("cooldown.example"),
"cooling entry must persist until cooldown expires"
);
}
#[test]
fn test_sweep_collects_success_reset_entry() {
let d = HostDiscipline::with_defaults();
let now = Instant::now();
d.state.insert(
"recovered.example".to_string(),
HostState::fresh(now, Duration::from_secs(1)),
);
d.sweep_now();
assert!(
!d.state.contains_key("recovered.example"),
"fully reset entry is equivalent to absent and must be collected"
);
}
fn opts_throttle_off(min_interval: Duration) -> PolitenessOptions {
PolitenessOptions {
min_interval,
max_interval: Duration::from_secs(60),
cooldown_default: Duration::from_secs(60),
bare403_threshold: 3,
autothrottle_enabled: false,
target_concurrency: 8,
backoff_base: Duration::ZERO,
backoff_max: Duration::from_secs(60),
stalled_threshold: 0,
jitter_ratio: 0.0,
}
}
fn opts_throttle_on(min_interval: Duration) -> PolitenessOptions {
PolitenessOptions {
autothrottle_enabled: true,
..opts_throttle_off(min_interval)
}
}
fn current_interval_of(d: &HostDiscipline, host: &str) -> Duration {
d.state
.get(host)
.map(|st| st.current_interval)
.unwrap_or_else(|| panic!("host {host} must have an entry"))
}
#[tokio::test]
async fn autothrottle_off_gate_interval_stays_min_interval() {
let min = Duration::from_millis(100);
let d = HostDiscipline::with_options(opts_throttle_off(min));
d.gate("t.example.com").await; for _ in 0..3 {
d.observe_success_with_latency("t.example.com", Duration::from_millis(1));
}
assert_eq!(
current_interval_of(&d, "t.example.com"),
min,
"autothrottle off must never move current_interval off min_interval"
);
d.gate("t.example.com").await;
let start = Instant::now();
d.gate("t.example.com").await;
let elapsed = start.elapsed();
assert!(
elapsed >= Duration::from_millis(90),
"gate gap must stay at min_interval, took {elapsed:?}"
);
assert!(
elapsed < Duration::from_millis(500),
"gate gap must not exceed min_interval, took {elapsed:?}"
);
}
#[tokio::test]
async fn autothrottle_converges_down_then_floors_at_min_interval() {
let min = Duration::from_millis(100);
let d = HostDiscipline::with_options(opts_throttle_on(min));
let host = "fast.example.com";
d.gate(host).await; d.observe_success_with_latency(host, Duration::from_secs(4));
assert_eq!(
current_interval_of(&d, host),
Duration::from_millis(500),
"first slow observation must raise interval to max(ema, target)"
);
let mut prev = current_interval_of(&d, host);
for _ in 0..3 {
d.observe_success_with_latency(host, Duration::from_millis(40));
let cur = current_interval_of(&d, host);
assert!(
cur < prev,
"interval must decrease monotonically on fast observations, {prev:?} -> {cur:?}"
);
assert!(
cur >= min,
"interval must never drop below min_interval, got {cur:?}"
);
prev = cur;
}
assert_eq!(
prev, min,
"EMA must floor at min_interval after enough fast observations"
);
}
#[tokio::test]
async fn autothrottle_converges_up_on_slow_responses() {
let d = HostDiscipline::with_options(opts_throttle_on(Duration::from_millis(100)));
let host = "slow.example.com";
d.gate(host).await; d.observe_success_with_latency(host, Duration::from_secs(4)); let after_first = current_interval_of(&d, host);
assert!(
after_first > Duration::from_millis(100),
"slow observation must raise interval above min, got {after_first:?}"
);
d.observe_success_with_latency(host, Duration::from_secs(40)); let after_second = current_interval_of(&d, host);
assert!(
after_second > after_first,
"larger latency must keep raising interval, {after_first:?} -> {after_second:?}"
);
assert!(
after_second <= Duration::from_secs(60),
"interval must stay clamped to max_interval, got {after_second:?}"
);
}
#[tokio::test]
async fn autothrottle_update_never_below_target() {
let d = HostDiscipline::with_options(opts_throttle_on(Duration::from_millis(100)));
let host = "bias.example.com";
d.gate(host).await; let latency = Duration::from_millis(40);
for _ in 0..5 {
d.observe_success_with_latency(host, latency);
let target = latency / 8;
assert!(
current_interval_of(&d, host) >= target,
"EMA bias: interval must stay >= target"
);
}
}
#[tokio::test]
async fn non_success_outcomes_leave_current_interval_unchanged() {
let d = HostDiscipline::with_options(opts_throttle_on(Duration::from_millis(100)));
let host = "err.example.com";
d.gate(host).await; d.observe_success_with_latency(host, Duration::from_secs(4)); let before = current_interval_of(&d, host);
d.observe(
host,
ObserveOutcome::RateLimited {
retry_after: Some(Duration::from_secs(30)),
},
);
d.observe(host, ObserveOutcome::Forbidden { challenge: false });
d.observe(host, ObserveOutcome::Forbidden { challenge: true });
d.observe(host, ObserveOutcome::Failure);
assert_eq!(
current_interval_of(&d, host),
before,
"error outcomes must not participate in latency convergence"
);
}
#[tokio::test]
async fn jitter_zero_releases_deterministically() {
let interval = Duration::from_millis(40);
let mut opts = opts_throttle_on(interval);
opts.jitter_ratio = 0.0;
let d = HostDiscipline::with_options(opts);
let host = "det.example.com";
let mut prev_slot = None;
for _ in 0..5 {
d.gate(host).await;
let slot = d.state.get(host).unwrap().next_allowed - interval;
if let Some(prev) = prev_slot {
assert_eq!(
slot - prev,
interval,
"no-jitter timetable must advance exactly one interval per release"
);
}
prev_slot = Some(slot);
}
}
#[tokio::test]
async fn jitter_half_randomizes_release_within_band() {
let interval = Duration::from_millis(40);
let mut opts = opts_throttle_on(interval);
opts.jitter_ratio = 0.5;
let d = Arc::new(HostDiscipline::with_options(opts));
let n = 16u32;
let mut handles = Vec::new();
for _ in 0..n {
let dd = Arc::clone(&d);
handles.push(tokio::spawn(async move {
dd.gate("jit.example.com").await;
Instant::now()
}));
}
let mut released: Vec<Instant> = Vec::new();
for handle in handles {
released.push(handle.await.expect("task join failed"));
}
released.sort();
let tolerance = Duration::from_millis(15);
let mut gaps = Vec::new();
for (i, pair) in released.windows(2).enumerate() {
let gap = pair[1].duration_since(pair[0]);
assert!(
gap <= interval * 2 + tolerance,
"jittered release #{i} gap {gap:?} must stay within [0,2]×interval"
);
gaps.push(gap);
}
let mean = gaps.iter().sum::<Duration>() / gaps.len() as u32;
let lo = interval * 7 / 10;
let hi = interval * 13 / 10;
assert!(
(lo..=hi).contains(&mean),
"mean gap {mean:?} must converge to interval {interval:?} (band {lo:?}..{hi:?})"
);
}
#[tokio::test]
async fn concurrent_gate_with_dynamic_interval_stays_spaced() {
let interval = Duration::from_millis(40);
let d = Arc::new(HostDiscipline::with_options(opts_throttle_on(interval)));
let host = "dyn.example.com";
d.gate(host).await;
d.observe_success_with_latency(host, Duration::from_millis(640));
let raised = current_interval_of(&d, host);
assert_eq!(
raised,
Duration::from_millis(80),
"precondition: interval raised to max(ema, target)"
);
let n = 8u32;
let mut handles = Vec::new();
for _ in 0..n {
let dd = Arc::clone(&d);
handles.push(tokio::spawn(async move {
dd.gate(host).await;
Instant::now()
}));
}
let mut released: Vec<Instant> = Vec::new();
for handle in handles {
released.push(handle.await.expect("task join failed"));
}
released.sort();
let jitter = Duration::from_millis(25);
for (i, pair) in released.windows(2).enumerate() {
let gap = pair[1].duration_since(pair[0]);
assert!(
gap + jitter >= raised,
"dynamic-interval release #{i} gap {gap:?} must be >= raised interval {raised:?}"
);
}
}
fn opts_backoff(base_ms: u64, max_ms: u64, stalled_threshold: u32) -> PolitenessOptions {
PolitenessOptions {
min_interval: Duration::from_millis(1),
max_interval: Duration::from_secs(60),
cooldown_default: Duration::from_secs(60),
bare403_threshold: 3,
autothrottle_enabled: false,
target_concurrency: 8,
backoff_base: Duration::from_millis(base_ms),
backoff_max: Duration::from_millis(max_ms),
stalled_threshold,
jitter_ratio: 0.0,
}
}
fn backoff_exponent_of(d: &HostDiscipline, host: &str) -> u32 {
d.state
.get(host)
.map(|st| st.backoff_exponent)
.unwrap_or_else(|| panic!("host {host} must have an entry"))
}
#[tokio::test]
async fn rate_limited_backoff_escalates_exponentially() {
let d = HostDiscipline::with_options(opts_backoff(30, 60_000, 0));
let host = "esc.example.com";
d.gate(host).await;
d.observe(host, ObserveOutcome::RateLimited { retry_after: None });
assert_eq!(backoff_exponent_of(&d, host), 1);
let wait1 = cooldown_remaining(&d, host);
assert!(
(Duration::from_millis(20)..=Duration::from_millis(45)).contains(&wait1),
"1st cooldown should be ~base (30ms), got {wait1:?}"
);
tokio::time::sleep(Duration::from_millis(50)).await;
d.observe(host, ObserveOutcome::RateLimited { retry_after: None });
assert_eq!(backoff_exponent_of(&d, host), 2);
let wait2 = cooldown_remaining(&d, host);
assert!(
(Duration::from_millis(50)..=Duration::from_millis(75)).contains(&wait2),
"2nd cooldown should be ~2×base (60ms), got {wait2:?}"
);
tokio::time::sleep(Duration::from_millis(80)).await;
d.observe(host, ObserveOutcome::RateLimited { retry_after: None });
assert_eq!(backoff_exponent_of(&d, host), 3);
let wait3 = cooldown_remaining(&d, host);
assert!(
(Duration::from_millis(110)..=Duration::from_millis(135)).contains(&wait3),
"3rd cooldown should be ~4×base (120ms), got {wait3:?}"
);
}
#[tokio::test]
async fn rate_limited_burst_within_window_does_not_advance_exponent() {
let d = HostDiscipline::with_options(opts_backoff(2_000, 60_000, 0));
let host = "burst.example.com";
d.gate(host).await;
d.observe(host, ObserveOutcome::RateLimited { retry_after: None });
let until_first = d.state.get(host).unwrap().cooldown_until.unwrap();
assert_eq!(backoff_exponent_of(&d, host), 1);
d.observe(host, ObserveOutcome::RateLimited { retry_after: None });
assert_eq!(
backoff_exponent_of(&d, host),
1,
"in-window repeat 429 must not advance the exponent"
);
let st = d.state.get(host).unwrap();
assert!(
st.cooldown_until.unwrap() >= until_first,
"burst refresh must not shorten the cooldown window"
);
}
#[tokio::test]
async fn rate_limited_retry_after_dominates_exponential() {
let d = HostDiscipline::with_options(opts_backoff(30, 60_000, 0));
let host = "ra.example.com";
d.gate(host).await;
d.observe(
host,
ObserveOutcome::RateLimited {
retry_after: Some(Duration::from_secs(30)),
},
);
let wait = cooldown_remaining(&d, host);
assert!(
wait >= Duration::from_secs(29),
"Retry-After must dominate the exponential floor, got {wait:?}"
);
}
#[tokio::test]
async fn rate_limited_backoff_caps_at_max() {
let d = HostDiscipline::with_options(opts_backoff(10, 40, 0));
let host = "cap.example.com";
d.gate(host).await;
for i in 0..8 {
tokio::time::sleep(Duration::from_millis(60)).await;
d.observe(host, ObserveOutcome::RateLimited { retry_after: None });
let wait = cooldown_remaining(&d, host);
assert!(
wait <= Duration::from_millis(60),
"hit #{i}: cooldown {wait:?} must be capped at backoff_max (40ms)"
);
}
}
#[tokio::test]
async fn success_resets_backoff_exponent() {
let d = HostDiscipline::with_options(opts_backoff(30, 60_000, 0));
let host = "reset.example.com";
d.gate(host).await;
d.observe(host, ObserveOutcome::RateLimited { retry_after: None });
assert_eq!(backoff_exponent_of(&d, host), 1);
d.observe(host, ObserveOutcome::Success);
assert_eq!(
backoff_exponent_of(&d, host),
0,
"Success must clear the backoff exponent"
);
d.observe(host, ObserveOutcome::RateLimited { retry_after: None });
let wait = cooldown_remaining(&d, host);
assert!(
(Duration::from_millis(20)..=Duration::from_millis(45)).contains(&wait),
"post-reset 429 must restart from base (~30ms), got {wait:?}"
);
}
#[test]
fn set_crawl_delay_first_value_wins() {
let d = HostDiscipline::with_defaults();
let host = "cd.example.com";
d.set_crawl_delay(host, Duration::from_secs(5));
d.set_crawl_delay(host, Duration::from_secs(1));
let stored = d
.state
.get(host)
.and_then(|st| st.crawl_delay)
.expect("crawl_delay must be stored");
assert_eq!(
stored,
Duration::from_secs(5),
"first crawl-delay must win, later values must not regress it"
);
}
#[tokio::test]
async fn crawl_delay_extends_gate_step() {
let d = HostDiscipline::with_options(opts_throttle_off(Duration::from_millis(100)));
let host = "cdgate.example.com";
d.set_crawl_delay(host, Duration::from_millis(300));
d.gate(host).await;
let start = Instant::now();
d.gate(host).await;
let elapsed = start.elapsed();
assert!(
elapsed >= Duration::from_millis(280),
"gate step must honor crawl-delay (300ms) over min_interval (100ms), took {elapsed:?}"
);
assert!(
elapsed < Duration::from_millis(600),
"gate step must not exceed crawl-delay, took {elapsed:?}"
);
}
#[tokio::test]
async fn crawl_delay_and_autothrottle_take_max() {
let d = HostDiscipline::with_options(opts_throttle_on(Duration::from_millis(100)));
let host = "cdat.example.com";
d.gate(host).await;
d.observe_success_with_latency(host, Duration::from_secs(4));
d.set_crawl_delay(host, Duration::from_millis(200));
d.gate(host).await;
let start = Instant::now();
d.gate(host).await;
let elapsed = start.elapsed();
assert!(
elapsed >= Duration::from_millis(480),
"autothrottle interval (500ms) must dominate crawl-delay (200ms), took {elapsed:?}"
);
}
#[test]
fn set_crawl_delay_creates_persistent_entry_lazily() {
let d = HostDiscipline::with_defaults();
let host = "lazy.example.com";
assert!(d.state.get(host).is_none(), "unknown host starts absent");
d.set_crawl_delay(host, Duration::from_secs(2));
assert!(
d.state.get(host).is_some(),
"set_crawl_delay must create the entry lazily"
);
d.sweep_now();
assert!(
d.state.contains_key(host),
"crawl-delay entry must survive sweep (legal constraint, not health signal)"
);
}
#[tokio::test]
async fn stalled_flag_set_at_threshold_and_cleared_by_success() {
let d = HostDiscipline::with_options(opts_backoff(30, 60_000, 3));
let host = "stall.example.com";
d.gate(host).await;
d.observe(host, ObserveOutcome::RateLimited { retry_after: None });
d.observe(host, ObserveOutcome::RateLimited { retry_after: None });
assert!(
!d.is_stalled(host),
"2 consecutive 429s below threshold=3 must not stall"
);
d.observe(host, ObserveOutcome::RateLimited { retry_after: None });
assert!(
d.is_stalled(host),
"3 consecutive 429s must trip the stalled flag"
);
d.observe(host, ObserveOutcome::Success);
assert!(!d.is_stalled(host), "Success must clear the stalled flag");
}
#[tokio::test]
async fn stalled_detection_disabled_at_zero_threshold() {
let d = HostDiscipline::with_options(opts_backoff(30, 60_000, 0));
let host = "nostall.example.com";
d.gate(host).await;
for _ in 0..5 {
d.observe(host, ObserveOutcome::RateLimited { retry_after: None });
tokio::time::sleep(Duration::from_millis(40)).await;
}
assert!(
!d.is_stalled(host),
"threshold=0 must disable stall detection"
);
}
}