use dashmap::DashMap;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::Semaphore;
use tracing::debug;
const MIN_DELAY_MS: u64 = 100;
const MAX_DELAY_MS: u64 = 30_000;
const DEFAULT_DELAY_MS: u64 = 200;
const ADDITIVE_INCREASE_MS: u64 = 20;
const MULTIPLICATIVE_DECREASE: f64 = 2.0;
#[derive(Debug, Clone)]
struct DomainRate {
current_delay_ms: u64,
last_request: Instant,
successes: u64,
rate_limits: u64,
errors: u64,
}
impl DomainRate {
fn new() -> Self {
Self {
current_delay_ms: DEFAULT_DELAY_MS,
last_request: Instant::now() - Duration::from_secs(60),
successes: 0,
rate_limits: 0,
errors: 0,
}
}
fn effective_delay(&self) -> Duration {
Duration::from_millis(self.current_delay_ms)
}
fn time_until_allowed(&self) -> Duration {
let elapsed = self.last_request.elapsed();
let delay = self.effective_delay();
if elapsed >= delay {
Duration::ZERO
} else {
delay - elapsed
}
}
fn mark_request_sent(&mut self) {
self.last_request = Instant::now();
}
fn record_success(&mut self) {
self.successes += 1;
self.current_delay_ms = self
.current_delay_ms
.saturating_sub(ADDITIVE_INCREASE_MS)
.max(MIN_DELAY_MS);
}
fn record_rate_limit(&mut self) {
self.rate_limits += 1;
self.current_delay_ms =
((self.current_delay_ms as f64 * MULTIPLICATIVE_DECREASE) as u64).min(MAX_DELAY_MS);
}
fn record_error(&mut self) {
self.errors += 1;
self.current_delay_ms = ((self.current_delay_ms as f64 * 1.5) as u64).min(MAX_DELAY_MS);
}
}
#[derive(Clone)]
pub struct AdaptiveRateLimiter {
domains: Arc<DashMap<String, DomainRate>>,
global_semaphore: Arc<Semaphore>,
}
impl AdaptiveRateLimiter {
pub fn new(max_global_concurrent: usize) -> Self {
Self {
domains: Arc::new(DashMap::new()),
global_semaphore: Arc::new(Semaphore::new(max_global_concurrent)),
}
}
pub async fn acquire(&self, domain: &str) -> RateLimitPermit {
let permit = self
.global_semaphore
.clone()
.acquire_owned()
.await
.expect("semaphore closed");
let wait_time = {
let entry = self
.domains
.entry(domain.to_string())
.or_insert_with(DomainRate::new);
let rate = entry.value();
rate.time_until_allowed()
};
if !wait_time.is_zero() {
debug!("Rate limiter: waiting {:?} for domain {domain}", wait_time);
tokio::time::sleep(wait_time).await;
}
if let Some(mut entry) = self.domains.get_mut(domain) {
entry.mark_request_sent();
}
RateLimitPermit {
_permit: permit,
domain: domain.to_string(),
limiter: self.clone(),
}
}
pub fn record_success(&self, domain: &str) {
if let Some(mut entry) = self.domains.get_mut(domain) {
entry.record_success();
debug!(
"Rate limiter {domain}: success (delay now {}ms)",
entry.current_delay_ms
);
}
}
pub fn record_rate_limit(&self, domain: &str) {
let entry = self
.domains
.entry(domain.to_string())
.or_insert_with(DomainRate::new);
let mut rate = entry;
rate.record_rate_limit();
debug!(
"Rate limiter {domain}: rate limited! (delay now {}ms)",
rate.current_delay_ms
);
}
pub fn record_error(&self, domain: &str) {
let entry = self
.domains
.entry(domain.to_string())
.or_insert_with(DomainRate::new);
let mut rate = entry;
rate.record_error();
debug!(
"Rate limiter {domain}: error (delay now {}ms)",
rate.current_delay_ms
);
}
pub fn domain_stats(&self, domain: &str) -> Option<DomainStats> {
self.domains.get(domain).map(|entry| DomainStats {
domain: domain.to_string(),
current_delay_ms: entry.current_delay_ms,
successes: entry.successes,
rate_limits: entry.rate_limits,
errors: entry.errors,
})
}
pub fn all_stats(&self) -> Vec<DomainStats> {
self.domains
.iter()
.map(|entry| DomainStats {
domain: entry.key().clone(),
current_delay_ms: entry.value().current_delay_ms,
successes: entry.value().successes,
rate_limits: entry.value().rate_limits,
errors: entry.value().errors,
})
.collect()
}
pub fn available_permits(&self) -> usize {
self.global_semaphore.available_permits()
}
}
impl Default for AdaptiveRateLimiter {
fn default() -> Self {
Self::new(50) }
}
pub struct RateLimitPermit {
_permit: tokio::sync::OwnedSemaphorePermit,
domain: String,
limiter: AdaptiveRateLimiter,
}
impl RateLimitPermit {
pub fn report_success(self) {
self.limiter.record_success(&self.domain);
}
pub fn report_rate_limited(self) {
self.limiter.record_rate_limit(&self.domain);
}
pub fn report_error(self) {
self.limiter.record_error(&self.domain);
}
}
#[derive(Debug, Clone)]
pub struct DomainStats {
pub domain: String,
pub current_delay_ms: u64,
pub successes: u64,
pub rate_limits: u64,
pub errors: u64,
}
impl DomainStats {
pub fn error_rate(&self) -> f64 {
let total = self.successes + self.rate_limits + self.errors;
if total == 0 {
0.0
} else {
(self.rate_limits + self.errors) as f64 / total as f64
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn rate_limiter_allows_requests() {
let rl = AdaptiveRateLimiter::new(10);
let permit = rl.acquire("example.com").await;
permit.report_success();
}
#[tokio::test]
async fn rate_limit_increases_delay() {
let rl = AdaptiveRateLimiter::new(10);
let permit = rl.acquire("slow.com").await;
permit.report_rate_limited();
let stats = rl.domain_stats("slow.com").unwrap();
assert!(stats.current_delay_ms > DEFAULT_DELAY_MS);
assert_eq!(stats.rate_limits, 1);
}
#[tokio::test]
async fn success_decreases_delay() {
let rl = AdaptiveRateLimiter::new(10);
let permit = rl.acquire("learn.com").await;
permit.report_rate_limited();
let high_delay = rl.domain_stats("learn.com").unwrap().current_delay_ms;
let permit = rl.acquire("learn.com").await;
permit.report_success();
let low_delay = rl.domain_stats("learn.com").unwrap().current_delay_ms;
assert!(low_delay < high_delay);
}
#[tokio::test]
async fn global_concurrency_limit() {
let rl = AdaptiveRateLimiter::new(2);
assert_eq!(rl.available_permits(), 2);
let _p1 = rl.acquire("a.com").await;
assert_eq!(rl.available_permits(), 1);
let _p2 = rl.acquire("b.com").await;
assert_eq!(rl.available_permits(), 0);
}
#[test]
fn domain_stats_error_rate() {
let stats = DomainStats {
domain: "test.com".into(),
current_delay_ms: 200,
successes: 8,
rate_limits: 1,
errors: 1,
};
assert!((stats.error_rate() - 0.2).abs() < 0.01);
}
}