scatter_proxy/
config.rs

1use std::collections::HashMap;
2use std::path::PathBuf;
3use std::time::Duration;
4
5/// Main configuration for ScatterProxy.
6pub struct ScatterProxyConfig {
7    /// URLs of proxy sources (line-delimited `ip:port` or `socks5://ip:port`).
8    pub sources: Vec<String>,
9    /// How often to re-fetch proxy sources (default: 10 min).
10    pub source_refresh_interval: Duration,
11    /// Per-(proxy, host) rate-limiting configuration.
12    pub rate_limit: RateLimitConfig,
13    /// Timeout for a single proxy connection attempt (default: 8s).
14    pub proxy_timeout: Duration,
15    /// Overall timeout for a task from submission to final failure (default: 60s).
16    pub task_timeout: Duration,
17    /// Number of concurrent proxy paths raced per request (default: 3).
18    pub max_concurrent_per_request: usize,
19    /// Global in-flight concurrency limit (default: 100).
20    pub max_inflight: usize,
21    /// Maximum scheduling attempts per task before giving up (default: 5).
22    pub max_attempts: usize,
23    /// Maximum number of pending tasks in the pool (default: 1000).
24    pub task_pool_capacity: usize,
25    /// Sliding window size for health tracking (default: 30).
26    pub health_window: usize,
27    /// Base cooldown duration after consecutive failures (default: 30s).
28    pub cooldown_base: Duration,
29    /// Maximum cooldown duration (default: 300s).
30    pub cooldown_max: Duration,
31    /// Number of consecutive failures before entering cooldown (default: 3).
32    pub cooldown_consecutive_fails: usize,
33    /// Minimum samples required before a proxy can be evicted (default: 30).
34    pub eviction_min_samples: usize,
35    /// Number of target errors before tripping the host circuit breaker (default: 10).
36    pub circuit_breaker_threshold: usize,
37    /// Interval between probe requests when a circuit breaker is open (default: 30s).
38    pub circuit_breaker_probe_interval: Duration,
39    /// Optional file path for persisting proxy state as JSON.
40    pub state_file: Option<PathBuf>,
41    /// How often to save state to disk (default: 5 min).
42    pub state_save_interval: Duration,
43    /// How often to log the metrics summary line (default: 30s).
44    pub metrics_log_interval: Duration,
45    /// Whether to prefer remote DNS resolution through the SOCKS5 proxy (default: true).
46    pub prefer_remote_dns: bool,
47}
48
49impl Default for ScatterProxyConfig {
50    fn default() -> Self {
51        Self {
52            sources: Vec::new(),
53            source_refresh_interval: Duration::from_secs(600),
54            rate_limit: RateLimitConfig::default(),
55            proxy_timeout: Duration::from_secs(8),
56            task_timeout: Duration::from_secs(60),
57            max_concurrent_per_request: 3,
58            max_inflight: 100,
59            max_attempts: 5,
60            task_pool_capacity: 1000,
61            health_window: 30,
62            cooldown_base: Duration::from_secs(30),
63            cooldown_max: Duration::from_secs(300),
64            cooldown_consecutive_fails: 3,
65            eviction_min_samples: 30,
66            circuit_breaker_threshold: 10,
67            circuit_breaker_probe_interval: Duration::from_secs(30),
68            state_file: None,
69            state_save_interval: Duration::from_secs(300),
70            metrics_log_interval: Duration::from_secs(30),
71            prefer_remote_dns: true,
72        }
73    }
74}
75
76/// Per-(proxy, host) rate-limiting configuration.
77pub struct RateLimitConfig {
78    /// Default minimum interval between requests through the same proxy to the same host (default: 500ms).
79    pub default_interval: Duration,
80    /// Per-host overrides for the minimum interval.
81    pub host_overrides: HashMap<String, Duration>,
82}
83
84impl Default for RateLimitConfig {
85    fn default() -> Self {
86        Self {
87            default_interval: Duration::from_millis(500),
88            host_overrides: HashMap::new(),
89        }
90    }
91}
92
93#[cfg(test)]
94mod tests {
95    use super::*;
96
97    #[test]
98    fn test_scatter_proxy_config_defaults() {
99        let cfg = ScatterProxyConfig::default();
100
101        assert!(cfg.sources.is_empty());
102        assert_eq!(cfg.source_refresh_interval, Duration::from_secs(600));
103        assert_eq!(cfg.proxy_timeout, Duration::from_secs(8));
104        assert_eq!(cfg.task_timeout, Duration::from_secs(60));
105        assert_eq!(cfg.max_concurrent_per_request, 3);
106        assert_eq!(cfg.max_inflight, 100);
107        assert_eq!(cfg.max_attempts, 5);
108        assert_eq!(cfg.task_pool_capacity, 1000);
109        assert_eq!(cfg.health_window, 30);
110        assert_eq!(cfg.cooldown_base, Duration::from_secs(30));
111        assert_eq!(cfg.cooldown_max, Duration::from_secs(300));
112        assert_eq!(cfg.cooldown_consecutive_fails, 3);
113        assert_eq!(cfg.eviction_min_samples, 30);
114        assert_eq!(cfg.circuit_breaker_threshold, 10);
115        assert_eq!(cfg.circuit_breaker_probe_interval, Duration::from_secs(30));
116        assert!(cfg.state_file.is_none());
117        assert_eq!(cfg.state_save_interval, Duration::from_secs(300));
118        assert_eq!(cfg.metrics_log_interval, Duration::from_secs(30));
119        assert!(cfg.prefer_remote_dns);
120    }
121
122    #[test]
123    fn test_rate_limit_config_defaults() {
124        let rl = RateLimitConfig::default();
125
126        assert_eq!(rl.default_interval, Duration::from_millis(500));
127        assert!(rl.host_overrides.is_empty());
128    }
129
130    #[test]
131    fn test_rate_limit_config_nested_in_scatter_proxy_config() {
132        let cfg = ScatterProxyConfig::default();
133
134        assert_eq!(cfg.rate_limit.default_interval, Duration::from_millis(500));
135        assert!(cfg.rate_limit.host_overrides.is_empty());
136    }
137
138    #[test]
139    fn test_config_can_be_customised() {
140        let cfg = ScatterProxyConfig {
141            sources: vec!["https://example.com/proxies.txt".into()],
142            max_concurrent_per_request: 5,
143            max_inflight: 200,
144            max_attempts: 10,
145            state_file: Some(PathBuf::from("/tmp/scatter.json")),
146            prefer_remote_dns: false,
147            rate_limit: RateLimitConfig {
148                default_interval: Duration::from_millis(250),
149                host_overrides: {
150                    let mut m = HashMap::new();
151                    m.insert("slow.example.com".into(), Duration::from_secs(2));
152                    m
153                },
154            },
155            ..ScatterProxyConfig::default()
156        };
157
158        assert_eq!(cfg.sources.len(), 1);
159        assert_eq!(cfg.max_concurrent_per_request, 5);
160        assert_eq!(cfg.max_inflight, 200);
161        assert_eq!(cfg.max_attempts, 10);
162        assert_eq!(cfg.state_file, Some(PathBuf::from("/tmp/scatter.json")));
163        assert!(!cfg.prefer_remote_dns);
164        assert_eq!(cfg.rate_limit.default_interval, Duration::from_millis(250));
165        assert_eq!(
166            cfg.rate_limit.host_overrides.get("slow.example.com"),
167            Some(&Duration::from_secs(2))
168        );
169        // fields that should still have defaults
170        assert_eq!(cfg.proxy_timeout, Duration::from_secs(8));
171        assert_eq!(cfg.task_timeout, Duration::from_secs(60));
172        assert_eq!(cfg.health_window, 30);
173    }
174
175    #[test]
176    fn test_cooldown_max_gte_cooldown_base() {
177        let cfg = ScatterProxyConfig::default();
178        assert!(cfg.cooldown_max >= cfg.cooldown_base);
179    }
180
181    #[test]
182    fn test_source_refresh_interval_is_10_minutes() {
183        let cfg = ScatterProxyConfig::default();
184        assert_eq!(cfg.source_refresh_interval.as_secs(), 10 * 60);
185    }
186
187    #[test]
188    fn test_state_save_interval_is_5_minutes() {
189        let cfg = ScatterProxyConfig::default();
190        assert_eq!(cfg.state_save_interval.as_secs(), 5 * 60);
191    }
192}
scatter_proxy/config.rs

scatter_proxy/
config.rs
