1use dashmap::DashMap;
4use parking_lot::RwLock;
5use serde::{Deserialize, Serialize};
6use std::path::Path;
7use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
8use std::sync::Arc;
9use std::time::{Duration, Instant};
10use tracing::{debug, info, warn};
11
12#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
14#[serde(rename_all = "lowercase")]
15pub enum ProxyProtocol {
16 #[default]
17 Http,
18 Https,
19 Socks5,
20}
21
22#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
24#[serde(rename_all = "lowercase")]
25pub enum ProxyStatus {
26 Active,
28 Cooldown,
30 Disabled,
32 Dead,
34}
35
36#[derive(Debug, Serialize)]
38pub struct ProxyStats {
39 pub host: String,
40 pub port: u16,
41 pub status: ProxyStatus,
42 pub success_count: u64,
43 pub fail_count: u64,
44 pub total_requests: u64,
45 pub avg_latency_ms: u64,
46 pub success_rate: f64,
47 pub last_used_secs_ago: Option<u64>,
48 pub cooldown_remaining_secs: Option<u64>,
49}
50
51#[derive(Debug)]
53pub struct ProxyEntry {
54 pub host: String,
55 pub port: u16,
56 pub username: Option<String>,
57 pub password: Option<String>,
58 pub protocol: ProxyProtocol,
59 status: RwLock<ProxyStatus>,
61 success_count: AtomicU64,
62 fail_count: AtomicU64,
63 consecutive_fails: AtomicU64,
64 total_latency_ms: AtomicU64,
65 last_used: RwLock<Option<Instant>>,
66 cooldown_until: RwLock<Option<Instant>>,
67}
68
69impl ProxyEntry {
70 fn new(host: String, port: u16, username: Option<String>, password: Option<String>) -> Self {
71 Self {
72 host,
73 port,
74 username,
75 password,
76 protocol: ProxyProtocol::Http,
77 status: RwLock::new(ProxyStatus::Active),
78 success_count: AtomicU64::new(0),
79 fail_count: AtomicU64::new(0),
80 consecutive_fails: AtomicU64::new(0),
81 total_latency_ms: AtomicU64::new(0),
82 last_used: RwLock::new(None),
83 cooldown_until: RwLock::new(None),
84 }
85 }
86
87 pub fn url(&self) -> String {
89 let scheme = match self.protocol {
90 ProxyProtocol::Http => "http",
91 ProxyProtocol::Https => "https",
92 ProxyProtocol::Socks5 => "socks5",
93 };
94 match (&self.username, &self.password) {
95 (Some(user), Some(pass)) => {
96 format!("{scheme}://{user}:{pass}@{}:{}", self.host, self.port)
97 }
98 _ => format!("{scheme}://{}:{}", self.host, self.port),
99 }
100 }
101
102 fn is_available(&self) -> bool {
104 let status = *self.status.read();
105 if status == ProxyStatus::Disabled || status == ProxyStatus::Dead {
106 return false;
107 }
108 if status == ProxyStatus::Cooldown {
109 if let Some(until) = *self.cooldown_until.read() {
110 if Instant::now() < until {
111 return false;
112 }
113 *self.status.write() = ProxyStatus::Active;
115 *self.cooldown_until.write() = None;
116 }
117 }
118 true
119 }
120
121 pub fn record_success(&self, latency_ms: u64) {
123 self.success_count.fetch_add(1, Ordering::Relaxed);
124 self.consecutive_fails.store(0, Ordering::Relaxed);
125 self.total_latency_ms
126 .fetch_add(latency_ms, Ordering::Relaxed);
127 *self.last_used.write() = Some(Instant::now());
128 let status = *self.status.read();
130 if status == ProxyStatus::Cooldown {
131 *self.status.write() = ProxyStatus::Active;
132 *self.cooldown_until.write() = None;
133 }
134 }
135
136 pub fn record_failure(&self) {
138 self.fail_count.fetch_add(1, Ordering::Relaxed);
139 let consecutive = self.consecutive_fails.fetch_add(1, Ordering::Relaxed) + 1;
140 *self.last_used.write() = Some(Instant::now());
141
142 if consecutive >= 10 {
143 *self.status.write() = ProxyStatus::Dead;
145 warn!(
146 "Proxy {}:{} marked as dead after {} consecutive failures",
147 self.host, self.port, consecutive
148 );
149 } else if consecutive >= 3 {
150 let cooldown_secs = 30 * (1u64 << (consecutive - 3)).min(16); *self.status.write() = ProxyStatus::Cooldown;
153 *self.cooldown_until.write() =
154 Some(Instant::now() + Duration::from_secs(cooldown_secs));
155 debug!(
156 "Proxy {}:{} in cooldown for {}s after {} failures",
157 self.host, self.port, cooldown_secs, consecutive
158 );
159 }
160 }
161
162 fn stats(&self) -> ProxyStats {
164 let success = self.success_count.load(Ordering::Relaxed);
165 let fail = self.fail_count.load(Ordering::Relaxed);
166 let total = success + fail;
167 let total_lat = self.total_latency_ms.load(Ordering::Relaxed);
168 let avg_lat = if success > 0 { total_lat / success } else { 0 };
169 let rate = if total > 0 {
170 success as f64 / total as f64
171 } else {
172 1.0
173 };
174
175 let last_used_secs = self.last_used.read().map(|t| t.elapsed().as_secs());
176
177 let cooldown_remaining = self.cooldown_until.read().and_then(|until| {
178 let now = Instant::now();
179 if now < until {
180 Some((until - now).as_secs())
181 } else {
182 None
183 }
184 });
185
186 ProxyStats {
187 host: self.host.clone(),
188 port: self.port,
189 status: *self.status.read(),
190 success_count: success,
191 fail_count: fail,
192 total_requests: total,
193 avg_latency_ms: avg_lat,
194 success_rate: rate,
195 last_used_secs_ago: last_used_secs,
196 cooldown_remaining_secs: cooldown_remaining,
197 }
198 }
199
200 pub fn set_status(&self, status: ProxyStatus) {
202 *self.status.write() = status;
203 if status == ProxyStatus::Active {
204 self.consecutive_fails.store(0, Ordering::Relaxed);
205 *self.cooldown_until.write() = None;
206 }
207 }
208}
209
210#[derive(Clone)]
212pub struct ProxyPool {
213 inner: Arc<ProxyPoolInner>,
214}
215
216struct ProxyPoolInner {
217 proxies: RwLock<Vec<Arc<ProxyEntry>>>,
218 next_index: AtomicUsize,
220 domain_assignments: DashMap<String, usize>,
222}
223
224impl ProxyPool {
225 pub fn new() -> Self {
227 Self {
228 inner: Arc::new(ProxyPoolInner {
229 proxies: RwLock::new(Vec::new()),
230 next_index: AtomicUsize::new(0),
231 domain_assignments: DashMap::new(),
232 }),
233 }
234 }
235
236 pub fn load_from_file(path: &Path) -> std::io::Result<Self> {
244 let content = std::fs::read_to_string(path)?;
245 let pool = Self::new();
246 let mut count = 0;
247
248 for line in content.lines() {
249 let line = line.trim();
250 if line.is_empty() || line.starts_with('#') {
251 continue;
252 }
253
254 if let Some(entry) = Self::parse_proxy_line(line) {
255 pool.add_proxy(entry);
256 count += 1;
257 } else {
258 warn!("Failed to parse proxy line: {line}");
259 }
260 }
261
262 info!("Loaded {count} proxies from {}", path.display());
263 Ok(pool)
264 }
265
266 fn parse_proxy_line(line: &str) -> Option<ProxyEntry> {
268 if line.contains("://") {
270 let url = url::Url::parse(line).ok()?;
271 let host = url.host_str()?.to_string();
272 let port = url.port().unwrap_or(8080);
273 let username = if url.username().is_empty() {
274 None
275 } else {
276 Some(url.username().to_string())
277 };
278 let password = url.password().map(|s| s.to_string());
279 let mut entry = ProxyEntry::new(host, port, username, password);
280 entry.protocol = match url.scheme() {
281 "socks5" => ProxyProtocol::Socks5,
282 "https" => ProxyProtocol::Https,
283 _ => ProxyProtocol::Http,
284 };
285 return Some(entry);
286 }
287
288 let parts: Vec<&str> = line.split(':').collect();
290 match parts.len() {
291 2 => {
292 let port = parts[1].parse().ok()?;
293 Some(ProxyEntry::new(parts[0].to_string(), port, None, None))
294 }
295 4 => {
296 let port = parts[1].parse().ok()?;
297 Some(ProxyEntry::new(
298 parts[0].to_string(),
299 port,
300 Some(parts[2].to_string()),
301 Some(parts[3].to_string()),
302 ))
303 }
304 _ => None,
305 }
306 }
307
308 pub fn add_proxy(&self, entry: ProxyEntry) {
310 self.inner.proxies.write().push(Arc::new(entry));
311 }
312
313 pub fn next_proxy(&self) -> Option<Arc<ProxyEntry>> {
315 let proxies = self.inner.proxies.read();
316 if proxies.is_empty() {
317 return None;
318 }
319
320 let len = proxies.len();
321 for _ in 0..len {
323 let idx = self.inner.next_index.fetch_add(1, Ordering::Relaxed) % len;
324 let proxy = &proxies[idx];
325 if proxy.is_available() {
326 return Some(Arc::clone(proxy));
327 }
328 }
329
330 warn!("All proxies unavailable, attempting cooldown recovery");
332 proxies
333 .iter()
334 .find(|p| {
335 *p.status.read() != ProxyStatus::Dead && *p.status.read() != ProxyStatus::Disabled
336 })
337 .cloned()
338 }
339
340 pub fn proxy_for_domain(&self, domain: &str) -> Option<Arc<ProxyEntry>> {
343 let proxies = self.inner.proxies.read();
344 if proxies.is_empty() {
345 return None;
346 }
347
348 if let Some(idx) = self.inner.domain_assignments.get(domain) {
350 let idx = *idx % proxies.len();
351 let proxy = &proxies[idx];
352 if proxy.is_available() {
353 return Some(Arc::clone(proxy));
354 }
355 self.inner.domain_assignments.remove(domain);
357 }
358
359 drop(proxies);
361 if let Some(proxy) = self.next_proxy() {
362 let proxies = self.inner.proxies.read();
363 if let Some(idx) = proxies
364 .iter()
365 .position(|p| std::ptr::eq(p.as_ref(), proxy.as_ref()))
366 {
367 self.inner
368 .domain_assignments
369 .insert(domain.to_string(), idx);
370 }
371 Some(proxy)
372 } else {
373 None
374 }
375 }
376
377 pub fn fresh_proxy_for_domain(&self, domain: &str) -> Option<Arc<ProxyEntry>> {
379 self.inner.domain_assignments.remove(domain);
380 self.proxy_for_domain(domain)
381 }
382
383 pub fn stats(&self) -> Vec<ProxyStats> {
385 self.inner
386 .proxies
387 .read()
388 .iter()
389 .map(|p| p.stats())
390 .collect()
391 }
392
393 pub fn pool_summary(&self) -> serde_json::Value {
395 let proxies = self.inner.proxies.read();
396 let total = proxies.len();
397 let active = proxies.iter().filter(|p| p.is_available()).count();
398 let dead = proxies
399 .iter()
400 .filter(|p| *p.status.read() == ProxyStatus::Dead)
401 .count();
402 let cooldown = proxies
403 .iter()
404 .filter(|p| *p.status.read() == ProxyStatus::Cooldown)
405 .count();
406
407 let total_success: u64 = proxies
408 .iter()
409 .map(|p| p.success_count.load(Ordering::Relaxed))
410 .sum();
411 let total_fail: u64 = proxies
412 .iter()
413 .map(|p| p.fail_count.load(Ordering::Relaxed))
414 .sum();
415 let total_reqs = total_success + total_fail;
416 let overall_rate = if total_reqs > 0 {
417 total_success as f64 / total_reqs as f64
418 } else {
419 1.0
420 };
421
422 serde_json::json!({
423 "total_proxies": total,
424 "active": active,
425 "cooldown": cooldown,
426 "dead": dead,
427 "disabled": total - active - dead - cooldown,
428 "total_requests": total_reqs,
429 "total_success": total_success,
430 "total_failures": total_fail,
431 "overall_success_rate": format!("{:.1}%", overall_rate * 100.0),
432 })
433 }
434
435 pub fn len(&self) -> usize {
437 self.inner.proxies.read().len()
438 }
439
440 pub fn is_empty(&self) -> bool {
442 self.inner.proxies.read().is_empty()
443 }
444
445 pub fn reset_all(&self) {
447 for proxy in self.inner.proxies.read().iter() {
448 proxy.set_status(ProxyStatus::Active);
449 }
450 self.inner.domain_assignments.clear();
451 info!("Reset all proxies to active status");
452 }
453
454 pub fn purge_dead(&self) -> usize {
456 let mut proxies = self.inner.proxies.write();
457 let before = proxies.len();
458 proxies.retain(|p| *p.status.read() != ProxyStatus::Dead);
459 let removed = before - proxies.len();
460 if removed > 0 {
461 info!("Purged {removed} dead proxies");
462 self.inner.domain_assignments.clear();
463 }
464 removed
465 }
466
467 pub fn build_client_with_proxy(
469 proxy: &ProxyEntry,
470 user_agent: &str,
471 timeout: Duration,
472 ) -> Result<reqwest::Client, reqwest::Error> {
473 let proxy_url = proxy.url();
474 let reqwest_proxy = reqwest::Proxy::all(&proxy_url)?;
475
476 reqwest::Client::builder()
477 .user_agent(user_agent)
478 .timeout(timeout)
479 .connect_timeout(Duration::from_secs(5))
480 .proxy(reqwest_proxy)
481 .redirect(reqwest::redirect::Policy::limited(10))
482 .gzip(true)
483 .brotli(true)
484 .build()
485 }
486}
487
488impl Default for ProxyPool {
489 fn default() -> Self {
490 Self::new()
491 }
492}
493
494#[cfg(test)]
495mod tests {
496 use super::*;
497
498 #[test]
499 fn parse_webshare_format() {
500 let entry = ProxyPool::parse_proxy_line("31.59.20.176:6754:nxrendke:2yq1w9nkgstq").unwrap();
501 assert_eq!(entry.host, "31.59.20.176");
502 assert_eq!(entry.port, 6754);
503 assert_eq!(entry.username.as_deref(), Some("nxrendke"));
504 assert_eq!(entry.password.as_deref(), Some("2yq1w9nkgstq"));
505 }
506
507 #[test]
508 fn parse_simple_format() {
509 let entry = ProxyPool::parse_proxy_line("1.2.3.4:8080").unwrap();
510 assert_eq!(entry.host, "1.2.3.4");
511 assert_eq!(entry.port, 8080);
512 assert!(entry.username.is_none());
513 }
514
515 #[test]
516 fn parse_url_format() {
517 let entry = ProxyPool::parse_proxy_line("http://user:pass@1.2.3.4:1080").unwrap();
518 assert_eq!(entry.host, "1.2.3.4");
519 assert_eq!(entry.port, 1080);
520 assert_eq!(entry.protocol, ProxyProtocol::Http);
521 assert_eq!(entry.username.as_deref(), Some("user"));
522 }
523
524 #[test]
525 fn round_robin_rotation() {
526 let pool = ProxyPool::new();
527 pool.add_proxy(ProxyEntry::new("1.1.1.1".into(), 8080, None, None));
528 pool.add_proxy(ProxyEntry::new("2.2.2.2".into(), 8080, None, None));
529 pool.add_proxy(ProxyEntry::new("3.3.3.3".into(), 8080, None, None));
530
531 let p1 = pool.next_proxy().unwrap();
532 let p2 = pool.next_proxy().unwrap();
533 let p3 = pool.next_proxy().unwrap();
534 let p4 = pool.next_proxy().unwrap();
535
536 assert_eq!(p1.host, "1.1.1.1");
537 assert_eq!(p2.host, "2.2.2.2");
538 assert_eq!(p3.host, "3.3.3.3");
539 assert_eq!(p4.host, "1.1.1.1"); }
541
542 #[test]
543 fn health_tracking() {
544 let entry = ProxyEntry::new("1.1.1.1".into(), 8080, None, None);
545 assert!(entry.is_available());
546
547 entry.record_success(100);
548 entry.record_success(200);
549 assert_eq!(entry.success_count.load(Ordering::Relaxed), 2);
550
551 entry.record_failure();
553 entry.record_failure();
554 entry.record_failure();
555 assert_eq!(*entry.status.read(), ProxyStatus::Cooldown);
556 }
557
558 #[test]
559 fn dead_after_many_failures() {
560 let entry = ProxyEntry::new("1.1.1.1".into(), 8080, None, None);
561 for _ in 0..10 {
562 entry.record_failure();
563 }
564 assert_eq!(*entry.status.read(), ProxyStatus::Dead);
565 assert!(!entry.is_available());
566 }
567
568 #[test]
569 fn proxy_url_generation() {
570 let entry = ProxyEntry::new(
571 "1.2.3.4".into(),
572 8080,
573 Some("user".into()),
574 Some("pass".into()),
575 );
576 assert_eq!(entry.url(), "http://user:pass@1.2.3.4:8080");
577
578 let entry2 = ProxyEntry::new("5.6.7.8".into(), 3128, None, None);
579 assert_eq!(entry2.url(), "http://5.6.7.8:3128");
580 }
581
582 #[test]
583 fn load_from_file() {
584 let dir = tempfile::tempdir().unwrap();
585 let path = dir.path().join("proxies.txt");
586 std::fs::write(
587 &path,
588 "# Comment line\n1.1.1.1:8080:user:pass\n2.2.2.2:3128\n\n",
589 )
590 .unwrap();
591
592 let pool = ProxyPool::load_from_file(&path).unwrap();
593 assert_eq!(pool.len(), 2);
594 }
595
596 #[test]
597 fn pool_summary_json() {
598 let pool = ProxyPool::new();
599 pool.add_proxy(ProxyEntry::new("1.1.1.1".into(), 8080, None, None));
600 let summary = pool.pool_summary();
601 assert_eq!(summary["total_proxies"], 1);
602 assert_eq!(summary["active"], 1);
603 }
604
605 #[test]
606 fn skip_unavailable_in_rotation() {
607 let pool = ProxyPool::new();
608 pool.add_proxy(ProxyEntry::new("1.1.1.1".into(), 8080, None, None));
609 pool.add_proxy(ProxyEntry::new("2.2.2.2".into(), 8080, None, None));
610
611 {
613 let proxies = pool.inner.proxies.read();
614 proxies[0].set_status(ProxyStatus::Dead);
615 }
616
617 let p = pool.next_proxy().unwrap();
619 assert_eq!(p.host, "2.2.2.2");
620 }
621
622 #[test]
623 fn fresh_proxy_for_domain_rotates_assignment() {
624 let pool = ProxyPool::new();
625 pool.add_proxy(ProxyEntry::new("1.1.1.1".into(), 8080, None, None));
626 pool.add_proxy(ProxyEntry::new("2.2.2.2".into(), 8080, None, None));
627
628 let first = pool.proxy_for_domain("duckduckgo.com").unwrap();
629 let second = pool.fresh_proxy_for_domain("duckduckgo.com").unwrap();
630
631 assert_ne!(first.host, second.host);
632 }
633}