proxyhunt 0.1.1

A fast, modern proxy scraper and checker
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99

use std::time::{Duration, Instant};
use anyhow::Result;
use reqwest::{Client, Proxy as ReqwestProxy};
use crate::proxy::{Proxy, ProxyProto};
use crate::cli::CheckArgs;
use crate::enrich::{Enricher, Enrichment};
use futures::StreamExt;
use indicatif::{ProgressBar, ProgressStyle};
use serde::Serialize;

#[derive(Debug, Clone, Serialize)]
pub struct CheckedProxy {
    pub proxy: Proxy,
    pub latency: Duration,
    pub status: bool,
    pub enrichment: Option<Enrichment>,
}

pub struct Checker {
    args: CheckArgs,
    enricher: Enricher,
}

impl Checker {
    pub fn new(args: CheckArgs) -> Self {
        let enricher = Enricher::new(args.geoip_db.as_ref());
        Self { args, enricher }
    }

    pub async fn check_all(&self, proxies: Vec<Proxy>) -> Vec<CheckedProxy> {
        let pb = ProgressBar::new(proxies.len() as u64);
        pb.set_style(ProgressStyle::default_bar()
            .template("{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} ({eta})")
            .unwrap()
            .progress_chars("#>-"));

        let results = futures::stream::iter(proxies)
            .map(|p| {
                let args = self.args.clone();
                let pb = pb.clone();
                async move {
                    let res = self.check_proxy(p, &args).await;
                    pb.inc(1);
                    res
                }
            })
            .buffer_unordered(self.args.concurrency)
            .collect::<Vec<CheckedProxy>>()
            .await;

        pb.finish_with_message("Done");

        let mut successful: Vec<CheckedProxy> = results.into_iter().filter(|r| r.status).collect();
        successful.sort_by_key(|r| r.latency);
        successful
    }

    async fn check_proxy(&self, proxy: Proxy, args: &CheckArgs) -> CheckedProxy {
        let start = Instant::now();
        let status = self.test_connection(&proxy, args).await.is_ok();
        let latency = start.elapsed();

        let enrichment = if !args.no_enrich {
            self.enricher.enrich(&proxy.host)
        } else {
            None
        };

        CheckedProxy {
            proxy,
            latency,
            status,
            enrichment,
        }
    }

    async fn test_connection(&self, proxy: &Proxy, args: &CheckArgs) -> Result<()> {
        let proxy_url = proxy.full_url();
        let reqwest_proxy = match proxy.proto {
            ProxyProto::Http => ReqwestProxy::all(&proxy_url)?,
            ProxyProto::Socks4 => ReqwestProxy::all(&proxy_url)?, // Reqwest handles socks4://
            ProxyProto::Socks5 => ReqwestProxy::all(&proxy_url)?, // Reqwest handles socks5://
        };

        let client = Client::builder()
            .proxy(reqwest_proxy)
            .timeout(Duration::from_secs(args.timeout))
            .connect_timeout(Duration::from_secs(args.connect_timeout))
            .danger_accept_invalid_certs(true)
            .build()?;

        let resp = client.get(&args.check_url).send().await?;
        if resp.status().is_success() {
            Ok(())
        } else {
            Err(anyhow::anyhow!("Status: {}", resp.status()))
        }
    }
}