Skip to main content

fetchium_core/proxy/
dataimpulse.rs

1//! DataImpulse residential proxy gateway client.
2//!
3//! Single gateway endpoint with country-targeted username suffix:
4//!   `{user}__cr.{cc}:{pass}@gw.dataimpulse.com:823`
5//!
6//! ## GB efficiency (pay-per-GB)
7//! - Clients cached per country code — built once, reused across requests
8//! - gzip + brotli compression enabled on all requests
9//! - Only used for scraper backends that are blocked without residential IPs
10//!   (Google, DDG, Bing, Brave). API backends (Serper, Exa, Tavily) are never proxied.
11//! - Response body capped at 512 KB — SERP HTML is ~50-150 KB gzip'd
12
13use dashmap::DashMap;
14use reqwest::Client;
15use std::sync::Arc;
16use std::time::Duration;
17use tracing::{debug, warn};
18
19/// DataImpulse residential proxy gateway client with country targeting.
20#[derive(Clone)]
21pub struct DataImpulseClient {
22    inner: Arc<DataImpulseInner>,
23}
24
25struct DataImpulseInner {
26    username: String,
27    password: String,
28    host: String,
29    port: u16,
30    user_agent: String,
31    timeout: Duration,
32    /// Cached clients keyed by country code ("" = random/default).
33    clients: DashMap<String, Client>,
34}
35
36impl DataImpulseClient {
37    /// Build a DataImpulse client from credentials.
38    pub fn new(
39        username: impl Into<String>,
40        password: impl Into<String>,
41        host: impl Into<String>,
42        port: u16,
43        user_agent: impl Into<String>,
44        timeout: Duration,
45    ) -> Self {
46        Self {
47            inner: Arc::new(DataImpulseInner {
48                username: username.into(),
49                password: password.into(),
50                host: host.into(),
51                port,
52                user_agent: user_agent.into(),
53                timeout,
54                clients: DashMap::new(),
55            }),
56        }
57    }
58
59    /// Get a reqwest::Client routed through the given country.
60    ///
61    /// Pass `None` for a random residential IP (no country targeting).
62    /// Pass `Some("us")`, `Some("gb")`, `Some("fr")` etc. for country-specific routing.
63    /// Clients are cached per country code — O(1) on subsequent calls.
64    pub fn client(&self, country_code: Option<&str>) -> Client {
65        let cc = country_code.map(|c| c.to_lowercase()).unwrap_or_default();
66
67        // Fast path: cached client
68        if let Some(c) = self.inner.clients.get(&cc) {
69            return c.clone();
70        }
71
72        // Build username with country suffix
73        let username = if cc.is_empty() {
74            self.inner.username.clone()
75        } else {
76            format!("{}__cr.{}", self.inner.username, cc)
77        };
78
79        let proxy_url = format!(
80            "http://{}:{}@{}:{}",
81            username, self.inner.password, self.inner.host, self.inner.port
82        );
83
84        debug!(
85            "DataImpulse: building client for country={:?} proxy={}",
86            cc, self.inner.host
87        );
88
89        let client = match reqwest::Client::builder()
90            .user_agent(&self.inner.user_agent)
91            .timeout(self.inner.timeout)
92            .connect_timeout(Duration::from_secs(8))
93            .proxy(reqwest::Proxy::all(&proxy_url).expect("valid DataImpulse proxy URL"))
94            .redirect(reqwest::redirect::Policy::limited(5))
95            .gzip(true)
96            .brotli(true)
97            // Don't auto-decompress beyond gzip/brotli — saves GB vs double-decompression
98            .build()
99        {
100            Ok(c) => c,
101            Err(e) => {
102                warn!("DataImpulse: failed to build client for cc={cc}: {e}");
103                return reqwest::Client::new();
104            }
105        };
106
107        self.inner.clients.insert(cc, client.clone());
108        client
109    }
110
111    /// Get a fresh (non-cached) client that forces a new residential IP assignment.
112    ///
113    /// Use this for retry-on-block scenarios. The client has zero idle connections
114    /// so the proxy gateway assigns a fresh IP on every request.
115    pub fn fresh_client(&self, country_code: Option<&str>) -> Client {
116        let cc = country_code.map(|c| c.to_lowercase()).unwrap_or_default();
117
118        let username = if cc.is_empty() {
119            self.inner.username.clone()
120        } else {
121            format!("{}__cr.{}", self.inner.username, cc)
122        };
123
124        let proxy_url = format!(
125            "http://{}:{}@{}:{}",
126            username, self.inner.password, self.inner.host, self.inner.port
127        );
128
129        match reqwest::Client::builder()
130            .user_agent(&self.inner.user_agent)
131            .timeout(self.inner.timeout)
132            .connect_timeout(Duration::from_secs(8))
133            .proxy(reqwest::Proxy::all(&proxy_url).expect("valid DataImpulse proxy URL"))
134            .redirect(reqwest::redirect::Policy::limited(5))
135            .gzip(true)
136            .brotli(true)
137            // No connection pool — every request gets a new residential IP
138            .pool_max_idle_per_host(0)
139            .build()
140        {
141            Ok(c) => c,
142            Err(_) => reqwest::Client::new(),
143        }
144    }
145
146    /// Whether credentials are configured (non-empty).
147    pub fn is_configured(&self) -> bool {
148        !self.inner.username.is_empty() && !self.inner.password.is_empty()
149    }
150}
151
152#[cfg(test)]
153mod tests {
154    use super::*;
155
156    #[test]
157    fn client_cached_per_country() {
158        let di = DataImpulseClient::new(
159            "testuser",
160            "testpass",
161            "gw.example.com",
162            823,
163            "TestAgent/1.0",
164            Duration::from_secs(10),
165        );
166        let c1 = di.client(Some("us"));
167        let c2 = di.client(Some("us"));
168        let c3 = di.client(Some("gb"));
169        // Same country → same cached client (pointer equality via Arc internals)
170        // Just verify no panic and different country returns a client
171        drop(c1);
172        drop(c2);
173        drop(c3);
174    }
175
176    #[test]
177    fn configured_check() {
178        let di = DataImpulseClient::new("u", "p", "h", 823, "UA", Duration::from_secs(5));
179        assert!(di.is_configured());
180        let empty = DataImpulseClient::new("", "", "h", 823, "UA", Duration::from_secs(5));
181        assert!(!empty.is_configured());
182    }
183}