stygian-proxy 0.13.5

High-performance, resilient proxy rotation for the Stygian scraping ecosystem.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
//! Core domain types for proxy management.

use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};

use serde::{Deserialize, Serialize};
use uuid::Uuid;

/// The protocol variant of a proxy endpoint.
///
/// # Example
/// ```
/// use stygian_proxy::types::ProxyType;
/// assert_eq!(ProxyType::Http, ProxyType::Http);
/// ```
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ProxyType {
    /// Plain HTTP proxy (CONNECT / forwarding).
    Http,
    /// HTTPS proxy over TLS.
    Https,
    #[cfg(feature = "socks")]
    /// SOCKS4 proxy (requires the `socks` feature).
    Socks4,
    #[cfg(feature = "socks")]
    /// SOCKS5 proxy (requires the `socks` feature).
    Socks5,
    /// CDN edge relay (`Cloudflare`, `CloudFront`, `Azure Front Door`, etc.).
    ///
    /// Traffic egresses through a CDN point-of-presence rather than a traditional proxy
    /// server.  Provider metadata is carried in
    /// [`ProxyCapabilities::cdn_provider`].
    CdnEdge,
}

/// TLS-profiled request mode for proxy-side HTTP operations.
///
/// Used by `tls-profiled` integrations to decide how strictly browser TLS
/// profiles should be mapped onto rustls.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ProfiledRequestMode {
    /// Broad compatibility: skip unknown entries and use safe fallbacks.
    Compatible,
    /// Profile-aware preset selected from the profile name.
    Preset,
    /// Strict cipher-suite mapping with compatibility group fallback.
    Strict,
    /// Strict cipher-suite + group mapping without fallback.
    StrictAll,
}

/// Protocol-level capabilities advertised by a proxy endpoint.
///
/// These flags are set when the proxy is registered and consulted during
/// capability-aware selection (see [`crate::manager::ProxyManager::acquire_with_capabilities`]).
///
/// # Example
/// ```
/// use stygian_proxy::types::ProxyCapabilities;
/// let caps = ProxyCapabilities::default();
/// assert!(!caps.supports_https_connect);
/// assert!(!caps.supports_socks5_udp);
/// assert!(!caps.supports_http3_tunnel);
/// ```
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub struct ProxyCapabilities {
    /// Proxy supports the `CONNECT` method for HTTPS tunnelling.
    #[serde(default)]
    pub supports_https_connect: bool,
    /// Proxy supports SOCKS5 with UDP relay (for UDP-based transports).
    #[serde(default)]
    pub supports_socks5_udp: bool,
    /// Proxy supports HTTP/3 (QUIC) tunnelling — future-compatible flag.
    #[serde(default)]
    pub supports_http3_tunnel: bool,
    /// Optional ISO-3166-1 alpha-2 country code for the proxy egress location.
    #[serde(default)]
    pub geo_country: Option<String>,
    /// Confidence score `[0.0, 1.0]` for the geo-location data.
    ///
    /// `None` means the provider did not supply confidence metadata.
    #[serde(default)]
    pub geo_confidence: Option<f32>,
    /// `true` when this proxy routes through a CDN edge node rather than a
    /// traditional SOCKS/HTTP proxy server.
    #[serde(default)]
    pub is_cdn_edge: bool,
    /// CDN provider name when `is_cdn_edge` is `true`.
    ///
    /// Advisory — used for monitoring and routing hints.
    /// Examples: `"cloudflare"`, `"cloudfront"`, `"azure-front-door"`.
    #[serde(default)]
    pub cdn_provider: Option<String>,
    /// TLS fingerprint profile this proxy presents toward the upstream target.
    ///
    /// Advisory identifier such as `"chrome-131"`, `"firefox-120"`, or
    /// `"curl"`.  Use with [`CapabilityRequirement::require_tls_profile`] to
    /// select proxies by their TLS stack identity.  `None` means unknown.
    #[serde(default)]
    pub tls_profile: Option<String>,
}

impl ProxyCapabilities {
    /// Returns `true` if every required flag in `req` is satisfied by `self`.
    ///
    /// # Example
    /// ```
    /// use stygian_proxy::types::{ProxyCapabilities, CapabilityRequirement};
    /// let caps = ProxyCapabilities { supports_https_connect: true, ..Default::default() };
    /// let req = CapabilityRequirement { require_https_connect: true, ..Default::default() };
    /// assert!(caps.satisfies(&req));
    /// let req2 = CapabilityRequirement { require_socks5_udp: true, ..Default::default() };
    /// assert!(!caps.satisfies(&req2));
    /// ```
    pub fn satisfies(&self, req: &CapabilityRequirement) -> bool {
        if req.require_https_connect && !self.supports_https_connect {
            return false;
        }
        if req.require_socks5_udp && !self.supports_socks5_udp {
            return false;
        }
        if req.require_http3_tunnel && !self.supports_http3_tunnel {
            return false;
        }
        if let Some(ref required_country) = req.require_geo_country
            && self.geo_country.as_deref() != Some(required_country.as_str())
        {
            return false;
        }
        if req.require_cdn_edge && !self.is_cdn_edge {
            return false;
        }
        if let Some(ref required_profile) = req.require_tls_profile
            && self.tls_profile.as_deref() != Some(required_profile.as_str())
        {
            return false;
        }
        true
    }
}

/// Required capability set used as a filter when acquiring a proxy.
///
/// All fields default to `false`/`None` — an empty requirement matches any proxy.
///
/// # Example
/// ```
/// use stygian_proxy::types::CapabilityRequirement;
/// let req = CapabilityRequirement::default();
/// // empty requirement — any proxy qualifies
/// assert!(!req.require_https_connect);
/// ```
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub struct CapabilityRequirement {
    /// Require `supports_https_connect`.
    #[serde(default)]
    pub require_https_connect: bool,
    /// Require `supports_socks5_udp`.
    #[serde(default)]
    pub require_socks5_udp: bool,
    /// Require `supports_http3_tunnel`.
    #[serde(default)]
    pub require_http3_tunnel: bool,
    /// Require a specific egress country (ISO-3166-1 alpha-2).
    #[serde(default)]
    pub require_geo_country: Option<String>,
    /// Require a CDN-edge proxy (`is_cdn_edge` must be `true`).
    #[serde(default)]
    pub require_cdn_edge: bool,
    /// Require a specific TLS fingerprint profile.
    ///
    /// When `Some`, only proxies whose [`ProxyCapabilities::tls_profile`]
    /// matches this value exactly are eligible.  Examples: `"chrome-131"`,
    /// `"firefox-120"`, `"curl"`.
    #[serde(default)]
    pub require_tls_profile: Option<String>,
}

/// The protocol routing path resolved for an outbound request.
///
/// Returned by [`crate::routing::resolve_routing_path`] to indicate how the
/// proxy should forward the connection.
///
/// # Example
/// ```
/// use stygian_proxy::types::RoutingPath;
/// let path = RoutingPath::H1H2OverTcp;
/// assert_eq!(format!("{path:?}"), "H1H2OverTcp");
/// ```
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum RoutingPath {
    /// HTTP/1.1 or HTTP/2 multiplexed over a TCP CONNECT tunnel.
    H1H2OverTcp,
    /// HTTP/3 (QUIC) over a UDP relay — requires `supports_http3_tunnel`.
    H3OverUdp,
    /// Persistent TCP CONNECT tunnel — connection is kept alive between requests.
    ///
    /// Selected when [`crate::routing::TransportPreference::PersistentTcp`] is used.
    PersistentTcp,
}

/// A proxy endpoint with optional authentication credentials.
///
/// `Debug` output masks `password` to prevent accidental credential logging.
///
/// # Example
/// ```
/// use stygian_proxy::types::{Proxy, ProxyType, ProxyCapabilities};
/// let p = Proxy {
///     url: "http://proxy.example.com:8080".into(),
///     proxy_type: ProxyType::Http,
///     username: Some("alice".into()),
///     password: Some("secret".into()),
///     weight: 1,
///     tags: vec!["prod".into()],
///     capabilities: ProxyCapabilities::default(),
/// };
/// let debug = format!("{p:?}");
/// assert!(debug.contains("***"), "password must be masked in Debug output");
/// ```
#[derive(Clone, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub struct Proxy {
    /// The proxy URL, e.g. `http://proxy.example.com:8080`.
    pub url: String,
    pub proxy_type: ProxyType,
    pub username: Option<String>,
    pub password: Option<String>,
    /// Relative selection weight for weighted rotation (default: `1`).
    pub weight: u32,
    /// User-defined tags for filtering and grouping.
    pub tags: Vec<String>,
    /// Protocol-level capabilities advertised by this proxy.
    #[serde(default)]
    pub capabilities: ProxyCapabilities,
}

impl std::fmt::Debug for Proxy {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("Proxy")
            .field("url", &self.url)
            .field("proxy_type", &self.proxy_type)
            .field("username", &self.username)
            .field("password", &self.password.as_deref().map(|_| "***"))
            .field("weight", &self.weight)
            .field("tags", &self.tags)
            .field("capabilities", &self.capabilities)
            .finish()
    }
}

/// A [`Proxy`] with a stable identity and insertion timestamp.
///
/// # Example
/// ```
/// use stygian_proxy::types::{Proxy, ProxyType, ProxyRecord};
/// let proxy = Proxy {
///     url: "http://proxy.example.com:8080".into(),
///     proxy_type: ProxyType::Http,
///     username: None,
///     password: None,
///     weight: 1,
///     tags: vec![],
///     capabilities: Default::default(),
/// };
/// let record = ProxyRecord::new(proxy);
/// assert!(!record.id.is_nil());
/// ```
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub struct ProxyRecord {
    pub id: Uuid,
    pub proxy: Proxy,
    /// Wall-clock time the proxy was added. Not serialized — `Instant` is
    /// not meaningfully portable; defaults to `Instant::now()` on deserialization.
    #[serde(skip, default = "Instant::now")]
    pub added_at: Instant,
}

impl ProxyRecord {
    /// Create a new [`ProxyRecord`] wrapping `proxy` with a freshly generated UUID.
    pub fn new(proxy: Proxy) -> Self {
        Self {
            id: Uuid::new_v4(),
            proxy,
            added_at: Instant::now(),
        }
    }
}

/// Per-proxy runtime metrics using lock-free atomic counters.
///
/// Intended to be shared via `Arc<ProxyMetrics>`.
///
/// # Example
/// ```
/// use stygian_proxy::types::ProxyMetrics;
/// let m = ProxyMetrics::default();
/// assert_eq!(m.success_rate(), 0.0);
/// assert_eq!(m.avg_latency_ms(), 0.0);
/// ```
#[derive(Debug, Default)]
pub struct ProxyMetrics {
    pub requests_total: AtomicU64,
    pub successes: AtomicU64,
    pub failures: AtomicU64,
    pub total_latency_ms: AtomicU64,
}

impl ProxyMetrics {
    /// Cast a `u64` counter to `f64` for ratio computation.
    ///
    /// `u64` can represent values up to ~1.8 × 10¹⁹; `f64` has 53-bit
    /// mantissa, so precision loss begins around 9 × 10¹⁵.  For long-running
    /// proxies that number is never reached in practice, and direct casting
    /// preserves ratios correctly (unlike saturating to `u32::MAX`).
    #[allow(clippy::cast_precision_loss)]
    const fn u64_as_f64(value: u64) -> f64 {
        value as f64
    }

    /// Returns the fraction of requests that succeeded, in `[0.0, 1.0]`.
    ///
    /// Returns `0.0` when no requests have been recorded.
    ///
    /// # Example
    /// ```
    /// use stygian_proxy::types::ProxyMetrics;
    /// use std::sync::atomic::Ordering;
    /// let m = ProxyMetrics::default();
    /// m.requests_total.store(10, Ordering::Relaxed);
    /// m.successes.store(8, Ordering::Relaxed);
    /// assert!((m.success_rate() - 0.8).abs() < f64::EPSILON);
    /// ```
    pub fn success_rate(&self) -> f64 {
        let total = self.requests_total.load(Ordering::Relaxed);
        if total == 0 {
            return 0.0;
        }
        Self::u64_as_f64(self.successes.load(Ordering::Relaxed)) / Self::u64_as_f64(total)
    }

    /// Returns the average request latency in milliseconds.
    ///
    /// Returns `0.0` when no requests have been recorded.
    ///
    /// # Example
    /// ```
    /// use stygian_proxy::types::ProxyMetrics;
    /// use std::sync::atomic::Ordering;
    /// let m = ProxyMetrics::default();
    /// m.requests_total.store(4, Ordering::Relaxed);
    /// m.total_latency_ms.store(400, Ordering::Relaxed);
    /// assert!((m.avg_latency_ms() - 100.0).abs() < f64::EPSILON);
    /// ```
    pub fn avg_latency_ms(&self) -> f64 {
        let total = self.requests_total.load(Ordering::Relaxed);
        if total == 0 {
            return 0.0;
        }
        Self::u64_as_f64(self.total_latency_ms.load(Ordering::Relaxed)) / Self::u64_as_f64(total)
    }
}

mod serde_duration_secs {
    use serde::{Deserialize, Deserializer, Serialize, Serializer};
    use std::time::Duration;

    pub fn serialize<S: Serializer>(d: &Duration, s: S) -> Result<S::Ok, S::Error> {
        d.as_secs().serialize(s)
    }

    pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Duration, D::Error> {
        Ok(Duration::from_secs(u64::deserialize(d)?))
    }
}

/// Configuration governing health checking and circuit-breaker behaviour.
///
/// Duration fields serialize as integer seconds for TOML/JSON compatibility.
///
/// # Example
/// ```
/// use stygian_proxy::types::ProxyConfig;
/// use std::time::Duration;
/// let cfg = ProxyConfig::default();
/// assert_eq!(cfg.health_check_url, "https://httpbin.org/ip");
/// assert_eq!(cfg.health_check_interval, Duration::from_secs(60));
/// assert_eq!(cfg.health_check_timeout, Duration::from_secs(5));
/// assert_eq!(cfg.circuit_open_threshold, 5);
/// assert_eq!(cfg.circuit_half_open_after, Duration::from_secs(30));
/// assert!(cfg.profiled_request_mode.is_none());
/// assert_eq!(cfg.health_check_jitter_pct, 0.20_f32);
/// assert!(cfg.max_requests_per_connection.is_none());
/// assert!(cfg.connection_max_age_secs.is_none());
/// ```
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub struct ProxyConfig {
    /// URL called during health checks to verify proxy liveness.
    pub health_check_url: String,
    /// How often to run health checks (seconds).
    #[serde(with = "serde_duration_secs")]
    pub health_check_interval: Duration,
    /// Per-probe HTTP timeout (seconds).
    #[serde(with = "serde_duration_secs")]
    pub health_check_timeout: Duration,
    /// Jitter factor applied to the health-check sleep window.
    ///
    /// `0.20` distributes each check window uniformly over
    /// `interval × [0.80, 1.20)`, preventing synchronised fleet-wide check
    /// storms.  Set to `0.0` to disable jitter.  Clamped to `[0.0, 0.99]`
    /// at runtime.
    ///
    /// Default: `0.20` (±20 %).
    #[serde(default = "default_health_check_jitter_pct")]
    pub health_check_jitter_pct: f32,
    /// Consecutive failures before the circuit trips to OPEN.
    pub circuit_open_threshold: u32,
    /// How long to wait in OPEN before transitioning to HALF-OPEN (seconds).
    #[serde(with = "serde_duration_secs")]
    pub circuit_half_open_after: Duration,
    /// Sticky-session policy for domain→proxy binding.
    #[serde(default)]
    pub sticky_policy: crate::session::StickyPolicy,
    /// Optional default mode for TLS-profiled helper clients.
    ///
    /// When set and `tls-profiled` is enabled, `ProxyManager` initializes its
    /// `HealthChecker` with a Chrome-profiled requester using this mode.
    ///
    /// Ignored when `tls-profiled` is disabled.
    #[serde(default)]
    pub profiled_request_mode: Option<ProfiledRequestMode>,
    /// Maximum requests routed through one persistent TCP connection before it
    /// is recycled.  `None` means no limit.  Only consulted when
    /// [`crate::routing::TransportPreference::PersistentTcp`] is active.
    #[serde(default)]
    pub max_requests_per_connection: Option<u32>,
    /// Maximum age of a persistent TCP connection in seconds before it is
    /// replaced.  `None` means no age limit.
    #[serde(default)]
    pub connection_max_age_secs: Option<u64>,
}

const fn default_health_check_jitter_pct() -> f32 {
    0.20
}

impl Default for ProxyConfig {
    fn default() -> Self {
        Self {
            health_check_url: "https://httpbin.org/ip".into(),
            health_check_interval: Duration::from_mins(1),
            health_check_timeout: Duration::from_secs(5),
            health_check_jitter_pct: 0.20,
            circuit_open_threshold: 5,
            circuit_half_open_after: Duration::from_secs(30),
            sticky_policy: crate::session::StickyPolicy::default(),
            profiled_request_mode: None,
            max_requests_per_connection: None,
            connection_max_age_secs: None,
        }
    }
}