stygian_proxy/vendor_quirks.rs
1//! Vendor-specific proxy URL quirks.
2//!
3//! ## Why vendor-specific quirks matter
4//!
5//! Most public proxy providers serve plain HTTP on a port that *looks*
6//! like an HTTPS port. Using `https://` against such a proxy causes the
7//! TLS layer to attempt a handshake on top of an already-running TLS
8//! session, producing `BoringSSL`'s `WRONG_VERSION_NUMBER` error.
9//! The 2026 guide flags this as a high-frequency footgun
10//! (`docs/dev/project/scraping-guide-2026-llm-context.md` L2840, the
11//! "Crawlera/Zyte proxy bug"):
12//! guide flags this as a high-frequency footgun
13//! (`docs/dev/project/scraping-guide-2026-llm-context.md` L2840, the
14//! "Crawlera/Zyte proxy bug"):
15//!
16//! > Port 8011 speaks plain HTTP. Both http:// and https:// keys must use
17//! > http:// scheme. Using https:// causes BoringSSL WRONG_VERSION_NUMBER
18//! > (TLS-over-TLS failure). Fix: `'https': 'http://key:@proxy.crawlera.com:8011/'`
19//!
20//! Operators hit this trap on three common providers:
21//!
22//! - `Crawlera` / `Zyte` Smart Proxy Manager (`*.crawlera.com:8011`,
23//! `*.zyte.com:8011`) — must be `http://` even when scraping `https://`
24//! targets.
25//! - `Bright Data` `brd.superproxy.io:22225` — username must follow the
26//! `brd-customer-<id>-session-<session_id>` pattern; missing the
27//! `-session-<id>` suffix silently collapses all traffic into the
28//! default session pool.
29//! - `IPRoyal` residential gateway — username must carry a country flag
30//! (e.g. `user-country-US`) for the egress IP to honour the request.
31//!
32//! [`VENDOR_QUIRKS`] encodes the four documented cases as a `const` slice
33//! so the table is zero-cost at runtime; [`check`] walks the slice using
34//! pure pointer/length compares and returns all matches in a small
35//! [`Vec`]. Hard-error quirks (like `Crawlera` 8011 + `https://`) are
36//! surfaced at ingest time by `validate_proxy_url`,
37//! which rejects the URL outright; warning-severity quirks are logged
38//! and the URL is accepted.
39//!
40//! ## Security note
41//!
42//! Quirks match on `host:port` only — the password component of the URL
43//! is never inspected, logged, or echoed in any error or warning. The
44//! quirk descriptions are static `&'static str` slices with no
45//! credentials baked in.
46//!
47//! ## Hot path
48//!
49//! [`check`] is a `pub fn` (the return type is `Vec<QuirkMatch>` per the
50//! task spec). In the common case where the URL host is not in the
51//! built-in table, the function returns an empty `Vec::new()` without
52//! iterating the table — the `Vec` allocation is skipped entirely. The
53//! table itself is a `const` slice, so there is no I/O, no locks, and
54//! no parsing beyond the [`ProxyUrl`] construction done by the caller.
55//!
56//! ## Example
57//!
58//! ```
59//! use stygian_proxy::vendor_quirks::{check, ProxyUrl, QuirkSeverity};
60//!
61//! // The classic Crawlera 8011 + https:// footgun.
62//! let url = ProxyUrl::parse("https://user:pass@proxy.crawlera.com:8011")
63//! .expect("parses");
64//! let matches = check(&url);
65//! assert_eq!(matches.len(), 1);
66//! assert_eq!(matches[0].severity, QuirkSeverity::Error);
67//! assert_eq!(matches[0].required_scheme, stygian_proxy::vendor_quirks::Scheme::Http);
68//! ```
69
70use std::str::FromStr;
71
72use serde::{Deserialize, Serialize};
73use thiserror::Error;
74
75// ─────────────────────────────────────────────────────────────────────────────
76// Scheme
77// ─────────────────────────────────────────────────────────────────────────────
78
79/// URL scheme of a proxy endpoint.
80///
81/// Mirrors the subset of [`crate::types::ProxyType`] that is reachable
82/// as a URL scheme. Used by [`VendorQuirk`] to express provider-specific
83/// scheme requirements (e.g. "Crawlera port 8011 is plain HTTP even
84/// though it serves HTTPS targets").
85///
86/// # Example
87///
88/// ```
89/// use std::str::FromStr;
90/// use stygian_proxy::vendor_quirks::Scheme;
91/// assert_eq!(Scheme::Http.as_str(), "http");
92/// assert_eq!(Scheme::Https.as_str(), "https");
93/// assert_eq!(Scheme::from_str("http").ok(), Some(Scheme::Http));
94/// assert!(Scheme::from_str("nope").is_err());
95/// ```
96#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
97#[serde(rename_all = "snake_case")]
98pub enum Scheme {
99 /// Plain HTTP (`http://`).
100 #[default]
101 Http,
102 /// HTTPS (`https://`).
103 Https,
104 /// SOCKS4 (`socks4://`) — only when the `socks` feature is enabled.
105 #[cfg(feature = "socks")]
106 Socks4,
107 /// SOCKS5 (`socks5://`) — only when the `socks` feature is enabled.
108 #[cfg(feature = "socks")]
109 Socks5,
110}
111
112impl Scheme {
113 /// Returns the canonical wire form of the scheme (e.g. `"http"`).
114 ///
115 /// # Example
116 ///
117 /// ```
118 /// use stygian_proxy::vendor_quirks::Scheme;
119 /// assert_eq!(Scheme::Http.as_str(), "http");
120 /// assert_eq!(Scheme::Https.as_str(), "https");
121 /// ```
122 #[must_use]
123 pub const fn as_str(self) -> &'static str {
124 match self {
125 Self::Http => "http",
126 Self::Https => "https",
127 #[cfg(feature = "socks")]
128 Self::Socks4 => "socks4",
129 #[cfg(feature = "socks")]
130 Self::Socks5 => "socks5",
131 }
132 }
133}
134
135impl FromStr for Scheme {
136 type Err = ();
137
138 /// Parse a [`Scheme`] from its wire form (e.g. `"http"`).
139 ///
140 /// Returns `Err(())` for any unknown scheme. The validator
141 /// surfaces the unknown scheme upstream as a structured error
142 /// rather than panicking.
143 fn from_str(s: &str) -> Result<Self, Self::Err> {
144 match s {
145 "http" => Ok(Self::Http),
146 "https" => Ok(Self::Https),
147 #[cfg(feature = "socks")]
148 "socks4" => Ok(Self::Socks4),
149 #[cfg(feature = "socks")]
150 "socks5" => Ok(Self::Socks5),
151 _ => Err(()),
152 }
153 }
154}
155
156// ─────────────────────────────────────────────────────────────────────────────
157// QuirkSeverity
158// ─────────────────────────────────────────────────────────────────────────────
159
160/// Severity classification for a [`QuirkMatch`].
161///
162/// The ingest flow in `validate_proxy_url` treats the
163/// severity as the action gate:
164///
165/// - [`QuirkSeverity::Error`] — hard-error quirks reject the URL outright.
166/// - [`QuirkSeverity::Warning`] — warning quirks are logged but the URL
167/// is accepted.
168/// - [`QuirkSeverity::Info`] — informational quirks are recorded for
169/// observability and the URL is accepted.
170///
171/// # Example
172///
173/// ```
174/// use stygian_proxy::vendor_quirks::QuirkSeverity;
175/// assert_eq!(QuirkSeverity::Error, QuirkSeverity::Error);
176/// assert_ne!(QuirkSeverity::Warning, QuirkSeverity::Info);
177/// ```
178#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
179#[serde(rename_all = "snake_case")]
180pub enum QuirkSeverity {
181 /// Informational — the URL is accepted; the quirk is recorded.
182 Info,
183 /// Warning — the URL is accepted; the quirk is logged.
184 Warning,
185 /// Error — the URL is rejected by the ingest validator.
186 Error,
187}
188
189// ─────────────────────────────────────────────────────────────────────────────
190// VendorQuirk
191// ─────────────────────────────────────────────────────────────────────────────
192
193/// A vendor-specific proxy URL rule, declared as a `const` table record.
194///
195/// Quirks match on `host_suffix:port` (no credentials). The semantics
196/// are severity-driven:
197///
198/// - For [`QuirkSeverity::Error`], the quirk fires when the URL's
199/// `scheme != required_scheme` (e.g. `Crawlera` 8011 + `https://`).
200/// - For [`QuirkSeverity::Warning`] and [`QuirkSeverity::Info`], the
201/// quirk fires on every `host_suffix:port` match (the `required_scheme`
202/// field is informational and not used for the trigger).
203///
204/// `Copy` so the [`VENDOR_QUIRKS`] `const` slice can be iterated without
205/// moving records out.
206///
207/// # Example
208///
209/// ```
210/// use stygian_proxy::vendor_quirks::{Scheme, VendorQuirk, QuirkSeverity, VENDOR_QUIRKS};
211///
212/// let crawlera = VENDOR_QUIRKS
213/// .iter()
214/// .find(|q| q.host_suffix == "crawlera.com")
215/// .expect("Crawlera quirk seeded");
216/// assert_eq!(crawlera.port, Some(8011));
217/// assert_eq!(crawlera.required_scheme, Scheme::Http);
218/// assert_eq!(crawlera.severity, QuirkSeverity::Error);
219///
220/// // Quirk descriptions never include the password component.
221/// assert!(!crawlera.description.contains("pass"));
222/// assert!(!crawlera.description.contains('@'));
223/// let _: VendorQuirk = *crawlera; // `Copy` for const-slice iteration
224/// ```
225#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
226pub struct VendorQuirk {
227 /// Host suffix that triggers the quirk (e.g. `"crawlera.com"`,
228 /// `"brd.superproxy.io"`). Matched as an exact host name OR as a
229 /// subdomain boundary (e.g. `proxy.crawlera.com` matches
230 /// `crawlera.com`).
231 pub host_suffix: &'static str,
232 /// Port that triggers the quirk. `None` matches any port.
233 pub port: Option<u16>,
234 /// Scheme that the provider requires on this port. Used by
235 /// [`QuirkSeverity::Error`] quirks to detect a scheme mismatch
236 /// (the reject trigger). Ignored by `Warning` / `Info` quirks.
237 pub required_scheme: Scheme,
238 /// Human-readable description of the quirk and the failure mode it
239 /// prevents. Used as the structured error reason in
240 /// `validate_proxy_url`.
241 pub description: &'static str,
242 /// Severity classification — drives whether the URL is rejected,
243 /// warned, or just recorded.
244 pub severity: QuirkSeverity,
245}
246
247// ─────────────────────────────────────────────────────────────────────────────
248// Built-in quirk table
249// ─────────────────────────────────────────────────────────────────────────────
250
251/// The first hard-error quirk — `Crawlera` port 8011 is plain HTTP.
252///
253/// Using `https://` against `proxy.crawlera.com:8011` causes
254/// `BoringSSL WRONG_VERSION_NUMBER` (TLS-over-TLS failure). Both
255/// `http://` and `https://` keys in the client config must use the
256/// `http://` scheme for the `Crawlera` port. See module docs.
257pub const CRAWLERA_8011_QUIRK: VendorQuirk = VendorQuirk {
258 host_suffix: "crawlera.com",
259 port: Some(8011),
260 required_scheme: Scheme::Http,
261 description: "Crawlera port 8011 is plain HTTP — using https:// causes BoringSSL WRONG_VERSION_NUMBER (TLS-over-TLS failure). Use http:// on both http and https scraping keys.",
262 severity: QuirkSeverity::Error,
263};
264
265/// `Zyte` Smart Proxy Manager port 8011 has the same plain-HTTP trap
266/// as `Crawlera` — `Zyte` operates the same upstream port range.
267pub const ZYTE_8011_QUIRK: VendorQuirk = VendorQuirk {
268 host_suffix: "zyte.com",
269 port: Some(8011),
270 required_scheme: Scheme::Http,
271 description: "Zyte Smart Proxy Manager port 8011 is plain HTTP — same WRONG_VERSION_NUMBER trap as Crawlera. Use http:// scheme on both http and https keys.",
272 severity: QuirkSeverity::Error,
273};
274
275/// `Bright Data` residential / datacenter super-proxy on port 22225.
276///
277/// The username must follow the `brd-customer-<id>-session-<id>`
278/// pattern for session isolation. A missing `-session-<id>` suffix
279/// silently collapses traffic into the default session pool.
280pub const BRD_SUPERPROXY_QUIRK: VendorQuirk = VendorQuirk {
281 host_suffix: "brd.superproxy.io",
282 port: Some(22225),
283 required_scheme: Scheme::Http,
284 description: "Bright Data brd.superproxy.io:22225 requires a brd-customer-<id>-session-<id> username; a missing -session-<id> suffix silently collapses traffic into the default session pool.",
285 severity: QuirkSeverity::Warning,
286};
287
288/// `IPRoyal` residential traffic requires a country flag in the username.
289///
290/// Example: `user-country-US`. Without the flag, the gateway ignores
291/// the requested exit country and serves a random residential IP. The
292/// `port = None` field matches any `IPRoyal` gateway port.
293pub const IPROYAL_QUIRK: VendorQuirk = VendorQuirk {
294 host_suffix: "iproyal.com",
295 port: None,
296 required_scheme: Scheme::Http,
297 description: "IPRoyal residential traffic requires a country flag in the username (e.g. user-country-US). Without the flag, the gateway ignores the requested exit country.",
298 severity: QuirkSeverity::Warning,
299};
300
301/// Every built-in [`VendorQuirk`], in declaration order.
302///
303/// New quirks should be added above this slice so the constant
304/// declarations remain the single source of truth, then included in
305/// the slice. The slice is `const`-constructible so the table is
306/// zero-cost at runtime.
307pub const VENDOR_QUIRKS: &[VendorQuirk] = &[
308 CRAWLERA_8011_QUIRK,
309 ZYTE_8011_QUIRK,
310 BRD_SUPERPROXY_QUIRK,
311 IPROYAL_QUIRK,
312];
313
314// ─────────────────────────────────────────────────────────────────────────────
315// ProxyUrl
316// ─────────────────────────────────────────────────────────────────────────────
317
318/// A parsed proxy URL — the canonical input shape for [`check`].
319///
320/// The struct is intentionally small (six `String` / `Option` fields,
321/// no `Vec`) and is built once per ingest by
322/// [`ProxyUrl::parse`]. The owned `String` fields make the type
323/// `'static`-safe; the [`check`] function only reads from it and never
324/// mutates. **Credentials are carried for completeness** but
325/// [`check`] never inspects the password component.
326///
327/// # Example
328///
329/// ```
330/// use stygian_proxy::vendor_quirks::{ProxyUrl, Scheme};
331///
332/// let p = ProxyUrl::parse("http://user:pass@proxy.crawlera.com:8011/path")
333/// .expect("parses");
334/// assert_eq!(p.scheme, Scheme::Http);
335/// assert_eq!(p.host, "proxy.crawlera.com");
336/// assert_eq!(p.port, Some(8011));
337/// assert_eq!(p.username.as_deref(), Some("user"));
338/// assert_eq!(p.password.as_deref(), Some("pass"));
339/// assert_eq!(p.path.as_deref(), Some("/path"));
340///
341/// let p = ProxyUrl::parse("https://proxy.test:8443").expect("parses");
342/// assert_eq!(p.scheme, Scheme::Https);
343/// assert_eq!(p.host, "proxy.test");
344/// assert_eq!(p.port, Some(8443));
345/// ```
346#[derive(Debug, Clone, PartialEq, Eq)]
347pub struct ProxyUrl {
348 /// URL scheme.
349 pub scheme: Scheme,
350 /// Host portion of the URL (lower-case, no brackets).
351 pub host: String,
352 /// Optional port. `None` means the scheme default (80 for HTTP, 443
353 /// for HTTPS).
354 pub port: Option<u16>,
355 /// Optional user-info username. Never logged by [`check`].
356 pub username: Option<String>,
357 /// Optional user-info password. **Never logged, matched, or echoed
358 /// by [`check`].**
359 pub password: Option<String>,
360 /// Optional path component (e.g. `"/"` for `http://host:80/`).
361 pub path: Option<String>,
362}
363
364/// Errors emitted by [`ProxyUrl::parse`].
365///
366/// Distinct from the host/port validation errors in
367/// `validate_proxy_url` (which produce
368/// `ProxyError::InvalidProxyUrl`) so a caller can
369/// decide whether to surface the parse failure as an upstream error
370/// or attempt recovery.
371#[derive(Debug, Error, PartialEq, Eq)]
372pub enum ParseError {
373 /// The URL did not contain a `"://"` separator.
374 #[error("missing scheme separator '://' in URL `{0}`")]
375 MissingSchemeSeparator(String),
376 /// The scheme is not in the supported set (e.g. `"ftp"`,
377 /// `"file"`).
378 #[error("unsupported scheme `{0}` in URL `{1}`")]
379 UnsupportedScheme(String, String),
380 /// The host portion is empty.
381 #[error("empty host in URL `{0}`")]
382 EmptyHost(String),
383 /// The explicit port is out of range `[1, 65535]`.
384 #[error("port `{port}` is out of range [1, 65535] in URL `{url}`")]
385 PortOutOfRange {
386 /// The offending port string.
387 port: String,
388 /// The URL that was being parsed.
389 url: String,
390 },
391 /// The explicit port was not a valid `u16` integer.
392 #[error("non-numeric port `{0}` in URL `{1}`")]
393 NonNumericPort(String, String),
394 /// The URL has an unmatched `[` in the host position (IPv6 literal
395 /// without a closing bracket).
396 #[error("unclosed IPv6 bracket in URL `{0}`")]
397 UnclosedIpv6Bracket(String),
398}
399
400impl ProxyUrl {
401 /// Parse a proxy URL into a [`ProxyUrl`].
402 ///
403 /// Recognises `http://`, `https://`, and (when the `socks` feature
404 /// is enabled) `socks4://` / `socks5://` schemes. IPv6 literals in
405 /// brackets (e.g. `http://[::1]:8080`) are supported. User-info is
406 /// split on the first `:` after the `//` separator so passwords
407 /// containing colons survive intact.
408 ///
409 /// # Errors
410 ///
411 /// Returns [`ParseError`] when the URL is structurally invalid.
412 /// The error message includes the original URL for diagnostics.
413 ///
414 /// # Example
415 ///
416 /// ```
417 /// use stygian_proxy::vendor_quirks::{ProxyUrl, Scheme};
418 ///
419 /// let p = ProxyUrl::parse("http://user:pa:ss@host:8080").unwrap();
420 /// assert_eq!(p.scheme, Scheme::Http);
421 /// assert_eq!(p.host, "host");
422 /// assert_eq!(p.port, Some(8080));
423 /// assert_eq!(p.username.as_deref(), Some("user"));
424 /// // Passwords containing colons are preserved.
425 /// assert_eq!(p.password.as_deref(), Some("pa:ss"));
426 /// ```
427 pub fn parse(url: &str) -> Result<Self, ParseError> {
428 // 1. Split scheme from the rest.
429 let (scheme_str, rest) = url
430 .split_once("://")
431 .ok_or_else(|| ParseError::MissingSchemeSeparator(url.to_owned()))?;
432
433 let scheme = Scheme::from_str(scheme_str)
434 .map_err(|()| ParseError::UnsupportedScheme(scheme_str.to_owned(), url.to_owned()))?;
435
436 // 2. Split user-info from authority: the part before the first
437 // `@` after the `://` is the user-info. The part after is
438 // host[:port][/path].
439 let (userinfo, authority_with_path) = match rest.split_once('@') {
440 Some((ui, auth)) => (ui, auth),
441 None => ("", rest),
442 };
443
444 // 3. Split authority from path. The authority ends at the first
445 // `/` after the user-info, or at end-of-string.
446 let (authority, path) = match authority_with_path.split_once('/') {
447 Some((a, p)) => (a, Some(format!("/{p}"))),
448 None => (authority_with_path, None),
449 };
450
451 // 4. Parse user-info into username + password (split on the
452 // FIRST `:` so passwords with colons survive).
453 let (username, password) = if userinfo.is_empty() {
454 (None, None)
455 } else {
456 match userinfo.split_once(':') {
457 Some((u, p)) => (Some(u.to_owned()), Some(p.to_owned())),
458 None => (Some(userinfo.to_owned()), None),
459 }
460 };
461
462 // 5. Split host from port, handling IPv6 brackets.
463 let (host, port_str) = if let Some(stripped) = authority.strip_prefix('[') {
464 // IPv6 literal: read until `]`.
465 let close = stripped
466 .find(']')
467 .ok_or_else(|| ParseError::UnclosedIpv6Bracket(url.to_owned()))?;
468 let host = &stripped[..close];
469 let after = &stripped[close + 1..];
470 let port_str = after.strip_prefix(':').unwrap_or("");
471 (host.to_owned(), port_str)
472 } else {
473 match authority.rsplit_once(':') {
474 Some((h, p)) => (h.to_owned(), p),
475 None => (authority.to_owned(), ""),
476 }
477 };
478
479 if host.is_empty() {
480 return Err(ParseError::EmptyHost(url.to_owned()));
481 }
482
483 // 6. Parse the optional port.
484 let port = if port_str.is_empty() {
485 None
486 } else {
487 let parsed: u32 = port_str
488 .parse()
489 .map_err(|_| ParseError::NonNumericPort(port_str.to_owned(), url.to_owned()))?;
490 if parsed == 0 || parsed > 65535 {
491 return Err(ParseError::PortOutOfRange {
492 port: port_str.to_owned(),
493 url: url.to_owned(),
494 });
495 }
496 // Truncation is safe: `parsed <= 65535 < u16::MAX`.
497 #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
498 let as_u16 = parsed as u16;
499 Some(as_u16)
500 };
501
502 Ok(Self {
503 scheme,
504 host: host.to_ascii_lowercase(),
505 port,
506 username,
507 password,
508 path,
509 })
510 }
511}
512
513// ─────────────────────────────────────────────────────────────────────────────
514// QuirkMatch
515// ─────────────────────────────────────────────────────────────────────────────
516
517/// A single vendor-quirk match produced by [`check`].
518///
519/// Carries the static [`VendorQuirk`] record (for `severity` /
520/// `host_suffix` / `description` / `required_scheme`) plus the
521/// observed values from the [`ProxyUrl`] that triggered the match.
522/// **The password component of the URL is never copied into the
523/// match** — the security note at the module top is binding.
524///
525/// The struct is `Clone` so matches can be retained across log
526/// emissions and HTTP responses without lifetime gymnastics.
527///
528/// # Example
529///
530/// ```
531/// use stygian_proxy::vendor_quirks::{check, ProxyUrl, QuirkSeverity};
532///
533/// let url = ProxyUrl::parse("http://user@brd.superproxy.io:22225").unwrap();
534/// let m = &check(&url)[0];
535/// assert_eq!(m.severity, QuirkSeverity::Warning);
536/// assert_eq!(m.host_suffix, "brd.superproxy.io");
537/// assert_eq!(m.observed_scheme, stygian_proxy::vendor_quirks::Scheme::Http);
538/// assert_eq!(m.observed_port, Some(22225));
539/// ```
540#[derive(Debug, Clone, PartialEq, Eq)]
541pub struct QuirkMatch {
542 /// Severity of the matched quirk (mirrors `quirk.severity`).
543 pub severity: QuirkSeverity,
544 /// Host suffix that triggered the match (mirrors `quirk.host_suffix`).
545 pub host_suffix: &'static str,
546 /// Required scheme per the quirk (mirrors `quirk.required_scheme`).
547 pub required_scheme: Scheme,
548 /// Human-readable description (mirrors `quirk.description`).
549 pub description: &'static str,
550 /// The URL's observed scheme at match time.
551 pub observed_scheme: Scheme,
552 /// The URL's observed port at match time.
553 pub observed_port: Option<u16>,
554}
555
556// ─────────────────────────────────────────────────────────────────────────────
557// check
558// ─────────────────────────────────────────────────────────────────────────────
559
560/// Returns every [`VendorQuirk`] that matches `url`.
561///
562/// The function walks the const [`VENDOR_QUIRKS`] slice using pure
563/// pointer/length compares (no I/O, no locks, no per-call
564/// allocations beyond the returned `Vec`). In the common case where
565/// the URL host is not in the built-in table, the function short-
566/// circuits on the first non-matching `host_suffix` and returns
567/// `Vec::new()` without further allocation beyond the empty `Vec`
568/// itself.
569///
570/// Quirk triggers:
571///
572/// - For [`QuirkSeverity::Error`] quirks: the quirk fires when
573/// `url.scheme != quirk.required_scheme` AND the host:port matches.
574/// - For [`QuirkSeverity::Warning`] / [`QuirkSeverity::Info`] quirks:
575/// the quirk fires whenever the host:port matches (the
576/// `required_scheme` field is informational).
577///
578/// Host matching is **subdomain-aware**: `proxy.crawlera.com`
579/// matches the `"crawlera.com"` suffix, but `"mycrawlera.com"` does
580/// not (no subdomain boundary).
581///
582/// # Example
583///
584/// ```
585/// use stygian_proxy::vendor_quirks::{check, ProxyUrl, QuirkSeverity, Scheme};
586///
587/// // Crawlera 8011 + https → 1 Error match (scheme mismatch).
588/// let url = ProxyUrl::parse("https://user:pass@proxy.crawlera.com:8011").unwrap();
589/// let m = check(&url);
590/// assert_eq!(m.len(), 1);
591/// assert_eq!(m[0].severity, QuirkSeverity::Error);
592/// assert_eq!(m[0].required_scheme, Scheme::Http);
593///
594/// // Crawlera 8011 + http → 0 matches (compliant URL).
595/// let url = ProxyUrl::parse("http://user:pass@proxy.crawlera.com:8011").unwrap();
596/// assert!(check(&url).is_empty());
597///
598/// // Bright Data super-proxy → 1 Warning regardless of username.
599/// let url = ProxyUrl::parse("http://user@brd.superproxy.io:22225").unwrap();
600/// assert_eq!(check(&url).len(), 1);
601/// assert_eq!(check(&url)[0].severity, QuirkSeverity::Warning);
602/// ```
603#[must_use]
604pub fn check(url: &ProxyUrl) -> Vec<QuirkMatch> {
605 let mut out: Vec<QuirkMatch> = Vec::new();
606 for quirk in VENDOR_QUIRKS {
607 if !host_suffix_matches(&url.host, quirk.host_suffix) {
608 continue;
609 }
610 if let Some(required_port) = quirk.port
611 && url.port != Some(required_port)
612 {
613 continue;
614 }
615 match quirk.severity {
616 QuirkSeverity::Error => {
617 // Hard-error quirks only fire on scheme mismatch.
618 if url.scheme == quirk.required_scheme {
619 continue;
620 }
621 }
622 QuirkSeverity::Warning | QuirkSeverity::Info => {
623 // Warning / Info quirks fire on every host:port match.
624 }
625 }
626 out.push(QuirkMatch {
627 severity: quirk.severity,
628 host_suffix: quirk.host_suffix,
629 required_scheme: quirk.required_scheme,
630 description: quirk.description,
631 observed_scheme: url.scheme,
632 observed_port: url.port,
633 });
634 }
635 out
636}
637
638/// `true` when `host` equals `suffix` or is a strict subdomain of
639/// `suffix` (e.g. `proxy.crawlera.com` matches `crawlera.com`).
640///
641/// `mycrawlera.com` does NOT match `crawlera.com` — the subdomain
642/// boundary must be a `.` character.
643fn host_suffix_matches(host: &str, suffix: &str) -> bool {
644 if host == suffix {
645 return true;
646 }
647 if host.len() <= suffix.len() + 1 {
648 return false;
649 }
650 // The character immediately before the suffix must be a `.` —
651 // guards against `mycrawlera.com` matching `crawlera.com`.
652 match host.as_bytes().get(host.len() - suffix.len() - 1) {
653 Some(b'.') => host.ends_with(suffix),
654 _ => false,
655 }
656}
657
658// ─────────────────────────────────────────────────────────────────────────────
659// Tests
660// ─────────────────────────────────────────────────────────────────────────────
661
662#[cfg(test)]
663#[allow(
664 clippy::unwrap_used,
665 clippy::expect_used,
666 clippy::panic,
667 clippy::indexing_slicing
668)] // deterministic test fixtures for URL parsing + quirk matching
669mod tests {
670 use super::*;
671
672 // ── Scheme ──────────────────────────────────────────────────────────────
673
674 #[test]
675 fn scheme_default_is_http() {
676 assert_eq!(Scheme::default(), Scheme::Http);
677 }
678
679 #[test]
680 fn scheme_as_str_matches_wire_format() {
681 assert_eq!(Scheme::Http.as_str(), "http");
682 assert_eq!(Scheme::Https.as_str(), "https");
683 }
684
685 #[test]
686 fn scheme_from_str_round_trip() {
687 for scheme in [Scheme::Http, Scheme::Https] {
688 assert_eq!(Scheme::from_str(scheme.as_str()), Ok(scheme));
689 }
690 assert_eq!(Scheme::from_str("nope"), Err(()));
691 }
692
693 // ── ProxyUrl parsing ────────────────────────────────────────────────────
694
695 #[test]
696 fn parse_simple_http_url() {
697 let p = ProxyUrl::parse("http://proxy.example.com:8080").unwrap();
698 assert_eq!(p.scheme, Scheme::Http);
699 assert_eq!(p.host, "proxy.example.com");
700 assert_eq!(p.port, Some(8080));
701 assert!(p.username.is_none());
702 assert!(p.password.is_none());
703 assert!(p.path.is_none());
704 }
705
706 #[test]
707 fn parse_https_url() {
708 let p = ProxyUrl::parse("https://proxy.example.com:8443/path").unwrap();
709 assert_eq!(p.scheme, Scheme::Https);
710 assert_eq!(p.host, "proxy.example.com");
711 assert_eq!(p.port, Some(8443));
712 assert_eq!(p.path.as_deref(), Some("/path"));
713 }
714
715 #[test]
716 fn parse_url_with_user_info() {
717 let p = ProxyUrl::parse("http://user:pass@proxy.test:3128").unwrap();
718 assert_eq!(p.scheme, Scheme::Http);
719 assert_eq!(p.host, "proxy.test");
720 assert_eq!(p.port, Some(3128));
721 assert_eq!(p.username.as_deref(), Some("user"));
722 assert_eq!(p.password.as_deref(), Some("pass"));
723 }
724
725 #[test]
726 fn parse_url_with_password_containing_colon() {
727 let p = ProxyUrl::parse("http://user:pa:ss@proxy.test:3128").unwrap();
728 assert_eq!(p.username.as_deref(), Some("user"));
729 // Passwords with colons must be preserved intact.
730 assert_eq!(p.password.as_deref(), Some("pa:ss"));
731 }
732
733 #[test]
734 fn parse_url_with_username_only() {
735 let p = ProxyUrl::parse("http://user@proxy.test:3128").unwrap();
736 assert_eq!(p.username.as_deref(), Some("user"));
737 assert!(p.password.is_none());
738 }
739
740 #[test]
741 fn parse_url_default_ports_are_optional() {
742 let p = ProxyUrl::parse("http://proxy.test").unwrap();
743 assert_eq!(p.port, None);
744 }
745
746 #[test]
747 fn parse_url_lowercases_host() {
748 let p = ProxyUrl::parse("http://PROXY.Test:8080").unwrap();
749 assert_eq!(p.host, "proxy.test");
750 }
751
752 #[test]
753 fn parse_url_with_trailing_slash() {
754 let p = ProxyUrl::parse("http://proxy.test:8080/").unwrap();
755 assert_eq!(p.path.as_deref(), Some("/"));
756 }
757
758 #[test]
759 fn parse_ipv6_url_with_brackets() {
760 let p = ProxyUrl::parse("http://[::1]:8080").unwrap();
761 assert_eq!(p.host, "::1");
762 assert_eq!(p.port, Some(8080));
763 }
764
765 #[test]
766 fn parse_missing_scheme_separator_is_error() {
767 let err = ProxyUrl::parse("not-a-url").unwrap_err();
768 assert!(matches!(err, ParseError::MissingSchemeSeparator(_)));
769 }
770
771 #[test]
772 fn parse_unsupported_scheme_is_error() {
773 let err = ProxyUrl::parse("ftp://host:21").unwrap_err();
774 assert!(matches!(err, ParseError::UnsupportedScheme(ref s, _) if s == "ftp"));
775 }
776
777 #[test]
778 fn parse_empty_host_is_error() {
779 let err = ProxyUrl::parse("http://:8080").unwrap_err();
780 assert!(matches!(err, ParseError::EmptyHost(_)));
781 }
782
783 #[test]
784 fn parse_port_out_of_range_is_error() {
785 let err = ProxyUrl::parse("http://host:99999").unwrap_err();
786 assert!(matches!(err, ParseError::PortOutOfRange { .. }));
787 }
788
789 #[test]
790 fn parse_zero_port_is_error() {
791 let err = ProxyUrl::parse("http://host:0").unwrap_err();
792 assert!(matches!(err, ParseError::PortOutOfRange { .. }));
793 }
794
795 #[test]
796 fn parse_non_numeric_port_is_error() {
797 let err = ProxyUrl::parse("http://host:abc").unwrap_err();
798 assert!(matches!(err, ParseError::NonNumericPort(ref s, _) if s == "abc"));
799 }
800
801 // ── VENDOR_QUIRKS table ──────────────────────────────────────────────────
802
803 #[test]
804 fn vendor_quirks_table_seeded_with_documented_providers() {
805 // Crawlera and Zyte are hard-error quirks (TLS-over-TLS trap).
806 assert_eq!(VENDOR_QUIRKS.len(), 4);
807 assert!(VENDOR_QUIRKS.iter().any(|q| q.host_suffix == "crawlera.com"
808 && q.port == Some(8011)
809 && q.required_scheme == Scheme::Http
810 && q.severity == QuirkSeverity::Error));
811 assert!(VENDOR_QUIRKS.iter().any(|q| q.host_suffix == "zyte.com"
812 && q.port == Some(8011)
813 && q.required_scheme == Scheme::Http
814 && q.severity == QuirkSeverity::Error));
815 // Bright Data and IPRoyal are warning quirks (username format
816 // warnings).
817 assert!(
818 VENDOR_QUIRKS
819 .iter()
820 .any(|q| q.host_suffix == "brd.superproxy.io"
821 && q.port == Some(22225)
822 && q.severity == QuirkSeverity::Warning)
823 );
824 assert!(
825 VENDOR_QUIRKS
826 .iter()
827 .any(|q| q.host_suffix == "iproyal.com" && q.severity == QuirkSeverity::Warning)
828 );
829 }
830
831 #[test]
832 fn vendor_quirks_table_is_const_constructible() {
833 // The slice can be evaluated in a const context, proving
834 // every record is a const literal.
835 const _: [VendorQuirk; 4] = [
836 VENDOR_QUIRKS[0],
837 VENDOR_QUIRKS[1],
838 VENDOR_QUIRKS[2],
839 VENDOR_QUIRKS[3],
840 ];
841 }
842
843 #[test]
844 fn vendor_quirks_descriptions_never_contain_credentials() {
845 // Security: descriptions are static text from the const table;
846 // they must never include password material.
847 for q in VENDOR_QUIRKS {
848 assert!(
849 !q.description.contains('@'),
850 "quirk has '@': {}",
851 q.description
852 );
853 assert!(
854 !q.description.contains("pass"),
855 "quirk mentions 'pass': {}",
856 q.description
857 );
858 }
859 }
860
861 // ── check: Crawlera / Zyte ───────────────────────────────────────────────
862
863 #[test]
864 fn check_crawlera_https_returns_error_match() {
865 let url = ProxyUrl::parse("https://user:pass@proxy.crawlera.com:8011").unwrap();
866 let m = check(&url);
867 assert_eq!(m.len(), 1);
868 assert_eq!(m[0].severity, QuirkSeverity::Error);
869 assert_eq!(m[0].required_scheme, Scheme::Http);
870 assert_eq!(m[0].host_suffix, "crawlera.com");
871 assert_eq!(m[0].observed_scheme, Scheme::Https);
872 assert_eq!(m[0].observed_port, Some(8011));
873 // The password is never echoed.
874 assert!(!m[0].description.contains("pass"));
875 }
876
877 #[test]
878 fn check_crawlera_http_compliant_returns_no_match() {
879 let url = ProxyUrl::parse("http://user:pass@proxy.crawlera.com:8011").unwrap();
880 assert!(check(&url).is_empty());
881 }
882
883 #[test]
884 fn check_crawlera_subdomain_matches() {
885 // `proxy.crawlera.com` is a subdomain of `crawlera.com`.
886 let url = ProxyUrl::parse("https://k:@proxy.crawlera.com:8011").unwrap();
887 let m = check(&url);
888 assert_eq!(m.len(), 1);
889 assert_eq!(m[0].severity, QuirkSeverity::Error);
890 }
891
892 #[test]
893 fn check_zyte_https_returns_error_match() {
894 let url = ProxyUrl::parse("https://apikey:@proxy.zyte.com:8011").unwrap();
895 let m = check(&url);
896 assert_eq!(m.len(), 1);
897 assert_eq!(m[0].severity, QuirkSeverity::Error);
898 assert_eq!(m[0].host_suffix, "zyte.com");
899 }
900
901 #[test]
902 fn check_zyte_http_compliant_returns_no_match() {
903 let url = ProxyUrl::parse("http://apikey:@proxy.zyte.com:8011").unwrap();
904 assert!(check(&url).is_empty());
905 }
906
907 // ── check: Bright Data ───────────────────────────────────────────────────
908
909 #[test]
910 fn check_bright_data_with_session_id_returns_warning() {
911 let url = ProxyUrl::parse("http://brd-customer-1-session-abc123@brd.superproxy.io:22225")
912 .unwrap();
913 let m = check(&url);
914 assert_eq!(m.len(), 1);
915 assert_eq!(m[0].severity, QuirkSeverity::Warning);
916 assert_eq!(m[0].host_suffix, "brd.superproxy.io");
917 }
918
919 #[test]
920 fn check_bright_data_without_session_id_returns_warning() {
921 let url = ProxyUrl::parse("http://user@brd.superproxy.io:22225").unwrap();
922 let m = check(&url);
923 assert_eq!(m.len(), 1);
924 assert_eq!(m[0].severity, QuirkSeverity::Warning);
925 }
926
927 #[test]
928 fn check_bright_data_wrong_port_returns_no_match() {
929 let url = ProxyUrl::parse("http://user@brd.superproxy.io:9999").unwrap();
930 assert!(check(&url).is_empty());
931 }
932
933 // ── check: IPRoyal ───────────────────────────────────────────────────────
934
935 #[test]
936 fn check_iproyal_returns_warning() {
937 let url = ProxyUrl::parse("http://user:pass@residential.iproyal.com:12321").unwrap();
938 let m = check(&url);
939 assert_eq!(m.len(), 1);
940 assert_eq!(m[0].severity, QuirkSeverity::Warning);
941 assert_eq!(m[0].host_suffix, "iproyal.com");
942 }
943
944 // ── check: no false positives ────────────────────────────────────────────
945
946 #[test]
947 fn check_unknown_host_returns_empty() {
948 let url = ProxyUrl::parse("http://user:pass@some-unrelated-host.example:8080").unwrap();
949 assert!(check(&url).is_empty());
950 }
951
952 #[test]
953 fn check_empty_url_returns_empty() {
954 // A URL with no matching host_suffix produces no quirks, even
955 // when the rest of the URL is unusual.
956 let url = ProxyUrl::parse("http://1.2.3.4:80").unwrap();
957 assert!(check(&url).is_empty());
958 }
959
960 #[test]
961 fn check_host_substring_does_not_match() {
962 // `mycrawlera.com` should NOT match the `crawlera.com` suffix
963 // (no `.` boundary).
964 let url = ProxyUrl::parse("https://user:pass@mycrawlera.com:8011").unwrap();
965 assert!(check(&url).is_empty());
966 }
967
968 #[test]
969 fn check_crawlera_8011_https_only_fires_for_8011() {
970 // Crawlera on a non-8011 port is not a quirk match.
971 let url = ProxyUrl::parse("https://user:pass@proxy.crawlera.com:9000").unwrap();
972 assert!(check(&url).is_empty());
973 }
974
975 // ── check: zero-allocation on empty result ───────────────────────────────
976
977 /// Sanity: the `check` function returns an empty `Vec` (no
978 /// allocations beyond the empty `Vec::new()` itself) for URLs
979 /// that don't match any quirk.
980 #[test]
981 fn check_unknown_host_returns_empty_vec() {
982 let url = ProxyUrl::parse("http://unrelated.example:80").unwrap();
983 let m = check(&url);
984 assert!(m.is_empty());
985 assert_eq!(m.capacity(), 0);
986 }
987
988 // ── ProxyUrl default port handling ───────────────────────────────────────
989
990 #[test]
991 fn validate_quirk_with_no_port_matches_any_port() {
992 // IPRoyal quirk has `port = None` so it matches any port.
993 let url = ProxyUrl::parse("http://user:pass@residential.iproyal.com:54321").unwrap();
994 let m = check(&url);
995 assert_eq!(m.len(), 1);
996 assert_eq!(m[0].host_suffix, "iproyal.com");
997 }
998
999 // ── module-level quirks count ────────────────────────────────────────────
1000
1001 #[test]
1002 fn all_known_quirks_slice_includes_every_constant() {
1003 // The `ALL_KNOWN_*` companion-slice pattern (mirrored from
1004 // `types::well_known::ALL_KNOWN_ASNS`) lets callers iterate
1005 // every constant without re-listing it.
1006 assert!(VENDOR_QUIRKS.contains(&CRAWLERA_8011_QUIRK));
1007 assert!(VENDOR_QUIRKS.contains(&ZYTE_8011_QUIRK));
1008 assert!(VENDOR_QUIRKS.contains(&BRD_SUPERPROXY_QUIRK));
1009 assert!(VENDOR_QUIRKS.contains(&IPROYAL_QUIRK));
1010 }
1011}