use std::str::FromStr;
use serde::{Deserialize, Serialize};
use thiserror::Error;
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Scheme {
#[default]
Http,
Https,
#[cfg(feature = "socks")]
Socks4,
#[cfg(feature = "socks")]
Socks5,
}
impl Scheme {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::Http => "http",
Self::Https => "https",
#[cfg(feature = "socks")]
Self::Socks4 => "socks4",
#[cfg(feature = "socks")]
Self::Socks5 => "socks5",
}
}
}
impl FromStr for Scheme {
type Err = ();
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"http" => Ok(Self::Http),
"https" => Ok(Self::Https),
#[cfg(feature = "socks")]
"socks4" => Ok(Self::Socks4),
#[cfg(feature = "socks")]
"socks5" => Ok(Self::Socks5),
_ => Err(()),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum QuirkSeverity {
Info,
Warning,
Error,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct VendorQuirk {
pub host_suffix: &'static str,
pub port: Option<u16>,
pub required_scheme: Scheme,
pub description: &'static str,
pub severity: QuirkSeverity,
}
pub const CRAWLERA_8011_QUIRK: VendorQuirk = VendorQuirk {
host_suffix: "crawlera.com",
port: Some(8011),
required_scheme: Scheme::Http,
description: "Crawlera port 8011 is plain HTTP — using https:// causes BoringSSL WRONG_VERSION_NUMBER (TLS-over-TLS failure). Use http:// on both http and https scraping keys.",
severity: QuirkSeverity::Error,
};
pub const ZYTE_8011_QUIRK: VendorQuirk = VendorQuirk {
host_suffix: "zyte.com",
port: Some(8011),
required_scheme: Scheme::Http,
description: "Zyte Smart Proxy Manager port 8011 is plain HTTP — same WRONG_VERSION_NUMBER trap as Crawlera. Use http:// scheme on both http and https keys.",
severity: QuirkSeverity::Error,
};
pub const BRD_SUPERPROXY_QUIRK: VendorQuirk = VendorQuirk {
host_suffix: "brd.superproxy.io",
port: Some(22225),
required_scheme: Scheme::Http,
description: "Bright Data brd.superproxy.io:22225 requires a brd-customer-<id>-session-<id> username; a missing -session-<id> suffix silently collapses traffic into the default session pool.",
severity: QuirkSeverity::Warning,
};
pub const IPROYAL_QUIRK: VendorQuirk = VendorQuirk {
host_suffix: "iproyal.com",
port: None,
required_scheme: Scheme::Http,
description: "IPRoyal residential traffic requires a country flag in the username (e.g. user-country-US). Without the flag, the gateway ignores the requested exit country.",
severity: QuirkSeverity::Warning,
};
pub const VENDOR_QUIRKS: &[VendorQuirk] = &[
CRAWLERA_8011_QUIRK,
ZYTE_8011_QUIRK,
BRD_SUPERPROXY_QUIRK,
IPROYAL_QUIRK,
];
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProxyUrl {
pub scheme: Scheme,
pub host: String,
pub port: Option<u16>,
pub username: Option<String>,
pub password: Option<String>,
pub path: Option<String>,
}
#[derive(Debug, Error, PartialEq, Eq)]
pub enum ParseError {
#[error("missing scheme separator '://' in URL `{0}`")]
MissingSchemeSeparator(String),
#[error("unsupported scheme `{0}` in URL `{1}`")]
UnsupportedScheme(String, String),
#[error("empty host in URL `{0}`")]
EmptyHost(String),
#[error("port `{port}` is out of range [1, 65535] in URL `{url}`")]
PortOutOfRange {
port: String,
url: String,
},
#[error("non-numeric port `{0}` in URL `{1}`")]
NonNumericPort(String, String),
#[error("unclosed IPv6 bracket in URL `{0}`")]
UnclosedIpv6Bracket(String),
}
impl ProxyUrl {
pub fn parse(url: &str) -> Result<Self, ParseError> {
let (scheme_str, rest) = url
.split_once("://")
.ok_or_else(|| ParseError::MissingSchemeSeparator(url.to_owned()))?;
let scheme = Scheme::from_str(scheme_str)
.map_err(|()| ParseError::UnsupportedScheme(scheme_str.to_owned(), url.to_owned()))?;
let (userinfo, authority_with_path) = match rest.split_once('@') {
Some((ui, auth)) => (ui, auth),
None => ("", rest),
};
let (authority, path) = match authority_with_path.split_once('/') {
Some((a, p)) => (a, Some(format!("/{p}"))),
None => (authority_with_path, None),
};
let (username, password) = if userinfo.is_empty() {
(None, None)
} else {
match userinfo.split_once(':') {
Some((u, p)) => (Some(u.to_owned()), Some(p.to_owned())),
None => (Some(userinfo.to_owned()), None),
}
};
let (host, port_str) = if let Some(stripped) = authority.strip_prefix('[') {
let close = stripped
.find(']')
.ok_or_else(|| ParseError::UnclosedIpv6Bracket(url.to_owned()))?;
let host = &stripped[..close];
let after = &stripped[close + 1..];
let port_str = after.strip_prefix(':').unwrap_or("");
(host.to_owned(), port_str)
} else {
match authority.rsplit_once(':') {
Some((h, p)) => (h.to_owned(), p),
None => (authority.to_owned(), ""),
}
};
if host.is_empty() {
return Err(ParseError::EmptyHost(url.to_owned()));
}
let port = if port_str.is_empty() {
None
} else {
let parsed: u32 = port_str
.parse()
.map_err(|_| ParseError::NonNumericPort(port_str.to_owned(), url.to_owned()))?;
if parsed == 0 || parsed > 65535 {
return Err(ParseError::PortOutOfRange {
port: port_str.to_owned(),
url: url.to_owned(),
});
}
#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
let as_u16 = parsed as u16;
Some(as_u16)
};
Ok(Self {
scheme,
host: host.to_ascii_lowercase(),
port,
username,
password,
path,
})
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct QuirkMatch {
pub severity: QuirkSeverity,
pub host_suffix: &'static str,
pub required_scheme: Scheme,
pub description: &'static str,
pub observed_scheme: Scheme,
pub observed_port: Option<u16>,
}
#[must_use]
pub fn check(url: &ProxyUrl) -> Vec<QuirkMatch> {
let mut out: Vec<QuirkMatch> = Vec::new();
for quirk in VENDOR_QUIRKS {
if !host_suffix_matches(&url.host, quirk.host_suffix) {
continue;
}
if let Some(required_port) = quirk.port
&& url.port != Some(required_port)
{
continue;
}
match quirk.severity {
QuirkSeverity::Error => {
if url.scheme == quirk.required_scheme {
continue;
}
}
QuirkSeverity::Warning | QuirkSeverity::Info => {
}
}
out.push(QuirkMatch {
severity: quirk.severity,
host_suffix: quirk.host_suffix,
required_scheme: quirk.required_scheme,
description: quirk.description,
observed_scheme: url.scheme,
observed_port: url.port,
});
}
out
}
fn host_suffix_matches(host: &str, suffix: &str) -> bool {
if host == suffix {
return true;
}
if host.len() <= suffix.len() + 1 {
return false;
}
match host.as_bytes().get(host.len() - suffix.len() - 1) {
Some(b'.') => host.ends_with(suffix),
_ => false,
}
}
#[cfg(test)]
#[allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::panic,
clippy::indexing_slicing
)] mod tests {
use super::*;
#[test]
fn scheme_default_is_http() {
assert_eq!(Scheme::default(), Scheme::Http);
}
#[test]
fn scheme_as_str_matches_wire_format() {
assert_eq!(Scheme::Http.as_str(), "http");
assert_eq!(Scheme::Https.as_str(), "https");
}
#[test]
fn scheme_from_str_round_trip() {
for scheme in [Scheme::Http, Scheme::Https] {
assert_eq!(Scheme::from_str(scheme.as_str()), Ok(scheme));
}
assert_eq!(Scheme::from_str("nope"), Err(()));
}
#[test]
fn parse_simple_http_url() {
let p = ProxyUrl::parse("http://proxy.example.com:8080").unwrap();
assert_eq!(p.scheme, Scheme::Http);
assert_eq!(p.host, "proxy.example.com");
assert_eq!(p.port, Some(8080));
assert!(p.username.is_none());
assert!(p.password.is_none());
assert!(p.path.is_none());
}
#[test]
fn parse_https_url() {
let p = ProxyUrl::parse("https://proxy.example.com:8443/path").unwrap();
assert_eq!(p.scheme, Scheme::Https);
assert_eq!(p.host, "proxy.example.com");
assert_eq!(p.port, Some(8443));
assert_eq!(p.path.as_deref(), Some("/path"));
}
#[test]
fn parse_url_with_user_info() {
let p = ProxyUrl::parse("http://user:pass@proxy.test:3128").unwrap();
assert_eq!(p.scheme, Scheme::Http);
assert_eq!(p.host, "proxy.test");
assert_eq!(p.port, Some(3128));
assert_eq!(p.username.as_deref(), Some("user"));
assert_eq!(p.password.as_deref(), Some("pass"));
}
#[test]
fn parse_url_with_password_containing_colon() {
let p = ProxyUrl::parse("http://user:pa:ss@proxy.test:3128").unwrap();
assert_eq!(p.username.as_deref(), Some("user"));
assert_eq!(p.password.as_deref(), Some("pa:ss"));
}
#[test]
fn parse_url_with_username_only() {
let p = ProxyUrl::parse("http://user@proxy.test:3128").unwrap();
assert_eq!(p.username.as_deref(), Some("user"));
assert!(p.password.is_none());
}
#[test]
fn parse_url_default_ports_are_optional() {
let p = ProxyUrl::parse("http://proxy.test").unwrap();
assert_eq!(p.port, None);
}
#[test]
fn parse_url_lowercases_host() {
let p = ProxyUrl::parse("http://PROXY.Test:8080").unwrap();
assert_eq!(p.host, "proxy.test");
}
#[test]
fn parse_url_with_trailing_slash() {
let p = ProxyUrl::parse("http://proxy.test:8080/").unwrap();
assert_eq!(p.path.as_deref(), Some("/"));
}
#[test]
fn parse_ipv6_url_with_brackets() {
let p = ProxyUrl::parse("http://[::1]:8080").unwrap();
assert_eq!(p.host, "::1");
assert_eq!(p.port, Some(8080));
}
#[test]
fn parse_missing_scheme_separator_is_error() {
let err = ProxyUrl::parse("not-a-url").unwrap_err();
assert!(matches!(err, ParseError::MissingSchemeSeparator(_)));
}
#[test]
fn parse_unsupported_scheme_is_error() {
let err = ProxyUrl::parse("ftp://host:21").unwrap_err();
assert!(matches!(err, ParseError::UnsupportedScheme(ref s, _) if s == "ftp"));
}
#[test]
fn parse_empty_host_is_error() {
let err = ProxyUrl::parse("http://:8080").unwrap_err();
assert!(matches!(err, ParseError::EmptyHost(_)));
}
#[test]
fn parse_port_out_of_range_is_error() {
let err = ProxyUrl::parse("http://host:99999").unwrap_err();
assert!(matches!(err, ParseError::PortOutOfRange { .. }));
}
#[test]
fn parse_zero_port_is_error() {
let err = ProxyUrl::parse("http://host:0").unwrap_err();
assert!(matches!(err, ParseError::PortOutOfRange { .. }));
}
#[test]
fn parse_non_numeric_port_is_error() {
let err = ProxyUrl::parse("http://host:abc").unwrap_err();
assert!(matches!(err, ParseError::NonNumericPort(ref s, _) if s == "abc"));
}
#[test]
fn vendor_quirks_table_seeded_with_documented_providers() {
assert_eq!(VENDOR_QUIRKS.len(), 4);
assert!(VENDOR_QUIRKS.iter().any(|q| q.host_suffix == "crawlera.com"
&& q.port == Some(8011)
&& q.required_scheme == Scheme::Http
&& q.severity == QuirkSeverity::Error));
assert!(VENDOR_QUIRKS.iter().any(|q| q.host_suffix == "zyte.com"
&& q.port == Some(8011)
&& q.required_scheme == Scheme::Http
&& q.severity == QuirkSeverity::Error));
assert!(
VENDOR_QUIRKS
.iter()
.any(|q| q.host_suffix == "brd.superproxy.io"
&& q.port == Some(22225)
&& q.severity == QuirkSeverity::Warning)
);
assert!(
VENDOR_QUIRKS
.iter()
.any(|q| q.host_suffix == "iproyal.com" && q.severity == QuirkSeverity::Warning)
);
}
#[test]
fn vendor_quirks_table_is_const_constructible() {
const _: [VendorQuirk; 4] = [
VENDOR_QUIRKS[0],
VENDOR_QUIRKS[1],
VENDOR_QUIRKS[2],
VENDOR_QUIRKS[3],
];
}
#[test]
fn vendor_quirks_descriptions_never_contain_credentials() {
for q in VENDOR_QUIRKS {
assert!(
!q.description.contains('@'),
"quirk has '@': {}",
q.description
);
assert!(
!q.description.contains("pass"),
"quirk mentions 'pass': {}",
q.description
);
}
}
#[test]
fn check_crawlera_https_returns_error_match() {
let url = ProxyUrl::parse("https://user:pass@proxy.crawlera.com:8011").unwrap();
let m = check(&url);
assert_eq!(m.len(), 1);
assert_eq!(m[0].severity, QuirkSeverity::Error);
assert_eq!(m[0].required_scheme, Scheme::Http);
assert_eq!(m[0].host_suffix, "crawlera.com");
assert_eq!(m[0].observed_scheme, Scheme::Https);
assert_eq!(m[0].observed_port, Some(8011));
assert!(!m[0].description.contains("pass"));
}
#[test]
fn check_crawlera_http_compliant_returns_no_match() {
let url = ProxyUrl::parse("http://user:pass@proxy.crawlera.com:8011").unwrap();
assert!(check(&url).is_empty());
}
#[test]
fn check_crawlera_subdomain_matches() {
let url = ProxyUrl::parse("https://k:@proxy.crawlera.com:8011").unwrap();
let m = check(&url);
assert_eq!(m.len(), 1);
assert_eq!(m[0].severity, QuirkSeverity::Error);
}
#[test]
fn check_zyte_https_returns_error_match() {
let url = ProxyUrl::parse("https://apikey:@proxy.zyte.com:8011").unwrap();
let m = check(&url);
assert_eq!(m.len(), 1);
assert_eq!(m[0].severity, QuirkSeverity::Error);
assert_eq!(m[0].host_suffix, "zyte.com");
}
#[test]
fn check_zyte_http_compliant_returns_no_match() {
let url = ProxyUrl::parse("http://apikey:@proxy.zyte.com:8011").unwrap();
assert!(check(&url).is_empty());
}
#[test]
fn check_bright_data_with_session_id_returns_warning() {
let url = ProxyUrl::parse("http://brd-customer-1-session-abc123@brd.superproxy.io:22225")
.unwrap();
let m = check(&url);
assert_eq!(m.len(), 1);
assert_eq!(m[0].severity, QuirkSeverity::Warning);
assert_eq!(m[0].host_suffix, "brd.superproxy.io");
}
#[test]
fn check_bright_data_without_session_id_returns_warning() {
let url = ProxyUrl::parse("http://user@brd.superproxy.io:22225").unwrap();
let m = check(&url);
assert_eq!(m.len(), 1);
assert_eq!(m[0].severity, QuirkSeverity::Warning);
}
#[test]
fn check_bright_data_wrong_port_returns_no_match() {
let url = ProxyUrl::parse("http://user@brd.superproxy.io:9999").unwrap();
assert!(check(&url).is_empty());
}
#[test]
fn check_iproyal_returns_warning() {
let url = ProxyUrl::parse("http://user:pass@residential.iproyal.com:12321").unwrap();
let m = check(&url);
assert_eq!(m.len(), 1);
assert_eq!(m[0].severity, QuirkSeverity::Warning);
assert_eq!(m[0].host_suffix, "iproyal.com");
}
#[test]
fn check_unknown_host_returns_empty() {
let url = ProxyUrl::parse("http://user:pass@some-unrelated-host.example:8080").unwrap();
assert!(check(&url).is_empty());
}
#[test]
fn check_empty_url_returns_empty() {
let url = ProxyUrl::parse("http://1.2.3.4:80").unwrap();
assert!(check(&url).is_empty());
}
#[test]
fn check_host_substring_does_not_match() {
let url = ProxyUrl::parse("https://user:pass@mycrawlera.com:8011").unwrap();
assert!(check(&url).is_empty());
}
#[test]
fn check_crawlera_8011_https_only_fires_for_8011() {
let url = ProxyUrl::parse("https://user:pass@proxy.crawlera.com:9000").unwrap();
assert!(check(&url).is_empty());
}
#[test]
fn check_unknown_host_returns_empty_vec() {
let url = ProxyUrl::parse("http://unrelated.example:80").unwrap();
let m = check(&url);
assert!(m.is_empty());
assert_eq!(m.capacity(), 0);
}
#[test]
fn validate_quirk_with_no_port_matches_any_port() {
let url = ProxyUrl::parse("http://user:pass@residential.iproyal.com:54321").unwrap();
let m = check(&url);
assert_eq!(m.len(), 1);
assert_eq!(m[0].host_suffix, "iproyal.com");
}
#[test]
fn all_known_quirks_slice_includes_every_constant() {
assert!(VENDOR_QUIRKS.contains(&CRAWLERA_8011_QUIRK));
assert!(VENDOR_QUIRKS.contains(&ZYTE_8011_QUIRK));
assert!(VENDOR_QUIRKS.contains(&BRD_SUPERPROXY_QUIRK));
assert!(VENDOR_QUIRKS.contains(&IPROYAL_QUIRK));
}
}