use crate::utils::url::UrlError;
use std::collections::BTreeMap;
use url::Url;
#[derive(Debug, Clone, Copy, Default)]
pub struct UrlNormalizer {
strip_query: bool,
}
impl UrlNormalizer {
pub fn new(strip_query: bool) -> Self {
Self { strip_query }
}
pub fn with_default() -> Self {
Self::new(false)
}
pub fn normalize(&self, url: &str) -> Result<String, UrlError> {
let trimmed = url.trim();
if trimmed.is_empty() {
return Err(UrlError::InvalidUrl("empty URL".to_string()));
}
let mut normalized = parse_and_validate_http_url(trimmed)?;
normalized.set_fragment(None);
lowercase_host(&mut normalized)?;
normalize_path(&mut normalized);
self.normalize_query(&mut normalized);
Ok(normalized.to_string())
}
fn normalize_query(&self, normalized: &mut Url) {
if self.strip_query {
normalized.set_query(None);
} else if let Some(q) = normalized.query() {
let filtered = Self::strip_known_tracking(q);
let sorted = Self::sort_query(&filtered);
normalized.set_query(Some(&sorted));
}
}
const TRACKING_EXACT_KEYS: &[&str] = &[
"fbclid", "gclid", "msclkid", "dclid", "twclid", "mc_eid", "_ga",
];
fn strip_known_tracking(query: &str) -> String {
let pairs: Vec<(String, String)> = match serde_urlencoded::from_str(query) {
Ok(pairs) => pairs,
Err(_) => return query.to_string(),
};
let kept: Vec<(String, String)> = pairs
.into_iter()
.filter(|(k, _)| {
!k.starts_with("utm_") && !Self::TRACKING_EXACT_KEYS.contains(&k.as_str())
})
.collect();
serde_urlencoded::to_string(&kept).unwrap_or_else(|_| query.to_string())
}
fn sort_query(query: &str) -> String {
let pairs: Vec<(String, String)> = match serde_urlencoded::from_str(query) {
Ok(p) => p,
Err(_) => return query.to_string(),
};
let mut grouped: BTreeMap<String, Vec<String>> = BTreeMap::new();
for (k, v) in pairs {
grouped.entry(k).or_default().push(v);
}
let total_pairs: usize = grouped.values().map(|v| v.len()).sum();
let mut parts: Vec<String> = Vec::with_capacity(total_pairs);
for (k, vs) in grouped {
for v in vs {
parts.push(format!("{}={}", k, v));
}
}
parts.join("&")
}
pub fn permutations(&self, url: &str) -> Vec<String> {
let parsed = match Url::parse(url.trim()) {
Ok(u) => u,
Err(_) => return vec![url.to_string()],
};
if !matches!(parsed.scheme(), "http" | "https") {
return vec![url.to_string()];
}
let mut variants = Self::expand_variants(&parsed);
variants.sort_unstable();
variants.dedup();
variants
}
fn expand_variants(parsed: &Url) -> Vec<String> {
let mut variants: Vec<String> = Vec::with_capacity(16);
let host = parsed.host_str().unwrap_or("");
let path = parsed.path();
let query = parsed.query();
let host_variants: Vec<String> = if let Some(rest) = host.strip_prefix("www.") {
vec![host.to_string(), rest.to_string()] } else {
vec![host.to_string(), format!("www.{}", host)] };
let scheme_variants = ["https", "http"];
let path_variants = Self::path_variants(path);
for scheme in scheme_variants {
for host_v in &host_variants {
for path_v in &path_variants {
let mut candidate = Url::parse(&format!("{}://{}{}", scheme, host_v, path_v))
.unwrap_or_else(|_| parsed.clone());
candidate.set_query(query);
candidate.set_fragment(None);
variants.push(candidate.to_string());
}
}
}
variants
}
fn path_variants(path: &str) -> Vec<String> {
let mut variants: Vec<String> = Vec::with_capacity(4);
variants.push(path.to_string());
for suffix in ["/index.html", "/index.htm", "/index.php"] {
if let Some(stripped) = path.strip_suffix(suffix) {
let stripped = if stripped.is_empty() { "/" } else { stripped };
variants.push(stripped.to_string());
} else {
variants.push(format!("{}{}", path, suffix));
}
}
variants.sort_unstable();
variants.dedup();
variants
}
}
fn parse_and_validate_http_url(trimmed: &str) -> Result<Url, UrlError> {
let parsed =
Url::parse(trimmed).map_err(|e| UrlError::InvalidUrl(format!("parse failed: {e}")))?;
if !matches!(parsed.scheme(), "http" | "https") {
return Err(UrlError::InvalidUrl(format!(
"unsupported scheme: {} (only http/https)",
parsed.scheme()
)));
}
Ok(parsed)
}
fn lowercase_host(normalized: &mut Url) -> Result<(), UrlError> {
if let Some(host) = normalized.host_str() {
let lower = host.to_lowercase();
if lower != host {
normalized
.set_host(Some(&lower))
.map_err(|e| UrlError::InvalidUrl(format!("set_host failed: {e}")))?;
}
}
Ok(())
}
fn normalize_path(normalized: &mut Url) {
let path = normalized.path().to_string();
if path.len() > 1 && path.ends_with('/') {
let new_path = path.trim_end_matches('/');
normalized.set_path(if new_path.is_empty() { "/" } else { new_path });
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_normalize_lowercase_scheme_and_host() {
let n = UrlNormalizer::new(false);
assert_eq!(
n.normalize("HTTPS://Example.COM/Path").unwrap(),
"https://example.com/Path"
);
}
#[test]
fn test_normalize_preserves_path_case() {
let n = UrlNormalizer::new(false);
assert_eq!(
n.normalize("https://example.com/CamelCasePath").unwrap(),
"https://example.com/CamelCasePath"
);
}
#[test]
fn test_normalize_strips_fragment() {
let n = UrlNormalizer::new(false);
assert_eq!(
n.normalize("https://example.com/path#section").unwrap(),
"https://example.com/path"
);
}
#[test]
fn test_normalize_strips_fragment_with_query() {
let n = UrlNormalizer::new(false);
assert_eq!(
n.normalize("https://example.com/path?a=1#frag").unwrap(),
"https://example.com/path?a=1"
);
}
#[test]
fn test_normalize_removes_trailing_slash() {
let n = UrlNormalizer::new(false);
assert_eq!(
n.normalize("https://example.com/path/").unwrap(),
"https://example.com/path"
);
}
#[test]
fn test_normalize_preserves_root_slash() {
let n = UrlNormalizer::new(false);
assert_eq!(
n.normalize("https://example.com/").unwrap(),
"https://example.com/"
);
}
#[test]
fn test_normalize_no_path_implicit_root() {
let n = UrlNormalizer::new(false);
assert_eq!(
n.normalize("https://example.com").unwrap(),
"https://example.com/"
);
}
#[test]
fn test_normalize_multiple_trailing_slashes() {
let n = UrlNormalizer::new(false);
assert_eq!(
n.normalize("https://example.com/path//").unwrap(),
"https://example.com/path"
);
}
#[test]
fn test_normalize_sorts_query() {
let n = UrlNormalizer::new(false);
assert_eq!(
n.normalize("https://example.com/path?b=2&a=1").unwrap(),
"https://example.com/path?a=1&b=2"
);
}
#[test]
fn test_normalize_sorts_query_stable_multi_values() {
let n = UrlNormalizer::new(false);
assert_eq!(
n.normalize("https://example.com/path?b=2&a=1&b=3").unwrap(),
"https://example.com/path?a=1&b=2&b=3"
);
}
#[test]
fn test_normalize_equivalent_urls_to_same_string() {
let n = UrlNormalizer::new(false);
let a = n
.normalize("HTTPS://Example.COM/Path/?b=2&a=1#frag")
.unwrap();
let b = n.normalize("https://example.com/Path?a=1&b=2").unwrap();
assert_eq!(a, b);
}
#[test]
fn test_normalize_strip_query_true() {
let n = UrlNormalizer::new(true);
assert_eq!(
n.normalize("https://example.com/path?a=1&b=2").unwrap(),
"https://example.com/path"
);
}
#[test]
fn test_normalize_strip_query_false_keeps_query() {
let n = UrlNormalizer::new(false);
assert_eq!(
n.normalize("https://example.com/path?a=1&b=2").unwrap(),
"https://example.com/path?a=1&b=2"
);
}
#[test]
fn test_normalize_strip_query_equivalent_urls() {
let n = UrlNormalizer::new(true);
let a = n.normalize("https://example.com/path?a=1").unwrap();
let b = n.normalize("https://example.com/path?b=2").unwrap();
assert_eq!(a, b);
assert_eq!(a, "https://example.com/path");
}
#[test]
fn test_normalize_empty_url_returns_err() {
let n = UrlNormalizer::new(false);
assert!(n.normalize("").is_err());
}
#[test]
fn test_normalize_whitespace_only_returns_err() {
let n = UrlNormalizer::new(false);
assert!(n.normalize(" ").is_err());
}
#[test]
fn test_normalize_invalid_url_returns_err() {
let n = UrlNormalizer::new(false);
assert!(n.normalize("not a url").is_err());
}
#[test]
fn test_normalize_unsupported_scheme_returns_err() {
let n = UrlNormalizer::new(false);
let result = n.normalize("ftp://example.com/file");
assert!(result.is_err() || result.is_ok());
if let Ok(s) = result {
assert!(
s.starts_with("http"),
"non-http should be rejected, got: {s}"
);
}
}
#[test]
fn test_normalize_preserves_port() {
let n = UrlNormalizer::new(false);
assert_eq!(
n.normalize("https://example.com:8080/path").unwrap(),
"https://example.com:8080/path"
);
}
#[test]
fn test_normalize_idempotent() {
let n = UrlNormalizer::new(false);
let once = n
.normalize("HTTPS://Example.COM/Path/?b=2&a=1#frag")
.unwrap();
let twice = n.normalize(&once).unwrap();
assert_eq!(once, twice);
}
#[test]
fn test_permutations_includes_original() {
let n = UrlNormalizer::new(false);
let perms = n.permutations("https://example.com/path");
assert!(perms.iter().any(|u| u.contains("example.com/path")));
}
#[test]
fn test_permutations_www_and_non_www() {
let n = UrlNormalizer::new(false);
let perms = n.permutations("https://example.com/path");
assert!(
perms.iter().any(|u| u.contains("://example.com")),
"missing non-www"
);
assert!(
perms.iter().any(|u| u.contains("://www.example.com")),
"missing www"
);
}
#[test]
fn test_permutations_http_and_https() {
let n = UrlNormalizer::new(false);
let perms = n.permutations("https://example.com/path");
let has_https = perms.iter().any(|u| u.starts_with("https://"));
let has_http = perms.iter().any(|u| u.starts_with("http://"));
assert!(has_https, "missing https variant");
assert!(has_http, "missing http variant");
}
#[test]
fn test_permutations_index_html() {
let n = UrlNormalizer::new(false);
let perms = n.permutations("https://example.com/path");
assert!(
perms.iter().any(|u| u.contains("/path/index.html")),
"missing index.html variant"
);
assert!(
perms.iter().any(|u| u.contains("/path/index.htm")),
"missing index.htm variant"
);
}
#[test]
fn test_permutations_strips_index_html() {
let n = UrlNormalizer::new(false);
let perms = n.permutations("https://example.com/path/index.html");
assert!(
perms
.iter()
.any(|u| u == "https://example.com/path" || u == "https://example.com/path/"),
"missing stripped index.html variant, got: {:?}",
perms
);
}
#[test]
fn test_permutations_from_www_includes_non_www() {
let n = UrlNormalizer::new(false);
let perms = n.permutations("https://www.example.com/path");
assert!(
perms
.iter()
.any(|u| u.contains("://example.com/path") && !u.contains("www.")),
"missing non-www variant"
);
}
#[test]
fn test_permutations_deduplicates() {
let n = UrlNormalizer::new(false);
let perms = n.permutations("https://example.com/");
let mut sorted = perms.clone();
sorted.sort();
let original_len = perms.len();
sorted.dedup();
assert_eq!(sorted.len(), original_len, "duplicates found");
}
#[test]
fn test_permutations_invalid_url_returns_original() {
let n = UrlNormalizer::new(false);
let perms = n.permutations("not a url");
assert_eq!(perms, vec!["not a url".to_string()]);
}
#[test]
fn test_permutations_non_http_scheme_returns_original() {
let n = UrlNormalizer::new(false);
let perms = n.permutations("ftp://example.com/file");
assert_eq!(perms, vec!["ftp://example.com/file".to_string()]);
}
#[test]
fn test_permutations_preserves_query() {
let n = UrlNormalizer::new(false);
let perms = n.permutations("https://example.com/path?a=1");
for p in &perms {
assert!(p.contains("?a=1"), "missing query in variant: {}", p);
}
}
#[test]
fn test_permutations_count_at_least_4() {
let n = UrlNormalizer::new(false);
let perms = n.permutations("https://example.com/path");
assert!(
perms.len() >= 4,
"expected >= 4 variants, got: {}",
perms.len()
);
}
#[test]
fn test_new_default_keeps_query() {
let n = UrlNormalizer::with_default();
assert_eq!(
n.normalize("https://example.com/path?a=1").unwrap(),
"https://example.com/path?a=1"
);
}
#[test]
fn test_clone_copy() {
let n1 = UrlNormalizer::new(true);
let n2 = n1; assert_eq!(
n1.normalize("https://example.com/path?a=1").unwrap(),
n2.normalize("https://example.com/path?a=1").unwrap()
);
}
#[test]
fn test_strip_tracking_utm_prefixed_keys() {
let n = UrlNormalizer::with_default();
let a = n
.normalize("https://example.com/page?utm_source=x&id=1")
.unwrap();
let b = n.normalize("https://example.com/page?id=1").unwrap();
assert_eq!(a, b, "utm_* variants must normalize to the same key");
assert!(!a.contains("utm_"), "utm params must be stripped, got {a}");
assert!(
a.contains("id=1"),
"non-tracking params must survive, got {a}"
);
}
#[test]
fn test_strip_tracking_exact_keys() {
let n = UrlNormalizer::with_default();
let a = n
.normalize("https://example.com/p?fbclid=abc&gclid=def&id=2")
.unwrap();
assert!(!a.contains("fbclid") && !a.contains("gclid"), "got {a}");
assert!(a.contains("id=2"), "got {a}");
}
#[test]
fn test_malformed_query_falls_back_without_panic() {
let n = UrlNormalizer::with_default();
let result = n.normalize("https://example.com/p?%%%bad&id=3");
assert!(result.is_ok(), "malformed query must not error");
}
}