use log::warn;
const MAX_URL_LENGTH: usize = 2048;
pub fn validate_and_normalize_url(url: &str) -> Option<String> {
if url.len() > MAX_URL_LENGTH {
warn!(
"Skipping URL exceeding maximum length ({} > {}): {}...",
url.len(),
MAX_URL_LENGTH,
&url[..50.min(url.len())]
);
return None;
}
let normalized = if url.contains("://") {
url.to_string()
} else {
format!("https://{url}")
};
if normalized.len() > MAX_URL_LENGTH {
warn!(
"Skipping normalized URL exceeding maximum length ({} > {}): {}...",
normalized.len(),
MAX_URL_LENGTH,
&normalized[..50.min(normalized.len())]
);
return None;
}
if let Ok(parsed) = url::Url::parse(&normalized) {
match parsed.scheme() {
"http" | "https" => Some(normalized),
_ => {
warn!("Skipping unsupported scheme for URL: {url}");
None
}
}
} else {
warn!("Skipping invalid URL: {url}");
None
}
}
#[cfg(test)]
mod tests {
use super::validate_and_normalize_url;
#[test]
fn test_validate_and_normalize_url_adds_https() {
let result = validate_and_normalize_url("example.com");
assert_eq!(result, Some("https://example.com".to_string()));
}
#[test]
fn test_validate_and_normalize_url_preserves_https() {
let result = validate_and_normalize_url("https://example.com");
assert_eq!(result, Some("https://example.com".to_string()));
}
#[test]
fn test_validate_and_normalize_url_preserves_http() {
let result = validate_and_normalize_url("http://example.com");
assert_eq!(result, Some("http://example.com".to_string()));
}
#[test]
fn test_validate_and_normalize_url_rejects_unsupported_scheme() {
let result = validate_and_normalize_url("not a url at all!!!");
assert_eq!(result, None);
}
#[test]
fn test_validate_and_normalize_url_rejects_invalid_url() {
let result = validate_and_normalize_url("not a valid url!!!");
assert_eq!(result, None);
}
#[test]
fn test_validate_and_normalize_url_with_path() {
let result = validate_and_normalize_url("example.com/path?query=value");
assert_eq!(
result,
Some("https://example.com/path?query=value".to_string())
);
}
#[test]
fn test_validate_and_normalize_url_with_port() {
let result = validate_and_normalize_url("example.com:8080");
assert_eq!(result, Some("https://example.com:8080".to_string()));
}
#[test]
fn test_validate_and_normalize_url_ipv6() {
let result = validate_and_normalize_url("http://[2001:db8::1]");
assert_eq!(result, Some("http://[2001:db8::1]".to_string()));
let result = validate_and_normalize_url("[2001:db8::1]");
assert_eq!(result, Some("https://[2001:db8::1]".to_string()));
let result = validate_and_normalize_url("https://[2001:db8::1]:8080");
assert_eq!(result, Some("https://[2001:db8::1]:8080".to_string()));
}
#[test]
fn test_validate_and_normalize_url_ipv6_with_path() {
let result = validate_and_normalize_url("[2001:db8::1]/path/to/resource");
assert_eq!(
result,
Some("https://[2001:db8::1]/path/to/resource".to_string())
);
}
#[test]
fn test_validate_and_normalize_url_internationalized_domain() {
let result = validate_and_normalize_url("http://xn--mnchen-3ya.de");
if let Some(url_str) = &result {
assert!(url_str.starts_with("http://"));
}
let result = validate_and_normalize_url("http://例え.テスト");
if let Some(url_str) = &result {
assert!(url_str.starts_with("http://"));
}
}
#[test]
fn test_validate_and_normalize_url_complex_paths() {
let result = validate_and_normalize_url("example.com/path/to/resource?key=value&other=123");
assert_eq!(
result,
Some("https://example.com/path/to/resource?key=value&other=123".to_string())
);
let result = validate_and_normalize_url("example.com/path#fragment");
assert_eq!(
result,
Some("https://example.com/path#fragment".to_string())
);
}
#[test]
fn test_validate_and_normalize_url_unsupported_schemes() {
assert_eq!(validate_and_normalize_url("ftp://example.com"), None);
assert_eq!(validate_and_normalize_url("file:///path/to/file"), None);
let mailto_result = validate_and_normalize_url("mailto:test@example.com");
if let Some(ref u) = mailto_result {
assert!(u.starts_with("http://") || u.starts_with("https://"));
}
assert_eq!(
validate_and_normalize_url("http://example.com"),
Some("http://example.com".to_string())
);
assert_eq!(
validate_and_normalize_url("https://example.com"),
Some("https://example.com".to_string())
);
}
#[test]
fn test_validate_and_normalize_url_edge_cases() {
assert_eq!(validate_and_normalize_url(""), None);
assert_eq!(validate_and_normalize_url(" "), None);
assert_eq!(validate_and_normalize_url("https://"), None);
}
#[test]
fn test_validate_and_normalize_url_subdomain() {
let result = validate_and_normalize_url("subdomain.example.com");
assert_eq!(result, Some("https://subdomain.example.com".to_string()));
let result = validate_and_normalize_url("www.example.com");
assert_eq!(result, Some("https://www.example.com".to_string()));
}
#[test]
fn test_validate_and_normalize_url_with_userinfo() {
let result = validate_and_normalize_url("https://user:pass@example.com");
assert_eq!(result, Some("https://user:pass@example.com".to_string()));
let result = validate_and_normalize_url("https://user@example.com");
assert_eq!(result, Some("https://user@example.com".to_string()));
}
#[test]
fn test_validate_and_normalize_url_http_preserved() {
let result = validate_and_normalize_url("http://example.com");
assert_eq!(result, Some("http://example.com".to_string()));
assert!(!result.unwrap().starts_with("https://"));
}
#[test]
fn test_validate_and_normalize_url_https_preserved() {
let result = validate_and_normalize_url("https://example.com");
assert_eq!(result, Some("https://example.com".to_string()));
}
#[test]
fn test_validate_and_normalize_url_malformed() {
assert_eq!(validate_and_normalize_url("://example.com"), None);
assert_eq!(validate_and_normalize_url("http://"), None);
}
#[test]
fn test_validate_and_normalize_url_special_characters() {
assert_eq!(
validate_and_normalize_url("example.com/path%20with%20spaces"),
Some("https://example.com/path%20with%20spaces".to_string())
);
assert_eq!(
validate_and_normalize_url("example.com/path+with+plus"),
Some("https://example.com/path+with+plus".to_string())
);
}
#[test]
fn test_validate_and_normalize_url_rejects_too_long_url() {
let long_path = "a".repeat(2100);
let long_url = format!("https://example.com/{}", long_path);
let result = validate_and_normalize_url(&long_url);
assert_eq!(result, None, "Should reject URL exceeding maximum length");
}
#[test]
fn test_validate_and_normalize_url_accepts_url_at_limit() {
let path = "a".repeat(2028);
let url_at_limit = format!("https://example.com/{}", path);
assert_eq!(
url_at_limit.len(),
2048,
"URL should be exactly 2048 characters"
);
let result = validate_and_normalize_url(&url_at_limit);
assert!(
result.is_some(),
"Should accept URL at maximum length (2048 chars)"
);
}
#[test]
fn test_validate_and_normalize_url_rejects_too_long_url_after_normalization() {
let path = "a".repeat(2045); let url = format!("example.com/{}", path);
let result = validate_and_normalize_url(&url);
assert_eq!(
result, None,
"Should reject URL that exceeds limit after normalization"
);
}
use proptest::prelude::*;
proptest! {
#[test]
fn test_url_normalization_idempotent(url in "[a-z]{3,20}\\.[a-z]{2,5}") {
let normalized1 = validate_and_normalize_url(&url);
if let Some(n1) = normalized1 {
let normalized2 = validate_and_normalize_url(&n1);
prop_assert_eq!(Some(n1.clone()), normalized2,
"Normalizing twice should produce same result");
}
}
#[test]
fn test_url_length_validation(
domain in "[a-z]{3,20}\\.[a-z]{2,5}",
path in prop::collection::vec("[a-z]{1,10}", 0..200)
) {
let url = format!("https://{}/{}", domain, path.join("/"));
let result = validate_and_normalize_url(&url);
if url.len() <= 2048 {
prop_assert!(result.is_some(),
"Valid URL under limit should normalize successfully");
} else {
prop_assert!(result.is_none(),
"URL over 2048 chars should be rejected");
}
}
#[test]
fn test_url_scheme_handling(domain in "[a-z]{3,20}\\.[a-z]{2,5}") {
let no_scheme = validate_and_normalize_url(&domain);
prop_assert!(no_scheme.is_some());
prop_assert!(no_scheme.unwrap().starts_with("https://"));
let http_url = format!("http://{}", domain);
let with_http = validate_and_normalize_url(&http_url);
prop_assert!(with_http.is_some());
prop_assert!(with_http.unwrap().starts_with("http://"));
}
#[test]
fn test_url_special_chars_no_panic(
domain in "[a-z]{3,20}\\.[a-z]{2,5}",
path in "[^/]{0,100}"
) {
let url = format!("https://{}/{}", domain, path);
let _result = validate_and_normalize_url(&url);
}
#[test]
fn test_url_with_query_and_fragment(
domain in "[a-z]{3,20}\\.[a-z]{2,5}",
query in "[a-z]{0,50}",
fragment in "[a-z]{0,50}"
) {
let url = format!("{}?query={}&key=value#{}", domain, query, fragment);
let result = validate_and_normalize_url(&url);
if let Some(normalized) = result {
prop_assert!(normalized.starts_with("https://"));
prop_assert!(normalized.contains(&domain));
}
}
#[test]
fn test_url_port_validation(
domain in "[a-z]{3,20}\\.[a-z]{2,5}",
port in 1u16..=65535
) {
let url = format!("{}:{}", domain, port);
let result = validate_and_normalize_url(&url);
prop_assert!(result.is_some());
if let Some(normalized) = result {
prop_assert!(normalized.contains(&port.to_string()));
}
}
#[test]
fn test_url_subdomain_normalization(
subdomain in "[a-z]{2,10}",
domain in "[a-z]{3,15}",
tld in "(com|org|net)"
) {
let url = format!("{}.{}.{}", subdomain, domain, tld);
let result = validate_and_normalize_url(&url);
prop_assert!(result.is_some());
if let Some(normalized) = result {
prop_assert!(normalized.starts_with("https://"));
prop_assert!(normalized.contains(&subdomain));
prop_assert!(normalized.contains(&domain));
prop_assert!(normalized.contains(&tld));
}
}
}
}