use regex::Regex;
use std::collections::HashSet;
use crate::domain::root_domain;
use crate::fetch::cookies::extract_cookie_infos;
use crate::fetch::dns::{AdditionalDnsData, TlsDnsData};
use crate::fetch::external_scripts::collect_script_hosts;
use crate::fetch::response::{HtmlData, ResponseData};
use crate::storage::{PersistedUrlRecord, UrlRecord};
fn extract_csp_domains(csp: &str) -> Vec<(String, String, Option<String>)> {
let mut results = Vec::new();
let mut seen = HashSet::new();
static CSP_DOMAIN_RE: std::sync::LazyLock<Regex> = std::sync::LazyLock::new(|| {
Regex::new(
r"(?:https?://)?(\*\.)?([a-zA-Z0-9][-a-zA-Z0-9]*(?:\.[a-zA-Z0-9][-a-zA-Z0-9]*)+)",
)
.expect("CSP domain regex")
});
let domain_re = &*CSP_DOMAIN_RE;
for directive_part in csp.split(';') {
let parts: Vec<&str> = directive_part.trim().splitn(2, ' ').collect();
let directive = (*parts.first().unwrap_or(&"")).to_string();
let values = parts.get(1).unwrap_or(&"");
for cap in domain_re.captures_iter(values) {
if let Some(fqdn_match) = cap.get(2) {
let fqdn = fqdn_match.as_str().to_lowercase();
if fqdn == "self" || fqdn == "none" || fqdn == "unsafe-inline" {
continue;
}
let key = (directive.clone(), fqdn.clone());
if seen.insert(key) {
let reg = root_domain(&fqdn);
results.push((directive.clone(), fqdn, reg));
}
}
}
}
results
}
pub(crate) fn build_url_record(
resp_data: &ResponseData,
html_data: &HtmlData,
tls_dns_data: &TlsDnsData,
additional_dns: &AdditionalDnsData,
elapsed: f64,
timestamp: i64,
run_id: Option<&String>,
) -> UrlRecord {
UrlRecord {
initial_domain: resp_data.initial_domain.clone(),
final_domain: resp_data.final_domain.clone(),
initial_url: Some(resp_data.initial_url.clone()),
final_url: Some(resp_data.final_url.clone()),
ip_address: tls_dns_data.ip_address.clone().unwrap_or_default(),
reverse_dns_name: tls_dns_data.reverse_dns_name.clone(),
status: resp_data.status,
status_desc: resp_data.status_desc.clone(),
response_time: elapsed,
title: html_data.title.clone(),
description: html_data
.description
.as_ref()
.filter(|d| !d.is_empty())
.cloned(),
meta_robots: html_data.meta_robots.clone(),
tls_version: tls_dns_data.tls_version,
ssl_cert_subject: tls_dns_data.subject.clone(),
ssl_cert_issuer: tls_dns_data.issuer.clone(),
ssl_cert_valid_from: tls_dns_data.valid_from,
ssl_cert_valid_to: tls_dns_data.valid_to,
timestamp,
nameservers: additional_dns.nameservers.clone(),
txt_records: additional_dns.txt_records.clone(),
mx_records: additional_dns.mx_records.clone(),
spf_record: additional_dns.spf_record.clone(),
dmarc_record: additional_dns.dmarc_record.clone(),
cipher_suite: tls_dns_data.cipher_suite.clone(),
key_algorithm: tls_dns_data.key_algorithm.clone(),
run_id: run_id.cloned(),
body_sha256: resp_data.body_sha256.clone(),
body_truncated: resp_data.body_truncated,
external_scripts_eligible: html_data.external_scripts_eligible,
external_scripts_scanned: html_data.external_scripts_scanned,
content_length: resp_data.content_length,
http_version: resp_data.http_version.clone(),
content_type: resp_data.content_type.clone(),
canonical_url: html_data.canonical_url.clone(),
cert_fingerprint_sha256: tls_dns_data.cert_fingerprint_sha256.clone(),
cert_serial_number: tls_dns_data.cert_serial_number.clone(),
cert_is_self_signed: tls_dns_data.cert_is_self_signed,
cert_is_wildcard: tls_dns_data.cert_is_wildcard,
cert_is_mismatched: None, meta_refresh_url: html_data.meta_refresh_url.clone(),
hsts_max_age: None,
hsts_include_subdomains: None,
hsts_preload: None,
mta_sts_record: additional_dns.mta_sts_record.clone(),
tls_rpt_record: additional_dns.tls_rpt_record.clone(),
bimi_record: additional_dns.bimi_record.clone(),
cdn_provider: None,
}
}
pub struct PersistedUrlRecordParams {
pub record: UrlRecord,
pub resp_data: ResponseData,
pub html_data: HtmlData,
pub tls_dns_data: TlsDnsData,
pub technologies_vec: Vec<crate::fingerprint::DetectedTechnology>,
pub redirect_chain: Vec<(String, u16)>,
pub partial_failures: Vec<(crate::error_handling::ErrorType, String)>,
pub geoip_data: Option<(String, crate::geoip::GeoIpResult)>,
pub whois_data: Option<crate::whois::WhoisResult>,
pub timestamp: i64,
pub run_id: Option<String>,
pub favicon: Option<crate::fetch::favicon::FaviconData>,
pub additional_dns: crate::fetch::dns::AdditionalDnsData,
pub well_known: crate::fetch::well_known::WellKnownData,
}
pub(crate) fn build_persisted_url_record(
mut params: PersistedUrlRecordParams,
) -> PersistedUrlRecord {
let oids_set: std::collections::HashSet<String> =
params.tls_dns_data.oids.take().unwrap_or_default();
let partial_failure_records: Vec<crate::storage::models::UrlPartialFailureRecord> = params
.partial_failures
.into_iter()
.map(|(error_type, error_message)| {
crate::storage::models::UrlPartialFailureRecord {
url_status_id: 0, error_type,
error_message,
timestamp: params.timestamp,
run_id: params.run_id.clone(),
}
})
.collect();
let sans_vec: Vec<String> = params
.tls_dns_data
.subject_alternative_names
.take()
.unwrap_or_default();
let analytics_ids = std::mem::take(&mut params.html_data.analytics_ids);
let structured_data = std::mem::take(&mut params.html_data.structured_data);
let social_media_links = std::mem::take(&mut params.html_data.social_media_links);
let contact_links = std::mem::take(&mut params.html_data.contact_links);
let exposed_secrets = std::mem::take(&mut params.html_data.exposed_secrets);
let security_headers = std::mem::take(&mut params.resp_data.security_headers);
let http_headers = std::mem::take(&mut params.resp_data.http_headers);
let csp_domains = security_headers
.get("Content-Security-Policy")
.map(|csp| extract_csp_domains(csp))
.unwrap_or_default();
let cookies = extract_cookie_infos(¶ms.resp_data.headers);
let script_hosts = collect_script_hosts(
¶ms.resp_data.final_url,
¶ms.html_data.script_sources,
);
if let Some(hsts_value) = security_headers
.iter()
.find(|(k, _)| k.eq_ignore_ascii_case("strict-transport-security"))
.map(|(_, v)| v.as_str())
{
let d = crate::security::parse_hsts_directive(hsts_value);
params.record.hsts_max_age = d.max_age.map(|v| i64::try_from(v).unwrap_or(i64::MAX));
params.record.hsts_include_subdomains = Some(d.include_subdomains);
params.record.hsts_preload = Some(d.preload);
}
params.record.cdn_provider = crate::fetch::cdn::detect_cdn_provider(
&http_headers,
&security_headers,
params.additional_dns.cname_chain.as_deref(),
params.additional_dns.nameservers.as_deref(),
);
let domain = ¶ms.resp_data.host;
if !domain.is_empty() && params.tls_dns_data.tls_version.is_some() {
let cn = params.tls_dns_data.subject.as_deref().unwrap_or("");
let matches_san = sans_vec.iter().any(|san| {
if let Some(wildcard_base) = san.strip_prefix("*.") {
domain == wildcard_base
|| domain.strip_suffix(wildcard_base).is_some_and(|prefix| {
prefix.ends_with('.') && !prefix[..prefix.len() - 1].contains('.')
})
} else {
domain == san
}
});
let matches_cn = cn
.split(',')
.filter_map(|part| part.trim().strip_prefix("CN="))
.any(|cn_val| cn_val.trim() == domain);
params.record.cert_is_mismatched = Some(!matches_san && !matches_cn);
}
PersistedUrlRecord {
url_record: params.record,
security_headers,
http_headers,
oids: oids_set,
redirect_chain: params.redirect_chain,
technologies: params.technologies_vec,
subject_alternative_names: sans_vec,
analytics_ids,
geoip: params.geoip_data,
structured_data: Some(structured_data),
social_media_links,
contact_links,
exposed_secrets,
whois: params.whois_data,
partial_failures: partial_failure_records,
favicon: params.favicon,
cname_records: params.additional_dns.cname_chain,
aaaa_records: params.additional_dns.aaaa_records,
caa_records: params.additional_dns.caa_records,
csp_domains,
cookies,
resource_hints: std::mem::take(&mut params.html_data.resource_hints),
script_hosts,
security_txt: params.well_known.security_txt,
robots_txt: params.well_known.robots_txt,
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::error_handling::ErrorType;
use crate::fetch::dns::{AdditionalDnsData, TlsDnsData};
use crate::fetch::response::{HtmlData, ResponseData};
use crate::geoip::GeoIpResult;
use crate::parse::StructuredData;
use crate::whois::WhoisResult;
use chrono::NaiveDateTime;
use proptest::prelude::*;
use reqwest::header::HeaderMap;
use std::collections::{HashMap, HashSet};
fn create_test_response_data() -> ResponseData {
let mut headers = HeaderMap::new();
headers.insert(
reqwest::header::CONTENT_SECURITY_POLICY,
"default-src 'self'".parse().unwrap(),
);
ResponseData {
initial_url: "https://example.com".to_string(),
final_url: "https://example.com".to_string(),
initial_domain: "example.com".to_string(),
final_domain: "example.com".to_string(),
host: "example.com".to_string(),
status: 200,
status_desc: "OK".to_string(),
headers,
security_headers: HashMap::new(),
http_headers: HashMap::new(),
body: std::sync::Arc::<str>::from("<html><head><title>Test</title></head></html>"),
body_sha256: None,
body_truncated: false,
content_length: None,
http_version: Some("HTTP/2".to_string()),
content_type: Some("text/html".to_string()),
}
}
fn create_test_html_data() -> HtmlData {
HtmlData {
title: "Test Page".to_string(),
description: Some("Test description".to_string()),
structured_data: StructuredData::default(),
social_media_links: vec![],
contact_links: vec![],
exposed_secrets: vec![],
analytics_ids: vec![],
meta_tags: HashMap::new(),
script_sources: vec![],
script_tag_ids: HashSet::new(),
inline_script_text: String::new(),
external_scripts_eligible: 0,
external_scripts_scanned: 0,
favicon_url: None,
canonical_url: None,
meta_refresh_url: None,
meta_robots: None,
resource_hints: Vec::new(),
}
}
fn create_test_tls_dns_data() -> TlsDnsData {
TlsDnsData {
tls_version: Some(crate::models::TlsVersion::Tls13),
subject: Some("CN=example.com".to_string()),
issuer: Some("CN=Let's Encrypt".to_string()),
valid_from: Some(
NaiveDateTime::parse_from_str("2024-01-01 00:00:00", "%Y-%m-%d %H:%M:%S").unwrap(),
),
valid_to: Some(
NaiveDateTime::parse_from_str("2024-12-31 23:59:59", "%Y-%m-%d %H:%M:%S").unwrap(),
),
oids: Some(HashSet::new()),
cipher_suite: Some("TLS_AES_256_GCM_SHA384".to_string()),
key_algorithm: Some(crate::models::KeyAlgorithm::RSA),
subject_alternative_names: Some(vec!["example.com".to_string()]),
ip_address: Some("192.0.2.1".to_string()),
reverse_dns_name: Some("example.com".to_string()),
cert_fingerprint_sha256: None,
cert_serial_number: None,
cert_is_self_signed: None,
cert_is_wildcard: None,
}
}
fn create_test_additional_dns_data() -> AdditionalDnsData {
AdditionalDnsData {
nameservers: Some("ns1.example.com,ns2.example.com".to_string()),
txt_records: Some("v=spf1 include:_spf.google.com ~all".to_string()),
mx_records: Some("10 mail.example.com".to_string()),
spf_record: Some("v=spf1 include:_spf.google.com ~all".to_string()),
dmarc_record: Some("v=DMARC1; p=none".to_string()),
mta_sts_record: None,
tls_rpt_record: None,
bimi_record: None,
cname_chain: None,
aaaa_records: None,
caa_records: None,
}
}
#[test]
fn test_build_url_record_basic() {
let resp_data = create_test_response_data();
let html_data = create_test_html_data();
let tls_dns_data = create_test_tls_dns_data();
let additional_dns = create_test_additional_dns_data();
let run_id = Some("test-run-123".to_string());
let record = build_url_record(
&resp_data,
&html_data,
&tls_dns_data,
&additional_dns,
1.5,
1234567890,
run_id.as_ref(),
);
assert_eq!(record.initial_domain, "example.com");
assert_eq!(record.final_domain, "example.com");
assert_eq!(record.ip_address, "192.0.2.1");
assert_eq!(record.status, 200);
assert_eq!(record.title, "Test Page");
assert_eq!(record.description, Some("Test description".to_string()));
assert_eq!(record.tls_version, Some(crate::models::TlsVersion::Tls13));
assert_eq!(record.run_id, run_id);
}
#[test]
fn test_build_url_record_empty_strings_normalized() {
let resp_data = create_test_response_data();
let mut html_data = create_test_html_data();
html_data.description = Some("".to_string());
let tls_dns_data = create_test_tls_dns_data();
let additional_dns = create_test_additional_dns_data();
let record = build_url_record(
&resp_data,
&html_data,
&tls_dns_data,
&additional_dns,
1.5,
1234567890,
None,
);
assert_eq!(record.description, None);
}
#[test]
fn test_build_url_record_none_fields() {
let resp_data = create_test_response_data();
let mut html_data = create_test_html_data();
html_data.description = None;
let tls_dns_data = create_test_tls_dns_data();
let additional_dns = create_test_additional_dns_data();
let record = build_url_record(
&resp_data,
&html_data,
&tls_dns_data,
&additional_dns,
1.5,
1234567890,
None,
);
assert_eq!(record.description, None);
}
#[test]
fn test_build_persisted_url_record_basic() {
let resp_data = create_test_response_data();
let html_data = create_test_html_data();
let tls_dns_data = create_test_tls_dns_data();
let additional_dns = create_test_additional_dns_data();
let run_id = Some("test-run-123".to_string());
let url_record = build_url_record(
&resp_data,
&html_data,
&tls_dns_data,
&additional_dns,
1.5,
1234567890,
run_id.as_ref(),
);
let technologies = vec![
crate::fingerprint::DetectedTechnology {
name: "WordPress".to_string(),
version: None,
category: None,
is_implied: false,
detection_source: None,
},
crate::fingerprint::DetectedTechnology {
name: "PHP".to_string(),
version: None,
category: None,
is_implied: false,
detection_source: None,
},
];
let redirect_chain = vec![("https://example.com".to_string(), 200)];
let partial_failures = vec![(ErrorType::DnsNsLookupError, "DNS lookup failed".to_string())];
let geoip_data = Some((
"192.0.2.1".to_string(),
GeoIpResult {
country_code: Some("US".to_string()),
country_name: Some("United States".to_string()),
..Default::default()
},
));
let whois_data = Some(WhoisResult {
registrar: Some("Example Registrar".to_string()),
..Default::default()
});
let persisted_record = build_persisted_url_record(PersistedUrlRecordParams {
record: url_record,
resp_data,
html_data,
tls_dns_data,
technologies_vec: technologies.clone(),
redirect_chain,
partial_failures,
geoip_data,
whois_data,
timestamp: 1234567890,
run_id,
favicon: None,
well_known: crate::fetch::well_known::WellKnownData::default(),
additional_dns: create_test_additional_dns_data(),
});
assert_eq!(persisted_record.url_record.final_domain, "example.com");
assert_eq!(persisted_record.technologies.len(), 2);
assert!(persisted_record
.technologies
.iter()
.any(|t| t.name == "PHP"));
assert_eq!(persisted_record.redirect_chain.len(), 1);
assert_eq!(persisted_record.partial_failures.len(), 1);
assert!(persisted_record.geoip.is_some());
assert!(persisted_record.whois.is_some());
}
#[test]
fn test_build_persisted_url_record_empty_oids() {
let resp_data = create_test_response_data();
let html_data = create_test_html_data();
let mut tls_dns_data = create_test_tls_dns_data();
tls_dns_data.oids = None; let additional_dns = create_test_additional_dns_data();
let url_record = build_url_record(
&resp_data,
&html_data,
&tls_dns_data,
&additional_dns,
1.5,
1234567890,
None,
);
let persisted_record = build_persisted_url_record(PersistedUrlRecordParams {
record: url_record,
resp_data,
html_data,
tls_dns_data,
technologies_vec: vec![],
redirect_chain: vec![],
partial_failures: vec![],
geoip_data: None,
whois_data: None,
timestamp: 1234567890,
run_id: None,
favicon: None,
well_known: crate::fetch::well_known::WellKnownData::default(),
additional_dns: create_test_additional_dns_data(),
});
assert!(persisted_record.oids.is_empty());
}
#[test]
fn test_build_persisted_url_record_empty_sans() {
let resp_data = create_test_response_data();
let html_data = create_test_html_data();
let mut tls_dns_data = create_test_tls_dns_data();
tls_dns_data.subject_alternative_names = None; let additional_dns = create_test_additional_dns_data();
let url_record = build_url_record(
&resp_data,
&html_data,
&tls_dns_data,
&additional_dns,
1.5,
1234567890,
None,
);
let persisted_record = build_persisted_url_record(PersistedUrlRecordParams {
record: url_record,
resp_data,
html_data,
tls_dns_data,
technologies_vec: vec![],
redirect_chain: vec![],
partial_failures: vec![],
geoip_data: None,
whois_data: None,
timestamp: 1234567890,
run_id: None,
favicon: None,
well_known: crate::fetch::well_known::WellKnownData::default(),
additional_dns: create_test_additional_dns_data(),
});
assert!(persisted_record.subject_alternative_names.is_empty());
}
#[test]
fn test_build_persisted_url_record_partial_failures() {
let resp_data = create_test_response_data();
let html_data = create_test_html_data();
let tls_dns_data = create_test_tls_dns_data();
let additional_dns = create_test_additional_dns_data();
let url_record = build_url_record(
&resp_data,
&html_data,
&tls_dns_data,
&additional_dns,
1.5,
1234567890,
None,
);
let partial_failures = vec![
(ErrorType::DnsNsLookupError, "NS lookup failed".to_string()),
(
ErrorType::DnsTxtLookupError,
"TXT lookup failed".to_string(),
),
];
let persisted_record = build_persisted_url_record(PersistedUrlRecordParams {
record: url_record,
resp_data,
html_data,
tls_dns_data,
technologies_vec: vec![],
redirect_chain: vec![],
partial_failures,
geoip_data: None,
whois_data: None,
timestamp: 1234567890,
run_id: None,
favicon: None,
well_known: crate::fetch::well_known::WellKnownData::default(),
additional_dns: create_test_additional_dns_data(),
});
assert_eq!(persisted_record.partial_failures.len(), 2);
assert_eq!(
persisted_record.partial_failures[0].error_type,
ErrorType::DnsNsLookupError
);
assert_eq!(
persisted_record.partial_failures[1].error_type,
ErrorType::DnsTxtLookupError
);
}
#[test]
fn test_build_url_record_whitespace_only_strings() {
let resp_data = create_test_response_data();
let mut html_data = create_test_html_data();
html_data.description = Some("\t\n".to_string()); let tls_dns_data = create_test_tls_dns_data();
let additional_dns = create_test_additional_dns_data();
let record = build_url_record(
&resp_data,
&html_data,
&tls_dns_data,
&additional_dns,
1.5,
1234567890,
None,
);
assert_eq!(record.description, Some("\t\n".to_string()));
}
#[test]
fn test_build_persisted_url_record_duplicate_oids() {
let resp_data = create_test_response_data();
let html_data = create_test_html_data();
let mut tls_dns_data = create_test_tls_dns_data();
let mut oids = HashSet::new();
oids.insert("1.2.3.4".to_string());
oids.insert("1.2.3.4".to_string()); oids.insert("5.6.7.8".to_string());
tls_dns_data.oids = Some(oids);
let additional_dns = create_test_additional_dns_data();
let url_record = build_url_record(
&resp_data,
&html_data,
&tls_dns_data,
&additional_dns,
1.5,
1234567890,
None,
);
let persisted_record = build_persisted_url_record(PersistedUrlRecordParams {
record: url_record,
resp_data,
html_data,
tls_dns_data,
technologies_vec: vec![],
redirect_chain: vec![],
partial_failures: vec![],
geoip_data: None,
whois_data: None,
timestamp: 1234567890,
run_id: None,
favicon: None,
well_known: crate::fetch::well_known::WellKnownData::default(),
additional_dns: create_test_additional_dns_data(),
});
assert_eq!(persisted_record.oids.len(), 2); assert!(persisted_record.oids.contains("1.2.3.4"));
assert!(persisted_record.oids.contains("5.6.7.8"));
}
#[test]
fn test_build_persisted_url_record_duplicate_sans() {
let resp_data = create_test_response_data();
let html_data = create_test_html_data();
let mut tls_dns_data = create_test_tls_dns_data();
tls_dns_data.subject_alternative_names = Some(vec![
"example.com".to_string(),
"www.example.com".to_string(),
"example.com".to_string(), ]);
let additional_dns = create_test_additional_dns_data();
let url_record = build_url_record(
&resp_data,
&html_data,
&tls_dns_data,
&additional_dns,
1.5,
1234567890,
None,
);
let persisted_record = build_persisted_url_record(PersistedUrlRecordParams {
record: url_record,
resp_data,
html_data,
tls_dns_data,
technologies_vec: vec![],
redirect_chain: vec![],
partial_failures: vec![],
geoip_data: None,
whois_data: None,
timestamp: 1234567890,
run_id: None,
favicon: None,
well_known: crate::fetch::well_known::WellKnownData::default(),
additional_dns: create_test_additional_dns_data(),
});
assert_eq!(persisted_record.subject_alternative_names.len(), 3);
assert_eq!(persisted_record.subject_alternative_names[0], "example.com");
assert_eq!(persisted_record.subject_alternative_names[2], "example.com");
}
#[test]
fn test_build_persisted_url_record_large_redirect_chain() {
let resp_data = create_test_response_data();
let html_data = create_test_html_data();
let tls_dns_data = create_test_tls_dns_data();
let additional_dns = create_test_additional_dns_data();
let url_record = build_url_record(
&resp_data,
&html_data,
&tls_dns_data,
&additional_dns,
1.5,
1234567890,
None,
);
let redirect_chain: Vec<(String, u16)> = (0..100)
.map(|i| (format!("https://example.com/redirect{}", i), 302))
.collect();
let persisted_record = build_persisted_url_record(PersistedUrlRecordParams {
record: url_record,
resp_data,
html_data,
tls_dns_data,
technologies_vec: vec![],
redirect_chain,
partial_failures: vec![],
geoip_data: None,
whois_data: None,
timestamp: 1234567890,
run_id: None,
favicon: None,
well_known: crate::fetch::well_known::WellKnownData::default(),
additional_dns: create_test_additional_dns_data(),
});
assert_eq!(persisted_record.redirect_chain.len(), 100);
assert_eq!(
persisted_record.redirect_chain[0].0,
"https://example.com/redirect0"
);
assert_eq!(
persisted_record.redirect_chain[99].0,
"https://example.com/redirect99"
);
}
#[test]
fn test_build_persisted_url_record_run_id_propagation() {
let resp_data = create_test_response_data();
let html_data = create_test_html_data();
let tls_dns_data = create_test_tls_dns_data();
let additional_dns = create_test_additional_dns_data();
let url_record = build_url_record(
&resp_data,
&html_data,
&tls_dns_data,
&additional_dns,
1.5,
1234567890,
None,
);
let run_id = Some("test-run-456".to_string());
let partial_failures = vec![(ErrorType::DnsNsLookupError, "Test error".to_string())];
let persisted_record = build_persisted_url_record(PersistedUrlRecordParams {
record: url_record,
resp_data,
html_data,
tls_dns_data,
technologies_vec: vec![],
redirect_chain: vec![],
partial_failures,
geoip_data: None,
whois_data: None,
timestamp: 1234567890,
run_id: run_id.clone(),
favicon: None,
well_known: crate::fetch::well_known::WellKnownData::default(),
additional_dns: create_test_additional_dns_data(),
});
assert_eq!(persisted_record.partial_failures.len(), 1);
assert_eq!(persisted_record.partial_failures[0].run_id, run_id);
}
#[rstest::rstest]
#[case::happy(
"default-src https://cdn.example.com; script-src 'self' https://*.example.net none 'unsafe-inline'",
&[
("default-src", "cdn.example.com", Some("example.com")),
("script-src", "example.net", Some("example.net")),
]
)]
#[case::uppercased_host(
"default-src HTTPS://CDN.EXAMPLE.COM",
&[("default-src", "cdn.example.com", Some("example.com"))]
)]
fn extract_csp_domains_happy_path(
#[case] csp: &str,
#[case] expected: &[(&str, &str, Option<&str>)],
) {
let got = extract_csp_domains(csp);
let got: Vec<(&str, &str, Option<&str>)> = got
.iter()
.map(|(d, f, r)| (d.as_str(), f.as_str(), r.as_deref()))
.collect();
assert_eq!(got, expected);
}
#[rstest::rstest]
#[case::empty("")]
#[case::semicolons(";;;")]
#[case::no_space("default-srchttps://cdn.example.com")]
#[case::javascript("javascript:")]
#[case::data("data:")]
fn extract_csp_domains_hostile_omits_hosts(#[case] csp: &str) {
assert!(extract_csp_domains(csp).is_empty());
}
#[test]
fn extract_csp_domains_documentation_ip() {
let got = extract_csp_domains("default-src http://192.0.2.1");
assert_eq!(
got,
vec![(
"default-src".to_string(),
"192.0.2.1".to_string(),
Some("2.1".to_string()),
)]
);
}
proptest! {
#[test]
fn extract_csp_domains_random_ascii_never_panics(
s in prop::collection::vec(0u8..=127, 0..128)
.prop_map(|b| String::from_utf8(b).expect("ASCII"))
) {
let rows = extract_csp_domains(&s);
for (_, fqdn, _) in &rows {
prop_assert_eq!(fqdn, &fqdn.to_lowercase());
}
}
}
}