use std::time::Duration;
use crate::parse::ExposedSecret;
use crate::security::validate_url_safe;
pub const MAX_SCRIPT_FETCH_PER_PAGE: usize = 10;
pub const MAX_SCRIPT_BODY_BYTES: usize = 1024 * 1024;
pub const SCRIPT_FETCH_TIMEOUT_SECS: u64 = 5;
const KNOWN_THIRD_PARTY_SCRIPT_HOST_SUFFIXES: &[&str] = &[
"js.stripe.com",
"consent.cookiebot.com",
"assets.squarespace.com",
"www.google-analytics.com",
"www.googletagmanager.com",
"connect.facebook.net",
"maps.googleapis.com",
"cdn.jsdelivr.net",
"cdnjs.cloudflare.com",
"unpkg.com",
"ajax.googleapis.com",
"fast.wistia.com",
"hcaptcha.com",
"www.google.com",
"imasdk.googleapis.com",
"static.cloudflareinsights.com",
"cdn.weglot.com",
];
#[derive(Debug, Default)]
pub struct ExternalScriptScanResult {
pub secrets: Vec<ExposedSecret>,
pub eligible: u32,
pub scanned: u32,
pub script_bodies_text: String,
}
pub async fn scan_external_scripts(
client: &reqwest::Client,
page_url: &str,
script_sources: &[String],
allow_localhost: bool,
) -> ExternalScriptScanResult {
if script_sources.is_empty() {
return ExternalScriptScanResult::default();
}
let eligible: Vec<String> = script_sources
.iter()
.filter_map(|src| resolve_script_url(page_url, src))
.filter(|abs| abs.starts_with("http://") || abs.starts_with("https://"))
.filter(|abs| allow_localhost || validate_url_safe(abs).is_ok())
.filter(|abs| is_first_party_script(page_url, abs))
.collect();
let eligible_count = u32::try_from(eligible.len()).unwrap_or(u32::MAX);
let resolved: Vec<String> = eligible
.into_iter()
.take(MAX_SCRIPT_FETCH_PER_PAGE)
.collect();
if resolved.is_empty() {
return ExternalScriptScanResult {
secrets: Vec::new(),
eligible: eligible_count,
scanned: 0,
script_bodies_text: String::new(),
};
}
log::debug!(
"scan_external_scripts: page={} eligible={} fetching={}",
page_url,
eligible_count,
resolved.len()
);
let fetches = resolved.iter().map(|url| async move {
let body_opt = fetch_script_body(client, url).await;
body_opt.map(|body| (url.clone(), body))
});
let results = futures::future::join_all(fetches).await;
let owned: Vec<(String, String)> = results.into_iter().flatten().collect();
let scanned = u32::try_from(owned.len()).unwrap_or(u32::MAX);
let (all_secrets, script_bodies_text) = tokio::task::spawn_blocking(move || {
let mut all_secrets: Vec<ExposedSecret> = Vec::new();
let mut body_parts: Vec<String> = Vec::new();
for (url, body) in owned {
if !body.is_empty() {
body_parts.push(body.to_lowercase());
}
let mut found = crate::parse::detect_exposed_secrets(&body);
if found.is_empty() {
continue;
}
let location: std::borrow::Cow<'static, str> =
std::borrow::Cow::Owned(format!("external_script:{url}"));
for secret in &mut found {
secret.location = location.clone();
}
all_secrets.extend(found);
}
(all_secrets, body_parts.join("\n"))
})
.await
.unwrap_or_else(|e| {
log::warn!("External-script secret scan join failed: {e}");
(Vec::new(), String::new())
});
ExternalScriptScanResult {
secrets: all_secrets,
eligible: eligible_count,
scanned,
script_bodies_text,
}
}
fn is_first_party_script(page_url: &str, script_url: &str) -> bool {
let Some(page_host) = url_host(page_url) else {
return false;
};
let Some(script_host) = url_host(script_url) else {
return false;
};
if is_known_third_party_script_host(&script_host) {
return false;
}
if page_host == script_host {
return true;
}
let Some(page_root) = crate::domain::root_domain(&page_host) else {
return false;
};
let Some(script_root) = crate::domain::root_domain(&script_host) else {
return false;
};
page_root.eq_ignore_ascii_case(&script_root)
}
fn url_host(url: &str) -> Option<String> {
url::Url::parse(url)
.ok()
.and_then(|u| u.host_str().map(str::to_lowercase))
}
fn is_known_third_party_script_host(host: &str) -> bool {
let host = host.trim_end_matches('.').to_ascii_lowercase();
KNOWN_THIRD_PARTY_SCRIPT_HOST_SUFFIXES.iter().any(|suffix| {
host == *suffix
|| host
.strip_suffix(suffix)
.is_some_and(|prefix| prefix.ends_with('.'))
})
}
fn resolve_script_url(page_url: &str, src: &str) -> Option<String> {
let trimmed = src.trim();
if trimmed.is_empty() {
return None;
}
if let Ok(abs) = url::Url::parse(trimmed) {
return Some(abs.to_string());
}
let base = url::Url::parse(page_url).ok()?;
base.join(trimmed).ok().map(|u| u.to_string())
}
pub(crate) fn collect_script_hosts(
page_url: &str,
script_sources: &[String],
) -> Vec<crate::storage::ScriptHostInfo> {
let mut seen = std::collections::HashSet::new();
let mut out = Vec::new();
for src in script_sources {
let Some(abs) = resolve_script_url(page_url, src) else {
continue;
};
let Some(host) = url_host(&abs) else {
continue;
};
if !seen.insert(host.clone()) {
continue;
}
let registrable = crate::domain::root_domain(&host);
let is_first_party = is_first_party_script(page_url, &abs);
out.push(crate::storage::ScriptHostInfo {
host,
registrable_domain: registrable,
is_first_party,
});
}
out
}
async fn fetch_script_body(client: &reqwest::Client, url: &str) -> Option<String> {
let resp = match tokio::time::timeout(
Duration::from_secs(SCRIPT_FETCH_TIMEOUT_SECS),
client.get(url).send(),
)
.await
{
Ok(Ok(r)) => r,
Ok(Err(e)) => {
log::debug!("external_script: fetch error for {url}: {e}");
return None;
}
Err(_) => {
log::debug!("external_script: fetch timed out for {url}");
return None;
}
};
if !resp.status().is_success() {
log::debug!(
"external_script: non-success status {} for {url}",
resp.status()
);
return None;
}
let content_type = resp
.headers()
.get(reqwest::header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.map(std::string::ToString::to_string);
use crate::fetch::stream::{stream_bytes_with_limit, OnLimit};
let streamed =
match stream_bytes_with_limit(resp, MAX_SCRIPT_BODY_BYTES, OnLimit::Truncate, url).await {
Ok(s) => s,
Err(e) => {
log::debug!("external_script: chunk error for {url}: {e}");
return None;
}
};
if streamed.is_truncated() {
log::debug!("external_script: truncating {url} at size cap");
}
let accumulated = streamed.into_bytes();
Some(decode_script_body(&accumulated, content_type.as_deref()))
}
fn decode_script_body(bytes: &[u8], content_type: Option<&str>) -> String {
crate::fetch::charset::decode_body(bytes, content_type, false)
}
#[cfg(test)]
mod tests {
use super::*;
use httptest::{matchers::*, responders::*, Expectation, Server};
#[test]
fn test_resolve_script_url_absolute() {
let resolved =
resolve_script_url("https://example.com/page", "https://cdn.example.org/x.js");
assert_eq!(resolved.as_deref(), Some("https://cdn.example.org/x.js"));
}
#[test]
fn test_resolve_script_url_protocol_relative() {
let resolved = resolve_script_url("https://example.com/page", "//cdn.example.org/x.js");
assert_eq!(resolved.as_deref(), Some("https://cdn.example.org/x.js"));
}
#[test]
fn test_resolve_script_url_path_relative() {
let resolved = resolve_script_url("https://example.com/dir/page.html", "static/main.js");
assert_eq!(
resolved.as_deref(),
Some("https://example.com/dir/static/main.js")
);
}
#[test]
fn test_resolve_script_url_root_relative() {
let resolved = resolve_script_url("https://example.com/page", "/assets/main.js");
assert_eq!(
resolved.as_deref(),
Some("https://example.com/assets/main.js")
);
}
#[test]
fn test_resolve_script_url_empty_returns_none() {
assert!(resolve_script_url("https://example.com/", "").is_none());
assert!(resolve_script_url("https://example.com/", " ").is_none());
}
#[test]
fn test_decode_script_body_uses_charset() {
let bytes = b"\x93secret\x94";
let decoded =
decode_script_body(bytes, Some("application/javascript; charset=windows-1252"));
assert!(decoded.contains('\u{201C}'));
assert!(!decoded.contains('\u{FFFD}'));
}
#[test]
fn test_is_first_party_script_same_etld() {
assert!(is_first_party_script(
"https://www.example.com/page",
"https://static.example.com/app.js"
));
assert!(is_first_party_script(
"https://example.co.uk/page",
"https://cdn.example.co.uk/main.js"
));
}
#[test]
fn test_collect_script_hosts_first_and_third_party() {
let hosts = collect_script_hosts(
"https://www.example.com/page",
&[
"/app.js".to_string(),
"https://www.googletagmanager.com/gtm.js".to_string(),
"https://static.example.com/bundle.js".to_string(),
"/app.js".to_string(),
],
);
assert!(
hosts
.iter()
.any(|h| h.host == "www.example.com" && h.is_first_party),
"relative first-party host; got {hosts:?}"
);
assert!(
hosts
.iter()
.any(|h| h.host == "static.example.com" && h.is_first_party),
"same eTLD+1; got {hosts:?}"
);
assert!(
hosts
.iter()
.any(|h| h.host == "www.googletagmanager.com" && !h.is_first_party),
"GTM must be third-party; got {hosts:?}"
);
assert_eq!(
hosts.iter().filter(|h| h.host == "www.example.com").count(),
1,
"duplicate hosts collapsed"
);
}
#[test]
fn test_is_first_party_script_rejects_third_party() {
assert!(!is_first_party_script(
"https://www.example.com/page",
"https://js.stripe.com/v3/"
));
assert!(!is_first_party_script(
"https://www.example.com/page",
"https://consent.cookiebot.com/uc.js"
));
assert!(!is_first_party_script(
"https://www.example.com/page",
"https://cdn.other.com/lib.js"
));
}
#[test]
fn test_is_first_party_script_rejects_cdn_even_on_matching_suffix() {
assert!(!is_first_party_script(
"https://stripe.com/docs",
"https://js.stripe.com/v3/"
));
}
#[tokio::test]
async fn test_scan_external_scripts_finds_secret_and_tags_location() {
let server = Server::run();
let aws_key = format!("AKIA{}", "IOSFODNN7EXAMPL2");
let body_template = "var aws = '__AWS__';";
let bound_body = body_template.replace("__AWS__", &aws_key);
server.expect(
Expectation::matching(request::method_path("GET", "/main.js")).respond_with(
status_code(200)
.insert_header("Content-Type", "application/javascript; charset=utf-8")
.body(bound_body),
),
);
let client = reqwest::Client::new();
let server_url = server.url("/main.js").to_string();
let page_url = server.url("/page").to_string();
let result = scan_external_scripts(
&client,
&page_url,
std::slice::from_ref(&server_url),
true, )
.await;
let found = result
.secrets
.iter()
.find(|s| s.secret_type == "aws-access-token")
.unwrap_or_else(|| panic!("expected aws-access-token; got {:?}", result.secrets));
assert_eq!(found.matched_value, aws_key);
assert!(
found.location.starts_with("external_script:"),
"location should be tagged external_script:..., got {:?}",
found.location
);
assert!(found.location.contains(&server_url));
assert_eq!(result.eligible, 1);
assert_eq!(result.scanned, 1);
assert!(
result.script_bodies_text.contains("var aws"),
"fetched body text must be retained for tech fingerprints; got {:?}",
result.script_bodies_text
);
}
#[tokio::test]
async fn test_scan_external_scripts_retains_body_without_secrets() {
let server = Server::run();
let marker = "function webpackJsonpCallback(data) { return data; }";
server.expect(
Expectation::matching(request::method_path("GET", "/app.js")).respond_with(
status_code(200)
.insert_header("Content-Type", "application/javascript")
.body(marker),
),
);
let client = reqwest::Client::new();
let server_url = server.url("/app.js").to_string();
let page_url = server.url("/page").to_string();
let result =
scan_external_scripts(&client, &page_url, std::slice::from_ref(&server_url), true)
.await;
assert!(result.secrets.is_empty());
assert_eq!(result.scanned, 1);
assert!(
result
.script_bodies_text
.contains("function webpackjsonpcallback"),
"lowercase body haystack expected; got {:?}",
result.script_bodies_text
);
}
#[tokio::test]
async fn test_scan_external_scripts_skips_third_party_cdn() {
let client = reqwest::Client::new();
let result = scan_external_scripts(
&client,
"https://example.com/page",
&["https://js.stripe.com/v3/".to_string()],
false,
)
.await;
assert!(
result.secrets.is_empty(),
"third-party CDN scripts must not be fetched"
);
assert_eq!(result.eligible, 0);
assert_eq!(result.scanned, 0);
}
#[tokio::test]
async fn test_scan_external_scripts_skips_loopback_when_disallowed() {
let server = Server::run();
let client = reqwest::Client::new();
let server_url = server.url("/main.js").to_string();
let page_url = server.url("/page").to_string();
let result = scan_external_scripts(
&client,
&page_url,
&[server_url],
false, )
.await;
assert!(result.secrets.is_empty());
assert_eq!(result.eligible, 0);
}
#[tokio::test]
async fn test_scan_external_scripts_caps_at_max_per_page() {
let server = Server::run();
let page_url = server.url("/page").to_string();
let mut urls = Vec::new();
for i in 0..(MAX_SCRIPT_FETCH_PER_PAGE + 5) {
let path = format!("/s{i}.js");
urls.push(server.url(&path).to_string());
if i < MAX_SCRIPT_FETCH_PER_PAGE {
server.expect(
Expectation::matching(request::method_path("GET", path))
.times(1)
.respond_with(status_code(200).body("// no secrets")),
);
}
}
let client = reqwest::Client::new();
let result = scan_external_scripts(&client, &page_url, &urls, true).await;
assert_eq!(
result.eligible as usize,
MAX_SCRIPT_FETCH_PER_PAGE + 5,
"eligible counts all first-party candidates before the cap"
);
assert_eq!(
result.scanned as usize, MAX_SCRIPT_FETCH_PER_PAGE,
"scanned must equal per-page cap; got {}",
result.scanned
);
}
}