use super::patterns::{
AntiBotTech, NEAR_EMPTY_BODY_LEN, TIER1_REGEXES, TIER2_AUTOMATON, TIER2_BODY_SIZE_LIMIT,
TIER3_ANY_TAG, TIER3_MIN_SIGNALS, TIER3_NO_BODY, TIER3_SCRIPT_BLOCK, TIER3_SCRIPT_HEAVY_BYTES,
TIER3_SCRIPT_HEAVY_VISIBLE_MAX, TIER3_VISIBLE_TEXT_MIN,
};
use once_cell::sync::{Lazy, OnceCell};
use regex::Regex;
use reqwest::header::HeaderMap;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Detection {
pub tech: AntiBotTech,
pub reason: String,
pub needs_browser: bool,
pub retry_after_secs: Option<u64>,
}
impl Detection {
fn new(tech: AntiBotTech, reason: impl Into<String>, needs_browser: bool) -> Self {
Self {
tech,
reason: reason.into(),
needs_browser,
retry_after_secs: None,
}
}
}
static JSON_PREFIX: Lazy<Regex> =
Lazy::new(|| Regex::new(r"^\s*[\[\{]").expect("antibot json-prefix regex"));
static XML_PREFIX: Lazy<Regex> =
Lazy::new(|| Regex::new(r"^\s*<\?xml\b").expect("antibot xml-prefix regex"));
fn content_type_is_data(ct_lower: &str) -> bool {
ct_lower.contains("application/json")
|| ct_lower.contains("text/xml")
|| ct_lower.contains("application/xml")
|| ct_lower.contains("application/atom+xml")
|| ct_lower.contains("application/rss+xml")
}
fn looks_like_data(body: &str, headers: &HeaderMap) -> bool {
if let Some(ct) = headers.get(reqwest::header::CONTENT_TYPE) {
if let Ok(ct_str) = ct.to_str() {
if content_type_is_data(&ct_str.to_ascii_lowercase()) {
return true;
}
}
}
JSON_PREFIX.is_match(body) || XML_PREFIX.is_match(body)
}
fn parse_retry_after(headers: &HeaderMap) -> Option<u64> {
crate::utils::retry_after::parse_retry_after_value(
headers.get(reqwest::header::RETRY_AFTER)?.to_str().ok()?,
)
}
static STYLE_BLOCK: Lazy<Regex> =
Lazy::new(|| Regex::new(r"(?is)<style\b[^>]*>.*?</style>").expect("antibot style-block regex"));
fn strip_scripts_and_styles(body: &str) -> String {
let s = TIER3_SCRIPT_BLOCK.replace_all(body, "");
STYLE_BLOCK.replace_all(&s, "").to_string()
}
fn tier1_match(body: &str, stripped_cache: &OnceCell<String>) -> Option<(AntiBotTech, String)> {
for (re, tech) in TIER1_REGEXES.iter() {
if re.is_match(body) {
return Some((*tech, tier1_reason(*tech)));
}
}
if body.len() > TIER2_BODY_SIZE_LIMIT {
let stripped = stripped_cache.get_or_init(|| strip_scripts_and_styles(body));
for (re, tech) in TIER1_REGEXES.iter() {
if re.is_match(stripped) {
return Some((*tech, tier1_reason(*tech)));
}
}
}
None
}
fn tier1_reason(tech: AntiBotTech) -> String {
match tech {
AntiBotTech::Cloudflare => "cloudflare challenge structure matched".to_string(),
AntiBotTech::Akamai => "akamai reference / bmbl signature matched".to_string(),
AntiBotTech::PerimeterX => "perimeterx captcha / cookie matched".to_string(),
AntiBotTech::DataDome => "datadome cookie / host matched".to_string(),
AntiBotTech::Imperva => "incapsula cookie matched".to_string(),
AntiBotTech::Sucuri => "sucuri firewall signature matched".to_string(),
AntiBotTech::Kasada => "kasada pointdefense signature matched".to_string(),
AntiBotTech::AwsWaf => "aws waf captcha / cookie matched".to_string(),
AntiBotTech::RateLimited => "http 429 rate limit".to_string(),
AntiBotTech::StructuralBlock => "structural block / js shell detected".to_string(),
AntiBotTech::Unknown => "unknown antibot signature".to_string(),
}
}
fn tier3_signals(body: &str, stripped_cache: &OnceCell<String>) -> (usize, Vec<&'static str>) {
let mut signals: Vec<&'static str> = Vec::new();
let stripped_of_scripts = stripped_cache.get_or_init(|| strip_scripts_and_styles(body));
let stripped = TIER3_ANY_TAG.replace_all(stripped_of_scripts, "");
if !TIER3_NO_BODY.is_match(body) {
signals.push("missing <body> tag");
}
let visible_len = stripped.trim().len();
if visible_len < TIER3_VISIBLE_TEXT_MIN {
signals.push("visible text < 50 chars");
}
let script_bytes: usize = TIER3_SCRIPT_BLOCK
.find_iter(body)
.map(|m| m.as_str().len())
.sum();
if script_bytes > TIER3_SCRIPT_HEAVY_BYTES && visible_len < TIER3_SCRIPT_HEAVY_VISIBLE_MAX {
signals.push("script-heavy with minimal content");
}
(signals.len(), signals)
}
fn waf_needs_browser(tech: AntiBotTech) -> bool {
matches!(
tech,
AntiBotTech::Cloudflare
| AntiBotTech::Akamai
| AntiBotTech::PerimeterX
| AntiBotTech::DataDome
| AntiBotTech::Imperva
| AntiBotTech::Kasada
| AntiBotTech::AwsWaf
)
}
fn tier2_short_body_detection(status: u16, body: &str) -> Option<Detection> {
let is_error = (400..500).contains(&status) || (500..600).contains(&status);
if !(is_error && body.len() < TIER2_BODY_SIZE_LIMIT) {
return None;
}
if let Some(mat) = TIER2_AUTOMATON.find(body) {
let matched = &body[mat.start()..mat.end()];
return Some(Detection::new(
AntiBotTech::Unknown,
format!("tier2 generic keyword matched: {}", matched),
false,
));
}
Some(Detection::new(
AntiBotTech::Unknown,
format!("HTTP {} short body without data content", status),
false,
))
}
fn near_empty_js_shell(body: &str, stripped_cache: &OnceCell<String>) -> Option<Detection> {
if body.len() >= NEAR_EMPTY_BODY_LEN {
return None;
}
let stripped_of_scripts = stripped_cache.get_or_init(|| strip_scripts_and_styles(body));
let stripped = TIER3_ANY_TAG.replace_all(stripped_of_scripts, "");
if stripped.trim().len() < TIER3_VISIBLE_TEXT_MIN {
Some(Detection::new(
AntiBotTech::StructuralBlock,
"near-empty body likely JS shell",
true,
))
} else {
None
}
}
fn tier3_classify(body: &str, stripped_cache: &OnceCell<String>) -> Option<Detection> {
let (count, signals) = tier3_signals(body, stripped_cache);
if count >= TIER3_MIN_SIGNALS {
Some(Detection::new(
AntiBotTech::StructuralBlock,
format!("tier3 structural signals: {}", signals.join(", ")),
true,
))
} else {
None
}
}
fn blocked_status_detection(status: u16) -> Option<Detection> {
if status != 403 && status != 503 {
return None;
}
let needs_browser = true;
let reason = if status == 403 {
"HTTP 403 blocked without data-HTML body"
} else {
"HTTP 503 service unavailable block"
};
Some(Detection::new(
AntiBotTech::Unknown,
reason.to_string(),
needs_browser,
))
}
fn structural_detection(
status: u16,
body: &str,
stripped_cache: &OnceCell<String>,
) -> Option<Detection> {
if !(200..300).contains(&status) {
return None;
}
if let Some(detection) = near_empty_js_shell(body, stripped_cache) {
return Some(detection);
}
tier3_classify(body, stripped_cache)
}
pub fn classify(status: u16, body: &str, headers: &HeaderMap, _url: &str) -> Option<Detection> {
let stripped_cache: OnceCell<String> = OnceCell::new();
if status == 429 {
let mut detection = Detection::new(AntiBotTech::RateLimited, "HTTP 429 rate limited", true);
detection.retry_after_secs = parse_retry_after(headers);
return Some(detection);
}
if looks_like_data(body, headers) {
return None;
}
if let Some((tech, hint)) = tier1_match(body, &stripped_cache) {
let needs_browser = waf_needs_browser(tech);
return Some(Detection::new(tech, hint, needs_browser));
}
if let Some(detection) = blocked_status_detection(status) {
return Some(detection);
}
if let Some(detection) = tier2_short_body_detection(status, body) {
return Some(detection);
}
if let Some(detection) = structural_detection(status, body, &stripped_cache) {
return Some(detection);
}
None
}
#[cfg(test)]
mod tests {
use super::*;
use reqwest::header::{HeaderMap, HeaderValue};
fn empty_headers() -> HeaderMap {
HeaderMap::new()
}
fn html_headers() -> HeaderMap {
let mut h = HeaderMap::new();
h.insert(
reqwest::header::CONTENT_TYPE,
HeaderValue::from_static("text/html; charset=utf-8"),
);
h
}
fn json_headers() -> HeaderMap {
let mut h = HeaderMap::new();
h.insert(
reqwest::header::CONTENT_TYPE,
HeaderValue::from_static("application/json; charset=utf-8"),
);
h
}
#[test]
fn classify_429_returns_rate_limited_needs_browser() {
let det = classify(429, "rate limited", &empty_headers(), "https://example.com")
.expect("429 should detect");
assert_eq!(det.tech, AntiBotTech::RateLimited);
assert!(det.needs_browser, "rate limited should need browser");
}
#[test]
fn classify_429_parses_numeric_retry_after() {
let mut h = HeaderMap::new();
h.insert(reqwest::header::RETRY_AFTER, HeaderValue::from_static("30"));
let det =
classify(429, "rate limited", &h, "https://example.com").expect("429 should detect");
assert_eq!(det.retry_after_secs, Some(30));
}
#[test]
fn classify_429_without_retry_after_header_yields_none() {
let det = classify(429, "rate limited", &empty_headers(), "https://example.com")
.expect("429 should detect");
assert_eq!(det.retry_after_secs, None);
}
#[test]
fn classify_429_retry_after_clamped_to_600() {
let mut h = HeaderMap::new();
h.insert(
reqwest::header::RETRY_AFTER,
HeaderValue::from_static("99999"),
);
let det =
classify(429, "rate limited", &h, "https://example.com").expect("429 should detect");
assert_eq!(det.retry_after_secs, Some(600));
}
#[test]
fn classify_429_parses_http_date_retry_after() {
let future = chrono::Utc::now() + chrono::Duration::seconds(60);
let value = future.format("%a, %d %b %Y %H:%M:%S GMT").to_string();
let mut h = HeaderMap::new();
h.insert(
reqwest::header::RETRY_AFTER,
HeaderValue::from_str(&value).expect("http-date is a valid header value"),
);
let det =
classify(429, "rate limited", &h, "https://example.com").expect("429 should detect");
let secs = det
.retry_after_secs
.expect("future http-date should parse to a wait");
assert!(secs <= 60, "http-date delta should be ~60s, got {secs}");
}
#[test]
fn classify_429_past_http_date_retry_after_is_zero() {
let past = chrono::Utc::now() - chrono::Duration::seconds(120);
let value = past.format("%a, %d %b %Y %H:%M:%S GMT").to_string();
let mut h = HeaderMap::new();
h.insert(
reqwest::header::RETRY_AFTER,
HeaderValue::from_str(&value).expect("http-date is a valid header value"),
);
let det =
classify(429, "rate limited", &h, "https://example.com").expect("429 should detect");
assert_eq!(
det.retry_after_secs,
Some(0),
"past http-date means retryable now"
);
}
#[test]
fn non_429_detection_has_no_retry_after() {
let det =
classify(403, "", &html_headers(), "https://example.com").expect("403 should detect");
assert_eq!(det.retry_after_secs, None);
}
#[test]
fn classify_json_response_returns_none() {
let body = r#"{"error":"blocked","reason":"captcha"}"#;
let det = classify(200, body, &json_headers(), "https://api.example.com/v1");
assert!(det.is_none(), "JSON response should not be flagged");
}
#[test]
fn classify_429_json_body_returns_rate_limited() {
let body = r#"{"status":429}"#;
let det = classify(429, body, &json_headers(), "https://api.example.com/v1")
.expect("429 JSON body should still classify as rate limited");
assert_eq!(det.tech, AntiBotTech::RateLimited);
assert!(det.needs_browser, "rate limited should need browser");
}
#[test]
fn classify_xml_response_returns_none() {
let body = "<?xml version=\"1.0\"?><error>blocked</error>";
let det = classify(403, body, &empty_headers(), "https://example.com/feed");
assert!(det.is_none(), "XML response should not be flagged");
}
#[test]
fn classify_cloudflare_tier1_match() {
let body = concat!(
"<html><head><title>Just a moment...</title></head>",
"<body><script src=\"/cdn-cgi/challenge-platform/h/g/orchestrate/jsch/v1\"></script></body></html>"
);
let det = classify(403, body, &html_headers(), "https://example.com")
.expect("cloudflare body should detect");
assert_eq!(det.tech, AntiBotTech::Cloudflare);
assert!(det.needs_browser);
}
#[test]
fn classify_akamai_tier1_match() {
let body = "<html><body>Reference #18.abc.456</body></html>";
let det = classify(403, body, &html_headers(), "https://example.com")
.expect("akamai body should detect");
assert_eq!(det.tech, AntiBotTech::Akamai);
assert!(det.needs_browser);
}
#[test]
fn classify_normal_back_wording_not_akamai() {
let body = "<html><body><button onclick=\"goBack()\" class=\"btn_back\">返回</button>\
<p>This is a normal product page with plenty of visible text so the \
near-empty structural detector will never fire on it.</p></body></html>";
assert!(
classify(200, body, &html_headers(), "https://example.com").is_none(),
"normal page containing go_back/btn_back must not be flagged as anti-bot"
);
}
#[test]
fn classify_akamai_bot_manager_cookie_markers() {
let body = "<html><body><script>document.cookie=\"_abck=8AC40FDA...~-1~\";</script>\
<p>content filler for length</p></body></html>";
let det = classify(200, body, &html_headers(), "https://example.com")
.expect("_abck marker should detect on 200");
assert_eq!(det.tech, AntiBotTech::Akamai);
assert!(det.needs_browser);
}
#[test]
fn classify_datadome_tier1_match_overrides_status() {
let body = "<html><body><script src=\"https://ct.captcha-delivery.com/c.js\"></script></body></html>";
let det = classify(200, body, &html_headers(), "https://example.com")
.expect("datadome tier1 should detect regardless of status");
assert_eq!(det.tech, AntiBotTech::DataDome);
}
#[test]
fn classify_tier1_strips_large_page_for_deep_scan() {
let noise = "<script>".to_string() + &"x".repeat(6 * 1024) + "</script>";
let body = format!(
"<html><head>{}</head><body><script src=\"/cdn-cgi/challenge-platform/x\"></script></body></html>",
noise
);
let det = classify(200, &body, &html_headers(), "https://example.com")
.expect("tier1 deep scan on large page should detect");
assert_eq!(det.tech, AntiBotTech::Cloudflare);
}
#[test]
fn classify_403_empty_body_returns_block() {
let det = classify(403, "", &html_headers(), "https://example.com")
.expect("403 empty body should detect");
assert_eq!(det.tech, AntiBotTech::Unknown);
assert!(det.reason.contains("403"));
}
#[test]
fn classify_503_short_body_returns_block() {
let det = classify(
503,
"Service Unavailable",
&html_headers(),
"https://example.com",
)
.expect("503 short body should detect");
assert_eq!(det.tech, AntiBotTech::Unknown);
}
#[test]
fn classify_4xx_tier2_keyword_match() {
let det = classify(
400,
"You have been blocked by our system",
&html_headers(),
"https://example.com",
)
.expect("4xx tier2 keyword should detect");
assert_eq!(det.tech, AntiBotTech::Unknown);
assert!(det.reason.contains("tier2"));
}
#[test]
fn classify_4xx_short_no_keyword_still_blocks() {
let det = classify(400, "bad request", &html_headers(), "https://example.com")
.expect("4xx short body should still block");
assert_eq!(det.tech, AntiBotTech::Unknown);
}
#[test]
fn classify_200_near_empty_returns_structural_block() {
let body = "<html><body><script>boot()</script></body></html>";
let det = classify(200, body, &html_headers(), "https://example.com")
.expect("200 near-empty body should detect");
assert_eq!(det.tech, AntiBotTech::StructuralBlock);
assert!(det.needs_browser);
}
#[test]
fn classify_200_script_heavy_no_content_returns_structural_block() {
let big_script = format!("<script>{}</script>", "var x=1;".repeat(1500)); let body = format!("<html><body>{}</body></html>", big_script);
let det = classify(200, &body, &html_headers(), "https://example.com")
.expect("script-heavy 200 should detect");
assert_eq!(det.tech, AntiBotTech::StructuralBlock);
assert!(det.reason.contains("tier3") || det.reason.contains("near-empty"));
}
#[test]
fn classify_200_missing_body_and_low_text_returns_structural_block() {
let body = "<html><head><title>Checking your browser</title></head></html>";
let det = classify(200, body, &html_headers(), "https://example.com")
.expect("missing body + low text should detect");
assert_eq!(det.tech, AntiBotTech::StructuralBlock);
}
#[test]
fn classify_normal_large_html_returns_none() {
let body = format!(
"<html><head><title>Example Domain</title></head>\
<body><h1>Example Domain</h1>\
<p>{}</p>\
<a href=\"https://www.iana.org/domains/example\">More information...</a>\
</body></html>",
"This domain is for use in illustrative examples in documents. ".repeat(20)
);
let det = classify(200, &body, &html_headers(), "https://example.com");
assert!(det.is_none(), "normal large page should not detect");
}
#[test]
fn classify_normal_200_returns_none() {
let body = "<html><body><h1>Welcome to example.com</h1>\
<p>This is a legitimate page with enough visible text content to pass Tier3.</p>\
</body></html>";
let det = classify(200, body, &html_headers(), "https://example.com");
assert!(det.is_none());
}
#[test]
fn looks_like_data_json_content_type() {
assert!(looks_like_data("not json", &json_headers()));
}
#[test]
fn looks_like_data_xml_body_prefix() {
let mut h = HeaderMap::new();
h.insert(
reqwest::header::CONTENT_TYPE,
HeaderValue::from_static("text/html"),
);
assert!(looks_like_data("<?xml version=\"1.0\"?><rss></rss>", &h));
}
#[test]
fn looks_like_data_html_returns_false() {
assert!(!looks_like_data(
"<html><body>hi</body></html>",
&html_headers()
));
}
#[test]
fn detection_new_basic() {
let d = Detection::new(AntiBotTech::Cloudflare, "test", true);
assert_eq!(d.tech, AntiBotTech::Cloudflare);
assert_eq!(d.reason, "test");
assert!(d.needs_browser);
}
}