use crate::patterns::binary;
use crate::patterns::{self, RegexThreat, ViewFilter};
use crate::preprocessor;
use crate::scan_budget::{
DEFAULT_MAX_SCAN_SECONDS, SCAN_TIME_BUDGET_EXHAUSTED_PATTERN, ScanTimeBudget,
};
use crate::semantic::{self, AnalysisResult, AttackKeywords, AttackStructures};
#[derive(Debug, Clone, Copy)]
pub struct DetectConfig {
pub max_content_length: usize,
pub max_full_scan_bytes: usize,
pub preserve_attack_patterns: bool,
pub semantic_threshold: f64,
pub threat_score_threshold: f64,
pub binary_min_run_length: usize,
pub max_scan_values: usize,
pub max_scan_chars: usize,
pub max_json_depth: usize,
}
impl Default for DetectConfig {
fn default() -> Self {
Self {
max_content_length: 10_000,
max_full_scan_bytes: 262_144,
preserve_attack_patterns: true,
semantic_threshold: 0.7,
threat_score_threshold: 1.0,
binary_min_run_length: 16,
max_scan_values: 512,
max_scan_chars: 65_536,
max_json_depth: 32,
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub enum Threat {
Regex(RegexThreat),
Semantic(SemanticThreat),
PatternTimeout(RegexThreat),
}
#[derive(Debug, Clone, PartialEq)]
pub struct SemanticThreat {
pub attack_type: String,
pub score: f64,
pub fallback: bool,
pub analysis: AnalysisResult,
}
#[derive(Debug, Clone, PartialEq)]
pub struct DetectVerdict {
pub is_threat: bool,
pub threat_score: f64,
pub threats: Vec<Threat>,
pub original_length: usize,
pub processed_length: usize,
}
const KNOWN_CONTEXTS: &[&str] = &[
"query_param",
"header",
"url_path",
"request_body",
"unknown",
];
pub const EMBEDDED_JSON_LEAF_CONTEXT_SUFFIX: &str = ":embedded_json";
#[must_use]
pub fn normalize_context(context: &str) -> &str {
let head = context.split(':').next().unwrap_or("");
if KNOWN_CONTEXTS.contains(&head) {
head
} else {
"unknown"
}
}
fn regex_anomaly(regex_threats: &[RegexThreat]) -> f64 {
regex_threats.iter().map(|t| t.weight).sum()
}
#[must_use]
fn scan_pass(
view_content: &str,
filter: ViewFilter,
normalized_context: &str,
validator_context: &str,
budget: &mut ScanTimeBudget,
) -> Vec<RegexThreat> {
let skip_filter = matches!(normalized_context, "unknown" | "request_body");
patterns::scan_view_budgeted(
view_content,
filter,
normalized_context,
skip_filter,
validator_context,
budget,
)
}
#[must_use]
fn drop_view_duplicate_threats(
seen_threats: &[RegexThreat],
new_threats: Vec<RegexThreat>,
) -> Vec<RegexThreat> {
let mut seen: std::collections::HashSet<(String, String)> = seen_threats
.iter()
.map(|threat| (threat.pattern.clone(), threat.match_text.clone()))
.collect();
let mut kept = Vec::new();
for threat in new_threats {
if seen.insert((threat.pattern.clone(), threat.match_text.clone())) {
kept.push(threat);
}
}
kept
}
fn semantic_threats(
processed: &str,
raw_content: &str,
config: &DetectConfig,
) -> Vec<SemanticThreat> {
if binary::looks_like_binary_content(raw_content) {
return Vec::new();
}
let semantic_budget = config.max_content_length;
let input: String = processed.chars().take(semantic_budget).collect();
let analysis = semantic::analyze(
&input,
&AttackKeywords::default(),
&AttackStructures::default(),
);
let score = semantic::get_threat_score(&analysis);
let mut threats = Vec::new();
if score > config.semantic_threshold {
let mut probs: Vec<(&str, f64)> = analysis
.attack_probabilities
.iter()
.map(|(k, v)| (*k, *v))
.collect();
probs.sort_unstable_by(|a, b| a.0.cmp(b.0));
for (attack_type, probability) in probs {
if probability >= config.semantic_threshold {
threats.push(SemanticThreat {
attack_type: attack_type.to_owned(),
score: probability,
fallback: false,
analysis: analysis.clone(),
});
}
}
if threats.is_empty() && score >= config.semantic_threshold {
threats.push(SemanticThreat {
attack_type: "suspicious".to_owned(),
score,
fallback: true,
analysis,
});
}
}
threats
}
#[must_use]
pub fn detect(content: &str, request_context: &str, config: &DetectConfig) -> DetectVerdict {
detect_with_semantic_skip(content, request_context, config, false)
}
#[must_use]
pub fn detect_with_semantic_skip(
content: &str,
request_context: &str,
config: &DetectConfig,
skip_semantic: bool,
) -> DetectVerdict {
let mut budget = ScanTimeBudget::new(DEFAULT_MAX_SCAN_SECONDS);
detect_with_budget(content, request_context, config, skip_semantic, &mut budget)
}
#[must_use]
pub fn detect_with_budget(
content: &str,
request_context: &str,
config: &DetectConfig,
skip_semantic: bool,
budget: &mut ScanTimeBudget,
) -> DetectVerdict {
let original_length = content.chars().count();
let (processed, decoded, decode_budget_exhausted) = preprocessor::preprocess_with_decoded(
content,
config.max_full_scan_bytes,
config.preserve_attack_patterns,
config.max_content_length,
);
let processed_length = processed.chars().count();
let normalized = normalize_context(request_context);
let validator_context = if request_context.ends_with(EMBEDDED_JSON_LEAF_CONTEXT_SUFFIX) {
format!("{normalized}{EMBEDDED_JSON_LEAF_CONTEXT_SUFFIX}")
} else {
normalized.to_owned()
};
let mut regex_threats: Vec<RegexThreat> = Vec::new();
regex_threats.extend(scan_pass(
&processed,
ViewFilter::Processed,
normalized,
&validator_context,
budget,
));
let raw_view = preprocessor::preprocess_signal_preserving(
content,
config.max_full_scan_bytes,
config.preserve_attack_patterns,
config.max_content_length,
);
let raw_pass = scan_pass(
&raw_view,
ViewFilter::Raw,
normalized,
&validator_context,
budget,
);
regex_threats.extend(drop_view_duplicate_threats(®ex_threats, raw_pass));
if let Some(threat) = patterns::decoded_view_traversal_threat(&processed, &raw_view) {
regex_threats.push(threat);
}
let url_decoded_view = preprocessor::truncate_safely(
&decoded,
config.max_full_scan_bytes,
config.preserve_attack_patterns,
config.max_content_length,
);
regex_threats.extend(scan_pass(
&url_decoded_view,
ViewFilter::UrlDecoded,
normalized,
&validator_context,
budget,
));
if decode_budget_exhausted {
regex_threats.push(patterns::decode_budget_exhausted_threat());
}
let additive_view = preprocessor::short_base64_additive_view(
content,
config.max_full_scan_bytes,
config.preserve_attack_patterns,
config.max_content_length,
);
if !additive_view.is_empty() {
regex_threats.extend(scan_pass(
&additive_view,
ViewFilter::All,
normalized,
&validator_context,
budget,
));
}
let semantic = if skip_semantic {
Vec::new()
} else {
semantic_threats(&processed, content, config)
};
score_verdict(
regex_threats,
semantic,
config,
original_length,
processed_length,
)
}
fn score_verdict(
regex_threats: Vec<RegexThreat>,
semantic: Vec<SemanticThreat>,
config: &DetectConfig,
original_length: usize,
processed_length: usize,
) -> DetectVerdict {
let anomaly = regex_anomaly(®ex_threats);
let is_threat = anomaly >= config.threat_score_threshold || !semantic.is_empty();
let threat_score = if regex_threats.is_empty() && semantic.is_empty() {
0.0
} else {
let semantic_max = semantic.iter().map(|s| s.score).fold(0.0_f64, f64::max);
(anomaly.max(semantic_max)).min(1.0)
};
let mut threats: Vec<Threat> = regex_threats
.into_iter()
.map(|threat| {
if threat.pattern == SCAN_TIME_BUDGET_EXHAUSTED_PATTERN {
Threat::PatternTimeout(threat)
} else {
Threat::Regex(threat)
}
})
.collect();
threats.extend(semantic.into_iter().map(Threat::Semantic));
DetectVerdict {
is_threat,
threat_score,
threats,
original_length,
processed_length,
}
}
#[cfg(test)]
mod tests {
use super::*;
fn corpus_config() -> DetectConfig {
DetectConfig {
max_content_length: 10_000,
max_full_scan_bytes: 262_144,
preserve_attack_patterns: true,
semantic_threshold: 0.7,
threat_score_threshold: 1.0,
binary_min_run_length: 16,
max_scan_values: 512,
max_scan_chars: 65_536,
max_json_depth: 32,
}
}
#[test]
fn the_config_defaults_carry_the_reference_values() {
let config = DetectConfig::default();
assert_eq!(config.max_content_length, 10_000);
assert_eq!(config.max_full_scan_bytes, 262_144);
assert!(config.preserve_attack_patterns);
assert!((config.semantic_threshold - 0.7).abs() < f64::EPSILON);
assert!((config.threat_score_threshold - 1.0).abs() < f64::EPSILON);
assert_eq!(config.binary_min_run_length, 16);
assert_eq!(config.max_scan_values, 512);
assert_eq!(config.max_scan_chars, 65_536);
assert_eq!(config.max_json_depth, 32);
}
#[test]
fn traversal_in_url_path_context() {
let v = detect("/files?name=../../etc/passwd", "url_path", &corpus_config());
assert!(v.is_threat);
assert!((v.threat_score - 1.0).abs() < f64::EPSILON);
assert_eq!(v.threats.len(), 2);
assert_eq!(v.original_length, 28);
assert_eq!(v.processed_length, 28);
}
#[test]
fn benign_content_is_clean() {
let v = detect("hello world", "request_body", &corpus_config());
assert!(!v.is_threat);
assert!((v.threat_score - 0.0).abs() < f64::EPSILON);
assert!(v.threats.is_empty());
}
fn is_sqli_comment_threat(t: &Threat) -> bool {
matches!(t, Threat::Regex(r) if r.category == "sqli" && r.pattern.contains("/\\*"))
}
#[test]
fn context_filtering_gates_sqli_narrow() {
let hit = detect("a/**/b", "request_body", &corpus_config());
assert!(hit.threats.iter().any(is_sqli_comment_threat));
let miss = detect("/files?name=../../etc/passwd", "url_path", &corpus_config());
assert!(
miss.threats
.iter()
.any(|t| matches!(t, Threat::Regex(r) if r.category == "dir_traversal"))
);
assert!(!miss.threats.iter().any(is_sqli_comment_threat));
}
#[test]
fn unknown_context_normalizes() {
assert_eq!(normalize_context("weird:thing"), "unknown");
assert_eq!(
normalize_context("query_param:embedded_json"),
"query_param"
);
assert_eq!(normalize_context("header"), "header");
}
#[test]
fn raw_view_only_patterns_need_the_raw_view() {
let v = detect("/search?q=%2e%2e%2f", "url_path", &corpus_config());
assert!(
v.threats
.iter()
.any(|t| matches!(t, Threat::Regex(r) if r.category == "path_traversal"))
);
}
#[test]
fn regex_anomaly_sums_weights() {
let v = detect("SELECT * FROM users", "request_body", &corpus_config());
assert!(v.is_threat);
let semantic_side = detect(&mixed_content(), "request_body", &corpus_config());
let semantic_threat = semantic_side
.threats
.iter()
.find(|t| matches!(t, Threat::Semantic(_)))
.expect("the mixed payload carries a semantic threat");
let anomaly: f64 = v
.threats
.iter()
.chain(std::iter::once(semantic_threat))
.filter_map(|t| match t {
Threat::Regex(r) | Threat::PatternTimeout(r) => Some(r.weight),
Threat::Semantic(_) => None,
})
.sum();
assert!((anomaly - 1.0).abs() < 1e-9, "anomaly {anomaly}");
let mixed = detect(&mixed_content(), "request_body", &corpus_config());
assert!(
mixed
.threats
.iter()
.any(|t| matches!(t, Threat::Semantic(_)))
);
let mixed_anomaly: f64 = mixed
.threats
.iter()
.filter_map(|t| match t {
Threat::Regex(r) | Threat::PatternTimeout(r) => Some(r.weight),
Threat::Semantic(_) => None,
})
.sum();
assert!(mixed_anomaly > 0.0, "regex hits still carry weight");
assert!(mixed.is_threat);
}
fn mixed_content() -> String {
format!(
"select union insert update delete drop from where order group having concat \
substring database table column (1 OR 1=1) {}",
"A".repeat(120)
)
}
#[test]
fn an_embedded_json_leaf_context_keeps_its_suffix() {
let v = detect(
"SELECT * FROM users",
"request_body:embedded_json",
&corpus_config(),
);
assert!(v.is_threat);
}
#[test]
fn a_short_base64_view_scans_as_an_additive_pass() {
let v = detect("JHt9", "request_body", &corpus_config());
assert!(v.is_threat, "unexpected verdict: {v:?}");
assert!(
v.threats
.iter()
.any(|t| matches!(t, Threat::Regex(r) if r.category == "cmd_injection")),
"verdict: {v:?}"
);
}
#[test]
fn binary_raw_content_never_yields_semantic_threats() {
let processed = "\0".repeat(16);
assert!(
semantic_threats(&processed, &processed, &corpus_config()).is_empty(),
"binary content is exempt from the semantic pass"
);
}
#[test]
fn an_exhausted_decode_budget_appends_its_marker_threat() {
let content = format!("%25{}", "25".repeat(15));
let v = detect(&content, "url_path", &corpus_config());
assert!(
v.threats.iter().any(|t| matches!(
t,
Threat::Regex(r) if r.pattern == "decode_budget_exhausted"
)),
"verdict: {v:?}"
);
}
}
#[cfg(test)]
mod gap_tests {
use super::*;
fn corpus_config() -> DetectConfig {
DetectConfig {
max_content_length: 10_000,
max_full_scan_bytes: 262_144,
preserve_attack_patterns: true,
semantic_threshold: 0.7,
threat_score_threshold: 1.0,
binary_min_run_length: 16,
max_scan_values: 512,
max_scan_chars: 65_536,
max_json_depth: 32,
}
}
#[test]
fn a_semantic_fallback_threat_reports_suspicious() {
let mut content = String::from("from where drop select ");
content.push_str(&"A".repeat(120));
content.push_str(" ({}$x==) foo(");
let v = detect(&content, "request_body", &corpus_config());
assert!(v.is_threat, "verdict: {v:?}");
assert!(
v.threats.iter().any(
|t| matches!(t, Threat::Semantic(s) if s.attack_type == "suspicious" && s.fallback)
),
"verdict: {v:?}"
);
}
#[test]
fn a_mixed_verdict_scores_the_semantic_maximum_against_the_anomaly() {
let content = format!(
"select union insert update delete drop from where order group having concat \
substring database table column (1 OR 1=1) {}",
"A".repeat(120)
);
let v = detect(&content, "request_body", &corpus_config());
assert!(v.is_threat);
assert!(
v.threats
.iter()
.any(|t| matches!(t, Threat::Regex(r) if r.category == "sqli"))
);
assert!(v.threats.iter().any(|t| matches!(t, Threat::Semantic(_))));
let semantic_max = v
.threats
.iter()
.filter_map(|t| match t {
Threat::Semantic(s) => Some(s.score),
Threat::Regex(_) | Threat::PatternTimeout(_) => None,
})
.fold(0.0_f64, f64::max);
let anomaly = v
.threats
.iter()
.filter_map(|t| match t {
Threat::Regex(r) | Threat::PatternTimeout(r) => Some(r.weight),
Threat::Semantic(_) => None,
})
.sum::<f64>();
assert!((v.threat_score - anomaly.max(semantic_max).min(1.0)).abs() < 1e-9);
}
}
#[cfg(test)]
mod scan_budget_tests {
use super::*;
use crate::scan_budget::{SCAN_TIME_BUDGET_EXHAUSTED_PATTERN, ScanTimeBudget};
use std::cell::Cell;
thread_local! {
static NOW: Cell<f64> = const { Cell::new(0.0) };
}
fn fake_clock() -> f64 {
NOW.with(|now| {
now.set(now.get() + 1.0);
now.get()
})
}
fn budget_threats(verdict: &DetectVerdict) -> Vec<&RegexThreat> {
verdict
.threats
.iter()
.filter_map(|t| match t {
Threat::PatternTimeout(r) => Some(r),
_ => None,
})
.collect()
}
#[test]
fn an_exhausted_budget_records_the_reference_pattern_timeout_threat() {
let mut budget = ScanTimeBudget::with_clock(Some(0.1), fake_clock);
let verdict = detect_with_budget(
"hello world",
"query_param",
&DetectConfig::default(),
false,
&mut budget,
);
let threats = budget_threats(&verdict);
assert!(!threats.is_empty(), "verdict: {verdict:?}");
let threat = threats[0];
assert_eq!(threat.pattern, SCAN_TIME_BUDGET_EXHAUSTED_PATTERN);
assert_eq!(threat.match_text, "");
assert_eq!(threat.position, 0);
assert_eq!(threat.category, "custom");
assert!((threat.weight - 1.0).abs() < f64::EPSILON);
assert!(verdict.is_threat, "the weighted threat blocks");
assert!(
!verdict
.threats
.iter()
.any(|t| matches!(t, Threat::Regex(_))),
"the scan stopped after the first pattern"
);
}
#[test]
fn the_budget_is_shared_across_values() {
let mut budget = ScanTimeBudget::with_clock(Some(2.5), fake_clock);
let config = DetectConfig::default();
let first = detect_with_budget("hello", "query_param", &config, false, &mut budget);
let second = detect_with_budget("world", "query_param", &config, false, &mut budget);
let recorded = budget_threats(&first)
.into_iter()
.chain(budget_threats(&second))
.any(|t| t.pattern == SCAN_TIME_BUDGET_EXHAUSTED_PATTERN);
assert!(recorded);
assert!(second.is_threat);
}
#[test]
fn within_budget_nothing_changes() {
let config = DetectConfig::default();
let benign = detect("page=2&sort=asc", "query_param", &config);
assert!(!benign.is_threat);
assert!(budget_threats(&benign).is_empty());
let attack = detect("<script>alert(1)</script>", "query_param", &config);
assert!(attack.is_threat);
assert!(budget_threats(&attack).is_empty());
assert!(
attack
.threats
.iter()
.any(|t| matches!(t, Threat::Regex(r) if r.category == "xss"))
);
}
#[test]
fn a_zero_cap_exhausts_before_the_first_pattern() {
let verdict = detect_with_budget(
"hello world",
"query_param",
&DetectConfig::default(),
false,
&mut ScanTimeBudget::new(0.0),
);
assert_eq!(budget_threats(&verdict).len(), 2, "verdict: {verdict:?}");
assert!(
verdict
.threats
.iter()
.all(|t| matches!(t, Threat::PatternTimeout(_)))
);
}
}