threatflux_string_analysis/
analyzer.rs

1//! String analysis functionality.
2
3use crate::patterns::{MAX_PATTERNS, Pattern};
4use crate::types::{
5    AnalysisError, AnalysisResult, DEFAULT_MAX_INDICATORS_PER_STRING, DEFAULT_SUSPICIOUS_ENTROPY,
6};
7use serde::{Deserialize, Serialize};
8use std::cmp::Reverse;
9use std::collections::BTreeSet;
10
11pub(crate) const MIN_ENTROPY_INPUT_BYTES: usize = 12;
12const MAX_MATCHED_TEXT_BYTES: usize = 512;
13
14/// Explainable evidence that contributed to a suspicious result.
15#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
16#[serde(deny_unknown_fields)]
17pub struct SuspiciousIndicator {
18    /// Stable name of the matching pattern or built-in heuristic.
19    pub pattern_name: String,
20    /// Human-readable explanation of the indicator.
21    pub description: String,
22    /// Severity from 0 through 10.
23    pub severity: u8,
24    /// Bounded excerpt of the first match, when applicable.
25    pub matched_text: Option<String>,
26    /// Whether `matched_text` was shortened to the evidence byte limit.
27    pub matched_text_truncated: bool,
28}
29
30/// Result of analyzing one string value.
31#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
32#[serde(deny_unknown_fields)]
33pub struct StringAnalysis {
34    /// Shannon entropy calculated over UTF-8 bytes.
35    pub entropy: f64,
36    /// Deterministically ordered categories assigned by matching patterns.
37    pub categories: BTreeSet<String>,
38    /// Bounded suspicious evidence retained in pattern evaluation order.
39    pub suspicious_indicators: Vec<SuspiciousIndicator>,
40    /// Whether suspicious evidence was omitted or declared without retained detail.
41    pub indicators_truncated: bool,
42    /// Whether at least one suspicious signal was retained, omitted, or declared.
43    pub is_suspicious: bool,
44}
45
46/// Interface for deterministic, thread-safe string analyzers.
47pub trait StringAnalyzer: Send + Sync {
48    /// Analyze a string and return categories and suspicious evidence.
49    fn analyze(&self, value: &str) -> StringAnalysis;
50
51    /// Check whether analysis emits any suspicious evidence.
52    fn is_suspicious(&self, value: &str) -> bool {
53        self.analyze(value).is_suspicious
54    }
55
56    /// Calculate Shannon entropy over the value's UTF-8 bytes.
57    fn calculate_entropy(&self, value: &str) -> f64;
58
59    /// Return patterns in evaluation order.
60    fn get_patterns(&self) -> &[Pattern];
61
62    /// Validate and append a uniquely named compiled pattern.
63    fn add_pattern(&mut self, pattern: Pattern) -> AnalysisResult<()>;
64}
65
66/// Default byte-entropy and regex-based analyzer.
67pub struct DefaultStringAnalyzer {
68    patterns: Vec<Pattern>,
69    entropy_threshold: f64,
70    max_indicators: usize,
71}
72
73impl DefaultStringAnalyzer {
74    /// Create an analyzer without custom or built-in patterns.
75    pub fn new() -> Self {
76        Self {
77            patterns: Vec::new(),
78            entropy_threshold: DEFAULT_SUSPICIOUS_ENTROPY,
79            max_indicators: DEFAULT_MAX_INDICATORS_PER_STRING,
80        }
81    }
82
83    /// Set and validate the high-entropy threshold.
84    pub fn with_entropy_threshold(mut self, threshold: f64) -> AnalysisResult<Self> {
85        validate_entropy_threshold(threshold)?;
86        self.entropy_threshold = threshold;
87        Ok(self)
88    }
89
90    /// Set and validate the suspicious evidence limit.
91    pub fn with_max_indicators(mut self, max_indicators: usize) -> AnalysisResult<Self> {
92        if max_indicators == 0 {
93            return Err(AnalysisError::InvalidConfiguration {
94                field: "max_indicators_per_string",
95                reason: "must be greater than zero".to_string(),
96            });
97        }
98        self.max_indicators = max_indicators;
99        Ok(self)
100    }
101
102    /// Replace the current pattern set after validating names and limits.
103    pub fn with_patterns(mut self, patterns: Vec<Pattern>) -> AnalysisResult<Self> {
104        if patterns.len() > MAX_PATTERNS {
105            return Err(AnalysisError::CapacityExceeded {
106                resource: "analyzer patterns",
107                limit: MAX_PATTERNS,
108            });
109        }
110
111        let mut names = BTreeSet::new();
112        for pattern in &patterns {
113            pattern.validate()?;
114            if !names.insert(pattern.name.clone()) {
115                return Err(AnalysisError::DuplicateName {
116                    kind: "pattern",
117                    name: pattern.name.to_string(),
118                });
119            }
120        }
121        self.patterns = patterns.into_iter().map(Pattern::compact).collect();
122        Ok(self)
123    }
124}
125
126impl StringAnalyzer for DefaultStringAnalyzer {
127    fn analyze(&self, value: &str) -> StringAnalysis {
128        let entropy = self.calculate_entropy(value);
129        let mut retained_indicators = Vec::new();
130        let mut categories = BTreeSet::new();
131        let mut indicators_truncated = false;
132        let mut indicator_order = 0usize;
133
134        for pattern in &self.patterns {
135            let Some(matched) = pattern.regex.find(value) else {
136                continue;
137            };
138            categories.insert(pattern.category.clone());
139            if pattern.is_suspicious {
140                let (matched_text, matched_text_truncated) = bounded_excerpt(matched.as_str());
141                retain_indicator(
142                    &mut retained_indicators,
143                    &mut indicators_truncated,
144                    self.max_indicators,
145                    indicator_order,
146                    SuspiciousIndicator {
147                        pattern_name: pattern.name.clone(),
148                        description: pattern.description.clone(),
149                        severity: pattern.severity,
150                        matched_text: Some(matched_text),
151                        matched_text_truncated,
152                    },
153                );
154                indicator_order = indicator_order.saturating_add(1);
155            }
156        }
157
158        if entropy >= self.entropy_threshold && value.len() >= MIN_ENTROPY_INPUT_BYTES {
159            retain_indicator(
160                &mut retained_indicators,
161                &mut indicators_truncated,
162                self.max_indicators,
163                indicator_order,
164                SuspiciousIndicator {
165                    pattern_name: "high_entropy".to_string(),
166                    description: format!(
167                        "Byte entropy {entropy:.2} meets the configured {threshold:.2} threshold",
168                        threshold = self.entropy_threshold,
169                    ),
170                    severity: 6,
171                    matched_text: None,
172                    matched_text_truncated: false,
173                },
174            );
175            indicator_order = indicator_order.saturating_add(1);
176        }
177
178        if value
179            .chars()
180            .any(|character| character.is_control() && !matches!(character, '\n' | '\r' | '\t'))
181        {
182            retain_indicator(
183                &mut retained_indicators,
184                &mut indicators_truncated,
185                self.max_indicators,
186                indicator_order,
187                SuspiciousIndicator {
188                    pattern_name: "non_printable_characters".to_string(),
189                    description: "Contains control characters other than tab or line breaks"
190                        .to_string(),
191                    severity: 5,
192                    matched_text: None,
193                    matched_text_truncated: false,
194                },
195            );
196        }
197
198        retained_indicators.sort_by_key(|ranked| ranked.order);
199        let suspicious_indicators: Vec<_> = retained_indicators
200            .into_iter()
201            .map(|ranked| ranked.indicator)
202            .collect();
203        let is_suspicious = !suspicious_indicators.is_empty() || indicators_truncated;
204        StringAnalysis {
205            entropy,
206            categories,
207            suspicious_indicators,
208            indicators_truncated,
209            is_suspicious,
210        }
211    }
212
213    fn calculate_entropy(&self, value: &str) -> f64 {
214        let bytes = value.as_bytes();
215        if bytes.is_empty() {
216            return 0.0;
217        }
218
219        let mut byte_counts = [0usize; 256];
220        for &byte in bytes {
221            byte_counts[usize::from(byte)] = byte_counts[usize::from(byte)].saturating_add(1);
222        }
223
224        let length = bytes.len() as f64;
225        byte_counts
226            .into_iter()
227            .filter(|count| *count != 0)
228            .fold(0.0, |entropy, count| {
229                let probability = count as f64 / length;
230                entropy - probability * probability.log2()
231            })
232    }
233
234    fn get_patterns(&self) -> &[Pattern] {
235        &self.patterns
236    }
237
238    fn add_pattern(&mut self, pattern: Pattern) -> AnalysisResult<()> {
239        pattern.validate()?;
240        if self.patterns.len() >= MAX_PATTERNS {
241            return Err(AnalysisError::CapacityExceeded {
242                resource: "analyzer patterns",
243                limit: MAX_PATTERNS,
244            });
245        }
246        if self
247            .patterns
248            .iter()
249            .any(|existing| existing.name == pattern.name)
250        {
251            return Err(AnalysisError::DuplicateName {
252                kind: "pattern",
253                name: pattern.name.to_string(),
254            });
255        }
256        self.patterns.push(pattern.compact());
257        Ok(())
258    }
259}
260
261impl Default for DefaultStringAnalyzer {
262    fn default() -> Self {
263        Self::new()
264    }
265}
266
267fn validate_entropy_threshold(threshold: f64) -> AnalysisResult<()> {
268    if threshold.is_finite() && (0.0..=8.0).contains(&threshold) {
269        Ok(())
270    } else {
271        Err(AnalysisError::InvalidConfiguration {
272            field: "min_suspicious_entropy",
273            reason: "must be finite and between 0 and 8 inclusive".to_string(),
274        })
275    }
276}
277
278struct RankedIndicator {
279    order: usize,
280    indicator: SuspiciousIndicator,
281}
282
283fn retain_indicator(
284    indicators: &mut Vec<RankedIndicator>,
285    truncated: &mut bool,
286    limit: usize,
287    order: usize,
288    indicator: SuspiciousIndicator,
289) {
290    if indicators.len() < limit {
291        indicators.push(RankedIndicator { order, indicator });
292        return;
293    }
294
295    *truncated = true;
296    let Some((lowest_index, lowest)) = indicators
297        .iter()
298        .enumerate()
299        .min_by_key(|(_, ranked)| (ranked.indicator.severity, Reverse(ranked.order)))
300    else {
301        return;
302    };
303    if indicator.severity > lowest.indicator.severity {
304        indicators[lowest_index] = RankedIndicator { order, indicator };
305    }
306}
307
308fn bounded_excerpt(value: &str) -> (String, bool) {
309    if value.len() <= MAX_MATCHED_TEXT_BYTES {
310        return (value.to_string(), false);
311    }
312
313    let mut end = MAX_MATCHED_TEXT_BYTES;
314    while !value.is_char_boundary(end) {
315        end -= 1;
316    }
317    (value[..end].to_string(), true)
318}
threatflux_string_analysis/analyzer.rs

threatflux_string_analysis/
analyzer.rs
