sentri_core/
test_infrastructure.rs

1/// Comprehensive Test Infrastructure for Sentri Detectors
2///
3/// Provides unified testing framework for validating detector implementations
4/// across EVM, Solana, and Move chains with consistent test patterns.
5use std::collections::HashMap;
6
7/// Test result for a detector
8#[derive(Debug, Clone)]
9pub struct DetectorTestResult {
10    /// Detector name
11    pub detector_name: String,
12    /// Chain (evm, solana, move)
13    pub chain: String,
14    /// Number of vulnerable patterns tested
15    pub vulnerable_count: usize,
16    /// Number of safe patterns tested
17    pub safe_count: usize,
18    /// Number of true positives (correctly detected vulnerabilities)
19    pub true_positives: usize,
20    /// Number of false positives (safe code detected as vulnerable)
21    pub false_positives: usize,
22    /// Number of false negatives (vulnerable code not detected)
23    pub false_negatives: usize,
24    /// Test execution time in ms
25    pub execution_time_ms: u64,
26}
27
28impl DetectorTestResult {
29    /// Calculate precision (TP / (TP + FP))
30    pub fn precision(&self) -> f64 {
31        let total = self.true_positives + self.false_positives;
32        if total == 0 {
33            1.0
34        } else {
35            self.true_positives as f64 / total as f64
36        }
37    }
38
39    /// Calculate recall (TP / (TP + FN))
40    pub fn recall(&self) -> f64 {
41        let total = self.true_positives + self.false_negatives;
42        if total == 0 {
43            1.0
44        } else {
45            self.true_positives as f64 / total as f64
46        }
47    }
48
49    /// Calculate F1 score
50    pub fn f1_score(&self) -> f64 {
51        let p = self.precision();
52        let r = self.recall();
53        if p + r == 0.0 {
54            0.0
55        } else {
56            2.0 * p * r / (p + r)
57        }
58    }
59
60    /// Check if test passes quality threshold
61    pub fn passes_quality_threshold(&self) -> bool {
62        self.precision() >= 0.90 && self.recall() >= 0.90 && self.f1_score() >= 0.90
63    }
64}
65
66/// Test case for detector validation
67#[derive(Debug, Clone)]
68pub struct DetectorTestCase {
69    /// Test name
70    pub name: String,
71    /// Test code pattern
72    pub code: String,
73    /// Whether this code is vulnerable
74    pub is_vulnerable: bool,
75    /// Expected findings count
76    pub expected_findings: usize,
77    /// Category (vulnerable, safe, edge_case, real_exploit)
78    pub category: String,
79}
80
81/// Comprehensive test suite for all detectors
82pub struct DetectorTestSuite {
83    test_cases: HashMap<String, Vec<DetectorTestCase>>,
84    results: HashMap<String, DetectorTestResult>,
85}
86
87impl Default for DetectorTestSuite {
88    fn default() -> Self {
89        Self::new()
90    }
91}
92
93impl DetectorTestSuite {
94    /// Create new test suite
95    pub fn new() -> Self {
96        Self {
97            test_cases: HashMap::new(),
98            results: HashMap::new(),
99        }
100    }
101
102    /// Add test case to suite
103    pub fn add_test_case(&mut self, detector_name: String, test_case: DetectorTestCase) {
104        self.test_cases
105            .entry(detector_name)
106            .or_default()
107            .push(test_case);
108    }
109
110    /// Get test cases for detector
111    pub fn get_test_cases(&self, detector_name: &str) -> Option<&[DetectorTestCase]> {
112        self.test_cases.get(detector_name).map(|v| v.as_slice())
113    }
114
115    /// Store test result
116    pub fn add_result(&mut self, detector_name: String, result: DetectorTestResult) {
117        self.results.insert(detector_name, result);
118    }
119
120    /// Get test result for detector
121    pub fn get_result(&self, detector_name: &str) -> Option<&DetectorTestResult> {
122        self.results.get(detector_name)
123    }
124
125    /// Generate test report
126    pub fn generate_report(&self) -> String {
127        let mut report = "# Detector Test Report\n\n".to_string();
128        report.push_str("## Summary\n\n");
129
130        let mut total_tests = 0;
131        let mut passed_tests = 0;
132        let mut avg_precision = 0.0;
133        let mut avg_recall = 0.0;
134        let mut avg_f1 = 0.0;
135
136        for (detector, result) in &self.results {
137            report.push_str(&format!("### {} ({})\n", detector, result.chain));
138
139            report.push_str(&format!(
140                "- **Vulnerable Patterns Tested:** {}\n",
141                result.vulnerable_count
142            ));
143            report.push_str(&format!(
144                "- **Safe Patterns Tested:** {}\n",
145                result.safe_count
146            ));
147            report.push_str(&format!(
148                "- **True Positives:** {}\n",
149                result.true_positives
150            ));
151            report.push_str(&format!(
152                "- **False Positives:** {}\n",
153                result.false_positives
154            ));
155            report.push_str(&format!(
156                "- **False Negatives:** {}\n",
157                result.false_negatives
158            ));
159            report.push_str(&format!(
160                "- **Precision:** {:.2}%\n",
161                result.precision() * 100.0
162            ));
163            report.push_str(&format!("- **Recall:** {:.2}%\n", result.recall() * 100.0));
164            report.push_str(&format!("- **F1 Score:** {:.4}\n", result.f1_score()));
165            report.push_str(&format!(
166                "- **Execution Time:** {} ms\n",
167                result.execution_time_ms
168            ));
169
170            let status = if result.passes_quality_threshold() {
171                "✓ PASS"
172            } else {
173                "✗ FAIL"
174            };
175            report.push_str(&format!("- **Status:** {}\n\n", status));
176
177            total_tests += 1;
178            if result.passes_quality_threshold() {
179                passed_tests += 1;
180            }
181
182            avg_precision += result.precision();
183            avg_recall += result.recall();
184            avg_f1 += result.f1_score();
185        }
186
187        if !self.results.is_empty() {
188            let count = self.results.len() as f64;
189            report.push_str("\n## Aggregate Metrics\n\n");
190            report.push_str(&format!(
191                "- **Total Detectors Tested:** {}\n",
192                self.results.len()
193            ));
194            report.push_str(&format!(
195                "- **Passed Quality Threshold:** {}/{}\n",
196                passed_tests, total_tests
197            ));
198            report.push_str(&format!(
199                "- **Average Precision:** {:.2}%\n",
200                (avg_precision / count) * 100.0
201            ));
202            report.push_str(&format!(
203                "- **Average Recall:** {:.2}%\n",
204                (avg_recall / count) * 100.0
205            ));
206            report.push_str(&format!("- **Average F1 Score:** {:.4}\n", avg_f1 / count));
207        }
208
209        report
210    }
211}
212
213/// Test infrastructure utilities
214pub mod utils {
215    /// Extract code snippet from finding
216    pub fn extract_code_context(code: &str, line: usize, context_lines: usize) -> String {
217        let lines: Vec<&str> = code.lines().collect();
218        let start = line.saturating_sub(context_lines);
219        let end = std::cmp::min(line + context_lines, lines.len());
220
221        lines[start..end].join("\n")
222    }
223
224    /// Compare two findings for equality
225    pub fn findings_match(f1: &crate::Finding, f2: &crate::Finding) -> bool {
226        f1.invariant_id == f2.invariant_id && f1.file == f2.file && f1.line == f2.line
227    }
228
229    /// Generate test code variant
230    pub fn generate_variant(base_code: &str, mutation: &str) -> String {
231        base_code.replace("// MUTATION_POINT", mutation)
232    }
233}
234
235#[cfg(test)]
236mod tests {
237    use super::*;
238
239    #[test]
240    fn test_result_metrics() {
241        let result = DetectorTestResult {
242            detector_name: "test".to_string(),
243            chain: "evm".to_string(),
244            vulnerable_count: 50,
245            safe_count: 50,
246            true_positives: 48,
247            false_positives: 1,
248            false_negatives: 2,
249            execution_time_ms: 150,
250        };
251
252        assert!(result.precision() > 0.95);
253        assert!(result.recall() > 0.95);
254        assert!(result.f1_score() > 0.95);
255    }
256
257    #[test]
258    fn test_suite_stores_cases() {
259        let mut suite = DetectorTestSuite::new();
260        let test_case = DetectorTestCase {
261            name: "vulnerable_case".to_string(),
262            code: "malicious code".to_string(),
263            is_vulnerable: true,
264            expected_findings: 1,
265            category: "vulnerable".to_string(),
266        };
267
268        suite.add_test_case("detector1".to_string(), test_case);
269        assert!(suite.get_test_cases("detector1").is_some());
270    }
271
272    #[test]
273    fn test_suite_generates_report() {
274        let mut suite = DetectorTestSuite::new();
275        let result = DetectorTestResult {
276            detector_name: "test_detector".to_string(),
277            chain: "evm".to_string(),
278            vulnerable_count: 50,
279            safe_count: 50,
280            true_positives: 45,
281            false_positives: 2,
282            false_negatives: 5,
283            execution_time_ms: 200,
284        };
285
286        suite.add_result("test_detector".to_string(), result);
287        let report = suite.generate_report();
288
289        assert!(report.contains("test_detector"));
290        assert!(report.contains("Precision"));
291        assert!(report.contains("Recall"));
292    }
293}
sentri_core/test_infrastructure.rs

sentri_core/
test_infrastructure.rs
