stakpak_shared/secrets/
mod.rs

1pub mod gitleaks;
2use crate::helper::generate_simple_id;
3/// Re-export the gitleaks initialization function for external access
4pub use gitleaks::initialize_gitleaks_config;
5use gitleaks::{DetectedSecret, detect_secrets};
6use std::collections::HashMap;
7use std::fmt;
8
9/// A result containing both the redacted string and the mapping of redaction keys to original secrets
10#[derive(Debug, Clone)]
11pub struct RedactionResult {
12    /// The input string with secrets replaced by redaction keys
13    pub redacted_string: String,
14    /// Mapping from redaction key to the original secret value
15    pub redaction_map: HashMap<String, String>,
16}
17
18impl RedactionResult {
19    pub fn new(redacted_string: String, redaction_map: HashMap<String, String>) -> Self {
20        Self {
21            redacted_string,
22            redaction_map,
23        }
24    }
25}
26
27impl fmt::Display for RedactionResult {
28    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
29        write!(f, "{}", self.redacted_string)
30    }
31}
32
33/// Redacts secrets from the input string and returns both the redacted string and redaction mapping
34///
35/// When privacy_mode is enabled, also detects and redacts private data like IP addresses and AWS account IDs
36pub fn redact_secrets(
37    content: &str,
38    path: Option<&str>,
39    old_redaction_map: &HashMap<String, String>,
40    privacy_mode: bool,
41) -> RedactionResult {
42    // Skip redaction if content already contains redacted secrets (avoid double redaction)
43    if content.contains("[REDACTED_SECRET:") {
44        return RedactionResult::new(content.to_string(), HashMap::new());
45    }
46
47    let mut secrets = detect_secrets(content, path, privacy_mode);
48
49    let mut redaction_map = old_redaction_map.clone();
50    let mut reverse_redaction_map: HashMap<String, String> = old_redaction_map
51        .clone()
52        .into_iter()
53        .map(|(k, v)| (v, k))
54        .collect();
55
56    for (original_secret, redaction_key) in &reverse_redaction_map {
57        // Extract rule_id from redaction_key format: [REDACTED_SECRET:rule_id:id]
58        let key_parts = redaction_key.split(':').collect::<Vec<&str>>();
59        if key_parts.len() == 3 {
60            let rule_id = key_parts[1].to_string();
61            if let Some(start) = content.find(original_secret) {
62                let end = start + original_secret.len();
63                secrets.push(DetectedSecret {
64                    rule_id,
65                    value: original_secret.clone(),
66                    start_pos: start,
67                    end_pos: end,
68                });
69            }
70        }
71    }
72
73    if secrets.is_empty() {
74        return RedactionResult::new(content.to_string(), HashMap::new());
75    }
76
77    let mut redacted_string = content.to_string();
78
79    // Deduplicate overlapping secrets - keep the longest one
80    let mut deduplicated_secrets: Vec<DetectedSecret> = Vec::new();
81    let mut sorted_by_start = secrets;
82    sorted_by_start.sort_by(|a, b| a.start_pos.cmp(&b.start_pos));
83
84    for secret in sorted_by_start {
85        let mut should_add = true;
86        let mut to_remove = Vec::new();
87
88        for (i, existing) in deduplicated_secrets.iter().enumerate() {
89            // Check if secrets overlap
90            let overlaps =
91                secret.start_pos < existing.end_pos && secret.end_pos > existing.start_pos;
92
93            if overlaps {
94                // Keep the longer secret (more specific)
95                if secret.value.len() > existing.value.len() {
96                    to_remove.push(i);
97                } else {
98                    should_add = false;
99                    break;
100                }
101            }
102        }
103
104        // Remove secrets that should be replaced by this longer one
105        for &i in to_remove.iter().rev() {
106            deduplicated_secrets.remove(i);
107        }
108
109        if should_add {
110            deduplicated_secrets.push(secret);
111        }
112    }
113
114    // Sort by position in reverse order to avoid index shifting issues
115    deduplicated_secrets.sort_by(|a, b| b.start_pos.cmp(&a.start_pos));
116
117    for secret in deduplicated_secrets {
118        // Validate character boundaries before replacement
119        if !content.is_char_boundary(secret.start_pos) || !content.is_char_boundary(secret.end_pos)
120        {
121            continue;
122        }
123
124        // Validate positions are within bounds
125        if secret.start_pos >= redacted_string.len() || secret.end_pos > redacted_string.len() {
126            continue;
127        }
128
129        // make sure same secrets have the same redaction key within the same file
130        // without making the hash content dependent (content addressable)
131        let redaction_key = if let Some(existing_key) = reverse_redaction_map.get(&secret.value) {
132            existing_key.clone()
133        } else {
134            let key = generate_redaction_key(&secret.rule_id);
135            // Store the mapping (only once per unique secret value)
136            redaction_map.insert(key.clone(), secret.value.clone());
137            reverse_redaction_map.insert(secret.value, key.clone());
138            key
139        };
140
141        // Replace the secret in the string
142        redacted_string.replace_range(secret.start_pos..secret.end_pos, &redaction_key);
143    }
144
145    RedactionResult::new(redacted_string, redaction_map)
146}
147
148/// Restores secrets in a redacted string using the provided redaction map
149pub fn restore_secrets(redacted_string: &str, redaction_map: &HashMap<String, String>) -> String {
150    let mut restored = redacted_string.to_string();
151
152    for (redaction_key, original_value) in redaction_map {
153        restored = restored.replace(redaction_key, original_value);
154    }
155
156    restored
157}
158
159/// Redacts a specific password value from the content without running secret detection
160pub fn redact_password(
161    content: &str,
162    password: &str,
163    old_redaction_map: &HashMap<String, String>,
164) -> RedactionResult {
165    if password.is_empty() {
166        return RedactionResult::new(content.to_string(), HashMap::new());
167    }
168
169    // Skip redaction if content already contains redacted secrets (avoid double redaction)
170    if content.contains("[REDACTED_SECRET:") {
171        return RedactionResult::new(content.to_string(), HashMap::new());
172    }
173
174    let mut redacted_string = content.to_string();
175    let mut redaction_map = old_redaction_map.clone();
176    let mut reverse_redaction_map: HashMap<String, String> = old_redaction_map
177        .clone()
178        .into_iter()
179        .map(|(k, v)| (v, k))
180        .collect();
181
182    // Check if we already have a redaction key for this password
183    let redaction_key = if let Some(existing_key) = reverse_redaction_map.get(password) {
184        existing_key.clone()
185    } else {
186        let key = generate_redaction_key("password");
187        // Store the mapping
188        redaction_map.insert(key.clone(), password.to_string());
189        reverse_redaction_map.insert(password.to_string(), key.clone());
190        key
191    };
192
193    // Replace all occurrences of the password
194    redacted_string = redacted_string.replace(password, &redaction_key);
195
196    RedactionResult::new(redacted_string, redaction_map)
197}
198
199/// Generates a random redaction key
200fn generate_redaction_key(rule_id: &str) -> String {
201    let id = generate_simple_id(6);
202    format!("[REDACTED_SECRET:{rule_id}:{id}]")
203}
204
205#[cfg(test)]
206mod tests {
207    use regex::Regex;
208
209    use crate::secrets::gitleaks::{
210        GITLEAKS_CONFIG, calculate_entropy, contains_any_keyword, create_simple_api_key_regex,
211        is_allowed_by_rule_allowlist, should_allow_match,
212    };
213
214    use super::*;
215
216    fn fake_aws_access_key() -> String {
217        ["AKIA", "IOSFODNN7EX23PLE"].concat()
218    }
219
220    fn fake_aws_access_key_alt() -> String {
221        ["AKIA", "IOSFODNN7REALKEY"].concat()
222    }
223
224    fn fake_aws_access_key_example() -> String {
225        ["AKIA", "IOSFODNN7EXAMPLE"].concat()
226    }
227
228    fn fake_github_token() -> String {
229        ["ghp", "_1234567890abcdef", "1234567890abcdef", "12345678"].concat()
230    }
231
232    fn fake_github_token_short() -> String {
233        ["ghp", "_1234567890abcdef"].concat()
234    }
235
236    fn fake_api_key_long() -> String {
237        ["abc123def456", "ghi789jkl012", "mno345pqr678"].concat()
238    }
239
240    fn fake_api_key() -> String {
241        ["abc123def456", "ghi789jklmnop"].concat()
242    }
243
244    fn fake_secret_token() -> String {
245        ["Kx9mP2nQ8rT4", "vW7yZ3cF6hJ1", "lN5sA0bD8eF"].concat()
246    }
247
248    fn fake_secret_token_long() -> String {
249        ["Kx9mP2nQ8rT4", "vW7yZ3cF6hJ1", "lN5sA0bD8eF2gH5jK"].concat()
250    }
251
252    fn fake_password_secret() -> String {
253        ["super", "secret", "password", "123456"].concat()
254    }
255
256    #[test]
257    fn test_redaction_key_generation() {
258        let key1 = generate_redaction_key("test");
259        let key2 = generate_redaction_key("my-rule");
260
261        // Keys should be different
262        assert_ne!(key1, key2);
263
264        // Keys should follow the expected format
265        assert!(key1.starts_with("[REDACTED_SECRET:test:"));
266        assert!(key1.ends_with("]"));
267        assert!(key2.starts_with("[REDACTED_SECRET:my-rule:"));
268        assert!(key2.ends_with("]"));
269    }
270
271    #[test]
272    fn test_empty_input() {
273        let result = redact_secrets("", None, &HashMap::new(), false);
274        assert_eq!(result.redacted_string, "");
275        assert!(result.redaction_map.is_empty());
276    }
277
278    #[test]
279    fn test_restore_secrets() {
280        let mut redaction_map = HashMap::new();
281        redaction_map.insert("[REDACTED_abc123]".to_string(), "secret123".to_string());
282        redaction_map.insert("[REDACTED_def456]".to_string(), "api_key_xyz".to_string());
283
284        let redacted = "Password is [REDACTED_abc123] and key is [REDACTED_def456]";
285        let restored = restore_secrets(redacted, &redaction_map);
286
287        assert_eq!(restored, "Password is secret123 and key is api_key_xyz");
288    }
289
290    #[test]
291    fn test_redaction_result_display() {
292        let mut redaction_map = HashMap::new();
293        redaction_map.insert("[REDACTED_test]".to_string(), "secret".to_string());
294
295        let result = RedactionResult::new("Hello [REDACTED_test]".to_string(), redaction_map);
296        assert_eq!(format!("{}", result), "Hello [REDACTED_test]");
297    }
298
299    #[test]
300    fn test_redact_secrets_with_api_key() {
301        // Use a pattern that matches the generic-api-key rule
302        let input = format!("export API_KEY={}", fake_api_key_long());
303        let result = redact_secrets(&input, None, &HashMap::new(), false);
304
305        // Should detect the API key and redact it
306        assert!(!result.redaction_map.is_empty());
307        assert!(result.redacted_string.contains("[REDACTED_"));
308        println!("Input: {}", input);
309        println!("Redacted: {}", result.redacted_string);
310        println!("Mapping: {:?}", result.redaction_map);
311    }
312
313    #[test]
314    fn test_redact_secrets_with_aws_key() {
315        let input = format!("AWS_ACCESS_KEY_ID={}", fake_aws_access_key());
316        let result = redact_secrets(&input, None, &HashMap::new(), false);
317
318        // Should detect the AWS access key
319        assert!(!result.redaction_map.is_empty());
320        println!("Input: {}", input);
321        println!("Redacted: {}", result.redacted_string);
322        println!("Mapping: {:?}", result.redaction_map);
323    }
324
325    #[test]
326    fn test_redaction_identical_secrets() {
327        let aws_key = fake_aws_access_key();
328        let input = format!(
329            "\n        export AWS_ACCESS_KEY_ID={aws_key}\n        export AWS_ACCESS_KEY_ID_2={aws_key}\n        "
330        );
331        let result = redact_secrets(&input, None, &HashMap::new(), false);
332
333        assert_eq!(result.redaction_map.len(), 1);
334    }
335
336    #[test]
337    fn test_redaction_identical_secrets_different_contexts() {
338        let aws_key = fake_aws_access_key();
339        let input_1 = format!("\n        export AWS_ACCESS_KEY_ID={aws_key}\n        ");
340        let input_2 = format!("\n        export SOME_OTHER_SECRET={aws_key}\n        ");
341        let result_1 = redact_secrets(&input_1, None, &HashMap::new(), false);
342        let result_2 = redact_secrets(&input_2, None, &result_1.redaction_map, false);
343
344        assert_eq!(result_1.redaction_map, result_2.redaction_map);
345    }
346
347    #[test]
348    fn test_redact_secrets_with_github_token() {
349        let input = format!("GITHUB_TOKEN={}", fake_github_token());
350        let result = redact_secrets(&input, None, &HashMap::new(), false);
351
352        // Should detect the GitHub PAT
353        assert!(!result.redaction_map.is_empty());
354        println!("Input: {}", input);
355        println!("Redacted: {}", result.redacted_string);
356        println!("Mapping: {:?}", result.redaction_map);
357    }
358
359    #[test]
360    fn test_no_secrets() {
361        let input = "This is just a normal string with no secrets";
362        let result = redact_secrets(input, None, &HashMap::new(), false);
363
364        // Should not detect any secrets
365        assert_eq!(result.redaction_map.len(), 0);
366        assert_eq!(result.redacted_string, input);
367    }
368
369    #[test]
370    fn test_debug_generic_api_key() {
371        let config = &*GITLEAKS_CONFIG;
372
373        // Find the generic-api-key rule
374        let generic_rule = config.rules.iter().find(|r| r.id == "generic-api-key");
375        if let Some(rule) = generic_rule {
376            println!("Generic API Key Rule:");
377            println!("  Regex: {:?}", rule.regex);
378            println!("  Entropy: {:?}", rule.entropy);
379            println!("  Keywords: {:?}", rule.keywords);
380
381            // Test the regex directly first
382            if let Some(regex_pattern) = &rule.regex {
383                if let Ok(regex) = Regex::new(regex_pattern) {
384                    let test_input = format!("API_KEY={}", fake_api_key_long());
385                    println!("\nTesting regex directly:");
386                    println!("  Input: {}", test_input);
387
388                    for mat in regex.find_iter(&test_input) {
389                        println!("  Raw match: '{}'", mat.as_str());
390                        println!("  Match position: {}-{}", mat.start(), mat.end());
391
392                        // Check captures
393                        if let Some(captures) = regex.captures(mat.as_str()) {
394                            for (i, cap) in captures.iter().enumerate() {
395                                if let Some(cap) = cap {
396                                    println!("  Capture {}: '{}'", i, cap.as_str());
397                                    if i == 1 {
398                                        let entropy = calculate_entropy(cap.as_str());
399                                        println!("  Entropy of capture 1: {:.2}", entropy);
400                                    }
401                                }
402                            }
403                        }
404                    }
405                }
406            } else {
407                println!("  No regex pattern (path-based rule)");
408            }
409
410            // Test various input patterns
411            let test_inputs = vec![
412                format!("API_KEY={}", fake_api_key_long()),
413                "api_key=RaNd0mH1ghEnTr0pyV4luE567890abcdef".to_string(),
414                format!("access_key={}", fake_secret_token_long()),
415                "secret_token=1234567890abcdef1234567890abcdef".to_string(),
416                "password=9k2L8pMvB3nQ7rX1ZdF5GhJwY4AsPo6C".to_string(),
417            ];
418
419            for input in test_inputs {
420                println!("\nTesting input: {}", input);
421                let result = redact_secrets(&input, None, &HashMap::new(), false);
422                println!("  Detected secrets: {}", result.redaction_map.len());
423                if !result.redaction_map.is_empty() {
424                    println!("  Redacted: {}", result.redacted_string);
425                }
426            }
427        } else {
428            println!("Generic API key rule not found!");
429        }
430    }
431
432    #[test]
433    fn test_simple_regex_match() {
434        // Test a very simple case that should definitely match
435        let input = "key=abcdefghijklmnop";
436        println!("Testing simple input: {}", input);
437
438        let config = &*GITLEAKS_CONFIG;
439        let generic_rule = config
440            .rules
441            .iter()
442            .find(|r| r.id == "generic-api-key")
443            .unwrap();
444
445        if let Some(regex_pattern) = &generic_rule.regex {
446            if let Ok(regex) = Regex::new(regex_pattern) {
447                println!("Regex pattern: {}", regex_pattern);
448
449                if regex.is_match(input) {
450                    println!("✓ Regex MATCHES the input!");
451
452                    for mat in regex.find_iter(input) {
453                        println!("Match found: '{}'", mat.as_str());
454
455                        if let Some(captures) = regex.captures(mat.as_str()) {
456                            println!("Full capture groups:");
457                            for (i, cap) in captures.iter().enumerate() {
458                                if let Some(cap) = cap {
459                                    println!("  Group {}: '{}'", i, cap.as_str());
460                                    if i == 1 {
461                                        let entropy = calculate_entropy(cap.as_str());
462                                        println!("  Entropy: {:.2} (threshold: 3.5)", entropy);
463                                    }
464                                }
465                            }
466                        }
467                    }
468                } else {
469                    println!("✗ Regex does NOT match the input");
470                }
471            }
472        } else {
473            println!("Rule has no regex pattern (path-based rule)");
474        }
475
476        // Also test the full redact_secrets function
477        let result = redact_secrets(input, None, &HashMap::new(), false);
478        println!(
479            "Full function result: {} secrets detected",
480            result.redaction_map.len()
481        );
482    }
483
484    #[test]
485    fn test_regex_breakdown() {
486        let config = &*GITLEAKS_CONFIG;
487        let generic_rule = config
488            .rules
489            .iter()
490            .find(|r| r.id == "generic-api-key")
491            .unwrap();
492
493        if let Some(regex_pattern) = &generic_rule.regex {
494            println!("Full regex: {}", regex_pattern);
495
496            // Let's break down the regex and test each part
497            let test_inputs = vec![
498                "key=abcdefghijklmnop",
499                "api_key=abcdefghijklmnop",
500                "secret=abcdefghijklmnop",
501                "token=abcdefghijklmnop",
502                "password=abcdefghijklmnop",
503                "access_key=abcdefghijklmnop",
504            ];
505
506            for input in test_inputs {
507                println!("\nTesting: '{}'", input);
508
509                // Test if the regex matches at all
510                if let Ok(regex) = Regex::new(regex_pattern) {
511                    let matches: Vec<_> = regex.find_iter(input).collect();
512                    println!("  Matches found: {}", matches.len());
513
514                    for (i, mat) in matches.iter().enumerate() {
515                        println!("  Match {}: '{}'", i, mat.as_str());
516
517                        // Test captures
518                        if let Some(captures) = regex.captures(mat.as_str()) {
519                            for (j, cap) in captures.iter().enumerate() {
520                                if let Some(cap) = cap {
521                                    println!("    Capture {}: '{}'", j, cap.as_str());
522                                    if j == 1 {
523                                        let entropy = calculate_entropy(cap.as_str());
524                                        println!("    Entropy: {:.2} (threshold: 3.5)", entropy);
525                                        if entropy >= 3.5 {
526                                            println!("    ✓ Entropy check PASSED");
527                                        } else {
528                                            println!("    ✗ Entropy check FAILED");
529                                        }
530                                    }
531                                }
532                            }
533                        }
534                    }
535                }
536            }
537        } else {
538            println!("Rule has no regex pattern (path-based rule)");
539        }
540
541        // Also test with a known working pattern from AWS
542        println!("\nTesting AWS pattern that we know works:");
543        let aws_input = format!("AWS_ACCESS_KEY_ID={}", fake_aws_access_key_example());
544        println!("Input: {}", aws_input);
545
546        let aws_rule = config
547            .rules
548            .iter()
549            .find(|r| r.id == "aws-access-token")
550            .unwrap();
551        if let Some(aws_regex_pattern) = &aws_rule.regex {
552            if let Ok(regex) = Regex::new(aws_regex_pattern) {
553                for mat in regex.find_iter(&aws_input) {
554                    println!("AWS Match: '{}'", mat.as_str());
555                    if let Some(captures) = regex.captures(mat.as_str()) {
556                        for (i, cap) in captures.iter().enumerate() {
557                            if let Some(cap) = cap {
558                                println!("  AWS Capture {}: '{}'", i, cap.as_str());
559                            }
560                        }
561                    }
562                }
563            }
564        } else {
565            println!("AWS rule has no regex pattern");
566        }
567    }
568
569    #[test]
570    fn test_working_api_key_patterns() {
571        let config = &*GITLEAKS_CONFIG;
572        let generic_rule = config
573            .rules
574            .iter()
575            .find(|r| r.id == "generic-api-key")
576            .unwrap();
577
578        // Get the compiled regex
579        let regex = generic_rule
580            .compiled_regex
581            .as_ref()
582            .expect("Regex should be compiled");
583
584        // Create test patterns that should match the regex structure
585        let test_inputs = vec![
586            // Pattern: prefix + keyword + separator + value + terminator
587            format!("myapp_api_key = \"{}\"", fake_api_key()),
588            format!("export SECRET_TOKEN={}", fake_secret_token()),
589            "app.auth.password: 9k2L8pMvB3nQ7rX1ZdF5GhJwY4AsPo6C8mN".to_string(),
590            "config.access_key=\"RaNd0mH1ghEnTr0pyV4luE567890abcdef\";".to_string(),
591            "DB_CREDENTIALS=xy9mP2nQ8rT4vW7yZ3cF6hJ1lN5sAdefghij".to_string(),
592        ];
593
594        for input in test_inputs {
595            println!("\nTesting: '{}'", input);
596
597            let matches: Vec<_> = regex.find_iter(&input).collect();
598            println!("  Matches found: {}", matches.len());
599
600            for (i, mat) in matches.iter().enumerate() {
601                println!("  Match {}: '{}'", i, mat.as_str());
602
603                if let Some(captures) = regex.captures(mat.as_str()) {
604                    for (j, cap) in captures.iter().enumerate() {
605                        if let Some(cap) = cap {
606                            println!("    Capture {}: '{}'", j, cap.as_str());
607                            if j == 1 {
608                                let entropy = calculate_entropy(cap.as_str());
609                                println!("    Entropy: {:.2} (threshold: 3.5)", entropy);
610
611                                // Also check if it would be allowed by allowlists
612                                let allowed = should_allow_match(
613                                    &input,
614                                    None,
615                                    mat.as_str(),
616                                    mat.start(),
617                                    mat.end(),
618                                    generic_rule,
619                                    &config.allowlist,
620                                );
621                                println!("    Allowed by allowlist: {}", allowed);
622                            }
623                        }
624                    }
625                }
626            }
627
628            // Test the full redact_secrets function
629            let result = redact_secrets(&input, None, &HashMap::new(), false);
630            println!(
631                "  Full function detected: {} secrets",
632                result.redaction_map.len()
633            );
634            if !result.redaction_map.is_empty() {
635                println!("  Redacted result: {}", result.redacted_string);
636            }
637        }
638    }
639
640    #[test]
641    fn test_regex_components() {
642        // Test individual components of the generic API key regex
643        let test_input = format!("export API_KEY={}", fake_secret_token());
644        println!("Testing input: {}", test_input);
645
646        // Test simpler regex patterns step by step
647        let test_patterns = vec![
648            (r"API_KEY", "Simple keyword match"),
649            (r"(?i)api_key", "Case insensitive keyword"),
650            (r"(?i).*key.*", "Any text with 'key'"),
651            (r"(?i).*key\s*=", "Key with equals"),
652            (r"(?i).*key\s*=\s*\w+", "Key with value"),
653            (
654                r"(?i)[\w.-]*(?:key).*?=.*?(\w{10,})",
655                "Complex pattern with capture",
656            ),
657        ];
658
659        for (pattern, description) in test_patterns {
660            println!("\nTesting pattern: {} ({})", pattern, description);
661
662            match Regex::new(pattern) {
663                Ok(regex) => {
664                    if regex.is_match(&test_input) {
665                        println!("  ✓ MATCHES");
666                        for mat in regex.find_iter(&test_input) {
667                            println!("    Full match: '{}'", mat.as_str());
668                        }
669                        if let Some(captures) = regex.captures(&test_input) {
670                            for (i, cap) in captures.iter().enumerate() {
671                                if let Some(cap) = cap {
672                                    println!("    Capture {}: '{}'", i, cap.as_str());
673                                }
674                            }
675                        }
676                    } else {
677                        println!("  ✗ NO MATCH");
678                    }
679                }
680                Err(e) => println!("  Error: {}", e),
681            }
682        }
683
684        // Test if there's an issue with the actual gitleaks regex compilation
685        let config = &*GITLEAKS_CONFIG;
686        let generic_rule = config
687            .rules
688            .iter()
689            .find(|r| r.id == "generic-api-key")
690            .unwrap();
691
692        println!("\nTesting actual gitleaks regex:");
693        if let Some(regex_pattern) = &generic_rule.regex {
694            match Regex::new(regex_pattern) {
695                Ok(regex) => {
696                    println!("  ✓ Regex compiles successfully");
697                    println!("  Testing against: {}", test_input);
698                    if regex.is_match(&test_input) {
699                        println!("  ✓ MATCHES");
700                    } else {
701                        println!("  ✗ NO MATCH");
702                    }
703                }
704                Err(e) => println!("  ✗ Regex compilation error: {}", e),
705            }
706        } else {
707            println!("  Rule has no regex pattern (path-based rule)");
708        }
709    }
710
711    #[test]
712    fn test_comprehensive_secrets_redaction() {
713        let aws_key = fake_aws_access_key_alt();
714        let github_token = fake_github_token();
715        let api_key = fake_api_key();
716        let secret_token = fake_secret_token();
717        let password = fake_password_secret();
718        let input = format!(
719            "\n# Configuration file with various secrets\nexport AWS_ACCESS_KEY_ID={aws_key}\nexport GITHUB_TOKEN={github_token}\nexport API_KEY={api_key}\nexport SECRET_TOKEN={secret_token}\nexport PASSWORD={password}\n\n# Some normal configuration\nexport DEBUG=true\nexport PORT=3000\n"
720        );
721
722        println!("Original input:\n{}", input);
723
724        let result = redact_secrets(&input, None, &HashMap::new(), false);
725
726        println!("Redacted output:\n{}", result.redacted_string);
727        println!("\nDetected {} secrets:", result.redaction_map.len());
728        for (key, value) in &result.redaction_map {
729            println!("  {} -> {}", key, value);
730        }
731
732        // Should detect at least 5 secrets: AWS key, GitHub token, API key, secret token, password
733        assert!(
734            result.redaction_map.len() >= 5,
735            "Should detect at least 5 secrets, found: {}",
736            result.redaction_map.len()
737        );
738
739        // Verify specific secrets are redacted
740        assert!(!result.redacted_string.contains(&aws_key));
741        assert!(!result.redacted_string.contains(&github_token));
742        assert!(!result.redacted_string.contains(&api_key));
743
744        // Verify normal config is preserved
745        assert!(result.redacted_string.contains("DEBUG=true"));
746        assert!(result.redacted_string.contains("PORT=3000"));
747    }
748
749    // Helper function for keyword validation tests
750    fn count_rules_that_would_process(input: &str) -> Vec<String> {
751        let config = &*GITLEAKS_CONFIG;
752        let mut rules = Vec::new();
753
754        for rule in &config.rules {
755            if rule.keywords.is_empty() || contains_any_keyword(input, &rule.keywords) {
756                rules.push(rule.id.clone());
757            }
758        }
759
760        rules
761    }
762
763    #[test]
764    fn test_keyword_filtering() {
765        println!("=== TESTING KEYWORD FILTERING ===");
766
767        let config = &*GITLEAKS_CONFIG;
768
769        // Find a rule that has keywords (like generic-api-key)
770        let generic_rule = config
771            .rules
772            .iter()
773            .find(|r| r.id == "generic-api-key")
774            .unwrap();
775        println!("Generic API Key rule keywords: {:?}", generic_rule.keywords);
776
777        // Test 1: Input with keywords should be processed
778        let input_with_keywords = format!("export API_KEY={}", fake_api_key());
779        let result1 = redact_secrets(&input_with_keywords, None, &HashMap::new(), false);
780        println!("\nTest 1 - Input WITH keywords:");
781        println!("  Input: {}", input_with_keywords);
782        println!(
783            "  Keywords present: {}",
784            contains_any_keyword(&input_with_keywords, &generic_rule.keywords)
785        );
786        println!("  Secrets detected: {}", result1.redaction_map.len());
787
788        // Test 2: Input without any keywords should NOT be processed for that rule
789        let input_without_keywords = "export DATABASE_URL=postgresql://user:pass@localhost/db";
790        let result2 = redact_secrets(input_without_keywords, None, &HashMap::new(), false);
791        println!("\nTest 2 - Input WITHOUT generic-api-key keywords:");
792        println!("  Input: {}", input_without_keywords);
793        println!(
794            "  Keywords present: {}",
795            contains_any_keyword(input_without_keywords, &generic_rule.keywords)
796        );
797        println!("  Secrets detected: {}", result2.redaction_map.len());
798
799        // Test 3: Input with different rule's keywords (AWS)
800        let aws_rule = config
801            .rules
802            .iter()
803            .find(|r| r.id == "aws-access-token")
804            .unwrap();
805        let aws_input = format!("AWS_ACCESS_KEY_ID={}", fake_aws_access_key_example());
806        let result3 = redact_secrets(&aws_input, None, &HashMap::new(), false);
807        println!("\nTest 3 - AWS input:");
808        println!("  Input: {}", aws_input);
809        println!("  AWS rule keywords: {:?}", aws_rule.keywords);
810        println!(
811            "  Keywords present: {}",
812            contains_any_keyword(&aws_input, &aws_rule.keywords)
813        );
814        println!("  Secrets detected: {}", result3.redaction_map.len());
815
816        // Validate that keyword filtering is working
817        assert!(
818            contains_any_keyword(&input_with_keywords, &generic_rule.keywords),
819            "API_KEY input should contain generic-api-key keywords"
820        );
821        assert!(
822            !contains_any_keyword(input_without_keywords, &generic_rule.keywords),
823            "DATABASE_URL input should NOT contain generic-api-key keywords"
824        );
825        assert!(
826            contains_any_keyword(&aws_input, &aws_rule.keywords),
827            "AWS input should contain AWS rule keywords"
828        );
829    }
830
831    #[test]
832    fn test_keyword_optimization_performance() {
833        println!("=== TESTING KEYWORD OPTIMIZATION PERFORMANCE ===");
834
835        let config = &*GITLEAKS_CONFIG;
836
837        // Test case 1: Input with NO keywords for any rule should be very fast
838        let no_keywords_input = "export DATABASE_CONNECTION=some_long_connection_string_that_has_no_common_secret_keywords";
839        println!("Testing input with no secret keywords:");
840        println!("  Input: {}", no_keywords_input);
841
842        let mut keyword_matches = 0;
843        for rule in &config.rules {
844            if contains_any_keyword(no_keywords_input, &rule.keywords) {
845                keyword_matches += 1;
846                println!("  Rule '{}' keywords match: {:?}", rule.id, rule.keywords);
847            }
848        }
849        println!(
850            "  Rules with matching keywords: {} out of {}",
851            keyword_matches,
852            config.rules.len()
853        );
854
855        let result = redact_secrets(no_keywords_input, None, &HashMap::new(), false);
856        println!("  Secrets detected: {}", result.redaction_map.len());
857
858        // Test case 2: Input with specific keywords should only process relevant rules
859        let specific_keywords_input = format!("export GITHUB_TOKEN={}", fake_github_token_short());
860        println!("\nTesting input with specific keywords (github):");
861        println!("  Input: {}", specific_keywords_input);
862
863        let mut matching_rules = Vec::new();
864        for rule in &config.rules {
865            if contains_any_keyword(&specific_keywords_input, &rule.keywords) {
866                matching_rules.push(&rule.id);
867            }
868        }
869        println!("  Rules that would be processed: {:?}", matching_rules);
870
871        let result = redact_secrets(&specific_keywords_input, None, &HashMap::new(), false);
872        println!("  Secrets detected: {}", result.redaction_map.len());
873
874        // Test case 3: Verify that rules without keywords are always processed
875        let rules_without_keywords: Vec<_> = config
876            .rules
877            .iter()
878            .filter(|rule| rule.keywords.is_empty())
879            .collect();
880        println!(
881            "\nRules without keywords (always processed): {}",
882            rules_without_keywords.len()
883        );
884        for rule in &rules_without_keywords {
885            println!("  - {}", rule.id);
886        }
887
888        // Assertions
889        assert!(
890            keyword_matches < config.rules.len(),
891            "Input with no keywords should not match all rules"
892        );
893        assert!(
894            !matching_rules.is_empty(),
895            "GitHub token input should match some rules"
896        );
897        assert!(
898            matching_rules.contains(&&"github-pat".to_string())
899                || matching_rules
900                    .iter()
901                    .any(|rule_id| rule_id.contains("github")),
902            "GitHub token should match GitHub-related rules"
903        );
904    }
905
906    #[test]
907    fn test_keyword_filtering_efficiency() {
908        println!("=== KEYWORD FILTERING EFFICIENCY TEST ===");
909
910        let config = &*GITLEAKS_CONFIG;
911        println!("Total rules in config: {}", config.rules.len());
912
913        // Test with input that has NO matching keywords
914        let non_secret_input = "export DATABASE_URL=localhost PORT=3000 DEBUG=true TIMEOUT=30";
915        println!("\nTesting non-secret input: {}", non_secret_input);
916
917        let mut rules_skipped = 0;
918        let mut rules_processed = 0;
919
920        for rule in &config.rules {
921            if rule.keywords.is_empty() || contains_any_keyword(non_secret_input, &rule.keywords) {
922                rules_processed += 1;
923            } else {
924                rules_skipped += 1;
925            }
926        }
927
928        println!(
929            "  Rules skipped due to keyword filtering: {}",
930            rules_skipped
931        );
932        println!("  Rules that would be processed: {}", rules_processed);
933        println!(
934            "  Efficiency gain: {:.1}% of rules skipped",
935            (rules_skipped as f64 / config.rules.len() as f64) * 100.0
936        );
937
938        // Verify no secrets are detected
939        let result = redact_secrets(non_secret_input, None, &HashMap::new(), false);
940        println!("  Secrets detected: {}", result.redaction_map.len());
941
942        // Now test with input that has relevant keywords
943        let secret_input = format!(
944            "export API_KEY={} SECRET_TOKEN=xyz789uvw012rst345def678",
945            fake_api_key()
946        );
947        println!("\nTesting input WITH secret keywords:");
948        println!("  Input: {}", secret_input);
949
950        let mut rules_with_keywords = 0;
951        for rule in &config.rules {
952            if contains_any_keyword(&secret_input, &rule.keywords) {
953                rules_with_keywords += 1;
954            }
955        }
956
957        println!("  Rules that match keywords: {}", rules_with_keywords);
958
959        let result = redact_secrets(&secret_input, None, &HashMap::new(), false);
960        println!("  Secrets detected: {}", result.redaction_map.len());
961
962        // Assertions
963        assert!(
964            rules_skipped > 0,
965            "Should skip at least some rules for non-secret input"
966        );
967        assert!(
968            rules_with_keywords > 0,
969            "Should find matching rules for secret input"
970        );
971        assert!(
972            !result.redaction_map.is_empty(),
973            "Should detect at least one secret"
974        );
975    }
976
977    #[test]
978    fn test_keyword_validation_summary() {
979        println!("=== KEYWORD VALIDATION SUMMARY ===");
980
981        let config = &*GITLEAKS_CONFIG;
982        let total_rules = config.rules.len();
983        println!("Total rules in gitleaks config: {}", total_rules);
984
985        // Test no keywords - should skip most rules
986        let no_keyword_input = "export DATABASE_URL=localhost PORT=3000";
987        println!("\n--- No keywords - should skip all rules ---");
988        println!("Input: {}", no_keyword_input);
989
990        let no_keyword_rules = count_rules_that_would_process(no_keyword_input);
991        println!(
992            "Rules that would be processed: {} out of {}",
993            no_keyword_rules.len(),
994            total_rules
995        );
996        println!("  Rules: {:?}", no_keyword_rules);
997
998        let no_keyword_secrets = detect_secrets(no_keyword_input, None, false);
999        println!(
1000            "Secrets detected: {} (expected: 0)",
1001            no_keyword_secrets.len()
1002        );
1003        assert_eq!(no_keyword_secrets.len(), 0, "Should not detect any secrets");
1004        println!("✅ Test passed");
1005
1006        // Test API keyword - should process generic-api-key rule
1007        let api_input = format!("export API_KEY={}", fake_api_key());
1008        println!("\n--- API keyword - should process generic-api-key rule ---");
1009        println!("Input: {}", api_input);
1010
1011        let api_rules = count_rules_that_would_process(&api_input);
1012        println!(
1013            "Rules that would be processed: {} out of {}",
1014            api_rules.len(),
1015            total_rules
1016        );
1017        println!("  Rules: {:?}", api_rules);
1018
1019        let api_secrets = detect_secrets(&api_input, None, false);
1020        println!("Secrets detected: {} (expected: 1)", api_secrets.len());
1021        assert!(!api_secrets.is_empty(), "Should detect at least 1 secrets");
1022        println!("✅ Test passed");
1023
1024        // Test AWS keyword - should process aws-access-token rule
1025        // Use a realistic AWS key that matches the pattern [A-Z2-7]{16}
1026        let aws_input = format!("AWS_ACCESS_KEY_ID={}", fake_aws_access_key_alt());
1027        println!("\n--- AWS keyword - should process aws-access-token rule ---");
1028        println!("Input: {}", aws_input);
1029
1030        let aws_rules = count_rules_that_would_process(&aws_input);
1031        println!(
1032            "Rules that would be processed: {} out of {}",
1033            aws_rules.len(),
1034            total_rules
1035        );
1036        println!("  Rules: {:?}", aws_rules);
1037
1038        let aws_secrets = detect_secrets(&aws_input, None, false);
1039        println!("Secrets detected: {} (expected: 1)", aws_secrets.len());
1040
1041        // Should detect AWS key
1042        assert!(!aws_secrets.is_empty(), "Should detect at least 1 secrets");
1043        println!("✅ Test passed");
1044    }
1045
1046    #[test]
1047    fn test_debug_missing_secrets() {
1048        println!("=== DEBUGGING MISSING SECRETS ===");
1049
1050        let test_cases = vec![
1051            format!("SECRET_TOKEN={}", fake_secret_token()),
1052            format!("PASSWORD={}", fake_password_secret()),
1053        ];
1054
1055        for input in test_cases {
1056            println!("\nTesting: {}", input);
1057
1058            // Check entropy first
1059            let parts: Vec<&str> = input.split('=').collect();
1060            if parts.len() == 2 {
1061                let secret_value = parts[1];
1062                let entropy = calculate_entropy(secret_value);
1063                println!("  Secret value: '{}'", secret_value);
1064                println!("  Entropy: {:.2} (threshold: 3.5)", entropy);
1065
1066                if entropy >= 3.5 {
1067                    println!("  ✓ Entropy check PASSED");
1068                } else {
1069                    println!("  ✗ Entropy check FAILED - this is why it's not detected");
1070                }
1071            }
1072
1073            // Test the fallback regex directly
1074            if let Ok(regex) = create_simple_api_key_regex() {
1075                println!("  Testing fallback regex:");
1076                if regex.is_match(&input) {
1077                    println!("    ✓ Fallback regex MATCHES");
1078                    for mat in regex.find_iter(&input) {
1079                        println!("    Match: '{}'", mat.as_str());
1080                        if let Some(captures) = regex.captures(mat.as_str()) {
1081                            for (i, cap) in captures.iter().enumerate() {
1082                                if let Some(cap) = cap {
1083                                    println!("      Capture {}: '{}'", i, cap.as_str());
1084                                }
1085                            }
1086                        }
1087
1088                        // Test allowlist checking
1089                        let config = &*GITLEAKS_CONFIG;
1090                        let generic_rule = config
1091                            .rules
1092                            .iter()
1093                            .find(|r| r.id == "generic-api-key")
1094                            .unwrap();
1095                        let allowed = should_allow_match(
1096                            &input,
1097                            None,
1098                            mat.as_str(),
1099                            mat.start(),
1100                            mat.end(),
1101                            generic_rule,
1102                            &config.allowlist,
1103                        );
1104                        println!("      Allowed by allowlist: {}", allowed);
1105                        if allowed {
1106                            println!(
1107                                "      ✗ FILTERED OUT by allowlist - this is why it's not detected"
1108                            );
1109                        }
1110                    }
1111                } else {
1112                    println!("    ✗ Fallback regex does NOT match");
1113                }
1114            }
1115
1116            // Test full detection
1117            let result = redact_secrets(&input, None, &HashMap::new(), false);
1118            println!(
1119                "  Full detection result: {} secrets",
1120                result.redaction_map.len()
1121            );
1122        }
1123    }
1124
1125    #[test]
1126    fn test_debug_allowlist_filtering() {
1127        println!("=== DEBUGGING ALLOWLIST FILTERING ===");
1128
1129        let test_cases = vec![
1130            format!("SECRET_TOKEN={}", fake_secret_token()),
1131            format!("PASSWORD={}", fake_password_secret()),
1132        ];
1133
1134        let config = &*GITLEAKS_CONFIG;
1135        let generic_rule = config
1136            .rules
1137            .iter()
1138            .find(|r| r.id == "generic-api-key")
1139            .unwrap();
1140
1141        for input in test_cases {
1142            println!("\nAnalyzing: {}", input);
1143
1144            if let Ok(regex) = create_simple_api_key_regex() {
1145                for mat in regex.find_iter(&input) {
1146                    let match_text = mat.as_str();
1147                    println!("  Match: '{}'", match_text);
1148
1149                    // Test global allowlist
1150                    if let Some(global_allowlist) = &config.allowlist {
1151                        println!("  Checking global allowlist:");
1152
1153                        // Test global regex patterns
1154                        if let Some(regexes) = &global_allowlist.regexes {
1155                            for (i, pattern) in regexes.iter().enumerate() {
1156                                if let Ok(regex) = Regex::new(pattern)
1157                                    && regex.is_match(match_text)
1158                                {
1159                                    println!("    ✗ FILTERED by global regex {}: '{}'", i, pattern);
1160                                }
1161                            }
1162                        }
1163
1164                        // Test global stopwords
1165                        if let Some(stopwords) = &global_allowlist.stopwords {
1166                            for stopword in stopwords {
1167                                if match_text.to_lowercase().contains(&stopword.to_lowercase()) {
1168                                    println!("    ✗ FILTERED by global stopword: '{}'", stopword);
1169                                }
1170                            }
1171                        }
1172                    }
1173
1174                    // Test rule-specific allowlists
1175                    if let Some(rule_allowlists) = &generic_rule.allowlists {
1176                        for (rule_idx, allowlist) in rule_allowlists.iter().enumerate() {
1177                            println!("  Checking rule allowlist {}:", rule_idx);
1178
1179                            // Test rule regex patterns
1180                            if let Some(regexes) = &allowlist.regexes {
1181                                for (i, pattern) in regexes.iter().enumerate() {
1182                                    if let Ok(regex) = Regex::new(pattern)
1183                                        && regex.is_match(match_text)
1184                                    {
1185                                        println!(
1186                                            "    ✗ FILTERED by rule regex {}: '{}'",
1187                                            i, pattern
1188                                        );
1189                                    }
1190                                }
1191                            }
1192
1193                            // Test rule stopwords
1194                            if let Some(stopwords) = &allowlist.stopwords {
1195                                for stopword in stopwords {
1196                                    if match_text.to_lowercase().contains(&stopword.to_lowercase())
1197                                    {
1198                                        println!("    ✗ FILTERED by rule stopword: '{}'", stopword);
1199                                    }
1200                                }
1201                            }
1202                        }
1203                    }
1204                }
1205            }
1206        }
1207    }
1208
1209    #[test]
1210    fn test_debug_new_allowlist_logic() {
1211        println!("=== DEBUGGING NEW ALLOWLIST LOGIC ===");
1212
1213        let test_cases = vec![
1214            format!("SECRET_TOKEN={}", fake_secret_token()),
1215            format!("PASSWORD={}", fake_password_secret()),
1216            "PASSWORD=password123".to_string(), // Should be filtered
1217            "API_KEY=example_key".to_string(),  // Should be filtered
1218        ];
1219
1220        let config = &*GITLEAKS_CONFIG;
1221        let generic_rule = config
1222            .rules
1223            .iter()
1224            .find(|r| r.id == "generic-api-key")
1225            .unwrap();
1226
1227        for input in test_cases {
1228            println!("\nTesting: {}", input);
1229
1230            if let Ok(regex) = create_simple_api_key_regex() {
1231                for mat in regex.find_iter(&input) {
1232                    let match_text = mat.as_str();
1233                    println!("  Match: '{}'", match_text);
1234
1235                    // Parse the KEY=VALUE
1236                    if let Some((_, value)) = match_text.split_once('=') {
1237                        println!("    Value: '{}'", value);
1238
1239                        // Test specific stopwords
1240                        let test_stopwords = ["token", "password", "super", "word"];
1241                        for stopword in test_stopwords {
1242                            let value_lower = value.to_lowercase();
1243                            let stopword_lower = stopword.to_lowercase();
1244
1245                            if value_lower == stopword_lower {
1246                                println!("    '{}' - Exact match: YES", stopword);
1247                            } else if value.len() < 15 && value_lower.contains(&stopword_lower) {
1248                                let without_stopword = value_lower.replace(&stopword_lower, "");
1249                                let is_simple = without_stopword.chars().all(|c| {
1250                                    c.is_ascii_digit() || "!@#$%^&*()_+-=[]{}|;:,.<>?".contains(c)
1251                                });
1252                                println!(
1253                                    "    '{}' - Short+contains: len={}, without='{}', simple={}",
1254                                    stopword,
1255                                    value.len(),
1256                                    without_stopword,
1257                                    is_simple
1258                                );
1259                            } else {
1260                                println!("    '{}' - No filter", stopword);
1261                            }
1262                        }
1263                    }
1264
1265                    // Test the actual allowlist
1266                    if let Some(rule_allowlists) = &generic_rule.allowlists {
1267                        for (rule_idx, allowlist) in rule_allowlists.iter().enumerate() {
1268                            let allowed = is_allowed_by_rule_allowlist(
1269                                &input,
1270                                None,
1271                                match_text,
1272                                mat.start(),
1273                                mat.end(),
1274                                allowlist,
1275                            );
1276                            println!("  Rule allowlist {}: allowed = {}", rule_idx, allowed);
1277                        }
1278                    }
1279                }
1280            }
1281        }
1282    }
1283
1284    #[test]
1285    fn test_redact_password_basic() {
1286        let content = "User password is supersecret123 and should be hidden";
1287        let password = "supersecret123";
1288        let result = redact_password(content, password, &HashMap::new());
1289
1290        // Should redact the password
1291        assert!(!result.redacted_string.contains(password));
1292        assert!(
1293            result
1294                .redacted_string
1295                .contains("[REDACTED_SECRET:password:")
1296        );
1297        assert_eq!(result.redaction_map.len(), 1);
1298
1299        // The redaction map should contain our password
1300        let redacted_password = result.redaction_map.values().next().unwrap();
1301        assert_eq!(redacted_password, password);
1302    }
1303
1304    #[test]
1305    fn test_redact_password_empty() {
1306        let content = "Some content without password";
1307        let password = "";
1308        let result = redact_password(content, password, &HashMap::new());
1309
1310        // Should not change anything
1311        assert_eq!(result.redacted_string, content);
1312        assert!(result.redaction_map.is_empty());
1313    }
1314
1315    #[test]
1316    fn test_redact_password_multiple_occurrences() {
1317        let content = "Password is mypass123 and again mypass123 appears here";
1318        let password = "mypass123";
1319        let result = redact_password(content, password, &HashMap::new());
1320
1321        // Should redact both occurrences with the same key
1322        assert!(!result.redacted_string.contains(password));
1323        assert_eq!(result.redaction_map.len(), 1);
1324
1325        // Count redaction keys in the result
1326        let redaction_key = result.redaction_map.keys().next().unwrap();
1327        let count = result.redacted_string.matches(redaction_key).count();
1328        assert_eq!(count, 2);
1329    }
1330
1331    #[test]
1332    fn test_redact_password_reuse_existing_key() {
1333        // Start with an existing redaction map
1334        let mut existing_map = HashMap::new();
1335        existing_map.insert(
1336            "[REDACTED_SECRET:password:abc123]".to_string(),
1337            "mypassword".to_string(),
1338        );
1339
1340        let content = "The password mypassword should use existing key";
1341        let password = "mypassword";
1342        let result = redact_password(content, password, &existing_map);
1343
1344        // Should reuse the existing key
1345        assert_eq!(result.redaction_map.len(), 1);
1346        assert!(
1347            result
1348                .redaction_map
1349                .contains_key("[REDACTED_SECRET:password:abc123]")
1350        );
1351        assert!(
1352            result
1353                .redacted_string
1354                .contains("[REDACTED_SECRET:password:abc123]")
1355        );
1356    }
1357
1358    #[test]
1359    fn test_redact_password_with_existing_different_secrets() {
1360        // Start with an existing redaction map containing different secrets
1361        let mut existing_map = HashMap::new();
1362        existing_map.insert(
1363            "[REDACTED_SECRET:api-key:xyz789]".to_string(),
1364            "some_api_key".to_string(),
1365        );
1366
1367        let content = "API key is some_api_key and password is newpassword123";
1368        let password = "newpassword123";
1369        let result = redact_password(content, password, &existing_map);
1370
1371        // Should preserve existing mapping and add new one
1372        assert_eq!(result.redaction_map.len(), 2);
1373        assert!(
1374            result
1375                .redaction_map
1376                .contains_key("[REDACTED_SECRET:api-key:xyz789]")
1377        );
1378        assert!(
1379            result
1380                .redaction_map
1381                .get("[REDACTED_SECRET:api-key:xyz789]")
1382                .unwrap()
1383                == "some_api_key"
1384        );
1385
1386        // Should add new password mapping
1387        let new_keys: Vec<_> = result
1388            .redaction_map
1389            .keys()
1390            .filter(|k| k.contains("password"))
1391            .collect();
1392        assert_eq!(new_keys.len(), 1);
1393        let password_key = new_keys[0];
1394        assert_eq!(
1395            result.redaction_map.get(password_key).unwrap(),
1396            "newpassword123"
1397        );
1398    }
1399
1400    #[test]
1401    fn test_redact_password_no_match() {
1402        let content = "This content has no matching password";
1403        let password = "notfound";
1404        let result = redact_password(content, password, &HashMap::new());
1405
1406        // Should still create a redaction key but content unchanged
1407        assert_eq!(result.redacted_string, content);
1408        assert_eq!(result.redaction_map.len(), 1);
1409        assert_eq!(result.redaction_map.values().next().unwrap(), "notfound");
1410    }
1411
1412    #[test]
1413    fn test_redact_password_integration_with_restore() {
1414        let content = "Login with username admin and password secret456";
1415        let password = "secret456";
1416        let result = redact_password(content, password, &HashMap::new());
1417
1418        // Redact the password
1419        assert!(!result.redacted_string.contains(password));
1420        assert!(result.redacted_string.contains("username admin"));
1421
1422        // Restore should bring back the original
1423        let restored = restore_secrets(&result.redacted_string, &result.redaction_map);
1424        assert_eq!(restored, content);
1425    }
1426
1427    #[test]
1428    fn test_redact_secrets_with_existing_redaction_map() {
1429        // Test that secrets in the existing redaction map get redacted even if not detected by detect_secrets
1430        let content = "The secret value is mysecretvalue123 and another is anothersecret456";
1431
1432        // First, test with empty map to prove the secret wouldn't normally be redacted
1433        let result_empty = redact_secrets(content, None, &HashMap::new(), false);
1434
1435        // Verify that mysecretvalue123 is NOT redacted when using empty map
1436        assert!(result_empty.redacted_string.contains("mysecretvalue123"));
1437        // Now create an existing redaction map with one of the secrets
1438        let mut existing_redaction_map = HashMap::new();
1439        existing_redaction_map.insert(
1440            "[REDACTED_SECRET:manual:abc123]".to_string(),
1441            "mysecretvalue123".to_string(),
1442        );
1443
1444        let result = redact_secrets(content, None, &existing_redaction_map, false);
1445
1446        // The secret from the existing map should be redacted
1447        assert!(
1448            result
1449                .redacted_string
1450                .contains("[REDACTED_SECRET:manual:abc123]")
1451        );
1452        assert!(!result.redacted_string.contains("mysecretvalue123"));
1453
1454        // The redaction map should contain the existing mapping
1455        assert!(
1456            result
1457                .redaction_map
1458                .contains_key("[REDACTED_SECRET:manual:abc123]")
1459        );
1460        assert_eq!(
1461            result
1462                .redaction_map
1463                .get("[REDACTED_SECRET:manual:abc123]")
1464                .unwrap(),
1465            "mysecretvalue123"
1466        );
1467    }
1468
1469    #[test]
1470    fn test_redact_secrets_skip_already_redacted() {
1471        // Content that already contains redacted secrets should not be double-redacted
1472        let content = "The password is [REDACTED_SECRET:password:abc123] and API key is [REDACTED_SECRET:api-key:xyz789]";
1473        let result = redact_secrets(content, None, &HashMap::new(), false);
1474
1475        // Should return content unchanged
1476        assert_eq!(result.redacted_string, content);
1477        // Should not add any new redactions
1478        assert!(result.redaction_map.is_empty());
1479    }
1480
1481    #[test]
1482    fn test_redact_password_skip_already_redacted() {
1483        // Content that already contains redacted secrets should not be double-redacted
1484        let content = "[REDACTED_SECRET:password:existing123]";
1485        let password = "newpassword";
1486        let result = redact_password(content, password, &HashMap::new());
1487
1488        // Should return content unchanged
1489        assert_eq!(result.redacted_string, content);
1490        // Should not add any new redactions
1491        assert!(result.redaction_map.is_empty());
1492    }
1493
1494    #[test]
1495    fn test_redact_secrets_skip_nested_redaction() {
1496        // Simulate what happens when local_tools redacts and proxy tries to redact again
1497        let original_password = "MySecureP@ssw0rd!";
1498
1499        // First redaction (simulating local_tools)
1500        let first_result = redact_password(original_password, original_password, &HashMap::new());
1501        assert!(
1502            first_result
1503                .redacted_string
1504                .contains("[REDACTED_SECRET:password:")
1505        );
1506
1507        // Second redaction attempt (simulating proxy) - should be skipped
1508        let second_result =
1509            redact_secrets(&first_result.redacted_string, None, &HashMap::new(), false);
1510
1511        // Should return the already-redacted content unchanged
1512        assert_eq!(second_result.redacted_string, first_result.redacted_string);
1513        assert!(second_result.redaction_map.is_empty());
1514    }
1515
1516    #[test]
1517    fn test_huawei_cloud_credentials_detection() {
1518        // Test Huawei Cloud credentials in CSV format
1519        // Using obviously fake test values (TESTHUAWEI prefix) to avoid GitHub push protection
1520        let csv_content = r#"User Name,Access Key Id,Secret Access Key
1521terraform,TESTHUAWEIKEY1234567,TestHuaweiSecretKey1234567890abcdefghij"#;
1522
1523        let result = redact_secrets(csv_content, None, &HashMap::new(), false);
1524
1525        println!("Input: {}", csv_content);
1526        println!("Redacted: {}", result.redacted_string);
1527        println!("Mapping: {:?}", result.redaction_map);
1528
1529        // Should detect both AK and SK
1530        assert!(
1531            !result.redaction_map.is_empty(),
1532            "Should detect Huawei credentials"
1533        );
1534
1535        // Verify AK is redacted (20 char uppercase alphanumeric)
1536        assert!(
1537            !result.redacted_string.contains("TESTHUAWEIKEY1234567"),
1538            "AK should be redacted"
1539        );
1540
1541        // Verify SK is redacted (40 char alphanumeric)
1542        assert!(
1543            !result
1544                .redacted_string
1545                .contains("TestHuaweiSecretKey1234567890abcdefghij"),
1546            "SK should be redacted"
1547        );
1548
1549        // Verify redaction keys are present
1550        assert!(
1551            result.redacted_string.contains("[REDACTED_SECRET:huawei-"),
1552            "Should contain Huawei redaction markers"
1553        );
1554    }
1555
1556    #[test]
1557    fn test_huawei_access_key_id_pattern() {
1558        // Test AK detection with "Access Key Id" keyword
1559        // Using obviously fake test value to avoid GitHub push protection
1560        // Must be exactly 20 chars to match the regex pattern
1561        let input = "Access Key Id: TESTHWCLOUD123456789";
1562        let result = redact_secrets(input, None, &HashMap::new(), false);
1563
1564        println!("Input: {}", input);
1565        println!("Redacted: {}", result.redacted_string);
1566
1567        assert!(
1568            !result.redaction_map.is_empty(),
1569            "Should detect Huawei AK with 'Access Key Id' keyword"
1570        );
1571        assert!(
1572            !result.redacted_string.contains("TESTHWCLOUD123456789"),
1573            "AK should be redacted"
1574        );
1575    }
1576
1577    #[test]
1578    fn test_huawei_secret_access_key_pattern() {
1579        // Test SK detection with "Secret Access Key" keyword
1580        // Using obviously fake test value to avoid GitHub push protection
1581        let input = "Secret Access Key: TestHwCloudSecretKey12345678901234567890";
1582        let result = redact_secrets(input, None, &HashMap::new(), false);
1583
1584        println!("Input: {}", input);
1585        println!("Redacted: {}", result.redacted_string);
1586
1587        assert!(
1588            !result.redaction_map.is_empty(),
1589            "Should detect Huawei SK with 'Secret Access Key' keyword"
1590        );
1591        assert!(
1592            !result
1593                .redacted_string
1594                .contains("TestHwCloudSecretKey12345678901234567890"),
1595            "SK should be redacted"
1596        );
1597    }
1598}
stakpak_shared/secrets/mod.rs

stakpak_shared/secrets/
mod.rs
