dd-sensitive-data-scanner 0.0.0

use crate::proximity_keywords::{
    CompiledExcludedProximityKeywords, CompiledIncludedProximityKeywords, contains_keyword_in_path,
    get_prefix_start, is_index_within_prefix,
};
use crate::scanner::metrics::RuleMetrics;
use crate::scanner::regex_rule::regex_cache_store::RegexCacheValue;
use crate::scanner::regex_rule::regex_store::SharedRegex;
use crate::scanner::{
    RuleResult, RuleStatus, StringMatchesCtx, get_next_regex_start, is_false_positive_match,
};
use crate::secondary_validation::Validator;
use crate::{CompiledRule, ExclusionCheck, Labels, Path, StringMatch};
use ahash::AHashMap;
use metrics::counter;
use regex_automata::Input;
use regex_automata::meta::Cache;
use regex_automata::util::captures::Captures;
use std::sync::Arc;

/// This is the internal representation of a rule after it has been validated / compiled.
#[derive(Clone)]
pub struct RegexCompiledRule {
    pub rule_index: usize,
    pub regex: SharedRegex,
    pub included_keywords: Option<CompiledIncludedProximityKeywords>,
    pub excluded_keywords: Option<CompiledExcludedProximityKeywords>,
    pub validator: Option<Arc<dyn Validator>>,
    pub metrics: RuleMetrics,
    pub pattern_capture_groups: Option<Vec<String>>,
}

impl CompiledRule for RegexCompiledRule {
    fn allow_scanner_to_exclude_namespace(&self) -> bool {
        false
    }

    fn get_string_matches(
        &self,
        content: &str,
        path: &Path,
        ctx: &mut StringMatchesCtx,
    ) -> RuleResult {
        match self.included_keywords {
            Some(ref included_keywords) => {
                self.get_string_matches_with_included_keywords(
                    content,
                    path,
                    ctx,
                    included_keywords,
                );
            }
            None => {
                let enable_debug = ctx.enable_debug_observability;
                let cache_value = ctx.regex_caches.get(&self.regex);
                let true_positive_search = self.true_positive_matches(
                    content,
                    0,
                    cache_value,
                    true,
                    ctx.exclusion_check,
                    ctx.excluded_matches,
                    path,
                    enable_debug,
                );
                for string_match in true_positive_search {
                    ctx.match_emitter.emit(string_match);
                }
            }
        }
        Ok(RuleStatus::Done)
    }

    fn should_exclude_multipass_v0(&self) -> bool {
        true
    }

    fn on_excluded_match_multipass_v0(
        &self,
        path: &Path,
        excluded_path: &str,
        enable_debug_observability: bool,
    ) {
        if enable_debug_observability {
            let labels = self.metrics.base_labels.clone_with_labels(Labels::new(&[
                ("sds_namespace", path.to_string()),
                ("sds_excluded_namespace", excluded_path.to_string()),
            ]));
            counter!("false_positive.multipass.excluded_match", labels).increment(1);
        } else {
            self.metrics.false_positive_excluded_attributes.increment(1);
        }
    }

    fn as_regex_rule(&self) -> Option<&RegexCompiledRule> {
        Some(self)
    }

    fn as_regex_rule_mut(&mut self) -> Option<&mut RegexCompiledRule> {
        Some(self)
    }
}

impl RegexCompiledRule {
    #[allow(clippy::too_many_arguments)]
    fn get_string_matches_with_included_keywords(
        &self,
        content: &str,
        path: &Path,
        ctx: &mut StringMatchesCtx,
        included_keywords: &CompiledIncludedProximityKeywords,
    ) {
        let enable_debug = ctx.enable_debug_observability;
        let mut included_keyword_matches = included_keywords.keyword_matches(content);

        'included_keyword_search: while let Some(included_keyword_match) =
            included_keyword_matches.next(ctx.regex_caches)
        {
            let cache_value = ctx.regex_caches.get(&self.regex);
            let true_positive_search = self.true_positive_matches(
                content,
                included_keyword_match.end,
                cache_value,
                false,
                ctx.exclusion_check,
                ctx.excluded_matches,
                path,
                enable_debug,
            );

            for true_positive_match in true_positive_search {
                if is_index_within_prefix(
                    content,
                    included_keyword_match.start,
                    true_positive_match.start,
                    included_keywords.look_ahead_character_count,
                ) {
                    // The match start might be further than the current start, so some chars
                    // can be skipped before the next included keyword scanning. The start
                    // is used instead of the end since the included keyword can overlap with
                    // a previous match (maybe this can be removed in the future?)
                    included_keyword_matches.skip_to(true_positive_match.start);
                    ctx.match_emitter.emit(true_positive_match);

                    // Continue search since another true positive could potentially be found within the same prefix
                } else {
                    // This match is ignored since it is not within the prefix, but
                    // the start of the match may be far ahead of the current start, so
                    // use it to reduce future scanning
                    let new_start = get_prefix_start(
                        true_positive_match.start,
                        included_keywords.look_ahead_character_count,
                        content,
                    )
                    .start;
                    included_keyword_matches.skip_to(new_start);
                    // Switch back to included keyword search, since we are past the prefix
                    continue 'included_keyword_search;
                }
            }
            // no more "true positive" matches were found in the entire string, so there's no need
            // to continue scanning for included keywords.
            break;
        }

        let mut has_verified_kws_in_path: Option<bool> = None;

        {
            let input = Input::new(content);
            let cache_value = ctx.regex_caches.get(&self.regex);
            if self
                .regex
                .search_half_with(&mut cache_value.cache, &input)
                .is_some()
            {
                has_verified_kws_in_path = Some(contains_keyword_in_path(
                    &path.sanitize(),
                    &included_keywords.keywords_pattern,
                ))
            }
        };

        if has_verified_kws_in_path.is_none() || has_verified_kws_in_path.is_some_and(|x| !x) {
            // We don't deal with true positives is in this case, because keywords don't match the path.
            // Return early.
            return;
        }

        let cache_value = ctx.regex_caches.get(&self.regex);

        let true_positive_search = self.true_positive_matches(
            content,
            0,
            cache_value,
            false,
            ctx.exclusion_check,
            ctx.excluded_matches,
            path,
            enable_debug,
        );

        for string_match in true_positive_search {
            ctx.match_emitter.emit(string_match);
        }
    }

    #[allow(clippy::too_many_arguments)]
    fn true_positive_matches<'a>(
        &'a self,
        content: &'a str,
        start: usize,
        cache: &'a mut RegexCacheValue,
        check_excluded_keywords: bool,
        exclusion_check: &'a ExclusionCheck<'a>,
        excluded_matches: &'a mut AHashMap<String, String>,
        scan_path: &'a Path<'a>,
        enable_debug_observability: bool,
    ) -> TruePositiveSearch<'a> {
        TruePositiveSearch {
            rule: self,
            content,
            start,
            cache: &mut cache.cache,
            captures: &mut cache.captures,
            check_excluded_keywords,
            exclusion_check,
            excluded_matches,
            scan_path,
            enable_debug_observability,
        }
    }
}

pub struct TruePositiveSearch<'a> {
    rule: &'a RegexCompiledRule,
    content: &'a str,
    start: usize,
    cache: &'a mut Cache,
    check_excluded_keywords: bool,
    exclusion_check: &'a ExclusionCheck<'a>,
    excluded_matches: &'a mut AHashMap<String, String>,
    captures: &'a mut Captures,
    scan_path: &'a Path<'a>,
    enable_debug_observability: bool,
}

impl TruePositiveSearch<'_> {
    fn perform_regex_scan(&mut self, input: &Input) -> Option<(usize, usize)> {
        match &self.rule.pattern_capture_groups {
            Some(capture_group) => {
                self.captures.clear();
                self.rule
                    .regex
                    .search_captures_with(self.cache, input, self.captures);
                self.captures
                    .get_group_by_name(
                        capture_group
                            .first()
                            .expect("pattern_capture_group is empty"),
                    )
                    .map(|span| (span.start, span.end))
            }
            None => self
                .rule
                .regex
                .search_with(self.cache, input)
                .map(|re_match| (re_match.start(), re_match.end())),
        }
    }
}

impl Iterator for TruePositiveSearch<'_> {
    type Item = StringMatch;

    fn next(&mut self) -> Option<Self::Item> {
        loop {
            if self.start > self.content.len() {
                return None;
            }
            let input = Input::new(self.content).range(self.start..);
            self.rule.regex.search_half_with(self.cache, &input)?;

            let regex_match_range = self.perform_regex_scan(&input)?;
            // this is only checking extra validators (e.g. checksums)
            let is_false_positive_match = is_false_positive_match(
                regex_match_range,
                self.rule,
                self.content,
                self.check_excluded_keywords,
            );

            if is_false_positive_match {
                if let Some(next) = get_next_regex_start(self.content, regex_match_range) {
                    self.start = next;
                } else {
                    // There are no more chars to scan
                    return None;
                }
            } else {
                // The next match will start at the end of this match. This is fine because
                // patterns that can match empty matches are rejected.
                self.start = regex_match_range.1;

                if self.exclusion_check.is_excluded(self.rule.rule_index) {
                    // Matches from excluded paths are saved and used to treat additional equal matches as false positives.
                    // Matches are checked against this `excluded_matches` set after all scanning has been done.
                    let match_str = &self.content[regex_match_range.0..regex_match_range.1];
                    if !self.excluded_matches.contains_key(match_str) {
                        let path = if self.enable_debug_observability {
                            self.scan_path.to_string()
                        } else {
                            String::new()
                        };
                        self.excluded_matches.insert(match_str.to_string(), path);
                    }
                } else {
                    return Some(StringMatch {
                        start: regex_match_range.0,
                        end: regex_match_range.1,
                        keyword: None,
                    });
                }
            }
        }
    }
}