use regex::Regex;
use std::sync::Arc;
pub(crate) const LARGE_FALLBACK_SCAN_THRESHOLD: usize = 10_000;
pub(crate) const MAX_WINDOW_DEDUP_ENTRIES: usize = 100_000;
pub(crate) const MAX_SCAN_CHUNK_BYTES: usize = 1024 * 1024;
pub(crate) const WINDOW_OVERLAP_BYTES: usize = 128 * 1024;
pub(crate) const FIRST_CAPTURE_GROUP_INDEX: usize = 1;
pub(crate) const FIRST_LINE_NUMBER: usize = 1;
pub(crate) const PREVIOUS_LINE_DISTANCE: usize = 1;
pub(crate) const MIN_LITERAL_PREFIX_CHARS: usize = 3;
pub(crate) const REGEX_SIZE_LIMIT_BYTES: usize = 1 << 20;
static REGEX_DFA_LIMIT_OVERRIDE: std::sync::atomic::AtomicUsize =
std::sync::atomic::AtomicUsize::new(0);
static LAZY_REGEX_COMPILE_EVENTS: std::sync::atomic::AtomicU64 =
std::sync::atomic::AtomicU64::new(0);
pub(crate) fn lazy_regex_compile_events() -> u64 {
LAZY_REGEX_COMPILE_EVENTS.load(std::sync::atomic::Ordering::Relaxed)
}
pub fn set_regex_dfa_limit(bytes: usize) {
REGEX_DFA_LIMIT_OVERRIDE.store(bytes, std::sync::atomic::Ordering::Relaxed);
}
#[must_use]
pub fn regex_dfa_limit_default() -> usize {
REGEX_SIZE_LIMIT_BYTES
}
#[must_use]
pub(crate) fn regex_dfa_limit() -> usize {
match REGEX_DFA_LIMIT_OVERRIDE.load(std::sync::atomic::Ordering::Relaxed) {
0 => REGEX_SIZE_LIMIT_BYTES,
n => n,
}
}
pub(crate) const HEX_CONTEXT_RADIUS_CHARS: usize = 20;
pub(crate) const MIN_HEX_MATCH_LEN: usize = 16;
pub(crate) const MIN_HEX_DIGITS_IN_MATCH: usize = 16;
pub(crate) const MIN_HEX_CONTEXT_DIGITS: usize = 8;
pub(crate) const MAX_HEX_CONTEXT_SEPARATORS: usize = 4;
#[derive(Debug, Clone)]
pub(crate) struct LineMapping {
pub(crate) start_offset: usize,
pub(crate) end_offset: usize,
pub(crate) line_number: usize,
pub(crate) original_start_offset: usize,
pub(crate) transport_decoded: bool,
}
#[cfg(not(feature = "multiline"))]
#[derive(Debug, Clone)]
pub(crate) struct PreprocessedText<'a> {
pub(crate) text: std::borrow::Cow<'a, str>,
pub(crate) mappings: Vec<LineMapping>,
}
#[cfg(not(feature = "multiline"))]
impl<'a> PreprocessedText<'a> {
pub(crate) fn line_for_offset(&self, offset: usize) -> Option<usize> {
let idx = self.mappings.partition_point(|m| m.start_offset <= offset);
if idx == 0 {
return None;
}
let m = &self.mappings[idx - 1];
if offset < m.end_offset {
Some(m.line_number)
} else {
None
}
}
pub(crate) fn source_offset_for_match(
&self,
source: &str,
offset: usize,
credential: &str,
) -> usize {
let idx = self.mappings.partition_point(|m| m.start_offset <= offset);
if idx == 0 {
return offset.min(source.len().saturating_sub(1));
}
let m = &self.mappings[idx - 1];
if offset >= m.end_offset {
return offset.min(source.len().saturating_sub(1));
}
source_offset_from_mapping(source, m, offset, credential)
}
pub(crate) fn transport_decoded_for_offset(&self, offset: usize) -> bool {
transport_decoded_for_offset(&self.mappings, offset)
}
pub(crate) fn passthrough(line: impl Into<std::borrow::Cow<'a, str>>) -> Self {
let line: std::borrow::Cow<'a, str> = line.into();
let end_offset = line.len();
Self {
text: line,
mappings: vec![LineMapping {
line_number: 1,
start_offset: 0,
end_offset,
original_start_offset: 0,
transport_decoded: false,
}],
}
}
}
pub(crate) fn transport_decoded_for_offset(mappings: &[LineMapping], offset: usize) -> bool {
let idx = mappings.partition_point(|mapping| mapping.start_offset <= offset);
idx.checked_sub(1)
.and_then(|index| mappings.get(index))
.is_some_and(|mapping| offset < mapping.end_offset && mapping.transport_decoded)
}
pub(crate) fn source_offset_from_mapping(
source: &str,
mapping: &LineMapping,
offset: usize,
credential: &str,
) -> usize {
if mapping.start_offset == mapping.original_start_offset && offset < source.len() {
return offset;
}
if let Some(line) = source_line_at(source, mapping.original_start_offset) {
if let Some(column) = line.find(credential) {
return mapping.original_start_offset + column;
}
}
let candidate = mapping
.original_start_offset
.saturating_add(offset.saturating_sub(mapping.start_offset));
if candidate < source.len() {
candidate
} else if mapping.original_start_offset < source.len() {
mapping.original_start_offset
} else {
source.len().saturating_sub(1)
}
}
pub(crate) fn source_line_at(source: &str, start: usize) -> Option<&str> {
if start >= source.len() {
return None;
}
let start = crate::engine::floor_char_boundary(source, start);
let rest = &source[start..];
let end = rest.find('\n').unwrap_or(rest.len()); let line = &rest[..end];
Some(line.strip_suffix('\r').unwrap_or(line)) }
#[cfg(feature = "multiline")]
pub(crate) type ScannerPreprocessedText<'a> = crate::multiline::PreprocessedText<'a>;
#[cfg(not(feature = "multiline"))]
pub(crate) type ScannerPreprocessedText<'a> = PreprocessedText<'a>;
#[derive(Debug, Clone)]
pub(crate) struct LazyRegex {
src: Arc<str>,
case_insensitive: bool,
cell: Arc<std::sync::OnceLock<Arc<Regex>>>,
has_literal_prefix: Arc<std::sync::OnceLock<bool>>,
has_distinctive_inner_literal: Arc<std::sync::OnceLock<bool>>,
}
impl LazyRegex {
#[cfg(test)]
pub(crate) fn detector(src: impl Into<Arc<str>>) -> Self {
Self {
src: src.into(),
case_insensitive: true,
cell: Arc::new(std::sync::OnceLock::new()),
has_literal_prefix: Arc::new(std::sync::OnceLock::new()),
has_distinctive_inner_literal: Arc::new(std::sync::OnceLock::new()),
}
}
pub(crate) fn detector_compiled(src: impl Into<Arc<str>>, compiled: Arc<Regex>) -> Self {
Self {
src: src.into(),
case_insensitive: true,
cell: Arc::new(std::sync::OnceLock::from(compiled)),
has_literal_prefix: Arc::new(std::sync::OnceLock::new()),
has_distinctive_inner_literal: Arc::new(std::sync::OnceLock::new()),
}
}
#[cfg(test)]
pub(crate) fn plain(src: impl Into<Arc<str>>) -> Self {
Self {
src: src.into(),
case_insensitive: false,
cell: Arc::new(std::sync::OnceLock::new()),
has_literal_prefix: Arc::new(std::sync::OnceLock::new()),
has_distinctive_inner_literal: Arc::new(std::sync::OnceLock::new()),
}
}
pub(crate) fn plain_compiled(src: impl Into<Arc<str>>, compiled: Arc<Regex>) -> Self {
Self {
src: src.into(),
case_insensitive: false,
cell: Arc::new(std::sync::OnceLock::from(compiled)),
has_literal_prefix: Arc::new(std::sync::OnceLock::new()),
has_distinctive_inner_literal: Arc::new(std::sync::OnceLock::new()),
}
}
pub(crate) fn as_str(&self) -> &str {
&self.src
}
#[must_use]
pub(crate) fn has_literal_prefix(&self) -> bool {
*self.has_literal_prefix.get_or_init(|| {
!crate::compiler::compiler_prefix::extract_literal_prefixes(&self.src).is_empty()
})
}
#[must_use]
pub(crate) fn has_distinctive_inner_literal(&self) -> bool {
*self.has_distinctive_inner_literal.get_or_init(|| {
crate::compiler::compiler_prefix::regex_has_required_literal_run(
&self.src,
crate::compiler::compiler_prefix::MIN_DISTINCTIVE_INFIX_CHARS,
)
})
}
pub(crate) fn is_case_insensitive(&self) -> bool {
self.case_insensitive
}
pub(crate) fn get(&self) -> &Regex {
self.cell
.get_or_init(|| {
LAZY_REGEX_COMPILE_EVENTS.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let built = if self.case_insensitive {
crate::compiler::compiler_compile::shared_regex(&self.src)
} else {
Regex::new(&self.src).map(Arc::new)
};
match built {
Ok(rx) => rx,
Err(error) => {
crate::prefilter_degrade::warn_prefilter_disabled(
&format!("detector regex first-use compile ({})", self.src),
&error,
);
never_match_sentinel()
}
}
})
.as_ref()
}
}
fn never_match_sentinel() -> Arc<Regex> {
static SENTINEL: std::sync::OnceLock<Arc<Regex>> = std::sync::OnceLock::new();
SENTINEL
.get_or_init(|| match Regex::new(r"\b\B") {
Ok(re) => Arc::new(re),
Err(error) => panic!("`\\b\\B` is a constant valid regex but failed to build: {error}"),
})
.clone()
}
#[derive(Debug, Clone)]
pub(crate) struct CompiledPattern {
pub detector_index: usize,
pub regex: LazyRegex,
pub group: Option<usize>,
pub client_safe: bool,
pub weak_anchor: bool,
pub structural_password_slot: bool,
pub match_proves_keyword_nearby: bool,
pub homoglyph_variant: bool,
}
impl CompiledPattern {
pub(crate) fn captures_exact_slot(&self, line: &str, start: usize, end: usize) -> bool {
self.regex.get().captures_iter(line).any(|captures| {
self.group
.and_then(|group| captures.get(group))
.is_some_and(|slot| slot.start() == start && slot.end() == end)
})
}
}
#[derive(Debug)]
pub(crate) struct CompiledCompanion {
pub(crate) name: Arc<str>,
pub(crate) regex: Regex,
pub(crate) capture_group: Option<usize>,
pub(crate) within_lines: usize,
pub(crate) required: bool,
}
#[cfg(feature = "entropy")]
pub(crate) use crate::scan_state::RawMatchPriority;
pub(crate) use crate::scan_state::ScanState;
pub use crate::scanner_config::{ScanExecutionRoute, ScannerConfig, ScannerTuningConfig};
#[cfg(feature = "ml")]
pub(crate) use crate::scan_state::ml_features_for_candidate;
#[cfg(feature = "ml")]
pub(crate) use crate::scan_state::MlPendingMatch;