use anyhow::{anyhow, Context, Result};
use regex::{Regex, RegexBuilder};
use serde::Deserialize;
use std::sync::OnceLock;
use toml::Value;
const REGEX_SIZE_LIMIT: usize = 64 * 1024 * 1024;
const UPSTREAM_TRAILING_BOUNDARY: &str = r#"(?:[\x60'"\s;]|\\[nr]|$)"#;
const RELAXED_TRAILING_BOUNDARY: &str = r#"(?:[\x60'"\s;&,)\]}></?#]|\\[nr]|$)"#;
fn relax_trailing_boundary(pattern: &str) -> String {
pattern.replace(UPSTREAM_TRAILING_BOUNDARY, RELAXED_TRAILING_BOUNDARY)
}
fn compile_rule_regex(pattern: &str) -> Result<Regex, regex::Error> {
RegexBuilder::new(&relax_trailing_boundary(pattern))
.size_limit(REGEX_SIZE_LIMIT)
.build()
}
#[derive(Debug, Default, Deserialize)]
#[serde(rename = "allowlist")]
#[allow(dead_code)]
pub struct AllowlistRaw {
pub paths: Option<Vec<String>>,
pub regexes: Option<Vec<String>>,
pub stopwords: Option<Vec<String>>,
}
#[derive(Debug, Default, Deserialize, Clone)]
#[allow(dead_code)]
pub struct RuleAllowlistRaw {
pub paths: Option<Vec<String>>,
pub regexes: Option<Vec<String>>,
pub stopwords: Option<Vec<String>>,
#[serde(rename = "regexTarget")]
pub regex_target: Option<String>,
pub condition: Option<String>,
}
#[derive(Debug, Deserialize, Clone)]
#[allow(dead_code)]
pub struct RuleRaw {
pub id: String,
pub description: String,
pub regex: Option<String>,
pub entropy: Option<f64>,
pub keywords: Option<Vec<String>>,
#[serde(rename = "secretGroup")]
pub secret_group: Option<u32>,
pub path: Option<String>,
pub allowlists: Option<Vec<RuleAllowlistRaw>>,
}
#[derive(Debug, Deserialize)]
#[allow(dead_code)]
pub struct GitleaksConfigRaw {
pub title: Option<String>,
#[serde(rename = "minVersion")]
pub min_version: Option<String>,
pub allowlist: Option<AllowlistRaw>,
pub rules: Vec<RuleRaw>,
}
pub struct CompiledGlobalAllowlist {
pub regexes: Vec<Regex>,
pub stopwords: Vec<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AllowlistCondition {
Or,
And,
}
pub struct CompiledRuleAllowlist {
pub regexes: Vec<Regex>,
pub stopwords: Vec<String>,
pub regex_target: Option<String>,
pub condition_and: bool,
pub has_paths: bool,
}
pub struct CompiledRule {
pub id: String,
pub regex: Regex,
pub entropy: Option<f64>,
pub keywords: Option<Vec<String>>,
pub keyword_pattern_ids: Option<Vec<u32>>,
pub secret_group: Option<u32>,
pub allowlists: Vec<CompiledRuleAllowlist>,
pub path: Option<String>,
}
pub struct KeywordPrefilter {
automaton: aho_corasick::AhoCorasick,
}
impl KeywordPrefilter {
pub fn matched_pattern_ids(&self, body: &str) -> std::collections::HashSet<u32> {
let mut found = std::collections::HashSet::new();
for m in self.automaton.find_overlapping_iter(body) {
#[allow(clippy::cast_possible_truncation)]
let pid = m.pattern().as_u32();
found.insert(pid);
}
found
}
pub fn any_id_present(rule_ids: &[u32], matched: &std::collections::HashSet<u32>) -> bool {
rule_ids.iter().any(|id| matched.contains(id))
}
}
pub struct GitleaksCompiled {
pub global_allowlist: CompiledGlobalAllowlist,
pub rules: Vec<CompiledRule>,
pub keyword_prefilter: Option<KeywordPrefilter>,
}
fn compile_allowlist_regexes(regexes: Option<Vec<String>>) -> Vec<Regex> {
regexes
.unwrap_or_default()
.into_iter()
.filter_map(|s| Regex::new(&s).ok())
.collect()
}
fn compile_global_allowlist(raw: Option<AllowlistRaw>) -> CompiledGlobalAllowlist {
let raw = raw.unwrap_or_default();
CompiledGlobalAllowlist {
regexes: compile_allowlist_regexes(raw.regexes),
stopwords: raw.stopwords.unwrap_or_default(),
}
}
fn parse_condition(condition: Option<&str>) -> AllowlistCondition {
match condition.map(str::to_uppercase).as_deref() {
Some("AND" | "&&") => AllowlistCondition::And,
_ => AllowlistCondition::Or,
}
}
fn compile_rule_allowlist(raw: &RuleAllowlistRaw) -> CompiledRuleAllowlist {
let condition_and = parse_condition(raw.condition.as_deref()) == AllowlistCondition::And;
let has_paths = raw.paths.as_ref().is_some_and(|p| !p.is_empty());
CompiledRuleAllowlist {
regexes: compile_allowlist_regexes(raw.regexes.clone()),
stopwords: raw.stopwords.clone().unwrap_or_default(),
regex_target: raw.regex_target.clone(),
condition_and,
has_paths,
}
}
fn rules_from_table(t: &toml::Table) -> Vec<RuleRaw> {
let mut out: Vec<RuleRaw> = Vec::new();
let mut pending_allowlists: Vec<RuleAllowlistRaw> = Vec::new();
for (key, val) in t {
if key == "allowlists" {
if let Value::Array(arr) = val {
for (i, tab) in arr.iter().enumerate() {
if let Value::Table(allow_t) = tab {
if let Some(a) = table_to_rule_allowlist(allow_t) {
if i == 0 {
if let Some(last) = out.last_mut() {
last.allowlists.get_or_insert_with(Vec::new).push(a);
} else {
pending_allowlists.push(a);
}
} else {
pending_allowlists.push(a);
}
}
}
}
}
continue;
}
if key.parse::<usize>().is_err() {
continue;
}
if let Value::Table(rule_t) = val {
if let Some(mut r) = table_to_rule(rule_t) {
if let Some(first) = pending_allowlists.first() {
r.allowlists = Some(vec![first.clone()]);
pending_allowlists = pending_allowlists.split_off(1);
}
out.push(r);
}
}
}
out
}
fn table_to_rule(t: &toml::Table) -> Option<RuleRaw> {
let s = toml::to_string(&Value::Table(t.clone())).ok()?;
toml::from_str(&s).ok()
}
fn table_to_rule_allowlist(t: &toml::Table) -> Option<RuleAllowlistRaw> {
let s = toml::to_string(&Value::Table(t.clone())).ok()?;
toml::from_str(&s).ok()
}
#[derive(Debug, Deserialize)]
struct AppendOverride {
#[serde(rename = "rule_id")]
rule_id: String,
#[serde(rename = "regexTarget")]
regex_target: Option<String>,
pub paths: Option<Vec<String>>,
pub regexes: Option<Vec<String>>,
pub stopwords: Option<Vec<String>>,
pub condition: Option<String>,
}
#[derive(Debug, Deserialize)]
struct OverridesFile {
append: Option<Vec<AppendOverride>>,
rules: Option<Vec<RuleRaw>>,
}
fn merge_overrides_into_rules(rules: &mut Vec<CompiledRule>, overlay_toml: &str) {
let file: OverridesFile = match toml::from_str(overlay_toml) {
Ok(v) => v,
Err(e) => {
log::warn!("Failed to parse gitleaks.overrides.toml: {e}");
return;
}
};
if let Some(new_rules) = file.rules {
for r in new_rules {
let Some(regex_str) = &r.regex else {
log::debug!("Skipping override rule '{}': no regex", r.id);
continue;
};
let regex = match Regex::new(regex_str) {
Ok(re) => re,
Err(e) => {
log::warn!("Skipping override rule '{}': invalid regex: {}", r.id, e);
continue;
}
};
let allowlists = r
.allowlists
.as_deref()
.unwrap_or(&[])
.iter()
.map(compile_rule_allowlist)
.collect();
let keywords = r.keywords.as_ref().map(|kws| {
kws.iter()
.map(|s| s.to_lowercase())
.collect::<Vec<String>>()
});
let compiled = CompiledRule {
id: r.id.clone(),
regex,
entropy: r.entropy,
keywords,
keyword_pattern_ids: None, secret_group: r.secret_group,
allowlists,
path: r.path.clone(),
};
if let Some(idx) = rules.iter().position(|existing| existing.id == r.id) {
log::debug!("Replacing override rule: {}", r.id);
rules[idx] = compiled;
} else {
log::debug!("Adding override rule: {}", r.id);
rules.push(compiled);
}
}
}
if let Some(list) = file.append {
for append in list {
let allowlist_raw = RuleAllowlistRaw {
paths: append.paths,
regexes: append.regexes,
stopwords: append.stopwords,
regex_target: append.regex_target,
condition: append.condition,
};
let compiled = compile_rule_allowlist(&allowlist_raw);
if let Some(rule) = rules.iter_mut().find(|r| r.id == append.rule_id) {
rule.allowlists.push(compiled);
} else {
log::debug!(
"Overrides reference rule_id '{}' which is not in config; skipping",
append.rule_id
);
}
}
}
}
fn load_compiled() -> Result<GitleaksCompiled> {
let toml_str = include_str!("../../config/gitleaks.toml");
let value: Value = toml::from_str(toml_str).context("Failed to parse gitleaks.toml")?;
let global_allowlist = value.get("allowlist").and_then(|v| {
toml::to_string(v)
.ok()
.and_then(|s| toml::from_str::<AllowlistRaw>(&s).ok())
});
let global_allowlist = compile_global_allowlist(global_allowlist);
let rules_val = value
.get("rules")
.ok_or_else(|| anyhow!("gitleaks.toml must have [rules]"))?;
let raw_rules: Vec<RuleRaw> = match rules_val {
Value::Array(_) => {
let raw: GitleaksConfigRaw =
toml::from_str(toml_str).context("Failed to parse gitleaks.toml rules")?;
raw.rules
}
Value::Table(t) => rules_from_table(t),
_ => return Err(anyhow!("gitleaks.toml rules must be array or table")),
};
let mut rules: Vec<CompiledRule> = raw_rules
.into_iter()
.filter_map(|r| {
let Some(regex_str) = &r.regex else {
log::debug!("Skipping gitleaks rule '{}': no regex (path-only)", r.id);
return None;
};
let regex = match compile_rule_regex(regex_str) {
Ok(re) => re,
Err(e) => {
log::warn!("Skipping gitleaks rule '{}': invalid regex: {}", r.id, e);
return None;
}
};
let allowlists = r
.allowlists
.as_deref()
.unwrap_or(&[])
.iter()
.map(compile_rule_allowlist)
.collect();
let keywords = r.keywords.as_ref().map(|kws| {
kws.iter()
.map(|s| s.to_lowercase())
.collect::<Vec<String>>()
});
Some(CompiledRule {
id: r.id,
regex,
entropy: r.entropy,
keywords,
keyword_pattern_ids: None, secret_group: r.secret_group,
allowlists,
path: r.path.clone(),
})
})
.collect();
if rules.is_empty() {
return Err(anyhow!("gitleaks.toml produced no valid rules"));
}
let overlay_toml = include_str!("../../config/gitleaks.overrides.toml");
merge_overrides_into_rules(&mut rules, overlay_toml);
let keyword_prefilter = build_keyword_prefilter(&mut rules);
Ok(GitleaksCompiled {
global_allowlist,
rules,
keyword_prefilter,
})
}
fn build_keyword_prefilter(rules: &mut [CompiledRule]) -> Option<KeywordPrefilter> {
use std::collections::HashMap;
let mut pattern_to_id: HashMap<String, u32> = HashMap::new();
let mut patterns: Vec<String> = Vec::new();
for rule in rules.iter_mut() {
let Some(kws) = rule.keywords.as_ref() else {
continue;
};
let mut ids: Vec<u32> = Vec::with_capacity(kws.len());
for kw in kws {
if kw.is_empty() {
continue;
}
let id = match pattern_to_id.get(kw) {
Some(&id) => id,
None => {
let id_usize = patterns.len();
let Ok(id) = u32::try_from(id_usize) else {
log::warn!("gitleaks keyword count exceeded u32; skipping prefilter");
return None;
};
patterns.push(kw.clone());
pattern_to_id.insert(kw.clone(), id);
id
}
};
ids.push(id);
}
if !ids.is_empty() {
ids.sort_unstable();
ids.dedup();
rule.keyword_pattern_ids = Some(ids);
}
}
if patterns.is_empty() {
return None;
}
let automaton = match aho_corasick::AhoCorasickBuilder::new()
.ascii_case_insensitive(true)
.match_kind(aho_corasick::MatchKind::Standard)
.build(&patterns)
{
Ok(a) => a,
Err(e) => {
log::warn!("Failed to build gitleaks keyword automaton: {e}; prefilter disabled");
for rule in rules.iter_mut() {
rule.keyword_pattern_ids = None;
}
return None;
}
};
Some(KeywordPrefilter { automaton })
}
static GITLEAKS_RESULT: OnceLock<std::result::Result<GitleaksCompiled, String>> = OnceLock::new();
fn ensure_loaded() -> &'static std::result::Result<GitleaksCompiled, String> {
GITLEAKS_RESULT.get_or_init(|| load_compiled().map_err(|e| format!("{e:?}")))
}
pub fn init_gitleaks() -> Result<()> {
match ensure_loaded() {
Ok(_) => Ok(()),
Err(s) => Err(anyhow!("{}", s)),
}
}
pub fn try_gitleaks() -> std::result::Result<&'static GitleaksCompiled, &'static str> {
match ensure_loaded() {
Ok(c) => Ok(c),
Err(s) => Err(s.as_str()),
}
}
pub fn gitleaks() -> &'static GitleaksCompiled {
match try_gitleaks() {
Ok(c) => c,
Err(e) => panic!(
"BUG: bundled config/gitleaks.toml failed to load: {e}. \
Use init_gitleaks() during scan startup to receive this as a Result."
),
}
}
#[cfg(test)]
mod tests {
use super::*;
use toml::Table;
#[test]
fn test_rules_table_allowlist_association() {
let mut rule_t = Table::new();
rule_t.insert("id".into(), Value::String("test-rule".into()));
rule_t.insert("description".into(), Value::String("Test".into()));
rule_t.insert("regex".into(), Value::String("[A-Z]+".into()));
let mut allow_t = Table::new();
allow_t.insert(
"regexes".into(),
Value::Array(vec![Value::String(".+EXAMPLE$".into())]),
);
let mut rules_t = Table::new();
rules_t.insert("0".into(), Value::Table(rule_t));
rules_t.insert(
"allowlists".into(),
Value::Array(vec![Value::Table(allow_t)]),
);
let rules = rules_from_table(&rules_t);
assert_eq!(rules.len(), 1, "expected one rule");
let allowlists = rules[0]
.allowlists
.as_ref()
.expect("rule should have allowlists");
assert_eq!(allowlists.len(), 1, "expected one allowlist");
let regexes = allowlists[0]
.regexes
.as_ref()
.expect("allowlist should have regexes");
assert_eq!(regexes.len(), 1);
assert!(
regexes[0].contains("EXAMPLE"),
"allowlist regex should match EXAMPLE suffix, got {:?}",
regexes[0]
);
}
#[test]
fn test_oversized_regex_rules_are_loaded() {
let config = gitleaks();
for expected_id in ["generic-api-key", "pypi-upload-token", "vault-batch-token"] {
assert!(
config.rules.iter().any(|r| r.id == expected_id),
"rule '{expected_id}' must be present in the bundled ruleset \
(was previously skipped due to compiled-regex size limit; \
see REGEX_SIZE_LIMIT in src/parse/gitleaks.rs)"
);
}
}
#[test]
fn test_keyword_prefilter_populated_for_keyworded_rules() {
let config = gitleaks();
let prefilter = config
.keyword_prefilter
.as_ref()
.expect("production config must have a keyword prefilter");
assert!(
prefilter.automaton.patterns_len() > 0,
"prefilter should have populated patterns"
);
for rule in &config.rules {
if let Some(kws) = &rule.keywords {
if kws.iter().any(|k| !k.is_empty()) {
let ids = rule
.keyword_pattern_ids
.as_ref()
.unwrap_or_else(|| panic!("rule '{}' has keywords but no IDs", rule.id));
assert!(
!ids.is_empty(),
"rule '{}' has keywords but its IDs vec is empty",
rule.id
);
for id in ids {
assert!(
(*id as usize) < prefilter.automaton.patterns_len(),
"rule '{}' has out-of-range pattern id {id}",
rule.id
);
}
}
}
}
}
#[test]
fn test_try_gitleaks_concurrent_first_call_returns_same_pointer() {
use std::sync::Arc;
use std::sync::Barrier;
let n = 16usize;
let barrier = Arc::new(Barrier::new(n));
let mut handles = Vec::with_capacity(n);
for _ in 0..n {
let b = Arc::clone(&barrier);
handles.push(std::thread::spawn(move || {
b.wait();
let cfg = try_gitleaks().expect("config must load on every thread");
std::ptr::from_ref(cfg) as usize
}));
}
let pointers: Vec<usize> = handles
.into_iter()
.map(|h| h.join().expect("thread panicked"))
.collect();
let first = pointers[0];
for (i, p) in pointers.iter().enumerate() {
assert_eq!(
*p, first,
"thread {i} observed a different GitleaksCompiled pointer ({p:#x} vs first {first:#x}); \
OnceLock did not serialise the first call as expected"
);
}
}
#[test]
fn test_keyword_prefilter_case_insensitive_match() {
let config = gitleaks();
let prefilter = config.keyword_prefilter.as_ref().expect("prefilter");
let sample = format!("AKIA{}", "IOSFODNN7EXAMPL2");
let upper_body = format!("blah {sample} trailing");
let lower_body = format!("blah {} trailing", sample.to_ascii_lowercase());
let upper = prefilter.matched_pattern_ids(&upper_body);
let lower = prefilter.matched_pattern_ids(&lower_body);
let mixed_suffix: String = sample
.chars()
.enumerate()
.map(|(i, c)| {
if i % 2 == 0 {
c.to_ascii_uppercase()
} else {
c.to_ascii_lowercase()
}
})
.collect();
let mixed = prefilter.matched_pattern_ids(&format!("blah {mixed_suffix} trailing"));
assert_eq!(upper, lower, "casing must not change matched id set");
assert_eq!(lower, mixed);
assert!(
!upper.is_empty(),
"the 'akia' keyword should fire on AKIA-prefixed strings"
);
}
#[test]
fn test_overlay_merge_appends_allowlists() {
let config = gitleaks();
let rule = config
.rules
.iter()
.find(|r| r.id == "sourcegraph-access-token")
.expect("sourcegraph-access-token rule should exist");
assert!(
!rule.allowlists.is_empty(),
"sourcegraph-access-token should have allowlists from overlay (gitleaks.overrides.toml)"
);
let has_line_target = rule
.allowlists
.iter()
.any(|a| a.regex_target.as_deref() == Some("line"));
assert!(
has_line_target,
"overlay defines regexTarget = 'line' for sourcegraph"
);
}
#[test]
fn test_overlay_replaces_sourcegraph_regex() {
let config = gitleaks();
let rule = config
.rules
.iter()
.find(|r| r.id == "sourcegraph-access-token")
.expect("sourcegraph-access-token rule should exist");
let count = config
.rules
.iter()
.filter(|r| r.id == "sourcegraph-access-token")
.count();
assert_eq!(
count, 1,
"replace must not leave duplicate sourcegraph rules"
);
let bare = concat!("a1b2c3d4e5f67890", "a1b2c3d4e5f67890a1b2c3d4");
assert!(
!rule.regex.is_match(bare),
"replaced sourcegraph rule must not match bare 40-hex"
);
let sgp_suffix = concat!("0123456789abcdef01234567", "89abcdef01234567");
let sgp = format!("sgp_{}_{}", "0123456789abcdef", sgp_suffix);
assert!(
rule.regex.is_match(&sgp),
"replaced sourcegraph rule must still match sgp_ tokens"
);
}
#[test]
fn test_overlay_replaced_rules_are_unique() {
let config = gitleaks();
let replaced = [
"sourcegraph-access-token",
"dropbox-api-token",
"square-access-token",
"datadog-access-token",
"facebook-page-access-token",
];
for id in replaced {
let count = config.rules.iter().filter(|r| r.id == id).count();
assert_eq!(
count, 1,
"overlay replace must leave exactly one '{id}' rule, found {count}"
);
}
let dropbox = config
.rules
.iter()
.find(|r| r.id == "dropbox-api-token")
.expect("dropbox-api-token");
let kws = dropbox.keywords.as_ref().expect("keywords");
assert!(
kws.iter().any(|k| k.contains("disabled")),
"disabled dropbox catch-all should use inert keywords; got {kws:?}"
);
assert!(
!dropbox.regex.is_match("dropbox=abcdefghijklmno"),
"disabled dropbox-api-token must not match 15-char identifiers"
);
}
}