use std::{io::Read, path::PathBuf, sync::OnceLock};
use anyhow::{Context, Result, anyhow, bail};
use flate2::read::GzDecoder;
use crate::rule::Confidence;
use crate::rules::Rules;
pub const BUILTIN_RULE_PROVENANCE: &str = include_str!("../generated/provenance.json");
const BUNDLE_MAGIC: &[u8] = b"KFRULES\x01";
const DEFAULT_RULE_BUNDLE: &[u8] = include_bytes!("../generated/builtin-rules.gz");
type BuiltinRuleFiles = Vec<(PathBuf, Vec<u8>)>;
type CachedBuiltinRuleFiles = Result<BuiltinRuleFiles, String>;
static BUILTIN_RULE_FILES: OnceLock<CachedBuiltinRuleFiles> = OnceLock::new();
pub fn get_builtin_rule_files() -> Result<Vec<(PathBuf, Vec<u8>)>> {
Ok(builtin_rule_files()?.to_vec())
}
fn builtin_rule_files() -> Result<&'static [(PathBuf, Vec<u8>)]> {
BUILTIN_RULE_FILES
.get_or_init(|| load_rule_files(DEFAULT_RULE_BUNDLE).map_err(|err| format!("{err:#}")))
.as_deref()
.map_err(|err| anyhow!("failed to load embedded builtin rules: {err}"))
}
pub fn get_betterleaks_rule_files() -> Result<Vec<(PathBuf, Vec<u8>)>> {
get_builtin_rule_files()
}
fn load_rule_files(bundle: &[u8]) -> Result<BuiltinRuleFiles> {
let mut decoded = Vec::new();
GzDecoder::new(bundle)
.read_to_end(&mut decoded)
.context("failed to decompress embedded rule bundle")?;
let mut cursor = BundleCursor::new(&decoded);
if cursor.take(BUNDLE_MAGIC.len())? != BUNDLE_MAGIC {
bail!("embedded rule bundle has an invalid header");
}
let mut files = Vec::new();
loop {
let path_len = usize::try_from(cursor.u32()?)
.map_err(|_| anyhow!("embedded rule path length exceeds platform limits"))?;
if path_len == 0 {
break;
}
let contents_len = usize::try_from(cursor.u64()?)
.map_err(|_| anyhow!("embedded rule contents length exceeds platform limits"))?;
let path = std::str::from_utf8(cursor.take(path_len)?)
.context("embedded rule bundle contains a non-UTF-8 path")?;
files.push((PathBuf::from(path), cursor.take(contents_len)?.to_vec()));
}
if !cursor.is_empty() {
bail!("embedded rule bundle has trailing data");
}
Ok(files)
}
pub fn get_builtin_rules(confidence: Option<Confidence>) -> Result<Rules> {
let confidence = confidence.unwrap_or(Confidence::Medium);
let files = builtin_rule_files()?;
Rules::from_paths_and_contents(
files.iter().map(|(path, contents)| (path.as_path(), contents.as_slice())),
confidence,
)
}
pub fn get_betterleaks_rules(confidence: Option<Confidence>) -> Result<Rules> {
get_builtin_rules(confidence)
}
struct BundleCursor<'a> {
remaining: &'a [u8],
}
impl<'a> BundleCursor<'a> {
fn new(contents: &'a [u8]) -> Self {
Self { remaining: contents }
}
fn is_empty(&self) -> bool {
self.remaining.is_empty()
}
fn take(&mut self, length: usize) -> Result<&'a [u8]> {
if self.remaining.len() < length {
bail!("embedded rule bundle is truncated");
}
let (result, remaining) = self.remaining.split_at(length);
self.remaining = remaining;
Ok(result)
}
fn u32(&mut self) -> Result<u32> {
let bytes: [u8; 4] = self.take(4)?.try_into().expect("slice length is checked");
Ok(u32::from_le_bytes(bytes))
}
fn u64(&mut self) -> Result<u64> {
let bytes: [u8; 8] = self.take(8)?.try_into().expect("slice length is checked");
Ok(u64::from_le_bytes(bytes))
}
}
#[cfg(test)]
mod test {
use super::*;
use crate::{
BetterleaksAccessMapHandler, EthereumValidation, Revocation, TlsMode, Validation,
betterleaks_filter::{BetterleaksFilterContext, evaluate_filter},
};
fn filter_discards(
rule: &crate::RuleSyntax,
path: &str,
secret: &str,
full_match: &str,
captures: &[(&str, &str)],
) -> bool {
evaluate_filter(
rule.betterleaks_filter.as_ref().expect("rule should have a filter"),
&BetterleaksFilterContext {
attributes: None,
path,
secret,
full_match,
line: full_match,
fragment_raw: full_match,
match_start_idx: 0,
match_end_idx: full_match.len(),
match_line_start_idx: 0,
match_line_end_idx: full_match.len(),
rule_id: &rule.id,
description: &rule.name,
captures: captures
.iter()
.map(|(name, value)| ((*name).to_string(), (*value).to_string()))
.collect(),
},
)
.unwrap()
.discard
}
#[test]
fn generic_api_key_requires_explicit_low_confidence_selection() {
let id = "betterleaks.generic-api-key";
let low = get_builtin_rules(Some(Confidence::Low)).unwrap();
assert_eq!(low.rules[id].confidence, Confidence::Low);
for minimum in [None, Some(Confidence::Medium), Some(Confidence::High)] {
let selected = get_builtin_rules(minimum).unwrap();
assert!(!selected.rules.contains_key(id), "generic API key selected at {minimum:?}");
}
let defaults = get_builtin_rules(None).unwrap();
assert_eq!(
defaults.rules["betterleaks.generic-credential-uri"].confidence,
Confidence::Medium
);
}
#[test]
fn builtin_filters_have_no_confidence_changing_calls_or_unused_uri_bindings() {
let rules = get_builtin_rules(Some(Confidence::Low)).unwrap();
for rule in rules.rules.values() {
let filter = serde_json::to_string(&rule.betterleaks_filter).unwrap();
assert!(!filter.contains("setConfidence"), "legacy confidence helper in {}", rule.id);
}
assert_eq!(rules.rules["betterleaks.generic-api-key"].confidence, Confidence::Low);
assert_eq!(
rules.rules["betterleaks.generic-credential-uri"].confidence,
Confidence::Medium
);
let uri = &rules.rules["betterleaks.generic-credential-uri"];
let filter = serde_json::to_string(uri.betterleaks_filter.as_ref().unwrap()).unwrap();
for name in [
"level",
"isExampleValue",
"isSyntheticCredentialTuple",
"isTestOrDocumentationSource",
"hasLowConfidenceHost",
] {
let identifier = serde_json::to_string(&crate::BetterleaksExpr::Identifier {
value: name.to_string(),
})
.unwrap();
assert!(!filter.contains(&identifier), "unused URI reference {name}");
assert!(!filter.contains(&format!(r#""name":"{name}""#)), "unused URI binding {name}");
}
}
#[test]
fn placeholder_private_keys_and_mongodb_passwords_are_filtered() {
let rules = get_builtin_rules(Some(Confidence::Low)).unwrap();
let key_rule = &rules.rules["betterleaks.private-key"];
for body in [
"",
"\nPRIVATE_KEY\n",
r#"\n"
+ "PRIVATE_KEY\n"
+ ""#,
"\nYWJjZA==\n",
"\nfoo\nbar\nfoobar\nbarfoo\n",
"\neHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4\n",
] {
let pem = format!("-----BEGIN PRIVATE KEY-----{body}-----END PRIVATE KEY-----");
assert!(filter_discards(key_rule, "fixture.java", &pem, &pem, &[]), "{pem}");
}
let numbered = (1..=96)
.map(|index| {
format!(" {index}{}\n", ["foo", "bar", "foobar", "barfoo"][(index - 1) % 4])
})
.collect::<String>();
let pem =
format!("-----BEGIN RSA PRIVATE KEY-----\n{numbered}-----END RSA PRIVATE KEY-----");
assert!(filter_discards(key_rule, "ini_test.go", &pem, &pem, &[]));
let body = "MC4CAQAwBQYDK2VwBCIEIAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8g";
for body in [
body.to_string(),
body.as_bytes()
.chunks(16)
.map(|chunk| std::str::from_utf8(chunk).unwrap())
.collect::<Vec<_>>()
.join("\n"),
] {
let pem = format!("-----BEGIN PRIVATE KEY-----\n{body}\n-----END PRIVATE KEY-----");
assert!(!filter_discards(key_rule, "fixture.pem", &pem, &pem, &[]), "{pem}");
for prefix in ["# ", "+", " * ", "> ", "// ", "; "] {
let represented = pem.replace('\n', &format!("\n{prefix}"));
assert!(
!filter_discards(key_rule, "fixture.pem", &represented, &represented, &[]),
"{prefix}"
);
}
}
let opaque = r#"-----BEGIN PRIVATE KEY-----", "begin key marker"),
Pair.of(expectedKeyString, "private key pem"),
Pair.of("-----END PRIVATE KEY-----"#;
assert!(filter_discards(key_rule, "fixture.java", opaque, opaque, &[]));
let with_material = opaque.replace("expectedKeyString", &format!("\"{body}\""));
assert!(!filter_discards(key_rule, "fixture.java", &with_material, &with_material, &[]));
let mongo = &rules.rules["betterleaks.mongodb-connection-string"];
for password in [
"$(AWS_SECRET_ACCESS_KEY)",
"$(MONGODB_PASSWORD)",
"${MONGODB_PASSWORD}",
"$MONGODB_PASSWORD",
"{{ MONGODB_PASSWORD }}",
] {
let uri = format!("mongodb+srv://$(MONGODB_USERNAME):{password}@cluster.invalid/db"); assert!(filter_discards(mongo, "fixture.yml", &uri, &uri, &[]), "{uri}");
}
for uri in [
"mongodb://$(MONGODB_USERNAME):r4nd0mLiteralSecret@cluster.invalid/db", "mongodb://alice:r4nd0mLiteralSecret@$(MONGODB_HOST)/db",
"mongodb://alice:literal$(suffix)@cluster.invalid/db", "mongodb://alice:dollar%24literal@cluster.invalid/db", ] {
assert!(!filter_discards(mongo, "fixture.yml", uri, uri, &[]), "{uri}");
}
let mut selected = Rules::new();
for rule in [key_rule, mongo] {
selected.rules.insert(rule.id.clone(), rule.clone());
}
crate::RulesDatabase::from_rule_collection(selected).unwrap();
}
#[test]
fn bare_provider_rules_detect_extracted_tokens_and_compile() {
let rules = get_builtin_rules(Some(Confidence::Low)).unwrap();
let cases = [
("deepseek-api-key", format!("sk-{}", "0123456789abcdef".repeat(2))),
("kimi-api-key", format!("sk-{}", "AbcD0123_-xy".repeat(4))),
("zai-api-key", format!("{}.AbcD0123efGH4567", "0123456789abcdef".repeat(2))),
("voyageai-api-key", format!("pa-{}QrstUv9", "AbcD0123_-xy".repeat(3))),
];
let mut selected = Rules::new();
for (id, token) in &cases {
let rule = &rules.rules[&format!("betterleaks.{id}")];
let regex =
regex::bytes::RegexBuilder::new(&rule.pattern).unicode(false).build().unwrap();
for input in
[token.clone(), format!("VAR_0000 = \"{token}\""), format!("{token} # provider")]
{
let captures =
regex.captures(input.as_bytes()).unwrap_or_else(|| panic!("{id}: {input}"));
assert_eq!(captures.get(1).unwrap().as_bytes(), token.as_bytes());
}
for input in [
format!("x{token}"),
format!("_{token}"),
format!("-{token}"),
format!("{token}x"),
format!("{token}_"),
] {
assert!(!regex.is_match(input.as_bytes()), "{id}: {input}");
}
assert!(matches!(rule.validation, Some(Validation::Betterleaks(_))));
assert!(!filter_discards(rule, "fixture.txt", token, token, &[]), "{id}");
selected.rules.insert(rule.id.clone(), rule.clone());
}
let db = crate::RulesDatabase::from_rule_collection(selected).unwrap();
let mut scanner = kingfisher_vectorscan::BlockScanner::new(db.vectorscan_db()).unwrap();
for (id, token) in cases {
let input = format!("{token}\n{token}");
let mut hits = 0;
scanner
.scan(input.as_bytes(), |index, from, to, _flags| {
if db.get_rule(index as usize).unwrap().id() == format!("betterleaks.{id}") {
let candidate = &input.as_bytes()[from as usize..to as usize];
let captures =
db.anchored_regexes()[index as usize].captures(candidate).unwrap();
assert_eq!(captures.get(1).unwrap().as_bytes(), token.as_bytes());
hits += 1;
}
kingfisher_vectorscan::Scan::Continue
})
.unwrap();
assert!(hits >= 2, "{id}: missing adjacent bare tokens");
}
}
#[test]
fn box_client_tokens_are_filtered_case_insensitively() {
let rules = get_builtin_rules(Some(Confidence::Low)).unwrap();
let rule = &rules.rules["betterleaks.box-api-access-token"];
let token = "aB3dE5fG7hJ9kL2mN4pQ6rS8tU0vW1xY";
assert!(filter_discards(
rule,
"fixture.env",
token,
&format!("BOX_CLIENT_TOKEN={token}"),
&[]
));
assert!(!filter_discards(
rule,
"fixture.env",
token,
&format!("BOX_API_TOKEN={token}"),
&[]
));
}
#[test]
fn refreshed_betterleaks_formats_detect_and_compile() {
let rules = get_builtin_rules(Some(Confidence::Low)).unwrap();
let cases = [
("bitbucket-data-center-token", format!("BBDC-{}", "Ab3D+eF/9gHk2mNp".repeat(2))),
("cloudflare-api-key.2", format!("cfut_{}deadbeef", "Ab3DeF9gHk".repeat(4))),
("cloudflare-api-key.2", format!("cfat_{}deadbeef", "Ab3DeF9gHk".repeat(4))),
(
"tableau-personal-access-token.1",
format!("+{}==:{}", "aB3dE5fG7hJ9kL2mN4pQ6", "aB3dE5fG7hJ9kL2m".repeat(2)),
),
(
"tableau-personal-access-token.1",
format!("/{}==:{}", "aB3dE5fG7hJ9kL2mN4pQ6", "aB3dE5fG7hJ9kL2m".repeat(2)),
),
];
let mut selected = Rules::new();
for (id, token) in &cases {
let rule = &rules.rules[&format!("betterleaks.{id}")];
let regex = rule.as_regex().unwrap();
for input in [token.clone(), format!("TOKEN=\"{token}\"")] {
let captures = regex.captures(input.as_bytes()).unwrap();
assert_eq!(captures.get(1).unwrap().as_bytes(), token.as_bytes(), "{id}");
assert!(!filter_discards(rule, "fixture.txt", token, &input, &[]), "{id}");
}
for input in [format!("x{token}"), format!("{token}x")] {
if *id != "bitbucket-data-center-token" || input.starts_with('x') {
assert!(!regex.is_match(input.as_bytes()), "{id}: {input}");
}
}
selected.rules.insert(rule.id.clone(), rule.clone());
}
let cf = &rules.rules["betterleaks.cloudflare-api-key.2"];
assert!(matches!(cf.validation, Some(Validation::Betterleaks(_))));
let account = cf.depends_on_rule.first().unwrap().as_ref().unwrap();
assert_eq!(account.rule_id, "betterleaks.cloudflare-account-id.1");
assert!(account.optional);
assert_eq!(account.within.as_deref(), Some("5L"));
assert!(!rules.rules[&account.rule_id].visible);
assert!(!rules.rules.contains_key("betterleaks.cloudflare-api-key"));
let db = crate::RulesDatabase::from_rule_collection(selected).unwrap();
let mut scanner = kingfisher_vectorscan::BlockScanner::new(db.vectorscan_db()).unwrap();
for (id, token) in cases {
let mut found = false;
scanner
.scan(token.as_bytes(), |index, from, to, _| {
if db.get_rule(index as usize).unwrap().id() == format!("betterleaks.{id}") {
let captures = db.anchored_regexes()[index as usize]
.captures(&token.as_bytes()[from as usize..to as usize])
.unwrap();
assert_eq!(captures.get(1).unwrap().as_bytes(), token.as_bytes());
found = true;
}
kingfisher_vectorscan::Scan::Continue
})
.unwrap();
assert!(found, "{id}: Vectorscan did not detect {token}");
}
}
#[test]
fn betterleaks_rc5_detectors_detect_and_compile() {
let rules = get_builtin_rules(Some(Confidence::Low)).unwrap();
let base32 = "abcdefghijklmnopqrstuvwxyz234567";
let tailscale_secret = "Ab3DeF9gHk2mNp6Qr8St0Vw1Xy4Za7Bc";
let cases = [
(
"labelbox-api-key",
format!("lbx_{}A", "Ab3DeF9gHk2mNp".repeat(3)),
"lbx_Ab3DeF9gHk2mNp",
),
(
"labelbox-recursion-api-key",
format!("rma_{}A", "Ab3DeF9gHk2mNp".repeat(3)),
"rma_Ab3DeF9gHk2mNp",
),
(
"nuget-api-key",
format!("oy2a{}q{}e", &base32[..15], &base32[..25]),
"oy2qabcdefghijklmnopabcdefghijklmnopqrstuvwxye",
),
(
"tailscale-auth-key",
format!("tskey-auth-kA1b2C3d4CNTRL-{tailscale_secret}"),
"tskey-auth-kA1b2C3d4CNTRL-short",
),
(
"tailscale-key",
format!("tskey-kA1b2C3d4CNTRL-{tailscale_secret}"),
"tskey-kA1b2C3d4CNTRL-short",
),
(
"tailscale-oauth-client-secret",
format!("tskey-client-kA1b2C3d4CNTRL-{tailscale_secret}"),
"tskey-client-kA1b2C3d4CNTRL-short",
),
(
"tailscale-scim-key",
format!("tskey-scim-A1b2C3d4E5f6-{tailscale_secret}"),
"tskey-scim-A1b2C3d4E5f6-short",
),
(
"tailscale-webhook-secret",
format!("tskey-webhook-kA1b2C3d4CNTRL-{tailscale_secret}"),
"tskey-webhook-kA1b2C3d4CNTRL-short",
),
];
let mut selected = Rules::new();
for (id, token, negative) in &cases {
let rule = &rules.rules[&format!("betterleaks.{id}")];
assert!(rule.visible, "{id}");
assert!(!rule.betterleaks_path_only, "{id}");
assert!(matches!(rule.validation, Some(Validation::Betterleaks(_))), "{id}");
assert_eq!(rule.betterleaks_secret_group, Some(0), "{id}");
let regex = rule.as_regex().unwrap();
assert_eq!(regex.captures_len(), 2, "{id}: expected one secret capture");
for input in [token.clone(), format!("TOKEN=\"{token}\"")] {
let captures = regex.captures(input.as_bytes()).unwrap_or_else(|| panic!("{id}"));
assert_eq!(captures.get(1).unwrap().as_bytes(), token.as_bytes(), "{id}");
assert!(!filter_discards(rule, "fixture.txt", token, &input, &[]), "{id}");
}
assert!(!regex.is_match(negative.as_bytes()), "{id}");
selected.rules.insert(rule.id.clone(), rule.clone());
}
let db = crate::RulesDatabase::from_rule_collection(selected).unwrap();
let mut scanner = kingfisher_vectorscan::BlockScanner::new(db.vectorscan_db()).unwrap();
for (id, token, _) in cases {
let mut found = false;
scanner
.scan(token.as_bytes(), |index, from, to, _| {
if db.get_rule(index as usize).unwrap().id() == format!("betterleaks.{id}") {
let captures = db.anchored_regexes()[index as usize]
.captures(&token.as_bytes()[from as usize..to as usize])
.unwrap();
assert_eq!(captures.get(1).unwrap().as_bytes(), token.as_bytes());
found = true;
}
kingfisher_vectorscan::Scan::Continue
})
.unwrap();
assert!(found, "{id}: Vectorscan did not detect the token");
}
}
#[test]
fn betterleaks_rc7_mapbox_tokens_detect_filter_and_compile() {
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
let rules = get_builtin_rules(Some(Confidence::Low)).unwrap();
assert_eq!(rules.num_rules(), 503);
let provenance: serde_json::Value = serde_json::from_str(BUILTIN_RULE_PROVENANCE).unwrap();
assert_eq!(
provenance["betterleaks"]["revision"],
"8f376ff9c31ed5309ac448fe601c5dd21b376155"
);
assert_eq!(
rules.rules.values().filter(|rule| rule.id.starts_with("veles.")).count(),
26,
"retain all selected Veles detectors and their four hidden component rules"
);
assert_eq!(
rules
.rules
.values()
.filter(|rule| rule.id.starts_with("veles.") && rule.visible)
.count(),
22,
"the rc.7 upgrade must retain all selected Veles detectors"
);
let signature = "Ab3DeF9gHk2mNp6Qr8St0A";
let mut selected = Rules::new();
let mut fixtures = Vec::new();
for (id, prefix) in [("mapbox-api-token", "pk"), ("mapbox-secret-token", "sk")] {
let rule = &rules.rules[&format!("betterleaks.{id}")];
assert!(rule.visible);
assert_eq!(rule.confidence, Confidence::High);
let Some(Validation::Betterleaks(validation)) = &rule.validation else {
panic!("{id} should retain upstream validation")
};
assert!(validation.analyze.is_some(), "{id}");
let regex = rule.as_regex().unwrap();
for username in ["fixture", "fixturex", "fixturexy", "fixture地图"] {
let payload = format!(r#"{{"u":"{username}","a":"clAb3DeF9gHk2mNp6Qr8St0Vw1"}}"#);
let token = format!("{prefix}.{}.{signature}", URL_SAFE_NO_PAD.encode(&payload));
for input in [token.clone(), format!("MAPBOX_TOKEN=\"{token}\"")] {
let captures =
regex.captures(input.as_bytes()).unwrap_or_else(|| panic!("{id}"));
assert_eq!(captures.get(1).unwrap().as_bytes(), token.as_bytes());
assert!(!filter_discards(rule, "fixture.txt", &token, &input, &[]), "{id}");
}
assert!(!regex.is_match(format!("x{token}").as_bytes()), "{id}");
assert!(!regex.is_match(format!("{token}_").as_bytes()), "{id}");
fixtures.push((rule.id.clone(), token));
}
for payload in [
r#"{"u":"fixture","b":"clAb3DeF9gHk2mNp6Qr8St0Vw1"}"#,
r#"{"u":"fixture","a":"clAb3DeF9gHk2mNp6Qr8St0Vw1","extra":true}"#,
r#"{"u":"","a":"clAb3DeF9gHk2mNp6Qr8St0Vw1"}"#,
r#"{"u":"fixture","a":"clAb3DeF9gHk2mNp6Qr8St0Vw1!"}"#,
r#"{"u":"fixture\\name","a":"clAb3DeF9gHk2mNp6Qr8St0Vw1"}"#,
] {
let token = format!("{prefix}.{}.{signature}", URL_SAFE_NO_PAD.encode(payload));
assert!(regex.is_match(token.as_bytes()), "{id}: expected a candidate");
assert!(
filter_discards(rule, "fixture.txt", &token, &token, &[]),
"{id}: {payload}"
);
}
let low_entropy_payload =
format!(r#"{{"u":"{}","a":"{}"}}"#, "a".repeat(120), "a".repeat(20));
let token = format!(
"{prefix}.{}.{}",
URL_SAFE_NO_PAD.encode(low_entropy_payload),
"A".repeat(22)
);
assert!(regex.is_match(token.as_bytes()), "{id}");
assert!(filter_discards(rule, "fixture.txt", &token, &token, &[]), "{id}");
selected.rules.insert(rule.id.clone(), rule.clone());
}
let db = crate::RulesDatabase::from_rule_collection(selected).unwrap();
let mut scanner = kingfisher_vectorscan::BlockScanner::new(db.vectorscan_db()).unwrap();
for (id, token) in fixtures {
let mut found = false;
scanner
.scan(token.as_bytes(), |index, _, _, _| {
if db.get_rule(index as usize).unwrap().id() == id {
found = true;
}
kingfisher_vectorscan::Scan::Continue
})
.unwrap();
assert!(found, "{id}: Vectorscan did not detect the token");
}
}
#[test]
fn veles_precedence_filters_only_suppress_covered_formats() {
let rules = get_builtin_rules(Some(Confidence::Low)).unwrap();
let uri = &rules.rules["betterleaks.generic-credential-uri"];
let password = "Ab3DeF9gHk2mNp6Qr8St0Vw1";
for (veles_id, destination) in [
("bitbucketcredentials", "bitbucket.org/team/repo.git"),
("codecatalystcredentials", "git.us-east-1.codecatalyst.aws/v1/space/project/repo"),
("codecommitcredentials", "git-codecommit.us-east-1.amazonaws.com/v1/repos/repo"),
] {
let url = format!("https://fixture:{password}@{destination}");
let veles = &rules.rules[&format!("veles.secrets/{veles_id}")];
let regex = veles.as_regex().unwrap();
let captures = regex.captures(url.as_bytes()).unwrap();
assert_eq!(captures.get(1).unwrap().as_bytes(), url.as_bytes());
for path in [
"project/.git/config",
r"C:\project\.git\config",
".git-credentials",
"shell_history",
] {
assert!(
filter_discards(uri, path, password, &url, &[("uri", &url)]),
"{veles_id}: {path}"
);
}
assert!(
!filter_discards(uri, "README.md", password, &url, &[("uri", &url)]),
"{veles_id}"
);
}
for (case, (scheme, username, destination)) in [
("http", "fixture", "bitbucket.org/team/repo.git"),
("HTTPS", "fixture", "bitbucket.org/team/repo.git"),
("https", "", "bitbucket.org/team/repo.git"),
("https", "fixture", "notbitbucket.org/team/repo.git"),
("https", "fixture", "bitbucket.org.evil.invalid/team/repo.git"),
("https", "fixture", "bitbucket.org:443/team/repo.git"),
("https", "fixture", "bitbucket.org/team/repo.GIT"),
("https", "fixture", "bitbucket.org/team/repo"),
("https", "fixture", "codecatalyst.aws/v1/repo"),
("https", "fixture", "git.codecatalyst.aws.evil.invalid/v1/repo"),
("https", "fixture", "git-codecommit.us_east_1.amazonaws.com/v1/repos/repo"),
("https", "fixture", "git-codecommit.us-east-1.amazonaws.com/"),
]
.into_iter()
.enumerate()
{
let url = format!("{scheme}://{username}:{password}@{destination}");
assert!(
uri.as_regex().unwrap().is_match(url.as_bytes()),
"expected URI candidate for negative fixture {case}"
);
assert!(
!filter_discards(uri, "project/.git/config", password, &url, &[("uri", &url)]),
"negative fixture {case} must not be suppressed"
);
}
let openrouter = &rules.rules["betterleaks.openrouter-api-key"];
for (prefix, discarded) in [("sk-or-v1-", true), ("SK-OR-V1-", false)] {
let token = format!("{prefix}{}", "a1b2c3d4e5f60789".repeat(4));
assert!(openrouter.as_regex().unwrap().is_match(token.as_bytes()));
assert_eq!(filter_discards(openrouter, "fixture.txt", &token, &token, &[]), discarded);
}
let heroku = &rules.rules["betterleaks.heroku-api-key-v2"];
let veles_heroku = &rules.rules["veles.secrets/herokuplatformkey"];
let body: String = "Ab3DeF9gHk2mNp6Qr8St0Vw1".chars().cycle().take(57).collect();
for (last, covered) in [("A", true), ("_", true), ("-", false)] {
let token = format!("HRKU-AA{body}{last}");
for suffix in ["", "\n", " ", "\"", ";"] {
let input = format!("{token}{suffix}");
let captures = heroku.as_regex().unwrap().captures(input.as_bytes()).unwrap();
assert_eq!(captures.get(1).unwrap().as_bytes(), token.as_bytes());
assert_eq!(veles_heroku.as_regex().unwrap().is_match(input.as_bytes()), covered);
assert_eq!(
filter_discards(heroku, "fixture.txt", &token, &input, &[]),
covered,
"Heroku suffix: {last:?}/{suffix:?}"
);
}
}
for id in ["betterleaks.heroku-api-key-v2", "betterleaks.slack-app-token"] {
assert!(rules.rules[id].betterleaks_filter.is_some(), "{id}");
assert!(rules.rules[id].revocation.is_some(), "{id}");
}
}
#[test]
fn test_get_default_rules() {
assert!(get_builtin_rules(None).unwrap().num_rules() >= 400);
}
#[test]
fn connection_string_rules_are_available_at_default_confidence() {
let rules = get_builtin_rules(None).unwrap();
let high_confidence = get_builtin_rules(Some(Confidence::High)).unwrap();
for suffix in ["unquoted", "double-quoted", "single-quoted", "odbc-braced"] {
let id = format!("kingfisher.connection-string.password.{suffix}");
let rule = &rules.rules[&id];
assert_eq!(rule.confidence, Confidence::Medium);
assert!(rule.visible);
assert_eq!(rule.as_anchored_regex().unwrap().captures_len(), 2);
assert!(rule.validation.is_none());
assert!(rule.revocation.is_none());
assert!(!high_confidence.rules.contains_key(&id));
}
}
#[test]
fn builtin_catalog_excludes_filename_only_detectors() {
let rules = get_builtin_rules(Some(Confidence::Low)).unwrap();
assert!(!rules.rules.contains_key("betterleaks.pkcs12-file"));
for rule in rules.rules.values() {
assert!(!rule.betterleaks_path_only, "unexpected filename-only rule {}", rule.id);
}
let provenance: serde_json::Value = serde_json::from_str(BUILTIN_RULE_PROVENANCE).unwrap();
assert!(provenance["rules"].get("betterleaks.pkcs12-file").is_none());
}
#[test]
fn bundled_rule_files_are_sorted_and_complete() {
let files = get_builtin_rule_files().unwrap();
assert_eq!(files.len(), 3);
assert!(files.windows(2).all(|pair| pair[0].0 <= pair[1].0));
}
#[test]
fn readable_files_load_identically_to_embedded_catalog() {
let manifest: serde_json::Value = serde_json::from_str(BUILTIN_RULE_PROVENANCE).unwrap();
let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("generated");
let paths: std::collections::BTreeSet<_> = manifest["rules"]
.as_object()
.unwrap()
.values()
.map(|record| root.join(record["generated_file"].as_str().unwrap()))
.collect();
assert_eq!(
paths,
[
root.join("rules/betterleaks.yml"),
root.join("rules/kingfisher.yml"),
root.join("rules/veles.yml"),
]
.into_iter()
.collect()
);
let exported = Rules::from_yaml_files(paths, Confidence::Low).unwrap();
let embedded = get_builtin_rules(Some(Confidence::Low)).unwrap();
assert_eq!(
serde_json::to_value(&exported.rules).unwrap(),
serde_json::to_value(&embedded.rules).unwrap()
);
assert_eq!(
serde_json::to_value(&exported.betterleaks_prefilter).unwrap(),
serde_json::to_value(&embedded.betterleaks_prefilter).unwrap()
);
}
#[test]
fn builtin_rule_files_are_cached() {
assert!(std::ptr::eq(builtin_rule_files().unwrap(), builtin_rule_files().unwrap()));
}
#[test]
fn default_rules_are_namespaced_by_source() {
let rules = get_builtin_rules(Some(Confidence::Low)).unwrap();
assert!(rules.num_rules() >= 400);
assert!(rules.rules.keys().all(|id| {
id.starts_with("betterleaks.")
|| id.starts_with("veles.")
|| id.starts_with("kingfisher.")
}));
for rule in rules.rules.values() {
for dependency in rule.depends_on_rule.iter().flatten() {
assert!(
rules.rules.contains_key(&dependency.rule_id),
"{} depends on missing rule {}",
rule.id,
dependency.rule_id
);
}
}
let app_config = rules
.rules
.get("betterleaks.azure-app-configuration-connection-string")
.expect("generated Betterleaks rule should exist");
let regex = app_config.as_regex().unwrap();
let captures = regex
.captures(
b"Endpoint=https://demo.azconfig.io;Id=abcde;Secret=ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789",
)
.expect("generated Betterleaks rule should match");
assert_eq!(
captures.get(app_config.betterleaks_secret_group.unwrap()).unwrap().as_bytes(),
b"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
);
assert!(rules.betterleaks_prefilter.is_some());
assert!(
rules.rules["betterleaks.browserstack-access-key.1"]
.depends_on_rule
.iter()
.flatten()
.any(|dep| dep.variable == "BROWSERSTACK_USERNAME_1" && dep.verify_candidates)
);
assert!(
rules.rules["betterleaks.clickhouse-cloud-api-secret-key"]
.depends_on_rule
.iter()
.flatten()
.any(|dep| dep.variable == "CLICKHOUSE_CLOUD_KEY_ID" && dep.verify_candidates)
);
assert!(
rules.rules["betterleaks.mongodb-atlas-service-account-secret"]
.depends_on_rule
.iter()
.flatten()
.any(|dep| dep.variable == "MONGODB_ATLAS_SERVICE_ACCOUNT_ID"
&& dep.verify_candidates)
);
assert!(
rules.rules["betterleaks.planetscale-api-token"]
.depends_on_rule
.iter()
.flatten()
.any(|dep| dep.variable == "PLANETSCALE_ID" && dep.verify_candidates)
);
assert!(
rules.rules["betterleaks.razorpay-key-secret.1"]
.depends_on_rule
.iter()
.flatten()
.any(|dep| dep.variable == "RAZORPAY_KEY_ID_1" && dep.verify_candidates)
);
assert!(
rules.rules["betterleaks.wiz-client-secret.1"]
.depends_on_rule
.iter()
.flatten()
.any(|dep| dep.variable == "WIZ_CLIENT_ID_1" && dep.verify_candidates)
);
let aws = rules.rules.get("betterleaks.aws-access-token").unwrap();
assert!(
aws.depends_on_rule
.iter()
.flatten()
.any(|dep| dep.variable == "AWS_SECRET_ACCESS_KEY" && dep.verify_candidates)
);
assert!(matches!(aws.revocation, Some(Revocation::AWS)));
let Some(Validation::Betterleaks(aws_validation)) = &aws.validation else {
panic!("AWS should use Betterleaks validation");
};
assert_eq!(
aws_validation.capabilities.access_map.as_ref().unwrap().handler,
BetterleaksAccessMapHandler::Aws
);
let bindings = aws_validation.capabilities.revocation_bindings.as_ref().unwrap();
assert_eq!(bindings.secret, "components.aws-secret-access-key");
assert_eq!(bindings.variables["AKID"], "finding.secret");
let session = rules
.rules
.get("betterleaks.aws-session-token")
.expect("AWS session-token compatibility rule should exist");
assert_eq!(session.name, "AWS Session Token");
assert!(session.depends_on_rule.iter().flatten().all(|dep| dep.verify_candidates));
assert!(matches!(session.validation, Some(Validation::AWS)));
let session_regex = session.as_regex().unwrap();
let session_captures = session_regex
.captures(br#""AWS_SESSION_TOKEN": "Ab9xQ7mN2kLp4RsT""#)
.expect("AWS session-token rule should match a quoted structured key");
assert_eq!(session_captures.get(1).unwrap().as_bytes(), b"Ab9xQ7mN2kLp4RsT");
assert_eq!(
session
.depends_on_rule
.iter()
.flatten()
.map(|dependency| (dependency.rule_id.as_str(), dependency.variable.as_str()))
.collect::<Vec<_>>(),
vec![
("betterleaks.aws-access-token", "AKID"),
("betterleaks.aws-secret-access-key", "AWS_SECRET_ACCESS_KEY"),
]
);
for (id, handler) in [
("betterleaks.auth0-client-secret.1", BetterleaksAccessMapHandler::Auth0),
("betterleaks.monday-api-token.1", BetterleaksAccessMapHandler::Monday),
("betterleaks.paypal-client-secret.1", BetterleaksAccessMapHandler::Paypal),
] {
let rule = rules.rules.get(id).expect("generated Betterleaks rule should exist");
let Some(Validation::Betterleaks(validation)) = &rule.validation else {
panic!("{id} should use Betterleaks validation");
};
let mapping = validation
.capabilities
.access_map
.as_ref()
.unwrap_or_else(|| panic!("{id} should have access-map capabilities"));
assert_eq!(mapping.handler, handler);
}
let auth0 = rules.rules["betterleaks.auth0-client-secret.1"]
.validation
.as_ref()
.and_then(|validation| match validation {
Validation::Betterleaks(validation) => validation.capabilities.access_map.as_ref(),
_ => None,
})
.expect("Auth0 access-map capabilities should exist");
assert_eq!(auth0.inputs["client_id"], "components.auth0-client-id.1");
assert_eq!(auth0.inputs["domain"], "components.auth0-domain.1");
let paypal = rules.rules["betterleaks.paypal-client-secret.1"]
.validation
.as_ref()
.and_then(|validation| match validation {
Validation::Betterleaks(validation) => validation.capabilities.access_map.as_ref(),
_ => None,
})
.expect("PayPal access-map capabilities should exist");
assert_eq!(paypal.inputs["client_id"], "components.paypal-client-id.1");
let google_api_key = rules.rules.get("betterleaks.gcp-api-key").unwrap();
let Some(Validation::Betterleaks(google_api_key_validation)) = &google_api_key.validation
else {
panic!("Google API key should use Betterleaks validation");
};
assert_eq!(
google_api_key_validation.capabilities.access_map.as_ref().unwrap().handler,
BetterleaksAccessMapHandler::GcpApiKey
);
let gcp = rules.rules.get("betterleaks.gcp-service-account").unwrap();
assert!(matches!(gcp.revocation, Some(Revocation::GCP)));
let generic = rules.rules.get("betterleaks.generic-api-key").expect("generic API-key rule");
assert_eq!(generic.confidence, Confidence::Low);
assert_eq!(generic.specificity, -100);
let generic_filter =
serde_json::to_string(generic.betterleaks_filter.as_ref().unwrap()).unwrap();
assert!(!generic_filter.contains("setConfidence"));
let level_identifier = serde_json::to_string(&crate::BetterleaksExpr::Identifier {
value: "level".to_string(),
})
.unwrap();
assert!(!generic_filter.contains(&level_identifier));
assert!(!generic_filter.contains(r#""name":"level""#));
for id in ["generic-password", "generic-username"] {
assert!(!rules.rules.contains_key(&format!("betterleaks.{id}")), "unexpected {id}");
}
let mongodb = rules.rules.get("betterleaks.mongodb-connection-string").unwrap();
assert!(matches!(mongodb.validation, Some(Validation::MongoDB)));
assert_eq!(mongodb.tls_mode, Some(TlsMode::Lax));
let jwt = rules.rules.get("betterleaks.jwt").unwrap();
assert!(matches!(jwt.validation, Some(Validation::JWT)));
assert_eq!(jwt.tls_mode, Some(TlsMode::Lax));
let private_key = rules.rules.get("betterleaks.private-key").unwrap();
assert!(matches!(private_key.validation, Some(Validation::Assumed)));
let polymarket = rules.rules.get("betterleaks.polymarket-private-key").unwrap();
assert!(matches!(
polymarket.validation,
Some(Validation::Ethereum(EthereumValidation::PrivateKey))
));
for id in [
"generic-credential-uri",
"gitlab-incoming-mail-address-token",
"circleci-project-token",
"cloudflare-api-key.1",
"digitalocean-pat",
"heroku-api-key-v2",
"npm-access-token",
"openrouter-api-key",
"postman-api-token",
"pypi-upload-token",
"sendgrid-api-token",
"slack-app-token",
"slack-config-access-token",
"slack-config-refresh-token",
"xai-api-key",
] {
assert!(rules.rules.contains_key(&format!("betterleaks.{id}")), "missing {id}");
}
assert!(rules.rules.contains_key("betterleaks.voyageai-api-key"));
let hcp = rules.rules.get("veles.secrets/hcpclientcredentials").unwrap();
assert!(matches!(hcp.validation, Some(Validation::Http(_))));
for id in [
"betterleaks.anthropic-api-key",
"betterleaks.gcp-api-key",
"betterleaks.github-app-token",
"betterleaks.notion-api-token",
"betterleaks.openai-api-key",
"betterleaks.resend-api-key.1",
"betterleaks.twilio-api-key",
"betterleaks.vercel-ai-gateway-key",
"betterleaks.doppler-api-token",
"betterleaks.heroku-api-key",
"betterleaks.heroku-api-key-v2",
"betterleaks.slack-legacy-token",
"betterleaks.slack-user-token",
"betterleaks.twitch-api-token",
"betterleaks.vercel-app-access-token",
"betterleaks.vercel-app-refresh-token",
] {
assert!(rules.rules[id].revocation.is_some(), "{id} should support revocation");
}
for id in ["betterleaks.rubygems-api-token", "betterleaks.anthropic-admin-api-key"] {
assert!(rules.rules[id].revocation.is_none(), "{id} needs an owner/partner flow");
}
for id in [
"veles.secrets/bitbucketcredentials",
"veles.secrets/circleciproject",
"veles.secrets/cloudflareapitoken",
"veles.secrets/codecatalystcredentials",
"veles.secrets/codecommitcredentials",
"veles.secrets/digitaloceanapikey",
"veles.secrets/grokxaiapikey",
"veles.secrets/herokuplatformkey",
"veles.secrets/npmjsaccesstoken",
"veles.secrets/openrouter",
"veles.secrets/postmanapikey",
"veles.secrets/postmancollectiontoken",
"veles.secrets/sendgrid",
"veles.secrets/slackappconfigaccesstoken",
"veles.secrets/slackappconfigrefreshtoken",
"veles.secrets/slackappleveltoken",
] {
let rule = &rules.rules[id];
assert!(rule.visible, "{id} should be the visible detector for its format");
assert!(matches!(rule.validation, Some(Validation::Http(_))));
}
for id in [
"veles.secrets/cloudflareapitoken",
"veles.secrets/digitaloceanapikey",
"veles.secrets/herokuplatformkey",
"veles.secrets/npmjsaccesstoken",
"veles.secrets/slackappleveltoken",
] {
assert!(rules.rules[id].revocation.is_some(), "{id} should support revocation");
}
for id in [
"betterleaks.cloudflare-api-key.1",
"betterleaks.digitalocean-pat",
"betterleaks.generic-credential-uri",
"betterleaks.heroku-api-key-v2",
"betterleaks.npm-access-token",
"betterleaks.openrouter-api-key",
"betterleaks.postman-api-token",
"betterleaks.sendgrid-api-token",
"betterleaks.slack-app-token",
"betterleaks.slack-config-access-token",
"betterleaks.slack-config-refresh-token",
"betterleaks.xai-api-key",
] {
assert!(
rules.rules[id].betterleaks_filter.is_some(),
"{id} should retain a Veles source-precedence filter"
);
}
assert!(matches!(
rules.rules["betterleaks.circleci-project-token"].validation,
Some(Validation::Betterleaks(_))
));
let npm = &rules.rules["betterleaks.npm-access-token"];
assert!(filter_discards(
npm,
"config.env",
"npm_ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789",
"npm_ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789",
&[],
));
let cloudflare = &rules.rules["betterleaks.cloudflare-api-key.1"];
let token = ["aB3dE5fG", "7hJ9kL2m", "N4pQ6rS8", "tU0vW1xY", "-z_C9dEf"].concat();
assert!(filter_discards(
cloudflare,
"cloudflare.env",
&token,
&format!("CF_API_TOKEN={token}"),
&[],
));
assert!(filter_discards(
cloudflare,
"application.env",
&token,
&format!("CLOUDFLARE_API_TOKEN={token}"),
&[],
));
let uri = &rules.rules["betterleaks.generic-credential-uri"];
assert!(matches!(uri.validation, Some(Validation::CredentialUri)));
assert_eq!(uri.tls_mode, Some(TlsMode::Lax));
let git_uri = "https://user:password@bitbucket.org/team/repo.git";
let uri_captures = [("uri", git_uri), ("host", "bitbucket.org"), ("password", "password")];
assert!(filter_discards(uri, "project/.git/config", "password", git_uri, &uri_captures,));
assert!(!filter_discards(uri, "README.md", "password", git_uri, &uri_captures));
}
#[test]
fn legacy_aliases_all_resolve_against_the_builtin_catalog() {
let rules = get_builtin_rules(Some(Confidence::Low)).unwrap();
let ids: Vec<&str> = rules.rules.keys().map(String::as_str).collect();
let mut unresolved = Vec::new();
for (family, selectors) in crate::legacy_aliases::legacy_aliases() {
for selector in selectors {
let matched = ids.iter().any(|id| {
*id == selector
|| (id.starts_with(selector.as_str())
&& matches!(id.as_bytes().get(selector.len()), Some(b'.' | b'-')))
});
if !matched {
unresolved.push(format!("{family} -> {selector}"));
}
}
}
assert!(
unresolved.is_empty(),
"legacy rule aliases no longer resolve against the built-in catalog \
(upstream rename, or coverage regression):\n {}",
unresolved.join("\n ")
);
}
}