pub(crate) const TOKEN_PREFIXES: &[(&str, &str)] = &[
("ghp_", "GitHub personal access token"),
("gho_", "GitHub OAuth"),
("ghu_", "GitHub user-to-server"),
("ghs_", "GitHub server-to-server"),
("ghr_", "GitHub refresh"),
("github_pat_", "GitHub fine-grained PAT"),
("glpat-", "GitLab"),
("sk-", "OpenAI & friends"),
("sk_live_", "Stripe secret"),
("sk_test_", "Stripe test secret"),
("rk_live_", "Stripe restricted"),
(
"pk_live_",
"Stripe publishable (not secret, but nobody wants it in history)",
),
("xox", "Slack (xoxb-, xoxp-, xoxa-, …)"),
("AKIA", "AWS access key id"),
("ASIA", "AWS temporary access key id"),
("AIza", "Google API key"),
("ya29.", "Google OAuth access token"),
("npm_", "npm automation token"),
("dop_v1_", "DigitalOcean"),
("shpat_", "Shopify private app token"),
("shpss_", "Shopify shared secret"),
("SG.", "SendGrid"),
("hf_", "Hugging Face"),
("gsk_", "Groq"),
("sk-ant-", "Anthropic"),
("dckr_pat_", "Docker Hub personal access token"),
("nfp_", "Netlify personal access token"),
("fo1_", "Fly.io"),
("hvs.", "HashiCorp Vault service token"),
("pypi-", "PyPI API token"),
("rubygems_", "RubyGems API key"),
("sq0atp-", "Square access token"),
("sq0csp-", "Square OAuth secret"),
("AGE-SECRET-KEY-", "age encryption identity"),
("Bearer ", "an Authorization header, pasted whole"),
("-----BEGIN", "PEM: private keys, certificates"),
];
const MIN_ENTROPY_LEN: usize = 20;
const MIN_ENTROPY_BITS: f64 = 3.2;
pub fn is_probably_secret(text: &str) -> bool {
let text = text.trim();
if text.is_empty() {
return false;
}
if text.starts_with("-----BEGIN") {
return true;
}
if TOKEN_PREFIXES.iter().any(|(prefix, _)| text.starts_with(prefix)) {
return true;
}
if text.contains(char::is_whitespace) {
return false;
}
if text.contains("://") || text.starts_with("www.") || text.contains('\\') || text.starts_with('/') {
return false;
}
if is_jwt(text) {
return true;
}
text.chars().count() >= MIN_ENTROPY_LEN && character_classes(text) >= 3 && entropy_bits(text) >= MIN_ENTROPY_BITS
}
#[cfg(feature = "rules")]
pub(crate) fn is_entropy_candidate(token: &str) -> bool {
if token.contains("://") || token.starts_with("www.") || token.contains('\\') || token.starts_with('/') {
return false;
}
token.chars().count() >= MIN_ENTROPY_LEN && character_classes(token) >= 3 && entropy_bits(token) >= MIN_ENTROPY_BITS
}
fn is_jwt(text: &str) -> bool {
let parts: Vec<&str> = text.split('.').collect();
parts.len() == 3
&& parts[0].starts_with("ey")
&& parts.iter().all(|part| {
!part.is_empty()
&& part
.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '=')
})
}
fn character_classes(text: &str) -> u8 {
let (mut lower, mut upper, mut digit, mut symbol) = (false, false, false, false);
for c in text.chars() {
match c {
c if c.is_lowercase() => lower = true,
c if c.is_uppercase() => upper = true,
c if c.is_ascii_digit() => digit = true,
_ => symbol = true,
}
}
u8::from(lower) + u8::from(upper) + u8::from(digit) + u8::from(symbol)
}
fn entropy_bits(text: &str) -> f64 {
let mut ascii = [0u32; 128];
let mut other: Option<std::collections::HashMap<char, u32>> = None;
let mut total = 0u64;
for c in text.chars() {
let code = c as u32;
if code < 128 {
ascii[code as usize] += 1;
} else {
*other.get_or_insert_with(Default::default).entry(c).or_insert(0) += 1;
}
total += 1;
}
if total == 0 {
return 0.0;
}
let total = total as f64;
let term = |count: u32| -> f64 {
if count == 0 {
return 0.0;
}
let p = f64::from(count) / total;
-p * p.log2()
};
ascii.iter().copied().map(term).sum::<f64>()
+ other
.iter()
.flat_map(|map| map.values().copied())
.map(term)
.sum::<f64>()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn vendor_tokens_and_keys_are_caught_outright() {
assert!(is_probably_secret("ghp_16C7e42F292c6912E7710c838347Ae178B4a"));
assert!(is_probably_secret("sk-ant-api03-abcDEF123456"));
assert!(is_probably_secret("AKIAIOSFODNN7EXAMPLE"));
assert!(is_probably_secret("xoxb-1234-5678-abcdefghijklmnop"));
assert!(is_probably_secret(
"-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1r\n-----END"
));
assert!(is_probably_secret(
"eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0In0.dBjftJeZ4CVPmB92K27uhbUJU1p1r_wW1gFWFOEjXk"
));
}
#[test]
fn random_looking_tokens_are_caught_by_shape() {
assert!(is_probably_secret("Xk7pQm2Rv9Ls4Tz8Wn3Yb6Hd"));
assert!(is_probably_secret("aG9sZG9udGhpc2lzYmFzZTY0ISE=Zm9v"));
}
#[test]
fn ordinary_copies_are_kept() {
assert!(!is_probably_secret(
"https://github.com/klappstuhlpy/funke/releases/tag/v0.3.1"
));
assert!(!is_probably_secret(r"C:\Users\bened\Documents\Coding\funke\README.md"));
assert!(!is_probably_secret(
"cargo clippy --workspace --all-targets -- -D warnings"
));
assert!(!is_probably_secret("bigbenwashere@gmail.com"));
assert!(!is_probably_secret("Guten Morgen, wie geht es dir?"));
assert!(!is_probably_secret("supercalifragilistic"));
assert!(!is_probably_secret("42"));
assert!(!is_probably_secret(""));
}
#[test]
fn a_short_human_password_is_not_detectable_by_shape() {
assert!(!is_probably_secret("Sommer2024!"));
}
#[test]
fn entropy_handles_non_ascii() {
assert!((entropy_bits("äöüß") - 2.0).abs() < 1e-9);
assert_eq!(entropy_bits(&"é".repeat(100_000)), 0.0);
assert_eq!(entropy_bits(""), 0.0);
}
}