use std::borrow::Cow;
use std::fmt;
use std::ops::Range;
use serde_json::Value;
#[cfg(test)]
mod tests;
pub const REDACTED: &str = "[REDACTED]";
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub enum Kind {
OpenAiApiKey,
AnthropicApiKey,
GoogleApiKey,
AwsAccessKey,
GitHubToken,
GitLabToken,
SlackToken,
StripeKey,
HuggingFaceToken,
JsonWebToken,
PrivateKey,
Assigned,
}
impl Kind {
pub fn name(self) -> &'static str {
match self {
Self::OpenAiApiKey => "OpenAI API key",
Self::AnthropicApiKey => "Anthropic API key",
Self::GoogleApiKey => "Google API key",
Self::AwsAccessKey => "AWS access key",
Self::GitHubToken => "GitHub token",
Self::GitLabToken => "GitLab token",
Self::SlackToken => "Slack token",
Self::StripeKey => "Stripe key",
Self::HuggingFaceToken => "Hugging Face token",
Self::JsonWebToken => "JSON Web Token",
Self::PrivateKey => "private key",
Self::Assigned => "assigned secret",
}
}
}
impl fmt::Display for Kind {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.name())
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct SecretMatch {
pub kind: Kind,
pub range: Range<usize>,
}
pub fn find(input: &str) -> impl Iterator<Item = SecretMatch> + '_ {
let bytes = input.as_bytes();
let mut at = 0;
let mut ended = 0;
std::iter::from_fn(move || {
while let Some(&byte) = bytes.get(at) {
if !is_token_byte(byte) {
at += 1;
continue;
}
let rest = bytes.get(at..).unwrap_or_default();
let len = rest
.iter()
.position(|&byte| !is_token_byte(byte))
.unwrap_or(rest.len());
let token = rest.get(..len).unwrap_or_default();
if may_be_secret(token)
&& (starts_token(input, at) || at == ended)
&& let Some(rest) = input.get(at..)
{
let found = secret_at(rest).map(|(kind, len)| (kind, 0..len));
let found = found.or_else(|| Some((Kind::Assigned, assigned(rest)?)));
if let Some((kind, range)) = found {
let range = at + range.start..at + range.end;
(at, ended) = (range.end, range.end);
return Some(SecretMatch { kind, range });
}
}
at += len;
}
None
})
}
pub fn has_secret(input: &str) -> bool {
find(input).next().is_some()
}
pub fn text(input: &str) -> Cow<'_, str> {
text_with(input, |_| REDACTED.to_owned())
}
pub fn text_with(input: &str, mut replace: impl FnMut(&str) -> String) -> Cow<'_, str> {
let mut found = find(input).peekable();
if found.peek().is_none() {
return Cow::Borrowed(input);
}
let mut out = String::with_capacity(input.len());
let mut from = 0;
for secret in found {
out.push_str(input.get(from..secret.range.start).unwrap_or_default());
from = secret.range.end;
out.push_str(&replace(input.get(secret.range).unwrap_or_default()));
}
out.push_str(input.get(from..).unwrap_or_default());
Cow::Owned(out)
}
const HELD_MOST: usize = 16 * 1024;
#[derive(Clone, Debug, Default)]
pub struct Streamed {
held: String,
}
impl Streamed {
pub fn push(&mut self, piece: &str) -> String {
self.held.push_str(piece);
let rest = self.held.split_off(settled(&self.held));
std::mem::replace(&mut self.held, rest)
}
pub fn finish(&mut self) -> String {
std::mem::take(&mut self.held)
}
}
fn settled(input: &str) -> usize {
let word = input
.char_indices()
.rfind(|&(_, ch)| !is_token_char(ch) && !matches!(ch, '.' | '+' | '/'))
.map_or(0, |(at, ch)| at + ch.len_utf8());
let line_of = |at: usize| {
input
.get(..at)
.and_then(|before| before.rfind('\n'))
.map_or(0, |at| at + 1)
};
let line = line_of(input.len());
let last = input.get(line..).unwrap_or_default();
let named = SECRET_NAMES.iter().any(|name| {
last.match_indices(name).any(|(at, _)| {
let after = last.get(at + name.len()..).unwrap_or_default();
!after.starts_with(is_token_char)
})
});
let mut cut = word;
if named || last.contains("-----BEGIN ") {
cut = cut.min(line);
}
let found: Vec<_> = find(input).collect();
for found in found.iter().rev() {
let past = cut < found.range.end || found.range.end == input.len();
if found.range.start < cut && past {
cut = line_of(found.range.start);
}
}
let least = input.len().saturating_sub(HELD_MOST);
let cut = input.ceil_char_boundary(cut.max(least));
let whole = |found: &SecretMatch| {
let (start, end) = (found.range.start, found.range.end);
if found.kind != Kind::Assigned {
return start..end;
}
let before = input.get(..start).unwrap_or_default();
let named = before.trim_end_matches(['=', ':', '"', '\'', ' ', '\t']);
let quoted = input
.get(end..)
.is_some_and(|after| after.starts_with(['"', '\'']));
named.trim_end_matches(is_token_char).len()..end + usize::from(quoted)
};
let through = found
.iter()
.map(whole)
.find(|whole| whole.start < cut && cut < whole.end && whole.end < input.len());
through.map_or(cut, |whole| whole.end)
}
pub fn json(value: &mut Value) -> bool {
json_with(value, &mut |_| REDACTED.to_owned())
}
fn json_with(value: &mut Value, replace: &mut impl FnMut(&str) -> String) -> bool {
match value {
Value::String(string) => {
let mut changed = match text_with(string, &mut *replace) {
Cow::Owned(redacted) => {
*string = redacted;
true
}
Cow::Borrowed(_) => false,
};
if string.trim_start().starts_with(['{', '['])
&& let Ok(mut inner) = serde_json::from_str::<Value>(string)
&& json_with(&mut inner, replace)
{
*string = inner.to_string();
changed = true;
}
changed
}
Value::Array(items) => items
.iter_mut()
.fold(false, |any, item| json_with(item, replace) | any),
Value::Object(fields) => fields
.values_mut()
.fold(false, |any, field| json_with(field, replace) | any),
Value::Null | Value::Bool(_) | Value::Number(_) => false,
}
}
fn secret_at(rest: &str) -> Option<(Kind, usize)> {
if rest.starts_with("-----BEGIN ") {
return private_key(rest).map(|len| (Kind::PrivateKey, len));
}
if let Some(len) = json_web_token(rest) {
return Some((Kind::JsonWebToken, len));
}
let token = rest.get(..token_len(rest))?;
if let Some(len) = aws_access_key(token) {
return Some((Kind::AwsAccessKey, len));
}
let kind = TOKENS.iter().find_map(|&(prefix, min, kind)| {
let tail = token.strip_prefix(prefix)?;
is_plausible(tail, min).then_some(kind)
})?;
Some((kind, token.len()))
}
const TOKENS: &[(&str, usize, Kind)] = &[
("sk-ant-", 16, Kind::AnthropicApiKey),
("sk-proj-", 16, Kind::OpenAiApiKey),
("sk-svcacct-", 16, Kind::OpenAiApiKey),
("sk-", 32, Kind::OpenAiApiKey),
("AIza", 20, Kind::GoogleApiKey),
("github_pat_", 22, Kind::GitHubToken),
("ghp_", 30, Kind::GitHubToken),
("gho_", 30, Kind::GitHubToken),
("ghu_", 30, Kind::GitHubToken),
("ghs_", 30, Kind::GitHubToken),
("ghr_", 30, Kind::GitHubToken),
("glpat-", 20, Kind::GitLabToken),
("xoxa-", 10, Kind::SlackToken),
("xoxb-", 10, Kind::SlackToken),
("xoxp-", 10, Kind::SlackToken),
("xoxr-", 10, Kind::SlackToken),
("xoxs-", 10, Kind::SlackToken),
("sk_live_", 16, Kind::StripeKey),
("sk_test_", 16, Kind::StripeKey),
("rk_live_", 16, Kind::StripeKey),
("rk_test_", 16, Kind::StripeKey),
("hf_", 30, Kind::HuggingFaceToken),
];
const SECRET_NAMES: &[&str] = &[
"SECRET",
"TOKEN",
"PASSWORD",
"PASSWD",
"API_KEY",
"ACCESS_KEY",
"SECRET_KEY",
"PRIVATE_KEY",
];
fn assigned(rest: &str) -> Option<Range<usize>> {
let name = rest.get(..token_len(rest))?;
let upper_snake = name.starts_with(|ch: char| ch.is_ascii_uppercase())
&& name
.chars()
.all(|ch| ch.is_ascii_uppercase() || ch.is_ascii_digit() || ch == '_');
let secret_named = SECRET_NAMES.iter().any(|end| {
name.strip_suffix(end)
.is_some_and(|head| head.is_empty() || head.ends_with('_'))
});
if !upper_snake || !secret_named {
return None;
}
let after = rest.get(name.len()..)?;
let after = after.strip_prefix(['"', '\'']).unwrap_or(after);
let after = after.trim_start_matches([' ', '\t']);
let after = after.strip_prefix(['=', ':'])?;
let after = after.trim_start_matches([' ', '\t']);
let start = rest.len() - after.len();
let plausible = |value: &str| {
value.len() >= 8
&& value.chars().all(|ch| ch.is_ascii_graphic() || ch == ' ')
&& !value.starts_with('$')
&& !value.chars().all(|ch| ch.is_ascii_digit())
&& !value.contains(REDACTED)
&& secret_at(value).is_none()
};
let (from, value) = match after.chars().next()? {
quote @ ('"' | '\'') => {
let inner = after.get(1..)?;
(start + 1, inner.get(..inner.find(quote)?)?)
}
_ => {
let whole = after.get(..after.find(ends_value).unwrap_or(after.len()))?;
let trimmed = whole.trim_end_matches(['.', ',', ':', '!', '?', ')']);
(start, if trimmed.len() >= 8 { trimmed } else { whole })
}
};
plausible(value).then(|| from..from + value.len())
}
fn ends_value(ch: char) -> bool {
ch.is_whitespace() || matches!(ch, '"' | '\'' | '`' | ',' | ';' | '&')
}
fn is_plausible(tail: &str, min: usize) -> bool {
tail.len() >= min && tail.chars().any(|ch| ch.is_ascii_alphabetic())
}
fn aws_access_key(token: &str) -> Option<usize> {
let tail = token
.strip_prefix("AKIA")
.or_else(|| token.strip_prefix("ASIA"))?;
let len = tail
.chars()
.take_while(|ch| ch.is_ascii_uppercase() || ch.is_ascii_digit())
.count();
(len == 16).then_some(4 + len)
}
fn json_web_token(rest: &str) -> Option<usize> {
let mut len = 0;
for _ in 0..2 {
let part = rest.get(len..)?;
let segment = token_len(part);
if !part.starts_with("eyJ") || segment < 10 || !part.get(segment..)?.starts_with('.') {
return None;
}
len += segment + 1;
}
Some(len + token_len(rest.get(len..)?))
}
fn may_be_secret(token: &[u8]) -> bool {
let first = token.first().copied().unwrap_or_default();
let second = token.get(1).copied().unwrap_or_default();
let shaped = matches!(
(first, second),
(b's' | b'r', b'k')
| (b'A', b'I' | b'K' | b'S')
| (b'g', b'h' | b'i' | b'l')
| (b'x', b'o')
| (b'h', b'f')
| (b'e', b'y')
| (b'-', b'-')
);
let named =
first.is_ascii_uppercase() && matches!(token.last(), Some(b'T' | b'N' | b'D' | b'Y'));
shaped || named
}
fn is_token_byte(byte: u8) -> bool {
byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-')
}
fn private_key(rest: &str) -> Option<usize> {
let header = rest.get("-----BEGIN ".len()..)?;
let label = header.split_once("-----")?.0;
let is_key = label.ends_with("PRIVATE KEY")
&& label.chars().all(|ch| ch.is_ascii_uppercase() || ch == ' ');
if !is_key {
return None;
}
let end = format!("-----END {label}-----");
Some(
rest.find(end.as_str())
.map_or(rest.len(), |at| at + end.len()),
)
}
fn token_len(rest: &str) -> usize {
rest.find(|ch| !is_token_char(ch)).unwrap_or(rest.len())
}
fn is_token_char(ch: char) -> bool {
ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-')
}
fn starts_token(input: &str, at: usize) -> bool {
input
.get(..at)
.and_then(|before| before.chars().next_back())
.is_none_or(|ch| !is_token_char(ch) && !matches!(ch, '+' | '/'))
}