use std::sync::LazyLock;
use regex::{Captures, Regex};
use crate::schema::Memory;
mod assign;
mod keys;
use assign::redact_assignments;
use keys::redact_private_keys;
#[non_exhaustive]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum SecretKind {
PrivateKey,
AwsAccessKeyId,
AwsSecretKey,
GithubToken,
ApiKey,
SlackToken,
SlackWebhook,
GoogleApiKey,
Jwt,
BearerToken,
BasicAuth,
UrlCredentials,
Secret,
}
impl SecretKind {
pub fn as_str(self) -> &'static str {
match self {
SecretKind::PrivateKey => "private_key",
SecretKind::AwsAccessKeyId => "aws_access_key_id",
SecretKind::AwsSecretKey => "aws_secret_key",
SecretKind::GithubToken => "github_token",
SecretKind::ApiKey => "api_key",
SecretKind::SlackToken => "slack_token",
SecretKind::SlackWebhook => "slack_webhook",
SecretKind::GoogleApiKey => "google_api_key",
SecretKind::Jwt => "jwt",
SecretKind::BearerToken => "bearer_token",
SecretKind::BasicAuth => "basic_auth",
SecretKind::UrlCredentials => "url_credentials",
SecretKind::Secret => "secret",
}
}
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Finding {
pub kind: SecretKind,
pub count: usize,
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Redacted {
pub text: String,
pub findings: Vec<Finding>,
}
impl Redacted {
pub fn changed(&self) -> bool {
!self.findings.is_empty()
}
}
pub fn redact(text: &str) -> Redacted {
let mut findings = Vec::new();
let mut current = redact_private_keys(text, &mut findings);
for rule in RULES.iter() {
if let Some(next) = rule.apply(current.as_deref().unwrap_or(text), &mut findings) {
current = Some(next);
}
}
if let Some(next) = redact_assignments(current.as_deref().unwrap_or(text), &mut findings) {
current = Some(next);
}
Redacted {
text: current.unwrap_or_else(|| text.to_string()),
findings,
}
}
pub fn redact_for_store(text: &str) -> String {
let redacted = redact(text);
if redacted.changed() {
let kinds = redacted
.findings
.iter()
.map(|f| format!("{}={}", f.kind.as_str(), f.count))
.collect::<Vec<_>>()
.join(",");
tracing::info!(%kinds, "redacted secrets before storing");
}
redacted.text
}
fn record(findings: &mut Vec<Finding>, kind: SecretKind, count: usize) {
match findings.iter_mut().find(|f| f.kind == kind) {
Some(f) => f.count += count,
None => findings.push(Finding { kind, count }),
}
}
fn marker(kind: SecretKind) -> String {
format!("[REDACTED:{}]", kind.as_str())
}
type Accept = fn(&Captures, &str) -> bool;
struct Rule {
kind: SecretKind,
re: Regex,
accept: Accept,
}
impl Rule {
fn new(kind: SecretKind, pattern: &str, accept: Accept) -> Self {
Self {
kind,
re: Regex::new(pattern).expect("redaction pattern compiles"),
accept,
}
}
fn apply(&self, text: &str, findings: &mut Vec<Finding>) -> Option<String> {
let mut out = String::new();
let mut last = 0;
let mut count = 0;
let mut pos = 0;
while let Some(caps) = self.re.captures_at(text, pos) {
let whole = caps.get(0).expect("group 0 always matches");
let span = caps.name("secret").unwrap_or(whole);
let (start, end) = (span.start(), span.end());
let accepted = start >= last
&& !text[end..].starts_with("[REDACTED:")
&& (self.accept)(&caps, span.as_str());
if !accepted {
pos = if start > whole.start() && span.len() <= MAX_VALUE {
start
} else {
whole.end()
};
continue;
}
out.push_str(&text[last..start]);
out.push_str(&marker(self.kind));
last = end;
pos = end.max(whole.end());
count += 1;
}
if count == 0 {
return None;
}
out.push_str(&text[last..]);
record(findings, self.kind, count);
Some(out)
}
}
const MAX_VALUE: usize = 512;
fn always(_: &Captures, _: &str) -> bool {
true
}
fn group<'t>(caps: &Captures<'t>, name: &str) -> &'t str {
caps.name(name).map_or("", |m| m.as_str())
}
fn has_random_run(s: &[u8], min: usize, member: fn(u8) -> bool) -> bool {
let (mut run, mut upper, mut lower, mut digit) = (0, false, false, false);
for &b in s {
if member(b) {
run += 1;
upper |= b.is_ascii_uppercase();
lower |= b.is_ascii_lowercase();
digit |= b.is_ascii_digit();
} else {
(run, upper, lower, digit) = (0, false, false, false);
}
if run >= min && upper && lower && digit {
return true;
}
}
false
}
static TYPE_NAME: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(r"^(?:[A-Z]{2,}[0-9]*)?(?:[A-Z][a-z]+[0-9]*)+$").expect("pattern compiles")
});
fn has_digit(s: &str) -> bool {
s.bytes().any(|b| b.is_ascii_digit())
}
fn mixed_case_with_digit(s: &str) -> bool {
s.bytes().any(|b| b.is_ascii_uppercase())
&& s.bytes().any(|b| b.is_ascii_lowercase())
&& has_digit(s)
}
fn is_placeholder(s: &str) -> bool {
s.starts_with(['$', '{', '<', '*', '%'])
}
fn sk_key_body(caps: &Captures, _: &str) -> bool {
let body = group(caps, "body");
let lowercase_run = body
.split(|c: char| !(c.is_ascii_lowercase() || c.is_ascii_digit()))
.any(|run| {
run.len() >= 32 && has_digit(run) && run.bytes().any(|b| b.is_ascii_lowercase())
});
mixed_case_with_digit(body) || lowercase_run || has_uuid(body)
}
fn url_credentials(caps: &Captures, _: &str) -> bool {
let user = group(caps, "user");
let pass = caps
.name("pass")
.or_else(|| caps.name("slashed"))
.map_or("", |m| m.as_str());
let port = caps.name("slashed").is_some_and(|m| {
let digits = m.as_str().bytes().take_while(u8::is_ascii_digit).count();
digits > 0 && m.as_str()[digits..].starts_with('/')
});
let stand_in = pass == user
|| ["password", "pass", "secret", "changeme"]
.iter()
.any(|word| pass.eq_ignore_ascii_case(word))
|| pass.bytes().all(|b| matches!(b, b'*' | b'x' | b'X'));
let scp_path = caps.name("slashed").is_some() && user.contains('@');
caps.name("digest").is_none()
&& !scp_path
&& !is_placeholder(user)
&& !is_placeholder(pass)
&& !port
&& !stand_in
}
fn has_uuid(s: &str) -> bool {
UUID.is_match(s)
}
static UUID: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")
.expect("pattern compiles")
});
const MYSQL_COMMAND: &str = r#"(?m)(?:(?:^|[ \t])(?:[^\s\-]\S*|--)[ \t]+|(?:^|[;&|(`"']|\$|\\n)[ \t]*)(?:\S*bin/)?(?:mysql(?:dump|admin|import|check|show)?|mariadb)[ \t]"#;
const TOKEN_START: &str = r"(?:^|[^A-Za-z0-9_\]]|\\[nrt])";
static RULES: LazyLock<Vec<Rule>> = LazyLock::new(|| {
let token = |pattern: &str| format!("{TOKEN_START}(?P<secret>{pattern})");
vec![
Rule::new(
SecretKind::Jwt,
&token(r"eyJ[A-Za-z0-9_\-]{10,}\.eyJ[A-Za-z0-9_\-]{10,}\.[A-Za-z0-9_\-]*"),
always,
),
Rule::new(
SecretKind::GithubToken,
&token(r"gh[pousr]_[A-Za-z0-9]{36,}|github_pat_[A-Za-z0-9_]{22,}"),
always,
),
Rule::new(
SecretKind::ApiKey,
&token(r"sk-(?P<body>[A-Za-z0-9_\-]{20,})"),
sk_key_body,
),
Rule::new(
SecretKind::ApiKey,
&token(r"[rs]k_(?:live|test)_[A-Za-z0-9]{20,}"),
always,
),
Rule::new(
SecretKind::ApiKey,
&token(r"glpat-[A-Za-z0-9_\-]{20,}|npm_[A-Za-z0-9]{36,}|hf_[A-Za-z0-9]{34,}"),
always,
),
Rule::new(
SecretKind::SlackToken,
&token(
r"xoxd-[A-Za-z0-9%+/=\-]{10,}|(?:xoxe\.)?(?:xox[abposrce]|xapp)-[A-Za-z0-9\-]{10,}",
),
|_, value| {
let digit_run = value
.split(|c: char| !c.is_ascii_digit())
.any(|run| run.len() >= 8);
digit_run || has_random_run(value.as_bytes(), 16, |b| b.is_ascii_alphanumeric())
},
),
Rule::new(
SecretKind::SlackWebhook,
r"https://hooks\.slack\.com/services/[A-Za-z0-9/_\-]{20,}",
always,
),
Rule::new(
SecretKind::GoogleApiKey,
&token(r"AIza[0-9A-Za-z_\-]{35}"),
always,
),
Rule::new(
SecretKind::AwsAccessKeyId,
&token(
r"(?:AKIA|ASIA|ABIA|ACCA|AGPA|AIDA|AIPA|ANPA|ANVA|APKA|AROA|ASCA)[A-Z0-9]{16}\b",
),
always,
),
Rule::new(
SecretKind::AwsSecretKey,
r"\[REDACTED:aws_access_key_id\][ \t:,]+(?P<secret>[A-Za-z0-9/+=]{40})(?:[^A-Za-z0-9/+=]|$)",
always,
),
Rule::new(
SecretKind::AwsSecretKey,
r#"(?i)(?:aws[_ ]?)?secret[_ ]?(?:access[_ ]?)?key(?:\\?["'])?(?:[ \t]*\[[^\]\n]*\])?(?:[ \t]*(?:=|:=|:|=>)[ \t]*|[ \t]+)(?:\\?["'])?(?P<secret>[A-Za-z0-9/+=]{40})(?:[^A-Za-z0-9/+=]|$)"#,
always,
),
Rule::new(
SecretKind::BearerToken,
r#"(?i)\b(?:bearer|authorization(?:\\?["'])?\s*:\s*(?:\\?["'])?token)\s+(?P<secret>[A-Za-z0-9\-._~+/]{16,}=*)"#,
|_, value| {
has_digit(value)
|| (value.len() >= 20
&& value.bytes().any(|b| b.is_ascii_uppercase())
&& value.bytes().any(|b| b.is_ascii_lowercase())
&& !TYPE_NAME.is_match(value))
},
),
Rule::new(
SecretKind::BasicAuth,
r#"(?i)\bauthorization(?:\\?["'])?\s*:\s*(?:\\?["'])?basic\s+(?P<secret>[A-Za-z0-9+/]{8,}={0,2})"#,
always,
),
Rule::new(
SecretKind::UrlCredentials,
r#"\b[A-Za-z][A-Za-z0-9+.\-]*://(?P<secret>(?P<user>[^\s:/\[\]"'<>]*):(?:(?P<pass>[^\s/?#\[\]"'<>]+)|(?P<slashed>[^\s?#\[\]"'<>@]+)))@(?P<digest>sha(?:256|512):)?"#,
url_credentials,
),
Rule::new(
SecretKind::BasicAuth,
r#"\bcurl\b[^\n|;&]*?\s(?:-u|--user)[ \t]*["']?(?P<user>[^\s:'"]+):(?P<secret>[^\s'"\[]+)"#,
|caps, value| !is_placeholder(group(caps, "user")) && !is_placeholder(value),
),
Rule::new(
SecretKind::Secret,
&format!(r#"{MYSQL_COMMAND}(?:[^\n|;&]*?[ \t])?-p["']?(?P<secret>[^\s'"\[]+)"#),
|_, value| !is_placeholder(value),
),
Rule::new(
SecretKind::Secret,
r#"\b(?:docker|podman|nerdctl|helm(?:[ \t]+registry)?)[ \t]+login\b[^\n|;&]*?\s-p[ \t]+["']?(?P<secret>[^\s'"\[]+)"#,
|_, value| !is_placeholder(value),
),
Rule::new(
SecretKind::Secret,
r#"\bsshpass[ \t]+-p[ \t]*["']?(?P<secret>[^\s'"\[]+)"#,
|_, value| !is_placeholder(value),
),
Rule::new(
SecretKind::Secret,
r"\b(?:machine\s+\S+|default)\s+login\s+\S+\s+password\s+(?P<secret>[^\s\[]+)",
|_, value| !is_placeholder(value),
),
]
});
pub fn redact_memory(memory: &mut Memory) -> bool {
let content = redact_for_store(&memory.content);
let mut changed = content != memory.content;
memory.content = content;
if let Some(context) = memory.context.as_deref() {
let redacted = redact_for_store(context);
changed |= redacted != context;
memory.context = Some(redacted);
}
changed
}
#[cfg(test)]
mod tests;