use regex::Regex;
use std::sync::OnceLock;
pub(crate) fn redact_sensitive_text(text: &str) -> String {
let out = data_url_base64_regex().replace_all(text, "data:<redacted>;base64,<redacted>");
let out = secret_token_regex().replace_all(&out, "<redacted>");
let out = bearer_regex().replace_all(&out, "${prefix}<redacted>");
assignment_regex()
.replace_all(&out, "${key}${sep}<redacted>")
.into_owned()
}
#[derive(Debug, Default, Clone)]
pub(crate) struct StreamingRedactor {
pending: String,
suppressing_secret_until: Option<char>,
redaction_marker_pending: bool,
}
impl StreamingRedactor {
const HOLD_BACK_CHARS: usize = 96;
const MAX_PENDING_CHARS: usize = Self::HOLD_BACK_CHARS * 8;
pub(crate) fn push(&mut self, text: &str) -> String {
self.pending.push_str(text);
let mut emitted = String::new();
loop {
if self.suppressing_secret_until.is_some() {
if self.redaction_marker_pending {
emitted.push_str(&self.start_pending_secret());
if self.redaction_marker_pending {
break;
}
}
self.drain_suppressed_secret();
if self.suppressing_secret_until.is_none() {
continue;
}
break;
}
let char_count = self.pending.chars().count();
if char_count <= Self::HOLD_BACK_CHARS {
if pending_may_start_secret(&self.pending) {
break;
}
emitted.push_str(&redact_sensitive_text(&std::mem::take(&mut self.pending)));
break;
}
let emit_chars = char_count - Self::HOLD_BACK_CHARS;
let split = char_split_index(&self.pending, emit_chars);
let candidate = &self.pending[..split];
let safe_split = candidate
.rfind(|ch: char| ch.is_whitespace() || matches!(ch, ',' | ';' | '}' | ']'))
.map(|index| index + self.pending[index..].chars().next().unwrap().len_utf8())
.unwrap_or(0);
let value_start = forced_secret_value_start(&self.pending);
if safe_split != 0
&& value_start.is_none()
&& forced_secret_introducer(&self.pending).is_none()
{
emitted.push_str(&redact_sensitive_text(
&self.pending.drain(..safe_split).collect::<String>(),
));
continue;
}
if char_count <= Self::MAX_PENDING_CHARS {
break;
}
let forced_split = char_split_index(&self.pending, char_count - Self::HOLD_BACK_CHARS);
if let Some((value_start, terminator)) =
value_start.filter(|(value_start, _)| *value_start < forced_split)
{
let mut redacted = self.pending.drain(..value_start).collect::<String>();
redacted.push_str("<redacted>");
if matches!(terminator, '"' | '\'') {
self.pending.drain(..terminator.len_utf8());
}
self.suppressing_secret_until = Some(terminator);
emitted.push_str(&redacted);
continue;
}
if let Some((prefix_end, terminator)) = forced_secret_introducer(&self.pending) {
emitted.push_str(&self.pending.drain(..prefix_end).collect::<String>());
self.suppressing_secret_until = Some(terminator);
self.redaction_marker_pending = true;
continue;
}
emitted.push_str(&redact_sensitive_text(
&self.pending.drain(..forced_split).collect::<String>(),
));
}
emitted
}
fn start_pending_secret(&mut self) -> String {
let Some(value_start) = self.pending.find(|ch: char| !ch.is_whitespace()) else {
return std::mem::take(&mut self.pending);
};
let value_start_char = self.pending[value_start..].chars().next().unwrap();
if matches!(value_start_char, '"' | '\'') {
self.suppressing_secret_until = Some(value_start_char);
}
let mut emitted = self.pending.drain(..value_start).collect::<String>();
let terminator = self
.suppressing_secret_until
.expect("suppression terminator");
emitted.push_str("<redacted>");
if matches!(terminator, '"' | '\'') {
self.pending.drain(..terminator.len_utf8());
}
self.redaction_marker_pending = false;
emitted
}
fn drain_suppressed_secret(&mut self) -> String {
let terminator = self
.suppressing_secret_until
.expect("suppression terminator");
let delimiter = self.pending.find(|ch: char| {
if terminator == ' ' {
ch.is_whitespace() || matches!(ch, ',' | ';' | '}' | ']')
} else {
ch == terminator
}
});
match delimiter {
Some(index) => {
let split = if terminator == ' ' {
index
} else {
index + self.pending[index..].chars().next().unwrap().len_utf8()
};
self.pending.drain(..split);
self.suppressing_secret_until = None;
}
None => {
let char_count = self.pending.chars().count();
if char_count > Self::HOLD_BACK_CHARS {
let split = char_split_index(&self.pending, char_count - Self::HOLD_BACK_CHARS);
self.pending.drain(..split);
}
}
}
String::new()
}
pub(crate) fn drain(&mut self) -> String {
if self.suppressing_secret_until.is_some() {
let mut emitted = String::new();
if self.redaction_marker_pending {
emitted.push_str(&self.start_pending_secret());
}
if self.suppressing_secret_until.is_some() {
if let Some(index) = self.pending.find(|ch: char| {
if self.suppressing_secret_until == Some(' ') {
ch.is_whitespace() || matches!(ch, ',' | ';' | '}' | ']')
} else {
self.suppressing_secret_until == Some(ch)
}
}) {
let split = if self.suppressing_secret_until == Some(' ') {
index
} else {
index + self.pending[index..].chars().next().unwrap().len_utf8()
};
self.pending.drain(..split);
self.suppressing_secret_until = None;
} else {
self.pending.clear();
}
}
return emitted;
}
let pending = std::mem::take(&mut self.pending);
if let Some((value_start, terminator)) =
forced_secret_value_start(&pending).filter(|(value_start, terminator)| {
matches!(terminator, '"' | '\'')
&& !pending[*value_start + terminator.len_utf8()..].contains(*terminator)
})
{
let mut emitted = pending[..value_start].to_owned();
emitted.push_str("<redacted>");
self.suppressing_secret_until = Some(terminator);
return emitted;
}
if pending_may_start_secret(&pending) {
self.pending = pending;
return String::new();
}
redact_sensitive_text(&pending)
}
pub(crate) fn flush(&mut self) -> String {
if self.suppressing_secret_until.is_some() {
let mut emitted = String::new();
if self.redaction_marker_pending {
emitted.push_str(&self.start_pending_secret());
}
self.drain_suppressed_secret();
if self.suppressing_secret_until.is_some() {
self.pending.clear();
self.suppressing_secret_until = None;
self.redaction_marker_pending = false;
return emitted;
}
if !self.pending.is_empty() {
emitted.push_str(&redact_sensitive_text(&std::mem::take(&mut self.pending)));
}
return emitted;
}
let pending = std::mem::take(&mut self.pending);
if let Some((value_start, _)) =
forced_secret_value_start(&pending).filter(|(value_start, terminator)| {
matches!(terminator, '"' | '\'')
&& !pending[*value_start + terminator.len_utf8()..].contains(*terminator)
})
{
let mut emitted = pending[..value_start].to_owned();
emitted.push_str("<redacted>");
return emitted;
}
redact_sensitive_text(&pending)
}
}
fn forced_secret_introducer(text: &str) -> Option<(usize, char)> {
let lower = text.to_ascii_lowercase();
if lower.starts_with("bearer")
&& lower["bearer".len()..]
.chars()
.next()
.is_some_and(char::is_whitespace)
{
let end = text["bearer".len()..]
.find(|ch: char| !ch.is_whitespace())
.unwrap_or(text.len() - "bearer".len())
+ "bearer".len();
return Some((end, ' '));
}
let separator = text.find(['=', ':'])?;
let key =
text[..separator].trim_matches(|ch: char| ch.is_whitespace() || matches!(ch, '"' | '\''));
if !is_credential_like_key(key) {
return None;
}
let end = text[separator + 1..]
.find(|ch: char| !ch.is_whitespace())
.unwrap_or(text.len() - separator - 1)
+ separator
+ 1;
Some((end, ' '))
}
fn forced_secret_value_start(text: &str) -> Option<(usize, char)> {
let (prefix_end, _) = forced_secret_introducer(text)?;
let value_start = text[prefix_end..].find(|ch: char| !ch.is_whitespace())? + prefix_end;
let value_start_char = text[value_start..].chars().next()?;
Some((
value_start,
if matches!(value_start_char, '"' | '\'') {
value_start_char
} else {
' '
},
))
}
fn char_split_index(text: &str, chars: usize) -> usize {
text.char_indices()
.nth(chars)
.map(|(index, _)| index)
.unwrap_or(text.len())
}
fn pending_may_start_secret(text: &str) -> bool {
if let Some((_, _terminator)) =
forced_secret_value_start(text).filter(|(value_start, terminator)| {
matches!(terminator, '"' | '\'')
&& !text[*value_start + terminator.len_utf8()..].contains(*terminator)
})
{
return true;
}
let trimmed_end = text.trim_end_matches(char::is_whitespace);
let token = trimmed_end
.rsplit(|ch: char| ch.is_whitespace() || matches!(ch, ',' | ';' | '{' | '['))
.next()
.unwrap_or(text)
.trim_matches(|ch: char| matches!(ch, '"' | '\''));
let normalized = token.to_ascii_lowercase().replace('-', "_");
if let Some(separator) = trimmed_end.find(['=', ':']) {
let key = trimmed_end[..separator]
.trim_matches(|ch: char| ch.is_whitespace() || matches!(ch, '"' | '\''));
if is_credential_like_key(key)
&& (redact_sensitive_text(text) == text || !text.ends_with(char::is_whitespace))
{
return true;
}
}
if trimmed_end.len() != text.len()
&& normalized != "bearer"
&& redact_sensitive_text(text) != text
{
return false;
}
if normalized.is_empty() {
return false;
}
if trimmed_end
.split_whitespace()
.rev()
.nth(1)
.is_some_and(|word| word.eq_ignore_ascii_case("bearer"))
{
return true;
}
if normalized == "bearer" || normalized.starts_with("bearer ") {
return true;
}
if normalized.contains('=') || normalized.contains(':') {
return normalized.split(['=', ':']).next().is_some_and(|key| {
let key = key.trim_matches(|ch: char| matches!(ch, '"' | '\''));
is_credential_like_key(key)
});
}
if normalized == "s"
|| normalized == "sk"
|| normalized.starts_with("sk_")
|| normalized == "e"
|| normalized == "ey"
|| normalized.starts_with("eyj")
{
return true;
}
[
"api_",
"api_k",
"api_ke",
"api_key",
"apikey",
"authorization",
"bearer",
"password",
"passwd",
"secret",
"token",
"access_",
"access_token",
"refresh_",
"refresh_token",
"auth_",
"auth_token",
"id_",
"id_token",
"client_",
"client_secret",
"exa_",
"exa_api_key",
"account_",
"account_id",
"accountid",
"chatgpt_",
"chatgpt_account_id",
]
.iter()
.any(|prefix| prefix.starts_with(normalized.as_str()) || normalized.starts_with(prefix))
}
pub(crate) fn is_credential_like_key(key: &str) -> bool {
let normalized = key.to_ascii_lowercase().replace('-', "_");
matches!(
normalized.as_str(),
"authorization"
| "api_key"
| "apikey"
| "password"
| "passwd"
| "secret"
| "token"
| "access_token"
| "accesstoken"
| "refresh_token"
| "refreshtoken"
| "auth_token"
| "authtoken"
| "id_token"
| "idtoken"
| "client_secret"
| "clientsecret"
| "account_id"
| "accountid"
| "chatgpt_account_id"
) || normalized.ends_with("_api_key")
|| normalized.ends_with("_secret")
}
fn data_url_base64_regex() -> &'static Regex {
static RE: OnceLock<Regex> = OnceLock::new();
RE.get_or_init(|| {
Regex::new(r"data:[A-Za-z0-9.+_-]+/[A-Za-z0-9.+_-]+;base64,[A-Za-z0-9+/=_-]+").unwrap()
})
}
fn secret_token_regex() -> &'static Regex {
static REGEX: OnceLock<Regex> = OnceLock::new();
REGEX.get_or_init(|| Regex::new(r"\b(?:sk-[A-Za-z0-9_.-]+|eyJ[A-Za-z0-9_.-]+)\b").unwrap())
}
fn bearer_regex() -> &'static Regex {
static REGEX: OnceLock<Regex> = OnceLock::new();
REGEX.get_or_init(|| Regex::new(r"(?i)(?P<prefix>\bbearer\s+)[A-Za-z0-9._~+/=-]{8,}").unwrap())
}
fn assignment_regex() -> &'static Regex {
static REGEX: OnceLock<Regex> = OnceLock::new();
REGEX.get_or_init(|| {
Regex::new(
r#"(?ix)
(?P<key>["']?\b(?:authorization|exa[_-]?api[_-]?key|api[_-]?key|password|passwd|secret|token|access[_-]?token|refresh[_-]?token|auth[_-]?token|id[_-]?token|client[_-]?secret|account[_-]?id|accountId|chatgpt-account-id)\b["']?)
(?P<sep>\s*[:=]\s*)
(?P<value>"[^"]*"|'[^']*'|[^\s,;}]+)
"#,
)
.unwrap()
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn redact_sensitive_text_redacts_base64_data_urls() {
let text = redact_sensitive_text("before data:image/png;base64,YWJj after");
assert_eq!(text, "before data:<redacted>;base64,<redacted> after");
assert!(!text.contains("data:image/png;base64"));
assert!(!text.contains("YWJj"));
}
#[test]
fn streaming_redactor_emits_long_delimiter_free_text_before_flush() {
let mut redactor = StreamingRedactor::default();
let mut emitted = String::new();
for _ in 0..10 {
emitted.push_str(&redactor.push(&"x".repeat(StreamingRedactor::MAX_PENDING_CHARS)));
}
emitted.push_str(&redactor.flush());
assert!(!emitted.is_empty());
assert_eq!(emitted.len(), StreamingRedactor::MAX_PENDING_CHARS * 10);
}
#[test]
fn streaming_redactor_suppresses_long_delimiter_free_secret_value() {
let mut redactor = StreamingRedactor::default();
let secret = "s".repeat(StreamingRedactor::MAX_PENDING_CHARS * 3);
let mut emitted = String::new();
emitted.push_str(&redactor.push("api_key="));
for chunk in secret.as_bytes().chunks(StreamingRedactor::HOLD_BACK_CHARS) {
emitted.push_str(&redactor.push(std::str::from_utf8(chunk).unwrap()));
}
emitted.push_str(&redactor.flush());
assert!(emitted.contains("api_key=<redacted>"), "{emitted}");
assert!(
!emitted.contains(&secret[..StreamingRedactor::MAX_PENDING_CHARS]),
"{emitted}"
);
}
#[test]
fn streaming_redactor_holds_split_sk_secret_until_flush() {
let mut redactor = StreamingRedactor::default();
let mut emitted = String::new();
emitted.push_str(&redactor.push("sk-"));
assert_eq!(emitted, "");
emitted.push_str(&redactor.push("abc123secret"));
assert_eq!(emitted, "");
emitted.push_str(&redactor.flush());
assert_eq!(emitted, "<redacted>");
assert!(!emitted.contains("sk-"));
assert!(!emitted.contains("abc123secret"));
}
#[test]
fn streaming_redactor_holds_split_eyj_secret_until_flush() {
let mut redactor = StreamingRedactor::default();
let mut emitted = String::new();
emitted.push_str(&redactor.push("eyJ"));
assert_eq!(emitted, "");
emitted.push_str(&redactor.push("abc123secret"));
assert_eq!(emitted, "");
emitted.push_str(&redactor.flush());
assert_eq!(emitted, "<redacted>");
assert!(!emitted.contains("eyJ"));
assert!(!emitted.contains("abc123secret"));
}
#[test]
fn generic_assignments_remain_unchanged_across_streaming_boundaries() {
for (prefix, value) in [("key", "value"), ("code", "200"), ("code", "ok")] {
let input = format!("{prefix}: {value}");
assert_eq!(redact_sensitive_text(&input), input);
let mut redactor = StreamingRedactor::default();
let mut streamed = String::new();
for chunk in input.as_bytes().chunks(2) {
streamed.push_str(&redactor.push(std::str::from_utf8(chunk).unwrap()));
}
streamed.push_str(&redactor.flush());
assert_eq!(streamed, input);
}
}
#[test]
fn streaming_redaction_matches_whole_text_at_every_secret_boundary() {
for (input, secret) in [
("api_key=api-secret", "api-secret"),
("Authorization: Bearer abcdefghijk", "abcdefghijk"),
("Bearer abcdefghijk", "abcdefghijk"),
("sk-abc123secret", "sk-abc123secret"),
("eyJabc123secret", "eyJabc123secret"),
("exa_api_key=provider-secret", "provider-secret"),
] {
let expected = redact_sensitive_text(input);
for split in 1..input.len() {
let mut redactor = StreamingRedactor::default();
let mut streamed = redactor.push(&input[..split]);
streamed.push_str(&redactor.push(&input[split..]));
streamed.push_str(&redactor.flush());
assert_eq!(streamed, expected, "{input:?} split at {split}");
assert!(!streamed.contains(secret), "{input:?} split at {split}");
}
}
}
#[test]
fn forced_drain_preserves_credential_context_for_long_values() {
let cases = [
("Bearer ", "bearer-secret", " after"),
("api_key = ", "assignment-secret", " next"),
(
"password = \"",
"quoted password with whitespace ",
"\" next",
),
];
for (prefix, value, suffix) in cases {
let input = format!(
"{prefix}{}{suffix}",
value.repeat(StreamingRedactor::MAX_PENDING_CHARS)
);
let expected = redact_sensitive_text(&input);
let splits = if prefix.contains("password") {
vec![prefix.len(), prefix.len() + 1, input.len() / 2]
} else {
(1..input.len()).collect()
};
for split in splits {
let mut redactor = StreamingRedactor::default();
let mut streamed = redactor.push(&input[..split]);
streamed.push_str(&redactor.push(&input[split..]));
streamed.push_str(&redactor.flush());
assert_eq!(streamed, expected, "{prefix:?} split at {split}");
assert!(!streamed.contains(value), "{prefix:?} split at {split}");
}
}
}
#[test]
fn oversized_credential_separators_keep_redaction_context() {
for (prefix, value, suffix) in [
("Bearer ", "bearer-secret", " after"),
("api_key=", "assignment-secret", " next"),
] {
let input = format!(
"{prefix}{}{value}{suffix}",
" ".repeat(StreamingRedactor::MAX_PENDING_CHARS + 1)
);
let expected = redact_sensitive_text(&input);
let mut redactor = StreamingRedactor::default();
let mut streamed = String::new();
for chunk in input.chars().map(|ch| ch.to_string()) {
streamed.push_str(&redactor.push(&chunk));
}
streamed.push_str(&redactor.flush());
assert_eq!(streamed, expected, "{prefix:?}");
assert!(!streamed.contains(value), "{prefix:?}: {streamed:?}");
}
}
#[test]
fn single_push_flush_preserves_redaction_marker_and_suffix() {
for (input, secret) in [
(
format!(
"Bearer {}bearer-secret after",
" ".repeat(StreamingRedactor::MAX_PENDING_CHARS + 1)
),
"bearer-secret",
),
(
format!(
"api_key={}api-secret next",
" ".repeat(StreamingRedactor::MAX_PENDING_CHARS + 1)
),
"api-secret",
),
(
format!(
"api_key = \"{}quoted-secret with spaces\" next",
" ".repeat(StreamingRedactor::MAX_PENDING_CHARS + 1)
),
"quoted-secret with spaces",
),
] {
let expected = redact_sensitive_text(&input);
let mut redactor = StreamingRedactor::default();
let streamed = format!("{}{}", redactor.push(&input), redactor.flush());
assert_eq!(streamed, expected, "{input:?}");
assert!(streamed.contains("<redacted>"), "{streamed:?}");
assert!(!streamed.contains(secret), "{streamed:?}");
assert!(streamed.ends_with(" next") || streamed.ends_with(" after"));
}
}
#[test]
fn credential_assignments_remain_redacted_after_policy_narrowing() {
let input =
"api_key = api-secret authorization: Bearer abcdefghijk exa_api_key=provider-secret";
let redacted = redact_sensitive_text(input);
assert!(
redacted
== "api_key = <redacted> authorization: <redacted> <redacted> exa_api_key=<redacted>",
"{redacted:?}"
);
}
#[test]
fn redact_sensitive_text_handles_common_secret_shapes_without_counts() {
let redacted = redact_sensitive_text(
r#"Bearer abcdefghijk password = "open sesame" token=tokensecret {"accountId":"acct-secret-123","access_token":"plain-secret","api_key":"api-secret","input_tokens":42,"token_estimate":9,"status":"error"} cwd=/tmp/project id=call_123"#,
);
assert!(redacted.contains("Bearer <redacted>"));
assert!(redacted.contains("password = <redacted>"));
assert!(redacted.contains("token=<redacted>"));
assert!(redacted.contains(r#""input_tokens":42"#));
assert!(redacted.contains(r#""token_estimate":9"#));
assert!(!redacted.contains("abcdefghijk"));
assert!(!redacted.contains("open sesame"));
assert!(!redacted.contains("tokensecret"));
}
#[test]
fn credential_like_key_does_not_match_usage_counts() {
assert!(!is_credential_like_key("input_tokens"));
assert!(!is_credential_like_key("output_token_count"));
assert!(!is_credential_like_key("token_estimate"));
assert!(is_credential_like_key("access_token"));
assert!(is_credential_like_key("api_key"));
assert!(!is_credential_like_key("key"));
}
#[test]
fn streaming_redactor_redacts_matched_and_unmatched_quoted_values() {
for quote in ['"', '\''] {
let input = format!("api_key = {quote}quoted value with spaces{quote} trailing");
let expected = redact_sensitive_text(&input);
let mut redactor = StreamingRedactor::default();
let streamed = format!("{}{}", redactor.push(&input), redactor.flush());
assert_eq!(streamed, expected, "{input:?}");
let input = format!("api_key = {quote}unmatched quoted value");
let mut redactor = StreamingRedactor::default();
let streamed = format!("{}{}", redactor.push(&input), redactor.flush());
assert_eq!(streamed, "api_key = <redacted>", "{input:?}");
assert!(!streamed.contains("unmatched quoted value"));
}
}
#[test]
fn push_keeps_persistent_pending_within_strict_bound() {
let suffix = "é".repeat(StreamingRedactor::MAX_PENDING_CHARS * 3);
let input = format!("ordinary, {suffix}");
let mut redactor = StreamingRedactor::default();
let emitted = redactor.push(&input);
assert!(redactor.pending.chars().count() <= StreamingRedactor::MAX_PENDING_CHARS);
assert_eq!(format!("{emitted}{}", redactor.flush()), input);
}
#[test]
fn nonterminal_drain_preserves_quoted_suppression_until_finish() {
for quote in ['"', '\''] {
let mut redactor = StreamingRedactor::default();
let mut output = redactor.push(&format!("api_key={quote}first\n"));
output.push_str(&redactor.drain());
output.push_str(&redactor.push(&format!("second{quote} suffix")));
output.push_str(&redactor.flush());
assert_eq!(output.matches("<redacted>").count(), 1, "{output:?}");
assert!(!output.contains("first"), "{output:?}");
assert!(!output.contains("second"), "{output:?}");
assert!(output.ends_with(" suffix"), "{output:?}");
}
}
}