1use std::sync::LazyLock;
2
3use regex::{Captures, Regex};
4
5use crate::schema::Memory;
6
7mod assign;
8mod keys;
9
10use assign::redact_assignments;
11use keys::redact_private_keys;
12
13#[non_exhaustive]
14#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
15pub enum SecretKind {
16 PrivateKey,
17 AwsAccessKeyId,
18 AwsSecretKey,
19 GithubToken,
20 ApiKey,
21 SlackToken,
22 SlackWebhook,
23 GoogleApiKey,
24 Jwt,
25 BearerToken,
26 BasicAuth,
27 UrlCredentials,
28 Secret,
29}
30
31impl SecretKind {
32 pub fn as_str(self) -> &'static str {
34 match self {
35 SecretKind::PrivateKey => "private_key",
36 SecretKind::AwsAccessKeyId => "aws_access_key_id",
37 SecretKind::AwsSecretKey => "aws_secret_key",
38 SecretKind::GithubToken => "github_token",
39 SecretKind::ApiKey => "api_key",
40 SecretKind::SlackToken => "slack_token",
41 SecretKind::SlackWebhook => "slack_webhook",
42 SecretKind::GoogleApiKey => "google_api_key",
43 SecretKind::Jwt => "jwt",
44 SecretKind::BearerToken => "bearer_token",
45 SecretKind::BasicAuth => "basic_auth",
46 SecretKind::UrlCredentials => "url_credentials",
47 SecretKind::Secret => "secret",
48 }
49 }
50}
51
52#[non_exhaustive]
53#[derive(Debug, Clone, PartialEq, Eq)]
54pub struct Finding {
55 pub kind: SecretKind,
56 pub count: usize,
57}
58
59#[non_exhaustive]
60#[derive(Debug, Clone, PartialEq, Eq)]
61pub struct Redacted {
62 pub text: String,
63 pub findings: Vec<Finding>,
65}
66
67impl Redacted {
68 pub fn changed(&self) -> bool {
69 !self.findings.is_empty()
70 }
71}
72
73pub fn redact(text: &str) -> Redacted {
76 let mut findings = Vec::new();
77 let mut current = redact_private_keys(text, &mut findings);
78 for rule in RULES.iter() {
79 if let Some(next) = rule.apply(current.as_deref().unwrap_or(text), &mut findings) {
80 current = Some(next);
81 }
82 }
83 if let Some(next) = redact_assignments(current.as_deref().unwrap_or(text), &mut findings) {
84 current = Some(next);
85 }
86 Redacted {
87 text: current.unwrap_or_else(|| text.to_string()),
88 findings,
89 }
90}
91
92pub fn redact_for_store(text: &str) -> String {
95 let redacted = redact(text);
96 if redacted.changed() {
97 let kinds = redacted
98 .findings
99 .iter()
100 .map(|f| format!("{}={}", f.kind.as_str(), f.count))
101 .collect::<Vec<_>>()
102 .join(",");
103 tracing::info!(%kinds, "redacted secrets before storing");
104 }
105 redacted.text
106}
107
108fn record(findings: &mut Vec<Finding>, kind: SecretKind, count: usize) {
109 match findings.iter_mut().find(|f| f.kind == kind) {
110 Some(f) => f.count += count,
111 None => findings.push(Finding { kind, count }),
112 }
113}
114
115fn marker(kind: SecretKind) -> String {
116 format!("[REDACTED:{}]", kind.as_str())
117}
118
119type Accept = fn(&Captures, &str) -> bool;
121
122struct Rule {
123 kind: SecretKind,
124 re: Regex,
125 accept: Accept,
126}
127
128impl Rule {
129 fn new(kind: SecretKind, pattern: &str, accept: Accept) -> Self {
130 Self {
131 kind,
132 re: Regex::new(pattern).expect("redaction pattern compiles"),
133 accept,
134 }
135 }
136
137 fn apply(&self, text: &str, findings: &mut Vec<Finding>) -> Option<String> {
146 let mut out = String::new();
147 let mut last = 0;
148 let mut count = 0;
149 let mut pos = 0;
150 while let Some(caps) = self.re.captures_at(text, pos) {
151 let whole = caps.get(0).expect("group 0 always matches");
152 let span = caps.name("secret").unwrap_or(whole);
153 let (start, end) = (span.start(), span.end());
154 let accepted = start >= last
155 && !text[end..].starts_with("[REDACTED:")
156 && (self.accept)(&caps, span.as_str());
157 if !accepted {
158 pos = if start > whole.start() && span.len() <= MAX_VALUE {
159 start
160 } else {
161 whole.end()
162 };
163 continue;
164 }
165 out.push_str(&text[last..start]);
166 out.push_str(&marker(self.kind));
167 last = end;
168 pos = end.max(whole.end());
169 count += 1;
170 }
171 if count == 0 {
172 return None;
173 }
174 out.push_str(&text[last..]);
175 record(findings, self.kind, count);
176 Some(out)
177 }
178}
179
180const MAX_VALUE: usize = 512;
183
184fn always(_: &Captures, _: &str) -> bool {
185 true
186}
187
188fn group<'t>(caps: &Captures<'t>, name: &str) -> &'t str {
189 caps.name(name).map_or("", |m| m.as_str())
190}
191
192fn has_random_run(s: &[u8], min: usize, member: fn(u8) -> bool) -> bool {
195 let (mut run, mut upper, mut lower, mut digit) = (0, false, false, false);
196 for &b in s {
197 if member(b) {
198 run += 1;
199 upper |= b.is_ascii_uppercase();
200 lower |= b.is_ascii_lowercase();
201 digit |= b.is_ascii_digit();
202 } else {
203 (run, upper, lower, digit) = (0, false, false, false);
204 }
205 if run >= min && upper && lower && digit {
206 return true;
207 }
208 }
209 false
210}
211
212static TYPE_NAME: LazyLock<Regex> = LazyLock::new(|| {
216 Regex::new(r"^(?:[A-Z]{2,}[0-9]*)?(?:[A-Z][a-z]+[0-9]*)+$").expect("pattern compiles")
217});
218
219fn has_digit(s: &str) -> bool {
220 s.bytes().any(|b| b.is_ascii_digit())
221}
222
223fn mixed_case_with_digit(s: &str) -> bool {
224 s.bytes().any(|b| b.is_ascii_uppercase())
225 && s.bytes().any(|b| b.is_ascii_lowercase())
226 && has_digit(s)
227}
228
229fn is_placeholder(s: &str) -> bool {
230 s.starts_with(['$', '{', '<', '*', '%'])
231}
232
233fn sk_key_body(caps: &Captures, _: &str) -> bool {
234 let body = group(caps, "body");
235 let lowercase_run = body
236 .split(|c: char| !(c.is_ascii_lowercase() || c.is_ascii_digit()))
237 .any(|run| {
238 run.len() >= 32 && has_digit(run) && run.bytes().any(|b| b.is_ascii_lowercase())
239 });
240 mixed_case_with_digit(body) || lowercase_run || has_uuid(body)
241}
242
243fn url_credentials(caps: &Captures, _: &str) -> bool {
247 let user = group(caps, "user");
248 let pass = caps
249 .name("pass")
250 .or_else(|| caps.name("slashed"))
251 .map_or("", |m| m.as_str());
252 let port = caps.name("slashed").is_some_and(|m| {
253 let digits = m.as_str().bytes().take_while(u8::is_ascii_digit).count();
254 digits > 0 && m.as_str()[digits..].starts_with('/')
255 });
256 let stand_in = pass == user
257 || ["password", "pass", "secret", "changeme"]
258 .iter()
259 .any(|word| pass.eq_ignore_ascii_case(word))
260 || pass.bytes().all(|b| matches!(b, b'*' | b'x' | b'X'));
261 let scp_path = caps.name("slashed").is_some() && user.contains('@');
262 caps.name("digest").is_none()
263 && !scp_path
264 && !is_placeholder(user)
265 && !is_placeholder(pass)
266 && !port
267 && !stand_in
268}
269
270fn has_uuid(s: &str) -> bool {
272 UUID.is_match(s)
273}
274
275static UUID: LazyLock<Regex> = LazyLock::new(|| {
276 Regex::new(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")
277 .expect("pattern compiles")
278});
279
280fn mysql_command() -> String {
287 let arg_flags = "ugpChUrtDRT";
288 format!(
289 r#"(?m)(?:(?:^|[ \t])(?:[^\s\-]\S*|--)[ \t]+|\b(?:sudo|doas)(?:[ \t]+(?:-[{arg_flags}][ \t]+[^\s\-]\S*|-[{arg_flags}]\S+|-[^\s\-{arg_flags}]\S*|--\S+))*[ \t]+|(?:^|[;&|(`"']|\$|\\n)[ \t]*)(?:\S*bin/)?(?:mysql(?:dump|admin|import|check|show)?|mariadb)(?:[ \t]|["'][ \t]*,[ \t]*)"#
290 )
291}
292
293const TOKEN_START: &str = r"(?:^|[^A-Za-z0-9_\]]|\\[nrt])";
298
299static RULES: LazyLock<Vec<Rule>> = LazyLock::new(|| {
300 let token = |pattern: &str| format!("{TOKEN_START}(?P<secret>{pattern})");
301 vec![
302 Rule::new(
303 SecretKind::Jwt,
304 &token(r"eyJ[A-Za-z0-9_\-]{10,}\.eyJ[A-Za-z0-9_\-]{10,}\.[A-Za-z0-9_\-]*"),
305 always,
306 ),
307 Rule::new(
308 SecretKind::GithubToken,
309 &token(r"gh[pousr]_[A-Za-z0-9]{36,}|github_pat_[A-Za-z0-9_]{22,}"),
310 always,
311 ),
312 Rule::new(
313 SecretKind::ApiKey,
314 &token(r"sk-(?P<body>[A-Za-z0-9_\-]{20,})"),
315 sk_key_body,
316 ),
317 Rule::new(
318 SecretKind::ApiKey,
319 &token(r"[rs]k_(?:live|test)_[A-Za-z0-9]{20,}"),
320 always,
321 ),
322 Rule::new(
323 SecretKind::ApiKey,
324 &token(r"glpat-[A-Za-z0-9_\-]{20,}|npm_[A-Za-z0-9]{36,}|hf_[A-Za-z0-9]{34,}"),
325 always,
326 ),
327 Rule::new(
328 SecretKind::SlackToken,
329 &token(
330 r"xoxd-[A-Za-z0-9%+/=\-]{10,}|(?:xoxe\.)?(?:xox[abposrce]|xapp)-[A-Za-z0-9\-]{10,}",
331 ),
332 |_, value| {
333 let digit_run = value
334 .split(|c: char| !c.is_ascii_digit())
335 .any(|run| run.len() >= 8);
336 digit_run || has_random_run(value.as_bytes(), 16, |b| b.is_ascii_alphanumeric())
337 },
338 ),
339 Rule::new(
340 SecretKind::Secret,
341 &token(r"tskey-[a-z]+-[A-Za-z0-9_\-]{16,}"),
342 |_, value| has_digit(value),
343 ),
344 Rule::new(
345 SecretKind::SlackWebhook,
346 r"https://hooks\.slack\.com/services/[A-Za-z0-9/_\-]{20,}",
347 always,
348 ),
349 Rule::new(
350 SecretKind::GoogleApiKey,
351 &token(r"AIza[0-9A-Za-z_\-]{35}"),
352 always,
353 ),
354 Rule::new(
355 SecretKind::AwsAccessKeyId,
356 &token(
357 r"(?:AKIA|ASIA|ABIA|ACCA|AGPA|AIDA|AIPA|ANPA|ANVA|APKA|AROA|ASCA)[A-Z0-9]{16}\b",
358 ),
359 always,
360 ),
361 Rule::new(
362 SecretKind::AwsSecretKey,
363 r"\[REDACTED:aws_access_key_id\][ \t:,]+(?P<secret>[A-Za-z0-9/+=]{40})(?:[^A-Za-z0-9/+=]|$)",
364 always,
365 ),
366 Rule::new(
367 SecretKind::AwsSecretKey,
368 r#"(?i)(?:aws[_ ]?)?secret[_ ]?(?:access[_ ]?)?key(?:\\?["'])?(?:[ \t]*\[[^\]\n]*\])?(?:[ \t]*(?:=|:=|:|=>)[ \t]*|[ \t]+)(?:\\?["'])?(?P<secret>[A-Za-z0-9/+=]{40})(?:[^A-Za-z0-9/+=]|$)"#,
369 always,
370 ),
371 Rule::new(
372 SecretKind::BearerToken,
373 r#"(?i)\b(?:bearer|authorization(?:\\?["'])?\s*:\s*(?:\\?["'])?token)\s+(?P<secret>[A-Za-z0-9\-._~+/]{16,}=*)"#,
374 |_, value| {
375 has_digit(value)
376 || (value.len() >= 20
377 && value.bytes().any(|b| b.is_ascii_uppercase())
378 && value.bytes().any(|b| b.is_ascii_lowercase())
379 && !TYPE_NAME.is_match(value))
380 },
381 ),
382 Rule::new(
383 SecretKind::BasicAuth,
384 r#"(?i)\bauthorization(?:\\?["'])?\s*:\s*(?:\\?["'])?basic\s+(?P<secret>[A-Za-z0-9+/]{8,}={0,2})"#,
385 always,
386 ),
387 Rule::new(
388 SecretKind::UrlCredentials,
389 r#"\b[A-Za-z][A-Za-z0-9+.\-]*://(?P<secret>(?P<user>[^\s:/\[\]"'<>]*):(?:(?P<pass>[^\s/?#\[\]"'<>]+)|(?P<slashed>[^\s?#\[\]"'<>@]+)))@(?P<digest>sha(?:256|512):)?"#,
390 url_credentials,
391 ),
392 Rule::new(
393 SecretKind::BasicAuth,
394 r#"\bcurl\b[^\n|;&]*?\s(?:-[A-Za-z0-9]*u|--user)[ \t]*["']?(?P<user>[^\s:'"]+):(?P<secret>[^\s'"\[]+)"#,
395 |caps, value| !is_placeholder(group(caps, "user")) && !is_placeholder(value),
396 ),
397 Rule::new(
398 SecretKind::Secret,
399 &format!(
400 r#"{}(?:[^\n|;&]*?[ \t,])?["']?-p["']?(?P<secret>[^\s'",\[][^\s'"\[]*)"#,
401 mysql_command()
402 ),
403 |_, value| !is_placeholder(value),
404 ),
405 Rule::new(
406 SecretKind::Secret,
407 r#"\b(?:docker|podman|nerdctl|skopeo|buildah|oras|helm(?:[ \t]+registry)?)[ \t]+login\b[^\n|;&]*?\s-p[ \t]*["']?(?P<secret>[^\s'"\[]+)"#,
408 |_, value| !is_placeholder(value),
409 ),
410 Rule::new(
411 SecretKind::Secret,
412 r#"\bsshpass(?:[ \t]+(?:-[fdP][ \t]+[^\s\-]\S*|-[^\s\-p]\S*))*[ \t]+-p[ \t]*["']?(?P<secret>[^\s'"\[]+)"#,
413 |_, value| !is_placeholder(value),
414 ),
415 Rule::new(
416 SecretKind::Secret,
417 r"\b(?:machine\s+\S+|default)\s+login\s+\S+\s+password\s+(?P<secret>[^\s\[]+)",
418 |_, value| !is_placeholder(value),
419 ),
420 ]
421});
422
423pub fn redact_memory(memory: &mut Memory) -> bool {
425 let content = redact_for_store(&memory.content);
426 let mut changed = content != memory.content;
427 memory.content = content;
428 if let Some(context) = memory.context.as_deref() {
429 let redacted = redact_for_store(context);
430 changed |= redacted != context;
431 memory.context = Some(redacted);
432 }
433 changed
434}
435
436#[cfg(test)]
437mod tests;