vtcode_commons/
sanitizer.rs

1//! Secret sanitization utilities for redacting sensitive information.
2//!
3//! Provides regex-based secret redaction for:
4//! - OpenAI API keys (`sk-...`)
5//! - AWS Access Key IDs (`AKIA...`)
6//! - GitHub App installation tokens (`ghs_...`, including ~520-char stateless JWTs)
7//! - Bearer tokens (`Bearer ...`)
8//! - Generic secret assignments (`api_key=...`, `password:...`, etc.)
9//!
10//! Use this module to sanitize text before logging, displaying in UI,
11//! or storing in session archives.
12//!
13//! GitHub App installation tokens are treated as opaque strings per
14//! <https://github.blog/changelog/2026-05-15-github-app-installation-tokens-per-request-override-header/>
15//! and <https://github.blog/changelog/2026-10-02-stateless-github-app-installation-tokens-rolled-out>:
16//! `ghs_`-prefixed JWTs (~520 chars, two dots) must not be subject to length
17//! assumptions and must be redacted even without a `Bearer` prefix.
18
19use regex::Regex;
20use std::sync::LazyLock;
21
22/// OpenAI API key pattern: sk- followed by alphanumeric characters
23static OPENAI_KEY_REGEX: LazyLock<Regex> = LazyLock::new(|| compile_regex(r"sk-[A-Za-z0-9_-]{16,}"));
24
25/// AWS Access Key ID pattern: AKIA followed by 16 alphanumeric characters
26static AWS_ACCESS_KEY_ID_REGEX: LazyLock<Regex> = LazyLock::new(|| compile_regex(r"\bAKIA[0-9A-Z]{16}\b"));
27
28/// GitHub App installation token pattern (recommended by GitHub):
29/// `ghs_[A-Za-z0-9\.\-_]{36,}` matches both legacy opaque tokens (~40 chars)
30/// and stateless JWT-format tokens (~520 chars, two dots).
31static GITHUB_APP_TOKEN_REGEX: LazyLock<Regex> = LazyLock::new(|| compile_regex(r"ghs_[A-Za-z0-9.\-_]{36,}"));
32
33/// Bearer token pattern: "Bearer " followed by token characters
34static BEARER_TOKEN_REGEX: LazyLock<Regex> = LazyLock::new(|| compile_regex(r"(?i)\bBearer\s+[A-Za-z0-9.\-_]{16,}\b"));
35
36/// Generic secret assignment pattern: key=value or key: value format
37/// Matches common secret key names like api_key, token, secret, password
38static SECRET_ASSIGNMENT_REGEX: LazyLock<Regex> = LazyLock::new(|| {
39    compile_regex(
40        r#"(?i)\b((?:[a-z0-9][a-z0-9_-]*?)?(?:api[\-_]?key|access[\-_]?key|client[\-_]?secret|credential|private[\-_]?key|token|secret|password|auth)[a-z0-9_-]*)\b(\s*[:=]\s*)(["']?)[^\s"']{8,}"#,
41    )
42});
43
44/// Maximum serialized size of a provider diagnostic after redaction.
45pub const PROVIDER_DIAGNOSTIC_MAX_BYTES: usize = 8 * 1024;
46const PROVIDER_DIAGNOSTIC_TRUNCATION_MARKER: &str = "… [diagnostic truncated]";
47
48/// Redact secrets and sensitive keys from a string.
49///
50/// This is a best-effort operation using well-known regex patterns.
51/// Redacted values are replaced with `[REDACTED_SECRET]`.
52///
53/// # Examples
54///
55/// ```
56/// use vtcode_commons::sanitizer::redact_secrets;
57///
58/// let input = format!("Found key: {}", concat!("sk-", "test1234567890abcdef"));
59/// let output = redact_secrets(input);
60/// assert_eq!(output, "Found key: [REDACTED_SECRET]");
61/// ```
62pub fn redact_secrets(input: String) -> String {
63    let r1 = OPENAI_KEY_REGEX.replace_all(&input, "[REDACTED_SECRET]");
64    let r2 = AWS_ACCESS_KEY_ID_REGEX.replace_all(&r1, "[REDACTED_SECRET]");
65    let r3 = GITHUB_APP_TOKEN_REGEX.replace_all(&r2, "[REDACTED_SECRET]");
66    let r4 = BEARER_TOKEN_REGEX.replace_all(&r3, "Bearer [REDACTED_SECRET]");
67    let r5 = SECRET_ASSIGNMENT_REGEX.replace_all(&r4, "$1$2$3[REDACTED_SECRET]");
68    // `into_owned` clones only when the final result is `Borrowed` (no regex
69    // matched at all); when any redaction occurred it moves the owned string
70    // without an extra allocation. Do NOT short-circuit on `Cow::Borrowed` —
71    // the final Cow is `Borrowed` whenever the *last* regex doesn't match,
72    // even if earlier regexes did, which would silently discard redactions.
73    r5.into_owned()
74}
75
76/// Redact secrets and return a bounded, UTF-8-safe provider diagnostic.
77///
78/// The input is sampled with a carry window so a secret beginning near the
79/// output boundary is still redacted before the final size limit is applied.
80pub fn sanitize_provider_diagnostic(input: impl AsRef<[u8]>) -> String {
81    let input = input.as_ref();
82    let sample_len = input.len().min(PROVIDER_DIAGNOSTIC_MAX_BYTES + STREAMING_REDACTION_CARRY_BYTES);
83    let sample = String::from_utf8_lossy(input.get(..sample_len).unwrap_or(input));
84    let redacted = redact_secrets(sample.into_owned());
85    if redacted.len() <= PROVIDER_DIAGNOSTIC_MAX_BYTES {
86        return redacted;
87    }
88
89    let content_limit = PROVIDER_DIAGNOSTIC_MAX_BYTES.saturating_sub(PROVIDER_DIAGNOSTIC_TRUNCATION_MARKER.len());
90    let end = redacted.floor_char_boundary(content_limit);
91    format!("{}{}", redacted.get(..end).unwrap_or(&redacted), PROVIDER_DIAGNOSTIC_TRUNCATION_MARKER)
92}
93
94/// Incrementally redact streamed output without retaining the full stream.
95///
96/// A bounded suffix is held between chunks so a secret split at an IO
97/// boundary is still matched by the same redaction rules as a complete line.
98#[derive(Debug, Default)]
99pub struct StreamingSecretRedactor {
100    pending: String,
101}
102
103const STREAMING_REDACTION_CARRY_BYTES: usize = 1_024;
104
105impl StreamingSecretRedactor {
106    /// Redact and return the safe prefix of `chunk`. The returned string may
107    /// be empty while the bounded carry window is being filled.
108    pub fn push(&mut self, chunk: &str) -> String {
109        self.pending.push_str(chunk);
110        if self.pending.len() <= STREAMING_REDACTION_CARRY_BYTES {
111            if !self.pending.contains('\n') {
112                return String::new();
113            }
114        }
115
116        let carry_split = self.pending.len().saturating_sub(STREAMING_REDACTION_CARRY_BYTES);
117        let line_split = self.pending.rfind('\n').map(|index| index + 1).unwrap_or(0);
118        let mut split_at = carry_split.max(line_split);
119        while split_at > 0 && !self.pending.is_char_boundary(split_at) {
120            split_at -= 1;
121        }
122        let prefix: String = self.pending.drain(..split_at).collect();
123        redact_secrets(prefix)
124    }
125
126    /// Redact and return the final carried suffix.
127    pub fn finish(self) -> String {
128        redact_secrets(self.pending)
129    }
130}
131
132#[allow(
133    clippy::panic,
134    reason = "Intentional compatibility, platform, or test-only suppression."
135)]
136fn compile_regex(pattern: &str) -> Regex {
137    match Regex::new(pattern) {
138        Ok(regex) => regex,
139        // Panic is acceptable thanks to the `load_regex` test
140        Err(err) => panic!("invalid regex pattern `{pattern}`: {err}"),
141    }
142}
143
144#[cfg(test)]
145mod tests {
146    use super::*;
147
148    #[test]
149    fn load_regex() {
150        // Verify all regex patterns compile without panicking
151        let _ = redact_secrets("test".to_string());
152    }
153
154    #[test]
155    fn redacts_openai_key() {
156        let input = format!("Found key: {}", concat!("sk-", "test1234567890abcdef"));
157        let output = redact_secrets(input);
158        assert_eq!(output, "Found key: [REDACTED_SECRET]");
159    }
160
161    #[test]
162    fn redacts_aws_access_key() {
163        // Assemble the documentation fixture at runtime so repository scans do not
164        // mistake it for a live credential.
165        let aws_key = concat!("AKIA", "IOSFODNN7EXAMPLE");
166        let input = format!(" creds: {aws_key} ");
167        let output = redact_secrets(input);
168        assert_eq!(output, " creds: [REDACTED_SECRET] ");
169    }
170
171    #[test]
172    fn redacts_bearer_token() {
173        let input = "Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9".to_string();
174        let output = redact_secrets(input);
175        assert_eq!(output, "Authorization: Bearer [REDACTED_SECRET]");
176    }
177
178    #[test]
179    fn redacts_github_app_installation_tokens() {
180        // Legacy opaque format (~40 chars, no dots) and stateless JWT format
181        // (~520 chars, two dots) per May/Oct 2026 changelogs. Both must be
182        // treated as opaque and redacted even without a Bearer prefix.
183        // Fixtures are assembled at runtime so scans do not flag them.
184        let legacy = format!("ghs_{}", "a".repeat(36));
185        let header = format!("ghs_{}", "b".repeat(20));
186        let payload = format!("{}.", "c".repeat(250));
187        let signature = "d".repeat(250);
188        let stateless = format!("{header}.{payload}{signature}");
189        assert!(stateless.len() >= 500, "fixture should model ~520-char token");
190        assert_eq!(stateless.matches('.').count(), 2);
191
192        for token in [&legacy, &stateless] {
193            let bare = redact_secrets(format!("saw token {token} in logs"));
194            assert_eq!(bare, "saw token [REDACTED_SECRET] in logs");
195            assert!(!bare.contains(token.as_str()));
196
197            let bearer = redact_secrets(format!("Authorization: Bearer {token}"));
198            assert!(!bearer.contains(token.as_str()));
199        }
200    }
201
202    #[test]
203    fn github_app_token_threshold_is_36_chars() {
204        // Locks the GitHub-recommended `ghs_[A-Za-z0-9.\-_]{36,}` boundary:
205        // 35 chars must not redact on its own, 36 chars must.
206        let short = format!("ghs_{}", "a".repeat(35));
207        let exact = format!("ghs_{}", "a".repeat(36));
208        assert_eq!(redact_secrets(format!("saw {short} here")), format!("saw {short} here"));
209        assert_eq!(redact_secrets(format!("saw {exact} here")), "saw [REDACTED_SECRET] here");
210    }
211
212    #[test]
213    fn redacts_github_app_token_split_across_stream_chunks() {
214        // A ~526-char stateless token fits in the 1_024-byte carry window, so
215        // halves split at an IO boundary must still redact on finish.
216        let token = format!("ghs_{}.{}.{}", "b".repeat(20), "c".repeat(250), "d".repeat(250));
217        let mid = token.floor_char_boundary(token.len() / 2);
218        let (first, second) = token.split_at(mid);
219        let mut redactor = StreamingSecretRedactor::default();
220        let mut output = redactor.push(first);
221        output.push_str(&redactor.push(&format!("{second}\n")));
222        output.push_str(&redactor.finish());
223        assert!(!output.contains(token.as_str()));
224        assert!(output.contains("[REDACTED_SECRET]"));
225    }
226
227    #[test]
228    fn redacts_api_key_assignment() {
229        let input = "api_key=sk-test12345678".to_string();
230        let output = redact_secrets(input);
231        assert_eq!(output, "api_key=[REDACTED_SECRET]");
232    }
233
234    #[test]
235    fn redacts_password_assignment() {
236        let input = "password: mysecretvalue".to_string();
237        let output = redact_secrets(input);
238        assert_eq!(output, "password: [REDACTED_SECRET]");
239    }
240
241    #[test]
242    fn redacts_token_in_quotes() {
243        let input = r#"token="abc123xyz789abcdef""#.to_string();
244        let output = redact_secrets(input);
245        assert_eq!(output, r#"token="[REDACTED_SECRET]""#);
246    }
247
248    #[test]
249    fn preserves_short_values() {
250        // Values under 8 characters should not be redacted
251        let input = "password: short".to_string();
252        let output = redact_secrets(input);
253        assert_eq!(output, "password: short");
254    }
255
256    #[test]
257    fn redacts_multiple_secrets() {
258        let openai_key = concat!("sk-", "test1234567890abcdef");
259        let aws_key = concat!("AKIA", "IOSFODNN7EXAMPLE");
260        let input = format!("Keys: {openai_key} and {aws_key}");
261        let output = redact_secrets(input);
262        // Verify both secrets are redacted
263        assert!(output.contains("[REDACTED_SECRET]"));
264        assert!(!output.contains(openai_key));
265        assert!(!output.contains(aws_key));
266    }
267
268    #[test]
269    fn preserves_non_secret_text() {
270        let input = "Hello world, this is normal text".to_string();
271        let output = redact_secrets(input);
272        assert_eq!(output, "Hello world, this is normal text");
273    }
274
275    #[test]
276    fn redacts_secrets_split_across_stream_chunks() {
277        let mut redactor = StreamingSecretRedactor::default();
278        let mut output = redactor.push("password=superse");
279        output.push_str(&redactor.push("cretvalue\n"));
280        output.push_str(&redactor.finish());
281
282        assert_eq!(output, "password=[REDACTED_SECRET]\n");
283        assert!(!output.contains("supersecretvalue"));
284    }
285
286    #[test]
287    fn provider_diagnostic_is_bounded_utf8_safe_and_redacted() {
288        let mut input = b"api_key=diagnostic-secret-value Bearer abcdefghijklmnop ".to_vec();
289        input.extend(std::iter::repeat_n(b'x', 20_000));
290        input.extend_from_slice("終端".as_bytes());
291        input.push(0xff);
292
293        let output = sanitize_provider_diagnostic(input);
294
295        assert!(output.len() <= PROVIDER_DIAGNOSTIC_MAX_BYTES);
296        assert!(output.is_char_boundary(output.len()));
297        assert!(!output.contains("diagnostic-secret-value"));
298        assert!(!output.contains("abcdefghijklmnop"));
299    }
300}
vtcode_commons/sanitizer.rs

vtcode_commons/
sanitizer.rs
