lean_ctx/core/
redaction.rs

1macro_rules! static_regex {
2    ($pattern:expr_2021) => {{
3        static RE: std::sync::OnceLock<regex::Regex> = std::sync::OnceLock::new();
4        RE.get_or_init(|| {
5            regex::Regex::new($pattern).expect(concat!("BUG: invalid static regex: ", $pattern))
6        })
7    }};
8}
9
10pub fn redaction_enabled_for_active_role() -> bool {
11    let role = crate::core::roles::active_role();
12    if role.role.name == "admin" {
13        role.io.redact_outputs
14    } else {
15        // Contract: redaction never disabled for non-admin roles.
16        true
17    }
18}
19
20pub fn redact_text_if_enabled(input: &str) -> String {
21    if !redaction_enabled_for_active_role() {
22        return input.to_string();
23    }
24    redact_text_with_excludes(input, &config_exclude_patterns())
25}
26
27/// #718: unquoted identifier or property-access chains (`SvelteKit`,
28/// `inputEnv.POCKETBASE_SUPERUSER_PASSWORD`, `serverEnv.getStripeSecretKey`,
29/// `confirmRequiredEndpointKeySchema`) are code REFERENCES to a secret, never
30/// the literal value — the value lives in a gitignored `.env`. Digits make a
31/// token secret-shaped (base64/hex), so any digit keeps the redaction
32/// (conservative: `password=hunter2` stays covered).
33fn is_identifier_reference(value: &str) -> bool {
34    let v = value.trim();
35    if v.is_empty()
36        || v.starts_with('"')
37        || v.starts_with('\'')
38        || v.starts_with('`')
39        || v.contains(|c: char| c.is_ascii_digit())
40    {
41        return false;
42    }
43    v.split('.').all(|segment| {
44        let mut chars = segment.chars();
45        matches!(chars.next(), Some(c) if c.is_ascii_alphabetic() || c == '_' || c == '$')
46            && chars.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '$')
47    })
48}
49
50/// #718: obvious placeholder/example values (`ghp_change_me`, `your_key_here`,
51/// `<insert-token>`) are documentation, not secrets — `.env.example` files
52/// must survive ctx_read verbatim.
53fn is_placeholder_value(value: &str) -> bool {
54    let v = value
55        .trim()
56        .trim_matches(|c| c == '"' || c == '\'' || c == '`')
57        .to_ascii_lowercase();
58    if v.starts_with('<') && v.ends_with('>') {
59        return true;
60    }
61    const MARKERS: &[&str] = &[
62        "change_me",
63        "change-me",
64        "changeme",
65        "example",
66        "placeholder",
67        "your_",
68        "your-",
69        "xxx",
70        "dummy",
71        "sample",
72        "todo",
73        "fixme",
74        "replace_me",
75        "replace-me",
76    ];
77    MARKERS.iter().any(|m| v.contains(m))
78}
79
80/// Right-hand sides that look like `key: value` but are obviously not secrets:
81/// TypeScript type annotations and language literals. Redacting these corrupts
82/// source files read through `ctx_read` (GH #430), so the key/value rules skip
83/// them. Compared case-insensitively after trimming surrounding quotes.
84fn is_non_secret_literal(value: &str) -> bool {
85    let v = value
86        .trim()
87        .trim_matches(|c| c == '"' || c == '\'' || c == '`');
88    // Type expressions are never flat secret tokens: real keys/tokens are drawn
89    // from `[A-Za-z0-9+/=_-]`, whereas type annotations carry angle brackets,
90    // unions, arrays or call/object syntax. `password: Promise<string>` and
91    // `apiKey: Record<string, unknown>` must survive ctx_read verbatim (GH #430).
92    if v.contains(['<', '>', '|', '(', ')', '[', ']', '{', '}']) {
93        return true;
94    }
95    matches!(
96        v.to_ascii_lowercase().as_str(),
97        "" | "undefined"
98            | "null"
99            | "none"
100            | "nil"
101            | "true"
102            | "false"
103            | "string"
104            | "number"
105            | "boolean"
106            | "bigint"
107            | "symbol"
108            | "object"
109            | "any"
110            | "unknown"
111            | "never"
112            | "void"
113            | "nan"
114            | "date"
115    )
116}
117
118/// One redaction rule: a labelled regex plus how the match is rebuilt.
119struct Rule {
120    label: &'static str,
121    re: &'static regex::Regex,
122    /// When set, group 1 is a prefix to keep and group 2 is the secret value;
123    /// the match is left untouched if that value is a non-secret literal
124    /// (`password: undefined`), an identifier reference
125    /// (`serverEnv.getStripeSecretKey`) or a placeholder (`ghp_change_me`) —
126    /// see `is_benign_secret_value` (GH #430, #718).
127    guard_value: bool,
128}
129
130/// Combined benign-value check for key/value secret rules (#430 + #718):
131/// language literals and type annotations, unquoted identifier/property
132/// references, and documentation placeholders are never redacted. Quoted
133/// string values stay protected — they ARE literal values.
134pub(crate) fn is_benign_secret_value(value: &str) -> bool {
135    is_non_secret_literal(value) || is_identifier_reference(value) || is_placeholder_value(value)
136}
137
138/// The single source of truth for secret patterns. `shell::redact` delegates
139/// here so the two layers can never drift apart again.
140///
141/// #718 word boundaries: the key/value alternations start with
142/// `(?:^|[^a-z0-9])` (consumed into the kept prefix — the regex crate has no
143/// lookbehind) so camelCase subwords (`superuserPassword`,
144/// `getStripeSecretKey`) never trigger a rule, while SNAKE_CASE env names
145/// (`GITHUB_FEEDBACK_TOKEN`) still do: `_` remains a permitted predecessor.
146fn redaction_rules() -> Vec<Rule> {
147    vec![
148        Rule {
149            label: "Bearer token",
150            re: static_regex!(r"(?i)(bearer\s+)[a-zA-Z0-9\-_\.]{8,}"),
151            guard_value: false,
152        },
153        Rule {
154            label: "Authorization header",
155            re: static_regex!(r"(?i)(authorization:\s*(?:basic|bearer|token)\s+)[^\s\r\n]+"),
156            guard_value: false,
157        },
158        // Key/value secrets: group 1 = predecessor + `name=`/`name: ` prefix
159        // (kept), group 2 = the value (redacted unless benign — GH #430/#718).
160        Rule {
161            label: "API key param",
162            re: static_regex!(
163                r#"(?im)((?:^|[^a-z0-9])(?:api[_-]?key|apikey|access[_-]?key|secret[_-]?key|token|password|passwd|pwd|secret)\s*[=:]\s*)([^\s\r\n,;&"']+)"#
164            ),
165            guard_value: true,
166        },
167        // Whole token is the secret — no prefix group, so the entire match is
168        // replaced. (Previously group 1 captured the key itself and leaked it.)
169        Rule {
170            label: "AWS key",
171            re: static_regex!(r"AKIA[0-9A-Z]{12,}"),
172            guard_value: false,
173        },
174        Rule {
175            label: "Private key block",
176            re: static_regex!(
177                r"(?s)(-----BEGIN\s+(?:RSA\s+)?PRIVATE\s+KEY-----).+?-----END\s+(?:RSA\s+)?PRIVATE\s+KEY-----"
178            ),
179            guard_value: false,
180        },
181        Rule {
182            label: "GitHub token",
183            re: static_regex!(r"(gh[pousr]_)[a-zA-Z0-9]{20,}"),
184            guard_value: false,
185        },
186        // Group 1 = prefix (kept), group 2 = the 32+ char value. Guarded since
187        // #718: 32-char identifiers like `confirmRequiredEndpointKeySchema`
188        // are references, not secrets.
189        Rule {
190            label: "Generic long secret",
191            re: static_regex!(
192                r#"(?im)((?:^|[^a-z0-9])(?:key|token|secret|password|credential|auth)\s*[=:]\s*)(['"]?[a-zA-Z0-9+/=\-_]{32,}['"]?)"#
193            ),
194            guard_value: true,
195        },
196    ]
197}
198
199pub fn redact_text(input: &str) -> String {
200    redact_text_with_excludes(input, &[])
201}
202
203/// #718: `redact_text` with subtractive user patterns from
204/// `[secret_detection].exclude_patterns` — a match covered by any exclude
205/// regex is kept verbatim, so known-safe naming conventions can be carved out
206/// without disabling secret detection wholesale.
207pub fn redact_text_with_excludes(input: &str, excludes: &[regex::Regex]) -> String {
208    let mut out = input.to_string();
209    for rule in redaction_rules() {
210        out = rule
211            .re
212            .replace_all(&out, |caps: &regex::Captures| {
213                let whole = caps.get(0).map_or("", |m| m.as_str());
214                if excludes.iter().any(|ex| ex.is_match(whole)) {
215                    return whole.to_string();
216                }
217                if rule.guard_value
218                    && let Some(value) = caps.get(2)
219                    && is_benign_secret_value(value.as_str())
220                {
221                    // Not a secret (identifier reference, literal, placeholder)
222                    // — keep verbatim (#430, #718).
223                    return whole.to_string();
224                }
225                match caps.get(1) {
226                    Some(prefix) => format!("{}[REDACTED:{}]", prefix.as_str(), rule.label),
227                    None => format!("[REDACTED:{}]", rule.label),
228                }
229            })
230            .to_string();
231    }
232    out
233}
234
235/// Compile the configured `exclude_patterns` (#718). Invalid regexes are
236/// skipped — a broken exclude must never disable redaction.
237pub fn config_exclude_patterns() -> Vec<regex::Regex> {
238    crate::core::config::Config::load()
239        .secret_detection
240        .exclude_patterns
241        .iter()
242        .filter_map(|p| regex::Regex::new(p).ok())
243        .collect()
244}
245
246/// Apply caller-supplied policy redaction patterns on top of the built-in
247/// secret rules: each regex match becomes `[REDACTED:<label>]`. Returns the
248/// transformed text and the number of redactions applied (for audit counts).
249///
250/// Used by context policy packs (GL #673) so a pack's `[redaction]` block
251/// actually removes matching content from what the model sees. The patterns are
252/// the pack's `[redaction]` entries, precompiled by
253/// [`crate::core::policy::runtime`].
254#[must_use]
255pub fn redact_with_patterns(input: &str, patterns: &[(String, regex::Regex)]) -> (String, usize) {
256    let mut out = input.to_string();
257    let mut hits = 0usize;
258    for (label, re) in patterns {
259        let mut local = 0usize;
260        out = re
261            .replace_all(&out, |_caps: &regex::Captures| {
262                local += 1;
263                format!("[REDACTED:{label}]")
264            })
265            .to_string();
266        hits += local;
267    }
268    (out, hits)
269}
270
271#[cfg(test)]
272mod tests {
273    use super::*;
274
275    #[test]
276    fn redacts_bearer_token() {
277        let s = "Authorization: Bearer abcdefghijklmnopqrstuvwxyz012345";
278        let out = redact_text(s);
279        assert!(out.contains("[REDACTED"));
280        assert!(!out.contains("abcdefghijklmnopqrstuvwxyz"));
281    }
282
283    #[test]
284    fn redacts_private_key_block() {
285        let s = "-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----";
286        let out = redact_text(s);
287        assert!(out.contains("[REDACTED"));
288        assert!(!out.contains("\nabc\n"));
289    }
290
291    #[test]
292    fn redacts_api_key_param_value() {
293        let out = redact_text("password=hunter2-super-secret-value");
294        assert!(
295            out.contains("password=[REDACTED:API key param]"),
296            "got: {out}"
297        );
298        assert!(!out.contains("hunter2"));
299    }
300
301    /// GH #430: TypeScript type annotations and language literals must NOT be
302    /// redacted — over-eager masking corrupted source files read via ctx_read.
303    #[test]
304    fn keeps_non_secret_literals() {
305        for s in [
306            "password: undefined",
307            "secret: string",
308            "token: null",
309            "apiKey: boolean",
310            "password = false",
311            "secret: any",
312            "let pwd: number = 1",
313        ] {
314            assert_eq!(redact_text(s), s, "must not redact non-secret literal: {s}");
315        }
316    }
317
318    /// GH #430: TS type annotations (generics, unions, arrays, function/object
319    /// types) carry angle brackets / brackets that real secret tokens never do,
320    /// so they must survive verbatim even when the key looks sensitive.
321    #[test]
322    fn keeps_type_annotations() {
323        for s in [
324            "password: Promise<string>",
325            "apiKey: Record<string, unknown>",
326            "token: string[]",
327            "secret: () => void",
328            "password: string | undefined",
329            "credential: { value: string }",
330        ] {
331            assert_eq!(redact_text(s), s, "must not redact type annotation: {s}");
332        }
333    }
334
335    /// Whole-token secrets must be removed, not annotated in place — previously
336    /// the closure kept group 1 (the key itself) and only appended `[REDACTED]`.
337    #[test]
338    fn fully_redacts_aws_key() {
339        let out = redact_text("AKIAIOSFODNN7EXAMPLE");
340        assert!(
341            !out.contains("AKIAIOSFODNN7EXAMPLE"),
342            "AWS key leaked: {out}"
343        );
344        assert!(out.contains("[REDACTED:AWS key]"));
345    }
346
347    #[test]
348    fn fully_redacts_generic_long_secret() {
349        // `credential=` is not covered by the API-key-param rule, so this
350        // exercises the generic fallback (the previously leaky path).
351        let secret = "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6"; // 32 chars
352        let out = redact_text(&format!("credential={secret}"));
353        assert!(!out.contains(secret), "long secret leaked: {out}");
354        assert!(
355            out.contains("credential=[REDACTED:Generic long secret]"),
356            "got: {out}"
357        );
358    }
359
360    #[test]
361    fn redacts_github_token_keeping_prefix() {
362        let out = redact_text("ghp_abcdefghijklmnopqrstuvwxyz0123");
363        assert!(out.starts_with("ghp_[REDACTED:GitHub token]"), "got: {out}");
364        assert!(!out.contains("abcdefghijklmnopqrstuvwxyz"));
365    }
366
367    // ── #718: benign identifier references, prose and placeholders ──
368
369    /// Repro 1: prose that mentions a keyword must not have the following
370    /// word redacted — "token: SvelteKit's…" is documentation, not a secret.
371    #[test]
372    fn keeps_prose_identifier_after_keyword() {
373        let s = "the CSRF token: SvelteKit's native origin-check on form actions";
374        assert_eq!(redact_text(s), s, "prose must survive verbatim");
375    }
376
377    /// Repro 2: camelCase subwords must not trigger the keyword alternation,
378    /// and identifier/property-access RHS values are references, not secrets.
379    #[test]
380    fn keeps_identifier_and_property_references() {
381        for s in [
382            "superuserPassword: inputEnv.POCKETBASE_SUPERUSER_PASSWORD",
383            "export const getStripeSecretKey = serverEnv.getStripeSecretKey;",
384            "const apiKey = config.stripeApiKey",
385        ] {
386            assert_eq!(redact_text(s), s, "identifier reference redacted: {s}");
387        }
388    }
389
390    /// Repro 3: a 32+ char identifier (Zod schema name) is a reference —
391    /// "Generic long secret" needs the same value guard as the API-key rule.
392    #[test]
393    fn keeps_long_schema_identifier() {
394        let s = "endpoint_key: confirmRequiredEndpointKeySchema,";
395        assert_eq!(redact_text(s), s, "schema identifier must not be redacted");
396    }
397
398    /// Repro 4: obvious placeholder values (.env.example) are documentation.
399    #[test]
400    fn keeps_placeholder_values() {
401        for s in [
402            "GITHUB_FEEDBACK_TOKEN=ghp_change_me",
403            "API_KEY=your_key_here",
404            "password=<insert-password>",
405            "SECRET_KEY=xxxxxxxx",
406        ] {
407            assert_eq!(redact_text(s), s, "placeholder redacted: {s}");
408        }
409    }
410
411    /// The flip side: real secret-shaped values must STILL be redacted after
412    /// the #718 guards.
413    #[test]
414    fn still_redacts_real_secret_values() {
415        // Digit-bearing value after a snake_case env name.
416        let out = redact_text("GITHUB_TOKEN=ghpA1b2c3d4e5f6g7h8");
417        assert!(!out.contains("ghpA1b2c3d4e5f6g7h8"), "leaked: {out}");
418        // SNAKE_CASE env assignment with digits (the _ predecessor stays a
419        // word boundary that MATCHES).
420        let out = redact_text("MY_SECRET=abc123def456ghi789");
421        assert!(!out.contains("abc123def456ghi789"), "leaked: {out}");
422        // Quoted 32+ char literal: a quoted value is never an identifier
423        // reference, so the Generic-long-secret guard keeps redacting it.
424        let quoted = "key: 'abcdefghijklmnopqrstuvwxyzabcdef'";
425        let out = redact_text(quoted);
426        assert!(
427            !out.contains("abcdefghijklmnopqrstuvwxyzabcdef"),
428            "leaked: {out}"
429        );
430    }
431
432    /// #718: exclude_patterns carve matches out subtractively.
433    #[test]
434    fn exclude_patterns_skip_matching_redactions() {
435        let excludes = vec![regex::Regex::new(r"LCTX_TEST_\w+").unwrap()];
436        let input = "token=LCTX_TEST_a1b2c3d4e5";
437        assert_eq!(
438            redact_text_with_excludes(input, &excludes),
439            input,
440            "excluded match must stay verbatim"
441        );
442        // Without the exclude the same value IS redacted (digits → secret).
443        assert!(redact_text(input).contains("[REDACTED"));
444    }
445
446    #[test]
447    fn identifier_and_placeholder_heuristics() {
448        assert!(is_identifier_reference("serverEnv.getStripeSecretKey"));
449        assert!(is_identifier_reference("confirmRequiredEndpointKeySchema"));
450        assert!(is_identifier_reference("$scope._private"));
451        assert!(!is_identifier_reference("abc123"), "digits → secret-shaped");
452        assert!(!is_identifier_reference("\"quoted\""), "literal value");
453        assert!(!is_identifier_reference("a-b"), "dash is not identifier");
454        assert!(is_placeholder_value("ghp_change_me"));
455        assert!(is_placeholder_value("<token>"));
456        assert!(is_placeholder_value("your_api_key_123"));
457        assert!(!is_placeholder_value("A1b2C3d4E5f6G7h8"));
458    }
459
460    #[test]
461    fn policy_patterns_redact_with_label_and_count() {
462        let patterns = vec![(
463            "employee_id".to_string(),
464            regex::Regex::new(r"EMP-\d{4}").unwrap(),
465        )];
466        let (out, hits) = redact_with_patterns("user EMP-1234 and EMP-5678", &patterns);
467        assert_eq!(hits, 2);
468        assert!(!out.contains("EMP-1234"));
469        assert!(out.contains("[REDACTED:employee_id]"));
470    }
471
472    #[test]
473    fn policy_patterns_noop_when_no_match() {
474        let patterns = vec![("iban".to_string(), regex::Regex::new(r"CH\d{2}").unwrap())];
475        let (out, hits) = redact_with_patterns("nothing sensitive here", &patterns);
476        assert_eq!(hits, 0);
477        assert_eq!(out, "nothing sensitive here");
478    }
479}
lean_ctx/core/redaction.rs

lean_ctx/core/
redaction.rs
