lean_ctx/core/
redaction.rs

1macro_rules! static_regex {
2    ($pattern:expr_2021) => {{
3        static RE: std::sync::OnceLock<regex::Regex> = std::sync::OnceLock::new();
4        RE.get_or_init(|| {
5            regex::Regex::new($pattern).expect(concat!("BUG: invalid static regex: ", $pattern))
6        })
7    }};
8}
9
10pub fn redaction_enabled_for_active_role() -> bool {
11    let role = crate::core::roles::active_role();
12    if role.role.name == "admin" {
13        role.io.redact_outputs
14    } else {
15        // Contract: redaction never disabled for non-admin roles.
16        true
17    }
18}
19
20pub fn redact_text_if_enabled(input: &str) -> String {
21    if !redaction_enabled_for_active_role() {
22        return input.to_string();
23    }
24    redact_text_with_excludes(input, &config_exclude_patterns())
25}
26
27/// #718: unquoted identifier or property-access chains (`SvelteKit`,
28/// `inputEnv.POCKETBASE_SUPERUSER_PASSWORD`, `serverEnv.getStripeSecretKey`,
29/// `confirmRequiredEndpointKeySchema`) are code REFERENCES to a secret, never
30/// the literal value — the value lives in a gitignored `.env`. Digits make a
31/// token secret-shaped (base64/hex), so any digit keeps the redaction
32/// (conservative: `password=hunter2` stays covered).
33/// #827: detect pure numeric values — integers, floats, scientific notation.
34/// These are never secrets even when the key contains `token` or `key`.
35fn looks_like_number(v: &str) -> bool {
36    if v.is_empty() {
37        return false;
38    }
39    let s = v.trim_start_matches(['+', '-']);
40    if s.is_empty() {
41        return false;
42    }
43    // Integer, float, or scientific notation (1.4e-06, 0.5, 600, 1e10)
44    s.parse::<f64>().is_ok()
45        && s.chars()
46            .all(|c| c.is_ascii_digit() || c == '.' || c == 'e' || c == 'E' || c == '+' || c == '-')
47}
48
49/// #827: env-variable reference patterns that should not be redacted.
50/// Matches: `os.environ/NAME`, `os.getenv("NAME")`, `process.env.NAME`,
51/// `inputEnv.NAME`, `${NAME}`, `$NAME`, `%NAME%` (Windows).
52fn is_env_reference(v: &str) -> bool {
53    if v.starts_with("os.environ/")
54        || v.starts_with("os.getenv(")
55        || v.starts_with("process.env.")
56        || v.starts_with("System.getenv(")
57        || v.starts_with("ENV[")
58        || v.starts_with("env(")
59    {
60        return true;
61    }
62    // Variable interpolation: ${VAR}, $VAR, %VAR%
63    if (v.starts_with("${") && v.ends_with('}'))
64        || (v.starts_with('$')
65            && v[1..]
66                .chars()
67                .all(|c| c.is_ascii_alphanumeric() || c == '_'))
68        || (v.starts_with('%') && v.ends_with('%') && v.len() > 2)
69    {
70        return true;
71    }
72    // Dotted identifier chains with an env-like prefix
73    if let Some(prefix) = v.split('.').next() {
74        let pl = prefix.to_ascii_lowercase();
75        if matches!(
76            pl.as_str(),
77            "env" | "inputenv" | "serverenv" | "secrets" | "vars" | "environ"
78        ) && v.contains('.')
79        {
80            return true;
81        }
82    }
83    false
84}
85
86fn is_identifier_reference(value: &str) -> bool {
87    let v = value.trim();
88    if v.is_empty() || v.starts_with('"') || v.starts_with('\'') || v.starts_with('`') {
89        return false;
90    }
91    // #827: env-variable reference patterns are not secrets.
92    // `os.environ/MY_SERVICE_API_KEY`, `process.env.NAME`, `${VAR}`, `$VAR`.
93    if is_env_reference(v) {
94        return true;
95    }
96    if v.contains(|c: char| c.is_ascii_digit()) {
97        return false;
98    }
99    v.split('.').all(|segment| {
100        let mut chars = segment.chars();
101        matches!(chars.next(), Some(c) if c.is_ascii_alphabetic() || c == '_' || c == '$')
102            && chars.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '$')
103    })
104}
105
106/// #718: obvious placeholder/example values (`ghp_change_me`, `your_key_here`,
107/// `<insert-token>`) are documentation, not secrets — `.env.example` files
108/// must survive ctx_read verbatim.
109fn is_placeholder_value(value: &str) -> bool {
110    let v = value
111        .trim()
112        .trim_matches(|c| c == '"' || c == '\'' || c == '`')
113        .to_ascii_lowercase();
114    if v.starts_with('<') && v.ends_with('>') {
115        return true;
116    }
117    const MARKERS: &[&str] = &[
118        "change_me",
119        "change-me",
120        "changeme",
121        "example",
122        "placeholder",
123        "your_",
124        "your-",
125        "xxx",
126        "dummy",
127        "sample",
128        "todo",
129        "fixme",
130        "replace_me",
131        "replace-me",
132    ];
133    MARKERS.iter().any(|m| v.contains(m))
134}
135
136/// Right-hand sides that look like `key: value` but are obviously not secrets:
137/// TypeScript type annotations and language literals. Redacting these corrupts
138/// source files read through `ctx_read` (GH #430), so the key/value rules skip
139/// them. Compared case-insensitively after trimming surrounding quotes.
140fn is_non_secret_literal(value: &str) -> bool {
141    let v = value
142        .trim()
143        .trim_matches(|c| c == '"' || c == '\'' || c == '`');
144    // #827: pure numbers (integer, float, scientific notation) are never secrets.
145    // `input_cost_per_token: 1.4e-06` must not be redacted.
146    if looks_like_number(v) {
147        return true;
148    }
149    // Type expressions are never flat secret tokens: real keys/tokens are drawn
150    // from `[A-Za-z0-9+/=_-]`, whereas type annotations carry angle brackets,
151    // unions, arrays or call/object syntax. `password: Promise<string>` and
152    // `apiKey: Record<string, unknown>` must survive ctx_read verbatim (GH #430).
153    if v.contains(['<', '>', '|', '(', ')', '[', ']', '{', '}']) {
154        return true;
155    }
156    matches!(
157        v.to_ascii_lowercase().as_str(),
158        "" | "undefined"
159            | "null"
160            | "none"
161            | "nil"
162            | "true"
163            | "false"
164            | "string"
165            | "number"
166            | "boolean"
167            | "bigint"
168            | "symbol"
169            | "object"
170            | "any"
171            | "unknown"
172            | "never"
173            | "void"
174            | "nan"
175            | "date"
176    )
177}
178
179/// One redaction rule: a labelled regex plus how the match is rebuilt.
180struct Rule {
181    label: &'static str,
182    re: &'static regex::Regex,
183    /// When set, group 1 is a prefix to keep and group 2 is the secret value;
184    /// the match is left untouched if that value is a non-secret literal
185    /// (`password: undefined`), an identifier reference
186    /// (`serverEnv.getStripeSecretKey`) or a placeholder (`ghp_change_me`) —
187    /// see `is_benign_secret_value` (GH #430, #718).
188    guard_value: bool,
189}
190
191/// Combined benign-value check for key/value secret rules (#430 + #718):
192/// language literals and type annotations, unquoted identifier/property
193/// references, and documentation placeholders are never redacted. Quoted
194/// string values stay protected — they ARE literal values.
195pub(crate) fn is_benign_secret_value(value: &str) -> bool {
196    is_non_secret_literal(value) || is_identifier_reference(value) || is_placeholder_value(value)
197}
198
199/// The single source of truth for secret patterns. `shell::redact` delegates
200/// here so the two layers can never drift apart again.
201///
202/// #718 word boundaries: the key/value alternations start with
203/// `(?:^|[^a-z0-9])` (consumed into the kept prefix — the regex crate has no
204/// lookbehind) so camelCase subwords (`superuserPassword`,
205/// `getStripeSecretKey`) never trigger a rule, while SNAKE_CASE env names
206/// (`GITHUB_FEEDBACK_TOKEN`) still do: `_` remains a permitted predecessor.
207fn redaction_rules() -> Vec<Rule> {
208    vec![
209        Rule {
210            label: "Bearer token",
211            re: static_regex!(r"(?i)(bearer\s+)[a-zA-Z0-9\-_\.]{8,}"),
212            guard_value: false,
213        },
214        Rule {
215            label: "Authorization header",
216            re: static_regex!(r"(?i)(authorization:\s*(?:basic|bearer|token)\s+)[^\s\r\n]+"),
217            guard_value: false,
218        },
219        // Key/value secrets: group 1 = predecessor + `name=`/`name: ` prefix
220        // (kept), group 2 = the value (redacted unless benign — GH #430/#718).
221        Rule {
222            label: "API key param",
223            re: static_regex!(
224                r#"(?im)((?:^|[^a-z0-9])(?:api[_-]?key|apikey|access[_-]?key|secret[_-]?key|token|password|passwd|pwd|secret)\s*[=:]\s*)([^\s\r\n,;&"']+)"#
225            ),
226            guard_value: true,
227        },
228        // Whole token is the secret — no prefix group, so the entire match is
229        // replaced. (Previously group 1 captured the key itself and leaked it.)
230        Rule {
231            label: "AWS key",
232            re: static_regex!(r"AKIA[0-9A-Z]{12,}"),
233            guard_value: false,
234        },
235        Rule {
236            label: "Private key block",
237            re: static_regex!(
238                r"(?s)(-----BEGIN\s+(?:RSA\s+)?PRIVATE\s+KEY-----).+?-----END\s+(?:RSA\s+)?PRIVATE\s+KEY-----"
239            ),
240            guard_value: false,
241        },
242        Rule {
243            label: "GitHub token",
244            re: static_regex!(r"(gh[pousr]_)[a-zA-Z0-9]{20,}"),
245            guard_value: false,
246        },
247        // Group 1 = prefix (kept), group 2 = the 32+ char value. Guarded since
248        // #718: 32-char identifiers like `confirmRequiredEndpointKeySchema`
249        // are references, not secrets.
250        Rule {
251            label: "Generic long secret",
252            re: static_regex!(
253                r#"(?im)((?:^|[^a-z0-9])(?:key|token|secret|password|credential|auth)\s*[=:]\s*)(['"]?[a-zA-Z0-9+/=\-_]{32,}['"]?)"#
254            ),
255            guard_value: true,
256        },
257    ]
258}
259
260pub fn redact_text(input: &str) -> String {
261    redact_text_with_excludes(input, &[])
262}
263
264/// #718: `redact_text` with subtractive user patterns from
265/// `[secret_detection].exclude_patterns` — a match covered by any exclude
266/// regex is kept verbatim, so known-safe naming conventions can be carved out
267/// without disabling secret detection wholesale.
268pub fn redact_text_with_excludes(input: &str, excludes: &[regex::Regex]) -> String {
269    let mut out = input.to_string();
270    for rule in redaction_rules() {
271        out = rule
272            .re
273            .replace_all(&out, |caps: &regex::Captures| {
274                let whole = caps.get(0).map_or("", |m| m.as_str());
275                if excludes.iter().any(|ex| ex.is_match(whole)) {
276                    return whole.to_string();
277                }
278                if rule.guard_value
279                    && let Some(value) = caps.get(2)
280                    && is_benign_secret_value(value.as_str())
281                {
282                    // Not a secret (identifier reference, literal, placeholder)
283                    // — keep verbatim (#430, #718).
284                    return whole.to_string();
285                }
286                match caps.get(1) {
287                    Some(prefix) => format!("{}[REDACTED:{}]", prefix.as_str(), rule.label),
288                    None => format!("[REDACTED:{}]", rule.label),
289                }
290            })
291            .to_string();
292    }
293    out
294}
295
296/// Compile the configured `exclude_patterns` (#718). Invalid regexes are
297/// skipped — a broken exclude must never disable redaction.
298pub fn config_exclude_patterns() -> Vec<regex::Regex> {
299    crate::core::config::Config::load()
300        .secret_detection
301        .exclude_patterns
302        .iter()
303        .filter_map(|p| regex::Regex::new(p).ok())
304        .collect()
305}
306
307/// Apply caller-supplied policy redaction patterns on top of the built-in
308/// secret rules: each regex match becomes `[REDACTED:<label>]`. Returns the
309/// transformed text and the number of redactions applied (for audit counts).
310///
311/// Used by context policy packs (GL #673) so a pack's `[redaction]` block
312/// actually removes matching content from what the model sees. The patterns are
313/// the pack's `[redaction]` entries, precompiled by
314/// [`crate::core::policy::runtime`].
315#[must_use]
316pub fn redact_with_patterns(input: &str, patterns: &[(String, regex::Regex)]) -> (String, usize) {
317    let mut out = input.to_string();
318    let mut hits = 0usize;
319    for (label, re) in patterns {
320        let mut local = 0usize;
321        out = re
322            .replace_all(&out, |_caps: &regex::Captures| {
323                local += 1;
324                format!("[REDACTED:{label}]")
325            })
326            .to_string();
327        hits += local;
328    }
329    (out, hits)
330}
331
332#[cfg(test)]
333mod tests {
334    use super::*;
335
336    #[test]
337    fn redacts_bearer_token() {
338        let s = "Authorization: Bearer abcdefghijklmnopqrstuvwxyz012345";
339        let out = redact_text(s);
340        assert!(out.contains("[REDACTED"));
341        assert!(!out.contains("abcdefghijklmnopqrstuvwxyz"));
342    }
343
344    #[test]
345    fn redacts_private_key_block() {
346        let s = "-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----";
347        let out = redact_text(s);
348        assert!(out.contains("[REDACTED"));
349        assert!(!out.contains("\nabc\n"));
350    }
351
352    #[test]
353    fn redacts_api_key_param_value() {
354        let out = redact_text("password=hunter2-super-secret-value");
355        assert!(
356            out.contains("password=[REDACTED:API key param]"),
357            "got: {out}"
358        );
359        assert!(!out.contains("hunter2"));
360    }
361
362    /// GH #430: TypeScript type annotations and language literals must NOT be
363    /// redacted — over-eager masking corrupted source files read via ctx_read.
364    #[test]
365    fn keeps_non_secret_literals() {
366        for s in [
367            "password: undefined",
368            "secret: string",
369            "token: null",
370            "apiKey: boolean",
371            "password = false",
372            "secret: any",
373            "let pwd: number = 1",
374        ] {
375            assert_eq!(redact_text(s), s, "must not redact non-secret literal: {s}");
376        }
377    }
378
379    /// GH #430: TS type annotations (generics, unions, arrays, function/object
380    /// types) carry angle brackets / brackets that real secret tokens never do,
381    /// so they must survive verbatim even when the key looks sensitive.
382    #[test]
383    fn keeps_type_annotations() {
384        for s in [
385            "password: Promise<string>",
386            "apiKey: Record<string, unknown>",
387            "token: string[]",
388            "secret: () => void",
389            "password: string | undefined",
390            "credential: { value: string }",
391        ] {
392            assert_eq!(redact_text(s), s, "must not redact type annotation: {s}");
393        }
394    }
395
396    /// Whole-token secrets must be removed, not annotated in place — previously
397    /// the closure kept group 1 (the key itself) and only appended `[REDACTED]`.
398    #[test]
399    fn fully_redacts_aws_key() {
400        let out = redact_text("AKIAIOSFODNN7EXAMPLE");
401        assert!(
402            !out.contains("AKIAIOSFODNN7EXAMPLE"),
403            "AWS key leaked: {out}"
404        );
405        assert!(out.contains("[REDACTED:AWS key]"));
406    }
407
408    #[test]
409    fn fully_redacts_generic_long_secret() {
410        // `credential=` is not covered by the API-key-param rule, so this
411        // exercises the generic fallback (the previously leaky path).
412        let secret = "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6"; // 32 chars
413        let out = redact_text(&format!("credential={secret}"));
414        assert!(!out.contains(secret), "long secret leaked: {out}");
415        assert!(
416            out.contains("credential=[REDACTED:Generic long secret]"),
417            "got: {out}"
418        );
419    }
420
421    #[test]
422    fn redacts_github_token_keeping_prefix() {
423        let out = redact_text("ghp_abcdefghijklmnopqrstuvwxyz0123");
424        assert!(out.starts_with("ghp_[REDACTED:GitHub token]"), "got: {out}");
425        assert!(!out.contains("abcdefghijklmnopqrstuvwxyz"));
426    }
427
428    // ── #718: benign identifier references, prose and placeholders ──
429
430    /// Repro 1: prose that mentions a keyword must not have the following
431    /// word redacted — "token: SvelteKit's…" is documentation, not a secret.
432    #[test]
433    fn keeps_prose_identifier_after_keyword() {
434        let s = "the CSRF token: SvelteKit's native origin-check on form actions";
435        assert_eq!(redact_text(s), s, "prose must survive verbatim");
436    }
437
438    /// Repro 2: camelCase subwords must not trigger the keyword alternation,
439    /// and identifier/property-access RHS values are references, not secrets.
440    #[test]
441    fn keeps_identifier_and_property_references() {
442        for s in [
443            "superuserPassword: inputEnv.POCKETBASE_SUPERUSER_PASSWORD",
444            "export const getStripeSecretKey = serverEnv.getStripeSecretKey;",
445            "const apiKey = config.stripeApiKey",
446        ] {
447            assert_eq!(redact_text(s), s, "identifier reference redacted: {s}");
448        }
449    }
450
451    /// Repro 3: a 32+ char identifier (Zod schema name) is a reference —
452    /// "Generic long secret" needs the same value guard as the API-key rule.
453    #[test]
454    fn keeps_long_schema_identifier() {
455        let s = "endpoint_key: confirmRequiredEndpointKeySchema,";
456        assert_eq!(redact_text(s), s, "schema identifier must not be redacted");
457    }
458
459    /// Repro 4: obvious placeholder values (.env.example) are documentation.
460    #[test]
461    fn keeps_placeholder_values() {
462        for s in [
463            "GITHUB_FEEDBACK_TOKEN=ghp_change_me",
464            "API_KEY=your_key_here",
465            "password=<insert-password>",
466            "SECRET_KEY=xxxxxxxx",
467        ] {
468            assert_eq!(redact_text(s), s, "placeholder redacted: {s}");
469        }
470    }
471
472    /// The flip side: real secret-shaped values must STILL be redacted after
473    /// the #718 guards.
474    #[test]
475    fn still_redacts_real_secret_values() {
476        // Digit-bearing value after a snake_case env name.
477        let out = redact_text("GITHUB_TOKEN=ghpA1b2c3d4e5f6g7h8");
478        assert!(!out.contains("ghpA1b2c3d4e5f6g7h8"), "leaked: {out}");
479        // SNAKE_CASE env assignment with digits (the _ predecessor stays a
480        // word boundary that MATCHES).
481        let out = redact_text("MY_SECRET=abc123def456ghi789");
482        assert!(!out.contains("abc123def456ghi789"), "leaked: {out}");
483        // Quoted 32+ char literal: a quoted value is never an identifier
484        // reference, so the Generic-long-secret guard keeps redacting it.
485        let quoted = "key: 'abcdefghijklmnopqrstuvwxyzabcdef'";
486        let out = redact_text(quoted);
487        assert!(
488            !out.contains("abcdefghijklmnopqrstuvwxyzabcdef"),
489            "leaked: {out}"
490        );
491    }
492
493    /// #718: exclude_patterns carve matches out subtractively.
494    #[test]
495    fn exclude_patterns_skip_matching_redactions() {
496        let excludes = vec![regex::Regex::new(r"LCTX_TEST_\w+").unwrap()];
497        let input = "token=LCTX_TEST_a1b2c3d4e5";
498        assert_eq!(
499            redact_text_with_excludes(input, &excludes),
500            input,
501            "excluded match must stay verbatim"
502        );
503        // Without the exclude the same value IS redacted (digits → secret).
504        assert!(redact_text(input).contains("[REDACTED"));
505    }
506
507    #[test]
508    fn identifier_and_placeholder_heuristics() {
509        assert!(is_identifier_reference("serverEnv.getStripeSecretKey"));
510        assert!(is_identifier_reference("confirmRequiredEndpointKeySchema"));
511        assert!(is_identifier_reference("$scope._private"));
512        assert!(!is_identifier_reference("abc123"), "digits → secret-shaped");
513        assert!(!is_identifier_reference("\"quoted\""), "literal value");
514        assert!(!is_identifier_reference("a-b"), "dash is not identifier");
515        assert!(is_placeholder_value("ghp_change_me"));
516        assert!(is_placeholder_value("<token>"));
517        assert!(is_placeholder_value("your_api_key_123"));
518        assert!(!is_placeholder_value("A1b2C3d4E5f6G7h8"));
519    }
520
521    #[test]
522    fn policy_patterns_redact_with_label_and_count() {
523        let patterns = vec![(
524            "employee_id".to_string(),
525            regex::Regex::new(r"EMP-\d{4}").unwrap(),
526        )];
527        let (out, hits) = redact_with_patterns("user EMP-1234 and EMP-5678", &patterns);
528        assert_eq!(hits, 2);
529        assert!(!out.contains("EMP-1234"));
530        assert!(out.contains("[REDACTED:employee_id]"));
531    }
532
533    #[test]
534    fn policy_patterns_noop_when_no_match() {
535        let patterns = vec![("iban".to_string(), regex::Regex::new(r"CH\d{2}").unwrap())];
536        let (out, hits) = redact_with_patterns("nothing sensitive here", &patterns);
537        assert_eq!(hits, 0);
538        assert_eq!(out, "nothing sensitive here");
539    }
540
541    /// GH #827: scientific notation and plain numbers after keys containing
542    /// `token` must not be redacted — these are pricing/cost fields.
543    #[test]
544    fn keeps_numeric_values_827() {
545        for (input, desc) in [
546            ("input_cost_per_token: 1.4e-06", "scientific notation"),
547            ("output_cost_per_token: 4.4e-06", "scientific notation"),
548            (
549                "cache_read_input_token_cost: 1.9e-07",
550                "scientific notation",
551            ),
552            ("token: 600", "plain integer"),
553            ("secret: 0.5", "decimal float"),
554            ("api_key: 42", "small integer"),
555            ("password: 3.14", "pi float"),
556        ] {
557            assert_eq!(
558                redact_text(input),
559                input,
560                "must not redact numeric value ({desc}): {input}"
561            );
562        }
563    }
564
565    /// GH #827: env-variable references must not be redacted — they are
566    /// pointers to secrets, not the secrets themselves.
567    #[test]
568    fn keeps_env_references_827() {
569        for (input, desc) in [
570            (
571                "api_key: os.environ/MY_SERVICE_API_KEY",
572                "Python os.environ/",
573            ),
574            ("secret: os.getenv(MY_KEY)", "Python os.getenv()"),
575            ("token: process.env.API_TOKEN", "Node process.env"),
576            (
577                "password: inputEnv.POCKETBASE_SUPERUSER_PASSWORD",
578                "inputEnv dot ref",
579            ),
580            ("api_key: ${MY_API_KEY}", "shell interpolation ${}"),
581            ("secret: $MY_SECRET", "shell $VAR"),
582            ("token: %API_TOKEN%", "Windows %VAR%"),
583            ("api_key: ENV[API_KEY]", "Ruby ENV[]"),
584            ("secret: env(SECRET_KEY)", "Laravel env()"),
585            ("password: System.getenv(DB_PASS)", "Java System.getenv"),
586        ] {
587            assert_eq!(
588                redact_text(input),
589                input,
590                "must not redact env ref ({desc}): {input}"
591            );
592        }
593    }
594
595    /// GH #827: real secrets must STILL be redacted (regression guard).
596    #[test]
597    fn still_redacts_real_secrets_827() {
598        for input in [
599            "api_key: sk-1234567890abcdef1234567890abcdef",
600            "password: hunter2-super-secret-value",
601            "token: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.payload.signature",
602        ] {
603            let out = redact_text(input);
604            assert!(
605                out.contains("[REDACTED"),
606                "must redact real secret: {input} -> {out}"
607            );
608        }
609    }
610}
lean_ctx/core/redaction.rs

lean_ctx/core/
redaction.rs
