llm_browser_testkit/
redact.rs

1//! Secret redaction for every report sink.
2//!
3//! All run output flows through [`crate::reporting::Reporter`]. The
4//! [`Redactor`] held by the reporter replaces known secret values with
5//! `[REDACTED]` before any sink (console, NDJSON, JUnit, GitHub, Perfetto)
6//! sees the text, so a leaked API key or token can never make it into a log
7//! or CI report.
8//!
9//! Secrets come from three places, all funneled into one redactor:
10//!
11//! - **Config-derived** — `collect_secrets_from_scenario_config` gathers
12//!   `llm_api_key`, per-endpoint API keys, Entra client secrets, AWS static
13//!   credentials, and the values of sensitive-named headers, and the runner
14//!   registers them on every [`crate::runner::ScenarioRunner`] it builds.
15//! - **Runtime-obtained** — tokens fetched by
16//!   [`crate::auth`](auth) (token commands, header commands, Entra
17//!   client-credentials and managed identity) are pushed into a
18//!   process-global observed-secret registry via [`observe_secret`], which
19//!   every [`Redactor::redact`] consults. They are registered before the
20//!   `LlmCallFinished` event that describes the call is emitted, so the
21//!   very event that echoes a token is redacted.
22//! - **Explicit extras** — callers add literal values via
23//!   `Reporter::add_redaction_secret` (the CLI's `--redact` /
24//!   `HARNESS_REDACT`), for secrets not present in the config (URL query
25//!   tokens, scenario-embedded test data).
26//!
27//! Redaction is exact-match, case-sensitive substring replacement. Values
28//! shorter than [`MIN_SECRET_LEN`] are skipped when derived from config or
29//! runtime sources so short, common strings (e.g. `"dev"`) do not destroy
30//! log readability — explicit extras always apply.
31//!
32//! Raw `eprintln!` sites that bypass the reporter (the `#[browser_test]`
33//! run-report strings, MCP/A2A server startup banners, the cost report)
34//! are intentionally out of scope: they carry no secret-bearing text today.
35
36use std::collections::{HashMap, HashSet, VecDeque};
37use std::sync::{LazyLock, PoisonError, RwLock};
38
39use crate::events::TestEvent;
40use crate::scenario::ScenarioConfig;
41
42/// Replacement text for every redacted value.
43const REDACTED: &str = "[REDACTED]";
44
45/// Shortest secret worth registering from config-derived or runtime
46/// sources. Explicit extras (`add_secret` with `min_len 0`) always apply.
47const MIN_SECRET_LEN: usize = 6;
48
49/// Maximum number of runtime-observed secrets kept in the global registry,
50/// bounding memory in long-running MCP/A2A server processes. Oldest
51/// entries are dropped first.
52const OBSERVED_CAP: usize = 512;
53
54/// Header names whose values are treated as secrets (compared lowercased).
55/// Innocuous headers (`x-org`, `content-type`) are NOT registered.
56const SENSITIVE_HEADER_NAMES: &[&str] = &[
57    "authorization",
58    "proxy-authorization",
59    "api-key",
60    "api_key",
61    "x-api-key",
62    "x-auth-token",
63    "authentication",
64    "token",
65    "cookie",
66    "x-amz-security-token",
67];
68
69/// Ordered set of runtime-observed secrets: `set` enables O(1) dedupe and
70/// lookup, `order` (insertion order) enables dropping the oldest entry when
71/// the cap is reached. `HashSet` alone has no ordering.
72#[derive(Default)]
73struct ObservedRegistry {
74    set: HashSet<String>,
75    order: VecDeque<String>,
76}
77
78impl ObservedRegistry {
79    /// Inserts a value, dropping the oldest entry at [`OBSERVED_CAP`].
80    fn insert(&mut self, value: String) {
81        if self.set.contains(&value) {
82            return;
83        }
84        if self.order.len() >= OBSERVED_CAP {
85            if let Some(oldest) = self.order.pop_front() {
86                self.set.remove(&oldest);
87            }
88        }
89        self.set.insert(value.clone());
90        self.order.push_back(value);
91    }
92
93    /// Iterates the registered secrets in insertion order.
94    fn iter(&self) -> impl Iterator<Item = &str> {
95        self.order.iter().map(String::as_str)
96    }
97}
98
99/// Process-global registry of runtime-observed secrets, shared by every
100/// reporter in the process (including macro-path default reporters).
101static OBSERVED: LazyLock<RwLock<ObservedRegistry>> =
102    LazyLock::new(|| RwLock::new(ObservedRegistry::default()));
103
104/// Registers a runtime-obtained secret so every subsequent
105/// [`Redactor::redact`] call replaces it: token-command output,
106/// header-command output, and Entra/IMDS access tokens.
107///
108/// Values shorter than [`MIN_SECRET_LEN`] are ignored; duplicates are
109/// deduplicated; the registry is capped at [`OBSERVED_CAP`] entries.
110pub fn observe_secret(value: &str) {
111    if value.len() < MIN_SECRET_LEN {
112        return;
113    }
114    OBSERVED
115        .write()
116        .unwrap_or_else(PoisonError::into_inner)
117        .insert(value.to_owned());
118}
119
120/// Replaces known secret values in a string with `[REDACTED]`.
121#[derive(Default)]
122pub struct Redactor {
123    secrets: Vec<String>,
124}
125
126impl Redactor {
127    /// Creates an empty redactor.
128    #[must_use]
129    pub fn new() -> Self {
130        Self::default()
131    }
132
133    /// Records a secret value when it is at least `min_len` characters
134    /// long (and non-empty). Explicit extras pass `min_len 0` so they
135    /// always apply.
136    pub fn add_secret(&mut self, value: &str, min_len: usize) {
137        if value.is_empty() || value.len() < min_len {
138            return;
139        }
140        if self.secrets.iter().any(|s| s == value) {
141            return;
142        }
143        self.secrets.push(value.to_owned());
144    }
145
146    /// Records config-derived secrets, applying [`MIN_SECRET_LEN`].
147    pub fn add_secret_values(&mut self, values: impl IntoIterator<Item = String>) {
148        for value in values {
149            self.add_secret(&value, MIN_SECRET_LEN);
150        }
151    }
152
153    /// Whether no local secrets are registered (runtime-observed secrets
154    /// may still apply).
155    #[must_use]
156    pub const fn is_empty(&self) -> bool {
157        self.secrets.is_empty()
158    }
159
160    /// Replaces every occurrence of every registered secret (local list
161    /// first, then the process-global observed-secret registry) with
162    /// `[REDACTED]`. Infallible: a broken registry can never fail an emit.
163    #[must_use]
164    pub fn redact(&self, input: &str) -> String {
165        let mut out = input.to_owned();
166        for secret in &self.secrets {
167            out = out.replace(secret, REDACTED);
168        }
169        {
170            let observed = OBSERVED.read().unwrap_or_else(PoisonError::into_inner);
171            for secret in observed.iter() {
172                out = out.replace(secret, REDACTED);
173            }
174        }
175        out
176    }
177
178    /// Returns a copy of the event with every `String` field redacted;
179    /// numeric/enum fields are copied verbatim.
180    #[must_use]
181    #[allow(clippy::too_many_lines)]
182    pub fn redact_event(&self, event: &TestEvent) -> TestEvent {
183        match event {
184            TestEvent::RunStarted { total_tests } => TestEvent::RunStarted {
185                total_tests: *total_tests,
186            },
187            TestEvent::TestStarted { test } => TestEvent::TestStarted {
188                test: self.redact(test),
189            },
190            TestEvent::StepStarted { test, index, label } => TestEvent::StepStarted {
191                test: self.redact(test),
192                index: *index,
193                label: self.redact(label),
194            },
195            TestEvent::StepFinished {
196                test,
197                index,
198                label,
199                status,
200                duration_ms,
201                message,
202                diagnostics,
203                screenshot,
204            } => TestEvent::StepFinished {
205                test: self.redact(test),
206                index: *index,
207                label: self.redact(label),
208                status: *status,
209                duration_ms: *duration_ms,
210                message: self.redact(message),
211                diagnostics: diagnostics.as_deref().map(|s| self.redact(s)),
212                screenshot: screenshot.as_deref().map(|s| self.redact(s)),
213            },
214            TestEvent::LlmCallStarted {
215                test,
216                index,
217                endpoint,
218                model,
219                purpose,
220            } => TestEvent::LlmCallStarted {
221                test: self.redact(test),
222                index: *index,
223                endpoint: self.redact(endpoint),
224                model: self.redact(model),
225                purpose: self.redact(purpose),
226            },
227            TestEvent::LlmCallFinished {
228                test,
229                index,
230                endpoint,
231                model,
232                purpose,
233                ok,
234                duration_ms,
235                input_tokens,
236                output_tokens,
237                cost,
238                error,
239            } => TestEvent::LlmCallFinished {
240                test: self.redact(test),
241                index: *index,
242                endpoint: self.redact(endpoint),
243                model: self.redact(model),
244                purpose: self.redact(purpose),
245                ok: *ok,
246                duration_ms: *duration_ms,
247                input_tokens: *input_tokens,
248                output_tokens: *output_tokens,
249                cost: *cost,
250                error: error.as_deref().map(|s| self.redact(s)),
251            },
252            TestEvent::TestFinished {
253                test,
254                passed,
255                failed,
256                skipped,
257                duration_ms,
258                cost,
259                tokens,
260                calls,
261            } => TestEvent::TestFinished {
262                test: self.redact(test),
263                passed: *passed,
264                failed: *failed,
265                skipped: *skipped,
266                duration_ms: *duration_ms,
267                cost: *cost,
268                tokens: *tokens,
269                calls: *calls,
270            },
271            TestEvent::RunFinished {
272                tests_passed,
273                tests_failed,
274                steps_passed,
275                steps_failed,
276                steps_skipped,
277                total_cost,
278                total_tokens,
279                total_calls,
280            } => TestEvent::RunFinished {
281                tests_passed: *tests_passed,
282                tests_failed: *tests_failed,
283                steps_passed: *steps_passed,
284                steps_failed: *steps_failed,
285                steps_skipped: *steps_skipped,
286                total_cost: *total_cost,
287                total_tokens: *total_tokens,
288                total_calls: *total_calls,
289            },
290            TestEvent::Warning { message } => TestEvent::Warning {
291                message: self.redact(message),
292            },
293        }
294    }
295}
296
297/// Gathers the startup-known secrets of a (merged) scenario config:
298/// LLM API keys, endpoint credentials, and sensitive header values.
299///
300/// Returns raw strings; the [`MIN_SECRET_LEN`] guard is applied at insert
301/// time by [`Redactor::add_secret_values`].
302#[must_use]
303pub fn collect_secrets_from_scenario_config(cfg: &ScenarioConfig) -> Vec<String> {
304    let mut secrets: Vec<String> = Vec::new();
305    if let Some(key) = &cfg.llm_api_key {
306        secrets.push(key.clone());
307    }
308    collect_sensitive_header_values(&cfg.llm_headers, &mut secrets);
309    for ep in cfg.endpoints.values() {
310        if let Some(key) = &ep.api_key {
311            secrets.push(key.clone());
312        }
313        if let Some(secret) = &ep.auth.client_secret {
314            secrets.push(secret.clone());
315        }
316        if let Some(key) = &ep.aws.secret_access_key {
317            secrets.push(key.clone());
318        }
319        if let Some(token) = &ep.aws.session_token {
320            secrets.push(token.clone());
321        }
322        collect_sensitive_header_values(&ep.headers, &mut secrets);
323    }
324    secrets
325}
326
327/// Pushes the values of sensitive-named headers into `out` (matched
328/// case-insensitively).
329fn collect_sensitive_header_values(headers: &HashMap<String, String>, out: &mut Vec<String>) {
330    for (name, value) in headers {
331        if SENSITIVE_HEADER_NAMES
332            .iter()
333            .any(|n| name.to_ascii_lowercase() == *n)
334        {
335            out.push(value.clone());
336        }
337    }
338}
339
340#[cfg(test)]
341mod tests {
342    use std::collections::HashMap;
343
344    use super::{
345        collect_secrets_from_scenario_config, observe_secret, Redactor, MIN_SECRET_LEN,
346        OBSERVED_CAP,
347    };
348    use crate::events::{StepStatus, TestEvent};
349    use crate::scenario::ScenarioConfig;
350
351    #[test]
352    fn test_redact_exact_replacement() {
353        let mut r = Redactor::new();
354        r.add_secret("sk-secret-key", 0);
355        assert_eq!(r.redact("Bearer sk-secret-key"), "Bearer [REDACTED]");
356    }
357
358    #[test]
359    fn test_redact_multiple_secrets() {
360        let mut r = Redactor::new();
361        r.add_secret("key-one", 0);
362        r.add_secret("token-two", 0);
363        let out = r.redact("key-one then token-two then key-one again");
364        assert_eq!(out, "[REDACTED] then [REDACTED] then [REDACTED] again");
365    }
366
367    #[test]
368    fn test_redact_secret_mid_string() {
369        let mut r = Redactor::new();
370        r.add_secret("abc123", 0);
371        assert_eq!(
372            r.redact("https://app.example.com/login?token=abc123&next=/x"),
373            "https://app.example.com/login?token=[REDACTED]&next=/x"
374        );
375    }
376
377    #[test]
378    fn test_redact_empty_redactor_unchanged() {
379        let r = Redactor::new();
380        let input = "plain hello world 12345";
381        assert_eq!(r.redact(input), input);
382    }
383
384    #[test]
385    fn test_redact_empty_secret_ignored() {
386        let mut r = Redactor::new();
387        r.add_secret("", 0);
388        assert_eq!(r.redact("a secret-leak here"), "a secret-leak here");
389    }
390
391    #[test]
392    fn test_length_guard_skips_short_config_secrets() {
393        let cfg = ScenarioConfig {
394            llm_api_key: Some("short".into()), // 5 chars < MIN_SECRET_LEN
395            ..ScenarioConfig::default()
396        };
397        let mut r = Redactor::new();
398        r.add_secret_values(collect_secrets_from_scenario_config(&cfg));
399        assert!(r.is_empty());
400        assert_eq!(r.redact("token short here"), "token short here");
401    }
402
403    #[test]
404    fn test_explicit_min_len_zero_always_applies() {
405        let mut r = Redactor::new();
406        r.add_secret("short", 0);
407        assert!(!r.is_empty());
408        assert_eq!(r.redact("token short here"), "token [REDACTED] here");
409    }
410
411    #[test]
412    fn test_collect_secrets_from_config() {
413        let mut headers = HashMap::new();
414        headers.insert("X-API-Key".to_owned(), "hdr-key-value".to_owned());
415        headers.insert("x-org".to_owned(), "acme".to_owned());
416        headers.insert("content-type".to_owned(), "application/json".to_owned());
417        let mut endpoints = HashMap::new();
418        let ep = crate::scenario::EndpointConfig {
419            api_key: Some("endpoint-key".to_owned()),
420            headers: HashMap::from([("Authorization".to_owned(), "Bearer ep-bearer".to_owned())]),
421            auth: crate::scenario::AuthConfig {
422                client_secret: Some("entra-secret".to_owned()),
423                ..crate::scenario::AuthConfig::default()
424            },
425            aws: crate::scenario::AwsConfig {
426                secret_access_key: Some("aws-secret".to_owned()),
427                session_token: Some("aws-session".to_owned()),
428                ..crate::scenario::AwsConfig::default()
429            },
430            fallbacks: vec!["fallback".to_owned()],
431            ..crate::scenario::EndpointConfig::default()
432        };
433        endpoints.insert("primary".to_owned(), ep);
434        endpoints.insert(
435            "fallback".to_owned(),
436            crate::scenario::EndpointConfig {
437                api_key: Some("fallback-key".to_owned()),
438                ..crate::scenario::EndpointConfig::default()
439            },
440        );
441        let cfg = ScenarioConfig {
442            llm_api_key: Some("global-key".to_owned()),
443            llm_headers: headers,
444            endpoints,
445            ..ScenarioConfig::default()
446        };
447        let secrets = collect_secrets_from_scenario_config(&cfg);
448        for expected in [
449            "global-key",
450            "hdr-key-value",
451            "endpoint-key",
452            "Bearer ep-bearer",
453            "entra-secret",
454            "aws-secret",
455            "aws-session",
456            "fallback-key",
457        ] {
458            assert!(secrets.iter().any(|s| s == expected), "missing {expected}");
459        }
460        assert!(
461            secrets
462                .iter()
463                .all(|s| s != "acme" && s != "application/json"),
464            "innocuous header values must not be registered"
465        );
466    }
467
468    #[test]
469    fn test_registry_observe_and_redact() {
470        observe_secret("registry-token-123456");
471        let r = Redactor::new();
472        assert_eq!(
473            r.redact("error body: registry-token-123456"),
474            "error body: [REDACTED]"
475        );
476    }
477
478    #[test]
479    fn test_registry_deduplicates() {
480        observe_secret("registry-dedup-abcdef");
481        observe_secret("registry-dedup-abcdef");
482        let r = Redactor::new();
483        assert_eq!(r.redact("registry-dedup-abcdef"), "[REDACTED]");
484    }
485
486    #[test]
487    fn test_registry_min_len_guard() {
488        observe_secret("tiny");
489        let r = Redactor::new();
490        assert_eq!(r.redact("token tiny here"), "token tiny here");
491    }
492
493    #[test]
494    fn test_registry_cap_drops_oldest() {
495        let r = Redactor::new();
496        for i in 0..OBSERVED_CAP + 10 {
497            observe_secret(&format!("registry-cap-secret-{i:03}"));
498        }
499        // The first 10 entries were evicted...
500        assert_eq!(
501            r.redact("registry-cap-secret-000"),
502            "registry-cap-secret-000"
503        );
504        // ...while the newest entries still redact.
505        assert_eq!(r.redact("registry-cap-secret-521"), "[REDACTED]");
506    }
507
508    #[test]
509    fn test_redact_event_step_finished() {
510        let mut r = Redactor::new();
511        r.add_secret("tok-query-secret", 0);
512        let event = TestEvent::StepFinished {
513            test: "login".into(),
514            index: 0,
515            label: "[navigate] /dashboard?token=tok-query-secret".into(),
516            status: StepStatus::Failed,
517            duration_ms: 100,
518            message: "element not found at /x?token=tok-query-secret".into(),
519            diagnostics: Some("url: /x?token=tok-query-secret".into()),
520            screenshot: Some("artifacts/login.png".into()),
521        };
522        let redacted = r.redact_event(&event);
523        let TestEvent::StepFinished {
524            label,
525            message,
526            diagnostics,
527            screenshot,
528            status,
529            ..
530        } = redacted
531        else {
532            panic!("expected step_finished");
533        };
534        assert_eq!(label, "[navigate] /dashboard?token=[REDACTED]");
535        assert_eq!(message, "element not found at /x?token=[REDACTED]");
536        assert_eq!(diagnostics.as_deref(), Some("url: /x?token=[REDACTED]"));
537        assert_eq!(screenshot.as_deref(), Some("artifacts/login.png"));
538        assert_eq!(status, StepStatus::Failed);
539    }
540
541    #[test]
542    fn test_redact_event_llm_call_error_with_registry_token() {
543        observe_secret("runtime-token-987654");
544        let mut r = Redactor::new();
545        r.add_secret("sk-static-key", 0);
546        let event = TestEvent::LlmCallFinished {
547            test: "login".into(),
548            index: 0,
549            endpoint: "default".into(),
550            model: "deepseek".into(),
551            purpose: "targeting".into(),
552            ok: false,
553            duration_ms: 900,
554            input_tokens: 0,
555            output_tokens: 0,
556            cost: 0.0,
557            error: Some("HTTP 401: Bearer sk-static-key invalid (got runtime-token-987654)".into()),
558        };
559        let redacted = r.redact_event(&event);
560        let TestEvent::LlmCallFinished { error, .. } = redacted else {
561            panic!("expected llm_call_finished");
562        };
563        assert_eq!(
564            error.as_deref(),
565            Some("HTTP 401: Bearer [REDACTED] invalid (got [REDACTED])")
566        );
567    }
568
569    #[test]
570    fn test_redact_event_preserves_non_string_fields() {
571        let r = Redactor::new();
572        let event = TestEvent::TestFinished {
573            test: "t".into(),
574            passed: 1,
575            failed: 0,
576            skipped: 0,
577            duration_ms: 12,
578            cost: 0.0042,
579            tokens: 42,
580            calls: 1,
581        };
582        let redacted = r.redact_event(&event);
583        let TestEvent::TestFinished {
584            test,
585            passed,
586            failed,
587            duration_ms,
588            cost,
589            tokens,
590            calls,
591            ..
592        } = redacted
593        else {
594            panic!("expected test_finished");
595        };
596        assert_eq!(test, "t");
597        assert_eq!((passed, failed), (1, 0));
598        assert_eq!(duration_ms, 12);
599        #[allow(clippy::float_cmp)]
600        {
601            assert_eq!(cost, 0.0042);
602        }
603        assert_eq!((tokens, calls), (42, 1));
604    }
605
606    #[test]
607    fn test_min_secret_len_constant_matches_guard() {
608        // Sanity: the default guard applies to `add_secret_values`.
609        let mut r = Redactor::new();
610        r.add_secret_values(["12345".to_owned(), "123456".to_owned()]);
611        assert_eq!(r.secrets.len(), 1);
612        assert_eq!(r.secrets[0], "123456");
613        assert!(!r.is_empty());
614        assert_eq!(MIN_SECRET_LEN, 6);
615    }
616}
llm_browser_testkit/redact.rs

llm_browser_testkit/
redact.rs
