llm_browser_testkit/
redact.rs

1//! Secret redaction for every report sink.
2//!
3//! All run output flows through [`crate::reporting::Reporter`]. The
4//! [`Redactor`] held by the reporter replaces known secret values with
5//! `[REDACTED]` before any sink (console, NDJSON, JUnit, GitHub, Perfetto)
6//! sees the text, so a leaked API key or token can never make it into a log
7//! or CI report.
8//!
9//! Secrets come from three places, all funneled into one redactor:
10//!
11//! - **Config-derived** — `collect_secrets_from_scenario_config` gathers
12//!   `llm_api_key`, per-endpoint API keys, Entra client secrets, AWS static
13//!   credentials, and the values of sensitive-named headers, and the runner
14//!   registers them on every [`crate::runner::ScenarioRunner`] it builds.
15//! - **Runtime-obtained** — tokens fetched by
16//!   [`crate::auth`](auth) (token commands, header commands, Entra
17//!   client-credentials and managed identity) are pushed into a
18//!   process-global observed-secret registry via [`observe_secret`], which
19//!   every [`Redactor::redact`] consults. They are registered before the
20//!   `LlmCallFinished` event that describes the call is emitted, so the
21//!   very event that echoes a token is redacted.
22//! - **Explicit extras** — callers add literal values via
23//!   `Reporter::add_redaction_secret` (the CLI's `--redact` /
24//!   `HARNESS_REDACT`), for secrets not present in the config (URL query
25//!   tokens, scenario-embedded test data).
26//!
27//! Redaction is exact-match, case-sensitive substring replacement. Values
28//! shorter than [`MIN_SECRET_LEN`] are skipped when derived from config or
29//! runtime sources so short, common strings (e.g. `"dev"`) do not destroy
30//! log readability — explicit extras always apply.
31//!
32//! Raw `eprintln!` sites that bypass the reporter (the `#[browser_test]`
33//! run-report strings, MCP/A2A server startup banners, the cost report)
34//! are intentionally out of scope: they carry no secret-bearing text today.
35
36use std::collections::{HashMap, HashSet, VecDeque};
37use std::sync::{LazyLock, PoisonError, RwLock};
38
39use crate::events::TestEvent;
40use crate::scenario::ScenarioConfig;
41
42/// Replacement text for every redacted value.
43const REDACTED: &str = "[REDACTED]";
44
45/// Shortest secret worth registering from config-derived or runtime
46/// sources. Explicit extras (`add_secret` with `min_len 0`) always apply.
47const MIN_SECRET_LEN: usize = 6;
48
49/// Maximum number of runtime-observed secrets kept in the global registry,
50/// bounding memory in long-running MCP/A2A server processes. Oldest
51/// entries are dropped first.
52const OBSERVED_CAP: usize = 512;
53
54/// Header names whose values are treated as secrets (compared lowercased).
55/// Innocuous headers (`x-org`, `content-type`) are NOT registered.
56const SENSITIVE_HEADER_NAMES: &[&str] = &[
57    "authorization",
58    "proxy-authorization",
59    "api-key",
60    "api_key",
61    "x-api-key",
62    "x-auth-token",
63    "authentication",
64    "token",
65    "cookie",
66    "x-amz-security-token",
67];
68
69/// Ordered set of runtime-observed secrets: `set` enables O(1) dedupe and
70/// lookup, `order` (insertion order) enables dropping the oldest entry when
71/// the cap is reached. `HashSet` alone has no ordering.
72#[derive(Default)]
73struct ObservedRegistry {
74    set: HashSet<String>,
75    order: VecDeque<String>,
76}
77
78impl ObservedRegistry {
79    /// Inserts a value, dropping the oldest entry at [`OBSERVED_CAP`].
80    fn insert(&mut self, value: String) {
81        if self.set.contains(&value) {
82            return;
83        }
84        if self.order.len() >= OBSERVED_CAP {
85            if let Some(oldest) = self.order.pop_front() {
86                self.set.remove(&oldest);
87            }
88        }
89        self.set.insert(value.clone());
90        self.order.push_back(value);
91    }
92
93    /// Iterates the registered secrets in insertion order.
94    fn iter(&self) -> impl Iterator<Item = &str> {
95        self.order.iter().map(String::as_str)
96    }
97}
98
99/// Process-global registry of runtime-observed secrets, shared by every
100/// reporter in the process (including macro-path default reporters).
101static OBSERVED: LazyLock<RwLock<ObservedRegistry>> =
102    LazyLock::new(|| RwLock::new(ObservedRegistry::default()));
103
104/// Registers a runtime-obtained secret so every subsequent
105/// [`Redactor::redact`] call replaces it: token-command output,
106/// header-command output, and Entra/IMDS access tokens.
107///
108/// Values shorter than [`MIN_SECRET_LEN`] are ignored; duplicates are
109/// deduplicated; the registry is capped at [`OBSERVED_CAP`] entries.
110pub fn observe_secret(value: &str) {
111    if value.len() < MIN_SECRET_LEN {
112        return;
113    }
114    OBSERVED
115        .write()
116        .unwrap_or_else(PoisonError::into_inner)
117        .insert(value.to_owned());
118}
119
120/// Replaces known secret values in a string with `[REDACTED]`.
121#[derive(Default)]
122pub struct Redactor {
123    secrets: Vec<String>,
124}
125
126impl Redactor {
127    /// Creates an empty redactor.
128    #[must_use]
129    pub fn new() -> Self {
130        Self::default()
131    }
132
133    /// Records a secret value when it is at least `min_len` characters
134    /// long (and non-empty). Explicit extras pass `min_len 0` so they
135    /// always apply.
136    pub fn add_secret(&mut self, value: &str, min_len: usize) {
137        if value.is_empty() || value.len() < min_len {
138            return;
139        }
140        if self.secrets.iter().any(|s| s == value) {
141            return;
142        }
143        self.secrets.push(value.to_owned());
144    }
145
146    /// Records config-derived secrets, applying [`MIN_SECRET_LEN`].
147    pub fn add_secret_values(&mut self, values: impl IntoIterator<Item = String>) {
148        for value in values {
149            self.add_secret(&value, MIN_SECRET_LEN);
150        }
151    }
152
153    /// Whether no local secrets are registered (runtime-observed secrets
154    /// may still apply).
155    #[must_use]
156    pub const fn is_empty(&self) -> bool {
157        self.secrets.is_empty()
158    }
159
160    /// Replaces every occurrence of every registered secret (local list
161    /// first, then the process-global observed-secret registry) with
162    /// `[REDACTED]`. Infallible: a broken registry can never fail an emit.
163    #[must_use]
164    pub fn redact(&self, input: &str) -> String {
165        let mut out = input.to_owned();
166        for secret in &self.secrets {
167            out = out.replace(secret, REDACTED);
168        }
169        {
170            let observed = OBSERVED.read().unwrap_or_else(PoisonError::into_inner);
171            for secret in observed.iter() {
172                out = out.replace(secret, REDACTED);
173            }
174        }
175        out
176    }
177
178    /// Returns a copy of the event with every `String` field redacted;
179    /// numeric/enum fields are copied verbatim.
180    #[must_use]
181    #[allow(clippy::too_many_lines)]
182    pub fn redact_event(&self, event: &TestEvent) -> TestEvent {
183        match event {
184            TestEvent::RunStarted { total_tests } => TestEvent::RunStarted {
185                total_tests: *total_tests,
186            },
187            TestEvent::TestStarted { test } => TestEvent::TestStarted {
188                test: self.redact(test),
189            },
190            TestEvent::StepStarted { test, index, label } => TestEvent::StepStarted {
191                test: self.redact(test),
192                index: *index,
193                label: self.redact(label),
194            },
195            TestEvent::StepFinished {
196                test,
197                index,
198                label,
199                status,
200                duration_ms,
201                message,
202                diagnostics,
203                screenshot,
204            } => TestEvent::StepFinished {
205                test: self.redact(test),
206                index: *index,
207                label: self.redact(label),
208                status: *status,
209                duration_ms: *duration_ms,
210                message: self.redact(message),
211                diagnostics: diagnostics.as_deref().map(|s| self.redact(s)),
212                screenshot: screenshot.as_deref().map(|s| self.redact(s)),
213            },
214            TestEvent::LlmCallStarted {
215                test,
216                index,
217                endpoint,
218                model,
219                purpose,
220            } => TestEvent::LlmCallStarted {
221                test: self.redact(test),
222                index: *index,
223                endpoint: self.redact(endpoint),
224                model: self.redact(model),
225                purpose: self.redact(purpose),
226            },
227            TestEvent::LlmCallFinished {
228                test,
229                index,
230                endpoint,
231                model,
232                purpose,
233                ok,
234                duration_ms,
235                input_tokens,
236                output_tokens,
237                cached_input_tokens,
238                cost,
239                error,
240            } => TestEvent::LlmCallFinished {
241                test: self.redact(test),
242                index: *index,
243                endpoint: self.redact(endpoint),
244                model: self.redact(model),
245                purpose: self.redact(purpose),
246                ok: *ok,
247                duration_ms: *duration_ms,
248                input_tokens: *input_tokens,
249                output_tokens: *output_tokens,
250                cached_input_tokens: *cached_input_tokens,
251                cost: *cost,
252                error: error.as_deref().map(|s| self.redact(s)),
253            },
254            TestEvent::TestFinished {
255                test,
256                passed,
257                failed,
258                skipped,
259                duration_ms,
260                cost,
261                tokens,
262                input_tokens,
263                output_tokens,
264                cached_input_tokens,
265                models,
266                calls,
267            } => TestEvent::TestFinished {
268                test: self.redact(test),
269                passed: *passed,
270                failed: *failed,
271                skipped: *skipped,
272                duration_ms: *duration_ms,
273                cost: *cost,
274                tokens: *tokens,
275                input_tokens: *input_tokens,
276                output_tokens: *output_tokens,
277                cached_input_tokens: *cached_input_tokens,
278                models: models.iter().map(|m| self.redact(m)).collect(),
279                calls: *calls,
280            },
281            TestEvent::RunFinished {
282                tests_passed,
283                tests_failed,
284                steps_passed,
285                steps_failed,
286                steps_skipped,
287                total_cost,
288                total_tokens,
289                total_input_tokens,
290                total_output_tokens,
291                total_cached_input_tokens,
292                models,
293                total_calls,
294            } => TestEvent::RunFinished {
295                tests_passed: *tests_passed,
296                tests_failed: *tests_failed,
297                steps_passed: *steps_passed,
298                steps_failed: *steps_failed,
299                steps_skipped: *steps_skipped,
300                total_cost: *total_cost,
301                total_tokens: *total_tokens,
302                total_input_tokens: *total_input_tokens,
303                total_output_tokens: *total_output_tokens,
304                total_cached_input_tokens: *total_cached_input_tokens,
305                models: models.iter().map(|m| self.redact(m)).collect(),
306                total_calls: *total_calls,
307            },
308            TestEvent::Warning { message } => TestEvent::Warning {
309                message: self.redact(message),
310            },
311        }
312    }
313}
314
315/// Gathers the startup-known secrets of a (merged) scenario config:
316/// LLM API keys, endpoint credentials, and sensitive header values.
317///
318/// Returns raw strings; the [`MIN_SECRET_LEN`] guard is applied at insert
319/// time by [`Redactor::add_secret_values`].
320#[must_use]
321pub fn collect_secrets_from_scenario_config(cfg: &ScenarioConfig) -> Vec<String> {
322    let mut secrets: Vec<String> = Vec::new();
323    if let Some(key) = &cfg.llm_api_key {
324        secrets.push(key.clone());
325    }
326    collect_sensitive_header_values(&cfg.llm_headers, &mut secrets);
327    for ep in cfg.endpoints.values() {
328        if let Some(key) = &ep.api_key {
329            secrets.push(key.clone());
330        }
331        if let Some(secret) = &ep.auth.client_secret {
332            secrets.push(secret.clone());
333        }
334        if let Some(key) = &ep.aws.secret_access_key {
335            secrets.push(key.clone());
336        }
337        if let Some(token) = &ep.aws.session_token {
338            secrets.push(token.clone());
339        }
340        collect_sensitive_header_values(&ep.headers, &mut secrets);
341    }
342    secrets
343}
344
345/// Pushes the values of sensitive-named headers into `out` (matched
346/// case-insensitively).
347fn collect_sensitive_header_values(headers: &HashMap<String, String>, out: &mut Vec<String>) {
348    for (name, value) in headers {
349        if SENSITIVE_HEADER_NAMES
350            .iter()
351            .any(|n| name.to_ascii_lowercase() == *n)
352        {
353            out.push(value.clone());
354        }
355    }
356}
357
358#[cfg(test)]
359mod tests {
360    use std::collections::HashMap;
361
362    use super::{
363        collect_secrets_from_scenario_config, observe_secret, Redactor, MIN_SECRET_LEN,
364        OBSERVED_CAP,
365    };
366    use crate::events::{StepStatus, TestEvent};
367    use crate::scenario::ScenarioConfig;
368
369    #[test]
370    fn test_redact_exact_replacement() {
371        let mut r = Redactor::new();
372        r.add_secret("sk-secret-key", 0);
373        assert_eq!(r.redact("Bearer sk-secret-key"), "Bearer [REDACTED]");
374    }
375
376    #[test]
377    fn test_redact_multiple_secrets() {
378        let mut r = Redactor::new();
379        r.add_secret("key-one", 0);
380        r.add_secret("token-two", 0);
381        let out = r.redact("key-one then token-two then key-one again");
382        assert_eq!(out, "[REDACTED] then [REDACTED] then [REDACTED] again");
383    }
384
385    #[test]
386    fn test_redact_secret_mid_string() {
387        let mut r = Redactor::new();
388        r.add_secret("abc123", 0);
389        assert_eq!(
390            r.redact("https://app.example.com/login?token=abc123&next=/x"),
391            "https://app.example.com/login?token=[REDACTED]&next=/x"
392        );
393    }
394
395    #[test]
396    fn test_redact_empty_redactor_unchanged() {
397        let r = Redactor::new();
398        let input = "plain hello world 12345";
399        assert_eq!(r.redact(input), input);
400    }
401
402    #[test]
403    fn test_redact_empty_secret_ignored() {
404        let mut r = Redactor::new();
405        r.add_secret("", 0);
406        assert_eq!(r.redact("a secret-leak here"), "a secret-leak here");
407    }
408
409    #[test]
410    fn test_length_guard_skips_short_config_secrets() {
411        let cfg = ScenarioConfig {
412            llm_api_key: Some("short".into()), // 5 chars < MIN_SECRET_LEN
413            ..ScenarioConfig::default()
414        };
415        let mut r = Redactor::new();
416        r.add_secret_values(collect_secrets_from_scenario_config(&cfg));
417        assert!(r.is_empty());
418        assert_eq!(r.redact("token short here"), "token short here");
419    }
420
421    #[test]
422    fn test_explicit_min_len_zero_always_applies() {
423        let mut r = Redactor::new();
424        r.add_secret("short", 0);
425        assert!(!r.is_empty());
426        assert_eq!(r.redact("token short here"), "token [REDACTED] here");
427    }
428
429    #[test]
430    fn test_collect_secrets_from_config() {
431        let mut headers = HashMap::new();
432        headers.insert("X-API-Key".to_owned(), "hdr-key-value".to_owned());
433        headers.insert("x-org".to_owned(), "acme".to_owned());
434        headers.insert("content-type".to_owned(), "application/json".to_owned());
435        let mut endpoints = HashMap::new();
436        let ep = crate::scenario::EndpointConfig {
437            api_key: Some("endpoint-key".to_owned()),
438            headers: HashMap::from([("Authorization".to_owned(), "Bearer ep-bearer".to_owned())]),
439            auth: crate::scenario::AuthConfig {
440                client_secret: Some("entra-secret".to_owned()),
441                ..crate::scenario::AuthConfig::default()
442            },
443            aws: crate::scenario::AwsConfig {
444                secret_access_key: Some("aws-secret".to_owned()),
445                session_token: Some("aws-session".to_owned()),
446                ..crate::scenario::AwsConfig::default()
447            },
448            fallbacks: vec!["fallback".to_owned()],
449            ..crate::scenario::EndpointConfig::default()
450        };
451        endpoints.insert("primary".to_owned(), ep);
452        endpoints.insert(
453            "fallback".to_owned(),
454            crate::scenario::EndpointConfig {
455                api_key: Some("fallback-key".to_owned()),
456                ..crate::scenario::EndpointConfig::default()
457            },
458        );
459        let cfg = ScenarioConfig {
460            llm_api_key: Some("global-key".to_owned()),
461            llm_headers: headers,
462            endpoints,
463            ..ScenarioConfig::default()
464        };
465        let secrets = collect_secrets_from_scenario_config(&cfg);
466        for expected in [
467            "global-key",
468            "hdr-key-value",
469            "endpoint-key",
470            "Bearer ep-bearer",
471            "entra-secret",
472            "aws-secret",
473            "aws-session",
474            "fallback-key",
475        ] {
476            assert!(secrets.iter().any(|s| s == expected), "missing {expected}");
477        }
478        assert!(
479            secrets
480                .iter()
481                .all(|s| s != "acme" && s != "application/json"),
482            "innocuous header values must not be registered"
483        );
484    }
485
486    #[test]
487    fn test_registry_observe_and_redact() {
488        observe_secret("registry-token-123456");
489        let r = Redactor::new();
490        assert_eq!(
491            r.redact("error body: registry-token-123456"),
492            "error body: [REDACTED]"
493        );
494    }
495
496    #[test]
497    fn test_registry_deduplicates() {
498        observe_secret("registry-dedup-abcdef");
499        observe_secret("registry-dedup-abcdef");
500        let r = Redactor::new();
501        assert_eq!(r.redact("registry-dedup-abcdef"), "[REDACTED]");
502    }
503
504    #[test]
505    fn test_registry_min_len_guard() {
506        observe_secret("tiny");
507        let r = Redactor::new();
508        assert_eq!(r.redact("token tiny here"), "token tiny here");
509    }
510
511    #[test]
512    fn test_registry_cap_drops_oldest() {
513        let r = Redactor::new();
514        for i in 0..OBSERVED_CAP + 10 {
515            observe_secret(&format!("registry-cap-secret-{i:03}"));
516        }
517        // The first 10 entries were evicted...
518        assert_eq!(
519            r.redact("registry-cap-secret-000"),
520            "registry-cap-secret-000"
521        );
522        // ...while the newest entries still redact.
523        assert_eq!(r.redact("registry-cap-secret-521"), "[REDACTED]");
524    }
525
526    #[test]
527    fn test_redact_event_step_finished() {
528        let mut r = Redactor::new();
529        r.add_secret("tok-query-secret", 0);
530        let event = TestEvent::StepFinished {
531            test: "login".into(),
532            index: 0,
533            label: "[navigate] /dashboard?token=tok-query-secret".into(),
534            status: StepStatus::Failed,
535            duration_ms: 100,
536            message: "element not found at /x?token=tok-query-secret".into(),
537            diagnostics: Some("url: /x?token=tok-query-secret".into()),
538            screenshot: Some("artifacts/login.png".into()),
539        };
540        let redacted = r.redact_event(&event);
541        let TestEvent::StepFinished {
542            label,
543            message,
544            diagnostics,
545            screenshot,
546            status,
547            ..
548        } = redacted
549        else {
550            panic!("expected step_finished");
551        };
552        assert_eq!(label, "[navigate] /dashboard?token=[REDACTED]");
553        assert_eq!(message, "element not found at /x?token=[REDACTED]");
554        assert_eq!(diagnostics.as_deref(), Some("url: /x?token=[REDACTED]"));
555        assert_eq!(screenshot.as_deref(), Some("artifacts/login.png"));
556        assert_eq!(status, StepStatus::Failed);
557    }
558
559    #[test]
560    fn test_redact_event_llm_call_error_with_registry_token() {
561        observe_secret("runtime-token-987654");
562        let mut r = Redactor::new();
563        r.add_secret("sk-static-key", 0);
564        let event = TestEvent::LlmCallFinished {
565            test: "login".into(),
566            index: 0,
567            endpoint: "default".into(),
568            model: "deepseek".into(),
569            purpose: "targeting".into(),
570            ok: false,
571            duration_ms: 900,
572            input_tokens: 0,
573            output_tokens: 0,
574            cached_input_tokens: 0,
575            cost: 0.0,
576            error: Some("HTTP 401: Bearer sk-static-key invalid (got runtime-token-987654)".into()),
577        };
578        let redacted = r.redact_event(&event);
579        let TestEvent::LlmCallFinished { error, .. } = redacted else {
580            panic!("expected llm_call_finished");
581        };
582        assert_eq!(
583            error.as_deref(),
584            Some("HTTP 401: Bearer [REDACTED] invalid (got [REDACTED])")
585        );
586    }
587
588    #[test]
589    fn test_redact_event_preserves_non_string_fields() {
590        let r = Redactor::new();
591        let event = TestEvent::TestFinished {
592            test: "t".into(),
593            passed: 1,
594            failed: 0,
595            skipped: 0,
596            duration_ms: 12,
597            cost: 0.0042,
598            tokens: 42,
599            input_tokens: 30,
600            output_tokens: 12,
601            cached_input_tokens: 5,
602            models: vec!["deepseek".into()],
603            calls: 1,
604        };
605        let redacted = r.redact_event(&event);
606        let TestEvent::TestFinished {
607            test,
608            passed,
609            failed,
610            duration_ms,
611            cost,
612            tokens,
613            calls,
614            ..
615        } = redacted
616        else {
617            panic!("expected test_finished");
618        };
619        assert_eq!(test, "t");
620        assert_eq!((passed, failed), (1, 0));
621        assert_eq!(duration_ms, 12);
622        #[allow(clippy::float_cmp)]
623        {
624            assert_eq!(cost, 0.0042);
625        }
626        assert_eq!((tokens, calls), (42, 1));
627    }
628
629    #[test]
630    fn test_min_secret_len_constant_matches_guard() {
631        // Sanity: the default guard applies to `add_secret_values`.
632        let mut r = Redactor::new();
633        r.add_secret_values(["12345".to_owned(), "123456".to_owned()]);
634        assert_eq!(r.secrets.len(), 1);
635        assert_eq!(r.secrets[0], "123456");
636        assert!(!r.is_empty());
637        assert_eq!(MIN_SECRET_LEN, 6);
638    }
639}
llm_browser_testkit/redact.rs

llm_browser_testkit/
redact.rs
