llm_browser_testkit/
redact.rs

1//! Secret redaction for every report sink.
2//!
3//! All run output flows through [`crate::reporting::Reporter`]. The
4//! [`Redactor`] held by the reporter replaces known secret values with
5//! `[REDACTED]` before any sink (console, NDJSON, JUnit, GitHub, Perfetto)
6//! sees the text, so a leaked API key or token can never make it into a log
7//! or CI report.
8//!
9//! Secrets come from three places, all funneled into one redactor:
10//!
11//! - **Config-derived** — `collect_secrets_from_scenario_config` gathers
12//!   `llm_api_key`, per-endpoint API keys, Entra client secrets, AWS static
13//!   credentials, and the values of sensitive-named headers, and the runner
14//!   registers them on every [`crate::runner::ScenarioRunner`] it builds.
15//! - **Runtime-obtained** — tokens fetched by
16//!   [`crate::auth`](auth) (token commands, header commands, Entra
17//!   client-credentials and managed identity) are pushed into a
18//!   process-global observed-secret registry via [`observe_secret`], which
19//!   every [`Redactor::redact`] consults. They are registered before the
20//!   `LlmCallFinished` event that describes the call is emitted, so the
21//!   very event that echoes a token is redacted.
22//! - **Explicit extras** — callers add literal values via
23//!   `Reporter::add_redaction_secret` (the CLI's `--redact` /
24//!   `HARNESS_REDACT`), for secrets not present in the config (URL query
25//!   tokens, scenario-embedded test data).
26//!
27//! Redaction is exact-match, case-sensitive substring replacement. Values
28//! shorter than [`MIN_SECRET_LEN`] are skipped when derived from config or
29//! runtime sources so short, common strings (e.g. `"dev"`) do not destroy
30//! log readability — explicit extras always apply.
31//!
32//! Raw `eprintln!` sites that bypass the reporter (the `#[browser_test]`
33//! run-report strings, MCP/A2A server startup banners, the cost report)
34//! are intentionally out of scope: they carry no secret-bearing text today.
35
36use std::collections::{HashMap, HashSet, VecDeque};
37use std::sync::{LazyLock, PoisonError, RwLock};
38
39use crate::events::TestEvent;
40use crate::scenario::ScenarioConfig;
41
42/// Replacement text for every redacted value.
43const REDACTED: &str = "[REDACTED]";
44
45/// Shortest secret worth registering from config-derived or runtime
46/// sources. Explicit extras (`add_secret` with `min_len 0`) always apply.
47const MIN_SECRET_LEN: usize = 6;
48
49/// Maximum number of runtime-observed secrets kept in the global registry,
50/// bounding memory in long-running MCP/A2A server processes. Oldest
51/// entries are dropped first.
52const OBSERVED_CAP: usize = 512;
53
54/// Header names whose values are treated as secrets (compared lowercased).
55/// Innocuous headers (`x-org`, `content-type`) are NOT registered.
56const SENSITIVE_HEADER_NAMES: &[&str] = &[
57    "authorization",
58    "proxy-authorization",
59    "api-key",
60    "api_key",
61    "x-api-key",
62    "x-auth-token",
63    "authentication",
64    "token",
65    "cookie",
66    "x-amz-security-token",
67];
68
69/// Ordered set of runtime-observed secrets: `set` enables O(1) dedupe and
70/// lookup, `order` (insertion order) enables dropping the oldest entry when
71/// the cap is reached. `HashSet` alone has no ordering.
72#[derive(Default)]
73struct ObservedRegistry {
74    set: HashSet<String>,
75    order: VecDeque<String>,
76}
77
78impl ObservedRegistry {
79    /// Inserts a value, dropping the oldest entry at [`OBSERVED_CAP`].
80    fn insert(&mut self, value: String) {
81        if self.set.contains(&value) {
82            return;
83        }
84        if self.order.len() >= OBSERVED_CAP {
85            if let Some(oldest) = self.order.pop_front() {
86                self.set.remove(&oldest);
87            }
88        }
89        self.set.insert(value.clone());
90        self.order.push_back(value);
91    }
92
93    /// Iterates the registered secrets in insertion order.
94    fn iter(&self) -> impl Iterator<Item = &str> {
95        self.order.iter().map(String::as_str)
96    }
97}
98
99/// Process-global registry of runtime-observed secrets, shared by every
100/// reporter in the process (including macro-path default reporters).
101static OBSERVED: LazyLock<RwLock<ObservedRegistry>> =
102    LazyLock::new(|| RwLock::new(ObservedRegistry::default()));
103
104/// Registers a runtime-obtained secret so every subsequent
105/// [`Redactor::redact`] call replaces it: token-command output,
106/// header-command output, and Entra/IMDS access tokens.
107///
108/// Values shorter than [`MIN_SECRET_LEN`] are ignored; duplicates are
109/// deduplicated; the registry is capped at [`OBSERVED_CAP`] entries.
110pub fn observe_secret(value: &str) {
111    if value.len() < MIN_SECRET_LEN {
112        return;
113    }
114    OBSERVED
115        .write()
116        .unwrap_or_else(PoisonError::into_inner)
117        .insert(value.to_owned());
118}
119
120/// Replaces known secret values in a string with `[REDACTED]`.
121#[derive(Default)]
122pub struct Redactor {
123    secrets: Vec<String>,
124}
125
126impl Redactor {
127    /// Creates an empty redactor.
128    #[must_use]
129    pub fn new() -> Self {
130        Self::default()
131    }
132
133    /// Records a secret value when it is at least `min_len` characters
134    /// long (and non-empty). Explicit extras pass `min_len 0` so they
135    /// always apply.
136    pub fn add_secret(&mut self, value: &str, min_len: usize) {
137        if value.is_empty() || value.len() < min_len {
138            return;
139        }
140        if self.secrets.iter().any(|s| s == value) {
141            return;
142        }
143        self.secrets.push(value.to_owned());
144    }
145
146    /// Records config-derived secrets, applying [`MIN_SECRET_LEN`].
147    pub fn add_secret_values(&mut self, values: impl IntoIterator<Item = String>) {
148        for value in values {
149            self.add_secret(&value, MIN_SECRET_LEN);
150        }
151    }
152
153    /// Whether no local secrets are registered (runtime-observed secrets
154    /// may still apply).
155    #[must_use]
156    pub const fn is_empty(&self) -> bool {
157        self.secrets.is_empty()
158    }
159
160    /// Replaces every occurrence of every registered secret (local list
161    /// first, then the process-global observed-secret registry) with
162    /// `[REDACTED]`. Infallible: a broken registry can never fail an emit.
163    #[must_use]
164    pub fn redact(&self, input: &str) -> String {
165        let mut out = input.to_owned();
166        for secret in &self.secrets {
167            out = out.replace(secret, REDACTED);
168        }
169        {
170            let observed = OBSERVED.read().unwrap_or_else(PoisonError::into_inner);
171            for secret in observed.iter() {
172                out = out.replace(secret, REDACTED);
173            }
174        }
175        out
176    }
177
178    /// Returns a copy of the event with every `String` field redacted;
179    /// numeric/enum fields are copied verbatim.
180    #[must_use]
181    #[allow(clippy::too_many_lines)]
182    pub fn redact_event(&self, event: &TestEvent) -> TestEvent {
183        match event {
184            TestEvent::RunStarted { total_tests } => TestEvent::RunStarted {
185                total_tests: *total_tests,
186            },
187            TestEvent::TestStarted { test } => TestEvent::TestStarted {
188                test: self.redact(test),
189            },
190            TestEvent::StepStarted { test, index, label } => TestEvent::StepStarted {
191                test: self.redact(test),
192                index: *index,
193                label: self.redact(label),
194            },
195            TestEvent::StepFinished {
196                test,
197                index,
198                label,
199                status,
200                duration_ms,
201                message,
202                diagnostics,
203                screenshot,
204            } => TestEvent::StepFinished {
205                test: self.redact(test),
206                index: *index,
207                label: self.redact(label),
208                status: *status,
209                duration_ms: *duration_ms,
210                message: self.redact(message),
211                diagnostics: diagnostics.as_deref().map(|s| self.redact(s)),
212                screenshot: screenshot.as_deref().map(|s| self.redact(s)),
213            },
214            TestEvent::LlmCallStarted {
215                test,
216                index,
217                endpoint,
218                model,
219                purpose,
220            } => TestEvent::LlmCallStarted {
221                test: self.redact(test),
222                index: *index,
223                endpoint: self.redact(endpoint),
224                model: self.redact(model),
225                purpose: self.redact(purpose),
226            },
227            TestEvent::LlmCallFinished {
228                test,
229                index,
230                endpoint,
231                model,
232                purpose,
233                ok,
234                duration_ms,
235                input_tokens,
236                output_tokens,
237                cached_input_tokens,
238                cache_creation_input_tokens,
239                cost,
240                error,
241            } => TestEvent::LlmCallFinished {
242                test: self.redact(test),
243                index: *index,
244                endpoint: self.redact(endpoint),
245                model: self.redact(model),
246                purpose: self.redact(purpose),
247                ok: *ok,
248                duration_ms: *duration_ms,
249                input_tokens: *input_tokens,
250                output_tokens: *output_tokens,
251                cached_input_tokens: *cached_input_tokens,
252                cache_creation_input_tokens: *cache_creation_input_tokens,
253                cost: *cost,
254                error: error.as_deref().map(|s| self.redact(s)),
255            },
256            TestEvent::TestFinished {
257                test,
258                passed,
259                failed,
260                skipped,
261                duration_ms,
262                cost,
263                tokens,
264                input_tokens,
265                output_tokens,
266                cached_input_tokens,
267                cache_creation_input_tokens,
268                models,
269                calls,
270            } => TestEvent::TestFinished {
271                test: self.redact(test),
272                passed: *passed,
273                failed: *failed,
274                skipped: *skipped,
275                duration_ms: *duration_ms,
276                cost: *cost,
277                tokens: *tokens,
278                input_tokens: *input_tokens,
279                output_tokens: *output_tokens,
280                cached_input_tokens: *cached_input_tokens,
281                cache_creation_input_tokens: *cache_creation_input_tokens,
282                models: models.iter().map(|m| self.redact(m)).collect(),
283                calls: *calls,
284            },
285            TestEvent::RunFinished {
286                tests_passed,
287                tests_failed,
288                steps_passed,
289                steps_failed,
290                steps_skipped,
291                total_cost,
292                total_tokens,
293                total_input_tokens,
294                total_output_tokens,
295                total_cached_input_tokens,
296                total_cache_creation_input_tokens,
297                models,
298                total_calls,
299            } => TestEvent::RunFinished {
300                tests_passed: *tests_passed,
301                tests_failed: *tests_failed,
302                steps_passed: *steps_passed,
303                steps_failed: *steps_failed,
304                steps_skipped: *steps_skipped,
305                total_cost: *total_cost,
306                total_tokens: *total_tokens,
307                total_input_tokens: *total_input_tokens,
308                total_output_tokens: *total_output_tokens,
309                total_cached_input_tokens: *total_cached_input_tokens,
310                total_cache_creation_input_tokens: *total_cache_creation_input_tokens,
311                models: models.iter().map(|m| self.redact(m)).collect(),
312                total_calls: *total_calls,
313            },
314            TestEvent::Warning { message } => TestEvent::Warning {
315                message: self.redact(message),
316            },
317        }
318    }
319}
320
321/// Gathers the startup-known secrets of a (merged) scenario config:
322/// LLM API keys, endpoint credentials, and sensitive header values.
323///
324/// Returns raw strings; the [`MIN_SECRET_LEN`] guard is applied at insert
325/// time by [`Redactor::add_secret_values`].
326#[must_use]
327pub fn collect_secrets_from_scenario_config(cfg: &ScenarioConfig) -> Vec<String> {
328    let mut secrets: Vec<String> = Vec::new();
329    if let Some(key) = &cfg.llm_api_key {
330        secrets.push(key.clone());
331    }
332    collect_sensitive_header_values(&cfg.llm_headers, &mut secrets);
333    for ep in cfg.endpoints.values() {
334        if let Some(key) = &ep.api_key {
335            secrets.push(key.clone());
336        }
337        if let Some(secret) = &ep.auth.client_secret {
338            secrets.push(secret.clone());
339        }
340        if let Some(key) = &ep.aws.secret_access_key {
341            secrets.push(key.clone());
342        }
343        if let Some(token) = &ep.aws.session_token {
344            secrets.push(token.clone());
345        }
346        collect_sensitive_header_values(&ep.headers, &mut secrets);
347    }
348    secrets
349}
350
351/// Pushes the values of sensitive-named headers into `out` (matched
352/// case-insensitively).
353fn collect_sensitive_header_values(headers: &HashMap<String, String>, out: &mut Vec<String>) {
354    for (name, value) in headers {
355        if SENSITIVE_HEADER_NAMES
356            .iter()
357            .any(|n| name.to_ascii_lowercase() == *n)
358        {
359            out.push(value.clone());
360        }
361    }
362}
363
364#[cfg(test)]
365mod tests {
366    use std::collections::HashMap;
367
368    use super::{
369        collect_secrets_from_scenario_config, observe_secret, Redactor, MIN_SECRET_LEN,
370        OBSERVED_CAP,
371    };
372    use crate::events::{StepStatus, TestEvent};
373    use crate::scenario::ScenarioConfig;
374
375    #[test]
376    fn test_redact_exact_replacement() {
377        let mut r = Redactor::new();
378        r.add_secret("sk-secret-key", 0);
379        assert_eq!(r.redact("Bearer sk-secret-key"), "Bearer [REDACTED]");
380    }
381
382    #[test]
383    fn test_redact_multiple_secrets() {
384        let mut r = Redactor::new();
385        r.add_secret("key-one", 0);
386        r.add_secret("token-two", 0);
387        let out = r.redact("key-one then token-two then key-one again");
388        assert_eq!(out, "[REDACTED] then [REDACTED] then [REDACTED] again");
389    }
390
391    #[test]
392    fn test_redact_secret_mid_string() {
393        let mut r = Redactor::new();
394        r.add_secret("abc123", 0);
395        assert_eq!(
396            r.redact("https://app.example.com/login?token=abc123&next=/x"),
397            "https://app.example.com/login?token=[REDACTED]&next=/x"
398        );
399    }
400
401    #[test]
402    fn test_redact_empty_redactor_unchanged() {
403        let r = Redactor::new();
404        let input = "plain hello world 12345";
405        assert_eq!(r.redact(input), input);
406    }
407
408    #[test]
409    fn test_redact_empty_secret_ignored() {
410        let mut r = Redactor::new();
411        r.add_secret("", 0);
412        assert_eq!(r.redact("a secret-leak here"), "a secret-leak here");
413    }
414
415    #[test]
416    fn test_length_guard_skips_short_config_secrets() {
417        let cfg = ScenarioConfig {
418            llm_api_key: Some("short".into()), // 5 chars < MIN_SECRET_LEN
419            ..ScenarioConfig::default()
420        };
421        let mut r = Redactor::new();
422        r.add_secret_values(collect_secrets_from_scenario_config(&cfg));
423        assert!(r.is_empty());
424        assert_eq!(r.redact("token short here"), "token short here");
425    }
426
427    #[test]
428    fn test_explicit_min_len_zero_always_applies() {
429        let mut r = Redactor::new();
430        r.add_secret("short", 0);
431        assert!(!r.is_empty());
432        assert_eq!(r.redact("token short here"), "token [REDACTED] here");
433    }
434
435    #[test]
436    fn test_collect_secrets_from_config() {
437        let mut headers = HashMap::new();
438        headers.insert("X-API-Key".to_owned(), "hdr-key-value".to_owned());
439        headers.insert("x-org".to_owned(), "acme".to_owned());
440        headers.insert("content-type".to_owned(), "application/json".to_owned());
441        let mut endpoints = HashMap::new();
442        let ep = crate::scenario::EndpointConfig {
443            api_key: Some("endpoint-key".to_owned()),
444            headers: HashMap::from([("Authorization".to_owned(), "Bearer ep-bearer".to_owned())]),
445            auth: crate::scenario::AuthConfig {
446                client_secret: Some("entra-secret".to_owned()),
447                ..crate::scenario::AuthConfig::default()
448            },
449            aws: crate::scenario::AwsConfig {
450                secret_access_key: Some("aws-secret".to_owned()),
451                session_token: Some("aws-session".to_owned()),
452                ..crate::scenario::AwsConfig::default()
453            },
454            fallbacks: vec!["fallback".to_owned()],
455            ..crate::scenario::EndpointConfig::default()
456        };
457        endpoints.insert("primary".to_owned(), ep);
458        endpoints.insert(
459            "fallback".to_owned(),
460            crate::scenario::EndpointConfig {
461                api_key: Some("fallback-key".to_owned()),
462                ..crate::scenario::EndpointConfig::default()
463            },
464        );
465        let cfg = ScenarioConfig {
466            llm_api_key: Some("global-key".to_owned()),
467            llm_headers: headers,
468            endpoints,
469            ..ScenarioConfig::default()
470        };
471        let secrets = collect_secrets_from_scenario_config(&cfg);
472        for expected in [
473            "global-key",
474            "hdr-key-value",
475            "endpoint-key",
476            "Bearer ep-bearer",
477            "entra-secret",
478            "aws-secret",
479            "aws-session",
480            "fallback-key",
481        ] {
482            assert!(secrets.iter().any(|s| s == expected), "missing {expected}");
483        }
484        assert!(
485            secrets
486                .iter()
487                .all(|s| s != "acme" && s != "application/json"),
488            "innocuous header values must not be registered"
489        );
490    }
491
492    #[test]
493    fn test_registry_observe_and_redact() {
494        observe_secret("registry-token-123456");
495        let r = Redactor::new();
496        assert_eq!(
497            r.redact("error body: registry-token-123456"),
498            "error body: [REDACTED]"
499        );
500    }
501
502    #[test]
503    fn test_registry_deduplicates() {
504        observe_secret("registry-dedup-abcdef");
505        observe_secret("registry-dedup-abcdef");
506        let r = Redactor::new();
507        assert_eq!(r.redact("registry-dedup-abcdef"), "[REDACTED]");
508    }
509
510    #[test]
511    fn test_registry_min_len_guard() {
512        observe_secret("tiny");
513        let r = Redactor::new();
514        assert_eq!(r.redact("token tiny here"), "token tiny here");
515    }
516
517    #[test]
518    fn test_registry_cap_drops_oldest() {
519        let r = Redactor::new();
520        for i in 0..OBSERVED_CAP + 10 {
521            observe_secret(&format!("registry-cap-secret-{i:03}"));
522        }
523        // The first 10 entries were evicted...
524        assert_eq!(
525            r.redact("registry-cap-secret-000"),
526            "registry-cap-secret-000"
527        );
528        // ...while the newest entries still redact.
529        assert_eq!(r.redact("registry-cap-secret-521"), "[REDACTED]");
530    }
531
532    #[test]
533    fn test_redact_event_step_finished() {
534        let mut r = Redactor::new();
535        r.add_secret("tok-query-secret", 0);
536        let event = TestEvent::StepFinished {
537            test: "login".into(),
538            index: 0,
539            label: "[navigate] /dashboard?token=tok-query-secret".into(),
540            status: StepStatus::Failed,
541            duration_ms: 100,
542            message: "element not found at /x?token=tok-query-secret".into(),
543            diagnostics: Some("url: /x?token=tok-query-secret".into()),
544            screenshot: Some("artifacts/login.png".into()),
545        };
546        let redacted = r.redact_event(&event);
547        let TestEvent::StepFinished {
548            label,
549            message,
550            diagnostics,
551            screenshot,
552            status,
553            ..
554        } = redacted
555        else {
556            panic!("expected step_finished");
557        };
558        assert_eq!(label, "[navigate] /dashboard?token=[REDACTED]");
559        assert_eq!(message, "element not found at /x?token=[REDACTED]");
560        assert_eq!(diagnostics.as_deref(), Some("url: /x?token=[REDACTED]"));
561        assert_eq!(screenshot.as_deref(), Some("artifacts/login.png"));
562        assert_eq!(status, StepStatus::Failed);
563    }
564
565    #[test]
566    fn test_redact_event_llm_call_error_with_registry_token() {
567        observe_secret("runtime-token-987654");
568        let mut r = Redactor::new();
569        r.add_secret("sk-static-key", 0);
570        let event = TestEvent::LlmCallFinished {
571            test: "login".into(),
572            index: 0,
573            endpoint: "default".into(),
574            model: "deepseek".into(),
575            purpose: "targeting".into(),
576            ok: false,
577            duration_ms: 900,
578            input_tokens: 0,
579            output_tokens: 0,
580            cached_input_tokens: 0,
581            cache_creation_input_tokens: 0,
582            cost: 0.0,
583            error: Some("HTTP 401: Bearer sk-static-key invalid (got runtime-token-987654)".into()),
584        };
585        let redacted = r.redact_event(&event);
586        let TestEvent::LlmCallFinished { error, .. } = redacted else {
587            panic!("expected llm_call_finished");
588        };
589        assert_eq!(
590            error.as_deref(),
591            Some("HTTP 401: Bearer [REDACTED] invalid (got [REDACTED])")
592        );
593    }
594
595    #[test]
596    fn test_redact_event_preserves_non_string_fields() {
597        let r = Redactor::new();
598        let event = TestEvent::TestFinished {
599            test: "t".into(),
600            passed: 1,
601            failed: 0,
602            skipped: 0,
603            duration_ms: 12,
604            cost: 0.0042,
605            tokens: 42,
606            input_tokens: 30,
607            output_tokens: 12,
608            cached_input_tokens: 5,
609            cache_creation_input_tokens: 3,
610            models: vec!["deepseek".into()],
611            calls: 1,
612        };
613        let redacted = r.redact_event(&event);
614        let TestEvent::TestFinished {
615            test,
616            passed,
617            failed,
618            duration_ms,
619            cost,
620            tokens,
621            calls,
622            ..
623        } = redacted
624        else {
625            panic!("expected test_finished");
626        };
627        assert_eq!(test, "t");
628        assert_eq!((passed, failed), (1, 0));
629        assert_eq!(duration_ms, 12);
630        #[allow(clippy::float_cmp)]
631        {
632            assert_eq!(cost, 0.0042);
633        }
634        assert_eq!((tokens, calls), (42, 1));
635    }
636
637    #[test]
638    fn test_min_secret_len_constant_matches_guard() {
639        // Sanity: the default guard applies to `add_secret_values`.
640        let mut r = Redactor::new();
641        r.add_secret_values(["12345".to_owned(), "123456".to_owned()]);
642        assert_eq!(r.secrets.len(), 1);
643        assert_eq!(r.secrets[0], "123456");
644        assert!(!r.is_empty());
645        assert_eq!(MIN_SECRET_LEN, 6);
646    }
647}
llm_browser_testkit/redact.rs

llm_browser_testkit/
redact.rs
