1mod bridge_fragments;
4mod credential_triggers;
5mod entropy;
6mod escaped_json;
7mod false_positive_filters;
8mod known_patterns;
9mod masking;
10mod submitted_atom_digest;
11
12use bridge_fragments::{
13 bridge_fragment_chain, contains_bounded_word, contains_word, is_bridge_fragment_shape,
14 trailing_bridge_fragment_cut,
15};
16use credential_triggers::{
17 assignment_credential_trigger, build_match, extract_token, find_trigger,
18 inline_credential_trigger, is_assignment_label_gap, is_base64_content_hash,
19 is_credential_token_label, is_lookup_key_label, is_pure_hex, is_structured_identifier,
20 is_uuid_canonical, shannon_entropy, strip_delimiters, value_candidates, wrapper_strip_repeated,
21};
22#[cfg(test)]
23use credential_triggers::{LOOKUP_KEY_LABELS, POSITION_TOKEN_LABELS, RECEIPT_TOKEN_LABELS};
24#[cfg(test)]
25use entropy::TRIGGER_WINDOW;
26use entropy::{
27 check_entropy_heuristic, tokenize_entropy_tokens, EntropyScanContext, COMPOUND_TRIGGER_WORDS,
28 ENTROPY_THRESHOLD, HEX_CREDENTIAL_LENGTHS, MAX_BRIDGE_FRAGMENTS, MAX_BRIDGE_GLUE_TOKENS,
29 MIN_BRIDGE_FRAGMENT_LEN, MIN_ENTROPY_LEN, TRIGGER_WORDS,
30};
31use false_positive_filters::{
32 after_last_sentence_boundary, before_first_sentence_boundary,
33 has_clause_credential_label_with_inline, has_direct_repository_credential_label_with_inline,
34 has_immediate_credential_label, is_aws_resource_name, is_environment_name,
35 is_git_revision_reference, is_labeled_sha256_digest, is_latex_fragment_without_credential_run,
36 is_latex_prose_macro, is_plausible_file_path, is_prose_code_reference,
37 is_repository_revision_reference, is_vcs_marker_before_hex, normalized_hex_credential_span,
38 ClauseValueKind,
39};
40#[cfg(test)]
41use false_positive_filters::{is_clause_narrative_gerund, is_clause_narrative_participle};
42use known_patterns::check_known_patterns;
43#[cfg(doc)]
44use known_patterns::find_url_userinfo;
45#[cfg(test)]
46use known_patterns::{find_prefix_token, is_filename_shaped_prefix_match, PREFIX_DETECTORS};
47pub use masking::{
48 bounded_masked_log_text, mask_bounded, mask_secrets, BoundedMask, MASK_WINDOW_CHARS,
49};
50#[cfg(test)]
51use masking::{collect_mask_spans, MAX_LOG_TEXT_OUTPUT_CHARS, TRUNCATION_MARKER};
52use masking::{extend_across_invisible_bridge, MAX_LOG_TEXT_MASK_INPUT_CHARS};
53
54pub use submitted_atom_digest::masked_submitted_atom_digest_v1;
55
56#[derive(Debug, Clone, PartialEq, Eq)]
63pub struct SecretMatch {
64 pub detector: &'static str,
66 pub trigger: Option<&'static str>,
68 pub masked: String,
70 pub location: Option<String>,
76}
77
78impl std::fmt::Display for SecretMatch {
79 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
80 write!(f, "content matches secret pattern {}", self.detector)?;
81 if let Some(trigger) = self.trigger {
82 write!(f, " near '{trigger}'")?;
83 }
84 if let Some(location) = &self.location {
85 write!(f, " in {location}")?;
86 }
87 write!(f, ". {}", block_guidance(self.detector))
88 }
89}
90
91fn block_guidance(detector: &'static str) -> &'static str {
100 match detector {
101 "url-userinfo" => {
102 "Placeholders in URL credential positions still match this pattern. \
103 Replace the whole URL with an environment-variable name or config key, \
104 or remove the entire user/password segment before writing."
105 }
106 "high-entropy-token"
107 | "uuid-near-trigger"
108 | "content-hash-near-trigger"
109 | "hex-credential-token" => {
110 "If this is a real credential, remove it before writing. If it is not \
111 (e.g. a file path, UUID, or hash that happens to sit near a word like \
112 key/secret/auth/token), put the candidate in a separate sentence or \
113 paragraph from those words, or express a source hash as an explicit \
114 commit/revision reference."
115 }
116 "nested-escaped-json" => {
117 "This value holds JSON text with escaped strings nested deeper than the \
118 gate decodes. Store the inner document unescaped, or flatten the nesting, \
119 before writing."
120 }
121 _ => {
122 "If this is a real credential, remove it before writing; store secrets \
123 in an env var or secrets manager instead."
124 }
125 }
126}
127pub fn check(content: &str) -> Result<(), SecretMatch> {
132 check_decoded(content, 0)
133}
134
135const MAX_ESCAPED_JSON_DEPTH: usize = 8;
138
139fn check_decoded(content: &str, depth: usize) -> Result<(), SecretMatch> {
140 if let Some(m) = scan(content) {
141 return Err(m);
142 }
143 for decoded in escaped_json::escaped_strings(content) {
144 if depth == MAX_ESCAPED_JSON_DEPTH {
145 return Err(build_match("nested-escaped-json", &decoded));
146 }
147 check_decoded(&decoded, depth + 1)?;
148 }
149 Ok(())
150}
151const REDACTION_MARKER: &str = "***MASKED***";
155
156const MAX_MASK_SCAN_WORK_BYTES: usize = MAX_LOG_TEXT_MASK_INPUT_CHARS * 2;
163
164#[cfg(test)]
165thread_local! {
166 static ENTROPY_TOKENIZATION_COUNT: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
167}
168
169#[cfg(test)]
181fn scan_match(text: &str) -> Option<(&str, &'static str)> {
182 let context = EntropyScanContext::new(text);
183 scan_from(text, 0, &context)
184}
185
186fn scan_from<'a>(
195 text: &'a str,
196 from: usize,
197 context: &EntropyScanContext<'a>,
198) -> Option<(&'a str, &'static str)> {
199 scan_from_with_trigger(text, from, context).map(|(slice, detector, _)| (slice, detector))
200}
201
202fn scan_from_with_trigger<'a>(
203 text: &'a str,
204 from: usize,
205 context: &EntropyScanContext<'a>,
206) -> Option<(&'a str, &'static str, Option<&'static str>)> {
207 let mut best =
208 check_known_patterns(&text[from..]).map(|(slice, detector)| (slice, detector, None));
209 if let Some(candidate) = check_entropy_heuristic(text, from, context) {
210 if best
211 .as_ref()
212 .is_none_or(|current| candidate.0.as_ptr() < current.0.as_ptr())
213 {
214 best = Some(candidate);
215 }
216 }
217 best
218}
219
220fn keep_leftmost<'a>(
226 best: &mut Option<(&'a str, &'static str)>,
227 cand: Option<(&'a str, &'static str)>,
228 base: usize,
229) {
230 if let Some((slice, name)) = cand {
231 let start = slice.as_ptr() as usize - base;
232 let replace = match *best {
233 Some((incumbent, _)) => start < (incumbent.as_ptr() as usize - base),
234 None => true,
235 };
236 if replace {
237 *best = Some((slice, name));
238 }
239 }
240}
241
242pub fn scan(text: &str) -> Option<SecretMatch> {
244 let context = EntropyScanContext::new(text);
245 scan_from_with_trigger(text, 0, &context).map(|(slice, detector, trigger)| {
246 let mut matched = build_match(detector, slice);
247 matched.trigger = trigger;
248 matched
249 })
250}
251
252#[cfg(test)]
253#[path = "secret_scan/escaped_json_tests.rs"]
254mod escaped_json_tests;
255#[cfg(test)]
256#[path = "secret_scan/issue_2655_tests.rs"]
257mod issue_2655_tests;
258#[cfg(test)]
259#[path = "secret_scan/tests.rs"]
260mod tests;
261
262#[derive(Debug, Clone, Copy, PartialEq, Eq)]
265pub enum RedactionSurface {
266 GitIngest,
268 SessionMirror,
270 WorkspaceMirror,
272 McpDiagnostic,
274 GateProbe,
276}
277
278#[derive(Debug, Clone, Copy, PartialEq, Eq)]
280pub enum RedactionSurfaceMode {
281 PermanentMaskOnly,
283}
284
285#[derive(Debug, Clone, Copy, PartialEq, Eq)]
287pub struct RedactionSurfaceContract {
288 pub mode: RedactionSurfaceMode,
289 pub final_stored_target: Option<&'static str>,
291 pub stamp_property: Option<&'static str>,
293 pub atomic_success_event: Option<&'static str>,
295}
296
297pub const GIT_INGEST_STORED_TARGET: &str = "final git-ingest entity/note fields";
300
301pub const SESSION_MIRROR_STORED_TARGET: &str =
308 "session_messages.text, session_messages.raw, sessions.cwd, sessions.git_branch, \
309 and sessions.slug";
310
311pub const WORKSPACE_MIRROR_STORED_TARGET: &str = "workspace mirror document.description";
315
316pub const fn redaction_surface_contract(surface: RedactionSurface) -> RedactionSurfaceContract {
318 let final_stored_target = match surface {
319 RedactionSurface::GitIngest => Some(GIT_INGEST_STORED_TARGET),
320 RedactionSurface::SessionMirror => Some(SESSION_MIRROR_STORED_TARGET),
321 RedactionSurface::WorkspaceMirror => Some(WORKSPACE_MIRROR_STORED_TARGET),
322 RedactionSurface::McpDiagnostic | RedactionSurface::GateProbe => None,
323 };
324
325 RedactionSurfaceContract {
326 mode: RedactionSurfaceMode::PermanentMaskOnly,
327 final_stored_target,
328 stamp_property: None,
329 atomic_success_event: None,
330 }
331}
332
333pub fn mask_for_redaction_surface(
339 surface: RedactionSurface,
340 text: &str,
341) -> std::borrow::Cow<'_, str> {
342 match redaction_surface_contract(surface).mode {
343 RedactionSurfaceMode::PermanentMaskOnly => mask_secrets(text),
344 }
345}
346
347#[derive(Debug, Clone, Copy, PartialEq, Eq)]
349pub struct OriginMappingError;
350
351impl std::fmt::Display for OriginMappingError {
352 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
353 f.write_str("secret scan origin mapping failed")
354 }
355}
356
357impl std::error::Error for OriginMappingError {}
358
359pub struct PropertyScanContext<'a> {
361 canonical: &'a str,
362 scalars: Vec<entropy::JsonScalarContext>,
363}
364
365impl<'a> PropertyScanContext<'a> {
366 pub fn new(canonical: &'a str) -> Result<Self, OriginMappingError> {
367 serde_json::from_str::<serde::de::IgnoredAny>(canonical).map_err(|_| OriginMappingError)?;
368 Ok(Self {
369 canonical,
370 scalars: entropy::json_scalar_contexts(canonical),
371 })
372 }
373
374 pub fn scan_atom(
376 &self,
377 quoted_span: std::ops::Range<usize>,
378 ) -> Result<Vec<SecretMatch>, OriginMappingError> {
379 let quoted = self
380 .canonical
381 .get(quoted_span.clone())
382 .ok_or(OriginMappingError)?;
383 if !quoted.starts_with('"') || !quoted.ends_with('"') || quoted.len() < 2 {
384 return Err(OriginMappingError);
385 }
386 let original: String = serde_json::from_str(quoted).map_err(|_| OriginMappingError)?;
387 let view = map_decoded_span(
388 self.canonical,
389 None,
390 quoted_span.start + 1..quoted_span.end - 1,
391 &original,
392 )?;
393 let label = self
394 .scalars
395 .binary_search_by_key(&(quoted_span.start + 1), |scalar| scalar.value_start)
396 .ok()
397 .map(|index| self.scalars[index])
398 .filter(|scalar| scalar.end == quoted_span.end - 1)
399 .and_then(|scalar| scalar.label);
400 let mut matches = Vec::new();
401 scan_origin_views(view, label, 0, &mut matches)?;
402 Ok(matches)
403 }
404}
405
406struct OriginView {
407 text: String,
408 origins: Vec<std::ops::Range<usize>>,
409}
410
411fn map_decoded_span(
412 input: &str,
413 parent: Option<&[std::ops::Range<usize>]>,
414 span: std::ops::Range<usize>,
415 expected: &str,
416) -> Result<OriginView, OriginMappingError> {
417 let source = input.get(span.clone()).ok_or(OriginMappingError)?;
418 if parent.is_some_and(|origins| origins.len() != input.len()) {
419 return Err(OriginMappingError);
420 }
421 let mut text = String::new();
422 let mut origins = Vec::new();
423 let mut cursor = 0;
424 while cursor < source.len() {
425 let start = cursor;
426 let first = source[cursor..].chars().next().ok_or(OriginMappingError)?;
427 let ch = if first != '\\' {
428 if first == '"' || first < ' ' {
429 return Err(OriginMappingError);
430 }
431 cursor += first.len_utf8();
432 first
433 } else {
434 let tail = &source[cursor..];
435 let (len, valid) = escaped_json::origin_token(tail);
436 if !valid {
437 return Err(OriginMappingError);
438 }
439 cursor += len;
440 let literal = format!("\"{}\"", &source[start..cursor]);
441 let decoded: String = serde_json::from_str(&literal).map_err(|_| OriginMappingError)?;
442 let mut chars = decoded.chars();
443 let ch = chars.next().ok_or(OriginMappingError)?;
444 if chars.next().is_some() {
445 return Err(OriginMappingError);
446 }
447 ch
448 };
449 let start = span.start + start;
450 let end = span.start + cursor;
451 let origin = if let Some(parent) = parent {
452 let slice = parent.get(start..end).ok_or(OriginMappingError)?;
453 let first = slice.first().ok_or(OriginMappingError)?;
454 let last = slice.last().ok_or(OriginMappingError)?;
455 if first.start >= first.end
456 || last.start >= last.end
457 || slice
458 .windows(2)
459 .any(|pair| pair[0].start > pair[1].start || pair[0].end > pair[1].end)
460 {
461 return Err(OriginMappingError);
462 }
463 first.start..last.end
464 } else {
465 start..end
466 };
467 if origins
468 .last()
469 .is_some_and(|prior: &std::ops::Range<usize>| {
470 prior.start > origin.start || prior.end > origin.end
471 })
472 {
473 return Err(OriginMappingError);
474 }
475 text.push(ch);
476 origins.extend(std::iter::repeat_n(origin, ch.len_utf8()));
477 }
478 if text != expected || origins.len() != text.len() {
479 return Err(OriginMappingError);
480 }
481 Ok(OriginView { text, origins })
482}
483
484fn scan_origin_views(
485 view: OriginView,
486 label: Option<(usize, &'static str)>,
487 depth: usize,
488 matches: &mut Vec<SecretMatch>,
489) -> Result<(), OriginMappingError> {
490 if let Some(matched) = scan(&view.text) {
491 matches.push(matched);
492 }
493 if let Some(label) = label {
494 let context = EntropyScanContext::with_owning(&view.text, &view.origins, label);
495 if let Some((value, detector, trigger)) = check_entropy_heuristic(&view.text, 0, &context) {
496 let mut matched = build_match(detector, value);
497 matched.trigger = trigger;
498 matches.push(matched);
499 }
500 }
501 for decoded in escaped_json::escaped_views(&view.text) {
502 let child = map_decoded_span(
503 &view.text,
504 Some(&view.origins),
505 decoded.source,
506 &decoded.text,
507 )?;
508 if depth == MAX_ESCAPED_JSON_DEPTH {
509 matches.push(build_match("nested-escaped-json", &child.text));
510 } else {
511 scan_origin_views(child, label, depth + 1, matches)?;
512 }
513 }
514 Ok(())
515}