Skip to main content

khive_storage/
secret_scan.rs

1//! Pure credential detection, decoding, masking and safe match evidence.
2
3mod bridge_fragments;
4mod credential_triggers;
5mod entropy;
6mod escaped_json;
7mod false_positive_filters;
8mod known_patterns;
9mod masking;
10mod submitted_atom_digest;
11
12use bridge_fragments::{
13    bridge_fragment_chain, contains_bounded_word, contains_word, is_bridge_fragment_shape,
14    trailing_bridge_fragment_cut,
15};
16use credential_triggers::{
17    assignment_credential_trigger, build_match, extract_token, find_trigger,
18    inline_credential_trigger, is_assignment_label_gap, is_base64_content_hash,
19    is_credential_token_label, is_lookup_key_label, is_pure_hex, is_structured_identifier,
20    is_uuid_canonical, shannon_entropy, strip_delimiters, value_candidates, wrapper_strip_repeated,
21};
22#[cfg(test)]
23use credential_triggers::{LOOKUP_KEY_LABELS, POSITION_TOKEN_LABELS, RECEIPT_TOKEN_LABELS};
24#[cfg(test)]
25use entropy::TRIGGER_WINDOW;
26use entropy::{
27    check_entropy_heuristic, tokenize_entropy_tokens, EntropyScanContext, COMPOUND_TRIGGER_WORDS,
28    ENTROPY_THRESHOLD, HEX_CREDENTIAL_LENGTHS, MAX_BRIDGE_FRAGMENTS, MAX_BRIDGE_GLUE_TOKENS,
29    MIN_BRIDGE_FRAGMENT_LEN, MIN_ENTROPY_LEN, TRIGGER_WORDS,
30};
31use false_positive_filters::{
32    after_last_sentence_boundary, before_first_sentence_boundary,
33    has_clause_credential_label_with_inline, has_direct_repository_credential_label_with_inline,
34    has_immediate_credential_label, is_aws_resource_name, is_environment_name,
35    is_git_revision_reference, is_labeled_sha256_digest, is_latex_fragment_without_credential_run,
36    is_latex_prose_macro, is_plausible_file_path, is_prose_code_reference,
37    is_repository_revision_reference, is_vcs_marker_before_hex, normalized_hex_credential_span,
38    ClauseValueKind,
39};
40#[cfg(test)]
41use false_positive_filters::{is_clause_narrative_gerund, is_clause_narrative_participle};
42use known_patterns::check_known_patterns;
43#[cfg(doc)]
44use known_patterns::find_url_userinfo;
45#[cfg(test)]
46use known_patterns::{find_prefix_token, is_filename_shaped_prefix_match, PREFIX_DETECTORS};
47pub use masking::{
48    bounded_masked_log_text, mask_bounded, mask_secrets, BoundedMask, MASK_WINDOW_CHARS,
49};
50#[cfg(test)]
51use masking::{collect_mask_spans, MAX_LOG_TEXT_OUTPUT_CHARS, TRUNCATION_MARKER};
52use masking::{extend_across_invisible_bridge, MAX_LOG_TEXT_MASK_INPUT_CHARS};
53
54pub use submitted_atom_digest::masked_submitted_atom_digest_v1;
55
56// ─── Public API ──────────────────────────────────────────────────────────────
57
58/// Returned when a write would store credential-looking content.
59///
60/// Carries the detector name and a masked excerpt (`first6...Nchars`).  The
61/// full candidate is never stored in the error.
62#[derive(Debug, Clone, PartialEq, Eq)]
63pub struct SecretMatch {
64    /// Human-readable name of the detector that fired.
65    pub detector: &'static str,
66    /// Canonical trigger from the matched context; known-prefix rules need none.
67    pub trigger: Option<&'static str>,
68    /// `first6...N` — the first 6 chars of the match followed by the total length.
69    pub masked: String,
70    /// Which record and field the match came from. `None` only where the
71    /// caller scanned exactly one string and nothing else, so there is one
72    /// candidate. A single-record verb that scans several fields still needs
73    /// this: the writer sees one refusal and cannot tell whether it was the
74    /// name, the content, a tag or a property that matched.
75    pub location: Option<String>,
76}
77
78impl std::fmt::Display for SecretMatch {
79    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
80        write!(f, "content matches secret pattern {}", self.detector)?;
81        if let Some(trigger) = self.trigger {
82            write!(f, " near '{trigger}'")?;
83        }
84        if let Some(location) = &self.location {
85            write!(f, " in {location}")?;
86        }
87        write!(f, ". {}", block_guidance(self.detector))
88    }
89}
90
91/// Actionable, caller-visible guidance for a hard block, keyed by detector
92/// name. If the content genuinely is a credential, remove it. If it
93/// is not — the common case for the detectors below, which key off SHAPE
94/// near a trigger word rather than a known credential prefix — the fix is to
95/// break up the flagged token so it no longer reads as one contiguous
96/// high-entropy value: separate it from words like key/secret/auth/token with
97/// a sentence or paragraph boundary, or use an explicit repository revision
98/// reference for a source hash.
99fn block_guidance(detector: &'static str) -> &'static str {
100    match detector {
101        "url-userinfo" => {
102            "Placeholders in URL credential positions still match this pattern. \
103             Replace the whole URL with an environment-variable name or config key, \
104             or remove the entire user/password segment before writing."
105        }
106        "high-entropy-token"
107        | "uuid-near-trigger"
108        | "content-hash-near-trigger"
109        | "hex-credential-token" => {
110            "If this is a real credential, remove it before writing. If it is not \
111             (e.g. a file path, UUID, or hash that happens to sit near a word like \
112             key/secret/auth/token), put the candidate in a separate sentence or \
113             paragraph from those words, or express a source hash as an explicit \
114             commit/revision reference."
115        }
116        "nested-escaped-json" => {
117            "This value holds JSON text with escaped strings nested deeper than the \
118             gate decodes. Store the inner document unescaped, or flatten the nesting, \
119             before writing."
120        }
121        _ => {
122            "If this is a real credential, remove it before writing; store secrets \
123              in an env var or secrets manager instead."
124        }
125    }
126}
127/// Hard-block content from being written.
128///
129/// Returns `Err(SecretMatch)` on the first match found, or
130/// `Ok(())` if no secret pattern fires.
131pub fn check(content: &str) -> Result<(), SecretMatch> {
132    check_decoded(content, 0)
133}
134
135/// JSON documents nested inside escaped string values deeper than this are
136/// refused rather than passed unread.
137const MAX_ESCAPED_JSON_DEPTH: usize = 8;
138
139fn check_decoded(content: &str, depth: usize) -> Result<(), SecretMatch> {
140    if let Some(m) = scan(content) {
141        return Err(m);
142    }
143    for decoded in escaped_json::escaped_strings(content) {
144        if depth == MAX_ESCAPED_JSON_DEPTH {
145            return Err(build_match("nested-escaped-json", &decoded));
146        }
147        check_decoded(&decoded, depth + 1)?;
148    }
149    Ok(())
150}
151// ─── Scanner ─────────────────────────────────────────────────────────────────
152
153/// Marker substituted for a detected secret span by [`mask_secrets`].
154const REDACTION_MARKER: &str = "***MASKED***";
155
156/// Maximum cumulative bytes revisited by the per-pass detector sweeps while masking
157/// one input. Entropy tokens are materialized once, but resuming inside a token
158/// rebuilds its member candidates and revisits their context, so that token's
159/// full prefix is also charged. This permits two full-size passes over the 1 MiB
160/// ASCII log-input case; the first pass is always allowed for larger or multibyte
161/// callers. Once exhausted, the remainder is redacted wholesale.
162const MAX_MASK_SCAN_WORK_BYTES: usize = MAX_LOG_TEXT_MASK_INPUT_CHARS * 2;
163
164#[cfg(test)]
165thread_local! {
166    static ENTROPY_TOKENIZATION_COUNT: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
167}
168
169/// Return the LEFTMOST secret in `text` as `(matched_slice, detector)`.
170///
171/// The matched slice borrows from `text`, so the caller can recover its byte
172/// span via pointer arithmetic — this is what lets [`mask_secrets`] redact in
173/// place while [`scan`] only needs the masked excerpt.
174///
175/// "Leftmost" (smallest start offset), NOT first-by-detector-priority, is the
176/// load-bearing contract: [`mask_secrets`] copies the text *before* each match
177/// verbatim, so a non-leftmost match would leak an earlier secret detected by a
178/// lower-priority detector (e.g. an `sk-ant-` key sitting to the left of a
179/// `ghp_` token). Both detector layers are folded through [`keep_leftmost`].
180#[cfg(test)]
181fn scan_match(text: &str) -> Option<(&str, &'static str)> {
182    let context = EntropyScanContext::new(text);
183    scan_from(text, 0, &context)
184}
185
186/// Like [`scan_match`], but only returns secrets whose span starts at or after
187/// `from`, while still evaluating Layer-2 trigger context against the FULL
188/// `text`. [`mask_secrets`] calls this with an advancing `from` so that an
189/// entropy token is detected even when its only trigger word sits to the left of
190/// an already-redacted earlier secret. Layer-1 known patterns are context-free,
191/// so scanning the `&text[from..]` suffix is equivalent; offsets recovered via
192/// pointer arithmetic against the original `text` base stay absolute. The
193/// pre-tokenized entropy view is shared by all passes.
194fn scan_from<'a>(
195    text: &'a str,
196    from: usize,
197    context: &EntropyScanContext<'a>,
198) -> Option<(&'a str, &'static str)> {
199    scan_from_with_trigger(text, from, context).map(|(slice, detector, _)| (slice, detector))
200}
201
202fn scan_from_with_trigger<'a>(
203    text: &'a str,
204    from: usize,
205    context: &EntropyScanContext<'a>,
206) -> Option<(&'a str, &'static str, Option<&'static str>)> {
207    let mut best =
208        check_known_patterns(&text[from..]).map(|(slice, detector)| (slice, detector, None));
209    if let Some(candidate) = check_entropy_heuristic(text, from, context) {
210        if best
211            .as_ref()
212            .is_none_or(|current| candidate.0.as_ptr() < current.0.as_ptr())
213        {
214            best = Some(candidate);
215        }
216    }
217    best
218}
219
220/// Replace `best` with `cand` when `cand` starts earlier in the original text
221/// (`base` is the start address of that text). On a tie the incumbent wins, so
222/// callers offer more-specific detectors first. This is what makes
223/// [`check_known_patterns`] and [`scan_match`] return the leftmost secret span
224/// rather than the first detector that happens to match anywhere.
225fn keep_leftmost<'a>(
226    best: &mut Option<(&'a str, &'static str)>,
227    cand: Option<(&'a str, &'static str)>,
228    base: usize,
229) {
230    if let Some((slice, name)) = cand {
231        let start = slice.as_ptr() as usize - base;
232        let replace = match *best {
233            Some((incumbent, _)) => start < (incumbent.as_ptr() as usize - base),
234            None => true,
235        };
236        if replace {
237            *best = Some((slice, name));
238        }
239    }
240}
241
242/// Return the first `SecretMatch` found in `text`, or `None`.
243pub fn scan(text: &str) -> Option<SecretMatch> {
244    let context = EntropyScanContext::new(text);
245    scan_from_with_trigger(text, 0, &context).map(|(slice, detector, trigger)| {
246        let mut matched = build_match(detector, slice);
247        matched.trigger = trigger;
248        matched
249    })
250}
251
252#[cfg(test)]
253#[path = "secret_scan/escaped_json_tests.rs"]
254mod escaped_json_tests;
255#[cfg(test)]
256#[path = "secret_scan/issue_2655_tests.rs"]
257mod issue_2655_tests;
258#[cfg(test)]
259#[path = "secret_scan/tests.rs"]
260mod tests;
261
262/// A named redact-not-block surface whose contract is intentionally separate
263/// from manifest-backed write admission (ADR-115 Amendment 2).
264#[derive(Debug, Clone, Copy, PartialEq, Eq)]
265pub enum RedactionSurface {
266    /// Git ingestion stores only masked commit, issue, and pull-request text.
267    GitIngest,
268    /// Session mirroring stores only masked `text` and `raw` projections.
269    SessionMirror,
270    /// Workspace mirroring stores masked document descriptions.
271    WorkspaceMirror,
272    /// MCP diagnostics are bounded caller-visible transport data, not records.
273    McpDiagnostic,
274    /// The kg `scan` verb's masked preview: caller-visible, never stored.
275    GateProbe,
276}
277
278/// Whether a redaction surface can consume a secret-gate exemption.
279#[derive(Debug, Clone, Copy, PartialEq, Eq)]
280pub enum RedactionSurfaceMode {
281    /// Always apply the canonical masker; never synthesize a stamp or event.
282    PermanentMaskOnly,
283}
284
285/// Machine-readable contract for a named redaction surface.
286#[derive(Debug, Clone, Copy, PartialEq, Eq)]
287pub struct RedactionSurfaceContract {
288    pub mode: RedactionSurfaceMode,
289    /// Durable target containing the masked result, if this is a write surface.
290    pub final_stored_target: Option<&'static str>,
291    /// Reserved stamp location; absent for permanent mask-only surfaces.
292    pub stamp_property: Option<&'static str>,
293    /// Atomic exemption-success event; absent when admission cannot occur.
294    pub atomic_success_event: Option<&'static str>,
295}
296
297/// Final stored target for [`RedactionSurface::GitIngest`] — see
298/// [`redaction_surface_contract`].
299pub const GIT_INGEST_STORED_TARGET: &str = "final git-ingest entity/note fields";
300
301/// Final stored target for [`RedactionSurface::SessionMirror`] — see
302/// [`redaction_surface_contract`]. Names every column the session mirror
303/// writes a masked provider-export projection into, not just the message
304/// body columns. `cwd`/`git_branch` live only on `sessions`, keyed per
305/// session — `session_messages` carries no such columns of its own (see the
306/// `session_messages` DDL in `khive-pack-session`).
307pub const SESSION_MIRROR_STORED_TARGET: &str =
308    "session_messages.text, session_messages.raw, sessions.cwd, sessions.git_branch, \
309     and sessions.slug";
310
311/// Final stored target for [`RedactionSurface::WorkspaceMirror`] — see
312/// [`redaction_surface_contract`]. The mirror passes masked text into the
313/// document entity's `description`; source URI, type, and checksum are separate.
314pub const WORKSPACE_MIRROR_STORED_TARGET: &str = "workspace mirror document.description";
315
316/// Return the closed contract for a named redact-not-block surface.
317pub const fn redaction_surface_contract(surface: RedactionSurface) -> RedactionSurfaceContract {
318    let final_stored_target = match surface {
319        RedactionSurface::GitIngest => Some(GIT_INGEST_STORED_TARGET),
320        RedactionSurface::SessionMirror => Some(SESSION_MIRROR_STORED_TARGET),
321        RedactionSurface::WorkspaceMirror => Some(WORKSPACE_MIRROR_STORED_TARGET),
322        RedactionSurface::McpDiagnostic | RedactionSurface::GateProbe => None,
323    };
324
325    RedactionSurfaceContract {
326        mode: RedactionSurfaceMode::PermanentMaskOnly,
327        final_stored_target,
328        stamp_property: None,
329        atomic_success_event: None,
330    }
331}
332
333/// Apply the canonical detector to a named permanent mask-only surface.
334///
335/// This wrapper makes the non-admission decision executable at each call site:
336/// it has no manifest input, cannot return an exemption outcome, and cannot
337/// synthesize the runtime-owned `khive:secret_gate` property or success event.
338pub fn mask_for_redaction_surface(
339    surface: RedactionSurface,
340    text: &str,
341) -> std::borrow::Cow<'_, str> {
342    match redaction_surface_contract(surface).mode {
343        RedactionSurfaceMode::PermanentMaskOnly => mask_secrets(text),
344    }
345}
346
347/// Invalid or ambiguous coordinates in contextual detector evidence.
348#[derive(Debug, Clone, Copy, PartialEq, Eq)]
349pub struct OriginMappingError;
350
351impl std::fmt::Display for OriginMappingError {
352    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
353        f.write_str("secret scan origin mapping failed")
354    }
355}
356
357impl std::error::Error for OriginMappingError {}
358
359/// Read-only scalar attribution for one canonical JSON property document.
360pub struct PropertyScanContext<'a> {
361    canonical: &'a str,
362    scalars: Vec<entropy::JsonScalarContext>,
363}
364
365impl<'a> PropertyScanContext<'a> {
366    pub fn new(canonical: &'a str) -> Result<Self, OriginMappingError> {
367        serde_json::from_str::<serde::de::IgnoredAny>(canonical).map_err(|_| OriginMappingError)?;
368        Ok(Self {
369            canonical,
370            scalars: entropy::json_scalar_contexts(canonical),
371        })
372    }
373
374    /// Scan one complete quoted key or value, retaining all of its view refusals.
375    pub fn scan_atom(
376        &self,
377        quoted_span: std::ops::Range<usize>,
378    ) -> Result<Vec<SecretMatch>, OriginMappingError> {
379        let quoted = self
380            .canonical
381            .get(quoted_span.clone())
382            .ok_or(OriginMappingError)?;
383        if !quoted.starts_with('"') || !quoted.ends_with('"') || quoted.len() < 2 {
384            return Err(OriginMappingError);
385        }
386        let original: String = serde_json::from_str(quoted).map_err(|_| OriginMappingError)?;
387        let view = map_decoded_span(
388            self.canonical,
389            None,
390            quoted_span.start + 1..quoted_span.end - 1,
391            &original,
392        )?;
393        let label = self
394            .scalars
395            .binary_search_by_key(&(quoted_span.start + 1), |scalar| scalar.value_start)
396            .ok()
397            .map(|index| self.scalars[index])
398            .filter(|scalar| scalar.end == quoted_span.end - 1)
399            .and_then(|scalar| scalar.label);
400        let mut matches = Vec::new();
401        scan_origin_views(view, label, 0, &mut matches)?;
402        Ok(matches)
403    }
404}
405
406struct OriginView {
407    text: String,
408    origins: Vec<std::ops::Range<usize>>,
409}
410
411fn map_decoded_span(
412    input: &str,
413    parent: Option<&[std::ops::Range<usize>]>,
414    span: std::ops::Range<usize>,
415    expected: &str,
416) -> Result<OriginView, OriginMappingError> {
417    let source = input.get(span.clone()).ok_or(OriginMappingError)?;
418    if parent.is_some_and(|origins| origins.len() != input.len()) {
419        return Err(OriginMappingError);
420    }
421    let mut text = String::new();
422    let mut origins = Vec::new();
423    let mut cursor = 0;
424    while cursor < source.len() {
425        let start = cursor;
426        let first = source[cursor..].chars().next().ok_or(OriginMappingError)?;
427        let ch = if first != '\\' {
428            if first == '"' || first < ' ' {
429                return Err(OriginMappingError);
430            }
431            cursor += first.len_utf8();
432            first
433        } else {
434            let tail = &source[cursor..];
435            let (len, valid) = escaped_json::origin_token(tail);
436            if !valid {
437                return Err(OriginMappingError);
438            }
439            cursor += len;
440            let literal = format!("\"{}\"", &source[start..cursor]);
441            let decoded: String = serde_json::from_str(&literal).map_err(|_| OriginMappingError)?;
442            let mut chars = decoded.chars();
443            let ch = chars.next().ok_or(OriginMappingError)?;
444            if chars.next().is_some() {
445                return Err(OriginMappingError);
446            }
447            ch
448        };
449        let start = span.start + start;
450        let end = span.start + cursor;
451        let origin = if let Some(parent) = parent {
452            let slice = parent.get(start..end).ok_or(OriginMappingError)?;
453            let first = slice.first().ok_or(OriginMappingError)?;
454            let last = slice.last().ok_or(OriginMappingError)?;
455            if first.start >= first.end
456                || last.start >= last.end
457                || slice
458                    .windows(2)
459                    .any(|pair| pair[0].start > pair[1].start || pair[0].end > pair[1].end)
460            {
461                return Err(OriginMappingError);
462            }
463            first.start..last.end
464        } else {
465            start..end
466        };
467        if origins
468            .last()
469            .is_some_and(|prior: &std::ops::Range<usize>| {
470                prior.start > origin.start || prior.end > origin.end
471            })
472        {
473            return Err(OriginMappingError);
474        }
475        text.push(ch);
476        origins.extend(std::iter::repeat_n(origin, ch.len_utf8()));
477    }
478    if text != expected || origins.len() != text.len() {
479        return Err(OriginMappingError);
480    }
481    Ok(OriginView { text, origins })
482}
483
484fn scan_origin_views(
485    view: OriginView,
486    label: Option<(usize, &'static str)>,
487    depth: usize,
488    matches: &mut Vec<SecretMatch>,
489) -> Result<(), OriginMappingError> {
490    if let Some(matched) = scan(&view.text) {
491        matches.push(matched);
492    }
493    if let Some(label) = label {
494        let context = EntropyScanContext::with_owning(&view.text, &view.origins, label);
495        if let Some((value, detector, trigger)) = check_entropy_heuristic(&view.text, 0, &context) {
496            let mut matched = build_match(detector, value);
497            matched.trigger = trigger;
498            matches.push(matched);
499        }
500    }
501    for decoded in escaped_json::escaped_views(&view.text) {
502        let child = map_decoded_span(
503            &view.text,
504            Some(&view.origins),
505            decoded.source,
506            &decoded.text,
507        )?;
508        if depth == MAX_ESCAPED_JSON_DEPTH {
509            matches.push(build_match("nested-escaped-json", &child.text));
510        } else {
511            scan_origin_views(child, label, depth + 1, matches)?;
512        }
513    }
514    Ok(())
515}