khive_runtime/secret_gate.rs
1//! Write-time secret detection gate.
2//!
3//! Scans caller-supplied content strings before any storage write. A match
4//! causes a hard `RuntimeError::SecretDetected` that names the detector and
5//! carries a masked excerpt — it never echoes the full candidate back.
6//!
7//! Scope: **credentials only** — API keys, tokens, private keys, passwords,
8//! and connection strings with embedded credentials. General PII (emails,
9//! phone numbers, company names) is intentionally NOT blocked.
10//!
11//! Detection is layered, cheap-first:
12//! 1. **Known-prefix / known-shape patterns** — AWS AKIA/ASIA, GitHub tokens,
13//! OpenAI `sk-proj-`, Anthropic `sk-ant-`, Stripe live keys, Fly.io tokens,
14//! Vercel secrets, Slack `xox*`, JWT triples, PEM private-key headers, Age
15//! secret keys, URL userinfo (`scheme://user:pass@`).
16//! 2. **High-entropy token heuristic** — base64/hex/base64url runs ≥ 24 chars
17//! near a trigger word (key, secret, password, credential, bearer, auth,
18//! apikey, api_key, access_key, private_key). The word `token` alone is
19//! NOT a trigger, to avoid blocking `tokenizer_*`, `token_count`, etc.
20//!
21//! Credential-shaped labels and assignments dominate the allowlist below.
22//! Public VCS revisions and plausible file paths remain exempt in ordinary
23//! technical prose, while path segments are still scanned independently.
24//!
25//! Full exemption rules (hex/UUID/SRI-hash passes, non-ASCII token
26//! delimiting, structured-identifier decomposition, trigger word-boundary
27//! matching, the underscore-boundary asymmetry between bare trigger words and
28//! the word `token`, and the adversarial-corpus rationale for why some
29//! false positives are accepted) are documented in full in
30//! `docs/api/secret_gate.md#module-level-detection-algorithm` — read that before
31//! changing any detection or exemption logic in this file.
32//!
33//! The caller-visible block message (`SecretMatch`'s `Display` impl) also
34//! carries actionable guidance (`block_guidance`) to split or reword the
35//! flagged token.
36//!
37//! A production-corpus replay harness (`corpus_replay`, `#[ignore]`d, run via
38//! `KHIVE_REPLAY_DB=<path> cargo test ... -- --ignored --nocapture`) measures
39//! the detector's block rate against real note/entity content; see the
40//! harness's own output for current numbers rather than a point-in-time count
41//! here, which would drift as the corpus changes. A checked-in, sanitized
42//! snapshot of that replay (per-detector block counts and sha256 digests of
43//! blocked content, never the content itself) lives at
44//! `tests/data/secret_gate_corpus_manifest.md`, generated by
45//! `corpus_replay::generate_corpus_manifest`.
46
47use crate::error::{RuntimeError, RuntimeResult};
48
49// ─── Public API ──────────────────────────────────────────────────────────────
50
51/// Returned when a write would store credential-looking content.
52///
53/// Carries the detector name and a masked excerpt (`first6...Nchars`). The
54/// full candidate is never stored in the error.
55#[derive(Debug, Clone, PartialEq, Eq)]
56pub struct SecretMatch {
57 /// Human-readable name of the detector that fired.
58 pub detector: &'static str,
59 /// `first6...N` — the first 6 chars of the match followed by the total length.
60 pub masked: String,
61}
62
63impl std::fmt::Display for SecretMatch {
64 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
65 write!(
66 f,
67 "content matches secret pattern {} at masked excerpt {}. {}",
68 self.detector,
69 self.masked,
70 block_guidance(self.detector)
71 )
72 }
73}
74
75/// Actionable, caller-visible guidance for a hard block, keyed by detector
76/// name. If the content genuinely is a credential, remove it. If it
77/// is not — the common case for the detectors below, which key off SHAPE
78/// near a trigger word rather than a known credential prefix — the fix is to
79/// break up the flagged token so it no longer reads as one contiguous
80/// high-entropy value: put it on its own line away from words like
81/// key/secret/auth/token, or insert a space/punctuation inside a long path or
82/// identifier (e.g. `workspaces / 20260705 / topic` instead of one glued
83/// token).
84fn block_guidance(detector: &'static str) -> &'static str {
85 match detector {
86 "high-entropy-token"
87 | "uuid-near-trigger"
88 | "content-hash-near-trigger"
89 | "hex-credential-token" => {
90 "If this is a real credential, remove it before writing. If it is not \
91 (e.g. a file path, UUID, or hash that happens to sit near a word like \
92 key/secret/auth/token), reword so the token is not glued directly next \
93 to that word — e.g. move it to its own line, or insert a space or word \
94 between them."
95 }
96 _ => {
97 "If this is a real credential, remove it before writing; store secrets \
98 in an env var or secrets manager instead."
99 }
100 }
101}
102
103/// Hard-block content from being written.
104///
105/// Returns `Err(RuntimeError::SecretDetected)` on the first match found, or
106/// `Ok(())` if no secret pattern fires.
107pub fn check(content: &str) -> RuntimeResult<()> {
108 if let Some(m) = scan(content) {
109 return Err(RuntimeError::SecretDetected(m));
110 }
111 Ok(())
112}
113
114/// Recursively scan a JSON value for credential-shaped strings.
115///
116/// Walks every string leaf (object values, array elements, nested objects).
117/// Returns `Err(RuntimeError::SecretDetected)` on the first match found.
118/// `None` / null / numeric / boolean JSON values are skipped.
119pub fn check_json(value: &serde_json::Value) -> RuntimeResult<()> {
120 scan_json_value(value)
121}
122
123/// Scan a string-tagged slice (entity/note tags).
124///
125/// Each tag string is scanned individually.
126pub fn check_tags(tags: &[String]) -> RuntimeResult<()> {
127 for tag in tags {
128 check(tag)?;
129 }
130 Ok(())
131}
132
133fn scan_json_value(value: &serde_json::Value) -> RuntimeResult<()> {
134 match value {
135 serde_json::Value::String(s) => check(s),
136 serde_json::Value::Array(arr) => {
137 for v in arr {
138 scan_json_value(v)?;
139 }
140 Ok(())
141 }
142 serde_json::Value::Object(map) => {
143 for (k, v) in map {
144 // Scan both the key (a credential can appear as a JSON key name)
145 // and the value recursively.
146 check(k)?;
147 scan_json_value(v)?;
148 }
149 Ok(())
150 }
151 _ => Ok(()),
152 }
153}
154
155// ─── Scanner ─────────────────────────────────────────────────────────────────
156
157/// Marker substituted for a detected secret span by [`mask_secrets`].
158const REDACTION_MARKER: &str = "***MASKED***";
159
160/// Return the LEFTMOST secret in `text` as `(matched_slice, detector)`.
161///
162/// The matched slice borrows from `text`, so the caller can recover its byte
163/// span via pointer arithmetic — this is what lets [`mask_secrets`] redact in
164/// place while [`scan`] only needs the masked excerpt.
165///
166/// "Leftmost" (smallest start offset), NOT first-by-detector-priority, is the
167/// load-bearing contract: [`mask_secrets`] copies the text *before* each match
168/// verbatim, so a non-leftmost match would leak an earlier secret detected by a
169/// lower-priority detector (e.g. an `sk-ant-` key sitting to the left of a
170/// `ghp_` token). Both detector layers are folded through [`keep_leftmost`].
171fn scan_match(text: &str) -> Option<(&str, &'static str)> {
172 scan_from(text, 0)
173}
174
175/// Like [`scan_match`], but only returns secrets whose span starts at or after
176/// `from`, while still evaluating Layer-2 trigger context against the FULL
177/// `text`. [`mask_secrets`] calls this with an advancing `from` so that an
178/// entropy token is detected even when its only trigger word sits to the left of
179/// an already-redacted earlier secret. Layer-1 known patterns are context-free,
180/// so scanning the `&text[from..]` suffix is equivalent; offsets recovered via
181/// pointer arithmetic against the original `text` base stay absolute.
182fn scan_from(text: &str, from: usize) -> Option<(&str, &'static str)> {
183 let base = text.as_ptr() as usize;
184 // Layer 1: known prefix / shape patterns. Context-free → suffix scan; the
185 // returned slice still borrows from the same allocation, so its absolute
186 // offset is `slice.as_ptr() - base`.
187 let mut best = check_known_patterns(&text[from..]);
188 // Layer 2: entropy heuristic on long tokens near trigger words. Evaluated
189 // over the full text (so left-of-`from` trigger words count) but only tokens
190 // at offset >= from are returned; kept only if left of the best known match.
191 keep_leftmost(&mut best, check_entropy_heuristic(text, from), base);
192 best
193}
194
195/// Replace `best` with `cand` when `cand` starts earlier in the original text
196/// (`base` is the start address of that text). On a tie the incumbent wins, so
197/// callers offer more-specific detectors first. This is what makes
198/// [`check_known_patterns`] and [`scan_match`] return the leftmost secret span
199/// rather than the first detector that happens to match anywhere.
200fn keep_leftmost<'a>(
201 best: &mut Option<(&'a str, &'static str)>,
202 cand: Option<(&'a str, &'static str)>,
203 base: usize,
204) {
205 if let Some((slice, name)) = cand {
206 let start = slice.as_ptr() as usize - base;
207 let replace = match *best {
208 Some((incumbent, _)) => start < (incumbent.as_ptr() as usize - base),
209 None => true,
210 };
211 if replace {
212 *best = Some((slice, name));
213 }
214 }
215}
216
217/// Return the first `SecretMatch` found in `text`, or `None`.
218fn scan(text: &str) -> Option<SecretMatch> {
219 scan_match(text).map(|(slice, detector)| build_match(detector, slice))
220}
221
222/// Redact every detected secret span in `text`, replacing each with
223/// `***MASKED***`.
224///
225/// This is the masking counterpart to [`check`]: where `check` hard-blocks a
226/// write on the first match, `mask_secrets` is for content that must be STORED
227/// with credentials stripped (the session mirror). It reuses the SAME
228/// canonical detector set as `check`/`scan`, so callers must never maintain a
229/// second, weaker masker.
230///
231/// Returns `Cow::Borrowed` when no secret is present (the common case), avoiding
232/// an allocation. Spans are discovered left to right against the ORIGINAL text,
233/// always evaluating trigger context over the full input — a high-entropy value
234/// whose only trigger word sits to the left of an earlier-redacted secret is
235/// still detected. See `docs/api/secret_gate.md#mask_secrets` for the scan-cursor
236/// mechanics.
237pub fn mask_secrets(text: &str) -> std::borrow::Cow<'_, str> {
238 let base = text.as_ptr() as usize;
239 // Collect every secret span (absolute byte offsets into `text`) before
240 // writing any output, so trigger-context detection always sees the original
241 // string rather than the suffix after the previous redaction.
242 let mut spans: Vec<(usize, usize)> = Vec::new();
243 let mut from = 0;
244 while from < text.len() {
245 match scan_from(text, from) {
246 Some((sub, _detector)) => {
247 let start = sub.as_ptr() as usize - base;
248 // The prefix detectors return whitespace-delimited tokens, so a
249 // credential glued to structural punctuation (JSON quotes/braces,
250 // sentence commas) carries that trailing punctuation into the
251 // match. Trim a conservative trailing set that can never be part
252 // of a credential, so redacting does not consume surrounding JSON
253 // or prose structure. `=` `/` `+` `.` `-` `_` are intentionally
254 // NOT trimmed — they are valid base64/JWT/key characters.
255 let core_len = sub
256 .trim_end_matches(['"', '\'', '`', '}', ']', ')', ',', ';'])
257 .len();
258 let end = start + core_len.max(1);
259 spans.push((start, end));
260 // `scan_from` only returns matches with start >= from, and `end`
261 // is strictly greater than `start`, so `from` strictly advances.
262 from = end;
263 }
264 None => break,
265 }
266 }
267 if spans.is_empty() {
268 return std::borrow::Cow::Borrowed(text);
269 }
270 let mut out = String::with_capacity(text.len());
271 let mut cursor = 0;
272 for (start, end) in spans {
273 // Spans are non-overlapping and ascending (each starts at/after the prior
274 // `end`); `max(cursor)` is a defensive guard, never load-bearing.
275 let start = start.max(cursor);
276 out.push_str(&text[cursor..start]);
277 out.push_str(REDACTION_MARKER);
278 cursor = end.max(cursor);
279 }
280 out.push_str(&text[cursor..]);
281 std::borrow::Cow::Owned(out)
282}
283
284// ─── Layer 1: known patterns ─────────────────────────────────────────────────
285
286/// Each entry: (detector_name, needle, min_total_token_len).
287///
288/// The needle must appear as a word-boundary-adjacent prefix in the token.
289/// `min_total_token_len` is the minimum length the token (needle + remainder)
290/// must have — prevents the prefix alone triggering without a payload.
291const PREFIX_DETECTORS: &[(&str, &str, usize)] = &[
292 // AWS
293 ("aws-access-key-id", "AKIA", 20),
294 ("aws-access-key-id", "ASIA", 20),
295 // GitHub tokens: personal-access (ghp_), OAuth (gho_), GitHub App
296 // user-to-server (ghu_), server-to-server (ghs_), refresh (ghr_), and the
297 // fine-grained PAT (github_pat_). All but github_pat_ share the gh*_ + 36+
298 // base62 shape.
299 ("github-token", "ghp_", 36),
300 ("github-token", "gho_", 36),
301 ("github-token", "ghu_", 36),
302 ("github-token", "ghs_", 36),
303 ("github-token", "ghr_", 36),
304 ("github-token", "github_pat_", 20),
305 // OpenAI
306 ("openai-api-key", "sk-proj-", 40),
307 // NOTE: bare "sk-" also matches Anthropic/Stripe below; put it last so
308 // the more-specific detectors fire first when both would match.
309 // Anthropic
310 ("anthropic-api-key", "sk-ant-", 20),
311 // Stripe live keys
312 ("stripe-secret-key", "sk_live_", 30),
313 ("stripe-restricted-key", "rk_live_", 30),
314 // Fly.io (fm2_ prefix only — FlyV1 handled separately because it embeds a space)
315 ("fly-token", "fm2_", 20),
316 // Vercel
317 ("vercel-token", "vercel_", 20),
318 // Slack
319 ("slack-token", "xoxb-", 40),
320 ("slack-token", "xoxa-", 40),
321 ("slack-token", "xoxp-", 40),
322 ("slack-token", "xoxr-", 40),
323 ("slack-token", "xoxs-", 40),
324 // Age secret key
325 ("age-secret-key", "AGE-SECRET-KEY-", 60),
326];
327
328/// Known safe compound words that start with `sk-` but are not credentials.
329/// E.g. scikit-learn slugs such as `sk-learn`, `sk-image`, `sk-lego`.
330const SK_SAFE_PREFIXES: &[&str] = &["sk-learn", "sk-image", "sk-lego", "sk-base", "sk-misc"];
331
332/// Shape-based patterns checked with custom logic.
333///
334/// Returns the LEFTMOST match across every detector (see [`keep_leftmost`]). The
335/// detectors are still offered in priority order, so two detectors that match at
336/// the SAME offset (e.g. bare `sk-` and the more-specific `sk-ant-`) resolve to
337/// the first-offered one.
338fn check_known_patterns(text: &str) -> Option<(&str, &'static str)> {
339 let base = text.as_ptr() as usize;
340 let mut best: Option<(&str, &'static str)> = None;
341
342 // --- Prefix patterns ---
343 for &(name, needle, min_len) in PREFIX_DETECTORS {
344 keep_leftmost(
345 &mut best,
346 find_prefix_token(text, needle, min_len).map(|m| (m, name)),
347 base,
348 );
349 }
350
351 // --- Bare `sk-` (after all more-specific sk- detectors above) ---
352 // Require length ≥ 30 AND exclude known safe scikit/library compound words.
353 if let Some(token) = find_prefix_token(text, "sk-", 30) {
354 if !SK_SAFE_PREFIXES.iter().any(|safe| token.starts_with(safe)) {
355 keep_leftmost(&mut best, Some((token, "openai-api-key")), base);
356 }
357 }
358
359 // --- Fly.io FlyV1 token: "FlyV1 <base64-payload>" ---
360 // The format embeds a space, so the generic prefix extractor (which stops at
361 // whitespace) cannot measure the combined length. Check for `FlyV1 ` followed
362 // by ≥ 4 non-whitespace characters as the payload.
363 if let Some(pos) = text.find("FlyV1 ") {
364 let at_boundary = pos == 0 || {
365 text[..pos]
366 .chars()
367 .next_back()
368 .is_none_or(|c| !c.is_ascii_alphanumeric())
369 };
370 if at_boundary {
371 let payload_start = pos + 6; // skip "FlyV1 "
372 let payload = extract_token(&text[payload_start..]);
373 if payload.len() >= 4 {
374 let candidate = &text[pos..payload_start + payload.len()];
375 keep_leftmost(&mut best, Some((candidate, "fly-token")), base);
376 }
377 }
378 }
379
380 // --- PEM private key block ---
381 // "-----BEGIN <TYPE> PRIVATE KEY-----"
382 if text.contains("-----BEGIN") && text.contains("PRIVATE KEY-----") {
383 if let Some(pos) = text.find("-----BEGIN") {
384 // Measure only the key block itself (up to END marker or end-of-string),
385 // not the rest of the surrounding text, so build_match reports the
386 // block length rather than the remaining string length.
387 let block_end = text[pos..]
388 .find("-----END")
389 .map(|rel| {
390 text[pos + rel..]
391 .find('\n')
392 .map(|l| pos + rel + l + 1)
393 .unwrap_or(text.len())
394 })
395 .unwrap_or(text.len());
396 let excerpt = &text[pos..block_end];
397 keep_leftmost(&mut best, Some((excerpt, "pem-private-key")), base);
398 }
399 }
400
401 // --- JWT triple: eyJ...eyJ...eyJ (header.payload.signature) ---
402 // A JWT starts with "eyJ" (base64url of `{"`) and has exactly two dots.
403 keep_leftmost(&mut best, find_jwt(text).map(|m| (m, "jwt")), base);
404
405 // --- URL userinfo: scheme://user:pass@host ---
406 keep_leftmost(
407 &mut best,
408 find_url_userinfo(text).map(|m| (m, "url-userinfo")),
409 base,
410 );
411
412 best
413}
414
415/// Locate the first token in `text` that starts with `needle` and has a
416/// total length >= `min_len`. Returns a slice of the full token on match.
417fn find_prefix_token<'a>(text: &'a str, needle: &str, min_len: usize) -> Option<&'a str> {
418 let mut start = 0;
419 while let Some(rel) = text[start..].find(needle) {
420 let abs = start + rel;
421 // Require that the needle starts at a token boundary (start-of-string
422 // or preceded by a non-ASCII-alphanumeric char). The needles are ASCII,
423 // so only an ASCII alphanumeric can be a real continuation of the same
424 // token; CJK/accented text (which Rust counts as `is_alphanumeric`) must
425 // act as a delimiter, else a secret glued to non-Latin prose (`数据AKIA…`)
426 // is missed.
427 let at_boundary = abs == 0 || {
428 let prev = text[..abs].chars().next_back().unwrap_or(' ');
429 !prev.is_ascii_alphanumeric()
430 };
431 if at_boundary {
432 let token = extract_token(&text[abs..]);
433 if token.len() >= min_len && !is_filename_shaped_prefix_match(token, needle) {
434 return Some(token);
435 }
436 }
437 start = abs + needle.len().max(1);
438 }
439 None
440}
441
442/// Known source-file extensions that can terminate an ordinary provider-
443/// prefixed filename. This is deliberately a closed set: an unknown suffix
444/// is not evidence strong enough to suppress a context-free prefix detector.
445const SOURCE_FILE_EXTENSIONS: &[&str] =
446 &[".py", ".rs", ".ts", ".js", ".sh", ".md", ".toml", ".json"];
447
448/// Returns `true` for a lowercase source filename after a known provider
449/// prefix, such as `vercel_deployment_monitor.py`.
450///
451/// A source extension alone is not enough. The payload before it must contain
452/// only lowercase ASCII letters and filename/path punctuation; any uppercase
453/// letter or digit is credential-value evidence and keeps the prefix match
454/// fail-closed. Outer Markdown/prose punctuation is ignored, but never any
455/// payload byte inside the filename itself.
456fn is_filename_shaped_prefix_match(token: &str, needle: &str) -> bool {
457 let token = token.trim_end_matches(|c: char| {
458 matches!(
459 c,
460 '`' | '"' | '\'' | ')' | ']' | '}' | '>' | ',' | ';' | ':' | '!' | '?'
461 )
462 });
463 let token = token.strip_suffix('.').unwrap_or(token);
464 let Some(payload) = token.strip_prefix(needle) else {
465 return false;
466 };
467 let Some(stem) = SOURCE_FILE_EXTENSIONS
468 .iter()
469 .find_map(|extension| payload.strip_suffix(extension))
470 else {
471 return false;
472 };
473
474 stem.bytes().any(|byte| byte.is_ascii_lowercase())
475 && stem
476 .bytes()
477 .any(|byte| matches!(byte, b'_' | b'-' | b'/' | b'.'))
478 && stem
479 .bytes()
480 .all(|byte| byte.is_ascii_lowercase() || matches!(byte, b'_' | b'-' | b'/' | b'.'))
481}
482
483/// Scan for a JWT pattern: at least two "eyJ" segments separated by a `.`
484/// character, with each segment at least 10 chars.
485fn find_jwt(text: &str) -> Option<&str> {
486 let bytes = text.as_bytes();
487 let mut i = 0;
488 while i + 4 < bytes.len() {
489 if bytes[i..].starts_with(b"eyJ") {
490 // Find the end of this JWT (whitespace or string end).
491 let end = bytes[i..]
492 .iter()
493 .position(|&b| b == b' ' || b == b'\n' || b == b'\r' || b == b'\t')
494 .map(|p| i + p)
495 .unwrap_or(bytes.len());
496 let candidate = &text[i..end];
497 // Must have at least 2 dots and 3 eyJ-prefixed segments.
498 let dots = candidate.as_bytes().iter().filter(|&&b| b == b'.').count();
499 if dots >= 2 {
500 let parts: Vec<&str> = candidate.splitn(3, '.').collect();
501 if parts.len() == 3
502 && parts[0].starts_with("eyJ")
503 && parts[1].starts_with("eyJ")
504 && parts[0].len() >= 10
505 && parts[1].len() >= 10
506 {
507 return Some(candidate);
508 }
509 }
510 i = end + 1;
511 } else {
512 i += 1;
513 }
514 }
515 None
516}
517
518/// Detect `scheme://user:pass@host` patterns where the `user:pass` portion
519/// contains actual credentials (both user and pass non-empty).
520fn find_url_userinfo(text: &str) -> Option<&str> {
521 let mut search = text;
522 let mut base = 0usize;
523 while let Some(at_rel) = search.find("://") {
524 let at_abs = base + at_rel;
525 // After `://`, look for `@` before the next `/`, `?`, ` `, or newline.
526 let rest_start = at_abs + 3;
527 let rest = &text[rest_start..];
528 if let Some(at_pos) = rest.find('@') {
529 let userinfo = &rest[..at_pos];
530 // Must contain a colon and both sides non-empty.
531 if let Some(colon) = userinfo.find(':') {
532 let user = &userinfo[..colon];
533 let pass = &userinfo[colon + 1..];
534 if !user.is_empty() && !pass.is_empty() && pass.len() >= 4 {
535 // Return a slice starting from the scheme. Walk back from
536 // `at_abs` to the first non-scheme char and resume just past
537 // it. Use `char_indices` and skip by the separator's full
538 // UTF-8 width: a multibyte separator (e.g. CJK prose before a
539 // credential URL) would otherwise leave `scheme_start` inside
540 // the codepoint and panic the slice below.
541 let scheme_start = text[..at_abs]
542 .char_indices()
543 .rev()
544 .find(|(_, c)| {
545 !c.is_ascii_alphanumeric() && *c != '+' && *c != '-' && *c != '.'
546 })
547 .map(|(idx, c)| idx + c.len_utf8())
548 .unwrap_or(0);
549 // Ensure there are no spaces in userinfo (not a code snippet).
550 if !userinfo.contains(' ') && !userinfo.contains('\n') {
551 let end = rest_start
552 + at_pos
553 + 1
554 + rest[at_pos + 1..]
555 .find([' ', '\n', '\r'])
556 .unwrap_or(rest[at_pos + 1..].len());
557 return Some(&text[scheme_start..end.min(text.len())]);
558 }
559 }
560 }
561 }
562 base = at_abs + 3;
563 search = &text[base..];
564 }
565 None
566}
567
568// ─── Layer 2: entropy heuristic ─────────────────────────────────────────────
569
570/// Trigger words checked as a bounded standalone word (see
571/// [`contains_bounded_word`]). `token` is deliberately excluded — see
572/// `has_standalone_token`/`has_token_assignment` instead.
573/// See `docs/api/secret_gate.md#trigger_words` for the substring-collision
574/// rationale (issues #577 / #632).
575const TRIGGER_WORDS: &[&str] = &[
576 "key",
577 "secret",
578 "password",
579 "passwd",
580 "credential",
581 "bearer",
582 "auth",
583 "apikey",
584];
585
586/// Compound triggers that retain suffix matching inside credential labels.
587/// Their underscore separator disambiguates them from ordinary prose, and
588/// suffixes are common in versioned credential names such as `api_keyv2`.
589const COMPOUND_TRIGGER_WORDS: &[&str] = &["api_key", "access_key", "private_key"];
590
591/// Minimum token length to apply the entropy check.
592const MIN_ENTROPY_LEN: usize = 24;
593
594/// Shannon entropy threshold (bits per character) above which a token is
595/// considered high-entropy. 7.0 corresponds to ~99% utilisation of a
596/// 128-symbol alphabet — typical for random base64/hex.
597const ENTROPY_THRESHOLD: f64 = 4.5;
598
599/// Window around a trigger word in which a high-entropy token must appear.
600const TRIGGER_WINDOW: usize = 120;
601
602/// Credential-shaped exact hex lengths (AWS secret key, SHA-256/git SHA
603/// doubled, SHA-512 hex, etc.) — checked against a whole token, a single
604/// separator-delimited run, and a normalized (separator-stripped)
605/// concatenation of adjacent hex runs/tokens; see
606/// [`contains_normalized_hex_credential`].
607const HEX_CREDENTIAL_LENGTHS: &[usize] = &[32, 40, 64, 128];
608
609/// Largest number of tokenizer fragments (inclusive of the anchor token)
610/// [`bridge_fragment_chain`] will concatenate when checking whether a
611/// trigger-adjacent short token is one piece of a delimiter-split credential.
612/// The bound is on FRAGMENT COUNT, deliberately NOT on gap byte length:
613/// a byte-length gap bound is defeated outright by
614/// repeating the delimiter (e.g. three U+200B in a row instead of one), while
615/// a fragment count cannot be bypassed that way — repeating the delimiter
616/// inside a single gap never creates a new fragment. This still bounds the
617/// reconstruction to a small local neighborhood (never a document-wide scan):
618/// each extension consumes one real fragment plus up to
619/// [`MAX_BRIDGE_GLUE_TOKENS`] delimiter-only glue tokens crossed to reach it,
620/// and the walk stops the moment a gap contains an ASCII alphanumeric character.
621const MAX_BRIDGE_FRAGMENTS: usize = 6;
622
623/// Maximum number of delimiter-only tokens (see [`is_delimiter_only_token`])
624/// the walk may absorb as glue, in ONE direction, while searching for the
625/// next real fragment across them. Glue tokens do
626/// not count against [`MAX_BRIDGE_FRAGMENTS`] — they carry none of the
627/// credential's own characters — but the search across them still needs its
628/// own bound, or a document seeded with a long run of punctuation-only
629/// tokens (`--- --- --- ...`) could turn the walk into a document-wide scan
630/// instead of the small local neighborhood the module doc promises.
631const MAX_BRIDGE_GLUE_TOKENS: usize = 6;
632
633/// Shortest bare token treated as a plausible FRAGMENT of a separator-split
634/// credential (see the `is_bridge_candidate` check in
635/// [`check_entropy_heuristic`]). Below this, common short words (`dead`,
636/// `beef`, `cafe`, or their base64-alphabet equivalents) would let ordinary
637/// prose feed the bridge checks. Applies to hex AND generic alphanumeric
638/// fragments alike — a
639/// base64/base64url-shaped credential half is exactly as plausible a bridge
640/// candidate as a hex half; the entropy/length decision made over the
641/// reconstructed chain, not this floor, is what keeps ordinary short prose
642/// fragments from being flagged.
643const MIN_BRIDGE_FRAGMENT_LEN: usize = 8;
644
645/// Largest index `<= i` that lies on a UTF-8 char boundary of `s`. Stable
646/// replacement for the unstable `str::floor_char_boundary`; used to snap
647/// byte-offset windows that may land inside a multibyte char before slicing.
648fn floor_char_boundary(s: &str, i: usize) -> usize {
649 let mut i = i.min(s.len());
650 while i > 0 && !s.is_char_boundary(i) {
651 i -= 1;
652 }
653 i
654}
655
656/// `from` restricts which tokens may be RETURNED (only those starting at or
657/// after `from`), but the trigger-context window is still computed over the full
658/// `text`. This lets [`mask_secrets`] advance past an earlier redaction without
659/// losing a trigger word that sat to the left of it.
660fn check_entropy_heuristic(text: &str, from: usize) -> Option<(&str, &'static str)> {
661 // Tokenize into maximal ASCII non-whitespace runs, recording each run's byte
662 // offset. Non-ASCII characters are delimiters (alongside ASCII whitespace):
663 // real base64/hex/base64url credentials are ASCII, so splitting on non-ASCII
664 // isolates an ASCII credential glued to CJK text/punctuation/fullwidth
665 // whitespace, while a run of natural-language CJK yields no ASCII run long
666 // enough to trip the length floor below. On pure-ASCII input this is
667 // identical to `split_ascii_whitespace`.
668 let tokens: Vec<(usize, &str)> = text
669 .split(|c: char| c.is_ascii_whitespace() || !c.is_ascii())
670 .filter(|t| !t.is_empty())
671 .map(|t| {
672 let offset = t.as_ptr() as usize - text.as_ptr() as usize;
673 (offset, t)
674 })
675 .collect();
676
677 for (idx, &(tok_offset, raw_token)) in tokens.iter().enumerate() {
678 // Strip common delimiters that wrap the actual value.
679 let token = strip_delimiters(raw_token);
680 // Only RETURN tokens at or after `from` (already-redacted spans lie
681 // before it); the trigger window below still spans the full text.
682 let token_offset = token.as_ptr() as usize - text.as_ptr() as usize;
683 if token_offset < from {
684 continue;
685 }
686 // A token below MIN_ENTROPY_LEN is still let through when it is
687 // itself a plausible FRAGMENT of a separator-split credential (a
688 // bare alphanumeric run of at least MIN_BRIDGE_FRAGMENT_LEN) — see
689 // the normalized-hex-concatenation and fragment-chain-bridge checks
690 // below. Gating on hex-only fragments would let a base64/base64url-shaped
691 // credential half through untouched: neither half of a Unicode-split
692 // base64-like secret is pure hex, so both would be skipped here before
693 // ever reaching the near-trigger checks. Gating on "any alphanumeric run"
694 // instead of
695 // "any run" still keeps ordinary prose out: a token containing
696 // spaces, punctuation, or other separators already split into
697 // smaller pieces by the tokenizer above, and the bridge/entropy
698 // decision applied to the reconstructed chain below is what actually
699 // filters unrelated short fragments — not this length floor.
700 let is_bridge_candidate = is_bridge_fragment_shape(token);
701 if token.len() < MIN_ENTROPY_LEN && !is_bridge_candidate {
702 continue;
703 }
704
705 // `token` is ASCII here (non-ASCII was split out at tokenization), so
706 // `shannon_entropy` over its bytes is a true per-character entropy.
707
708 // Compute the trigger window before any shape-based allowlist decision.
709 // UUID and base64 content-hash exemptions remain trigger-sensitive.
710 // VCS revisions and file paths use narrower syntactic context below.
711 let window_start = floor_char_boundary(text, tok_offset.saturating_sub(TRIGGER_WINDOW));
712 let window_end = floor_char_boundary(text, tok_offset + raw_token.len() + TRIGGER_WINDOW);
713 let window = &text[window_start..window_end];
714 let raw_start = tok_offset - window_start;
715 let raw_end = raw_start + raw_token.len();
716
717 // A high-entropy candidate must not provide its own surrounding
718 // trigger context. This matters for paths whose slugs contain a real
719 // trigger token, such as `ADR-051-cli-auth-and-kg-git-workflow.md`.
720 // Search the prose on either side independently, then preserve inline
721 // credential shapes through a focused assignment/config check.
722 let near_trigger = contains_trigger(&window[..raw_start])
723 || contains_trigger(&window[raw_end..])
724 || has_inline_credential_trigger(raw_token);
725
726 // UUID canonical form and sha-prefixed base64 content hashes (SRI /
727 // npm lockfile integrity) are allowlisted only outside trigger
728 // context. Near a trigger, both shapes fall through to detection
729 // below instead of being silently passed.
730 //
731 // A UUID's own character entropy cannot be relied on to catch it once
732 // it falls through: hex digits cap at log2(16) = 4.0 bits/char, which
733 // never reaches ENTROPY_THRESHOLD (4.5) regardless of token length.
734 // The explicit checks immediately below are what actually block a
735 // UUID-shaped or hash-shaped token in trigger context; letting it run
736 // into the generic entropy computation at the bottom of this loop
737 // would silently readmit it. A corpus replay of ~19k real notes/docs
738 // measured exactly one benign token (an internal task `area_id` UUID
739 // co-occurring with the word "auth" inside `authorized_write`) newly
740 // blocked by this rule — an accepted false positive, not a systemic
741 // regression.
742 //
743 // Both exact-shape checkers require the WHOLE token to match, so a
744 // credential glued to ordinary storage syntax (`api_key=<uuid>`,
745 // `(<uuid>)`, `{"api_key":"<uuid>"}`, a trailing sentence period,
746 // a doubled assignment, or a label itself containing `:`/`=`)
747 // would otherwise never reach them: `strip_delimiters` above only
748 // trims `"'`:=,;` at the token's OUTER ends, not braces/parens, and
749 // not an internal `=`/`:` from an assignment form. `value_candidates`
750 // enumerates every plausible value extraction from those glued forms
751 // specifically for this pair of checks — it does not replace `token`
752 // for any other check in this loop (entropy, hex, structured-
753 // identifier), none of which require an exact shape match. This is a
754 // small bounded iteration over separator positions in one token, not
755 // an allocation-heavy scan.
756 if near_trigger && value_candidates(token).any(is_uuid_canonical) {
757 return Some((token, "uuid-near-trigger"));
758 }
759 if near_trigger && value_candidates(token).any(is_base64_content_hash) {
760 return Some((token, "content-hash-near-trigger"));
761 }
762 if !near_trigger && (is_uuid_canonical(token) || is_base64_content_hash(token)) {
763 continue;
764 }
765
766 // Pure hex tokens (git SHA, checksum digests) are allowlisted when they
767 // are not near a trigger. Trigger-adjacent Git revisions require an
768 // explicit VCS coordinate marker.
769 if !near_trigger && is_pure_hex(token) {
770 continue;
771 }
772
773 // A 40-hex value attached to a VCS coordinate marker (`commit`,
774 // `revision`, `rev`, `sha`) is a public VCS coordinate, not a
775 // credential — but ONLY when the surrounding clause carries no
776 // credential label ("api key value is commit <hex>" is a labeled
777 // credential wearing a marker). The exemption is a flag, not an early
778 // `continue`: it must skip ONLY the hex-credential-shape checks
779 // below, never fragment reconstruction — an exempted anchor that
780 // skipped reconstruction would let a marker-adjacent fragment hide a
781 // split credential (the same anchor-ordering rule the file-path
782 // exemption pins via
783 // `blocks_unicode_split_hex_credential_with_path_shaped_anchor`).
784 // Marker-word form: the token IS a bare `commit`/`revision`/`rev`/
785 // `sha` whose next token is the 40-hex value. A fixed marker word is
786 // not attacker-controlled credential material, so a full `continue`
787 // is safe here — without it the marker would anchor fragment
788 // reconstruction and re-accumulate its own legitimate neighboring
789 // revision. The hex value itself is a separate token and receives
790 // the full check sequence on its own iteration.
791 if is_vcs_marker_before_hex(text, raw_token)
792 && !has_clause_credential_label(
793 text,
794 token_offset,
795 raw_token,
796 ClauseValueKind::VcsReference,
797 )
798 {
799 continue;
800 }
801
802 let vcs_reference_exempt = is_git_revision_reference(text, token_offset, raw_token)
803 && !has_clause_credential_label(
804 text,
805 token_offset,
806 raw_token,
807 ClauseValueKind::VcsReference,
808 );
809
810 // Hex API keys (AWS secret access key, Stripe test keys, random hex
811 // tokens) are pure hex yet are real credentials. The entropy heuristic
812 // cannot catch them — hex alphabet maxes at log2(16) = 4.0 bits/char,
813 // which is always below ENTROPY_THRESHOLD (4.5). A credential-shaped
814 // hex token (32 / 40 / 64 / 128 chars) near a trigger word is always
815 // flagged. Generic "sha" or "hash" prose does not rescue a token;
816 // `is_git_revision_reference` requires a 40-hex value attached to a
817 // VCS coordinate marker.
818 if !vcs_reference_exempt
819 && near_trigger
820 && is_pure_hex(token)
821 && HEX_CREDENTIAL_LENGTHS.contains(&token.len())
822 {
823 return Some((token, "hex-credential-token"));
824 }
825
826 // A genuine credential can be diluted below the WHOLE-TOKEN-AVERAGE
827 // entropy/hex checks above by low-entropy filler segments sharing the
828 // same whitespace token (issue #1044) — e.g. a 40-char hex payload or
829 // a random run as one `/`-delimited path segment among several short
830 // filler segments (`vault/<payload>/rotate.md`,
831 // `a/b/c/d/<payload>/e/f.rs`). Decomposing on every non-alphanumeric
832 // separator — the same run split `is_structured_identifier` uses —
833 // and re-running the hex-credential-length and entropy checks against
834 // each individual run independently of its surrounding filler closes
835 // that gap. Only `MIN_ENTROPY_LEN`+ runs are considered, so short
836 // natural-language path segments (the common case, verified against
837 // the #1040 measurement corpus: none of its real path false positives
838 // contain a single run this long) never trip it. This does NOT touch
839 // the `is_structured_identifier` exemption itself, which stays scoped
840 // to `!near_trigger` per the module doc's soundness argument — a run
841 // this long clearing its own entropy/hex check is evidence independent
842 // of that exemption's word-shape rule.
843 if near_trigger {
844 // `vcs_reference_exempt` also covers the single-token forms these
845 // per-token shape checks would re-flag (`rev:<hex>` is one token
846 // whose 40-hex run and normalized-hex accumulation both match);
847 // it does NOT cover fragment reconstruction below.
848 for run in token.split(|c: char| !c.is_ascii_alphanumeric()) {
849 if run.len() < MIN_ENTROPY_LEN {
850 continue;
851 }
852 if !vcs_reference_exempt
853 && is_pure_hex(run)
854 && HEX_CREDENTIAL_LENGTHS.contains(&run.len())
855 {
856 return Some((token, "hex-credential-token"));
857 }
858 if shannon_entropy(run.as_bytes()) >= ENTROPY_THRESHOLD {
859 return Some((token, "high-entropy-token"));
860 }
861 }
862
863 // #1062: the per-run loop above still misses a credential hex
864 // payload split into MULTIPLE runs each individually below
865 // MIN_ENTROPY_LEN — e.g. `0123456789abcdef0123/456789abcdef01234567`
866 // (two 20-char hex runs joined by `/`): neither run alone reaches
867 // the 24-char floor or a HEX_CREDENTIAL_LENGTHS value, the whole
868 // token is not pure hex (the `/` breaks it), and the whole-token
869 // entropy is capped below ENTROPY_THRESHOLD by the small
870 // hex-plus-separator alphabet. Concatenating consecutive pure-hex
871 // runs (dropping the separators) and re-checking the combined
872 // length against the same HEX_CREDENTIAL_LENGTHS allowlist closes
873 // this gap without widening the allowlist itself. Bounded to the
874 // runs inside this one token, not a document-wide scan.
875 if !vcs_reference_exempt && contains_normalized_hex_credential(token) {
876 return Some((token, "hex-credential-token"));
877 }
878
879 // Unicode/multi-fragment variant of the same bypass (#1062): a
880 // non-ASCII separator (e.g. U+200B) is a TOKENIZER delimiter (see
881 // the tokenizer comment above `tokens`), so it splits the payload
882 // into SEPARATE tokens instead of surviving inside one — the
883 // concatenation check above never sees the halves together.
884 // Bridging only ONE adjacent pair (`idx` with `idx + 1`, or `idx`
885 // with `idx - 1`) and bounding the gap by raw byte length is
886 // insufficient: repeating the delimiter (e.g. three U+200B in a
887 // row) exceeds any small fixed byte-length gap bound, and a
888 // three-token split means no single adjacent pair ever reaches
889 // the full credential length. `bridge_fragment_chain` fixes both:
890 // it walks outward in BOTH directions across a bounded CHAIN of
891 // fragments (MAX_BRIDGE_FRAGMENTS, not a byte-length gap), so
892 // repeating the delimiter cannot buy an attacker anything. A
893 // delimiter-only token sitting between two Unicode gaps (`---`
894 // glue) is itself transparent to the walk — see
895 // [`is_delimiter_only_token`] — so it is absorbed as gap material
896 // rather than treated as a chain-terminating non-fragment token.
897 // Every token found while extending from the anchor must itself
898 // be [`is_bridge_fragment_shape`] (alphanumeric,
899 // MIN_BRIDGE_FRAGMENT_LEN+): this is what stops the walk at a
900 // short trigger/glue word (`key`, `api`, `for`, 3-4 chars) rather
901 // than dragging prose into the reconstruction. The anchor is
902 // included unconditionally, so shape exemptions that can admit a
903 // long non-fragment anchor must run after this reconstruction.
904 //
905 // ACTUAL GUARANTEE (not "every three-or-more-way split is
906 // reconstructed", which this function does NOT provide):
907 // reconstruction covers splits of
908 // up to MAX_BRIDGE_FRAGMENTS real fragments (each individually
909 // meeting MIN_BRIDGE_FRAGMENT_LEN), where a single gap between
910 // two fragments may be any byte length but spans at most
911 // MAX_BRIDGE_GLUE_TOKENS delimiter-only intermediary tokens per
912 // probe direction. A split into MORE than MAX_BRIDGE_FRAGMENTS
913 // real fragments, into fragments individually below
914 // MIN_BRIDGE_FRAGMENT_LEN, or across MORE than
915 // MAX_BRIDGE_GLUE_TOKENS delimiter-only tokens in one gap, is an
916 // accepted residual limitation of the local-neighborhood bound,
917 // not a soundness gap to close here: per ADR-096 / ADR-115, this
918 // gate is accidental-persistence hygiene on a single-principal
919 // same-uid host, not defense against a same-uid adversary
920 // hand-splitting a credential to evade it — that adversary can
921 // write the DB directly. This module does not itself establish
922 // that the host is same-uid; it inherits that from the daemon's
923 // accept-site refusal of any peer uid other than its own
924 // (`khive-runtime/src/daemon.rs`, peer-credential check before the
925 // first frame is read). If that refusal is ever removed or
926 // weakened, the residual limitation documented here stops being
927 // residual. See
928 // `allows_seven_way_hex_split_beyond_fragment_cap_documented_limitation`
929 // and `allows_six_way_sub_floor_hex_split_documented_limitation`.
930 //
931 // The chain is checked two ways. `contains_normalized_hex_credential`
932 // runs over the fragments joined by a plain space — a non-
933 // alphanumeric separator the function already treats as a run
934 // boundary, so it accumulates only genuinely-adjacent hex runs
935 // exactly as it does for a single token's internal `/`-split
936 // runs. Separately, the fragments are
937 // concatenated WITHOUT a separator and the SAME whole-token
938 // entropy decision a genuine single-token high-entropy candidate
939 // must clear is applied to that reconstruction — this catches
940 // the case where a base64/base64url-shaped credential split by the same
941 // tokenizer-delimiter mechanism is not pure hex, so the
942 // hex-length check alone never catches it, but its reconstructed
943 // entropy does. Unrelated short fragments near a trigger (e.g.
944 // two short git SHAs cited in the same sentence) either fail the
945 // per-fragment shape gate or reconstruct to well under
946 // MIN_ENTROPY_LEN, so they stay allowed; see
947 // `blocks_separator_split_three_way_hex_credential_mixed_case`
948 // and `allows_unrelated_short_fragments_cited_near_a_trigger_word`.
949 // A vcs-exempt anchor skips reconstruction FROM ITSELF: the chain
950 // would re-accumulate the anchor's own legitimate 40-hex (any
951 // adjacent prose word is fragment-shaped enough to open a chain)
952 // and re-flag every benign marker-adjacent revision. A genuinely
953 // split credential hiding one fragment behind a marker is still
954 // caught: every OTHER fragment anchors its own chain, walks both
955 // directions, and accumulates the exempted fragment's hex into
956 // the total (see
957 // `blocks_split_hex_credential_with_marker_adjacent_fragment`).
958 if !vcs_reference_exempt && tokens.len() > 1 {
959 let fragments = bridge_fragment_chain(&tokens, text, idx);
960 if fragments.len() > 1 {
961 let hex_probe = fragments.join(" ");
962 if contains_normalized_hex_credential(&hex_probe) {
963 return Some((token, "hex-credential-token"));
964 }
965 let concatenated: String = fragments.concat();
966 if concatenated.len() >= MIN_ENTROPY_LEN
967 && concatenated.bytes().all(|b| b.is_ascii_alphanumeric())
968 && shannon_entropy(concatenated.as_bytes()) >= ENTROPY_THRESHOLD
969 {
970 return Some((token, "high-entropy-token"));
971 }
972 }
973 }
974
975 if is_plausible_file_path(token)
976 && !has_clause_credential_label(
977 text,
978 token_offset,
979 raw_token,
980 ClauseValueKind::FilePath,
981 )
982 {
983 continue;
984 }
985 }
986
987 // Other structured identifiers (branch names, doc slugs, snake_case
988 // identifiers) are exempted from the entropy check — see the module
989 // doc and `is_structured_identifier`. Must come after the
990 // UUID/content-hash and hex-credential-token checks above (neither of
991 // which it weakens) and before the entropy computation, since a
992 // identifier can exceed ENTROPY_THRESHOLD on Shannon entropy alone.
993 // This general exemption applies only outside trigger context; the
994 // narrower file-path exemption above also requires the absence of an
995 // immediate credential label.
996 if !near_trigger && is_structured_identifier(token) {
997 continue;
998 }
999
1000 let entropy = shannon_entropy(token.as_bytes());
1001 if entropy < ENTROPY_THRESHOLD {
1002 continue;
1003 }
1004
1005 // High-entropy token in trigger context — flag it.
1006 if near_trigger {
1007 return Some((token, "high-entropy-token"));
1008 }
1009 }
1010 None
1011}
1012
1013/// `true` when every byte of `run` is an ASCII hex digit. The
1014/// no-minimum-length, no-`0x`-prefix building block for
1015/// [`contains_normalized_hex_credential`]'s intra-token run decomposition.
1016/// Unlike [`is_pure_hex`] this has no 8-char floor of its own — a legitimate
1017/// credential split across separators can leave a shorter individual run
1018/// that still must sum correctly with its neighbors.
1019fn is_hex_run(run: &str) -> bool {
1020 !run.is_empty() && run.bytes().all(|b| b.is_ascii_hexdigit())
1021}
1022
1023const VCS_MARKERS: &[&str] = &["commit", "revision", "rev", "sha"];
1024
1025/// Marker-word form of a VCS reference: `raw_token` is itself a bare marker
1026/// and the next token in `text` is a 40-hex value.
1027fn is_vcs_marker_before_hex(text: &str, raw_token: &str) -> bool {
1028 let token = wrapper_strip_repeated(raw_token);
1029 let marker = strip_delimiters(token);
1030 if !VCS_MARKERS
1031 .iter()
1032 .any(|candidate| marker.eq_ignore_ascii_case(candidate))
1033 {
1034 return false;
1035 }
1036 let raw_offset = raw_token.as_ptr() as usize - text.as_ptr() as usize;
1037 let next = text[raw_offset + raw_token.len()..].trim_start();
1038 let next = wrapper_strip_repeated(extract_token(next));
1039 next.len() == 40 && next.bytes().all(|b| b.is_ascii_hexdigit())
1040}
1041
1042fn is_git_revision_reference(text: &str, token_offset: usize, raw_token: &str) -> bool {
1043 const MARKERS: &[&str] = VCS_MARKERS;
1044
1045 let token = wrapper_strip_repeated(raw_token);
1046 if is_vcs_marker_before_hex(text, raw_token) {
1047 return true;
1048 }
1049
1050 if token.len() == 40 && token.bytes().all(|b| b.is_ascii_hexdigit()) {
1051 let marker = trailing_identifier(&text[..token_offset]);
1052 return MARKERS
1053 .iter()
1054 .any(|candidate| marker.eq_ignore_ascii_case(candidate));
1055 }
1056
1057 let Some((label, value)) = token.rsplit_once(':') else {
1058 return false;
1059 };
1060 let label = wrapper_strip_repeated(label);
1061 let value = wrapper_strip_repeated(value);
1062 MARKERS
1063 .iter()
1064 .any(|candidate| label.eq_ignore_ascii_case(candidate))
1065 && value.len() == 40
1066 && value.bytes().all(|b| b.is_ascii_hexdigit())
1067}
1068
1069fn trailing_identifier(text: &str) -> &str {
1070 let trimmed = text.trim_end_matches(|c: char| !c.is_ascii_alphanumeric() && c != '_');
1071 trimmed
1072 .rsplit(|c: char| !c.is_ascii_alphanumeric() && c != '_')
1073 .next()
1074 .unwrap_or_default()
1075}
1076
1077/// Words the clause walk in [`has_clause_credential_label`] steps over when
1078/// searching backwards for a credential label. Connectors are the words that
1079/// commonly sit between a label and its value in natural assignment prose
1080/// ("api key value is X", "the token was X"); the VCS coordinate markers are
1081/// included so the marker itself cannot shield an earlier label from the
1082/// walk ("api key value is commit <hex>"); prepositions, determiners, and
1083/// possessives are the glue of noun-compound label qualifiers ("api key for
1084/// our production deploy: X") and carry no content of their own.
1085const LABEL_CLAUSE_SKIP_WORDS: &[&str] = &[
1086 "commit",
1087 "revision",
1088 "rev",
1089 "sha",
1090 "is",
1091 "was",
1092 "are",
1093 "were",
1094 "be",
1095 "been",
1096 "being",
1097 "value",
1098 "values",
1099 "the",
1100 "a",
1101 "an",
1102 "this",
1103 "that",
1104 "it",
1105 "its",
1106 "as",
1107 "here",
1108 "now",
1109 "currently",
1110 "equals",
1111 "for",
1112 "of",
1113 "to",
1114 "in",
1115 "on",
1116 "at",
1117 "by",
1118 "with",
1119 "from",
1120 "per",
1121 "and",
1122 "or",
1123 "our",
1124 "my",
1125 "your",
1126 "their",
1127];
1128
1129/// Maximum identifiers the clause walk examines. Bounds the scan cost to one
1130/// short assignment clause. Exhausting the budget is NOT evidence of absence:
1131/// when a value delimiter was crossed, running out of steps fails CLOSED
1132/// (treated as credential-labeled) — a truncated scan cannot prove the clause
1133/// is unlabeled, and any exhaustion-fails-open rule re-admits the labeled
1134/// value bypass one natural word past the budget. Sized so a label separated
1135/// from its value by connectors plus a dotted version qualifier ("api key
1136/// v1.2 value is commit <hex>" — the version costs two identifier steps)
1137/// stays in range without exhaustion.
1138const LABEL_CLAUSE_WALK_LIMIT: usize = 8;
1139
1140/// File-path candidates may walk across at most this many content identifiers
1141/// without a `:`/`=` delimiter. Two reaches the direct-label shapes that need
1142/// protection (`auth scanner found <path>`) while keeping the walk local.
1143const FILE_PATH_NO_DELIMITER_CONTENT_LIMIT: usize = 2;
1144
1145/// The two narrow trigger-context exemptions need different no-delimiter
1146/// behavior. VCS coordinates preserve the strict prose guard that keeps
1147/// `the key changes are in commit <sha>` readable; file paths admit the
1148/// bounded bridge above so a short label cannot disguise a credential value.
1149#[derive(Clone, Copy, Debug, Eq, PartialEq)]
1150enum ClauseValueKind {
1151 VcsReference,
1152 FilePath,
1153}
1154
1155/// Sentence/paragraph boundary inside a clause-walk gap. `;`, `!`, `?`, and
1156/// blank lines always end the clause. `.` ends it only when it is not
1157/// immediately followed by an alphanumeric character: a dot tight between
1158/// identifier fragments ("v1.2") is intra-token punctuation, while a dot at
1159/// the end of the gap abuts the next identifier (gaps end where the adjacent
1160/// identifier begins) and is likewise intra-token.
1161fn gap_has_sentence_boundary(gap: &str) -> bool {
1162 if gap.contains("\n\n") || gap.contains("\r\n\r\n") || gap.contains([';', '!', '?']) {
1163 return true;
1164 }
1165 let bytes = gap.as_bytes();
1166 bytes.iter().enumerate().any(|(i, b)| {
1167 *b == b'.'
1168 && bytes
1169 .get(i + 1)
1170 .is_some_and(|next| !next.is_ascii_alphanumeric())
1171 })
1172}
1173
1174/// Version-shaped identifier fragment ("2", "v1", "12") — the pieces a dotted
1175/// version qualifier like `v1.2` splits into under identifier extraction.
1176/// Treated as connector material so a versioned label ("api key v1.2 value
1177/// is …") stays reachable.
1178fn is_version_fragment(word: &str) -> bool {
1179 let digits = word
1180 .strip_prefix('v')
1181 .or_else(|| word.strip_prefix('V'))
1182 .unwrap_or(word);
1183 !digits.is_empty() && digits.bytes().all(|b| b.is_ascii_digit())
1184}
1185
1186/// Hex-run identifier long enough to be credential material rather than a
1187/// word ("0123456789abcdef01234567"). Treated as connector material by the
1188/// clause walk: a separator-split payload fragment sitting between the
1189/// candidate value and its label is value material, not a label word that
1190/// ends the clause.
1191fn is_hex_fragment_word(word: &str) -> bool {
1192 word.len() >= 12 && word.bytes().all(|b| b.is_ascii_hexdigit())
1193}
1194
1195/// Allocation-free case-insensitive substring search for ASCII identifiers.
1196fn contains_ascii_case_insensitive(haystack: &str, needle: &str) -> bool {
1197 haystack
1198 .as_bytes()
1199 .windows(needle.len())
1200 .any(|window| window.eq_ignore_ascii_case(needle.as_bytes()))
1201}
1202
1203/// A trailing identifier can contain only ASCII alphanumerics and `_`, so
1204/// splitting on `_` exactly preserves the bare-trigger boundary rule used by
1205/// [`contains_bounded_word`] without allocating a lowercase copy.
1206fn clause_label_has_credential_trigger(label: &str) -> bool {
1207 label.eq_ignore_ascii_case("token")
1208 || COMPOUND_TRIGGER_WORDS
1209 .iter()
1210 .any(|trigger| contains_ascii_case_insensitive(label, trigger))
1211 || label.split('_').any(|part| {
1212 TRIGGER_WORDS
1213 .iter()
1214 .any(|trigger| part.eq_ignore_ascii_case(trigger))
1215 })
1216}
1217
1218fn is_label_clause_skip_word(label: &str) -> bool {
1219 LABEL_CLAUSE_SKIP_WORDS
1220 .iter()
1221 .any(|word| label.eq_ignore_ascii_case(word))
1222}
1223
1224/// Words ending in the byte sequence `ed` are only a cheap proxy for a
1225/// regular English participle. Keep known lexical false matches explicit so
1226/// a noun such as `hundred` cannot become evidence that a credential label is
1227/// merely narrative prose. Irregular forms such as `found` deliberately do
1228/// not qualify: they must not shield `api key found <value>`.
1229fn is_clause_narrative_participle(label: &str) -> bool {
1230 const NON_PARTICIPLE_ED_WORDS: &[&str] = &["hundred"];
1231
1232 label.len() >= 5
1233 && label
1234 .get(label.len().saturating_sub(2)..)
1235 .is_some_and(|suffix| suffix.eq_ignore_ascii_case("ed"))
1236 && !NON_PARTICIPLE_ED_WORDS
1237 .iter()
1238 .any(|word| label.eq_ignore_ascii_case(word))
1239}
1240
1241/// Cheap regular-gerund proxy used only after the no-delimiter file-path walk
1242/// has crossed an explicit `in` on the value side (`api_key handling in
1243/// <path>`). Adjacency alone is never narrative evidence: `api key handling
1244/// <value>` must keep walking to the direct credential label.
1245fn is_clause_narrative_gerund(label: &str) -> bool {
1246 label.len() >= 6
1247 && label
1248 .get(label.len().saturating_sub(3)..)
1249 .is_some_and(|suffix| suffix.eq_ignore_ascii_case("ing"))
1250}
1251
1252/// `true` when the candidate token sits in credential-value syntax: an inline
1253/// credential shape on the token itself, or a credential label reachable by
1254/// walking backwards through the current clause. The walk steps over
1255/// [`LABEL_CLAUSE_SKIP_WORDS`], version fragments, and long hex fragments,
1256/// and stops at a sentence/paragraph boundary (see
1257/// [`gap_has_sentence_boundary`]) — a label on the far side of a boundary is
1258/// prose context, not this value's label. Crossing a value delimiter (`:` or
1259/// `=`, including one attached to a VCS marker: "deploy sha: <hex>" is still
1260/// assignment syntax) additionally lets the walk step over content words
1261/// outside those sets, bounded only by [`LABEL_CLAUSE_WALK_LIMIT`], the
1262/// sentence boundary, and the past-participle stop: "label with qualifiers:
1263/// value" names the value regardless of how many qualifier nouns the label
1264/// carries ("api key for production deploy: X", "api key for shared
1265/// encrypted deploy: X"). A per-clause content-word cap was tried here and
1266/// removed — any cap re-admits the labeled-value bypass one natural
1267/// qualifier past the cap. For the same reason, exhausting the walk budget
1268/// after crossing a delimiter fails CLOSED: the clause is assignment-shaped
1269/// and its head was never scanned, so it is treated as credential-labeled.
1270/// A regular past-participle content word ("flagged", "introduced") ends the walk —
1271/// verb-phrase prose narrates an action on the value rather than labeling it
1272/// ("the auth scanner flagged this file: <path>", "one extra token was
1273/// introduced by sha: <hex>"). Coordinating conjunctions are transparent to
1274/// that position test — "shared and encrypted deploy" keeps both participles
1275/// in adjective position. Without a delimiter, VCS references step over only
1276/// the closed connector sets, preserving ordinary prose such as "the key
1277/// changes are in commit <hex>". File paths additionally step over at most
1278/// [`FILE_PATH_NO_DELIMITER_CONTENT_LIMIT`] content words, closing direct
1279/// label shapes such as "auth scanner found <path>" without broadening the
1280/// VCS tier.
1281fn has_clause_credential_label(
1282 text: &str,
1283 token_offset: usize,
1284 raw_token: &str,
1285 value_kind: ClauseValueKind,
1286) -> bool {
1287 if has_inline_credential_trigger(raw_token) {
1288 return true;
1289 }
1290
1291 let mut rest = &text[..token_offset];
1292 let mut crossed_value_delimiter = false;
1293 let mut no_delimiter_content_words = 0usize;
1294 // Whether the previously processed identifier (the one nearer the value)
1295 // was the literal preposition `in`. This starts false deliberately:
1296 // adjacency to the value must not make a gerund narrative evidence.
1297 let mut arrived_through_in_preposition = false;
1298 // Whether the previously processed identifier (the one nearer the value)
1299 // was connector material. Starts true: step 0 is adjacent to the value or
1300 // its delimiter.
1301 let mut arrived_through_connector = true;
1302 for step in 0..LABEL_CLAUSE_WALK_LIMIT {
1303 let label = trailing_identifier(rest);
1304 if label.is_empty() {
1305 return false;
1306 }
1307 let gap = &rest[label.as_ptr() as usize - rest.as_ptr() as usize + label.len()..];
1308 if step > 0 && gap_has_sentence_boundary(gap) {
1309 return false;
1310 }
1311 if gap.contains([':', '=']) {
1312 crossed_value_delimiter = true;
1313 }
1314 if clause_label_has_credential_trigger(label) {
1315 return true;
1316 }
1317 let skippable = is_label_clause_skip_word(label)
1318 || is_version_fragment(label)
1319 || is_hex_fragment_word(label);
1320 if !skippable {
1321 // This barrier applies to delimiter-bearing clauses and to the
1322 // bounded file-path bridge. It intentionally recognizes regular
1323 // `-ed` forms only: `found` is a bridge word, not a narrative
1324 // shield for a direct credential label.
1325 let no_delimiter_file_path_narrative = !crossed_value_delimiter
1326 && value_kind == ClauseValueKind::FilePath
1327 && arrived_through_in_preposition
1328 && is_clause_narrative_gerund(label);
1329 // Delimiter-bearing clauses and VCS coordinates preserve their
1330 // existing direct-participle semantics. A no-delimiter file path
1331 // must first process real value-side context: the synthetic
1332 // initial connector state alone cannot let `api key leaked
1333 // <value>` stop before reaching the trigger.
1334 let regular_participle_narrative = is_clause_narrative_participle(label)
1335 && (crossed_value_delimiter || value_kind != ClauseValueKind::FilePath || step > 0);
1336 if arrived_through_connector
1337 && (regular_participle_narrative || no_delimiter_file_path_narrative)
1338 {
1339 return false;
1340 }
1341 if !crossed_value_delimiter {
1342 if value_kind != ClauseValueKind::FilePath
1343 || no_delimiter_content_words >= FILE_PATH_NO_DELIMITER_CONTENT_LIMIT
1344 {
1345 return false;
1346 }
1347 no_delimiter_content_words += 1;
1348 }
1349 }
1350 // Coordinating conjunctions are transparent to participle-position
1351 // classification: in "shared and encrypted deploy" the coordination
1352 // as a whole is followed by a content noun, so "shared" is still an
1353 // adjective — the conjunction preserves the arrived state instead of
1354 // marking connector position.
1355 if !label.eq_ignore_ascii_case("and") && !label.eq_ignore_ascii_case("or") {
1356 arrived_through_connector = skippable;
1357 }
1358 arrived_through_in_preposition = label.eq_ignore_ascii_case("in");
1359 let start = label.as_ptr() as usize - rest.as_ptr() as usize;
1360 rest = &rest[..start];
1361 }
1362 // Budget exhausted. A clause that crossed a value delimiter and ran out
1363 // of steps without a sentence boundary or verb-position participle is
1364 // assignment-shaped with an unscanned head — fail closed rather than let
1365 // clause length launder a labeled credential into the exemptions.
1366 crossed_value_delimiter
1367}
1368
1369fn is_plausible_file_path(token: &str) -> bool {
1370 let token = token.trim_start_matches(|c: char| {
1371 matches!(
1372 c,
1373 '"' | '\'' | '`' | '(' | ')' | '[' | ']' | '{' | '}' | ',' | ';'
1374 )
1375 });
1376 let token = token.trim_end_matches(|c: char| {
1377 matches!(
1378 c,
1379 '"' | '\'' | '`' | '(' | ')' | '[' | ']' | '{' | '}' | ',' | ';' | '.'
1380 )
1381 });
1382 let path = if let Some(angle_path) = token.strip_prefix('<') {
1383 let Some(close) = angle_path.rfind('>') else {
1384 return false;
1385 };
1386 if !is_line_location_suffix(&angle_path[close + 1..]) {
1387 return false;
1388 }
1389 &angle_path[..close]
1390 } else if let Some((path, suffix)) = token.rsplit_once(':') {
1391 if is_line_location_suffix(suffix) {
1392 path
1393 } else {
1394 token
1395 }
1396 } else {
1397 token
1398 };
1399
1400 if path.contains("://")
1401 || !path.contains('/')
1402 || !path
1403 .bytes()
1404 .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'/' | b'.' | b'_' | b'-' | b'~'))
1405 {
1406 return false;
1407 }
1408
1409 let segment_count = path
1410 .split('/')
1411 .filter(|segment| !segment.is_empty())
1412 .count();
1413 segment_count >= 2 || (path.starts_with('/') && segment_count == 1)
1414}
1415
1416fn is_line_location_suffix(suffix: &str) -> bool {
1417 if suffix.is_empty() {
1418 return true;
1419 }
1420 let suffix = suffix.strip_prefix(':').unwrap_or(suffix);
1421 let suffix = suffix.strip_prefix('~').unwrap_or(suffix);
1422 let mut parts = suffix.split('-');
1423 let Some(start) = parts.next() else {
1424 return false;
1425 };
1426 !start.is_empty()
1427 && start.bytes().all(|b| b.is_ascii_digit())
1428 && parts
1429 .next()
1430 .is_none_or(|end| !end.is_empty() && end.bytes().all(|b| b.is_ascii_digit()))
1431 && parts.next().is_none()
1432}
1433
1434/// `true` when concatenating consecutive pure-hex runs in `token` — splitting
1435/// on every non-alphanumeric character and dropping the separators
1436/// themselves — reaches one of [`HEX_CREDENTIAL_LENGTHS`].
1437///
1438/// Closes the separator-dilution bypass (#1062) where a credential-length
1439/// hex payload is spread across multiple runs each individually below
1440/// [`MIN_ENTROPY_LEN`]: `0123456789abcdef0123/456789abcdef01234567` is two
1441/// 20-char hex runs that never individually reach 24 chars or a
1442/// credential-length boundary, but normalize to one 40-char hex sequence.
1443/// A non-hex, non-empty run resets the running sum — this only bridges
1444/// ADJACENT hex runs, not hex fragments scattered across unrelated filler.
1445fn contains_normalized_hex_credential(token: &str) -> bool {
1446 let mut concatenated_len = 0usize;
1447 for run in token.split(|c: char| !c.is_ascii_alphanumeric()) {
1448 if run.is_empty() {
1449 continue;
1450 }
1451 if is_hex_run(run) {
1452 concatenated_len += run.len();
1453 if HEX_CREDENTIAL_LENGTHS.contains(&concatenated_len) {
1454 return true;
1455 }
1456 } else {
1457 concatenated_len = 0;
1458 }
1459 }
1460 false
1461}
1462
1463/// `true` when the gap `text[gap_start..gap_end]` between two adjacent
1464/// tokenizer tokens holds no ASCII alphanumeric character — the shape a
1465/// tokenizer-delimiting separator (ASCII whitespace, or a non-ASCII
1466/// character such as U+200B; see the tokenizer comment in
1467/// [`check_entropy_heuristic`]) leaves behind when it splits one credential
1468/// payload into two tokens. Deliberately UNBOUNDED on gap byte length
1469/// (#1062: a byte-length bound here is defeated outright by
1470/// repeating the delimiter character) — [`bridge_fragment_chain`] is what
1471/// keeps the overall reconstruction bounded, via [`MAX_BRIDGE_FRAGMENTS`],
1472/// not this check. This still never bridges tokens separated by a genuine
1473/// word or sentence: any real word in the gap contains an ASCII alphanumeric
1474/// character and fails the check immediately.
1475fn adjacent_gap_is_bridgeable(text: &str, gap_start: usize, gap_end: usize) -> bool {
1476 gap_end >= gap_start && !text[gap_start..gap_end].contains(|c: char| c.is_ascii_alphanumeric())
1477}
1478
1479/// `true` when `s` is shaped like a plausible FRAGMENT of a separator-split
1480/// credential: alphanumeric-only and at least [`MIN_BRIDGE_FRAGMENT_LEN`]
1481/// bytes. Shared by the short-token anchor-admission check in
1482/// [`check_entropy_heuristic`] and by [`bridge_fragment_chain`]'s outward
1483/// walk. A long anchor can reach reconstruction without satisfying this
1484/// shape, but every neighboring fragment merged into its chain must satisfy
1485/// it. This stops the walk at a short trigger/glue word (`key`, `api`, `for`)
1486/// sitting immediately beside the real fragments (#1062).
1487fn is_bridge_fragment_shape(s: &str) -> bool {
1488 s.len() >= MIN_BRIDGE_FRAGMENT_LEN && s.bytes().all(|b| b.is_ascii_alphanumeric())
1489}
1490
1491/// `true` when `s` holds no ASCII alphanumeric character at all — the same
1492/// predicate [`adjacent_gap_is_bridgeable`] applies to the byte-range GAP
1493/// between two tokenizer tokens, applied here to a tokenizer TOKEN itself
1494/// (`s` is always non-empty: the tokenizer filters empty tokens). A
1495/// delimiter-only token such as `---` sitting between two Unicode-separator
1496/// gaps (#1062) carries none of a credential's own
1497/// characters — it is exactly as transparent to reconstruction as the
1498/// surrounding whitespace/Unicode gaps are, so [`bridge_fragment_chain`]
1499/// treats it as glue to walk across, not as a chain-terminating non-fragment
1500/// token. A token can never be both this and [`is_bridge_fragment_shape`]:
1501/// the latter requires only alphanumeric bytes, this requires none.
1502fn is_delimiter_only_token(s: &str) -> bool {
1503 !s.bytes().any(|b| b.is_ascii_alphanumeric())
1504}
1505
1506/// Looks outward from `tokens[edge]` in `dir` (`-1` = toward index 0, `+1` =
1507/// toward the end) for the next [`is_bridge_fragment_shape`] token, walking
1508/// transparently across up to [`MAX_BRIDGE_GLUE_TOKENS`] consecutive
1509/// [`is_delimiter_only_token`] glue tokens along the way. Every gap crossed
1510/// — including the ones on either side of a glue token — must be
1511/// [`adjacent_gap_is_bridgeable`]. Returns the found fragment's index, or
1512/// `None` if the walk runs off the end of `tokens`, meets a token that is
1513/// neither a fragment nor glue, meets a non-bridgeable gap, or exhausts the
1514/// glue budget before finding a fragment.
1515fn probe_bridge_fragment(
1516 tokens: &[(usize, &str)],
1517 text: &str,
1518 edge: usize,
1519 dir: isize,
1520) -> Option<usize> {
1521 let mut i = edge;
1522 let mut glue_skipped = 0usize;
1523 loop {
1524 let next_i = i.checked_add_signed(dir)?;
1525 if next_i >= tokens.len() {
1526 return None;
1527 }
1528 let (lo, hi) = if dir < 0 { (next_i, i) } else { (i, next_i) };
1529 let (lo_offset, lo_raw) = tokens[lo];
1530 let (hi_offset, _) = tokens[hi];
1531 let gap_start = lo_offset + lo_raw.len();
1532 if !adjacent_gap_is_bridgeable(text, gap_start, hi_offset) {
1533 return None;
1534 }
1535 let candidate = strip_delimiters(tokens[next_i].1);
1536 if is_bridge_fragment_shape(candidate) {
1537 return Some(next_i);
1538 }
1539 if is_delimiter_only_token(candidate) && glue_skipped < MAX_BRIDGE_GLUE_TOKENS {
1540 glue_skipped += 1;
1541 i = next_i;
1542 continue;
1543 }
1544 return None;
1545 }
1546}
1547
1548/// Reconstructs the bounded chain of tokenizer fragments containing
1549/// `tokens[anchor_idx]`, by walking outward in both directions via
1550/// [`probe_bridge_fragment`] until the chain has reached
1551/// [`MAX_BRIDGE_FRAGMENTS`] real fragments or neither direction can extend
1552/// further. Returns each REAL fragment's [`strip_delimiters`]-ed body, in
1553/// document order, for the caller to recombine — any delimiter-only glue
1554/// tokens absorbed along the way (#1062) are dropped from the
1555/// result entirely, so a caller joining fragments with a space
1556/// ([`contains_normalized_hex_credential`]) or concatenating them directly
1557/// (the generic entropy check) sees only the genuine fragments, exactly as
1558/// if the glue were more gap. Extends both directions every iteration so a
1559/// credential split with fragments on both sides of the anchor (e.g. the
1560/// anchor is the MIDDLE fragment of a three-way split) is fully
1561/// reconstructed, not just one side of it. A length-1 result means no
1562/// extension was possible — callers should skip further work in that case.
1563/// The anchor itself is included without a fragment-shape check; only outward
1564/// extensions are admitted through [`probe_bridge_fragment`].
1565fn bridge_fragment_chain<'a>(
1566 tokens: &[(usize, &'a str)],
1567 text: &str,
1568 anchor_idx: usize,
1569) -> Vec<&'a str> {
1570 let mut start = anchor_idx;
1571 let mut end = anchor_idx;
1572 let mut fragment_count = 1usize;
1573
1574 loop {
1575 let mut extended = false;
1576 if fragment_count < MAX_BRIDGE_FRAGMENTS && start > 0 {
1577 if let Some(new_start) = probe_bridge_fragment(tokens, text, start, -1) {
1578 start = new_start;
1579 fragment_count += 1;
1580 extended = true;
1581 }
1582 }
1583 if fragment_count < MAX_BRIDGE_FRAGMENTS && end + 1 < tokens.len() {
1584 if let Some(new_end) = probe_bridge_fragment(tokens, text, end, 1) {
1585 end = new_end;
1586 fragment_count += 1;
1587 extended = true;
1588 }
1589 }
1590 if !extended {
1591 break;
1592 }
1593 }
1594
1595 tokens[start..=end]
1596 .iter()
1597 .map(|&(_, raw)| strip_delimiters(raw))
1598 .filter(|stripped| !is_delimiter_only_token(stripped))
1599 .collect()
1600}
1601
1602/// Returns `true` when `low_window` contains `needle` as a standalone word —
1603/// bounded on both sides by a character outside the word-char set (or
1604/// start/end of string) — rather than merely as a substring.
1605/// `underscore_is_word_char` selects the boundary rule the caller needs; see
1606/// `docs/api/secret_gate.md#contains_word` for the two deliberately different
1607/// rules and why each caller needs its own.
1608fn contains_word(low_window: &str, needle: &str, underscore_is_word_char: bool) -> bool {
1609 let is_word_char = |c: char| c.is_ascii_alphanumeric() || (underscore_is_word_char && c == '_');
1610 let mut start = 0;
1611 while let Some(rel) = low_window[start..].find(needle) {
1612 let abs = start + rel;
1613 let before_ok = abs == 0
1614 || low_window[..abs]
1615 .chars()
1616 .next_back()
1617 .is_none_or(|c| !is_word_char(c));
1618 let after_end = abs + needle.len();
1619 let after_ok = after_end >= low_window.len()
1620 || low_window[after_end..]
1621 .chars()
1622 .next()
1623 .is_none_or(|c| !is_word_char(c));
1624 if before_ok && after_ok {
1625 return true;
1626 }
1627 start = abs + needle.len().max(1);
1628 }
1629 false
1630}
1631
1632/// Returns `true` when `low_window` contains the bare trigger word `needle`
1633/// as a standalone word, with underscore treated as a BOUNDARY (see
1634/// [`contains_word`]) — so `secret_key=…`/`auth_token=…`/`signing_key=…`
1635/// still match (on the `secret`/`auth`/`key` half), while pure letter-joined
1636/// collisions like `authorized`/`authentication`/`monkey`/`keyword` do not.
1637fn contains_bounded_word(low_window: &str, needle: &str) -> bool {
1638 contains_word(low_window, needle, false)
1639}
1640
1641/// Returns `true` when a compound credential label begins at an identifier
1642/// boundary. The trailing edge is deliberately unbounded so version suffixes
1643/// and larger underscore-composed labels remain protected.
1644fn contains_compound_trigger(low_text: &str) -> bool {
1645 COMPOUND_TRIGGER_WORDS.iter().any(|needle| {
1646 let mut start = 0;
1647 while let Some(rel) = low_text[start..].find(needle) {
1648 let abs = start + rel;
1649 let before_ok = abs == 0
1650 || low_text[..abs]
1651 .chars()
1652 .next_back()
1653 .is_none_or(|c| !c.is_ascii_alphanumeric());
1654 if before_ok {
1655 return true;
1656 }
1657 start = abs + needle.len();
1658 }
1659 false
1660 })
1661}
1662
1663/// Returns `true` when `text` contains a boundary-delimited credential trigger.
1664fn contains_trigger(text: &str) -> bool {
1665 let low = text.to_ascii_lowercase();
1666 TRIGGER_WORDS
1667 .iter()
1668 .any(|tw| contains_bounded_word(&low, tw))
1669 || contains_compound_trigger(&low)
1670 || has_standalone_token(&low)
1671 || has_token_assignment(&low)
1672 || has_assignment_credential_trigger(&low)
1673}
1674
1675/// Detect a credential-bearing assignment label before an `=` or `:`.
1676///
1677/// The separator may be preceded by whitespace or a JSON quote. Compound
1678/// triggers deliberately retain substring matching inside the label so common
1679/// version suffixes such as `api_keyv2` remain protected.
1680fn has_assignment_credential_trigger(low_text: &str) -> bool {
1681 low_text.char_indices().any(|(index, ch)| {
1682 if !matches!(ch, '=' | ':') {
1683 return false;
1684 }
1685 let before =
1686 low_text[..index].trim_end_matches(|c: char| !c.is_ascii_alphanumeric() && c != '_');
1687 let label = before
1688 .rsplit(|c: char| !c.is_ascii_alphanumeric() && c != '_')
1689 .next()
1690 .unwrap_or_default();
1691 COMPOUND_TRIGGER_WORDS
1692 .iter()
1693 .any(|needle| label.contains(needle))
1694 || TRIGGER_WORDS
1695 .iter()
1696 .any(|tw| contains_bounded_word(label, tw))
1697 || label == "token"
1698 })
1699}
1700
1701/// Detect credential labels embedded in the same whitespace token as a value.
1702///
1703/// The surrounding-context scan deliberately excludes the candidate token so
1704/// a trigger word inside a path cannot make that path self-trigger. Credential
1705/// assignments still need to fire when no whitespace separates label and
1706/// value, including JSON-like forms. Underscore-delimited config identifiers
1707/// without an assignment are retained for compatibility with shapes such as
1708/// `session_secret_<value>`.
1709fn has_inline_credential_trigger(raw_token: &str) -> bool {
1710 let low = raw_token.to_ascii_lowercase();
1711
1712 if has_assignment_credential_trigger(&low) {
1713 return true;
1714 }
1715
1716 !low.contains(['/', '-', '.'])
1717 && low.contains('_')
1718 && (COMPOUND_TRIGGER_WORDS
1719 .iter()
1720 .any(|needle| low.contains(needle))
1721 || TRIGGER_WORDS
1722 .iter()
1723 .any(|tw| contains_bounded_word(&low, tw)))
1724}
1725
1726/// Returns `true` when `low_window` contains the word `token` as a standalone
1727/// word, with underscore treated as a WORD CHARACTER / continuation (see
1728/// [`contains_word`]) — but NOT as part of compound identifiers such as
1729/// `tokenizer`, `token_count`, or `next_token`. This underscore-as-
1730/// continuation rule is deliberately different from
1731/// [`contains_bounded_word`]: `token` alone is not a
1732/// credential trigger (it fires on too many benign technical terms), so it
1733/// needs the narrower, underscore-inclusive standalone-word definition,
1734/// whereas the bare `TRIGGER_WORDS` need underscore-joined compounds like
1735/// `secret_key` to still register.
1736fn has_standalone_token(low_window: &str) -> bool {
1737 contains_word(low_window, "token", true)
1738}
1739
1740/// Returns `true` when `low_window` contains the assignment form `token=` or
1741/// `token:` where the `token` identifier has a word boundary BEFORE it.
1742///
1743/// This is boundary-aware so that compound identifiers like `next_token:` or
1744/// `pagination_token=` do NOT trigger — only a standalone `token=`/`token:`
1745/// at the start of a field name does.
1746///
1747/// Examples that return `true`: `token=<value>`, `token: <value>`,
1748/// `"token": "<value>"` (JSON key-value pairs).
1749/// Examples that return `false`: `next_token: <value>`,
1750/// `pagination_token=<value>`, `token_count: <value>`.
1751fn has_token_assignment(low_window: &str) -> bool {
1752 let needle = "token";
1753 let mut start = 0;
1754 while let Some(rel) = low_window[start..].find(needle) {
1755 let abs = start + rel;
1756 // Require a word boundary BEFORE `token`.
1757 let before_ok = abs == 0
1758 || low_window[..abs]
1759 .chars()
1760 .next_back()
1761 .is_none_or(|c| !c.is_ascii_alphanumeric() && c != '_');
1762 let after_end = abs + needle.len();
1763 // Require `=` or `:` immediately after `token` (possibly with surrounding
1764 // whitespace or quotes stripped by the time we see the lowercased window).
1765 let after_char = low_window[after_end..].chars().next();
1766 let after_is_assign = matches!(after_char, Some('=') | Some(':'));
1767 if before_ok && after_is_assign {
1768 return true;
1769 }
1770 start = abs + needle.len().max(1);
1771 }
1772 false
1773}
1774
1775// ─── Allowlist helpers ───────────────────────────────────────────────────────
1776
1777/// Returns `true` for pure-hex tokens (case-insensitive, optional `0x`/`0X` prefix,
1778/// 8–128 chars) — git SHAs, checksum digests, uuid-hex without hyphens.
1779///
1780/// This helper is used with context: pure-hex tokens near credential trigger words
1781/// are NOT allowlisted (see `check_entropy_heuristic`). Only call this function
1782/// when you have already confirmed no trigger context is nearby.
1783fn is_pure_hex(token: &str) -> bool {
1784 let hex_part = token
1785 .strip_prefix("0x")
1786 .or(token.strip_prefix("0X"))
1787 .unwrap_or(token);
1788 hex_part.len() >= 8 && hex_part.len() <= 128 && hex_part.bytes().all(|b| b.is_ascii_hexdigit())
1789}
1790
1791/// Returns `true` for tokens that are unambiguous base64/base64url content
1792/// hashes with an explicit `sha<N>-` prefix (SRI hash, npm lockfile integrity).
1793/// Bare base64 of the same length WITHOUT the prefix is NOT allowlisted — see
1794/// `docs/api/secret_gate.md#is_base64_content_hash` for the full criteria list and
1795/// why the explicit prefix is required.
1796fn is_base64_content_hash(token: &str) -> bool {
1797 // Known vendor prefixes — never allowlist even if they look like base64.
1798 // Includes bare `sk-` to prevent OpenAI-shaped tokens from being allowlisted.
1799 const VENDOR_PREFIXES: &[&str] = &[
1800 "sk-",
1801 "rk_live_",
1802 "fm2_",
1803 "vercel_",
1804 "xoxb-",
1805 "xoxa-",
1806 "xoxp-",
1807 "xoxr-",
1808 "xoxs-",
1809 "ghp_",
1810 "gho_",
1811 "ghu_",
1812 "ghs_",
1813 "ghr_",
1814 "github_pat_",
1815 "AKIA",
1816 "ASIA",
1817 "AGE-SECRET-KEY-",
1818 "FlyV1",
1819 ];
1820 if VENDOR_PREFIXES.iter().any(|p| token.starts_with(p)) {
1821 return false;
1822 }
1823 // Require an explicit SRI `sha[0-9]+-` prefix. Bare base64 at sha-length
1824 // is NOT allowlisted — it is indistinguishable from a real API token.
1825 let body = if let Some(rest) = token.strip_prefix("sha") {
1826 // rest starts with digits followed by '-'
1827 let dash = rest.find('-').unwrap_or(rest.len());
1828 let digits = &rest[..dash];
1829 if !digits.is_empty() && digits.bytes().all(|b| b.is_ascii_digit()) && dash < rest.len() {
1830 &rest[dash + 1..] // everything after "sha<digits>-"
1831 } else {
1832 return false; // no valid sha<N>- prefix → not a known content hash
1833 }
1834 } else {
1835 return false; // no sha prefix → not allowlisted
1836 };
1837 // Strip optional padding (at most 2 `=`).
1838 let stripped = body.trim_end_matches('=');
1839 let pad_removed = body.len() - stripped.len();
1840 if pad_removed > 2 {
1841 return false;
1842 }
1843 // Accept only SHA-family content-hash lengths (43, 64, 86–88 chars unpadded).
1844 let n = stripped.len();
1845 if n != 43 && n != 64 && !(86..=88).contains(&n) {
1846 return false;
1847 }
1848 // Accept both standard-base64 and URL-safe-base64 alphabets.
1849 stripped
1850 .bytes()
1851 .all(|b| b.is_ascii_alphanumeric() || b == b'+' || b == b'/' || b == b'-' || b == b'_')
1852}
1853
1854/// Structural separators that gate entry into [`is_structured_identifier`]
1855/// (rule 1: the token must contain at least one of these). The actual run
1856/// decomposition (rule 2) splits on every non-alphanumeric character, not
1857/// just these four — see the doc comment on `is_structured_identifier`.
1858const STRUCTURAL_SEPARATORS: [char; 4] = ['/', '-', '_', '.'];
1859
1860/// Largest length a single path/branch/identifier segment (a "run" between
1861/// separators) may have and still be considered word-shaped.
1862const MAX_RUN_LEN: usize = 24;
1863
1864/// Runs whose letter portion is at or below this length skip the
1865/// case-transition-density check: density is not a meaningful signal on very
1866/// short runs (e.g. `R1`, `v2`, `ADR`).
1867const DENSITY_EXEMPT_LETTER_LEN: usize = 4;
1868
1869/// Maximum case-transition density (transitions divided by letter_count - 1)
1870/// a run's letter portion may have and still be considered word-shaped.
1871const MAX_CASE_TRANSITION_DENSITY: f64 = 0.3;
1872
1873/// Returns `true` when `token` is shaped like a file path, branch name, or
1874/// other structured identifier rather than a high-entropy secret (word-shaped
1875/// runs separated by `/`, `-`, `_`, `.`). Exempts from the entropy heuristic
1876/// ONLY outside trigger context — see the module doc and
1877/// `docs/api/secret_gate.md#is_structured_identifier` for the run-shape criteria.
1878fn is_structured_identifier(token: &str) -> bool {
1879 if !token.contains(|c: char| STRUCTURAL_SEPARATORS.contains(&c)) {
1880 return false;
1881 }
1882 let runs: Vec<&str> = token
1883 .split(|c: char| !c.is_ascii_alphanumeric())
1884 .filter(|r| !r.is_empty())
1885 .collect();
1886 runs.len() >= 2 && runs.iter().all(|run| is_word_shaped_run(run))
1887}
1888
1889/// A single run (segment between structural separators) is word-shaped when
1890/// it matches `[A-Za-z]+[0-9]*` or `[0-9]+`, is at most [`MAX_RUN_LEN`] chars,
1891/// and (for the letters-then-digits form) its letter portion has a low
1892/// case-transition density.
1893fn is_word_shaped_run(run: &str) -> bool {
1894 if run.is_empty() || run.len() > MAX_RUN_LEN {
1895 return false;
1896 }
1897 let bytes = run.as_bytes();
1898 if bytes.iter().all(|b| b.is_ascii_digit()) {
1899 return true;
1900 }
1901 let letter_end = bytes
1902 .iter()
1903 .position(|b| !b.is_ascii_alphabetic())
1904 .unwrap_or(bytes.len());
1905 // A run that does not start with a letter, and is not pure digits (ruled
1906 // out above), mixes digits and letters in a shape other than
1907 // letters-then-digits — not word-shaped.
1908 if letter_end == 0 {
1909 return false;
1910 }
1911 // Everything after the leading letters must be digits only (no further
1912 // letters), else the run is not the `[A-Za-z]+[0-9]*` shape.
1913 if !bytes[letter_end..].iter().all(|b| b.is_ascii_digit()) {
1914 return false;
1915 }
1916 case_transition_density_ok(&run[..letter_end])
1917}
1918
1919/// `true` when the case-transition density of `letters` (an all-ASCII-letter
1920/// string) is at or below [`MAX_CASE_TRANSITION_DENSITY`]. A transition is an
1921/// adjacent letter pair where one side is uppercase and the other is not.
1922/// Runs with few enough letters pass automatically (see
1923/// [`DENSITY_EXEMPT_LETTER_LEN`]) since density is noisy on short strings.
1924fn case_transition_density_ok(letters: &str) -> bool {
1925 let chars: Vec<char> = letters.chars().collect();
1926 if chars.len() <= DENSITY_EXEMPT_LETTER_LEN {
1927 return true;
1928 }
1929 let transitions = chars
1930 .windows(2)
1931 .filter(|w| w[0].is_ascii_uppercase() != w[1].is_ascii_uppercase())
1932 .count();
1933 let density = transitions as f64 / (chars.len() - 1) as f64;
1934 density <= MAX_CASE_TRANSITION_DENSITY
1935}
1936
1937/// `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`
1938fn is_uuid_canonical(s: &str) -> bool {
1939 let b = s.as_bytes();
1940 if b.len() != 36 {
1941 return false;
1942 }
1943 b[8] == b'-'
1944 && b[13] == b'-'
1945 && b[18] == b'-'
1946 && b[23] == b'-'
1947 && b[..8].iter().all(|c| c.is_ascii_hexdigit())
1948 && b[9..13].iter().all(|c| c.is_ascii_hexdigit())
1949 && b[14..18].iter().all(|c| c.is_ascii_hexdigit())
1950 && b[19..23].iter().all(|c| c.is_ascii_hexdigit())
1951 && b[24..].iter().all(|c| c.is_ascii_hexdigit())
1952}
1953
1954/// Strip common wrapping characters (`"`, `'`, `` ` ``, `:`, `=`) from both ends.
1955fn strip_delimiters(s: &str) -> &str {
1956 s.trim_matches(|c| matches!(c, '"' | '\'' | '`' | ':' | '=' | ',' | ';'))
1957}
1958
1959/// Strip `{}()[]"'.,;` from both ends of `s`, repeatedly (JSON nests one
1960/// wrapper inside another).
1961fn strip_wrappers(s: &str) -> &str {
1962 s.trim_matches(|c: char| {
1963 matches!(
1964 c,
1965 '{' | '}' | '(' | ')' | '[' | ']' | '"' | '\'' | '`' | '.' | ',' | ';'
1966 )
1967 })
1968}
1969
1970fn wrapper_strip_repeated(token: &str) -> &str {
1971 let mut cur = token;
1972 loop {
1973 let next = strip_wrappers(cur);
1974 if next == cur {
1975 return cur;
1976 }
1977 cur = next;
1978 }
1979}
1980
1981/// Yields every candidate value an assignment/wrapper-glued token could
1982/// contain, for the near-trigger UUID/content-hash exact-shape checks only.
1983/// See `docs/api/secret_gate.md#value_candidates` for why every `=`/`:` suffix
1984/// must be tried rather than just the first or last.
1985fn value_candidates(token: &str) -> impl Iterator<Item = &str> {
1986 let cur = wrapper_strip_repeated(token);
1987 std::iter::once(cur).chain(cur.char_indices().filter_map(move |(i, c)| {
1988 if c == '=' || c == ':' {
1989 let after = strip_wrappers(&cur[i + c.len_utf8()..]);
1990 if !after.is_empty() {
1991 return Some(after);
1992 }
1993 }
1994 None
1995 }))
1996}
1997
1998// ─── Utilities ───────────────────────────────────────────────────────────────
1999
2000/// Extract a contiguous token (non-whitespace chars) starting at the beginning of `s`.
2001fn extract_token(s: &str) -> &str {
2002 let end = s
2003 .find(|c: char| c.is_whitespace() || c == '\n' || c == '\r')
2004 .unwrap_or(s.len());
2005 &s[..end]
2006}
2007
2008/// Shannon entropy in bits per character.
2009///
2010/// H = -∑ p_i log2(p_i)
2011fn shannon_entropy(bytes: &[u8]) -> f64 {
2012 if bytes.is_empty() {
2013 return 0.0;
2014 }
2015 let mut counts = [0u32; 256];
2016 for &b in bytes {
2017 counts[b as usize] += 1;
2018 }
2019 let len = bytes.len() as f64;
2020 counts
2021 .iter()
2022 .filter(|&&c| c > 0)
2023 .map(|&c| {
2024 let p = c as f64 / len;
2025 -p * p.log2()
2026 })
2027 .sum()
2028}
2029
2030/// Build a `SecretMatch` from a detector name and the candidate string.
2031///
2032/// The masked excerpt is: first 6 chars + "..." + total length.
2033/// Never includes more than 6 chars of the actual value.
2034fn build_match(detector: &'static str, candidate: &str) -> SecretMatch {
2035 let chars: Vec<char> = candidate.chars().collect();
2036 let preview: String = chars.iter().take(6).collect();
2037 let masked = format!("{}...{}chars", preview, chars.len());
2038 SecretMatch { detector, masked }
2039}
2040
2041// ─── Tests ───────────────────────────────────────────────────────────────────
2042
2043#[cfg(test)]
2044mod tests {
2045 use super::*;
2046
2047 #[test]
2048 fn blocks_aws_akia() {
2049 // FAKE key: prefix is real shape, 16-char suffix invented.
2050 let fake = "AKIAFAKEKEY1234567890";
2051 assert!(scan(fake).is_some(), "AKIA must be caught");
2052 let m = scan(fake).unwrap();
2053 assert_eq!(m.detector, "aws-access-key-id");
2054 // Masked excerpt must not echo the full key.
2055 assert!(
2056 !m.masked.contains("FAKEKEY1234567890"),
2057 "must not echo the secret: {}",
2058 m.masked
2059 );
2060 }
2061
2062 #[test]
2063 fn blocks_aws_asia() {
2064 let fake = "ASIAFAKEKEY00000000000";
2065 let m = scan(fake);
2066 assert!(m.is_some(), "ASIA must be caught");
2067 assert_eq!(m.unwrap().detector, "aws-access-key-id");
2068 }
2069
2070 #[test]
2071 fn blocks_github_ghp() {
2072 // 36 chars total to pass min_len.
2073 let fake = "ghp_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
2074 assert!(scan(fake).is_some(), "ghp_ must be caught");
2075 }
2076
2077 #[test]
2078 fn blocks_github_gho() {
2079 let fake = "gho_BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB";
2080 assert!(scan(fake).is_some(), "gho_ must be caught");
2081 }
2082
2083 #[test]
2084 fn blocks_github_pat() {
2085 let fake = "github_pat_AAAAAABBBBBBCCCCCC";
2086 assert!(scan(fake).is_some(), "github_pat_ must be caught");
2087 }
2088
2089 #[test]
2090 fn blocks_openai_sk() {
2091 let fake = "sk-aaaaaabbbbbbccccccddddddeeeeeeffffgg";
2092 assert!(scan(fake).is_some(), "sk- must be caught");
2093 }
2094
2095 #[test]
2096 fn blocks_anthropic_sk_ant() {
2097 let fake = "sk-ant-api03-AAAAAAAAAAAAAAA";
2098 assert!(scan(fake).is_some(), "sk-ant- must be caught");
2099 assert_eq!(scan(fake).unwrap().detector, "anthropic-api-key");
2100 }
2101
2102 #[test]
2103 fn blocks_stripe_live() {
2104 let fake = "sk_live_FAKESTRIPE0000000000000"; // gitleaks:allow
2105 assert!(scan(fake).is_some(), "sk_live_ must be caught");
2106 assert_eq!(scan(fake).unwrap().detector, "stripe-secret-key");
2107 }
2108
2109 #[test]
2110 fn blocks_stripe_restricted() {
2111 let fake = "rk_live_FAKESTRIPE0000000000000"; // gitleaks:allow
2112 assert!(scan(fake).is_some(), "rk_live_ must be caught");
2113 assert_eq!(scan(fake).unwrap().detector, "stripe-restricted-key");
2114 }
2115
2116 #[test]
2117 fn blocks_fly_flyv1() {
2118 let fake = "FlyV1 FAKEFLYTOKEN000000000000000000";
2119 assert!(scan(fake).is_some(), "FlyV1 must be caught");
2120 assert_eq!(scan(fake).unwrap().detector, "fly-token");
2121 }
2122
2123 #[test]
2124 fn blocks_fly_fm2() {
2125 let fake = "fm2_FAKEFLYTOKEN00000000000000000";
2126 assert!(scan(fake).is_some(), "fm2_ must be caught");
2127 assert_eq!(scan(fake).unwrap().detector, "fly-token");
2128 }
2129
2130 #[test]
2131 fn blocks_vercel_token() {
2132 let fake = "vercel_FAKETOKEN00000000000000000";
2133 assert!(scan(fake).is_some(), "vercel_ must be caught");
2134 assert_eq!(scan(fake).unwrap().detector, "vercel-token");
2135 }
2136
2137 #[test]
2138 fn prefix_detectors_allow_lowercase_source_filenames() {
2139 let filename_body = "provider_integration_monitoring_adapter_configuration.py";
2140
2141 for &(_, needle, min_len) in PREFIX_DETECTORS {
2142 let candidate = format!("{needle}{filename_body}");
2143 assert!(
2144 candidate.len() >= min_len,
2145 "negative control must exercise {needle}'s length tier"
2146 );
2147 assert!(
2148 find_prefix_token(&candidate, needle, min_len).is_none(),
2149 "lowercase source filename must not match prefix {needle}: {candidate}"
2150 );
2151 assert!(
2152 check(&candidate).is_ok(),
2153 "lowercase source filename must pass the canonical scanner: {candidate}, got {:?}",
2154 scan(&candidate)
2155 );
2156 }
2157 }
2158
2159 #[test]
2160 fn prefix_detectors_keep_value_shaped_payloads_with_source_suffixes() {
2161 for &(_, needle, min_len) in PREFIX_DETECTORS {
2162 let required_payload_len = min_len.saturating_sub(needle.len()).max(2);
2163 let value_payload = "A1".repeat(required_payload_len.div_ceil(2));
2164 let candidate = format!("{needle}{value_payload}.py");
2165 assert!(
2166 find_prefix_token(&candidate, needle, min_len).is_some(),
2167 "uppercase/digit value evidence must preserve prefix {needle}: {candidate}"
2168 );
2169 }
2170 }
2171
2172 #[test]
2173 fn prefix_detectors_keep_lowercase_single_run_payloads_with_source_suffixes() {
2174 for &(_, needle, min_len) in PREFIX_DETECTORS {
2175 let required_payload_len = min_len.saturating_sub(needle.len()).max(24);
2176 let value_payload = "a".repeat(required_payload_len);
2177 let candidate = format!("{needle}{value_payload}.py");
2178 assert!(
2179 find_prefix_token(&candidate, needle, min_len).is_some(),
2180 "an extension alone must not exempt prefix {needle}: {candidate}"
2181 );
2182 }
2183 }
2184
2185 #[test]
2186 fn allows_vercel_prefixed_source_filename_on_every_scanner_surface() {
2187 let content = "review `vercel_deployment_monitoring_adapter.py` before release";
2188
2189 assert!(check(content).is_ok(), "write gate must allow filename");
2190 assert!(scan(content).is_none(), "scanner must not report filename");
2191 assert_eq!(
2192 mask_secrets(content).as_ref(),
2193 content,
2194 "masking surface must preserve the filename byte-for-byte"
2195 );
2196 }
2197
2198 #[test]
2199 fn blocks_slack_xoxb() {
2200 let fake = "xoxb-FAKE-SLACKTOKEN-000000000000000000000000";
2201 assert!(scan(fake).is_some(), "xoxb- must be caught");
2202 assert_eq!(scan(fake).unwrap().detector, "slack-token");
2203 }
2204
2205 #[test]
2206 fn blocks_pem_private_key() {
2207 // Split the header so the literal detector-trigger string is not present
2208 // verbatim in source — pre-commit's detect-private-key hook would fire.
2209 // The gate detects it at runtime because scan() sees the assembled string.
2210 let header = ["-----BEGIN RSA", " PRIVATE KEY-----"].concat(); // gitleaks:allow
2211 let fake = format!("{}\nMIIEo\u{2026}\n-----END RSA PRIVATE KEY-----", header);
2212 assert!(scan(&fake).is_some(), "PEM private key must be caught");
2213 assert_eq!(scan(&fake).unwrap().detector, "pem-private-key");
2214 }
2215
2216 #[test]
2217 fn blocks_pem_ec_private_key() {
2218 let header = ["-----BEGIN EC", " PRIVATE KEY-----"].concat(); // gitleaks:allow
2219 let fake = format!("{}\nMHQCAQEE\u{2026}\n-----END EC PRIVATE KEY-----", header);
2220 assert!(scan(&fake).is_some(), "EC PEM must be caught");
2221 }
2222
2223 #[test]
2224 fn blocks_age_secret_key() {
2225 // AGE-SECRET-KEY- followed by 59 base32 chars (Bech32m body).
2226 let fake = "AGE-SECRET-KEY-1QQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQ";
2227 assert!(scan(fake).is_some(), "AGE-SECRET-KEY- must be caught");
2228 assert_eq!(scan(fake).unwrap().detector, "age-secret-key");
2229 }
2230
2231 #[test]
2232 fn blocks_jwt_triple() {
2233 // Synthetic JWT structure: header.payload.signature (no real key).
2234 let fake = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.FAKE_SIG_XXXXXXXXXXXX"; // gitleaks:allow
2235 assert!(scan(fake).is_some(), "JWT triple must be caught");
2236 assert_eq!(scan(fake).unwrap().detector, "jwt");
2237 }
2238
2239 #[test]
2240 fn blocks_url_userinfo() {
2241 let fake = "postgresql://dbuser:S3cr3tP4ss@db.example.com:5432/mydb";
2242 assert!(scan(fake).is_some(), "URL userinfo must be caught");
2243 assert_eq!(scan(fake).unwrap().detector, "url-userinfo");
2244 }
2245
2246 #[test]
2247 fn blocks_high_entropy_near_bearer_word() {
2248 // 32 random-looking base64 chars adjacent to the word "bearer".
2249 let fake = "Bearer token: Xk9mZ2vQpLrT8nJwYuAeHfBsDcGiONvM"; // gitleaks:allow
2250 assert!(
2251 scan(fake).is_some(),
2252 "high-entropy value near 'bearer' must be caught"
2253 );
2254 assert_eq!(scan(fake).unwrap().detector, "high-entropy-token");
2255 }
2256
2257 #[test]
2258 fn blocks_high_entropy_near_secret_word() {
2259 let fake = "secret=Xk9mZ2vQpLrT8nJwYuAeHfBsDcGiONvM"; // gitleaks:allow
2260 assert!(
2261 scan(fake).is_some(),
2262 "high-entropy value near 'secret' must be caught"
2263 );
2264 }
2265
2266 #[test]
2267 fn error_message_masks_secret() {
2268 let fake = "ghp_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
2269 let m = scan(fake).unwrap();
2270 // Masked form: first 6 chars + "...N chars".
2271 // Must NOT contain the full suffix.
2272 let masked = &m.masked;
2273 assert!(
2274 !masked.contains("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"),
2275 "mask must not echo the full secret value; got: {masked}"
2276 );
2277 // Must start with "ghp_AA" (first 6 chars of the token).
2278 assert!(
2279 masked.starts_with("ghp_AA"),
2280 "mask must show first 6 chars; got: {masked}"
2281 );
2282 }
2283
2284 // ── False-positive suite ─────────────────────────────────────────────────
2285
2286 #[test]
2287 fn allows_sha256_hex() {
2288 // 64-char lowercase hex — typical sha256 digest.
2289 let sha = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
2290 assert!(
2291 scan(sha).is_none(),
2292 "sha256 hex must pass (allowlisted); fired: {:?}",
2293 scan(sha)
2294 );
2295 }
2296
2297 #[test]
2298 fn allows_uuid() {
2299 let uuid = "550e8400-e29b-41d4-a716-446655440000";
2300 assert!(
2301 scan(uuid).is_none(),
2302 "UUID must pass; fired: {:?}",
2303 scan(uuid)
2304 );
2305 }
2306
2307 #[test]
2308 fn allows_git_sha() {
2309 // 40-char lowercase git SHA.
2310 let sha = "d362950a3c9b1a4cb47d97f1623e38f1a1e6bcdf";
2311 assert!(
2312 scan(sha).is_none(),
2313 "git SHA must pass; fired: {:?}",
2314 scan(sha)
2315 );
2316 }
2317
2318 #[test]
2319 fn allows_normal_prose() {
2320 let prose =
2321 "The FlashAttention paper introduces IO-aware tiling for transformer self-attention.";
2322 assert!(scan(prose).is_none(), "normal prose must pass");
2323 }
2324
2325 #[test]
2326 fn allows_code_snippet() {
2327 let code = r#"fn create_entity(name: &str, kind: &str) -> RuntimeResult<Entity> {
2328 self.validate_entity_kind(kind)?;
2329 Ok(Entity::new("local", kind, name))
2330}"#;
2331 assert!(
2332 scan(code).is_none(),
2333 "code snippet must pass; fired: {:?}",
2334 scan(code)
2335 );
2336 }
2337
2338 #[test]
2339 fn allows_long_url_without_credentials() {
2340 let url = "https://docs.example.com/api/v2/entities?kind=concept&limit=100";
2341 assert!(scan(url).is_none(), "URL without userinfo must pass");
2342 }
2343
2344 #[test]
2345 fn allows_base64_image_stub() {
2346 // Realistic short base64 data URI stub — no trigger words, below threshold length.
2347 let b64 = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVQI12NgAAIABQ";
2348 assert!(
2349 scan(b64).is_none(),
2350 "base64 image stub without trigger word must pass; fired: {:?}",
2351 scan(b64)
2352 );
2353 }
2354
2355 #[test]
2356 fn allows_long_plain_url() {
2357 let url = "https://api.github.com/repos/ohdearquant/khive/pulls/76/comments?per_page=100";
2358 assert!(
2359 scan(url).is_none(),
2360 "plain URL must pass; fired: {:?}",
2361 scan(url)
2362 );
2363 }
2364
2365 #[test]
2366 fn allows_manifest_content_hash() {
2367 // A string like what appears in Cargo.lock or npm lockfiles.
2368 let line =
2369 "checksum = \"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\"";
2370 assert!(
2371 scan(line).is_none(),
2372 "manifest content hash line must pass; fired: {:?}",
2373 scan(line)
2374 );
2375 }
2376
2377 #[test]
2378 fn masked_excerpt_format() {
2379 let fake = "AKIAFAKEKEY1234567890";
2380 let m = scan(fake).unwrap();
2381 // Format: first6...Nchars
2382 assert!(m.masked.contains("..."), "masked must contain '...'");
2383 assert!(m.masked.ends_with("chars"), "masked must end with 'chars'");
2384 }
2385
2386 // ── Gate function ────────────────────────────────────────────────────────
2387
2388 #[test]
2389 fn check_returns_ok_for_safe_content() {
2390 assert!(check("A normal memory note about LoRA.").is_ok());
2391 }
2392
2393 #[test]
2394 fn check_returns_err_for_secret() {
2395 let fake = "AKIAFAKEKEY1234567890";
2396 let result = check(fake);
2397 assert!(result.is_err(), "check must fail for AKIA key");
2398 let err = result.unwrap_err();
2399 assert!(
2400 matches!(err, RuntimeError::SecretDetected(_)),
2401 "error variant must be SecretDetected"
2402 );
2403 }
2404
2405 // ── Entropy helpers ──────────────────────────────────────────────────────
2406
2407 #[test]
2408 fn entropy_of_uniform_string_is_zero() {
2409 let s = "aaaaaaaaaaaaaaaa";
2410 assert!(shannon_entropy(s.as_bytes()) < 0.01);
2411 }
2412
2413 #[test]
2414 fn entropy_of_random_bytes_is_high() {
2415 // A truly random-looking string should exceed 4.5 bits/char.
2416 let s = b"X9kZ2vQpLrT8nJwYuAeHfBsDcGiONvM1"; // 32 mixed base64 chars
2417 assert!(shannon_entropy(s) > 4.5, "entropy={}", shannon_entropy(s));
2418 }
2419
2420 #[test]
2421 fn cjk_prose_near_trigger_is_not_flagged() {
2422 // Regression: a multibyte CJK run (~19 chars = 57 bytes) clears the
2423 // byte-length floor, and `shannon_entropy` over UTF-8 bytes reads it as
2424 // high-entropy — so a Chinese title near the `auth` trigger word used to
2425 // false-positive as `high-entropy-token`. Non-ASCII tokens are now
2426 // skipped by the entropy heuristic: real base64/hex credentials are
2427 // ASCII, so this cannot hide a secret.
2428 let content = "更新 auth 配置数据库连接管理系统核心模块设计文档";
2429 assert!(
2430 check(content).is_ok(),
2431 "CJK prose near a trigger word must not be flagged as a secret"
2432 );
2433 }
2434
2435 #[test]
2436 fn ascii_secret_near_trigger_still_flagged() {
2437 // The non-ASCII skip must NOT weaken detection of genuine ASCII
2438 // high-entropy credentials near a trigger word.
2439 let content = "api_key X9kZ2vQpLrT8nJwYuAeHfBsDcGiONvM1";
2440 assert!(
2441 check(content).is_err(),
2442 "ASCII high-entropy token near a trigger word must still be blocked"
2443 );
2444 }
2445
2446 #[test]
2447 fn ascii_secret_in_cjk_context_does_not_panic_and_is_flagged() {
2448 // The ±120-byte trigger window around an ASCII token can land in the
2449 // middle of a multibyte CJK character when the token is embedded in
2450 // non-Latin prose. Slicing on a non-char-boundary would panic — the
2451 // window bounds are snapped via `floor_char_boundary`. Detection of
2452 // the genuine ASCII secret must still fire.
2453 let cjk = "数据库连接管理系统核心模块设计文档".repeat(6); // 17 chars × 6 = 306 bytes
2454 // The leading single-byte `x` breaks 3-byte CJK alignment so the window
2455 // start (token_offset - 120) lands mid-character without the snap.
2456 let content = format!("{cjk}x api_key X9kZ2vQpLrT8nJwYuAeHfBsDcGiONvM1 {cjk}");
2457 assert!(
2458 check(&content).is_err(),
2459 "ASCII secret in CJK context must still be blocked (and must not panic)"
2460 );
2461 }
2462
2463 #[test]
2464 fn ascii_secret_glued_to_cjk_is_still_flagged() {
2465 // Regression: a prefixless high-entropy credential glued (no ASCII
2466 // whitespace) to CJK text, CJK brackets/quotes, a fullwidth space, or a
2467 // fullwidth colon used to slip through, because the whole whitespace token
2468 // contained a non-ASCII byte and was skipped wholesale. Non-ASCII is now
2469 // a token delimiter, so the ASCII credential run is isolated and
2470 // entropy-checked while the surrounding ±120-byte window still sees the
2471 // trigger word.
2472 let secret = "X9kZ2vQpLrT8nJwYuAeHfBsDcGiONvM1"; // gitleaks:allow
2473 let cases = [
2474 format!("api_key {secret}数据"), // CJK suffix glued to the token
2475 format!("api_key 「{secret}」"), // CJK brackets wrap the token
2476 format!("api_key {secret}"), // U+3000 ideographic space separator
2477 format!("api_key:{secret}"), // U+FF1A fullwidth colon separator
2478 format!("数据{secret}更新 api_key"), // CJK-glued prefix, trigger after
2479 ];
2480 for content in &cases {
2481 assert!(
2482 check(content).is_err(),
2483 "ASCII secret glued to CJK must be blocked: {content:?}"
2484 );
2485 }
2486 }
2487
2488 #[test]
2489 fn high_entropy_ascii_run_without_trigger_is_not_flagged() {
2490 // The non-ASCII-as-delimiter change must not weaken the trigger-context
2491 // discipline: a high-entropy ASCII run isolated from CJK prose but NOT
2492 // near a credential trigger word is still allowed (only the tokenizer
2493 // changed, not the `near_trigger` gate).
2494 let secret = "X9kZ2vQpLrT8nJwYuAeHfBsDcGiONvM1"; // gitleaks:allow
2495 let content = format!("数据库连接{secret}核心模块设计文档");
2496 assert!(
2497 check(&content).is_ok(),
2498 "high-entropy ASCII run with no trigger word must not be flagged"
2499 );
2500 }
2501
2502 #[test]
2503 fn known_prefix_secret_glued_after_cjk_is_still_flagged() {
2504 // A Layer-1 known-prefix secret glued directly after
2505 // CJK prose (no ASCII whitespace) was missed, because the prefix boundary
2506 // check used `is_alphanumeric` — which Rust counts true for CJK — so the
2507 // preceding ideograph was not treated as a delimiter. These credentials
2508 // must be caught with no nearby ASCII trigger word, on the left side too.
2509 let cases = [
2510 "数据AKIAIOSFODNN7EXAMPLE", // gitleaks:allow
2511 "令牌github_pat_11ABCDEFG0HIJKLMNOPQR", // gitleaks:allow
2512 "密钥sk-ant-api03-AAAAAAAAAAAAAAAAAA", // gitleaks:allow
2513 "配置FlyV1 fm2_AAAABBBBCCCCDDDD", // gitleaks:allow
2514 ];
2515 for content in cases {
2516 assert!(
2517 check(content).is_err(),
2518 "known-prefix secret glued after CJK must be blocked: {content:?}"
2519 );
2520 }
2521 }
2522
2523 #[test]
2524 fn url_userinfo_after_cjk_does_not_panic_and_is_flagged() {
2525 // A credential URL glued after CJK prose panicked,
2526 // because scheme_start was (separator byte index + 1) — one byte into a
2527 // multibyte CJK separator — and the slice fell on a non-char boundary.
2528 // The public check() API must return a controlled error, never panic.
2529 let cases = [
2530 "数据postgresql://dbuser:S3cr3tP4ss@db.example.com/db", // gitleaks:allow
2531 "配置mysql://root:hunter2pw@10.0.0.1:3306/app", // gitleaks:allow
2532 "连接redis://svc:V3ryS3cretPw@cache.internal:6379", // gitleaks:allow
2533 ];
2534 for content in cases {
2535 assert!(
2536 check(content).is_err(),
2537 "credential URL after CJK must be blocked, not panic: {content:?}"
2538 );
2539 }
2540 }
2541
2542 #[test]
2543 fn non_ascii_glued_token_trigger_is_still_flagged() {
2544 // `token=`/`token:`/standalone `token` glued directly
2545 // after non-ASCII prose was missed because has_standalone_token /
2546 // has_token_assignment used is_alphanumeric for the word boundary — CJK,
2547 // accented letters, and fullwidth digits all count as alphanumeric in
2548 // Rust, so the preceding char was not seen as a boundary and the `token`
2549 // trigger was suppressed, leaving the high-entropy value unflagged.
2550 let opaque = "Xk9mZ2vQpLrT8nJwYuAeHfBsDcGiONvMabcdef"; // gitleaks:allow
2551 let blocked = [
2552 format!("数据token={opaque}"), // CJK + assignment form, ASCII '='
2553 format!("配置token: {opaque}"), // CJK + assignment form, ASCII ':'
2554 format!("密钥token {opaque}"), // CJK + standalone-word form
2555 format!("résumétoken: {opaque}"), // accented letter before `token`
2556 format!("1token: {opaque}"), // fullwidth digit before `token`
2557 ];
2558 for content in &blocked {
2559 assert!(
2560 check(content).is_err(),
2561 "non-ASCII-glued token trigger must flag the value: {content:?}"
2562 );
2563 }
2564 // Compound identifiers stay excluded — the `_` boundary rule is unchanged
2565 // and an ASCII letter before `token` is still a continuation, so these
2566 // (including the pure-ASCII `servicetoken:`) must still pass.
2567 let allowed = [
2568 format!("数据next_token: {opaque}"),
2569 format!("数据token_count: {opaque}"),
2570 format!("servicetoken: {opaque}"),
2571 ];
2572 for content in &allowed {
2573 assert!(
2574 check(content).is_ok(),
2575 "compound token identifier must not be flagged: {content:?}"
2576 );
2577 }
2578 }
2579
2580 #[test]
2581 fn allowlist_passes_sha256() {
2582 // A plain sha256 hex digest passes via `is_pure_hex` (not `is_allowlisted`
2583 // because hex is now context-dependent; this tests the primitive directly).
2584 let sha = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
2585 assert!(is_pure_hex(sha));
2586 }
2587
2588 #[test]
2589 fn allowlist_passes_uuid_canonical() {
2590 assert!(is_uuid_canonical("550e8400-e29b-41d4-a716-446655440000"));
2591 }
2592
2593 #[test]
2594 fn allowlist_does_not_pass_mixed_token() {
2595 // A token that starts with letters but mixes in non-hex chars.
2596 assert!(!is_pure_hex("sk-aaaaaabbbbbbccccccddddddeeeeeeffffgg"));
2597 }
2598
2599 // ── Structured-field gate helpers ────────────────────────────────────────
2600
2601 #[test]
2602 fn check_json_blocks_secret_in_object_value() {
2603 let props = serde_json::json!({ "api_key": "AKIAFAKEKEY1234567890" });
2604 assert!(
2605 check_json(&props).is_err(),
2606 "secret in properties object value must be blocked"
2607 );
2608 }
2609
2610 #[test]
2611 fn check_json_blocks_secret_in_nested_object() {
2612 let props = serde_json::json!({ "credentials": { "token": "sk-proj-FAKEKEY00000000000000000000000000000000" } }); // gitleaks:allow
2613 assert!(
2614 check_json(&props).is_err(),
2615 "secret in nested properties object must be blocked"
2616 );
2617 }
2618
2619 #[test]
2620 fn check_json_blocks_secret_in_array() {
2621 let props = serde_json::json!(["normal", "AKIAFAKEKEY1234567890"]);
2622 assert!(
2623 check_json(&props).is_err(),
2624 "secret in JSON array must be blocked"
2625 );
2626 }
2627
2628 #[test]
2629 fn check_json_passes_safe_properties() {
2630 let props = serde_json::json!({
2631 "domain": "attention",
2632 "status": "researched",
2633 "year": 2024
2634 });
2635 assert!(
2636 check_json(&props).is_ok(),
2637 "normal properties must pass; fired: {:?}",
2638 check_json(&props).err()
2639 );
2640 }
2641
2642 #[test]
2643 fn check_tags_blocks_credential_tag() {
2644 let tags = vec![
2645 "type:concept".to_string(),
2646 "AKIAFAKEKEY1234567890".to_string(),
2647 ];
2648 assert!(
2649 check_tags(&tags).is_err(),
2650 "credential-shaped tag must be blocked"
2651 );
2652 }
2653
2654 #[test]
2655 fn check_tags_passes_normal_tags() {
2656 let tags = vec!["type:concept".to_string(), "domain:attention".to_string()];
2657 assert!(
2658 check_tags(&tags).is_ok(),
2659 "normal tags must pass; fired: {:?}",
2660 check_tags(&tags).err()
2661 );
2662 }
2663
2664 // ── False-positive: sk-learn and scikit-learn slugs ──────────────────────
2665
2666 #[test]
2667 fn allows_sk_learn_prose() {
2668 // scikit-learn slug used as an entity name or knowledge atom.
2669 let texts = &[
2670 "sk-learn is a Python machine learning library",
2671 "sk-learn-compatible transformer pipeline reference",
2672 "sk-learn scikit-learn estimator interface",
2673 ];
2674 for t in texts {
2675 assert!(
2676 scan(t).is_none(),
2677 "sk-learn prose must pass; fired: {:?} on {:?}",
2678 scan(t),
2679 t
2680 );
2681 }
2682 }
2683
2684 #[test]
2685 fn blocks_openai_sk_proj_not_confused_with_sk_learn() {
2686 // Real OpenAI key shape must still be caught.
2687 let fake = "sk-proj-FAKEKEY00000000000000000000000000000000"; // gitleaks:allow
2688 assert!(
2689 scan(fake).is_some(),
2690 "sk-proj- key must still be caught after sk-learn exemption"
2691 );
2692 }
2693
2694 // ── False-positive: SRI / tokenizer hash metadata ────────────────────────
2695
2696 #[test]
2697 fn blocks_sri_hash_near_key_word_accepted_fp() {
2698 // SRI hash as used in HTML integrity attributes (sha384, base64-encoded),
2699 // placed directly beside the trigger word "key". The content-hash
2700 // allowlist is a prose-context exemption, not unconditional: near a
2701 // credential trigger, a sha-prefixed hash falls through to the explicit
2702 // near-trigger content-hash detector like any other high-entropy
2703 // candidate. This is an accepted false positive on a real but rare
2704 // shape (an integrity hash literally next to the word "key").
2705 let line = "integrity key: sha384-oqVuAfXRKap7fdgcCY5uykM6+R9GqQ8K/uxy9rx7HNQlGYl1kPzQho1wx4JwY8wC";
2706 assert!(
2707 scan(line).is_some(),
2708 "SRI hash near trigger word 'key' must now be blocked (accepted FP); passed unexpectedly"
2709 );
2710 }
2711
2712 #[test]
2713 fn allows_base64_tokenizer_hash_metadata() {
2714 // Tokenizer metadata containing a base64 hash near technical keywords.
2715 let line = "tokenizer_vocab_hash: Xk9mZ2vQpLrT8nJwYuAeHfBsDcGiONvM"; // gitleaks:allow
2716 assert!(
2717 scan(line).is_none(),
2718 "tokenizer hash metadata must pass; fired: {:?}",
2719 scan(line)
2720 );
2721 }
2722
2723 #[test]
2724 fn allows_npm_lockfile_integrity() {
2725 // npm lockfile integrity line with sha512 base64url hash (86 base64 chars + ==).
2726 // sha512 digest = 64 bytes → base64 = 88 chars (86 unpadded + ==).
2727 let body_86 = "Xk9mZ2vQpLrT8nJwYuAeHfBsDcGiONvM1234567890abcdefghijklmnopqrstuvwxABCDEFGHIJKLMNOPQRST";
2728 assert_eq!(body_86.len(), 86, "test body must be exactly 86 chars");
2729 let line = format!(
2730 "resolved: https://registry.npmjs.org/foo/-/foo-1.0.0.tgz\nintegrity: sha512-{body_86}=="
2731 );
2732 assert!(
2733 scan(&line).is_none(),
2734 "npm lockfile integrity must pass; fired: {:?}",
2735 scan(&line)
2736 );
2737 }
2738
2739 // ── False-positive: tokenizer vs token trigger word ─────────────────────
2740
2741 #[test]
2742 fn allows_tokenizer_vocab_hash_no_block() {
2743 // `tokenizer_vocab_hash` contains the substring "token" but NOT as a
2744 // standalone word (followed by 'i' which is alphanumeric), so the
2745 // standalone-token boundary check must not fire here.
2746 let line = "tokenizer_vocab_hash = Xk9mZ2vQpLrT8nJwYuAeHfBsDcGiONvM"; // gitleaks:allow
2747 assert!(
2748 scan(line).is_none(),
2749 "tokenizer_vocab_hash must pass; 'token' is only standalone-word matched; fired: {:?}",
2750 scan(line)
2751 );
2752 }
2753
2754 // ── True-positives: bare base64 at sha-lengths near trigger words ────────
2755
2756 #[test]
2757 fn blocks_bare_base64url_43chars_near_key() {
2758 // A 43-char base64url token (= sha256 body length) near the word "key".
2759 // Without a sha<N>- prefix this MUST be caught, not allowlisted.
2760 let token_43 = "wJalrXUtnFEMI-K7MDENGbPxRfiCYEXAMPLEKEYX123"; // gitleaks:allow
2761 assert_eq!(token_43.len(), 43, "test token must be exactly 43 chars");
2762 let line = format!("api key {token_43}");
2763 assert!(
2764 scan(&line).is_some(),
2765 "43-char base64url token near 'key' must be caught (no sha-prefix = not a hash); fired: {:?}",
2766 scan(&line)
2767 );
2768 }
2769
2770 #[test]
2771 fn blocks_bare_base64url_64chars_near_secret() {
2772 // A 64-char base64url token (= sha384 body length) near "secret".
2773 // Must be caught without sha<N>- prefix.
2774 let token_64 = "wJalrXUtnFEMI-K7MDENGbPxRfiCYEXAMPLEKEYX123wJalrXUtnFEMI-K7MDENa"; // gitleaks:allow
2775 assert_eq!(token_64.len(), 64, "test token must be exactly 64 chars");
2776 let line = format!("secret: {token_64}");
2777 assert!(
2778 scan(&line).is_some(),
2779 "64-char base64url token near 'secret' must be caught; got: {:?}",
2780 scan(&line)
2781 );
2782 }
2783
2784 #[test]
2785 fn blocks_bare_base64url_86chars_near_auth() {
2786 // An 86-char base64url token (= sha512 body length) near "auth".
2787 // Must be caught without sha<N>- prefix.
2788 let token_86 = "wJalrXUtnFEMI-K7MDENGbPxRfiCYEXAMPLEKEYX123wJalrXUtnFEMI-K7MDENwJalrXUtnFEMI-K7MDENabc"; // gitleaks:allow
2789 assert_eq!(token_86.len(), 86, "test token must be exactly 86 chars");
2790 let line = format!("auth header {token_86}");
2791 assert!(
2792 scan(&line).is_some(),
2793 "86-char base64url token near 'auth' must be caught; got: {:?}",
2794 scan(&line)
2795 );
2796 }
2797
2798 // ── True-positives: standalone `token` trigger ───────────────────────────
2799
2800 #[test]
2801 fn blocks_service_token_opaque_value() {
2802 // "service token <opaque-high-entropy>" — `token` as a standalone word
2803 // with a high-entropy value must be caught.
2804 let opaque = "Xk9mZ2vQpLrT8nJwYuAeHfBsDcGiONvMabcdef"; // gitleaks:allow
2805 assert!(
2806 opaque.len() >= 24,
2807 "opaque must be long enough for entropy check"
2808 );
2809 let line = format!("service token {opaque}");
2810 assert!(
2811 scan(&line).is_some(),
2812 "service token <opaque> must be caught by standalone 'token' check; got: {:?}",
2813 scan(&line)
2814 );
2815 }
2816
2817 #[test]
2818 fn blocks_token_equals_credential() {
2819 // `token=<high-entropy>` (assignment form) must be caught via has_token_assignment.
2820 let opaque = "Xk9mZ2vQpLrT8nJwYuAeHfBsDcGiONvMabcdef"; // gitleaks:allow
2821 let line = format!("token={opaque}");
2822 assert!(
2823 scan(&line).is_some(),
2824 "token=<value> must be caught via token= trigger; got: {:?}",
2825 scan(&line)
2826 );
2827 }
2828
2829 #[test]
2830 fn blocks_token_colon_credential() {
2831 // `token: <high-entropy>` (key-value form) must be caught via has_token_assignment.
2832 let opaque = "Xk9mZ2vQpLrT8nJwYuAeHfBsDcGiONvMabcdef"; // gitleaks:allow
2833 let line = format!("token: {opaque}");
2834 assert!(
2835 scan(&line).is_some(),
2836 "token: <value> must be caught via token: trigger; got: {:?}",
2837 scan(&line)
2838 );
2839 }
2840
2841 #[test]
2842 fn allows_next_token_technical_context() {
2843 // `next_token` is a technical term; the high-entropy value here has low
2844 // entropy anyway, so it must pass.
2845 let line = "next_token: cursor-page-2-abcdef12345678";
2846 assert!(
2847 scan(line).is_none(),
2848 "next_token technical context must not be blocked; fired: {:?}",
2849 scan(line)
2850 );
2851 }
2852
2853 // ── Boundary-aware token= / token: (compound identifiers must pass) ─────
2854
2855 #[test]
2856 fn allows_next_token_high_entropy_cursor() {
2857 // `next_token:` with a realistic high-entropy pagination cursor must NOT be
2858 // blocked. `next_token` has `_token` suffix — not a standalone assignment form.
2859 let cursor = "Xk9mZ2vQpLrT8nJwYuAeHfBsDcGiONvMabcdef"; // gitleaks:allow
2860 let line = format!("next_token: {cursor}");
2861 assert!(
2862 scan(&line).is_none(),
2863 "next_token with high-entropy cursor must pass (compound identifier); fired: {:?}",
2864 scan(&line)
2865 );
2866 }
2867
2868 #[test]
2869 fn allows_token_count_high_entropy() {
2870 // `token_count:` with a high-entropy value must NOT be blocked.
2871 // `token_count` has `token_` prefix — the word boundary after `token` is `_`,
2872 // which is excluded by has_token_assignment.
2873 let opaque = "Xk9mZ2vQpLrT8nJwYuAeHfBsDcGiONvMabcdef"; // gitleaks:allow
2874 let line = format!("token_count: {opaque}");
2875 assert!(
2876 scan(&line).is_none(),
2877 "token_count with high-entropy value must pass; fired: {:?}",
2878 scan(&line)
2879 );
2880 }
2881
2882 // ── Hex allowlist is not applied when trigger context is present ────────
2883 //
2884 // Pure hex strings have a theoretical maximum entropy of log2(16) = 4.0 bits/char,
2885 // which is below the ENTROPY_THRESHOLD of 4.5. That means pure hex tokens cannot
2886 // reach the entropy threshold and will never be flagged by the heuristic alone.
2887 //
2888 // However, the hex allowlist was previously applied BEFORE the trigger window was
2889 // computed, meaning a future threshold reduction or edge case could silently
2890 // skip credential-context hex. The fix: compute trigger context first; only
2891 // apply the hex allowlist when NOT near a trigger. The tests below verify the
2892 // structural change is in place by confirming that non-pure-hex high-entropy
2893 // tokens near triggers are caught (showing the trigger path is live), and that
2894 // purely hex tokens near triggers still correctly pass (entropy too low to flag).
2895
2896 #[test]
2897 fn hex_near_key_blocked_in_credential_context() {
2898 // A pure-hex 32-char token near "api key" is a credential-shaped hex
2899 // token in trigger context. Entropy alone cannot flag it (hex max =
2900 // 4.0 < 4.5 threshold), but the explicit hex-credential-token path
2901 // must catch it.
2902 let hex32 = "4f9c2e8a1d3b5c7e9f0a2b4d6e8c0a2b";
2903 assert_eq!(hex32.len(), 32);
2904 let line = format!("api key {hex32}");
2905 assert!(
2906 scan(&line).is_some(),
2907 "32-char pure hex near 'api key' must be blocked; got None"
2908 );
2909 }
2910
2911 #[test]
2912 fn hex_credential_lengths_blocked_near_trigger() {
2913 // Verify all four credential-shaped lengths are caught near a trigger.
2914 let hex40 = "a3f5c2e9d1b8047e63a1f4c2d5b6e8f1a9c3d2e4";
2915 let hex64 = "1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b";
2916 let hex128 = format!("{hex64}{hex64}");
2917 assert_eq!(hex40.len(), 40);
2918 assert_eq!(hex64.len(), 64);
2919 assert_eq!(hex128.len(), 128);
2920
2921 for (label, hex) in &[
2922 ("hex40", hex40),
2923 ("hex64", hex64),
2924 ("hex128", hex128.as_str()),
2925 ] {
2926 let line = format!("secret key: {hex}");
2927 assert!(
2928 scan(&line).is_some(),
2929 "{label} near 'secret key' must be blocked; got None"
2930 );
2931 }
2932 }
2933
2934 #[test]
2935 fn hex_blocked_when_trigger_and_hash_word_coexist() {
2936 // Credential trigger dominates: adding "hash" or "sha" to the window does
2937 // not rescue a pure-hex token when a credential trigger is also present.
2938 // An attacker controlling the prose could otherwise bypass the gate with
2939 // one extra word, so the hash-word exception must NOT apply in trigger context.
2940 let hex32 = "4f9c2e8a1d3b5c7e9f0a2b4d6e8c0a2b";
2941 let key_hash_line = format!("api key hash {hex32}");
2942 let secret_sha_line = format!("secret sha {hex32}");
2943 assert!(
2944 scan(&key_hash_line).is_some(),
2945 "'api key hash <hex32>' must be blocked; got None"
2946 );
2947 assert!(
2948 scan(&secret_sha_line).is_some(),
2949 "'secret sha <hex32>' must be blocked; got None"
2950 );
2951 }
2952
2953 #[test]
2954 fn hex_near_sha_context_word_allowed() {
2955 // A 40-char hex with "sha" or "commit" in the window — but no credential
2956 // trigger — must be allowed (git SHA or content hash in normal prose).
2957 let hex40 = "da39a3ee5e6b4b0d3255bfef95601890afd80709";
2958 let sha_line = format!("sha1: {hex40}");
2959 let commit_line = format!("commit sha {hex40}");
2960 assert!(
2961 scan(&sha_line).is_none(),
2962 "hex40 near 'sha1' context must be allowed; fired: {:?}",
2963 scan(&sha_line)
2964 );
2965 assert!(
2966 scan(&commit_line).is_none(),
2967 "hex40 near 'commit sha' context must be allowed; fired: {:?}",
2968 scan(&commit_line)
2969 );
2970 }
2971
2972 #[test]
2973 fn allows_git_revision_reference_near_ordinary_key_and_token_prose() {
2974 let revision = "d362950a3c9b1a4cb47d97f1623e38f1a1e6bcdf";
2975 let contents = [
2976 format!("the primary key behavior is pinned to commit {revision}"),
2977 format!("revision: {revision} emits one extra token"),
2978 format!("primary key behavior at revision:{revision};"),
2979 format!("configuration key\n\nrev {revision}."),
2980 format!("one extra token was introduced by sha: {revision}"),
2981 ];
2982 for content in &contents {
2983 assert!(
2984 check(content).is_ok(),
2985 "git revision reference in technical prose must pass: {content:?}, got {:?}",
2986 scan(content)
2987 );
2988 }
2989 }
2990
2991 #[test]
2992 fn git_revision_reference_does_not_exempt_credential_assignments() {
2993 let revision = "d362950a3c9b1a4cb47d97f1623e38f1a1e6bcdf";
2994 for content in [
2995 format!("api_key={revision}"),
2996 format!("secret key: {revision}"),
2997 format!("token={revision}"),
2998 format!("api key hash {revision}"),
2999 ] {
3000 assert!(
3001 check(&content).is_err(),
3002 "credential assignment must remain blocked: {content:?}"
3003 );
3004 }
3005 }
3006
3007 #[test]
3008 fn blocks_forty_hex_after_credential_phrase_with_vcs_marker() {
3009 let revision = "d362950a3c9b1a4cb47d97f1623e38f1a1e6bcdf";
3010 for content in [
3011 format!("api key value is commit {revision}"),
3012 format!("api key value is revision {revision}"),
3013 format!("the secret is rev {revision}"),
3014 format!("token value sha {revision}"),
3015 ] {
3016 assert_eq!(
3017 scan(&content).map(|matched| matched.detector),
3018 Some("hex-credential-token"),
3019 "a credential phrase must not be hidden by connector words before \
3020 a VCS marker: {content:?}"
3021 );
3022 }
3023 }
3024
3025 #[test]
3026 fn blocks_unicode_punctuation_between_vcs_marker_and_forty_hex() {
3027 let revision = "d362950a3c9b1a4cb47d97f1623e38f1a1e6bcdf";
3028 let content = format!("api_key value commit\u{200B}{revision}");
3029 assert!(
3030 check(&content).is_err(),
3031 "a zero-width space between marker and value must not rescue a \
3032 labeled credential: {content:?}, got {:?}",
3033 scan(&content)
3034 );
3035 }
3036
3037 #[test]
3038 fn blocks_slash_bearing_base64_credential_in_value_syntax() {
3039 // 40-char standard-base64-alphabet value whose `/` splits it into two
3040 // runs (19 and 20 bytes) each below MIN_ENTROPY_LEN — per-run checks
3041 // never see it, so the block must come from refusing the path
3042 // exemption for a credential-value clause and applying whole-token
3043 // entropy.
3044 let content = "api key value is Xk9mZ2vQpLrT8nJwYuA/HfBsDcGiONvMabcdefgh";
3045 assert_eq!(
3046 scan(content).map(|matched| matched.detector),
3047 Some("high-entropy-token"),
3048 "slash-bearing base64 credential in value syntax must be blocked"
3049 );
3050 }
3051
3052 #[test]
3053 fn blocks_angle_bracket_line_range_base64_credential_in_value_syntax() {
3054 let content = "api key value is <Xk9mZ2vQpLrT8nJwYuA/HfBsDcGiONvMabcdefgh>:~97-103";
3055 assert_eq!(
3056 scan(content).map(|matched| matched.detector),
3057 Some("high-entropy-token"),
3058 "angle-bracket/line-range dressing must not exempt a credential in \
3059 value syntax"
3060 );
3061 }
3062
3063 #[test]
3064 fn blocks_split_hex_credential_with_marker_adjacent_fragment() {
3065 // A 64-hex credential split 40+24 by a zero-width space, with the
3066 // first fragment hiding behind a VCS marker. The marker-adjacent
3067 // fragment is exempt as its own anchor, but the second fragment
3068 // anchors its own reconstruction chain, walks back across the gap,
3069 // and accumulates 40+24=64 — the symmetric-anchor property the
3070 // vcs-exempt bridge skip relies on.
3071 let content = concat!(
3072 "api token context: commit ",
3073 "d362950a3c9b1a4cb47d97f1623e38f1a1e6bcdf",
3074 "\u{200B}",
3075 "0123456789abcdef01234567"
3076 );
3077 assert_eq!(
3078 scan(content).map(|matched| matched.detector),
3079 Some("hex-credential-token"),
3080 "split credential with a marker-adjacent fragment must be blocked"
3081 );
3082 }
3083
3084 #[test]
3085 fn blocks_forty_hex_behind_qualified_label_with_delimiter() {
3086 // A label with qualifier words the connector set cannot enumerate
3087 // ("for deploy") followed by a value delimiter is assignment syntax:
3088 // once the walk crosses the `:`/`=`, every label-side identifier is
3089 // stepped over until the trigger word.
3090 let revision = "d362950a3c9b1a4cb47d97f1623e38f1a1e6bcdf";
3091 for content in [
3092 format!("api key for deploy: commit {revision}"),
3093 format!("prod api key for deploy = commit {revision}"),
3094 ] {
3095 assert_eq!(
3096 scan(&content).map(|matched| matched.detector),
3097 Some("hex-credential-token"),
3098 "a qualified label before a value delimiter must not be \
3099 hidden from the exemption guard: {content:?}"
3100 );
3101 }
3102 }
3103
3104 #[test]
3105 fn blocks_slash_base64_behind_qualified_label_with_delimiter() {
3106 let content = "api key for deploy: Xk9mZ2vQpLrT8nJwYuA/HfBsDcGiONvMabcdefgh";
3107 assert_eq!(
3108 scan(content).map(|matched| matched.detector),
3109 Some("high-entropy-token"),
3110 "a qualified label before a value delimiter must refuse the path \
3111 exemption for a slash-bearing base64 credential"
3112 );
3113 }
3114
3115 #[test]
3116 fn blocks_forty_hex_behind_versioned_label() {
3117 // `v1.2` splits into version fragments under identifier extraction;
3118 // the intra-token dot must not read as a sentence boundary and the
3119 // fragments must be stepped over like connector words.
3120 let revision = "d362950a3c9b1a4cb47d97f1623e38f1a1e6bcdf";
3121 let content = format!("api key v1.2 value is commit {revision}");
3122 assert_eq!(
3123 scan(&content).map(|matched| matched.detector),
3124 Some("hex-credential-token"),
3125 "a versioned credential label must stay reachable through its \
3126 version fragments: {content:?}"
3127 );
3128 }
3129
3130 #[test]
3131 fn blocks_labeled_inline_marker_split_credential() {
3132 // The r2 medium probes: `marker:value` inline forms carrying a split
3133 // credential, with a credential label ahead of a value delimiter.
3134 // The clause guard disables the VCS exemption, and whole-token
3135 // normalized-hex accumulation fires on the fused token.
3136 for content in [
3137 concat!(
3138 "api token context: rev:",
3139 "d362950a3c9b1a4cb47d97f1623e38f1a1e6bcdf",
3140 "\u{200B}",
3141 "0123456789abcdef01234567"
3142 )
3143 .to_string(),
3144 concat!(
3145 "api token context: ",
3146 "0123456789abcdef01234567",
3147 "\u{200B}",
3148 "rev:d362950a3c9b1a4cb47d97f1623e38f1a1e6bcdf"
3149 )
3150 .to_string(),
3151 ] {
3152 assert!(
3153 check(&content).is_err(),
3154 "a labeled inline-marker split credential must be blocked: \
3155 {content:?}, got {:?}",
3156 scan(&content)
3157 );
3158 }
3159 }
3160
3161 #[test]
3162 fn blocks_forty_hex_behind_multiword_label_and_marker_delimiter() {
3163 // Coverage: qualifier nouns beyond the two-word case are
3164 // reachable once prepositions/possessives read as glue, and a
3165 // delimiter attached to the VCS marker itself ("deploy sha: <hex>")
3166 // is assignment syntax like any other delimiter.
3167 let revision = "d362950a3c9b1a4cb47d97f1623e38f1a1e6bcdf";
3168 for content in [
3169 format!("api key for deploy sha: {revision}"),
3170 format!("prod api key for deploy sha: {revision}"),
3171 format!("api key for production deploy: commit {revision}"),
3172 ] {
3173 assert_eq!(
3174 scan(&content).map(|matched| matched.detector),
3175 Some("hex-credential-token"),
3176 "a natural multiword credential label must not be hidden by \
3177 glue words or a marker-attached delimiter: {content:?}"
3178 );
3179 }
3180 }
3181
3182 #[test]
3183 fn blocks_slash_base64_behind_possessive_qualified_label() {
3184 let content = "api key for our production deploy: Xk9mZ2vQpLrT8nJwYuA/HfBsDcGiONvMabcdefgh";
3185 assert_eq!(
3186 scan(content).map(|matched| matched.detector),
3187 Some("high-entropy-token"),
3188 "possessive-qualified credential label must refuse the path \
3189 exemption"
3190 );
3191 }
3192
3193 #[test]
3194 fn blocks_forty_hex_behind_participial_adjective_qualifier() {
3195 // Coverage: a past-participle word in ADJECTIVE position
3196 // (followed by a content noun: "shared deploy", "encrypted backup")
3197 // is a label qualifier, not verb-phrase prose, and must not end the
3198 // walk.
3199 let revision = "d362950a3c9b1a4cb47d97f1623e38f1a1e6bcdf";
3200 let content = format!("api key for shared deploy: commit {revision}");
3201 assert_eq!(
3202 scan(&content).map(|matched| matched.detector),
3203 Some("hex-credential-token"),
3204 "a participial-adjective label qualifier must stay walkable: \
3205 {content:?}"
3206 );
3207 }
3208
3209 #[test]
3210 fn blocks_slash_base64_behind_participial_adjective_qualifier() {
3211 let content = "api key for encrypted backup: Xk9mZ2vQpLrT8nJwYuA/HfBsDcGiONvMabcdefgh";
3212 assert_eq!(
3213 scan(content).map(|matched| matched.detector),
3214 Some("high-entropy-token"),
3215 "a participial-adjective label qualifier must refuse the path \
3216 exemption"
3217 );
3218 }
3219
3220 #[test]
3221 fn blocks_forty_hex_behind_chained_qualifier_label() {
3222 // Coverage: a chain of qualifiers between the value and
3223 // the trigger ("shared encrypted deploy") must not exhaust the walk
3224 // before the label head is reached. Any per-clause content-word cap
3225 // re-admits this bypass one qualifier past the cap.
3226 let revision = "d362950a3c9b1a4cb47d97f1623e38f1a1e6bcdf";
3227 let content = format!("api key for shared encrypted deploy: commit {revision}");
3228 assert_eq!(
3229 scan(&content).map(|matched| matched.detector),
3230 Some("hex-credential-token"),
3231 "a chained-qualifier credential label must stay walkable: \
3232 {content:?}"
3233 );
3234 }
3235
3236 #[test]
3237 fn blocks_slash_base64_behind_chained_qualifier_label() {
3238 let content =
3239 "api key for shared encrypted deploy: Xk9mZ2vQpLrT8nJwYuA/HfBsDcGiONvMabcdefgh";
3240 assert_eq!(
3241 scan(content).map(|matched| matched.detector),
3242 Some("high-entropy-token"),
3243 "a chained-qualifier credential label must refuse the path \
3244 exemption"
3245 );
3246 }
3247
3248 #[test]
3249 fn blocks_over_limit_qualified_label_fails_closed() {
3250 // Coverage: labels whose clause exhausts the walk budget
3251 // (a marker, an extra qualifier, or an interleaved glue word pushes
3252 // the trigger past the limit). Exhaustion after a value delimiter
3253 // fails closed — clause length must not launder a labeled credential
3254 // into the exemptions.
3255 let revision = "d362950a3c9b1a4cb47d97f1623e38f1a1e6bcdf";
3256 let opaque = "Xk9mZ2vQpLrT8nJwYuA/HfBsDcGiONvMabcdefgh";
3257 for content in [
3258 format!("api key for the new shared encrypted staging deploy: commit {revision}"),
3259 format!("api key for the new shared encrypted regional staging deploy: {opaque}"),
3260 format!("api key for the new shared and encrypted staging deploy: {opaque}"),
3261 format!(
3262 "api key for the new shared encrypted regional staging deploy: commit {revision}"
3263 ),
3264 format!("api key for the new shared and encrypted staging deploy: commit {revision}"),
3265 format!("api key for the new shared encrypted staging deploy: {opaque}"),
3266 ] {
3267 assert!(
3268 check(&content).is_err(),
3269 "an over-limit qualified credential label must fail closed: \
3270 {content:?}, got {:?}",
3271 scan(&content)
3272 );
3273 }
3274 }
3275
3276 #[test]
3277 fn accepted_false_positive_topical_trigger_before_delimited_path() {
3278 // The clause walk treats ANY reachable pre-delimiter trigger as a
3279 // credential label — it has no grammar to tell a label head ("api
3280 // key ...") from a topical object ("testing auth against parser").
3281 // Distinguishing them would reopen the labeled-value bypasses, so
3282 // this ordinary prose shape blocks. Accepted false positive,
3283 // conservative direction; documented in docs/api/secret_gate.md.
3284 let content = "results from testing auth against parser: \
3285 internal/workspaces/20260701/cloud-rebuild/R1-repo-audit.md";
3286 assert!(
3287 check(content).is_err(),
3288 "accepted-FP contract changed: topical trigger before a \
3289 delimited path no longer blocks — update the docs if deliberate"
3290 );
3291 }
3292
3293 #[test]
3294 fn blocks_participle_before_trigger_word() {
3295 // Ordering contract: a past-participle word BEFORE the trigger never
3296 // matters — the walk reaches the trigger first. Pinned so the
3297 // verb-position rule cannot regress into shielding these labels.
3298 let revision = "d362950a3c9b1a4cb47d97f1623e38f1a1e6bcdf";
3299 let opaque = "Xk9mZ2vQpLrT8nJwYuA/HfBsDcGiONvMabcdefgh";
3300 for content in [
3301 format!("shared key: {revision}"),
3302 format!("generated api key: {revision}"),
3303 format!("encrypted token = {opaque}"),
3304 ] {
3305 assert!(
3306 check(&content).is_err(),
3307 "a participle before the trigger word must not shield the \
3308 label: {content:?}, got {:?}",
3309 scan(&content)
3310 );
3311 }
3312 }
3313
3314 #[test]
3315 fn accepted_false_positive_docs_path_behind_attributive_trigger_and_delimiter() {
3316 // "auth setup: <path>" carries a trigger word in clause range ahead
3317 // of a value delimiter; the walk cannot distinguish an attributive
3318 // trigger ("auth setup") from a label head ("api key ...") without
3319 // reopening the labeled-value bypasses, so this ordinary prose shape
3320 // blocks. Accepted false positive — conservative direction under the
3321 // threat model; documented in docs/api/secret_gate.md.
3322 let content = "see the docs for auth setup: \
3323 internal/workspaces/20260701/cloud-rebuild/R1-repo-audit.md";
3324 assert!(
3325 check(content).is_err(),
3326 "accepted-FP contract changed: attributive trigger before a \
3327 delimited path no longer blocks — update the docs if deliberate"
3328 );
3329 }
3330
3331 #[test]
3332 fn allows_verb_phrase_prose_with_delimiter_before_trigger_word() {
3333 // Past-participle content words are verb-phrase evidence: the clause
3334 // narrates an action on the value instead of labeling it. These are
3335 // the false-positive shapes the exemptions exist for, and they must
3336 // survive the marker-attached-delimiter and glue-word widenings.
3337 let revision = "d362950a3c9b1a4cb47d97f1623e38f1a1e6bcdf";
3338 for content in [
3339 format!("one extra token was introduced by sha: {revision}"),
3340 format!("the api key was rotated. deploy notes reference commit {revision}"),
3341 format!("api key updated: commit {revision}"),
3342 ] {
3343 assert!(
3344 check(&content).is_ok(),
3345 "verb-phrase prose must keep the VCS exemption: {content:?}, \
3346 got {:?}",
3347 scan(&content)
3348 );
3349 }
3350 }
3351
3352 #[test]
3353 fn allows_unlabeled_unknown_connector_without_delimiter() {
3354 // VCS coordinates retain the strict no-delimiter tier: an identifier
3355 // outside the connector set ends the walk. The bounded bridge used by
3356 // file paths must not re-block this ordinary revision prose.
3357 let revision = "d362950a3c9b1a4cb47d97f1623e38f1a1e6bcdf";
3358 for content in [
3359 format!("the key changes are in commit {revision}"),
3360 format!("deployed at commit {revision}"),
3361 ] {
3362 assert!(
3363 check(&content).is_ok(),
3364 "prose without a value delimiter must keep the VCS exemption: \
3365 {content:?}, got {:?}",
3366 scan(&content)
3367 );
3368 }
3369 }
3370
3371 #[test]
3372 fn blocks_direct_no_delimiter_labels_for_both_file_path_value_families() {
3373 let values = [
3374 "Xk9mZ2vQpLrT8nJwYuA/HfBsDcGiONvMabcdefgh",
3375 "internal/workspaces/20260701/cloud-rebuild/R1-repo-audit.md",
3376 ];
3377
3378 for value in values {
3379 for label in ["api key found", "auth scanner found", "secret note"] {
3380 let content = format!("{label} {value}");
3381 assert!(
3382 check(&content).is_err(),
3383 "a direct no-delimiter label must refuse the file-path exemption: \
3384 {content:?}, got {:?}",
3385 scan(&content)
3386 );
3387 }
3388 }
3389 }
3390
3391 #[test]
3392 fn blocks_direct_gerund_and_see_labels_for_both_file_path_value_families() {
3393 let values = [
3394 "Xk9mZ2vQpLrT8nJwYuA/HfBsDcGiONvMabcdefgh",
3395 "internal/workspaces/20260701/cloud-rebuild/R1-repo-audit.md",
3396 ];
3397
3398 for value in values {
3399 for label in [
3400 "api key handling",
3401 "auth scanner handling",
3402 "api key see",
3403 "key: see",
3404 ] {
3405 let content = format!("{label} {value}");
3406 assert!(
3407 check(&content).is_err(),
3408 "narrative-looking adjacency must not shield a direct credential label: \
3409 {content:?}, got {:?}",
3410 scan(&content)
3411 );
3412 }
3413 }
3414 }
3415
3416 #[test]
3417 fn blocks_direct_participle_labels_for_both_file_path_value_families() {
3418 let values = [
3419 "Xk9mZ2vQpLrT8nJwYuA/HfBsDcGiONvMabcdefgh",
3420 "internal/workspaces/20260701/cloud-rebuild/R1-repo-audit.md",
3421 ];
3422
3423 for value in values {
3424 for label in ["api key leaked", "auth scanner leaked"] {
3425 let content = format!("{label} {value}");
3426 assert!(
3427 check(&content).is_err(),
3428 "step-zero participle adjacency must not shield a direct credential label: \
3429 {content:?}, got {:?}",
3430 scan(&content)
3431 );
3432 }
3433 }
3434 }
3435
3436 #[test]
3437 fn allows_no_delimiter_narrative_file_path_prose() {
3438 // The value-side `file`/`this` identifiers provide real walk context
3439 // before `flagged`; the step-zero direct-label tightening must not
3440 // remove this required narrative exemption.
3441 let content = "the auth scanner flagged this file \
3442 internal/workspaces/20260701/cloud-rebuild/R1-repo-audit.md";
3443
3444 assert!(
3445 check(content).is_ok(),
3446 "a regular participle in narrative position must preserve the path: got {:?}",
3447 scan(content)
3448 );
3449 }
3450
3451 #[test]
3452 fn suffix_word_hundred_is_not_a_narrative_participle() {
3453 assert!(is_clause_narrative_participle("flagged"));
3454 assert!(is_clause_narrative_participle("INTRODUCED"));
3455 assert!(!is_clause_narrative_participle("found"));
3456 assert!(!is_clause_narrative_participle("hundred"));
3457 assert!(is_clause_narrative_gerund("HANDLING"));
3458 assert!(!is_clause_narrative_gerund("found"));
3459
3460 for value in [
3461 "Xk9mZ2vQpLrT8nJwYuA/HfBsDcGiONvMabcdefgh",
3462 "internal/workspaces/20260701/cloud-rebuild/R1-repo-audit.md",
3463 ] {
3464 let content = format!("api key hundred: {value}");
3465 assert!(
3466 check(&content).is_err(),
3467 "a lexical -ed suffix must not shield a credential label: \
3468 {content:?}, got {:?}",
3469 scan(&content)
3470 );
3471 }
3472 }
3473
3474 #[test]
3475 fn allows_vcs_reference_with_prose_boundary_before_trigger_word() {
3476 // A sentence or paragraph boundary between a trigger word and the
3477 // marker/value clause means the trigger is prose context, not this
3478 // value's label — the clause walk must stop at the boundary.
3479 let revision = "d362950a3c9b1a4cb47d97f1623e38f1a1e6bcdf";
3480 for content in [
3481 format!("configuration key\n\nrev {revision}."),
3482 format!("rotate the api key. the fix is commit {revision}"),
3483 ] {
3484 assert!(
3485 check(&content).is_ok(),
3486 "boundary-separated trigger prose must not block a VCS \
3487 reference: {content:?}, got {:?}",
3488 scan(&content)
3489 );
3490 }
3491 }
3492
3493 #[test]
3494 fn blocks_sha_revision_marker_immediately_labeled_as_api_key() {
3495 let revision = "d362950a3c9b1a4cb47d97f1623e38f1a1e6bcdf";
3496 let content = format!("api_key sha:{revision}");
3497
3498 assert!(
3499 check(&content).is_err(),
3500 "credential-labeled SHA value must remain blocked: {content:?}"
3501 );
3502 }
3503
3504 #[test]
3505 fn blocks_rev_revision_marker_immediately_labeled_as_token() {
3506 let revision = "d362950a3c9b1a4cb47d97f1623e38f1a1e6bcdf";
3507 let content = format!("token rev:{revision}");
3508
3509 assert!(
3510 check(&content).is_err(),
3511 "credential-labeled revision value must remain blocked: {content:?}"
3512 );
3513 }
3514
3515 #[test]
3516 fn hex64_near_hash_context_allowed() {
3517 // A 64-char hex near "sha256" or "hash" — with no credential trigger —
3518 // must be allowed (content digest in normal prose).
3519 let hex64 = "1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b";
3520 let sha_line = format!("sha256: {hex64}");
3521 let hash_line = format!("hash value {hex64}");
3522 assert!(
3523 scan(&sha_line).is_none(),
3524 "hex64 near 'sha256' must be allowed; fired: {:?}",
3525 scan(&sha_line)
3526 );
3527 assert!(
3528 scan(&hash_line).is_none(),
3529 "hex64 near 'hash' must be allowed; fired: {:?}",
3530 scan(&hash_line)
3531 );
3532 }
3533
3534 #[test]
3535 fn blocks_high_entropy_hex_like_token_near_key() {
3536 // A token whose character set exceeds pure hex (contains mixed-case, digits,
3537 // and non-hex chars) that ALSO passes `is_pure_hex = false` AND has high
3538 // entropy AND appears near "key" MUST be caught. This is the realistic
3539 // real-world case: hex-looking API tokens often mix case and non-hex chars.
3540 // Example: a 32-char mixed-charset token near "api key".
3541 let mixed = "Xk9mZ2vQpLrT8nJwYuAeHfBsDcGiONvM"; // gitleaks:allow — not pure hex
3542 assert!(!is_pure_hex(mixed), "test token must not be pure hex");
3543 let line = format!("api key {mixed}");
3544 assert!(
3545 scan(&line).is_some(),
3546 "mixed-charset high-entropy token near 'api key' must be caught; got: {:?}",
3547 scan(&line)
3548 );
3549 }
3550
3551 #[test]
3552 fn allows_hex40_without_trigger() {
3553 // 40-char hex string in a neutral context (no trigger word) must still pass —
3554 // it's likely a git commit SHA or content hash.
3555 let hex40 = "da39a3ee5e6b4b0d3255bfef95601890afd80709";
3556 let line = format!("commit: {hex40}");
3557 assert!(
3558 scan(&line).is_none(),
3559 "40-char hex without trigger word must pass; fired: {:?}",
3560 scan(&line)
3561 );
3562 }
3563
3564 // ── check_json scans object keys ─────────────────────────────────────────
3565
3566 #[test]
3567 fn check_json_blocks_secret_in_object_key() {
3568 // A credential used as a JSON object key (not a value) must be caught.
3569 let props = serde_json::json!({ "ghp_FakeGitHubToken0000000000000000000": "redacted" }); // gitleaks:allow
3570 assert!(
3571 check_json(&props).is_err(),
3572 "credential as JSON object key must be blocked"
3573 );
3574 }
3575
3576 #[test]
3577 fn check_json_blocks_nested_secret_key() {
3578 // Nested credential key must be caught.
3579 let props = serde_json::json!({
3580 "metadata": {
3581 "AKIAFAKEKEY000000000": "value" // gitleaks:allow
3582 }
3583 });
3584 assert!(
3585 check_json(&props).is_err(),
3586 "nested credential as JSON object key must be blocked"
3587 );
3588 }
3589
3590 // ── PEM masking format ───────────────────────────────────────────────────
3591
3592 #[test]
3593 fn pem_masked_excerpt_reflects_block_length_not_rest_of_string() {
3594 let header = ["-----BEGIN RSA", " PRIVATE KEY-----"].concat(); // gitleaks:allow
3595 let fake = format!(
3596 "{}\nMIIEo\u{2026}\n-----END RSA PRIVATE KEY-----\nsome trailing text that is very long",
3597 header
3598 );
3599 let m = scan(&fake).unwrap();
3600 assert_eq!(m.detector, "pem-private-key");
3601 // The masked length should reflect only the key block, not the whole string.
3602 // "some trailing text that is very long" is ~37 chars; total string is much longer.
3603 // The block ends after "-----END RSA PRIVATE KEY-----\n".
3604 // We just verify it is shorter than the full string length.
3605 let full_len = fake.chars().count();
3606 let reported_len: usize = m
3607 .masked
3608 .trim_end_matches("chars")
3609 .rsplit("...")
3610 .next()
3611 .and_then(|s| s.parse().ok())
3612 .unwrap_or(full_len + 1);
3613 assert!(
3614 reported_len < full_len,
3615 "masked length ({reported_len}) should be less than full string length ({full_len})"
3616 );
3617 }
3618
3619 // ── UTF-8 char-boundary reproduction tests ───────────────────────────────
3620 //
3621 // These tests verify that no code path in secret_gate panics when multibyte
3622 // UTF-8 characters (emoji, CJK, accented Latin) appear at positions where
3623 // byte-level slicing could land mid-codepoint. Each test targets a specific
3624 // code path. A panic means the bug is live; a pass means the path is safe.
3625
3626 /// `build_match` masked preview: if the detected candidate starts with
3627 /// multibyte chars the "first 6 chars" preview must not slice on a byte
3628 /// boundary that falls mid-codepoint. build_match already uses
3629 /// `chars().take(6)`, but we exercise it with emoji-prefixed candidates.
3630 #[test]
3631 fn utf8_build_match_preview_multibyte_prefix_no_panic() {
3632 // "🔑" = 4 bytes; repeat 3 times = 12 bytes for only 3 chars.
3633 // A ghp_-prefixed token with an emoji: let's construct a scenario where
3634 // a known-prefix secret is immediately adjacent to multibyte content so
3635 // that build_match receives a slice starting at a multibyte char.
3636 // PEM block with multibyte chars in the body exercises build_match on a
3637 // candidate that may contain non-ASCII.
3638 let header = ["-----BEGIN RSA", " PRIVATE KEY-----"].concat(); // gitleaks:allow
3639 let fake = format!("{}\n🔑密钥\n-----END RSA PRIVATE KEY-----", header);
3640 // Must not panic; mask must not echo full body.
3641 let m = scan(&fake);
3642 assert!(m.is_some(), "PEM with emoji body must still be caught");
3643 let m = m.unwrap();
3644 assert!(
3645 !m.masked.contains("🔑密钥"),
3646 "mask must not echo the emoji body"
3647 );
3648 }
3649
3650 /// `extract_token` called with a string starting with multibyte chars:
3651 /// the FlyV1 handler calls `extract_token(&text[payload_start..])` where
3652 /// `payload_start` is just past "FlyV1 " (ASCII). If the payload is ASCII
3653 /// this is trivially safe, but we verify it cannot panic when the rest of
3654 /// the text after the payload contains multibyte chars.
3655 #[test]
3656 fn utf8_extract_token_multibyte_suffix_no_panic() {
3657 // "FlyV1 ABCDEFGHIJ密钥" — the payload is "ABCDEFGHIJ密钥"; extract_token
3658 // must stop at the ideographic chars (which are NOT ASCII whitespace) and
3659 // return the whole glued run without panicking.
3660 let text = "FlyV1 ABCDEFGHIJ密钥";
3661 // scan() must not panic.
3662 let _ = scan(text);
3663 }
3664
3665 /// `find_prefix_token` with multibyte chars immediately before and after
3666 /// the known prefix: checks text[..abs] boundary slices and
3667 /// extract_token(&text[abs..]) do not panic.
3668 #[test]
3669 fn utf8_prefix_detector_multibyte_adjacent_no_panic() {
3670 // 🔑 (4 bytes) immediately before AKIA: boundary at abs = 4, which is a
3671 // valid char boundary (end of the emoji). extract_token sees ASCII from abs.
3672 let text = "🔑AKIAFAKEKEY00000000000000";
3673 let _ = scan(text); // must not panic
3674
3675 // é (U+00E9 = 2 bytes) immediately before ghp_:
3676 let text2 = "éghp_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
3677 let _ = scan(text2); // must not panic
3678
3679 // Emoji immediately after the token — extract_token ends at the emoji
3680 // (non-whitespace, but non-ASCII acts as delimiter in entropy heuristic).
3681 // For prefix tokens extract_token stops at ASCII whitespace only, so the
3682 // emoji would be included in the token length measurement.
3683 let text3 = "AKIAFAKEKEY00000000000000🔑";
3684 let _ = scan(text3); // must not panic
3685 }
3686
3687 /// `find_jwt` with multibyte chars as "whitespace" adjacent to a JWT-like
3688 /// candidate: `i = end + 1` could skip into a multibyte char if `end`
3689 /// pointed at a non-ASCII byte. The position() search only looks for ASCII
3690 /// whitespace bytes, so a multibyte space (U+3000) is NOT found — `end`
3691 /// equals bytes.len() and `i = bytes.len() + 1` exits the loop. Still
3692 /// verify no panic on CJK-surrounded JWT-like content.
3693 #[test]
3694 fn utf8_jwt_multibyte_adjacent_no_panic() {
3695 // A (fake) JWT-like triple surrounded by CJK text.
3696 let jwt = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.FAKE_SIG_XXXXXXXXXXXX"; // gitleaks:allow
3697 let text = format!("数据{jwt}密钥");
3698 let _ = scan(&text); // must not panic
3699
3700 // JWT followed by ideographic space (U+3000 = 3 bytes 0xE3 0x80 0x80) —
3701 // not matched by the ASCII-whitespace position() search.
3702 let text2 = format!("{jwt}\u{3000}morecontent");
3703 let _ = scan(&text2); // must not panic
3704
3705 // JWT followed by emoji
3706 let text3 = format!("{jwt}🔑");
3707 let _ = scan(&text3); // must not panic
3708 }
3709
3710 /// `find_url_userinfo` with multibyte chars between "://" and "@":
3711 /// `at_pos` from `rest.find('@')` and `colon` from `userinfo.find(':')` are
3712 /// ASCII markers (char boundaries), but `scheme_start` calculation uses
3713 /// char_indices().rev() which must handle multibyte chars in the scheme
3714 /// prefix correctly.
3715 #[test]
3716 fn utf8_url_userinfo_multibyte_scheme_no_panic() {
3717 // CJK glued to a credential URL — the scheme_start walker must not place
3718 // the start inside a multibyte codepoint.
3719 let cases = [
3720 "🔑postgresql://dbuser:S3cr3tP4ss@db.example.com/db", // gitleaks:allow
3721 "密钥mysql://root:hunter2pw@10.0.0.1:3306/app", // gitleaks:allow
3722 "éredis://svc:V3ryS3cretPw@cache.internal:6379", // gitleaks:allow
3723 ];
3724 for text in &cases {
3725 // Must not panic and must detect the credential.
3726 let result = scan(text);
3727 assert!(
3728 result.is_some(),
3729 "URL credential after multibyte must be caught: {text:?}"
3730 );
3731 }
3732 }
3733
3734 /// `check_entropy_heuristic` window slicing with multibyte content at the
3735 /// ±TRIGGER_WINDOW boundary: `floor_char_boundary` must prevent slicing
3736 /// on a non-char boundary.
3737 #[test]
3738 fn utf8_entropy_window_multibyte_boundary_no_panic() {
3739 // Construct content where the TRIGGER_WINDOW (120 bytes) boundary falls
3740 // inside a 3-byte CJK character. Repeat "数" (U+6570 = 3 bytes) to fill
3741 // exactly 119 bytes, then add an ASCII trigger word + high-entropy token.
3742 // Window start: token_offset - 120 = lands inside one of the CJK chars.
3743 let cjk_fill = "数".repeat(39); // 39 × 3 = 117 bytes
3744 assert_eq!(cjk_fill.len(), 117);
3745 // Pad with 2 more ASCII chars ("xy") so that the 120-byte window lands at
3746 // byte 119 which is the second byte of the 40th "数" — mid-multibyte.
3747 let secret = "Xk9mZ2vQpLrT8nJwYuAeHfBsDcGiONvM1"; // gitleaks:allow
3748 let content = format!("{cjk_fill}xy key {secret}");
3749 let _ = scan(&content); // must not panic
3750
3751 // Also test the right edge: token ends at byte offset, window_end =
3752 // token_offset + raw_token.len() + 120 may land mid-multibyte.
3753 let content2 = format!("key {secret}{cjk_fill}xy");
3754 let _ = scan(&content2); // must not panic
3755 }
3756
3757 /// `check()` top-level fuzz: a large batch of inputs with multibyte
3758 /// characters at various offsets to catch any remaining panic sites.
3759 /// All results must be either Ok or Err (not a panic).
3760 #[test]
3761 fn utf8_no_panic_property_test() {
3762 let multibyte_items = [
3763 "🔑", // 4-byte emoji
3764 "密", // 3-byte CJK
3765 "é", // 2-byte accented Latin
3766 "\u{3000}", // 3-byte ideographic space
3767 "🇺🇸", // 8-byte emoji flag (two surrogate-like scalars)
3768 ];
3769 let secrets = [
3770 "AKIAFAKEKEY00000000000000",
3771 "ghp_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
3772 "sk-ant-api03-AAAAAAAAAAAAAAA",
3773 "Xk9mZ2vQpLrT8nJwYuAeHfBsDcGiONvM1",
3774 "FlyV1 fm2_AAAABBBBCCCCDDDDEEEEFFFF",
3775 ];
3776 for mb in &multibyte_items {
3777 for secret in &secrets {
3778 for sep in &["", " ", "\n"] {
3779 // multibyte before secret
3780 let s = format!("{mb}{sep}{secret}");
3781 let _ = check(&s);
3782 // multibyte after secret
3783 let s = format!("{secret}{sep}{mb}");
3784 let _ = check(&s);
3785 // multibyte both sides
3786 let s = format!("{mb}{sep}{secret}{sep}{mb}");
3787 let _ = check(&s);
3788 // repeated multibyte filling TRIGGER_WINDOW boundary
3789 let fill = mb.repeat(50);
3790 let s = format!("{fill} api_key {secret} {fill}");
3791 let _ = check(&s);
3792 }
3793 }
3794 }
3795 }
3796
3797 // ── mask_secrets: in-place redaction reusing the canonical detector ───────
3798
3799 #[test]
3800 fn mask_secrets_borrows_clean_text() {
3801 let clean = "The FlashAttention paper introduces IO-aware tiling.";
3802 let masked = mask_secrets(clean);
3803 assert!(
3804 matches!(masked, std::borrow::Cow::Borrowed(_)),
3805 "clean text must not allocate"
3806 );
3807 assert_eq!(masked, clean);
3808 }
3809
3810 #[test]
3811 fn mask_secrets_redacts_shapes_the_old_mirror_regex_missed() {
3812 // These are exactly the detectors the session mirror's previous local
3813 // regex did NOT cover, which is why it now shares this masker.
3814 let cases = [
3815 "key: sk-proj-FAKEKEY00000000000000000000000000000000", // gitleaks:allow
3816 "cred ASIAFAKEKEY00000000000", // gitleaks:allow
3817 "stripe sk_live_FAKESTRIPE0000000000000", // gitleaks:allow
3818 "db postgresql://dbuser:S3cr3tP4ss@db.example.com/db", // gitleaks:allow
3819 ];
3820 for c in &cases {
3821 let masked = mask_secrets(c);
3822 assert!(
3823 masked.contains(REDACTION_MARKER),
3824 "must redact: {c:?} -> {masked:?}"
3825 );
3826 }
3827 }
3828
3829 #[test]
3830 fn mask_secrets_redacts_every_span_and_keeps_prose() {
3831 let line =
3832 "first sk-ant-api03-AAAAAAAAAAAAAAA then ghp_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA end";
3833 let masked = mask_secrets(line);
3834 assert!(
3835 !masked.contains("sk-ant-api03") && !masked.contains("ghp_AAAA"),
3836 "no secret may survive: {masked}"
3837 );
3838 assert_eq!(
3839 masked.matches(REDACTION_MARKER).count(),
3840 2,
3841 "both secrets must be redacted: {masked}"
3842 );
3843 assert!(masked.starts_with("first "), "prose preserved: {masked}");
3844 assert!(masked.ends_with(" end"), "prose preserved: {masked}");
3845 }
3846
3847 #[test]
3848 fn mask_secrets_output_passes_check() {
3849 // The masked output must itself be clean — no credential left for the
3850 // write-time gate to catch.
3851 let line = "token=ghp_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA and AKIAFAKEKEY1234567890";
3852 let masked = mask_secrets(line).into_owned();
3853 assert!(
3854 check(&masked).is_ok(),
3855 "masked output must pass the gate: {masked}"
3856 );
3857 }
3858
3859 #[test]
3860 fn mask_secrets_redacts_entropy_secret_left_of_known_secret() {
3861 // Cross-layer leftmost regression: a Layer-2 entropy secret sits to the
3862 // LEFT of a Layer-1 known-prefix secret. A scan that short-circuits on
3863 // the first known match (or returns first-by-detector-priority) would
3864 // redact `ghp_…` and copy the entropy token before it verbatim — leaking
3865 // it. `scan_match` must fold both layers through leftmost selection.
3866 let line =
3867 "secret=Xk9mZ2vQpLrT8nJwYuAeHfBsDcGiONvM and ghp_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; // gitleaks:allow
3868 let masked = mask_secrets(line).into_owned();
3869 assert!(
3870 !masked.contains("Xk9mZ2vQpLrT8nJwYuAeHfBsDcGiONvM") && !masked.contains("ghp_AAAA"),
3871 "neither the entropy secret nor the known secret may survive: {masked}"
3872 );
3873 assert_eq!(
3874 masked.matches(REDACTION_MARKER).count(),
3875 2,
3876 "both secrets must be redacted exactly once: {masked}"
3877 );
3878 assert!(
3879 check(&masked).is_ok(),
3880 "masked output must pass the gate: {masked}"
3881 );
3882 }
3883
3884 #[test]
3885 fn github_app_token_families_are_masked() {
3886 // ghu_ (user-to-server), ghs_ (server-to-server), and ghr_ (refresh)
3887 // GitHub App tokens are real credential families. They are
3888 // context-free: no trigger word needed.
3889 let cases = [
3890 "ghu_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", // gitleaks:allow
3891 "ghs_BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB", // gitleaks:allow
3892 "ghr_CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC", // gitleaks:allow
3893 ];
3894 for token in &cases {
3895 assert!(
3896 check(token).is_err(),
3897 "gate must hard-block GitHub App token {token}"
3898 );
3899 let line = format!("auth: {token} trailing");
3900 let masked = mask_secrets(&line).into_owned();
3901 assert!(
3902 !masked.contains(token),
3903 "GitHub App token must not survive masking: {masked}"
3904 );
3905 assert!(
3906 check(&masked).is_ok(),
3907 "masked output must pass the gate: {masked}"
3908 );
3909 }
3910 }
3911
3912 #[test]
3913 fn mask_secrets_redacts_entropy_token_whose_trigger_is_left_of_earlier_secret() {
3914 // The entropy detector only fires near a
3915 // trigger word. When the trigger (`api_key`) sits to the LEFT of an
3916 // earlier known-prefix secret (`ghp_…`), a masker that rescans only the
3917 // suffix after each redaction loses that context and leaks the later
3918 // high-entropy token. Spans must be discovered against the ORIGINAL text.
3919 let line =
3920 "api_key ghp_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA Xk9mZ2vQpLrT8nJwYuAeHfBsDcGiONvM1"; // gitleaks:allow
3921 let masked = mask_secrets(line).into_owned();
3922 assert!(
3923 !masked.contains("ghp_AAAA"),
3924 "the known secret must be redacted: {masked}"
3925 );
3926 assert!(
3927 !masked.contains("Xk9mZ2vQpLrT8nJwYuAeHfBsDcGiONvM1"),
3928 "the later entropy token must be redacted even though its trigger \
3929 word sits left of the earlier redaction: {masked}"
3930 );
3931 assert_eq!(
3932 masked.matches(REDACTION_MARKER).count(),
3933 2,
3934 "both secrets must be redacted exactly once: {masked}"
3935 );
3936 assert!(
3937 check(&masked).is_ok(),
3938 "masked output must pass the gate: {masked}"
3939 );
3940 }
3941
3942 // ── Structured identifiers: file paths / branch names ───────────────────
3943
3944 #[test]
3945 fn allows_high_entropy_file_path_near_secret_word() {
3946 let content =
3947 "workspace path fable-ops/ADR-DRAFT-adr079-slices234.md for the secret gate bug";
3948 assert!(
3949 check(content).is_ok(),
3950 "structured file path in technical prose must pass; got {:?}",
3951 scan(content)
3952 );
3953 }
3954
3955 #[test]
3956 fn allows_high_entropy_workspace_path_before_later_key_word() {
3957 let content =
3958 "see internal/workspaces/20260701/adr079-slices234/PACKET.md for the key behavior";
3959 assert!(
3960 check(content).is_ok(),
3961 "a path before a later topical key word must pass; got {:?}",
3962 scan(content)
3963 );
3964 }
3965
3966 #[test]
3967 fn allows_high_entropy_short_run_path_near_auth_word() {
3968 let content =
3969 "auth work saved at internal/workspaces/20260701/cloud-rebuild/R1-repo-audit.md";
3970 assert!(
3971 check(content).is_ok(),
3972 "path with a short run in technical prose must pass; got {:?}",
3973 scan(content)
3974 );
3975 }
3976
3977 #[test]
3978 fn allows_branch_and_review_filename_near_key_word() {
3979 let content =
3980 "branch feat-session-mirror pushed, see release_notes_v2.md for the key findings";
3981 assert!(
3982 check(content).is_ok(),
3983 "branch name and review filename near 'key' must not be blocked; fired: {:?}",
3984 scan(content)
3985 );
3986 }
3987
3988 #[test]
3989 fn allows_adr_doc_path_near_password_word() {
3990 let content = "password reset doc: docs/adr/ADR-055-epistemic-edge-relations.md";
3991 assert!(
3992 check(content).is_ok(),
3993 "ADR doc path near 'password' must not be blocked; fired: {:?}",
3994 scan(content)
3995 );
3996 }
3997
3998 #[test]
3999 fn allows_source_file_path_near_credential_word() {
4000 let content = "credential handling code crates/khive-pack-session/src/mirror/ingest.rs";
4001 assert!(
4002 check(content).is_ok(),
4003 "source file path near 'credential' must not be blocked; fired: {:?}",
4004 scan(content)
4005 );
4006 }
4007
4008 #[test]
4009 fn allows_long_snake_case_identifier_near_key_word() {
4010 let content = "api key handling lives in check_entropy_heuristic_impl";
4011 assert!(
4012 check(content).is_ok(),
4013 "snake_case identifier near 'key' must not be blocked; fired: {:?}",
4014 scan(content)
4015 );
4016 }
4017
4018 // ── Structured-identifier exemption: catch-suite regression ─────────────
4019
4020 #[test]
4021 fn hyphenated_random_secret_is_not_a_structured_identifier() {
4022 // Same token as `blocks_bare_base64url_43chars_near_key`: hyphenated
4023 // but not word-shaped. The second run exceeds the 24-char run cap,
4024 // and the first run's case-transition density (~0.42) exceeds the
4025 // 0.3 threshold on its own, so this must not be exempted and the
4026 // existing catch-suite test must keep blocking it.
4027 assert!(!is_structured_identifier(
4028 "wJalrXUtnFEMI-K7MDENGbPxRfiCYEXAMPLEKEYX123"
4029 ));
4030 let line = "api key wJalrXUtnFEMI-K7MDENGbPxRfiCYEXAMPLEKEYX123";
4031 assert!(
4032 scan(line).is_some(),
4033 "hyphenated random secret must still be blocked; got: {:?}",
4034 scan(line)
4035 );
4036 }
4037
4038 // ── Structured-identifier exemption: direct unit tests ───────────────────
4039
4040 #[test]
4041 fn structured_identifier_true_for_repro_paths() {
4042 let paths = [
4043 "fable-ops/ADR-DRAFT-adr079-slices234.md",
4044 "internal/workspaces/20260701/adr079-slices234/PACKET.md",
4045 "internal/workspaces/20260701/cloud-rebuild/R1-repo-audit.md",
4046 "release_notes_v2.md",
4047 "docs/adr/ADR-055-epistemic-edge-relations.md",
4048 "crates/khive-pack-session/src/mirror/ingest.rs",
4049 "check_entropy_heuristic_impl",
4050 ];
4051 for p in paths {
4052 assert!(
4053 is_structured_identifier(p),
4054 "expected structured identifier: {p}"
4055 );
4056 }
4057 }
4058
4059 #[test]
4060 fn structured_identifier_false_without_separator() {
4061 // No `/`, `-`, `_`, or `.` present — fails rule 1 outright.
4062 assert!(!is_structured_identifier(
4063 "Xk9mZ2vQpLrT8nJwYuAeHfBsDcGiONvM"
4064 ));
4065 }
4066
4067 #[test]
4068 fn structured_identifier_false_for_leetspeak_digit_interleaving() {
4069 // Digits interleaved with letters within a run (not a trailing digit
4070 // suffix) fail the `[A-Za-z]+[0-9]*` / `[0-9]+` shape check.
4071 assert!(!is_structured_identifier("S3cr3t-P4ssw0rd-t0ken-here!"));
4072 }
4073
4074 #[test]
4075 fn structured_identifier_false_for_run_over_length_cap() {
4076 // A 26-char single alphabetic run between separators fails the
4077 // 24-char per-run length cap even though it is otherwise trivially
4078 // word-shaped (uniform lowercase, zero case transitions).
4079 let long_run = "a".repeat(26);
4080 let token = format!("prefix-{long_run}-suffix");
4081 assert!(!is_structured_identifier(&token));
4082 }
4083
4084 // ── Credential-labeled structured identifiers remain blocked ────────────
4085 //
4086 // Narrower fixes that keep some exemption alive in trigger context (e.g.
4087 // requiring a trailing file-extension run, or requiring >= 2 path-shaped
4088 // runs with a low average per-run letters-only entropy) are all
4089 // attacker-defeatable: a random credential can be extension-suffixed, or
4090 // split/padded into short runs that drive each run's own entropy toward
4091 // its length ceiling (log2(run_len)), which real short path words already
4092 // sit at. Shannon entropy over an attacker-chosen run boundary cannot
4093 // distinguish "distinct letters that spell an English word" from
4094 // "distinct letters chosen adversarially" — both hit the same
4095 // log2(length) ceiling — an upper bound shared by every string of that
4096 // length, not a claim that entropy is content-independent at a fixed
4097 // length (two different N-character strings can and do score
4098 // differently; neither is distinguishable from the other by a threshold
4099 // alone once both sit near that shared ceiling). So no aggregation is
4100 // sound. Plausible file paths are therefore exempt only after their
4101 // individual runs have passed the entropy and normalized-hex checks, and
4102 // only when the path is not immediately labeled as a credential value.
4103 // The tests below guard that credential-label boundary.
4104
4105 #[test]
4106 fn blocks_separator_secret_access_key_bypass() {
4107 // Adversarial bypass shape: an AWS-secret-key-like value split into
4108 // separator-delimited word-shaped runs to dodge the entropy check.
4109 let content = "secret_access_key abcdefghij/klmnopqrst/uvwxyzabcd/efghijk";
4110 assert!(
4111 check(content).is_err(),
4112 "AWS Secret Access Key shaped bypass must be blocked: {:?}",
4113 scan(content)
4114 );
4115 }
4116
4117 #[test]
4118 fn blocks_adversarial_lowercase_only_separator_token_near_access_key() {
4119 let content = "access_key qrstuvwxyz/abcdefghij/klmnopqrst/uvwxyzab";
4120 assert!(
4121 check(content).is_err(),
4122 "lowercase-only separator-delimited high-entropy token near \
4123 'access_key' must be blocked: {:?}",
4124 scan(content)
4125 );
4126 }
4127
4128 #[test]
4129 fn blocks_adversarial_digit_and_word_mixed_token_near_api_key() {
4130 // A mix of pure-digit runs and letters-then-digits runs (both
4131 // individually word-shaped) whose combined alphabet diversity crosses
4132 // the entropy threshold.
4133 let content = "api_key attaycofrsm827/festwqjhc493/8261947350/qwikjzx982";
4134 assert!(
4135 check(content).is_err(),
4136 "digit-and-word-mixed high-entropy token near 'api_key' must be blocked: {:?}",
4137 scan(content)
4138 );
4139 }
4140
4141 #[test]
4142 fn blocks_adversarial_token_assignment_separator_delimited_secret() {
4143 let content = "token=zxkqwmvbpl/trfhysjgnc/dweiaoutkz-mnbvcxzlk";
4144 assert!(
4145 check(content).is_err(),
4146 "token= with lowercase-only separator-delimited high-entropy value \
4147 must be blocked: {:?}",
4148 scan(content)
4149 );
4150 }
4151
4152 #[test]
4153 fn blocks_extension_suffix_bypass_secret_access_key() {
4154 // A file-extension check alone would exempt this: appending `.md`
4155 // to a random credential must not bypass detection.
4156 let content = "secret_access_key abcdefghij/klmnopqrst/uvwxyzabcd/efghijk.md";
4157 assert!(
4158 check(content).is_err(),
4159 "extension-suffixed AWS Secret Access Key shaped bypass must be blocked: {:?}",
4160 scan(content)
4161 );
4162 }
4163
4164 #[test]
4165 fn blocks_extension_suffix_bypass_token_assignment() {
4166 let content = "token=zxkqwmvbpl/trfhysjgnc/dweiaoutkz-mnbvcxzlk.rs";
4167 assert!(
4168 check(content).is_err(),
4169 "extension-suffixed token= bypass must be blocked: {:?}",
4170 scan(content)
4171 );
4172 }
4173
4174 #[test]
4175 fn blocks_unsuffixed_separator_split_credential_bypasses() {
4176 let cases = [
4177 "secret_access_key abcdefghij/klmnopqrst/uvwxyzabcd/efghijk",
4178 "token=zxkqwmvbpl/trfhysjgnc/dweiaoutkz-mnbvcxzlk",
4179 ];
4180 for content in cases {
4181 assert!(
4182 check(content).is_err(),
4183 "bypass string must still be blocked: {content:?}, got: {:?}",
4184 scan(content)
4185 );
4186 }
4187 }
4188
4189 #[test]
4190 fn blocks_digit_run_suffix_bypass_attempt() {
4191 let cases = [
4192 "secret_access_key abcdefghij/klmnopqrst/uvwxyzabcd/efghijk2024",
4193 "secret_access_key abcdefghij2024/klmnopqrst/uvwxyzabcd/efghijk.md",
4194 ];
4195 for content in cases {
4196 assert!(
4197 check(content).is_err(),
4198 "digit-run-suffixed bypass attempt must be blocked: {content:?}, got: {:?}",
4199 scan(content)
4200 );
4201 }
4202 }
4203
4204 #[test]
4205 fn blocks_low_entropy_padding_run_bypass_attempts() {
4206 // A low-entropy padding run (`aaaa`) inserted before short/digit-shaped
4207 // runs would drag any AVERAGE per-run entropy signal below its
4208 // threshold. With the exemption dropped entirely, these must be
4209 // blocked purely on full-token entropy, same as any other
4210 // near-trigger high-entropy token.
4211 let cases = [
4212 "secret_access_key abcdefghij/klmnopqrst/uvwxyzabcd/efghijk/aaaa/R1.md",
4213 "token=zxkqwmvbpl/trfhysjgnc/dweiaoutkz/mnbvcxzlk/aaaa/R1.rs",
4214 "secret_access_key abcdefghij/klmnopqrst/uvwxyzabcd/efghijk/aaaa/bbbb/R1.md",
4215 "token=zxkqwmvbpl/trfhysjgnc/dweiaoutkz/mnbvcxzlk/aaaa/bbbb/R1.rs",
4216 ];
4217 for content in cases {
4218 assert!(
4219 check(content).is_err(),
4220 "padding-run bypass attempt must be blocked: {content:?}, got: {:?}",
4221 scan(content)
4222 );
4223 }
4224 }
4225
4226 #[test]
4227 fn blocks_run_splitting_bypass_attempts() {
4228 // Splitting a credential into short (4-6 char) runs drives EVERY
4229 // run's own letters-only entropy toward log2(run_len), which ordinary
4230 // short English path words already sit at or near: this is exactly
4231 // why any per-run entropy ceiling is unsound as an exemption signal.
4232 // With the exemption dropped, these are blocked on full-token entropy
4233 // regardless of run shape.
4234 let cases = [
4235 "secret_access_key abcd/efgh/ijkl/mnop/qrst/uvwx/yzab/cdef.md",
4236 "secret_access_key abcde/fghij/klmno/pqrst/uvwxy/zabcd.md",
4237 "secret_access_key abcdef/ghijkl/mnopqr/stuvwx/yzabcd.md",
4238 ];
4239 for content in cases {
4240 assert!(
4241 check(content).is_err(),
4242 "run-splitting bypass attempt must be blocked: {content:?}, got: {:?}",
4243 scan(content)
4244 );
4245 }
4246 }
4247
4248 #[test]
4249 fn blocks_separator_split_generic_hex_credential_ascii() {
4250 // Two 20-char hex runs joined by `/` — individually
4251 // below MIN_ENTROPY_LEN (24) and neither is a HEX_CREDENTIAL_LENGTHS
4252 // length on its own; the whole token is not pure hex (the `/` breaks
4253 // it) and its entropy is capped below ENTROPY_THRESHOLD by the
4254 // 17-symbol hex-plus-separator alphabet, so none of the prior checks
4255 // caught it. Concatenating the two runs (dropping the separator)
4256 // normalizes to one 40-char hex sequence — a HEX_CREDENTIAL_LENGTHS
4257 // value.
4258 let content = "api key 0123456789abcdef0123/456789abcdef01234567";
4259 assert!(
4260 check(content).is_err(),
4261 "separator-split hex credential must be blocked: got {:?}",
4262 scan(content)
4263 );
4264 }
4265
4266 #[test]
4267 fn blocks_separator_split_generic_hex_credential_unicode_separator() {
4268 // Same shape as above, but the separator is a non-ASCII character
4269 // (U+200B zero-width space) instead of `/`: the tokenizer treats
4270 // every non-ASCII character as a delimiter (see the tokenizer
4271 // comment in `check_entropy_heuristic`), so this splits the payload
4272 // into TWO tokens rather than leaving it inside one — the
4273 // intra-token concatenation above never sees both halves together.
4274 // The adjacent-token bridge must catch it the same way.
4275 let content = "api key 0123456789abcdef0123\u{200B}456789abcdef01234567";
4276 assert!(
4277 check(content).is_err(),
4278 "Unicode-separator split hex credential must be blocked: got {:?}",
4279 scan(content)
4280 );
4281 }
4282
4283 #[test]
4284 fn blocks_unicode_split_hex_credential_with_path_shaped_anchor() {
4285 let content =
4286 "api key handling uses source/x/0123456789abcdef0123\u{200B}456789abcdef01234567";
4287 let tokens: Vec<(usize, &str)> = content
4288 .split(|c: char| c.is_ascii_whitespace() || !c.is_ascii())
4289 .filter(|token| !token.is_empty())
4290 .map(|token| (token.as_ptr() as usize - content.as_ptr() as usize, token))
4291 .collect();
4292 let anchor = tokens
4293 .iter()
4294 .position(|(_, token)| token.starts_with("source/"))
4295 .expect("path anchor must be tokenized");
4296 assert!(is_plausible_file_path(tokens[anchor].1));
4297 let fragments = bridge_fragment_chain(&tokens, content, anchor);
4298 assert_eq!(fragments.len(), 2);
4299 assert!(contains_normalized_hex_credential(&fragments.join(" ")));
4300 assert!(
4301 check(content).is_err(),
4302 "path-shaped anchor must not bypass fragment reconstruction: got {:?}",
4303 scan(content)
4304 );
4305 }
4306
4307 #[test]
4308 fn blocks_separator_split_hex_credential_repeated_unicode_gap() {
4309 // Three U+200B zero-width spaces in a row
4310 // (9 bytes) would exceed a fixed byte-length gap bound (e.g. 8 bytes),
4311 // leaving the two 20-char hex halves unbridged. The fragment-chain
4312 // bridge is bounded by fragment COUNT (MAX_BRIDGE_FRAGMENTS), not
4313 // gap byte length, so repeating the delimiter buys an attacker
4314 // nothing: the gap between the two fragments still contains zero
4315 // ASCII alphanumeric characters, so it is still one bridgeable gap
4316 // regardless of how many times the delimiter repeats inside it.
4317 let content = "api key 0123456789abcdef0123\u{200B}\u{200B}\u{200B}456789abcdef01234567";
4318 assert!(
4319 check(content).is_err(),
4320 "repeated-Unicode-gap split hex credential must be blocked: got {:?}",
4321 scan(content)
4322 );
4323 }
4324
4325 #[test]
4326 fn blocks_separator_split_three_way_hex_credential_mixed_case() {
4327 // A 40-char mixed-case hex credential split
4328 // into THREE tokens by two single-U+200B gaps. Bridging only one
4329 // adjacent pair (`idx` with `idx + 1`, or `idx` with
4330 // `idx - 1`) would never reach the full
4331 // 40 chars. `bridge_fragment_chain` walks a bounded chain in both
4332 // directions, so starting from the first fragment reconstructs all
4333 // three. `is_ascii_hexdigit` accepts both cases, so the mixed-case
4334 // split (`AAAA...` alongside lowercase `cccc...`) must still
4335 // normalize to one 40-char hex sequence.
4336 let content = "api key AAAA1111bbbb22\u{200B}22cccc3333ddd\u{200B}d4444eeee5555";
4337 assert!(
4338 check(content).is_err(),
4339 "three-way mixed-case Unicode-split hex credential must be blocked: got {:?}",
4340 scan(content)
4341 );
4342 }
4343
4344 #[test]
4345 fn blocks_separator_split_base64_like_unicode_credential() {
4346 // A base64-like credential (mixed-case
4347 // alphanumeric, not hex) split by one U+200B into two 20-char
4348 // halves. A hex-only bridge candidacy gate would only admit pure-hex
4349 // short tokens, so neither half here (mixed-case, non-hex letters
4350 // like `X`, `k`, `Z`) ever reached the near-trigger bridge checks at
4351 // all. `is_bridge_candidate` now admits any short alphanumeric
4352 // token, and the reconstructed chain is checked against the SAME
4353 // whole-token entropy decision a genuine single-token high-entropy
4354 // candidate must clear — closing the hex-only gap without widening
4355 // detection to non-alphanumeric noise.
4356 let content = "api key Xk9mZ2vQpLrT8nJwYuAe\u{200B}HfBsDcGiONvMabcdefgh";
4357 assert!(
4358 check(content).is_err(),
4359 "base64-like Unicode-split credential must be blocked: got {:?}",
4360 scan(content)
4361 );
4362 }
4363
4364 #[test]
4365 fn blocks_punctuation_glue_between_two_unicode_gaps() {
4366 // Two 20-char hex fragments separated by a
4367 // punctuation-only token (`---`) sandwiched between two U+200B
4368 // gaps. `adjacent_gap_is_bridgeable` already accepts any
4369 // non-alphanumeric GAP between tokens; before this fix, `---` was
4370 // tokenized as its own TOKEN (not gap text), failed
4371 // `is_bridge_fragment_shape` (not alphanumeric), and stopped the
4372 // walk before it ever reached the second hex fragment — the two
4373 // real fragments were never joined, contradicting the bridge's own
4374 // stated intent that delimiter-only material is transparent to
4375 // reconstruction. `is_delimiter_only_token` now lets the walk
4376 // absorb `---` as glue and continue to the fragment on its far
4377 // side, without counting it against `MAX_BRIDGE_FRAGMENTS`.
4378 let content = "api key 0123456789abcdef0123\u{200B}---\u{200B}456789abcdef01234567";
4379 assert!(
4380 check(content).is_err(),
4381 "punctuation-glue split hex credential between two Unicode gaps must be \
4382 blocked: got {:?}",
4383 scan(content)
4384 );
4385 }
4386
4387 #[test]
4388 fn allows_seven_way_hex_split_beyond_fragment_cap_documented_limitation() {
4389 // A 64-hex credential split into SEVEN
4390 // Unicode-separated fragments (each meeting MIN_BRIDGE_FRAGMENT_LEN)
4391 // exceeds MAX_BRIDGE_FRAGMENTS (6), so no chain the walk can build
4392 // ever reconstructs the full 64 chars. This is an ACCEPTED RESIDUAL
4393 // of the local-neighborhood bound, not a defect to fix here: per
4394 // ADR-096 / ADR-115 the secret gate is accidental-persistence
4395 // hygiene on a single-principal same-uid host, not defense against a
4396 // same-uid adversary hand-splitting a credential to evade it — that
4397 // adversary could write the DB directly instead. This test pins the
4398 // boundary so a future reader does not mistake it for an
4399 // unaddressed bypass.
4400 let content = "api key 012345678\u{200B}9abcdef01\u{200B}23456789a\u{200B}bcdef0123\u{200B}456789abc\u{200B}def012345\u{200B}6789abcdef";
4401 assert!(
4402 check(content).is_ok(),
4403 "seven-way hex split beyond MAX_BRIDGE_FRAGMENTS is a documented residual \
4404 limitation and must stay allowed: got {:?}",
4405 scan(content)
4406 );
4407 }
4408
4409 #[test]
4410 fn allows_six_way_sub_floor_hex_split_documented_limitation() {
4411 // A 40-hex credential split into six
4412 // Unicode-separated fragments each individually below
4413 // MIN_BRIDGE_FRAGMENT_LEN (8) — 7/7/7/7/6/6 characters. Every
4414 // fragment fails `is_bridge_candidate`, so none ever reaches
4415 // `bridge_fragment_chain` in the first place. Same accepted-residual
4416 // rationale as the seven-way split above: a same-uid adversary
4417 // splitting fragments this small to evade the gate can equally
4418 // write the DB directly. This test pins the boundary.
4419 let content = "api key 0123456\u{200B}789abcd\u{200B}ef01234\u{200B}56789ab\u{200B}cdef01\u{200B}234567";
4420 assert!(
4421 check(content).is_ok(),
4422 "six-way sub-MIN_BRIDGE_FRAGMENT_LEN hex split is a documented residual \
4423 limitation and must stay allowed: got {:?}",
4424 scan(content)
4425 );
4426 }
4427
4428 #[test]
4429 fn allows_unrelated_short_fragments_cited_near_a_trigger_word() {
4430 // False-positive guard: ordinary prose
4431 // citing two SEPARATE short hex/base64-ish identifiers (e.g. two
4432 // unrelated git SHA prefixes) near a trigger word must NOT combine
4433 // into a block just because a delimiter-only gap between them makes
4434 // them bridge-eligible. Each fragment is well under
4435 // MIN_BRIDGE_FRAGMENT_LEN's credential-length neighborhood, and the
4436 // reconstructed concatenation (16 chars) is neither a
4437 // HEX_CREDENTIAL_LENGTHS value nor at MIN_ENTROPY_LEN (24), so it
4438 // must stay allowed exactly like a real single fragment that short
4439 // would.
4440 let content = "api key: see commits abc12345, def67890 for the fix";
4441 assert!(
4442 check(content).is_ok(),
4443 "unrelated short fragments cited near a trigger word must stay allowed: \
4444 fired {:?}",
4445 scan(content)
4446 );
4447 }
4448
4449 #[test]
4450 fn allows_unrelated_short_base64_like_fragments_cited_near_a_trigger_word() {
4451 // Same guard as above, for the newly-widened non-hex/base64-like
4452 // bridge path specifically: two short mixed-case alphanumeric build
4453 // identifiers separated by a plain space near a trigger word.
4454 // Reconstructed length (12 chars) is far under MIN_ENTROPY_LEN (24),
4455 // so the generic entropy reconstruction must not fire.
4456 let content = "api key: build ids Ab3Kf9 and Xy7Lm2 do not match";
4457 assert!(
4458 check(content).is_ok(),
4459 "unrelated short base64-like fragments cited near a trigger word must \
4460 stay allowed: fired {:?}",
4461 scan(content)
4462 );
4463 }
4464
4465 #[test]
4466 fn allows_scattered_short_hex_runs_that_do_not_sum_to_a_credential_length() {
4467 // False-positive guard: short hex-looking runs
4468 // that happen to sit near a trigger word must NOT be flagged just
4469 // because they exist — only when their normalized concatenation
4470 // actually lands on a HEX_CREDENTIAL_LENGTHS value. Three
4471 // independent 8-char runs (running total 8, 16, 24) never hit
4472 // 32/40/64/128, and the whole-token entropy check that follows stays
4473 // below ENTROPY_THRESHOLD for this path-shaped content.
4474 let content = "auth config lives in abc12345/de678901/fa234567.md";
4475 assert!(
4476 check(content).is_ok(),
4477 "scattered short hex runs that never sum to a credential length \
4478 must stay allowed: fired {:?}",
4479 scan(content)
4480 );
4481 }
4482
4483 #[test]
4484 fn allows_fp_paths_whose_full_token_entropy_is_already_below_threshold() {
4485 // 4 of the 7 original FP-repro paths stay OK near a trigger word even
4486 // with NO structured-identifier exemption at all, because their own
4487 // full-token Shannon entropy already reads below ENTROPY_THRESHOLD
4488 // (4.5) — the exemption was never load-bearing for these regardless
4489 // of which version of it existed.
4490 let paths = [
4491 "release_notes_v2.md",
4492 "docs/adr/ADR-055-epistemic-edge-relations.md",
4493 "crates/khive-pack-session/src/mirror/ingest.rs",
4494 "check_entropy_heuristic_impl",
4495 ];
4496 for p in paths {
4497 let content = format!("api_key handling in {p}");
4498 assert!(
4499 check(&content).is_ok(),
4500 "{p} must stay allowed near 'api_key' (full-token entropy already \
4501 below threshold): fired {:?}",
4502 scan(&content)
4503 );
4504 }
4505 }
4506
4507 #[test]
4508 fn allows_adr_draft_path_near_trigger() {
4509 let content = "api_key handling in fable-ops/ADR-DRAFT-adr079-slices234.md";
4510 assert!(
4511 check(content).is_ok(),
4512 "ADR-DRAFT path in technical prose must pass; got {:?}",
4513 scan(content)
4514 );
4515 }
4516
4517 #[test]
4518 fn allows_workspace_packet_path_near_trigger() {
4519 let content = "api_key handling in internal/workspaces/20260701/adr079-slices234/PACKET.md";
4520 assert!(
4521 check(content).is_ok(),
4522 "workspace path in technical prose must pass; got {:?}",
4523 scan(content)
4524 );
4525 }
4526
4527 #[test]
4528 fn allows_high_entropy_repo_audit_path_near_api_key() {
4529 let content =
4530 "api_key handling in internal/workspaces/20260701/cloud-rebuild/R1-repo-audit.md";
4531 assert!(
4532 check(content).is_ok(),
4533 "repository path in technical prose must pass; got {:?}",
4534 scan(content)
4535 );
4536 }
4537
4538 #[test]
4539 fn allows_source_paths_near_ordinary_key_and_token_prose() {
4540 let contents = [
4541 "see <a/path/to/file.py>:~97-103 lists it as a real checkpoint-supplied key",
4542 "see <a/path/to/file.py>:~97-103 emits one extra token",
4543 "see /workspace/src/checkpoint_loader.rs:97-103\n\nconfiguration key behavior",
4544 "the token behavior is implemented in src/runtime/secret_gate.rs:618-914.",
4545 ];
4546 for content in contents {
4547 assert!(
4548 check(content).is_ok(),
4549 "source path in technical prose must pass: {content:?}, got {:?}",
4550 scan(content)
4551 );
4552 }
4553 }
4554
4555 #[test]
4556 fn allows_adr_authorization_path_in_markdown_ingest() {
4557 let content =
4558 "- Evidence: `docs/adr/C-ADR-007-authorization-server.md:70` defines token handling.";
4559 assert!(
4560 check(content).is_ok(),
4561 "ADR path citation in technical markdown must pass: got {:?}",
4562 scan(content)
4563 );
4564 }
4565
4566 #[test]
4567 fn blocks_markdown_ingest_bare_opaque_value_near_token() {
4568 let content = concat!(
4569 "- Review evidence for token handling: ",
4570 "Xk9mZ2vQpLrT8nJwYuAeHfBs",
4571 "DcGiONvM1qPrStUvWxYz23456789"
4572 );
4573 assert_eq!(
4574 scan(content).map(|matched| matched.detector),
4575 Some("high-entropy-token"),
4576 "bare opaque value near token must remain blocked"
4577 );
4578 }
4579
4580 // ── UUID / content-hash allowlists are prose-context only ───────────────
4581
4582 #[test]
4583 fn blocks_uuid_directly_labeled_as_api_key() {
4584 let content = "api_key 550e8400-e29b-41d4-a716-446655440000";
4585 assert!(
4586 check(content).is_err(),
4587 "UUID-shaped token labeled api_key must be blocked; got {:?}",
4588 scan(content)
4589 );
4590 }
4591
4592 #[test]
4593 fn blocks_sha256_content_hash_labeled_as_secret() {
4594 let content = "secret sha256-ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopq";
4595 assert!(
4596 check(content).is_err(),
4597 "sha256-prefixed hash labeled secret must be blocked; got {:?}",
4598 scan(content)
4599 );
4600 }
4601
4602 #[test]
4603 fn blocks_sha384_content_hash_labeled_as_api_key() {
4604 let content =
4605 "api_key sha384-ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
4606 assert!(
4607 check(content).is_err(),
4608 "sha384-prefixed hash labeled api_key must be blocked; got {:?}",
4609 scan(content)
4610 );
4611 }
4612
4613 #[test]
4614 fn blocks_sha512_content_hash_labeled_as_auth() {
4615 let content = "auth sha512-ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/ABCDEFGHIJKLMNOPQRSTUV";
4616 assert!(
4617 check(content).is_err(),
4618 "sha512-prefixed hash labeled auth must be blocked; got {:?}",
4619 scan(content)
4620 );
4621 }
4622
4623 #[test]
4624 fn allows_uuid_with_no_trigger_within_window() {
4625 // Common benign shape: a UUID (e.g. an internal record id) with no
4626 // credential trigger word anywhere in the surrounding window stays
4627 // allowed — the allowlist still applies outside trigger context.
4628 let content =
4629 "task 550e8400-e29b-41d4-a716-446655440000 was created and assigned to the team";
4630 assert!(
4631 check(content).is_ok(),
4632 "UUID with no nearby trigger word must stay allowed; got {:?}",
4633 scan(content)
4634 );
4635 }
4636
4637 #[test]
4638 fn allows_area_id_uuid_near_authorized_substring() {
4639 // An internal task `area_id` UUID field sitting within the trigger
4640 // window of the SUBSTRING "auth" inside
4641 // `authorized_write_requires_dominance` is not a genuine mention of
4642 // the word "auth": it is a pure substring collision with
4643 // "authorized". Bare trigger words match at a word boundary (see
4644 // `contains_bounded_word`), so `auth` does not match inside
4645 // `authorized`; this UUID has no trigger in its window and passes via
4646 // the ordinary out-of-context UUID allowlist.
4647 let content = "area_id: cfcea31d-6f50-4fd1-ad6d-5f160de1694c\n\n## Problem\nReduce Lion microkernel axioms. Converted authorized_write_requires_dominance from axiom to theorem.";
4648 assert!(
4649 check(content).is_ok(),
4650 "internal area_id UUID near the 'authorized' substring \
4651 (not a genuine 'auth' mention) must now pass; got {:?}",
4652 scan(content)
4653 );
4654 }
4655
4656 // ── UUID/hash value extraction from assignment and wrapper syntax ───────
4657
4658 #[test]
4659 fn blocks_uuid_glued_to_assignment_equals() {
4660 let content = "api_key=550e8400-e29b-41d4-a716-446655440000";
4661 assert!(
4662 check(content).is_err(),
4663 "UUID glued via '=' to a trigger word must be blocked; got {:?}",
4664 scan(content)
4665 );
4666 }
4667
4668 #[test]
4669 fn blocks_uuid_with_trailing_sentence_period() {
4670 let content = "api_key 550e8400-e29b-41d4-a716-446655440000.";
4671 assert!(
4672 check(content).is_err(),
4673 "UUID with a trailing sentence period near a trigger must be blocked; got {:?}",
4674 scan(content)
4675 );
4676 }
4677
4678 #[test]
4679 fn blocks_uuid_wrapped_in_parens() {
4680 let content = "api_key (550e8400-e29b-41d4-a716-446655440000)";
4681 assert!(
4682 check(content).is_err(),
4683 "UUID wrapped in parens near a trigger must be blocked; got {:?}",
4684 scan(content)
4685 );
4686 }
4687
4688 #[test]
4689 fn blocks_uuid_in_json_object() {
4690 let content = "{\"api_key\":\"550e8400-e29b-41d4-a716-446655440000\"}";
4691 assert!(
4692 check(content).is_err(),
4693 "UUID in a JSON-ish object near a trigger key must be blocked; got {:?}",
4694 scan(content)
4695 );
4696 }
4697
4698 #[test]
4699 fn blocks_content_hash_glued_to_assignment_equals() {
4700 let content = "secret=sha256-ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopq";
4701 assert!(
4702 check(content).is_err(),
4703 "sha256-prefixed hash glued via '=' to a trigger word must be blocked; \
4704 got {:?}",
4705 scan(content)
4706 );
4707 }
4708
4709 #[test]
4710 fn blocks_content_hash_with_trailing_sentence_period() {
4711 let content = "secret sha256-ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopq.";
4712 assert!(
4713 check(content).is_err(),
4714 "sha256-prefixed hash with a trailing period near a trigger must be \
4715 blocked; got {:?}",
4716 scan(content)
4717 );
4718 }
4719
4720 #[test]
4721 fn blocks_content_hash_wrapped_in_parens() {
4722 let content = "secret (sha256-ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopq)";
4723 assert!(
4724 check(content).is_err(),
4725 "sha256-prefixed hash wrapped in parens near a trigger must be blocked; \
4726 got {:?}",
4727 scan(content)
4728 );
4729 }
4730
4731 #[test]
4732 fn blocks_content_hash_in_json_object() {
4733 let content = "{\"secret\":\"sha256-ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopq\"}";
4734 assert!(
4735 check(content).is_err(),
4736 "sha256-prefixed hash in a JSON-ish object near a trigger key must be \
4737 blocked; got {:?}",
4738 scan(content)
4739 );
4740 }
4741
4742 #[test]
4743 fn allows_uuid_wrapped_in_parens_with_no_trigger_nearby() {
4744 // Control: the prose allowlist must survive for wrapper syntax when
4745 // there is no credential trigger word anywhere in the window — only
4746 // the trigger-context extraction changed, not the outside-context
4747 // allowlist itself.
4748 let content = "wrapper (550e8400-e29b-41d4-a716-446655440000) present";
4749 assert!(
4750 check(content).is_ok(),
4751 "UUID wrapped in parens with no trigger word nearby must stay allowed; \
4752 got {:?}",
4753 scan(content)
4754 );
4755 }
4756
4757 #[test]
4758 fn blocks_padded_content_hash_glued_to_assignment_with_trailing_period() {
4759 // A padded base64 value ends in its own `=`, which is also a valid
4760 // separator character — `value_candidates` must enumerate the
4761 // suffix after every `=`/`:`, not assume any single separator
4762 // position, so the true value is recovered regardless of which
4763 // separator happens to sit where.
4764 let content = "secret=sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=.";
4765 assert!(
4766 check(content).is_err(),
4767 "padded sha256 hash glued via '=' with a trailing period must be \
4768 blocked; got {:?}",
4769 scan(content)
4770 );
4771 }
4772
4773 #[test]
4774 fn blocks_padded_content_hash_in_json_object() {
4775 let content = "{\"secret\":\"sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\"}";
4776 assert!(
4777 check(content).is_err(),
4778 "padded sha256 hash in a JSON-ish object near a trigger key must be \
4779 blocked; got {:?}",
4780 scan(content)
4781 );
4782 }
4783
4784 #[test]
4785 fn blocks_uuid_when_json_label_itself_contains_colon() {
4786 // The label can itself contain the separator character
4787 // (`"api:key"` rather than `"api_key"`); the first `:` after
4788 // wrapper-stripping then lands inside the label, not at the
4789 // label/value boundary. value_candidates must still surface the
4790 // bare UUID as a later suffix candidate.
4791 let content = "{\"api:key\":\"550e8400-e29b-41d4-a716-446655440000\"}";
4792 assert!(
4793 check(content).is_err(),
4794 "UUID must be blocked even when the JSON label contains ':'; got {:?}",
4795 scan(content)
4796 );
4797 }
4798
4799 #[test]
4800 fn blocks_uuid_when_json_label_itself_contains_equals() {
4801 let content = "{\"api=key\":\"550e8400-e29b-41d4-a716-446655440000\"}";
4802 assert!(
4803 check(content).is_err(),
4804 "UUID must be blocked even when the JSON label contains '='; got {:?}",
4805 scan(content)
4806 );
4807 }
4808
4809 #[test]
4810 fn blocks_uuid_behind_doubled_assignment() {
4811 // key=label=value: the first `=` lands between two labels, not at
4812 // the true value boundary.
4813 let content = "api_key=label=550e8400-e29b-41d4-a716-446655440000"; // gitleaks:allow
4814 assert!(
4815 check(content).is_err(),
4816 "UUID must be blocked behind a doubled assignment; got {:?}",
4817 scan(content)
4818 );
4819 }
4820
4821 #[test]
4822 fn blocks_padded_content_hash_behind_doubled_assignment_equals() {
4823 let content = "secret=label=sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=.";
4824 assert!(
4825 check(content).is_err(),
4826 "padded content hash must be blocked behind a doubled '=' assignment; \
4827 got {:?}",
4828 scan(content)
4829 );
4830 }
4831
4832 #[test]
4833 fn blocks_padded_content_hash_behind_doubled_assignment_colon() {
4834 let content = "secret:label=sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=.";
4835 assert!(
4836 check(content).is_err(),
4837 "padded content hash must be blocked behind a doubled ':'+'=' \
4838 assignment; got {:?}",
4839 scan(content)
4840 );
4841 }
4842
4843 #[test]
4844 fn allows_benign_url_with_scheme_and_path_separators() {
4845 // `value_candidates`'s any-suffix semantics must not block ordinary
4846 // URLs, whose `://` and `/` characters produce several suffix
4847 // candidates but none of them are UUID- or content-hash-shaped.
4848 // Placed near a real trigger word ("key") so the check actually
4849 // exercises the trigger-context path rather than being skipped
4850 // outright.
4851 let content = "api_key endpoint=https://example.test/resource/for/testing";
4852 assert!(
4853 check(content).is_ok(),
4854 "a benign URL near a trigger word must stay allowed; got {:?}",
4855 scan(content)
4856 );
4857 }
4858
4859 // ── Trigger word-boundary matching ──────────────────────────────────────
4860
4861 #[test]
4862 fn allows_trigger_substrings_inside_benign_path_slugs() {
4863 let paths = [
4864 "docs/_archive/adr_v0/ADR-051-cli-auth-and-kg-git-workflow.md",
4865 "docs/platform/oauth-callback-docs-and-redirect-handling-v2.md",
4866 "docs/research/author-attribution-and-collaboration-notes.md",
4867 "docs/security/passwordless-authentication-overview-v3.md",
4868 "docs/platform/private_keynote-authoring-guide-v2.md",
4869 ];
4870 for path in paths {
4871 assert!(
4872 check(path).is_ok(),
4873 "trigger substring inside a benign path slug must not make the path \
4874 its own credential context: {path:?}, got {:?}",
4875 scan(path)
4876 );
4877 }
4878 }
4879
4880 #[test]
4881 fn blocks_inline_auth_assignment_with_high_entropy_value() {
4882 let content = "auth=Xk9mZ2vQpLrT8nJwYuAeHfBsDcGiONvMabcdef"; // gitleaks:allow
4883 assert!(
4884 check(content).is_err(),
4885 "auth=<high-entropy-value> must still be blocked; got {:?}",
4886 scan(content)
4887 );
4888 }
4889
4890 #[test]
4891 fn blocks_suffix_bearing_compound_credential_assignments() {
4892 let opaque = "Xk9mZ2vQpLrT8nJwYuAeHfBsDcGiONvMabcdef"; // gitleaks:allow
4893 let cases = [
4894 format!("api_keyv2={opaque}"),
4895 format!("access_keyv2={opaque}"),
4896 format!("private_keyv2={opaque}"),
4897 format!("API_KEYV2={opaque}"),
4898 format!(r#"{{"private_keyv2":"{opaque}"}}"#),
4899 ];
4900 for content in &cases {
4901 assert!(
4902 check(content).is_err(),
4903 "suffix-bearing compound credential assignment must be blocked: \
4904 {content:?}, got {:?}",
4905 scan(content)
4906 );
4907 }
4908 }
4909
4910 #[test]
4911 fn blocks_spaced_suffix_bearing_compound_credential_assignments() {
4912 let opaque = "Xk9mZ2vQpLrT8nJwYuAeHfBsDcGiONvMabcdef"; // gitleaks:allow
4913 for label in ["api_keyv2", "access_keyv2", "private_keyv2"] {
4914 let cases = [
4915 format!("{label} = {opaque}"),
4916 format!("{label} : {opaque}"),
4917 format!(r#"{{"{label}": "{opaque}"}}"#),
4918 ];
4919 for content in &cases {
4920 assert!(
4921 check(content).is_err(),
4922 "spaced suffix-bearing compound credential assignment must be \
4923 blocked: {content:?}, got {:?}",
4924 scan(content)
4925 );
4926 }
4927 }
4928 }
4929
4930 #[test]
4931 fn blocks_suffix_bearing_compound_credentials_without_assignment_separator() {
4932 let opaque = "Xk9mZ2vQpLrT8nJwYuAeHfBsDcGiONvMabcdef"; // gitleaks:allow
4933 for label in ["api_keyv2", "access_keyv2", "private_keyv2"] {
4934 let cases = [format!("{label} {opaque}"), format!("{label}{opaque}")];
4935 for content in &cases {
4936 assert!(
4937 check(content).is_err(),
4938 "suffix-bearing compound credential without an assignment separator must be \
4939 blocked: {content:?}, got {:?}",
4940 scan(content)
4941 );
4942 assert!(
4943 mask_secrets(content).contains(REDACTION_MARKER),
4944 "shared secret masker must redact separator-free compound credential: \
4945 {content:?}"
4946 );
4947 }
4948 }
4949
4950 let prefixed = "xapi_keyv2=Xk9mZ2vQpLrT8nJwYuAeHfBsDcGiONvMabcdef"; // gitleaks:allow
4951 assert!(
4952 check(prefixed).is_err(),
4953 "prefix-bearing compound credential assignment must be blocked: \
4954 {prefixed:?}, got {:?}",
4955 scan(prefixed)
4956 );
4957 }
4958
4959 #[test]
4960 fn allows_authorized_and_authentication_prose_near_uuid() {
4961 // The word-boundary fix directly: "auth" no longer matches the
4962 // substring inside "authorized"/"authentication", so ordinary prose
4963 // using those words does not poison the trigger window for a nearby
4964 // UUID or other allowlisted shape.
4965 let cases = [
4966 "authorized_write_requires_dominance was converted from axiom to theorem, id 550e8400-e29b-41d4-a716-446655440000",
4967 "authentication flow diagram lives at 550e8400-e29b-41d4-a716-446655440000",
4968 ];
4969 for content in cases {
4970 assert!(
4971 check(content).is_ok(),
4972 "'authorized'/'authentication' substring must not trigger the \
4973 entropy heuristic: {content:?}, got {:?}",
4974 scan(content)
4975 );
4976 }
4977 }
4978
4979 #[test]
4980 fn allows_turkey_monkey_keyword_prose_near_uuid() {
4981 // Other bare-word substring collisions in TRIGGER_WORDS ("key") must
4982 // likewise not fire on ordinary English words that merely contain it.
4983 let cases = [
4984 "the turkey and monkey story references id 550e8400-e29b-41d4-a716-446655440000",
4985 "keyword research doc: 550e8400-e29b-41d4-a716-446655440000",
4986 ];
4987 for content in cases {
4988 assert!(
4989 check(content).is_ok(),
4990 "'turkey'/'monkey'/'keyword' substring must not trigger the \
4991 entropy heuristic: {content:?}, got {:?}",
4992 scan(content)
4993 );
4994 }
4995 }
4996
4997 #[test]
4998 fn blocks_opaque_tokens_near_standalone_trigger_words() {
4999 // Word-boundary matching only removes SUBSTRING collisions; a genuine
5000 // standalone trigger word must still dominate exactly as before.
5001 let opaque = "Xk9mZ2vQpLrT8nJwYuAeHfBsDcGiONvMabcdef"; // gitleaks:allow
5002 let cases = [
5003 format!("auth header {opaque}"),
5004 format!("the key is {opaque}"),
5005 format!("secret value: {opaque}"),
5006 ];
5007 for content in &cases {
5008 assert!(
5009 check(content).is_err(),
5010 "a genuine standalone trigger word must still block: {content:?}, \
5011 got {:?}",
5012 scan(content)
5013 );
5014 }
5015 }
5016
5017 #[test]
5018 fn accepted_false_positive_workspace_artifact_path_behind_attributive_trigger() {
5019 // "secret gate false positive repro: <path>" carries a trigger word
5020 // in clause range ahead of a value delimiter. The clause walk no
5021 // longer caps content words after a delimiter (any cap re-admits the
5022 // chained-qualifier labeled-value bypass), so this meta-prose shape
5023 // blocks. Accepted false positive — same attributive-trigger class
5024 // as the auth-setup docs path; documented in docs/api/secret_gate.md.
5025 let content = "writing up the secret gate false positive repro: \
5026 .workspace/20260101/fix-secret-gate-trigger-false-positive/MEASUREMENT_REPORT.md";
5027 assert!(
5028 check(content).is_err(),
5029 "accepted-FP contract changed: attributive trigger before a \
5030 delimited path no longer blocks — update the docs if deliberate"
5031 );
5032 }
5033
5034 #[test]
5035 fn accepted_false_positive_archive_doc_path_behind_attributive_trigger() {
5036 // "secret scanner archive notes: <path>" — attributive trigger two
5037 // qualifiers ahead of the delimiter. Same accepted-FP class as
5038 // above; the walk cannot tell an attributive trigger from a label
5039 // head without reopening the chained-qualifier bypass.
5040 let content =
5041 "secret scanner archive notes: docs/_archive/ADR051-TenantEncryption-v2Notes.md";
5042 assert!(
5043 check(content).is_err(),
5044 "accepted-FP contract changed: attributive trigger before a \
5045 delimited path no longer blocks — update the docs if deliberate"
5046 );
5047 }
5048
5049 #[test]
5050 fn allows_absolute_path_near_standalone_auth() {
5051 let content = "the auth scanner flagged this file: /home/user/projects/workspace/SessionNotes20260107/AuthGateFollowup2.md";
5052 assert!(
5053 check(content).is_ok(),
5054 "absolute path in technical prose must pass; got {:?}",
5055 scan(content)
5056 );
5057 }
5058
5059 #[test]
5060 fn blocks_assignment_shaped_credential_disguised_as_path_near_api_key() {
5061 // Adversarial negative: a credential-shaped value glued via '='
5062 // directly to a trigger word must not be exempted just because it is
5063 // path-shaped (separator-delimited, word-shaped runs) and looks
5064 // superficially like the technical paths above. The compound label
5065 // `api_key` makes this a credential value, so it must block.
5066 let content = "api_key=/home/user/workspaces/2026/topic-name-example/SECRET_VALUE_HERE.md";
5067 assert!(
5068 check(content).is_err(),
5069 "assignment-shaped credential disguised as a path must still be \
5070 blocked: {content:?}, got {:?}",
5071 scan(content)
5072 );
5073 }
5074
5075 #[test]
5076 fn blocks_separator_split_secret_access_key_compound() {
5077 // Adversarial negative: a separator-split bypass shape must still be
5078 // blocked. The `secret` and `key` entries both match because underscore
5079 // is a boundary for bare `TRIGGER_WORDS`. This asserts the end-to-end
5080 // outcome.
5081 let content = "secret_access_key abcdefghij/klmnopqrst/uvwxyzabcd/efghijk.md";
5082 assert!(
5083 check(content).is_err(),
5084 "secret_access_key bypass shape must still be blocked: {content:?}, \
5085 got {:?}",
5086 scan(content)
5087 );
5088 }
5089
5090 // ── Underscore is a BOUNDARY for bare TRIGGER_WORDS, not a continuation ─
5091 //
5092 // Bare TRIGGER_WORDS are word-boundary-aware, but underscore must be
5093 // treated as a boundary rather than a word character (continuation) for
5094 // this set specifically: the opposite of `has_standalone_token`'s rule
5095 // for `token`. Treating underscore as a continuation would silently drop
5096 // detection of extremely common underscore-joined credential-config
5097 // compounds (`SECRET_KEY=`, `auth_token=`, `signing_key=`,
5098 // `session_secret_...`), since `secret`/`key`/`auth` would never be
5099 // bounded by `_` under that rule. Fixed by treating `_` as a boundary for
5100 // the bare `TRIGGER_WORDS` check specifically (see `contains_word`'s
5101 // `underscore_is_word_char` parameter), while leaving
5102 // `has_standalone_token`'s `token`-specific underscore-as-continuation
5103 // rule (the `tokenizer`/`next_token`/`token_count` exemption) unchanged.
5104
5105 #[test]
5106 fn blocks_secret_key_assignment_when_underscore_bounds_trigger() {
5107 // `SECRET_KEY=<value>` must block via the plain-substring `secret`
5108 // trigger even though `secret` is followed by `_` rather than a
5109 // non-word-char boundary.
5110 let content = "SECRET_KEY=dGhpc2lzYXNlY3JldGtleXZhbHVlMTIzNDU2Nzg5MA=="; // gitleaks:allow
5111 assert!(
5112 check(content).is_err(),
5113 "SECRET_KEY=<value> (Django/Flask-style config) must still be \
5114 blocked: {content:?}, got {:?}",
5115 scan(content)
5116 );
5117 }
5118
5119 #[test]
5120 fn blocks_auth_token_assignment_when_underscore_bounds_trigger() {
5121 let opaque = "Xk9mZ2vQpLrT8nJwYuAeHfBsDcGiONvMabcdef"; // gitleaks:allow
5122 let content = format!("auth_token={opaque}");
5123 assert!(
5124 check(&content).is_err(),
5125 "auth_token=<value> must still be blocked: {content:?}, got {:?}",
5126 scan(&content)
5127 );
5128 }
5129
5130 #[test]
5131 fn blocks_underscore_joined_session_secret_and_signing_key_compounds() {
5132 let opaque = "Xk9mZ2vQpLrT8nJwYuAeHfBsDcGiONvMabcdef"; // gitleaks:allow
5133 let cases = [
5134 format!("session_secret_{opaque}"),
5135 format!("signing_key={opaque}"),
5136 ];
5137 for content in &cases {
5138 assert!(
5139 check(content).is_err(),
5140 "underscore-joined credential compound must still be blocked: \
5141 {content:?}, got {:?}",
5142 scan(content)
5143 );
5144 }
5145 }
5146
5147 #[test]
5148 fn allows_letter_joined_trigger_substrings_in_benign_prose() {
5149 // The letter-joined substring-collision exemption must survive the
5150 // underscore-as-boundary change, since that change only affects the
5151 // underscore character, not letter-joined words.
5152 let cases = [
5153 "authorized_write_requires_dominance was converted from axiom to theorem, id 550e8400-e29b-41d4-a716-446655440000",
5154 "authentication flow diagram lives at 550e8400-e29b-41d4-a716-446655440000",
5155 "the turkey and monkey story references id 550e8400-e29b-41d4-a716-446655440000",
5156 "keyword research doc: 550e8400-e29b-41d4-a716-446655440000",
5157 ];
5158 for content in cases {
5159 assert!(
5160 check(content).is_ok(),
5161 "letter-joined substring collision must stay exempt: \
5162 {content:?}, got {:?}",
5163 scan(content)
5164 );
5165 }
5166 }
5167
5168 #[test]
5169 fn block_message_carries_actionable_guidance() {
5170 let fake = "AKIAFAKEKEY1234567890";
5171 let m = scan(fake).unwrap();
5172 let rendered = m.to_string();
5173 assert!(
5174 rendered.contains("real credential"),
5175 "block message must carry actionable guidance: {rendered}"
5176 );
5177 }
5178
5179 #[test]
5180 fn block_message_shape_guidance_mentions_rewording() {
5181 let opaque = "Xk9mZ2vQpLrT8nJwYuAeHfBsDcGiONvMabcdef"; // gitleaks:allow
5182 let content = format!("auth header {opaque}");
5183 let m = scan(&content).unwrap();
5184 let rendered = m.to_string();
5185 assert!(
5186 rendered.contains("reword") || rendered.contains("own line"),
5187 "shape-based detector guidance must suggest rewording/splitting: {rendered}"
5188 );
5189 }
5190}
5191
5192// ─── Corpus replay harness (manual, opt-in) ─────────────────────────────────
5193//
5194// Measures how many real note/entity strings the gate blocks, so a detector
5195// change can be evaluated against production content rather than intuition
5196// (see the module doc). Opens the target database
5197// STRICTLY read-only (`SQLITE_OPEN_READ_ONLY`) and never mutates it. Point
5198// `KHIVE_REPLAY_DB` at a copy or a live KG database file path; the harness
5199// never writes, locks aggressively, or deletes anything.
5200//
5201// Run with: `KHIVE_REPLAY_DB=/path/to/khive.db cargo test -p khive-runtime \
5202// --release -- --ignored --nocapture corpus_replay`
5203#[cfg(test)]
5204mod corpus_replay {
5205 use super::*;
5206 use rusqlite::{Connection, OpenFlags};
5207
5208 // ── Whole-token-average entropy dilution (issue #1044, false-negative) ──
5209
5210 #[test]
5211 fn blocks_hex_credential_diluted_by_filler_path_segments() {
5212 // A real 40-char hex credential as one path segment among low-entropy
5213 // filler segments. Whole-token-average entropy is diluted below
5214 // ENTROPY_THRESHOLD, and the whole token is not pure hex (it has `/`
5215 // and `.` in it), so neither the whole-token entropy check nor the
5216 // whole-token hex-credential-token check catches it — only a per-run
5217 // check does.
5218 let line = "api key vault/9f8e7d6c5b4a39281706f5e4d3c2b1a09f8e7d6c/rotate.md";
5219 let m = scan(line);
5220 assert!(
5221 m.is_some(),
5222 "hex credential diluted by filler path segments must be blocked; got None"
5223 );
5224 assert_eq!(m.unwrap().detector, "hex-credential-token");
5225 }
5226
5227 #[test]
5228 fn blocks_chopped_secret_padded_by_filler_runs() {
5229 // A random high-entropy run planted among short filler runs. Whole-
5230 // token-average entropy is diluted below threshold by the filler.
5231 let line = "secret path a/b/c/d/Xk9mZ2vQpLrT8nJwYuAeHfBsDcGiONvM/e/f.rs"; // gitleaks:allow
5232 let m = scan(line);
5233 assert!(
5234 m.is_some(),
5235 "chopped/padded secret among filler runs must be blocked; got None"
5236 );
5237 assert_eq!(m.unwrap().detector, "high-entropy-token");
5238 }
5239
5240 #[test]
5241 fn allows_real_paths_with_no_run_reaching_min_entropy_len() {
5242 // Regression guard: the #1040 measurement corpus's real path false
5243 // positives never contain a single run >= MIN_ENTROPY_LEN, so the new
5244 // per-run check introduced for #1044 must not newly block them.
5245 let contents = [
5246 "branch feat-session-mirror pushed, see release_notes_v2.md for the key findings",
5247 "password reset doc: docs/adr/ADR-055-epistemic-edge-relations.md",
5248 "credential handling code crates/khive-pack-session/src/mirror/ingest.rs",
5249 "api key handling lives in check_entropy_heuristic_impl",
5250 ];
5251 for content in contents {
5252 assert!(
5253 check(content).is_ok(),
5254 "real path with no long run must still pass; fired: {:?}",
5255 scan(content)
5256 );
5257 }
5258 }
5259
5260 #[test]
5261 #[ignore]
5262 fn replay_against_corpus() {
5263 let db_path = std::env::var("KHIVE_REPLAY_DB")
5264 .expect("set KHIVE_REPLAY_DB=/path/to/khive.db to run the corpus replay (read-only)");
5265 let conn = Connection::open_with_flags(
5266 &db_path,
5267 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
5268 )
5269 .expect("open corpus DB read-only");
5270
5271 let mut total = 0usize;
5272 let mut blocked = 0usize;
5273 let mut samples: Vec<String> = Vec::new();
5274
5275 let mut collect = |sql: &str| {
5276 let mut stmt = conn.prepare(sql).expect("prepare replay query");
5277 let mut rows = stmt.query([]).expect("query replay rows");
5278 while let Some(row) = rows.next().expect("read replay row") {
5279 let content: Option<String> = row.get(0).unwrap_or(None);
5280 let Some(content) = content else { continue };
5281 if content.is_empty() {
5282 continue;
5283 }
5284 total += 1;
5285 if let Some(m) = scan(&content) {
5286 blocked += 1;
5287 if samples.len() < 30 {
5288 samples.push(format!(
5289 "{} :: {}",
5290 m,
5291 content.chars().take(160).collect::<String>()
5292 ));
5293 }
5294 }
5295 }
5296 };
5297
5298 collect("SELECT content FROM notes WHERE deleted_at IS NULL");
5299 collect("SELECT description FROM entities WHERE deleted_at IS NULL");
5300
5301 eprintln!("corpus replay: {blocked}/{total} strings blocked");
5302 for s in &samples {
5303 eprintln!(" BLOCKED: {s}");
5304 }
5305 }
5306
5307 /// Generates the sanitized aggregate corpus manifest checked in at
5308 /// `tests/data/secret_gate_corpus_manifest.md` (#1062): per-
5309 /// detector block counts plus a sha256 of each blocked candidate's
5310 /// content — never the candidate text itself, so the manifest carries no
5311 /// production data. This is a POINT-IN-TIME generator, not a CI check:
5312 /// re-run it manually (`KHIVE_REPLAY_DB=... cargo test -p khive-runtime
5313 /// --release -- --ignored --nocapture generate_corpus_manifest`) and
5314 /// hand-update the checked-in file when the detector set changes enough
5315 /// to warrant a fresh snapshot. `replay_against_corpus` above stays the
5316 /// human-readable spot-check with a few truncated samples; this is the
5317 /// reproducible-evidence counterpart the #1040/#1056 drop rationale in
5318 /// the PR body cites.
5319 #[test]
5320 #[ignore]
5321 fn generate_corpus_manifest() {
5322 use sha2::{Digest, Sha256};
5323 use std::collections::BTreeMap;
5324
5325 let db_path = std::env::var("KHIVE_REPLAY_DB")
5326 .expect("set KHIVE_REPLAY_DB=/path/to/khive.db to run the corpus replay (read-only)");
5327 let conn = Connection::open_with_flags(
5328 &db_path,
5329 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
5330 )
5331 .expect("open corpus DB read-only");
5332
5333 let mut total = 0usize;
5334 let mut counts_by_detector: BTreeMap<&'static str, usize> = BTreeMap::new();
5335 let mut hashes_by_detector: BTreeMap<&'static str, Vec<String>> = BTreeMap::new();
5336 let mut max_run_reaching_min_entropy_len = 0usize;
5337
5338 let mut collect = |sql: &str| {
5339 let mut stmt = conn.prepare(sql).expect("prepare replay query");
5340 let mut rows = stmt.query([]).expect("query replay rows");
5341 while let Some(row) = rows.next().expect("read replay row") {
5342 let content: Option<String> = row.get(0).unwrap_or(None);
5343 let Some(content) = content else { continue };
5344 if content.is_empty() {
5345 continue;
5346 }
5347 total += 1;
5348 // Track the #1040 soundness claim directly against the same
5349 // corpus this replay scans: the longest run any path-shaped
5350 // token contributes, so the "no real path false positive
5351 // reaches MIN_ENTROPY_LEN" claim is checked against actual
5352 // data rather than asserted.
5353 for token in content.split_whitespace() {
5354 for run in token.split(|c: char| !c.is_ascii_alphanumeric()) {
5355 max_run_reaching_min_entropy_len =
5356 max_run_reaching_min_entropy_len.max(run.len());
5357 }
5358 }
5359 if let Some(m) = scan(&content) {
5360 *counts_by_detector.entry(m.detector).or_insert(0) += 1;
5361 let hash = format!("{:x}", Sha256::digest(content.as_bytes()));
5362 hashes_by_detector.entry(m.detector).or_default().push(hash);
5363 }
5364 }
5365 };
5366
5367 collect("SELECT content FROM notes WHERE deleted_at IS NULL");
5368 collect("SELECT description FROM entities WHERE deleted_at IS NULL");
5369
5370 let blocked: usize = counts_by_detector.values().sum();
5371 println!("total_scanned: {total}");
5372 println!("total_blocked: {blocked}");
5373 println!("longest_alphanumeric_run_in_corpus: {max_run_reaching_min_entropy_len}");
5374 println!("counts_by_detector:");
5375 for (detector, count) in &counts_by_detector {
5376 println!(" {detector}: {count}");
5377 }
5378 println!("blocked_content_sha256_by_detector:");
5379 for (detector, hashes) in &hashes_by_detector {
5380 println!(" {detector}:");
5381 for hash in hashes {
5382 println!(" {hash}");
5383 }
5384 }
5385 }
5386}