keyhog_scanner/
telemetry.rs

1//! Lightweight per-scan telemetry.
2//!
3//! Two purposes:
4//!
5//! 1. **Always-on counters** for things the reporter wants to surface
6//!    even on a default run (e.g. "no secrets, but 3 example/test keys
7//!    were suppressed - pass `--dogfood` to see them"). These are
8//!    cheap atomic increments.
9//! 2. **Opt-in event capture** (`enable_dogfood()`) - the engine logs
10//!    per-decision detail so a user can answer "why didn't keyhog fire
11//!    on my fixture?" without rebuilding with debug instrumentation.
12//!
13//! Single-process CLI scans use the process-global `OnceLock<Telemetry>` as
14//! the lightest container. Long-lived daemon workers use [`ScanTelemetry`]
15//! scopes so concurrent client scans do not share counts/events.
16
17#[cfg(feature = "decode")]
18use keyhog_core::ChunkMetadata;
19use serde::{Deserialize, Serialize};
20use std::borrow::Cow;
21use std::cell::RefCell;
22use std::collections::{BTreeMap, HashSet};
23use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
24use std::sync::{Arc, Mutex, OnceLock};
25
26/// A single dogfood event. Variants are intentionally narrow - anything
27/// scanner-internal that would help a user understand a missed or
28/// suppressed credential should go here.
29#[derive(Debug, Clone, Serialize, Deserialize)]
30#[serde(tag = "kind", rename_all = "snake_case")]
31pub enum DogfoodEvent {
32    /// A credential was matched but suppressed as a known example /
33    /// placeholder (e.g. ends with `EXAMPLE`, is a sequential
34    /// placeholder, contains a `DUMMY`/`FAKE`/`MOCK` token).
35    ///
36    /// `reason` is `Cow<'static, str>` so callers can pass a literal
37    /// without allocating (`Cow::Borrowed("ends_with_EXAMPLE")`),
38    /// while the daemon-protocol deserialize path can also produce
39    /// owned values from over-the-wire JSON.
40    ExampleSuppressed {
41        detector: String,
42        path: Option<String>,
43        credential_redacted: String,
44        reason: Cow<'static, str>,
45    },
46    /// A credential was matched but suppressed by a SHAPE / heuristic / marker
47    /// gate in the suppression cascade (UUID-v4, bare-hex digest, base64 blob,
48    /// repetitive run, dashed serial, template placeholder, DUMMY/PLACEHOLDER
49    /// word, doc-marker substring, …) other than the example-token counter
50    /// path. These gates are recall-affecting: a real secret that happens to
51    /// wear a suppressed shape is dropped here, so `--dogfood` must report it
52    /// (the `--help` contract: "whether a match was made and silenced, or never
53    /// reached the engine"). `reason` is the gate name (e.g.
54    /// `Cow::Borrowed("uuid_v4_shape")`). No detector field: the suppression
55    /// cascade adjudicates on shape/markers, not detector identity, so naming a
56    /// detector here would be a guess.
57    ShapeSuppressed {
58        path: Option<String>,
59        credential_redacted: String,
60        reason: Cow<'static, str>,
61    },
62    /// A bounded static-recovery grammar recognized a candidate expression but
63    /// rejected malformed or unsupported literal data. The original source is
64    /// still scanned. No source bytes are retained in this event.
65    StaticRecoveryRejected {
66        path: Option<String>,
67        expression_offset: usize,
68        decoder: Cow<'static, str>,
69        reason: Cow<'static, str>,
70    },
71}
72
73/// Typed reasons emitted when bounded static recovery cannot evaluate a
74/// recognized JavaScript literal expression.
75#[derive(Debug, Clone, Copy, PartialEq, Eq)]
76pub(crate) enum StaticRecoveryRejection {
77    LiteralByteArrayElement,
78    JsonBase64,
79    JsonUtf8,
80    JsonByteArray,
81    XorPlaintextUtf8,
82    StringJoinJson,
83    BufferBase64,
84    BufferHex,
85    AesKeyLength,
86    AesIvLength,
87    AesCiphertextBlockLength,
88    AesPadding,
89    AesPlaintextUtf8,
90}
91
92impl StaticRecoveryRejection {
93    const ALL: [Self; 13] = [
94        Self::LiteralByteArrayElement,
95        Self::JsonBase64,
96        Self::JsonUtf8,
97        Self::JsonByteArray,
98        Self::XorPlaintextUtf8,
99        Self::StringJoinJson,
100        Self::BufferBase64,
101        Self::BufferHex,
102        Self::AesKeyLength,
103        Self::AesIvLength,
104        Self::AesCiphertextBlockLength,
105        Self::AesPadding,
106        Self::AesPlaintextUtf8,
107    ];
108
109    const fn index(self) -> usize {
110        match self {
111            Self::LiteralByteArrayElement => 0,
112            Self::JsonBase64 => 1,
113            Self::JsonUtf8 => 2,
114            Self::JsonByteArray => 3,
115            Self::XorPlaintextUtf8 => 4,
116            Self::StringJoinJson => 5,
117            Self::BufferBase64 => 6,
118            Self::BufferHex => 7,
119            Self::AesKeyLength => 8,
120            Self::AesIvLength => 9,
121            Self::AesCiphertextBlockLength => 10,
122            Self::AesPadding => 11,
123            Self::AesPlaintextUtf8 => 12,
124        }
125    }
126
127    pub(crate) const fn as_str(self) -> &'static str {
128        match self {
129            Self::LiteralByteArrayElement => "literal_byte_array_element",
130            Self::JsonBase64 => "json_base64",
131            Self::JsonUtf8 => "json_utf8",
132            Self::JsonByteArray => "json_byte_array",
133            Self::XorPlaintextUtf8 => "xor_plaintext_utf8",
134            Self::StringJoinJson => "string_join_json",
135            Self::BufferBase64 => "buffer_base64",
136            Self::BufferHex => "buffer_hex",
137            Self::AesKeyLength => "aes_key_length",
138            Self::AesIvLength => "aes_iv_length",
139            Self::AesCiphertextBlockLength => "aes_ciphertext_block_length",
140            Self::AesPadding => "aes_padding",
141            Self::AesPlaintextUtf8 => "aes_plaintext_utf8",
142        }
143    }
144}
145
146/// Maximum retained detail events per scan. Aggregate counters continue past
147/// this limit and the omitted count is surfaced in the trace.
148pub const DOGFOOD_DETAIL_EVENT_LIMIT: usize = 1024;
149
150fn record_dropped_detail(counter: &AtomicUsize) {
151    let mut current = counter.load(Ordering::Relaxed);
152    while current != usize::MAX {
153        match counter.compare_exchange_weak(
154            current,
155            current + 1,
156            Ordering::Relaxed,
157            Ordering::Relaxed,
158        ) {
159            Ok(_) => return,
160            Err(observed) => current = observed,
161        }
162    }
163}
164
165fn push_dogfood_detail(
166    events: &Mutex<Vec<DogfoodEvent>>,
167    detail_events_dropped: &AtomicUsize,
168    event: DogfoodEvent,
169) -> bool {
170    match events.lock() {
171        Ok(mut events) if events.len() < DOGFOOD_DETAIL_EVENT_LIMIT => {
172            events.push(event);
173            true
174        }
175        Ok(_) | Err(_) => {
176            // LAW10: a full or poisoned detail buffer increments the operator-visible dropped-detail counter below.
177            record_dropped_detail(detail_events_dropped);
178            false
179        }
180    }
181}
182
183fn recover_telemetry_lock<'a, T>(mutex: &'a Mutex<T>) -> std::sync::MutexGuard<'a, T> {
184    match mutex.lock() {
185        Ok(guard) => guard,
186        Err(poisoned) => {
187            let guard = poisoned.into_inner();
188            mutex.clear_poison();
189            guard
190        }
191    }
192}
193
194#[derive(Default)]
195struct StaticRecoveryTelemetry {
196    counts: [AtomicU64; StaticRecoveryRejection::ALL.len()],
197}
198
199#[derive(Debug, Clone, PartialEq, Eq, Hash)]
200enum EmittedDogfoodKey {
201    Suppression(String),
202    #[cfg(feature = "decode")]
203    StaticRecovery {
204        source_type: Arc<str>,
205        path: Option<Arc<str>>,
206        commit: Option<Arc<str>>,
207        expression_offset: usize,
208        reason: &'static str,
209    },
210}
211
212impl StaticRecoveryTelemetry {
213    fn record(&self, reason: StaticRecoveryRejection) {
214        self.add(reason, 1);
215    }
216
217    fn add(&self, reason: StaticRecoveryRejection, amount: u64) {
218        let counter = &self.counts[reason.index()];
219        let mut current = counter.load(Ordering::Relaxed);
220        while current != u64::MAX {
221            let next = current.saturating_add(amount);
222            match counter.compare_exchange_weak(current, next, Ordering::Relaxed, Ordering::Relaxed)
223            {
224                Ok(_) => return,
225                Err(observed) => current = observed,
226            }
227        }
228    }
229
230    fn snapshot(&self) -> BTreeMap<String, u64> {
231        StaticRecoveryRejection::ALL
232            .iter()
233            .filter_map(|reason| {
234                let count = self.counts[reason.index()].load(Ordering::Relaxed);
235                (count != 0).then(|| (reason.as_str().to_owned(), count))
236            })
237            .collect()
238    }
239
240    fn reset(&self) {
241        for count in &self.counts {
242            count.store(0, Ordering::Relaxed);
243        }
244    }
245}
246
247#[derive(Default)]
248struct Telemetry {
249    dogfood_enabled: AtomicBool,
250    example_suppressions: AtomicUsize,
251    events: Mutex<Vec<DogfoodEvent>>,
252    /// Namespaced keys for events already emitted by this trace. Suppression
253    /// events key by credential hash. Static recovery keys by path and reason.
254    /// The same credential is adjudicated by several pipeline stages (the
255    /// example-token gate AND a shape/weak-anchor gate can both drop the same
256    /// `AKIA…EXAMPLE` key), so without this the `--dogfood` trace emitted one
257    /// event per STAGE (duplicate noise for one logical suppression (KH-GAP-091)).
258    /// Keyed without the reason/stage so the FIRST stage to record a credential
259    /// wins and later stages are deduped; the example counter keeps its own
260    /// (reason-keyed) dedup so per-stage COUNTS are unaffected.
261    emitted_suppression_events: Mutex<HashSet<EmittedDogfoodKey>>,
262    detail_events_dropped: AtomicUsize,
263    static_recovery: StaticRecoveryTelemetry,
264}
265
266/// Per-request scanner telemetry used by daemon scan workers.
267///
268/// The regular CLI process still uses the process-global telemetry cell because
269/// it runs one scan per process. A daemon serves many client requests in one
270/// process, so each request owns one `ScanTelemetry` and installs it with
271/// [`with_scan_telemetry`] for the duration of the scan. Recorders then route
272/// counts/events into that scope instead of the process-global cell.
273#[derive(Default)]
274pub struct ScanTelemetry {
275    dogfood_enabled: AtomicBool,
276    example_suppressions: AtomicUsize,
277    events: Mutex<Vec<DogfoodEvent>>,
278    emitted_suppression_events: Mutex<HashSet<EmittedDogfoodKey>>,
279    detail_events_dropped: AtomicUsize,
280    static_recovery: StaticRecoveryTelemetry,
281}
282
283impl ScanTelemetry {
284    pub fn new() -> Self {
285        Self::default()
286    }
287
288    pub fn enable_dogfood(&self) {
289        self.dogfood_enabled.store(true, Ordering::Relaxed);
290    }
291
292    fn is_dogfood_enabled(&self) -> bool {
293        self.dogfood_enabled.load(Ordering::Relaxed)
294    }
295
296    fn example_suppression_count(&self) -> usize {
297        self.example_suppressions.load(Ordering::Relaxed)
298    }
299
300    fn drain_events(&self) -> Vec<DogfoodEvent> {
301        drain_event_buffers(&self.events, &self.emitted_suppression_events)
302    }
303
304    pub fn drain(&self) -> ScanTelemetrySnapshot {
305        ScanTelemetrySnapshot {
306            example_suppressions: self.example_suppression_count() as u64,
307            dogfood_events: self.drain_events(),
308            dogfood_detail_events_dropped: self.detail_events_dropped.load(Ordering::Relaxed)
309                as u64,
310            static_recovery_rejections: self.static_recovery.snapshot(),
311        }
312    }
313}
314
315pub struct ScanTelemetrySnapshot {
316    pub example_suppressions: u64,
317    pub dogfood_events: Vec<DogfoodEvent>,
318    pub dogfood_detail_events_dropped: u64,
319    pub static_recovery_rejections: BTreeMap<String, u64>,
320}
321
322thread_local! {
323    static CURRENT_SCAN_TELEMETRY: RefCell<Option<Arc<ScanTelemetry>>> = RefCell::new(None);
324}
325
326struct ScanTelemetryRestore {
327    previous: Option<Arc<ScanTelemetry>>,
328}
329
330impl Drop for ScanTelemetryRestore {
331    fn drop(&mut self) {
332        let previous = self.previous.take();
333        CURRENT_SCAN_TELEMETRY.with(|slot| {
334            *slot.borrow_mut() = previous;
335        });
336    }
337}
338
339/// Run `f` with `telemetry` installed for scanner telemetry recorders on this
340/// thread. Nested scopes restore the previous owner on drop, including during
341/// unwinding.
342pub fn with_scan_telemetry<R>(telemetry: &Arc<ScanTelemetry>, f: impl FnOnce() -> R) -> R {
343    let previous = CURRENT_SCAN_TELEMETRY.with(|slot| {
344        let mut slot = slot.borrow_mut();
345        slot.replace(Arc::clone(telemetry))
346    });
347    let _restore = ScanTelemetryRestore { previous };
348    f()
349}
350
351fn current_scan_telemetry() -> Option<Arc<ScanTelemetry>> {
352    CURRENT_SCAN_TELEMETRY.with(|slot| slot.borrow().clone())
353}
354
355/// Capture the request-scoped telemetry owner before dispatching work to a
356/// thread pool. Rayon workers do not inherit thread-local state automatically.
357pub(crate) fn capture_scan_telemetry() -> Option<Arc<ScanTelemetry>> {
358    current_scan_telemetry()
359}
360
361/// Install a captured request scope for one worker closure. When no request
362/// scope exists, execute directly so normal CLI scans retain the global path.
363pub(crate) fn with_captured_scan_telemetry<R>(
364    telemetry: Option<&Arc<ScanTelemetry>>,
365    f: impl FnOnce() -> R,
366) -> R {
367    match telemetry {
368        Some(telemetry) => with_scan_telemetry(telemetry, f),
369        None => f(),
370    }
371}
372
373fn current_scan_dogfood_enabled() -> Option<bool> {
374    CURRENT_SCAN_TELEMETRY.with(|slot| {
375        slot.borrow()
376            .as_ref()
377            .map(|telemetry| telemetry.is_dogfood_enabled())
378    })
379}
380
381// Global lock-free telemetry counters (KH-116)
382static FILES_SCANNED: AtomicUsize = AtomicUsize::new(0);
383static BYTES_SCANNED: AtomicUsize = AtomicUsize::new(0);
384static SKIPPED_FILES: AtomicUsize = AtomicUsize::new(0);
385static TOTAL_MATCHES: AtomicUsize = AtomicUsize::new(0);
386static GPU_DISPATCHES: AtomicUsize = AtomicUsize::new(0);
387/// Files that MATCHED a structured-format heuristic (k8s Secret, Terraform
388/// state, Jupyter notebook, docker-compose) but FAILED to parse, so the
389/// structured decode-through (e.g. base64-encoded secrets inside a k8s `data:`
390/// block) was NOT applied. The raw text is still scanned, so this is not a total
391/// miss, but credentials only reachable via the structured decode are silently
392/// lost on the offending file. Counted (not just `tracing::debug!`-logged, which
393/// is filtered out at default verbosity) so the scan can surface the coverage
394/// gap loudly at completion (Law 10).
395static STRUCTURED_PARSE_FAILURES: AtomicUsize = AtomicUsize::new(0);
396/// A chunk matched a structured decode-through format (k8s Secret /
397/// docker-compose / tfstate / Jupyter notebook) but exceeded
398/// `MAX_STRUCTURED_PARSE_BYTES`, so its structured decode-through (base64
399/// `data:` decoding) was skipped. Distinct from a parse FAILURE: the file is
400/// well-formed, just too large for the structured pass. The raw bytes are still
401/// scanned, but the regular scan does not recover base64-encoded values, so this
402/// is a real recall gap the reporter must surface (Law 10) rather than the bare
403/// `return None` that previously dropped it silently.
404static STRUCTURED_OVERSIZE_SKIPS: AtomicUsize = AtomicUsize::new(0);
405/// Decode-through work was truncated by a safety budget/cap. The raw chunk is
406/// still scanned, but secrets only reachable after an omitted recursive decode
407/// layer may be missed, so the CLI must surface this as a coverage gap.
408static DECODE_TRUNCATIONS: AtomicUsize = AtomicUsize::new(0);
409#[cfg(test)]
410thread_local! {
411    static THREAD_DECODE_TRUNCATIONS: std::cell::Cell<usize> =
412        const { std::cell::Cell::new(0) };
413}
414/// A trigger bitmap or compiled pattern-index side table referenced a pattern
415/// outside the compiled pattern bitmap. That loses phase-2 admission/expansion
416/// coverage for the affected pattern, so the operator must see the partial scan.
417static INVALID_PATTERN_INDEX_SKIPS: AtomicUsize = AtomicUsize::new(0);
418/// Cross-chunk boundary reassembly could not run because the caller supplied a
419/// result vector with different cardinality than the chunk vector.
420static BOUNDARY_RESULT_CARDINALITY_MISMATCHES: AtomicUsize = AtomicUsize::new(0);
421/// Multiline/structured reassembly produced a synthetic finding mapping whose
422/// source line was not present in the caller-provided line-offset table.
423static LINE_OFFSET_MAPPING_MISMATCHES: AtomicUsize = AtomicUsize::new(0);
424/// A configured per-chunk deadline elapsed before the scanner completed every
425/// detection and post-processing stage for that chunk.
426static CHUNK_DEADLINE_ABORTS: AtomicUsize = AtomicUsize::new(0);
427
428/// Scanner coverage gap recorded when a scanner-owned transform did not run to
429/// full coverage. These are not source skips: raw bytes still flow through the
430/// scanner, but structured/decode-only secrets may be missed.
431#[derive(Debug, Clone, Copy, PartialEq, Eq)]
432pub(crate) enum ScannerCoverageGapEvent {
433    StructuredParseFailure,
434    StructuredOversizeSkip,
435    DecodeTruncation,
436    InvalidPatternIndexSkip,
437    BoundaryResultCardinalityMismatch,
438    LineOffsetMappingMismatch,
439    ChunkDeadlineAbort,
440}
441
442impl ScannerCoverageGapEvent {
443    /// Every variant, so the per-scan reset owner (`reset_for_scan`) can zero the
444    /// full coverage-gap counter set without a new gap counter ever being forgotten.
445    pub(crate) const ALL: [Self; 7] = [
446        Self::StructuredParseFailure,
447        Self::StructuredOversizeSkip,
448        Self::DecodeTruncation,
449        Self::InvalidPatternIndexSkip,
450        Self::BoundaryResultCardinalityMismatch,
451        Self::LineOffsetMappingMismatch,
452        Self::ChunkDeadlineAbort,
453    ];
454
455    pub(crate) fn counter(self) -> &'static AtomicUsize {
456        match self {
457            Self::StructuredParseFailure => &STRUCTURED_PARSE_FAILURES,
458            Self::StructuredOversizeSkip => &STRUCTURED_OVERSIZE_SKIPS,
459            Self::DecodeTruncation => &DECODE_TRUNCATIONS,
460            Self::InvalidPatternIndexSkip => &INVALID_PATTERN_INDEX_SKIPS,
461            Self::BoundaryResultCardinalityMismatch => &BOUNDARY_RESULT_CARDINALITY_MISMATCHES,
462            Self::LineOffsetMappingMismatch => &LINE_OFFSET_MAPPING_MISMATCHES,
463            Self::ChunkDeadlineAbort => &CHUNK_DEADLINE_ABORTS,
464        }
465    }
466
467    const fn label(self) -> &'static str {
468        match self {
469            Self::StructuredParseFailure => "structured_parse_failures",
470            Self::StructuredOversizeSkip => "structured_oversize_skips",
471            Self::DecodeTruncation => "decode_truncations",
472            Self::InvalidPatternIndexSkip => "invalid_pattern_index_skips",
473            Self::BoundaryResultCardinalityMismatch => "boundary_result_cardinality_mismatches",
474            Self::LineOffsetMappingMismatch => "line_offset_mapping_mismatches",
475            Self::ChunkDeadlineAbort => "chunk_deadline_aborts",
476        }
477    }
478}
479
480/// Exact scanner-owned coverage-gap counters at one point in time.
481///
482/// Taking a saturating delta around a scan gives autoroute and other embedding
483/// surfaces a typed completeness receipt without resetting process-wide state.
484#[derive(Clone, Copy, Default, Eq, PartialEq)]
485pub struct ScannerCoverageSnapshot {
486    counts: [usize; ScannerCoverageGapEvent::ALL.len()],
487}
488
489impl ScannerCoverageSnapshot {
490    #[must_use]
491    pub fn capture() -> Self {
492        Self {
493            counts: std::array::from_fn(|index| {
494                ScannerCoverageGapEvent::ALL[index]
495                    .counter()
496                    .load(Ordering::Relaxed)
497            }),
498        }
499    }
500
501    #[must_use]
502    pub fn saturating_delta(self, earlier: Self) -> Self {
503        Self {
504            counts: std::array::from_fn(|index| {
505                self.counts[index].saturating_sub(earlier.counts[index])
506            }),
507        }
508    }
509
510    #[must_use]
511    pub fn is_empty(self) -> bool {
512        self.counts.iter().all(|count| *count == 0)
513    }
514}
515
516impl std::fmt::Debug for ScannerCoverageSnapshot {
517    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
518        let mut gaps = formatter.debug_map();
519        for (event, count) in ScannerCoverageGapEvent::ALL.into_iter().zip(self.counts) {
520            if count > 0 {
521                gaps.entry(&event.label(), &count);
522            }
523        }
524        gaps.finish()
525    }
526}
527
528/// Receipt proving a scanner coverage gap passed through the typed recorder.
529#[derive(Debug, Clone, Copy, PartialEq, Eq)]
530#[must_use = "scanner coverage gaps must be recorded through the typed recorder so partial coverage remains surfaced"]
531pub(crate) struct RecordedScannerCoverageGap {
532    event: ScannerCoverageGapEvent,
533    previous: usize,
534    delta: usize,
535}
536
537pub(crate) fn record_scanner_coverage_gap(
538    event: ScannerCoverageGapEvent,
539) -> RecordedScannerCoverageGap {
540    let previous = event.counter().fetch_add(1, Ordering::Relaxed);
541    RecordedScannerCoverageGap {
542        event,
543        previous,
544        delta: 1,
545    }
546}
547
548// Global static dogfood capability flag for fast opt-in checking (KH-120)
549static DOGFOOD_ENABLED: AtomicBool = AtomicBool::new(false);
550
551fn cell() -> &'static Telemetry {
552    static CELL: OnceLock<Telemetry> = OnceLock::new();
553    CELL.get_or_init(Telemetry::default)
554}
555
556/// Enable dogfood event capture for the current process. Idempotent.
557pub fn enable_dogfood() {
558    DOGFOOD_ENABLED.store(true, Ordering::Relaxed);
559    cell().dogfood_enabled.store(true, Ordering::Relaxed);
560}
561
562pub fn is_dogfood_enabled() -> bool {
563    if let Some(enabled) = current_scan_dogfood_enabled() {
564        return enabled;
565    }
566    DOGFOOD_ENABLED.load(Ordering::Relaxed)
567}
568
569/// Record one example/placeholder suppression. The default path is only the
570/// per-scan atomic counter; hash/lock/redaction work is reserved for opt-in
571/// `--dogfood` event capture.
572pub fn record_example_suppression(
573    detector: &str,
574    path: Option<&str>,
575    credential: &str,
576    reason: &'static str,
577) {
578    if let Some(t) = current_scan_telemetry() {
579        record_example_suppression_in(
580            &t.example_suppressions,
581            &t.events,
582            &t.emitted_suppression_events,
583            &t.detail_events_dropped,
584            detector,
585            path,
586            credential,
587            reason,
588        );
589        return;
590    }
591
592    let t = cell();
593    record_example_suppression_in(
594        &t.example_suppressions,
595        &t.events,
596        &t.emitted_suppression_events,
597        &t.detail_events_dropped,
598        detector,
599        path,
600        credential,
601        reason,
602    );
603}
604
605fn record_example_suppression_in(
606    example_suppressions: &AtomicUsize,
607    events: &Mutex<Vec<DogfoodEvent>>,
608    emitted_suppression_events: &Mutex<HashSet<EmittedDogfoodKey>>,
609    detail_events_dropped: &AtomicUsize,
610    detector: &str,
611    path: Option<&str>,
612    credential: &str,
613    reason: &'static str,
614) {
615    example_suppressions.fetch_add(1, Ordering::Relaxed);
616
617    // KH-120: Wrap dogfood logging events behind static capability flags to eliminate overhead during silent scans.
618    if !is_dogfood_enabled() {
619        return;
620    }
621
622    let credential_hash = keyhog_core::hex_encode(&keyhog_core::sha256_hash(credential));
623    // One EVENT per credential across all stages (KH-GAP-091): if a later
624    // shape gate already recorded this same credential, or vice-versa, don't emit
625    // a duplicate. First stage to reach it wins.
626    if !mark_suppression_event_emitted(
627        emitted_suppression_events,
628        detail_events_dropped,
629        &credential_hash,
630    ) {
631        return;
632    }
633
634    // KH-disc: use the single canonical redaction policy (`keyhog_core::redact`)
635    // so dogfood output matches finding output - the bespoke 6-char-prefix
636    // helper leaked up to 6 of 8 bytes of short credentials.
637    let redacted = keyhog_core::redact(credential).into_owned();
638    push_dogfood_detail(
639        events,
640        detail_events_dropped,
641        DogfoodEvent::ExampleSuppressed {
642            detector: detector.to_string(),
643            path: path.map(str::to_string),
644            credential_redacted: redacted,
645            reason: Cow::Borrowed(reason),
646        },
647    );
648}
649
650/// Insert `credential_hash` into the shared emitted-event set, returning `true`
651/// only the FIRST time a given credential VALUE is seen this scan. Both
652/// suppression recorders gate their `events.push` on this so the `--dogfood`
653/// trace carries one event per logical suppression rather than one per pipeline
654/// stage. The key is the credential hash ALONE, not `path\0hash`: because one
655/// logical drop of a credential can be recorded by several stages with
656/// INCONSISTENT path context (an early gate knows the file; a later
657/// entropy/fallback stage records `path=None`); keying on path would let those
658/// re-emit as duplicate events for the same logical suppression (KH-GAP-091).
659/// The shared detail budget bounds both this set and the event vector. Once it
660/// is exhausted, the exact dropped-detail counter remains operator-visible.
661fn mark_suppression_event_emitted(
662    emitted_suppression_events: &Mutex<HashSet<EmittedDogfoodKey>>,
663    detail_events_dropped: &AtomicUsize,
664    credential_hash: &str,
665) -> bool {
666    match emitted_suppression_events.lock() {
667        Ok(mut emitted) => {
668            let key = EmittedDogfoodKey::Suppression(credential_hash.to_owned());
669            if emitted.contains(&key) {
670                return false;
671            }
672            if emitted.len() >= DOGFOOD_DETAIL_EVENT_LIMIT {
673                record_dropped_detail(detail_events_dropped);
674                return false;
675            }
676            emitted.insert(key)
677        }
678        Err(_) => {
679            // LAW10: poisoned diagnostic dedup increments the surfaced omitted-detail counter; findings and exact aggregates remain intact.
680            record_dropped_detail(detail_events_dropped);
681            false // LAW10: poisoned diagnostic dedup is surfaced as one omitted detail; finding and exact aggregate counters are unchanged.
682        }
683    }
684}
685
686/// Record one SHAPE / heuristic suppression (UUID, bare-hex, base64 blob,
687/// repetitive run, …) for the `--dogfood` trace. Unlike
688/// [`record_example_suppression`] this is on the HOT suppression path (every
689/// candidate that hits a shape gate), so it is **zero-cost when dogfood is
690/// off**: the `is_dogfood_enabled()` atomic load short-circuits before any
691/// hashing / locking. It also does NOT bump the example-suppression counter -
692/// the reporter's "N example keys suppressed" summary stays example-only; shape
693/// drops are a `--dogfood`-only diagnostic. Dedup reuses the shared seen-set
694/// (keyed with a `shape\0` prefix so it can't collide with example keys).
695pub(crate) fn record_shape_suppression(path: Option<&str>, credential: &str, reason: &'static str) {
696    // Cheap atomic first - the common (no-dogfood) scan pays nothing beyond this.
697    if !is_dogfood_enabled() {
698        return;
699    }
700    if let Some(t) = current_scan_telemetry() {
701        record_shape_suppression_in(
702            &t.events,
703            &t.emitted_suppression_events,
704            &t.detail_events_dropped,
705            path,
706            credential,
707            reason,
708        );
709        return;
710    }
711    let t = cell();
712    record_shape_suppression_in(
713        &t.events,
714        &t.emitted_suppression_events,
715        &t.detail_events_dropped,
716        path,
717        credential,
718        reason,
719    );
720}
721
722/// Record a static-recovery rejection in the dogfood trace. Deduplication keeps
723/// repeated references to the same rejected expression from producing noise.
724#[cfg(feature = "decode")]
725pub(crate) fn record_static_recovery_rejection(
726    metadata: &ChunkMetadata,
727    expression_offset: usize,
728    reason: StaticRecoveryRejection,
729) {
730    if !is_dogfood_enabled() {
731        return;
732    }
733    if let Some(t) = current_scan_telemetry() {
734        t.static_recovery.record(reason);
735        if !mark_static_recovery_event_emitted(
736            &t.emitted_suppression_events,
737            &t.detail_events_dropped,
738            metadata,
739            expression_offset,
740            reason,
741        ) {
742            return;
743        }
744        push_dogfood_detail(
745            &t.events,
746            &t.detail_events_dropped,
747            static_recovery_event(metadata, expression_offset, reason),
748        );
749        return;
750    }
751    let t = cell();
752    t.static_recovery.record(reason);
753    if !mark_static_recovery_event_emitted(
754        &t.emitted_suppression_events,
755        &t.detail_events_dropped,
756        metadata,
757        expression_offset,
758        reason,
759    ) {
760        return;
761    }
762    push_dogfood_detail(
763        &t.events,
764        &t.detail_events_dropped,
765        static_recovery_event(metadata, expression_offset, reason),
766    );
767}
768
769#[cfg(feature = "decode")]
770fn static_recovery_event(
771    metadata: &ChunkMetadata,
772    expression_offset: usize,
773    reason: StaticRecoveryRejection,
774) -> DogfoodEvent {
775    DogfoodEvent::StaticRecoveryRejected {
776        path: metadata.path.as_deref().map(str::to_owned),
777        expression_offset,
778        decoder: Cow::Borrowed("javascript-static"),
779        reason: Cow::Borrowed(reason.as_str()),
780    }
781}
782
783#[cfg(feature = "decode")]
784fn mark_static_recovery_event_emitted(
785    emitted_events: &Mutex<HashSet<EmittedDogfoodKey>>,
786    detail_events_dropped: &AtomicUsize,
787    metadata: &ChunkMetadata,
788    expression_offset: usize,
789    reason: StaticRecoveryRejection,
790) -> bool {
791    let key = EmittedDogfoodKey::StaticRecovery {
792        source_type: Arc::clone(&metadata.source_type),
793        path: metadata.path.clone(),
794        commit: metadata.commit.clone(),
795        expression_offset,
796        reason: reason.as_str(),
797    };
798    match emitted_events.lock() {
799        Ok(mut emitted) => {
800            if emitted.contains(&key) {
801                return false;
802            }
803            if emitted.len() >= DOGFOOD_DETAIL_EVENT_LIMIT {
804                record_dropped_detail(detail_events_dropped);
805                return false;
806            }
807            emitted.insert(key)
808        }
809        Err(_) => {
810            // LAW10: poisoned diagnostic dedup increments the surfaced omitted-detail counter; findings and exact aggregates remain intact.
811            record_dropped_detail(detail_events_dropped);
812            false // LAW10: poisoned diagnostic dedup is surfaced as one omitted detail; scan findings and exact rejection counters are unchanged.
813        }
814    }
815}
816
817fn record_shape_suppression_in(
818    events: &Mutex<Vec<DogfoodEvent>>,
819    emitted_suppression_events: &Mutex<HashSet<EmittedDogfoodKey>>,
820    detail_events_dropped: &AtomicUsize,
821    path: Option<&str>,
822    credential: &str,
823    reason: &'static str,
824) {
825    let credential_hash = keyhog_core::hex_encode(&keyhog_core::sha256_hash(credential));
826    // One EVENT per credential across ALL stages (KH-GAP-091): a credential
827    // the example-token gate already recorded (e.g. `AKIA…EXAMPLE`, which is also
828    // a weak-anchor shape) must not emit a second shape event for the same
829    // logical drop. The shared emitted-set also collapses the same shape gate
830    // firing twice for one credential, so this fully replaces the old
831    // reason-keyed dedup.
832    if !mark_suppression_event_emitted(
833        emitted_suppression_events,
834        detail_events_dropped,
835        &credential_hash,
836    ) {
837        return;
838    }
839    let redacted = keyhog_core::redact(credential).into_owned();
840    push_dogfood_detail(
841        events,
842        detail_events_dropped,
843        DogfoodEvent::ShapeSuppressed {
844            path: path.map(str::to_string),
845            credential_redacted: redacted,
846            reason: Cow::Borrowed(reason),
847        },
848    );
849}
850
851/// Count of example/placeholder credentials suppressed during this scan.
852pub fn example_suppression_count() -> usize {
853    cell().example_suppressions.load(Ordering::Relaxed)
854}
855
856/// Zero the suppression counter without disturbing the dogfood
857/// enable-flag or any in-flight events. Used by the daemon between
858/// scan requests so per-request counts don't accumulate across
859/// clients - the count we ship over the wire belongs to one scan.
860#[cfg(test)]
861pub(crate) fn reset_example_suppression_count() {
862    cell().example_suppressions.store(0, Ordering::Relaxed);
863}
864
865/// Add `n` to the suppression counter without recording an event.
866/// Used by the daemon client to merge a daemon-side count into the
867/// CLI's own counter so the reporter's empty-findings summary fires
868/// correctly across the IPC boundary.
869pub fn add_example_suppressions(n: usize) {
870    cell().example_suppressions.fetch_add(n, Ordering::Relaxed);
871}
872
873/// Record that a file matched a structured-format heuristic but failed to parse,
874/// so its structured decode-through was not applied (see
875/// [`struct@STRUCTURED_PARSE_FAILURES`]). Always counts (not dogfood-gated): this
876/// is a recall-coverage fact the reporter surfaces unconditionally, like the
877/// walker skip counters.
878pub(crate) fn record_structured_parse_failure() {
879    let _receipt = record_scanner_coverage_gap(ScannerCoverageGapEvent::StructuredParseFailure);
880}
881
882/// Count of files that matched a structured format but failed to parse this scan.
883pub fn structured_parse_failure_count() -> usize {
884    STRUCTURED_PARSE_FAILURES.load(Ordering::Relaxed)
885}
886
887/// Record that a well-formed structured decode-through file (k8s Secret /
888/// docker-compose / tfstate / Jupyter notebook) exceeded
889/// `MAX_STRUCTURED_PARSE_BYTES`, so its base64 `data:` decode-through was
890/// skipped. Always counts: like a parse failure this is a recall-coverage fact
891/// the reporter surfaces unconditionally (Law 10), not a silent `return None`.
892pub(crate) fn record_structured_oversize_skip() {
893    let _receipt = record_scanner_coverage_gap(ScannerCoverageGapEvent::StructuredOversizeSkip);
894}
895
896/// Count of decode-through structured files skipped this scan for exceeding the
897/// structured-parse size cap.
898pub fn structured_oversize_skip_count() -> usize {
899    STRUCTURED_OVERSIZE_SKIPS.load(Ordering::Relaxed)
900}
901
902/// Record that recursive decode-through stopped before exhausting all available
903/// decoder output because a safety budget/cap fired.
904pub(crate) fn record_decode_truncation() {
905    let _receipt = record_scanner_coverage_gap(ScannerCoverageGapEvent::DecodeTruncation);
906    #[cfg(test)]
907    THREAD_DECODE_TRUNCATIONS.with(|count| count.set(count.get() + 1));
908}
909
910/// Count of decode roots truncated by safety budgets/caps this scan.
911#[cfg(not(test))]
912pub fn decode_truncation_count() -> usize {
913    DECODE_TRUNCATIONS.load(Ordering::Relaxed)
914}
915
916/// Count of decode roots truncated by safety budgets/caps on the current test
917/// thread. Production still records the global counter; tests read this local
918/// view so parallel decode-budget probes cannot pollute exact assertions.
919#[cfg(test)]
920pub fn decode_truncation_count() -> usize {
921    THREAD_DECODE_TRUNCATIONS.with(|count| count.get())
922}
923
924/// Record that compiled pattern-index side data referenced an out-of-range
925/// pattern and the affected expansion/admission edge had to be skipped.
926pub(crate) fn record_invalid_pattern_index_skip() {
927    let _receipt = record_scanner_coverage_gap(ScannerCoverageGapEvent::InvalidPatternIndexSkip);
928}
929
930/// Count of compiled-pattern expansion/admission edges skipped by invalid
931/// pattern indices this scan.
932pub fn invalid_pattern_index_skip_count() -> usize {
933    INVALID_PATTERN_INDEX_SKIPS.load(Ordering::Relaxed)
934}
935
936/// Record that boundary reassembly was skipped because caller-provided chunk
937/// and result slices no longer had the same cardinality.
938pub(crate) fn record_boundary_result_cardinality_mismatch() {
939    let _receipt =
940        record_scanner_coverage_gap(ScannerCoverageGapEvent::BoundaryResultCardinalityMismatch);
941}
942
943/// Count of boundary-reassembly passes skipped by chunk/result cardinality
944/// mismatch this scan.
945pub fn boundary_result_cardinality_mismatch_count() -> usize {
946    BOUNDARY_RESULT_CARDINALITY_MISMATCHES.load(Ordering::Relaxed)
947}
948
949/// Record that source line attribution fell back because a synthetic multiline
950/// mapping could not find its line in the original line-offset table.
951#[cfg(feature = "multiline")]
952pub(crate) fn record_line_offset_mapping_mismatch() {
953    let _receipt = record_scanner_coverage_gap(ScannerCoverageGapEvent::LineOffsetMappingMismatch);
954}
955
956/// Record that a configured deadline stopped a chunk before full coverage.
957pub(crate) fn record_chunk_deadline_abort() {
958    let _receipt = record_scanner_coverage_gap(ScannerCoverageGapEvent::ChunkDeadlineAbort);
959}
960
961/// Count of chunks that stopped before full coverage because their deadline elapsed.
962pub fn chunk_deadline_abort_count() -> usize {
963    CHUNK_DEADLINE_ABORTS.load(Ordering::Relaxed)
964}
965
966/// Count of synthetic multiline/structured mapping attribution mismatches this
967/// scan.
968pub fn line_offset_mapping_mismatch_count() -> usize {
969    LINE_OFFSET_MAPPING_MISMATCHES.load(Ordering::Relaxed)
970}
971
972/// Append events into the per-process buffer without going through the
973/// `record_example_suppression` path (no counter bump, no dogfood
974/// enable-check). Used by the daemon client to replay events captured
975/// on the daemon side, so `--dogfood` output works in daemon mode.
976pub fn append_events<I: IntoIterator<Item = DogfoodEvent>>(events: I) {
977    append_event_details(events, true);
978}
979
980/// Append detail events transported with exact daemon aggregates.
981///
982/// Unlike [`append_events`], this does not infer static-recovery counts from
983/// the bounded detail list. Call [`merge_daemon_aggregates`] once for the same
984/// response so retained details and exact totals cannot be double-counted.
985pub fn append_daemon_events<I: IntoIterator<Item = DogfoodEvent>>(events: I) {
986    append_event_details(events, false);
987}
988
989fn append_event_details<I: IntoIterator<Item = DogfoodEvent>>(
990    events: I,
991    infer_static_recovery_counts: bool,
992) {
993    let t = cell();
994    for event in events {
995        if infer_static_recovery_counts {
996            let DogfoodEvent::StaticRecoveryRejected { reason, .. } = &event else {
997                push_dogfood_detail(&t.events, &t.detail_events_dropped, event);
998                continue;
999            };
1000            if let Some(reason) = StaticRecoveryRejection::ALL
1001                .iter()
1002                .find(|candidate| candidate.as_str() == reason.as_ref())
1003            {
1004                t.static_recovery.record(*reason);
1005            }
1006        }
1007        push_dogfood_detail(&t.events, &t.detail_events_dropped, event);
1008    }
1009}
1010
1011/// Merge exact dogfood aggregates returned by a compatible daemon scan.
1012///
1013/// Detail events are transported separately through [`append_daemon_events`]. This
1014/// method validates every typed rejection reason before mutating process state,
1015/// so a response from an incompatible newer daemon fails instead of producing
1016/// a plausible but incomplete trace.
1017pub fn merge_daemon_aggregates(
1018    static_recovery_rejections: &BTreeMap<String, u64>,
1019    detail_events_dropped: u64,
1020) -> Result<(), String> {
1021    let mut resolved = Vec::with_capacity(static_recovery_rejections.len());
1022    for (name, count) in static_recovery_rejections {
1023        let Some(reason) = StaticRecoveryRejection::ALL
1024            .iter()
1025            .copied()
1026            .find(|candidate| candidate.as_str() == name)
1027        else {
1028            return Err(format!(
1029                "daemon returned unknown static-recovery rejection reason {name:?}; restart it with this KeyHog build"
1030            ));
1031        };
1032        resolved.push((reason, *count));
1033    }
1034
1035    let telemetry = cell();
1036    for (reason, count) in resolved {
1037        telemetry.static_recovery.add(reason, count);
1038    }
1039    let dropped = usize::try_from(detail_events_dropped).unwrap_or(usize::MAX); // LAW10: wire counts wider than this host can represent remain surfaced at the largest representable count; scan findings are unchanged.
1040    let counter = &telemetry.detail_events_dropped;
1041    let mut current = counter.load(Ordering::Relaxed);
1042    while current != usize::MAX {
1043        match counter.compare_exchange_weak(
1044            current,
1045            current.saturating_add(dropped),
1046            Ordering::Relaxed,
1047            Ordering::Relaxed,
1048        ) {
1049            Ok(_) => break,
1050            Err(observed) => current = observed,
1051        }
1052    }
1053    Ok(())
1054}
1055
1056/// Exact per-reason static-recovery rejection counts for the current process
1057/// scan. Detail-event deduplication and retention limits never change these
1058/// aggregates.
1059pub fn static_recovery_rejection_counts() -> BTreeMap<String, u64> {
1060    cell().static_recovery.snapshot()
1061}
1062
1063/// Number of dogfood detail events omitted after the bounded trace filled.
1064pub fn dogfood_detail_events_dropped() -> usize {
1065    cell().detail_events_dropped.load(Ordering::Relaxed)
1066}
1067
1068/// Drain and return all captured dogfood events. Returns empty when
1069/// `enable_dogfood()` was never called.
1070pub fn drain_events() -> Vec<DogfoodEvent> {
1071    let t = cell();
1072    drain_event_buffers(&t.events, &t.emitted_suppression_events)
1073}
1074
1075fn drain_event_buffers(
1076    events: &Mutex<Vec<DogfoodEvent>>,
1077    emitted_suppression_events: &Mutex<HashSet<EmittedDogfoodKey>>,
1078) -> Vec<DogfoodEvent> {
1079    // The drained batch is one complete trace; the next scan must be able to emit
1080    // its own events for the same credentials, so clear the per-credential
1081    // emitted-event dedup alongside the drain.
1082    recover_telemetry_lock(emitted_suppression_events).clear();
1083    std::mem::take(&mut *recover_telemetry_lock(events))
1084}
1085
1086// Telemetry recording helpers (KH-116)
1087pub(crate) fn record_file_scanned(bytes: usize) {
1088    FILES_SCANNED.fetch_add(1, Ordering::Relaxed);
1089    BYTES_SCANNED.fetch_add(bytes, Ordering::Relaxed);
1090}
1091
1092pub(crate) fn global_scan_counts() -> (usize, usize) {
1093    (
1094        FILES_SCANNED.load(Ordering::Relaxed),
1095        BYTES_SCANNED.load(Ordering::Relaxed),
1096    )
1097}
1098
1099pub(crate) fn record_file_skipped() {
1100    SKIPPED_FILES.fetch_add(1, Ordering::Relaxed);
1101}
1102
1103pub(crate) fn record_match_found() {
1104    TOTAL_MATCHES.fetch_add(1, Ordering::Relaxed);
1105}
1106
1107pub(crate) fn record_gpu_dispatch() {
1108    GPU_DISPATCHES.fetch_add(1, Ordering::Relaxed);
1109}
1110
1111/// Reset process-global telemetry that is scoped to one scan.
1112///
1113/// Long-lived callers (CLI library use, daemon-style harnesses, and integration
1114/// tests) must not let a previous scan's suppression count, dogfood flag, or
1115/// coverage-gap counters change the next scan's report. Scoped daemon telemetry
1116/// (`with_scan_telemetry`) remains isolated by its caller-owned handle.
1117pub fn reset_for_scan() {
1118    let t = cell();
1119    DOGFOOD_ENABLED.store(false, Ordering::Relaxed);
1120    t.dogfood_enabled.store(false, Ordering::Relaxed);
1121    t.example_suppressions.store(0, Ordering::Relaxed);
1122    t.detail_events_dropped.store(0, Ordering::Relaxed);
1123    t.static_recovery.reset();
1124    FILES_SCANNED.store(0, Ordering::Relaxed);
1125    BYTES_SCANNED.store(0, Ordering::Relaxed);
1126    SKIPPED_FILES.store(0, Ordering::Relaxed);
1127    TOTAL_MATCHES.store(0, Ordering::Relaxed);
1128    GPU_DISPATCHES.store(0, Ordering::Relaxed);
1129    for gap in ScannerCoverageGapEvent::ALL {
1130        gap.counter().store(0, Ordering::Relaxed);
1131    }
1132    #[cfg(test)]
1133    THREAD_DECODE_TRUNCATIONS.with(|count| count.set(0));
1134    recover_telemetry_lock(&t.events).clear();
1135    recover_telemetry_lock(&t.emitted_suppression_events).clear();
1136    CURRENT_SCAN_TELEMETRY.with(|slot| {
1137        *slot.borrow_mut() = None;
1138    });
1139}
1140
1141#[cfg(test)]
1142#[doc(hidden)]
1143pub mod testing {
1144    use std::sync::Arc;
1145
1146    /// Reset all telemetry state. Test-only facade for integration tests.
1147    pub fn reset() {
1148        super::reset_for_scan();
1149    }
1150
1151    pub(crate) fn poison_events(telemetry: &Arc<super::ScanTelemetry>) {
1152        let telemetry = Arc::clone(telemetry);
1153        let _ = std::thread::spawn(move || {
1154            // LAW10: this cfg(test) helper has no production runtime effect; it joins an expected panic to poison a disposable scoped buffer.
1155            let Ok(_events) = telemetry.events.lock() else {
1156                panic!("fresh telemetry event buffer was already poisoned");
1157            };
1158            panic!("poison scoped telemetry event buffer");
1159        })
1160        .join();
1161    }
1162}
keyhog_scanner/telemetry.rs

keyhog_scanner/
telemetry.rs
