keyhog-scanner 0.5.41

//! Random-token vs dictionary-identifier discriminator (KH-L-0413).
//!
//! The generic keyword bridge suppresses any value the identifier/type-name
//! shape gates flag (`pure_identifier_no_digit`, `pure_identifier`,
//! `type_name_shape`, `word_separated_identifier`). Those gates exist to drop
//! code references: `password = getUserName`, `secret = configValue`: but they
//! ALSO drop a large pool of REAL random passwords that happen to be all-letters
//! with no digit (`GRAPHITE_PASS=gjbubxsu`, `password="ufnlbbavawsdeecn"`,
//! `ftp://user:pxidztpv`): on CredData ~1114 keyword-anchored positives, measured
//! via the KH-L-0412 `--dogfood` trace.
//!
//! The two classes are SHAPE-identical (lowercase, no digit), so the only sound
//! discriminator is LANGUAGE STRUCTURE: a real random password has improbable
//! letter adjacencies (`gjb`, `xs`, `dz`), a dictionary identifier is built from
//! pronounceable English fragments (`get`, `user`, `config`). We score the mean
//! adjacent-bigram log-probability of the value's alphabetic runs against an
//! English bigram model (`data/english_bigram_logprob.bin`, generated by
//! `ml/gen_bigram_model.py` from a standard wordlist). A value whose letters are
//! collectively IMPROBABLE under English (mean log-prob below a threshold) is a
//! random token; a pronounceable one is a dictionary identifier.
//!
//! This is the SOUND half of the lever: lifting the identifier gates
//! unconditionally recovered +1023 CredData TP but added +3554 FP (precision
//! 0.60→0.40) by surfacing every `password = someVariable`; gating the lift on
//! `is_random_token` keeps the random passwords while leaving the identifier
//! references suppressed. Verified on BOTH bench corpora before landing.

use std::sync::LazyLock;

/// English bigram log-probabilities, row-major `[a][b]` over `'a'..='z'`,
/// little-endian f32. Generated + committed by `ml/gen_bigram_model.py`; the
/// `.bin` is the reproducible source of truth (host-independent).
const BIGRAM_LOGPROB_BYTES: &[u8] = include_bytes!("../../data/english_bigram_logprob.bin");

/// Parsed 26×26 model. Built once; the hot path indexes `[a*26 + b]`.
static BIGRAM_LOGPROB: LazyLock<[f32; 676]> = LazyLock::new(|| {
    assert_eq!(
        BIGRAM_LOGPROB_BYTES.len(),
        676 * 4,
        "english_bigram_logprob.bin must hold exactly 26*26 little-endian f32 \
         (regenerate with ml/gen_bigram_model.py)"
    );
    let mut t = [0.0f32; 676];
    for (i, slot) in t.iter_mut().enumerate() {
        let b = &BIGRAM_LOGPROB_BYTES[i * 4..i * 4 + 4];
        *slot = f32::from_le_bytes([b[0], b[1], b[2], b[3]]);
    }
    t
});

/// Minimum alphabetic characters before a randomness verdict is meaningful.
/// Below this, English bigram statistics are too sparse to separate a short
/// random password from a short identifier, so we return `false` (NOT random ⇒
/// the identifier gate keeps suppressing (fail safe toward precision)).
pub(crate) const MIN_ALPHA: usize = 6;

/// Mean adjacent-bigram log-probability at or below which a token's letters are
/// collectively too improbable for English ⇒ a random token, not a dictionary
/// identifier. Calibrated on the CredData real-password vs identifier pools so
/// random passwords (≤ ~−7) pass while dictionary identifiers (≥ ~−6.4) do not;
/// the model's clean separation gap sits around −6.7.
const RANDOM_LOGPROB_THRESHOLD: f32 = -6.85;

/// Minimum DISTINCT lowercase letters a value must have for a `random` verdict.
/// A 1–2 distinct-letter token (`aaaaaaaa`, `xzxzxzxz`, `qqqqwwww`) has
/// improbable English bigrams, it would pass the log-prob threshold, but it is
/// a repetitive / alternating PATTERN, not a random token. Without this guard the
/// discriminator is only sound downstream of a caller-side entropy/diversity
/// floor (the generic bridge has one; the api.rs weak-anchor path does not). The
/// floor of 3 is data-calibrated: all 1285 CredData random passwords the
/// discriminator recovers have ≥ 4 distinct letters (min 4, e.g. `ttqqrqjt`),
/// while every blind-spot pattern has ≤ 2, so 3 separates them with margin and
/// drops no real password.
pub(crate) const MIN_DISTINCT_LETTERS: usize = 3;

#[derive(Debug, Clone, Copy)]
pub(crate) struct RandomTokenEvidence {
    mean_bigram_logprob: Option<f32>,
    distinct_letters: usize,
}

impl RandomTokenEvidence {
    pub(crate) fn analyze(value: &str) -> Self {
        let table = &*BIGRAM_LOGPROB;
        let bytes = value.as_bytes();
        let mut sum = 0.0f32;
        let mut pairs = 0u32;
        let mut alpha = 0usize;
        let mut prev: Option<usize> = None;
        let mut seen_letters = 0u32;
        for &byte in bytes {
            if byte.is_ascii_alphabetic() {
                alpha += 1;
                let idx = (byte.to_ascii_lowercase() - b'a') as usize;
                seen_letters |= 1u32 << idx;
                if let Some(p) = prev {
                    sum += table[p * 26 + idx];
                    pairs += 1;
                }
                prev = Some(idx);
            } else {
                // digit / symbol ends the current alphabetic run
                prev = None;
            }
        }
        let mean_bigram_logprob = if alpha < MIN_ALPHA || pairs == 0 {
            None
        } else {
            Some(sum / pairs as f32)
        };
        Self {
            mean_bigram_logprob,
            distinct_letters: seen_letters.count_ones() as usize,
        }
    }

    #[inline]
    pub(crate) fn mean_bigram_logprob(self) -> Option<f32> {
        self.mean_bigram_logprob
    }

    #[inline]
    pub(crate) fn distinct_letters(self) -> usize {
        self.distinct_letters
    }

    #[inline]
    pub(crate) fn is_random_token(self) -> bool {
        self.mean_bigram_logprob()
            .is_some_and(|score| score <= RANDOM_LOGPROB_THRESHOLD)
            && self.distinct_letters >= MIN_DISTINCT_LETTERS
    }
}

#[derive(Debug, Clone, Copy)]
pub(crate) struct TokenRandomness<'a> {
    candidate: &'a str,
    candidate_evidence: RandomTokenEvidence,
}

impl<'a> TokenRandomness<'a> {
    #[inline]
    pub(crate) fn for_candidate(candidate: &'a str) -> Self {
        Self {
            candidate,
            candidate_evidence: RandomTokenEvidence::analyze(candidate),
        }
    }

    #[inline]
    pub(crate) fn evidence_for(&self, value: &str) -> RandomTokenEvidence {
        if std::ptr::eq(value.as_ptr(), self.candidate.as_ptr())
            && value.len() == self.candidate.len()
        {
            self.candidate_evidence
        } else {
            RandomTokenEvidence::analyze(value)
        }
    }

    #[inline]
    pub(crate) fn is_random_token(&self, value: &str) -> bool {
        self.evidence_for(value).is_random_token()
    }
}

/// `true` iff `value` reads as a RANDOM token (real credential) rather than a
/// pronounceable dictionary identifier (code reference) OR a low-diversity
/// repetitive pattern. Fails safe to `false` (treat as NOT random ⇒ keep
/// suppressing) when the value is too short/sparse to judge or has too few
/// distinct letters (soundness over reach, independent of any caller-side floor).
pub(crate) fn is_random_token(value: &str) -> bool {
    RandomTokenEvidence::analyze(value).is_random_token()
}

/// `true` iff the bigram model is CONFIDENT that `value` is a pronounceable
/// English dictionary word, it has at least [`MIN_ALPHA`] alphabetic chars AND
/// its mean adjacent-bigram log-probability sits ABOVE the random threshold
/// (`password`, `secret`, `welcome`, `admin1234`).
///
/// This is the deliberate mirror image of [`is_random_token`], NOT merely its
/// negation: `!is_random_token` is also true for a SHORT token the model cannot
/// judge (`mean_bigram_logprob == None`, the fail-safe). This predicate stays
/// `false` there, so it can only ever DROP a value the model is sure is English
/// it never suppresses a short random password on a fail-safe, and a random
/// token (`pxidztpv`, score ≤ −6.85) is below the threshold so it is kept.
///
/// Used by the strong-anchor structural detectors (e.g. `url-credentials`,
/// whose regex proves a `scheme://user:<x>@host` credential SLOT but cannot
/// itself tell the literal placeholder word `password` from a real secret) to
/// drop the dictionary-word placeholders the Tier-B randomness floor would have
/// caught (without that floor's length penalty on short random passwords).
pub(crate) fn is_confident_dictionary_word(value: &str) -> bool {
    // A real English word carries at least one letter OUTSIDE the hex alphabet
    // (`g..=z`). A pure-hex digest (`08c0fee0abeb…`) is built only from `a..f`
    // plus digits, yet its `a..f` adjacencies (`ab`, `be`, `de`, `ea`) score as
    // probable English, without this guard the model misreads every hex key as
    // a dictionary word and the placeholder gate would suppress real hex secrets.
    let has_non_hex_letter = value
        .bytes()
        .any(|b| (b'g'..=b'z').contains(&b.to_ascii_lowercase()));
    has_non_hex_letter
        && RandomTokenEvidence::analyze(value)
            .mean_bigram_logprob()
            .is_some_and(|score| score > RANDOM_LOGPROB_THRESHOLD)
}

/// `true` iff `value` has FEWER than [`MIN_DISTINCT_LETTERS`] distinct ASCII
/// letters, a repetitive / alternating / digit-only MASK (`xxxxxxxx`, `aaaaaa`,
/// `ababab`, `12345678`), never a real password.
///
/// This is the soundness companion to [`is_confident_dictionary_word`] for the
/// strong-anchor structural-password-slot family. Those detectors are
/// `is_service_anchored`, so the post-match pipeline sets `bypass_shape_gates`
/// and SKIPS the Tier-B repetitive-run / repeated-block gates that normally drop
/// a `--password xxxxxxxx` mask. `is_confident_dictionary_word` cannot catch a
/// mask (its bigrams are improbable English, so the model is NOT confident it is
/// a word), so without this guard the strong anchor would surface the mask as a
/// false positive. A genuinely-short random password (`i8cr1w!`, 4 distinct
/// letters) clears the floor and is kept, the same `MIN_DISTINCT_LETTERS = 3`
/// boundary [`is_random_token`] uses, so the two paths agree byte-for-byte.
pub(crate) fn has_low_letter_diversity(value: &str) -> bool {
    RandomTokenEvidence::analyze(value).distinct_letters() < MIN_DISTINCT_LETTERS
}

/// Shared decision for the CONTIGUOUS identifier/type-name shape gates
/// (KH-L-0413): keep the gate engaged (`true` ⇒ the value stays suppressed)
/// UNLESS the value reads as a random token, in which case lift it (`false` ⇒
/// recover the value). The single source of truth for the gate so the scan-time
/// generic bridge (`phase2_generic_shape`) and the post-process weak-anchor
/// path (`suppression::api::suppress_named_detector_finding`) agree
/// byte-for-byte (both wrap the SAME `is_random_token`, never a second copy).
///
/// Used ONLY for the contiguous gates (`pure_identifier` / `type_name`), whose
/// own predicates already reject digit-bearing values; the WORD-SEPARATED gate
/// needs the stricter [`keep_word_separated_gate_with_randomness`].
#[inline]
pub(crate) fn keep_identifier_gate_with_randomness(
    value: &str,
    randomness: &TokenRandomness<'_>,
) -> bool {
    !randomness.is_random_token(value)
}

/// Stricter sibling of [`keep_identifier_gate_with_randomness`] for the
/// WORD-SEPARATED identifier gate (KH-L-0414). The randomness model is an
/// ENGLISH-WORD model, and a multi-segment programmer identifier with embedded
/// digits / uppercase splits into SHORT acronym fragments (`d2i_PKCS7_bio` →
/// `pkcs`, `curlx_memdup0` → `memdup`) that the model mis-scores as random
/// so `is_random_token` alone is unsound here. Real CredData word-separated
/// passwords are uniformly all-lowercase letters + `_`/`-` separators
/// (`abxnj_gjvpuqzo`, `aapqhgn-qhuuc-trnmf`); requiring that shape BEFORE
/// trusting the randomness verdict recovers 141 real passwords while keeping
/// every acronym / product-key identifier (`d2i_PKCS7_bio`, `sqlite3_malloc64`,
/// `2iw9-n01w-Mc4V-faEC`) suppressed. Returns `true` (stay suppressed) for
/// anything that is not an all-lowercase-letter (+ separator) random token.
#[inline]
pub(crate) fn keep_word_separated_gate_with_randomness(
    value: &str,
    randomness: &TokenRandomness<'_>,
) -> bool {
    // Any digit / uppercase / non-ASCII byte ⇒ not the clean lowercase password
    // shape ⇒ keep the gate engaged (the acronym / product-key class the English
    // model would mis-lift).
    if !value
        .bytes()
        .all(|b| b.is_ascii_lowercase() || b == b'_' || b == b'-')
    {
        return true;
    }
    !randomness.is_random_token(value)
}