truman-core 0.1.0

Core types, filters, and parallel scanner for Truman secret detection
Documentation
//! Shannon entropy helpers for high-entropy detectors.
//!
//! Mirrors `HighEntropyStringsPlugin.calculate_shannon_entropy` (Python):
//! entropy is computed over the detector's `charset`, not over the
//! characters present in the data.

/// Shannon entropy of `data`, measured against `charset`.
///
/// For each character `c` in `charset`: `p = count(data, c) / len(data)`,
/// summing `-p * log2(p)` for `p > 0`. Returns `0.0` for empty data.
/// An empty charset also yields `0.0`.
pub fn shannon_entropy(data: &str, charset: &str) -> f64 {
    shannon_entropy_bytes(data, charset.as_bytes())
}

/// Byte-oriented variant; `charset` is a byte set.
pub fn shannon_entropy_bytes(data: &str, charset: &[u8]) -> f64 {
    if data.is_empty() || charset.is_empty() {
        return 0.0;
    }
    let len = data.len() as f64;
    let bytes = data.as_bytes();
    let mut entropy = 0.0;
    for &c in charset {
        let count = bytes.iter().filter(|&&b| b == c).count() as f64;
        if count > 0.0 {
            let p = count / len;
            entropy += -p * p.log2();
        }
    }
    entropy
}

/// Hex entropy with the Python `HexHighEntropyString` integer penalty:
///
/// if the whole string parses as an integer (and `len > 1`),
/// subtract `1.2 / log2(len)`. This pushes all-digit strings below the
/// default hex limit (3.0) unless they are long. Mirrors
/// `HexHighEntropyString.calculate_shannon_entropy`: `len == 1` never
/// penalized; `int(data)` success (base-10, optional sign) penalizes.
pub fn hex_shannon_entropy(data: &str) -> f64 {
    const HEX: &str = "0123456789abcdefABCDEF";
    let mut entropy = shannon_entropy(data, HEX);
    if data.len() <= 1 {
        return entropy;
    }
    // Python `int(data)`: base-10 with optional surrounding whitespace and
    // sign / underscores. Approximate: trim, then try i128, then a manual
    // strict-digit check for overlong numerics i128 cannot hold.
    let t = data.trim();
    let parses = t.parse::<i128>().is_ok() || {
        let s = t.strip_prefix(['+', '-']).unwrap_or(t).replace('_', "");
        !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit())
    };
    if parses {
        entropy -= 1.2 / (data.len() as f64).log2();
    }
    entropy
}

/// Base64-ish entropy (letters + digits + `+/`, url-safe `-_`, padding `=`).
pub fn base64_shannon_entropy(data: &str) -> f64 {
    const B64: &str = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/\\-_=";
    shannon_entropy(data, B64)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn empty_is_zero() {
        assert_eq!(shannon_entropy("", "abc"), 0.0);
        assert_eq!(shannon_entropy("abc", ""), 0.0);
        assert_eq!(hex_shannon_entropy(""), 0.0);
        assert_eq!(base64_shannon_entropy(""), 0.0);
    }

    #[test]
    fn uniform_binary_entropy_is_one() {
        // "01" over charset "01": p=0.5 each -> entropy 1.0
        let e = shannon_entropy("01", "01");
        assert!((e - 1.0).abs() < 1e-9, "got {e}");
    }

    #[test]
    fn repeated_char_is_zero() {
        // Cross-checked with Python: se('aaaa', b64) == 0.0
        assert!((base64_shannon_entropy("aaaa") - 0.0).abs() < 1e-9);
        assert!((shannon_entropy("aaaa", "ab") - 0.0).abs() < 1e-9);
    }

    #[test]
    fn two_distinct_chars_is_one() {
        // Cross-checked: se('ab', b64) == 1.0
        assert!((base64_shannon_entropy("ab") - 1.0).abs() < 1e-9);
    }

    #[test]
    fn charset_limits_scope() {
        // Characters outside the charset contribute nothing (denominator is
        // still len(data), mirroring Python `data.count(x)/len(data)`).
        let e = shannon_entropy("ab!!", "ab");
        // p_a = p_b = 2/4 = 0.5 → 1.0; '!' ignored.
        assert!((e - 1.0).abs() < 1e-9, "got {e}");
    }

    #[test]
    fn hex_reference_values_match_python() {
        // Values produced by the Python reference implementation:
        // se('0123456789', hexdigits) == 3.321928094887362
        let plain = shannon_entropy("0123456789", "0123456789abcdefABCDEF");
        assert!((plain - 3.321928094887362).abs() < 1e-9, "got {plain}");
        // se('deadbeef', hexdigits) == 2.1556390622295662
        let d = shannon_entropy("deadbeef", "0123456789abcdefABCDEF");
        assert!((d - 2.1556390622295662).abs() < 1e-9, "got {d}");
        // se('ABCDEF', hexdigits) == 2.584962500721156
        let u = shannon_entropy("ABCDEF", "0123456789abcdefABCDEF");
        assert!((u - 2.584962500721156).abs() < 1e-9, "got {u}");
    }

    #[test]
    fn hex_int_penalty_lowers_entropy() {
        let plain = shannon_entropy("0123456789", "0123456789abcdefABCDEF");
        let penalized = hex_shannon_entropy("0123456789");
        // Python: 3.321928094887362 - 1.2/log2(10) == 2.9606921000905846
        assert!(
            (penalized - 2.9606921000905846).abs() < 1e-9,
            "got {penalized}"
        );
        assert!(penalized < plain, "{penalized} < {plain}");
        // Below the default hex limit (3.0), as Python intends.
        assert!(penalized < 3.0);
    }

    #[test]
    fn hex_single_char_has_no_penalty() {
        // Python returns early for len == 1.
        let s = "5";
        assert!(
            (hex_shannon_entropy(s) - shannon_entropy(s, "0123456789abcdefABCDEF")).abs() < 1e-9
        );
    }

    #[test]
    fn hex_non_numeric_has_no_penalty() {
        let s = "abcdef0123";
        assert!(
            (hex_shannon_entropy(s) - shannon_entropy(s, "0123456789abcdefABCDEF")).abs() < 1e-9
        );
        let s2 = "deadbeef";
        assert!(
            (hex_shannon_entropy(s2) - shannon_entropy(s2, "0123456789abcdefABCDEF")).abs() < 1e-9
        );
    }

    #[test]
    fn hex_long_numeric_still_penalized() {
        // Longer than i128 range (~39 digits max): fallback digit check.
        let s = "1234567890".repeat(10); // 100 digits
        let plain = shannon_entropy(&s, "0123456789abcdefABCDEF");
        let penalized = hex_shannon_entropy(&s);
        let expected = plain - 1.2 / (s.len() as f64).log2();
        assert!((penalized - expected).abs() < 1e-9, "got {penalized}");
    }

    #[test]
    fn base64_urlsafe_charset() {
        // '+', '/', '-', '_', '=' and backslash are all in the charset.
        for s in ["a+b/c", "a-b_c", "abc=", r"a\b"] {
            assert!(
                base64_shannon_entropy(s) > 0.0,
                "{s} should have nonzero entropy"
            );
        }
    }
}