pub fn shannon_entropy(data: &str, charset: &str) -> f64 {
shannon_entropy_bytes(data, charset.as_bytes())
}
pub fn shannon_entropy_bytes(data: &str, charset: &[u8]) -> f64 {
if data.is_empty() || charset.is_empty() {
return 0.0;
}
let len = data.len() as f64;
let bytes = data.as_bytes();
let mut entropy = 0.0;
for &c in charset {
let count = bytes.iter().filter(|&&b| b == c).count() as f64;
if count > 0.0 {
let p = count / len;
entropy += -p * p.log2();
}
}
entropy
}
pub fn hex_shannon_entropy(data: &str) -> f64 {
const HEX: &str = "0123456789abcdefABCDEF";
let mut entropy = shannon_entropy(data, HEX);
if data.len() <= 1 {
return entropy;
}
let t = data.trim();
let parses = t.parse::<i128>().is_ok() || {
let s = t.strip_prefix(['+', '-']).unwrap_or(t).replace('_', "");
!s.is_empty() && s.bytes().all(|b| b.is_ascii_digit())
};
if parses {
entropy -= 1.2 / (data.len() as f64).log2();
}
entropy
}
pub fn base64_shannon_entropy(data: &str) -> f64 {
const B64: &str = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/\\-_=";
shannon_entropy(data, B64)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn empty_is_zero() {
assert_eq!(shannon_entropy("", "abc"), 0.0);
assert_eq!(shannon_entropy("abc", ""), 0.0);
assert_eq!(hex_shannon_entropy(""), 0.0);
assert_eq!(base64_shannon_entropy(""), 0.0);
}
#[test]
fn uniform_binary_entropy_is_one() {
let e = shannon_entropy("01", "01");
assert!((e - 1.0).abs() < 1e-9, "got {e}");
}
#[test]
fn repeated_char_is_zero() {
assert!((base64_shannon_entropy("aaaa") - 0.0).abs() < 1e-9);
assert!((shannon_entropy("aaaa", "ab") - 0.0).abs() < 1e-9);
}
#[test]
fn two_distinct_chars_is_one() {
assert!((base64_shannon_entropy("ab") - 1.0).abs() < 1e-9);
}
#[test]
fn charset_limits_scope() {
let e = shannon_entropy("ab!!", "ab");
assert!((e - 1.0).abs() < 1e-9, "got {e}");
}
#[test]
fn hex_reference_values_match_python() {
let plain = shannon_entropy("0123456789", "0123456789abcdefABCDEF");
assert!((plain - 3.321928094887362).abs() < 1e-9, "got {plain}");
let d = shannon_entropy("deadbeef", "0123456789abcdefABCDEF");
assert!((d - 2.1556390622295662).abs() < 1e-9, "got {d}");
let u = shannon_entropy("ABCDEF", "0123456789abcdefABCDEF");
assert!((u - 2.584962500721156).abs() < 1e-9, "got {u}");
}
#[test]
fn hex_int_penalty_lowers_entropy() {
let plain = shannon_entropy("0123456789", "0123456789abcdefABCDEF");
let penalized = hex_shannon_entropy("0123456789");
assert!(
(penalized - 2.9606921000905846).abs() < 1e-9,
"got {penalized}"
);
assert!(penalized < plain, "{penalized} < {plain}");
assert!(penalized < 3.0);
}
#[test]
fn hex_single_char_has_no_penalty() {
let s = "5";
assert!(
(hex_shannon_entropy(s) - shannon_entropy(s, "0123456789abcdefABCDEF")).abs() < 1e-9
);
}
#[test]
fn hex_non_numeric_has_no_penalty() {
let s = "abcdef0123";
assert!(
(hex_shannon_entropy(s) - shannon_entropy(s, "0123456789abcdefABCDEF")).abs() < 1e-9
);
let s2 = "deadbeef";
assert!(
(hex_shannon_entropy(s2) - shannon_entropy(s2, "0123456789abcdefABCDEF")).abs() < 1e-9
);
}
#[test]
fn hex_long_numeric_still_penalized() {
let s = "1234567890".repeat(10); let plain = shannon_entropy(&s, "0123456789abcdefABCDEF");
let penalized = hex_shannon_entropy(&s);
let expected = plain - 1.2 / (s.len() as f64).log2();
assert!((penalized - expected).abs() < 1e-9, "got {penalized}");
}
#[test]
fn base64_urlsafe_charset() {
for s in ["a+b/c", "a-b_c", "abc=", r"a\b"] {
assert!(
base64_shannon_entropy(s) > 0.0,
"{s} should have nonzero entropy"
);
}
}
}