systemprompt_security/policy/secrets/
entropy.rs

1//! The high-entropy backstop for credentials carrying no vendor prefix.
2//!
3//! A random base64 blob pasted into a prompt matches none of
4//! [`super::SECRET_PATTERNS`] but still reads as machine-generated key
5//! material. Randomness alone cannot say so: a serialised protobuf, a base64
6//! JSON envelope, and a 32-byte key are all dense mixed-case base64 of similar
7//! measured entropy. [`is_structured_payload`] supplies the missing
8//! discriminator — key material decodes to bytes with no readable structure,
9//! whereas a tool result decodes to text or to a self-consistent wire format.
10//!
11//! Copyright (c) systemprompt.io — Business Source License 1.1.
12//! See <https://systemprompt.io> for licensing details.
13
14use base64::Engine as _;
15use base64::engine::general_purpose::{STANDARD_NO_PAD, URL_SAFE_NO_PAD};
16use regex::Regex;
17
18pub const DEFAULT_MIN_LEN: usize = 32;
19
20/// Measured entropy as a fraction of the token length's ceiling.
21///
22/// Why: Shannon entropy is bounded by the number of symbols actually sampled,
23/// so a raw bits-per-byte threshold is far laxer on a 32-char token (ceiling
24/// 5.0) than on a 128-char one (ceiling 6.0). Dividing by that ceiling makes
25/// one number mean the same thing at every length. 0.80 sits below the
26/// measured floor (0.816 over 1000 samples) for random base64 key material of
27/// 24 bytes and up, and above base64-encoded English prose.
28pub const DEFAULT_THRESHOLD: f64 = 0.80;
29
30const TOKEN_DELIMITERS: &str = "\"'`()[]{}<>,;:";
31const TOKEN_CHARSET_EXTRA: &str = "+/=_-";
32const ENTROPY_CEILING_SYMBOLS: usize = 64;
33
34const MIN_STRUCTURED_LEN: usize = 16;
35const MAX_FIELD_NUMBER: u64 = 64;
36const MAX_NESTING_DEPTH: u32 = 4;
37const MIN_NESTED_PAYLOAD_LEN: usize = 4;
38const TEXT_RATIO_NUMERATOR: usize = 9;
39const TEXT_RATIO_DENOMINATOR: usize = 10;
40
41/// Tunables for the heuristic, read from the `secret_scan` policy's `entropy`
42/// block. [`Default`] reproduces the built-in behaviour, which is what every
43/// caller outside the policy chain gets.
44#[derive(Debug, Clone)]
45pub struct EntropyConfig {
46    pub enabled: bool,
47    pub min_len: usize,
48    pub threshold: f64,
49    pub allowlist: Vec<Regex>,
50}
51
52impl Default for EntropyConfig {
53    fn default() -> Self {
54        Self {
55            enabled: true,
56            min_len: DEFAULT_MIN_LEN,
57            threshold: DEFAULT_THRESHOLD,
58            allowlist: Vec::new(),
59        }
60    }
61}
62
63#[must_use]
64pub fn find_high_entropy_token<'a>(text: &'a str, config: &EntropyConfig) -> Option<&'a str> {
65    if !config.enabled {
66        return None;
67    }
68    text.split(|c: char| c.is_whitespace() || TOKEN_DELIMITERS.contains(c))
69        .find(|token| is_credential_shaped(token, config))
70}
71
72fn is_credential_shaped(token: &str, config: &EntropyConfig) -> bool {
73    token.len() >= config.min_len
74        && token
75            .chars()
76            .all(|c| c.is_ascii_alphanumeric() || TOKEN_CHARSET_EXTRA.contains(c))
77        && token.chars().any(|c| c.is_ascii_uppercase())
78        && token.chars().any(|c| c.is_ascii_lowercase())
79        && token.chars().any(|c| c.is_ascii_digit())
80        && entropy_ratio(token) >= config.threshold
81        && !config.allowlist.iter().any(|re| re.is_match(token))
82        && !is_structured_payload(token)
83}
84
85fn shannon_entropy(s: &str) -> f64 {
86    let mut counts = [0u32; 256];
87    let bytes = s.as_bytes();
88    for &b in bytes {
89        counts[usize::from(b)] += 1;
90    }
91    let len = bytes.len() as f64;
92    counts
93        .iter()
94        .filter(|&&c| c > 0)
95        .map(|&c| {
96            let p = f64::from(c) / len;
97            -p * p.log2()
98        })
99        .sum()
100}
101
102fn entropy_ratio(s: &str) -> f64 {
103    let symbols = s.len().min(ENTROPY_CEILING_SYMBOLS);
104    let ceiling = (symbols as f64).log2();
105    if ceiling <= 0.0 {
106        return 0.0;
107    }
108    shannon_entropy(s) / ceiling
109}
110
111fn is_structured_payload(token: &str) -> bool {
112    decode_base64(token).is_some_and(|bytes| {
113        bytes.len() >= MIN_STRUCTURED_LEN && (is_mostly_text(&bytes) || is_protobuf(&bytes, 0))
114    })
115}
116
117fn decode_base64(token: &str) -> Option<Vec<u8>> {
118    let body = token.trim_end_matches('=');
119    STANDARD_NO_PAD
120        .decode(body)
121        .or_else(|_| URL_SAFE_NO_PAD.decode(body))
122        .ok()
123}
124
125fn is_mostly_text(bytes: &[u8]) -> bool {
126    if bytes.is_empty() {
127        return false;
128    }
129    let printable = bytes
130        .iter()
131        .filter(|&&b| matches!(b, b'\t' | b'\n' | b'\r') || (0x20..0x7f).contains(&b))
132        .count();
133    printable * TEXT_RATIO_DENOMINATOR >= bytes.len() * TEXT_RATIO_NUMERATOR
134}
135
136// Why: exact buffer consumption alone is a weak signal on a short random blob,
137// so a decode only counts as protobuf when it also carries at least two fields
138// and one length-delimited payload that is itself text or protobuf. Random key
139// material clears all three by accident far less than one time in a hundred; a
140// real serialised message clears them by construction.
141fn is_protobuf(bytes: &[u8], depth: u32) -> bool {
142    if depth > MAX_NESTING_DEPTH || bytes.len() < MIN_STRUCTURED_LEN {
143        return false;
144    }
145    parse_protobuf(bytes, depth).is_some_and(|parsed| parsed.fields >= 2 && parsed.nested_structure)
146}
147
148struct ParsedMessage {
149    fields: usize,
150    nested_structure: bool,
151}
152
153fn parse_protobuf(bytes: &[u8], depth: u32) -> Option<ParsedMessage> {
154    let mut cursor = 0usize;
155    let mut parsed = ParsedMessage {
156        fields: 0,
157        nested_structure: false,
158    };
159    while cursor < bytes.len() {
160        let (tag, after_tag) = read_varint(bytes, cursor)?;
161        let field_number = tag >> 3;
162        if field_number == 0 || field_number > MAX_FIELD_NUMBER {
163            return None;
164        }
165        cursor = match tag & 7 {
166            0 => read_varint(bytes, after_tag)?.1,
167            1 => advance(bytes, after_tag, 8)?,
168            5 => advance(bytes, after_tag, 4)?,
169            2 => {
170                let (len, after_len) = read_varint(bytes, after_tag)?;
171                let len = usize::try_from(len).ok()?;
172                let end = advance(bytes, after_len, len)?;
173                let payload = bytes.get(after_len..end)?;
174                if payload.len() >= MIN_NESTED_PAYLOAD_LEN
175                    && (is_mostly_text(payload) || is_protobuf(payload, depth + 1))
176                {
177                    parsed.nested_structure = true;
178                }
179                end
180            },
181            _ => return None,
182        };
183        parsed.fields += 1;
184    }
185    Some(parsed)
186}
187
188fn advance(bytes: &[u8], cursor: usize, by: usize) -> Option<usize> {
189    let end = cursor.checked_add(by)?;
190    (end <= bytes.len()).then_some(end)
191}
192
193fn read_varint(bytes: &[u8], start: usize) -> Option<(u64, usize)> {
194    let mut value = 0u64;
195    let mut shift = 0u32;
196    let mut cursor = start;
197    loop {
198        let byte = *bytes.get(cursor)?;
199        cursor += 1;
200        value |= u64::from(byte & 0x7f) << shift;
201        if byte & 0x80 == 0 {
202            return Some((value, cursor));
203        }
204        shift += 7;
205        if shift >= 64 {
206            return None;
207        }
208    }
209}
systemprompt_security/policy/secrets/entropy.rs

systemprompt_security/policy/secrets/
entropy.rs
