systemprompt_security/policy/secrets/
entropy.rs

1//! The high-entropy backstop for credentials carrying no vendor prefix.
2//!
3//! A random base64 blob pasted into a prompt matches none of
4//! [`super::SECRET_PATTERNS`] but still reads as machine-generated key
5//! material. Randomness alone cannot say so: a serialised protobuf, a base64
6//! JSON envelope, and a 32-byte key are all dense mixed-case base64 of similar
7//! measured entropy. [`is_structured_payload`] supplies the missing
8//! discriminator — key material decodes to bytes with no readable structure,
9//! whereas a tool result decodes to text or to a self-consistent wire format.
10//!
11//! Copyright (c) systemprompt.io — Business Source License 1.1.
12//! See <https://systemprompt.io> for licensing details.
13
14use base64::Engine as _;
15use base64::engine::general_purpose::{STANDARD_NO_PAD, URL_SAFE_NO_PAD};
16use regex::Regex;
17
18pub const DEFAULT_MIN_LEN: usize = 32;
19
20pub const DEFAULT_THRESHOLD: f64 = 0.80;
21
22const TOKEN_DELIMITERS: &str = "\"'`()[]{}<>,;:";
23const TOKEN_CHARSET_EXTRA: &str = "+/=_-";
24const ENTROPY_CEILING_SYMBOLS: usize = 64;
25
26const MIN_STRUCTURED_LEN: usize = 16;
27const MAX_PAYLOAD_PREFIX_LEN: usize = 10;
28const DIGEST_LENGTHS: [(&str, usize); 3] = [("sha256", 32), ("sha384", 48), ("sha512", 64)];
29const MAX_FIELD_NUMBER: u64 = 64;
30const MAX_NESTING_DEPTH: u32 = 4;
31const MIN_NESTED_PAYLOAD_LEN: usize = 4;
32const TEXT_RATIO_NUMERATOR: usize = 9;
33const TEXT_RATIO_DENOMINATOR: usize = 10;
34
35/// Tunables for the heuristic, read from the `secret_scan` policy's `entropy`
36/// block. [`Default`] reproduces the built-in behaviour, which is what every
37/// caller outside the policy chain gets.
38#[derive(Debug, Clone)]
39pub struct EntropyConfig {
40    pub enabled: bool,
41    pub min_len: usize,
42    pub threshold: f64,
43    pub allowlist: Vec<Regex>,
44}
45
46impl Default for EntropyConfig {
47    fn default() -> Self {
48        Self {
49            enabled: true,
50            min_len: DEFAULT_MIN_LEN,
51            threshold: DEFAULT_THRESHOLD,
52            allowlist: Vec::new(),
53        }
54    }
55}
56
57#[must_use]
58pub fn find_high_entropy_token<'a>(text: &'a str, config: &EntropyConfig) -> Option<&'a str> {
59    if !config.enabled {
60        return None;
61    }
62    text.split(|c: char| c.is_whitespace() || TOKEN_DELIMITERS.contains(c))
63        .find(|token| is_credential_shaped(token, config))
64}
65
66fn is_credential_shaped(token: &str, config: &EntropyConfig) -> bool {
67    token.len() >= config.min_len
68        && token
69            .chars()
70            .all(|c| c.is_ascii_alphanumeric() || TOKEN_CHARSET_EXTRA.contains(c))
71        && token.chars().any(|c| c.is_ascii_uppercase())
72        && token.chars().any(|c| c.is_ascii_lowercase())
73        && token.chars().any(|c| c.is_ascii_digit())
74        && entropy_ratio(token) >= config.threshold
75        && !config.allowlist.iter().any(|re| re.is_match(token))
76        && !is_verified_digest(token)
77        && !is_structured_payload(token)
78}
79
80// Why: an SRI hash (`sha384-<base64>`) is public integrity metadata, not key
81// material, but its payload is dense base64 that clears every entropy check.
82// The exoneration is length-verified rather than prefix-trusted: a credential
83// smuggled behind a `sha384-` prefix decodes to the wrong byte count and is
84// still reported.
85fn is_verified_digest(token: &str) -> bool {
86    let Some((prefix, payload)) = token.split_once('-') else {
87        return false;
88    };
89    DIGEST_LENGTHS
90        .iter()
91        .find(|(algo, _)| algo.eq_ignore_ascii_case(prefix))
92        .is_some_and(|&(_, digest_len)| {
93            decode_base64(payload).is_some_and(|bytes| bytes.len() == digest_len)
94        })
95}
96
97fn shannon_entropy(s: &str) -> f64 {
98    let mut counts = [0u32; 256];
99    let bytes = s.as_bytes();
100    for &b in bytes {
101        counts[usize::from(b)] += 1;
102    }
103    let len = bytes.len() as f64;
104    counts
105        .iter()
106        .filter(|&&c| c > 0)
107        .map(|&c| {
108            let p = f64::from(c) / len;
109            -p * p.log2()
110        })
111        .sum()
112}
113
114fn entropy_ratio(s: &str) -> f64 {
115    let symbols = s.len().min(ENTROPY_CEILING_SYMBOLS);
116    let ceiling = (symbols as f64).log2();
117    if ceiling <= 0.0 {
118        return 0.0;
119    }
120    shannon_entropy(s) / ceiling
121}
122
123fn is_structured_payload(token: &str) -> bool {
124    decoded_payload(token).is_some_and(|bytes| {
125        bytes.len() >= MIN_STRUCTURED_LEN && (is_mostly_text(&bytes) || is_protobuf(&bytes, 0))
126    })
127}
128
129// Why: a `name-<base64>` token never decodes as a whole — the prefix is not
130// base64 — which used to defeat the structured-payload discriminator for
131// exactly the prefixed-payload shapes it exists to exonerate. A short
132// alphanumeric prefix is stripped and the remainder given the same chance.
133fn decoded_payload(token: &str) -> Option<Vec<u8>> {
134    decode_base64(token).or_else(|| {
135        let (prefix, payload) = token.split_once('-')?;
136        let plausible_prefix = prefix.len() <= MAX_PAYLOAD_PREFIX_LEN
137            && prefix.chars().all(|c| c.is_ascii_alphanumeric());
138        plausible_prefix.then(|| decode_base64(payload)).flatten()
139    })
140}
141
142fn decode_base64(token: &str) -> Option<Vec<u8>> {
143    let body = token.trim_end_matches('=');
144    STANDARD_NO_PAD
145        .decode(body)
146        .or_else(|_| URL_SAFE_NO_PAD.decode(body))
147        .ok()
148}
149
150fn is_mostly_text(bytes: &[u8]) -> bool {
151    if bytes.is_empty() {
152        return false;
153    }
154    let printable = bytes
155        .iter()
156        .filter(|&&b| matches!(b, b'\t' | b'\n' | b'\r') || (0x20..0x7f).contains(&b))
157        .count();
158    printable * TEXT_RATIO_DENOMINATOR >= bytes.len() * TEXT_RATIO_NUMERATOR
159}
160
161// Why: exact buffer consumption alone is a weak signal on a short random blob,
162// so a decode only counts as protobuf when it also carries at least two fields
163// and one length-delimited payload that is itself text or protobuf. Random key
164// material clears all three by accident far less than one time in a hundred; a
165// real serialised message clears them by construction.
166fn is_protobuf(bytes: &[u8], depth: u32) -> bool {
167    if depth > MAX_NESTING_DEPTH || bytes.len() < MIN_STRUCTURED_LEN {
168        return false;
169    }
170    parse_protobuf(bytes, depth).is_some_and(|parsed| parsed.fields >= 2 && parsed.nested_structure)
171}
172
173struct ParsedMessage {
174    fields: usize,
175    nested_structure: bool,
176}
177
178fn parse_protobuf(bytes: &[u8], depth: u32) -> Option<ParsedMessage> {
179    let mut cursor = 0usize;
180    let mut parsed = ParsedMessage {
181        fields: 0,
182        nested_structure: false,
183    };
184    while cursor < bytes.len() {
185        let (tag, after_tag) = read_varint(bytes, cursor)?;
186        let field_number = tag >> 3;
187        if field_number == 0 || field_number > MAX_FIELD_NUMBER {
188            return None;
189        }
190        cursor = match tag & 7 {
191            0 => read_varint(bytes, after_tag)?.1,
192            1 => advance(bytes, after_tag, 8)?,
193            5 => advance(bytes, after_tag, 4)?,
194            2 => {
195                let (len, after_len) = read_varint(bytes, after_tag)?;
196                let len = usize::try_from(len).ok()?;
197                let end = advance(bytes, after_len, len)?;
198                let payload = bytes.get(after_len..end)?;
199                if payload.len() >= MIN_NESTED_PAYLOAD_LEN
200                    && (is_mostly_text(payload) || is_protobuf(payload, depth + 1))
201                {
202                    parsed.nested_structure = true;
203                }
204                end
205            },
206            _ => return None,
207        };
208        parsed.fields += 1;
209    }
210    Some(parsed)
211}
212
213fn advance(bytes: &[u8], cursor: usize, by: usize) -> Option<usize> {
214    let end = cursor.checked_add(by)?;
215    (end <= bytes.len()).then_some(end)
216}
217
218fn read_varint(bytes: &[u8], start: usize) -> Option<(u64, usize)> {
219    let mut value = 0u64;
220    let mut shift = 0u32;
221    let mut cursor = start;
222    loop {
223        let byte = *bytes.get(cursor)?;
224        cursor += 1;
225        value |= u64::from(byte & 0x7f) << shift;
226        if byte & 0x80 == 0 {
227            return Some((value, cursor));
228        }
229        shift += 7;
230        if shift >= 64 {
231            return None;
232        }
233    }
234}
systemprompt_security/policy/secrets/entropy.rs

systemprompt_security/policy/secrets/
entropy.rs
