systemprompt_security/policy/secrets/
entropy.rs

1//! The high-entropy backstop for credentials carrying no vendor prefix.
2//!
3//! A random base64 blob pasted into a prompt matches none of
4//! [`super::SECRET_PATTERNS`] but still reads as machine-generated key
5//! material. Randomness alone cannot say so: a serialised protobuf, a base64
6//! JSON envelope, and a 32-byte key are all dense mixed-case base64 of similar
7//! measured entropy. [`is_structured_payload`] supplies the missing
8//! discriminator — key material decodes to bytes with no readable structure,
9//! whereas a tool result decodes to text or to a self-consistent wire format.
10//!
11//! `/` is a legal token character, so an absolute path is scored as one token
12//! rather than split on its separators. A macOS `$TMPDIR`
13//! (`/var/folders/<12>/<30>/T/`) clears every check that way — the random
14//! segment supplies the entropy, and the `T` supplies the uppercase the shape
15//! test demands — so [`is_filesystem_path`] exempts path-shaped tokens. Claude
16//! Code carries those paths in its system prompt, and without the exemption
17//! every request from an affected Mac is denied.
18//!
19//! Copyright (c) systemprompt.io — Business Source License 1.1.
20//! See <https://systemprompt.io> for licensing details.
21
22use base64::Engine as _;
23use base64::engine::general_purpose::{STANDARD_NO_PAD, URL_SAFE_NO_PAD};
24use regex::Regex;
25
26pub const DEFAULT_MIN_LEN: usize = 32;
27
28pub const DEFAULT_THRESHOLD: f64 = 0.80;
29
30const TOKEN_DELIMITERS: &str = "\"'`()[]{}<>,;:";
31const TOKEN_CHARSET_EXTRA: &str = "+/=_-";
32const ENTROPY_CEILING_SYMBOLS: usize = 64;
33
34const MIN_STRUCTURED_LEN: usize = 16;
35const MAX_PAYLOAD_PREFIX_LEN: usize = 10;
36const DIGEST_LENGTHS: [(&str, usize); 3] = [("sha256", 32), ("sha384", 48), ("sha512", 64)];
37const MAX_FIELD_NUMBER: u64 = 64;
38const MAX_NESTING_DEPTH: u32 = 4;
39const MIN_NESTED_PAYLOAD_LEN: usize = 4;
40const TEXT_RATIO_NUMERATOR: usize = 9;
41const TEXT_RATIO_DENOMINATOR: usize = 10;
42
43/// Tunables for the heuristic, read from the `secret_scan` policy's `entropy`
44/// block. [`Default`] reproduces the built-in behaviour, which is what every
45/// caller outside the policy chain gets.
46#[derive(Debug, Clone)]
47pub struct EntropyConfig {
48    pub enabled: bool,
49    pub min_len: usize,
50    pub threshold: f64,
51    pub allowlist: Vec<Regex>,
52}
53
54impl Default for EntropyConfig {
55    fn default() -> Self {
56        Self {
57            enabled: true,
58            min_len: DEFAULT_MIN_LEN,
59            threshold: DEFAULT_THRESHOLD,
60            allowlist: Vec::new(),
61        }
62    }
63}
64
65#[must_use]
66pub fn find_high_entropy_token<'a>(text: &'a str, config: &EntropyConfig) -> Option<&'a str> {
67    if !config.enabled {
68        return None;
69    }
70    text.split(|c: char| c.is_whitespace() || TOKEN_DELIMITERS.contains(c))
71        .find(|token| is_credential_shaped(token, config))
72}
73
74fn is_credential_shaped(token: &str, config: &EntropyConfig) -> bool {
75    has_credential_shape(token, config)
76        && !config.allowlist.iter().any(|re| re.is_match(token))
77        && !is_verified_digest(token)
78        && !is_structured_payload(token)
79        && !is_filesystem_path(token, config)
80}
81
82fn has_credential_shape(token: &str, config: &EntropyConfig) -> bool {
83    token.len() >= config.min_len
84        && token
85            .chars()
86            .all(|c| c.is_ascii_alphanumeric() || TOKEN_CHARSET_EXTRA.contains(c))
87        && token.chars().any(|c| c.is_ascii_uppercase())
88        && token.chars().any(|c| c.is_ascii_lowercase())
89        && token.chars().any(|c| c.is_ascii_digit())
90        && entropy_ratio(token) >= config.threshold
91}
92
93// Why: a whole path is one token, so exonerating it wholesale would let key
94// material hide in a segment — each segment is scored separately instead.
95// `+` and `=` are base64's own characters and effectively never appear in a
96// path, so their presence keeps the token in scope. Windows paths need no
97// handling: `:` is a delimiter and `\` is outside the charset, so they never
98// survive tokenisation as a single token.
99fn is_filesystem_path(token: &str, config: &EntropyConfig) -> bool {
100    token.starts_with('/')
101        && token.matches('/').count() >= 2
102        && !token.contains(['+', '='])
103        && !token
104            .split('/')
105            .any(|segment| has_credential_shape(segment, config))
106}
107
108// Why: an SRI hash (`sha384-<base64>`) is public integrity metadata, not key
109// material, but its payload is dense base64 that clears every entropy check.
110// The exoneration is length-verified rather than prefix-trusted: a credential
111// smuggled behind a `sha384-` prefix decodes to the wrong byte count and is
112// still reported.
113fn is_verified_digest(token: &str) -> bool {
114    let Some((prefix, payload)) = token.split_once('-') else {
115        return false;
116    };
117    DIGEST_LENGTHS
118        .iter()
119        .find(|(algo, _)| algo.eq_ignore_ascii_case(prefix))
120        .is_some_and(|&(_, digest_len)| {
121            decode_base64(payload).is_some_and(|bytes| bytes.len() == digest_len)
122        })
123}
124
125fn shannon_entropy(s: &str) -> f64 {
126    let mut counts = [0u32; 256];
127    let bytes = s.as_bytes();
128    for &b in bytes {
129        counts[usize::from(b)] += 1;
130    }
131    let len = bytes.len() as f64;
132    counts
133        .iter()
134        .filter(|&&c| c > 0)
135        .map(|&c| {
136            let p = f64::from(c) / len;
137            -p * p.log2()
138        })
139        .sum()
140}
141
142fn entropy_ratio(s: &str) -> f64 {
143    let symbols = s.len().min(ENTROPY_CEILING_SYMBOLS);
144    let ceiling = (symbols as f64).log2();
145    if ceiling <= 0.0 {
146        return 0.0;
147    }
148    shannon_entropy(s) / ceiling
149}
150
151fn is_structured_payload(token: &str) -> bool {
152    decoded_payload(token).is_some_and(|bytes| {
153        bytes.len() >= MIN_STRUCTURED_LEN && (is_mostly_text(&bytes) || is_protobuf(&bytes, 0))
154    })
155}
156
157// Why: a `name-<base64>` token never decodes as a whole — the prefix is not
158// base64 — which used to defeat the structured-payload discriminator for
159// exactly the prefixed-payload shapes it exists to exonerate. A short
160// alphanumeric prefix is stripped and the remainder given the same chance.
161fn decoded_payload(token: &str) -> Option<Vec<u8>> {
162    decode_base64(token).or_else(|| {
163        let (prefix, payload) = token.split_once('-')?;
164        let plausible_prefix = prefix.len() <= MAX_PAYLOAD_PREFIX_LEN
165            && prefix.chars().all(|c| c.is_ascii_alphanumeric());
166        plausible_prefix.then(|| decode_base64(payload)).flatten()
167    })
168}
169
170fn decode_base64(token: &str) -> Option<Vec<u8>> {
171    let body = token.trim_end_matches('=');
172    STANDARD_NO_PAD
173        .decode(body)
174        .or_else(|_| URL_SAFE_NO_PAD.decode(body))
175        .ok()
176}
177
178fn is_mostly_text(bytes: &[u8]) -> bool {
179    if bytes.is_empty() {
180        return false;
181    }
182    let printable = bytes
183        .iter()
184        .filter(|&&b| matches!(b, b'\t' | b'\n' | b'\r') || (0x20..0x7f).contains(&b))
185        .count();
186    printable * TEXT_RATIO_DENOMINATOR >= bytes.len() * TEXT_RATIO_NUMERATOR
187}
188
189// Why: exact buffer consumption alone is a weak signal on a short random blob,
190// so a decode only counts as protobuf when it also carries at least two fields
191// and one length-delimited payload that is itself text or protobuf. Random key
192// material clears all three by accident far less than one time in a hundred; a
193// real serialised message clears them by construction.
194fn is_protobuf(bytes: &[u8], depth: u32) -> bool {
195    if depth > MAX_NESTING_DEPTH || bytes.len() < MIN_STRUCTURED_LEN {
196        return false;
197    }
198    parse_protobuf(bytes, depth).is_some_and(|parsed| parsed.fields >= 2 && parsed.nested_structure)
199}
200
201struct ParsedMessage {
202    fields: usize,
203    nested_structure: bool,
204}
205
206fn parse_protobuf(bytes: &[u8], depth: u32) -> Option<ParsedMessage> {
207    let mut cursor = 0usize;
208    let mut parsed = ParsedMessage {
209        fields: 0,
210        nested_structure: false,
211    };
212    while cursor < bytes.len() {
213        let (tag, after_tag) = read_varint(bytes, cursor)?;
214        let field_number = tag >> 3;
215        if field_number == 0 || field_number > MAX_FIELD_NUMBER {
216            return None;
217        }
218        cursor = match tag & 7 {
219            0 => read_varint(bytes, after_tag)?.1,
220            1 => advance(bytes, after_tag, 8)?,
221            5 => advance(bytes, after_tag, 4)?,
222            2 => {
223                let (len, after_len) = read_varint(bytes, after_tag)?;
224                let len = usize::try_from(len).ok()?;
225                let end = advance(bytes, after_len, len)?;
226                let payload = bytes.get(after_len..end)?;
227                if payload.len() >= MIN_NESTED_PAYLOAD_LEN
228                    && (is_mostly_text(payload) || is_protobuf(payload, depth + 1))
229                {
230                    parsed.nested_structure = true;
231                }
232                end
233            },
234            _ => return None,
235        };
236        parsed.fields += 1;
237    }
238    Some(parsed)
239}
240
241fn advance(bytes: &[u8], cursor: usize, by: usize) -> Option<usize> {
242    let end = cursor.checked_add(by)?;
243    (end <= bytes.len()).then_some(end)
244}
245
246fn read_varint(bytes: &[u8], start: usize) -> Option<(u64, usize)> {
247    let mut value = 0u64;
248    let mut shift = 0u32;
249    let mut cursor = start;
250    loop {
251        let byte = *bytes.get(cursor)?;
252        cursor += 1;
253        value |= u64::from(byte & 0x7f) << shift;
254        if byte & 0x80 == 0 {
255            return Some((value, cursor));
256        }
257        shift += 7;
258        if shift >= 64 {
259            return None;
260        }
261    }
262}
systemprompt_security/policy/secrets/entropy.rs

systemprompt_security/policy/secrets/
entropy.rs
