guard_core_engine/
binary_islands.rs

1//! Binary islands reduction, ported from `guard_core/detection_engine/
2//! binary_islands.py` (upstream commit 5f399234).
3//!
4//! A multipart file-part payload whose binary artifact characters fill at
5//! least a fifth of it is reduced to its printable runs before pattern
6//! scanning: compressed or encrypted upload bytes stop producing attack-shaped
7//! matches whose rate grows with file size, while text genuinely embedded in
8//! an upload (a script inside a PDF, a stored path inside an archive) forms
9//! printable runs past the minimum length and is still scanned in full.
10//!
11//! String model: like the binary artifact gate ([`crate::patterns::binary`]),
12//! artifact classes are evaluated over decoded code points. Invalid UTF-8
13//! bytes of an engine body surface as U+FFFD at the decode boundary (Python
14//! decodes with surrogateescape, so one binary byte is one surrogate rune
15//! there and one U+FFFD per maximal invalid run here); U+FFFD is part of the
16//! artifact class and is excluded from the printable-run class, so binary
17//! bytes end islands exactly like the surrogateescape range does in Python.
18
19/// `_BINARY_LIKE_ARTIFACT_RATIO`: a payload is binary-like when artifact
20/// characters make up at least this fraction of it.
21pub const BINARY_LIKE_ARTIFACT_RATIO: f64 = 0.2;
22
23/// `value_is_binary_like`: the artifact-character ratio of the content
24/// reaches [`BINARY_LIKE_ARTIFACT_RATIO`].
25#[must_use]
26pub fn value_is_binary_like(content: &str) -> bool {
27    if content.is_empty() {
28        return false;
29    }
30    let total = content.chars().count();
31    let artifacts = content
32        .chars()
33        .filter(|c| crate::patterns::binary::is_binary_artifact(*c))
34        .count();
35    (artifacts as f64) / (total as f64) >= BINARY_LIKE_ARTIFACT_RATIO
36}
37
38/// Whether `r` belongs to the island run class `_ISLAND_RUN_RE` from
39/// `guard_core/detection_engine/binary_islands.py`.
40///
41/// That class is tab, newline, carriage return, printable ASCII, and the wide
42/// non-control Unicode ranges. The Unicode replacement character (and every
43/// other artifact) is excluded, so binary bytes end a run.
44#[must_use]
45pub fn printable_island_rune(r: char) -> bool {
46    let cp = r as u32;
47    matches!(cp, 0x09 | 0x0A | 0x0D)
48        || (0x20..=0x7E).contains(&cp)
49        || (0xA1..=0xD7FF).contains(&cp)
50        || (0xE000..=0xFFFC).contains(&cp)
51        || (0xFFFE..=0xFFFF).contains(&cp)
52        || (0x1_0000..=0x10_FFFF).contains(&cp)
53}
54
55/// `extract_binary_islands`: the maximal printable runs of the content whose
56/// code-point length reaches `min_run_length`, in order.
57///
58/// A minimum of 1 or below keeps the whole content (the reference returns
59/// `[content]`).
60#[must_use]
61pub fn extract_binary_islands(content: &str, min_run_length: usize) -> Vec<&str> {
62    if min_run_length <= 1 {
63        return vec![content];
64    }
65    let mut islands = Vec::new();
66    let mut run_start: Option<usize> = None;
67    let mut run_len = 0_usize;
68    for (offset, r) in content.char_indices() {
69        if printable_island_rune(r) {
70            if run_start.is_none() {
71                run_start = Some(offset);
72            }
73            run_len += 1;
74            continue;
75        }
76        if let Some(start) = run_start
77            && run_len >= min_run_length
78        {
79            islands.push(&content[start..offset]);
80        }
81        run_start = None;
82        run_len = 0;
83    }
84    if let Some(start) = run_start
85        && run_len >= min_run_length
86    {
87        islands.push(&content[start..]);
88    }
89    islands
90}
91
92#[cfg(test)]
93mod tests {
94    use super::*;
95
96    // The Python tests decode binary fixtures with surrogateescape; the
97    // engine's string model maps undecodable bytes to U+FFFD at the decode
98    // boundary, so the fixtures here embed U+FFFD (or ASCII control bytes,
99    // which are artifacts too) directly.
100
101    #[test]
102    fn extract_keeps_runs_at_or_above_min_length() {
103        let content = "\u{FFFD}abc\u{FFFD}".to_owned() + &"x".repeat(16) + "\u{FFFD}def\u{FFFD}";
104        assert_eq!(extract_binary_islands(&content, 16), vec!["x".repeat(16)]);
105    }
106
107    #[test]
108    fn extract_returns_runs_separately() {
109        let content =
110            "\u{FFFD}".to_owned() + &"a".repeat(16) + "\u{FFFD}" + &"b".repeat(16) + "\u{FFFD}";
111        assert_eq!(
112            extract_binary_islands(&content, 16),
113            vec!["a".repeat(16), "b".repeat(16)]
114        );
115    }
116
117    #[test]
118    fn extract_preserves_non_ascii_text_runs() {
119        let text = "Caf\u{e9} r\u{e9}sum\u{e9} na\u{ef}ve d\u{e9}cor s\u{e9}lection";
120        assert_eq!(extract_binary_islands(text, 16), vec![text]);
121    }
122
123    #[test]
124    fn extract_below_min_run_length_returns_content() {
125        let content = "anything\u{0}at all";
126        assert_eq!(extract_binary_islands(content, 1), vec![content]);
127    }
128
129    #[test]
130    fn extract_keeps_tab_newline_carriage_return_inside_runs() {
131        let content = "\u{0}select 1\nfrom t\r\nwhere x=1\u{0}";
132        assert_eq!(
133            extract_binary_islands(content, 16),
134            vec!["select 1\nfrom t\r\nwhere x=1"]
135        );
136    }
137
138    #[test]
139    fn binary_like_rejects_text_and_accepts_noise() {
140        assert!(!value_is_binary_like(""));
141        assert!(!value_is_binary_like(
142            "plain text body with attack 1 OR 1=1"
143        ));
144        assert!(!value_is_binary_like("one null\u{0}byte"));
145        assert!(value_is_binary_like(&noise_string(7, 4096)));
146    }
147
148    #[test]
149    fn replacement_character_is_an_artifact_and_ends_runs() {
150        // U+FFFD counts toward the artifact ratio ...
151        assert!(value_is_binary_like(&("\u{FFFD}".repeat(2) + "ab")));
152        // ... and is excluded from the printable-run class (the Python class
153        // deliberately skips U+FFFD: its run range is E000..FFFC, FFFE..).
154        let content = "a".repeat(16);
155        let islands = extract_binary_islands(&content, 16);
156        assert_eq!(islands, vec!["a".repeat(16)]);
157        let split = "\u{FFFD}x".repeat(16);
158        assert!(extract_binary_islands(&split, 16).is_empty());
159    }
160
161    /// Deterministic pseudo-random "noise" text: the engine's string model of
162    /// random raw bytes (ASCII bytes pass through, with the control range
163    /// counting as artifacts; every other byte decodes to U+FFFD, also an
164    /// artifact).
165    fn noise_string(seed: u64, size: usize) -> String {
166        let mut state = seed.wrapping_mul(0x9E37_79B9_7F4A_7C15).max(1);
167        let mut out = String::with_capacity(size);
168        for _ in 0..size {
169            state ^= state << 13;
170            state ^= state >> 7;
171            state ^= state << 17;
172            let byte = (state % 256) as u8;
173            if byte < 0x80 {
174                out.push(byte as char);
175            } else {
176                out.push('\u{FFFD}');
177            }
178        }
179        out
180    }
181}
guard_core_engine/binary_islands.rs

guard_core_engine/
binary_islands.rs
