guard_core_engine/patterns/
binary.rs

1//! Binary artifact density gate and binary-content heuristic, ported from
2//! `guard_core/detection_engine/binary_prefix.py` and `semantic.py`
3//! (`looks_like_binary_content`).
4//!
5//! String-model mapping: the reference scans decoded text where undecodable
6//! bytes surface as surrogateescape code points (U+DC80..U+DCFF) and mojibake
7//! as U+FFFD; both classes count as artifacts. The engine scans `&str` (always
8//! valid UTF-8), so undecodable bytes never reach this module as surrogates;
9//! the port's decode boundary represents them as U+FFFD (or drops them), which
10//! is an artifact class here. The surrogate branch is kept for parity with the
11//! reference class definition but is unreachable for `char`. All window
12//! arithmetic runs on code-point indices, the same index space the corpus
13//! mandates for threat positions.
14
15/// Cumulative artifact-count prefix: `counts[i]` = artifacts in `s[..i]`.
16/// Built over code points; indexing is by code-point index (the same space
17/// the structural matchers and threat positions use).
18#[must_use]
19pub fn build_binary_prefix(text: &str) -> Vec<u32> {
20    let mut counts = Vec::with_capacity(text.chars().count() + 1);
21    counts.push(0u32);
22    let mut count = 0u32;
23    for c in text.chars() {
24        if is_binary_artifact(c) {
25            count += 1;
26        }
27        counts.push(count);
28    }
29    counts
30}
31
32const BINARY_DENSITY_RADIUS: usize = 64;
33const BINARY_DENSITY_LIMIT: u32 = 4;
34
35#[must_use]
36pub fn match_is_binary_density(
37    binary_prefix: &[u32],
38    match_start: usize,
39    match_end: usize,
40) -> bool {
41    if binary_prefix.is_empty() {
42        return false;
43    }
44    let high = (match_end + BINARY_DENSITY_RADIUS).min(binary_prefix.len() - 1);
45    let low = match_start.saturating_sub(BINARY_DENSITY_RADIUS);
46    binary_prefix[high].saturating_sub(binary_prefix[low]) >= BINARY_DENSITY_LIMIT
47}
48
49#[must_use]
50pub(crate) fn is_binary_artifact(c: char) -> bool {
51    let cp = c as u32;
52    // control characters (except tab/LF/CR) and DEL
53    if (cp < 0x20 && !matches!(cp, 0x09 | 0x0A | 0x0D)) || cp == 0x7F {
54        return true;
55    }
56    // replacement character
57    if cp == 0xFFFD {
58        return true;
59    }
60    #[cfg(not(coverage))] // unreachable: Rust chars exclude the surrogate
61    // range, so the surrogateescape byte marks never appear as chars
62    if (0xDC80..=0xDCFF).contains(&cp) {
63        return true;
64    }
65    // Latin-1/Latin-Ext-A artifact, minus the text allowlist
66    if (0x80..=0x024F).contains(&cp) {
67        return !is_text_allowlist(c);
68    }
69    false
70}
71
72/// The only characters in `0x80..=0x024F` treated as text. Mirrors the
73/// reference class complement exactly: within `0x80..=0xBF` only the
74/// punctuation/marker singles are text, and all of `0xC0..=0xFF` (including
75/// the multiplication and division signs) plus `0x100..=0x17F` are text.
76#[must_use]
77fn is_text_allowlist(c: char) -> bool {
78    let cp = c as u32;
79    if (0x00C0..=0x00FF).contains(&cp) {
80        return true;
81    }
82    if matches!(
83        cp,
84        0x00A3 | 0x00A5 | 0x00AA | 0x00B0 | 0x00B1 | 0x00B2 | 0x00B3 | 0x00B5 | 0x00B9 | 0x00BA
85    ) {
86        return true;
87    }
88    (0x0100..=0x017F).contains(&cp)
89}
90
91const BINARY_CONTENT_RATIO_THRESHOLD: f64 = 0.2;
92
93/// `looks_like_binary_content` from semantic.py.
94#[must_use]
95pub fn looks_like_binary_content(content: &str) -> bool {
96    if content.is_empty() {
97        return false;
98    }
99    let mut non_text = 0usize;
100    let mut len = 0usize;
101    for c in content.chars() {
102        len += 1;
103        let cp = c as u32;
104        let is_ws = matches!(c, '\t' | '\r' | '\n');
105        let printable = !c.is_control() && !matches!(cp, 0xFFFD);
106        if !is_ws && (!printable || cp == 0xFFFD) {
107            non_text += 1;
108        }
109    }
110    let ratio = non_text as f64 / len as f64;
111    ratio >= BINARY_CONTENT_RATIO_THRESHOLD
112}
113
114#[cfg(test)]
115mod tests {
116    use super::*;
117
118    #[test]
119    fn pure_text_has_zero_artifacts() {
120        let prefix = build_binary_prefix("hello <script> world\t\n");
121        assert_eq!(prefix.last(), Some(&0));
122    }
123
124    #[test]
125    fn control_bytes_count() {
126        let prefix = build_binary_prefix("a\u{1}b\u{2}c\u{3}d\u{4}");
127        assert!(match_is_binary_density(&prefix, 0, 8));
128    }
129
130    #[test]
131    fn accented_text_is_allowed() {
132        let prefix = build_binary_prefix("caf\u{00e9} cr\u{00e8}me br\u{00fb}l\u{00e9}");
133        assert_eq!(prefix.last(), Some(&0));
134    }
135
136    #[test]
137    fn artifact_classes_match_reference_exactly() {
138        // The reference `_BINARY_ARTIFACT_RE` class set, collapsed to ranges
139        // over the BMP ranges it covers (verified against guard-core 4.0.3).
140        // Everything below 0x300 outside these ranges is text.
141        const ARTIFACT_RANGES: &[(u32, u32)] = &[
142            (0x00, 0x08),
143            (0x0B, 0x0C),
144            (0x0E, 0x1F),
145            (0x7F, 0xA2),
146            (0xA4, 0xA4),
147            (0xA6, 0xA9),
148            (0xAB, 0xAF),
149            (0xB4, 0xB4),
150            (0xB6, 0xB8),
151            (0xBB, 0xBF),
152            (0x0180, 0x024F),
153        ];
154        let is_ref_artifact = |cp: u32| {
155            ARTIFACT_RANGES
156                .iter()
157                .any(|(lo, hi)| (*lo..=*hi).contains(&cp))
158        };
159        for cp in 0u32..=0x3000 {
160            let c = char::from_u32(cp).expect("BMP scalar");
161            assert_eq!(
162                is_binary_artifact(c),
163                is_ref_artifact(cp),
164                "artifact mismatch at U+{cp:04X}"
165            );
166        }
167    }
168
169    #[test]
170    fn latin1_punctuation_signs_are_text() {
171        // U+00D7 (multiplication sign) and U+00F7 (division sign) sit in the
172        // reference's text allowlist (all of 0xC0..=0xFF is text).
173        let prefix = build_binary_prefix("3 \u{00d7} 4 \u{00f7} 2");
174        assert_eq!(prefix.last(), Some(&0));
175    }
176}
177
178#[cfg(test)]
179mod coverage_tests {
180    use super::*;
181
182    #[test]
183    fn density_gate_needs_a_prefix_and_enough_high_bytes() {
184        // an empty prefix never gates
185        assert!(!match_is_binary_density(&[], 0, 1));
186        // fewer than the limit of high bytes in the window is fine
187        let prefix = vec![0_u32, 0, 0, 0, 3, 3];
188        assert!(!match_is_binary_density(&prefix, 2, 3));
189        // a dense window gates
190        let dense = vec![0_u32, 9, 9, 9, 9, 9];
191        assert!(match_is_binary_density(&dense, 2, 3));
192    }
193
194    #[test]
195    fn artifact_chars_cover_control_replacement_surrogate_and_latin() {
196        assert!(is_binary_artifact('\u{1}'));
197        assert!(is_binary_artifact('\u{7f}'));
198        assert!(is_binary_artifact('\u{fffd}'));
199        assert!(!is_binary_artifact(
200            char::from_u32(0x0301).expect("in range")
201        ));
202        assert!(is_binary_artifact('\u{80}'));
203        assert!(!is_binary_artifact('\t'));
204        assert!(!is_binary_artifact('a'));
205    }
206}
guard_core_engine/patterns/binary.rs

guard_core_engine/patterns/
binary.rs
