provenant/utils/file/
encoding.rs

1// SPDX-FileCopyrightText: nexB Inc. and others
2// SPDX-FileCopyrightText: Provenant contributors
3// SPDX-License-Identifier: Apache-2.0
4// Derived from ScanCode Toolkit (Apache-2.0); modified. See NOTICE.
5
6//! Byte-to-text decoding and "does this look like text" heuristics shared by
7//! classification and text-extraction logic.
8
9const BINARY_CONTROL_CHAR_THRESHOLD_DIVISOR: usize = 10;
10
11pub(super) const CORRUPTED_UTF16_BOM_PREFIX: &[u8] = &[0xEF, 0xBF, 0xBD, 0xEF, 0xBF, 0xBD];
12
13/// Decode a byte buffer to a String, trying UTF-16 first when the byte shape
14/// strongly suggests it, then UTF-8, then Latin-1.
15///
16/// Latin-1 (ISO-8859-1) maps bytes 0x00-0xFF directly to Unicode U+0000-U+00FF,
17/// so it can decode any byte sequence. This matches Python ScanCode's use of
18/// `UnicodeDammit` which auto-detects encoding with Latin-1 as fallback.
19pub fn decode_bytes_to_string(bytes: &[u8]) -> String {
20    if let Some(decoded) = decode_utf16_text(bytes) {
21        return decoded;
22    }
23
24    match String::from_utf8(bytes.to_vec()) {
25        Ok(s) => s,
26        Err(e) => {
27            let bytes = e.into_bytes();
28            if has_binary_control_chars(&bytes) {
29                return String::new();
30            }
31            bytes.iter().map(|&b| b as char).collect()
32        }
33    }
34}
35
36pub(super) fn is_utf8_text(bytes: &[u8]) -> bool {
37    std::str::from_utf8(bytes).is_ok()
38}
39
40fn strip_corrupted_utf16_bom_prefix(bytes: &[u8]) -> &[u8] {
41    bytes
42        .strip_prefix(CORRUPTED_UTF16_BOM_PREFIX)
43        .unwrap_or(bytes)
44}
45
46fn decode_utf16_units(bytes: &[u8], is_le: bool, require_text_shape: bool) -> Option<String> {
47    if bytes.is_empty() || !bytes.len().is_multiple_of(2) {
48        return None;
49    }
50
51    let code_units: Vec<u16> = bytes
52        .chunks_exact(2)
53        .map(|chunk| {
54            if is_le {
55                u16::from_le_bytes([chunk[0], chunk[1]])
56            } else {
57                u16::from_be_bytes([chunk[0], chunk[1]])
58            }
59        })
60        .collect();
61
62    let decoded = std::char::decode_utf16(code_units)
63        .collect::<Result<String, _>>()
64        .ok()?;
65
66    if !require_text_shape {
67        return (!decoded.contains('\0')).then_some(decoded);
68    }
69
70    if !looks_like_decoded_text(&decoded) {
71        return None;
72    }
73
74    Some(decoded)
75}
76
77pub(super) fn looks_like_decoded_text(decoded: &str) -> bool {
78    if decoded
79        .chars()
80        .any(|ch| ch.is_control() && !matches!(ch, '\n' | '\r' | '\t'))
81    {
82        return false;
83    }
84
85    let visible = decoded
86        .chars()
87        .filter(|ch| !ch.is_control() || matches!(ch, '\n' | '\r' | '\t'))
88        .count();
89    if visible < 3 || decoded.contains('\0') {
90        return false;
91    }
92
93    let alpha = decoded.chars().filter(|ch| ch.is_alphabetic()).count();
94    let punctuation = decoded
95        .chars()
96        .filter(|ch| {
97            matches!(
98                ch,
99                '{' | '}'
100                    | '['
101                    | ']'
102                    | '<'
103                    | '>'
104                    | '('
105                    | ')'
106                    | ':'
107                    | ';'
108                    | ','
109                    | '"'
110                    | '\''
111                    | '/'
112                    | '='
113                    | '-'
114                    | '_'
115                    | '#'
116                    | '!'
117            )
118        })
119        .count();
120    let whitespace = decoded.chars().filter(|ch| ch.is_whitespace()).count();
121
122    let textish = alpha + punctuation + whitespace;
123    textish + (visible / 5) >= visible && (alpha > 0 || punctuation >= 2)
124}
125
126fn detect_utf16_endianness(bytes: &[u8]) -> Option<bool> {
127    let stripped = strip_corrupted_utf16_bom_prefix(bytes);
128    if stripped.len() < 4 || !stripped.len().is_multiple_of(2) {
129        return None;
130    }
131
132    let pair_count = stripped.len() / 2;
133    let even_zero = stripped.iter().step_by(2).filter(|&&b| b == 0).count();
134    let odd_zero = stripped
135        .iter()
136        .skip(1)
137        .step_by(2)
138        .filter(|&&b| b == 0)
139        .count();
140
141    let looks_like_be = even_zero * 3 >= pair_count && odd_zero * 6 <= pair_count;
142    let looks_like_le = odd_zero * 3 >= pair_count && even_zero * 6 <= pair_count;
143
144    match (looks_like_le, looks_like_be) {
145        (true, false) => Some(true),
146        (false, true) => Some(false),
147        (true, true) => Some(true),
148        (false, false) => None,
149    }
150}
151
152pub(super) fn decode_utf16_text(bytes: &[u8]) -> Option<String> {
153    if let Some(decoded) = decode_utf16_bom_text(bytes) {
154        return Some(decoded);
155    }
156
157    let stripped = strip_corrupted_utf16_bom_prefix(bytes);
158    match detect_utf16_endianness(bytes) {
159        Some(true) => decode_utf16_units(stripped, true, true),
160        Some(false) => decode_utf16_units(stripped, false, true),
161        None => None,
162    }
163}
164
165pub(super) fn decode_utf16_json_text(bytes: &[u8]) -> Option<String> {
166    if bytes.len() >= 2 {
167        let (is_le, body) = match bytes {
168            [0xFF, 0xFE, rest @ ..] => (true, rest),
169            [0xFE, 0xFF, rest @ ..] => (false, rest),
170            _ => {
171                let stripped = strip_corrupted_utf16_bom_prefix(bytes);
172                return match detect_utf16_endianness(bytes) {
173                    Some(true) => decode_utf16_units(stripped, true, false),
174                    Some(false) => decode_utf16_units(stripped, false, false),
175                    None => None,
176                };
177            }
178        };
179
180        if body.is_empty() || !body.len().is_multiple_of(2) {
181            return None;
182        }
183
184        return decode_utf16_units(body, is_le, false);
185    }
186
187    None
188}
189
190fn decode_utf16_bom_text(bytes: &[u8]) -> Option<String> {
191    if bytes.len() < 2 || !bytes.len().is_multiple_of(2) {
192        return None;
193    }
194
195    let (is_le, body) = match bytes {
196        [0xFF, 0xFE, rest @ ..] => (true, rest),
197        [0xFE, 0xFF, rest @ ..] => (false, rest),
198        _ => return None,
199    };
200
201    if body.is_empty() || body.len() % 2 != 0 {
202        return None;
203    }
204
205    decode_utf16_units(body, is_le, true)
206}
207
208pub(super) fn has_binary_control_chars(bytes: &[u8]) -> bool {
209    let control_count = bytes
210        .iter()
211        .filter(|&&b| b < 0x09 || (b > 0x0D && b < 0x20))
212        .count();
213    control_count > bytes.len() / BINARY_CONTROL_CHAR_THRESHOLD_DIVISOR
214}
215
216pub(super) fn has_decodable_text(bytes: &[u8]) -> bool {
217    bytes.is_empty()
218        || is_utf8_text(bytes)
219        || decode_utf16_text(bytes).is_some()
220        || !has_binary_control_chars(bytes)
221}
222
223pub(super) fn looks_like_textual_bytes(bytes: &[u8]) -> bool {
224    if bytes.is_empty() || is_utf8_text(bytes) {
225        return true;
226    }
227    if let Some(decoded) = decode_utf16_text(bytes) {
228        return decoded
229            .chars()
230            .any(|ch| !ch.is_control() || matches!(ch, '\n' | '\r' | '\t'));
231    }
232
233    let printable_count = bytes
234        .iter()
235        .filter(|&&b| matches!(b, b'\n' | b'\r' | b'\t') || (0x20..=0x7e).contains(&b))
236        .count();
237    printable_count * 2 >= bytes.len()
238}
provenant/utils/file/encoding.rs

provenant/utils/file/
encoding.rs
