provenant/utils/file/
encoding.rs

1// SPDX-FileCopyrightText: nexB Inc. and others
2// ScanCode is a trademark of nexB Inc.
3// SPDX-FileCopyrightText: Provenant contributors
4// SPDX-License-Identifier: Apache-2.0
5// Derived from ScanCode Toolkit (Apache-2.0); modified. See NOTICE.
6
7//! Byte-to-text decoding and "does this look like text" heuristics shared by
8//! classification and text-extraction logic.
9
10const BINARY_CONTROL_CHAR_THRESHOLD_DIVISOR: usize = 10;
11
12pub(super) const CORRUPTED_UTF16_BOM_PREFIX: &[u8] = &[0xEF, 0xBF, 0xBD, 0xEF, 0xBF, 0xBD];
13
14/// Diagnostic message emitted when invalid-UTF-8 input is dropped from
15/// detection because it exceeds the binary-control-char threshold. Surfaced via
16/// the scanner's structured `scan_diagnostics` channel so the skip is
17/// observable rather than silent.
18pub(super) const NEAR_BINARY_SKIP_DIAGNOSTIC: &str = "Text skipped from license/copyright detection: invalid UTF-8 with too many control bytes (likely binary)";
19
20/// Decode a byte buffer to a String, trying UTF-16 first when the byte shape
21/// strongly suggests it, then UTF-8, then Latin-1.
22///
23/// Latin-1 (ISO-8859-1) maps bytes 0x00-0xFF directly to Unicode U+0000-U+00FF,
24/// so it can decode any byte sequence. This matches Python ScanCode's use of
25/// `UnicodeDammit` which auto-detects encoding with Latin-1 as fallback.
26pub fn decode_bytes_to_string(bytes: &[u8]) -> String {
27    decode_bytes_to_string_with_diagnostic(bytes).0
28}
29
30/// Like [`decode_bytes_to_string`], but also returns a structured diagnostic
31/// when the result is the empty string because the input looked like binary
32/// (invalid UTF-8 with a high control-byte ratio). The decoded result is
33/// unchanged; only the optional diagnostic is added so the silent skip becomes
34/// observable in scan output.
35pub(super) fn decode_bytes_to_string_with_diagnostic(bytes: &[u8]) -> (String, Option<String>) {
36    if let Some(decoded) = decode_utf16_text(bytes) {
37        return (decoded, None);
38    }
39
40    match String::from_utf8(bytes.to_vec()) {
41        Ok(s) => (s, None),
42        Err(e) => {
43            let bytes = e.into_bytes();
44            if has_binary_control_chars(&bytes) {
45                return (String::new(), Some(NEAR_BINARY_SKIP_DIAGNOSTIC.to_string()));
46            }
47            (bytes.iter().map(|&b| b as char).collect(), None)
48        }
49    }
50}
51
52pub(super) fn is_utf8_text(bytes: &[u8]) -> bool {
53    std::str::from_utf8(bytes).is_ok()
54}
55
56fn strip_corrupted_utf16_bom_prefix(bytes: &[u8]) -> &[u8] {
57    bytes
58        .strip_prefix(CORRUPTED_UTF16_BOM_PREFIX)
59        .unwrap_or(bytes)
60}
61
62fn decode_utf16_units(bytes: &[u8], is_le: bool, require_text_shape: bool) -> Option<String> {
63    if bytes.is_empty() || !bytes.len().is_multiple_of(2) {
64        return None;
65    }
66
67    let code_units: Vec<u16> = bytes
68        .chunks_exact(2)
69        .map(|chunk| {
70            if is_le {
71                u16::from_le_bytes([chunk[0], chunk[1]])
72            } else {
73                u16::from_be_bytes([chunk[0], chunk[1]])
74            }
75        })
76        .collect();
77
78    let decoded = std::char::decode_utf16(code_units)
79        .collect::<Result<String, _>>()
80        .ok()?;
81
82    if !require_text_shape {
83        return (!decoded.contains('\0')).then_some(decoded);
84    }
85
86    if !looks_like_decoded_text(&decoded) {
87        return None;
88    }
89
90    Some(decoded)
91}
92
93pub(super) fn looks_like_decoded_text(decoded: &str) -> bool {
94    if decoded
95        .chars()
96        .any(|ch| ch.is_control() && !matches!(ch, '\n' | '\r' | '\t'))
97    {
98        return false;
99    }
100
101    let visible = decoded
102        .chars()
103        .filter(|ch| !ch.is_control() || matches!(ch, '\n' | '\r' | '\t'))
104        .count();
105    if visible < 3 || decoded.contains('\0') {
106        return false;
107    }
108
109    let alpha = decoded.chars().filter(|ch| ch.is_alphabetic()).count();
110    let punctuation = decoded
111        .chars()
112        .filter(|ch| {
113            matches!(
114                ch,
115                '{' | '}'
116                    | '['
117                    | ']'
118                    | '<'
119                    | '>'
120                    | '('
121                    | ')'
122                    | ':'
123                    | ';'
124                    | ','
125                    | '"'
126                    | '\''
127                    | '/'
128                    | '='
129                    | '-'
130                    | '_'
131                    | '#'
132                    | '!'
133            )
134        })
135        .count();
136    let whitespace = decoded.chars().filter(|ch| ch.is_whitespace()).count();
137
138    let textish = alpha + punctuation + whitespace;
139    textish + (visible / 5) >= visible && (alpha > 0 || punctuation >= 2)
140}
141
142fn detect_utf16_endianness(bytes: &[u8]) -> Option<bool> {
143    let stripped = strip_corrupted_utf16_bom_prefix(bytes);
144    if stripped.len() < 4 || !stripped.len().is_multiple_of(2) {
145        return None;
146    }
147
148    let pair_count = stripped.len() / 2;
149    let even_zero = stripped.iter().step_by(2).filter(|&&b| b == 0).count();
150    let odd_zero = stripped
151        .iter()
152        .skip(1)
153        .step_by(2)
154        .filter(|&&b| b == 0)
155        .count();
156
157    let looks_like_be = even_zero * 3 >= pair_count && odd_zero * 6 <= pair_count;
158    let looks_like_le = odd_zero * 3 >= pair_count && even_zero * 6 <= pair_count;
159
160    match (looks_like_le, looks_like_be) {
161        (true, false) => Some(true),
162        (false, true) => Some(false),
163        (true, true) => Some(true),
164        (false, false) => None,
165    }
166}
167
168pub(super) fn decode_utf16_text(bytes: &[u8]) -> Option<String> {
169    if let Some(decoded) = decode_utf16_bom_text(bytes) {
170        return Some(decoded);
171    }
172
173    let stripped = strip_corrupted_utf16_bom_prefix(bytes);
174    match detect_utf16_endianness(bytes) {
175        Some(true) => decode_utf16_units(stripped, true, true),
176        Some(false) => decode_utf16_units(stripped, false, true),
177        None => None,
178    }
179}
180
181pub(super) fn decode_utf16_json_text(bytes: &[u8]) -> Option<String> {
182    if bytes.len() >= 2 {
183        let (is_le, body) = match bytes {
184            [0xFF, 0xFE, rest @ ..] => (true, rest),
185            [0xFE, 0xFF, rest @ ..] => (false, rest),
186            _ => {
187                let stripped = strip_corrupted_utf16_bom_prefix(bytes);
188                return match detect_utf16_endianness(bytes) {
189                    Some(true) => decode_utf16_units(stripped, true, false),
190                    Some(false) => decode_utf16_units(stripped, false, false),
191                    None => None,
192                };
193            }
194        };
195
196        if body.is_empty() || !body.len().is_multiple_of(2) {
197            return None;
198        }
199
200        return decode_utf16_units(body, is_le, false);
201    }
202
203    None
204}
205
206fn decode_utf16_bom_text(bytes: &[u8]) -> Option<String> {
207    if bytes.len() < 2 || !bytes.len().is_multiple_of(2) {
208        return None;
209    }
210
211    let (is_le, body) = match bytes {
212        [0xFF, 0xFE, rest @ ..] => (true, rest),
213        [0xFE, 0xFF, rest @ ..] => (false, rest),
214        _ => return None,
215    };
216
217    if body.is_empty() || body.len() % 2 != 0 {
218        return None;
219    }
220
221    decode_utf16_units(body, is_le, true)
222}
223
224pub(super) fn has_binary_control_chars(bytes: &[u8]) -> bool {
225    let control_count = bytes
226        .iter()
227        .filter(|&&b| b < 0x09 || (b > 0x0D && b < 0x20))
228        .count();
229    control_count > bytes.len() / BINARY_CONTROL_CHAR_THRESHOLD_DIVISOR
230}
231
232pub(super) fn has_decodable_text(bytes: &[u8]) -> bool {
233    bytes.is_empty()
234        || is_utf8_text(bytes)
235        || decode_utf16_text(bytes).is_some()
236        || !has_binary_control_chars(bytes)
237}
238
239pub(super) fn looks_like_textual_bytes(bytes: &[u8]) -> bool {
240    if bytes.is_empty() || is_utf8_text(bytes) {
241        return true;
242    }
243    if let Some(decoded) = decode_utf16_text(bytes) {
244        return decoded
245            .chars()
246            .any(|ch| !ch.is_control() || matches!(ch, '\n' | '\r' | '\t'));
247    }
248
249    let printable_count = bytes
250        .iter()
251        .filter(|&&b| matches!(b, b'\n' | b'\r' | b'\t') || (0x20..=0x7e).contains(&b))
252        .count();
253    printable_count * 2 >= bytes.len()
254}
provenant/utils/file/encoding.rs

provenant/utils/file/
encoding.rs
