provenant/utils/file/
encoding.rs

1// SPDX-FileCopyrightText: nexB Inc. and others
2// SPDX-FileCopyrightText: Provenant contributors
3// SPDX-License-Identifier: Apache-2.0
4// Derived from ScanCode Toolkit (Apache-2.0); modified. See NOTICE.
5
6//! Byte-to-text decoding and "does this look like text" heuristics shared by
7//! classification and text-extraction logic.
8
9const BINARY_CONTROL_CHAR_THRESHOLD_DIVISOR: usize = 10;
10
11pub(super) const CORRUPTED_UTF16_BOM_PREFIX: &[u8] = &[0xEF, 0xBF, 0xBD, 0xEF, 0xBF, 0xBD];
12
13/// Diagnostic message emitted when invalid-UTF-8 input is dropped from
14/// detection because it exceeds the binary-control-char threshold. Surfaced via
15/// the scanner's structured `scan_diagnostics` channel so the skip is
16/// observable rather than silent.
17pub(super) const NEAR_BINARY_SKIP_DIAGNOSTIC: &str = "Text skipped from license/copyright detection: invalid UTF-8 with too many control bytes (likely binary)";
18
19/// Decode a byte buffer to a String, trying UTF-16 first when the byte shape
20/// strongly suggests it, then UTF-8, then Latin-1.
21///
22/// Latin-1 (ISO-8859-1) maps bytes 0x00-0xFF directly to Unicode U+0000-U+00FF,
23/// so it can decode any byte sequence. This matches Python ScanCode's use of
24/// `UnicodeDammit` which auto-detects encoding with Latin-1 as fallback.
25pub fn decode_bytes_to_string(bytes: &[u8]) -> String {
26    decode_bytes_to_string_with_diagnostic(bytes).0
27}
28
29/// Like [`decode_bytes_to_string`], but also returns a structured diagnostic
30/// when the result is the empty string because the input looked like binary
31/// (invalid UTF-8 with a high control-byte ratio). The decoded result is
32/// unchanged; only the optional diagnostic is added so the silent skip becomes
33/// observable in scan output.
34pub(super) fn decode_bytes_to_string_with_diagnostic(bytes: &[u8]) -> (String, Option<String>) {
35    if let Some(decoded) = decode_utf16_text(bytes) {
36        return (decoded, None);
37    }
38
39    match String::from_utf8(bytes.to_vec()) {
40        Ok(s) => (s, None),
41        Err(e) => {
42            let bytes = e.into_bytes();
43            if has_binary_control_chars(&bytes) {
44                return (String::new(), Some(NEAR_BINARY_SKIP_DIAGNOSTIC.to_string()));
45            }
46            (bytes.iter().map(|&b| b as char).collect(), None)
47        }
48    }
49}
50
51pub(super) fn is_utf8_text(bytes: &[u8]) -> bool {
52    std::str::from_utf8(bytes).is_ok()
53}
54
55fn strip_corrupted_utf16_bom_prefix(bytes: &[u8]) -> &[u8] {
56    bytes
57        .strip_prefix(CORRUPTED_UTF16_BOM_PREFIX)
58        .unwrap_or(bytes)
59}
60
61fn decode_utf16_units(bytes: &[u8], is_le: bool, require_text_shape: bool) -> Option<String> {
62    if bytes.is_empty() || !bytes.len().is_multiple_of(2) {
63        return None;
64    }
65
66    let code_units: Vec<u16> = bytes
67        .chunks_exact(2)
68        .map(|chunk| {
69            if is_le {
70                u16::from_le_bytes([chunk[0], chunk[1]])
71            } else {
72                u16::from_be_bytes([chunk[0], chunk[1]])
73            }
74        })
75        .collect();
76
77    let decoded = std::char::decode_utf16(code_units)
78        .collect::<Result<String, _>>()
79        .ok()?;
80
81    if !require_text_shape {
82        return (!decoded.contains('\0')).then_some(decoded);
83    }
84
85    if !looks_like_decoded_text(&decoded) {
86        return None;
87    }
88
89    Some(decoded)
90}
91
92pub(super) fn looks_like_decoded_text(decoded: &str) -> bool {
93    if decoded
94        .chars()
95        .any(|ch| ch.is_control() && !matches!(ch, '\n' | '\r' | '\t'))
96    {
97        return false;
98    }
99
100    let visible = decoded
101        .chars()
102        .filter(|ch| !ch.is_control() || matches!(ch, '\n' | '\r' | '\t'))
103        .count();
104    if visible < 3 || decoded.contains('\0') {
105        return false;
106    }
107
108    let alpha = decoded.chars().filter(|ch| ch.is_alphabetic()).count();
109    let punctuation = decoded
110        .chars()
111        .filter(|ch| {
112            matches!(
113                ch,
114                '{' | '}'
115                    | '['
116                    | ']'
117                    | '<'
118                    | '>'
119                    | '('
120                    | ')'
121                    | ':'
122                    | ';'
123                    | ','
124                    | '"'
125                    | '\''
126                    | '/'
127                    | '='
128                    | '-'
129                    | '_'
130                    | '#'
131                    | '!'
132            )
133        })
134        .count();
135    let whitespace = decoded.chars().filter(|ch| ch.is_whitespace()).count();
136
137    let textish = alpha + punctuation + whitespace;
138    textish + (visible / 5) >= visible && (alpha > 0 || punctuation >= 2)
139}
140
141fn detect_utf16_endianness(bytes: &[u8]) -> Option<bool> {
142    let stripped = strip_corrupted_utf16_bom_prefix(bytes);
143    if stripped.len() < 4 || !stripped.len().is_multiple_of(2) {
144        return None;
145    }
146
147    let pair_count = stripped.len() / 2;
148    let even_zero = stripped.iter().step_by(2).filter(|&&b| b == 0).count();
149    let odd_zero = stripped
150        .iter()
151        .skip(1)
152        .step_by(2)
153        .filter(|&&b| b == 0)
154        .count();
155
156    let looks_like_be = even_zero * 3 >= pair_count && odd_zero * 6 <= pair_count;
157    let looks_like_le = odd_zero * 3 >= pair_count && even_zero * 6 <= pair_count;
158
159    match (looks_like_le, looks_like_be) {
160        (true, false) => Some(true),
161        (false, true) => Some(false),
162        (true, true) => Some(true),
163        (false, false) => None,
164    }
165}
166
167pub(super) fn decode_utf16_text(bytes: &[u8]) -> Option<String> {
168    if let Some(decoded) = decode_utf16_bom_text(bytes) {
169        return Some(decoded);
170    }
171
172    let stripped = strip_corrupted_utf16_bom_prefix(bytes);
173    match detect_utf16_endianness(bytes) {
174        Some(true) => decode_utf16_units(stripped, true, true),
175        Some(false) => decode_utf16_units(stripped, false, true),
176        None => None,
177    }
178}
179
180pub(super) fn decode_utf16_json_text(bytes: &[u8]) -> Option<String> {
181    if bytes.len() >= 2 {
182        let (is_le, body) = match bytes {
183            [0xFF, 0xFE, rest @ ..] => (true, rest),
184            [0xFE, 0xFF, rest @ ..] => (false, rest),
185            _ => {
186                let stripped = strip_corrupted_utf16_bom_prefix(bytes);
187                return match detect_utf16_endianness(bytes) {
188                    Some(true) => decode_utf16_units(stripped, true, false),
189                    Some(false) => decode_utf16_units(stripped, false, false),
190                    None => None,
191                };
192            }
193        };
194
195        if body.is_empty() || !body.len().is_multiple_of(2) {
196            return None;
197        }
198
199        return decode_utf16_units(body, is_le, false);
200    }
201
202    None
203}
204
205fn decode_utf16_bom_text(bytes: &[u8]) -> Option<String> {
206    if bytes.len() < 2 || !bytes.len().is_multiple_of(2) {
207        return None;
208    }
209
210    let (is_le, body) = match bytes {
211        [0xFF, 0xFE, rest @ ..] => (true, rest),
212        [0xFE, 0xFF, rest @ ..] => (false, rest),
213        _ => return None,
214    };
215
216    if body.is_empty() || body.len() % 2 != 0 {
217        return None;
218    }
219
220    decode_utf16_units(body, is_le, true)
221}
222
223pub(super) fn has_binary_control_chars(bytes: &[u8]) -> bool {
224    let control_count = bytes
225        .iter()
226        .filter(|&&b| b < 0x09 || (b > 0x0D && b < 0x20))
227        .count();
228    control_count > bytes.len() / BINARY_CONTROL_CHAR_THRESHOLD_DIVISOR
229}
230
231pub(super) fn has_decodable_text(bytes: &[u8]) -> bool {
232    bytes.is_empty()
233        || is_utf8_text(bytes)
234        || decode_utf16_text(bytes).is_some()
235        || !has_binary_control_chars(bytes)
236}
237
238pub(super) fn looks_like_textual_bytes(bytes: &[u8]) -> bool {
239    if bytes.is_empty() || is_utf8_text(bytes) {
240        return true;
241    }
242    if let Some(decoded) = decode_utf16_text(bytes) {
243        return decoded
244            .chars()
245            .any(|ch| !ch.is_control() || matches!(ch, '\n' | '\r' | '\t'));
246    }
247
248    let printable_count = bytes
249        .iter()
250        .filter(|&&b| matches!(b, b'\n' | b'\r' | b'\t') || (0x20..=0x7e).contains(&b))
251        .count();
252    printable_count * 2 >= bytes.len()
253}
provenant/utils/file/encoding.rs

provenant/utils/file/
encoding.rs
