provenant/utils/file/
encoding.rs

1// SPDX-FileCopyrightText: nexB Inc. and others
2// ScanCode is a trademark of nexB Inc.
3// SPDX-FileCopyrightText: Provenant contributors
4// SPDX-License-Identifier: Apache-2.0
5// Derived from ScanCode Toolkit (Apache-2.0); modified. See NOTICE.
6
7//! Byte-to-text decoding and "does this look like text" heuristics shared by
8//! classification and text-extraction logic.
9
10use crate::models::ScanDiagnostic;
11
12const BINARY_CONTROL_CHAR_THRESHOLD_DIVISOR: usize = 10;
13
14pub(super) const CORRUPTED_UTF16_BOM_PREFIX: &[u8] = &[0xEF, 0xBF, 0xBD, 0xEF, 0xBF, 0xBD];
15
16/// Diagnostic message emitted when invalid-UTF-8 input is dropped from
17/// detection because it exceeds the binary-control-char threshold. Surfaced via
18/// the scanner's structured `scan_diagnostics` channel so the skip is
19/// observable rather than silent. This is an informational outcome (Provenant
20/// correctly declined text detection on a binary file), not a failure, so it is
21/// recorded at [`DiagnosticSeverity::Info`](crate::models::DiagnosticSeverity)
22/// and excluded from the ScanCode-compatible `scan_errors` field.
23pub(super) const NEAR_BINARY_SKIP_DIAGNOSTIC: &str = "Text skipped from license/copyright detection: invalid UTF-8 with too many control bytes (likely binary)";
24
25/// Decode a byte buffer to a String, trying UTF-16 first when the byte shape
26/// strongly suggests it, then UTF-8, then Latin-1.
27///
28/// Latin-1 (ISO-8859-1) maps bytes 0x00-0xFF directly to Unicode U+0000-U+00FF,
29/// so it can decode any byte sequence. This matches Python ScanCode's use of
30/// `UnicodeDammit` which auto-detects encoding with Latin-1 as fallback.
31pub fn decode_bytes_to_string(bytes: &[u8]) -> String {
32    decode_bytes_to_string_with_diagnostic(bytes).0
33}
34
35/// Like [`decode_bytes_to_string`], but also returns a structured diagnostic
36/// when the result is the empty string because the input looked like binary
37/// (invalid UTF-8 with a high control-byte ratio). The decoded result is
38/// unchanged; only the optional diagnostic is added so the silent skip becomes
39/// observable in scan output.
40pub(super) fn decode_bytes_to_string_with_diagnostic(
41    bytes: &[u8],
42) -> (String, Option<ScanDiagnostic>) {
43    if let Some(decoded) = decode_utf16_text(bytes) {
44        return (decoded, None);
45    }
46
47    match String::from_utf8(bytes.to_vec()) {
48        Ok(s) => (s, None),
49        Err(e) => {
50            let bytes = e.into_bytes();
51            if has_binary_control_chars(&bytes) {
52                return (
53                    String::new(),
54                    Some(ScanDiagnostic::info(NEAR_BINARY_SKIP_DIAGNOSTIC)),
55                );
56            }
57            (bytes.iter().map(|&b| b as char).collect(), None)
58        }
59    }
60}
61
62pub(super) fn is_utf8_text(bytes: &[u8]) -> bool {
63    std::str::from_utf8(bytes).is_ok()
64}
65
66fn strip_corrupted_utf16_bom_prefix(bytes: &[u8]) -> &[u8] {
67    bytes
68        .strip_prefix(CORRUPTED_UTF16_BOM_PREFIX)
69        .unwrap_or(bytes)
70}
71
72fn decode_utf16_units(bytes: &[u8], is_le: bool, require_text_shape: bool) -> Option<String> {
73    if bytes.is_empty() || !bytes.len().is_multiple_of(2) {
74        return None;
75    }
76
77    let code_units: Vec<u16> = bytes
78        .chunks_exact(2)
79        .map(|chunk| {
80            if is_le {
81                u16::from_le_bytes([chunk[0], chunk[1]])
82            } else {
83                u16::from_be_bytes([chunk[0], chunk[1]])
84            }
85        })
86        .collect();
87
88    let decoded = std::char::decode_utf16(code_units)
89        .collect::<Result<String, _>>()
90        .ok()?;
91
92    if !require_text_shape {
93        return (!decoded.contains('\0')).then_some(decoded);
94    }
95
96    if !looks_like_decoded_text(&decoded) {
97        return None;
98    }
99
100    Some(decoded)
101}
102
103pub(super) fn looks_like_decoded_text(decoded: &str) -> bool {
104    if decoded
105        .chars()
106        .any(|ch| ch.is_control() && !matches!(ch, '\n' | '\r' | '\t'))
107    {
108        return false;
109    }
110
111    let visible = decoded
112        .chars()
113        .filter(|ch| !ch.is_control() || matches!(ch, '\n' | '\r' | '\t'))
114        .count();
115    if visible < 3 || decoded.contains('\0') {
116        return false;
117    }
118
119    let alpha = decoded.chars().filter(|ch| ch.is_alphabetic()).count();
120    let punctuation = decoded
121        .chars()
122        .filter(|ch| {
123            matches!(
124                ch,
125                '{' | '}'
126                    | '['
127                    | ']'
128                    | '<'
129                    | '>'
130                    | '('
131                    | ')'
132                    | ':'
133                    | ';'
134                    | ','
135                    | '"'
136                    | '\''
137                    | '/'
138                    | '='
139                    | '-'
140                    | '_'
141                    | '#'
142                    | '!'
143            )
144        })
145        .count();
146    let whitespace = decoded.chars().filter(|ch| ch.is_whitespace()).count();
147
148    let textish = alpha + punctuation + whitespace;
149    textish + (visible / 5) >= visible && (alpha > 0 || punctuation >= 2)
150}
151
152fn detect_utf16_endianness(bytes: &[u8]) -> Option<bool> {
153    let stripped = strip_corrupted_utf16_bom_prefix(bytes);
154    if stripped.len() < 4 || !stripped.len().is_multiple_of(2) {
155        return None;
156    }
157
158    let pair_count = stripped.len() / 2;
159    let even_zero = stripped.iter().step_by(2).filter(|&&b| b == 0).count();
160    let odd_zero = stripped
161        .iter()
162        .skip(1)
163        .step_by(2)
164        .filter(|&&b| b == 0)
165        .count();
166
167    let looks_like_be = even_zero * 3 >= pair_count && odd_zero * 6 <= pair_count;
168    let looks_like_le = odd_zero * 3 >= pair_count && even_zero * 6 <= pair_count;
169
170    match (looks_like_le, looks_like_be) {
171        (true, false) => Some(true),
172        (false, true) => Some(false),
173        (true, true) => Some(true),
174        (false, false) => None,
175    }
176}
177
178pub(super) fn decode_utf16_text(bytes: &[u8]) -> Option<String> {
179    if let Some(decoded) = decode_utf16_bom_text(bytes) {
180        return Some(decoded);
181    }
182
183    let stripped = strip_corrupted_utf16_bom_prefix(bytes);
184    match detect_utf16_endianness(bytes) {
185        Some(true) => decode_utf16_units(stripped, true, true),
186        Some(false) => decode_utf16_units(stripped, false, true),
187        None => None,
188    }
189}
190
191pub(super) fn decode_utf16_json_text(bytes: &[u8]) -> Option<String> {
192    if bytes.len() >= 2 {
193        let (is_le, body) = match bytes {
194            [0xFF, 0xFE, rest @ ..] => (true, rest),
195            [0xFE, 0xFF, rest @ ..] => (false, rest),
196            _ => {
197                let stripped = strip_corrupted_utf16_bom_prefix(bytes);
198                return match detect_utf16_endianness(bytes) {
199                    Some(true) => decode_utf16_units(stripped, true, false),
200                    Some(false) => decode_utf16_units(stripped, false, false),
201                    None => None,
202                };
203            }
204        };
205
206        if body.is_empty() || !body.len().is_multiple_of(2) {
207            return None;
208        }
209
210        return decode_utf16_units(body, is_le, false);
211    }
212
213    None
214}
215
216fn decode_utf16_bom_text(bytes: &[u8]) -> Option<String> {
217    if bytes.len() < 2 || !bytes.len().is_multiple_of(2) {
218        return None;
219    }
220
221    let (is_le, body) = match bytes {
222        [0xFF, 0xFE, rest @ ..] => (true, rest),
223        [0xFE, 0xFF, rest @ ..] => (false, rest),
224        _ => return None,
225    };
226
227    if body.is_empty() || body.len() % 2 != 0 {
228        return None;
229    }
230
231    decode_utf16_units(body, is_le, true)
232}
233
234pub(super) fn has_binary_control_chars(bytes: &[u8]) -> bool {
235    let control_count = bytes
236        .iter()
237        .filter(|&&b| b < 0x09 || (b > 0x0D && b < 0x20))
238        .count();
239    control_count > bytes.len() / BINARY_CONTROL_CHAR_THRESHOLD_DIVISOR
240}
241
242pub(super) fn has_decodable_text(bytes: &[u8]) -> bool {
243    bytes.is_empty()
244        || is_utf8_text(bytes)
245        || decode_utf16_text(bytes).is_some()
246        || !has_binary_control_chars(bytes)
247}
248
249pub(super) fn looks_like_textual_bytes(bytes: &[u8]) -> bool {
250    if bytes.is_empty() || is_utf8_text(bytes) {
251        return true;
252    }
253    if let Some(decoded) = decode_utf16_text(bytes) {
254        return decoded
255            .chars()
256            .any(|ch| !ch.is_control() || matches!(ch, '\n' | '\r' | '\t'));
257    }
258
259    let printable_count = bytes
260        .iter()
261        .filter(|&&b| matches!(b, b'\n' | b'\r' | b'\t') || (0x20..=0x7e).contains(&b))
262        .count();
263    printable_count * 2 >= bytes.len()
264}
provenant/utils/file/encoding.rs

provenant/utils/file/
encoding.rs
