Skip to main content

dataprof_csv/
encoding.rs

1//! Encoding diagnosis for text input that failed UTF-8 decoding.
2
3use std::path::Path;
4
5/// What [`diagnose_non_utf8`] found about a file that failed UTF-8 decoding.
6#[derive(Debug, Clone, PartialEq, Eq)]
7pub struct EncodingDiagnostic {
8    /// Best-effort encoding guess (e.g. `"latin-1"`, `"windows-1252"`, `"utf-16le"`).
9    pub guess: &'static str,
10    /// Byte offset of the first invalid UTF-8 sequence, when found within the
11    /// sampled prefix. `None` means the sampled prefix was valid UTF-8 and the
12    /// offending byte lies further into the file.
13    pub first_invalid_byte: Option<usize>,
14}
15
16impl EncodingDiagnostic {
17    /// Human-readable detail for an error message, without echoing file contents.
18    pub fn detail(&self) -> String {
19        // Plain ASCII only: this string ends up in an encoding-error message that
20        // may be printed to a non-UTF-8 terminal, so it must never add mojibake.
21        match self.first_invalid_byte {
22            Some(offset) => format!(
23                "looks like {}; first invalid UTF-8 byte at offset {offset}",
24                self.guess
25            ),
26            None => format!(
27                "looks like {}; contains bytes that are not valid UTF-8",
28                self.guess
29            ),
30        }
31    }
32}
33
34/// Prefix budget for [`diagnose_non_utf8`]. Up to three additional bytes finish
35/// a UTF-8 sequence crossing its boundary, keeping the error path bounded.
36const ENCODING_SNIFF_LIMIT: usize = 1 << 20; // 1 MiB
37
38/// Inspect a file that failed UTF-8 decoding and guess its encoding.
39///
40/// Returns `None` when the sampled prefix is valid UTF-8 (so a failure that is
41/// *not* an encoding problem is never mislabeled as one). BOM-prefixed UTF-16 is
42/// reported directly; otherwise a byte in `0x80..=0x9F` points at windows-1252
43/// and anything else at latin-1 — a best-effort guess, not a definitive verdict.
44pub fn diagnose_non_utf8(path: &Path) -> Option<EncodingDiagnostic> {
45    use std::io::Read;
46
47    // decode-audit: unknown — this runs only to enrich an error the caller
48    // already has. If the file cannot be reopened/read for sniffing, returning
49    // None keeps that original engine error, so the failure is surfaced, not lost.
50    let mut file = std::fs::File::open(path).ok()?;
51    let mut sample = Vec::new();
52    // decode-audit: unknown — as above; a read failure abandons the guess and
53    // preserves the caller's original error rather than swallowing it.
54    file.by_ref()
55        .take(ENCODING_SNIFF_LIMIT as u64)
56        .read_to_end(&mut sample)
57        .ok()?;
58
59    if sample.starts_with(&[0xFF, 0xFE]) {
60        return Some(EncodingDiagnostic {
61            guess: "utf-16le",
62            first_invalid_byte: Some(0),
63        });
64    }
65    if sample.starts_with(&[0xFE, 0xFF]) {
66        return Some(EncodingDiagnostic {
67            guess: "utf-16be",
68            first_invalid_byte: Some(0),
69        });
70    }
71
72    // A valid-UTF-8 prefix means the fault, if any, is beyond the sample — do not
73    // claim an encoding problem we cannot see.
74    // A capped prefix can split a valid four-byte UTF-8 sequence. Read at most
75    // three more bytes before deciding whether that sequence is invalid; at
76    // EOF an incomplete sequence is a real encoding error.
77    if sample.len() == ENCODING_SNIFF_LIMIT
78        && let Err(err) = std::str::from_utf8(&sample)
79        && err.error_len().is_none()
80    {
81        let sequence_start = err.valid_up_to();
82        // An incomplete UTF-8 error always starts at a valid leading byte.
83        let sequence_length = match sample[sequence_start] {
84            0xF0..=0xF4 => 4,
85            0xE0..=0xEF => 3,
86            _ => 2,
87        };
88        let remaining = sequence_length - (sample.len() - sequence_start);
89        // decode-audit: unknown — an unsuccessful lookahead preserves the
90        // original engine error, just like an unsuccessful prefix read.
91        file.take(remaining as u64).read_to_end(&mut sample).ok()?;
92    }
93    let first_invalid_byte = match std::str::from_utf8(&sample) {
94        Ok(_) => return None,
95        Err(err) => Some(err.valid_up_to()),
96    };
97
98    let has_c1_controls = sample.iter().any(|&b| (0x80..=0x9F).contains(&b));
99    let guess = if has_c1_controls {
100        "windows-1252"
101    } else {
102        "latin-1"
103    };
104
105    Some(EncodingDiagnostic {
106        guess,
107        first_invalid_byte,
108    })
109}
110
111#[cfg(test)]
112mod tests {
113    use super::*;
114    use std::io::Write;
115    use tempfile::NamedTempFile;
116
117    fn write_bytes(bytes: &[u8]) -> NamedTempFile {
118        let mut f = NamedTempFile::new().unwrap();
119        f.write_all(bytes).unwrap();
120        f.flush().unwrap();
121        f
122    }
123
124    #[test]
125    fn diagnose_non_utf8_flags_latin1() {
126        // "café" in latin-1: 0xE9 (é) is a UTF-8 leading byte for a 3-byte
127        // sequence, but here it is not followed by continuation bytes, so the
128        // stream is invalid UTF-8 at that position.
129        let f = write_bytes(b"name\ncaf\xE9\n");
130        let d = diagnose_non_utf8(f.path()).expect("latin-1 should be flagged");
131        assert_eq!(d.guess, "latin-1");
132        assert_eq!(d.first_invalid_byte, Some(8));
133    }
134
135    #[test]
136    fn diagnose_non_utf8_flags_windows1252() {
137        // 0x93/0x94 are smart quotes in windows-1252 (C1 range), invalid UTF-8.
138        let f = write_bytes(b"quote\n\x93hi\x94\n");
139        let d = diagnose_non_utf8(f.path()).expect("windows-1252 should be flagged");
140        assert_eq!(d.guess, "windows-1252");
141        assert_eq!(d.first_invalid_byte, Some(6));
142    }
143
144    #[test]
145    fn diagnose_non_utf8_flags_utf16_bom() {
146        let f = write_bytes(&[0xFF, 0xFE, b'a', 0x00]);
147        let d = diagnose_non_utf8(f.path()).expect("utf-16 BOM should be flagged");
148        assert_eq!(d.guess, "utf-16le");
149    }
150
151    #[test]
152    fn diagnose_non_utf8_ignores_clean_utf8() {
153        let f = write_bytes("name\ncafé\n".as_bytes());
154        assert!(diagnose_non_utf8(f.path()).is_none());
155    }
156
157    #[test]
158    fn encoding_diagnostic_detail_never_leaks_bytes() {
159        let d = EncodingDiagnostic {
160            guess: "latin-1",
161            first_invalid_byte: Some(8),
162        };
163        let detail = d.detail();
164        assert!(detail.contains("latin-1"));
165        assert!(detail.contains("offset 8"));
166    }
167}