1use std::path::Path;
4
5#[derive(Debug, Clone, PartialEq, Eq)]
7pub struct EncodingDiagnostic {
8 pub guess: &'static str,
10 pub first_invalid_byte: Option<usize>,
14}
15
16impl EncodingDiagnostic {
17 pub fn detail(&self) -> String {
19 match self.first_invalid_byte {
22 Some(offset) => format!(
23 "looks like {}; first invalid UTF-8 byte at offset {offset}",
24 self.guess
25 ),
26 None => format!(
27 "looks like {}; contains bytes that are not valid UTF-8",
28 self.guess
29 ),
30 }
31 }
32}
33
34const ENCODING_SNIFF_LIMIT: usize = 1 << 20; pub fn diagnose_non_utf8(path: &Path) -> Option<EncodingDiagnostic> {
45 use std::io::Read;
46
47 let mut file = std::fs::File::open(path).ok()?;
51 let mut sample = Vec::new();
52 file.by_ref()
55 .take(ENCODING_SNIFF_LIMIT as u64)
56 .read_to_end(&mut sample)
57 .ok()?;
58
59 if sample.starts_with(&[0xFF, 0xFE]) {
60 return Some(EncodingDiagnostic {
61 guess: "utf-16le",
62 first_invalid_byte: Some(0),
63 });
64 }
65 if sample.starts_with(&[0xFE, 0xFF]) {
66 return Some(EncodingDiagnostic {
67 guess: "utf-16be",
68 first_invalid_byte: Some(0),
69 });
70 }
71
72 if sample.len() == ENCODING_SNIFF_LIMIT
78 && let Err(err) = std::str::from_utf8(&sample)
79 && err.error_len().is_none()
80 {
81 let sequence_start = err.valid_up_to();
82 let sequence_length = match sample[sequence_start] {
84 0xF0..=0xF4 => 4,
85 0xE0..=0xEF => 3,
86 _ => 2,
87 };
88 let remaining = sequence_length - (sample.len() - sequence_start);
89 file.take(remaining as u64).read_to_end(&mut sample).ok()?;
92 }
93 let first_invalid_byte = match std::str::from_utf8(&sample) {
94 Ok(_) => return None,
95 Err(err) => Some(err.valid_up_to()),
96 };
97
98 let has_c1_controls = sample.iter().any(|&b| (0x80..=0x9F).contains(&b));
99 let guess = if has_c1_controls {
100 "windows-1252"
101 } else {
102 "latin-1"
103 };
104
105 Some(EncodingDiagnostic {
106 guess,
107 first_invalid_byte,
108 })
109}
110
111#[cfg(test)]
112mod tests {
113 use super::*;
114 use std::io::Write;
115 use tempfile::NamedTempFile;
116
117 fn write_bytes(bytes: &[u8]) -> NamedTempFile {
118 let mut f = NamedTempFile::new().unwrap();
119 f.write_all(bytes).unwrap();
120 f.flush().unwrap();
121 f
122 }
123
124 #[test]
125 fn diagnose_non_utf8_flags_latin1() {
126 let f = write_bytes(b"name\ncaf\xE9\n");
130 let d = diagnose_non_utf8(f.path()).expect("latin-1 should be flagged");
131 assert_eq!(d.guess, "latin-1");
132 assert_eq!(d.first_invalid_byte, Some(8));
133 }
134
135 #[test]
136 fn diagnose_non_utf8_flags_windows1252() {
137 let f = write_bytes(b"quote\n\x93hi\x94\n");
139 let d = diagnose_non_utf8(f.path()).expect("windows-1252 should be flagged");
140 assert_eq!(d.guess, "windows-1252");
141 assert_eq!(d.first_invalid_byte, Some(6));
142 }
143
144 #[test]
145 fn diagnose_non_utf8_flags_utf16_bom() {
146 let f = write_bytes(&[0xFF, 0xFE, b'a', 0x00]);
147 let d = diagnose_non_utf8(f.path()).expect("utf-16 BOM should be flagged");
148 assert_eq!(d.guess, "utf-16le");
149 }
150
151 #[test]
152 fn diagnose_non_utf8_ignores_clean_utf8() {
153 let f = write_bytes("name\ncafé\n".as_bytes());
154 assert!(diagnose_non_utf8(f.path()).is_none());
155 }
156
157 #[test]
158 fn encoding_diagnostic_detail_never_leaks_bytes() {
159 let d = EncodingDiagnostic {
160 guess: "latin-1",
161 first_invalid_byte: Some(8),
162 };
163 let detail = d.detail();
164 assert!(detail.contains("latin-1"));
165 assert!(detail.contains("offset 8"));
166 }
167}