use std::path::Path;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct EncodingDiagnostic {
pub guess: &'static str,
pub first_invalid_byte: Option<usize>,
}
impl EncodingDiagnostic {
pub fn detail(&self) -> String {
match self.first_invalid_byte {
Some(offset) => format!(
"looks like {}; first invalid UTF-8 byte at offset {offset}",
self.guess
),
None => format!(
"looks like {}; contains bytes that are not valid UTF-8",
self.guess
),
}
}
}
const ENCODING_SNIFF_LIMIT: usize = 1 << 20;
pub fn diagnose_non_utf8(path: &Path) -> Option<EncodingDiagnostic> {
use std::io::Read;
let mut file = std::fs::File::open(path).ok()?;
let mut sample = Vec::new();
file.by_ref()
.take(ENCODING_SNIFF_LIMIT as u64)
.read_to_end(&mut sample)
.ok()?;
if sample.starts_with(&[0xFF, 0xFE]) {
return Some(EncodingDiagnostic {
guess: "utf-16le",
first_invalid_byte: Some(0),
});
}
if sample.starts_with(&[0xFE, 0xFF]) {
return Some(EncodingDiagnostic {
guess: "utf-16be",
first_invalid_byte: Some(0),
});
}
if sample.len() == ENCODING_SNIFF_LIMIT
&& let Err(err) = std::str::from_utf8(&sample)
&& err.error_len().is_none()
{
let sequence_start = err.valid_up_to();
let sequence_length = match sample[sequence_start] {
0xF0..=0xF4 => 4,
0xE0..=0xEF => 3,
_ => 2,
};
let remaining = sequence_length - (sample.len() - sequence_start);
file.take(remaining as u64).read_to_end(&mut sample).ok()?;
}
let first_invalid_byte = match std::str::from_utf8(&sample) {
Ok(_) => return None,
Err(err) => Some(err.valid_up_to()),
};
let has_c1_controls = sample.iter().any(|&b| (0x80..=0x9F).contains(&b));
let guess = if has_c1_controls {
"windows-1252"
} else {
"latin-1"
};
Some(EncodingDiagnostic {
guess,
first_invalid_byte,
})
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write;
use tempfile::NamedTempFile;
fn write_bytes(bytes: &[u8]) -> NamedTempFile {
let mut f = NamedTempFile::new().unwrap();
f.write_all(bytes).unwrap();
f.flush().unwrap();
f
}
#[test]
fn diagnose_non_utf8_flags_latin1() {
let f = write_bytes(b"name\ncaf\xE9\n");
let d = diagnose_non_utf8(f.path()).expect("latin-1 should be flagged");
assert_eq!(d.guess, "latin-1");
assert_eq!(d.first_invalid_byte, Some(8));
}
#[test]
fn diagnose_non_utf8_flags_windows1252() {
let f = write_bytes(b"quote\n\x93hi\x94\n");
let d = diagnose_non_utf8(f.path()).expect("windows-1252 should be flagged");
assert_eq!(d.guess, "windows-1252");
assert_eq!(d.first_invalid_byte, Some(6));
}
#[test]
fn diagnose_non_utf8_flags_utf16_bom() {
let f = write_bytes(&[0xFF, 0xFE, b'a', 0x00]);
let d = diagnose_non_utf8(f.path()).expect("utf-16 BOM should be flagged");
assert_eq!(d.guess, "utf-16le");
}
#[test]
fn diagnose_non_utf8_ignores_clean_utf8() {
let f = write_bytes("name\ncafé\n".as_bytes());
assert!(diagnose_non_utf8(f.path()).is_none());
}
#[test]
fn encoding_diagnostic_detail_never_leaks_bytes() {
let d = EncodingDiagnostic {
guess: "latin-1",
first_invalid_byte: Some(8),
};
let detail = d.detail();
assert!(detail.contains("latin-1"));
assert!(detail.contains("offset 8"));
}
}