#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum Encoding {
#[default]
Iso8859_1,
Utf8,
}
impl Encoding {
#[must_use]
pub fn decode(self, bytes: &[u8]) -> String {
match self {
Self::Iso8859_1 => bytes.iter().map(|&b| char::from(b)).collect(),
Self::Utf8 => String::from_utf8_lossy(bytes).into_owned(),
}
}
}
const BOM: &[u8] = &[0xEF, 0xBB, 0xBF];
#[must_use]
pub fn strip_bom(input: &[u8]) -> &[u8] {
input.strip_prefix(BOM).unwrap_or(input)
}
#[must_use]
pub fn split_lines(input: &[u8]) -> Vec<&[u8]> {
let input = strip_bom(input);
if input.is_empty() {
return Vec::new();
}
let mut lines: Vec<&[u8]> = input.split(|&b| b == b'\n').collect();
if lines.last().is_some_and(|l| l.is_empty()) {
lines.pop();
}
for line in &mut lines {
if let [rest @ .., b'\r'] = *line {
*line = rest;
}
}
lines
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn iso_8859_1_maps_bytes_to_code_points() {
assert_eq!(Encoding::Iso8859_1.decode(b"caf\xE9"), "café");
}
#[test]
fn utf_8_decodes_natively_and_never_panics() {
assert_eq!(Encoding::Utf8.decode("café".as_bytes()), "café");
assert!(Encoding::Utf8.decode(b"caf\xE9").contains('\u{FFFD}'));
}
#[test]
fn splits_lf_and_crlf_and_strips_bom() {
assert_eq!(split_lines(b"a\nb\n"), vec![&b"a"[..], &b"b"[..]]);
assert_eq!(split_lines(b"a\r\nb"), vec![&b"a"[..], &b"b"[..]]);
assert_eq!(split_lines(b"\xEF\xBB\xBFa\n"), vec![&b"a"[..]]);
assert!(split_lines(b"").is_empty());
}
#[test]
fn keeps_interior_blank_lines() {
assert_eq!(
split_lines(b"a\n\nb\n"),
vec![&b"a"[..], &b""[..], &b"b"[..]]
);
}
}