rich_ext/
encoding.rs

1//! Explicit text decoding for CLI and application inputs.
2//!
3//! This extension never guesses a headerless encoding. Callers retain their
4//! existing default decoding policy unless an encoding is explicitly selected.
5
6use std::io::{Error, ErrorKind, Result};
7
8/// Supported explicit text encodings. All decoding is strict.
9#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10pub enum Encoding {
11    Utf8,
12    /// Requires a UTF-16 byte-order mark.
13    Utf16,
14    Utf16Le,
15    Utf16Be,
16}
17
18impl std::str::FromStr for Encoding {
19    type Err = String;
20
21    fn from_str(name: &str) -> std::result::Result<Self, Self::Err> {
22        match name.to_ascii_lowercase().as_str() {
23            "utf-8" => Ok(Self::Utf8),
24            "utf-16" => Ok(Self::Utf16),
25            "utf-16le" => Ok(Self::Utf16Le),
26            "utf-16be" => Ok(Self::Utf16Be),
27            _ => Err(format!(
28                "unsupported encoding {name:?}; use utf-8, utf-16, utf-16le or utf-16be"
29            )),
30        }
31    }
32}
33
34fn invalid(message: &str) -> Error {
35    Error::new(ErrorKind::InvalidData, message)
36}
37
38/// Recognize a likely UTF-16 BOM, excluding ambiguous UTF-32 signatures.
39pub fn has_utf16_bom(bytes: &[u8]) -> bool {
40    !bytes.starts_with(&[0xff, 0xfe, 0, 0])
41        && (bytes.starts_with(&[0xff, 0xfe]) || bytes.starts_with(&[0xfe, 0xff]))
42}
43
44impl Encoding {
45    /// Decode bytes, consuming a matching BOM and rejecting malformed data.
46    /// `Utf16` requires a BOM; endian-specific variants also accept headerless
47    /// data, but reject a contradictory BOM. Generic `Utf16` rejects ambiguous
48    /// UTF-32 signatures; select an endian explicitly for BOM + leading NUL.
49    /// Newline policy belongs to callers.
50    pub fn decode(self, bytes: &[u8]) -> Result<String> {
51        if self == Self::Utf8 {
52            let bytes = bytes.strip_prefix(&[0xef, 0xbb, 0xbf]).unwrap_or(bytes);
53            return std::str::from_utf8(bytes)
54                .map(str::to_owned)
55                .map_err(|_| invalid("input is not valid UTF-8"));
56        }
57        if self == Self::Utf16
58            && (bytes.starts_with(&[0xff, 0xfe, 0, 0]) || bytes.starts_with(&[0, 0, 0xfe, 0xff]))
59        {
60            return Err(invalid(
61                "UTF-32 is not supported; convert the input to UTF-8",
62            ));
63        }
64        let bom = if bytes.starts_with(&[0xff, 0xfe]) {
65            Some(Self::Utf16Le)
66        } else if bytes.starts_with(&[0xfe, 0xff]) {
67            Some(Self::Utf16Be)
68        } else {
69            None
70        };
71        let endian = match (self, bom) {
72            (Self::Utf16, None) => {
73                return Err(invalid(
74                    "UTF-16 needs a BOM; select --encoding utf-16le or utf-16be for headerless input",
75                ));
76            }
77            (Self::Utf16, Some(endian)) => endian,
78            (selected, Some(endian)) if selected != endian => {
79                return Err(invalid("UTF-16 BOM conflicts with the selected encoding"));
80            }
81            (selected, _) => selected,
82        };
83        let bytes = if bom.is_some() { &bytes[2..] } else { bytes };
84        if bytes.len() % 2 != 0 {
85            return Err(invalid("invalid UTF-16: odd byte count"));
86        }
87        let units = bytes.as_chunks::<2>().0.iter().map(|&pair| {
88            if endian == Self::Utf16Le {
89                u16::from_le_bytes(pair)
90            } else {
91                u16::from_be_bytes(pair)
92            }
93        });
94        char::decode_utf16(units)
95            .collect::<std::result::Result<String, _>>()
96            .map_err(|_| invalid("invalid UTF-16: unpaired surrogate"))
97    }
98}
99
100#[cfg(test)]
101mod tests {
102    use super::*;
103
104    #[test]
105    fn explicit_endianness_preserves_unicode() {
106        let text = "Hello 漢字 🙂\r\n";
107        for encoding in [Encoding::Utf16Le, Encoding::Utf16Be] {
108            let mut bytes: Vec<u8> = text
109                .encode_utf16()
110                .flat_map(|unit| {
111                    if encoding == Encoding::Utf16Le {
112                        unit.to_le_bytes()
113                    } else {
114                        unit.to_be_bytes()
115                    }
116                })
117                .collect();
118            assert_eq!(encoding.decode(&bytes).unwrap(), text);
119            assert!(Encoding::Utf16.decode(&bytes).is_err());
120            let bom = if encoding == Encoding::Utf16Le {
121                [0xff, 0xfe]
122            } else {
123                [0xfe, 0xff]
124            };
125            bytes.splice(0..0, bom);
126            assert_eq!(Encoding::Utf16.decode(&bytes).unwrap(), text);
127            assert_eq!(encoding.decode(&bytes).unwrap(), text);
128        }
129    }
130
131    #[test]
132    fn malformed_and_conflicting_input_is_rejected() {
133        assert!(Encoding::Utf16Le.decode(&[1]).is_err());
134        assert!(Encoding::Utf16Le.decode(&[0, 0xd8]).is_err());
135        assert!(Encoding::Utf16Be.decode(&[0xff, 0xfe, 65, 0]).is_err());
136        assert!(Encoding::Utf16.decode(&[0xff, 0xfe, 0, 0]).is_err());
137        assert!(!has_utf16_bom(&[0xff, 0xfe, 0, 0]));
138        assert_eq!(Encoding::Utf16Le.decode(&[0xff, 0xfe, 0, 0]).unwrap(), "\0");
139        assert!(Encoding::Utf8.decode(&[0xff]).is_err());
140        assert_eq!(
141            Encoding::Utf8.decode(b"\xef\xbb\xbfhello").unwrap(),
142            "hello"
143        );
144    }
145}
rich_ext/encoding.rs

rich_ext/
encoding.rs
