mail_parser/decoders/charsets/
mod.rs

1/*
2 * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
3 *
4 * SPDX-License-Identifier: Apache-2.0 OR MIT
5 */
6
7//! Charset conversion to UTF-8.
8//!
9//! [`Charset::from_label`] maps a MIME or WHATWG label to a [`Charset`], and
10//! [`Charset::decode`] converts bytes. The single-byte charsets, UTF-7 and
11//! UTF-16 are built in; the multi-byte charsets (Shift_JIS, EUC-JP,
12//! ISO-2022-JP, EUC-KR, Big5, GBK, GB18030) and a few legacy single-byte
13//! ones need the `full_encoding` feature, which uses `encoding_rs`.
14//!
15//! ```
16//! use mail_parser::Charset;
17//!
18//! let charset = Charset::from_label(b"ISO-8859-1").expect("known label");
19//! assert_eq!(charset.decode(b"caf\xe9"), "caf\u{e9}");
20//! assert_eq!(Charset::from_label(b"x-unknown"), None);
21//! ```
22
23mod ascii;
24mod label;
25mod multi_byte;
26mod single_byte;
27mod tables;
28mod utf16;
29mod utf7;
30mod utf8;
31
32use std::borrow::Cow;
33
34use multi_byte::MultiByte;
35use single_byte::Table;
36use utf16::Endian;
37
38const MAX_LABEL_LEN: usize = 45;
39const UTF8_LABEL: &[u8] = b"utf-8";
40
41/// A charset decoder function, as returned by [`charset_decoder`].
42pub type DecoderFnc = fn(&[u8]) -> String;
43
44/// A charset that text can be converted from.
45#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
46#[non_exhaustive]
47pub enum Charset {
48    /// UTF-8, also used for text without a charset or with an unknown one.
49    #[default]
50    Utf8,
51    /// UTF-7 (RFC 2152).
52    Utf7,
53    /// UTF-16 with a byte order mark; little-endian without one.
54    Utf16,
55    /// UTF-16, little-endian.
56    Utf16Le,
57    /// UTF-16, big-endian.
58    Utf16Be,
59    /// ISO-8859-2 (Latin-2, Central European).
60    Iso8859_2,
61    /// ISO-8859-3 (Latin-3, South European).
62    Iso8859_3,
63    /// ISO-8859-4 (Latin-4, North European).
64    Iso8859_4,
65    /// ISO-8859-5 (Cyrillic).
66    Iso8859_5,
67    /// ISO-8859-6 (Arabic).
68    Iso8859_6,
69    /// ISO-8859-7 (Greek).
70    Iso8859_7,
71    /// ISO-8859-8 (Hebrew).
72    Iso8859_8,
73    /// ISO-8859-9 (Latin-5, Turkish), strictly as the standard defines it.
74    /// No label resolves here: `iso-8859-9` and its aliases resolve to
75    /// [`Charset::Windows1254`], the superset senders actually send.
76    Iso8859_9,
77    /// ISO-8859-10 (Latin-6, Nordic).
78    Iso8859_10,
79    /// ISO-8859-13 (Latin-7, Baltic).
80    Iso8859_13,
81    /// ISO-8859-14 (Latin-8, Celtic).
82    Iso8859_14,
83    /// ISO-8859-15 (Latin-9).
84    Iso8859_15,
85    /// ISO-8859-16 (Latin-10, South-Eastern European).
86    Iso8859_16,
87    /// windows-1250 (Central European).
88    Windows1250,
89    /// windows-1251 (Cyrillic).
90    Windows1251,
91    /// windows-1252 (Western European), also used for the ISO-8859-1 and
92    /// US-ASCII labels, as browsers do.
93    Windows1252,
94    /// windows-1253 (Greek).
95    Windows1253,
96    /// windows-1254 (Turkish).
97    Windows1254,
98    /// windows-1255 (Hebrew).
99    Windows1255,
100    /// windows-1256 (Arabic).
101    Windows1256,
102    /// windows-1257 (Baltic).
103    Windows1257,
104    /// windows-1258 (Vietnamese).
105    Windows1258,
106    /// KOI8-R (Russian).
107    Koi8R,
108    /// KOI8-U (Ukrainian).
109    Koi8U,
110    /// Mac OS Roman.
111    Macintosh,
112    /// IBM code page 850 (DOS Latin-1).
113    Ibm850,
114    /// TIS-620 (Thai), strictly as the standard defines it. No label resolves
115    /// here: `tis-620` and `iso-8859-11` resolve to [`Charset::Windows874`],
116    /// the superset senders actually send.
117    Tis620,
118    /// Shift_JIS (Japanese); needs the `full_encoding` feature.
119    ShiftJis,
120    /// Big5 (Traditional Chinese); needs the `full_encoding` feature.
121    Big5,
122    /// EUC-JP (Japanese); needs the `full_encoding` feature.
123    EucJp,
124    /// EUC-KR (Korean); needs the `full_encoding` feature.
125    EucKr,
126    /// GB18030 (Simplified Chinese); needs the `full_encoding` feature.
127    Gb18030,
128    /// GBK (Simplified Chinese); needs the `full_encoding` feature.
129    Gbk,
130    /// ISO-2022-JP (Japanese); needs the `full_encoding` feature.
131    Iso2022Jp,
132    /// windows-874 (Thai). Also the target of the `tis-620` and `iso-8859-11`
133    /// labels, since senders use them for code page 874 bytes.
134    Windows874,
135    /// IBM code page 866 (DOS Cyrillic); needs the `full_encoding` feature.
136    Ibm866,
137    /// x-mac-cyrillic; needs the `full_encoding` feature.
138    XMacCyrillic,
139    /// x-user-defined; needs the `full_encoding` feature.
140    XUserDefined,
141    /// The WHATWG replacement encoding, for labels of charsets that are not
142    /// decoded on purpose (ISO-2022-KR, HZ-GB-2312, ...): the text becomes one
143    /// U+FFFD. Needs the `full_encoding` feature.
144    Replacement,
145}
146
147enum Decoder {
148    Utf8,
149    Utf7,
150    Utf16Bom,
151    Utf16(Endian),
152    SingleByte(&'static Table),
153    MultiByte(MultiByte),
154}
155
156impl Charset {
157    /// The charset for a MIME or WHATWG label, ASCII case-insensitive;
158    /// `None` for an unknown label.
159    pub fn from_label(label: &[u8]) -> Option<Charset> {
160        label::lookup(label.get(..MAX_LABEL_LEN).unwrap_or(label))
161    }
162
163    /// Converts `bytes` to text; byte sequences that are not valid in the
164    /// charset become U+FFFD. Borrowed when the bytes are already the text
165    /// (valid UTF-8, or ASCII in a single-byte charset).
166    pub fn decode(self, bytes: &[u8]) -> Cow<'_, str> {
167        match self.decoder() {
168            Decoder::Utf8 => utf8::decode(bytes),
169            Decoder::Utf7 => utf7::decode(bytes),
170            Decoder::Utf16Bom => Cow::Owned(utf16::decode_bom(bytes)),
171            Decoder::Utf16(endian) => Cow::Owned(endian.decode(bytes)),
172            Decoder::SingleByte(table) => table.decode(bytes),
173            Decoder::MultiByte(encoding) => encoding.decode(bytes),
174        }
175    }
176
177    /// Converts `bytes` to text like [`Charset::decode`], reusing the vector
178    /// when the bytes are already the text.
179    pub fn decode_owned(self, bytes: Vec<u8>) -> String {
180        let decoded = match self.decoder() {
181            Decoder::Utf8 => return utf8::decode_owned(bytes),
182            Decoder::Utf7 => utf7::decode(&bytes),
183            Decoder::Utf16Bom => return utf16::decode_bom(&bytes),
184            Decoder::Utf16(endian) => return endian.decode(&bytes),
185            Decoder::SingleByte(table) => table.decode(&bytes),
186            Decoder::MultiByte(encoding) => encoding.decode(&bytes),
187        };
188        let text = match decoded {
189            Cow::Owned(text) => return text,
190            Cow::Borrowed(text) => text,
191        };
192        if text.len() == bytes.len() {
193            utf8::decode_owned(bytes)
194        } else {
195            text.to_owned()
196        }
197    }
198
199    pub(crate) fn decode_prefix(self, bytes: &[u8]) -> Cow<'_, str> {
200        match self.decoder() {
201            Decoder::Utf8 => utf8::decode(utf8::complete_prefix(bytes)),
202            Decoder::Utf7 => utf7::decode(utf7::complete_prefix(bytes)),
203            Decoder::Utf16Bom => Cow::Owned(utf16::decode_bom(utf16::complete_prefix_bom(bytes))),
204            Decoder::Utf16(endian) => Cow::Owned(endian.decode(endian.complete_prefix(bytes))),
205            Decoder::SingleByte(table) => table.decode(bytes),
206            Decoder::MultiByte(encoding) => encoding.decode_prefix(bytes),
207        }
208    }
209
210    /// Converts `bytes` like [`Charset::decode`], appending the text to
211    /// `out`.
212    pub fn decode_append(self, bytes: &[u8], out: &mut String) {
213        match self.decoder() {
214            Decoder::Utf8 => utf8::decode_append(bytes, out),
215            Decoder::Utf7 => utf7::decode_append(bytes, out),
216            Decoder::Utf16Bom => utf16::decode_bom_append(bytes, out),
217            Decoder::Utf16(endian) => endian.decode_append(bytes, out),
218            Decoder::SingleByte(table) => table.decode_append(bytes, out),
219            Decoder::MultiByte(encoding) => encoding.decode_append(bytes, out),
220        }
221    }
222
223    pub(crate) fn is_supported(self) -> bool {
224        cfg!(feature = "full_encoding") || !matches!(self.decoder(), Decoder::MultiByte(_))
225    }
226
227    /// Converts `bytes` to text like [`Charset::decode`] and reports whether any sequence was not valid in the charset.
228    pub fn decode_checked(self, bytes: &[u8]) -> (Cow<'_, str>, bool) {
229        match self.decoder() {
230            Decoder::Utf8 => utf8::decode_checked(bytes),
231            Decoder::Utf7 => utf7::decode_checked(bytes),
232            Decoder::Utf16Bom => {
233                let (text, malformed) = utf16::decode_bom_checked(bytes);
234                (Cow::Owned(text), malformed)
235            }
236            Decoder::Utf16(endian) => {
237                let (text, malformed) = endian.decode_checked(bytes);
238                (Cow::Owned(text), malformed)
239            }
240            Decoder::SingleByte(table) => {
241                let text = table.decode(bytes);
242                let malformed = text.contains(char::REPLACEMENT_CHARACTER);
243                (text, malformed)
244            }
245            Decoder::MultiByte(encoding) => encoding.decode_checked(bytes),
246        }
247    }
248
249    pub(crate) fn decode_owned_checked(self, bytes: Vec<u8>) -> (String, bool) {
250        if let Decoder::Utf8 = self.decoder() {
251            return utf8::decode_owned_checked(bytes);
252        }
253        let (decoded, malformed) = self.decode_checked(&bytes);
254        let text = match decoded {
255            Cow::Owned(text) => return (text, malformed),
256            Cow::Borrowed(text) => text,
257        };
258        if text.len() == bytes.len() {
259            (utf8::decode_owned(bytes), malformed)
260        } else {
261            (text.to_owned(), malformed)
262        }
263    }
264
265    pub(crate) fn decode_cow(self, bytes: Cow<'_, [u8]>) -> Cow<'_, str> {
266        match bytes {
267            Cow::Borrowed(bytes) => self.decode(bytes),
268            Cow::Owned(bytes) => Cow::Owned(self.decode_owned(bytes)),
269        }
270    }
271
272    pub(crate) fn decode_cow_checked(self, bytes: Cow<'_, [u8]>) -> (Cow<'_, str>, bool) {
273        match bytes {
274            Cow::Borrowed(bytes) => self.decode_checked(bytes),
275            Cow::Owned(bytes) => {
276                let (text, malformed) = self.decode_owned_checked(bytes);
277                (Cow::Owned(text), malformed)
278            }
279        }
280    }
281
282    fn decoder(self) -> Decoder {
283        match self {
284            Charset::Utf8 => Decoder::Utf8,
285            Charset::Utf7 => Decoder::Utf7,
286            Charset::Utf16 => Decoder::Utf16Bom,
287            Charset::Utf16Le => Decoder::Utf16(Endian::Little),
288            Charset::Utf16Be => Decoder::Utf16(Endian::Big),
289            Charset::Iso8859_2 => Decoder::SingleByte(&tables::ISO_8859_2),
290            Charset::Iso8859_3 => Decoder::SingleByte(&tables::ISO_8859_3),
291            Charset::Iso8859_4 => Decoder::SingleByte(&tables::ISO_8859_4),
292            Charset::Iso8859_5 => Decoder::SingleByte(&tables::ISO_8859_5),
293            Charset::Iso8859_6 => Decoder::SingleByte(&tables::ISO_8859_6),
294            Charset::Iso8859_7 => Decoder::SingleByte(&tables::ISO_8859_7),
295            Charset::Iso8859_8 => Decoder::SingleByte(&tables::ISO_8859_8),
296            Charset::Iso8859_9 => Decoder::SingleByte(&tables::ISO_8859_9),
297            Charset::Iso8859_10 => Decoder::SingleByte(&tables::ISO_8859_10),
298            Charset::Iso8859_13 => Decoder::SingleByte(&tables::ISO_8859_13),
299            Charset::Iso8859_14 => Decoder::SingleByte(&tables::ISO_8859_14),
300            Charset::Iso8859_15 => Decoder::SingleByte(&tables::ISO_8859_15),
301            Charset::Iso8859_16 => Decoder::SingleByte(&tables::ISO_8859_16),
302            Charset::Windows1250 => Decoder::SingleByte(&tables::CP1250),
303            Charset::Windows1251 => Decoder::SingleByte(&tables::CP1251),
304            Charset::Windows1252 => Decoder::SingleByte(&tables::CP1252),
305            Charset::Windows1253 => Decoder::SingleByte(&tables::CP1253),
306            Charset::Windows1254 => Decoder::SingleByte(&tables::CP1254),
307            Charset::Windows1255 => Decoder::SingleByte(&tables::CP1255),
308            Charset::Windows1256 => Decoder::SingleByte(&tables::CP1256),
309            Charset::Windows1257 => Decoder::SingleByte(&tables::CP1257),
310            Charset::Windows1258 => Decoder::SingleByte(&tables::CP1258),
311            Charset::Koi8R => Decoder::SingleByte(&tables::KOI8_R),
312            Charset::Koi8U => Decoder::SingleByte(&tables::KOI8_U),
313            Charset::Macintosh => Decoder::SingleByte(&tables::MACINTOSH),
314            Charset::Ibm850 => Decoder::SingleByte(&tables::IBM850),
315            Charset::Tis620 => Decoder::SingleByte(&tables::TIS_620),
316            Charset::ShiftJis => Decoder::MultiByte(MultiByte::ShiftJis),
317            Charset::Big5 => Decoder::MultiByte(MultiByte::Big5),
318            Charset::EucJp => Decoder::MultiByte(MultiByte::EucJp),
319            Charset::EucKr => Decoder::MultiByte(MultiByte::EucKr),
320            Charset::Gb18030 => Decoder::MultiByte(MultiByte::Gb18030),
321            Charset::Gbk => Decoder::MultiByte(MultiByte::Gbk),
322            Charset::Iso2022Jp => Decoder::MultiByte(MultiByte::Iso2022Jp),
323            Charset::Windows874 => Decoder::SingleByte(&tables::WINDOWS_874),
324            Charset::Ibm866 => Decoder::MultiByte(MultiByte::Ibm866),
325            Charset::XMacCyrillic => Decoder::MultiByte(MultiByte::XMacCyrillic),
326            Charset::XUserDefined => Decoder::MultiByte(MultiByte::XUserDefined),
327            Charset::Replacement => Decoder::MultiByte(MultiByte::Replacement),
328        }
329    }
330}
331
332/// Converts `bytes` from the charset with this label; UTF-8 when the label
333/// is unknown.
334pub fn decode<'x>(charset: &[u8], bytes: &'x [u8]) -> Cow<'x, str> {
335    Charset::from_label(charset)
336        .unwrap_or_default()
337        .decode(bytes)
338}
339
340/// Converts `bytes` from the charset with this label (UTF-8 when the label
341/// is unknown), appending the text to `out`. The signature fits the charset
342/// callback of the `encodify::rfc2047` decoders.
343pub fn decode_append(charset: &[u8], bytes: &[u8], out: &mut String) {
344    let charset = if charset.eq_ignore_ascii_case(UTF8_LABEL) {
345        Charset::Utf8
346    } else {
347        Charset::from_label(charset).unwrap_or_default()
348    };
349    charset.decode_append(bytes, out);
350}
351
352/// The 0.11 decoder lookup: a function converting text from the charset
353/// with this label, or `None` for UTF-8 and unknown labels.
354/// [`Charset::from_label`] and [`Charset::decode`] do the same without a
355/// function pointer and borrow when they can.
356pub fn charset_decoder(charset: &[u8]) -> Option<DecoderFnc> {
357    macro_rules! decoder_fnc {
358        ($charset:expr, $($variant:ident),+) => {
359            match $charset {
360                Charset::Utf8 => None,
361                $(Charset::$variant => Some(|bytes| Charset::$variant.decode(bytes).into_owned()),)+
362            }
363        };
364    }
365
366    decoder_fnc!(
367        Charset::from_label(charset)?,
368        Utf7,
369        Utf16,
370        Utf16Le,
371        Utf16Be,
372        Iso8859_2,
373        Iso8859_3,
374        Iso8859_4,
375        Iso8859_5,
376        Iso8859_6,
377        Iso8859_7,
378        Iso8859_8,
379        Iso8859_9,
380        Iso8859_10,
381        Iso8859_13,
382        Iso8859_14,
383        Iso8859_15,
384        Iso8859_16,
385        Windows1250,
386        Windows1251,
387        Windows1252,
388        Windows1253,
389        Windows1254,
390        Windows1255,
391        Windows1256,
392        Windows1257,
393        Windows1258,
394        Koi8R,
395        Koi8U,
396        Macintosh,
397        Ibm850,
398        Tis620,
399        ShiftJis,
400        Big5,
401        EucJp,
402        EucKr,
403        Gb18030,
404        Gbk,
405        Iso2022Jp,
406        Windows874,
407        Ibm866,
408        XMacCyrillic,
409        XUserDefined,
410        Replacement
411    )
412}
413
414#[cfg(test)]
415mod tests {
416    use super::{Charset, charset_decoder, decode_append, utf8};
417    use crate::scan::tests::Rng;
418    use std::borrow::Cow;
419
420    const ALL: [Charset; 44] = [
421        Charset::Utf8,
422        Charset::Utf7,
423        Charset::Utf16,
424        Charset::Utf16Le,
425        Charset::Utf16Be,
426        Charset::Iso8859_2,
427        Charset::Iso8859_3,
428        Charset::Iso8859_4,
429        Charset::Iso8859_5,
430        Charset::Iso8859_6,
431        Charset::Iso8859_7,
432        Charset::Iso8859_8,
433        Charset::Iso8859_9,
434        Charset::Iso8859_10,
435        Charset::Iso8859_13,
436        Charset::Iso8859_14,
437        Charset::Iso8859_15,
438        Charset::Iso8859_16,
439        Charset::Windows1250,
440        Charset::Windows1251,
441        Charset::Windows1252,
442        Charset::Windows1253,
443        Charset::Windows1254,
444        Charset::Windows1255,
445        Charset::Windows1256,
446        Charset::Windows1257,
447        Charset::Windows1258,
448        Charset::Koi8R,
449        Charset::Koi8U,
450        Charset::Macintosh,
451        Charset::Ibm850,
452        Charset::Tis620,
453        Charset::ShiftJis,
454        Charset::Big5,
455        Charset::EucJp,
456        Charset::EucKr,
457        Charset::Gb18030,
458        Charset::Gbk,
459        Charset::Iso2022Jp,
460        Charset::Windows874,
461        Charset::Ibm866,
462        Charset::XMacCyrillic,
463        Charset::XUserDefined,
464        Charset::Replacement,
465    ];
466
467    const CHECKED_SAMPLES: usize = if cfg!(miri) { 30 } else { 3_000 };
468
469    #[test]
470    fn checked_decoding_returns_the_same_text() {
471        let alphabet =
472            b"aZ09 +-/=\r\n\x00\x1b$B(\x80\x81\xa4\xa5\xbf\xc3\xa9\xd8\xdc\xe9\xef\xfe\xff";
473        let mut rng = Rng(0x0dec_0de0_c4ec_4ed0);
474        for charset in ALL {
475            for _ in 0..CHECKED_SAMPLES {
476                let len = rng.below(24);
477                let bytes: Vec<u8> = (0..len)
478                    .map(|_| {
479                        alphabet
480                            .get(rng.below(alphabet.len()))
481                            .copied()
482                            .unwrap_or(b'a')
483                    })
484                    .collect();
485                let (checked, malformed) = charset.decode_checked(&bytes);
486                assert_eq!(checked, charset.decode(&bytes), "{charset:?} {bytes:?}");
487                let (owned, owned_malformed) = charset.decode_owned_checked(bytes.clone());
488                assert_eq!(
489                    owned,
490                    charset.decode_owned(bytes.clone()),
491                    "{charset:?} {bytes:?}"
492                );
493                assert_eq!(owned_malformed, malformed, "{charset:?} {bytes:?}");
494                if !malformed
495                    && matches!(charset, Charset::Utf8 | Charset::Utf16Le | Charset::Utf16Be)
496                {
497                    assert!(
498                        !checked.contains(char::REPLACEMENT_CHARACTER),
499                        "{charset:?} {bytes:?}"
500                    );
501                }
502                if !charset.is_supported() {
503                    assert_eq!(malformed, matches!(checked, Cow::Owned(_)));
504                }
505            }
506        }
507    }
508
509    #[test]
510    fn owned_decoding_matches_borrowed_decoding() {
511        let mut rng = Rng(0x0dec_0de0_c4ec_4ed1);
512        for charset in ALL {
513            for len in [
514                utf8::SIMD_MIN - 1,
515                utf8::SIMD_MIN,
516                utf8::SIMD_MIN + 1,
517                3 * utf8::SIMD_MIN,
518            ] {
519                for rare in [None, Some(b'\xe9'), Some(b'\xff'), Some(b'\xc3')] {
520                    let mut bytes: Vec<u8> = (0..len).map(|_| b'a' + rng.below(26) as u8).collect();
521                    if let (Some(byte), Some(slot)) = (rare, bytes.get_mut(rng.below(len))) {
522                        *slot = byte;
523                    }
524                    let expected = charset.decode(&bytes).into_owned();
525                    let (_, malformed) = charset.decode_checked(&bytes);
526                    assert_eq!(
527                        charset.decode_owned(bytes.clone()),
528                        expected,
529                        "{charset:?} {len}"
530                    );
531                    assert_eq!(
532                        charset.decode_owned_checked(bytes),
533                        (expected, malformed),
534                        "{charset:?} {len}"
535                    );
536                }
537            }
538        }
539    }
540
541    #[test]
542    fn malformed_charset_sequences() {
543        let cases: [(Charset, &[u8], bool); 27] = [
544            (Charset::Utf8, b"caf\xc3\xa9", false),
545            (Charset::Utf8, b"caf\xe9", true),
546            (Charset::Utf8, b"\xef\xbf\xbd", false),
547            (Charset::Utf8, b"", false),
548            (Charset::Iso8859_3, b"caf\xe9", false),
549            (Charset::Iso8859_3, b"\xa5", true),
550            (Charset::Windows1252, b"\x80\xff", false),
551            (Charset::Windows1252, b"\x81", true),
552            (Charset::Utf16Le, b"a\x00b\x00", false),
553            (Charset::Utf16Le, b"a\x00b", true),
554            (Charset::Utf16Le, b"\x00\xd8a\x00", true),
555            (Charset::Utf16Be, b"\xff\xfd", false),
556            (Charset::Utf16, b"\xfe\xff\x00a", false),
557            (Charset::Utf16, b"\xfe\xff\x00a\x00", true),
558            (Charset::Utf7, b"Hi Mom -+Jjo--!", false),
559            (Charset::Utf7, b"+AKM-1", false),
560            (Charset::Utf7, b"Hi +AKM", true),
561            (Charset::Utf7, b"caf\xc3\xa9", true),
562            (Charset::Utf7, b"+2D0-", true),
563            (Charset::Utf7, b"+A-", true),
564            (Charset::Utf7, b"a+", true),
565            (Charset::Utf7, b"1 +- 1", false),
566            (Charset::Utf7, b"x +AKM y", false),
567            (Charset::Utf7, b"+AKM\xe9", true),
568            (Charset::Utf7, b"+//0-", false),
569            (Charset::ShiftJis, b"plain", false),
570            (Charset::Replacement, b"", false),
571        ];
572        for (charset, bytes, expected) in cases {
573            let malformed = charset.decode_checked(bytes).1;
574            assert_eq!(malformed, expected, "{charset:?} {bytes:?}");
575            assert_eq!(
576                charset.decode_owned_checked(bytes.to_vec()).1,
577                malformed,
578                "{charset:?} {bytes:?}"
579            );
580        }
581        #[cfg(feature = "full_encoding")]
582        {
583            assert!(!Charset::ShiftJis.decode_checked(b"\x83n\x83\x8D").1);
584            assert!(Charset::ShiftJis.decode_checked(b"\x83").1);
585            assert!(Charset::Replacement.decode_checked(b"abc").1);
586            assert!(Charset::ShiftJis.is_supported());
587        }
588        #[cfg(not(feature = "full_encoding"))]
589        assert!(!Charset::ShiftJis.is_supported());
590        assert!(Charset::Iso8859_2.is_supported() && Charset::Utf16.is_supported());
591    }
592
593    #[test]
594    fn utf8_label_shortcut_matches_lookup() {
595        for label in ["utf-8", "UTF-8", "Utf-8", "utf8", "iso-8859-1", "x", ""] {
596            for bytes in [&b"caf\xc3\xa9"[..], b"caf\xe9 \xff", b"\xc3", b""] {
597                let mut expected = String::new();
598                Charset::from_label(label.as_bytes())
599                    .unwrap_or_default()
600                    .decode_append(bytes, &mut expected);
601                let mut actual = String::new();
602                decode_append(label.as_bytes(), bytes, &mut actual);
603                assert_eq!(actual, expected, "{label:?} {bytes:?}");
604            }
605        }
606    }
607
608    #[test]
609    fn decode_charset() {
610        let inputs = [
611            ("iso-8859-1", b"\xe1\xe9\xed\xf3\xfa".to_vec(), "áéíóú"),
612            ("iso-8859-1", b"\x805.4bn".to_vec(), "€5.4bn"),
613            ("latin1", b"\x805.4bn".to_vec(), "€5.4bn"),
614            ("iso88591", b"\x805.4bn".to_vec(), "€5.4bn"),
615            ("us-ascii", b"\x805.4bn".to_vec(), "€5.4bn"),
616            ("iso8859-5", b"\xbf\xe0\xd8\xd2\xd5\xe2, \xdc\xd8\xe0".to_vec(), "Привет, мир"),
617            ("cp1252", b"\xa1El \xf1and\xfa comi\xf3 \xf1oquis!".to_vec(), "¡El ñandú comió ñoquis!"),
618            ("iso-8859-5", b"\xbf\xe0\xd8\xd2\xd5\xe2, \xdc\xd8\xe0".to_vec(), "Привет, мир"),
619            ("iso-8859-6", b"\xe5\xd1\xcd\xc8\xc7 \xc8\xc7\xe4\xd9\xc7\xe4\xe5".to_vec(),"مرحبا بالعالم"),
620            ("iso-8859-7", b"\xc3\xe5\xe9\xdc \xf3\xef\xf5 \xca\xfc\xf3\xec\xe5".to_vec(),"Γειά σου Κόσμε"),
621            ("iso-8859-8", b"\xf9\xec\xe5\xed \xf2\xe5\xec\xed".to_vec(),"שלום עולם"),
622            ("iso-8859-11", b"\xc3\xcb\xd1\xca\xca\xd3\xcb\xc3\xd1\xba\xcd\xd1\xa1\xa2\xc3\xd0\xe4\xb7\xc2\xb7\xd5\xe8\xe3\xaa\xe9\xa1\xd1\xba\xa4\xcd\xc1\xbe\xd4\xc7\xe0\xb5\xcd\xc3\xec".to_vec(),"รหัสสำหรับอักขระไทยที่ใช้กับคอมพิวเตอร์"),
623            ("windows-1250", b"Zelo rada grem v sla\x9a\xe8i\xe8arno".to_vec(),"Zelo rada grem v slaščičarno"),
624            ("windows-1251", b"\xcf\xf0\xe8\xe2\xe5\xf2, \xec\xe8\xf0".to_vec(),"Привет, мир"),
625            ("windows-1252", b"\xa1El \xf1and\xfa comi\xf3 \xf1oquis!".to_vec(),"¡El ñandú comió ñoquis!"),
626            ("windows-1253", b"\xca\xf9\xe4\xe9\xea\xef\xdf \xd3\xf9\xea\xf1\xdc\xf4\xe7\xf2 \xf3\xf4\xef Rust".to_vec(),"Κωδικοί Σωκράτης στο Rust"),
627            ("windows-1254", b"Kebab\xfdm\xfd baharatl\xfd yapma".to_vec(),"Kebabımı baharatlı yapma"),
628            ("windows-1255", b"\xf9\xec\xe5\xed \xf2\xe5\xec\xed".to_vec(),"שלום עולם"),
629            ("windows-1256", b"\xe3\xd1\xcd\xc8\xc7 \xc8\xc7\xe1\xda\xc7\xe1\xe3".to_vec(),"مرحبا بالعالم"),
630            ("windows-1257", b"Mu h\xf5ljuk on angerjaid t\xe4is".to_vec(),"Mu hõljuk on angerjaid täis"),
631            ("windows-1258", b"Xin ch\xe0o".to_vec(),"Xin chào"),
632            ("macintosh", b"\x87\x8e\x92\x97\x9c".to_vec(),"áéíóú"),
633            ("ibm850", b"\x9b\x9c\x9d\x9e".to_vec(),"ø£Ø×"),
634            ("koi8-r", b"\xf0\xd2\xc9\xd7\xc5\xd4, \xcd\xc9\xd2".to_vec(),"Привет, мир"),
635            ("koi8-u", b"\xf0\xd2\xc9\xd7\xa6\xd4 \xf3\xd7\xa6\xd4".to_vec(),"Привіт Світ"),
636            // Positions that ISO 8859-7:2003 added over the 1987 edition.
637            ("iso-8859-7", b"\xa4\xa5\xaa".to_vec(),"€₯ͺ"),
638            // 0xa5 and 0xab were transposed.
639            ("iso-8859-16", b"\xa5\xab".to_vec(),"„«"),
640            // 0xdb became the euro sign in Mac OS 8.5; 0xf0 is the Apple logo.
641            ("macintosh", b"\xc6\xcd\xdb\xf0\xf6\xf7".to_vec(),"∆Õ€\u{f8ff}ˆ˜"),
642            // Hebrew point holam haser for vav, a later windows-1255 addition.
643            ("windows-1255", b"\xca".to_vec(),"\u{5ba}"),
644            // The iso-8859-9 and tis-620 labels resolve to the Windows
645            // supersets, so bytes in 0x80..=0xa0 decode instead of being lost.
646            ("iso-8859-9", b"\x93Merhaba\x94 \x96 d\xfcnya".to_vec(),"“Merhaba” – dünya"),
647            ("iso-8859-9", b"\x805.4bn".to_vec(),"€5.4bn"),
648            ("latin5", b"\x805.4bn".to_vec(),"€5.4bn"),
649            ("tis-620", b"\x93\xca\xc7\xd1\xca\xb4\xd5\x94".to_vec(),"“สวัสดี”"),
650            ("iso-8859-11", b"\x805.4bn".to_vec(),"€5.4bn"),
651            // Thai text decodes as before; only the 0x80..=0xa0 band changed.
652            ("tis-620", b"\xc3\xcb\xd1\xca".to_vec(),"รหัส"),
653            ("utf-7", b"+ZYeB9FH6ckh5Pg-, 1980.".to_vec(),"文致出版社, 1980."),
654            ("utf-16le", b"\xcf0\xed0\xfc0\xfb0\xef0\xfc0\xeb0\xc90".to_vec(),"ハロー・ワールド"),
655            ("utf-16be", b"0\xcf0\xed0\xfc0\xfb0\xef0\xfc0\xeb0\xc9".to_vec(),"ハロー・ワールド"),
656            ("utf-16", b"\xff\xfe\xe1\x00\xe9\x00\xed\x00\xf3\x00\xfa\x00".to_vec(),"áéíóú"),
657            ("utf-16", b"\xfe\xff\x00\xe1\x00\xe9\x00\xed\x00\xf3\x00\xfa".to_vec(),"áéíóú"),
658
659            #[cfg(feature = "full_encoding")]
660            ("shift_jis", b"\x83n\x83\x8D\x81[\x81E\x83\x8F\x81[\x83\x8B\x83h".to_vec(),"ハロー・ワールド"),
661            #[cfg(feature = "full_encoding")]
662            ("big5", b"\xa7A\xa6n\xa1A\xa5@\xac\xc9".to_vec(),"你好，世界"),
663            #[cfg(feature = "full_encoding")]
664            ("euc-jp", b"\xa5\xcf\xa5\xed\xa1\xbc\xa1\xa6\xa5\xef\xa1\xbc\xa5\xeb\xa5\xc9".to_vec(),"ハロー・ワールド"),
665            #[cfg(feature = "full_encoding")]
666            ("euc-kr", b"\xbe\xc8\xb3\xe7\xc7\xcf\xbc\xbc\xbf\xe4 \xbc\xbc\xb0\xe8".to_vec(),"안녕하세요 세계"),
667            #[cfg(feature = "full_encoding")]
668            ("cp949", b"\xbe\xc8\xb3\xe7\xc7\xcf\xbc\xbc\xbf\xe4 \xbc\xbc\xb0\xe8".to_vec(),"안녕하세요 세계"),
669            #[cfg(feature = "full_encoding")]
670            ("uhc", b"\xbe\xc8\xb3\xe7\xc7\xcf\xbc\xbc\xbf\xe4 \xbc\xbc\xb0\xe8".to_vec(),"안녕하세요 세계"),
671            #[cfg(feature = "full_encoding")]
672            ("x-windows-949", b"\xbe\xc8\xb3\xe7\xc7\xcf\xbc\xbc\xbf\xe4 \xbc\xbc\xb0\xe8".to_vec(),"안녕하세요 세계"),
673            #[cfg(feature = "full_encoding")]
674            ("windows-949", b"\xbe\xc8\xb3\xe7\xc7\xcf\xbc\xbc\xbf\xe4 \xbc\xbc\xb0\xe8".to_vec(),"안녕하세요 세계"),
675            #[cfg(feature = "full_encoding")]
676            ("iso-2022-jp", b"\x1b$B%O%m!<!&%o!<%k%I\x1b(B".to_vec(),"ハロー・ワールド"),
677            #[cfg(feature = "full_encoding")]
678            ("gbk", b"\xc4\xe3\xba\xc3\xa3\xac\xca\xc0\xbd\xe7".to_vec(),"你好，世界"),
679            #[cfg(feature = "full_encoding")]
680            ("gb18030", b"\xc4\xe3\xba\xc3\xa3\xac\xca\xc0\xbd\xe7".to_vec(),"你好，世界"),
681            #[cfg(feature = "full_encoding")]
682            ("x-mac-cyrillic", b"\x8f\xf0\xe8\xe2\xe5\xf2".to_vec(),"Привет"),
683            #[cfg(feature = "full_encoding")]
684            ("x-user-defined", b"\x80\xff".to_vec(),"\u{f780}\u{f7ff}"),
685            #[cfg(feature = "full_encoding")]
686            ("iso-2022-kr", b"\x1b$)Cabcd".to_vec(),"\u{fffd}"),
687            #[cfg(feature = "full_encoding")]
688            ("hz-gb-2312", b"~{...~}".to_vec(),"\u{fffd}"),
689            ];
690
691        for (label, bytes, expected) in inputs {
692            let decoder = charset_decoder(label.as_bytes()).expect(label);
693            assert_eq!(decoder(&bytes), expected, "{label}");
694            let charset = Charset::from_label(label.as_bytes()).expect(label);
695            assert_eq!(charset.decode(&bytes), expected, "{label}");
696            let mut out = String::from(">");
697            charset.decode_append(&bytes, &mut out);
698            assert_eq!(out.strip_prefix('>'), Some(expected), "{label}");
699        }
700    }
701
702    #[test]
703    fn decoder_charset() {
704        for input in ["gbk", "extended_unix_code_packed_format_for_japanese"] {
705            if !input.is_empty() {
706                assert!(
707                    charset_decoder(input.as_bytes()).is_some(),
708                    "Failed for {input}",
709                );
710            }
711        }
712    }
713
714    #[test]
715    fn decoder_charset_encoding_rs_labels() {
716        let supported = [
717            "l9",
718            "koi",
719            "koi8",
720            "sjis",
721            "ucs-2",
722            "ms932",
723            "ascii",
724            "x-gbk",
725            "cp1250",
726            "cp1251",
727            "cp1252",
728            "cp1253",
729            "cp1254",
730            "cp1255",
731            "cp1256",
732            "cp1257",
733            "cp1258",
734            "visual",
735            "korean",
736            "x-sjis",
737            "ksc5601",
738            "gb_2312",
739            "dos-874",
740            "cn-big5",
741            "unicode",
742            "chinese",
743            "logical",
744            "koi8-ru",
745            "x-cp1250",
746            "ksc_5601",
747            "x-cp1251",
748            "iso88591",
749            "csgb2312",
750            "x-cp1252",
751            "iso88592",
752            "x-cp1253",
753            "iso88593",
754            "x-cp1254",
755            "iso88594",
756            "x-cp1255",
757            "iso88595",
758            "x-x-big5",
759            "x-cp1256",
760            "iso88596",
761            "x-cp1257",
762            "iso88597",
763            "x-cp1258",
764            "iso88598",
765            "iso88599",
766            "us-ascii",
767            "x-euc-jp",
768            "iso885910",
769            "iso8859-1",
770            "iso885911",
771            "iso8859-2",
772            "iso8859-3",
773            "iso885913",
774            "iso8859-4",
775            "iso885914",
776            "iso8859-5",
777            "iso885915",
778            "iso8859-6",
779            "iso8859-7",
780            "iso8859-8",
781            "iso-ir-58",
782            "iso8859-9",
783            "csunicode",
784            "iso8859-10",
785            "gb_2312-80",
786            "iso8859-11",
787            "iso8859-13",
788            "iso8859-14",
789            "iso8859-15",
790            "iso-ir-149",
791            "big5-hkscs",
792            "windows-949",
793            "cp949",
794            "uhc",
795            "x-windows-949",
796            "csisolatin9",
797            "csiso88596e",
798            "csiso88598e",
799            "unicodefffe",
800            "unicodefeff",
801            "csiso88596i",
802            "csiso88598i",
803            "windows-31j",
804            "x-mac-roman",
805            "sun_eu_greek",
806            "csksc56011987",
807            "ansi_x3.4-1968",
808            "csiso58gb231280",
809            "iso-10646-ucs-2",
810            "iso-8859-6-e",
811            "iso-8859-8-e",
812            "iso-8859-6-i",
813            "replacement",
814            "iso-2022-kr",
815            "csiso2022kr",
816            "iso-2022-cn",
817            "iso-2022-cn-ext",
818            "hz-gb-2312",
819            "x-user-defined",
820            "x-mac-cyrillic",
821            "x-mac-ukrainian",
822        ];
823        for input in supported {
824            assert!(
825                charset_decoder(input.as_bytes()).is_some(),
826                "Expected a decoder for {input}",
827            );
828        }
829
830        let unsupported = [
831            "utf8",
832            "utf-8",
833            "unicode11utf8",
834            "unicode20utf8",
835            "x-unicode20utf8",
836            "unicode-1-1-utf-8",
837        ];
838        for input in unsupported {
839            assert!(
840                charset_decoder(input.as_bytes()).is_none(),
841                "Did not expect a decoder for {input}",
842            );
843            assert_eq!(
844                Charset::from_label(input.as_bytes()),
845                Some(Charset::Utf8),
846                "{input}"
847            );
848        }
849        for input in ["", "unknown", "utf-9", "x-unknown-8bit"] {
850            assert_eq!(Charset::from_label(input.as_bytes()), None, "{input}");
851        }
852    }
853
854    #[test]
855    fn decode_utf7() {
856        let inputs = [
857            ("Hello, World+ACE-", "Hello, World!"),
858            ("Hi Mom -+Jjo--!", "Hi Mom -☺-!"),
859            ("+ZeVnLIqe-", "日本語"),
860            ("Item 3 is +AKM-1.", "Item 3 is £1."),
861            ("Plus minus +- -+ +--", "Plus minus + -+ +-"),
862            (
863                "+APw-ber ihre mi+AN8-liche Lage+ADs- +ACI-wir",
864                "über ihre mißliche Lage; \"wir",
865            ),
866            (
867                concat!(
868                    "+ACI-The sayings of Confucius,+ACI- James R. Ware, trans.  +U/BTFw-:\n",
869                    "+ZYeB9FH6ckh5Pg-, 1980.\n",
870                    "+Vttm+E6UfZM-, +W4tRQ066bOg-, +UxdOrA-:  +Ti1XC2b4Xpc-, 1990."
871                ),
872                concat!(
873                    "\"The sayings of Confucius,\" James R. Ware, trans.  台北:\n",
874                    "文致出版社, 1980.\n",
875                    "四書五經, 宋元人注, 北京:  中國書店, 1990."
876                ),
877            ),
878        ];
879
880        for (input, expected) in inputs {
881            assert_eq!(Charset::Utf7.decode(input.as_bytes()), expected);
882        }
883        for (input, expected) in [
884            ("+AKM.", "\u{a3}."),
885            ("x +AKM y", "x \u{a3} y"),
886            ("+AKM, then", "\u{a3}, then"),
887            ("+AKM\r\nnext", "\u{a3}\r\nnext"),
888            ("+AKM-- dash", "\u{a3}- dash"),
889            ("1 +- 1", "1 + 1"),
890            ("+!", "+!"),
891            ("+A.", "\u{fffd}."),
892            ("caf\u{e9}", "caf\u{c3}\u{a9}"),
893            ("Hi +AKM", "Hi "),
894            ("+A-", "\u{fffd}"),
895            ("+2D3eAA-", "\u{1f600}"),
896            ("+2D0-", "\u{fffd}"),
897        ] {
898            assert_eq!(Charset::Utf7.decode(input.as_bytes()), expected, "{input}");
899        }
900    }
901
902    #[cfg(feature = "full_encoding")]
903    #[test]
904    fn encoding_rs_charsets_drop_a_utf8_bom() {
905        for (bytes, expected) in [
906            (&b"\xef\xbb\xbf"[..], ""),
907            (b"\xef\xbb\xbfabc\xd0\x96", "abc\u{416}"),
908        ] {
909            for charset in [Charset::Ibm866, Charset::ShiftJis, Charset::XMacCyrillic] {
910                let decoded = charset.decode(bytes);
911                assert!(matches!(decoded, Cow::Borrowed(_)), "{charset:?} {bytes:?}");
912                assert_eq!(decoded, expected, "{charset:?} {bytes:?}");
913                assert_eq!(charset.decode_owned(bytes.to_vec()), expected);
914                let mut appended = String::from("<");
915                charset.decode_append(bytes, &mut appended);
916                assert_eq!(appended.strip_prefix('<'), Some(expected));
917                assert_eq!(charset.decode_checked(bytes).0, expected);
918            }
919        }
920    }
921
922    #[test]
923    fn utf16_dangling_byte_is_replaced() {
924        for (charset, bytes, expected) in [
925            (Charset::Utf16Le, &b"a\x00b\x00c"[..], "ab\u{fffd}"),
926            (Charset::Utf16Be, b"\x00a\x00b\x00", "ab\u{fffd}"),
927            (Charset::Utf16, b"\xff\xfea\x00b", "a\u{fffd}"),
928            (Charset::Utf16, b"\xfe\xff\x00a\x00", "a\u{fffd}"),
929            (Charset::Utf16Le, b"\x00\xd8a", "\u{fffd}"),
930            (Charset::Utf16Be, b"\xd8\x00\x00a\x00", "\u{fffd}a\u{fffd}"),
931            (Charset::Utf16Le, b"x", "\u{fffd}"),
932            (Charset::Utf16Le, b"a\x00", "a"),
933        ] {
934            assert_eq!(charset.decode(bytes), expected, "{charset:?} {bytes:?}");
935            assert_eq!(charset.decode_owned(bytes.to_vec()), expected);
936            let mut appended = String::from("<");
937            charset.decode_append(bytes, &mut appended);
938            assert_eq!(appended.strip_prefix('<'), Some(expected));
939            let (checked, malformed) = charset.decode_checked(bytes);
940            assert_eq!(checked, expected, "{charset:?} {bytes:?}");
941            assert_eq!(malformed, expected.contains('\u{fffd}'));
942        }
943    }
944}
mail_parser/decoders/charsets/mod.rs

mail_parser/decoders/charsets/
mod.rs
