oxixml_encoding/label.rs
1//! The [`Encoding`] enum and IANA/WHATWG label resolution.
2
3use crate::decoder::Decoder;
4
5/// A character encoding recognized by this crate.
6///
7/// The original core set is exactly what an XML processor needs to deal with in practice:
8/// `UTF-8` and `UTF-16` (in both byte orders) are the two encodings XML 1.0 requires every
9/// conformant processor to support; [`Encoding::Latin1`] and [`Encoding::Windows1252`] cover the
10/// overwhelming majority of legacy (`encoding="..."`) documents seen in the wild, and
11/// [`Encoding::Ascii`] is useful as a strict validation target.
12///
13/// Beyond that core, this crate also implements the rest of the
14/// [WHATWG Encoding Standard](https://encoding.spec.whatwg.org/)'s legacy single-byte encodings
15/// (the full `ISO-8859-*`, `windows-125x`, `KOI8-*`, `IBM866`, `macintosh` and `x-mac-cyrillic`
16/// families), its three legacy Japanese multi-byte encodings (Shift_JIS, EUC-JP, and
17/// ISO-2022-JP), and its legacy CJK multi-byte encodings (GBK, gb18030, Big5, EUC-KR),
18/// prioritized for COOLJAPAN. See [`Encoding::ALL`] for the complete list.
19///
20/// Unlike the WHATWG Encoding Standard, which folds `ISO-8859-1` into `windows-1252` for web
21/// compatibility, this crate keeps them distinct: XML tooling (libxml2, Xerces, .NET, the JDK,
22/// ...) has always treated `ISO-8859-1` as the "pure" Latin-1 repertoire (bytes `0x00..=0xFF` map
23/// to `U+0000..=U+00FF` unchanged) and `windows-1252` as the Microsoft superset that additionally
24/// assigns the `0x80..=0x9F` range to punctuation such as smart quotes and the euro sign. The
25/// same reasoning applies to [`Encoding::Iso8859_9`] versus [`Encoding::Windows1254`]: the WHATWG
26/// standard folds the `"iso-8859-9"`/`"latin5"` labels into windows-1254 too, but this crate keeps
27/// pure ISO-8859-9 (ISO/IEC 8859-9:1999, a.k.a. Latin-5) as its own encoding for the same reason.
28#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
29pub enum Encoding {
30 /// UTF-8: the ASCII-compatible variable-width encoding of Unicode, and the default assumed
31 /// by XML 1.0 in the absence of a byte-order mark or `encoding` declaration.
32 Utf8,
33 /// UTF-16, little-endian byte order.
34 Utf16Le,
35 /// UTF-16, big-endian byte order.
36 Utf16Be,
37 /// ISO-8859-1 (Latin-1): each byte `b` decodes to the scalar value `U+00xx` with `xx == b`.
38 Latin1,
39 /// windows-1252 (a.k.a. `cp1252`): ISO-8859-1 with the C1 control range `0x80..=0x9F`
40 /// reassigned to printable characters (the five positions Microsoft left unassigned keep
41 /// their Latin-1 fallback, per the [WHATWG index table][idx]).
42 ///
43 /// [idx]: https://encoding.spec.whatwg.org/index-windows-1252.txt
44 Windows1252,
45 /// US-ASCII: the 7-bit repertoire `U+0000..=U+007F`. Bytes `0x80..=0xFF` are not valid.
46 Ascii,
47
48 // --- ISO-8859 family (beyond -1, above) -------------------------------------------------
49 /// ISO-8859-2 (Latin-2): Central/Eastern European.
50 Iso8859_2,
51 /// ISO-8859-3 (Latin-3): South European (Maltese, Esperanto).
52 Iso8859_3,
53 /// ISO-8859-4 (Latin-4): North European (Baltic languages, pre-ISO-8859-13).
54 Iso8859_4,
55 /// ISO-8859-5: Cyrillic.
56 Iso8859_5,
57 /// ISO-8859-6: Arabic.
58 Iso8859_6,
59 /// ISO-8859-7: Greek.
60 Iso8859_7,
61 /// ISO-8859-8: Hebrew (visual order; this crate makes no distinction from the "logical
62 /// order" `ISO-8859-8-I` variant, since directionality is outside this crate's scope and
63 /// both share the same byte/code-point repertoire).
64 Iso8859_8,
65 /// ISO-8859-9 (Latin-5): identical to ISO-8859-1 except six positions replaced with the
66 /// Turkish letters `Ğ`/`ğ`, `İ`/`ı`, `Ş`/`ş`. See the type-level docs for why this is kept
67 /// distinct from [`Encoding::Windows1254`].
68 Iso8859_9,
69 /// ISO-8859-10 (Latin-6): Nordic (an alternative to -4 with full Icelandic coverage).
70 Iso8859_10,
71 /// ISO-8859-13 (Latin-7): Baltic.
72 Iso8859_13,
73 /// ISO-8859-14 (Latin-8): Celtic.
74 Iso8859_14,
75 /// ISO-8859-15 (Latin-9): Latin-1 with the euro sign and a handful of French/Finnish letters.
76 Iso8859_15,
77 /// ISO-8859-16 (Latin-10): South-Eastern European.
78 Iso8859_16,
79
80 // --- windows-125x family (beyond -1252, above) ------------------------------------------
81 /// windows-1250: Central/Eastern European.
82 Windows1250,
83 /// windows-1251: Cyrillic.
84 Windows1251,
85 /// windows-1253: Greek.
86 Windows1253,
87 /// windows-1254 (Turkish): the Microsoft superset of [`Encoding::Iso8859_9`].
88 Windows1254,
89 /// windows-1255: Hebrew.
90 Windows1255,
91 /// windows-1256: Arabic.
92 Windows1256,
93 /// windows-1257: Baltic.
94 Windows1257,
95 /// windows-1258: Vietnamese.
96 Windows1258,
97
98 // --- Other single-byte encodings ---------------------------------------------------------
99 /// KOI8-R: Russian Cyrillic.
100 Koi8R,
101 /// KOI8-U: Ukrainian Cyrillic (a KOI8-R variant with a handful of Ukrainian letters).
102 Koi8U,
103 /// IBM866 (code page 866): Cyrillic, MS-DOS.
104 Ibm866,
105 /// macintosh (a.k.a. "Mac OS Roman"): the classic Mac OS Western European encoding.
106 Macintosh,
107 /// x-mac-cyrillic (a.k.a. "Mac OS Cyrillic"): the classic Mac OS Cyrillic encoding.
108 XMacCyrillic,
109
110 // --- Legacy multi-byte Japanese encodings ------------------------------------------------
111 /// Shift_JIS (a.k.a. `MS932`/`Windows-31J`): the JIS X 0201 + JIS X 0208 encoding most
112 /// commonly used on Japanese Windows and in Japanese web content.
113 ShiftJis,
114 /// EUC-JP: the Unix-family Japanese encoding covering JIS X 0201, JIS X 0208, and (via its
115 /// `0x8F`-prefixed `SS3` sequences) JIS X 0212.
116 EucJp,
117 /// ISO-2022-JP: the 7-bit, escape-sequence-switched Japanese encoding historically used for
118 /// Japanese email (`Content-Transfer-Encoding: 7bit`-safe).
119 Iso2022Jp,
120
121 // --- Legacy multi-byte CJK encodings ------------------------------------------------------
122 /// GBK: the 2-byte-only Chinese (Simplified) encoding, a.k.a. `CP936`. Decodes identically to
123 /// [`Encoding::Gb18030`] (per the WHATWG Encoding Standard, "GBK's decoder is gb18030's
124 /// decoder"); its encoder is the more conservative one, refusing to emit the 4-byte sequences
125 /// [`Encoding::Gb18030`]'s encoder falls back to for code points outside GBK's repertoire.
126 Gbk,
127 /// gb18030: the Chinese national standard encoding. A superset of [`Encoding::Gbk`] with
128 /// 1/2/4-byte sequences whose *encoder* is total over Unicode -- every scalar value except
129 /// `U+E5E5` has a representation (see [`crate::encode_from_utf8`]). This does not quite mean
130 /// every scalar value round-trips through encode-then-decode: per the WHATWG gb18030 encoder
131 /// algorithm, 18 Private Use Area code points are deliberately encoded to a fixed legacy byte
132 /// pair (for compatibility with the GB18030-2005 standard) that decodes back to a *different*
133 /// code point under the current index -- see `crate::cjk::gb18030::encode`'s own docs for the
134 /// full list.
135 Gb18030,
136 /// Big5: the Traditional Chinese encoding historically used in Taiwan and Hong Kong, including
137 /// the Hong Kong Supplementary Character Set (HKSCS) extensions per the WHATWG index.
138 Big5,
139 /// EUC-KR: the Korean encoding covering KS X 1001 plus the Unified Hangul Code (a.k.a.
140 /// Windows-949) extension, per the WHATWG index.
141 EucKr,
142}
143
144impl Encoding {
145 /// Every encoding this crate supports, in declaration order. Useful for exhaustively testing
146 /// or listing supported encodings without having to enumerate the (growing) variant list by
147 /// hand.
148 ///
149 /// # Examples
150 ///
151 /// ```
152 /// use oxixml_encoding::Encoding;
153 ///
154 /// assert!(Encoding::ALL.contains(&Encoding::Utf8));
155 /// assert!(Encoding::ALL.contains(&Encoding::ShiftJis));
156 /// // Every entry's canonical name resolves back to itself.
157 /// for &encoding in Encoding::ALL {
158 /// assert_eq!(Encoding::for_label(encoding.name()), Some(encoding));
159 /// }
160 /// ```
161 pub const ALL: &'static [Encoding] = &[
162 Encoding::Utf8,
163 Encoding::Utf16Le,
164 Encoding::Utf16Be,
165 Encoding::Latin1,
166 Encoding::Windows1252,
167 Encoding::Ascii,
168 Encoding::Iso8859_2,
169 Encoding::Iso8859_3,
170 Encoding::Iso8859_4,
171 Encoding::Iso8859_5,
172 Encoding::Iso8859_6,
173 Encoding::Iso8859_7,
174 Encoding::Iso8859_8,
175 Encoding::Iso8859_9,
176 Encoding::Iso8859_10,
177 Encoding::Iso8859_13,
178 Encoding::Iso8859_14,
179 Encoding::Iso8859_15,
180 Encoding::Iso8859_16,
181 Encoding::Windows1250,
182 Encoding::Windows1251,
183 Encoding::Windows1253,
184 Encoding::Windows1254,
185 Encoding::Windows1255,
186 Encoding::Windows1256,
187 Encoding::Windows1257,
188 Encoding::Windows1258,
189 Encoding::Koi8R,
190 Encoding::Koi8U,
191 Encoding::Ibm866,
192 Encoding::Macintosh,
193 Encoding::XMacCyrillic,
194 Encoding::ShiftJis,
195 Encoding::EucJp,
196 Encoding::Iso2022Jp,
197 Encoding::Gbk,
198 Encoding::Gb18030,
199 Encoding::Big5,
200 Encoding::EucKr,
201 ];
202
203 /// Resolve an IANA/WHATWG charset label to an [`Encoding`], case-insensitively and after
204 /// trimming ASCII whitespace, mirroring the label tables in the
205 /// [WHATWG Encoding Standard](https://encoding.spec.whatwg.org/#names-and-labels) — with the
206 /// [`Encoding::Iso8859_9`]-versus-[`Encoding::Windows1254`] exception documented on
207 /// [`Encoding`] itself.
208 ///
209 /// Returns [`None`] for labels this crate does not resolve to a supported encoding (for
210 /// example `"utf-32"`, `"windows-874"` or `"ebcdic-cp-us"`).
211 ///
212 /// # Examples
213 ///
214 /// ```
215 /// use oxixml_encoding::Encoding;
216 ///
217 /// assert_eq!(Encoding::for_label("UTF-8"), Some(Encoding::Utf8));
218 /// assert_eq!(Encoding::for_label(" utf8 \t"), Some(Encoding::Utf8));
219 /// assert_eq!(Encoding::for_label("Windows-1252"), Some(Encoding::Windows1252));
220 /// assert_eq!(Encoding::for_label("iso-8859-1"), Some(Encoding::Latin1));
221 /// assert_eq!(Encoding::for_label("Shift_JIS"), Some(Encoding::ShiftJis));
222 /// assert_eq!(Encoding::for_label("EUC-JP"), Some(Encoding::EucJp));
223 /// assert_eq!(Encoding::for_label("gb18030"), Some(Encoding::Gb18030));
224 /// assert_eq!(Encoding::for_label("windows-874"), None);
225 /// ```
226 #[must_use]
227 pub fn for_label(label: &str) -> Option<Encoding> {
228 let trimmed = label.trim_matches(is_whatwg_ascii_whitespace);
229 if trimmed.is_empty() {
230 return None;
231 }
232 // WHATWG label matching is defined over the ASCII-lowercased label; avoid allocating in
233 // the common case where the caller already passed a canonical lowercase label.
234 if trimmed.bytes().all(|b| !b.is_ascii_uppercase()) {
235 return encoding_for_lowercase_label(trimmed);
236 }
237 let lowered = trimmed.to_ascii_lowercase();
238 encoding_for_lowercase_label(&lowered)
239 }
240
241 /// The canonical name of this encoding, as it should appear in generated `encoding="..."`
242 /// declarations or diagnostics.
243 ///
244 /// # Examples
245 ///
246 /// ```
247 /// use oxixml_encoding::Encoding;
248 ///
249 /// assert_eq!(Encoding::Utf8.name(), "UTF-8");
250 /// assert_eq!(Encoding::Windows1252.name(), "windows-1252");
251 /// assert_eq!(Encoding::ShiftJis.name(), "Shift_JIS");
252 /// ```
253 #[must_use]
254 pub const fn name(&self) -> &'static str {
255 match self {
256 Encoding::Utf8 => "UTF-8",
257 Encoding::Utf16Le => "UTF-16LE",
258 Encoding::Utf16Be => "UTF-16BE",
259 Encoding::Latin1 => "ISO-8859-1",
260 Encoding::Windows1252 => "windows-1252",
261 Encoding::Ascii => "US-ASCII",
262 Encoding::Iso8859_2 => "ISO-8859-2",
263 Encoding::Iso8859_3 => "ISO-8859-3",
264 Encoding::Iso8859_4 => "ISO-8859-4",
265 Encoding::Iso8859_5 => "ISO-8859-5",
266 Encoding::Iso8859_6 => "ISO-8859-6",
267 Encoding::Iso8859_7 => "ISO-8859-7",
268 Encoding::Iso8859_8 => "ISO-8859-8",
269 Encoding::Iso8859_9 => "ISO-8859-9",
270 Encoding::Iso8859_10 => "ISO-8859-10",
271 Encoding::Iso8859_13 => "ISO-8859-13",
272 Encoding::Iso8859_14 => "ISO-8859-14",
273 Encoding::Iso8859_15 => "ISO-8859-15",
274 Encoding::Iso8859_16 => "ISO-8859-16",
275 Encoding::Windows1250 => "windows-1250",
276 Encoding::Windows1251 => "windows-1251",
277 Encoding::Windows1253 => "windows-1253",
278 Encoding::Windows1254 => "windows-1254",
279 Encoding::Windows1255 => "windows-1255",
280 Encoding::Windows1256 => "windows-1256",
281 Encoding::Windows1257 => "windows-1257",
282 Encoding::Windows1258 => "windows-1258",
283 Encoding::Koi8R => "KOI8-R",
284 Encoding::Koi8U => "KOI8-U",
285 Encoding::Ibm866 => "IBM866",
286 Encoding::Macintosh => "macintosh",
287 Encoding::XMacCyrillic => "x-mac-cyrillic",
288 Encoding::ShiftJis => "Shift_JIS",
289 Encoding::EucJp => "EUC-JP",
290 Encoding::Iso2022Jp => "ISO-2022-JP",
291 Encoding::Gbk => "GBK",
292 Encoding::Gb18030 => "gb18030",
293 Encoding::Big5 => "Big5",
294 Encoding::EucKr => "EUC-KR",
295 }
296 }
297
298 /// Whether this encoding is ASCII-compatible in the sense relevant to XML prologue
299 /// scanning: every byte in `0x00..=0x7F` decodes to the identically-valued scalar, so an
300 /// ASCII-only substring (such as an `<?xml ... ?>` declaration) can be located by scanning
301 /// raw bytes without first performing a full decode.
302 ///
303 /// This holds for every single-byte encoding this crate supports (all of them agree with
304 /// ASCII below `0x80`, by construction — see [`Encoding::ALL`]), not only the original four.
305 /// It does *not* hold for the seven multi-byte encodings (the three legacy Japanese ones and
306 /// the four legacy CJK ones): even though each of them also encodes `0x00..=0x7F` as ASCII at
307 /// the start of the document (which is enough to make [`crate::sniff_xml_encoding`]'s
308 /// prologue scan work, since an `<?xml ...?>` declaration is required to be the first thing
309 /// in the entity and is itself pure ASCII), a byte elsewhere in the document is not reliably
310 /// ASCII-interpretable without tracking decoder state (ISO-2022-JP's shift state, or --
311 /// for the other six -- whether the byte is a lead or trail byte of a multi-byte sequence),
312 /// so this method intentionally answers `false` for all seven.
313 #[must_use]
314 pub const fn is_ascii_compatible_single_byte(&self) -> bool {
315 matches!(
316 self,
317 Encoding::Utf8
318 | Encoding::Latin1
319 | Encoding::Windows1252
320 | Encoding::Ascii
321 | Encoding::Iso8859_2
322 | Encoding::Iso8859_3
323 | Encoding::Iso8859_4
324 | Encoding::Iso8859_5
325 | Encoding::Iso8859_6
326 | Encoding::Iso8859_7
327 | Encoding::Iso8859_8
328 | Encoding::Iso8859_9
329 | Encoding::Iso8859_10
330 | Encoding::Iso8859_13
331 | Encoding::Iso8859_14
332 | Encoding::Iso8859_15
333 | Encoding::Iso8859_16
334 | Encoding::Windows1250
335 | Encoding::Windows1251
336 | Encoding::Windows1253
337 | Encoding::Windows1254
338 | Encoding::Windows1255
339 | Encoding::Windows1256
340 | Encoding::Windows1257
341 | Encoding::Windows1258
342 | Encoding::Koi8R
343 | Encoding::Koi8U
344 | Encoding::Ibm866
345 | Encoding::Macintosh
346 | Encoding::XMacCyrillic
347 )
348 }
349
350 /// Whether this encoding is one of the 26 legacy single-byte encodings backed by a generated
351 /// 128-entry upper-half table in `crate::tables::legacy_single_byte` (every [`Encoding`]
352 /// variant except UTF-8/UTF-16LE/UTF-16BE, the original [`Encoding::Latin1`]/
353 /// [`Encoding::Windows1252`]/[`Encoding::Ascii`] trio, and the seven multi-byte encodings —
354 /// the three legacy Japanese ones and the four legacy CJK ones).
355 #[must_use]
356 pub(crate) const fn is_legacy_single_byte_table(&self) -> bool {
357 matches!(
358 self,
359 Encoding::Iso8859_2
360 | Encoding::Iso8859_3
361 | Encoding::Iso8859_4
362 | Encoding::Iso8859_5
363 | Encoding::Iso8859_6
364 | Encoding::Iso8859_7
365 | Encoding::Iso8859_8
366 | Encoding::Iso8859_9
367 | Encoding::Iso8859_10
368 | Encoding::Iso8859_13
369 | Encoding::Iso8859_14
370 | Encoding::Iso8859_15
371 | Encoding::Iso8859_16
372 | Encoding::Windows1250
373 | Encoding::Windows1251
374 | Encoding::Windows1253
375 | Encoding::Windows1254
376 | Encoding::Windows1255
377 | Encoding::Windows1256
378 | Encoding::Windows1257
379 | Encoding::Windows1258
380 | Encoding::Koi8R
381 | Encoding::Koi8U
382 | Encoding::Ibm866
383 | Encoding::Macintosh
384 | Encoding::XMacCyrillic
385 )
386 }
387
388 /// Create a new incremental [`Decoder`] for this encoding.
389 ///
390 /// # Examples
391 ///
392 /// ```
393 /// use oxixml_encoding::Encoding;
394 ///
395 /// let mut decoder = Encoding::Utf8.new_decoder();
396 /// let mut out = String::new();
397 /// decoder.decode_to_string(b"hello", &mut out, true).expect("valid utf-8");
398 /// assert_eq!(out, "hello");
399 /// ```
400 #[must_use]
401 pub const fn new_decoder(self) -> Decoder {
402 Decoder::new(self)
403 }
404}
405
406impl std::fmt::Display for Encoding {
407 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
408 f.write_str(self.name())
409 }
410}
411
412/// WHATWG's definition of "ASCII whitespace" used when trimming labels: tab, line feed, form
413/// feed, carriage return and space.
414const fn is_whatwg_ascii_whitespace(c: char) -> bool {
415 matches!(c, '\t' | '\n' | '\x0C' | '\r' | ' ')
416}
417
418/// Resolve an already-trimmed, already-lowercased label. Kept as a `match` over `&str` (rather
419/// than a hash map) since the label set is small, fixed at compile time, and this keeps the
420/// crate free of any external dependency.
421///
422/// Label lists are transcribed from the WHATWG Encoding Standard
423/// ([`§4.2 "get an encoding"`](https://encoding.spec.whatwg.org/#names-and-labels), data dated
424/// 2024-09-18), with one deliberate deviation: the `"iso-8859-9"`/`"latin5"`-family labels
425/// resolve to [`Encoding::Iso8859_9`] here rather than [`Encoding::Windows1254`] (see the
426/// type-level docs on [`Encoding`]).
427fn encoding_for_lowercase_label(label: &str) -> Option<Encoding> {
428 match label {
429 // --- UTF-8 --------------------------------------------------------------------------
430 "unicode-1-1-utf-8" | "unicode11utf8" | "unicode20utf8" | "utf-8" | "utf8"
431 | "x-unicode20utf8" => Some(Encoding::Utf8),
432
433 // --- UTF-16, unspecified/little-endian ----------------------------------------------
434 // Per the WHATWG Encoding Standard, a bare "utf-16" label (no endianness suffix) is
435 // resolved as UTF-16LE for compatibility with the (much more common) little-endian
436 // content that historically omitted the suffix.
437 "utf-16" | "utf-16le" | "utf16" | "utf16le" | "ucs-2" | "unicode" | "iso-10646-ucs-2"
438 | "csunicode" => Some(Encoding::Utf16Le),
439
440 // --- UTF-16, big-endian ----------------------------------------------------------------
441 "utf-16be" | "utf16be" | "unicodefffe" => Some(Encoding::Utf16Be),
442
443 // --- ISO-8859-1 / Latin-1 --------------------------------------------------------------
444 "iso-8859-1" | "iso-ir-100" | "iso8859-1" | "iso88591" | "iso_8859-1"
445 | "iso_8859-1:1987" | "l1" | "latin1" | "csisolatin1" | "8859-1" | "88591" => {
446 Some(Encoding::Latin1)
447 }
448
449 // --- windows-1252 ------------------------------------------------------------------------
450 "windows-1252" | "cp1252" | "x-cp1252" => Some(Encoding::Windows1252),
451
452 // --- US-ASCII --------------------------------------------------------------------------
453 "us-ascii" | "ascii" | "ansi_x3.4-1968" | "ansi_x3.4-1986" | "iso646-us" | "iso-ir-6"
454 | "us" | "ibm367" | "cp367" | "csascii" => Some(Encoding::Ascii),
455
456 // --- ISO-8859-2 --------------------------------------------------------------------------
457 "csisolatin2" | "iso-8859-2" | "iso-ir-101" | "iso8859-2" | "iso88592" | "iso_8859-2"
458 | "iso_8859-2:1987" | "l2" | "latin2" => Some(Encoding::Iso8859_2),
459
460 // --- ISO-8859-3 --------------------------------------------------------------------------
461 "csisolatin3" | "iso-8859-3" | "iso-ir-109" | "iso8859-3" | "iso88593" | "iso_8859-3"
462 | "iso_8859-3:1988" | "l3" | "latin3" => Some(Encoding::Iso8859_3),
463
464 // --- ISO-8859-4 --------------------------------------------------------------------------
465 "csisolatin4" | "iso-8859-4" | "iso-ir-110" | "iso8859-4" | "iso88594" | "iso_8859-4"
466 | "iso_8859-4:1988" | "l4" | "latin4" => Some(Encoding::Iso8859_4),
467
468 // --- ISO-8859-5 --------------------------------------------------------------------------
469 "csisolatincyrillic" | "cyrillic" | "iso-8859-5" | "iso-ir-144" | "iso8859-5"
470 | "iso88595" | "iso_8859-5" | "iso_8859-5:1988" => Some(Encoding::Iso8859_5),
471
472 // --- ISO-8859-6 --------------------------------------------------------------------------
473 "arabic" | "asmo-708" | "csiso88596e" | "csiso88596i" | "csisolatinarabic" | "ecma-114"
474 | "iso-8859-6" | "iso-8859-6-e" | "iso-8859-6-i" | "iso-ir-127" | "iso8859-6"
475 | "iso88596" | "iso_8859-6" | "iso_8859-6:1987" => Some(Encoding::Iso8859_6),
476
477 // --- ISO-8859-7 --------------------------------------------------------------------------
478 "csisolatingreek" | "ecma-118" | "elot_928" | "greek" | "greek8" | "iso-8859-7"
479 | "iso-ir-126" | "iso8859-7" | "iso88597" | "iso_8859-7" | "iso_8859-7:1987"
480 | "sun_eu_greek" => Some(Encoding::Iso8859_7),
481
482 // --- ISO-8859-8 (and ISO-8859-8-I: same repertoire, direction only, out of scope) -------
483 "csiso88598e" | "csisolatinhebrew" | "hebrew" | "iso-8859-8" | "iso-8859-8-e"
484 | "iso-ir-138" | "iso8859-8" | "iso88598" | "iso_8859-8" | "iso_8859-8:1988" | "visual"
485 | "csiso88598i" | "iso-8859-8-i" | "logical" => Some(Encoding::Iso8859_8),
486
487 // --- ISO-8859-9 (kept distinct from windows-1254; see `Encoding` docs) -----------------
488 "csisolatin5" | "iso-8859-9" | "iso-ir-148" | "iso8859-9" | "iso88599" | "iso_8859-9"
489 | "iso_8859-9:1989" | "l5" | "latin5" => Some(Encoding::Iso8859_9),
490
491 // --- ISO-8859-10 -------------------------------------------------------------------------
492 "csisolatin6" | "iso-8859-10" | "iso-ir-157" | "iso8859-10" | "iso885910" | "l6"
493 | "latin6" => Some(Encoding::Iso8859_10),
494
495 // --- ISO-8859-13 -------------------------------------------------------------------------
496 "iso-8859-13" | "iso8859-13" | "iso885913" => Some(Encoding::Iso8859_13),
497
498 // --- ISO-8859-14 -------------------------------------------------------------------------
499 "iso-8859-14" | "iso8859-14" | "iso885914" => Some(Encoding::Iso8859_14),
500
501 // --- ISO-8859-15 -------------------------------------------------------------------------
502 "csisolatin9" | "iso-8859-15" | "iso8859-15" | "iso885915" | "iso_8859-15" | "l9" => {
503 Some(Encoding::Iso8859_15)
504 }
505
506 // --- ISO-8859-16 -------------------------------------------------------------------------
507 "iso-8859-16" => Some(Encoding::Iso8859_16),
508
509 // --- windows-1250 ------------------------------------------------------------------------
510 "cp1250" | "windows-1250" | "x-cp1250" => Some(Encoding::Windows1250),
511
512 // --- windows-1251 ------------------------------------------------------------------------
513 "cp1251" | "windows-1251" | "x-cp1251" => Some(Encoding::Windows1251),
514
515 // --- windows-1253 ------------------------------------------------------------------------
516 "cp1253" | "windows-1253" | "x-cp1253" => Some(Encoding::Windows1253),
517
518 // --- windows-1254 ------------------------------------------------------------------------
519 "cp1254" | "windows-1254" | "x-cp1254" => Some(Encoding::Windows1254),
520
521 // --- windows-1255 ------------------------------------------------------------------------
522 "cp1255" | "windows-1255" | "x-cp1255" => Some(Encoding::Windows1255),
523
524 // --- windows-1256 ------------------------------------------------------------------------
525 "cp1256" | "windows-1256" | "x-cp1256" => Some(Encoding::Windows1256),
526
527 // --- windows-1257 ------------------------------------------------------------------------
528 "cp1257" | "windows-1257" | "x-cp1257" => Some(Encoding::Windows1257),
529
530 // --- windows-1258 ------------------------------------------------------------------------
531 "cp1258" | "windows-1258" | "x-cp1258" => Some(Encoding::Windows1258),
532
533 // --- KOI8-R ------------------------------------------------------------------------------
534 "cskoi8r" | "koi" | "koi8" | "koi8-r" | "koi8_r" => Some(Encoding::Koi8R),
535
536 // --- KOI8-U ------------------------------------------------------------------------------
537 "koi8-ru" | "koi8-u" => Some(Encoding::Koi8U),
538
539 // --- IBM866 ------------------------------------------------------------------------------
540 "866" | "cp866" | "csibm866" | "ibm866" => Some(Encoding::Ibm866),
541
542 // --- macintosh ---------------------------------------------------------------------------
543 "csmacintosh" | "mac" | "macintosh" | "x-mac-roman" => Some(Encoding::Macintosh),
544
545 // --- x-mac-cyrillic ----------------------------------------------------------------------
546 "x-mac-cyrillic" | "x-mac-ukrainian" => Some(Encoding::XMacCyrillic),
547
548 // --- EUC-JP ------------------------------------------------------------------------------
549 "cseucpkdfmtjapanese" | "euc-jp" | "x-euc-jp" => Some(Encoding::EucJp),
550
551 // --- ISO-2022-JP -------------------------------------------------------------------------
552 "csiso2022jp" | "iso-2022-jp" => Some(Encoding::Iso2022Jp),
553
554 // --- Shift_JIS ---------------------------------------------------------------------------
555 "csshiftjis" | "ms932" | "ms_kanji" | "shift-jis" | "shift_jis" | "sjis"
556 | "windows-31j" | "x-sjis" => Some(Encoding::ShiftJis),
557
558 // --- GBK ---------------------------------------------------------------------------------
559 "chinese" | "csgb2312" | "csiso58gb231280" | "gb2312" | "gb_2312" | "gb_2312-80"
560 | "gbk" | "iso-ir-58" | "x-gbk" => Some(Encoding::Gbk),
561
562 // --- gb18030 -----------------------------------------------------------------------------
563 "gb18030" => Some(Encoding::Gb18030),
564
565 // --- Big5 --------------------------------------------------------------------------------
566 "big5" | "big5-hkscs" | "cn-big5" | "csbig5" | "x-x-big5" => Some(Encoding::Big5),
567
568 // --- EUC-KR ------------------------------------------------------------------------------
569 "cseuckr" | "csksc56011987" | "euc-kr" | "iso-ir-149" | "korean" | "ks_c_5601-1987"
570 | "ks_c_5601-1989" | "ksc5601" | "ksc_5601" | "windows-949" => Some(Encoding::EucKr),
571
572 _ => None,
573 }
574}
575
576#[cfg(test)]
577mod tests {
578 use super::*;
579
580 #[test]
581 fn for_label_is_case_insensitive() {
582 assert_eq!(Encoding::for_label("UTF-8"), Some(Encoding::Utf8));
583 assert_eq!(Encoding::for_label("Utf-8"), Some(Encoding::Utf8));
584 assert_eq!(Encoding::for_label("uTf8"), Some(Encoding::Utf8));
585 assert_eq!(
586 Encoding::for_label("WINDOWS-1252"),
587 Some(Encoding::Windows1252)
588 );
589 assert_eq!(Encoding::for_label("Iso-8859-1"), Some(Encoding::Latin1));
590 assert_eq!(Encoding::for_label("US-ASCII"), Some(Encoding::Ascii));
591 assert_eq!(Encoding::for_label("Shift_JIS"), Some(Encoding::ShiftJis));
592 assert_eq!(Encoding::for_label("SHIFT_JIS"), Some(Encoding::ShiftJis));
593 }
594
595 #[test]
596 fn for_label_trims_whatwg_whitespace() {
597 assert_eq!(Encoding::for_label(" utf-8\t\n"), Some(Encoding::Utf8));
598 assert_eq!(Encoding::for_label("\r\nlatin1 "), Some(Encoding::Latin1));
599 assert_eq!(
600 Encoding::for_label(" shift_jis \n"),
601 Some(Encoding::ShiftJis)
602 );
603 }
604
605 #[test]
606 fn for_label_unknown_returns_none() {
607 assert_eq!(Encoding::for_label("utf-32"), None);
608 assert_eq!(Encoding::for_label(""), None);
609 assert_eq!(Encoding::for_label(" "), None);
610 // Deliberately unsupported: Thai (windows-874/iso-8859-11) is outside this crate's
611 // current scope.
612 assert_eq!(Encoding::for_label("windows-874"), None);
613 assert_eq!(Encoding::for_label("iso-8859-11"), None);
614 }
615
616 #[test]
617 fn for_label_cjk_multi_byte_labels() {
618 assert_eq!(Encoding::for_label("gbk"), Some(Encoding::Gbk));
619 assert_eq!(Encoding::for_label("GBK"), Some(Encoding::Gbk));
620 assert_eq!(Encoding::for_label("gb2312"), Some(Encoding::Gbk));
621 assert_eq!(Encoding::for_label("x-gbk"), Some(Encoding::Gbk));
622 assert_eq!(Encoding::for_label("chinese"), Some(Encoding::Gbk));
623 assert_eq!(Encoding::for_label("gb18030"), Some(Encoding::Gb18030));
624 assert_eq!(Encoding::for_label("GB18030"), Some(Encoding::Gb18030));
625 assert_eq!(Encoding::for_label("big5"), Some(Encoding::Big5));
626 assert_eq!(Encoding::for_label("Big5"), Some(Encoding::Big5));
627 assert_eq!(Encoding::for_label("big5-hkscs"), Some(Encoding::Big5));
628 assert_eq!(Encoding::for_label("cn-big5"), Some(Encoding::Big5));
629 assert_eq!(Encoding::for_label("euc-kr"), Some(Encoding::EucKr));
630 assert_eq!(Encoding::for_label("EUC-KR"), Some(Encoding::EucKr));
631 assert_eq!(Encoding::for_label("windows-949"), Some(Encoding::EucKr));
632 assert_eq!(Encoding::for_label("ks_c_5601-1987"), Some(Encoding::EucKr));
633 assert_eq!(Encoding::for_label("korean"), Some(Encoding::EucKr));
634 }
635
636 #[test]
637 fn for_label_utf16_variants() {
638 assert_eq!(Encoding::for_label("utf-16"), Some(Encoding::Utf16Le));
639 assert_eq!(Encoding::for_label("utf-16le"), Some(Encoding::Utf16Le));
640 assert_eq!(Encoding::for_label("utf-16be"), Some(Encoding::Utf16Be));
641 assert_eq!(Encoding::for_label("unicodeFFFE"), Some(Encoding::Utf16Be));
642 }
643
644 #[test]
645 fn for_label_iso_8859_9_is_kept_distinct_from_windows_1254() {
646 assert_eq!(Encoding::for_label("iso-8859-9"), Some(Encoding::Iso8859_9));
647 assert_eq!(Encoding::for_label("latin5"), Some(Encoding::Iso8859_9));
648 assert_eq!(
649 Encoding::for_label("windows-1254"),
650 Some(Encoding::Windows1254)
651 );
652 assert_eq!(Encoding::for_label("cp1254"), Some(Encoding::Windows1254));
653 assert_ne!(Encoding::Iso8859_9, Encoding::Windows1254);
654 }
655
656 #[test]
657 fn for_label_japanese_multi_byte_labels() {
658 assert_eq!(Encoding::for_label("shift_jis"), Some(Encoding::ShiftJis));
659 assert_eq!(Encoding::for_label("sjis"), Some(Encoding::ShiftJis));
660 assert_eq!(Encoding::for_label("ms932"), Some(Encoding::ShiftJis));
661 assert_eq!(Encoding::for_label("windows-31j"), Some(Encoding::ShiftJis));
662 assert_eq!(Encoding::for_label("euc-jp"), Some(Encoding::EucJp));
663 assert_eq!(Encoding::for_label("x-euc-jp"), Some(Encoding::EucJp));
664 assert_eq!(
665 Encoding::for_label("iso-2022-jp"),
666 Some(Encoding::Iso2022Jp)
667 );
668 assert_eq!(
669 Encoding::for_label("csiso2022jp"),
670 Some(Encoding::Iso2022Jp)
671 );
672 }
673
674 #[test]
675 fn name_round_trips_through_for_label_for_every_supported_encoding() {
676 for &enc in Encoding::ALL {
677 assert_eq!(
678 Encoding::for_label(enc.name()),
679 Some(enc),
680 "name = {}",
681 enc.name()
682 );
683 // Case-insensitivity holds for every encoding's canonical name too.
684 assert_eq!(
685 Encoding::for_label(&enc.name().to_lowercase()),
686 Some(enc),
687 "lowercased name = {}",
688 enc.name()
689 );
690 assert_eq!(
691 Encoding::for_label(&enc.name().to_uppercase()),
692 Some(enc),
693 "uppercased name = {}",
694 enc.name()
695 );
696 }
697 }
698
699 #[test]
700 fn all_is_free_of_duplicates() {
701 for (i, &a) in Encoding::ALL.iter().enumerate() {
702 for &b in &Encoding::ALL[i + 1..] {
703 assert_ne!(a, b, "Encoding::ALL contains a duplicate: {a}");
704 }
705 }
706 }
707
708 #[test]
709 fn display_matches_name() {
710 assert_eq!(Encoding::Utf8.to_string(), "UTF-8");
711 assert_eq!(Encoding::Windows1252.to_string(), "windows-1252");
712 assert_eq!(Encoding::ShiftJis.to_string(), "Shift_JIS");
713 }
714
715 #[test]
716 fn is_ascii_compatible_single_byte_matches_expectation() {
717 assert!(Encoding::Utf8.is_ascii_compatible_single_byte());
718 assert!(Encoding::Latin1.is_ascii_compatible_single_byte());
719 assert!(Encoding::Windows1252.is_ascii_compatible_single_byte());
720 assert!(Encoding::Ascii.is_ascii_compatible_single_byte());
721 assert!(Encoding::Iso8859_2.is_ascii_compatible_single_byte());
722 assert!(Encoding::Windows1251.is_ascii_compatible_single_byte());
723 assert!(Encoding::Koi8R.is_ascii_compatible_single_byte());
724 assert!(!Encoding::Utf16Le.is_ascii_compatible_single_byte());
725 assert!(!Encoding::Utf16Be.is_ascii_compatible_single_byte());
726 assert!(!Encoding::ShiftJis.is_ascii_compatible_single_byte());
727 assert!(!Encoding::EucJp.is_ascii_compatible_single_byte());
728 assert!(!Encoding::Iso2022Jp.is_ascii_compatible_single_byte());
729 assert!(!Encoding::Gbk.is_ascii_compatible_single_byte());
730 assert!(!Encoding::Gb18030.is_ascii_compatible_single_byte());
731 assert!(!Encoding::Big5.is_ascii_compatible_single_byte());
732 assert!(!Encoding::EucKr.is_ascii_compatible_single_byte());
733 }
734
735 #[test]
736 fn is_legacy_single_byte_table_matches_expectation() {
737 assert!(Encoding::Iso8859_2.is_legacy_single_byte_table());
738 assert!(Encoding::Windows1250.is_legacy_single_byte_table());
739 assert!(Encoding::Koi8R.is_legacy_single_byte_table());
740 assert!(Encoding::Macintosh.is_legacy_single_byte_table());
741 assert!(!Encoding::Latin1.is_legacy_single_byte_table());
742 assert!(!Encoding::Windows1252.is_legacy_single_byte_table());
743 assert!(!Encoding::Ascii.is_legacy_single_byte_table());
744 assert!(!Encoding::Utf8.is_legacy_single_byte_table());
745 assert!(!Encoding::ShiftJis.is_legacy_single_byte_table());
746 assert!(!Encoding::EucJp.is_legacy_single_byte_table());
747 assert!(!Encoding::Iso2022Jp.is_legacy_single_byte_table());
748 assert!(!Encoding::Gbk.is_legacy_single_byte_table());
749 assert!(!Encoding::Gb18030.is_legacy_single_byte_table());
750 assert!(!Encoding::Big5.is_legacy_single_byte_table());
751 assert!(!Encoding::EucKr.is_legacy_single_byte_table());
752 }
753
754 #[test]
755 fn all_contains_every_variant_exactly_once_via_new_decoder_smoke_test() {
756 // Not exhaustive by construction (nothing enforces that `ALL` is kept in sync with the
757 // enum at compile time), but constructing a decoder for every entry at least catches an
758 // entry that was added to `ALL` without the corresponding `Decoder::new` support (which
759 // would panic here rather than silently mis-decode).
760 for &encoding in Encoding::ALL {
761 let _ = encoding.new_decoder();
762 }
763 }
764}