rusty_xml_parser/
encoding.rs

1//! Character encodings matching libxml2 `encoding.h` without iconv.
2//! 8-bit tables are byte-identical to v2.15.3 `codegen/charset.inc`.
3
4use crate::encoding_tables::*;
5use crate::error::{XmlError, XML_ERR_UNSUPPORTED_ENCODING};
6
7/// libxml2 `xmlCharEncoding` discriminants.
8#[derive(Clone, Copy, Debug, PartialEq, Eq)]
9#[repr(i32)]
10pub enum XmlCharEncoding {
11    Error = -1,
12    None = 0,
13    Utf8 = 1,
14    Utf16Le = 2,
15    Utf16Be = 3,
16    Ucs4Le = 4,
17    Ucs4Be = 5,
18    Ebcdic = 6,
19    Ucs4_2143 = 7,
20    Ucs4_3412 = 8,
21    Ucs2 = 9,
22    Iso8859_1 = 10,
23    Iso8859_2 = 11,
24    Iso8859_3 = 12,
25    Iso8859_4 = 13,
26    Iso8859_5 = 14,
27    Iso8859_6 = 15,
28    Iso8859_7 = 16,
29    Iso8859_8 = 17,
30    Iso8859_9 = 18,
31    Iso2022Jp = 19,
32    ShiftJis = 20,
33    EucJp = 21,
34    Ascii = 22,
35    Utf16 = 23,
36    Html = 24,
37    Iso8859_10 = 25,
38    Iso8859_11 = 26,
39    Iso8859_13 = 27,
40    Iso8859_14 = 28,
41    Iso8859_15 = 29,
42    Iso8859_16 = 30,
43    Windows1252 = 31,
44    Windows1250 = 32,
45    Windows1251 = 33,
46    Windows1253 = 34,
47    Windows1254 = 35,
48    Windows1255 = 36,
49    Windows1256 = 37,
50    Windows1257 = 38,
51    Windows1258 = 39,
52    Koi8R = 40,
53    Koi8U = 41,
54    Ibm866 = 42,
55    Macintosh = 43,
56}
57
58const NAME_MAP: &[(&str, XmlCharEncoding)] = &[
59    ("ascii", XmlCharEncoding::Ascii),
60    ("windows-1250", XmlCharEncoding::Windows1250),
61    ("cp1250", XmlCharEncoding::Windows1250),
62    ("windows1250", XmlCharEncoding::Windows1250),
63    ("windows-1251", XmlCharEncoding::Windows1251),
64    ("cp1251", XmlCharEncoding::Windows1251),
65    ("windows1251", XmlCharEncoding::Windows1251),
66    ("windows-1253", XmlCharEncoding::Windows1253),
67    ("cp1253", XmlCharEncoding::Windows1253),
68    ("windows1253", XmlCharEncoding::Windows1253),
69    ("windows-1254", XmlCharEncoding::Windows1254),
70    ("cp1254", XmlCharEncoding::Windows1254),
71    ("windows1254", XmlCharEncoding::Windows1254),
72    ("windows-1255", XmlCharEncoding::Windows1255),
73    ("cp1255", XmlCharEncoding::Windows1255),
74    ("windows1255", XmlCharEncoding::Windows1255),
75    ("windows-1256", XmlCharEncoding::Windows1256),
76    ("cp1256", XmlCharEncoding::Windows1256),
77    ("windows1256", XmlCharEncoding::Windows1256),
78    ("windows-1257", XmlCharEncoding::Windows1257),
79    ("cp1257", XmlCharEncoding::Windows1257),
80    ("windows1257", XmlCharEncoding::Windows1257),
81    ("windows-1258", XmlCharEncoding::Windows1258),
82    ("cp1258", XmlCharEncoding::Windows1258),
83    ("windows1258", XmlCharEncoding::Windows1258),
84    ("koi8-r", XmlCharEncoding::Koi8R),
85    ("koi8r", XmlCharEncoding::Koi8R),
86    ("cskoi8r", XmlCharEncoding::Koi8R),
87    ("koi8-u", XmlCharEncoding::Koi8U),
88    ("koi8u", XmlCharEncoding::Koi8U),
89    ("ibm866", XmlCharEncoding::Ibm866),
90    ("cp866", XmlCharEncoding::Ibm866),
91    ("866", XmlCharEncoding::Ibm866),
92    ("macintosh", XmlCharEncoding::Macintosh),
93    ("mac", XmlCharEncoding::Macintosh),
94    ("csmacintosh", XmlCharEncoding::Macintosh),
95
96    ("csisolatin1", XmlCharEncoding::Iso8859_1),
97    ("iso-8859-1", XmlCharEncoding::Iso8859_1),
98    ("iso-8859-2", XmlCharEncoding::Iso8859_2),
99    ("iso-8859-3", XmlCharEncoding::Iso8859_3),
100    ("iso-8859-4", XmlCharEncoding::Iso8859_4),
101    ("iso-8859-5", XmlCharEncoding::Iso8859_5),
102    ("iso-8859-6", XmlCharEncoding::Iso8859_6),
103    ("iso-8859-7", XmlCharEncoding::Iso8859_7),
104    ("iso-8859-8", XmlCharEncoding::Iso8859_8),
105    ("iso-8859-9", XmlCharEncoding::Iso8859_9),
106    ("iso-8859-10", XmlCharEncoding::Iso8859_10),
107    ("iso-8859-11", XmlCharEncoding::Iso8859_11),
108    ("iso-8859-13", XmlCharEncoding::Iso8859_13),
109    ("iso-8859-14", XmlCharEncoding::Iso8859_14),
110    ("iso-8859-15", XmlCharEncoding::Iso8859_15),
111    ("iso-8859-16", XmlCharEncoding::Iso8859_16),
112    ("iso8859-1", XmlCharEncoding::Iso8859_1),
113    ("iso_8859-1", XmlCharEncoding::Iso8859_1),
114    ("latin1", XmlCharEncoding::Iso8859_1),
115    ("us-ascii", XmlCharEncoding::Ascii),
116    ("utf-16", XmlCharEncoding::Utf16),
117    ("utf-16be", XmlCharEncoding::Utf16Be),
118    ("utf-16le", XmlCharEncoding::Utf16Le),
119    ("utf-8", XmlCharEncoding::Utf8),
120    ("utf16", XmlCharEncoding::Utf16),
121    ("utf8", XmlCharEncoding::Utf8),
122    ("unicode", XmlCharEncoding::Utf16),
123    ("ucs-2", XmlCharEncoding::Ucs2),
124    ("ucs-4", XmlCharEncoding::Ucs4Le),
125    ("ucs2", XmlCharEncoding::Ucs2),
126    ("ucs4", XmlCharEncoding::Ucs4Le),
127    ("windows-1252", XmlCharEncoding::Windows1252),
128    ("x-cp1252", XmlCharEncoding::Windows1252),
129    ("ibm037", XmlCharEncoding::Ebcdic),
130    ("ebcdic", XmlCharEncoding::Ebcdic),
131    ("iso-2022-jp", XmlCharEncoding::Iso2022Jp),
132    ("shift_jis", XmlCharEncoding::ShiftJis),
133    ("shift-jis", XmlCharEncoding::ShiftJis),
134    ("sjis", XmlCharEncoding::ShiftJis),
135    ("euc-jp", XmlCharEncoding::EucJp),
136];
137
138/// `xmlParseCharEncoding`.
139#[doc(alias = "xmlParseCharEncoding")]
140pub fn xml_parse_char_encoding(name: &str) -> XmlCharEncoding {
141    let lower = name.trim().to_ascii_lowercase();
142    for (n, enc) in NAME_MAP {
143        if *n == lower {
144            return if *enc == XmlCharEncoding::Utf16 {
145                XmlCharEncoding::Utf16Le
146            } else {
147                *enc
148            };
149        }
150    }
151    XmlCharEncoding::Error
152}
153
154/// `xmlGetCharEncodingName`.
155#[doc(alias = "xmlGetCharEncodingName")]
156pub fn xml_get_char_encoding_name(enc: XmlCharEncoding) -> Option<&'static str> {
157    Some(match enc {
158        XmlCharEncoding::Utf8 => "UTF-8",
159        XmlCharEncoding::Utf16Le | XmlCharEncoding::Utf16Be | XmlCharEncoding::Utf16 => "UTF-16",
160        XmlCharEncoding::Ucs4Le | XmlCharEncoding::Ucs4Be => "UCS-4",
161        XmlCharEncoding::Iso8859_1 => "ISO-8859-1",
162        XmlCharEncoding::Windows1250 => "windows-1250",
163        XmlCharEncoding::Windows1251 => "windows-1251",
164        XmlCharEncoding::Windows1253 => "windows-1253",
165        XmlCharEncoding::Windows1254 => "windows-1254",
166        XmlCharEncoding::Windows1255 => "windows-1255",
167        XmlCharEncoding::Windows1256 => "windows-1256",
168        XmlCharEncoding::Windows1257 => "windows-1257",
169        XmlCharEncoding::Windows1258 => "windows-1258",
170        XmlCharEncoding::Koi8R => "KOI8-R",
171        XmlCharEncoding::Koi8U => "KOI8-U",
172        XmlCharEncoding::Ibm866 => "IBM866",
173        XmlCharEncoding::Macintosh => "macintosh",
174        XmlCharEncoding::Iso8859_2 => "ISO-8859-2",
175        XmlCharEncoding::Iso8859_3 => "ISO-8859-3",
176        XmlCharEncoding::Iso8859_4 => "ISO-8859-4",
177        XmlCharEncoding::Iso8859_5 => "ISO-8859-5",
178        XmlCharEncoding::Iso8859_6 => "ISO-8859-6",
179        XmlCharEncoding::Iso8859_7 => "ISO-8859-7",
180        XmlCharEncoding::Iso8859_8 => "ISO-8859-8",
181        XmlCharEncoding::Iso8859_9 => "ISO-8859-9",
182        XmlCharEncoding::Iso8859_10 => "ISO-8859-10",
183        XmlCharEncoding::Iso8859_11 => "ISO-8859-11",
184        XmlCharEncoding::Iso8859_13 => "ISO-8859-13",
185        XmlCharEncoding::Iso8859_14 => "ISO-8859-14",
186        XmlCharEncoding::Iso8859_15 => "ISO-8859-15",
187        XmlCharEncoding::Iso8859_16 => "ISO-8859-16",
188        XmlCharEncoding::Ascii => "US-ASCII",
189        XmlCharEncoding::Windows1252 => "windows-1252",
190        XmlCharEncoding::Ebcdic => "IBM037",
191        XmlCharEncoding::Ucs2 => "UCS-2",
192        XmlCharEncoding::Iso2022Jp => "ISO-2022-JP",
193        XmlCharEncoding::ShiftJis => "Shift_JIS",
194        XmlCharEncoding::EucJp => "EUC-JP",
195        XmlCharEncoding::Html => "HTML",
196        XmlCharEncoding::None | XmlCharEncoding::Error | XmlCharEncoding::Ucs4_2143 | XmlCharEncoding::Ucs4_3412 => {
197            return None
198        }
199    })
200}
201
202/// `xmlDetectCharEncoding` — XML 1.0 appendix F plus libxml2's UTF-16 extras.
203#[doc(alias = "xmlDetectCharEncoding")]
204pub fn xml_detect_char_encoding(input: &[u8]) -> XmlCharEncoding {
205    if input.len() >= 4 {
206        let b = &input[..4];
207        if b == [0x00, 0x00, 0x00, 0x3C] {
208            return XmlCharEncoding::Ucs4Be;
209        }
210        if b == [0x3C, 0x00, 0x00, 0x00] {
211            return XmlCharEncoding::Ucs4Le;
212        }
213        if b == [0x00, 0x00, 0x3C, 0x00] {
214            return XmlCharEncoding::Ucs4_2143;
215        }
216        if b == [0x00, 0x3C, 0x00, 0x00] {
217            return XmlCharEncoding::Ucs4_3412;
218        }
219        if b == [0x4C, 0x6F, 0xA7, 0x94] {
220            return XmlCharEncoding::Ebcdic;
221        }
222        if b == [0x3C, 0x3F, 0x78, 0x6D] {
223            return XmlCharEncoding::Utf8;
224        }
225        if b == [0x3C, 0x00, 0x3F, 0x00] {
226            return XmlCharEncoding::Utf16Le;
227        }
228        if b == [0x00, 0x3C, 0x00, 0x3F] {
229            return XmlCharEncoding::Utf16Be;
230        }
231        if b == [0x00, 0x00, 0xFE, 0xFF] {
232            return XmlCharEncoding::Ucs4Be;
233        }
234        if b == [0xFF, 0xFE, 0x00, 0x00] {
235            return XmlCharEncoding::Ucs4Le;
236        }
237    }
238    if input.len() >= 3 && input[..3] == [0xEF, 0xBB, 0xBF] {
239        return XmlCharEncoding::Utf8;
240    }
241    if input.len() >= 2 {
242        if input[0] == 0xFE && input[1] == 0xFF {
243            return XmlCharEncoding::Utf16Be;
244        }
245        if input[0] == 0xFF && input[1] == 0xFE {
246            return XmlCharEncoding::Utf16Le;
247        }
248    }
249    XmlCharEncoding::None
250}
251
252fn eightbit_to_utf8(input: &[u8], table: &[u16; 128]) -> Vec<u8> {
253    let mut out = Vec::with_capacity(input.len() * 2);
254    for &b in input {
255        let cp = if b < 0x80 {
256            b as u32
257        } else {
258            table[(b - 0x80) as usize] as u32
259        };
260        if let Some(c) = char::from_u32(cp) {
261            let mut buf = [0u8; 4];
262            out.extend_from_slice(c.encode_utf8(&mut buf).as_bytes());
263        }
264    }
265    out
266}
267
268fn latin1_to_utf8(input: &[u8]) -> Vec<u8> {
269    let mut out = Vec::with_capacity(input.len() * 2);
270    for &b in input {
271        let c = b as char;
272        let mut buf = [0u8; 4];
273        out.extend_from_slice(c.encode_utf8(&mut buf).as_bytes());
274    }
275    out
276}
277
278fn utf16_to_utf8(input: &[u8], be: bool) -> Result<Vec<u8>, XmlError> {
279    let mut i = 0;
280    if input.len() >= 2 {
281        let bom = if be {
282            input[0] == 0xFE && input[1] == 0xFF
283        } else {
284            input[0] == 0xFF && input[1] == 0xFE
285        };
286        if bom {
287            i = 2;
288        }
289    }
290    let mut units = Vec::new();
291    while i + 1 < input.len() {
292        let u = if be {
293            u16::from_be_bytes([input[i], input[i + 1]])
294        } else {
295            u16::from_le_bytes([input[i], input[i + 1]])
296        };
297        units.push(u);
298        i += 2;
299    }
300    let s = String::from_utf16(&units).map_err(|_| {
301        XmlError::new(XML_ERR_UNSUPPORTED_ENCODING, "Invalid UTF-16", 0, 0)
302    })?;
303    Ok(s.into_bytes())
304}
305
306fn ucs4_to_utf8(input: &[u8], order: [usize; 4]) -> Result<Vec<u8>, XmlError> {
307    let mut i = 0;
308    if input.len() >= 4 {
309        let w = u32::from_be_bytes([input[0], input[1], input[2], input[3]]);
310        if w == 0xFEFF || w == 0xFFFE0000 {
311            i = 4;
312        }
313    }
314    let mut out = String::new();
315    while i + 3 < input.len() {
316        let b = [input[i], input[i + 1], input[i + 2], input[i + 3]];
317        let cp = u32::from_be_bytes([b[order[0]], b[order[1]], b[order[2]], b[order[3]]]);
318        match char::from_u32(cp) {
319            Some(c) if c != '\u{feff}' || !out.is_empty() => out.push(c),
320            Some(_) => {}
321            None => {
322                return Err(XmlError::new(
323                    XML_ERR_UNSUPPORTED_ENCODING,
324                    "Invalid UCS-4",
325                    0,
326                    0,
327                ));
328            }
329        }
330        i += 4;
331    }
332    Ok(out.into_bytes())
333}
334
335/// IBM037 (EBCDIC) — enough to convert XML documents detected as `4C 6F A7 94`.
336static CP037: [u16; 256] = [
337    0x00, 0x01, 0x02, 0x03, 0x9c, 0x09, 0x86, 0x7f, 0x97, 0x8d, 0x8e, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
338    0x10, 0x11, 0x12, 0x13, 0x9d, 0x85, 0x08, 0x87, 0x18, 0x19, 0x92, 0x8f, 0x1c, 0x1d, 0x1e, 0x1f,
339    0x80, 0x81, 0x82, 0x83, 0x84, 0x0a, 0x17, 0x1b, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x05, 0x06, 0x07,
340    0x90, 0x91, 0x16, 0x93, 0x94, 0x95, 0x96, 0x04, 0x98, 0x99, 0x9a, 0x9b, 0x14, 0x15, 0x9e, 0x1a,
341    0x20, 0xa0, 0xe2, 0xe4, 0xe0, 0xe1, 0xe3, 0xe5, 0xe7, 0xf1, 0xa2, 0x2e, 0x3c, 0x28, 0x2b, 0x7c,
342    0x26, 0xe9, 0xea, 0xeb, 0xe8, 0xed, 0xee, 0xef, 0xec, 0xdf, 0x21, 0x24, 0x2a, 0x29, 0x3b, 0xac,
343    0x2d, 0x2f, 0xc2, 0xc4, 0xc0, 0xc1, 0xc3, 0xc5, 0xc7, 0xd1, 0xa6, 0x2c, 0x25, 0x5f, 0x3e, 0x3f,
344    0xf8, 0xc9, 0xca, 0xcb, 0xc8, 0xcd, 0xce, 0xcf, 0xcc, 0x60, 0x3a, 0x23, 0x40, 0x27, 0x3d, 0x22,
345    0xd8, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xab, 0xbb, 0xf0, 0xfd, 0xfe, 0xb1,
346    0xb0, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0xaa, 0xba, 0xe6, 0xb8, 0xc6, 0xa4,
347    0xb5, 0x7e, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0xa1, 0xbf, 0xd0, 0xdd, 0xde, 0xae,
348    0x5e, 0xa3, 0xa5, 0xb7, 0xa9, 0xa7, 0xb6, 0xbc, 0xbd, 0xbe, 0x5b, 0x5d, 0xaf, 0xa8, 0xb4, 0xd7,
349    0x7b, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0xad, 0xf4, 0xf6, 0xf2, 0xf3, 0xf5,
350    0x7d, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0xb9, 0xfb, 0xfc, 0xf9, 0xfa, 0xff,
351    0x5c, 0xf7, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0xb2, 0xd4, 0xd6, 0xd2, 0xd3, 0xd5,
352    0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0xb3, 0xdb, 0xdc, 0xd9, 0xda, 0x9f,
353];
354
355fn table_for(enc: XmlCharEncoding) -> Option<&'static [u16; 128]> {
356    Some(match enc {
357        XmlCharEncoding::Iso8859_2 => &XML_UNICODE_ISO8859_2,
358        XmlCharEncoding::Iso8859_3 => &XML_UNICODE_ISO8859_3,
359        XmlCharEncoding::Iso8859_4 => &XML_UNICODE_ISO8859_4,
360        XmlCharEncoding::Iso8859_5 => &XML_UNICODE_ISO8859_5,
361        XmlCharEncoding::Iso8859_6 => &XML_UNICODE_ISO8859_6,
362        XmlCharEncoding::Iso8859_7 => &XML_UNICODE_ISO8859_7,
363        XmlCharEncoding::Iso8859_8 => &XML_UNICODE_ISO8859_8,
364        XmlCharEncoding::Iso8859_9 => &XML_UNICODE_ISO8859_9,
365        XmlCharEncoding::Iso8859_10 => &XML_UNICODE_ISO8859_10,
366        XmlCharEncoding::Iso8859_11 => &XML_UNICODE_ISO8859_11,
367        XmlCharEncoding::Iso8859_13 => &XML_UNICODE_ISO8859_13,
368        XmlCharEncoding::Iso8859_14 => &XML_UNICODE_ISO8859_14,
369        XmlCharEncoding::Iso8859_15 => &XML_UNICODE_ISO8859_15,
370        XmlCharEncoding::Iso8859_16 => &XML_UNICODE_ISO8859_16,
371        XmlCharEncoding::Windows1252 => &XML_UNICODE_windows_1252,
372        XmlCharEncoding::Windows1250 => &crate::encoding_tables_more::XML_UNICODE_WINDOWS1250,
373        XmlCharEncoding::Windows1251 => &crate::encoding_tables_more::XML_UNICODE_WINDOWS1251,
374        XmlCharEncoding::Windows1253 => &crate::encoding_tables_more::XML_UNICODE_WINDOWS1253,
375        XmlCharEncoding::Windows1254 => &crate::encoding_tables_more::XML_UNICODE_WINDOWS1254,
376        XmlCharEncoding::Windows1255 => &crate::encoding_tables_more::XML_UNICODE_WINDOWS1255,
377        XmlCharEncoding::Windows1256 => &crate::encoding_tables_more::XML_UNICODE_WINDOWS1256,
378        XmlCharEncoding::Windows1257 => &crate::encoding_tables_more::XML_UNICODE_WINDOWS1257,
379        XmlCharEncoding::Windows1258 => &crate::encoding_tables_more::XML_UNICODE_WINDOWS1258,
380        XmlCharEncoding::Koi8R => &crate::encoding_tables_more::XML_UNICODE_KOI8R,
381        XmlCharEncoding::Koi8U => &crate::encoding_tables_more::XML_UNICODE_KOI8U,
382        XmlCharEncoding::Ibm866 => &crate::encoding_tables_more::XML_UNICODE_IBM866,
383        XmlCharEncoding::Macintosh => &crate::encoding_tables_more::XML_UNICODE_MACINTOSH,
384
385        _ => return None,
386    })
387}
388
389fn sniff_encoding_decl(bytes: &[u8]) -> Option<XmlCharEncoding> {
390    // The declaration is ASCII. Scanning the bytes directly avoids building a
391    // 1 KB String and a second lowercased copy of it on every single parse.
392    let n = bytes.len().min(1024);
393    let head = &bytes[..n];
394    if head.len() < 8 {
395        return None;
396    }
397    let idx = head
398        .windows(8)
399        .position(|w| w.eq_ignore_ascii_case(b"encoding"))?;
400    let mut i = idx + 8;
401    while i < n && matches!(head[i], b' ' | b'\t' | b'\r' | b'\n' | b'=') {
402        i += 1;
403    }
404    if i >= n || (head[i] != b'"' && head[i] != b'\'') {
405        return None;
406    }
407    let q = head[i];
408    i += 1;
409    let start = i;
410    while i < n && head[i] != q {
411        i += 1;
412    }
413    if i >= n {
414        return None;
415    }
416    let name = std::str::from_utf8(&head[start..i]).ok()?;
417    xml_parse_char_encoding(name).into_option()
418}
419
420impl XmlCharEncoding {
421    fn into_option(self) -> Option<XmlCharEncoding> {
422        match self {
423            XmlCharEncoding::Error | XmlCharEncoding::None => None,
424            e => Some(e),
425        }
426    }
427}
428
429/// Convert `input` to UTF-8. `hint` is the `encoding` argument to `xmlReadMemory`.
430pub fn xml_convert_to_utf8(
431    input: &[u8],
432    hint: Option<&str>,
433) -> Result<(Vec<u8>, Option<String>), XmlError> {
434    let (c, n) = xml_convert_to_utf8_cow(input, hint)?;
435    Ok((c.into_owned(), n.map(str::to_string)))
436}
437
438/// As [`xml_convert_to_utf8`], but borrows the input when it is already UTF-8.
439/// The overwhelmingly common case is a UTF-8 document, and copying it whole
440/// before parsing costs one allocation and one full memcpy per parse.
441pub(crate) fn xml_convert_to_utf8_cow<'a>(
442    input: &'a [u8],
443    hint: Option<&str>,
444) -> Result<(std::borrow::Cow<'a, [u8]>, Option<&'static str>), XmlError> {
445    if input.len() >= 2 && input[0] == 0x1f && input[1] == 0x8b {
446        return Err(XmlError::new(
447            XML_ERR_UNSUPPORTED_ENCODING,
448            "gzip input requires the unzip feature / XML_PARSE_UNZIP",
449            0,
450            0,
451        ));
452    }
453    let mut enc = hint
454        .and_then(|h| xml_parse_char_encoding(h).into_option())
455        .unwrap_or_else(|| xml_detect_char_encoding(input));
456    if enc == XmlCharEncoding::None || enc == XmlCharEncoding::Utf8 {
457        if let Some(d) = sniff_encoding_decl(input) {
458            if d != XmlCharEncoding::Utf8 && d != XmlCharEncoding::Ascii {
459                enc = d;
460            }
461        }
462    }
463    if enc == XmlCharEncoding::None || enc == XmlCharEncoding::Utf8 || enc == XmlCharEncoding::Ascii
464    {
465        // Slice past a BOM rather than draining it, which memmoved the document.
466        let body = if input.starts_with(&[0xEF, 0xBB, 0xBF]) { &input[3..] } else { input };
467        return Ok((std::borrow::Cow::Borrowed(body), xml_get_char_encoding_name(enc)));
468    }
469    let converted = match enc {
470        XmlCharEncoding::Iso8859_1 => latin1_to_utf8(input),
471        XmlCharEncoding::Utf16Le | XmlCharEncoding::Utf16 => utf16_to_utf8(input, false)?,
472        XmlCharEncoding::Utf16Be => utf16_to_utf8(input, true)?,
473        XmlCharEncoding::Ucs4Be => ucs4_to_utf8(input, [0, 1, 2, 3])?,
474        XmlCharEncoding::Ucs4Le => ucs4_to_utf8(input, [3, 2, 1, 0])?,
475        XmlCharEncoding::Ucs4_2143 => ucs4_to_utf8(input, [1, 0, 3, 2])?,
476        XmlCharEncoding::Ucs4_3412 => ucs4_to_utf8(input, [2, 3, 0, 1])?,
477        XmlCharEncoding::Ucs2 => utf16_to_utf8(input, true)?,
478        XmlCharEncoding::Ebcdic => {
479            let mut out = Vec::new();
480            for &b in input {
481                let c = char::from_u32(CP037[b as usize] as u32).unwrap_or('\u{fffd}');
482                let mut buf = [0u8; 4];
483                out.extend_from_slice(c.encode_utf8(&mut buf).as_bytes());
484            }
485            out
486        }
487        other => {
488            if let Some(t) = table_for(other) {
489                eightbit_to_utf8(input, t)
490            } else {
491                return Err(XmlError::new(
492                    XML_ERR_UNSUPPORTED_ENCODING,
493                    format!(
494                        "Unsupported encoding {}",
495                        xml_get_char_encoding_name(other).unwrap_or("?")
496                    ),
497                    0,
498                    0,
499                ));
500            }
501        }
502    };
503    Ok((
504        std::borrow::Cow::Owned(converted),
505        xml_get_char_encoding_name(enc),
506    ))
507}
508
509#[cfg(test)]
510mod tests {
511    use super::*;
512
513    #[test]
514    fn latin1_converts() {
515        let (u, _) = xml_convert_to_utf8(&[0xE9], Some("ISO-8859-1")).unwrap();
516        assert_eq!(u, "é".as_bytes());
517    }
518
519    #[test]
520    fn utf16_le_bom() {
521        // BOM + `<a/>` in UTF-16LE
522        let mut b = vec![0xFF, 0xFE];
523        for c in "<a/>".encode_utf16() {
524            b.extend_from_slice(&c.to_le_bytes());
525        }
526        let (u, _) = xml_convert_to_utf8(&b, None).unwrap();
527        assert_eq!(std::str::from_utf8(&u).unwrap().trim_start_matches('\u{feff}'), "<a/>");
528    }
529
530    #[test]
531    fn iso8859_2_table() {
532        // 0xA1 → U+0104 LATIN CAPITAL LETTER A WITH OGONEK
533        let (u, _) = xml_convert_to_utf8(&[0xA1], Some("ISO-8859-2")).unwrap();
534        assert_eq!(u, "Ą".as_bytes());
535    }
536
537    #[test]
538    fn parse_encoding_names() {
539        assert_eq!(xml_parse_char_encoding("utf-8"), XmlCharEncoding::Utf8);
540        assert_eq!(xml_parse_char_encoding("latin1"), XmlCharEncoding::Iso8859_1);
541        assert_eq!(xml_parse_char_encoding("windows-1252"), XmlCharEncoding::Windows1252);
542    }
543}
544
rusty_xml_parser/encoding.rs

rusty_xml_parser/
encoding.rs
