rusty_xml_parser/
encoding.rs

1//! Character encodings matching libxml2 `encoding.h` without iconv.
2//! 8-bit tables are byte-identical to v2.15.3 `codegen/charset.inc`.
3
4use crate::encoding_tables::*;
5use crate::error::{XmlError, XML_ERR_UNSUPPORTED_ENCODING};
6
7/// libxml2 `xmlCharEncoding` discriminants.
8#[derive(Clone, Copy, Debug, PartialEq, Eq)]
9#[repr(i32)]
10pub enum XmlCharEncoding {
11    Error = -1,
12    None = 0,
13    Utf8 = 1,
14    Utf16Le = 2,
15    Utf16Be = 3,
16    Ucs4Le = 4,
17    Ucs4Be = 5,
18    Ebcdic = 6,
19    Ucs4_2143 = 7,
20    Ucs4_3412 = 8,
21    Ucs2 = 9,
22    Iso8859_1 = 10,
23    Iso8859_2 = 11,
24    Iso8859_3 = 12,
25    Iso8859_4 = 13,
26    Iso8859_5 = 14,
27    Iso8859_6 = 15,
28    Iso8859_7 = 16,
29    Iso8859_8 = 17,
30    Iso8859_9 = 18,
31    Iso2022Jp = 19,
32    ShiftJis = 20,
33    EucJp = 21,
34    Ascii = 22,
35    Utf16 = 23,
36    Html = 24,
37    Iso8859_10 = 25,
38    Iso8859_11 = 26,
39    Iso8859_13 = 27,
40    Iso8859_14 = 28,
41    Iso8859_15 = 29,
42    Iso8859_16 = 30,
43    Windows1252 = 31,
44}
45
46const NAME_MAP: &[(&str, XmlCharEncoding)] = &[
47    ("ascii", XmlCharEncoding::Ascii),
48    ("csisolatin1", XmlCharEncoding::Iso8859_1),
49    ("iso-8859-1", XmlCharEncoding::Iso8859_1),
50    ("iso-8859-2", XmlCharEncoding::Iso8859_2),
51    ("iso-8859-3", XmlCharEncoding::Iso8859_3),
52    ("iso-8859-4", XmlCharEncoding::Iso8859_4),
53    ("iso-8859-5", XmlCharEncoding::Iso8859_5),
54    ("iso-8859-6", XmlCharEncoding::Iso8859_6),
55    ("iso-8859-7", XmlCharEncoding::Iso8859_7),
56    ("iso-8859-8", XmlCharEncoding::Iso8859_8),
57    ("iso-8859-9", XmlCharEncoding::Iso8859_9),
58    ("iso-8859-10", XmlCharEncoding::Iso8859_10),
59    ("iso-8859-11", XmlCharEncoding::Iso8859_11),
60    ("iso-8859-13", XmlCharEncoding::Iso8859_13),
61    ("iso-8859-14", XmlCharEncoding::Iso8859_14),
62    ("iso-8859-15", XmlCharEncoding::Iso8859_15),
63    ("iso-8859-16", XmlCharEncoding::Iso8859_16),
64    ("iso8859-1", XmlCharEncoding::Iso8859_1),
65    ("iso_8859-1", XmlCharEncoding::Iso8859_1),
66    ("latin1", XmlCharEncoding::Iso8859_1),
67    ("us-ascii", XmlCharEncoding::Ascii),
68    ("utf-16", XmlCharEncoding::Utf16),
69    ("utf-16be", XmlCharEncoding::Utf16Be),
70    ("utf-16le", XmlCharEncoding::Utf16Le),
71    ("utf-8", XmlCharEncoding::Utf8),
72    ("utf16", XmlCharEncoding::Utf16),
73    ("utf8", XmlCharEncoding::Utf8),
74    ("unicode", XmlCharEncoding::Utf16),
75    ("ucs-2", XmlCharEncoding::Ucs2),
76    ("ucs-4", XmlCharEncoding::Ucs4Le),
77    ("ucs2", XmlCharEncoding::Ucs2),
78    ("ucs4", XmlCharEncoding::Ucs4Le),
79    ("windows-1252", XmlCharEncoding::Windows1252),
80    ("x-cp1252", XmlCharEncoding::Windows1252),
81    ("ibm037", XmlCharEncoding::Ebcdic),
82    ("ebcdic", XmlCharEncoding::Ebcdic),
83    ("iso-2022-jp", XmlCharEncoding::Iso2022Jp),
84    ("shift_jis", XmlCharEncoding::ShiftJis),
85    ("shift-jis", XmlCharEncoding::ShiftJis),
86    ("sjis", XmlCharEncoding::ShiftJis),
87    ("euc-jp", XmlCharEncoding::EucJp),
88];
89
90/// `xmlParseCharEncoding`.
91#[doc(alias = "xmlParseCharEncoding")]
92pub fn xml_parse_char_encoding(name: &str) -> XmlCharEncoding {
93    let lower = name.trim().to_ascii_lowercase();
94    for (n, enc) in NAME_MAP {
95        if *n == lower {
96            return if *enc == XmlCharEncoding::Utf16 {
97                XmlCharEncoding::Utf16Le
98            } else {
99                *enc
100            };
101        }
102    }
103    XmlCharEncoding::Error
104}
105
106/// `xmlGetCharEncodingName`.
107#[doc(alias = "xmlGetCharEncodingName")]
108pub fn xml_get_char_encoding_name(enc: XmlCharEncoding) -> Option<&'static str> {
109    Some(match enc {
110        XmlCharEncoding::Utf8 => "UTF-8",
111        XmlCharEncoding::Utf16Le | XmlCharEncoding::Utf16Be | XmlCharEncoding::Utf16 => "UTF-16",
112        XmlCharEncoding::Ucs4Le | XmlCharEncoding::Ucs4Be => "UCS-4",
113        XmlCharEncoding::Iso8859_1 => "ISO-8859-1",
114        XmlCharEncoding::Iso8859_2 => "ISO-8859-2",
115        XmlCharEncoding::Iso8859_3 => "ISO-8859-3",
116        XmlCharEncoding::Iso8859_4 => "ISO-8859-4",
117        XmlCharEncoding::Iso8859_5 => "ISO-8859-5",
118        XmlCharEncoding::Iso8859_6 => "ISO-8859-6",
119        XmlCharEncoding::Iso8859_7 => "ISO-8859-7",
120        XmlCharEncoding::Iso8859_8 => "ISO-8859-8",
121        XmlCharEncoding::Iso8859_9 => "ISO-8859-9",
122        XmlCharEncoding::Iso8859_10 => "ISO-8859-10",
123        XmlCharEncoding::Iso8859_11 => "ISO-8859-11",
124        XmlCharEncoding::Iso8859_13 => "ISO-8859-13",
125        XmlCharEncoding::Iso8859_14 => "ISO-8859-14",
126        XmlCharEncoding::Iso8859_15 => "ISO-8859-15",
127        XmlCharEncoding::Iso8859_16 => "ISO-8859-16",
128        XmlCharEncoding::Ascii => "US-ASCII",
129        XmlCharEncoding::Windows1252 => "windows-1252",
130        XmlCharEncoding::Ebcdic => "IBM037",
131        XmlCharEncoding::Ucs2 => "UCS-2",
132        XmlCharEncoding::Iso2022Jp => "ISO-2022-JP",
133        XmlCharEncoding::ShiftJis => "Shift_JIS",
134        XmlCharEncoding::EucJp => "EUC-JP",
135        XmlCharEncoding::Html => "HTML",
136        XmlCharEncoding::None | XmlCharEncoding::Error | XmlCharEncoding::Ucs4_2143 | XmlCharEncoding::Ucs4_3412 => {
137            return None
138        }
139    })
140}
141
142/// `xmlDetectCharEncoding` — XML 1.0 appendix F plus libxml2's UTF-16 extras.
143#[doc(alias = "xmlDetectCharEncoding")]
144pub fn xml_detect_char_encoding(input: &[u8]) -> XmlCharEncoding {
145    if input.len() >= 4 {
146        let b = &input[..4];
147        if b == [0x00, 0x00, 0x00, 0x3C] {
148            return XmlCharEncoding::Ucs4Be;
149        }
150        if b == [0x3C, 0x00, 0x00, 0x00] {
151            return XmlCharEncoding::Ucs4Le;
152        }
153        if b == [0x00, 0x00, 0x3C, 0x00] {
154            return XmlCharEncoding::Ucs4_2143;
155        }
156        if b == [0x00, 0x3C, 0x00, 0x00] {
157            return XmlCharEncoding::Ucs4_3412;
158        }
159        if b == [0x4C, 0x6F, 0xA7, 0x94] {
160            return XmlCharEncoding::Ebcdic;
161        }
162        if b == [0x3C, 0x3F, 0x78, 0x6D] {
163            return XmlCharEncoding::Utf8;
164        }
165        if b == [0x3C, 0x00, 0x3F, 0x00] {
166            return XmlCharEncoding::Utf16Le;
167        }
168        if b == [0x00, 0x3C, 0x00, 0x3F] {
169            return XmlCharEncoding::Utf16Be;
170        }
171        if b == [0x00, 0x00, 0xFE, 0xFF] {
172            return XmlCharEncoding::Ucs4Be;
173        }
174        if b == [0xFF, 0xFE, 0x00, 0x00] {
175            return XmlCharEncoding::Ucs4Le;
176        }
177    }
178    if input.len() >= 3 && input[..3] == [0xEF, 0xBB, 0xBF] {
179        return XmlCharEncoding::Utf8;
180    }
181    if input.len() >= 2 {
182        if input[0] == 0xFE && input[1] == 0xFF {
183            return XmlCharEncoding::Utf16Be;
184        }
185        if input[0] == 0xFF && input[1] == 0xFE {
186            return XmlCharEncoding::Utf16Le;
187        }
188    }
189    XmlCharEncoding::None
190}
191
192fn eightbit_to_utf8(input: &[u8], table: &[u16; 128]) -> Vec<u8> {
193    let mut out = Vec::with_capacity(input.len() * 2);
194    for &b in input {
195        let cp = if b < 0x80 {
196            b as u32
197        } else {
198            table[(b - 0x80) as usize] as u32
199        };
200        if let Some(c) = char::from_u32(cp) {
201            let mut buf = [0u8; 4];
202            out.extend_from_slice(c.encode_utf8(&mut buf).as_bytes());
203        }
204    }
205    out
206}
207
208fn latin1_to_utf8(input: &[u8]) -> Vec<u8> {
209    let mut out = Vec::with_capacity(input.len() * 2);
210    for &b in input {
211        let c = b as char;
212        let mut buf = [0u8; 4];
213        out.extend_from_slice(c.encode_utf8(&mut buf).as_bytes());
214    }
215    out
216}
217
218fn utf16_to_utf8(input: &[u8], be: bool) -> Result<Vec<u8>, XmlError> {
219    let mut i = 0;
220    if input.len() >= 2 {
221        let bom = if be {
222            input[0] == 0xFE && input[1] == 0xFF
223        } else {
224            input[0] == 0xFF && input[1] == 0xFE
225        };
226        if bom {
227            i = 2;
228        }
229    }
230    let mut units = Vec::new();
231    while i + 1 < input.len() {
232        let u = if be {
233            u16::from_be_bytes([input[i], input[i + 1]])
234        } else {
235            u16::from_le_bytes([input[i], input[i + 1]])
236        };
237        units.push(u);
238        i += 2;
239    }
240    let s = String::from_utf16(&units).map_err(|_| {
241        XmlError::new(XML_ERR_UNSUPPORTED_ENCODING, "Invalid UTF-16", 0, 0)
242    })?;
243    Ok(s.into_bytes())
244}
245
246fn ucs4_to_utf8(input: &[u8], order: [usize; 4]) -> Result<Vec<u8>, XmlError> {
247    let mut i = 0;
248    if input.len() >= 4 {
249        let w = u32::from_be_bytes([input[0], input[1], input[2], input[3]]);
250        if w == 0xFEFF || w == 0xFFFE0000 {
251            i = 4;
252        }
253    }
254    let mut out = String::new();
255    while i + 3 < input.len() {
256        let b = [input[i], input[i + 1], input[i + 2], input[i + 3]];
257        let cp = u32::from_be_bytes([b[order[0]], b[order[1]], b[order[2]], b[order[3]]]);
258        match char::from_u32(cp) {
259            Some(c) if c != '\u{feff}' || !out.is_empty() => out.push(c),
260            Some(_) => {}
261            None => {
262                return Err(XmlError::new(
263                    XML_ERR_UNSUPPORTED_ENCODING,
264                    "Invalid UCS-4",
265                    0,
266                    0,
267                ));
268            }
269        }
270        i += 4;
271    }
272    Ok(out.into_bytes())
273}
274
275/// IBM037 (EBCDIC) — enough to convert XML documents detected as `4C 6F A7 94`.
276static CP037: [u16; 256] = [
277    0x00, 0x01, 0x02, 0x03, 0x9c, 0x09, 0x86, 0x7f, 0x97, 0x8d, 0x8e, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
278    0x10, 0x11, 0x12, 0x13, 0x9d, 0x85, 0x08, 0x87, 0x18, 0x19, 0x92, 0x8f, 0x1c, 0x1d, 0x1e, 0x1f,
279    0x80, 0x81, 0x82, 0x83, 0x84, 0x0a, 0x17, 0x1b, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x05, 0x06, 0x07,
280    0x90, 0x91, 0x16, 0x93, 0x94, 0x95, 0x96, 0x04, 0x98, 0x99, 0x9a, 0x9b, 0x14, 0x15, 0x9e, 0x1a,
281    0x20, 0xa0, 0xe2, 0xe4, 0xe0, 0xe1, 0xe3, 0xe5, 0xe7, 0xf1, 0xa2, 0x2e, 0x3c, 0x28, 0x2b, 0x7c,
282    0x26, 0xe9, 0xea, 0xeb, 0xe8, 0xed, 0xee, 0xef, 0xec, 0xdf, 0x21, 0x24, 0x2a, 0x29, 0x3b, 0xac,
283    0x2d, 0x2f, 0xc2, 0xc4, 0xc0, 0xc1, 0xc3, 0xc5, 0xc7, 0xd1, 0xa6, 0x2c, 0x25, 0x5f, 0x3e, 0x3f,
284    0xf8, 0xc9, 0xca, 0xcb, 0xc8, 0xcd, 0xce, 0xcf, 0xcc, 0x60, 0x3a, 0x23, 0x40, 0x27, 0x3d, 0x22,
285    0xd8, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xab, 0xbb, 0xf0, 0xfd, 0xfe, 0xb1,
286    0xb0, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0xaa, 0xba, 0xe6, 0xb8, 0xc6, 0xa4,
287    0xb5, 0x7e, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0xa1, 0xbf, 0xd0, 0xdd, 0xde, 0xae,
288    0x5e, 0xa3, 0xa5, 0xb7, 0xa9, 0xa7, 0xb6, 0xbc, 0xbd, 0xbe, 0x5b, 0x5d, 0xaf, 0xa8, 0xb4, 0xd7,
289    0x7b, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0xad, 0xf4, 0xf6, 0xf2, 0xf3, 0xf5,
290    0x7d, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0xb9, 0xfb, 0xfc, 0xf9, 0xfa, 0xff,
291    0x5c, 0xf7, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0xb2, 0xd4, 0xd6, 0xd2, 0xd3, 0xd5,
292    0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0xb3, 0xdb, 0xdc, 0xd9, 0xda, 0x9f,
293];
294
295fn table_for(enc: XmlCharEncoding) -> Option<&'static [u16; 128]> {
296    Some(match enc {
297        XmlCharEncoding::Iso8859_2 => &XML_UNICODE_ISO8859_2,
298        XmlCharEncoding::Iso8859_3 => &XML_UNICODE_ISO8859_3,
299        XmlCharEncoding::Iso8859_4 => &XML_UNICODE_ISO8859_4,
300        XmlCharEncoding::Iso8859_5 => &XML_UNICODE_ISO8859_5,
301        XmlCharEncoding::Iso8859_6 => &XML_UNICODE_ISO8859_6,
302        XmlCharEncoding::Iso8859_7 => &XML_UNICODE_ISO8859_7,
303        XmlCharEncoding::Iso8859_8 => &XML_UNICODE_ISO8859_8,
304        XmlCharEncoding::Iso8859_9 => &XML_UNICODE_ISO8859_9,
305        XmlCharEncoding::Iso8859_10 => &XML_UNICODE_ISO8859_10,
306        XmlCharEncoding::Iso8859_11 => &XML_UNICODE_ISO8859_11,
307        XmlCharEncoding::Iso8859_13 => &XML_UNICODE_ISO8859_13,
308        XmlCharEncoding::Iso8859_14 => &XML_UNICODE_ISO8859_14,
309        XmlCharEncoding::Iso8859_15 => &XML_UNICODE_ISO8859_15,
310        XmlCharEncoding::Iso8859_16 => &XML_UNICODE_ISO8859_16,
311        XmlCharEncoding::Windows1252 => &XML_UNICODE_windows_1252,
312        _ => return None,
313    })
314}
315
316fn sniff_encoding_decl(bytes: &[u8]) -> Option<XmlCharEncoding> {
317    // The declaration is ASCII. Scanning the bytes directly avoids building a
318    // 1 KB String and a second lowercased copy of it on every single parse.
319    let n = bytes.len().min(1024);
320    let head = &bytes[..n];
321    if head.len() < 8 {
322        return None;
323    }
324    let idx = head
325        .windows(8)
326        .position(|w| w.eq_ignore_ascii_case(b"encoding"))?;
327    let mut i = idx + 8;
328    while i < n && matches!(head[i], b' ' | b'\t' | b'\r' | b'\n' | b'=') {
329        i += 1;
330    }
331    if i >= n || (head[i] != b'"' && head[i] != b'\'') {
332        return None;
333    }
334    let q = head[i];
335    i += 1;
336    let start = i;
337    while i < n && head[i] != q {
338        i += 1;
339    }
340    if i >= n {
341        return None;
342    }
343    let name = std::str::from_utf8(&head[start..i]).ok()?;
344    xml_parse_char_encoding(name).into_option()
345}
346
347impl XmlCharEncoding {
348    fn into_option(self) -> Option<XmlCharEncoding> {
349        match self {
350            XmlCharEncoding::Error | XmlCharEncoding::None => None,
351            e => Some(e),
352        }
353    }
354}
355
356/// Convert `input` to UTF-8. `hint` is the `encoding` argument to `xmlReadMemory`.
357pub fn xml_convert_to_utf8(
358    input: &[u8],
359    hint: Option<&str>,
360) -> Result<(Vec<u8>, Option<String>), XmlError> {
361    let (c, n) = xml_convert_to_utf8_cow(input, hint)?;
362    Ok((c.into_owned(), n.map(str::to_string)))
363}
364
365/// As [`xml_convert_to_utf8`], but borrows the input when it is already UTF-8.
366/// The overwhelmingly common case is a UTF-8 document, and copying it whole
367/// before parsing costs one allocation and one full memcpy per parse.
368pub(crate) fn xml_convert_to_utf8_cow<'a>(
369    input: &'a [u8],
370    hint: Option<&str>,
371) -> Result<(std::borrow::Cow<'a, [u8]>, Option<&'static str>), XmlError> {
372    if input.len() >= 2 && input[0] == 0x1f && input[1] == 0x8b {
373        return Err(XmlError::new(
374            XML_ERR_UNSUPPORTED_ENCODING,
375            "gzip input requires the unzip feature / XML_PARSE_UNZIP",
376            0,
377            0,
378        ));
379    }
380    let mut enc = hint
381        .and_then(|h| xml_parse_char_encoding(h).into_option())
382        .unwrap_or_else(|| xml_detect_char_encoding(input));
383    if enc == XmlCharEncoding::None || enc == XmlCharEncoding::Utf8 {
384        if let Some(d) = sniff_encoding_decl(input) {
385            if d != XmlCharEncoding::Utf8 && d != XmlCharEncoding::Ascii {
386                enc = d;
387            }
388        }
389    }
390    if enc == XmlCharEncoding::None || enc == XmlCharEncoding::Utf8 || enc == XmlCharEncoding::Ascii
391    {
392        // Slice past a BOM rather than draining it, which memmoved the document.
393        let body = if input.starts_with(&[0xEF, 0xBB, 0xBF]) { &input[3..] } else { input };
394        return Ok((std::borrow::Cow::Borrowed(body), xml_get_char_encoding_name(enc)));
395    }
396    let converted = match enc {
397        XmlCharEncoding::Iso8859_1 => latin1_to_utf8(input),
398        XmlCharEncoding::Utf16Le | XmlCharEncoding::Utf16 => utf16_to_utf8(input, false)?,
399        XmlCharEncoding::Utf16Be => utf16_to_utf8(input, true)?,
400        XmlCharEncoding::Ucs4Be => ucs4_to_utf8(input, [0, 1, 2, 3])?,
401        XmlCharEncoding::Ucs4Le => ucs4_to_utf8(input, [3, 2, 1, 0])?,
402        XmlCharEncoding::Ucs4_2143 => ucs4_to_utf8(input, [1, 0, 3, 2])?,
403        XmlCharEncoding::Ucs4_3412 => ucs4_to_utf8(input, [2, 3, 0, 1])?,
404        XmlCharEncoding::Ucs2 => utf16_to_utf8(input, true)?,
405        XmlCharEncoding::Ebcdic => {
406            let mut out = Vec::new();
407            for &b in input {
408                let c = char::from_u32(CP037[b as usize] as u32).unwrap_or('\u{fffd}');
409                let mut buf = [0u8; 4];
410                out.extend_from_slice(c.encode_utf8(&mut buf).as_bytes());
411            }
412            out
413        }
414        other => {
415            if let Some(t) = table_for(other) {
416                eightbit_to_utf8(input, t)
417            } else {
418                return Err(XmlError::new(
419                    XML_ERR_UNSUPPORTED_ENCODING,
420                    format!(
421                        "Unsupported encoding {}",
422                        xml_get_char_encoding_name(other).unwrap_or("?")
423                    ),
424                    0,
425                    0,
426                ));
427            }
428        }
429    };
430    Ok((
431        std::borrow::Cow::Owned(converted),
432        xml_get_char_encoding_name(enc),
433    ))
434}
435
436#[cfg(test)]
437mod tests {
438    use super::*;
439
440    #[test]
441    fn latin1_converts() {
442        let (u, _) = xml_convert_to_utf8(&[0xE9], Some("ISO-8859-1")).unwrap();
443        assert_eq!(u, "é".as_bytes());
444    }
445
446    #[test]
447    fn utf16_le_bom() {
448        // BOM + `<a/>` in UTF-16LE
449        let mut b = vec![0xFF, 0xFE];
450        for c in "<a/>".encode_utf16() {
451            b.extend_from_slice(&c.to_le_bytes());
452        }
453        let (u, _) = xml_convert_to_utf8(&b, None).unwrap();
454        assert_eq!(std::str::from_utf8(&u).unwrap().trim_start_matches('\u{feff}'), "<a/>");
455    }
456
457    #[test]
458    fn iso8859_2_table() {
459        // 0xA1 → U+0104 LATIN CAPITAL LETTER A WITH OGONEK
460        let (u, _) = xml_convert_to_utf8(&[0xA1], Some("ISO-8859-2")).unwrap();
461        assert_eq!(u, "Ą".as_bytes());
462    }
463
464    #[test]
465    fn parse_encoding_names() {
466        assert_eq!(xml_parse_char_encoding("utf-8"), XmlCharEncoding::Utf8);
467        assert_eq!(xml_parse_char_encoding("latin1"), XmlCharEncoding::Iso8859_1);
468        assert_eq!(xml_parse_char_encoding("windows-1252"), XmlCharEncoding::Windows1252);
469    }
470}
471
rusty_xml_parser/encoding.rs

rusty_xml_parser/
encoding.rs
