use super::{
DecoderFnc,
multi_byte::*,
single_byte::*,
utf::{decoder_utf7, decoder_utf16, decoder_utf16_be, decoder_utf16_le},
};
pub fn charset_decoder(charset: &[u8]) -> Option<DecoderFnc> {
let mut l_charset = [0u8; 45];
for (dest, src) in l_charset.iter_mut().zip(charset.iter()) {
*dest = match src {
b'A'..=b'Z' => *src + 32,
b'-' => b'_',
_ => *src,
};
}
hashify::tiny_map!(&l_charset[..charset.len().clamp(1, 45)],
"850" => decoder_ibm_850,
"866" => decoder_ibm866,
"ansi_x3.4_1968" => decoder_cp1252,
"arabic" => decoder_iso_8859_6,
"ascii" => decoder_cp1252,
"asmo_708" => decoder_iso_8859_6,
"big5" => decoder_big5,
"big5_hkscs" => decoder_big5,
"chinese" => decoder_gbk,
"cn_big5" => decoder_big5,
"cp1250" => decoder_cp1250,
"cp1251" => decoder_cp1251,
"cp1252" => decoder_cp1252,
"cp1253" => decoder_cp1253,
"cp1254" => decoder_cp1254,
"cp1255" => decoder_cp1255,
"cp1256" => decoder_cp1256,
"cp1257" => decoder_cp1257,
"cp1258" => decoder_cp1258,
"cp819" => decoder_cp1252,
"cp850" => decoder_ibm_850,
"cp866" => decoder_ibm866,
"cp936" => decoder_gbk,
"csbig5" => decoder_big5,
"cseuckr" => decoder_euc_kr,
"cseucpkdfmtjapanese" => decoder_euc_jp,
"csgb18030" => decoder_gb18030,
"csgb2312" => decoder_gbk,
"csgbk" => decoder_gbk,
"csibm866" => decoder_ibm866,
"csiso2022jp" => decoder_iso2022_jp,
"csiso2022kr" => decoder_replacement,
"csiso58gb231280" => decoder_gbk,
"csiso885913" => decoder_iso_8859_13,
"csiso885914" => decoder_iso_8859_14,
"csiso885915" => decoder_iso_8859_15,
"csiso885916" => decoder_iso_8859_16,
"csiso88596e" => decoder_iso_8859_6,
"csiso88596i" => decoder_iso_8859_6,
"csiso88598e" => decoder_iso_8859_8,
"csiso88598i" => decoder_iso_8859_8,
"csisolatin1" => decoder_cp1252,
"csisolatin2" => decoder_iso_8859_2,
"csisolatin3" => decoder_iso_8859_3,
"csisolatin4" => decoder_iso_8859_4,
"csisolatin5" => decoder_iso_8859_9,
"csisolatin6" => decoder_iso_8859_10,
"csisolatin9" => decoder_iso_8859_15,
"csisolatinarabic" => decoder_iso_8859_6,
"csisolatincyrillic" => decoder_iso_8859_5,
"csisolatingreek" => decoder_iso_8859_7,
"csisolatinhebrew" => decoder_iso_8859_8,
"cskoi8r" => decoder_koi8_r,
"cskoi8u" => decoder_koi8_u,
"csksc56011987" => decoder_euc_kr,
"csmacintosh" => decoder_macintosh,
"cspc850multilingual" => decoder_ibm_850,
"csshiftjis" => decoder_shift_jis,
"cstis620" => decoder_tis_620,
"csunicode" => decoder_utf16,
"csutf16" => decoder_utf16,
"csutf16be" => decoder_utf16_be,
"csutf16le" => decoder_utf16_le,
"csutf7" => decoder_utf7,
"cswindows1250" => decoder_cp1250,
"cswindows1251" => decoder_cp1251,
"cswindows1252" => decoder_cp1252,
"cswindows1253" => decoder_cp1253,
"cswindows1254" => decoder_cp1254,
"cswindows1255" => decoder_cp1255,
"cswindows1256" => decoder_cp1256,
"cswindows1257" => decoder_cp1257,
"cswindows1258" => decoder_cp1258,
"cswindows874" => decoder_windows874,
"cyrillic" => decoder_iso_8859_5,
"dos_874" => decoder_windows874,
"ecma_114" => decoder_iso_8859_6,
"ecma_118" => decoder_iso_8859_7,
"elot_928" => decoder_iso_8859_7,
"euc_jp" => decoder_euc_jp,
"euc_kr" => decoder_euc_kr,
"extended_unix_code_packed_format_for_japanese" => decoder_euc_jp,
"gb18030" => decoder_gb18030,
"gb2312" => decoder_gb18030,
"gb_2312" => decoder_gbk,
"gb_2312_80" => decoder_gbk,
"gbk" => decoder_gbk,
"greek" => decoder_iso_8859_7,
"greek8" => decoder_iso_8859_7,
"hebrew" => decoder_iso_8859_8,
"hz_gb_2312" => decoder_replacement,
"ibm819" => decoder_cp1252,
"ibm850" => decoder_ibm_850,
"ibm866" => decoder_ibm866,
"iso88591" => decoder_cp1252,
"iso885910" => decoder_iso_8859_10,
"iso885911" => decoder_tis_620,
"iso885913" => decoder_iso_8859_13,
"iso885914" => decoder_iso_8859_14,
"iso885915" => decoder_iso_8859_15,
"iso88592" => decoder_iso_8859_2,
"iso88593" => decoder_iso_8859_3,
"iso88594" => decoder_iso_8859_4,
"iso88595" => decoder_iso_8859_5,
"iso88596" => decoder_iso_8859_6,
"iso88597" => decoder_iso_8859_7,
"iso88598" => decoder_iso_8859_8,
"iso88599" => decoder_iso_8859_9,
"iso8859_1" => decoder_cp1252,
"iso8859_10" => decoder_iso_8859_10,
"iso8859_11" => decoder_tis_620,
"iso8859_13" => decoder_iso_8859_13,
"iso8859_14" => decoder_iso_8859_14,
"iso8859_15" => decoder_iso_8859_15,
"iso8859_2" => decoder_iso_8859_2,
"iso8859_3" => decoder_iso_8859_3,
"iso8859_4" => decoder_iso_8859_4,
"iso8859_5" => decoder_iso_8859_5,
"iso8859_6" => decoder_iso_8859_6,
"iso8859_7" => decoder_iso_8859_7,
"iso8859_8" => decoder_iso_8859_8,
"iso8859_9" => decoder_iso_8859_9,
"iso_10646_ucs_2" => decoder_utf16,
"iso_2022_cn" => decoder_replacement,
"iso_2022_cn_ext" => decoder_replacement,
"iso_2022_jp" => decoder_iso2022_jp,
"iso_2022_kr" => decoder_replacement,
"iso_8859_1" => decoder_cp1252,
"iso_8859_10" => decoder_iso_8859_10,
"iso_8859_10:1992" => decoder_iso_8859_10,
"iso_8859_11" => decoder_tis_620,
"iso_8859_13" => decoder_iso_8859_13,
"iso_8859_14" => decoder_iso_8859_14,
"iso_8859_14:1998" => decoder_iso_8859_14,
"iso_8859_15" => decoder_iso_8859_15,
"iso_8859_16" => decoder_iso_8859_16,
"iso_8859_16:2001" => decoder_iso_8859_16,
"iso_8859_1:1987" => decoder_cp1252,
"iso_8859_2" => decoder_iso_8859_2,
"iso_8859_2:1987" => decoder_iso_8859_2,
"iso_8859_3" => decoder_iso_8859_3,
"iso_8859_3:1988" => decoder_iso_8859_3,
"iso_8859_4" => decoder_iso_8859_4,
"iso_8859_4:1988" => decoder_iso_8859_4,
"iso_8859_5" => decoder_iso_8859_5,
"iso_8859_5:1988" => decoder_iso_8859_5,
"iso_8859_6" => decoder_iso_8859_6,
"iso_8859_6:1987" => decoder_iso_8859_6,
"iso_8859_6_e" => decoder_iso_8859_6,
"iso_8859_6_i" => decoder_iso_8859_6,
"iso_8859_7" => decoder_iso_8859_7,
"iso_8859_7:1987" => decoder_iso_8859_7,
"iso_8859_8" => decoder_iso_8859_8,
"iso_8859_8:1988" => decoder_iso_8859_8,
"iso_8859_8_e" => decoder_iso_8859_8,
"iso_8859_8_i" => decoder_iso_8859_8,
"iso_8859_9" => decoder_iso_8859_9,
"iso_8859_9:1989" => decoder_iso_8859_9,
"iso_celtic" => decoder_iso_8859_14,
"iso_ir_100" => decoder_cp1252,
"iso_ir_101" => decoder_iso_8859_2,
"iso_ir_109" => decoder_iso_8859_3,
"iso_ir_110" => decoder_iso_8859_4,
"iso_ir_126" => decoder_iso_8859_7,
"iso_ir_127" => decoder_iso_8859_6,
"iso_ir_138" => decoder_iso_8859_8,
"iso_ir_144" => decoder_iso_8859_5,
"iso_ir_148" => decoder_iso_8859_9,
"iso_ir_149" => decoder_euc_kr,
"iso_ir_157" => decoder_iso_8859_10,
"iso_ir_199" => decoder_iso_8859_14,
"iso_ir_226" => decoder_iso_8859_16,
"iso_ir_58" => decoder_gbk,
"koi" => decoder_koi8_r,
"koi8" => decoder_koi8_r,
"koi8_r" => decoder_koi8_r,
"koi8_ru" => decoder_koi8_u,
"koi8_u" => decoder_koi8_u,
"korean" => decoder_euc_kr,
"ks_c_5601_1987" => decoder_euc_kr,
"ks_c_5601_1989" => decoder_euc_kr,
"ksc5601" => decoder_euc_kr,
"ksc_5601" => decoder_euc_kr,
"l1" => decoder_cp1252,
"l10" => decoder_iso_8859_16,
"l2" => decoder_iso_8859_2,
"l3" => decoder_iso_8859_3,
"l4" => decoder_iso_8859_4,
"l5" => decoder_iso_8859_9,
"l6" => decoder_iso_8859_10,
"l8" => decoder_iso_8859_14,
"l9" => decoder_iso_8859_15,
"latin1" => decoder_cp1252,
"latin10" => decoder_iso_8859_16,
"latin2" => decoder_iso_8859_2,
"latin3" => decoder_iso_8859_3,
"latin4" => decoder_iso_8859_4,
"latin5" => decoder_iso_8859_9,
"latin6" => decoder_iso_8859_10,
"latin8" => decoder_iso_8859_14,
"latin_9" => decoder_iso_8859_15,
"logical" => decoder_iso_8859_8,
"mac" => decoder_macintosh,
"macintosh" => decoder_macintosh,
"ms932" => decoder_shift_jis,
"ms936" => decoder_gbk,
"ms_kanji" => decoder_shift_jis,
"replacement" => decoder_replacement,
"shift_jis" => decoder_shift_jis,
"sjis" => decoder_shift_jis,
"sun_eu_greek" => decoder_iso_8859_7,
"tis_620" => decoder_tis_620,
"ucs_2" => decoder_utf16,
"unicode" => decoder_utf16,
"unicodefeff" => decoder_utf16,
"unicodefffe" => decoder_utf16_be,
"us_ascii" => decoder_cp1252,
"utf_16" => decoder_utf16,
"utf_16be" => decoder_utf16_be,
"utf_16le" => decoder_utf16_le,
"utf_7" => decoder_utf7,
"visual" => decoder_iso_8859_8,
"windows_1250" => decoder_cp1250,
"windows_1251" => decoder_cp1251,
"windows_1252" => decoder_cp1252,
"windows_1253" => decoder_cp1253,
"windows_1254" => decoder_cp1254,
"windows_1255" => decoder_cp1255,
"windows_1256" => decoder_cp1256,
"windows_1257" => decoder_cp1257,
"windows_1258" => decoder_cp1258,
"windows_31j" => decoder_shift_jis,
"windows_874" => decoder_windows874,
"windows_936" => decoder_gbk,
"windows_949" => decoder_euc_kr,
"x_cp1250" => decoder_cp1250,
"x_cp1251" => decoder_cp1251,
"x_cp1252" => decoder_cp1252,
"x_cp1253" => decoder_cp1253,
"x_cp1254" => decoder_cp1254,
"x_cp1255" => decoder_cp1255,
"x_cp1256" => decoder_cp1256,
"x_cp1257" => decoder_cp1257,
"x_cp1258" => decoder_cp1258,
"x_euc_jp" => decoder_euc_jp,
"x_gbk" => decoder_gbk,
"x_mac_cyrillic" => decoder_x_mac_cyrillic,
"x_mac_roman" => decoder_macintosh,
"x_mac_ukrainian" => decoder_x_mac_cyrillic,
"x_sjis" => decoder_shift_jis,
"x_user_defined" => decoder_x_user_defined,
"x_x_big5" => decoder_big5,
)
}
#[cfg(test)]
mod tests {
use super::charset_decoder;
#[test]
fn decoder_charset() {
for input in ["gbk", "extended_unix_code_packed_format_for_japanese"] {
if !input.is_empty() {
assert!(
charset_decoder(input.as_bytes()).is_some(),
"Failed for {input}",
);
}
}
}
#[test]
fn decoder_charset_encoding_rs_labels() {
let supported = [
"l9", "koi", "koi8", "sjis", "ucs-2", "ms932", "ascii", "x-gbk", "cp1250", "cp1251",
"cp1252", "cp1253", "cp1254", "cp1255", "cp1256", "cp1257", "cp1258", "visual",
"korean", "x-sjis", "ksc5601", "gb_2312", "dos-874", "cn-big5", "unicode", "chinese",
"logical", "koi8-ru", "x-cp1250", "ksc_5601", "x-cp1251", "iso88591", "csgb2312",
"x-cp1252", "iso88592", "x-cp1253", "iso88593", "x-cp1254", "iso88594", "x-cp1255",
"iso88595", "x-x-big5", "x-cp1256", "iso88596", "x-cp1257", "iso88597", "x-cp1258",
"iso88598", "iso88599", "us-ascii", "x-euc-jp", "iso885910", "iso8859-1", "iso885911",
"iso8859-2", "iso8859-3", "iso885913", "iso8859-4", "iso885914", "iso8859-5",
"iso885915", "iso8859-6", "iso8859-7", "iso8859-8", "iso-ir-58", "iso8859-9",
"csunicode", "iso8859-10", "gb_2312-80", "iso8859-11", "iso8859-13", "iso8859-14",
"iso8859-15", "iso-ir-149", "big5-hkscs", "windows-949", "csisolatin9", "csiso88596e",
"csiso88598e", "unicodefffe", "unicodefeff", "csiso88596i", "csiso88598i", "windows-31j",
"x-mac-roman", "sun_eu_greek", "csksc56011987", "ansi_x3.4-1968", "csiso58gb231280",
"iso-10646-ucs-2", "iso-8859-6-e", "iso-8859-8-e", "iso-8859-6-i", "replacement",
"iso-2022-kr", "csiso2022kr", "iso-2022-cn", "iso-2022-cn-ext", "hz-gb-2312",
"x-user-defined", "x-mac-cyrillic", "x-mac-ukrainian",
];
for input in supported {
assert!(
charset_decoder(input.as_bytes()).is_some(),
"Expected a decoder for {input}",
);
}
let unsupported = [
"utf8", "utf-8", "unicode11utf8", "unicode20utf8", "x-unicode20utf8",
"unicode-1-1-utf-8",
];
for input in unsupported {
assert!(
charset_decoder(input.as_bytes()).is_none(),
"Did not expect a decoder for {input}",
);
}
}
}