#![cfg(all(feature = "alloc", feature = "whatwg"))]
use charcode::*;
fn decode(encoding: &'static Encoding, bytes: &[u8]) -> String {
encoding
.decode_with(bytes, DecodeOptions::new().bom(Bom::Ignore))
.0
.into_owned()
}
fn encode(encoding: &'static Encoding, text: &str) -> Vec<u8> {
encoding.encode_html_form(text).0.into_owned()
}
#[test]
fn labels_are_matched_case_insensitively_and_trimmed() {
assert_eq!(Encoding::for_label(b"UTF-8"), Some(UTF_8));
assert_eq!(Encoding::for_label(b"utf8"), Some(UTF_8));
assert_eq!(Encoding::for_label(b"\t\n\x0C\r UTF-8 \t"), Some(UTF_8));
assert_eq!(Encoding::for_label(b"LATIN1"), Some(WINDOWS_1252));
assert_eq!(Encoding::for_label(b"ascii"), Some(WINDOWS_1252));
assert_eq!(Encoding::for_label(b"iso-8859-1"), Some(WINDOWS_1252));
assert_eq!(Encoding::for_label(b"chinese"), Some(GBK));
assert_eq!(Encoding::for_label(b"utf-16"), Some(UTF_16LE));
}
#[test]
fn unknown_labels_are_rejected() {
assert_eq!(Encoding::for_label(b""), None);
assert_eq!(Encoding::for_label(b" "), None);
assert_eq!(Encoding::for_label(b"utf 8"), None);
assert_eq!(Encoding::for_label(b"nonsense"), None);
assert_eq!(Encoding::for_label(b"u tf-8"), None);
assert_eq!(Encoding::for_label(&[b'a'; 100]), None);
assert_eq!(Encoding::for_label(&[0xFF, 0xFE]), None);
}
#[test]
fn replacement_labels_can_be_filtered_out() {
for label in [
&b"csiso2022kr"[..],
b"hz-gb-2312",
b"iso-2022-cn",
b"iso-2022-cn-ext",
b"iso-2022-kr",
b"replacement",
] {
assert_eq!(Encoding::for_label(label), Some(REPLACEMENT), "{label:?}");
assert_eq!(Encoding::for_label_no_replacement(label), None, "{label:?}");
}
assert_eq!(
Encoding::for_label_no_replacement(b"iso-2022-jp"),
Some(ISO_2022_JP)
);
}
#[test]
fn every_encoding_is_reachable_by_its_own_name() {
#[cfg(not(any(feature = "dos", feature = "ebcdic", feature = "mac", feature = "misc")))]
assert_eq!(Encoding::all().len(), 40);
for &encoding in Encoding::all() {
assert_eq!(
Encoding::for_label(encoding.name().as_bytes()),
Some(encoding),
"{}",
encoding.name()
);
}
}
#[test]
fn output_encoding_replaces_the_three_that_cannot_encode() {
assert_eq!(REPLACEMENT.output_encoding(), UTF_8);
assert_eq!(UTF_16BE.output_encoding(), UTF_8);
assert_eq!(UTF_16LE.output_encoding(), UTF_8);
assert_eq!(WINDOWS_1252.output_encoding(), WINDOWS_1252);
for &encoding in Encoding::all() {
let output = encoding.output_encoding();
assert_eq!(output.output_encoding(), output, "{}", encoding.name());
}
let (bytes, encoding, _) = UTF_16LE.encode("hi").unwrap();
assert_eq!(&bytes[..], b"hi");
assert_eq!(encoding, UTF_8);
}
#[test]
fn a_bom_overrides_the_named_encoding() {
let (text, encoding, _) = WINDOWS_1252.decode(b"\xEF\xBB\xBFcaf\xC3\xA9");
assert_eq!(text, "caf\u{E9}");
assert_eq!(encoding, UTF_8);
let (text, encoding, _) = WINDOWS_1252.decode(b"\xFF\xFEa\x00b\x00");
assert_eq!(text, "ab");
assert_eq!(encoding, UTF_16LE);
let (text, encoding, _) = WINDOWS_1252.decode(b"\xFE\xFF\x00a\x00b");
assert_eq!(text, "ab");
assert_eq!(encoding, UTF_16BE);
}
#[test]
fn bom_removal_only_strips_this_encodings_own_mark() {
let remove = DecodeOptions::new().bom(Bom::Remove);
assert_eq!(UTF_8.decode_with(b"\xEF\xBB\xBFa", remove).0, "a");
assert_eq!(
UTF_8.decode_with(b"\xFF\xFEa", remove).0,
"\u{FFFD}\u{FFFD}a"
);
assert_eq!(
WINDOWS_1252.decode_with(b"\xEF\xBB\xBFa", remove).0,
"\u{EF}\u{BB}\u{BF}a"
);
}
#[test]
fn without_bom_handling_keeps_the_mark_as_content() {
assert_eq!(
UTF_8
.decode_with(b"\xEF\xBB\xBFa", DecodeOptions::new().bom(Bom::Ignore))
.0,
"\u{FEFF}a"
);
assert_eq!(Encoding::for_bom(b"\xEF\xBB\xBF"), Some((UTF_8, 3)));
assert_eq!(Encoding::for_bom(b"\xEF\xBB"), None);
assert_eq!(Encoding::for_bom(b""), None);
}
#[test]
fn a_bom_split_across_chunks_is_still_recognized() {
let mut decoder = WINDOWS_1252.new_decoder();
let mut text = String::new();
decoder.decode_to_string(b"\xEF", &mut text, false).unwrap();
decoder.decode_to_string(b"\xBB", &mut text, false).unwrap();
decoder
.decode_to_string(b"\xBFcaf\xC3\xA9", &mut text, true)
.unwrap();
assert_eq!(text, "caf\u{E9}");
assert_eq!(decoder.encoding(), UTF_8);
}
#[test]
fn a_near_miss_bom_is_decoded_as_content() {
let mut decoder = WINDOWS_1252.new_decoder();
let mut text = String::new();
decoder
.decode_to_string(b"\xEF\xBB", &mut text, false)
.unwrap();
decoder.decode_to_string(b"\x41", &mut text, true).unwrap();
assert_eq!(text, "\u{EF}\u{BB}A");
assert_eq!(decoder.encoding(), WINDOWS_1252);
}
#[test]
fn utf8_substitutes_maximal_subparts() {
assert_eq!(decode(UTF_8, b"\xF0\x9F\x98\x80"), "\u{1F600}");
assert_eq!(decode(UTF_8, b"\xF0\x9F\x98"), "\u{FFFD}");
assert_eq!(decode(UTF_8, b"\xE0\x80\x80"), "\u{FFFD}\u{FFFD}\u{FFFD}");
assert_eq!(decode(UTF_8, b"\xED\xA0\x80"), "\u{FFFD}\u{FFFD}\u{FFFD}");
assert_eq!(decode(UTF_8, b"\xC0\xAF"), "\u{FFFD}\u{FFFD}");
assert_eq!(
decode(UTF_8, b"\xF4\x90\x80\x80"),
"\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}"
);
assert_eq!(decode(UTF_8, b"a\xE2\x82b"), "a\u{FFFD}b");
assert_eq!(
String::from_utf8_lossy(b"\x80\xE1\xA0\xC0\x41"),
decode(UTF_8, b"\x80\xE1\xA0\xC0\x41")
);
}
#[test]
fn valid_utf8_is_borrowed_not_copied() {
let ignore = DecodeOptions::new().bom(Bom::Ignore);
let bytes = "caf\u{E9}".as_bytes();
assert!(matches!(
UTF_8.decode_with(bytes, ignore).0,
std::borrow::Cow::Borrowed(_)
));
assert!(matches!(
WINDOWS_1252.decode_with(b"ascii", ignore).0,
std::borrow::Cow::Borrowed(_)
));
assert!(matches!(
WINDOWS_1252.decode_with(b"caf\xE9", ignore).0,
std::borrow::Cow::Owned(_)
));
}
#[test]
fn single_byte_unmapped_bytes_become_replacements() {
assert_eq!(decode(WINDOWS_1252, b"\x81\x8D\x90"), "\u{81}\u{8D}\u{90}");
assert_eq!(decode(ISO_8859_6, b"\xA1"), "\u{FFFD}");
assert_eq!(decode(ISO_8859_6, b"\xA4"), "\u{A4}");
assert_eq!(ISO_8859_8_I.name(), "ISO-8859-8-I");
assert_eq!(decode(ISO_8859_8_I, b"\xE0"), decode(ISO_8859_8, b"\xE0"));
}
#[test]
fn gb18030_restores_bytes_from_an_incomplete_four_byte_sequence() {
assert_eq!(decode(GB18030, b"\x81\x30\x41"), "\u{FFFD}0A");
assert_eq!(decode(GB18030, b"\x81\x30\x81\x41"), "\u{FFFD}0\u{4E04}");
assert_eq!(decode(GB18030, b"\x81\x30\x81"), "\u{FFFD}");
assert_eq!(decode(GB18030, b"\x81"), "\u{FFFD}");
}
#[test]
fn gb18030_maps_0x80_to_the_euro_sign() {
assert_eq!(decode(GB18030, b"\x80"), "\u{20AC}");
assert_eq!(encode(GBK, "\u{20AC}"), b"\x80");
assert_eq!(encode(GB18030, "\u{20AC}"), b"\xA2\xE3");
}
#[test]
fn gb18030_four_byte_sequences_cover_the_rest_of_unicode() {
assert_eq!(decode(GB18030, b"\x90\x30\x81\x30"), "\u{10000}");
assert_eq!(encode(GB18030, "\u{10000}"), b"\x90\x30\x81\x30");
assert_eq!(decode(GB18030, b"\xE3\x32\x9A\x35"), "\u{10FFFF}");
assert_eq!(encode(GB18030, "\u{10FFFF}"), b"\xE3\x32\x9A\x35");
assert_eq!(decode(GB18030, b"\x84\x31\xA4\x39"), "\u{FFFF}");
assert_eq!(encode(GBK, "\u{10000}"), b"𐀀");
}
#[test]
fn gb18030_has_an_asymmetric_private_use_table() {
assert_eq!(decode(GB18030, b"\xA3\xA0"), "\u{3000}");
assert_eq!(encode(GB18030, "\u{E5E5}"), b"");
assert_eq!(encode(GB18030, "\u{E78D}"), b"\xA6\xD9");
assert_eq!(decode(GB18030, b"\xA6\xD9"), "\u{FE10}");
assert_eq!(encode(GB18030, "\u{E864}"), b"\xFE\xA0");
assert_eq!(encode(GB18030, "\u{E7C7}"), b"\x81\x35\xF4\x37");
assert_eq!(decode(GB18030, b"\x81\x35\xF4\x37"), "\u{E7C7}");
}
#[test]
fn big5_pointers_that_decode_to_two_scalar_values() {
assert_eq!(decode(BIG5, b"\x88\x62"), "\u{CA}\u{304}");
assert_eq!(decode(BIG5, b"\x88\x64"), "\u{CA}\u{30C}");
assert_eq!(decode(BIG5, b"\x88\xA3"), "\u{EA}\u{304}");
assert_eq!(decode(BIG5, b"\x88\xA5"), "\u{EA}\u{30C}");
}
#[test]
fn big5_encoder_avoids_the_hkscs_extensions() {
for (text, bytes) in [
("\u{2550}", &b"\xF9\xF9"[..]),
("\u{255E}", b"\xF9\xE9"),
("\u{2561}", b"\xF9\xEB"),
("\u{256A}", b"\xF9\xEA"),
("\u{5341}", b"\xA4\x51"),
("\u{5345}", b"\xA4\xCA"),
] {
assert_eq!(encode(BIG5, text), bytes, "{text:?}");
assert_eq!(decode(BIG5, bytes), text);
}
}
#[test]
fn big5_restores_an_ascii_trail_byte() {
assert_eq!(decode(BIG5, b"\x81\x21"), "\u{FFFD}!");
assert_eq!(decode(BIG5, b"\x81\xFF"), "\u{FFFD}");
}
#[test]
fn euc_jp_decodes_jis0212_but_never_encodes_it() {
assert_eq!(decode(EUC_JP, b"\x8F\xA2\xC2"), "\u{A1}");
assert_eq!(encode(EUC_JP, "\u{A1}"), b"¡");
assert_eq!(decode(EUC_JP, b"\x8E\xB1"), "\u{FF71}");
assert_eq!(encode(EUC_JP, "\u{FF71}"), b"\x8E\xB1");
}
#[test]
fn shift_jis_end_user_defined_characters() {
assert_eq!(decode(SHIFT_JIS, b"\xF0\x40"), "\u{E000}");
assert_eq!(decode(SHIFT_JIS, b"\xF9\xFC"), "\u{E757}");
assert_eq!(decode(SHIFT_JIS, b"\x80"), "\u{80}");
assert_eq!(decode(SHIFT_JIS, b"\xA0\x41"), "\u{FFFD}A");
}
#[test]
fn japanese_encoders_fold_the_yen_and_overline() {
assert_eq!(encode(SHIFT_JIS, "\u{A5}\u{203E}"), b"\x5C\x7E");
assert_eq!(encode(EUC_JP, "\u{A5}\u{203E}"), b"\x5C\x7E");
assert_eq!(encode(ISO_2022_JP, "\u{A5}"), b"\x1B(J\x5C\x1B(B");
}
#[test]
fn iso_2022_jp_concatenation_hazard_from_the_standard() {
let once = encode(ISO_2022_JP, "\u{A5}");
let twice: Vec<u8> = once.iter().chain(once.iter()).copied().collect();
assert_eq!(decode(ISO_2022_JP, &twice), "\u{A5}\u{FFFD}\u{A5}");
}
#[test]
fn iso_2022_jp_rejects_back_to_back_escapes() {
assert_eq!(decode(ISO_2022_JP, b"\x1B(B\x1B(B"), "\u{FFFD}");
assert_eq!(decode(ISO_2022_JP, b"\x1B(BA\x1B(B"), "A");
assert_eq!(decode(ISO_2022_JP, b"\x1B(Z"), "\u{FFFD}(Z");
assert_eq!(decode(ISO_2022_JP, b"\x1BA"), "\u{FFFD}A");
assert_eq!(decode(ISO_2022_JP, b"\x1B"), "\u{FFFD}");
}
#[test]
fn iso_2022_jp_encoder_returns_to_ascii_at_the_end() {
assert_eq!(encode(ISO_2022_JP, "\u{65E5}"), b"\x1B$B\x46\x7C\x1B(B");
assert_eq!(encode(ISO_2022_JP, "a\u{65E5}b"), b"a\x1B$B\x46\x7C\x1B(Bb");
assert_eq!(encode(ISO_2022_JP, "\x1B"), b"�");
assert_eq!(encode(ISO_2022_JP, "\x0E\x0F"), b"��");
assert_eq!(
decode(ISO_2022_JP, &encode(ISO_2022_JP, "\u{FF71}")),
"\u{30A2}"
);
}
#[test]
fn replacement_decodes_anything_to_one_error() {
assert_eq!(decode(REPLACEMENT, b""), "");
assert_eq!(decode(REPLACEMENT, b"a"), "\u{FFFD}");
assert_eq!(decode(REPLACEMENT, b"a long stretch of bytes"), "\u{FFFD}");
assert!(
!REPLACEMENT
.decode_with(b"x", DecodeOptions::new().bom(Bom::Ignore))
.2
.is_lossless()
);
}
#[test]
fn x_user_defined_maps_the_upper_half_to_private_use() {
assert_eq!(decode(X_USER_DEFINED, b"\x80\xFF"), "\u{F780}\u{F7FF}");
assert_eq!(encode(X_USER_DEFINED, "\u{F780}\u{F7FF}"), b"\x80\xFF");
assert_eq!(encode(X_USER_DEFINED, "\u{F77F}"), b"");
}
#[test]
fn unmappable_characters_become_numeric_references() {
let (bytes, _, tally) = WINDOWS_1252.encode_html_form("a\u{4E00}b");
assert_eq!(&bytes[..], b"a一b");
assert_eq!(tally.errors, 1);
assert_eq!(encode(WINDOWS_1252, "\u{10FFFF}"), b"");
let (_, _, tally) = WINDOWS_1252.encode_html_form("plain");
assert!(tally.is_lossless());
}
#[test]
fn without_replacement_reports_the_offending_character() {
let mut bytes = Vec::new();
let error = WINDOWS_1252
.new_encoder()
.encode_from_str("ab\u{4E00}", &mut bytes, true)
.unwrap_err();
assert_eq!(error.character, '\u{4E00}');
assert_eq!(error.offset, 5);
assert_eq!(bytes, b"ab");
let strict = DecodeOptions::new()
.bom(Bom::Ignore)
.malformed(Malformed::Fail);
assert_eq!(
WINDOWS_1252
.try_decode(b"caf\xE9", strict)
.map(|(t, _, _)| t),
Ok(std::borrow::Cow::Owned(String::from("caf\u{E9}")))
);
assert!(ISO_8859_6.try_decode(b"\xA1", strict).is_err());
}
#[test]
fn a_four_byte_output_buffer_is_enough_to_make_progress() {
let mut decoder = BIG5.new_decoder_with(DecodeOptions::new().bom(Bom::Ignore));
let mut buffer = [0u8; DECODER_MIN_BUFFER];
let (result, read, written) = decoder.decode_to_utf8(b"\x88\x62\x41", &mut buffer, true);
assert_eq!(result, DecoderResult::OutputFull);
assert_eq!((read, written), (2, 4));
assert_eq!(&buffer[..4], "\u{CA}\u{304}".as_bytes());
let mut encoder = GB18030.new_encoder();
let mut buffer = [0u8; ENCODER_MIN_BUFFER];
let (result, _, written) = encoder.encode_from_utf8("\u{10000}\u{10001}", &mut buffer, true);
assert_eq!(result, EncoderResult::OutputFull);
assert_eq!(&buffer[..written], b"\x900\x810");
}
#[test]
fn encodings_compare_and_print_by_name() {
assert_eq!(UTF_8, UTF_8);
assert_ne!(UTF_8, WINDOWS_1252);
assert_eq!(format!("{UTF_8}"), "UTF-8");
assert_eq!(format!("{UTF_8:?}"), "Encoding(UTF-8)");
assert!(WINDOWS_1252.is_single_byte());
assert!(X_USER_DEFINED.is_single_byte());
assert!(!BIG5.is_single_byte());
assert!(WINDOWS_1252.is_ascii_compatible());
assert!(!UTF_16LE.is_ascii_compatible());
assert!(!ISO_2022_JP.is_ascii_compatible());
assert!(!REPLACEMENT.is_ascii_compatible());
}
#[test]
fn the_whatwg_lookup_refuses_everything_outside_the_standard() {
assert_eq!(Encoding::for_whatwg_label(b"latin1"), Some(WINDOWS_1252));
assert_eq!(
Encoding::for_whatwg_label(b"\tShift-JIS\n"),
Some(SHIFT_JIS)
);
assert_eq!(Encoding::for_whatwg_label(b"hz-gb-2312"), Some(REPLACEMENT));
assert_eq!(
Encoding::for_whatwg_label_no_replacement(b"hz-gb-2312"),
None
);
for &encoding in Encoding::all() {
for label in encoding.labels() {
if let Some(found) = Encoding::for_whatwg_label(label.as_bytes()) {
assert!(found.is_whatwg(), "{label} reached {}", found.name());
}
}
}
for label in [
&b"cp437"[..],
b"ibm437",
b"cp037",
b"ebcdic-cp-us",
b"x-mac-greek",
b"atari-st",
b"kz-1048",
] {
assert_eq!(Encoding::for_whatwg_label(label), None, "{label:?}");
#[cfg(feature = "extras")]
assert!(Encoding::for_label(label).is_some(), "{label:?}");
}
}
#[test]
fn is_whatwg_marks_exactly_the_standards_encodings() {
for &encoding in Encoding::all() {
let standard = matches!(
encoding.name(),
"UTF-8" | "UTF-16BE" | "UTF-16LE" | "replacement" | "x-user-defined"
) || encoding.name().starts_with("ISO-8859")
|| encoding.name().starts_with("windows-")
|| encoding.name().starts_with("KOI8")
|| matches!(
encoding.name(),
"IBM866"
| "macintosh"
| "x-mac-cyrillic"
| "GBK"
| "gb18030"
| "Big5"
| "EUC-JP"
| "ISO-2022-JP"
| "Shift_JIS"
| "EUC-KR"
);
assert_eq!(encoding.is_whatwg(), standard, "{}", encoding.name());
}
}