charcode 0.1.5

//! Decoding every pointer of every index.
//!
//! Each test reconstructs the byte sequence for a pointer from the algorithm in
//! the standard, independently of the decoder, and checks that decoding it yields
//! the code point the index holds.  That catches a mistranscribed formula, which
//! is the failure mode the generated tables cannot protect against.

use alloc::string::String;

use crate::Encoding;
use crate::encodings::*;

/// Decodes with no BOM handling and no substitution, for tests that want to see
/// exactly what a byte sequence maps to.
fn decode_strict(encoding: &'static Encoding, bytes: &[u8]) -> Option<String> {
    encoding
        .try_decode(
            bytes,
            crate::DecodeOptions::new()
                .bom(crate::Bom::Ignore)
                .malformed(crate::Malformed::Fail),
        )
        .ok()
        .map(|(text, _, _)| text.into_owned())
}

fn expect_char(encoding: &'static Encoding, bytes: &[u8], code_point: u32) {
    let expected = char::from_u32(code_point).expect("index holds scalar values");
    let decoded = decode_strict(encoding, bytes);
    assert_eq!(
        decoded.as_deref(),
        Some(&*alloc::string::String::from(expected)),
        "{} should decode {bytes:02X?} to U+{code_point:04X}",
        encoding.name()
    );
}

#[cfg(feature = "single-byte")]
#[test]
fn single_byte_indexes() {
    use crate::tables::single_byte as sb;
    let cases: [(&'static Encoding, &[u16; 128]); 28] = [
        (IBM866, &sb::IBM866_DECODE),
        (ISO_8859_2, &sb::ISO_8859_2_DECODE),
        (ISO_8859_3, &sb::ISO_8859_3_DECODE),
        (ISO_8859_4, &sb::ISO_8859_4_DECODE),
        (ISO_8859_5, &sb::ISO_8859_5_DECODE),
        (ISO_8859_6, &sb::ISO_8859_6_DECODE),
        (ISO_8859_7, &sb::ISO_8859_7_DECODE),
        (ISO_8859_8, &sb::ISO_8859_8_DECODE),
        (ISO_8859_8_I, &sb::ISO_8859_8_DECODE),
        (ISO_8859_10, &sb::ISO_8859_10_DECODE),
        (ISO_8859_13, &sb::ISO_8859_13_DECODE),
        (ISO_8859_14, &sb::ISO_8859_14_DECODE),
        (ISO_8859_15, &sb::ISO_8859_15_DECODE),
        (ISO_8859_16, &sb::ISO_8859_16_DECODE),
        (KOI8_R, &sb::KOI8_R_DECODE),
        (KOI8_U, &sb::KOI8_U_DECODE),
        (MACINTOSH, &sb::MACINTOSH_DECODE),
        (WINDOWS_874, &sb::WINDOWS_874_DECODE),
        (WINDOWS_1250, &sb::WINDOWS_1250_DECODE),
        (WINDOWS_1251, &sb::WINDOWS_1251_DECODE),
        (WINDOWS_1252, &sb::WINDOWS_1252_DECODE),
        (WINDOWS_1253, &sb::WINDOWS_1253_DECODE),
        (WINDOWS_1254, &sb::WINDOWS_1254_DECODE),
        (WINDOWS_1255, &sb::WINDOWS_1255_DECODE),
        (WINDOWS_1256, &sb::WINDOWS_1256_DECODE),
        (WINDOWS_1257, &sb::WINDOWS_1257_DECODE),
        (WINDOWS_1258, &sb::WINDOWS_1258_DECODE),
        (X_MAC_CYRILLIC, &sb::X_MAC_CYRILLIC_DECODE),
    ];
    for (encoding, table) in cases {
        for byte in 0..=0x7Fu8 {
            expect_char(encoding, &[byte], u32::from(byte));
        }
        for (pointer, &code_point) in table.iter().enumerate() {
            let byte = (pointer + 0x80) as u8;
            if code_point == 0 {
                assert_eq!(
                    decode_strict(encoding, &[byte]),
                    None,
                    "{} byte {byte:02X} is unmapped",
                    encoding.name()
                );
            } else {
                expect_char(encoding, &[byte], u32::from(code_point));
            }
        }
    }
}

#[cfg(feature = "gb18030")]
#[test]
fn gb18030_two_byte_index() {
    use crate::tables::gb18030::GB18030_DECODE;
    for (pointer, &code_point) in GB18030_DECODE.iter().enumerate() {
        let lead = pointer / 190 + 0x81;
        let trail = pointer % 190;
        let trail = trail + if trail < 0x3F { 0x40 } else { 0x41 };
        assert!((0x81..=0xFE).contains(&lead) && trail != 0x7F);
        let bytes = [lead as u8, trail as u8];
        for encoding in [GB18030, GBK] {
            if code_point == 0 {
                assert_eq!(decode_strict(encoding, &bytes), None);
            } else {
                expect_char(encoding, &bytes, u32::from(code_point));
            }
        }
    }
}

#[cfg(feature = "gb18030")]
#[test]
fn gb18030_four_byte_ranges() {
    use crate::index;
    // Every pointer the four-byte form can express, at the resolution of the
    // range table's boundaries plus a step through each range.
    for pointer in (0..=1_237_575u32).step_by(97) {
        let b1 = pointer / (10 * 126 * 10);
        let rest = pointer % (10 * 126 * 10);
        let b2 = rest / (10 * 126);
        let rest = rest % (10 * 126);
        let b3 = rest / 10;
        let b4 = rest % 10;
        let bytes = [
            (b1 + 0x81) as u8,
            (b2 + 0x30) as u8,
            (b3 + 0x81) as u8,
            (b4 + 0x30) as u8,
        ];
        match index::gb18030_ranges_code_point(pointer) {
            Some(code_point) => expect_char(GB18030, &bytes, code_point),
            None => assert_eq!(
                decode_strict(GB18030, &bytes),
                None,
                "pointer {pointer} is unmapped"
            ),
        }
        // GBK's decoder is gb18030's, so it accepts four-byte sequences too.
        assert_eq!(decode_strict(GBK, &bytes), decode_strict(GB18030, &bytes));
    }
}

/// GB 2312-80 is index gb18030 narrowed to its own bytes and corrected at the
/// two pointers where the two charsets disagree.
#[cfg(feature = "gb18030")]
#[test]
fn gb2312_is_a_corrected_narrowing_of_gb18030() {
    use crate::GB2312;
    use crate::index;
    use crate::tables::gb2312::GB2312_DECODE_DELTA;
    use crate::tables::gb18030::GB18030_DECODE;

    let mut mapped = 0;
    for lead in 0x81..=0xFEu8 {
        for trail in 0x40..=0xFEu8 {
            let bytes = [lead, trail];
            let inside = (0xA1..=0xF7).contains(&lead) && (0xA1..=0xFE).contains(&trail);
            let decoded = decode_strict(GB2312, &bytes);
            if !inside {
                assert_eq!(decoded, None, "GB2312 should refuse {bytes:02X?}");
                continue;
            }
            let pointer = (usize::from(lead) - 0x81) * 190 + (usize::from(trail) - 0x41);
            // 0xFFFF in the delta means GB 2312 leaves the pointer unassigned
            // where GBK fills it in.
            let expected = match GB2312_DECODE_DELTA
                .binary_search_by_key(&(pointer as u16), |&(p, _)| p)
                .ok()
                .map(|i| GB2312_DECODE_DELTA[i].1)
            {
                Some(0xFFFF) => None,
                Some(code_point) => Some(u32::from(code_point)),
                None => index::code_point(&GB18030_DECODE, pointer),
            };
            match expected {
                Some(code_point) => {
                    mapped += 1;
                    expect_char(GB2312, &bytes, code_point);
                    // And it round-trips.
                    let c = char::from_u32(code_point).unwrap();
                    let mut out = alloc::vec::Vec::new();
                    GB2312
                        .new_encoder()
                        .encode_from_str(&alloc::string::String::from(c), &mut out, true)
                        .unwrap_or_else(|e| panic!("GB2312 cannot encode {e}"));
                    assert_eq!(out, bytes, "U+{code_point:04X}");
                }
                None => assert_eq!(decoded, None, "{bytes:02X?}"),
            }
        }
    }
    assert_eq!(mapped, 7445, "GB 2312-80 has 7445 code points");

    // The two corrections, spelled out.
    expect_char(GB2312, b"\xA1\xA4", 0x30FB);
    expect_char(GB2312, b"\xA1\xAA", 0x2015);
    expect_char(GBK, b"\xA1\xA4", 0x00B7);
    expect_char(GBK, b"\xA1\xAA", 0x2014);

    // GBK's four-byte form and its low lead bytes are not GB 2312's.
    assert_eq!(decode_strict(GB2312, b"\x84\x31\xA4\x39"), None);
    assert_eq!(decode_strict(GB2312, b"\x80"), None);
}

#[cfg(feature = "big5")]
#[test]
fn big5_index() {
    use crate::tables::big5::BIG5_DECODE;
    for (pointer, &code_point) in BIG5_DECODE.iter().enumerate() {
        let lead = pointer / 157 + 0x81;
        let trail = pointer % 157;
        let trail = trail + if trail < 0x3F { 0x40 } else { 0x62 };
        if lead > 0xFE {
            continue;
        }
        let bytes = [lead as u8, trail as u8];
        let pair = match pointer {
            1133 => Some("\u{00CA}\u{0304}"),
            1135 => Some("\u{00CA}\u{030C}"),
            1164 => Some("\u{00EA}\u{0304}"),
            1166 => Some("\u{00EA}\u{030C}"),
            _ => None,
        };
        if let Some(pair) = pair {
            assert_eq!(decode_strict(BIG5_HKSCS, &bytes).as_deref(), Some(pair));
        } else if code_point == 0 {
            assert_eq!(decode_strict(BIG5_HKSCS, &bytes), None, "pointer {pointer}");
        } else {
            expect_char(BIG5_HKSCS, &bytes, code_point);
        }
    }
}

#[cfg(feature = "euc-kr")]
#[test]
fn euc_kr_index() {
    use crate::tables::euc_kr::EUC_KR_DECODE;
    for (pointer, &code_point) in EUC_KR_DECODE.iter().enumerate() {
        let lead = pointer / 190 + 0x81;
        let trail = pointer % 190 + 0x41;
        let bytes = [lead as u8, trail as u8];
        if code_point == 0 {
            assert_eq!(decode_strict(EUC_KR, &bytes), None, "pointer {pointer}");
        } else {
            expect_char(EUC_KR, &bytes, u32::from(code_point));
        }
    }
}

#[cfg(feature = "euc-jp")]
#[test]
fn euc_jp_indexes() {
    use crate::index;
    use crate::tables::jis::{JIS0208_DECODE, JIS0212_DECODE};
    for pointer in 0..8836usize {
        let bytes = [(pointer / 94 + 0xA1) as u8, (pointer % 94 + 0xA1) as u8];
        match index::code_point(&JIS0208_DECODE, pointer) {
            Some(code_point) => expect_char(X_WHATWG_EUC_JP, &bytes, code_point),
            None => assert_eq!(
                decode_strict(X_WHATWG_EUC_JP, &bytes),
                None,
                "jis0208 {pointer}"
            ),
        }
        let bytes = [0x8F, bytes[0], bytes[1]];
        match index::code_point(&JIS0212_DECODE, pointer) {
            Some(code_point) => expect_char(X_WHATWG_EUC_JP, &bytes, code_point),
            None => assert_eq!(
                decode_strict(X_WHATWG_EUC_JP, &bytes),
                None,
                "jis0212 {pointer}"
            ),
        }
    }
    // Half-width katakana through the 0x8E prefix.
    for byte in 0xA1..=0xDFu8 {
        expect_char(
            X_WHATWG_EUC_JP,
            &[0x8E, byte],
            0xFF61 - 0xA1 + u32::from(byte),
        );
    }
}

#[cfg(feature = "shift-jis")]
#[test]
fn shift_jis_index() {
    use crate::index;
    use crate::tables::jis::JIS0208_DECODE;
    for pointer in 0..JIS0208_DECODE.len() {
        let lead = pointer / 188;
        let lead_offset = if lead < 0x1F { 0x81 } else { 0xC1 };
        let trail = pointer % 188;
        let offset = if trail < 0x3F { 0x40 } else { 0x41 };
        let (lead, trail) = ((lead + lead_offset) as u8, (trail + offset) as u8);
        let addressable = ((0x81..=0x9F).contains(&lead) || (0xE0..=0xFC).contains(&lead))
            && ((0x40..=0x7E).contains(&trail) || (0x80..=0xFC).contains(&trail));
        if !addressable {
            continue;
        }
        let bytes = [lead, trail];
        if (8836..=10715).contains(&pointer) {
            // The Windows end-user defined characters, which no index covers.
            expect_char(WINDOWS_31J, &bytes, 0xE000 - 8836 + pointer as u32);
        } else {
            match index::code_point(&JIS0208_DECODE, pointer) {
                Some(code_point) => expect_char(WINDOWS_31J, &bytes, code_point),
                None => assert_eq!(
                    decode_strict(WINDOWS_31J, &bytes),
                    None,
                    "pointer {pointer}"
                ),
            }
        }
    }
    for byte in 0xA1..=0xDFu8 {
        expect_char(WINDOWS_31J, &[byte], 0xFF61 - 0xA1 + u32::from(byte));
    }
    expect_char(WINDOWS_31J, &[0x80], 0x80);
}

#[cfg(feature = "iso-2022-jp")]
#[test]
fn iso_2022_jp_index() {
    use crate::index;
    use crate::tables::jis::JIS0208_DECODE;
    for pointer in 0..8836usize {
        let bytes = [
            0x1B,
            0x24,
            0x42,
            (pointer / 94 + 0x21) as u8,
            (pointer % 94 + 0x21) as u8,
        ];
        match index::code_point(&JIS0208_DECODE, pointer) {
            Some(code_point) => expect_char(X_WHATWG_ISO_2022_JP, &bytes, code_point),
            None => assert_eq!(
                decode_strict(X_WHATWG_ISO_2022_JP, &bytes),
                None,
                "pointer {pointer}"
            ),
        }
    }
    // The half-width katakana escape.
    for byte in 0x21..=0x5Fu8 {
        expect_char(
            X_WHATWG_ISO_2022_JP,
            &[0x1B, 0x28, 0x49, byte],
            0xFF61 - 0x21 + u32::from(byte),
        );
    }
    // The Roman escape remaps two ASCII bytes.
    expect_char(X_WHATWG_ISO_2022_JP, &[0x1B, 0x28, 0x4A, 0x5C], 0x00A5);
    expect_char(X_WHATWG_ISO_2022_JP, &[0x1B, 0x28, 0x4A, 0x7E], 0x203E);
}

#[test]
fn x_user_defined_index() {
    for byte in 0..=0xFFu8 {
        let expected = if byte.is_ascii() {
            u32::from(byte)
        } else {
            0xF780 + u32::from(byte) - 0x80
        };
        expect_char(X_USER_DEFINED, &[byte], expected);
    }
}

#[test]
fn utf_16_decodes_both_orders() {
    expect_char(UTF_16LE, &[0x41, 0x00], 0x41);
    expect_char(UTF_16BE, &[0x00, 0x41], 0x41);
    // A surrogate pair for U+1F600.
    expect_char(UTF_16LE, &[0x3D, 0xD8, 0x00, 0xDE], 0x1F600);
    expect_char(UTF_16BE, &[0xD8, 0x3D, 0xDE, 0x00], 0x1F600);
    // Unpaired surrogates are errors in both orders.
    assert_eq!(decode_strict(UTF_16LE, &[0x00, 0xD8]), None);
    assert_eq!(decode_strict(UTF_16BE, &[0xDC, 0x00]), None);
}

#[cfg(feature = "whatwg")]
#[test]
fn every_label_resolves() {
    use crate::tables::labels::LABELS;
    // The extras add to this, so the exact count only holds without them.
    #[cfg(not(any(feature = "dos", feature = "ebcdic", feature = "mac", feature = "misc")))]
    assert_eq!(LABELS.len(), 228);
    assert!(LABELS.len() >= 228);
    for entry in LABELS.iter() {
        let (label, expected) = (entry.text, entry.encoding);
        assert_eq!(
            Encoding::for_label(label.as_bytes()),
            Some(expected),
            "label {label:?}"
        );
        // The lookup is case-insensitive and trims ASCII whitespace.
        let upper = label.to_ascii_uppercase();
        assert_eq!(Encoding::for_label(upper.as_bytes()), Some(expected));
        let padded = alloc::format!("\t {label} \r\n");
        assert_eq!(Encoding::for_label(padded.as_bytes()), Some(expected));
    }
    // The table has to stay sorted for the binary search to be correct.
    assert!(LABELS.windows(2).all(|w| w[0].text < w[1].text));
}

#[cfg(feature = "gb18030")]
#[test]
fn gb18030_encoder_side_table() {
    // The eighteen private use code points the encoder maps asymmetrically.
    for (scalar, bytes) in [
        (0xE78Du32, [0xA6u8, 0xD9u8]),
        (0xE796, [0xA6, 0xF3]),
        (0xE81E, [0xFE, 0x59]),
        (0xE864, [0xFE, 0xA0]),
    ] {
        let c = char::from_u32(scalar).unwrap();
        let mut out = alloc::vec::Vec::new();
        GB18030
            .new_encoder()
            .encode_from_str(&alloc::string::String::from(c), &mut out, true)
            .expect("side table entries are encodable");
        assert_eq!(out, bytes, "U+{scalar:04X}");
    }
}