toklen 0.2.0

/// GPT-2 byte-to-unicode mapping table.
///
/// Each byte value (0–255) maps to a visible Unicode character.
/// Printable ASCII bytes (0x21–0x7E, plus 0xA1–0xAC, 0xAE–0xFF) stay as-is;
/// control characters and non-printable bytes are remapped to codepoints
/// starting from U+0100.
#[cfg(any(feature = "bpe", feature = "byte_level"))]
pub const BYTE_TO_CHAR: [char; 256] = {
    let mut table = ['\0'; 256];
    let mut next: u32 = 256;
    let mut i: usize = 0;

    while i < 256 {
        let b = i as u8;
        let nice = (b >= b'!' && b <= b'~') || (b >= 0xA1 && b <= 0xAC) || b >= 0xAE;
        let cp = if nice {
            i as u32
        } else {
            let cp = next;
            next += 1;
            cp
        };
        // SAFETY: all codepoints generated are valid Unicode scalars
        // (0x21–0x7E, 0xA1–0xAC, 0xAE–0xFF, 0x100–0x155).
        table[i] = char::from_u32(cp).expect("valid codepoint");
        i += 1;
    }

    table
};

/// Pre-computed UTF-8 bytes per byte-to-char entry ([byte0, byte1]).
#[cfg(feature = "byte_level")]
pub const BYTE_TO_UTF8: [[u8; 2]; 256] = {
    let mut table = [[0u8; 2]; 256];
    let mut i: usize = 0;

    while i < 256 {
        let cp = BYTE_TO_CHAR[i] as u32;
        if cp < 0x80 {
            table[i] = [cp as u8, 0];
        } else {
            table[i] = [(0xC0 | (cp >> 6)) as u8, (0x80 | (cp & 0x3F)) as u8];
        }
        i += 1;
    }

    table
};

/// Pre-computed UTF-8 encoding length per byte-to-char entry.
#[cfg(feature = "byte_level")]
pub const BYTE_TO_UTF8_LEN: [u8; 256] = {
    let mut table = [0u8; 256];
    let mut i: usize = 0;

    while i < 256 {
        let cp = BYTE_TO_CHAR[i] as u32;
        table[i] = if cp < 0x80 { 1 } else { 2 };
        i += 1;
    }

    table
};