ebustl-parser 0.7.0

//! Character code table 00, Latin alphabet (Tech 3264 Appendix 2).
//!
//! The table is based on ISO 6937/2-1983 Add.1:1989 and is implemented on top of textcode's
//! ISO 6937, which is a later ISO 6937 superset with a Euro sign, with these differences:
//!
//! - 24h is the currency sign `¤` and A4h is the dollar sign `$`. textcode decodes 24h as `$`
//!   and A4h as `€`.
//! - 7Fh, A6h, A8h, C0h and C9h are unassigned. textcode decodes A8h as `¤` and some of the
//!   others as U+0000.
//! - A diacritical mark (column C) not followed by a letter is decoded as its spacing form,
//!   which is how ISO 6937 conveys a stand-alone accent (diacritical mark followed by space).
//!   textcode gives U+FFFD and also drops the following byte.
//! - A letter with a diacritical mark missing from the table is decoded as the letter followed
//!   by the Unicode combining mark, instead of U+0000.

use textcode::{Iso6937, decode as decode_iso6937, encode as encode_iso6937};

/// Decoded in place of unassigned codes
const REPLACEMENT: char = char::REPLACEMENT_CHARACTER;
/// Encoded in place of characters missing from the table, as textcode does
const ENCODE_FALLBACK: u8 = b'?';

/// The diacritical marks in column C: (code, Unicode combining mark, spacing form)
const DIACRITICS: [(u8, char, char); 14] = [
    (0xC1, '\u{0300}', '`'),
    (0xC2, '\u{0301}', '´'),
    (0xC3, '\u{0302}', '^'),
    (0xC4, '\u{0303}', '~'),
    (0xC5, '\u{0304}', '¯'),
    (0xC6, '\u{0306}', '˘'),
    (0xC7, '\u{0307}', '˙'),
    (0xC8, '\u{0308}', '¨'),
    (0xCA, '\u{030A}', '˚'),
    (0xCB, '\u{0327}', '¸'),
    (0xCC, '\u{0332}', '_'),
    (0xCD, '\u{030B}', '˝'),
    (0xCE, '\u{0328}', '˛'),
    (0xCF, '\u{030C}', 'ˇ'),
];

/// Returns the combining mark and spacing form if `code` is a diacritical mark.
fn diacritic(code: u8) -> Option<(char, char)> {
    DIACRITICS
        .iter()
        .find(|(c, _, _)| *c == code)
        .map(|&(_, mark, spacing)| (mark, spacing))
}

/// Decodes character codes (no control codes) from character code table 00.
pub(crate) fn decode(data: &[u8]) -> String {
    let mut text = String::with_capacity(data.len());
    let mut i = 0;
    while i < data.len() {
        i += decode_next(&data[i..], &mut text);
    }
    text
}

/// Decodes the first character of `data` into `text` and returns the number of bytes used.
fn decode_next(data: &[u8], text: &mut String) -> usize {
    let code = data[0];
    let Some((mark, spacing)) = diacritic(code) else {
        text.push(decode_single(code));
        return 1;
    };
    // The diacritical mark precedes the character it applies to
    match data.get(1) {
        Some(&letter) if letter.is_ascii_alphabetic() => {
            push_accented_letter(text, code, letter, mark);
            2
        }
        // A diacritical mark followed by space is the stand-alone diacritical mark
        Some(b' ') => {
            text.push(spacing);
            2
        }
        // Nothing to apply the mark to, e.g. at the end of the text
        _ => {
            text.push(spacing);
            1
        }
    }
}

fn decode_single(code: u8) -> char {
    match code {
        0x24 => '¤',
        0xA4 => '$',
        0x7F | 0xA6 | 0xA8 | 0xC0 | 0xC9 => REPLACEMENT,
        _ => decode_iso6937::<Iso6937>([code])
            .chars()
            .next()
            .filter(|&c| c != '\0')
            .unwrap_or(REPLACEMENT),
    }
}

/// Pushes the precomposed letter if there is one, otherwise the letter and combining mark.
fn push_accented_letter(text: &mut String, code: u8, letter: u8, mark: char) {
    let composed = decode_iso6937::<Iso6937>([code, letter]);
    let mut chars = composed.chars();
    match (chars.next(), chars.next()) {
        (Some(c), None) if c != '\0' && c != REPLACEMENT => text.push(c),
        _ => {
            text.push(char::from(letter));
            text.push(mark);
        }
    }
}

/// Encodes text using character code table 00. Characters missing from the table are
/// encoded as `?`.
pub(crate) fn encode(text: &str) -> Vec<u8> {
    let mut data = Vec::with_capacity(text.len());
    for c in text.chars() {
        match c {
            '$' => data.push(0xA4),
            '¤' => data.push(0x24),
            // In textcode's ISO 6937 superset only, at the position of '$' in the table
            '€' => data.push(ENCODE_FALLBACK),
            _ => data.extend(encode_iso6937::<Iso6937>(c.encode_utf8(&mut [0; 4]))),
        }
    }
    data
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn ascii_letters_digits_and_punctuation() {
        assert_eq!("Hello, World! 123 #@", decode(b"Hello, World! 123 #@"));
    }

    #[test]
    fn currency_signs() {
        assert_eq!("¤", decode(&[0x24]));
        assert_eq!("$500,000", decode(b"\xA4500,000"));
        assert_eq!("£¥", decode(&[0xA3, 0xA5]));
    }

    #[test]
    fn unassigned_codes() {
        for code in [0x7F, 0xA6, 0xA8, 0xC0, 0xC9, 0xD8, 0xE5] {
            assert_eq!("\u{FFFD}", decode(&[code]), "code {code:02X}");
        }
    }

    #[test]
    fn column_a_to_f_symbols() {
        assert_eq!("¡«←↑→↓", decode(&[0xA1, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF]));
        // D0h is HORIZONTAL BAR (U+2015)
        assert_eq!("\u{2015}®©™♪", decode(&[0xD0, 0xD2, 0xD3, 0xD4, 0xD5]));
        assert_eq!(
            // E0h is OHM SIGN (U+2126), FFh is SOFT HYPHEN (U+00AD)
            "\u{2126}ÆØŒßþ\u{AD}",
            decode(&[0xE0, 0xE1, 0xE9, 0xEA, 0xFB, 0xFC, 0xFF])
        );
    }

    #[test]
    fn accented_letters() {
        assert_eq!("Ä", decode(&[0xC8, b'A']));
        assert_eq!("ê", decode(&[0xC3, b'e']));
        assert_eq!(
            "é à ç č ő ą",
            decode(b"\xC2e \xC1a \xCBc \xCFc \xCDo \xCEa")
        );
    }

    #[test]
    fn letter_without_precomposed_form() {
        assert_eq!("x\u{0308}", decode(&[0xC8, b'x']));
        assert_eq!("a\u{0332}", decode(&[0xCC, b'a']));
    }

    #[test]
    fn diacritic_followed_by_space_is_stand_alone() {
        // The following character must not be dropped
        assert_eq!("´A", decode(&[0xC2, b' ', b'A']));
    }

    #[test]
    fn diacritic_without_letter_is_stand_alone() {
        assert_eq!("^", decode(&[0xC3]));
        assert_eq!("^)", decode(&[0xC3, b')']));
        assert_eq!("^^", decode(&[0xC3, 0xC3]));
        assert_eq!("¨1", decode(&[0xC8, b'1']));
    }

    #[test]
    fn encode_currency_signs() {
        assert_eq!(vec![0xA4, b'5'], encode("$5"));
        assert_eq!(vec![0x24], encode("¤"));
        assert_eq!(vec![b'?'], encode("€"));
    }

    #[test]
    fn encode_accented_letters() {
        assert_eq!(vec![0xC2, b'e', 0xC8, b'A'], encode("éÄ"));
    }

    #[test]
    fn encode_decode_roundtrip() {
        let text = "Är det $5 eller 5 ¤? Ça coûte ¼ £, señor!";
        assert_eq!(text, decode(&encode(text)));
    }
}