icu_segmenter 2.3.0

// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).

use icu_properties::PropertyNamesLong;
use icu_segmenter::*;

struct TestContentIterator<LineIterator>(LineIterator);

struct TestData {
    original_line: String,
    chars: Vec<char>,
    break_result_utf8: Vec<usize>,
    break_result_utf16: Vec<usize>,
    break_result_latin1: Option<Vec<usize>>,
}

impl TestContentIterator<core::str::Split<'static, char>> {
    pub fn new(file: &'static str) -> Self {
        Self(file.split('\n'))
    }
}

impl<LineIterator: Iterator> Iterator for TestContentIterator<LineIterator>
where
    LineIterator::Item: Into<String>,
{
    type Item = TestData;

    fn next(&mut self) -> Option<Self::Item> {
        loop {
            let line: String = self.0.next()?.into();
            if line.is_empty() {
                // EOF
                return None;
            }
            if line.starts_with('#') {
                // Comment
                continue;
            }

            let mut r = line.split('#');
            let r = r.next();
            let v = r.unwrap().split_ascii_whitespace();
            let mut chars = Vec::new();
            let mut break_result_utf8 = Vec::new();
            let mut break_result_utf16 = Vec::new();
            let mut break_result_latin1 = Vec::new();

            let mut utf8_len = 0;
            let mut u16_len = 0;

            let mut ascii_only = true;
            for (count, item) in v.enumerate() {
                if count % 2 == 1 {
                    let ch = char::from_u32(u32::from_str_radix(item, 16).unwrap()).unwrap();
                    chars.push(ch);

                    utf8_len += ch.len_utf8();
                    u16_len += 1 + (ch as u32 > 0xFFFF) as usize;

                    if ch as u32 >= 0x100 {
                        ascii_only = false;
                    }
                } else if item != "\u{00d7}" {
                    assert_eq!(item, "\u{00f7}");
                    break_result_utf8.push(utf8_len);
                    break_result_utf16.push(u16_len);
                    break_result_latin1.push(chars.len());
                }
            }
            return Some(Self::Item {
                original_line: line,
                chars,
                break_result_utf8,
                break_result_utf16,
                break_result_latin1: if ascii_only {
                    Some(break_result_latin1)
                } else {
                    None
                },
            });
        }
    }
}

// TODO: The current line breaking implementation does not fully apply LB1, so we have to pre-process the
// test data to replace complex characters with equivalent non-complex characters. We should remove this
// once LB1 is fully implemented.
fn lb1_sa_replace(c: char) -> char {
    use icu_properties::{
        CodePointMapData,
        props::{GeneralCategory, LineBreak},
    };
    match CodePointMapData::new().get(c) {
        LineBreak::ComplexContext => match (CodePointMapData::new().get(c), c.len_utf8()) {
            (GeneralCategory::NonspacingMark | GeneralCategory::SpacingMark, 2) => {
                const _: () = assert!('\u{0300}'.len_utf8() == 2);
                '\u{0300}'
            }
            (GeneralCategory::NonspacingMark | GeneralCategory::SpacingMark, 3) => {
                const _: () = assert!('\u{081C}'.len_utf8() == 3);
                '\u{081C}'
            }
            (GeneralCategory::NonspacingMark | GeneralCategory::SpacingMark, 4) => {
                const _: () = assert!('\u{101FD}'.len_utf8() == 4);
                '\u{101FD}'
            }
            (_, 2) => {
                const _: () = assert!('À'.len_utf8() == 2);
                'À'
            }
            (_, 3) => {
                const _: () = assert!('ࠀ'.len_utf8() == 3);
                'ࠀ'
            }
            (_, 4) => {
                const _: () = assert!('𐀀'.len_utf8() == 4);
                '𐀀'
            }
            _ => unreachable!(),
        },
        _ => c,
    }
}

fn line_break_test(
    file: &'static str,
    segmenter: LineSegmenterBorrowed,
    allow_lb1_violation: bool,
) {
    let test_iter = TestContentIterator::new(file);
    for (i, mut test) in test_iter.enumerate() {
        let s: String = test
            .chars
            .into_iter()
            .map(if allow_lb1_violation {
                lb1_sa_replace
            } else {
                |ch| ch
            })
            .collect();
        let iter = segmenter.segment_str(&s);
        let result: Vec<usize> = iter.collect();
        // NOTE: For consistency with ICU4C and other Segmenters, we return a breakpoint at
        // index 0, despite UAX #14 suggesting otherwise. See issue #3283.
        if test.break_result_utf8.first() != Some(&0) {
            test.break_result_utf8.insert(0, 0);
        }
        if result != test.break_result_utf8 {
            use icu_properties::{
                CodePointMapData,
                props::{EastAsianWidth, GeneralCategory, LineBreak},
            };
            let lb = CodePointMapData::<LineBreak>::new();
            let lb_name = PropertyNamesLong::<LineBreak>::new();
            let gc = CodePointMapData::<GeneralCategory>::new();
            let gc_name = PropertyNamesLong::<GeneralCategory>::new();
            let eaw = CodePointMapData::<EastAsianWidth>::new();
            let eaw_name = PropertyNamesLong::<EastAsianWidth>::new();

            let mut iter = segmenter.segment_str(&s);
            // TODO(egg): It would be really nice to have Name here.
            println!(
                "  | A | E | Code pt. | Line_Break         | General_Category   | East_Asian_Width | Literal"
            );
            for (i, c) in s.char_indices() {
                let expected_break = test.break_result_utf8.contains(&i);
                let actual_break = result.contains(&i);
                if actual_break {
                    iter.next();
                }
                println!(
                    "{}| {} | {} | {:>8} | {:>18} | {:>18} | {:>16} | {}",
                    if actual_break != expected_break {
                        "😭"
                    } else {
                        "  "
                    },
                    if actual_break { "÷" } else { "×" },
                    if expected_break { "÷" } else { "×" },
                    format!("{:04X}", c as u32),
                    lb_name
                        .get(lb.get(c))
                        .unwrap_or(&format!("{:?}", lb.get(c))),
                    gc_name
                        .get(gc.get(c))
                        .unwrap_or(&format!("{:?}", gc.get(c))),
                    eaw_name
                        .get(eaw.get(c))
                        .unwrap_or(&format!("{:?}", eaw.get(c))),
                    c
                )
            }
            println!("Test case #{i}");
            panic!()
        }

        let result: Vec<usize> = segmenter
            .segment_utf16(&s.encode_utf16().collect::<Vec<u16>>())
            .collect();
        if test.break_result_utf16.first() != Some(&0) {
            test.break_result_utf16.insert(0, 0);
        }
        assert_eq!(
            result, test.break_result_utf16,
            "UTF16: {}",
            test.original_line
        );

        // Test data is Latin-1 character only, it can run for Latin-1 segmenter test.
        if let Some(mut break_result_latin1) = test.break_result_latin1 {
            let result: Vec<usize> = segmenter
                .segment_latin1(s.chars().map(|c| c as u8).collect::<Vec<_>>().as_slice())
                .collect();
            if break_result_latin1.first() != Some(&0) {
                break_result_latin1.insert(0, 0);
            }
            assert_eq!(
                result, break_result_latin1,
                "Latin1: {}",
                test.original_line
            );
        }
    }
}

#[test]
fn run_line_break_test() {
    line_break_test(
        include_str!("testdata/LineBreakTest_15.1.txt"),
        LineSegmenter::new_for_non_complex_scripts(Default::default()),
        true,
    );
    line_break_test(
        include_str!("testdata/LineBreakTest.txt"),
        LineSegmenter::new_17_for_non_complex_scripts(Default::default()),
        true,
    );
    line_break_test(
        include_str!("testdata/LineBreakTest.txt"),
        LineSegmenter::new_neo_for_non_complex_scripts(Default::default()),
        false,
    );
}

#[test]
fn run_line_break_extra_test() {
    line_break_test(
        include_str!("testdata/LineBreakExtraTest_15.1.txt"),
        LineSegmenter::new_for_non_complex_scripts(Default::default()),
        false,
    );
    line_break_test(
        include_str!("testdata/LineBreakExtraTest.txt"),
        LineSegmenter::new_17_for_non_complex_scripts(Default::default()),
        false,
    );
    line_break_test(
        include_str!("testdata/LineBreakExtraTest.txt"),
        LineSegmenter::new_neo_for_non_complex_scripts(Default::default()),
        false,
    );
}

#[test]
fn run_line_break_random_test() {
    line_break_test(
        include_str!("testdata/LineBreakRandomTest_15.1.txt"),
        LineSegmenter::new_for_non_complex_scripts(Default::default()),
        false,
    );
    line_break_test(
        include_str!("testdata/LineBreakRandomTest.txt"),
        LineSegmenter::new_17_for_non_complex_scripts(Default::default()),
        false,
    );
}

fn word_break_test(file: &'static str, segmenter: WordSegmenterBorrowed) {
    let test_iter = TestContentIterator::new(file);
    for (i, test) in test_iter.enumerate() {
        let s: String = test.chars.into_iter().collect();
        let iter = segmenter.segment_str(&s);
        let result: Vec<usize> = iter.collect();
        if result != test.break_result_utf8 {
            use icu_properties::{CodePointMapData, props::WordBreak};
            let wb = CodePointMapData::<WordBreak>::new();
            let wb_name = PropertyNamesLong::<WordBreak>::new();
            let mut iter = segmenter.segment_str(&s);
            // TODO(egg): It would be really nice to have Name here.
            println!("  | A | E | Code pt. |     Word_Break     | State | Literal");
            for (i, c) in s.char_indices() {
                let expected_break = test.break_result_utf8.contains(&i);
                let actual_break = result.contains(&i);
                if actual_break {
                    iter.next();
                }
                println!(
                    "{}| {} | {} | {:>8} | {:>18} | {} | {}",
                    if actual_break != expected_break {
                        "😭"
                    } else {
                        "  "
                    },
                    if actual_break { "÷" } else { "×" },
                    if expected_break { "÷" } else { "×" },
                    format!("{:04X}", c as u32),
                    wb_name
                        .get(wb.get(c))
                        .unwrap_or(&format!("{:?}", wb.get(c))),
                    // Placeholder for logging the state if exposed.
                    // Not "?????" to hide from clippy.
                    "?".repeat(5),
                    c
                )
            }
            println!("Test case #{i}");
            panic!()
        }

        let result: Vec<usize> = segmenter
            .segment_utf16(&s.encode_utf16().collect::<Vec<u16>>())
            .collect();
        assert_eq!(
            result, test.break_result_utf16,
            "UTF16: {}",
            test.original_line
        );

        // Test data is Latin-1 character only, it can run for Latin-1 segmenter test.
        if let Some(break_result_latin1) = test.break_result_latin1 {
            let result: Vec<usize> = segmenter
                .segment_latin1(s.chars().map(|c| c as u8).collect::<Vec<_>>().as_slice())
                .collect();
            assert_eq!(
                result, break_result_latin1,
                "Latin1: {}",
                test.original_line
            );
        }
    }
}

#[test]
fn run_word_break_test() {
    word_break_test(
        include_str!("testdata/WordBreakTest.txt"),
        WordSegmenter::new_for_non_complex_scripts(Default::default()),
    );
    word_break_test(
        include_str!("testdata/WordBreakTest.txt"),
        WordSegmenter::new_neo_for_non_complex_scripts(Default::default()),
    );
}

#[test]
fn run_word_break_extra_test() {
    word_break_test(
        include_str!("testdata/WordBreakExtraTest.txt"),
        WordSegmenter::new_for_non_complex_scripts(Default::default()),
    );
    word_break_test(
        include_str!("testdata/WordBreakExtraTest.txt"),
        WordSegmenter::new_neo_for_non_complex_scripts(Default::default()),
    );
}

#[test]
fn run_word_break_random_test() {
    word_break_test(
        include_str!("testdata/WordBreakRandomTest.txt"),
        WordSegmenter::new_for_non_complex_scripts(Default::default()),
    );
}

fn grapheme_break_test(file: &'static str, segmenter: GraphemeClusterSegmenterBorrowed) {
    let test_iter = TestContentIterator::new(file);
    for (i, test) in test_iter.enumerate() {
        let s: String = test.chars.into_iter().collect();
        let iter = segmenter.segment_str(&s);
        let result: Vec<usize> = iter.collect();
        if result != test.break_result_utf8 {
            use icu_properties::{CodePointMapData, props::GraphemeClusterBreak};
            let gcb = CodePointMapData::<GraphemeClusterBreak>::new();
            let gcb_name = PropertyNamesLong::<GraphemeClusterBreak>::new();
            let mut iter = segmenter.segment_str(&s);
            // TODO(egg): It would be really nice to have Name here.
            println!("  | A | E | Code pt. |            GCB | State | Literal");
            for (i, c) in s.char_indices() {
                let expected_break = test.break_result_utf8.contains(&i);
                let actual_break = result.contains(&i);
                if actual_break {
                    iter.next();
                }
                println!(
                    "{}| {} | {} | {:>8} | {:>14} | {} | {}",
                    if actual_break != expected_break {
                        "😭"
                    } else {
                        "  "
                    },
                    if actual_break { "÷" } else { "×" },
                    if expected_break { "÷" } else { "×" },
                    format!("{:04X}", c as u32),
                    gcb_name
                        .get(gcb.get(c))
                        .unwrap_or(&format!("{:?}", gcb.get(c))),
                    // Placeholder for logging the state if exposed.
                    // Not "?????" to hide from clippy.
                    "?".repeat(5),
                    c
                )
            }
            println!("Test case #{i}");
            panic!()
        }

        let result: Vec<usize> = segmenter
            .segment_utf16(&s.encode_utf16().collect::<Vec<u16>>())
            .collect();
        assert_eq!(
            result, test.break_result_utf16,
            "UTF16: {}",
            test.original_line
        );

        // Test data is Latin-1 character only, it can run for Latin-1 segmenter test.
        if let Some(break_result_latin1) = test.break_result_latin1 {
            let result: Vec<usize> = segmenter
                .segment_latin1(s.chars().map(|c| c as u8).collect::<Vec<_>>().as_slice())
                .collect();
            assert_eq!(
                result, break_result_latin1,
                "Latin1: {}",
                test.original_line
            );
        }
    }
}

#[test]
fn run_grapheme_break_test() {
    grapheme_break_test(
        include_str!("testdata/GraphemeBreakTest.txt"),
        GraphemeClusterSegmenter::new(),
    );
    grapheme_break_test(
        include_str!("testdata/GraphemeBreakTest.txt"),
        GraphemeClusterSegmenter::new_neo(),
    );
}

#[test]
fn run_grapheme_break_extra_test() {
    grapheme_break_test(
        include_str!("testdata/GraphemeBreakExtraTest.txt"),
        GraphemeClusterSegmenter::new(),
    );
    grapheme_break_test(
        include_str!("testdata/GraphemeBreakExtraTest.txt"),
        GraphemeClusterSegmenter::new_neo(),
    );
}

#[test]
fn run_grapheme_break_random_test() {
    grapheme_break_test(
        include_str!("testdata/GraphemeBreakRandomTest.txt"),
        GraphemeClusterSegmenter::new(),
    );
}

fn sentence_break_test(file: &'static str, segmenter: SentenceSegmenterBorrowed) {
    let test_iter = TestContentIterator::new(file);
    for (i, test) in test_iter.enumerate() {
        let s: String = test.chars.into_iter().collect();
        let iter = segmenter.segment_str(&s);
        let result: Vec<usize> = iter.collect();
        if result != test.break_result_utf8 {
            use icu_properties::{CodePointMapData, props::SentenceBreak};
            let sb = CodePointMapData::<SentenceBreak>::new();
            let sb_name = PropertyNamesLong::<SentenceBreak>::new();
            let mut iter = segmenter.segment_str(&s);
            // TODO(egg): It would be really nice to have Name here.
            println!("  | A | E | Code pt. | Sentence_Break | State | Literal");
            for (i, c) in s.char_indices() {
                let expected_break = test.break_result_utf8.contains(&i);
                let actual_break = result.contains(&i);
                if actual_break {
                    iter.next();
                }
                println!(
                    "{}| {} | {} | {:>8} | {:>14} | {} | {}",
                    if actual_break != expected_break {
                        "😭"
                    } else {
                        "  "
                    },
                    if actual_break { "÷" } else { "×" },
                    if expected_break { "÷" } else { "×" },
                    format!("{:04X}", c as u32),
                    sb_name
                        .get(sb.get(c))
                        .unwrap_or(&format!("{:?}", sb.get(c))),
                    // Placeholder for logging the state if exposed.
                    // Not "?????" to hide from clippy.
                    "?".repeat(5),
                    c
                )
            }
            println!("Test case #{i}");
            panic!()
        }

        let result: Vec<usize> = segmenter
            .segment_utf16(&s.encode_utf16().collect::<Vec<u16>>())
            .collect();
        assert_eq!(
            result, test.break_result_utf16,
            "UTF16: {}",
            test.original_line
        );

        // Test data is Latin-1 character only, it can run for Latin-1 segmenter test.
        if let Some(break_result_latin1) = test.break_result_latin1 {
            let result: Vec<usize> = segmenter
                .segment_latin1(s.chars().map(|c| c as u8).collect::<Vec<_>>().as_slice())
                .collect();
            assert_eq!(
                result, break_result_latin1,
                "Latin1: {}",
                test.original_line
            );
        }
    }
}

#[test]
fn run_sentence_break_test() {
    sentence_break_test(
        include_str!("testdata/SentenceBreakTest.txt"),
        SentenceSegmenter::new(Default::default()),
    );
    sentence_break_test(
        include_str!("testdata/SentenceBreakTest.txt"),
        SentenceSegmenter::new_neo(Default::default()),
    );
}

#[test]
fn run_sentence_break_extra_test() {
    sentence_break_test(
        include_str!("testdata/SentenceBreakExtraTest.txt"),
        SentenceSegmenter::new(Default::default()),
    );
    sentence_break_test(
        include_str!("testdata/SentenceBreakExtraTest.txt"),
        SentenceSegmenter::new_neo(Default::default()),
    );
}

#[test]
fn run_sentence_break_random_test() {
    sentence_break_test(
        include_str!("testdata/SentenceBreakRandomTest.txt"),
        SentenceSegmenter::new(Default::default()),
    );
}