use super::{ScriptRun, script_runs};
use crate::WritingSystem;
fn cut(text: &str) -> Vec<(WritingSystem, usize, &str)> {
script_runs(text).map(ScriptRun::into_components).collect()
}
#[test]
fn the_worked_example_cuts_where_the_script_changes() {
assert_eq!(
cut("Rust 2024 年鉴"),
vec![
(WritingSystem::Latn, 0, "Rust 2024 "),
(WritingSystem::Hani, 10, "年鉴"),
]
);
}
#[test]
fn preceding_wins_over_following() {
assert_eq!(
cut("日本 word"),
vec![
(WritingSystem::Hani, 0, "日本 "),
(WritingSystem::Latn, 7, "word")
]
);
assert_eq!(
cut("word 日本"),
vec![
(WritingSystem::Latn, 0, "word "),
(WritingSystem::Hani, 5, "日本")
]
);
}
#[test]
fn a_leading_contextual_stretch_joins_what_follows() {
assert_eq!(cut(" 年鉴"), vec![(WritingSystem::Hani, 0, " 年鉴")]);
assert_eq!(
cut(" \t\"hello\""),
vec![(WritingSystem::Latn, 0, " \t\"hello\"")]
);
assert_eq!(
cut("«Привет» hello"),
vec![
(WritingSystem::Cyrl, 0, "«Привет» "),
(WritingSystem::Latn, 17, "hello"),
]
);
}
#[test]
fn a_wholly_contextual_text_is_one_fallback_run() {
assert_eq!(cut("123 !!!"), vec![(WritingSystem::Zzzz, 0, "123 !!!")]);
assert_eq!(cut(" "), vec![(WritingSystem::Zzzz, 0, " ")]);
assert_eq!(cut("+-*/"), vec![(WritingSystem::Zzzz, 0, "+-*/")]);
assert_eq!(cut("\u{0301}"), vec![(WritingSystem::Zzzz, 0, "\u{0301}")]);
}
#[test]
fn an_unrostered_script_is_never_swallowed() {
assert_eq!(
cut("word ᚠᚢᚦ word"),
vec![
(WritingSystem::Latn, 0, "word "),
(WritingSystem::Zzzz, 5, "ᚠᚢᚦ "),
(WritingSystem::Latn, 15, "word"),
]
);
assert_eq!(
cut("中文 Հայերեն"),
vec![
(WritingSystem::Hani, 0, "中文 "),
(WritingSystem::Armn, 7, "Հայերեն"),
]
);
}
#[test]
fn combining_marks_do_not_cut() {
let decomposed = "cafe\u{0301}";
assert_eq!(cut(decomposed), vec![(WritingSystem::Latn, 0, decomposed)]);
assert_eq!(cut("café"), vec![(WritingSystem::Latn, 0, "café")]);
}
#[test]
fn japanese_cuts_into_its_three_scripts() {
assert_eq!(
cut("日本語のテキスト"),
vec![
(WritingSystem::Hani, 0, "日本語"),
(WritingSystem::Hira, 9, "の"),
(WritingSystem::Kana, 12, "テキスト"),
]
);
}
#[test]
fn an_empty_text_has_no_runs() {
assert_eq!(cut(""), vec![]);
assert_eq!(script_runs("").count(), 0);
}
#[test]
fn the_runs_are_total_contiguous_and_non_empty() {
for text in CORPUS {
let runs: Vec<_> = script_runs(text).collect();
let mut at = 0;
for run in &runs {
assert_eq!(run.at(), at, "a gap or an overlap in {text:?}");
assert!(!run.text().is_empty(), "an empty run in {text:?}");
assert_eq!(run.text(), &text[run.at()..run.end()], "in {text:?}");
at = run.end();
}
assert_eq!(at, text.len(), "the tail of {text:?} is uncovered");
let rebuilt: String = runs.iter().map(|run| run.text()).collect();
assert_eq!(rebuilt, *text, "the runs do not rebuild {text:?}");
}
}
#[test]
fn the_same_text_cuts_the_same_way() {
for text in CORPUS {
let once: Vec<_> = script_runs(text).collect();
let twice: Vec<_> = script_runs(text).collect();
assert_eq!(once, twice, "at {text:?}");
}
}
#[test]
fn neighbouring_runs_never_share_a_script() {
for text in CORPUS {
let runs: Vec<_> = script_runs(text).collect();
for pair in runs.windows(2) {
assert_ne!(
pair[0].script(),
pair[1].script(),
"an unnecessary cut in {text:?}"
);
}
}
}
#[test]
fn a_spent_iterator_stays_spent() {
let mut runs = script_runs("hi");
assert!(runs.next().is_some());
assert!(runs.next().is_none());
assert!(runs.next().is_none());
}
#[test]
fn the_size_hint_bounds_the_answer() {
for text in CORPUS {
let (low, high) = script_runs(text).size_hint();
let produced = script_runs(text).count();
assert!(produced >= low, "at {text:?}");
assert!(
produced <= high.expect("the hint has an upper bound"),
"at {text:?}"
);
}
}
#[test]
fn a_run_states_its_own_span() {
let run = ScriptRun::new(WritingSystem::Hani, 10, "年鉴");
assert_eq!(run.script(), &WritingSystem::Hani);
assert_eq!(run.at(), 10);
assert_eq!(run.text(), "年鉴");
assert_eq!(run.end(), 16);
assert_eq!(
run.clone().into_components(),
(WritingSystem::Hani, 10, "年鉴")
);
}
#[test]
fn a_concatenation_covers_what_its_halves_did() {
for left in CORPUS {
for right in CORPUS {
let joined = format!("{left}{right}");
let apart = script_runs(left).count() + script_runs(right).count();
let together = script_runs(&joined).count();
assert!(
together <= apart,
"joining {left:?} and {right:?} invented a boundary"
);
let rebuilt: String = script_runs(&joined)
.map(|run| run.text().to_owned())
.collect();
assert_eq!(rebuilt, joined);
}
}
}
const CORPUS: &[&str] = &[
"",
" ",
"a",
"hello world",
"Rust 2024 年鉴",
"123 !!!",
"日本語のテキスト",
"中文 English 混排",
" 年鉴",
"cafe\u{0301} 中文",
"Привет, мир! Hello, world!",
"مرحبا بالعالم hello",
"שלום עולם",
"Ελληνικά κείμενο",
"हिन्दी पाठ",
"ภาษาไทย",
"한국어 텍스트",
"word ᚠᚢᚦ word",
"🎉 emoji 🎉 中文 🎉",
"\u{0301}",
"アイウ ABC",
"a\u{200D}b",
"𠀀𠀁 surrogate-free astral han",
"ភាសាខ្មែរ ພາສາລາວ ဗမာစာ བོད་སྐད།",
"বাংলা ਪੰਜਾਬੀ ગુજરાતી ଓଡ଼ିଆ தமிழ் తెలుగు ಕನ್ನಡ മലയാളം සිංහල",
"Հայերեն ქართული ܣܘܪܝܝܐ ދިވެހި ᠮᠣᠩᠭᠣᠯ ᐃᓄᒃᑎᑐᑦ አማርኛ ㄅㄆㄇ",
];