extern crate yoin;
#[test]
fn tokenize_works() {
let input = "すもももももももものうち";
let tokenizer = yoin::ipadic::tokenizer();
let expected_tokens =
vec![("すもも",
vec!["名詞", "一般", "*", "*", "*", "*", "すもも", "スモモ", "スモモ"]),
("も", vec!["助詞", "係助詞", "*", "*", "*", "*", "も", "モ", "モ"]),
("もも",
vec!["名詞", "一般", "*", "*", "*", "*", "もも", "モモ", "モモ"]),
("も", vec!["助詞", "係助詞", "*", "*", "*", "*", "も", "モ", "モ"]),
("もも",
vec!["名詞", "一般", "*", "*", "*", "*", "もも", "モモ", "モモ"]),
("の", vec!["助詞", "連体化", "*", "*", "*", "*", "の", "ノ", "ノ"]),
("うち",
vec!["名詞",
"非自立",
"副詞可能",
"*",
"*",
"*",
"うち",
"ウチ",
"ウチ"])];
let tokens = tokenizer.tokenize(input);
assert_eq!(tokens.len(), expected_tokens.len());
for (t, (expected_surface, expected_features)) in tokens.into_iter().zip(expected_tokens) {
assert_eq!(t.surface(), expected_surface);
assert_eq!(t.features().collect::<Vec<_>>(), expected_features);
}
}