1use runmat_builtins::{
4 BuiltinCompletionPolicy, BuiltinDescriptor, BuiltinErrorDescriptor, BuiltinOutputMode,
5 BuiltinParamArity, BuiltinParamDescriptor, BuiltinParamType, BuiltinSignatureDescriptor,
6 CharArray, ResolveContext, StringArray, Type, Value,
7};
8use runmat_macros::runtime_builtin;
9
10use crate::builtins::strings::common::{char_row_to_string_slice, is_missing_string};
11use crate::builtins::strings::core::compat::scalar_text;
12use crate::builtins::strings::text_analytics::documents::{
13 tokenized_document_language, transform_tokenized_document, DocumentTokenType,
14 TOKENIZED_DOCUMENT_CLASS,
15};
16use crate::{build_runtime_error, gather_if_needed_async, make_cell_with_shape, BuiltinResult};
17
18const OUT_WORDS: [BuiltinParamDescriptor; 1] = [BuiltinParamDescriptor {
19 name: "updatedWords",
20 ty: BuiltinParamType::Any,
21 arity: BuiltinParamArity::Required,
22 default: None,
23 description: "Normalized words or tokenizedDocument object.",
24}];
25
26const IN_WORDS: [BuiltinParamDescriptor; 1] = [BuiltinParamDescriptor {
27 name: "words",
28 ty: BuiltinParamType::Any,
29 arity: BuiltinParamArity::Required,
30 default: None,
31 description: "Words to stem or lemmatize.",
32}];
33
34const IN_WORDS_REST: [BuiltinParamDescriptor; 2] = [
35 BuiltinParamDescriptor {
36 name: "words",
37 ty: BuiltinParamType::Any,
38 arity: BuiltinParamArity::Required,
39 default: None,
40 description: "Words to stem or lemmatize.",
41 },
42 BuiltinParamDescriptor {
43 name: "NameValue",
44 ty: BuiltinParamType::Any,
45 arity: BuiltinParamArity::Variadic,
46 default: None,
47 description: "Name-value options: Language and Style.",
48 },
49];
50
51const ERROR_INVALID_INPUT: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
52 code: "RM.NORMALIZEWORDS.INVALID_INPUT",
53 identifier: Some("RunMat:normalizeWords:InvalidInput"),
54 when: "Inputs are not a supported normalizeWords form.",
55 message: "normalizeWords: invalid input",
56};
57
58const ERRORS: [BuiltinErrorDescriptor; 1] = [ERROR_INVALID_INPUT];
59
60pub const NORMALIZE_WORDS_DESCRIPTOR: BuiltinDescriptor = BuiltinDescriptor {
61 signatures: &[
62 BuiltinSignatureDescriptor {
63 label: "updatedDocuments = normalizeWords(documents)",
64 inputs: &IN_WORDS,
65 outputs: &OUT_WORDS,
66 },
67 BuiltinSignatureDescriptor {
68 label: "updatedWords = normalizeWords(words)",
69 inputs: &IN_WORDS,
70 outputs: &OUT_WORDS,
71 },
72 BuiltinSignatureDescriptor {
73 label: "updatedWords = normalizeWords(words, Name, Value, ...)",
74 inputs: &IN_WORDS_REST,
75 outputs: &OUT_WORDS,
76 },
77 ],
78 output_mode: BuiltinOutputMode::Fixed,
79 completion_policy: BuiltinCompletionPolicy::Public,
80 errors: &ERRORS,
81};
82
83fn any_type(_args: &[Type], _ctx: &ResolveContext) -> Type {
84 Type::Unknown
85}
86
87fn normalize_error(message: impl Into<String>) -> crate::RuntimeError {
88 let mut builder = build_runtime_error(message).with_builtin("normalizeWords");
89 if let Some(identifier) = ERROR_INVALID_INPUT.identifier {
90 builder = builder.with_identifier(identifier);
91 }
92 builder.build()
93}
94
95#[runtime_builtin(
96 name = "normalizeWords",
97 category = "strings/text_analytics",
98 summary = "Stem or lemmatize standalone word arrays.",
99 keywords = "normalizeWords,stem,lemma,text analytics,words",
100 accel = "sink",
101 type_resolver(any_type),
102 descriptor(crate::builtins::strings::text_analytics::normalize::NORMALIZE_WORDS_DESCRIPTOR),
103 builtin_path = "crate::builtins::strings::text_analytics::normalize"
104)]
105async fn normalize_words_builtin(args: Vec<Value>) -> BuiltinResult<Value> {
106 let (words, options) = parse_args(args).await?;
107 normalize_words_value(words, options)
108}
109
110async fn parse_args(args: Vec<Value>) -> BuiltinResult<(Value, NormalizeOptions)> {
111 if args.is_empty() {
112 return Err(normalize_error(
113 "normalizeWords: expected words input and optional name-value pairs",
114 ));
115 }
116 if !(args.len() - 1).is_multiple_of(2) {
117 return Err(normalize_error(
118 "normalizeWords: name-value options must appear in pairs",
119 ));
120 }
121
122 let words = gather_if_needed_async(&args[0]).await.map_err(|err| {
123 normalize_error(format!(
124 "normalizeWords: failed to gather words input: {err}"
125 ))
126 })?;
127 let mut options = NormalizeOptions::default();
128 let mut idx = 1;
129 while idx < args.len() {
130 let name = gather_if_needed_async(&args[idx]).await.map_err(|err| {
131 normalize_error(format!(
132 "normalizeWords: failed to gather option name: {err}"
133 ))
134 })?;
135 let value = gather_if_needed_async(&args[idx + 1])
136 .await
137 .map_err(|err| {
138 normalize_error(format!(
139 "normalizeWords: failed to gather option value: {err}"
140 ))
141 })?;
142 let name = scalar_text(&name, "normalizeWords")
143 .map_err(|err| normalize_error(err.to_string()))?
144 .to_ascii_lowercase();
145 match name.as_str() {
146 "language" => {
147 let value = scalar_text(&value, "normalizeWords")
148 .map_err(|err| normalize_error(err.to_string()))?;
149 options.language = Language::parse(&value)?;
150 options.language_explicit = true;
151 }
152 "style" => {
153 let value = scalar_text(&value, "normalizeWords")
154 .map_err(|err| normalize_error(err.to_string()))?;
155 options.style = Style::parse(&value)?
156 }
157 _ => {
158 return Err(normalize_error(format!(
159 "normalizeWords: unsupported option '{name}'"
160 )));
161 }
162 }
163 idx += 2;
164 }
165 Ok((words, options))
166}
167
168#[derive(Clone, Copy, Debug, PartialEq, Eq)]
169struct NormalizeOptions {
170 language: Language,
171 language_explicit: bool,
172 style: Style,
173}
174
175impl Default for NormalizeOptions {
176 fn default() -> Self {
177 Self {
178 language: Language::English,
179 language_explicit: false,
180 style: Style::Stem,
181 }
182 }
183}
184
185impl NormalizeOptions {
186 fn validate_standalone(self) -> BuiltinResult<()> {
187 if self.style == Style::Lemma && self.language == Language::German {
188 return Err(normalize_error(
189 "normalizeWords: Style 'lemma' for standalone words supports English only; use tokenizedDocument for Japanese or Korean lemmatization",
190 ));
191 }
192 Ok(())
193 }
194}
195
196#[derive(Clone, Copy, Debug, PartialEq, Eq)]
197enum Language {
198 English,
199 German,
200}
201
202impl Language {
203 fn parse(value: &str) -> BuiltinResult<Self> {
204 match value.to_ascii_lowercase().as_str() {
205 "en" => Ok(Self::English),
206 "de" => Ok(Self::German),
207 "ja" | "ko" => Err(normalize_error(
208 "normalizeWords: standalone word input supports Language 'en' or 'de'; use tokenizedDocument for Japanese or Korean",
209 )),
210 other => Err(normalize_error(format!(
211 "normalizeWords: Language must be 'en' or 'de', got '{other}'"
212 ))),
213 }
214 }
215
216 fn parse_document_language(value: &str) -> BuiltinResult<Self> {
217 match value.to_ascii_lowercase().as_str() {
218 "en" => Ok(Self::English),
219 "de" => Ok(Self::German),
220 "ja" | "ko" => Err(normalize_error(
221 "normalizeWords: Japanese and Korean tokenizedDocument normalization requires MeCab-compatible token details and remains tracked",
222 )),
223 other => Err(normalize_error(format!(
224 "normalizeWords: unsupported tokenizedDocument language '{other}'"
225 ))),
226 }
227 }
228}
229
230#[derive(Clone, Copy, Debug, PartialEq, Eq)]
231enum Style {
232 Stem,
233 Lemma,
234}
235
236impl Style {
237 fn parse(value: &str) -> BuiltinResult<Self> {
238 match value.to_ascii_lowercase().as_str() {
239 "stem" => Ok(Self::Stem),
240 "lemma" => Ok(Self::Lemma),
241 other => Err(normalize_error(format!(
242 "normalizeWords: Style must be 'stem' or 'lemma', got '{other}'"
243 ))),
244 }
245 }
246}
247
248fn normalize_words_value(value: Value, options: NormalizeOptions) -> BuiltinResult<Value> {
249 match value {
250 Value::String(text) => {
251 options.validate_standalone()?;
252 Ok(Value::String(normalize_word_or_missing(&text, options)?))
253 }
254 Value::StringArray(array) => {
255 options.validate_standalone()?;
256 let data = array
257 .data
258 .iter()
259 .map(|text| normalize_word_or_missing(text, options))
260 .collect::<BuiltinResult<Vec<_>>>()?;
261 StringArray::new(data, array.shape)
262 .map(Value::StringArray)
263 .map_err(|err| normalize_error(format!("normalizeWords: {err}")))
264 }
265 Value::CharArray(array) => {
266 options.validate_standalone()?;
267 let rows = (0..array.rows)
268 .map(|row| {
269 normalize_word_or_missing(
270 &char_row_to_string_slice(&array.data, array.cols, row),
271 options,
272 )
273 })
274 .collect::<BuiltinResult<Vec<_>>>()?;
275 char_rows(rows)
276 }
277 Value::Cell(cell) => {
278 options.validate_standalone()?;
279 let shape = cell.shape.clone();
280 let data = cell
281 .data
282 .into_iter()
283 .map(|item| normalize_cell_item(item, options))
284 .collect::<BuiltinResult<Vec<_>>>()?;
285 make_cell_with_shape(data, shape).map_err(|err| normalize_error(err.to_string()))
286 }
287 Value::Object(object) if object.is_class(TOKENIZED_DOCUMENT_CLASS) => {
288 normalize_tokenized_document(&object, options)
289 }
290 Value::Object(object) => Err(normalize_error(format!(
291 "normalizeWords: expected tokenizedDocument object, got {}",
292 object.class_name
293 ))),
294 other => Err(normalize_error(format!(
295 "normalizeWords: expected string scalar/array, character vector/array, or cell text input, got {other:?}"
296 ))),
297 }
298}
299
300fn normalize_tokenized_document(
301 object: &runmat_builtins::ObjectInstance,
302 mut options: NormalizeOptions,
303) -> BuiltinResult<Value> {
304 if options.language_explicit {
305 return Err(normalize_error(
306 "normalizeWords: tokenizedDocument input uses document Language metadata; the Language option is only supported for standalone words",
307 ));
308 }
309 options.language = Language::parse_document_language(&tokenized_document_language(object))?;
310 transform_tokenized_document(object, "normalizeWords", |token, token_type| {
311 if matches!(
312 token_type,
313 DocumentTokenType::Letters | DocumentTokenType::Other
314 ) {
315 Ok(Some(normalize_word(token, options)))
316 } else {
317 Ok(Some(token.to_string()))
318 }
319 })
320}
321
322fn normalize_cell_item(value: Value, options: NormalizeOptions) -> BuiltinResult<Value> {
323 match value {
324 Value::String(text) => Ok(Value::String(normalize_word_or_missing(&text, options)?)),
325 Value::StringArray(array) if array.data.len() == 1 => Ok(Value::StringArray(
326 StringArray::new(
327 vec![normalize_word_or_missing(&array.data[0], options)?],
328 array.shape,
329 )
330 .map_err(|err| normalize_error(format!("normalizeWords: {err}")))?,
331 )),
332 Value::CharArray(array) if array.rows <= 1 => {
333 let text = if array.rows == 0 {
334 String::new()
335 } else {
336 char_row_to_string_slice(&array.data, array.cols, 0)
337 };
338 Ok(Value::CharArray(CharArray::new_row(&normalize_word_or_missing(
339 &text, options,
340 )?)))
341 }
342 other => Err(normalize_error(format!(
343 "normalizeWords: cell elements must be string scalars or character vectors, got {other:?}"
344 ))),
345 }
346}
347
348fn normalize_word_or_missing(text: &str, options: NormalizeOptions) -> BuiltinResult<String> {
349 if is_missing_string(text) {
350 return Ok(text.to_string());
351 }
352 let trimmed = text.trim();
353 if trimmed.split_whitespace().nth(1).is_some() {
354 return Err(normalize_error(format!(
355 "normalizeWords: each input element must contain a single word, got '{trimmed}'"
356 )));
357 }
358 Ok(normalize_word(trimmed, options))
359}
360
361fn normalize_word(word: &str, options: NormalizeOptions) -> String {
362 if !word.chars().any(|ch| ch.is_alphabetic()) {
363 return word.to_string();
364 }
365 match (options.language, options.style) {
366 (Language::English, Style::Stem) => porter_stem(&word.to_ascii_lowercase()),
367 (Language::English, Style::Lemma) => english_lemma(&word.to_ascii_lowercase()),
368 (Language::German, Style::Stem) => german_stem(&word.to_lowercase()),
369 (Language::German, Style::Lemma) => word.to_string(),
370 }
371}
372
373fn char_rows(rows: Vec<String>) -> BuiltinResult<Value> {
374 let row_count = rows.len();
375 let cols = rows
376 .iter()
377 .map(|row| row.chars().count())
378 .max()
379 .unwrap_or(0);
380 let mut data = Vec::with_capacity(row_count * cols);
381 for row in rows {
382 let mut chars = row.chars().collect::<Vec<_>>();
383 chars.resize(cols, ' ');
384 data.extend(chars);
385 }
386 CharArray::new(data, row_count, cols)
387 .map(Value::CharArray)
388 .map_err(|err| normalize_error(format!("normalizeWords: {err}")))
389}
390
391pub(in crate::builtins::strings::text_analytics) fn english_lemma(word: &str) -> String {
392 match word {
393 "am" | "are" | "is" | "was" | "were" | "be" | "been" | "being" => "be".to_string(),
394 "has" | "had" | "having" => "have".to_string(),
395 "does" | "did" | "done" | "doing" => "do".to_string(),
396 "ran" => "run".to_string(),
397 "children" => "child".to_string(),
398 "men" => "man".to_string(),
399 "women" => "woman".to_string(),
400 "mice" => "mouse".to_string(),
401 "geese" => "goose".to_string(),
402 "teeth" => "tooth".to_string(),
403 "feet" => "foot".to_string(),
404 "better" | "best" => "good".to_string(),
405 "worse" | "worst" => "bad".to_string(),
406 _ => english_lemma_rules(word),
407 }
408}
409
410fn english_lemma_rules(word: &str) -> String {
411 if word.len() <= 3 {
412 return word.to_string();
413 }
414 if let Some(base) = word.strip_suffix("ies") {
415 if base.len() > 1 {
416 return format!("{base}y");
417 }
418 }
419 if let Some(base) = word.strip_suffix("ves") {
420 if base.len() > 1 {
421 return format!("{base}f");
422 }
423 }
424 if let Some(base) = word.strip_suffix("ing") {
425 if base.len() >= 3 {
426 return undouble_final_consonant(base);
427 }
428 }
429 if let Some(base) = word.strip_suffix("ed") {
430 if base.len() >= 3 {
431 return undouble_final_consonant(base);
432 }
433 }
434 if let Some(base) = word.strip_suffix("es") {
435 if base.len() >= 3 {
436 return base.to_string();
437 }
438 }
439 if let Some(base) = word.strip_suffix('s') {
440 if base.len() >= 3 && !base.ends_with('s') {
441 return base.to_string();
442 }
443 }
444 word.to_string()
445}
446
447fn undouble_final_consonant(text: &str) -> String {
448 let mut chars = text.chars().collect::<Vec<_>>();
449 if chars.len() >= 2 {
450 let last = chars[chars.len() - 1];
451 let prev = chars[chars.len() - 2];
452 if last == prev && is_consonant_char(last) && !matches!(last, 's' | 'z' | 'l') {
453 chars.pop();
454 }
455 }
456 chars.into_iter().collect()
457}
458
459fn german_stem(word: &str) -> String {
460 let mut stem = word
461 .replace('ä', "a")
462 .replace('ö', "o")
463 .replace('ü', "u")
464 .replace('ß', "ss");
465 if stem.len() <= 3 {
466 return stem;
467 }
468 for suffix in [
469 "heiten", "keit", "lich", "isch", "ern", "em", "er", "en", "es", "e", "s",
470 ] {
471 if stem.ends_with(suffix) && stem.len() > suffix.len() + 2 {
472 let keep = stem.len() - suffix.len();
473 stem.truncate(keep);
474 break;
475 }
476 }
477 stem
478}
479
480fn porter_stem(word: &str) -> String {
481 if word.len() <= 2 {
482 return word.to_string();
483 }
484 let mut stem = word.to_string();
485 porter_step_1a(&mut stem);
486 porter_step_1b(&mut stem);
487 porter_step_1c(&mut stem);
488 porter_step_2(&mut stem);
489 porter_step_3(&mut stem);
490 porter_step_4(&mut stem);
491 porter_step_5(&mut stem);
492 stem
493}
494
495fn porter_step_1a(stem: &mut String) {
496 if replace_suffix(stem, "sses", "ss") {
497 return;
498 }
499 if replace_suffix(stem, "ies", "i") || stem.ends_with("ss") {
500 return;
501 }
502 if stem.ends_with('s') {
503 stem.pop();
504 }
505}
506
507fn porter_step_1b(stem: &mut String) {
508 if stem.ends_with("eed") {
509 let base = &stem[..stem.len() - 3];
510 if measure(base) > 0 {
511 stem.truncate(stem.len() - 1);
512 }
513 return;
514 }
515 let mut changed = false;
516 if stem.ends_with("ed") {
517 let base = &stem[..stem.len() - 2];
518 if contains_vowel(base) {
519 stem.truncate(stem.len() - 2);
520 changed = true;
521 }
522 } else if stem.ends_with("ing") {
523 let base = &stem[..stem.len() - 3];
524 if contains_vowel(base) {
525 stem.truncate(stem.len() - 3);
526 changed = true;
527 }
528 }
529 if changed {
530 if stem.ends_with("at") || stem.ends_with("bl") || stem.ends_with("iz") {
531 stem.push('e');
532 } else if ends_double_consonant(stem)
533 && !matches!(stem.chars().last(), Some('l' | 's' | 'z'))
534 {
535 stem.pop();
536 } else if measure(stem) == 1 && cvc(stem) {
537 stem.push('e');
538 }
539 }
540}
541
542fn porter_step_1c(stem: &mut String) {
543 if stem.ends_with('y') {
544 let base = &stem[..stem.len() - 1];
545 if contains_vowel(base) {
546 stem.pop();
547 stem.push('i');
548 }
549 }
550}
551
552fn porter_step_2(stem: &mut String) {
553 for (suffix, replacement) in [
554 ("ational", "ate"),
555 ("tional", "tion"),
556 ("enci", "ence"),
557 ("anci", "ance"),
558 ("izer", "ize"),
559 ("abli", "able"),
560 ("alli", "al"),
561 ("entli", "ent"),
562 ("eli", "e"),
563 ("ousli", "ous"),
564 ("ization", "ize"),
565 ("ation", "ate"),
566 ("ator", "ate"),
567 ("alism", "al"),
568 ("iveness", "ive"),
569 ("fulness", "ful"),
570 ("ousness", "ous"),
571 ("aliti", "al"),
572 ("iviti", "ive"),
573 ("biliti", "ble"),
574 ("logi", "log"),
575 ] {
576 if replace_suffix_if_measure(stem, suffix, replacement, 0) {
577 return;
578 }
579 }
580}
581
582fn porter_step_3(stem: &mut String) {
583 for (suffix, replacement) in [
584 ("icate", "ic"),
585 ("ative", ""),
586 ("alize", "al"),
587 ("iciti", "ic"),
588 ("ical", "ic"),
589 ("ful", ""),
590 ("ness", ""),
591 ] {
592 if replace_suffix_if_measure(stem, suffix, replacement, 0) {
593 return;
594 }
595 }
596}
597
598fn porter_step_4(stem: &mut String) {
599 for suffix in [
600 "ement", "ance", "ence", "able", "ible", "ment", "ant", "ent", "ism", "ate", "iti", "ous",
601 "ive", "ize", "al", "er", "ic",
602 ] {
603 if stem.ends_with(suffix) {
604 let base = &stem[..stem.len() - suffix.len()];
605 if measure(base) > 1 {
606 stem.truncate(base.len());
607 }
608 return;
609 }
610 }
611 if stem.ends_with("ion") {
612 let base = &stem[..stem.len() - 3];
613 if measure(base) > 1 && matches!(base.chars().last(), Some('s' | 't')) {
614 stem.truncate(base.len());
615 }
616 }
617}
618
619fn porter_step_5(stem: &mut String) {
620 if stem.ends_with('e') {
621 let base = &stem[..stem.len() - 1];
622 let m = measure(base);
623 if m > 1 || (m == 1 && !cvc(base)) {
624 stem.truncate(base.len());
625 }
626 }
627 if measure(stem) > 1 && stem.ends_with("ll") {
628 stem.pop();
629 }
630}
631
632fn replace_suffix(stem: &mut String, suffix: &str, replacement: &str) -> bool {
633 if stem.ends_with(suffix) {
634 let base_len = stem.len() - suffix.len();
635 stem.truncate(base_len);
636 stem.push_str(replacement);
637 true
638 } else {
639 false
640 }
641}
642
643fn replace_suffix_if_measure(
644 stem: &mut String,
645 suffix: &str,
646 replacement: &str,
647 min_measure: usize,
648) -> bool {
649 if stem.ends_with(suffix) {
650 let base = &stem[..stem.len() - suffix.len()];
651 if measure(base) > min_measure {
652 let base_len = base.len();
653 stem.truncate(base_len);
654 stem.push_str(replacement);
655 }
656 true
657 } else {
658 false
659 }
660}
661
662fn measure(word: &str) -> usize {
663 let chars = word.chars().collect::<Vec<_>>();
664 let mut count = 0;
665 let mut prev_vowel = false;
666 for idx in 0..chars.len() {
667 let vowel = is_vowel(&chars, idx);
668 if !vowel && prev_vowel {
669 count += 1;
670 }
671 prev_vowel = vowel;
672 }
673 count
674}
675
676fn contains_vowel(word: &str) -> bool {
677 let chars = word.chars().collect::<Vec<_>>();
678 (0..chars.len()).any(|idx| is_vowel(&chars, idx))
679}
680
681fn is_vowel(chars: &[char], idx: usize) -> bool {
682 match chars[idx] {
683 'a' | 'e' | 'i' | 'o' | 'u' => true,
684 'y' => idx > 0 && !is_vowel(chars, idx - 1),
685 _ => false,
686 }
687}
688
689fn is_consonant_char(ch: char) -> bool {
690 ch.is_ascii_alphabetic() && !matches!(ch, 'a' | 'e' | 'i' | 'o' | 'u')
691}
692
693fn ends_double_consonant(word: &str) -> bool {
694 let chars = word.chars().collect::<Vec<_>>();
695 chars.len() >= 2
696 && chars[chars.len() - 1] == chars[chars.len() - 2]
697 && is_consonant_char(chars[chars.len() - 1])
698}
699
700fn cvc(word: &str) -> bool {
701 let chars = word.chars().collect::<Vec<_>>();
702 if chars.len() < 3 {
703 return false;
704 }
705 let len = chars.len();
706 !is_vowel(&chars, len - 1)
707 && is_vowel(&chars, len - 2)
708 && !is_vowel(&chars, len - 3)
709 && !matches!(chars[len - 1], 'w' | 'x' | 'y')
710}
711
712#[cfg(test)]
713mod tests {
714 use super::*;
715 use runmat_builtins::{CellArray, ObjectInstance, Tensor};
716
717 use crate::builtins::strings::text_analytics::documents::documents_from_object;
718
719 fn run(args: Vec<Value>) -> BuiltinResult<Value> {
720 futures::executor::block_on(normalize_words_builtin(args))
721 }
722
723 fn tokenized_document(documents: Vec<Vec<&str>>, language: &str) -> Value {
724 let mut object = ObjectInstance::new(TOKENIZED_DOCUMENT_CLASS.to_string());
725 let cells = documents
726 .iter()
727 .map(|doc| {
728 StringArray::new(
729 doc.iter().map(|token| (*token).to_string()).collect(),
730 vec![1, doc.len()],
731 )
732 .map(Value::StringArray)
733 .unwrap()
734 })
735 .collect::<Vec<_>>();
736 object.properties.insert(
737 "Documents".to_string(),
738 Value::Cell(CellArray::new(cells, documents.len(), 1).unwrap()),
739 );
740 object.properties.insert(
741 "Shape".to_string(),
742 Value::Tensor(Tensor::new(vec![documents.len() as f64, 1.0], vec![1, 2]).unwrap()),
743 );
744 object.properties.insert(
745 "TokenizeMethod".to_string(),
746 Value::String("unicode".to_string()),
747 );
748 object
749 .properties
750 .insert("Language".to_string(), Value::String(language.to_string()));
751 Value::Object(object)
752 }
753
754 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
755 #[test]
756 fn stems_english_string_array_and_preserves_shape() {
757 let input = StringArray::new(
758 vec![
759 "strongly".to_string(),
760 "worded".to_string(),
761 "collections".to_string(),
762 "words".to_string(),
763 ],
764 vec![2, 2],
765 )
766 .unwrap();
767 let out = run(vec![Value::StringArray(input)]).expect("normalize");
768 let Value::StringArray(array) = out else {
769 panic!("expected string array");
770 };
771 assert_eq!(array.shape, vec![2, 2]);
772 assert_eq!(array.data, vec!["strongli", "word", "collect", "word"]);
773 }
774
775 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
776 #[test]
777 fn lemmatizes_english_char_vector() {
778 let out = run(vec![
779 Value::CharArray(CharArray::new_row("running")),
780 Value::String("Style".to_string()),
781 Value::String("lemma".to_string()),
782 ])
783 .expect("normalize");
784 assert_eq!(out, Value::CharArray(CharArray::new_row("run")));
785 }
786
787 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
788 #[test]
789 fn stems_german_word_array() {
790 let input =
791 StringArray::new(vec!["Morgen".to_string(), "guter".to_string()], vec![2, 1]).unwrap();
792 let out = run(vec![
793 Value::StringArray(input),
794 Value::CharArray(CharArray::new_row("Language")),
795 Value::CharArray(CharArray::new_row("de")),
796 ])
797 .expect("normalize");
798 let Value::StringArray(array) = out else {
799 panic!("expected string array");
800 };
801 assert_eq!(array.data, vec!["morg", "gut"]);
802 }
803
804 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
805 #[test]
806 fn repads_multi_row_char_array_after_normalization() {
807 let input = CharArray::new("runningworded ".chars().collect(), 2, "running".len()).unwrap();
808 let out = run(vec![Value::CharArray(input)]).expect("normalize");
809 let Value::CharArray(array) = out else {
810 panic!("expected char array");
811 };
812 assert_eq!(array.rows, 2);
813 assert_eq!(array.cols, 4);
814 assert_eq!(
815 array.data.into_iter().collect::<String>(),
816 "run word".to_string()
817 );
818 }
819
820 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
821 #[test]
822 fn preserves_cell_element_types() {
823 let cell = CellArray::new(
824 vec![
825 Value::CharArray(CharArray::new_row("running")),
826 Value::String("children".to_string()),
827 ],
828 1,
829 2,
830 )
831 .unwrap();
832 let out = run(vec![
833 Value::Cell(cell),
834 Value::String("Style".to_string()),
835 Value::String("lemma".to_string()),
836 ])
837 .expect("normalize");
838 let Value::Cell(cell) = out else {
839 panic!("expected cell");
840 };
841 assert_eq!(cell.shape, vec![1, 2]);
842 assert_eq!(cell.data[0], Value::CharArray(CharArray::new_row("run")));
843 assert_eq!(cell.data[1], Value::String("child".to_string()));
844 }
845
846 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
847 #[test]
848 fn preserves_missing_strings() {
849 let input = StringArray::new(vec!["<missing>".to_string()], vec![1, 1]).unwrap();
850 let out = run(vec![Value::StringArray(input)]).expect("normalize");
851 assert!(matches!(out, Value::StringArray(array) if array.data == vec!["<missing>"]));
852 }
853
854 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
855 #[test]
856 fn normalizes_tokenized_documents_and_preserves_complex_tokens() {
857 let out = run(vec![tokenized_document(
858 vec![vec!["a", "strongly", "worded", ".", "https://example.com"]],
859 "en",
860 )])
861 .expect("normalize documents");
862 let Value::Object(object) = out else {
863 panic!("expected tokenizedDocument object");
864 };
865 assert_eq!(
866 documents_from_object(&object, "test").unwrap(),
867 vec![vec!["a", "strongli", "word", ".", "https://example.com",]]
868 );
869 }
870
871 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
872 #[test]
873 fn rejects_language_override_for_tokenized_documents() {
874 let err = run(vec![
875 tokenized_document(vec![vec!["word"]], "en"),
876 Value::String("Language".to_string()),
877 Value::String("de".to_string()),
878 ])
879 .expect_err("expected Language option rejection");
880 assert!(err.to_string().contains("document Language metadata"));
881 }
882
883 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
884 #[test]
885 fn rejects_unsupported_language_for_standalone_words() {
886 let err = run(vec![
887 Value::String("word".to_string()),
888 Value::String("Language".to_string()),
889 Value::String("ja".to_string()),
890 ])
891 .expect_err("expected unsupported language");
892 assert!(err.to_string().contains("tokenizedDocument"));
893 }
894
895 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
896 #[test]
897 fn rejects_german_lemma_for_standalone_words() {
898 let err = run(vec![
899 Value::String("wort".to_string()),
900 Value::String("Language".to_string()),
901 Value::String("de".to_string()),
902 Value::String("Style".to_string()),
903 Value::String("lemma".to_string()),
904 ])
905 .expect_err("expected unsupported German lemma");
906 assert!(err.to_string().contains("English only"));
907 }
908
909 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
910 #[test]
911 fn wraps_non_text_option_errors_with_normalize_identifier() {
912 let err = run(vec![
913 Value::String("word".to_string()),
914 Value::Num(1.0),
915 Value::String("stem".to_string()),
916 ])
917 .expect_err("expected option name error");
918 assert!(err.to_string().contains("normalizeWords"));
919 assert_eq!(err.identifier(), Some("RunMat:normalizeWords:InvalidInput"));
920 }
921
922 #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
923 #[test]
924 fn rejects_multi_word_elements() {
925 let err = run(vec![Value::String("two words".to_string())])
926 .expect_err("expected single word error");
927 assert!(err.to_string().contains("single word"));
928 }
929}