runmat_runtime/builtins/strings/text_analytics/
stopwords.rs

1//! MATLAB-compatible `stopWords` builtin.
2
3use runmat_builtins::{
4    BuiltinCompletionPolicy, BuiltinDescriptor, BuiltinErrorDescriptor, BuiltinOutputMode,
5    BuiltinParamArity, BuiltinParamDescriptor, BuiltinParamType, BuiltinSignatureDescriptor,
6    ResolveContext, StringArray, Type, Value,
7};
8use runmat_macros::runtime_builtin;
9
10use crate::builtins::strings::core::compat::scalar_text;
11use crate::{build_runtime_error, gather_if_needed_async, BuiltinResult, RuntimeError};
12
13const OUT_WORDS: [BuiltinParamDescriptor; 1] = [BuiltinParamDescriptor {
14    name: "words",
15    ty: BuiltinParamType::Any,
16    arity: BuiltinParamArity::Required,
17    default: None,
18    description: "Stop word list.",
19}];
20
21const LANGUAGE_INPUTS: [BuiltinParamDescriptor; 2] = [
22    BuiltinParamDescriptor {
23        name: "Name",
24        ty: BuiltinParamType::StringScalar,
25        arity: BuiltinParamArity::Required,
26        default: Some("Language"),
27        description: "Language option name.",
28    },
29    BuiltinParamDescriptor {
30        name: "language",
31        ty: BuiltinParamType::StringScalar,
32        arity: BuiltinParamArity::Required,
33        default: Some("en"),
34        description: "Stop word language: en, ja, de, or ko.",
35    },
36];
37
38const ERROR_INVALID_INPUT: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
39    code: "RM.STOPWORDS.INVALID_INPUT",
40    identifier: Some("RunMat:stopWords:InvalidInput"),
41    when: "Inputs are not a supported stopWords form.",
42    message: "stopWords: invalid input",
43};
44
45const ERRORS: [BuiltinErrorDescriptor; 1] = [ERROR_INVALID_INPUT];
46
47pub const STOP_WORDS_DESCRIPTOR: BuiltinDescriptor = BuiltinDescriptor {
48    signatures: &[
49        BuiltinSignatureDescriptor {
50            label: "words = stopWords",
51            inputs: &[],
52            outputs: &OUT_WORDS,
53        },
54        BuiltinSignatureDescriptor {
55            label: "words = stopWords('Language', language)",
56            inputs: &LANGUAGE_INPUTS,
57            outputs: &OUT_WORDS,
58        },
59    ],
60    output_mode: BuiltinOutputMode::Fixed,
61    completion_policy: BuiltinCompletionPolicy::Public,
62    errors: &ERRORS,
63};
64
65fn string_array_type(_args: &[Type], _ctx: &ResolveContext) -> Type {
66    Type::String
67}
68
69fn stopwords_error(message: impl Into<String>) -> RuntimeError {
70    let mut builder = build_runtime_error(message).with_builtin("stopWords");
71    if let Some(identifier) = ERROR_INVALID_INPUT.identifier {
72        builder = builder.with_identifier(identifier);
73    }
74    builder.build()
75}
76
77#[runtime_builtin(
78    name = "stopWords",
79    category = "strings/text_analytics",
80    summary = "Return common stop words for supported Text Analytics languages.",
81    keywords = "stopWords,stop words,text analytics,language,string",
82    accel = "sink",
83    type_resolver(string_array_type),
84    descriptor(crate::builtins::strings::text_analytics::stopwords::STOP_WORDS_DESCRIPTOR),
85    builtin_path = "crate::builtins::strings::text_analytics::stopwords"
86)]
87async fn stop_words_builtin(args: Vec<Value>) -> BuiltinResult<Value> {
88    let language = parse_language_args(args).await?;
89    let words = stop_words_for_language(language);
90    let strings = words.iter().map(|word| (*word).to_string()).collect();
91    StringArray::new(strings, vec![words.len(), 1])
92        .map(Value::StringArray)
93        .map_err(|err| stopwords_error(format!("stopWords: {err}")))
94}
95
96async fn parse_language_args(args: Vec<Value>) -> BuiltinResult<StopWordsLanguage> {
97    match args.len() {
98        0 => Ok(StopWordsLanguage::English),
99        2 => {
100            let name = gather_if_needed_async(&args[0]).await.map_err(|err| {
101                stopwords_error(format!("stopWords: failed to gather option name: {err}"))
102            })?;
103            let value = gather_if_needed_async(&args[1]).await.map_err(|err| {
104                stopwords_error(format!("stopWords: failed to gather language value: {err}"))
105            })?;
106            let option = scalar_text(&name, "stopWords")?.to_ascii_lowercase();
107            if option != "language" {
108                return Err(stopwords_error(format!(
109                    "stopWords: unsupported option '{option}'"
110                )));
111            }
112            parse_language(&scalar_text(&value, "stopWords")?)
113        }
114        _ => Err(stopwords_error(
115            "stopWords: expected stopWords or stopWords('Language', language)",
116        )),
117    }
118}
119
120#[derive(Clone, Copy, Debug, PartialEq, Eq)]
121pub(in crate::builtins::strings::text_analytics) enum StopWordsLanguage {
122    English,
123    Japanese,
124    German,
125    Korean,
126}
127
128fn parse_language(language: &str) -> BuiltinResult<StopWordsLanguage> {
129    match language.to_ascii_lowercase().as_str() {
130        "en" => Ok(StopWordsLanguage::English),
131        "ja" => Ok(StopWordsLanguage::Japanese),
132        "de" => Ok(StopWordsLanguage::German),
133        "ko" => Ok(StopWordsLanguage::Korean),
134        other => Err(stopwords_error(format!(
135            "stopWords: language must be 'en', 'ja', 'de', or 'ko', got '{other}'"
136        ))),
137    }
138}
139
140pub(in crate::builtins::strings::text_analytics) fn stop_words_for_language(
141    language: StopWordsLanguage,
142) -> &'static [&'static str] {
143    match language {
144        StopWordsLanguage::English => ENGLISH_STOP_WORDS,
145        StopWordsLanguage::Japanese => JAPANESE_STOP_WORDS,
146        StopWordsLanguage::German => GERMAN_STOP_WORDS,
147        StopWordsLanguage::Korean => KOREAN_STOP_WORDS,
148    }
149}
150
151const ENGLISH_STOP_WORDS: &[&str] = &[
152    "a",
153    "about",
154    "above",
155    "across",
156    "after",
157    "all",
158    "along",
159    "also",
160    "am",
161    "an",
162    "and",
163    "any",
164    "are",
165    "aren't",
166    "arent",
167    "as",
168    "at",
169    "be",
170    "because",
171    "been",
172    "before",
173    "being",
174    "between",
175    "both",
176    "but",
177    "by",
178    "can",
179    "can't",
180    "cant",
181    "cannot",
182    "could",
183    "couldn't",
184    "couldnt",
185    "did",
186    "didn't",
187    "didnt",
188    "do",
189    "does",
190    "doesn't",
191    "doesnt",
192    "doing",
193    "done",
194    "don't",
195    "dont",
196    "during",
197    "each",
198    "either",
199    "for",
200    "from",
201    "given",
202    "had",
203    "has",
204    "have",
205    "having",
206    "he",
207    "he'd",
208    "hed",
209    "he'll",
210    "hell",
211    "her",
212    "here",
213    "hers",
214    "him",
215    "himself",
216    "his",
217    "how",
218    "however",
219    "i",
220    "i'd",
221    "id",
222    "i'll",
223    "ill",
224    "i'm",
225    "im",
226    "if",
227    "in",
228    "instead",
229    "into",
230    "is",
231    "isn't",
232    "isnt",
233    "it",
234    "it'll",
235    "itll",
236    "it's",
237    "its",
238    "i've",
239    "ive",
240    "let's",
241    "lets",
242    "may",
243    "me",
244    "more",
245    "most",
246    "much",
247    "must",
248    "my",
249    "no",
250    "not",
251    "now",
252    "of",
253    "on",
254    "one",
255    "only",
256    "or",
257    "other",
258    "our",
259    "out",
260    "over",
261    "said",
262    "says",
263    "see",
264    "she",
265    "she'd",
266    "shed",
267    "she'll",
268    "shell",
269    "should",
270    "since",
271    "so",
272    "some",
273    "such",
274    "than",
275    "that",
276    "the",
277    "their",
278    "them",
279    "then",
280    "there",
281    "therefore",
282    "these",
283    "they",
284    "this",
285    "those",
286    "through",
287    "to",
288    "too",
289    "towards",
290    "under",
291    "until",
292    "us",
293    "use",
294    "used",
295    "uses",
296    "using",
297    "very",
298    "want",
299    "was",
300    "wasn't",
301    "wasnt",
302    "we",
303    "we'd",
304    "wed",
305    "we'll",
306    "well",
307    "we're",
308    "were",
309    "we've",
310    "weve",
311    "what",
312    "what's",
313    "whats",
314    "when",
315    "where",
316    "whether",
317    "which",
318    "while",
319    "who",
320    "who'll",
321    "wholl",
322    "who's",
323    "whos",
324    "who've",
325    "whove",
326    "will",
327    "with",
328    "within",
329    "without",
330    "won't",
331    "wont",
332    "would",
333    "wouldn't",
334    "wouldnt",
335    "you",
336    "you'd",
337    "youd",
338    "you'll",
339    "youll",
340    "you're",
341    "youre",
342    "you've",
343    "youve",
344    "your",
345];
346
347const JAPANESE_STOP_WORDS: &[&str] = &[
348    "あそこ",
349    "あたり",
350    "あちら",
351    "あっち",
352    "あと",
353    "あなた",
354    "あれ",
355    "いくつ",
356    "いつ",
357    "いま",
358    "いや",
359    "いろいろ",
360    "うち",
361    "おおまか",
362    "おまえ",
363    "おれ",
364    "ここ",
365    "こちら",
366    "こっち",
367    "こと",
368    "これ",
369    "それ",
370    "そこ",
371    "そちら",
372    "そっち",
373    "ため",
374    "どこ",
375    "どこか",
376    "ところ",
377    "もの",
378    "よう",
379    "私",
380    "我々",
381    "彼",
382    "彼女",
383    "誰",
384    "何",
385    "一",
386    "二",
387    "三",
388    "四",
389    "五",
390    "六",
391    "七",
392    "八",
393    "九",
394    "十",
395    "上",
396    "下",
397    "前",
398    "後",
399    "左",
400    "右",
401    "中",
402    "外",
403    "する",
404    "した",
405    "して",
406    "される",
407    "です",
408    "ます",
409    "ませ",
410    "ない",
411    "なる",
412    "あり",
413    "ある",
414    "いる",
415    "そして",
416    "また",
417    "から",
418    "まで",
419    "より",
420    "ので",
421    "のに",
422    "ほど",
423    "ばかり",
424    "だけ",
425    "など",
426    "この",
427    "その",
428    "あの",
429    "どの",
430    "ここ",
431    "そこ",
432    "あそこ",
433    "ところ",
434    "場合",
435    "今回",
436    "以前",
437    "以後",
438];
439
440const GERMAN_STOP_WORDS: &[&str] = &[
441    "ab",
442    "aber",
443    "alle",
444    "allem",
445    "allen",
446    "aller",
447    "alles",
448    "als",
449    "also",
450    "am",
451    "an",
452    "andere",
453    "anderem",
454    "anderen",
455    "anderer",
456    "anderes",
457    "auch",
458    "auf",
459    "aus",
460    "bei",
461    "bin",
462    "bis",
463    "bist",
464    "da",
465    "damit",
466    "dann",
467    "das",
468    "dass",
469    "daß",
470    "dein",
471    "deine",
472    "deinem",
473    "deiner",
474    "deines",
475    "dem",
476    "den",
477    "denn",
478    "der",
479    "derer",
480    "des",
481    "dessen",
482    "dich",
483    "die",
484    "dies",
485    "diese",
486    "diesem",
487    "diesen",
488    "dieser",
489    "dieses",
490    "dir",
491    "doch",
492    "du",
493    "durch",
494    "ein",
495    "eine",
496    "einem",
497    "einen",
498    "einer",
499    "eines",
500    "er",
501    "es",
502    "euch",
503    "euer",
504    "eure",
505    "eurem",
506    "euren",
507    "eures",
508    "für",
509    "ganz",
510    "gar",
511    "habe",
512    "haben",
513    "hat",
514    "hatte",
515    "hattest",
516    "hattet",
517    "hätte",
518    "hättest",
519    "hättet",
520    "her",
521    "hin",
522    "ich",
523    "ihm",
524    "ihn",
525    "ihr",
526    "ihre",
527    "ihrem",
528    "ihren",
529    "ihrer",
530    "ihres",
531    "im",
532    "in",
533    "ins",
534    "ist",
535    "ja",
536    "jede",
537    "jedem",
538    "jeden",
539    "jeder",
540    "jedes",
541    "jene",
542    "jenem",
543    "jenen",
544    "jener",
545    "jenes",
546    "kann",
547    "kannst",
548    "kein",
549    "keine",
550    "keinem",
551    "keinen",
552    "keiner",
553    "keines",
554    "können",
555    "könnte",
556    "könnten",
557    "könntest",
558    "ließ",
559    "man",
560    "manche",
561    "manchem",
562    "manchen",
563    "mancher",
564    "manches",
565    "mehr",
566    "mein",
567    "meine",
568    "meinem",
569    "meinen",
570    "meiner",
571    "meines",
572    "mich",
573    "mir",
574    "mit",
575    "muss",
576    "musst",
577    "musste",
578    "muß",
579    "müssen",
580    "müssten",
581    "nach",
582    "nicht",
583    "nichts",
584    "noch",
585    "nun",
586    "nur",
587    "ob",
588    "oder",
589    "seid",
590    "sein",
591    "seine",
592    "seinem",
593    "seinen",
594    "seiner",
595    "seines",
596    "sich",
597    "sie",
598    "sind",
599    "so",
600    "um",
601    "und",
602    "uns",
603    "unter",
604    "vom",
605    "von",
606    "vor",
607    "war",
608    "waren",
609    "warst",
610    "warum",
611    "was",
612    "weil",
613    "welche",
614    "welchem",
615    "welchen",
616    "welcher",
617    "welches",
618    "wenn",
619    "wer",
620    "werde",
621    "werden",
622    "weshalb",
623    "wie",
624    "wieder",
625    "wir",
626    "wirst",
627    "wo",
628    "während",
629    "wieso",
630    "zu",
631    "zum",
632    "zur",
633    "über",
634];
635
636const KOREAN_STOP_WORDS: &[&str] = &[
637    "이",
638    "그",
639    "저",
640    "것",
641    "수",
642    "등",
643    "들",
644    "및",
645    "에서",
646    "으로",
647    "에게",
648    "께서",
649    "그리고",
650    "그러나",
651    "하지만",
652    "또한",
653    "또",
654    "더",
655    "더욱",
656    "가장",
657    "매우",
658    "너무",
659    "아주",
660    "좀",
661    "잘",
662    "못",
663    "안",
664    "않다",
665    "있다",
666    "없다",
667    "하다",
668    "되다",
669    "이다",
670    "아니다",
671    "같다",
672    "위해",
673    "통해",
674    "대한",
675    "대해",
676    "까지",
677    "부터",
678    "보다",
679    "처럼",
680    "만큼",
681    "하고",
682    "하며",
683    "해서",
684    "하는",
685    "한",
686    "된",
687    "되는",
688    "있는",
689    "없는",
690    "입니다",
691    "합니다",
692    "있습니다",
693    "그리고",
694    "그러면",
695    "따라서",
696    "때문",
697    "때문에",
698    "우리",
699    "너희",
700    "그들",
701    "자신",
702    "각",
703    "각각",
704    "모든",
705    "어떤",
706    "무슨",
707    "언제",
708    "어디",
709    "누구",
710    "무엇",
711    "왜",
712    "어떻게",
713];
714
715#[cfg(test)]
716mod tests {
717    use super::*;
718    use futures::executor::block_on;
719    use runmat_builtins::CharArray;
720
721    fn call(args: Vec<Value>) -> BuiltinResult<Value> {
722        block_on(stop_words_builtin(args))
723    }
724
725    fn words(value: Value) -> Vec<String> {
726        match value {
727            Value::StringArray(array) => {
728                assert_eq!(array.cols, 1);
729                array.data
730            }
731            other => panic!("expected string array, got {other:?}"),
732        }
733    }
734
735    #[test]
736    fn stop_words_default_returns_english_column() {
737        let out = words(call(Vec::new()).expect("stopWords"));
738        assert!(out.len() > 100);
739        assert_eq!(out.first().map(String::as_str), Some("a"));
740        assert!(out.contains(&"and".to_string()));
741        assert!(out.contains(&"the".to_string()));
742        assert!(out.contains(&"you're".to_string()));
743    }
744
745    #[test]
746    fn stop_words_language_option_supports_documented_languages() {
747        let german = words(
748            call(vec![
749                Value::String("Language".into()),
750                Value::CharArray(CharArray::new_row("de")),
751            ])
752            .expect("German stop words"),
753        );
754        assert!(german.contains(&"und".to_string()));
755        assert!(german.contains(&"über".to_string()));
756
757        let japanese = words(
758            call(vec![
759                Value::CharArray(CharArray::new_row("Language")),
760                Value::String("ja".into()),
761            ])
762            .expect("Japanese stop words"),
763        );
764        assert!(japanese.contains(&"これ".to_string()));
765        assert!(japanese.contains(&"する".to_string()));
766
767        let korean = words(
768            call(vec![
769                Value::String("language".into()),
770                Value::String("ko".into()),
771            ])
772            .expect("Korean stop words"),
773        );
774        assert!(korean.contains(&"그리고".to_string()));
775        assert!(korean.contains(&"합니다".to_string()));
776    }
777
778    #[test]
779    fn stop_words_rejects_unsupported_forms() {
780        let err = call(vec![Value::String("de".into())]).unwrap_err();
781        assert_eq!(err.identifier(), Some("RunMat:stopWords:InvalidInput"));
782        assert!(err.message().contains("expected stopWords"));
783
784        let err = call(vec![
785            Value::String("Locale".into()),
786            Value::String("en".into()),
787        ])
788        .unwrap_err();
789        assert_eq!(err.identifier(), Some("RunMat:stopWords:InvalidInput"));
790        assert!(err.message().contains("unsupported option"));
791
792        let err = call(vec![
793            Value::String("Language".into()),
794            Value::String("fr".into()),
795        ])
796        .unwrap_err();
797        assert_eq!(err.identifier(), Some("RunMat:stopWords:InvalidInput"));
798        assert!(err.message().contains("language must be"));
799    }
800}
runmat_runtime/builtins/strings/text_analytics/stopwords.rs

runmat_runtime/builtins/strings/text_analytics/
stopwords.rs
