novel_segment/
pipeline.rs

1//! Tokenizer / optimizer traits and default module names.
2
3use crate::segment::Segment;
4use crate::word::Word;
5
6pub const URL_TOKENIZER: &str = "URLTokenizer";
7pub const WILDCARD_TOKENIZER: &str = "WildcardTokenizer";
8pub const PUNCTUATION_TOKENIZER: &str = "PunctuationTokenizer";
9pub const FOREIGN_TOKENIZER: &str = "ForeignTokenizer";
10pub const DICT_TOKENIZER: &str = "DictTokenizer";
11pub const CHS_NAME_TOKENIZER: &str = "ChsNameTokenizer";
12pub const JP_SIMPLE_TOKENIZER: &str = "JpSimpleTokenizer";
13pub const ZHUYIN_TOKENIZER: &str = "ZhuyinTokenizer";
14pub const EMAIL_OPTIMIZER: &str = "EmailOptimizer";
15pub const CHS_NAME_OPTIMIZER: &str = "ChsNameOptimizer";
16pub const DICT_OPTIMIZER: &str = "DictOptimizer";
17pub const DATETIME_OPTIMIZER: &str = "DatetimeOptimizer";
18pub const FOREIGN_OPTIMIZER: &str = "ForeignOptimizer";
19pub const ZHT_SYNONYM_OPTIMIZER: &str = "ZhtSynonymOptimizer";
20pub const ADJECTIVE_OPTIMIZER: &str = "AdjectiveOptimizer";
21
22pub fn default_tokenizer_names(all_mod: bool) -> Vec<&'static str> {
23    let _ = all_mod;
24    vec![
25        URL_TOKENIZER,
26        WILDCARD_TOKENIZER,
27        PUNCTUATION_TOKENIZER,
28        FOREIGN_TOKENIZER,
29        DICT_TOKENIZER,
30        CHS_NAME_TOKENIZER,
31        JP_SIMPLE_TOKENIZER,
32        ZHUYIN_TOKENIZER,
33    ]
34}
35
36pub fn default_optimizer_names(all_mod: bool) -> Vec<&'static str> {
37    let mut v = vec![
38        EMAIL_OPTIMIZER,
39        CHS_NAME_OPTIMIZER,
40        DICT_OPTIMIZER,
41        DATETIME_OPTIMIZER,
42        FOREIGN_OPTIMIZER,
43        ADJECTIVE_OPTIMIZER,
44    ];
45    if all_mod {
46        // Match ENUM_SUBMODS: Zht sits before Adjective.
47        v.insert(v.len() - 1, ZHT_SYNONYM_OPTIMIZER);
48    }
49    v
50}
51
52pub trait Tokenizer: Send + Sync {
53    fn name(&self) -> &'static str;
54    fn split(&self, words: Vec<Word>, seg: &Segment) -> Vec<Word>;
55}
56
57pub trait Optimizer: Send + Sync {
58    fn name(&self) -> &'static str;
59    fn do_optimize(&self, words: Vec<Word>, seg: &Segment) -> Vec<Word>;
60}
61
62pub fn enabled(name: &str, disabled: &[String]) -> bool {
63    !disabled.iter().any(|d| d == name)
64}
65
66/// Apply `fn` only to unrecognized words (`p` missing or 0). Returns `None` to keep original.
67pub fn split_unknown(words: Vec<Word>, mut f: impl FnMut(&str) -> Option<Vec<Word>>) -> Vec<Word> {
68    let mut ret = Vec::new();
69    for word in words {
70        if word.is_recognized() {
71            ret.push(word);
72            continue;
73        }
74        match f(&word.w) {
75            Some(parts) => ret.extend(parts),
76            None => ret.push(word),
77        }
78    }
79    ret
80}
81
82/// Apply `fn` only when `p` is unset (`typeof p !== 'number'`).
83pub fn split_unset(words: Vec<Word>, mut f: impl FnMut(&str) -> Option<Vec<Word>>) -> Vec<Word> {
84    let mut ret = Vec::new();
85    for word in words {
86        if word.has_pos() {
87            ret.push(word);
88            continue;
89        }
90        match f(&word.w) {
91            Some(parts) => ret.extend(parts),
92            None => ret.push(word),
93        }
94    }
95    ret
96}
97
98pub fn splice_token(words: &mut Vec<Word>, i: usize, len: usize, nw: Word) {
99    let end = (i + len).min(words.len());
100    words.splice(i..end, std::iter::once(nw));
101}
102
103/// JS `sliceToken` / `createToken`: set `auto_create` when the merged word is not in TABLE.
104pub fn splice_created(words: &mut Vec<Word>, i: usize, len: usize, mut nw: Word, seg: &Segment) {
105    if !seg.table.table.contains_key(&nw.w) {
106        nw.auto_create = true;
107    }
108    let end = (i + len).min(words.len());
109    if nw.m.is_none() && i < end {
110        nw.m = Some(words[i..end].to_vec());
111    }
112    splice_token(words, i, len, nw);
113}
novel_segment/pipeline.rs

novel_segment/
pipeline.rs
