ws-segment-rs 0.1.0

Chinese word segmentation for web novels (Rust port of novel-segment)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

//! Built-in punctuation / separator characters for PunctuationTokenizer.

use once_cell::sync::Lazy;
use std::collections::{BTreeMap, HashMap};

const TABLE: &str = concat!(
    r#" ,.;+-|/\'":?<>[]{}=!@#$%^&*()~`"#,
    "。，、＇：∶；?‘’“”〝〞ˆˇ﹕︰﹔﹖﹑·¨….¸;！´？！～—ˉ｜‖＂〃｀@﹫¡¿﹏﹋﹌︴々﹟#﹩$﹠&﹪%*﹡﹢﹦",
    "﹤‐￣¯―﹨ˆ˜﹍﹎+=<­＿_-",
    r"\",
    "ˇ~﹉﹊（）〈〉‹›﹛﹜『』〖〗［］《》〔〕{}「」【】︵︷︿︹︽_﹁﹃︻︶︸",
    "﹀︺︾ˉ﹂﹄︼＋－×÷﹢﹣±／＝≈≡≠∧∨∑∏∪∩∈⊙⌒⊥∥∠∽≌＜＞≤≥≮≯∧∨√﹙﹚[]﹛﹜∫∮∝∞⊙∏",
    "┌┬┐┏┳┓╒╤╕─│├┼┤┣╋┫╞╪╡━┃└┴┘┗┻┛╘╧╛┄┆┅┇╭─╮┏━┓╔╦╗┈┊│╳│┃┃╠╬╣┉┋╰─╯┗━┛",
    "╚╩╝╲╱┞┟┠┡┢┦┧┨┩┪╉╊┭┮┯┰┱┲┵┶┷┸╇╈┹┺┽┾┿╀╁╂╃╄╅╆",
    "○◇□△▽☆●◆■▲▼★♠♥♦♣☼☺◘♀√☻◙♂×▁▂▃▄▅▆▇█⊙◎۞卍卐╱╲▁▏↖↗↑←↔◤◥╲╱▔▕↙↘↓→↕◣◢∷▒░℡™",
    "．・　※",
    "⋯",
    "丶",
);

/// length → punctuation → length
pub static STOPWORD2: Lazy<BTreeMap<usize, HashMap<String, usize>>> = Lazy::new(|| {
    let mut map: BTreeMap<usize, HashMap<String, usize>> = BTreeMap::new();
    let mut seen = std::collections::HashSet::new();
    for ch in TABLE.chars() {
        let w = ch.to_string();
        if w.is_empty() || !seen.insert(w.clone()) {
            continue;
        }
        let len = crate::text::char_len(&w);
        map.entry(len).or_default().insert(w, len);
    }
    map
});