Struct Segment 

pub struct Segment { /* private fields */ }

Chinese word segmenter.

Implementations

impl Segment

pub fn new(options: SegmentOptions) -> Self

Examples found in repository

examples/basic.rs (lines 4-7)

fn main() -> Result<(), novel_segment::Error> {
    let mut seg = Segment::new(novel_segment::SegmentOptions {
        auto_cjk: true,
        ..Default::default()
    });
    seg.use_default()?;
    let text = "这是一个基于Rust的中文分词模块。";
    let words = seg.do_segment(text, DoSegmentOptions::default());
    println!("{text}");
    println!(
        "{}",
        words
            .iter()
            .map(|w| format!("{}:{:x}", w.w, w.p.unwrap_or(0)))
            .collect::<Vec<_>>()
            .join(" / ")
    );

    let simple = seg.do_segment_simple(text, DoSegmentOptions::default());
    println!("{}", simple.join(" / "));

    seg.add_word("專有名詞", Some(novel_segment::POSTAG::D_N), Some(100.0))?;
    Ok(())
}

pub fn with_default() -> Result<Self>

new Segment(); useDefault() — auto_cjk stays false.

pub fn with_novel_default() -> Result<Self>

Test/CLI profile: autoCjk, all_mod, convertSynonym.

pub fn with_node_novel_default() -> Result<Self>

word.novel.test.ts: createSegment then wipe SYNONYM and reload with nodeNovelMode.

pub fn use_default(&mut self) -> Result<&mut Self>

Examples found in repository

examples/basic.rs (line 8)

fn main() -> Result<(), novel_segment::Error> {
    let mut seg = Segment::new(novel_segment::SegmentOptions {
        auto_cjk: true,
        ..Default::default()
    });
    seg.use_default()?;
    let text = "这是一个基于Rust的中文分词模块。";
    let words = seg.do_segment(text, DoSegmentOptions::default());
    println!("{text}");
    println!(
        "{}",
        words
            .iter()
            .map(|w| format!("{}:{:x}", w.w, w.p.unwrap_or(0)))
            .collect::<Vec<_>>()
            .join(" / ")
    );

    let simple = seg.do_segment_simple(text, DoSegmentOptions::default());
    println!("{}", simple.join(" / "));

    seg.add_word("專有名詞", Some(novel_segment::POSTAG::D_N), Some(100.0))?;
    Ok(())
}

pub fn use_default_modules(&mut self) -> &mut Self

pub fn use_tokenizer(&mut self, tok: Box<dyn Tokenizer>) -> &mut Self

pub fn use_optimizer(&mut self, opt: Box<dyn Optimizer>) -> &mut Self

pub fn use_default_dicts(&mut self) -> Result<&mut Self>

pub fn load_dict(&mut self, name: &str) -> Result<&mut Self>

pub fn load_dict_file(&mut self, path: impl AsRef<Path>) -> Result<&mut Self>

pub fn load_synonym_dict( &mut self, name: &str, skip_exists: bool, ) -> Result<&mut Self>

pub fn clear_synonym_dict(&mut self) -> &mut Self

pub fn use_default_synonym_dict( &mut self, node_novel_mode: bool, ) -> Result<&mut Self>

JS useDefaultSynonymDict.

pub fn load_stopword_dict(&mut self, name: &str) -> Result<&mut Self>

pub fn load_blacklist_dict(&mut self, name: &str) -> Result<&mut Self>

pub fn load_blacklist_optimizer_dict(&mut self, name: &str) -> Result<&mut Self>

pub fn load_blacklist_synonym_dict(&mut self, name: &str) -> Result<&mut Self>

pub fn do_blacklist(&mut self) -> &mut Self

pub fn add_blacklist(&mut self, word: &str) -> &mut Self

Remove a word from TABLE (JS addBlacklist).

pub fn add_word( &mut self, spec: &str, p: Option<u32>, f: Option<f64>, ) -> Result<&mut Self>

Add a dictionary entry (詞|詞性|詞權值 or just the word).

Examples found in repository

examples/basic.rs (line 24)

fn main() -> Result<(), novel_segment::Error> {
    let mut seg = Segment::new(novel_segment::SegmentOptions {
        auto_cjk: true,
        ..Default::default()
    });
    seg.use_default()?;
    let text = "这是一个基于Rust的中文分词模块。";
    let words = seg.do_segment(text, DoSegmentOptions::default());
    println!("{text}");
    println!(
        "{}",
        words
            .iter()
            .map(|w| format!("{}:{:x}", w.w, w.p.unwrap_or(0)))
            .collect::<Vec<_>>()
            .join(" / ")
    );

    let simple = seg.do_segment_simple(text, DoSegmentOptions::default());
    println!("{}", simple.join(" / "));

    seg.add_word("專有名詞", Some(novel_segment::POSTAG::D_N), Some(100.0))?;
    Ok(())
}

pub fn add_synonym(&mut self, canonical: &str, variants: &[&str]) -> &mut Self

pub fn do_segment(&self, text: &str, options: DoSegmentOptions) -> Vec<Word>

Examples found in repository

examples/basic.rs (line 10)

fn main() -> Result<(), novel_segment::Error> {
    let mut seg = Segment::new(novel_segment::SegmentOptions {
        auto_cjk: true,
        ..Default::default()
    });
    seg.use_default()?;
    let text = "这是一个基于Rust的中文分词模块。";
    let words = seg.do_segment(text, DoSegmentOptions::default());
    println!("{text}");
    println!(
        "{}",
        words
            .iter()
            .map(|w| format!("{}:{:x}", w.w, w.p.unwrap_or(0)))
            .collect::<Vec<_>>()
            .join(" / ")
    );

    let simple = seg.do_segment_simple(text, DoSegmentOptions::default());
    println!("{}", simple.join(" / "));

    seg.add_word("專有名詞", Some(novel_segment::POSTAG::D_N), Some(100.0))?;
    Ok(())
}

pub fn do_segment_simple( &self, text: &str, options: DoSegmentOptions, ) -> Vec<String>

Examples found in repository

examples/basic.rs (line 21)

fn main() -> Result<(), novel_segment::Error> {
    let mut seg = Segment::new(novel_segment::SegmentOptions {
        auto_cjk: true,
        ..Default::default()
    });
    seg.use_default()?;
    let text = "这是一个基于Rust的中文分词模块。";
    let words = seg.do_segment(text, DoSegmentOptions::default());
    println!("{text}");
    println!(
        "{}",
        words
            .iter()
            .map(|w| format!("{}:{:x}", w.w, w.p.unwrap_or(0)))
            .collect::<Vec<_>>()
            .join(" / ")
    );

    let simple = seg.do_segment_simple(text, DoSegmentOptions::default());
    println!("{}", simple.join(" / "));

    seg.add_word("專有名詞", Some(novel_segment::POSTAG::D_N), Some(100.0))?;
    Ok(())
}

Trait Implementations

impl Default for Segment

fn default() -> Self

Returns the “default value” for a type.