Struct tantivy_jieba::JiebaTokenizer

pub struct JiebaTokenizer;

Tokenize the text using jieba_rs.

Need to load dict on first tokenization.

Example

use tantivy::tokenizer::*;
let tokenizer = tantivy_jieba::JiebaTokenizer {};
let mut token_stream = tokenizer.token_stream("测试");
assert_eq!(token_stream.next().unwrap().text, "测试");
assert!(token_stream.next().is_none());

Register tantivy tokenizer

use tantivy::schema::Schema;
use tantivy::tokenizer::*;
use tantivy::Index;
let tokenizer = tantivy_jieba::JiebaTokenizer {};
let index = Index::create_in_ram(schema);
index.tokenizers()
     .register("jieba", tokenizer);

Trait Implementations

impl Clone for JiebaTokenizer

fn clone(&self) -> JiebaTokenizer

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Tokenizer for JiebaTokenizer

fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a>

Creates a token stream for a given str.