tantivy-meta-tokenizer 0.3.0

Easy and fast book searcher, create and search your private library.
Documentation 
1
2
3
4
5
6
7
8
9

use tantivy::tokenizer::{BoxTokenStream, SimpleTokenizer, Tokenizer};

lazy_static::lazy_static! {
    static ref SIMPLE: SimpleTokenizer = SimpleTokenizer;
}

pub fn token_stream(text: &str) -> BoxTokenStream {
    SIMPLE.token_stream(text)
}