Function rnltk::token::tokenize_sentence_configurable

pub fn tokenize_sentence_configurable(
    sentence: &str,
    config: TokenConfig
) -> Vec<String>

Tokenize sentence based on a given configuration.

This function will be deprecated in the future once rnltk hits version 1.0 and functionality will be moved to tokenize_sentence.

Examples

use rnltk::token;
 
let token_config = token::TokenConfig::default();
let text = "Why hello there. General Kenobi!";
let tokens = vec!["hello", "gener", "kenobi"];
let tokenized_text = token::tokenize_sentence_configurable(text, token_config);

assert_eq!(tokens, tokenized_text);