Function rnltk::token::tokenize_sentence

pub fn tokenize_sentence(sentence: &str) -> Vec<String>

Converts sentence to token vector.

Examples

use rnltk::token;
 
let text = "Why hello there. General Kenobi!";
let tokens = vec!["why", "hello", "there", "general", "kenobi"];
let tokenized_text = token::tokenize_sentence(text);

assert_eq!(tokens, tokenized_text);