Struct SimpleTokenizer 

pub struct SimpleTokenizer { /* private fields */ }

Simple tokenizer that splits text into words

Examples

use sklears_feature_extraction::text::preprocessing::SimpleTokenizer;

let tokenizer = SimpleTokenizer::new();
let tokens = tokenizer.tokenize("Hello, world! This is a test.");
assert_eq!(tokens, vec!["hello", "world", "this", "is", "a", "test"]);

Implementations

impl SimpleTokenizer

pub fn new() -> Self

Create a new simple tokenizer

pub fn lowercase(self, lowercase: bool) -> Self

Set whether to convert to lowercase

pub fn remove_punctuation(self, remove_punctuation: bool) -> Self

Set whether to remove punctuation

pub fn tokenize(&self, text: &str) -> Vec<String>

Tokenize text into words

Trait Implementations

impl Clone for SimpleTokenizer

fn clone(&self) -> Self

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for SimpleTokenizer

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for SimpleTokenizer

fn default() -> Self

Returns the “default value” for a type.