Struct Tokenize 

pub struct Tokenize { /* private fields */ }

Tokenize text into words using a specified delimiter

Implementations

impl Tokenize

pub fn new(delimiter: String) -> Self

Create a new tokenize transform with custom delimiter

pub fn whitespace() -> Self

Create tokenizer with whitespace delimiter

pub fn any_whitespace() -> Self

Create tokenizer that splits on any whitespace

Trait Implementations

impl Clone for Tokenize

fn clone(&self) -> Tokenize

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Tokenize

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Transform<String> for Tokenize

type Output = Vec<String>

fn transform(&self, input: String) -> Result<Self::Output>

Apply the transformation to the input

fn is_deterministic(&self) -> bool

Check if the transform is deterministic