Trait sudachiclone::tokenizer::CanTokenize

pub trait CanTokenize {
    fn tokenize<T: AsRef<str>>(
        &self, 
        text: T, 
        mode: &Option<SplitMode>, 
        logger: Option<Box<dyn Log>>
    ) -> Option<MorphemeList>;
}

Required methods

fn tokenize<T: AsRef<str>>(
    &self,
    text: T,
    mode: &Option<SplitMode>,
    logger: Option<Box<dyn Log>>
) -> Option<MorphemeList>

Loading content...

Implementations on Foreign Types

impl<'a, C: CanTokenize + ?Sized> CanTokenize for &'a C

fn tokenize<T: AsRef<str>>(
    &self,
    text: T,
    mode: &Option<SplitMode>,
    logger: Option<Box<dyn Log>>
) -> Option<MorphemeList>

Loading content...

Implementors

impl CanTokenize for Tokenizer

fn tokenize<T: AsRef<str>>(
    &self,
    text: T,
    mode: &Option<SplitMode>,
    logger: Option<Box<dyn Log>>
) -> Option<MorphemeList>

Loading content...