Struct Tokenizer 

pub struct Tokenizer(pub Tokenizer);

Tuple Fields

0: Tokenizer

Implementations

impl Tokenizer

pub fn txt2tokens2writer<W>(&self, txt: &str, wtr: &mut W) -> Result<(), Error>

where W: Write,

impl Tokenizer

pub fn json2tokens2writer<W>( &self, jline: &[u8], wtr: &mut W, ) -> Result<(), Error>

where W: Write,

impl Tokenizer

pub fn jsonl2tokens2writer<I, W>( &self, jsonl: I, wtr: &mut W, ) -> Result<(), Error>

where I: Iterator<Item = Result<Vec<u8>, Error>>, W: Write,

impl Tokenizer

pub fn stdin2jsonl2tokens2stdout(&self) -> Result<(), Error>

impl Tokenizer

pub fn new_default() -> Result<Self, Error>