pub type BatchTokenizerFn = Box<dyn Fn(&[PromptInput]) -> Result<Vec<Vec<u32>>, String> + Send>;Expand description
A batch tokenizer function that takes multiple prompt inputs and returns multiple token vectors. This is much faster than tokenizing one at a time.
Aliased Typeยง
pub struct BatchTokenizerFn(/* private fields */);