pub type AnyToken<'a> = TypedToken<'a, AnyGrammar>;Expand description
Token alias for grammar-independent tokenization pipelines.
Aliased Type§
pub struct AnyToken<'a> { /* private fields */ }pub type AnyToken<'a> = TypedToken<'a, AnyGrammar>;Token alias for grammar-independent tokenization pipelines.
pub struct AnyToken<'a> { /* private fields */ }