wordchipper 0.9.2

HPC Rust LLM Tokenizer Library
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
//! # [`SpanLexer`] mechanics for [`TextSpanner`](`crate::spanners::TextSpanner`) implementations.

pub mod logos;
pub(crate) mod regex_automata;

pub mod accelerators;
mod lexer_builder;
mod lexer_spanner;
mod span_lexer;

#[doc(inline)]
pub use lexer_builder::*;
#[doc(inline)]
pub use lexer_spanner::*;
#[doc(inline)]
pub use span_lexer::*;