wordchipper 0.9.2

HPC Rust LLM Tokenizer Library
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
//! # [`TokenSpanEncoder`] Utilities

pub mod span_encoders;

#[allow(clippy::module_inception)]
mod token_span_encoder;

#[doc(inline)]
pub use span_encoders::SpanEncoder;
#[doc(inline)]
pub use span_encoders::SpanEncoderSelector;
#[doc(inline)]
pub use token_span_encoder::*;