wordchipper 0.9.2

HPC Rust LLM Tokenizer Library
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

//! # Token Encoders
//!
//! ## Example
//!
//! ```rust,no_run
//! use std::sync::Arc;
//!
//! use wordchipper::{
//!     TokenEncoder,
//!     TokenEncoderOptions,
//!     TokenType,
//!     UnifiedTokenVocab,
//! };
//!
//! fn example<T: TokenType>(
//!     vocab: Arc<UnifiedTokenVocab<T>>,
//!     batch: &[&str],
//! ) -> Vec<Vec<T>> {
//!     let vocab1 = vocab.clone();
//!     let encoder = TokenEncoderOptions::default().build(vocab1);
//!     encoder.try_encode_batch(batch, None).unwrap()
//! }
//! ```

mod encoder_options;
#[cfg(any(test, feature = "testing"))]
pub mod testing;
mod token_encoder;
pub mod token_span_encoder;

#[doc(inline)]
pub use encoder_options::*;
#[doc(inline)]
pub use token_encoder::*;