rlt_core/tokenizer.rs
1//! Byte-level tokenizer with BOS/EOS specials. Keeps demos self-contained.
2
3use crate::{Result, RltError};
4
5/// Beginning-of-sequence token id (byte-level: 256).
6pub const BOS_ID: u32 = 256;
7/// End-of-sequence token id (byte-level: 257).
8pub const EOS_ID: u32 = 257;
9/// Minimum vocab size for this tokenizer (all bytes + BOS + EOS).
10pub const MIN_VOCAB: usize = 258;
11
12/// Byte-level tokenizer: token id = byte value; specials live above 255.
13#[derive(Clone, Debug)]
14pub struct ByteTokenizer {
15 vocab_size: usize,
16}
17
18impl ByteTokenizer {
19 /// Create a tokenizer; `vocab_size` must be >= [`MIN_VOCAB`].
20 ///
21 /// The byte-level scheme is fixed at [`MIN_VOCAB`] ids: with a larger
22 /// `vocab_size`, ids 258..vocab_size are never produced by
23 /// [`ByteTokenizer::encode`] (dead ids reserved for a future text-level
24 /// vocabulary extension).
25 pub fn new(vocab_size: usize) -> Result<Self> {
26 if vocab_size < MIN_VOCAB {
27 return Err(RltError::Token(format!(
28 "vocab_size {vocab_size} < {MIN_VOCAB} (need all byte values + BOS + EOS)"
29 )));
30 }
31 Ok(Self { vocab_size })
32 }
33
34 /// Encode text to token ids, optionally adding BOS/EOS.
35 pub fn encode(&self, text: &str, with_bos: bool, with_eos: bool) -> Vec<u32> {
36 let mut ids = Vec::with_capacity(text.len() + 2);
37 if with_bos {
38 ids.push(BOS_ID);
39 }
40 ids.extend(text.as_bytes().iter().map(|b| u32::from(*b)));
41 if with_eos {
42 ids.push(EOS_ID);
43 }
44 ids
45 }
46
47 /// Decode token ids back to text. Special tokens (ids >= 256) are skipped.
48 /// Bytes that do not form valid UTF-8 decode as the replacement character
49 /// (lossy decoding).
50 pub fn decode(&self, tokens: &[u32]) -> String {
51 let bytes: Vec<u8> = tokens
52 .iter()
53 .filter_map(|&t| u8::try_from(t).ok())
54 .collect();
55 String::from_utf8_lossy(&bytes).into_owned()
56 }
57
58 /// Vocabulary size this tokenizer was built for.
59 pub fn vocab_size(&self) -> usize {
60 self.vocab_size
61 }
62}