Skip to main content

rlt_core/
tokenizer.rs

1//! Byte-level tokenizer with BOS/EOS specials. Keeps demos self-contained.
2
3use crate::{Result, RltError};
4
5/// Beginning-of-sequence token id (byte-level: 256).
6pub const BOS_ID: u32 = 256;
7/// End-of-sequence token id (byte-level: 257).
8pub const EOS_ID: u32 = 257;
9/// Minimum vocab size for this tokenizer (all bytes + BOS + EOS).
10pub const MIN_VOCAB: usize = 258;
11
12/// Byte-level tokenizer: token id = byte value; specials live above 255.
13#[derive(Clone, Debug)]
14pub struct ByteTokenizer {
15    vocab_size: usize,
16}
17
18impl ByteTokenizer {
19    /// Create a tokenizer; `vocab_size` must be >= [`MIN_VOCAB`].
20    ///
21    /// The byte-level scheme is fixed at [`MIN_VOCAB`] ids: with a larger
22    /// `vocab_size`, ids 258..vocab_size are never produced by
23    /// [`ByteTokenizer::encode`] (dead ids reserved for a future text-level
24    /// vocabulary extension).
25    pub fn new(vocab_size: usize) -> Result<Self> {
26        if vocab_size < MIN_VOCAB {
27            return Err(RltError::Token(format!(
28                "vocab_size {vocab_size} < {MIN_VOCAB} (need all byte values + BOS + EOS)"
29            )));
30        }
31        Ok(Self { vocab_size })
32    }
33
34    /// Encode text to token ids, optionally adding BOS/EOS.
35    pub fn encode(&self, text: &str, with_bos: bool, with_eos: bool) -> Vec<u32> {
36        let mut ids = Vec::with_capacity(text.len() + 2);
37        if with_bos {
38            ids.push(BOS_ID);
39        }
40        ids.extend(text.as_bytes().iter().map(|b| u32::from(*b)));
41        if with_eos {
42            ids.push(EOS_ID);
43        }
44        ids
45    }
46
47    /// Decode token ids back to text. Special tokens (ids >= 256) are skipped.
48    /// Bytes that do not form valid UTF-8 decode as the replacement character
49    /// (lossy decoding).
50    pub fn decode(&self, tokens: &[u32]) -> String {
51        let bytes: Vec<u8> = tokens
52            .iter()
53            .filter_map(|&t| u8::try_from(t).ok())
54            .collect();
55        String::from_utf8_lossy(&bytes).into_owned()
56    }
57
58    /// Vocabulary size this tokenizer was built for.
59    pub fn vocab_size(&self) -> usize {
60        self.vocab_size
61    }
62}