rlx_moshi/
tokenizer.rs

1use anyhow::{Context, Result};
2use sentencepiece::SentencePieceProcessor;
3use std::path::Path;
4
5/// Moshi text tokenizer (SentencePiece, 32k vocab).
6pub struct MoshiTokenizer {
7    inner: SentencePieceProcessor,
8    pub text_start_token: u32,
9    pub text_pad_token: u32,
10}
11
12impl MoshiTokenizer {
13    pub fn open(path: impl AsRef<Path>) -> Result<Self> {
14        let inner = SentencePieceProcessor::open(path.as_ref())
15            .with_context(|| format!("open tokenizer {}", path.as_ref().display()))?;
16        Ok(Self {
17            inner,
18            text_start_token: 32_000,
19            text_pad_token: 3,
20        })
21    }
22
23    pub fn encode(&self, text: &str) -> Result<Vec<u32>> {
24        let ids = self
25            .inner
26            .encode(text)
27            .with_context(|| format!("encode {text:?}"))?;
28        Ok(ids.into_iter().map(|p| p.id).collect())
29    }
30
31    pub fn decode_piece(&self, id: u32) -> Result<String> {
32        Ok(self.inner.decode_piece_ids(&[id])?)
33    }
34
35    /// Build per-frame text tokens for one-way generation: pad + inner-monologue prefix.
36    pub fn prompt_frame_tokens(&self, prompt: &str, num_frames: usize) -> Result<Vec<u32>> {
37        let mut ids = self.encode(prompt)?;
38        if ids.is_empty() {
39            ids.push(self.text_pad_token);
40        }
41        let mut frames = Vec::with_capacity(num_frames);
42        for fi in 0..num_frames {
43            let tok = if fi == 0 {
44                self.text_start_token
45            } else if fi - 1 < ids.len() {
46                ids[fi - 1]
47            } else {
48                self.text_pad_token
49            };
50            frames.push(tok);
51        }
52        Ok(frames)
53    }
54}
rlx_moshi/tokenizer.rs

rlx_moshi/
tokenizer.rs
