rlx-moshi 0.2.11

Kyutai Moshi speech-to-speech LM (Helium + depth transformer) for RLX
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54

use anyhow::{Context, Result};
use sentencepiece::SentencePieceProcessor;
use std::path::Path;

/// Moshi text tokenizer (SentencePiece, 32k vocab).
pub struct MoshiTokenizer {
    inner: SentencePieceProcessor,
    pub text_start_token: u32,
    pub text_pad_token: u32,
}

impl MoshiTokenizer {
    pub fn open(path: impl AsRef<Path>) -> Result<Self> {
        let inner = SentencePieceProcessor::open(path.as_ref())
            .with_context(|| format!("open tokenizer {}", path.as_ref().display()))?;
        Ok(Self {
            inner,
            text_start_token: 32_000,
            text_pad_token: 3,
        })
    }

    pub fn encode(&self, text: &str) -> Result<Vec<u32>> {
        let ids = self
            .inner
            .encode(text)
            .with_context(|| format!("encode {text:?}"))?;
        Ok(ids.into_iter().map(|p| p.id).collect())
    }

    pub fn decode_piece(&self, id: u32) -> Result<String> {
        Ok(self.inner.decode_piece_ids(&[id])?)
    }

    /// Build per-frame text tokens for one-way generation: pad + inner-monologue prefix.
    pub fn prompt_frame_tokens(&self, prompt: &str, num_frames: usize) -> Result<Vec<u32>> {
        let mut ids = self.encode(prompt)?;
        if ids.is_empty() {
            ids.push(self.text_pad_token);
        }
        let mut frames = Vec::with_capacity(num_frames);
        for fi in 0..num_frames {
            let tok = if fi == 0 {
                self.text_start_token
            } else if fi - 1 < ids.len() {
                ids[fi - 1]
            } else {
                self.text_pad_token
            };
            frames.push(tok);
        }
        Ok(frames)
    }
}