deepseek-recipe-encoding 0.1.0

Conversation rendering and token encoding for DeepSeek V4 and V4.1 prompts.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

//! Encode rendered prompts into token ids.

use std::sync::Arc;

use tokenizers::Tokenizer;

/// Encode text into model token ids.
///
/// Implemented for [`Tokenizer`]; other tokenizers can implement it to back a
/// prompt encoding.
pub trait TokenizerEncoder: Send + Sync {
    /// Encode `text` into token ids without added special tokens.
    fn encode_ids(&self, text: &str) -> Result<Vec<u32>, String>;
}

impl TokenizerEncoder for Tokenizer {
    fn encode_ids(&self, text: &str) -> Result<Vec<u32>, String> {
        self.encode(text, false)
            .map(|encoding| encoding.get_ids().to_vec())
            .map_err(|error| error.to_string())
    }
}

impl<T: TokenizerEncoder> TokenizerEncoder for Arc<T> {
    fn encode_ids(&self, text: &str) -> Result<Vec<u32>, String> {
        (**self).encode_ids(text)
    }
}