deepseek-recipe-encoding 0.1.0

Conversation rendering and token encoding for DeepSeek V4 and V4.1 prompts.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

//! Model-specific conversation rendering and token encoding.
//!
//! V4 and V4.1 provide prompt rendering. Attach a tokenizer to an encoding to
//! encode conversations into token IDs.

use deepseek_recipe_core::conversation::Conversation;
use deepseek_recipe_core::multimodal::ImageSource;

pub use tokenizer::TokenizerEncoder;
pub use v4::{dsv4, dsv41};

pub mod v4;

mod tokenizer;

/// A rendered prompt and the images its placeholders refer to.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RenderedPrompt {
    /// Model input string.
    pub prompt: String,
    /// Image sources in the order their placeholders appear in the prompt.
    pub image_sources: Vec<ImageSource>,
}

/// Failure to encode a conversation.
#[derive(Debug, thiserror::Error)]
pub enum EncodingError {
    /// The rendered prompt could not be encoded.
    #[error("failed to encode conversation: {0}")]
    Encode(String),
    /// The encoding has no attached tokenizer.
    #[error("no tokenizer is attached to the encoding")]
    MissingTokenizer,
}

/// Render model prompts and encode conversations into token IDs.
pub trait PromptEncoding {
    /// Encode a conversation into model token IDs using the attached tokenizer.
    ///
    /// The tokenizer must be attached to the encoding before this call, through
    /// the encoding's `with_tokenizer` method. Returns
    /// [`EncodingError::MissingTokenizer`] when no tokenizer is attached.
    fn encode(&self, conversation: &Conversation) -> Result<Vec<u32>, EncodingError>;

    /// Render the conversation and the prefix for the next assistant turn.
    fn render_conversation(&self, conversation: &Conversation) -> RenderedPrompt;
}